MrKalzu/pando
1
0 Fork 0
Code Issues 2 Pull Requests Actions Packages Projects Releases Wiki Activity

Support for bigger clusters.

Browse Source
This commit is contained in:
Kalzu Rekku committed 2026-10-27 14:08:42 +02:00
1 parent be278b3c7c
commit a6d014601d
8 files changed
+1336 -323
Download Patch File Download Diff File Expand all files Collapse all files

No files matched your search

repl-test.sh
+341 -122
View File
@@ -1,24 +1,69 @@
#!/usr/bin/env bash
# ==============================================================================
# repl-test.sh: Integration harness for Records API & Replication (mTLS)
# Starts Primary (:3000), Replica (:3001), and Dashboard (:8080).
#
# Supports N instances in multiple topologies:
# star - one primary, all replicas pull directly from it (default)
# chain - a linear chain: primary -> r1 -> r2 -> ... -> rN
#
# Environment variable overrides:
# NODES=4 number of total nodes (1 primary + N-1 replicas; default: 2)
# BASE_PORT=3000 base port; nodes occupy BASE_PORT .. BASE_PORT+NODES-1
# TOPO=star|chain replication topology (default: star)
# TOKEN=... shared replication bearer token
# START_DASHBOARD=1 launch the web dashboard (default: 1)
# SKIP_SWITCHOVER=0 skip the switchover smoke test (default: 0)
# NO_WAIT=0 exit immediately after smoke tests instead of waiting
#
# Examples:
# bash repl-test.sh # 2 nodes, star, default ports
# NODES=3 bash repl-test.sh # 1 primary + 2 replicas, star
# NODES=4 TOPO=chain bash repl-test.sh # 4-node replication chain
# NODES=5 SKIP_SWITCHOVER=1 bash repl-test.sh
# ==============================================================================
set -euo pipefail
# --- Configuration & Overrides ---
# ---------------------------------------------------------------------------
# Configuration
# ---------------------------------------------------------------------------
ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)"
BIN="${BIN:-$ROOT_DIR/target/debug/rust_test}"
WORK="${WORK:-$ROOT_DIR/repl-test}"
PRIMARY_PORT="${PRIMARY_PORT:-3000}"
REPLICA_PORT="${REPLICA_PORT:-3001}"
BASE_PORT="${BASE_PORT:-3000}"
NODES="${NODES:-2}" # total node count (must be >= 2)
TOPO="${TOPO:-star}" # star | chain
TOKEN="${TOKEN:-change-me}"
DASHBOARD_PORT="${DASHBOARD_PORT:-8080}"
TOKEN="${TOKEN:-change-me}" # Replication bearer token
CORS_ORIGIN="${CORS_ORIGIN:-http://127.0.0.1:$DASHBOARD_PORT}"
START_DASHBOARD="${START_DASHBOARD:-1}"
SKIP_SWITCHOVER="${SKIP_SWITCHOVER:-0}"
NO_WAIT="${NO_WAIT:-0}"
if [[ "$NODES" -lt 2 ]]; then
echo "Error: NODES must be >= 2." >&2
exit 1
fi
# ---------------------------------------------------------------------------
# Node naming and port helpers
# ---------------------------------------------------------------------------
# node_name 0 -> "primary"
# node_name 1 -> "replica-1"
# node_name 2 -> "replica-2" ...
node_name() {
local idx=$1
if [[ $idx -eq 0 ]]; then echo "primary"; else echo "replica-$idx"; fi
}
# node_port 0 -> BASE_PORT
# node_port 1 -> BASE_PORT+1 ...
node_port() { echo $(( BASE_PORT + $1 )); }
PIDS=()
# --- Helper Utilities ---
# ---------------------------------------------------------------------------
# Helper utilities
# ---------------------------------------------------------------------------
check_port() {
local port=$1
if command -v lsof >/dev/null 2>&1; then
@@ -43,38 +88,119 @@ cleanup() {
}
trap cleanup EXIT INT TERM
# --- Pre-flight Checks ---
for port in "$PRIMARY_PORT" "$REPLICA_PORT"; do
json_field() {
# Usage: json_field <json_string> <field_name>
local json="$1" field="$2"
if command -v jq >/dev/null 2>&1; then
echo "$json" | jq -r ".${field} // empty"
else
echo "$json" | grep -o "\"${field}\":\"[^\"]*" | head -1 | cut -d'"' -f4
fi
}
color_echo() {
local color=$1; shift
local code
case "$color" in
red) code='\033[1;31m' ;;
green) code='\033[1;32m' ;;
yellow) code='\033[1;33m' ;;
blue) code='\033[1;34m' ;;
*) code='\033[0m' ;;
esac
echo -e "${code}$*\033[0m"
}
wait_up() {
local name=$1 port=$2
for _ in $(seq 1 80); do
if curl -k -sf "https://127.0.0.1:$port/api/now" >/dev/null 2>&1; then
return 0
fi
sleep 0.1
done
color_echo red "Error: $name failed to start on :$port" >&2
if [[ -f "$WORK/$name/log.txt" ]]; then
echo "--- Last 30 lines of $WORK/$name/log.txt ---" >&2
tail -n 30 "$WORK/$name/log.txt" >&2
fi
exit 1
}
mint_join_token() {
local primary_port=$1
local resp
resp=$(curl -k -sf -X POST "https://127.0.0.1:$primary_port/api/cluster/tokens" \
-H 'Content-Type: application/json' \
-d '{"ttl_secs": 300}')
json_field "$resp" "token"
}
wait_replica_caught_up() {
local name=$1 port=$2 target_seq=$3
for _ in $(seq 1 80); do
local out
out=$(curl -k -s "https://127.0.0.1:$port/api/replication/status" 2>/dev/null || true)
local applied
applied=$(json_field "$out" "sync.last_applied_seq" 2>/dev/null || echo "0")
# Fallback grep for nested .sync.last_applied_seq
if [[ -z "$applied" || "$applied" == "null" ]]; then
applied=$(echo "$out" | grep -o '"last_applied_seq":[0-9]*' | head -1 | cut -d: -f2)
fi
if [[ "${applied:-0}" -ge "$target_seq" ]]; then
return 0
fi
sleep 0.25
done
color_echo yellow " ⚠ WARNING: $name (:$port) did not reach seq $target_seq in time." >&2
return 1
}
# ---------------------------------------------------------------------------
# Pre-flight
# ---------------------------------------------------------------------------
for idx in $(seq 0 $(( NODES - 1 ))); do
port=$(node_port "$idx")
if check_port "$port"; then
echo "Error: Port $port is already in use by another process." >&2
color_echo red "Error: Port $port is already in use." >&2
exit 1
fi
done
if [[ -f "$ROOT_DIR/Cargo.toml" ]]; then
echo "Ensuring debug binary is up to date with cargo..."
( cd "$ROOT_DIR" && cargo build )
elif [[ ! -x "$BIN" ]]; then
echo "Error: Binary not found or not executable: $BIN" >&2
exit 1
if [[ "$START_DASHBOARD" -eq 1 ]]; then
if check_port "$DASHBOARD_PORT"; then
color_echo yellow "Dashboard port :$DASHBOARD_PORT is in use; dashboard will be skipped."
START_DASHBOARD=0
fi
fi
# --- Workspace & Schema Setup ---
rm -rf "$WORK"
mkdir -p "$WORK/primary" "$WORK/replica"
# Build
if [[ -f "$ROOT_DIR/Cargo.toml" ]]; then
echo "Ensuring debug binary is up to date..."
( cd "$ROOT_DIR" && cargo build )
elif [[ ! -x "$BIN" ]]; then
color_echo red "Error: Binary not found: $BIN" >&2; exit 1
fi
# Schema allows "msg" and "message", requiring integer "version"
for d in primary replica; do
cat > "$WORK/$d/config.json" <<'EOF'
# ---------------------------------------------------------------------------
# Workspace setup
# ---------------------------------------------------------------------------
rm -rf "$WORK"
for idx in $(seq 0 $(( NODES - 1 ))); do
name=$(node_name "$idx")
mkdir -p "$WORK/$name"
cat > "$WORK/$name/config.json" <<'CFGEOF'
{
"mode": "loose",
"fields": {
"version": { "type": "integer" }
}
}
EOF
CFGEOF
done
# ---------------------------------------------------------------------------
# start_node <name> <port> [extra env vars...]
# ---------------------------------------------------------------------------
start_node() {
local name=$1 port=$2
shift 2
@@ -88,142 +214,235 @@ start_node() {
PIDS+=("$!")
}
wait_up() {
local name=$1 port=$2
for _ in $(seq 1 60); do
# Note: -k used since cluster cert is generated on boot
if curl -k -sf "https://127.0.0.1:$port/api/now" >/dev/null 2>&1; then
return 0
fi
sleep 0.1
done
echo -e "\n\033[1;31mError: $name failed to start on :$port\033[0m" >&2
if [[ -f "$WORK/$name/log.txt" ]]; then
echo "--- Last 20 lines of $WORK/$name/log.txt ---" >&2
tail -n 20 "$WORK/$name/log.txt" >&2
fi
exit 1
}
# ==============================================================================
# 1. Launch Primary (Cluster CA & Server Cert generated automatically)
# 1. Start Primary
# ==============================================================================
echo -e "\033[1;32m==> Starting Primary on :$PRIMARY_PORT (HTTPS/mTLS)...\033[0m"
PRIMARY_PORT=$(node_port 0)
color_echo green "==> [1/N] Starting Primary on :$PRIMARY_PORT (HTTPS/mTLS)..."
start_node primary "$PRIMARY_PORT" APP_ROLE=primary
wait_up "primary" "$PRIMARY_PORT"
color_echo green " ✔ Primary is up."
# ==============================================================================
# 2. Test Dynamic Discovery: Retrieve Cluster Public CA
# 2. Retrieve CA cert (sanity check)
# ==============================================================================
echo -e "\n\033[1;34m==> Testing dynamic discovery: Fetching Cluster Public CA...\033[0m"
color_echo blue "\n==> Verifying Cluster CA is accessible..."
ca_resp=$(curl -k -sf "https://127.0.0.1:$PRIMARY_PORT/api/cluster/ca")
if echo "$ca_resp" | grep -q 'BEGIN CERTIFICATE'; then
echo -e " \033[1;32m✔ Cluster CA certificate downloaded successfully.\033[0m"
color_echo green " ✔ Cluster CA certificate downloaded."
else
echo -e " \033[1;31m✖ Failed to retrieve Cluster CA.\033[0m"
exit 1
color_echo red " ✖ Failed to retrieve Cluster CA – aborting." >&2; exit 1
fi
# ==============================================================================
# 3. Mint Short-Lived Join Token via Primary
# 3. Start replicas according to topology
#
# star topology: every replica pulls directly from the primary
# chain topology: replica-N pulls from replica-(N-1), which pulls from primary
# ==============================================================================
echo -e "\n\033[1;34m==> Minting short-lived Join Token from Primary...\033[0m"
token_resp=$(curl -k -sf -X POST "https://127.0.0.1:$PRIMARY_PORT/api/cluster/tokens" \
-H 'Content-Type: application/json' \
-d '{"ttl_secs": 120}')
if command -v jq >/dev/null 2>&1; then
JOIN_TOKEN=$(echo "$token_resp" | jq -r '.token')
else
JOIN_TOKEN=$(echo "$token_resp" | grep -o '"token":"[^"]*' | cut -d'"' -f4)
fi
# Arrays for later use
PORTS=() # PORTS[i] = port for node i
NAMES=() # NAMES[i] = name for node i
SOURCES=() # SOURCES[i] = upstream URL for node i (empty for primary)
echo -e " \033[1;32m✔ Obtained one-time join token: $JOIN_TOKEN\033[0m"
PORTS[0]=$PRIMARY_PORT
NAMES[0]="primary"
SOURCES[0]=""
# ==============================================================================
# 4. Launch Replica (Bootstraps mTLS identity using the Join Token)
# ==============================================================================
echo -e "\n\033[1;32m==> Starting Replica on :$REPLICA_PORT (Dynamic Join Enrollment)...\033[0m"
start_node replica "$REPLICA_PORT" \
APP_ROLE=replica \
APP_PRIMARY_URL="https://127.0.0.1:$PRIMARY_PORT" \
APP_JOIN_TOKEN="$JOIN_TOKEN"
wait_up "replica" "$REPLICA_PORT"
for idx in $(seq 1 $(( NODES - 1 ))); do
name=$(node_name "$idx")
port=$(node_port "$idx")
PORTS[$idx]=$port
NAMES[$idx]="$name"
# Verify credentials were saved to replica's SQLite database
if [[ -f "$WORK/replica/messages.db" ]]; then
echo -e " \033[1;32m✔ Replica enrolled and saved mTLS credentials in SQLite.\033[0m"
fi
case "$TOPO" in
chain)
# Replica pulls from the previous node in the chain
upstream_port=$(node_port $(( idx - 1 )))
upstream_url="https://127.0.0.1:$upstream_port"
upstream_name=$(node_name $(( idx - 1 )))
;;
star|*)
# Replica pulls directly from the primary
upstream_port=$PRIMARY_PORT
upstream_url="https://127.0.0.1:$PRIMARY_PORT"
upstream_name="primary"
;;
esac
# --- Launch Dashboard (if available) ---
if [[ "$START_DASHBOARD" -eq 1 && -f "$ROOT_DIR/dashboard.py" ]]; then
if check_port "$DASHBOARD_PORT"; then
echo -e "\033[1;33mDashboard port :$DASHBOARD_PORT is in use. Skipping dashboard.\033[0m"
else
echo -e "\033[1;32m==> Starting Dashboard on :$DASHBOARD_PORT\033[0m"
( cd "$ROOT_DIR" && exec python3 dashboard.py ) >"$WORK/dashboard.log" 2>&1 &
PIDS+=("$!")
SOURCES[$idx]="$upstream_url"
# The join token must come from the Cluster CA holder (always the primary)
color_echo blue "\n==> Minting join token for $name from primary..."
JOIN_TOKEN=$(mint_join_token "$PRIMARY_PORT")
color_echo green " ✔ Token: $JOIN_TOKEN"
color_echo green "\n==> [$idx/$(( NODES - 1 ))] Starting $name on :$port"
echo " Topology: $TOPO | upstream: $upstream_name ($upstream_url)"
# In chain topology the replication upstream differs from the CA issuer (primary).
# APP_JOIN_URL overrides enrollment so the new replica can get a cert from the
# actual primary even though it pulls data from an intermediate relay node.
start_node "$name" "$port" \
APP_ROLE=replica \
APP_PRIMARY_URL="$upstream_url" \
APP_JOIN_URL="https://127.0.0.1:$PRIMARY_PORT" \
APP_JOIN_TOKEN="$JOIN_TOKEN"
wait_up "$name" "$port"
color_echo green " ✔ $name is up and enrolled."
if [[ -f "$WORK/$name/messages.db" ]]; then
color_echo green " ✔ mTLS credentials persisted in SQLite."
fi
done
# ---------------------------------------------------------------------------
# Dashboard
# ---------------------------------------------------------------------------
if [[ "$START_DASHBOARD" -eq 1 && -f "$ROOT_DIR/dashboard.py" ]]; then
color_echo green "\n==> Starting Dashboard on :$DASHBOARD_PORT..."
( cd "$ROOT_DIR" && exec python3 dashboard.py ) >"$WORK/dashboard.log" 2>&1 &
PIDS+=("$!")
fi
# ==============================================================================
# 5. Write Smoke Test Records to Primary
# 4. Write smoke-test records to primary
# ==============================================================================
echo -e "\n\033[1;34m==> Writing 3 test records to Primary via HTTPS\033[0m"
for i in 1 2 3; do
RECORD_COUNT=3
color_echo blue "\n==> Writing $RECORD_COUNT test records to Primary..."
for i in $(seq 1 "$RECORD_COUNT"); do
curl -k -s -X POST "https://127.0.0.1:$PRIMARY_PORT/api/records" \
-H 'Content-Type: application/json' \
-d "{\"data\":{\"version\":1,\"msg\":\"hello $i\"}}" >/dev/null
echo " Record #$i written"
echo " Record #$i written"
done
# ==============================================================================
# 6. Await Replica mTLS Sync
# 5. Verify every replica has converged
# ==============================================================================
echo -e "\n\033[1;34m==> Verifying replica mTLS sync stream...\033[0m"
caught_up=0
for _ in $(seq 1 40); do
status_output=$(curl -k -s "https://127.0.0.1:$REPLICA_PORT/api/replication/status" || true)
if echo "$status_output" | grep -q '"last_applied_seq":3'; then
caught_up=1
break
color_echo blue "\n==> Waiting for all replicas to converge to seq $RECORD_COUNT..."
all_ok=1
for idx in $(seq 1 $(( NODES - 1 ))); do
name="${NAMES[$idx]}"
port="${PORTS[$idx]}"
echo -n " Checking $name (:$port)... "
if wait_replica_caught_up "$name" "$port" "$RECORD_COUNT"; then
color_echo green "✔ seq $RECORD_COUNT reached"
else
echo "✖ timeout"
all_ok=0
fi
done
if [[ $all_ok -eq 1 ]]; then
color_echo green "✔ All replicas are in sync."
else
color_echo yellow "⚠ Some replicas did not converge in time."
fi
# ==============================================================================
# 6. Read-only guard check on every replica
# ==============================================================================
color_echo blue "\n==> Verifying read-only guard on all replicas..."
for idx in $(seq 1 $(( NODES - 1 ))); do
name="${NAMES[$idx]}"
port="${PORTS[$idx]}"
http_code=$(curl -k -s -o /dev/null -w "%{http_code}" \
-X POST "https://127.0.0.1:$port/api/records" \
-H 'Content-Type: application/json' \
-d '{"data":{"version":1,"msg":"should be rejected"}}')
if [[ "$http_code" == "403" ]]; then
color_echo green " ✔ $name (:$port) → HTTP 403 Forbidden (correct)"
else
color_echo red " ✖ $name (:$port) → HTTP $http_code (expected 403)"
fi
sleep 0.25
done
if [[ $caught_up -eq 1 ]]; then
echo -e "\033[1;32m✔ SUCCESS: Replica caught up to seq 3 over mTLS!\033[0m"
else
echo -e "\033[1;31m✖ WARNING: Replica did not reach seq 3 in time.\033[0m"
echo "Replica Status Output: $status_output"
# ==============================================================================
# 7. Graceful switchover (primary -> first replica)
# Skipped if SKIP_SWITCHOVER=1 or only 1 replica exists (would leave no primary)
# ==============================================================================
if [[ "$SKIP_SWITCHOVER" -eq 0 && "$NODES" -ge 2 ]]; then
FIRST_REPLICA_PORT="${PORTS[1]}"
FIRST_REPLICA_NAME="${NAMES[1]}"
color_echo blue "\n==> Initiating Graceful Switchover: primary (:$PRIMARY_PORT) ─► $FIRST_REPLICA_NAME (:$FIRST_REPLICA_PORT)..."
switch_resp=$(curl -k -sf -X POST "https://127.0.0.1:$PRIMARY_PORT/api/admin/switchover" \
-H 'Content-Type: application/json' \
-d "{\"replica_url\":\"https://127.0.0.1:$FIRST_REPLICA_PORT\",\"timeout_secs\":15}" || true)
color_echo green " ✔ Switchover response: $switch_resp"
# Old primary is now replica – verify 403
http_code=$(curl -k -s -o /dev/null -w "%{http_code}" \
-X POST "https://127.0.0.1:$PRIMARY_PORT/api/records" \
-H 'Content-Type: application/json' \
-d '{"data":{"version":1,"msg":"should fail on demoted primary"}}')
if [[ "$http_code" == "403" ]]; then
color_echo green " ✔ Demoted primary returned HTTP 403 (read-only)"
else
color_echo red " ✖ Demoted primary returned HTTP $http_code (expected 403)"
fi
# Write to new primary
NEXT_SEQ=$(( RECORD_COUNT + 1 ))
color_echo blue "\n==> Writing record #$NEXT_SEQ to new primary ($FIRST_REPLICA_NAME :$FIRST_REPLICA_PORT)..."
write_resp=$(curl -k -sf -X POST "https://127.0.0.1:$FIRST_REPLICA_PORT/api/records" \
-H 'Content-Type: application/json' \
-d "{\"data\":{\"version\":1,\"msg\":\"post-switchover write\"}}" || true)
color_echo green " ✔ Created: $write_resp"
# After switchover in star topology, the other replicas still point to the old primary
# which now points to the new primary – they will converge via chaining.
# In chain topology they chain through naturally.
# Verify the demoted primary (now replica) catches up:
color_echo blue "\n==> Verifying all nodes converge to seq $NEXT_SEQ post-switchover..."
all_ok=1
for idx in $(seq 0 $(( NODES - 1 ))); do
port="${PORTS[$idx]}"
name="${NAMES[$idx]}"
if [[ $idx -eq 1 ]]; then continue; fi # new primary already has it
echo -n " Checking $name (:$port)... "
if wait_replica_caught_up "$name" "$port" "$NEXT_SEQ"; then
color_echo green "✔ seq $NEXT_SEQ reached"
else
echo "✖ timeout"
all_ok=0
fi
done
if [[ $all_ok -eq 1 ]]; then
color_echo green "✔ All nodes converged after switchover."
else
color_echo yellow "⚠ Some nodes did not converge after switchover."
fi
fi
# ==============================================================================
# 7. Verify Read-Only Safety Guard on Replica
# Ready Banner
# ==============================================================================
echo -e "\n\033[1;34m==> Testing Replica read-only safety guard:\033[0m"
http_code=$(curl -k -s -o /dev/null -w "%{http_code}" -X POST "https://127.0.0.1:$REPLICA_PORT/api/records" \
-H 'Content-Type: application/json' \
-d '{"data":{"version":1,"msg":"should fail"}}')
if [[ "$http_code" == "403" ]]; then
echo -e " \033[1;32m✔ HTTP 403 Forbidden received as expected.\033[0m"
else
echo -e " \033[1;31m✖ Expected HTTP 403, got $http_code.\033[0m"
echo ""
echo "======================================================================"
echo " Cluster is healthy • Topology: $TOPO • Nodes: $NODES"
echo ""
echo " Nodes:"
for idx in $(seq 0 $(( NODES - 1 ))); do
port="${PORTS[$idx]}"
name="${NAMES[$idx]}"
if [[ $idx -eq 0 ]]; then
echo " [$(( idx + 1 ))] Primary (:$port) https://127.0.0.1:$port"
else
src="${SOURCES[$idx]}"
echo " [$(( idx + 1 ))] Replica (:$port) https://127.0.0.1:$port <─ $src"
fi
done
echo ""
if [[ "$START_DASHBOARD" -eq 1 ]]; then
echo " Web Dashboard: http://127.0.0.1:$DASHBOARD_PORT"
fi
echo " Logs Dir: $WORK/"
echo ""
echo " Press Ctrl+C to stop all instances."
echo "======================================================================"
# --- Ready Banner ---
cat <<EOF
======================================================================
Cluster is healthy and running over mTLS:
- Primary API: https://127.0.0.1:$PRIMARY_PORT
- Replica API: https://127.0.0.1:$REPLICA_PORT
- Web Dashboard: http://127.0.0.1:$DASHBOARD_PORT
- Logs Dir: $WORK/
Press Ctrl+C to stop all instances.
======================================================================
EOF
wait
if [[ "$NO_WAIT" -ne 1 ]]; then
wait
fi