136 lines
4.4 KiB
Bash
Executable File
136 lines
4.4 KiB
Bash
Executable File
#!/usr/bin/env bash
|
|
set -euo pipefail
|
|
|
|
BASE_URL="${1:-http://127.0.0.1:8100}"
|
|
API_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
|
|
PYTHON_BIN="${PYTHON_BIN:-/opt/domaincheck/domainCheck/.venv/bin/python}"
|
|
SIM_SCRIPT="$API_DIR/deploy/multi-region/simulate_cluster_node.py"
|
|
PRUNE_SCRIPT="$API_DIR/deploy/multi-region/prune_cluster_nodes.sh"
|
|
RUNTIME_DIR="$API_DIR/runtime"
|
|
mkdir -p "$RUNTIME_DIR"
|
|
|
|
CTRL_LOG="$RUNTIME_DIR/rehearse-mainland-controller.log"
|
|
WORKER1_LOG="$RUNTIME_DIR/rehearse-mainland-worker-01.log"
|
|
WORKER2_LOG="$RUNTIME_DIR/rehearse-mainland-worker-02.log"
|
|
|
|
CTRL_NODE="mainland-controller-rehearsal"
|
|
WORKER1_NODE="mainland-worker-rehearsal-01"
|
|
WORKER2_NODE="mainland-worker-rehearsal-02"
|
|
|
|
cleanup() {
|
|
set +e
|
|
if [ -n "${CTRL_PID:-}" ]; then kill "$CTRL_PID" >/dev/null 2>&1 || true; fi
|
|
if [ -n "${WORKER1_PID:-}" ]; then kill "$WORKER1_PID" >/dev/null 2>&1 || true; fi
|
|
if [ -n "${WORKER2_PID:-}" ]; then kill "$WORKER2_PID" >/dev/null 2>&1 || true; fi
|
|
bash "$PRUNE_SCRIPT" --node-code "$CTRL_NODE" --node-code "$WORKER1_NODE" --node-code "$WORKER2_NODE" >/dev/null 2>&1 || true
|
|
}
|
|
trap cleanup EXIT
|
|
|
|
echo "[0/6] cleaning possible stale rehearsal nodes"
|
|
bash "$PRUNE_SCRIPT" --node-code "$CTRL_NODE" --node-code "$WORKER1_NODE" --node-code "$WORKER2_NODE" >/dev/null 2>&1 || true
|
|
|
|
echo "[1/6] start simulated mainland controller"
|
|
"$PYTHON_BIN" "$SIM_SCRIPT" \
|
|
--node-code "$CTRL_NODE" \
|
|
--region mainland \
|
|
--role control \
|
|
--status online \
|
|
--detail "多机演练 controller" \
|
|
--interval 15 >"$CTRL_LOG" 2>&1 &
|
|
CTRL_PID=$!
|
|
|
|
echo "[2/6] start simulated mainland workers"
|
|
"$PYTHON_BIN" "$SIM_SCRIPT" \
|
|
--node-code "$WORKER1_NODE" \
|
|
--region mainland \
|
|
--role worker \
|
|
--status busy \
|
|
--current-load 1 \
|
|
--phase "running" \
|
|
--detail "多机演练 worker-01" \
|
|
--job-id 1 \
|
|
--job-code "detect-rehearsal-cluster" \
|
|
--cycle-token "rehearsal001" \
|
|
--active-threads 5 \
|
|
--available-proxy-count 12 \
|
|
--interval 15 >"$WORKER1_LOG" 2>&1 &
|
|
WORKER1_PID=$!
|
|
|
|
"$PYTHON_BIN" "$SIM_SCRIPT" \
|
|
--node-code "$WORKER2_NODE" \
|
|
--region mainland \
|
|
--role worker \
|
|
--status online \
|
|
--current-load 0 \
|
|
--phase "idle" \
|
|
--detail "多机演练 worker-02" \
|
|
--job-id 0 \
|
|
--job-code "" \
|
|
--cycle-token "" \
|
|
--active-threads 0 \
|
|
--available-proxy-count 8 \
|
|
--interval 15 >"$WORKER2_LOG" 2>&1 &
|
|
WORKER2_PID=$!
|
|
|
|
sleep 2
|
|
|
|
echo "[3/6] fetch runtime readiness"
|
|
READINESS_JSON="$(curl -fsS "${BASE_URL}/api/v1/runtime/readiness")"
|
|
echo "$READINESS_JSON"
|
|
echo
|
|
|
|
echo "[4/6] fetch runtime cluster"
|
|
CLUSTER_JSON="$(curl -fsS "${BASE_URL}/api/v1/runtime/cluster")"
|
|
echo "$CLUSTER_JSON"
|
|
echo
|
|
|
|
echo "[5/6] assert rehearsal result"
|
|
"${PYTHON_BIN}" - <<'PY' "$READINESS_JSON" "$CLUSTER_JSON" "$CTRL_NODE" "$WORKER1_NODE" "$WORKER2_NODE"
|
|
import json
|
|
import sys
|
|
|
|
readiness = json.loads(sys.argv[1]).get("data", {})
|
|
cluster = json.loads(sys.argv[2]).get("data", {})
|
|
ctrl_node, worker1_node, worker2_node = sys.argv[3:6]
|
|
|
|
errors = []
|
|
if readiness.get("status") != "ready":
|
|
errors.append(f"readiness.status expected ready, got {readiness.get('status')!r}")
|
|
if not readiness.get("ready", False):
|
|
errors.append("readiness.ready expected true")
|
|
|
|
summary = cluster.get("summary") or {}
|
|
if int(summary.get("online_control_nodes", 0) or 0) < 2:
|
|
errors.append(f"online_control_nodes expected >= 2, got {summary.get('online_control_nodes')}")
|
|
if int(summary.get("online_worker_nodes", 0) or 0) < 2:
|
|
errors.append(f"online_worker_nodes expected >= 2, got {summary.get('online_worker_nodes')}")
|
|
|
|
nodes = {str(item.get('node_code') or ''): item for item in (cluster.get("nodes") or [])}
|
|
for node_code in (ctrl_node, worker1_node, worker2_node):
|
|
if node_code not in nodes:
|
|
errors.append(f"missing simulated node in cluster: {node_code}")
|
|
|
|
if worker1_node in nodes and str(nodes[worker1_node].get("status")) != "busy":
|
|
errors.append(f"{worker1_node} expected busy, got {nodes[worker1_node].get('status')!r}")
|
|
if worker2_node in nodes and str(nodes[worker2_node].get("status")) != "online":
|
|
errors.append(f"{worker2_node} expected online, got {nodes[worker2_node].get('status')!r}")
|
|
|
|
if errors:
|
|
print("rehearsal failed:")
|
|
for item in errors:
|
|
print(f"- {item}")
|
|
raise SystemExit(1)
|
|
|
|
print("rehearsal passed")
|
|
print(json.dumps({
|
|
"readiness": readiness,
|
|
"cluster_summary": summary,
|
|
}, ensure_ascii=False, indent=2))
|
|
PY
|
|
|
|
echo
|
|
echo "[6/6] rehearsal completed"
|
|
echo "controller log: $CTRL_LOG"
|
|
echo "worker1 log: $WORKER1_LOG"
|
|
echo "worker2 log: $WORKER2_LOG"
|