#!/usr/bin/env bash set -euo pipefail BASE_URL="${1:-http://127.0.0.1:8100}" API_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)" PYTHON_BIN="${PYTHON_BIN:-/opt/domaincheck/domainCheck/.venv/bin/python}" SIM_SCRIPT="$API_DIR/deploy/multi-region/simulate_cluster_node.py" PRUNE_SCRIPT="$API_DIR/deploy/multi-region/prune_cluster_nodes.sh" RUNTIME_DIR="$API_DIR/runtime" mkdir -p "$RUNTIME_DIR" CTRL_LOG="$RUNTIME_DIR/rehearse-mainland-controller.log" WORKER1_LOG="$RUNTIME_DIR/rehearse-mainland-worker-01.log" WORKER2_LOG="$RUNTIME_DIR/rehearse-mainland-worker-02.log" CTRL_NODE="mainland-controller-rehearsal" WORKER1_NODE="mainland-worker-rehearsal-01" WORKER2_NODE="mainland-worker-rehearsal-02" cleanup() { set +e if [ -n "${CTRL_PID:-}" ]; then kill "$CTRL_PID" >/dev/null 2>&1 || true; fi if [ -n "${WORKER1_PID:-}" ]; then kill "$WORKER1_PID" >/dev/null 2>&1 || true; fi if [ -n "${WORKER2_PID:-}" ]; then kill "$WORKER2_PID" >/dev/null 2>&1 || true; fi bash "$PRUNE_SCRIPT" --node-code "$CTRL_NODE" --node-code "$WORKER1_NODE" --node-code "$WORKER2_NODE" >/dev/null 2>&1 || true } trap cleanup EXIT echo "[0/6] cleaning possible stale rehearsal nodes" bash "$PRUNE_SCRIPT" --node-code "$CTRL_NODE" --node-code "$WORKER1_NODE" --node-code "$WORKER2_NODE" >/dev/null 2>&1 || true echo "[1/6] start simulated mainland controller" "$PYTHON_BIN" "$SIM_SCRIPT" \ --node-code "$CTRL_NODE" \ --region mainland \ --role control \ --status online \ --detail "多机演练 controller" \ --interval 15 >"$CTRL_LOG" 2>&1 & CTRL_PID=$! echo "[2/6] start simulated mainland workers" "$PYTHON_BIN" "$SIM_SCRIPT" \ --node-code "$WORKER1_NODE" \ --region mainland \ --role worker \ --status busy \ --current-load 1 \ --phase "running" \ --detail "多机演练 worker-01" \ --job-id 1 \ --job-code "detect-rehearsal-cluster" \ --cycle-token "rehearsal001" \ --active-threads 5 \ --available-proxy-count 12 \ --interval 15 >"$WORKER1_LOG" 2>&1 & WORKER1_PID=$! "$PYTHON_BIN" "$SIM_SCRIPT" \ --node-code "$WORKER2_NODE" \ --region mainland \ --role worker \ --status online \ --current-load 0 \ --phase "idle" \ --detail "多机演练 worker-02" \ --job-id 0 \ --job-code "" \ --cycle-token "" \ --active-threads 0 \ --available-proxy-count 8 \ --interval 15 >"$WORKER2_LOG" 2>&1 & WORKER2_PID=$! sleep 2 echo "[3/6] fetch runtime readiness" READINESS_JSON="$(curl -fsS "${BASE_URL}/api/v1/runtime/readiness")" echo "$READINESS_JSON" echo echo "[4/6] fetch runtime cluster" CLUSTER_JSON="$(curl -fsS "${BASE_URL}/api/v1/runtime/cluster")" echo "$CLUSTER_JSON" echo echo "[5/6] assert rehearsal result" "${PYTHON_BIN}" - <<'PY' "$READINESS_JSON" "$CLUSTER_JSON" "$CTRL_NODE" "$WORKER1_NODE" "$WORKER2_NODE" import json import sys readiness = json.loads(sys.argv[1]).get("data", {}) cluster = json.loads(sys.argv[2]).get("data", {}) ctrl_node, worker1_node, worker2_node = sys.argv[3:6] errors = [] if readiness.get("status") != "ready": errors.append(f"readiness.status expected ready, got {readiness.get('status')!r}") if not readiness.get("ready", False): errors.append("readiness.ready expected true") summary = cluster.get("summary") or {} if int(summary.get("online_control_nodes", 0) or 0) < 2: errors.append(f"online_control_nodes expected >= 2, got {summary.get('online_control_nodes')}") if int(summary.get("online_worker_nodes", 0) or 0) < 2: errors.append(f"online_worker_nodes expected >= 2, got {summary.get('online_worker_nodes')}") nodes = {str(item.get('node_code') or ''): item for item in (cluster.get("nodes") or [])} for node_code in (ctrl_node, worker1_node, worker2_node): if node_code not in nodes: errors.append(f"missing simulated node in cluster: {node_code}") if worker1_node in nodes and str(nodes[worker1_node].get("status")) != "busy": errors.append(f"{worker1_node} expected busy, got {nodes[worker1_node].get('status')!r}") if worker2_node in nodes and str(nodes[worker2_node].get("status")) != "online": errors.append(f"{worker2_node} expected online, got {nodes[worker2_node].get('status')!r}") if errors: print("rehearsal failed:") for item in errors: print(f"- {item}") raise SystemExit(1) print("rehearsal passed") print(json.dumps({ "readiness": readiness, "cluster_summary": summary, }, ensure_ascii=False, indent=2)) PY echo echo "[6/6] rehearsal completed" echo "controller log: $CTRL_LOG" echo "worker1 log: $WORKER1_LOG" echo "worker2 log: $WORKER2_LOG"