first

2026-04-16 21:35:47 +08:00
parent ff32aa50bf
commit ebf632e651
86 changed files with 14097 additions and 585 deletions
--- a/domain-api/deploy/multi-region/rehearse_multi_region.sh
+++ b/domain-api/deploy/multi-region/rehearse_multi_region.sh
@@ -0,0 +1,135 @@
+#!/usr/bin/env bash
+set -euo pipefail
+
+BASE_URL="${1:-http://127.0.0.1:8100}"
+API_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")/../.." && pwd)"
+PYTHON_BIN="${PYTHON_BIN:-/opt/domaincheck/domainCheck/.venv/bin/python}"
+SIM_SCRIPT="$API_DIR/deploy/multi-region/simulate_cluster_node.py"
+PRUNE_SCRIPT="$API_DIR/deploy/multi-region/prune_cluster_nodes.sh"
+RUNTIME_DIR="$API_DIR/runtime"
+mkdir -p "$RUNTIME_DIR"
+
+CTRL_LOG="$RUNTIME_DIR/rehearse-mainland-controller.log"
+WORKER1_LOG="$RUNTIME_DIR/rehearse-mainland-worker-01.log"
+WORKER2_LOG="$RUNTIME_DIR/rehearse-mainland-worker-02.log"
+
+CTRL_NODE="mainland-controller-rehearsal"
+WORKER1_NODE="mainland-worker-rehearsal-01"
+WORKER2_NODE="mainland-worker-rehearsal-02"
+
+cleanup() {
+  set +e
+  if [ -n "${CTRL_PID:-}" ]; then kill "$CTRL_PID" >/dev/null 2>&1 || true; fi
+  if [ -n "${WORKER1_PID:-}" ]; then kill "$WORKER1_PID" >/dev/null 2>&1 || true; fi
+  if [ -n "${WORKER2_PID:-}" ]; then kill "$WORKER2_PID" >/dev/null 2>&1 || true; fi
+  bash "$PRUNE_SCRIPT" --node-code "$CTRL_NODE" --node-code "$WORKER1_NODE" --node-code "$WORKER2_NODE" >/dev/null 2>&1 || true
+}
+trap cleanup EXIT
+
+echo "[0/6] cleaning possible stale rehearsal nodes"
+bash "$PRUNE_SCRIPT" --node-code "$CTRL_NODE" --node-code "$WORKER1_NODE" --node-code "$WORKER2_NODE" >/dev/null 2>&1 || true
+
+echo "[1/6] start simulated mainland controller"
+"$PYTHON_BIN" "$SIM_SCRIPT" \
+  --node-code "$CTRL_NODE" \
+  --region mainland \
+  --role control \
+  --status online \
+  --detail "多机演练 controller" \
+  --interval 15 >"$CTRL_LOG" 2>&1 &
+CTRL_PID=$!
+
+echo "[2/6] start simulated mainland workers"
+"$PYTHON_BIN" "$SIM_SCRIPT" \
+  --node-code "$WORKER1_NODE" \
+  --region mainland \
+  --role worker \
+  --status busy \
+  --current-load 1 \
+  --phase "running" \
+  --detail "多机演练 worker-01" \
+  --job-id 1 \
+  --job-code "detect-rehearsal-cluster" \
+  --cycle-token "rehearsal001" \
+  --active-threads 5 \
+  --available-proxy-count 12 \
+  --interval 15 >"$WORKER1_LOG" 2>&1 &
+WORKER1_PID=$!
+
+"$PYTHON_BIN" "$SIM_SCRIPT" \
+  --node-code "$WORKER2_NODE" \
+  --region mainland \
+  --role worker \
+  --status online \
+  --current-load 0 \
+  --phase "idle" \
+  --detail "多机演练 worker-02" \
+  --job-id 0 \
+  --job-code "" \
+  --cycle-token "" \
+  --active-threads 0 \
+  --available-proxy-count 8 \
+  --interval 15 >"$WORKER2_LOG" 2>&1 &
+WORKER2_PID=$!
+
+sleep 2
+
+echo "[3/6] fetch runtime readiness"
+READINESS_JSON="$(curl -fsS "${BASE_URL}/api/v1/runtime/readiness")"
+echo "$READINESS_JSON"
+echo
+
+echo "[4/6] fetch runtime cluster"
+CLUSTER_JSON="$(curl -fsS "${BASE_URL}/api/v1/runtime/cluster")"
+echo "$CLUSTER_JSON"
+echo
+
+echo "[5/6] assert rehearsal result"
+"${PYTHON_BIN}" - <<'PY' "$READINESS_JSON" "$CLUSTER_JSON" "$CTRL_NODE" "$WORKER1_NODE" "$WORKER2_NODE"
+import json
+import sys
+
+readiness = json.loads(sys.argv[1]).get("data", {})
+cluster = json.loads(sys.argv[2]).get("data", {})
+ctrl_node, worker1_node, worker2_node = sys.argv[3:6]
+
+errors = []
+if readiness.get("status") != "ready":
+    errors.append(f"readiness.status expected ready, got {readiness.get('status')!r}")
+if not readiness.get("ready", False):
+    errors.append("readiness.ready expected true")
+
+summary = cluster.get("summary") or {}
+if int(summary.get("online_control_nodes", 0) or 0) < 2:
+    errors.append(f"online_control_nodes expected >= 2, got {summary.get('online_control_nodes')}")
+if int(summary.get("online_worker_nodes", 0) or 0) < 2:
+    errors.append(f"online_worker_nodes expected >= 2, got {summary.get('online_worker_nodes')}")
+
+nodes = {str(item.get('node_code') or ''): item for item in (cluster.get("nodes") or [])}
+for node_code in (ctrl_node, worker1_node, worker2_node):
+    if node_code not in nodes:
+        errors.append(f"missing simulated node in cluster: {node_code}")
+
+if worker1_node in nodes and str(nodes[worker1_node].get("status")) != "busy":
+    errors.append(f"{worker1_node} expected busy, got {nodes[worker1_node].get('status')!r}")
+if worker2_node in nodes and str(nodes[worker2_node].get("status")) != "online":
+    errors.append(f"{worker2_node} expected online, got {nodes[worker2_node].get('status')!r}")
+
+if errors:
+    print("rehearsal failed:")
+    for item in errors:
+        print(f"- {item}")
+    raise SystemExit(1)
+
+print("rehearsal passed")
+print(json.dumps({
+    "readiness": readiness,
+    "cluster_summary": summary,
+}, ensure_ascii=False, indent=2))
+PY
+
+echo
+echo "[6/6] rehearsal completed"
+echo "controller log: $CTRL_LOG"
+echo "worker1 log: $WORKER1_LOG"
+echo "worker2 log: $WORKER2_LOG"