Files
getDomain/domain-api/deploy/multi-region/check_ops_plane.sh
2026-04-18 23:52:51 +08:00

610 lines
24 KiB
Bash
Raw Permalink Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
#!/usr/bin/env bash
set -euo pipefail
BASE_URL="${1:-http://127.0.0.1:8100}"
CURL_CONNECT_TIMEOUT="${CURL_CONNECT_TIMEOUT:-3}"
CURL_MAX_TIME="${CURL_MAX_TIME:-10}"
curl_json() {
curl -fsS \
--connect-timeout "${CURL_CONNECT_TIMEOUT}" \
--max-time "${CURL_MAX_TIME}" \
"$1"
}
curl_post_json() {
local url="$1"
local body="$2"
curl -fsS \
--connect-timeout "${CURL_CONNECT_TIMEOUT}" \
--max-time "${CURL_MAX_TIME}" \
-X POST "${url}" \
-H 'Content-Type: application/json' \
-d "${body}"
}
OVERVIEW_JSON="$(curl_json "${BASE_URL}/api/v1/ops/overview")"
NODES_JSON="$(curl_json "${BASE_URL}/api/v1/ops/nodes")"
JOBS_JSON="$(curl_json "${BASE_URL}/api/v1/ops/jobs")"
ACTIVITY_JSON="$(curl_json "${BASE_URL}/api/v1/ops/activity-stream")"
LAUNCHPAD_JSON="$(curl_json "${BASE_URL}/api/v1/ops/releases/launchpad")"
LATEST_RELEASE_JSON="$(curl_json "${BASE_URL}/api/v1/ops/releases/latest")"
POLICY_PREVIEW_JSON="$(
curl_post_json "${BASE_URL}/api/v1/ops/policy/preview" \
'{"action":"runtime.restart_api","target_type":"node","target_node_code":"mainland-controller-01"}'
)"
echo "[1/12] ops overview"
printf '%s\n\n' "${OVERVIEW_JSON}"
echo "[2/12] top recommendation"
python3 - <<'PY' "${OVERVIEW_JSON}"
import json
import sys
payload = json.loads(sys.argv[1])
data = payload.get("data") or {}
recommendation = dict(data.get("recommendation") or {})
print(json.dumps({
"source": str(recommendation.get("source") or ""),
"key": str(recommendation.get("key") or ""),
"priority": str(recommendation.get("priority") or ""),
"summary": str(recommendation.get("summary") or ""),
"primary_action_code": str(recommendation.get("primary_action_code") or ""),
"secondary_action_code": str(recommendation.get("secondary_action_code") or ""),
"node_codes": list(recommendation.get("node_codes") or []),
}, ensure_ascii=False, indent=2))
PY
echo
echo "[3/12] driver recommendations (condensed)"
python3 - <<'PY' "${OVERVIEW_JSON}"
import json
import sys
payload = json.loads(sys.argv[1])
data = payload.get("data") or {}
recommendations = list(data.get("driver_recommendations") or [])
summary = {
"total": len(recommendations),
"items": [
{
"key": str(item.get("key") or ""),
"title": str(item.get("title") or ""),
"level_label": str(item.get("level_label") or ""),
"primary_action_code": str(item.get("primary_action_code") or ""),
"secondary_action_code": str(item.get("secondary_action_code") or ""),
"meta_text": str(item.get("meta_text") or ""),
}
for item in recommendations[:8]
],
}
print(json.dumps(summary, ensure_ascii=False, indent=2))
PY
echo
echo "[4/12] ops capabilities"
curl_json "${BASE_URL}/api/v1/ops/capabilities"
echo
echo
echo "[5/12] ops blueprint"
curl_json "${BASE_URL}/api/v1/ops/blueprint"
echo
echo
echo "[6/12] ops runbook"
curl_json "${BASE_URL}/api/v1/ops/runbook"
echo
echo
echo "[7/12] ops nodes"
printf '%s\n' "${NODES_JSON}"
echo
echo
echo "[8/12] ops jobs"
printf '%s\n' "${JOBS_JSON}"
echo
echo
echo "[9/12] ops activity stream"
printf '%s\n' "${ACTIVITY_JSON}"
echo
echo
echo "[10/12] release launchpad"
printf '%s\n' "${LAUNCHPAD_JSON}"
echo
echo
echo "[11/12] latest release"
printf '%s\n' "${LATEST_RELEASE_JSON}"
echo
echo
echo "[12/12] policy preview sample"
printf '%s\n' "${POLICY_PREVIEW_JSON}"
echo
echo
echo "[13/13] condensed summary"
python3 - <<'PY' \
"${BASE_URL}" \
"${OVERVIEW_JSON}" \
"${NODES_JSON}" \
"${JOBS_JSON}" \
"${ACTIVITY_JSON}" \
"${LAUNCHPAD_JSON}" \
"${LATEST_RELEASE_JSON}" \
"${POLICY_PREVIEW_JSON}"
import json
import sys
def as_dict(payload):
try:
raw = json.loads(payload)
except Exception:
return {}
return raw.get("data") or {}
def ensure_dict(value):
return value if isinstance(value, dict) else {}
def ensure_list(value):
return value if isinstance(value, list) else []
def ensure_str(value):
return value if isinstance(value, str) else ""
def command(name, *parts):
tokens = [
"bash",
"domain-api/deploy/multi-region/drive_ops_center.sh",
name,
base_url,
]
tokens.extend([str(part) for part in parts if str(part or "").strip()])
return " ".join(tokens)
def release_command(name, *parts):
tokens = [
"bash",
"domain-api/deploy/multi-region/drive_release_hub.sh",
base_url,
name,
]
tokens.extend([str(part) for part in parts if str(part or "").strip()])
return " ".join(tokens)
def build_operator_decision():
remote_access_state_counts = ensure_dict(node_summary.get("remote_access_state_counts"))
pending_bootstrap = int(node_summary.get("pending_bootstrap", 0) or 0)
runtime_only = int(node_summary.get("runtime_only", 0) or 0)
online_total = int(node_summary.get("online", 0) or 0)
participating_total = int(scene_log.get("participating_node_count", 0) or 0)
launchpad_blocked = bool(launchpad_workers.get("blocked")) or bool(launchpad_controls.get("blocked"))
running_total = int(jobs_summary.get("running_count", 0) or 0)
queued_total = int(jobs_summary.get("queued_count", 0) or 0)
if scene_log.get("status") == "disabled" and participating_total > 0:
return {
"lane": "observability",
"priority": "high",
"reason_code": "scene_log_disabled",
"title": "先恢复远端现场日志可见性",
"summary": ensure_str(scene_log.get("summary")),
"next_focus": ensure_dict(scene_log.get("focus_ref")),
"primary_command_key": "scene_log",
"secondary_command_key": "driver_focus_preview",
}
if pending_bootstrap > 0 or remote_access_state_counts:
return {
"lane": "node_handover",
"priority": "high",
"reason_code": "managed_nodes_not_ready",
"title": "先补齐节点接管与远程可达性",
"summary": (
f"当前 managed node 共 {int(node_summary.get('managed_total', 0) or 0)} 台,"
f"pending_bootstrap={pending_bootstrap}runtime_only={runtime_only}。"
),
"next_focus": {
"kind": "managed_node_handover",
"node_code": first_attention_node_code,
} if first_attention_node_code else {},
"primary_command_key": "node_handover",
"secondary_command_key": "node_bootstrap_plan",
}
if launchpad_blocked or str(launchpad_status.get("status") or "") == "attention":
return {
"lane": "release",
"priority": "medium",
"reason_code": "release_launchpad_attention",
"title": "检查发布驾驶舱与 rollout 阻断",
"summary": ensure_str(launchpad_status.get("summary")),
"next_focus": release_focus_ref,
"primary_command_key": "release_launchpad",
"secondary_command_key": "release_preview_latest",
}
if running_total > 0 or queued_total > 0:
return {
"lane": "ops_jobs",
"priority": "medium",
"reason_code": "jobs_active",
"title": "继续盯运行中任务与活动流",
"summary": (
f"当前 running={running_total}queued={queued_total}"
f"activity_status={ensure_str(activity_summary.get('status')) or 'unknown'}。"
),
"next_focus": {
"kind": "ops_jobs",
"section": "activity_stream",
},
"primary_command_key": "jobs",
"secondary_command_key": "activity_preview",
}
return {
"lane": "steady",
"priority": "low",
"reason_code": "steady_state",
"title": "当前进入稳定观察态",
"summary": (
f"online={online_total}attention_nodes={len(attention_nodes)}"
f"scene_log_status={ensure_str(scene_log.get('status')) or 'standby'}。"
),
"next_focus": recommendation_focus_ref or ensure_dict(scene_log.get("focus_ref")),
"primary_command_key": "driver_feed",
"secondary_command_key": "activity_preview",
}
def normalize_scene_log_observation(overview_data):
driver_feed = ensure_dict(overview_data.get("driver_feed"))
scene_log_data = ensure_dict(driver_feed.get("scene_log_observation"))
if scene_log_data:
normalized = dict(scene_log_data)
normalized["status"] = ensure_str(normalized.get("status")) or "standby"
normalized["status_label"] = ensure_str(normalized.get("status_label")) or "待命"
normalized["summary"] = ensure_str(normalized.get("summary")) or "当前现场日志观察暂无额外结论。"
normalized["focus_ref"] = ensure_dict(normalized.get("focus_ref"))
normalized["target_node_code"] = ensure_str(normalized.get("target_node_code"))
normalized["participating_node_count"] = int(normalized.get("participating_node_count", 0) or 0)
normalized["covered_participating_node_count"] = int(normalized.get("covered_participating_node_count", 0) or 0)
normalized["line_count"] = int(normalized.get("line_count", 0) or 0)
return normalized
execution_scene = ensure_dict(overview_data.get("execution_scene"))
log_sync = ensure_dict(execution_scene.get("log_sync"))
participation = ensure_dict(execution_scene.get("participation_summary"))
missing_nodes = ensure_list(log_sync.get("missing_participating_nodes"))
participating_nodes = ensure_list(execution_scene.get("participating_nodes"))
target_node_code = ""
if missing_nodes:
target_node_code = ensure_str(missing_nodes[0])
elif participating_nodes:
target_node_code = ensure_str(ensure_dict(participating_nodes[0]).get("node_code"))
participating_total = int(
log_sync.get("participating_node_count", participation.get("participating_nodes", len(participating_nodes)))
or participation.get("participating_nodes", len(participating_nodes))
or len(participating_nodes)
or 0
)
covered_total = int(log_sync.get("covered_participating_node_count", 0) or 0)
line_count = int(log_sync.get("line_count", 0) or 0)
enabled = bool(log_sync.get("enabled", False))
mode = ensure_str(log_sync.get("mode")) or "key"
status = "standby"
status_label = "待命"
summary = "当前没有参与检测节点,现场日志观察处于待命状态。"
if participating_total > 0 and not enabled:
status = "disabled"
status_label = "未开启"
summary = (
f"当前有 {participating_total} 台参与节点,但远端日志回传仍关闭,"
"海外控制面还看不到节点级现场日志。"
)
elif participating_total > 0 and (
line_count <= 0 or covered_total <= 0 or int(log_sync.get("missing_participating_node_count", len(missing_nodes)) or len(missing_nodes)) > 0
):
status = "waiting_sample"
status_label = "等待样本"
summary = (
f"远端日志回传已经开启,但参与节点只覆盖 {covered_total}/{participating_total}"
f"样本 {line_count} 条,仍需继续观察或下钻节点现场日志。"
)
elif participating_total > 0:
status = "ready"
status_label = "可下钻"
summary = (
f"远端日志回传已经形成样本,当前覆盖 {covered_total}/{participating_total} 台参与节点,"
"可以直接下钻节点现场日志。"
)
focus_ref = {}
if target_node_code:
focus_ref = {
"kind": "node_scene_log",
"node_code": target_node_code,
"mode": mode,
"limit": 120 if status == "waiting_sample" else 80,
}
return {
"status": status,
"status_label": status_label,
"summary": summary,
"focus_ref": focus_ref,
"target_node_code": target_node_code,
"participating_node_count": participating_total,
"covered_participating_node_count": covered_total,
"line_count": line_count,
}
base_url = ensure_str(sys.argv[1]) or "http://127.0.0.1:8100"
overview = as_dict(sys.argv[2])
nodes = as_dict(sys.argv[3])
jobs = as_dict(sys.argv[4])
activity = as_dict(sys.argv[5])
launchpad = as_dict(sys.argv[6])
latest_release = as_dict(sys.argv[7])
policy_preview = as_dict(sys.argv[8])
recommendation = ensure_dict(overview.get("recommendation"))
scene_log = normalize_scene_log_observation(overview)
managed_nodes = ensure_list(nodes.get("managed_nodes"))
node_summary = ensure_dict(nodes.get("summary"))
jobs_summary = ensure_dict(jobs.get("summary"))
activity_summary = ensure_dict(activity.get("summary"))
launchpad_release = ensure_dict(launchpad.get("release"))
launchpad_rec = ensure_dict(launchpad.get("top_recommendation"))
launchpad_workers = ensure_dict(launchpad.get("worker_rollout_preview"))
launchpad_controls = ensure_dict(launchpad.get("control_rollout_preview"))
launchpad_status = ensure_dict(launchpad.get("launchpad_status"))
latest_release_entry = latest_release if isinstance(latest_release, dict) else {}
latest_package_preview = ensure_dict(launchpad.get("latest_package_preview"))
online_nodes = [
node for node in managed_nodes
if str(node.get("status") or "").lower() in {"online", "busy"}
]
attention_nodes = [
{
"node_code": str(node.get("node_code") or ""),
"status": str(node.get("status") or ""),
"role": str(node.get("role") or ""),
"stack_status_label": str(node.get("stack_status_label") or ""),
"recommended_action_code": str(node.get("recommended_action_code") or ""),
"recommended_action_label": str(node.get("recommended_action_label") or ""),
"primary_issue": str(node.get("primary_issue") or ""),
}
for node in managed_nodes
if str(node.get("status") or "").lower() not in {"online", "busy"}
or str(node.get("stack_status") or "").lower() not in {"healthy", "ready", ""}
]
activity_items = ensure_list(activity.get("items"))
recent_activity = [
{
"kind": str(item.get("kind") or ""),
"title": str(item.get("title") or ""),
"level": str(item.get("level") or ""),
"created_at": str(item.get("created_at") or ""),
}
for item in activity_items[:5]
]
job_items = ensure_list(jobs.get("items"))
running_jobs = [
{
"job_key": str(item.get("job_key") or ""),
"status": str(item.get("status") or ""),
"title": str(item.get("title") or ""),
"progress_text": str(item.get("progress_text") or ""),
}
for item in job_items
if str(item.get("status") or "").lower() in {"running", "queued", "waiting"}
][:5]
recommendation_focus_ref = ensure_dict(recommendation.get("focus_ref"))
first_attention = attention_nodes[0] if attention_nodes else {}
first_attention_node_code = ensure_str(first_attention.get("node_code"))
scene_log_target_node_code = ensure_str(scene_log.get("target_node_code"))
latest_release_version = str(latest_release_entry.get("version") or latest_release_entry.get("release_version") or "")
release_focus_ref = ensure_dict(launchpad_rec.get("focus_ref"))
release_id = str(release_focus_ref.get("release_id") or "")
operator_decision = build_operator_decision()
primary_command_key = ensure_str(operator_decision.get("primary_command_key"))
secondary_command_key = ensure_str(operator_decision.get("secondary_command_key"))
result = {
"overview": {
"top_recommendation": {
"source": str(recommendation.get("source") or ""),
"key": str(recommendation.get("key") or ""),
"priority": str(recommendation.get("priority") or ""),
"summary": str(recommendation.get("summary") or ""),
"primary_action_code": str(recommendation.get("primary_action_code") or ""),
"secondary_action_code": str(recommendation.get("secondary_action_code") or ""),
"focus_ref": recommendation.get("focus_ref") or {},
"node_codes": list(recommendation.get("node_codes") or []),
},
"scene_log_observation": {
"status": str(scene_log.get("status") or ""),
"status_label": str(scene_log.get("status_label") or ""),
"summary": str(scene_log.get("summary") or ""),
"target_node_code": str(scene_log.get("target_node_code") or ""),
"participating_node_count": int(scene_log.get("participating_node_count") or 0),
"covered_participating_node_count": int(scene_log.get("covered_participating_node_count") or 0),
"line_count": int(scene_log.get("line_count") or 0),
"focus_ref": scene_log.get("focus_ref") or {},
},
},
"nodes": {
"managed_total": int(node_summary.get("managed_total") or len(managed_nodes)),
"online_or_busy_total": len(online_nodes),
"offline_total": int(node_summary.get("offline_count") or 0),
"attention_total": len(attention_nodes),
"attention_nodes": attention_nodes[:8],
},
"jobs": {
"running_total": int(jobs_summary.get("running_count") or 0),
"queued_total": int(jobs_summary.get("queued_count") or 0),
"failed_total": int(jobs_summary.get("failed_count") or 0),
"recent_running_jobs": running_jobs,
},
"activity_stream": {
"total": int(activity_summary.get("total") or len(activity_items)),
"error_count": int(activity_summary.get("error_count") or 0),
"warning_count": int(activity_summary.get("warning_count") or 0),
"running_job_count": int(activity_summary.get("running_job_count") or 0),
"recent_items": recent_activity,
},
"operator_decision": {
"lane": ensure_str(operator_decision.get("lane")),
"priority": ensure_str(operator_decision.get("priority")),
"reason_code": ensure_str(operator_decision.get("reason_code")),
"title": ensure_str(operator_decision.get("title")),
"summary": ensure_str(operator_decision.get("summary")),
"next_focus": ensure_dict(operator_decision.get("next_focus")),
"primary_command_key": primary_command_key,
"secondary_command_key": secondary_command_key,
},
"recommended_commands": {
"stack_diagnosis": "bash domain-api/deploy/multi-region/check_ops_center_stack.sh",
"driver_feed": command("driver-feed"),
"activity_preview": command("activity-preview"),
"driver_focus_preview": command("driver-focus-preview"),
"driver_focus_run": command("driver-focus-run"),
"top_recommendation_preview": (
command("driver-preview", str(recommendation.get("primary_action_code") or ""))
if str(recommendation.get("primary_action_code") or "").strip()
else ""
),
"top_recommendation_focus_preview": (
command("driver-focus-preview", f"entry:{str(recommendation.get('key') or '')}")
if str(recommendation.get("key") or "").strip()
else ""
),
"scene_log": (
command("scene-node-log", scene_log_target_node_code, "120", "key")
if scene_log_target_node_code
else ""
),
"scene_log_direct": (
f"bash domain-api/deploy/multi-region/check_node_scene_log.sh {base_url} {scene_log_target_node_code} 120 key"
if scene_log_target_node_code
else ""
),
"node_handover": (
command("node-handover", first_attention_node_code)
if first_attention_node_code
else command("stack-first-gap-handover")
),
"node_bootstrap_plan": (
command("node-bootstrap-plan", first_attention_node_code)
if first_attention_node_code
else ""
),
"delivery_queue": (
command("delivery-queue", first_attention_node_code)
if first_attention_node_code
else ""
),
"release_launchpad": release_command("launchpad"),
"release_preview_latest": release_command("preview-latest"),
"release_preview_latest_worker": release_command("preview-latest-smart", "worker"),
"release_preview_latest_control": release_command("preview-latest-smart", "control"),
"release_focus": (
command("driver-preview", "focus_release_hub")
if str(launchpad_rec.get("action_code") or "").strip()
else ""
),
"release_preview_selected": (
release_command("preview-release", release_id, "worker")
if release_id
else ""
),
"jobs": command("activity-stream", "kind=ops_job"),
},
"next_actions": {
"primary_command": "",
"secondary_command": "",
},
"release_launchpad": {
"status": str(launchpad_status.get("status") or ""),
"status_label": str(launchpad_status.get("status_label") or ""),
"summary": str(launchpad_status.get("summary") or ""),
"recommended_action_code": str(launchpad_rec.get("action_code") or ""),
"recommended_action_label": str(launchpad_rec.get("action_label") or ""),
"recommended_execution_mode": str(launchpad_rec.get("execution_mode") or ""),
"recommended_execution_mode_label": str(launchpad_rec.get("execution_mode_label") or ""),
"focus_ref": launchpad_rec.get("focus_ref") or {},
"release_version": str(launchpad_release.get("version") or launchpad_release.get("release_version") or ""),
"worker_rollout_preview": {
"available": bool(launchpad_workers.get("available")),
"blocked": bool(launchpad_workers.get("blocked")),
"execution_mode": str(launchpad_workers.get("execution_mode") or ""),
"execution_mode_label": str(launchpad_workers.get("execution_mode_label") or ""),
"target_node_codes": list(launchpad_workers.get("target_node_codes") or []),
},
"control_rollout_preview": {
"available": bool(launchpad_controls.get("available")),
"blocked": bool(launchpad_controls.get("blocked")),
"execution_mode": str(launchpad_controls.get("execution_mode") or ""),
"execution_mode_label": str(launchpad_controls.get("execution_mode_label") or ""),
"target_node_codes": list(launchpad_controls.get("target_node_codes") or []),
},
},
"latest_release": {
"version": str(latest_release_entry.get("version") or latest_release_entry.get("release_version") or ""),
"status": ensure_str(latest_release_entry.get("status")),
"status_label": ensure_str(latest_release_entry.get("status_label")),
"summary": ensure_str(latest_release_entry.get("summary") or latest_release_entry.get("summary_text")),
},
"policy_preview_sample": {
"execution_mode": str(policy_preview.get("execution_mode") or ""),
"risk_level": str(policy_preview.get("risk_level") or ""),
"approval_required": bool(policy_preview.get("approval_required")),
"blocked": bool(policy_preview.get("blocked")),
"blocking_reasons": ensure_list(policy_preview.get("blocking_reasons"))[:5],
"warnings": ensure_list(policy_preview.get("warnings"))[:5],
},
"focus_refs": {
"top_recommendation": recommendation_focus_ref,
"scene_log": ensure_dict(scene_log.get("focus_ref")),
"release_hub": release_focus_ref,
},
"rollout_targets": {
"worker": list(launchpad_workers.get("target_node_codes") or []),
"control": list(launchpad_controls.get("target_node_codes") or []),
},
"package_preview": {
"available": bool(latest_package_preview.get("release")),
"preview_source": ensure_str(latest_package_preview.get("preview_source")),
"message": ensure_str(latest_package_preview.get("message")),
},
}
result["next_actions"]["primary_command"] = ensure_str(
ensure_dict(result.get("recommended_commands")).get(primary_command_key)
)
result["next_actions"]["secondary_command"] = ensure_str(
ensure_dict(result.get("recommended_commands")).get(secondary_command_key)
)
print(json.dumps(result, ensure_ascii=False, indent=2))
PY
echo