1273 lines
57 KiB
Bash
1273 lines
57 KiB
Bash
#!/usr/bin/env bash
|
||
set -euo pipefail
|
||
|
||
BASE_URL="${1:-http://127.0.0.1:8100}"
|
||
VIEW_MODE="${2:-full}"
|
||
CURL_CONNECT_TIMEOUT="${CURL_CONNECT_TIMEOUT:-3}"
|
||
CURL_MAX_TIME="${CURL_MAX_TIME:-8}"
|
||
|
||
FETCH_BODY=""
|
||
FETCH_STATUS="000"
|
||
|
||
extract_api_data_json() {
|
||
local raw_json="${1:-}"
|
||
python3 - "${raw_json}" <<'PY'
|
||
import json
|
||
import sys
|
||
|
||
raw_json = str(sys.argv[1] or "")
|
||
try:
|
||
payload = json.loads(raw_json)
|
||
except Exception:
|
||
raise SystemExit(1)
|
||
|
||
data = payload.get("data")
|
||
if not isinstance(data, dict):
|
||
raise SystemExit(1)
|
||
|
||
print(json.dumps(data, ensure_ascii=False, indent=2))
|
||
PY
|
||
}
|
||
|
||
fetch_json() {
|
||
local url="${1:-}"
|
||
local raw_text
|
||
raw_text="$(curl -sS \
|
||
--connect-timeout "${CURL_CONNECT_TIMEOUT}" \
|
||
--max-time "${CURL_MAX_TIME}" \
|
||
"${url}" \
|
||
-w $'\nHTTP_STATUS=%{http_code}' || true)"
|
||
FETCH_BODY="${raw_text%$'\n'HTTP_STATUS=*}"
|
||
FETCH_STATUS="${raw_text##*$'\n'HTTP_STATUS=}"
|
||
}
|
||
|
||
fetch_json "${BASE_URL}/api/v1/ops/stack-diagnosis?base_url=${BASE_URL}"
|
||
STACK_DIAGNOSIS_JSON="${FETCH_BODY}"
|
||
STACK_DIAGNOSIS_STATUS="${FETCH_STATUS}"
|
||
if [[ "${STACK_DIAGNOSIS_STATUS}" == 2* ]]; then
|
||
if STACK_DIAGNOSIS_DATA_JSON="$(extract_api_data_json "${STACK_DIAGNOSIS_JSON}" 2>/dev/null)"; then
|
||
if [[ "${VIEW_MODE}" != "summary" ]]; then
|
||
echo "[1/1] ops stack diagnosis (api contract)"
|
||
printf 'status=%s\n%s\n\n' "${STACK_DIAGNOSIS_STATUS}" "${STACK_DIAGNOSIS_JSON}"
|
||
fi
|
||
echo "[9/9] condensed stack summary"
|
||
STACK_BASE_URL="${BASE_URL}" python3 - <<'PY' "${STACK_DIAGNOSIS_DATA_JSON}"
|
||
import json
|
||
import os
|
||
import sys
|
||
|
||
|
||
def ensure_dict(value):
|
||
return value if isinstance(value, dict) else {}
|
||
|
||
|
||
def ensure_list(value):
|
||
return value if isinstance(value, list) else []
|
||
|
||
|
||
def ensure_str(value):
|
||
return value if isinstance(value, str) else ""
|
||
|
||
|
||
def command(*parts: str) -> str:
|
||
tokens = ["bash", "domain-api/deploy/multi-region/drive_ops_center.sh"]
|
||
tokens.extend([str(part).strip() for part in parts if str(part).strip()])
|
||
return " ".join(tokens)
|
||
|
||
|
||
def release_command(*parts: str) -> str:
|
||
tokens = ["bash", "domain-api/deploy/multi-region/drive_release_hub.sh"]
|
||
tokens.extend([str(part).strip() for part in parts if str(part).strip()])
|
||
return " ".join(tokens)
|
||
|
||
|
||
payload = json.loads(str(sys.argv[1] or "{}"))
|
||
base_url = ensure_str(os.environ.get("STACK_BASE_URL")) or "http://127.0.0.1:8100"
|
||
diagnosis = ensure_dict(payload.get("diagnosis"))
|
||
overview = ensure_dict(payload.get("overview"))
|
||
managed_nodes = ensure_dict(payload.get("managed_nodes"))
|
||
release_hub = ensure_dict(payload.get("release_hub"))
|
||
|
||
issues = ensure_list(diagnosis.get("issues"))
|
||
recommended_actions = ensure_list(diagnosis.get("recommended_actions"))
|
||
next_step = ensure_dict(diagnosis.get("next_step"))
|
||
backend_operator_decision = ensure_dict(diagnosis.get("operator_decision"))
|
||
backend_next_actions = ensure_dict(diagnosis.get("next_actions"))
|
||
backend_recommended_commands = ensure_dict(diagnosis.get("recommended_commands"))
|
||
log_sync = ensure_dict(overview.get("log_sync"))
|
||
problem_nodes = ensure_list(managed_nodes.get("problem_nodes"))
|
||
next_step_focus_ref = ensure_dict(next_step.get("focus_ref"))
|
||
launchpad_target_node_code = ensure_str(diagnosis.get("launchpad_recommended_target_node_code")) or ensure_str(next_step_focus_ref.get("node_code"))
|
||
launchpad_recovery_label = ensure_str(diagnosis.get("launchpad_recommended_recovery_label"))
|
||
launchpad_recovery_summary = ensure_str(diagnosis.get("launchpad_recommended_recovery_summary")) or ensure_str(next_step.get("reason"))
|
||
handover_next_action_code = ensure_str(next_step.get("action_code"))
|
||
|
||
scene_log_target_node_code = ""
|
||
missing_nodes = ensure_list(log_sync.get("missing_sample_node_codes"))
|
||
if missing_nodes:
|
||
scene_log_target_node_code = ensure_str(missing_nodes[0])
|
||
elif problem_nodes:
|
||
scene_log_target_node_code = ensure_str(ensure_dict(problem_nodes[0]).get("node_code"))
|
||
|
||
recommended_commands = {
|
||
"stack_summary": f"bash domain-api/deploy/multi-region/check_ops_center_stack.sh {base_url} summary",
|
||
"doctor": command("doctor", base_url),
|
||
"driver_feed": command("driver-feed", base_url),
|
||
"driver_focus_preview": command("driver-focus-preview", base_url),
|
||
"driver_focus_run": command("driver-focus-run", base_url),
|
||
"go_live_recover": command("go-live-recover", base_url),
|
||
"log_sync_recover": command("log-sync-recover", base_url, "key"),
|
||
"log_sync_recover_full": command("log-sync-recover", base_url, "full"),
|
||
"agent_gap_check": command("agent-gap-check", base_url),
|
||
"agent_gap_recover": command("agent-gap-recover", base_url),
|
||
"api_restart": command("runtime-refresh-recover", base_url),
|
||
"node_onboarding": (
|
||
command("node-onboarding", base_url, ensure_str(ensure_dict(problem_nodes[0]).get("node_code")))
|
||
if problem_nodes and ensure_str(ensure_dict(problem_nodes[0]).get("node_code"))
|
||
else ""
|
||
),
|
||
"node_recover": (
|
||
command("node-recover", base_url, ensure_str(ensure_dict(problem_nodes[0]).get("node_code")))
|
||
if problem_nodes and ensure_str(ensure_dict(problem_nodes[0]).get("node_code"))
|
||
else ""
|
||
),
|
||
"node_bootstrap_preview": (
|
||
command("node-bootstrap-preview", base_url, ensure_str(ensure_dict(problem_nodes[0]).get("node_code")))
|
||
if problem_nodes and ensure_str(ensure_dict(problem_nodes[0]).get("node_code"))
|
||
else ""
|
||
),
|
||
"node_bootstrap_run": (
|
||
command("node-bootstrap-run", base_url, launchpad_target_node_code, "cli")
|
||
if launchpad_target_node_code
|
||
else command("node-bootstrap-run", base_url, ensure_str(ensure_dict(problem_nodes[0]).get("node_code")), "cli")
|
||
if problem_nodes and ensure_str(ensure_dict(problem_nodes[0]).get("node_code"))
|
||
else ""
|
||
),
|
||
"node_acceptance_run": (
|
||
command("node-acceptance-run", base_url, launchpad_target_node_code, "cli")
|
||
if launchpad_target_node_code
|
||
else command("node-acceptance-run", base_url, ensure_str(ensure_dict(problem_nodes[0]).get("node_code")), "cli")
|
||
if launchpad_target_node_code or (problem_nodes and ensure_str(ensure_dict(problem_nodes[0]).get("node_code")))
|
||
else ""
|
||
),
|
||
"node_acceptance_plan": (
|
||
command("node-acceptance-plan", base_url, ensure_str(ensure_dict(problem_nodes[0]).get("node_code")))
|
||
if problem_nodes and ensure_str(ensure_dict(problem_nodes[0]).get("node_code"))
|
||
else ""
|
||
),
|
||
"scene_log": (
|
||
command("scene-node-log", base_url, scene_log_target_node_code, "120", "key")
|
||
if scene_log_target_node_code
|
||
else ""
|
||
),
|
||
"release_launchpad": release_command(base_url, "launchpad"),
|
||
"release_preview_latest": release_command(base_url, "preview-latest"),
|
||
"jobs": command("activity-stream", base_url, "kind=ops_job"),
|
||
"node_handover": (
|
||
command("node-handover", base_url, ensure_str(ensure_dict(problem_nodes[0]).get("node_code")))
|
||
if problem_nodes and ensure_str(ensure_dict(problem_nodes[0]).get("node_code"))
|
||
else command("stack-first-gap-handover", base_url)
|
||
),
|
||
}
|
||
|
||
operator_decision = {
|
||
"lane": "steady",
|
||
"priority": "low",
|
||
"reason_code": "steady_state",
|
||
"title": "当前进入稳定观察态",
|
||
"summary": "当前总检没有发现新的阻断问题，可继续按既有链路观察与推进。",
|
||
"next_focus": ensure_dict(next_step.get("focus_ref")),
|
||
"primary_command_key": "driver_feed",
|
||
"secondary_command_key": "jobs",
|
||
}
|
||
|
||
participating_total = int(log_sync.get("participating_nodes_total", 0) or 0)
|
||
remote_access_ready = int(managed_nodes.get("remote_access_ready", 0) or 0)
|
||
managed_enabled = int(managed_nodes.get("managed_enabled", 0) or 0)
|
||
default_rollout_gate_status = ensure_str(release_hub.get("default_rollout_gate_status"))
|
||
|
||
if not bool(log_sync.get("enabled")) and participating_total > 0:
|
||
operator_decision = {
|
||
"lane": "observability",
|
||
"priority": "high",
|
||
"reason_code": "scene_log_disabled",
|
||
"title": "先恢复远端现场日志可见性",
|
||
"summary": "当前已有参与检测节点，但远端日志回传仍关闭，海外控制面仍处于半盲态。",
|
||
"next_focus": {"kind": "node_scene_log", "node_code": scene_log_target_node_code, "mode": "key", "limit": 80}
|
||
if scene_log_target_node_code
|
||
else {},
|
||
"primary_command_key": "log_sync_recover",
|
||
"secondary_command_key": "scene_log",
|
||
}
|
||
elif managed_enabled > 0 and remote_access_ready == 0:
|
||
primary_command_key = "agent_gap_recover"
|
||
secondary_command_key = "node_handover"
|
||
title = "先补齐节点接管与 remote-agent 可达性"
|
||
summary = (
|
||
f"managed_enabled={managed_enabled}，remote_access_ready={remote_access_ready}；"
|
||
"当前还不能从海外控制面统一下发标准动作。"
|
||
)
|
||
if handover_next_action_code == "bootstrap_run" and launchpad_target_node_code:
|
||
primary_command_key = "node_bootstrap_run"
|
||
secondary_command_key = "node_handover"
|
||
title = "先跑首台节点接入收口"
|
||
summary = (
|
||
launchpad_recovery_summary
|
||
or join_text(
|
||
launchpad_recovery_label,
|
||
launchpad_target_node_code,
|
||
"当前真正阻断已经收敛到首台节点接入收口。"
|
||
)
|
||
)
|
||
elif handover_next_action_code == "run_acceptance" and launchpad_target_node_code:
|
||
primary_command_key = "node_acceptance_run"
|
||
secondary_command_key = "node_handover"
|
||
title = "先跑首台节点接管验收"
|
||
summary = (
|
||
launchpad_recovery_summary
|
||
or join_text(
|
||
launchpad_recovery_label,
|
||
launchpad_target_node_code,
|
||
"当前真正阻断已经收敛到首台节点接管验收。"
|
||
)
|
||
)
|
||
operator_decision = {
|
||
"lane": "node_handover",
|
||
"priority": "high",
|
||
"reason_code": "managed_nodes_agent_pending",
|
||
"title": title,
|
||
"summary": summary,
|
||
"next_focus": next_step_focus_ref,
|
||
"primary_command_key": primary_command_key,
|
||
"secondary_command_key": secondary_command_key,
|
||
}
|
||
elif default_rollout_gate_status == "blocked":
|
||
operator_decision = {
|
||
"lane": "release",
|
||
"priority": "medium",
|
||
"reason_code": "release_rollout_gate_blocked",
|
||
"title": "先检查发布门禁与 rollout 阻断",
|
||
"summary": ensure_str(release_hub.get("default_rollout_gate_summary")),
|
||
"next_focus": ensure_dict(release_hub.get("default_rollout_gate_focus_ref") or release_hub.get("focus_ref")),
|
||
"primary_command_key": "release_launchpad",
|
||
"secondary_command_key": "release_preview_latest",
|
||
}
|
||
elif any(ensure_str(ensure_dict(item).get("code")) == "runtime_build_schema_stale" for item in issues):
|
||
operator_decision = {
|
||
"lane": "runtime_recovery",
|
||
"priority": "high",
|
||
"reason_code": "runtime_build_schema_stale",
|
||
"title": "先执行运行时刷新与上线复核",
|
||
"summary": "当前已经识别到仓库代码与运行中 API schema 之间存在漂移，先走统一恢复链，再继续节点接管或发布判断。",
|
||
"next_focus": ensure_dict(next_step.get("focus_ref")),
|
||
"primary_command_key": "go_live_recover",
|
||
"secondary_command_key": "api_restart",
|
||
}
|
||
elif recommended_actions or issues:
|
||
operator_decision = {
|
||
"lane": "ops_jobs",
|
||
"priority": "medium",
|
||
"reason_code": "issues_need_attention",
|
||
"title": "先按总检建议逐项收口当前问题",
|
||
"summary": (
|
||
f"recommended_actions={len(recommended_actions)}，issues={len(issues)}，"
|
||
f"stack_status={ensure_str(diagnosis.get('stack_status')) or 'unknown'}。"
|
||
),
|
||
"next_focus": ensure_dict(next_step.get("focus_ref")),
|
||
"primary_command_key": "doctor",
|
||
"secondary_command_key": "jobs",
|
||
}
|
||
|
||
if backend_recommended_commands:
|
||
merged_recommended_commands = dict(recommended_commands)
|
||
merged_recommended_commands.update({
|
||
key: ensure_str(value)
|
||
for key, value in backend_recommended_commands.items()
|
||
if ensure_str(key) and ensure_str(value)
|
||
})
|
||
recommended_commands = merged_recommended_commands
|
||
|
||
if backend_operator_decision:
|
||
operator_decision = {
|
||
"lane": ensure_str(backend_operator_decision.get("lane")) or ensure_str(operator_decision.get("lane")),
|
||
"priority": ensure_str(backend_operator_decision.get("priority")) or ensure_str(operator_decision.get("priority")),
|
||
"reason_code": ensure_str(backend_operator_decision.get("reason_code")) or ensure_str(operator_decision.get("reason_code")),
|
||
"title": ensure_str(backend_operator_decision.get("title")) or ensure_str(operator_decision.get("title")),
|
||
"summary": ensure_str(backend_operator_decision.get("summary")) or ensure_str(operator_decision.get("summary")),
|
||
"next_focus": ensure_dict(backend_operator_decision.get("next_focus")) or ensure_dict(operator_decision.get("next_focus")),
|
||
"primary_command_key": ensure_str(backend_operator_decision.get("primary_command_key")) or ensure_str(operator_decision.get("primary_command_key")),
|
||
"secondary_command_key": ensure_str(backend_operator_decision.get("secondary_command_key")) or ensure_str(operator_decision.get("secondary_command_key")),
|
||
}
|
||
|
||
next_actions = {
|
||
"primary_command_key": ensure_str(operator_decision.get("primary_command_key")),
|
||
"primary_command": ensure_str(recommended_commands.get(ensure_str(operator_decision.get("primary_command_key")))),
|
||
"secondary_command_key": ensure_str(operator_decision.get("secondary_command_key")),
|
||
"secondary_command": ensure_str(recommended_commands.get(ensure_str(operator_decision.get("secondary_command_key")))),
|
||
}
|
||
|
||
if backend_next_actions:
|
||
next_actions["primary_command_key"] = ensure_str(backend_next_actions.get("primary_command_key")) or next_actions["primary_command_key"]
|
||
next_actions["secondary_command_key"] = ensure_str(backend_next_actions.get("secondary_command_key")) or next_actions["secondary_command_key"]
|
||
next_actions["primary_command"] = (
|
||
ensure_str(backend_next_actions.get("primary_command"))
|
||
or ensure_str(recommended_commands.get(next_actions["primary_command_key"]))
|
||
or next_actions["primary_command"]
|
||
)
|
||
next_actions["secondary_command"] = (
|
||
ensure_str(backend_next_actions.get("secondary_command"))
|
||
or ensure_str(recommended_commands.get(next_actions["secondary_command_key"]))
|
||
or next_actions["secondary_command"]
|
||
)
|
||
|
||
diagnosis["operator_decision"] = operator_decision
|
||
diagnosis["next_actions"] = next_actions
|
||
diagnosis["recommended_commands"] = recommended_commands
|
||
payload["diagnosis"] = diagnosis
|
||
print(json.dumps(payload, ensure_ascii=False, indent=2))
|
||
PY
|
||
exit 0
|
||
fi
|
||
fi
|
||
|
||
fetch_json "${BASE_URL}/health"
|
||
HEALTH_JSON="${FETCH_BODY}"
|
||
HEALTH_STATUS="${FETCH_STATUS}"
|
||
|
||
fetch_json "${BASE_URL}/api/v1/ops/contracts"
|
||
CONTRACTS_JSON="${FETCH_BODY}"
|
||
CONTRACTS_STATUS="${FETCH_STATUS}"
|
||
|
||
fetch_json "${BASE_URL}/api/v1/ops/link-snapshot"
|
||
LINK_SNAPSHOT_JSON="${FETCH_BODY}"
|
||
LINK_SNAPSHOT_STATUS="${FETCH_STATUS}"
|
||
|
||
fetch_json "${BASE_URL}/api/v1/ops/overview"
|
||
OVERVIEW_JSON="${FETCH_BODY}"
|
||
OVERVIEW_STATUS="${FETCH_STATUS}"
|
||
|
||
fetch_json "${BASE_URL}/api/v1/ops/nodes"
|
||
NODES_JSON="${FETCH_BODY}"
|
||
NODES_STATUS="${FETCH_STATUS}"
|
||
|
||
fetch_json "${BASE_URL}/api/v1/ops/releases/launchpad"
|
||
LAUNCHPAD_JSON="${FETCH_BODY}"
|
||
LAUNCHPAD_STATUS="${FETCH_STATUS}"
|
||
|
||
fetch_json "${BASE_URL}/api/v1/ops/playbook-runs?limit=6"
|
||
PLAYBOOK_RUNS_JSON="${FETCH_BODY}"
|
||
PLAYBOOK_RUNS_STATUS="${FETCH_STATUS}"
|
||
|
||
fetch_json "${BASE_URL}/api/v1/ops/activity-stream?limit=8"
|
||
ACTIVITY_JSON="${FETCH_BODY}"
|
||
ACTIVITY_STATUS="${FETCH_STATUS}"
|
||
|
||
if [[ "${VIEW_MODE}" != "summary" ]]; then
|
||
echo "[1/9] api health"
|
||
printf 'status=%s\n%s\n\n' "${HEALTH_STATUS}" "${HEALTH_JSON}"
|
||
|
||
echo "[2/9] ops contracts"
|
||
printf 'status=%s\n%s\n\n' "${CONTRACTS_STATUS}" "${CONTRACTS_JSON}"
|
||
|
||
echo "[3/9] ops link snapshot"
|
||
printf 'status=%s\n%s\n\n' "${LINK_SNAPSHOT_STATUS}" "${LINK_SNAPSHOT_JSON}"
|
||
|
||
echo "[4/9] ops overview"
|
||
printf 'status=%s\n%s\n\n' "${OVERVIEW_STATUS}" "${OVERVIEW_JSON}"
|
||
|
||
echo "[5/9] managed nodes"
|
||
printf 'status=%s\n%s\n\n' "${NODES_STATUS}" "${NODES_JSON}"
|
||
|
||
echo "[6/9] release launchpad"
|
||
printf 'status=%s\n%s\n\n' "${LAUNCHPAD_STATUS}" "${LAUNCHPAD_JSON}"
|
||
|
||
echo "[7/9] recent playbook runs"
|
||
printf 'status=%s\n%s\n\n' "${PLAYBOOK_RUNS_STATUS}" "${PLAYBOOK_RUNS_JSON}"
|
||
|
||
echo "[8/9] recent activity stream"
|
||
printf 'status=%s\n%s\n\n' "${ACTIVITY_STATUS}" "${ACTIVITY_JSON}"
|
||
fi
|
||
|
||
echo "[9/9] condensed stack summary"
|
||
STACK_BASE_URL="${BASE_URL}" python3 - <<'PY' \
|
||
"${STACK_DIAGNOSIS_JSON}" \
|
||
"${STACK_DIAGNOSIS_STATUS}" \
|
||
"${HEALTH_JSON}" \
|
||
"${HEALTH_STATUS}" \
|
||
"${CONTRACTS_JSON}" \
|
||
"${CONTRACTS_STATUS}" \
|
||
"${LINK_SNAPSHOT_JSON}" \
|
||
"${LINK_SNAPSHOT_STATUS}" \
|
||
"${OVERVIEW_JSON}" \
|
||
"${OVERVIEW_STATUS}" \
|
||
"${NODES_JSON}" \
|
||
"${NODES_STATUS}" \
|
||
"${LAUNCHPAD_JSON}" \
|
||
"${LAUNCHPAD_STATUS}" \
|
||
"${PLAYBOOK_RUNS_JSON}" \
|
||
"${PLAYBOOK_RUNS_STATUS}" \
|
||
"${ACTIVITY_JSON}" \
|
||
"${ACTIVITY_STATUS}"
|
||
import json
|
||
import os
|
||
import sys
|
||
|
||
|
||
def ensure_str(value):
|
||
return value if isinstance(value, str) else ""
|
||
|
||
|
||
def load_payload(raw_text: str) -> dict:
|
||
try:
|
||
return json.loads(raw_text)
|
||
except Exception:
|
||
return {}
|
||
|
||
|
||
def as_data(payload: dict) -> dict:
|
||
if isinstance(payload.get("data"), dict):
|
||
return dict(payload.get("data") or {})
|
||
return {}
|
||
|
||
|
||
def issue_rank(severity: str) -> int:
|
||
return {"blocked": 3, "warning": 2, "info": 1}.get(str(severity or "").strip(), 0)
|
||
|
||
|
||
def command(*parts: str) -> str:
|
||
tokens = ["bash", "domain-api/deploy/multi-region/drive_ops_center.sh"]
|
||
tokens.extend([str(part).strip() for part in parts if str(part).strip()])
|
||
return " ".join(tokens)
|
||
|
||
|
||
def release_command(*parts: str) -> str:
|
||
tokens = ["bash", "domain-api/deploy/multi-region/drive_release_hub.sh"]
|
||
tokens.extend([str(part).strip() for part in parts if str(part).strip()])
|
||
return " ".join(tokens)
|
||
|
||
|
||
stack_diagnosis_payload = load_payload(sys.argv[1])
|
||
STACK_DIAGNOSIS_STATUS = str(sys.argv[2] or "000")
|
||
health = load_payload(sys.argv[3])
|
||
health_status = str(sys.argv[4] or "000")
|
||
contracts_payload = load_payload(sys.argv[5])
|
||
contracts_status = str(sys.argv[6] or "000")
|
||
link_snapshot_payload = load_payload(sys.argv[7])
|
||
link_snapshot_status = str(sys.argv[8] or "000")
|
||
overview_payload = load_payload(sys.argv[9])
|
||
overview_status = str(sys.argv[10] or "000")
|
||
nodes_root_payload = load_payload(sys.argv[11])
|
||
nodes_status = str(sys.argv[12] or "000")
|
||
launchpad_payload = load_payload(sys.argv[13])
|
||
launchpad_status_code = str(sys.argv[14] or "000")
|
||
playbook_runs_root_payload = load_payload(sys.argv[15])
|
||
playbook_runs_status = str(sys.argv[16] or "000")
|
||
activity_root_payload = load_payload(sys.argv[17])
|
||
activity_status = str(sys.argv[18] or "000")
|
||
|
||
contracts = as_data(contracts_payload)
|
||
link_snapshot = as_data(link_snapshot_payload)
|
||
overview = as_data(overview_payload)
|
||
nodes_payload = as_data(nodes_root_payload)
|
||
launchpad = as_data(launchpad_payload)
|
||
playbook_runs_payload = as_data(playbook_runs_root_payload)
|
||
activity_payload = as_data(activity_root_payload)
|
||
stack_base_url = str(os.environ.get("STACK_BASE_URL") or "").strip()
|
||
health_build = dict(health.get("build") or {})
|
||
health_route_surface = dict(health_build.get("route_surface") or {})
|
||
stack_diagnosis = as_data(stack_diagnosis_payload)
|
||
diagnosis = dict(stack_diagnosis.get("diagnosis") or {})
|
||
|
||
contracts_rows = list(contracts.get("contracts") or [])
|
||
nodes = list(nodes_payload.get("nodes") or [])
|
||
nodes_summary = dict(nodes_payload.get("summary") or {})
|
||
playbook_runs = list(playbook_runs_payload.get("runs") or playbook_runs_payload.get("playbook_runs") or [])
|
||
activities = list(activity_payload.get("items") or activity_payload.get("activities") or [])
|
||
|
||
recommendation = dict(overview.get("recommendation") or {})
|
||
execution_scene = dict(overview.get("execution_scene") or {})
|
||
log_sync = dict(execution_scene.get("log_sync") or overview.get("log_sync") or {})
|
||
release_hub = dict(overview.get("release_hub") or {})
|
||
default_rollout_gate = dict(release_hub.get("default_rollout_gate") or {})
|
||
launchpad_status = dict(launchpad.get("launchpad_status") or {})
|
||
latest_release = dict(launchpad.get("latest_release") or release_hub.get("latest_release") or {})
|
||
|
||
problem_nodes = []
|
||
for node in nodes:
|
||
node_code = str(node.get("node_code") or "").strip()
|
||
if not node_code:
|
||
continue
|
||
queue_state = str(node.get("delivery_queue_state") or "").strip()
|
||
agent_state = str(node.get("agent_state") or "").strip()
|
||
remote_access_state = str(node.get("remote_access_state") or "").strip()
|
||
if queue_state == "dead_letter" or agent_state not in {"online", "online_busy"} or remote_access_state in {
|
||
"agent_pending",
|
||
"disabled",
|
||
"unmanaged",
|
||
}:
|
||
problem_nodes.append(
|
||
{
|
||
"node_code": node_code,
|
||
"agent_state": agent_state,
|
||
"remote_access_state": remote_access_state,
|
||
"delivery_queue_state": queue_state,
|
||
"participation_state": str(node.get("participation_state") or "").strip(),
|
||
}
|
||
)
|
||
|
||
problem_runs = [
|
||
{
|
||
"run_code": str(item.get("run_code") or "").strip(),
|
||
"status": str(item.get("status") or "").strip(),
|
||
"status_label": str(item.get("status_label") or "").strip(),
|
||
"focus_step_key": str(item.get("focus_step_key") or "").strip(),
|
||
"focus_summary": str(item.get("focus_summary") or item.get("summary_text") or "").strip(),
|
||
}
|
||
for item in playbook_runs
|
||
if str(item.get("status") or "").strip() not in {"success", "completed", "healthy"}
|
||
]
|
||
|
||
activity_counts = {}
|
||
for item in activities:
|
||
status_key = str(item.get("status") or item.get("job_status") or "unknown").strip() or "unknown"
|
||
activity_counts[status_key] = int(activity_counts.get(status_key, 0) or 0) + 1
|
||
|
||
issues = []
|
||
|
||
|
||
def add_issue(
|
||
*,
|
||
code: str,
|
||
severity: str,
|
||
layer: str,
|
||
summary: str,
|
||
detail: str = "",
|
||
action_code: str = "",
|
||
focus_ref=None,
|
||
commands=None,
|
||
) -> None:
|
||
issues.append(
|
||
{
|
||
"code": str(code or "").strip(),
|
||
"severity": str(severity or "").strip() or "warning",
|
||
"layer": str(layer or "").strip(),
|
||
"summary": str(summary or "").strip(),
|
||
"detail": str(detail or "").strip(),
|
||
"action_code": str(action_code or "").strip(),
|
||
"focus_ref": dict(focus_ref or {}),
|
||
"commands": [str(item).strip() for item in list(commands or []) if str(item).strip()],
|
||
}
|
||
)
|
||
|
||
|
||
surface_matrix = [
|
||
("stack_diagnosis", STACK_DIAGNOSIS_STATUS.startswith("2")),
|
||
("api", health_status.startswith("2")),
|
||
("contracts", contracts_status.startswith("2")),
|
||
("link_snapshot", link_snapshot_status.startswith("2")),
|
||
("overview", overview_status.startswith("2")),
|
||
("managed_nodes", nodes_status.startswith("2")),
|
||
("release_hub", launchpad_status_code.startswith("2")),
|
||
("playbook_runs", playbook_runs_status.startswith("2")),
|
||
("activity_stream", activity_status.startswith("2")),
|
||
]
|
||
surface_status_counts = {
|
||
"total": len(surface_matrix),
|
||
"available": sum(1 for _, available in surface_matrix if available),
|
||
"missing": sum(1 for _, available in surface_matrix if not available),
|
||
}
|
||
missing_surfaces = [name for name, available in surface_matrix if not available]
|
||
|
||
if not health_status.startswith("2"):
|
||
add_issue(
|
||
code="api_unreachable",
|
||
severity="blocked",
|
||
layer="api",
|
||
summary="控制面 API 当前不可用，总检无法进入正式判断链路。",
|
||
detail=f"/health 返回 HTTP {health_status}。",
|
||
commands=[
|
||
f"curl -s {stack_base_url}/health",
|
||
"systemctl status domaincheck-api --no-pager -l",
|
||
],
|
||
)
|
||
|
||
stack_diagnosis_hint = (
|
||
"当前 stack-diagnosis 没有成功返回，总检已退化为多 surface 拼装视图。"
|
||
)
|
||
if not STACK_DIAGNOSIS_STATUS.startswith("2"):
|
||
stack_diag_detail = (
|
||
f"/api/v1/ops/stack-diagnosis 返回 HTTP {STACK_DIAGNOSIS_STATUS}。"
|
||
if str(STACK_DIAGNOSIS_STATUS).isdigit() and str(STACK_DIAGNOSIS_STATUS) != "000"
|
||
else "/api/v1/ops/stack-diagnosis 当前不可达或请求超时。"
|
||
)
|
||
add_issue(
|
||
code="stack_diagnosis_unavailable",
|
||
severity="warning" if health_status.startswith("2") else "blocked",
|
||
layer="stack_diagnosis",
|
||
summary=stack_diagnosis_hint,
|
||
detail=(
|
||
stack_diag_detail
|
||
+ " 建议先单独复跑 stack-diagnosis / stack-next，确认是不是 API 慢、路由未挂齐，或者服务未切到最新版本。"
|
||
),
|
||
commands=[
|
||
f"curl -s {stack_base_url}/api/v1/ops/stack-diagnosis?base_url={stack_base_url}",
|
||
f"bash domain-api/deploy/multi-region/drive_ops_center.sh stack-next {stack_base_url}",
|
||
"systemctl status domaincheck-api --no-pager -l",
|
||
],
|
||
)
|
||
|
||
if health_status.startswith("2") and not contracts_status.startswith("2"):
|
||
add_issue(
|
||
code="ops_contracts_endpoint_missing",
|
||
severity="warning",
|
||
layer="contracts",
|
||
summary="控制面 API 已在线，但 ops/contracts 未正常返回，存在运行实例与当前代码契约不一致的风险。",
|
||
detail=(
|
||
f"/api/v1/ops/contracts 返回 HTTP {contracts_status}；"
|
||
"这通常意味着当前 systemd 进程没有跑到最新接口集，或部署包/服务重启未完全对齐。"
|
||
),
|
||
commands=[
|
||
f"curl -s {stack_base_url}/api/v1/ops/contracts",
|
||
"systemctl status domaincheck-api --no-pager -l",
|
||
f"bash domain-api/deploy/multi-region/drive_ops_center.sh runtime-refresh-recover {stack_base_url}",
|
||
],
|
||
)
|
||
|
||
if health_status.startswith("2") and not health_build:
|
||
add_issue(
|
||
code="api_build_identity_missing",
|
||
severity="warning",
|
||
layer="api",
|
||
summary="控制面 health 还没有暴露 build 指纹，当前不利于快速判断服务是否真的切到最新部署内容。",
|
||
detail="建议升级到包含 build + route_surface 指纹的新版本后再继续多机联调。",
|
||
commands=[
|
||
f"curl -s {stack_base_url}/health",
|
||
f"bash domain-api/deploy/multi-region/drive_ops_center.sh runtime-refresh-recover {stack_base_url}",
|
||
],
|
||
)
|
||
|
||
if health_status.startswith("2") and health_route_surface and not bool(health_route_surface.get("surface_complete", True)):
|
||
add_issue(
|
||
code="api_route_surface_incomplete",
|
||
severity="warning",
|
||
layer="api.routes",
|
||
summary="当前运行中的控制面实例没有挂齐关键协议路由，存在代码与服务实例不一致的风险。",
|
||
detail=(
|
||
"缺少路由: "
|
||
+ ", ".join([str(item).strip() for item in list(health_route_surface.get("missing_paths") or []) if str(item).strip()])
|
||
),
|
||
commands=[
|
||
f"curl -s {stack_base_url}/health",
|
||
f"bash domain-api/deploy/multi-region/drive_ops_center.sh runtime-refresh-recover {stack_base_url}",
|
||
],
|
||
)
|
||
|
||
if health_status.startswith("2") and missing_surfaces and any(
|
||
name in missing_surfaces for name in ("link_snapshot", "overview", "managed_nodes", "release_hub")
|
||
):
|
||
add_issue(
|
||
code="ops_surface_partial",
|
||
severity="warning",
|
||
layer="stack",
|
||
summary="海外单脑控制面关键观察层没有全部联通，总检已退化为部分收口。",
|
||
detail=f"当前缺失层: {', '.join(missing_surfaces)}",
|
||
commands=[
|
||
f"bash domain-api/deploy/multi-region/drive_ops_center.sh stack-diagnosis {stack_base_url} full",
|
||
f"bash domain-api/deploy/multi-region/drive_ops_center.sh doctor {stack_base_url}",
|
||
],
|
||
)
|
||
|
||
participating_nodes_total = int(log_sync.get("participating_nodes_total", log_sync.get("participating_node_count", 0)) or 0)
|
||
if bool(log_sync.get("enabled", False)) is False and participating_nodes_total > 0:
|
||
log_sync_action = ""
|
||
next_action_codes = list(link_snapshot.get("next_actions") or [])
|
||
for item in next_action_codes:
|
||
action_code = str(item.get("action_code") or "").strip()
|
||
if action_code.startswith("enable_log_sync"):
|
||
log_sync_action = action_code
|
||
break
|
||
if not log_sync_action:
|
||
log_sync_action = "enable_log_sync_key"
|
||
add_issue(
|
||
code="remote_log_sync_disabled",
|
||
severity="warning",
|
||
layer="overview.log_sync",
|
||
summary="现场已有参与检测节点，但远端日志回传仍关闭，海外控制面对执行细节仍处于半盲态。",
|
||
detail=(
|
||
f"参与检测节点 {participating_nodes_total} 台，"
|
||
f"已覆盖 {int(log_sync.get('covered_participating_nodes', log_sync.get('covered_participating_node_count', 0)) or 0)} 台。"
|
||
),
|
||
action_code=log_sync_action,
|
||
commands=[
|
||
f"bash domain-api/deploy/multi-region/drive_ops_center.sh driver-resolve {stack_base_url} {log_sync_action}",
|
||
],
|
||
)
|
||
|
||
managed_enabled = int(nodes_summary.get("managed_enabled", 0) or 0)
|
||
remote_access_ready = int(nodes_summary.get("remote_access_ready", 0) or 0)
|
||
if managed_enabled > 0 and remote_access_ready == 0:
|
||
add_issue(
|
||
code="managed_nodes_agent_pending",
|
||
severity="blocked",
|
||
layer="managed_nodes",
|
||
summary="托管节点虽然已经纳入 Ops Center，但 0 台 remote-agent 就绪，海外主控还不能统一下发标准执行动作。",
|
||
detail=(
|
||
f"managed_enabled={managed_enabled}，remote_access_ready={remote_access_ready}；"
|
||
"当前节点多数仍处于 agent_pending / runtime_only / pending_bootstrap。"
|
||
),
|
||
action_code=str(launchpad_status.get("recommended_action_code") or "fix_managed_nodes"),
|
||
focus_ref=dict(launchpad_status.get("focus_ref") or {}),
|
||
commands=[
|
||
f"bash domain-api/deploy/multi-region/drive_ops_center.sh driver-resolve {stack_base_url} fix_managed_nodes",
|
||
f"bash domain-api/deploy/multi-region/drive_ops_center.sh doctor {stack_base_url}",
|
||
],
|
||
)
|
||
|
||
queue_dead_letter_nodes = int(nodes_summary.get("queue_dead_letter_nodes", 0) or 0)
|
||
queue_dead_letter_records = int(nodes_summary.get("queue_dead_letter_records", 0) or 0)
|
||
if queue_dead_letter_nodes > 0 or queue_dead_letter_records > 0:
|
||
add_issue(
|
||
code="delivery_queue_dead_letter",
|
||
severity="warning",
|
||
layer="managed_nodes.queue",
|
||
summary="存在 Node Agent 回执死信，后续自动化动作可能反复失败或无法收敛。",
|
||
detail=(
|
||
f"queue_dead_letter_nodes={queue_dead_letter_nodes}，"
|
||
f"queue_dead_letter_records={queue_dead_letter_records}"
|
||
),
|
||
commands=[
|
||
f"bash domain-api/deploy/multi-region/drive_ops_center.sh doctor {stack_base_url}",
|
||
],
|
||
)
|
||
|
||
if str(default_rollout_gate.get("status") or "").strip() == "blocked":
|
||
add_issue(
|
||
code="release_rollout_gate_blocked",
|
||
severity="warning",
|
||
layer="release_hub",
|
||
summary="ReleaseHub 默认门禁当前阻断，说明虽然发布包已准备好，但正式 Rollout 还不具备条件。",
|
||
detail=str(default_rollout_gate.get("summary_text") or default_rollout_gate.get("summary") or "").strip(),
|
||
action_code=str(launchpad_status.get("recommended_action_code") or "").strip(),
|
||
focus_ref=dict(default_rollout_gate.get("focus_ref") or launchpad_status.get("focus_ref") or {}),
|
||
commands=[
|
||
f"bash domain-api/deploy/multi-region/drive_ops_center.sh release-launchpad {stack_base_url}",
|
||
f"bash domain-api/deploy/multi-region/drive_ops_center.sh driver-resolve {stack_base_url} {str(launchpad_status.get('recommended_action_code') or 'fix_managed_nodes').strip()}",
|
||
],
|
||
)
|
||
|
||
if problem_runs:
|
||
add_issue(
|
||
code="playbook_runs_need_attention",
|
||
severity="warning",
|
||
layer="playbook_runs",
|
||
summary="最近存在未完全收口的 playbook run，建议先确认执行序列是否停在中间步骤。",
|
||
detail=f"problem_runs_total={len(problem_runs)}",
|
||
commands=[
|
||
f"bash domain-api/deploy/multi-region/drive_ops_center.sh doctor {stack_base_url}",
|
||
],
|
||
)
|
||
|
||
ordered_action_candidates = [
|
||
{
|
||
"source": "overview.primary",
|
||
"action_code": str(recommendation.get("primary_action_code") or "").strip(),
|
||
"title": str(recommendation.get("title") or "").strip(),
|
||
"reason": "来自 overview.recommendation.primary_action_code",
|
||
"focus_ref": dict(recommendation.get("focus_ref") or {}),
|
||
},
|
||
{
|
||
"source": "release_hub.launchpad",
|
||
"action_code": str(launchpad_status.get("recommended_action_code") or "").strip(),
|
||
"title": str(launchpad_status.get("status_label") or "").strip(),
|
||
"reason": "来自 release_launchpad.launchpad_status.recommended_action_code",
|
||
"focus_ref": dict(launchpad_status.get("focus_ref") or {}),
|
||
},
|
||
{
|
||
"source": "overview.secondary",
|
||
"action_code": str(recommendation.get("secondary_action_code") or "").strip(),
|
||
"title": str(recommendation.get("title") or "").strip(),
|
||
"reason": "来自 overview.recommendation.secondary_action_code",
|
||
"focus_ref": dict(recommendation.get("focus_ref") or {}),
|
||
},
|
||
]
|
||
for issue in issues:
|
||
issue_action_code = str(issue.get("action_code") or "").strip()
|
||
if issue_action_code:
|
||
ordered_action_candidates.append(
|
||
{
|
||
"source": f"issue:{issue.get('code')}",
|
||
"action_code": issue_action_code,
|
||
"title": str(issue.get("summary") or "").strip(),
|
||
"reason": str(issue.get("detail") or issue.get("summary") or "").strip(),
|
||
"focus_ref": dict(issue.get("focus_ref") or {}),
|
||
}
|
||
)
|
||
|
||
recommended_actions = []
|
||
seen_action_codes = set()
|
||
for item in ordered_action_candidates:
|
||
action_code = str(item.get("action_code") or "").strip()
|
||
if not action_code or action_code in seen_action_codes:
|
||
continue
|
||
seen_action_codes.add(action_code)
|
||
recommended_actions.append(
|
||
{
|
||
"action_code": action_code,
|
||
"source": str(item.get("source") or "").strip(),
|
||
"title": str(item.get("title") or "").strip(),
|
||
"reason": str(item.get("reason") or "").strip(),
|
||
"focus_ref": dict(item.get("focus_ref") or {}),
|
||
"command": f"bash domain-api/deploy/multi-region/drive_ops_center.sh driver-resolve {stack_base_url} {action_code}",
|
||
}
|
||
)
|
||
|
||
blocking_issue_total = sum(1 for item in issues if str(item.get("severity") or "").strip() == "blocked")
|
||
warning_issue_total = sum(1 for item in issues if str(item.get("severity") or "").strip() == "warning")
|
||
info_issue_total = sum(1 for item in issues if str(item.get("severity") or "").strip() == "info")
|
||
|
||
surface_status = "healthy"
|
||
if not health_status.startswith("2"):
|
||
surface_status = "broken"
|
||
elif missing_surfaces:
|
||
surface_status = "partial"
|
||
|
||
automation_status = "ready"
|
||
if blocking_issue_total > 0:
|
||
automation_status = "blocked"
|
||
elif warning_issue_total > 0:
|
||
automation_status = "attention"
|
||
|
||
stack_status = "ready"
|
||
if surface_status == "broken" or automation_status == "blocked":
|
||
stack_status = "blocked"
|
||
elif surface_status != "healthy" or automation_status != "ready":
|
||
stack_status = "attention"
|
||
|
||
top_issue = None
|
||
if issues:
|
||
top_issue = sorted(
|
||
issues,
|
||
key=lambda item: (
|
||
-issue_rank(str(item.get("severity") or "").strip()),
|
||
str(item.get("code") or "").strip(),
|
||
),
|
||
)[0]
|
||
|
||
next_step = {}
|
||
if recommended_actions:
|
||
first_action = dict(recommended_actions[0] or {})
|
||
next_step = {
|
||
"action_code": str(first_action.get("action_code") or "").strip(),
|
||
"source": str(first_action.get("source") or "").strip(),
|
||
"reason": str(first_action.get("reason") or "").strip(),
|
||
"command": str(first_action.get("command") or "").strip(),
|
||
"focus_ref": dict(first_action.get("focus_ref") or {}),
|
||
}
|
||
elif top_issue:
|
||
next_step = {
|
||
"action_code": "",
|
||
"source": f"issue:{str(top_issue.get('code') or '').strip()}",
|
||
"reason": str(top_issue.get("summary") or "").strip(),
|
||
"command": "",
|
||
"focus_ref": dict(top_issue.get("focus_ref") or {}),
|
||
}
|
||
|
||
next_step_focus_ref = dict(next_step.get("focus_ref") or {})
|
||
launchpad_target_node_code = str(
|
||
diagnosis.get("launchpad_recommended_target_node_code")
|
||
or next_step_focus_ref.get("node_code")
|
||
or ""
|
||
).strip()
|
||
launchpad_recovery_label = str(diagnosis.get("launchpad_recommended_recovery_label") or "").strip()
|
||
launchpad_recovery_summary = str(diagnosis.get("launchpad_recommended_recovery_summary") or next_step.get("reason") or "").strip()
|
||
handover_next_action_code = str(next_step.get("action_code") or "").strip()
|
||
|
||
scene_log_missing_nodes = list(
|
||
log_sync.get("missing_sample_node_codes")
|
||
or log_sync.get("missing_participating_nodes")
|
||
or []
|
||
)
|
||
scene_log_target_node_code = ""
|
||
if scene_log_missing_nodes:
|
||
scene_log_target_node_code = str(scene_log_missing_nodes[0] or "").strip()
|
||
elif problem_nodes:
|
||
scene_log_target_node_code = str(problem_nodes[0].get("node_code") or "").strip()
|
||
|
||
recommended_commands = {
|
||
"stack_summary": f"bash domain-api/deploy/multi-region/check_ops_center_stack.sh {stack_base_url} summary",
|
||
"doctor": command("doctor", stack_base_url),
|
||
"driver_feed": command("driver-feed", stack_base_url),
|
||
"driver_focus_preview": command("driver-focus-preview", stack_base_url),
|
||
"driver_focus_run": command("driver-focus-run", stack_base_url),
|
||
"go_live_recover": command("go-live-recover", stack_base_url),
|
||
"log_sync_recover": command("log-sync-recover", stack_base_url, "key"),
|
||
"log_sync_recover_full": command("log-sync-recover", stack_base_url, "full"),
|
||
"agent_gap_check": command("agent-gap-check", stack_base_url),
|
||
"agent_gap_recover": command("agent-gap-recover", stack_base_url),
|
||
"api_restart": command("runtime-refresh-recover", stack_base_url),
|
||
"node_onboarding": (
|
||
command("node-onboarding", stack_base_url, str(problem_nodes[0].get("node_code") or "").strip())
|
||
if problem_nodes and str(problem_nodes[0].get("node_code") or "").strip()
|
||
else ""
|
||
),
|
||
"node_recover": (
|
||
command("node-recover", stack_base_url, str(problem_nodes[0].get("node_code") or "").strip())
|
||
if problem_nodes and str(problem_nodes[0].get("node_code") or "").strip()
|
||
else ""
|
||
),
|
||
"node_bootstrap_preview": (
|
||
command("node-bootstrap-preview", stack_base_url, str(problem_nodes[0].get("node_code") or "").strip())
|
||
if problem_nodes and str(problem_nodes[0].get("node_code") or "").strip()
|
||
else ""
|
||
),
|
||
"node_bootstrap_run": (
|
||
command("node-bootstrap-run", stack_base_url, launchpad_target_node_code, "cli")
|
||
if launchpad_target_node_code
|
||
else command("node-bootstrap-run", stack_base_url, str(problem_nodes[0].get("node_code") or "").strip(), "cli")
|
||
if launchpad_target_node_code or (problem_nodes and str(problem_nodes[0].get("node_code") or "").strip())
|
||
else ""
|
||
),
|
||
"node_acceptance_run": (
|
||
command("node-acceptance-run", stack_base_url, launchpad_target_node_code, "cli")
|
||
if launchpad_target_node_code
|
||
else command("node-acceptance-run", stack_base_url, str(problem_nodes[0].get("node_code") or "").strip(), "cli")
|
||
if launchpad_target_node_code or (problem_nodes and str(problem_nodes[0].get("node_code") or "").strip())
|
||
else ""
|
||
),
|
||
"node_acceptance_plan": (
|
||
command("node-acceptance-plan", stack_base_url, str(problem_nodes[0].get("node_code") or "").strip())
|
||
if problem_nodes and str(problem_nodes[0].get("node_code") or "").strip()
|
||
else ""
|
||
),
|
||
"scene_log": (
|
||
command("scene-node-log", stack_base_url, scene_log_target_node_code, "120", "key")
|
||
if scene_log_target_node_code
|
||
else ""
|
||
),
|
||
"release_launchpad": release_command(stack_base_url, "launchpad"),
|
||
"release_preview_latest": release_command(stack_base_url, "preview-latest"),
|
||
"jobs": command("activity-stream", stack_base_url, "kind=ops_job"),
|
||
"node_handover": (
|
||
command("node-handover", stack_base_url, str(problem_nodes[0].get("node_code") or "").strip())
|
||
if problem_nodes and str(problem_nodes[0].get("node_code") or "").strip()
|
||
else command("stack-first-gap-handover", stack_base_url)
|
||
),
|
||
}
|
||
|
||
operator_decision = {
|
||
"lane": "steady",
|
||
"priority": "low",
|
||
"reason_code": "steady_state",
|
||
"title": "当前进入稳定观察态",
|
||
"summary": "当前总检没有发现新的阻断问题，可继续按既有链路观察与推进。",
|
||
"next_focus": next_step_focus_ref,
|
||
"primary_command_key": "driver_feed",
|
||
"secondary_command_key": "jobs",
|
||
}
|
||
|
||
if str(log_sync.get("enabled", False)).lower() in {"false", ""} and participating_nodes_total > 0:
|
||
operator_decision = {
|
||
"lane": "observability",
|
||
"priority": "high",
|
||
"reason_code": "scene_log_disabled",
|
||
"title": "先恢复远端现场日志可见性",
|
||
"summary": "当前已有参与检测节点，但远端日志回传仍关闭，海外控制面仍处于半盲态。",
|
||
"next_focus": {"kind": "node_scene_log", "node_code": scene_log_target_node_code, "mode": "key", "limit": 80}
|
||
if scene_log_target_node_code
|
||
else {},
|
||
"primary_command_key": "log_sync_recover",
|
||
"secondary_command_key": "scene_log",
|
||
}
|
||
elif managed_enabled > 0 and remote_access_ready == 0:
|
||
primary_command_key = "agent_gap_recover"
|
||
secondary_command_key = "node_handover"
|
||
title = "先补齐节点接管与 remote-agent 可达性"
|
||
summary = (
|
||
f"managed_enabled={managed_enabled}，remote_access_ready={remote_access_ready}；"
|
||
"当前还不能从海外控制面统一下发标准动作。"
|
||
)
|
||
if handover_next_action_code == "bootstrap_run" and launchpad_target_node_code:
|
||
primary_command_key = "node_bootstrap_run"
|
||
secondary_command_key = "node_handover"
|
||
title = "先跑首台节点接入收口"
|
||
summary = (
|
||
launchpad_recovery_summary
|
||
or join_text(
|
||
launchpad_recovery_label,
|
||
launchpad_target_node_code,
|
||
"当前真正阻断已经收敛到首台节点接入收口。"
|
||
)
|
||
)
|
||
elif handover_next_action_code == "run_acceptance" and launchpad_target_node_code:
|
||
primary_command_key = "node_acceptance_run"
|
||
secondary_command_key = "node_handover"
|
||
title = "先跑首台节点接管验收"
|
||
summary = (
|
||
launchpad_recovery_summary
|
||
or join_text(
|
||
launchpad_recovery_label,
|
||
launchpad_target_node_code,
|
||
"当前真正阻断已经收敛到首台节点接管验收。"
|
||
)
|
||
)
|
||
operator_decision = {
|
||
"lane": "node_handover",
|
||
"priority": "high",
|
||
"reason_code": "managed_nodes_agent_pending",
|
||
"title": title,
|
||
"summary": summary,
|
||
"next_focus": next_step_focus_ref,
|
||
"primary_command_key": primary_command_key,
|
||
"secondary_command_key": secondary_command_key,
|
||
}
|
||
elif str(default_rollout_gate.get("status") or "").strip() == "blocked":
|
||
operator_decision = {
|
||
"lane": "release",
|
||
"priority": "medium",
|
||
"reason_code": "release_rollout_gate_blocked",
|
||
"title": "先检查发布门禁与 rollout 阻断",
|
||
"summary": str(default_rollout_gate.get("summary_text") or default_rollout_gate.get("summary") or "").strip(),
|
||
"next_focus": dict(default_rollout_gate.get("focus_ref") or launchpad_status.get("focus_ref") or {}),
|
||
"primary_command_key": "release_launchpad",
|
||
"secondary_command_key": "release_preview_latest",
|
||
}
|
||
elif any(str(item.get("code") or "").strip() == "runtime_build_schema_stale" for item in issues):
|
||
operator_decision = {
|
||
"lane": "runtime_recovery",
|
||
"priority": "high",
|
||
"reason_code": "runtime_build_schema_stale",
|
||
"title": "先执行运行时刷新与上线复核",
|
||
"summary": "当前已经识别到仓库代码与运行中 API schema 之间存在漂移，先走统一恢复链，再继续节点接管或发布判断。",
|
||
"next_focus": dict(next_step.get("focus_ref") or {}),
|
||
"primary_command_key": "go_live_recover",
|
||
"secondary_command_key": "api_restart",
|
||
}
|
||
elif problem_runs:
|
||
operator_decision = {
|
||
"lane": "ops_jobs",
|
||
"priority": "medium",
|
||
"reason_code": "playbook_runs_need_attention",
|
||
"title": "先确认未收口的 playbook run",
|
||
"summary": f"最近存在 {len(problem_runs)} 个未完全收口的 playbook run。",
|
||
"next_focus": dict(next_step.get("focus_ref") or {}),
|
||
"primary_command_key": "doctor",
|
||
"secondary_command_key": "jobs",
|
||
}
|
||
|
||
next_actions = {
|
||
"primary_command": str(recommended_commands.get(str(operator_decision.get("primary_command_key") or "").strip()) or "").strip(),
|
||
"secondary_command": str(recommended_commands.get(str(operator_decision.get("secondary_command_key") or "").strip()) or "").strip(),
|
||
}
|
||
|
||
quick_commands = [
|
||
f"bash domain-api/deploy/multi-region/drive_ops_center.sh stack-diagnosis {stack_base_url} summary",
|
||
f"bash domain-api/deploy/multi-region/drive_ops_center.sh doctor {stack_base_url}",
|
||
]
|
||
for action in recommended_actions[:3]:
|
||
command = str(action.get("command") or "").strip()
|
||
if command and command not in quick_commands:
|
||
quick_commands.append(command)
|
||
if not contracts_status.startswith("2"):
|
||
contract_check_command = f"curl -s {stack_base_url}/api/v1/ops/contracts"
|
||
if contract_check_command not in quick_commands:
|
||
quick_commands.append(contract_check_command)
|
||
|
||
operator_hints = []
|
||
if stack_status == "blocked":
|
||
operator_hints.append("当前总检已经能看到现场，但海外单脑自动化执行链路仍未正式打通，先处理阻断问题再继续 Rollout 或批量运维。")
|
||
elif stack_status == "attention":
|
||
operator_hints.append("当前总检可用，但仍有缺口；建议先按 next_step 收口，再进入更细粒度脚本。")
|
||
else:
|
||
operator_hints.append("当前海外单脑控制面总检已基本就绪，可以把它作为后续所有联调和发布的固定起手式。")
|
||
if not contracts_status.startswith("2") and health_status.startswith("2"):
|
||
operator_hints.append("health 正常但 contracts 缺失，优先怀疑运行中的 API 实例不是这份最新代码，或服务重启后未真正切到最新部署内容。")
|
||
if bool(log_sync.get("enabled", False)) is False and participating_nodes_total > 0:
|
||
operator_hints.append("远端日志回传当前关闭，后续如果继续多机联调，海外控制面会继续处于半盲态。")
|
||
|
||
summary = {
|
||
"diagnosis": {
|
||
"base_url": stack_base_url,
|
||
"surface_status": surface_status,
|
||
"automation_status": automation_status,
|
||
"stack_status": stack_status,
|
||
"issue_total": len(issues),
|
||
"blocking_issue_total": blocking_issue_total,
|
||
"warning_issue_total": warning_issue_total,
|
||
"info_issue_total": info_issue_total,
|
||
"missing_surfaces": missing_surfaces,
|
||
"launchpad_recommended_target_node_code": ensure_str(diagnosis.get("launchpad_recommended_target_node_code")),
|
||
"launchpad_recommended_recovery_label": ensure_str(diagnosis.get("launchpad_recommended_recovery_label")),
|
||
"launchpad_recommended_recovery_summary": ensure_str(diagnosis.get("launchpad_recommended_recovery_summary")),
|
||
"launchpad_onboarding_bootstrap_pending_nodes": int(diagnosis.get("launchpad_onboarding_bootstrap_pending_nodes", 0) or 0),
|
||
"launchpad_onboarding_acceptance_ready_nodes": int(diagnosis.get("launchpad_onboarding_acceptance_ready_nodes", 0) or 0),
|
||
"next_step": next_step,
|
||
"recommended_actions": recommended_actions[:5],
|
||
"operator_decision": operator_decision,
|
||
"next_actions": next_actions,
|
||
"issues": issues[:8],
|
||
"operator_hints": operator_hints,
|
||
"quick_commands": quick_commands,
|
||
"recommended_commands": recommended_commands,
|
||
},
|
||
"api": {
|
||
"stack_diagnosis_http_status": STACK_DIAGNOSIS_STATUS,
|
||
"stack_diagnosis_available": STACK_DIAGNOSIS_STATUS.startswith("2"),
|
||
"http_status": health_status,
|
||
"available": health_status.startswith("2"),
|
||
"service": str(health.get("service") or ""),
|
||
"version": str(health.get("version") or ""),
|
||
"api_prefix": str(health.get("api_prefix") or ""),
|
||
"worker_mode": str(health.get("worker_mode") or ""),
|
||
"build": {
|
||
"source": str(health_build.get("source") or ""),
|
||
"package_name": str(health_build.get("package_name") or ""),
|
||
"generated_at": str(health_build.get("generated_at") or ""),
|
||
"commit_sha": str(health_build.get("commit_sha") or ""),
|
||
"commit_ref": str(health_build.get("commit_ref") or ""),
|
||
"manifest_path": str(health_build.get("manifest_path") or ""),
|
||
"route_surface_complete": bool(health_route_surface.get("surface_complete", False)) if health_route_surface else False,
|
||
"route_surface_missing": [
|
||
str(item).strip()
|
||
for item in list(health_route_surface.get("missing_paths") or [])
|
||
if str(item).strip()
|
||
],
|
||
},
|
||
},
|
||
"surface_matrix": {
|
||
"status_counts": surface_status_counts,
|
||
"items": [
|
||
{"name": name, "available": bool(available)}
|
||
for name, available in surface_matrix
|
||
],
|
||
},
|
||
"contracts": {
|
||
"http_status": contracts_status,
|
||
"available": contracts_status.startswith("2"),
|
||
"registry_version": str(contracts.get("registry_version") or ""),
|
||
"schema_version": str(contracts.get("schema_version") or ""),
|
||
"contracts_total": len(contracts_rows),
|
||
"contract_keys": [str(item.get("key") or "").strip() for item in contracts_rows if str(item.get("key") or "").strip()],
|
||
"error": str(contracts_payload.get("detail") or contracts_payload.get("message") or ""),
|
||
},
|
||
"link_snapshot": {
|
||
"http_status": link_snapshot_status,
|
||
"available": link_snapshot_status.startswith("2"),
|
||
"status": str(link_snapshot.get("status") or ""),
|
||
"headline": str(link_snapshot.get("headline") or ""),
|
||
"next_action_codes": [
|
||
str(item.get("action_code") or "").strip()
|
||
for item in list(link_snapshot.get("next_actions") or [])
|
||
if str(item.get("action_code") or "").strip()
|
||
],
|
||
"error": str(link_snapshot_payload.get("detail") or link_snapshot_payload.get("message") or ""),
|
||
},
|
||
"overview": {
|
||
"http_status": overview_status,
|
||
"available": overview_status.startswith("2"),
|
||
"recommendation_key": str(recommendation.get("key") or ""),
|
||
"recommendation_title": str(recommendation.get("title") or ""),
|
||
"primary_action_code": str(recommendation.get("primary_action_code") or ""),
|
||
"secondary_action_code": str(recommendation.get("secondary_action_code") or ""),
|
||
"focus_ref": dict(recommendation.get("focus_ref") or {}),
|
||
"execution_scene_summary": str(
|
||
execution_scene.get("summary")
|
||
or (execution_scene.get("participation_summary") or {}).get("summary")
|
||
or execution_scene.get("phase_detail")
|
||
or ""
|
||
),
|
||
"execution_scene_counts": dict(execution_scene.get("counts") or {}),
|
||
"log_sync": {
|
||
"status": str(log_sync.get("status") or log_sync.get("state") or ""),
|
||
"status_label": str(log_sync.get("status_label") or ""),
|
||
"mode": str(log_sync.get("mode") or ""),
|
||
"mode_label": str(log_sync.get("mode_label") or ""),
|
||
"enabled": bool(log_sync.get("enabled", False)),
|
||
"covered_participating_nodes": int(
|
||
log_sync.get("covered_participating_nodes", log_sync.get("covered_participating_node_count", 0)) or 0
|
||
),
|
||
"participating_nodes_total": int(
|
||
log_sync.get("participating_nodes_total", log_sync.get("participating_node_count", 0)) or 0
|
||
),
|
||
"missing_sample_node_codes": list(
|
||
log_sync.get("missing_sample_node_codes")
|
||
or log_sync.get("missing_participating_nodes")
|
||
or []
|
||
),
|
||
},
|
||
"error": str(overview_payload.get("detail") or overview_payload.get("message") or ""),
|
||
},
|
||
"managed_nodes": {
|
||
"http_status": nodes_status,
|
||
"available": nodes_status.startswith("2"),
|
||
"total": int(nodes_summary.get("total", len(nodes)) or 0),
|
||
"managed_total": int(nodes_summary.get("managed_total", 0) or 0),
|
||
"managed_enabled": int(nodes_summary.get("managed_enabled", 0) or 0),
|
||
"online": int(nodes_summary.get("online", 0) or 0),
|
||
"agent_ready": int(nodes_summary.get("agent_ready", 0) or 0),
|
||
"ssh_ready": int(nodes_summary.get("ssh_ready", 0) or 0),
|
||
"remote_access_ready": int(nodes_summary.get("remote_access_ready", 0) or 0),
|
||
"participating": int(nodes_summary.get("participating", 0) or 0),
|
||
"dispatch_active": int(nodes_summary.get("dispatch_active", 0) or 0),
|
||
"standby": int(nodes_summary.get("standby", 0) or 0),
|
||
"load_syncing": int(nodes_summary.get("load_syncing", 0) or 0),
|
||
"queue_retrying_nodes": int(nodes_summary.get("queue_retrying_nodes", 0) or 0),
|
||
"queue_dead_letter_nodes": int(nodes_summary.get("queue_dead_letter_nodes", 0) or 0),
|
||
"queue_pending_records": int(nodes_summary.get("queue_pending_records", 0) or 0),
|
||
"queue_dead_letter_records": int(nodes_summary.get("queue_dead_letter_records", 0) or 0),
|
||
"agent_state_counts": dict(nodes_summary.get("status_counts") or {}),
|
||
"remote_access_state_counts": dict(nodes_summary.get("remote_access_state_counts") or {}),
|
||
"delivery_queue_state_counts": dict(nodes_summary.get("delivery_queue_state_counts") or {}),
|
||
"problem_nodes": problem_nodes[:8],
|
||
"error": str(nodes_root_payload.get("detail") or nodes_root_payload.get("message") or ""),
|
||
},
|
||
"release_hub": {
|
||
"http_status": launchpad_status_code,
|
||
"available": launchpad_status_code.startswith("2"),
|
||
"latest_release_version": str(latest_release.get("release_version") or ""),
|
||
"latest_release_status": str(latest_release.get("status") or ""),
|
||
"default_rollout_gate_status": str(default_rollout_gate.get("status") or ""),
|
||
"default_rollout_gate_label": str(default_rollout_gate.get("status_label") or ""),
|
||
"default_rollout_gate_summary": str(default_rollout_gate.get("summary_text") or default_rollout_gate.get("summary") or ""),
|
||
"default_rollout_gate_focus_ref": dict(default_rollout_gate.get("focus_ref") or {}),
|
||
"launchpad_status": str(launchpad_status.get("status") or ""),
|
||
"launchpad_status_label": str(launchpad_status.get("status_label") or ""),
|
||
"launchpad_summary": str(launchpad_status.get("summary_text") or launchpad_status.get("summary") or ""),
|
||
"recommended_action_code": str(launchpad_status.get("recommended_action_code") or ""),
|
||
"recommended_target_node_code": str(launchpad_status.get("recommended_target_node_code") or ""),
|
||
"recommended_recovery_label": str(launchpad_status.get("recommended_recovery_label") or ""),
|
||
"recommended_recovery_summary": str(launchpad_status.get("recommended_recovery_summary") or ""),
|
||
"onboarding_bootstrap_pending_nodes": int(launchpad_status.get("onboarding_bootstrap_pending_nodes", 0) or 0),
|
||
"onboarding_acceptance_ready_nodes": int(launchpad_status.get("onboarding_acceptance_ready_nodes", 0) or 0),
|
||
"recommended_execution_mode": str(launchpad_status.get("recommended_execution_mode") or ""),
|
||
"focus_ref": dict(launchpad_status.get("focus_ref") or {}),
|
||
"error": str(launchpad_payload.get("detail") or launchpad_payload.get("message") or ""),
|
||
},
|
||
"playbook_runs": {
|
||
"http_status": playbook_runs_status,
|
||
"available": playbook_runs_status.startswith("2"),
|
||
"recent_total": len(playbook_runs),
|
||
"problem_runs_total": len(problem_runs),
|
||
"problem_runs": problem_runs[:5],
|
||
"error": str(playbook_runs_root_payload.get("detail") or playbook_runs_root_payload.get("message") or ""),
|
||
},
|
||
"activity_stream": {
|
||
"http_status": activity_status,
|
||
"available": activity_status.startswith("2"),
|
||
"recent_total": len(activities),
|
||
"status_counts": activity_counts,
|
||
"top_items": [
|
||
{
|
||
"kind": str(item.get("kind") or "").strip(),
|
||
"status": str(item.get("status") or item.get("job_status") or "").strip(),
|
||
"summary": str(item.get("summary_text") or item.get("summary") or "").strip(),
|
||
"focus_ref": dict(item.get("focus_ref") or {}),
|
||
}
|
||
for item in activities[:5]
|
||
],
|
||
"error": str(activity_root_payload.get("detail") or activity_root_payload.get("message") or ""),
|
||
},
|
||
}
|
||
|
||
print(json.dumps(summary, ensure_ascii=False, indent=2))
|
||
PY
|