#!/usr/bin/env bash set -euo pipefail BASE_URL="${1:-http://127.0.0.1:8100}" VIEW_MODE="${2:-full}" CURL_CONNECT_TIMEOUT="${CURL_CONNECT_TIMEOUT:-3}" CURL_MAX_TIME="${CURL_MAX_TIME:-8}" FETCH_BODY="" FETCH_STATUS="000" extract_api_data_json() { local raw_json="${1:-}" python3 - "${raw_json}" <<'PY' import json import sys raw_json = str(sys.argv[1] or "") try: payload = json.loads(raw_json) except Exception: raise SystemExit(1) data = payload.get("data") if not isinstance(data, dict): raise SystemExit(1) print(json.dumps(data, ensure_ascii=False, indent=2)) PY } fetch_json() { local url="${1:-}" local raw_text raw_text="$(curl -sS \ --connect-timeout "${CURL_CONNECT_TIMEOUT}" \ --max-time "${CURL_MAX_TIME}" \ "${url}" \ -w $'\nHTTP_STATUS=%{http_code}' || true)" FETCH_BODY="${raw_text%$'\n'HTTP_STATUS=*}" FETCH_STATUS="${raw_text##*$'\n'HTTP_STATUS=}" } fetch_json "${BASE_URL}/api/v1/ops/stack-diagnosis?base_url=${BASE_URL}" STACK_DIAGNOSIS_JSON="${FETCH_BODY}" STACK_DIAGNOSIS_STATUS="${FETCH_STATUS}" if [[ "${STACK_DIAGNOSIS_STATUS}" == 2* ]]; then if STACK_DIAGNOSIS_DATA_JSON="$(extract_api_data_json "${STACK_DIAGNOSIS_JSON}" 2>/dev/null)"; then if [[ "${VIEW_MODE}" != "summary" ]]; then echo "[1/1] ops stack diagnosis (api contract)" printf 'status=%s\n%s\n\n' "${STACK_DIAGNOSIS_STATUS}" "${STACK_DIAGNOSIS_JSON}" fi echo "[9/9] condensed stack summary" STACK_BASE_URL="${BASE_URL}" python3 - <<'PY' "${STACK_DIAGNOSIS_DATA_JSON}" import json import os import sys def ensure_dict(value): return value if isinstance(value, dict) else {} def ensure_list(value): return value if isinstance(value, list) else [] def ensure_str(value): return value if isinstance(value, str) else "" def command(*parts: str) -> str: tokens = ["bash", "domain-api/deploy/multi-region/drive_ops_center.sh"] tokens.extend([str(part).strip() for part in parts if str(part).strip()]) return " ".join(tokens) def release_command(*parts: str) -> str: tokens = ["bash", "domain-api/deploy/multi-region/drive_release_hub.sh"] tokens.extend([str(part).strip() for part in parts if str(part).strip()]) return " ".join(tokens) payload = json.loads(str(sys.argv[1] or "{}")) base_url = ensure_str(os.environ.get("STACK_BASE_URL")) or "http://127.0.0.1:8100" diagnosis = ensure_dict(payload.get("diagnosis")) overview = ensure_dict(payload.get("overview")) managed_nodes = ensure_dict(payload.get("managed_nodes")) release_hub = ensure_dict(payload.get("release_hub")) issues = ensure_list(diagnosis.get("issues")) recommended_actions = ensure_list(diagnosis.get("recommended_actions")) next_step = ensure_dict(diagnosis.get("next_step")) backend_operator_decision = ensure_dict(diagnosis.get("operator_decision")) backend_next_actions = ensure_dict(diagnosis.get("next_actions")) backend_recommended_commands = ensure_dict(diagnosis.get("recommended_commands")) log_sync = ensure_dict(overview.get("log_sync")) problem_nodes = ensure_list(managed_nodes.get("problem_nodes")) next_step_focus_ref = ensure_dict(next_step.get("focus_ref")) launchpad_target_node_code = ensure_str(diagnosis.get("launchpad_recommended_target_node_code")) or ensure_str(next_step_focus_ref.get("node_code")) launchpad_recovery_label = ensure_str(diagnosis.get("launchpad_recommended_recovery_label")) launchpad_recovery_summary = ensure_str(diagnosis.get("launchpad_recommended_recovery_summary")) or ensure_str(next_step.get("reason")) handover_next_action_code = ensure_str(next_step.get("action_code")) scene_log_target_node_code = "" missing_nodes = ensure_list(log_sync.get("missing_sample_node_codes")) if missing_nodes: scene_log_target_node_code = ensure_str(missing_nodes[0]) elif problem_nodes: scene_log_target_node_code = ensure_str(ensure_dict(problem_nodes[0]).get("node_code")) recommended_commands = { "stack_summary": f"bash domain-api/deploy/multi-region/check_ops_center_stack.sh {base_url} summary", "doctor": command("doctor", base_url), "driver_feed": command("driver-feed", base_url), "driver_focus_preview": command("driver-focus-preview", base_url), "driver_focus_run": command("driver-focus-run", base_url), "go_live_recover": command("go-live-recover", base_url), "log_sync_recover": command("log-sync-recover", base_url, "key"), "log_sync_recover_full": command("log-sync-recover", base_url, "full"), "agent_gap_check": command("agent-gap-check", base_url), "agent_gap_recover": command("agent-gap-recover", base_url), "api_restart": command("runtime-refresh-recover", base_url), "node_onboarding": ( command("node-onboarding", base_url, ensure_str(ensure_dict(problem_nodes[0]).get("node_code"))) if problem_nodes and ensure_str(ensure_dict(problem_nodes[0]).get("node_code")) else "" ), "node_recover": ( command("node-recover", base_url, ensure_str(ensure_dict(problem_nodes[0]).get("node_code"))) if problem_nodes and ensure_str(ensure_dict(problem_nodes[0]).get("node_code")) else "" ), "node_bootstrap_preview": ( command("node-bootstrap-preview", base_url, ensure_str(ensure_dict(problem_nodes[0]).get("node_code"))) if problem_nodes and ensure_str(ensure_dict(problem_nodes[0]).get("node_code")) else "" ), "node_bootstrap_run": ( command("node-bootstrap-run", base_url, launchpad_target_node_code, "cli") if launchpad_target_node_code else command("node-bootstrap-run", base_url, ensure_str(ensure_dict(problem_nodes[0]).get("node_code")), "cli") if problem_nodes and ensure_str(ensure_dict(problem_nodes[0]).get("node_code")) else "" ), "node_acceptance_run": ( command("node-acceptance-run", base_url, launchpad_target_node_code, "cli") if launchpad_target_node_code else command("node-acceptance-run", base_url, ensure_str(ensure_dict(problem_nodes[0]).get("node_code")), "cli") if launchpad_target_node_code or (problem_nodes and ensure_str(ensure_dict(problem_nodes[0]).get("node_code"))) else "" ), "node_acceptance_plan": ( command("node-acceptance-plan", base_url, ensure_str(ensure_dict(problem_nodes[0]).get("node_code"))) if problem_nodes and ensure_str(ensure_dict(problem_nodes[0]).get("node_code")) else "" ), "scene_log": ( command("scene-node-log", base_url, scene_log_target_node_code, "120", "key") if scene_log_target_node_code else "" ), "release_launchpad": release_command(base_url, "launchpad"), "release_preview_latest": release_command(base_url, "preview-latest"), "jobs": command("activity-stream", base_url, "kind=ops_job"), "node_handover": ( command("node-handover", base_url, ensure_str(ensure_dict(problem_nodes[0]).get("node_code"))) if problem_nodes and ensure_str(ensure_dict(problem_nodes[0]).get("node_code")) else command("stack-first-gap-handover", base_url) ), } operator_decision = { "lane": "steady", "priority": "low", "reason_code": "steady_state", "title": "当前进入稳定观察态", "summary": "当前总检没有发现新的阻断问题,可继续按既有链路观察与推进。", "next_focus": ensure_dict(next_step.get("focus_ref")), "primary_command_key": "driver_feed", "secondary_command_key": "jobs", } participating_total = int(log_sync.get("participating_nodes_total", 0) or 0) remote_access_ready = int(managed_nodes.get("remote_access_ready", 0) or 0) managed_enabled = int(managed_nodes.get("managed_enabled", 0) or 0) default_rollout_gate_status = ensure_str(release_hub.get("default_rollout_gate_status")) if not bool(log_sync.get("enabled")) and participating_total > 0: operator_decision = { "lane": "observability", "priority": "high", "reason_code": "scene_log_disabled", "title": "先恢复远端现场日志可见性", "summary": "当前已有参与检测节点,但远端日志回传仍关闭,海外控制面仍处于半盲态。", "next_focus": {"kind": "node_scene_log", "node_code": scene_log_target_node_code, "mode": "key", "limit": 80} if scene_log_target_node_code else {}, "primary_command_key": "log_sync_recover", "secondary_command_key": "scene_log", } elif managed_enabled > 0 and remote_access_ready == 0: primary_command_key = "agent_gap_recover" secondary_command_key = "node_handover" title = "先补齐节点接管与 remote-agent 可达性" summary = ( f"managed_enabled={managed_enabled},remote_access_ready={remote_access_ready};" "当前还不能从海外控制面统一下发标准动作。" ) if handover_next_action_code == "bootstrap_run" and launchpad_target_node_code: primary_command_key = "node_bootstrap_run" secondary_command_key = "node_handover" title = "先跑首台节点接入收口" summary = ( launchpad_recovery_summary or join_text( launchpad_recovery_label, launchpad_target_node_code, "当前真正阻断已经收敛到首台节点接入收口。" ) ) elif handover_next_action_code == "run_acceptance" and launchpad_target_node_code: primary_command_key = "node_acceptance_run" secondary_command_key = "node_handover" title = "先跑首台节点接管验收" summary = ( launchpad_recovery_summary or join_text( launchpad_recovery_label, launchpad_target_node_code, "当前真正阻断已经收敛到首台节点接管验收。" ) ) operator_decision = { "lane": "node_handover", "priority": "high", "reason_code": "managed_nodes_agent_pending", "title": title, "summary": summary, "next_focus": next_step_focus_ref, "primary_command_key": primary_command_key, "secondary_command_key": secondary_command_key, } elif default_rollout_gate_status == "blocked": operator_decision = { "lane": "release", "priority": "medium", "reason_code": "release_rollout_gate_blocked", "title": "先检查发布门禁与 rollout 阻断", "summary": ensure_str(release_hub.get("default_rollout_gate_summary")), "next_focus": ensure_dict(release_hub.get("default_rollout_gate_focus_ref") or release_hub.get("focus_ref")), "primary_command_key": "release_launchpad", "secondary_command_key": "release_preview_latest", } elif any(ensure_str(ensure_dict(item).get("code")) == "runtime_build_schema_stale" for item in issues): operator_decision = { "lane": "runtime_recovery", "priority": "high", "reason_code": "runtime_build_schema_stale", "title": "先执行运行时刷新与上线复核", "summary": "当前已经识别到仓库代码与运行中 API schema 之间存在漂移,先走统一恢复链,再继续节点接管或发布判断。", "next_focus": ensure_dict(next_step.get("focus_ref")), "primary_command_key": "go_live_recover", "secondary_command_key": "api_restart", } elif recommended_actions or issues: operator_decision = { "lane": "ops_jobs", "priority": "medium", "reason_code": "issues_need_attention", "title": "先按总检建议逐项收口当前问题", "summary": ( f"recommended_actions={len(recommended_actions)},issues={len(issues)}," f"stack_status={ensure_str(diagnosis.get('stack_status')) or 'unknown'}。" ), "next_focus": ensure_dict(next_step.get("focus_ref")), "primary_command_key": "doctor", "secondary_command_key": "jobs", } if backend_recommended_commands: merged_recommended_commands = dict(recommended_commands) merged_recommended_commands.update({ key: ensure_str(value) for key, value in backend_recommended_commands.items() if ensure_str(key) and ensure_str(value) }) recommended_commands = merged_recommended_commands if backend_operator_decision: operator_decision = { "lane": ensure_str(backend_operator_decision.get("lane")) or ensure_str(operator_decision.get("lane")), "priority": ensure_str(backend_operator_decision.get("priority")) or ensure_str(operator_decision.get("priority")), "reason_code": ensure_str(backend_operator_decision.get("reason_code")) or ensure_str(operator_decision.get("reason_code")), "title": ensure_str(backend_operator_decision.get("title")) or ensure_str(operator_decision.get("title")), "summary": ensure_str(backend_operator_decision.get("summary")) or ensure_str(operator_decision.get("summary")), "next_focus": ensure_dict(backend_operator_decision.get("next_focus")) or ensure_dict(operator_decision.get("next_focus")), "primary_command_key": ensure_str(backend_operator_decision.get("primary_command_key")) or ensure_str(operator_decision.get("primary_command_key")), "secondary_command_key": ensure_str(backend_operator_decision.get("secondary_command_key")) or ensure_str(operator_decision.get("secondary_command_key")), } next_actions = { "primary_command_key": ensure_str(operator_decision.get("primary_command_key")), "primary_command": ensure_str(recommended_commands.get(ensure_str(operator_decision.get("primary_command_key")))), "secondary_command_key": ensure_str(operator_decision.get("secondary_command_key")), "secondary_command": ensure_str(recommended_commands.get(ensure_str(operator_decision.get("secondary_command_key")))), } if backend_next_actions: next_actions["primary_command_key"] = ensure_str(backend_next_actions.get("primary_command_key")) or next_actions["primary_command_key"] next_actions["secondary_command_key"] = ensure_str(backend_next_actions.get("secondary_command_key")) or next_actions["secondary_command_key"] next_actions["primary_command"] = ( ensure_str(backend_next_actions.get("primary_command")) or ensure_str(recommended_commands.get(next_actions["primary_command_key"])) or next_actions["primary_command"] ) next_actions["secondary_command"] = ( ensure_str(backend_next_actions.get("secondary_command")) or ensure_str(recommended_commands.get(next_actions["secondary_command_key"])) or next_actions["secondary_command"] ) diagnosis["operator_decision"] = operator_decision diagnosis["next_actions"] = next_actions diagnosis["recommended_commands"] = recommended_commands payload["diagnosis"] = diagnosis print(json.dumps(payload, ensure_ascii=False, indent=2)) PY exit 0 fi fi fetch_json "${BASE_URL}/health" HEALTH_JSON="${FETCH_BODY}" HEALTH_STATUS="${FETCH_STATUS}" fetch_json "${BASE_URL}/api/v1/ops/contracts" CONTRACTS_JSON="${FETCH_BODY}" CONTRACTS_STATUS="${FETCH_STATUS}" fetch_json "${BASE_URL}/api/v1/ops/link-snapshot" LINK_SNAPSHOT_JSON="${FETCH_BODY}" LINK_SNAPSHOT_STATUS="${FETCH_STATUS}" fetch_json "${BASE_URL}/api/v1/ops/overview" OVERVIEW_JSON="${FETCH_BODY}" OVERVIEW_STATUS="${FETCH_STATUS}" fetch_json "${BASE_URL}/api/v1/ops/nodes" NODES_JSON="${FETCH_BODY}" NODES_STATUS="${FETCH_STATUS}" fetch_json "${BASE_URL}/api/v1/ops/releases/launchpad" LAUNCHPAD_JSON="${FETCH_BODY}" LAUNCHPAD_STATUS="${FETCH_STATUS}" fetch_json "${BASE_URL}/api/v1/ops/playbook-runs?limit=6" PLAYBOOK_RUNS_JSON="${FETCH_BODY}" PLAYBOOK_RUNS_STATUS="${FETCH_STATUS}" fetch_json "${BASE_URL}/api/v1/ops/activity-stream?limit=8" ACTIVITY_JSON="${FETCH_BODY}" ACTIVITY_STATUS="${FETCH_STATUS}" if [[ "${VIEW_MODE}" != "summary" ]]; then echo "[1/9] api health" printf 'status=%s\n%s\n\n' "${HEALTH_STATUS}" "${HEALTH_JSON}" echo "[2/9] ops contracts" printf 'status=%s\n%s\n\n' "${CONTRACTS_STATUS}" "${CONTRACTS_JSON}" echo "[3/9] ops link snapshot" printf 'status=%s\n%s\n\n' "${LINK_SNAPSHOT_STATUS}" "${LINK_SNAPSHOT_JSON}" echo "[4/9] ops overview" printf 'status=%s\n%s\n\n' "${OVERVIEW_STATUS}" "${OVERVIEW_JSON}" echo "[5/9] managed nodes" printf 'status=%s\n%s\n\n' "${NODES_STATUS}" "${NODES_JSON}" echo "[6/9] release launchpad" printf 'status=%s\n%s\n\n' "${LAUNCHPAD_STATUS}" "${LAUNCHPAD_JSON}" echo "[7/9] recent playbook runs" printf 'status=%s\n%s\n\n' "${PLAYBOOK_RUNS_STATUS}" "${PLAYBOOK_RUNS_JSON}" echo "[8/9] recent activity stream" printf 'status=%s\n%s\n\n' "${ACTIVITY_STATUS}" "${ACTIVITY_JSON}" fi echo "[9/9] condensed stack summary" STACK_BASE_URL="${BASE_URL}" python3 - <<'PY' \ "${STACK_DIAGNOSIS_JSON}" \ "${STACK_DIAGNOSIS_STATUS}" \ "${HEALTH_JSON}" \ "${HEALTH_STATUS}" \ "${CONTRACTS_JSON}" \ "${CONTRACTS_STATUS}" \ "${LINK_SNAPSHOT_JSON}" \ "${LINK_SNAPSHOT_STATUS}" \ "${OVERVIEW_JSON}" \ "${OVERVIEW_STATUS}" \ "${NODES_JSON}" \ "${NODES_STATUS}" \ "${LAUNCHPAD_JSON}" \ "${LAUNCHPAD_STATUS}" \ "${PLAYBOOK_RUNS_JSON}" \ "${PLAYBOOK_RUNS_STATUS}" \ "${ACTIVITY_JSON}" \ "${ACTIVITY_STATUS}" import json import os import sys def ensure_str(value): return value if isinstance(value, str) else "" def load_payload(raw_text: str) -> dict: try: return json.loads(raw_text) except Exception: return {} def as_data(payload: dict) -> dict: if isinstance(payload.get("data"), dict): return dict(payload.get("data") or {}) return {} def issue_rank(severity: str) -> int: return {"blocked": 3, "warning": 2, "info": 1}.get(str(severity or "").strip(), 0) def command(*parts: str) -> str: tokens = ["bash", "domain-api/deploy/multi-region/drive_ops_center.sh"] tokens.extend([str(part).strip() for part in parts if str(part).strip()]) return " ".join(tokens) def release_command(*parts: str) -> str: tokens = ["bash", "domain-api/deploy/multi-region/drive_release_hub.sh"] tokens.extend([str(part).strip() for part in parts if str(part).strip()]) return " ".join(tokens) stack_diagnosis_payload = load_payload(sys.argv[1]) STACK_DIAGNOSIS_STATUS = str(sys.argv[2] or "000") health = load_payload(sys.argv[3]) health_status = str(sys.argv[4] or "000") contracts_payload = load_payload(sys.argv[5]) contracts_status = str(sys.argv[6] or "000") link_snapshot_payload = load_payload(sys.argv[7]) link_snapshot_status = str(sys.argv[8] or "000") overview_payload = load_payload(sys.argv[9]) overview_status = str(sys.argv[10] or "000") nodes_root_payload = load_payload(sys.argv[11]) nodes_status = str(sys.argv[12] or "000") launchpad_payload = load_payload(sys.argv[13]) launchpad_status_code = str(sys.argv[14] or "000") playbook_runs_root_payload = load_payload(sys.argv[15]) playbook_runs_status = str(sys.argv[16] or "000") activity_root_payload = load_payload(sys.argv[17]) activity_status = str(sys.argv[18] or "000") contracts = as_data(contracts_payload) link_snapshot = as_data(link_snapshot_payload) overview = as_data(overview_payload) nodes_payload = as_data(nodes_root_payload) launchpad = as_data(launchpad_payload) playbook_runs_payload = as_data(playbook_runs_root_payload) activity_payload = as_data(activity_root_payload) stack_base_url = str(os.environ.get("STACK_BASE_URL") or "").strip() health_build = dict(health.get("build") or {}) health_route_surface = dict(health_build.get("route_surface") or {}) stack_diagnosis = as_data(stack_diagnosis_payload) diagnosis = dict(stack_diagnosis.get("diagnosis") or {}) contracts_rows = list(contracts.get("contracts") or []) nodes = list(nodes_payload.get("nodes") or []) nodes_summary = dict(nodes_payload.get("summary") or {}) playbook_runs = list(playbook_runs_payload.get("runs") or playbook_runs_payload.get("playbook_runs") or []) activities = list(activity_payload.get("items") or activity_payload.get("activities") or []) recommendation = dict(overview.get("recommendation") or {}) execution_scene = dict(overview.get("execution_scene") or {}) log_sync = dict(execution_scene.get("log_sync") or overview.get("log_sync") or {}) release_hub = dict(overview.get("release_hub") or {}) default_rollout_gate = dict(release_hub.get("default_rollout_gate") or {}) launchpad_status = dict(launchpad.get("launchpad_status") or {}) latest_release = dict(launchpad.get("latest_release") or release_hub.get("latest_release") or {}) problem_nodes = [] for node in nodes: node_code = str(node.get("node_code") or "").strip() if not node_code: continue queue_state = str(node.get("delivery_queue_state") or "").strip() agent_state = str(node.get("agent_state") or "").strip() remote_access_state = str(node.get("remote_access_state") or "").strip() if queue_state == "dead_letter" or agent_state not in {"online", "online_busy"} or remote_access_state in { "agent_pending", "disabled", "unmanaged", }: problem_nodes.append( { "node_code": node_code, "agent_state": agent_state, "remote_access_state": remote_access_state, "delivery_queue_state": queue_state, "participation_state": str(node.get("participation_state") or "").strip(), } ) problem_runs = [ { "run_code": str(item.get("run_code") or "").strip(), "status": str(item.get("status") or "").strip(), "status_label": str(item.get("status_label") or "").strip(), "focus_step_key": str(item.get("focus_step_key") or "").strip(), "focus_summary": str(item.get("focus_summary") or item.get("summary_text") or "").strip(), } for item in playbook_runs if str(item.get("status") or "").strip() not in {"success", "completed", "healthy"} ] activity_counts = {} for item in activities: status_key = str(item.get("status") or item.get("job_status") or "unknown").strip() or "unknown" activity_counts[status_key] = int(activity_counts.get(status_key, 0) or 0) + 1 issues = [] def add_issue( *, code: str, severity: str, layer: str, summary: str, detail: str = "", action_code: str = "", focus_ref=None, commands=None, ) -> None: issues.append( { "code": str(code or "").strip(), "severity": str(severity or "").strip() or "warning", "layer": str(layer or "").strip(), "summary": str(summary or "").strip(), "detail": str(detail or "").strip(), "action_code": str(action_code or "").strip(), "focus_ref": dict(focus_ref or {}), "commands": [str(item).strip() for item in list(commands or []) if str(item).strip()], } ) surface_matrix = [ ("stack_diagnosis", STACK_DIAGNOSIS_STATUS.startswith("2")), ("api", health_status.startswith("2")), ("contracts", contracts_status.startswith("2")), ("link_snapshot", link_snapshot_status.startswith("2")), ("overview", overview_status.startswith("2")), ("managed_nodes", nodes_status.startswith("2")), ("release_hub", launchpad_status_code.startswith("2")), ("playbook_runs", playbook_runs_status.startswith("2")), ("activity_stream", activity_status.startswith("2")), ] surface_status_counts = { "total": len(surface_matrix), "available": sum(1 for _, available in surface_matrix if available), "missing": sum(1 for _, available in surface_matrix if not available), } missing_surfaces = [name for name, available in surface_matrix if not available] if not health_status.startswith("2"): add_issue( code="api_unreachable", severity="blocked", layer="api", summary="控制面 API 当前不可用,总检无法进入正式判断链路。", detail=f"/health 返回 HTTP {health_status}。", commands=[ f"curl -s {stack_base_url}/health", "systemctl status domaincheck-api --no-pager -l", ], ) stack_diagnosis_hint = ( "当前 stack-diagnosis 没有成功返回,总检已退化为多 surface 拼装视图。" ) if not STACK_DIAGNOSIS_STATUS.startswith("2"): stack_diag_detail = ( f"/api/v1/ops/stack-diagnosis 返回 HTTP {STACK_DIAGNOSIS_STATUS}。" if str(STACK_DIAGNOSIS_STATUS).isdigit() and str(STACK_DIAGNOSIS_STATUS) != "000" else "/api/v1/ops/stack-diagnosis 当前不可达或请求超时。" ) add_issue( code="stack_diagnosis_unavailable", severity="warning" if health_status.startswith("2") else "blocked", layer="stack_diagnosis", summary=stack_diagnosis_hint, detail=( stack_diag_detail + " 建议先单独复跑 stack-diagnosis / stack-next,确认是不是 API 慢、路由未挂齐,或者服务未切到最新版本。" ), commands=[ f"curl -s {stack_base_url}/api/v1/ops/stack-diagnosis?base_url={stack_base_url}", f"bash domain-api/deploy/multi-region/drive_ops_center.sh stack-next {stack_base_url}", "systemctl status domaincheck-api --no-pager -l", ], ) if health_status.startswith("2") and not contracts_status.startswith("2"): add_issue( code="ops_contracts_endpoint_missing", severity="warning", layer="contracts", summary="控制面 API 已在线,但 ops/contracts 未正常返回,存在运行实例与当前代码契约不一致的风险。", detail=( f"/api/v1/ops/contracts 返回 HTTP {contracts_status};" "这通常意味着当前 systemd 进程没有跑到最新接口集,或部署包/服务重启未完全对齐。" ), commands=[ f"curl -s {stack_base_url}/api/v1/ops/contracts", "systemctl status domaincheck-api --no-pager -l", f"bash domain-api/deploy/multi-region/drive_ops_center.sh runtime-refresh-recover {stack_base_url}", ], ) if health_status.startswith("2") and not health_build: add_issue( code="api_build_identity_missing", severity="warning", layer="api", summary="控制面 health 还没有暴露 build 指纹,当前不利于快速判断服务是否真的切到最新部署内容。", detail="建议升级到包含 build + route_surface 指纹的新版本后再继续多机联调。", commands=[ f"curl -s {stack_base_url}/health", f"bash domain-api/deploy/multi-region/drive_ops_center.sh runtime-refresh-recover {stack_base_url}", ], ) if health_status.startswith("2") and health_route_surface and not bool(health_route_surface.get("surface_complete", True)): add_issue( code="api_route_surface_incomplete", severity="warning", layer="api.routes", summary="当前运行中的控制面实例没有挂齐关键协议路由,存在代码与服务实例不一致的风险。", detail=( "缺少路由: " + ", ".join([str(item).strip() for item in list(health_route_surface.get("missing_paths") or []) if str(item).strip()]) ), commands=[ f"curl -s {stack_base_url}/health", f"bash domain-api/deploy/multi-region/drive_ops_center.sh runtime-refresh-recover {stack_base_url}", ], ) if health_status.startswith("2") and missing_surfaces and any( name in missing_surfaces for name in ("link_snapshot", "overview", "managed_nodes", "release_hub") ): add_issue( code="ops_surface_partial", severity="warning", layer="stack", summary="海外单脑控制面关键观察层没有全部联通,总检已退化为部分收口。", detail=f"当前缺失层: {', '.join(missing_surfaces)}", commands=[ f"bash domain-api/deploy/multi-region/drive_ops_center.sh stack-diagnosis {stack_base_url} full", f"bash domain-api/deploy/multi-region/drive_ops_center.sh doctor {stack_base_url}", ], ) participating_nodes_total = int(log_sync.get("participating_nodes_total", log_sync.get("participating_node_count", 0)) or 0) if bool(log_sync.get("enabled", False)) is False and participating_nodes_total > 0: log_sync_action = "" next_action_codes = list(link_snapshot.get("next_actions") or []) for item in next_action_codes: action_code = str(item.get("action_code") or "").strip() if action_code.startswith("enable_log_sync"): log_sync_action = action_code break if not log_sync_action: log_sync_action = "enable_log_sync_key" add_issue( code="remote_log_sync_disabled", severity="warning", layer="overview.log_sync", summary="现场已有参与检测节点,但远端日志回传仍关闭,海外控制面对执行细节仍处于半盲态。", detail=( f"参与检测节点 {participating_nodes_total} 台," f"已覆盖 {int(log_sync.get('covered_participating_nodes', log_sync.get('covered_participating_node_count', 0)) or 0)} 台。" ), action_code=log_sync_action, commands=[ f"bash domain-api/deploy/multi-region/drive_ops_center.sh driver-resolve {stack_base_url} {log_sync_action}", ], ) managed_enabled = int(nodes_summary.get("managed_enabled", 0) or 0) remote_access_ready = int(nodes_summary.get("remote_access_ready", 0) or 0) if managed_enabled > 0 and remote_access_ready == 0: add_issue( code="managed_nodes_agent_pending", severity="blocked", layer="managed_nodes", summary="托管节点虽然已经纳入 Ops Center,但 0 台 remote-agent 就绪,海外主控还不能统一下发标准执行动作。", detail=( f"managed_enabled={managed_enabled},remote_access_ready={remote_access_ready};" "当前节点多数仍处于 agent_pending / runtime_only / pending_bootstrap。" ), action_code=str(launchpad_status.get("recommended_action_code") or "fix_managed_nodes"), focus_ref=dict(launchpad_status.get("focus_ref") or {}), commands=[ f"bash domain-api/deploy/multi-region/drive_ops_center.sh driver-resolve {stack_base_url} fix_managed_nodes", f"bash domain-api/deploy/multi-region/drive_ops_center.sh doctor {stack_base_url}", ], ) queue_dead_letter_nodes = int(nodes_summary.get("queue_dead_letter_nodes", 0) or 0) queue_dead_letter_records = int(nodes_summary.get("queue_dead_letter_records", 0) or 0) if queue_dead_letter_nodes > 0 or queue_dead_letter_records > 0: add_issue( code="delivery_queue_dead_letter", severity="warning", layer="managed_nodes.queue", summary="存在 Node Agent 回执死信,后续自动化动作可能反复失败或无法收敛。", detail=( f"queue_dead_letter_nodes={queue_dead_letter_nodes}," f"queue_dead_letter_records={queue_dead_letter_records}" ), commands=[ f"bash domain-api/deploy/multi-region/drive_ops_center.sh doctor {stack_base_url}", ], ) if str(default_rollout_gate.get("status") or "").strip() == "blocked": add_issue( code="release_rollout_gate_blocked", severity="warning", layer="release_hub", summary="ReleaseHub 默认门禁当前阻断,说明虽然发布包已准备好,但正式 Rollout 还不具备条件。", detail=str(default_rollout_gate.get("summary_text") or default_rollout_gate.get("summary") or "").strip(), action_code=str(launchpad_status.get("recommended_action_code") or "").strip(), focus_ref=dict(default_rollout_gate.get("focus_ref") or launchpad_status.get("focus_ref") or {}), commands=[ f"bash domain-api/deploy/multi-region/drive_ops_center.sh release-launchpad {stack_base_url}", f"bash domain-api/deploy/multi-region/drive_ops_center.sh driver-resolve {stack_base_url} {str(launchpad_status.get('recommended_action_code') or 'fix_managed_nodes').strip()}", ], ) if problem_runs: add_issue( code="playbook_runs_need_attention", severity="warning", layer="playbook_runs", summary="最近存在未完全收口的 playbook run,建议先确认执行序列是否停在中间步骤。", detail=f"problem_runs_total={len(problem_runs)}", commands=[ f"bash domain-api/deploy/multi-region/drive_ops_center.sh doctor {stack_base_url}", ], ) ordered_action_candidates = [ { "source": "overview.primary", "action_code": str(recommendation.get("primary_action_code") or "").strip(), "title": str(recommendation.get("title") or "").strip(), "reason": "来自 overview.recommendation.primary_action_code", "focus_ref": dict(recommendation.get("focus_ref") or {}), }, { "source": "release_hub.launchpad", "action_code": str(launchpad_status.get("recommended_action_code") or "").strip(), "title": str(launchpad_status.get("status_label") or "").strip(), "reason": "来自 release_launchpad.launchpad_status.recommended_action_code", "focus_ref": dict(launchpad_status.get("focus_ref") or {}), }, { "source": "overview.secondary", "action_code": str(recommendation.get("secondary_action_code") or "").strip(), "title": str(recommendation.get("title") or "").strip(), "reason": "来自 overview.recommendation.secondary_action_code", "focus_ref": dict(recommendation.get("focus_ref") or {}), }, ] for issue in issues: issue_action_code = str(issue.get("action_code") or "").strip() if issue_action_code: ordered_action_candidates.append( { "source": f"issue:{issue.get('code')}", "action_code": issue_action_code, "title": str(issue.get("summary") or "").strip(), "reason": str(issue.get("detail") or issue.get("summary") or "").strip(), "focus_ref": dict(issue.get("focus_ref") or {}), } ) recommended_actions = [] seen_action_codes = set() for item in ordered_action_candidates: action_code = str(item.get("action_code") or "").strip() if not action_code or action_code in seen_action_codes: continue seen_action_codes.add(action_code) recommended_actions.append( { "action_code": action_code, "source": str(item.get("source") or "").strip(), "title": str(item.get("title") or "").strip(), "reason": str(item.get("reason") or "").strip(), "focus_ref": dict(item.get("focus_ref") or {}), "command": f"bash domain-api/deploy/multi-region/drive_ops_center.sh driver-resolve {stack_base_url} {action_code}", } ) blocking_issue_total = sum(1 for item in issues if str(item.get("severity") or "").strip() == "blocked") warning_issue_total = sum(1 for item in issues if str(item.get("severity") or "").strip() == "warning") info_issue_total = sum(1 for item in issues if str(item.get("severity") or "").strip() == "info") surface_status = "healthy" if not health_status.startswith("2"): surface_status = "broken" elif missing_surfaces: surface_status = "partial" automation_status = "ready" if blocking_issue_total > 0: automation_status = "blocked" elif warning_issue_total > 0: automation_status = "attention" stack_status = "ready" if surface_status == "broken" or automation_status == "blocked": stack_status = "blocked" elif surface_status != "healthy" or automation_status != "ready": stack_status = "attention" top_issue = None if issues: top_issue = sorted( issues, key=lambda item: ( -issue_rank(str(item.get("severity") or "").strip()), str(item.get("code") or "").strip(), ), )[0] next_step = {} if recommended_actions: first_action = dict(recommended_actions[0] or {}) next_step = { "action_code": str(first_action.get("action_code") or "").strip(), "source": str(first_action.get("source") or "").strip(), "reason": str(first_action.get("reason") or "").strip(), "command": str(first_action.get("command") or "").strip(), "focus_ref": dict(first_action.get("focus_ref") or {}), } elif top_issue: next_step = { "action_code": "", "source": f"issue:{str(top_issue.get('code') or '').strip()}", "reason": str(top_issue.get("summary") or "").strip(), "command": "", "focus_ref": dict(top_issue.get("focus_ref") or {}), } next_step_focus_ref = dict(next_step.get("focus_ref") or {}) launchpad_target_node_code = str( diagnosis.get("launchpad_recommended_target_node_code") or next_step_focus_ref.get("node_code") or "" ).strip() launchpad_recovery_label = str(diagnosis.get("launchpad_recommended_recovery_label") or "").strip() launchpad_recovery_summary = str(diagnosis.get("launchpad_recommended_recovery_summary") or next_step.get("reason") or "").strip() handover_next_action_code = str(next_step.get("action_code") or "").strip() scene_log_missing_nodes = list( log_sync.get("missing_sample_node_codes") or log_sync.get("missing_participating_nodes") or [] ) scene_log_target_node_code = "" if scene_log_missing_nodes: scene_log_target_node_code = str(scene_log_missing_nodes[0] or "").strip() elif problem_nodes: scene_log_target_node_code = str(problem_nodes[0].get("node_code") or "").strip() recommended_commands = { "stack_summary": f"bash domain-api/deploy/multi-region/check_ops_center_stack.sh {stack_base_url} summary", "doctor": command("doctor", stack_base_url), "driver_feed": command("driver-feed", stack_base_url), "driver_focus_preview": command("driver-focus-preview", stack_base_url), "driver_focus_run": command("driver-focus-run", stack_base_url), "go_live_recover": command("go-live-recover", stack_base_url), "log_sync_recover": command("log-sync-recover", stack_base_url, "key"), "log_sync_recover_full": command("log-sync-recover", stack_base_url, "full"), "agent_gap_check": command("agent-gap-check", stack_base_url), "agent_gap_recover": command("agent-gap-recover", stack_base_url), "api_restart": command("runtime-refresh-recover", stack_base_url), "node_onboarding": ( command("node-onboarding", stack_base_url, str(problem_nodes[0].get("node_code") or "").strip()) if problem_nodes and str(problem_nodes[0].get("node_code") or "").strip() else "" ), "node_recover": ( command("node-recover", stack_base_url, str(problem_nodes[0].get("node_code") or "").strip()) if problem_nodes and str(problem_nodes[0].get("node_code") or "").strip() else "" ), "node_bootstrap_preview": ( command("node-bootstrap-preview", stack_base_url, str(problem_nodes[0].get("node_code") or "").strip()) if problem_nodes and str(problem_nodes[0].get("node_code") or "").strip() else "" ), "node_bootstrap_run": ( command("node-bootstrap-run", stack_base_url, launchpad_target_node_code, "cli") if launchpad_target_node_code else command("node-bootstrap-run", stack_base_url, str(problem_nodes[0].get("node_code") or "").strip(), "cli") if launchpad_target_node_code or (problem_nodes and str(problem_nodes[0].get("node_code") or "").strip()) else "" ), "node_acceptance_run": ( command("node-acceptance-run", stack_base_url, launchpad_target_node_code, "cli") if launchpad_target_node_code else command("node-acceptance-run", stack_base_url, str(problem_nodes[0].get("node_code") or "").strip(), "cli") if launchpad_target_node_code or (problem_nodes and str(problem_nodes[0].get("node_code") or "").strip()) else "" ), "node_acceptance_plan": ( command("node-acceptance-plan", stack_base_url, str(problem_nodes[0].get("node_code") or "").strip()) if problem_nodes and str(problem_nodes[0].get("node_code") or "").strip() else "" ), "scene_log": ( command("scene-node-log", stack_base_url, scene_log_target_node_code, "120", "key") if scene_log_target_node_code else "" ), "release_launchpad": release_command(stack_base_url, "launchpad"), "release_preview_latest": release_command(stack_base_url, "preview-latest"), "jobs": command("activity-stream", stack_base_url, "kind=ops_job"), "node_handover": ( command("node-handover", stack_base_url, str(problem_nodes[0].get("node_code") or "").strip()) if problem_nodes and str(problem_nodes[0].get("node_code") or "").strip() else command("stack-first-gap-handover", stack_base_url) ), } operator_decision = { "lane": "steady", "priority": "low", "reason_code": "steady_state", "title": "当前进入稳定观察态", "summary": "当前总检没有发现新的阻断问题,可继续按既有链路观察与推进。", "next_focus": next_step_focus_ref, "primary_command_key": "driver_feed", "secondary_command_key": "jobs", } if str(log_sync.get("enabled", False)).lower() in {"false", ""} and participating_nodes_total > 0: operator_decision = { "lane": "observability", "priority": "high", "reason_code": "scene_log_disabled", "title": "先恢复远端现场日志可见性", "summary": "当前已有参与检测节点,但远端日志回传仍关闭,海外控制面仍处于半盲态。", "next_focus": {"kind": "node_scene_log", "node_code": scene_log_target_node_code, "mode": "key", "limit": 80} if scene_log_target_node_code else {}, "primary_command_key": "log_sync_recover", "secondary_command_key": "scene_log", } elif managed_enabled > 0 and remote_access_ready == 0: primary_command_key = "agent_gap_recover" secondary_command_key = "node_handover" title = "先补齐节点接管与 remote-agent 可达性" summary = ( f"managed_enabled={managed_enabled},remote_access_ready={remote_access_ready};" "当前还不能从海外控制面统一下发标准动作。" ) if handover_next_action_code == "bootstrap_run" and launchpad_target_node_code: primary_command_key = "node_bootstrap_run" secondary_command_key = "node_handover" title = "先跑首台节点接入收口" summary = ( launchpad_recovery_summary or join_text( launchpad_recovery_label, launchpad_target_node_code, "当前真正阻断已经收敛到首台节点接入收口。" ) ) elif handover_next_action_code == "run_acceptance" and launchpad_target_node_code: primary_command_key = "node_acceptance_run" secondary_command_key = "node_handover" title = "先跑首台节点接管验收" summary = ( launchpad_recovery_summary or join_text( launchpad_recovery_label, launchpad_target_node_code, "当前真正阻断已经收敛到首台节点接管验收。" ) ) operator_decision = { "lane": "node_handover", "priority": "high", "reason_code": "managed_nodes_agent_pending", "title": title, "summary": summary, "next_focus": next_step_focus_ref, "primary_command_key": primary_command_key, "secondary_command_key": secondary_command_key, } elif str(default_rollout_gate.get("status") or "").strip() == "blocked": operator_decision = { "lane": "release", "priority": "medium", "reason_code": "release_rollout_gate_blocked", "title": "先检查发布门禁与 rollout 阻断", "summary": str(default_rollout_gate.get("summary_text") or default_rollout_gate.get("summary") or "").strip(), "next_focus": dict(default_rollout_gate.get("focus_ref") or launchpad_status.get("focus_ref") or {}), "primary_command_key": "release_launchpad", "secondary_command_key": "release_preview_latest", } elif any(str(item.get("code") or "").strip() == "runtime_build_schema_stale" for item in issues): operator_decision = { "lane": "runtime_recovery", "priority": "high", "reason_code": "runtime_build_schema_stale", "title": "先执行运行时刷新与上线复核", "summary": "当前已经识别到仓库代码与运行中 API schema 之间存在漂移,先走统一恢复链,再继续节点接管或发布判断。", "next_focus": dict(next_step.get("focus_ref") or {}), "primary_command_key": "go_live_recover", "secondary_command_key": "api_restart", } elif problem_runs: operator_decision = { "lane": "ops_jobs", "priority": "medium", "reason_code": "playbook_runs_need_attention", "title": "先确认未收口的 playbook run", "summary": f"最近存在 {len(problem_runs)} 个未完全收口的 playbook run。", "next_focus": dict(next_step.get("focus_ref") or {}), "primary_command_key": "doctor", "secondary_command_key": "jobs", } next_actions = { "primary_command": str(recommended_commands.get(str(operator_decision.get("primary_command_key") or "").strip()) or "").strip(), "secondary_command": str(recommended_commands.get(str(operator_decision.get("secondary_command_key") or "").strip()) or "").strip(), } quick_commands = [ f"bash domain-api/deploy/multi-region/drive_ops_center.sh stack-diagnosis {stack_base_url} summary", f"bash domain-api/deploy/multi-region/drive_ops_center.sh doctor {stack_base_url}", ] for action in recommended_actions[:3]: command = str(action.get("command") or "").strip() if command and command not in quick_commands: quick_commands.append(command) if not contracts_status.startswith("2"): contract_check_command = f"curl -s {stack_base_url}/api/v1/ops/contracts" if contract_check_command not in quick_commands: quick_commands.append(contract_check_command) operator_hints = [] if stack_status == "blocked": operator_hints.append("当前总检已经能看到现场,但海外单脑自动化执行链路仍未正式打通,先处理阻断问题再继续 Rollout 或批量运维。") elif stack_status == "attention": operator_hints.append("当前总检可用,但仍有缺口;建议先按 next_step 收口,再进入更细粒度脚本。") else: operator_hints.append("当前海外单脑控制面总检已基本就绪,可以把它作为后续所有联调和发布的固定起手式。") if not contracts_status.startswith("2") and health_status.startswith("2"): operator_hints.append("health 正常但 contracts 缺失,优先怀疑运行中的 API 实例不是这份最新代码,或服务重启后未真正切到最新部署内容。") if bool(log_sync.get("enabled", False)) is False and participating_nodes_total > 0: operator_hints.append("远端日志回传当前关闭,后续如果继续多机联调,海外控制面会继续处于半盲态。") summary = { "diagnosis": { "base_url": stack_base_url, "surface_status": surface_status, "automation_status": automation_status, "stack_status": stack_status, "issue_total": len(issues), "blocking_issue_total": blocking_issue_total, "warning_issue_total": warning_issue_total, "info_issue_total": info_issue_total, "missing_surfaces": missing_surfaces, "launchpad_recommended_target_node_code": ensure_str(diagnosis.get("launchpad_recommended_target_node_code")), "launchpad_recommended_recovery_label": ensure_str(diagnosis.get("launchpad_recommended_recovery_label")), "launchpad_recommended_recovery_summary": ensure_str(diagnosis.get("launchpad_recommended_recovery_summary")), "launchpad_onboarding_bootstrap_pending_nodes": int(diagnosis.get("launchpad_onboarding_bootstrap_pending_nodes", 0) or 0), "launchpad_onboarding_acceptance_ready_nodes": int(diagnosis.get("launchpad_onboarding_acceptance_ready_nodes", 0) or 0), "next_step": next_step, "recommended_actions": recommended_actions[:5], "operator_decision": operator_decision, "next_actions": next_actions, "issues": issues[:8], "operator_hints": operator_hints, "quick_commands": quick_commands, "recommended_commands": recommended_commands, }, "api": { "stack_diagnosis_http_status": STACK_DIAGNOSIS_STATUS, "stack_diagnosis_available": STACK_DIAGNOSIS_STATUS.startswith("2"), "http_status": health_status, "available": health_status.startswith("2"), "service": str(health.get("service") or ""), "version": str(health.get("version") or ""), "api_prefix": str(health.get("api_prefix") or ""), "worker_mode": str(health.get("worker_mode") or ""), "build": { "source": str(health_build.get("source") or ""), "package_name": str(health_build.get("package_name") or ""), "generated_at": str(health_build.get("generated_at") or ""), "commit_sha": str(health_build.get("commit_sha") or ""), "commit_ref": str(health_build.get("commit_ref") or ""), "manifest_path": str(health_build.get("manifest_path") or ""), "route_surface_complete": bool(health_route_surface.get("surface_complete", False)) if health_route_surface else False, "route_surface_missing": [ str(item).strip() for item in list(health_route_surface.get("missing_paths") or []) if str(item).strip() ], }, }, "surface_matrix": { "status_counts": surface_status_counts, "items": [ {"name": name, "available": bool(available)} for name, available in surface_matrix ], }, "contracts": { "http_status": contracts_status, "available": contracts_status.startswith("2"), "registry_version": str(contracts.get("registry_version") or ""), "schema_version": str(contracts.get("schema_version") or ""), "contracts_total": len(contracts_rows), "contract_keys": [str(item.get("key") or "").strip() for item in contracts_rows if str(item.get("key") or "").strip()], "error": str(contracts_payload.get("detail") or contracts_payload.get("message") or ""), }, "link_snapshot": { "http_status": link_snapshot_status, "available": link_snapshot_status.startswith("2"), "status": str(link_snapshot.get("status") or ""), "headline": str(link_snapshot.get("headline") or ""), "next_action_codes": [ str(item.get("action_code") or "").strip() for item in list(link_snapshot.get("next_actions") or []) if str(item.get("action_code") or "").strip() ], "error": str(link_snapshot_payload.get("detail") or link_snapshot_payload.get("message") or ""), }, "overview": { "http_status": overview_status, "available": overview_status.startswith("2"), "recommendation_key": str(recommendation.get("key") or ""), "recommendation_title": str(recommendation.get("title") or ""), "primary_action_code": str(recommendation.get("primary_action_code") or ""), "secondary_action_code": str(recommendation.get("secondary_action_code") or ""), "focus_ref": dict(recommendation.get("focus_ref") or {}), "execution_scene_summary": str( execution_scene.get("summary") or (execution_scene.get("participation_summary") or {}).get("summary") or execution_scene.get("phase_detail") or "" ), "execution_scene_counts": dict(execution_scene.get("counts") or {}), "log_sync": { "status": str(log_sync.get("status") or log_sync.get("state") or ""), "status_label": str(log_sync.get("status_label") or ""), "mode": str(log_sync.get("mode") or ""), "mode_label": str(log_sync.get("mode_label") or ""), "enabled": bool(log_sync.get("enabled", False)), "covered_participating_nodes": int( log_sync.get("covered_participating_nodes", log_sync.get("covered_participating_node_count", 0)) or 0 ), "participating_nodes_total": int( log_sync.get("participating_nodes_total", log_sync.get("participating_node_count", 0)) or 0 ), "missing_sample_node_codes": list( log_sync.get("missing_sample_node_codes") or log_sync.get("missing_participating_nodes") or [] ), }, "error": str(overview_payload.get("detail") or overview_payload.get("message") or ""), }, "managed_nodes": { "http_status": nodes_status, "available": nodes_status.startswith("2"), "total": int(nodes_summary.get("total", len(nodes)) or 0), "managed_total": int(nodes_summary.get("managed_total", 0) or 0), "managed_enabled": int(nodes_summary.get("managed_enabled", 0) or 0), "online": int(nodes_summary.get("online", 0) or 0), "agent_ready": int(nodes_summary.get("agent_ready", 0) or 0), "ssh_ready": int(nodes_summary.get("ssh_ready", 0) or 0), "remote_access_ready": int(nodes_summary.get("remote_access_ready", 0) or 0), "participating": int(nodes_summary.get("participating", 0) or 0), "dispatch_active": int(nodes_summary.get("dispatch_active", 0) or 0), "standby": int(nodes_summary.get("standby", 0) or 0), "load_syncing": int(nodes_summary.get("load_syncing", 0) or 0), "queue_retrying_nodes": int(nodes_summary.get("queue_retrying_nodes", 0) or 0), "queue_dead_letter_nodes": int(nodes_summary.get("queue_dead_letter_nodes", 0) or 0), "queue_pending_records": int(nodes_summary.get("queue_pending_records", 0) or 0), "queue_dead_letter_records": int(nodes_summary.get("queue_dead_letter_records", 0) or 0), "agent_state_counts": dict(nodes_summary.get("status_counts") or {}), "remote_access_state_counts": dict(nodes_summary.get("remote_access_state_counts") or {}), "delivery_queue_state_counts": dict(nodes_summary.get("delivery_queue_state_counts") or {}), "problem_nodes": problem_nodes[:8], "error": str(nodes_root_payload.get("detail") or nodes_root_payload.get("message") or ""), }, "release_hub": { "http_status": launchpad_status_code, "available": launchpad_status_code.startswith("2"), "latest_release_version": str(latest_release.get("release_version") or ""), "latest_release_status": str(latest_release.get("status") or ""), "default_rollout_gate_status": str(default_rollout_gate.get("status") or ""), "default_rollout_gate_label": str(default_rollout_gate.get("status_label") or ""), "default_rollout_gate_summary": str(default_rollout_gate.get("summary_text") or default_rollout_gate.get("summary") or ""), "default_rollout_gate_focus_ref": dict(default_rollout_gate.get("focus_ref") or {}), "launchpad_status": str(launchpad_status.get("status") or ""), "launchpad_status_label": str(launchpad_status.get("status_label") or ""), "launchpad_summary": str(launchpad_status.get("summary_text") or launchpad_status.get("summary") or ""), "recommended_action_code": str(launchpad_status.get("recommended_action_code") or ""), "recommended_target_node_code": str(launchpad_status.get("recommended_target_node_code") or ""), "recommended_recovery_label": str(launchpad_status.get("recommended_recovery_label") or ""), "recommended_recovery_summary": str(launchpad_status.get("recommended_recovery_summary") or ""), "onboarding_bootstrap_pending_nodes": int(launchpad_status.get("onboarding_bootstrap_pending_nodes", 0) or 0), "onboarding_acceptance_ready_nodes": int(launchpad_status.get("onboarding_acceptance_ready_nodes", 0) or 0), "recommended_execution_mode": str(launchpad_status.get("recommended_execution_mode") or ""), "focus_ref": dict(launchpad_status.get("focus_ref") or {}), "error": str(launchpad_payload.get("detail") or launchpad_payload.get("message") or ""), }, "playbook_runs": { "http_status": playbook_runs_status, "available": playbook_runs_status.startswith("2"), "recent_total": len(playbook_runs), "problem_runs_total": len(problem_runs), "problem_runs": problem_runs[:5], "error": str(playbook_runs_root_payload.get("detail") or playbook_runs_root_payload.get("message") or ""), }, "activity_stream": { "http_status": activity_status, "available": activity_status.startswith("2"), "recent_total": len(activities), "status_counts": activity_counts, "top_items": [ { "kind": str(item.get("kind") or "").strip(), "status": str(item.get("status") or item.get("job_status") or "").strip(), "summary": str(item.get("summary_text") or item.get("summary") or "").strip(), "focus_ref": dict(item.get("focus_ref") or {}), } for item in activities[:5] ], "error": str(activity_root_payload.get("detail") or activity_root_payload.get("message") or ""), }, } print(json.dumps(summary, ensure_ascii=False, indent=2)) PY