from __future__ import annotations import json from datetime import datetime, timedelta from pathlib import Path import re import time from app.core.config import settings from app.core.files import runtime_root from app.services.ops_command_service import build_bash_command from app.services.ops_agent_service import ( execute_managed_node_onboarding_recovery, get_managed_node_handover, list_ops_job_events_for_jobs, list_managed_nodes_with_agent_state, ) from app.services.ops_execution_mode_service import execution_mode_label from app.services.ops_job_service import create_ops_job_batch, get_ops_job_summary, list_ops_jobs, sync_managed_nodes_from_cluster from app.services.ops_playbook_service import ( execute_ops_playbook, get_ops_playbook, get_ops_playbook_run, get_recent_ops_playbook_runs, list_ops_playbook_run_events, preview_ops_playbook, ) from app.services.ops_release_service import ( build_release_execution_mode_recommendation, create_release_and_smart_rollout_from_latest_package, get_latest_release_package_metadata, create_smart_release_rollout, get_latest_release, get_release_launchpad, get_release_summary, list_release_rollouts, prepare_latest_release_package, ) from app.services.ops_template_service import get_ops_action_template from app.services.build_info_service import get_runtime_build_info from app.services.runtime_status_service import get_runtime_status from app.services.runtime_settings_service import get_runtime_settings, update_runtime_settings from app.services.sync_record_service import get_sync_summary _OPS_INSPECTION_FETCH_LIMIT = 80 _OPS_ACTIVITY_FETCH_LIMIT = 18 _OPS_INSPECTION_ACTION_KEYS = ("health.snapshot", "logs.collect", "diagnostics.collect") _OPS_CONTRACT_REGISTRY_VERSION = "2026-04-18" _OPS_CONTRACT_SCHEMA_VERSION = "v1" _REMOTE_LOG_PREVIEW_LINE_RE = re.compile(r"^\[(?P[^\]]+)\]\s+\[(?P[^\]]+)\]\s+(?P.*)$") _BACKEND_DRIVER_ACTION_CODES = { "disable_log_sync", "enable_log_sync_key", "enable_log_sync_full", "replay_delivery_queue", "flush_delivery_queue", "run_scene_logs_key", "run_scene_logs_full", "open_playbook_dialog", "open_action_template_dialog", "focus_playbook_run", "focus_activity_item", "focus_latest_job_events", "open_playbook_run_latest_events", "handover_first_gap", "view_first_gap", "open_rollout_dialog", "open_release_dialog", "open_release_deploy_control", "open_release_deploy_worker", "open_release_deploy_custom", "create_smart_release_rollout_worker", "create_smart_release_rollout_control", "create_release_rollout_worker", "create_release_rollout_control", "focus_release_hub", "open_worker_logs_participating", "open_worker_logs_standby", "open_worker_logs", "run_inspection_participating", "run_inspection_standby", "run_standard_inspection", "open_diagnostics_participating", "open_diagnostics", "publish_latest_worker", "review_smart_rollout_preview", "review_control_rollout", "fix_rollout_blockers", "fix_managed_nodes", "bootstrap_run", "run_acceptance", "release_package", "release_prepare", "api-restart", } _SAFE_AUTO_DRIVER_ACTION_CODES = { "disable_log_sync", "enable_log_sync_key", "enable_log_sync_full", "run_scene_logs_key", "run_scene_logs_full", "open_playbook_dialog", "open_action_template_dialog", "focus_playbook_run", "focus_activity_item", "focus_latest_job_events", "open_playbook_run_latest_events", "handover_first_gap", "view_first_gap", "open_rollout_dialog", "open_release_dialog", "open_release_deploy_control", "open_release_deploy_worker", "open_release_deploy_custom", "focus_release_hub", "release_package", "release_prepare", "api-restart", "review_smart_rollout_preview", "review_control_rollout", "fix_rollout_blockers", "open_worker_logs_participating", "open_worker_logs_standby", "open_worker_logs", "open_diagnostics_participating", "open_diagnostics", "run_inspection_participating", "run_inspection_standby", "run_standard_inspection", } _GUARDED_AUTO_DRIVER_ACTION_CODES = { "replay_delivery_queue", "flush_delivery_queue", "publish_latest_worker", "create_smart_release_rollout_worker", "create_smart_release_rollout_control", "create_release_rollout_worker", "create_release_rollout_control", } _MIXED_DRIVER_ACTION_CODES = { "fix_managed_nodes", "bootstrap_run", "run_acceptance", } _UI_ONLY_DRIVER_ACTION_CODES = set() _OPS_CONTRACT_DEFINITIONS = ( { "key": "ops_job_contract", "title": "Ops Job Contract", "version": _OPS_CONTRACT_SCHEMA_VERSION, "status": "active", "summary": "冻结 ops job / step / event / policy / approve / cancel / dispatch 的正式执行对象 contract。", "schema_doc_path": "docs/schemas/ops_job_contract.md", "primary_endpoint": "/api/v1/ops/jobs", "discovery_endpoints": [ "/api/v1/ops/jobs", "/api/v1/ops/jobs/{job_id}", "/api/v1/ops/jobs/{job_id}/events", "/api/v1/ops/jobs", "/api/v1/ops/jobs/batch", "/api/v1/ops/policy/preview", "/api/v1/ops/jobs/{job_id}/approve", "/api/v1/ops/jobs/{job_id}/cancel", "/api/v1/ops/jobs/{job_id}/dispatch", ], "service_keys": [ "app.services.ops_job_service", "app.services.ops_agent_service", ], "consumers": [ "ops-center", "codex-driver", "cli", "node-agent", "release-hub", "playbook-runner", ], "related_contract_keys": [ "ops_agent_protocol", "release_hub_contract", "ops_playbook_contract", "ops_observability_contract", "ops_stack_diagnosis_contract", ], "notes": [ "ops job 是正式执行颗粒度对象,playbook run 与 rollout 最终都应收敛到它。", "approval / cancel / dispatch 不只是按钮动作,而是正式状态迁移。", "页面列表默认消费 compact job,详情页再读取 full job + steps + events。", ], }, { "key": "ops_agent_protocol", "title": "Node Agent Protocol", "version": _OPS_CONTRACT_SCHEMA_VERSION, "status": "active", "summary": "冻结海外控制面与大陆 Node Agent 之间的认证、注册、心跳、拉任务、完成回执与事件回放 contract。", "schema_doc_path": "docs/schemas/ops_agent_protocol.md", "primary_endpoint": "/api/v1/ops/agent/bootstrap-plan", "discovery_endpoints": [ "/api/v1/ops/agent/tokens", "/api/v1/ops/agent/bootstrap-plan", "/api/v1/ops/agent/register", "/api/v1/ops/agent/heartbeat", "/api/v1/ops/agent/pull", "/api/v1/ops/agent/jobs/{job_id}/start", "/api/v1/ops/agent/jobs/{job_id}/complete", "/api/v1/ops/agent/jobs/{job_id}/events", "/api/v1/ops/nodes/{node_code}/delivery-queue", "/api/v1/ops/nodes/{node_code}/delivery-queue/records", "/api/v1/ops/nodes/{node_code}/delivery-queue/flush", "/api/v1/ops/nodes/{node_code}/delivery-queue/replay", "/api/v1/ops/nodes/{node_code}/delivery-queue/records/{record_id}/replay", "/api/v1/ops/nodes/{node_code}/delivery-queue/records/{record_id}/discard", ], "service_keys": [ "app.services.ops_agent_service", "app.node_agent", ], "consumers": [ "ops-center", "codex-driver", "cli", "node-agent", ], "related_contract_keys": [ "ops_job_contract", "ops_observability_contract", "ops_stack_diagnosis_contract", ], "notes": [ "Agent 只执行结构化动作,不长期接受任意 shell。", "complete / events 回执已经具备 client_request_id / client_event_id 幂等语义。", "heartbeat 已纳入 delivery_queue 快照,可暴露 retrying / dead_letter 现场。", ], }, { "key": "release_hub_contract", "title": "Release Hub Contract", "version": _OPS_CONTRACT_SCHEMA_VERSION, "status": "active", "summary": "冻结 Release / Rollout / Launchpad / Default Gate 的对象模型与门禁口径。", "schema_doc_path": "docs/schemas/release_hub_contract.md", "primary_endpoint": "/api/v1/ops/releases/launchpad", "discovery_endpoints": [ "/api/v1/ops/overview", "/api/v1/ops/releases", "/api/v1/ops/releases/latest", "/api/v1/ops/releases/launchpad", "/api/v1/ops/releases/{release_id}", "/api/v1/ops/releases/{release_id}/rollouts", "/api/v1/ops/rollouts/{rollout_id}", "/api/v1/ops/rollouts/{rollout_id}/jobs", "/api/v1/ops/rollouts/{rollout_id}/advance", ], "service_keys": [ "app.services.ops_release_service", "app.services.ops_service", ], "consumers": [ "ops-center", "codex-driver", "cli", "release-hub", ], "related_contract_keys": [ "ops_job_contract", "ops_driver_contract", "ops_playbook_contract", "ops_stack_diagnosis_contract", ], "notes": [ "发布必须先有 Release,再有 Rollout,不回退到节点 git pull。", "default_rollout_gate 是首页、版本区和自动驾驶共用的统一门禁。", "Rollout 状态机应继续固定为 planned/running/awaiting_approval/ready_for_next_batch/halted/completed/completed_with_issues。", ], }, { "key": "ops_driver_contract", "title": "Ops Driver Contract", "version": _OPS_CONTRACT_SCHEMA_VERSION, "status": "active", "summary": "冻结 overview / driver-feed / codex-brief / driver-actions / runbook sequence 的统一驾驶 contract。", "schema_doc_path": "docs/schemas/ops_driver_contract.md", "primary_endpoint": "/api/v1/ops/driver-feed", "discovery_endpoints": [ "/api/v1/ops/overview", "/api/v1/ops/driver-feed", "/api/v1/ops/codex-brief", "/api/v1/ops/driver-actions/resolve", "/api/v1/ops/driver-actions/execute-resolved", "/api/v1/ops/codex-actions/resolve", "/api/v1/ops/codex-actions/execute", "/api/v1/ops/activity-stream", "/api/v1/ops/runbook", "/api/v1/ops/runbook/sequences/{sequence_key}/resolve", "/api/v1/ops/runbook/sequences/{sequence_key}/execute", "/api/v1/ops/driver-actions/preview", "/api/v1/ops/driver-actions/execute", ], "service_keys": [ "app.services.ops_service", "app.services.ops_playbook_service", ], "consumers": [ "ops-center", "codex-driver", "cli", ], "related_contract_keys": [ "release_hub_contract", "ops_playbook_contract", "ops_observability_contract", "ops_stack_diagnosis_contract", ], "notes": [ "后端负责推荐优先级和 ui_intent,页面不再自算。", "driver recommendation / runbook sequence / activity stream 是三条不同 contract。", "Codex 通过 codex-brief 判断 safe_auto / guarded_auto / ui_only / blocked。", "driver-actions/preview 负责统一生成执行链、请求体预览和主次动作 contract 视图。", ], }, { "key": "ops_playbook_contract", "title": "Ops Playbook Contract", "version": _OPS_CONTRACT_SCHEMA_VERSION, "status": "active", "summary": "冻结 playbook catalog / preview / run / events 的正式编排 contract。", "schema_doc_path": "docs/schemas/ops_playbook_contract.md", "primary_endpoint": "/api/v1/ops/playbooks", "discovery_endpoints": [ "/api/v1/ops/playbooks", "/api/v1/ops/playbooks/preview", "/api/v1/ops/playbooks/execute", "/api/v1/ops/playbook-runs", "/api/v1/ops/playbook-runs/{run_code}", "/api/v1/ops/playbook-runs/{run_code}/events", "/api/v1/ops/playbook-runs/{run_code}/rerun", "/api/v1/ops/playbook-runs/{run_code}/cancel", ], "service_keys": [ "app.services.ops_playbook_service", "app.services.ops_service", ], "consumers": [ "ops-center", "codex-driver", "cli", ], "related_contract_keys": [ "ops_job_contract", "release_hub_contract", "ops_driver_contract", "ops_observability_contract", "ops_stack_diagnosis_contract", ], "notes": [ "接管验收、标准巡检、现场观察应优先收口为 playbook run。", "playbook run 是 ops job 的编排聚合对象,不是单条任务。", "页面、CLI、Codex 应围绕同一份 preview / run / events contract 观察与执行。", ], }, { "key": "ops_observability_contract", "title": "Ops Observability Contract", "version": _OPS_CONTRACT_SCHEMA_VERSION, "status": "active", "summary": "冻结 execution scene / inspection overview / activity stream / delivery queue 的正式观察面 contract。", "schema_doc_path": "docs/schemas/ops_observability_contract.md", "primary_endpoint": "/api/v1/ops/overview", "discovery_endpoints": [ "/api/v1/ops/overview", "/api/v1/ops/inspection-overview", "/api/v1/ops/activity-stream", "/api/v1/ops/nodes/{node_code}/scene-log", "/api/v1/ops/nodes/{node_code}/delivery-queue", "/api/v1/ops/nodes/{node_code}/delivery-queue/records", "/api/v1/ops/nodes/{node_code}/delivery-queue/flush", "/api/v1/ops/nodes/{node_code}/delivery-queue/replay", "/api/v1/ops/nodes/{node_code}/delivery-queue/records/{record_id}/replay", "/api/v1/ops/nodes/{node_code}/delivery-queue/records/{record_id}/discard", ], "service_keys": [ "app.services.ops_service", "app.services.ops_agent_service", ], "consumers": [ "ops-center", "codex-driver", "cli", ], "related_contract_keys": [ "ops_job_contract", "ops_agent_protocol", "ops_driver_contract", "ops_playbook_contract", "ops_stack_diagnosis_contract", ], "notes": [ "execution scene 必须明确区分 dispatch_active / recent_only / standby / load_syncing。", "inspection overview 必须按节点收口最近 health / worker logs / diagnostics。", "delivery queue 当前允许 head_only,但不允许绕过 ops job 直接改远端记录。", ], }, { "key": "ops_stack_diagnosis_contract", "title": "Ops Stack Diagnosis Contract", "version": _OPS_CONTRACT_SCHEMA_VERSION, "status": "active", "summary": "冻结海外单脑总检入口的统一诊断 contract,供页面、CLI、Codex、按钮共享同一份第一现场判断。", "schema_doc_path": "docs/schemas/ops_stack_diagnosis_contract.md", "primary_endpoint": "/api/v1/ops/stack-diagnosis", "discovery_endpoints": [ "/api/v1/ops/go-live-summary", "/api/v1/ops/stack-diagnosis", "/api/v1/ops/contracts", "/api/v1/ops/link-snapshot", "/api/v1/ops/overview", "/api/v1/ops/nodes", "/api/v1/ops/releases/launchpad", "/api/v1/ops/playbook-runs", "/api/v1/ops/activity-stream", ], "service_keys": [ "app.services.ops_service", ], "consumers": [ "ops-center", "codex-driver", "cli", ], "related_contract_keys": [ "ops_job_contract", "ops_agent_protocol", "release_hub_contract", "ops_driver_contract", "ops_playbook_contract", "ops_observability_contract", ], "notes": [ "这是海外单脑控制面的固定起手式,不允许页面、CLI、Codex 再各自拼一份第一现场诊断。", "总检必须同时给出 stack_status、issues、next_step、quick_commands,不能只回一段说明文字。", "go-live-summary 是给页面 / CLI / Codex 直接消费的收口摘要层,stack-diagnosis 则保留更细的可解释结构。", "当总检 contract 可用后,check_ops_center_stack.sh 应优先直接消费它,而不是继续各算各的。", ], }, { "key": "ops_doctor_decision_contract", "title": "Ops Doctor Decision Contract", "version": _OPS_CONTRACT_SCHEMA_VERSION, "status": "active", "summary": "冻结 doctor-decision 主决策 contract,把总检主判断、建议处理面、下一步动作和证据摘要统一暴露给页面、CLI 和 Codex。", "schema_doc_path": "docs/schemas/ops_doctor_decision_contract.md", "primary_endpoint": "/api/v1/ops/doctor-decision", "discovery_endpoints": [ "/api/v1/ops/doctor-decision", "/api/v1/ops/stack-diagnosis", "/api/v1/ops/go-live-summary", "/api/v1/ops/go-live-bundle", "/api/v1/ops/contracts", ], "service_keys": [ "app.services.ops_service", ], "consumers": [ "ops-center", "codex-driver", "cli", ], "related_contract_keys": [ "ops_stack_diagnosis_contract", "ops_driver_contract", "ops_go_live_bundle_contract", "ops_go_live_signoff_contract", "release_hub_contract", ], "notes": [ "doctor-decision 是页面、CLI、Codex 的统一主决策入口,不应再让不同消费方各算各的“第一处理面”。", "优先读取最新导出的 doctor manifest / go-live bundle 中的 doctor_decision,再在缺 bundle 时回落到 live 总检兜底。", "返回值必须同时给出 status、preferred_surface、next_action_code 与推荐命令,不能只给一句提示文字。", ], }, { "key": "ops_go_live_signoff_contract", "title": "Ops Go-Live Signoff Contract", "version": _OPS_CONTRACT_SCHEMA_VERSION, "status": "active", "summary": "冻结发布前最终签收结论 contract,把收口、发布、自动化、launchpad 对齐状态压成同一份签字判断。", "schema_doc_path": "docs/schemas/ops_go_live_signoff_contract.md", "primary_endpoint": "/api/v1/ops/go-live-signoff", "discovery_endpoints": [ "/api/v1/ops/go-live-signoff", "/api/v1/ops/go-live-summary", "/api/v1/ops/stack-diagnosis", "/api/v1/ops/driver-feed", "/api/v1/ops/codex-brief", "/api/v1/ops/releases/launchpad", "/api/v1/ops/contracts", ], "service_keys": [ "app.services.ops_service", ], "consumers": [ "ops-center", "codex-driver", "cli", ], "related_contract_keys": [ "ops_doctor_decision_contract", "ops_stack_diagnosis_contract", "ops_driver_contract", "release_hub_contract", "ops_observability_contract", ], "notes": [ "signoff 不是替代 go-live-summary,而是在其基础上给出最终可否签字上线的统一结论。", "当 signoff_status=blocked 时,不允许页面再显示‘基本可上线’这类模糊表述。", "launchpad 摘要在 go-live-summary / driver-feed / codex-brief 之间若不一致,必须至少提升为 attention。", ], }, { "key": "ops_go_live_bundle_contract", "title": "Ops Go-Live Bundle Contract", "version": _OPS_CONTRACT_SCHEMA_VERSION, "status": "active", "summary": "冻结发布前证据包与 manifest 复核 contract,把 bundle 是否存在、manifest 状态、失败项和交付建议统一暴露给页面、CLI 和 Codex。", "schema_doc_path": "docs/schemas/ops_go_live_bundle_contract.md", "primary_endpoint": "/api/v1/ops/go-live-bundle", "discovery_endpoints": [ "/api/v1/ops/go-live-bundle", "/api/v1/ops/go-live-signoff", "/api/v1/ops/go-live-summary", "/api/v1/ops/contracts", ], "service_keys": [ "app.services.ops_service", ], "consumers": [ "ops-center", "codex-driver", "cli", ], "related_contract_keys": [ "ops_doctor_decision_contract", "ops_go_live_signoff_contract", "ops_stack_diagnosis_contract", "ops_driver_contract", "release_hub_contract", ], "notes": [ "页面不应再假设证据包已经导出,而是必须明确显示当前有没有 bundle、manifest 是否可复核。", "当 bundle 不存在时,应明确回落到 go-live-export / go-live-review / go-live-signoff 等固定命令,而不是提示人工自行找目录。", "这层只负责读取最新 bundle / manifest 与复核摘要,不直接在后端发起 shell 导出。", ], }, { "key": "ops_go_live_review_contract", "title": "Ops Go-Live Review Contract", "version": _OPS_CONTRACT_SCHEMA_VERSION, "status": "active", "summary": "冻结 go-live-review 复核结论 contract,把 bundle manifest 的正式复核结果、launchpad 对齐与下一步建议统一暴露给页面、CLI 和 Codex。", "schema_doc_path": "docs/schemas/ops_go_live_review_contract.md", "primary_endpoint": "/api/v1/ops/go-live-review", "discovery_endpoints": [ "/api/v1/ops/go-live-review", "/api/v1/ops/go-live-bundle", "/api/v1/ops/doctor-decision", "/api/v1/ops/go-live-signoff", "/api/v1/ops/contracts", ], "service_keys": [ "app.services.ops_service", ], "consumers": [ "ops-center", "codex-driver", "cli", ], "related_contract_keys": [ "ops_go_live_bundle_contract", "ops_doctor_decision_contract", "ops_go_live_signoff_contract", "ops_stack_diagnosis_contract", "ops_driver_contract", "release_hub_contract", ], "notes": [ "go-live-review 不是简单重复 bundle 状态,而是 bundle manifest 的正式复核结论层。", "当 launchpad 摘要在 go_live_summary / stack_diagnosis / driver_feed / codex_brief 之间漂移时,即使关键文件都存在,也必须至少回到 attention。", "页面和 Codex 应优先消费这层的 headline / recommended_next_steps,而不是自行拼 review 说明。", ], }, ) def _bool_label(value: bool) -> str: return "enabled" if value else "disabled" def get_ops_contract_registry() -> dict: contracts: list[dict] = [] contracts_by_key: dict[str, dict] = {} for raw_item in _OPS_CONTRACT_DEFINITIONS: contract = { "key": str(raw_item.get("key") or "").strip(), "title": str(raw_item.get("title") or "").strip(), "version": str(raw_item.get("version") or _OPS_CONTRACT_SCHEMA_VERSION).strip(), "status": str(raw_item.get("status") or "active").strip() or "active", "summary": str(raw_item.get("summary") or "").strip(), "schema_doc_path": str(raw_item.get("schema_doc_path") or "").strip(), "primary_endpoint": str(raw_item.get("primary_endpoint") or "").strip(), "discovery_endpoints": [str(item or "").strip() for item in list(raw_item.get("discovery_endpoints") or []) if str(item or "").strip()], "service_keys": [str(item or "").strip() for item in list(raw_item.get("service_keys") or []) if str(item or "").strip()], "consumers": [str(item or "").strip() for item in list(raw_item.get("consumers") or []) if str(item or "").strip()], "related_contract_keys": [ str(item or "").strip() for item in list(raw_item.get("related_contract_keys") or []) if str(item or "").strip() ], "notes": [str(item or "").strip() for item in list(raw_item.get("notes") or []) if str(item or "").strip()], } if not contract["key"]: continue contracts.append(contract) contracts_by_key[contract["key"]] = contract return { "registry_version": _OPS_CONTRACT_REGISTRY_VERSION, "schema_version": _OPS_CONTRACT_SCHEMA_VERSION, "contracts_total": len(contracts), "contracts": contracts, "contracts_by_key": contracts_by_key, "docs_root": "docs/schemas", "discovery_entrypoints": [ "/api/v1/ops/contracts", "/api/v1/ops/contracts/{contract_key}", "/api/v1/ops/stack-diagnosis", "/api/v1/ops/capabilities", "/api/v1/ops/overview", "/api/v1/ops/driver-feed", "/api/v1/ops/codex-brief", "/api/v1/ops/releases/launchpad", ], } def get_ops_contract_detail(contract_key: str) -> dict: normalized_contract_key = str(contract_key or "").strip() if not normalized_contract_key: return {} registry = get_ops_contract_registry() contracts_by_key = dict(registry.get("contracts_by_key") or {}) selected_contract = dict(contracts_by_key.get(normalized_contract_key) or {}) if not selected_contract: return {} related_contracts = [ dict(contracts_by_key.get(related_key) or {}) for related_key in list(selected_contract.get("related_contract_keys") or []) if str(related_key or "").strip() and dict(contracts_by_key.get(related_key) or {}) ] related_contracts = [ { "key": str(item.get("key") or "").strip(), "title": str(item.get("title") or "").strip(), "summary": str(item.get("summary") or "").strip(), "schema_doc_path": str(item.get("schema_doc_path") or "").strip(), "primary_endpoint": str(item.get("primary_endpoint") or "").strip(), } for item in related_contracts ] return { "registry_version": str(registry.get("registry_version") or _OPS_CONTRACT_REGISTRY_VERSION), "schema_version": str(registry.get("schema_version") or _OPS_CONTRACT_SCHEMA_VERSION), "docs_root": str(registry.get("docs_root") or "docs/schemas"), "detail_endpoint": f"/api/v1/ops/contracts/{normalized_contract_key}", "contract": selected_contract, "related_contracts": related_contracts, "related_contract_keys": [ str(item.get("key") or "").strip() for item in related_contracts if str(item.get("key") or "").strip() ], } def _normalize_ops_contract_keys(raw_values: list | tuple | set | None) -> list[str]: normalized_values: list[str] = [] seen_values: set[str] = set() for raw_value in list(raw_values or []): normalized_value = str(raw_value or "").strip() if not normalized_value or normalized_value in seen_values: continue seen_values.add(normalized_value) normalized_values.append(normalized_value) return normalized_values def _build_ops_contract_navigation( contract_keys: list | tuple | set | None, *, primary_contract_key: str = "", registry: dict | None = None, ) -> dict: registry_payload = dict(registry or get_ops_contract_registry()) contracts_by_key = dict(registry_payload.get("contracts_by_key") or {}) normalized_primary_contract_key = str(primary_contract_key or "").strip() ordered_contract_keys = _normalize_ops_contract_keys(contract_keys) if normalized_primary_contract_key: ordered_contract_keys = [normalized_primary_contract_key] + [ item for item in ordered_contract_keys if item != normalized_primary_contract_key ] contracts: list[dict] = [] for contract_key in ordered_contract_keys: raw_contract = dict(contracts_by_key.get(contract_key) or {}) if not raw_contract: continue related_contract_keys = _normalize_ops_contract_keys(raw_contract.get("related_contract_keys") or []) contracts.append( { "key": contract_key, "title": str(raw_contract.get("title") or "").strip(), "status": str(raw_contract.get("status") or "").strip(), "version": str(raw_contract.get("version") or _OPS_CONTRACT_SCHEMA_VERSION).strip(), "summary": str(raw_contract.get("summary") or "").strip(), "primary_endpoint": str(raw_contract.get("primary_endpoint") or "").strip(), "schema_doc_path": str(raw_contract.get("schema_doc_path") or "").strip(), "detail_endpoint": f"/api/v1/ops/contracts/{contract_key}", "discovery_endpoints": [ str(item).strip() for item in list(raw_contract.get("discovery_endpoints") or []) if str(item).strip() ], "related_contract_keys": related_contract_keys, } ) return { "detail_endpoint_pattern": "/api/v1/ops/contracts/{contract_key}", "primary_contract_key": ( normalized_primary_contract_key if normalized_primary_contract_key else str((contracts[0] or {}).get("key") or "").strip() ), "contract_keys": ( [str(item.get("key") or "").strip() for item in contracts if str(item.get("key") or "").strip()] if contracts else ordered_contract_keys ), "contracts": contracts, } def _ops_contract_keys_from_focus_ref(focus_ref: dict | None) -> list[str]: normalized_focus_ref = dict(focus_ref or {}) kind = str(normalized_focus_ref.get("kind") or "").strip() section = str(normalized_focus_ref.get("section") or "").strip() if kind in {"release_hub", "release", "rollout"} or section in { "release_launchpad", "release_detail", "default_rollout_gate", }: return ["release_hub_contract"] if kind in {"execution_scene", "ops_job", "ops_job_event", "activity_stream", "node_scene_log"}: return ["ops_observability_contract", "ops_stack_diagnosis_contract"] if kind in {"playbook_run", "playbook", "playbook_step"}: return ["ops_playbook_contract"] if kind in {"managed_node", "node_agent", "delivery_queue"}: return ["ops_agent_protocol"] return [] def _ops_contract_keys_from_action_code(action_code: str) -> list[str]: normalized_action_code = str(action_code or "").strip() if not normalized_action_code: return [] if normalized_action_code.startswith("enable_log_sync") or normalized_action_code in { "run_inspection_participating", "run_inspection_all", "focus_execution_scene", "focus_activity_item", }: return ["ops_observability_contract", "ops_stack_diagnosis_contract"] if normalized_action_code in { "fix_managed_nodes", "issue_agent_token", "create_bootstrap_plan", "open_agent_onboarding", "bootstrap_run", "run_acceptance", }: return ["ops_agent_protocol"] if normalized_action_code in { "open_release_dialog", "focus_release_hub", "create_release_rollout_worker", "create_release_rollout_control", "publish_latest_worker", "publish_latest_control", }: return ["release_hub_contract"] return [] def _ops_contract_keys_from_job_action(job_action: str) -> list[str]: normalized_job_action = str(job_action or "").strip() if not normalized_job_action: return [] if normalized_job_action in {"health.snapshot", "logs.collect", "diagnostics.collect", "service.status"}: return ["ops_observability_contract", "ops_stack_diagnosis_contract"] if normalized_job_action == "node.bootstrap" or normalized_job_action.startswith(("runtime.", "service.", "delivery.queue.")): return ["ops_agent_protocol"] if normalized_job_action == "deploy.release": return ["release_hub_contract"] return [] def _suggest_ops_contract_keys_for_codex_entry(entry: dict) -> list[str]: normalized_entry = dict(entry or {}) executor_kind = str(normalized_entry.get("executor_kind") or "driver_action").strip() or "driver_action" contract_keys: list[str] = ["ops_driver_contract"] if executor_kind == "runbook_sequence": contract_keys.append("ops_playbook_contract") for focus_ref in ( normalized_entry.get("focus_ref"), normalized_entry.get("primary_focus_ref"), normalized_entry.get("secondary_focus_ref"), ): contract_keys.extend(_ops_contract_keys_from_focus_ref(focus_ref)) for action_code in ( normalized_entry.get("primary_action_code"), normalized_entry.get("secondary_action_code"), normalized_entry.get("focus_action_code"), ): contract_keys.extend(_ops_contract_keys_from_action_code(str(action_code or "").strip())) return _normalize_ops_contract_keys(contract_keys) def _suggest_ops_contract_keys_for_activity_item(item: dict) -> list[str]: normalized_item = dict(item or {}) kind = str(normalized_item.get("kind") or "").strip() ui_intent = dict(normalized_item.get("ui_intent") or {}) contract_keys: list[str] = [] if kind in {"execution_scene", "log_sync"}: contract_keys.extend(["ops_observability_contract", "ops_stack_diagnosis_contract"]) elif kind in {"playbook_run", "runbook_sequence"}: contract_keys.append("ops_playbook_contract") if kind == "runbook_sequence": contract_keys.append("ops_driver_contract") elif kind == "rollout": contract_keys.append("release_hub_contract") elif kind == "ops_job": contract_keys.extend( _ops_contract_keys_from_job_action( str(normalized_item.get("action") or normalized_item.get("title") or "").strip() ) ) else: contract_keys.append("ops_observability_contract") for focus_ref in ( normalized_item.get("focus_ref"), normalized_item.get("source_focus_ref"), ): contract_keys.extend(_ops_contract_keys_from_focus_ref(focus_ref)) for action_code in ( ui_intent.get("driver_action_code"), normalized_item.get("primary_action_code"), normalized_item.get("secondary_action_code"), normalized_item.get("focus_action_code"), ): contract_keys.extend(_ops_contract_keys_from_action_code(str(action_code or "").strip())) if str(ui_intent.get("kind") or "").strip() == "focus_execution_scene": contract_keys.extend(["ops_observability_contract", "ops_stack_diagnosis_contract"]) return _normalize_ops_contract_keys(contract_keys) def _primary_ops_contract_key_for_activity_item(item: dict, contract_keys: list[str]) -> str: normalized_item = dict(item or {}) normalized_contract_keys = _normalize_ops_contract_keys(contract_keys) kind = str(normalized_item.get("kind") or "").strip() job_action = str(normalized_item.get("action") or normalized_item.get("title") or "").strip() if kind in {"playbook_run", "runbook_sequence"} and "ops_playbook_contract" in normalized_contract_keys: return "ops_playbook_contract" if kind == "rollout" and "release_hub_contract" in normalized_contract_keys: return "release_hub_contract" if kind == "ops_job": if job_action == "node.bootstrap" and "ops_agent_protocol" in normalized_contract_keys: return "ops_agent_protocol" if job_action == "deploy.release" and "release_hub_contract" in normalized_contract_keys: return "release_hub_contract" if "ops_observability_contract" in normalized_contract_keys: return "ops_observability_contract" if "ops_driver_contract" in normalized_contract_keys: return "ops_driver_contract" return normalized_contract_keys[0] if normalized_contract_keys else "" def _attach_driver_feed_contract_navigation(entry: dict) -> dict: normalized_entry = dict(entry or {}) contract_keys = _normalize_ops_contract_keys( list(normalized_entry.get("contract_keys") or []) + _suggest_ops_contract_keys_for_codex_entry(normalized_entry) ) return { **normalized_entry, "contract_keys": contract_keys, "contract_navigation": _build_ops_contract_navigation( contract_keys, primary_contract_key="ops_driver_contract", registry=get_ops_contract_registry(), ), } def _format_node_code_list(raw_values: object, *, limit: int = 5) -> str: values: list[str] = [] seen: set[str] = set() for item in list(raw_values or []): value = str(item or "").strip() if not value or value in seen: continue seen.add(value) values.append(value) if not values: return "-" visible = values[: max(1, int(limit or 5))] suffix = f" 等 {len(values)} 台" if len(values) > len(visible) else "" return "、".join(visible) + suffix def _normalize_driver_text_lines(raw_values: object, *, limit: int = 5) -> list[str]: normalized: list[str] = [] seen: set[str] = set() for item in list(raw_values or []): value = str(item or "").strip() if not value or value in seen: continue seen.add(value) normalized.append(value) return normalized[: max(1, int(limit or 5))] def _is_cluster_online_node(node: dict) -> bool: return str(node.get("cluster_status") or "").strip() in {"online", "busy"} def _is_inspection_visible_node(node: dict) -> bool: return bool(node.get("cluster_is_effective_worker", False)) and _is_cluster_online_node(node) def _is_inspection_eligible_node(node: dict) -> bool: return ( bool(node.get("is_managed", False)) and bool(node.get("is_enabled", False)) and bool(node.get("is_agent_online", False)) and _is_inspection_visible_node(node) ) def _inspection_job_bucket(job: dict) -> str: action = str(job.get("action") or "").strip() if action in {"health.snapshot", "diagnostics.collect"}: return action if action != "logs.collect": return "" service_name = str((job.get("payload") or {}).get("service_name") or "").strip().lower() if "worker" not in service_name: return "" return "logs.collect" def _slim_inspection_job(job: dict) -> dict: payload = dict(job.get("payload") or {}) return { "id": int(job.get("id") or 0), "job_code": str(job.get("job_code") or ""), "action": str(job.get("action") or ""), "status": str(job.get("status") or ""), "target_node_code": str(job.get("target_node_code") or ""), "created_at": str(job.get("created_at") or ""), "started_at": str(job.get("started_at") or ""), "finished_at": str(job.get("finished_at") or ""), "updated_at": str(job.get("updated_at") or ""), "payload": { "service_name": str(payload.get("service_name") or ""), "lines": int(payload.get("lines") or 0) if payload.get("lines") not in (None, "") else 0, }, "error_message": str(job.get("error_message") or ""), } def _inspection_status_label(status: str) -> str: normalized_status = str(status or "").strip() if not normalized_status: return "未收口" mapping = { "success": "成功", "failed": "失败", "blocked": "阻断", "cancelled": "已取消", "awaiting_approval": "待审批", "dispatching": "待派发", "running": "执行中", "queued": "排队中", } return mapping.get(normalized_status, normalized_status) def _build_inspection_contract_result(job: dict | None, *, action_label: str) -> dict: normalized_job = dict(job or {}) if not normalized_job: return { "status": "", "status_label": "未收口", "occurred_at": "", "job_code": "", "summary": f"最近没有 {action_label} 收口记录。", "error_message": "", } status = str(normalized_job.get("status") or "").strip() occurred_at = ( str(normalized_job.get("finished_at") or "").strip() or str(normalized_job.get("updated_at") or "").strip() or str(normalized_job.get("started_at") or "").strip() or str(normalized_job.get("created_at") or "").strip() ) job_code = str(normalized_job.get("job_code") or "").strip() error_message = str(normalized_job.get("error_message") or "").strip() if status == "success": summary = f"最近一次{action_label}收口成功。" elif status: summary = f"最近一次{action_label}状态为 {_inspection_status_label(status)}。" else: summary = f"最近没有 {action_label} 收口记录。" if error_message: summary = f"{summary} 错误:{error_message}" return { "status": status, "status_label": _inspection_status_label(status) if status else "未收口", "occurred_at": occurred_at, "job_code": job_code, "summary": summary, "error_message": error_message, } def _build_inspection_issue_summary( job_map: dict[str, dict], *, node_code: str = "", latest_job: dict | None = None, node: dict | None = None, ) -> dict: action_labels = { "health.snapshot": "健康快照", "logs.collect": "Worker 日志", "diagnostics.collect": "诊断包", } node_context = dict(node or {}) node_is_visible = _is_inspection_visible_node(node_context) node_is_eligible = bool(node_context.get("is_inspection_eligible", False)) or _is_inspection_eligible_node(node_context) node_is_participating = bool(node_context.get("cluster_detect_participating", False)) node_is_managed = bool(node_context.get("is_managed", False)) node_is_enabled = bool(node_context.get("is_enabled", False)) node_agent_online = bool(node_context.get("is_agent_online", False)) node_has_ssh_access = bool(node_context.get("has_ssh_access", False)) agent_state_label = str(node_context.get("agent_state_label") or "").strip() agent_state_reason = str(node_context.get("agent_state_reason") or "").strip() preferred_job = ( job_map.get("diagnostics.collect") or job_map.get("logs.collect") or job_map.get("health.snapshot") or dict(latest_job or {}) ) preferred_job_id = int(preferred_job.get("id") or 0) preferred_job_code = str(preferred_job.get("job_code") or "").strip() preferred_action = str(preferred_job.get("action") or "").strip() preferred_target_node_code = str(preferred_job.get("target_node_code") or node_code or "").strip() preferred_ui_intent = ( _build_driver_ui_intent( "job_events", job_id=preferred_job_id, job_code=preferred_job_code, action=preferred_action, target_node_code=preferred_target_node_code, ) if preferred_job_id > 0 else {} ) present_statuses = [ str((job_map.get(key) or {}).get("status") or "").strip() for key in _OPS_INSPECTION_ACTION_KEYS if str((job_map.get(key) or {}).get("status") or "").strip() ] if node_is_visible and not node_is_eligible: if not node_is_managed: return { "problem_kind": "handover_gap", "problem_label": "未接管", "problem_title": "执行节点未接管", "problem_level": "danger", "problem_keys": [], "summary": "节点在线且属于有效执行面,但还没纳入 Node Agent 接管,当前不能稳定下发标准巡检、发布或诊断动作。", "recommended_action": "纳管节点", "recommended_action_code": "handover_first_gap", "ui_intent": {}, } if not node_is_enabled: return { "problem_kind": "handover_gap", "problem_label": "已停用", "problem_title": "节点已停用", "problem_level": "warning", "problem_keys": [], "summary": "节点虽然仍在有效执行面中,但当前被标记为停用,先确认是否需要重新启用后再下发巡检或发布动作。", "recommended_action": "查看节点", "recommended_action_code": "view_first_gap", "ui_intent": {}, } if not node_agent_online: return { "problem_kind": "handover_gap", "problem_label": "SSH 已备好" if node_has_ssh_access else (agent_state_label or "待接入"), "problem_title": "Agent 未就绪", "problem_level": "danger", "problem_keys": [], "summary": ( f"节点已纳管,SSH 入口已备好,虽然仍未进入 Agent 在线状态,但已可通过 SSH 执行日志、诊断与部分服务控制。{agent_state_reason}" if node_has_ssh_access and agent_state_reason else ( "节点已纳管,SSH 入口已备好,当前已可通过 SSH 执行日志、诊断与部分服务控制;标准巡检编排、正式发布与 Rollout 仍优先依赖 Node Agent 在线。" if node_has_ssh_access else ( f"节点已纳管,但仍未进入 Agent 在线状态。{agent_state_reason}" if agent_state_reason else "节点已纳管,但仍未进入 Agent 在线状态,当前不能稳定执行标准巡检。" ) ) ), "recommended_action": "补接入", "recommended_action_code": "handover_first_gap", "ui_intent": {}, } if node_is_visible and node_is_participating and not present_statuses: return { "problem_kind": "participating_no_inspection", "problem_label": "现场未巡检", "problem_title": "参与检测但未形成巡检", "problem_level": "warning", "problem_keys": [], "summary": "节点当前正在真实参与检测,但还没有形成标准巡检记录,建议先看现场日志或补一轮标准巡检。", "recommended_action": "看现场日志", "recommended_action_code": "open_worker_logs_participating", "ui_intent": {}, } if node_is_visible and node_is_eligible and not node_is_participating and not present_statuses: return { "problem_kind": "standby_no_inspection", "problem_label": "待命未巡检", "problem_title": "在线待命但未巡检", "problem_level": "info", "problem_keys": [], "summary": "节点当前在线可执行,但尚未参与检测,也还没有形成标准巡检记录,适合先做一轮巡检确认是否正常待命。", "recommended_action": "补齐标准巡检", "recommended_action_code": "run_standard_inspection", "ui_intent": {}, } terminal_problem_actions = [ key for key in _OPS_INSPECTION_ACTION_KEYS if str((job_map.get(key) or {}).get("status") or "").strip() in {"failed", "blocked", "cancelled"} ] if terminal_problem_actions: key = terminal_problem_actions[0] job = job_map.get(key) or {} return { "problem_kind": "terminal_failure", "problem_label": "失败/阻断", "problem_title": f"{action_labels[key]}失败", "problem_level": "error", "problem_keys": terminal_problem_actions, "summary": f"{action_labels[key]}当前状态为 {_inspection_status_label(str(job.get('status') or ''))},优先查看事件并重跑标准巡检。", "recommended_action": "先看事件", "recommended_action_code": "focus_latest_job_events", "ui_intent": preferred_ui_intent, } running_actions = [ key for key in _OPS_INSPECTION_ACTION_KEYS if str((job_map.get(key) or {}).get("status") or "").strip() in {"queued", "dispatching", "running", "awaiting_approval"} ] if running_actions: key = running_actions[0] job = job_map.get(key) or {} return { "problem_kind": "inflight_execution", "problem_label": "执行中", "problem_title": f"{action_labels[key]}执行中", "problem_level": "warning", "problem_keys": running_actions, "summary": f"{action_labels[key]}仍在{_inspection_status_label(str(job.get('status') or ''))},先等待回执或检查对应节点事件流。", "recommended_action": "查看事件", "recommended_action_code": "focus_latest_job_events", "ui_intent": preferred_ui_intent, } missing_actions = [key for key in _OPS_INSPECTION_ACTION_KEYS if not str((job_map.get(key) or {}).get("status") or "").strip()] if missing_actions: missing_problem_map = { "health.snapshot": ("missing_health_snapshot", "缺健康快照", "健康快照缺口"), "logs.collect": ("missing_worker_logs", "缺 Worker 日志", "Worker 日志缺口"), "diagnostics.collect": ("missing_diagnostics", "缺诊断包", "诊断包缺口"), } if len(missing_actions) == 1: problem_kind, problem_label, problem_title = missing_problem_map.get( missing_actions[0], ("missing_inspection_steps", "巡检缺口", "巡检缺口"), ) else: problem_kind, problem_label, problem_title = ( "missing_inspection_steps", "巡检缺口", "巡检缺口", ) return { "problem_kind": problem_kind, "problem_label": problem_label, "problem_title": problem_title, "problem_level": "info", "problem_keys": missing_actions, "summary": f"当前仍缺少 {'、'.join(action_labels[key] for key in missing_actions)} 的巡检记录。", "recommended_action": "补齐标准巡检", "recommended_action_code": "run_standard_inspection", "ui_intent": {}, } return { "problem_kind": "healthy", "problem_label": "已收口", "problem_title": "巡检健康", "problem_level": "success", "problem_keys": [], "summary": "当前三类巡检均已成功回执。", "recommended_action": "当前无需处理", "recommended_action_code": "no_action", "ui_intent": {}, } def _slim_execution_scene_node(node: dict) -> dict: raw_participation_state = str(node.get("participation_state") or "").strip() is_dispatch_active = bool(node.get("is_dispatch_active", False)) if is_dispatch_active or raw_participation_state in {"running", "claimed", "dispatch_active"}: participation_bucket = "dispatch_active" participation_bucket_label = "执行/已领" elif raw_participation_state in {"recent_throughput", "recent_only"}: participation_bucket = "recent_only" participation_bucket_label = "近窗有吞吐" elif raw_participation_state == "load_syncing": participation_bucket = "load_syncing" participation_bucket_label = "负载待确认" elif raw_participation_state in {"standby", "idle"}: participation_bucket = "standby" participation_bucket_label = "在线待命" else: participation_bucket = raw_participation_state participation_bucket_label = str(node.get("participation_label") or "").strip() return { "node_code": str(node.get("node_code") or ""), "region": str(node.get("region") or ""), "role": str(node.get("role") or ""), "status": str(node.get("status") or ""), "current_load": int(node.get("current_load", 0) or 0), "last_heartbeat_at": str(node.get("last_heartbeat_at") or ""), "participation_state": raw_participation_state, "participation_state_raw": raw_participation_state, "participation_label": str(node.get("participation_label") or ""), "participation_reason": str(node.get("participation_reason") or node.get("standby_reason") or ""), "participation_bucket": participation_bucket, "participation_bucket_label": participation_bucket_label, "is_dispatch_active": is_dispatch_active, "items_total": int(node.get("items_total", 0) or 0), "items_claimed": int(node.get("items_claimed", 0) or 0), "items_running": int(node.get("items_running", 0) or 0), "items_completed": int(node.get("items_completed", 0) or 0), "items_failed": int(node.get("items_failed", 0) or 0), "processed_recent": int(node.get("processed_recent", 0) or 0), "processed_per_minute": float(node.get("processed_per_minute", 0) or 0), "detail": str(node.get("detail") or ""), "phase": str(node.get("phase") or ""), } def _preferred_release_label(release_summary: dict) -> str: active_by_channel = dict(release_summary.get("active_by_channel") or {}) for channel in ("stable", "beta", "canary"): version = str(active_by_channel.get(channel) or "").strip() if version: return f"{version} / {channel}" return "" def _preferred_release_for_ops() -> dict: for channel in ("stable", "beta", "canary"): release = get_latest_release(channel=channel) if int(release.get("id") or 0) > 0: return release return {} def _normalize_driver_node_codes(raw_node_codes: object) -> list[str]: normalized: list[str] = [] seen: set[str] = set() for item in list(raw_node_codes or []): node_code = str(item or "").strip() if not node_code or node_code in seen: continue seen.add(node_code) normalized.append(node_code) return normalized def _recommend_driver_execution_mode(node_codes: list[str]) -> dict: return _recommend_driver_execution_mode_for_supported_modes( node_codes, supported_modes=["remote-agent", "ssh"], default_mode="remote-agent", ) def _normalize_driver_supported_execution_modes( supported_modes: list[str] | tuple[str, ...] | None = None, *, default_mode: str = "remote-agent", ) -> list[str]: normalized_default_mode = str(default_mode or "").strip() or "remote-agent" normalized_modes: list[str] = [] seen: set[str] = set() for item in list(supported_modes or []): mode = str(item or "").strip() if not mode or mode in seen: continue seen.add(mode) normalized_modes.append(mode) if normalized_modes: return normalized_modes return [normalized_default_mode] def _recommend_driver_execution_mode_for_supported_modes( node_codes: list[str], *, supported_modes: list[str] | tuple[str, ...] | None = None, default_mode: str = "remote-agent", ) -> dict: normalized_node_codes = _normalize_driver_node_codes(node_codes) normalized_supported_modes = _normalize_driver_supported_execution_modes( supported_modes, default_mode=default_mode, ) normalized_default_mode = ( str(default_mode or "").strip() if str(default_mode or "").strip() in normalized_supported_modes else normalized_supported_modes[0] ) if len(normalized_supported_modes) == 1: only_mode = normalized_supported_modes[0] return { "execution_mode": only_mode, "reason": f"当前动作仅支持 {only_mode},保持该执行路径。", "resolved_nodes": [], "missing_node_codes": [], "supported_modes": normalized_supported_modes, } if not normalized_node_codes: return { "execution_mode": normalized_default_mode, "reason": f"当前未指定目标节点,默认继续使用 {normalized_default_mode}。", "resolved_nodes": [], "missing_node_codes": [], "supported_modes": normalized_supported_modes, } if ( "local-runtime" in normalized_supported_modes and normalized_node_codes and all(node_code == settings.node_code for node_code in normalized_node_codes) ): return { "execution_mode": "local-runtime", "reason": "当前目标节点就是本机控制面节点,推荐直接走 local-runtime。", "resolved_nodes": [], "missing_node_codes": [], "supported_modes": normalized_supported_modes, } managed_nodes_payload = list_managed_nodes_with_agent_state() managed_nodes = list(managed_nodes_payload.get("nodes") or []) managed_map = { str(item.get("node_code") or "").strip(): dict(item or {}) for item in managed_nodes if str(item.get("node_code") or "").strip() } resolved_nodes = [managed_map[node_code] for node_code in normalized_node_codes if node_code in managed_map] missing_node_codes = [node_code for node_code in normalized_node_codes if node_code not in managed_map] def _agent_ready(node: dict) -> bool: return ( bool(node.get("is_managed", False)) and bool(node.get("is_enabled", False)) and bool(node.get("is_agent_online", False)) ) def _ssh_ready(node: dict) -> bool: return ( bool(node.get("is_managed", False)) and bool(node.get("is_enabled", False)) and bool(node.get("has_ssh_access", False)) ) if ( "remote-agent" in normalized_supported_modes and resolved_nodes and len(resolved_nodes) == len(normalized_node_codes) and all(_agent_ready(node) for node in resolved_nodes) ): return { "execution_mode": "remote-agent", "reason": "当前目标节点都已进入 Agent 在线状态,默认沿用 remote-agent。", "resolved_nodes": resolved_nodes, "missing_node_codes": missing_node_codes, "supported_modes": normalized_supported_modes, } if ( "ssh" in normalized_supported_modes and resolved_nodes and len(resolved_nodes) == len(normalized_node_codes) and all(_ssh_ready(node) for node in resolved_nodes) ): return { "execution_mode": "ssh", "reason": "当前目标节点 SSH 已全部备好,自动切到 SSH 执行路径。", "resolved_nodes": resolved_nodes, "missing_node_codes": missing_node_codes, "supported_modes": normalized_supported_modes, } return { "execution_mode": normalized_default_mode, "reason": f"当前目标节点执行能力不一致,默认保守回退到 {normalized_default_mode}。", "resolved_nodes": resolved_nodes, "missing_node_codes": missing_node_codes, "supported_modes": normalized_supported_modes, } def _recommend_driver_execution_mode_for_playbook(playbook_key: str, node_codes: list[str]) -> dict: playbook = dict(get_ops_playbook(playbook_key) or {}) supported_modes = list(playbook.get("execution_modes") or []) default_mode = str(playbook.get("default_execution_mode") or "remote-agent").strip() or "remote-agent" return _recommend_driver_execution_mode_for_supported_modes( node_codes, supported_modes=supported_modes, default_mode=default_mode, ) def _recommend_driver_execution_mode_for_action_template(template_key: str, node_codes: list[str]) -> dict: template = dict(get_ops_action_template(template_key) or {}) supported_modes = list(template.get("execution_modes") or []) default_mode = str(template.get("default_execution_mode") or "remote-agent").strip() or "remote-agent" return _recommend_driver_execution_mode_for_supported_modes( node_codes, supported_modes=supported_modes, default_mode=default_mode, ) def _build_driver_ui_intent(kind: str, **payload: object) -> dict: return { "kind": str(kind or "").strip(), **payload, } def _normalize_focus_ref(focus_ref: object) -> dict: if not isinstance(focus_ref, dict): return {} normalized: dict[str, object] = {} for raw_key, raw_value in focus_ref.items(): key = str(raw_key or "").strip() if not key: continue if isinstance(raw_value, bool): normalized[key] = raw_value continue if isinstance(raw_value, int): if raw_value: normalized[key] = raw_value continue if isinstance(raw_value, float): if raw_value: normalized[key] = raw_value continue if isinstance(raw_value, str): value = raw_value.strip() if value: normalized[key] = value continue if raw_value is not None: normalized[key] = raw_value return normalized def _find_activity_stream_item(activity_key: str, *, kind: str = "", scan_limit: int = 200) -> dict: normalized_activity_key = str(activity_key or "").strip() normalized_kind = str(kind or "").strip() if not normalized_activity_key: return {} activity_stream = get_ops_activity_stream(limit=min(max(scan_limit, 20), 100), scan_limit=max(scan_limit, 80)) for item in list(activity_stream.get("items") or []): if str(item.get("activity_key") or "").strip() != normalized_activity_key: continue if normalized_kind and str(item.get("kind") or "").strip() != normalized_kind: continue return dict(item or {}) return {} def _merge_focus_ref(focus_ref: object, **extra: object) -> dict: merged = _normalize_focus_ref(focus_ref) for raw_key, raw_value in extra.items(): key = str(raw_key or "").strip() if not key: continue if isinstance(raw_value, bool): merged[key] = raw_value continue if isinstance(raw_value, int): if raw_value: merged[key] = raw_value continue if isinstance(raw_value, float): if raw_value: merged[key] = raw_value continue if isinstance(raw_value, str): value = raw_value.strip() if value: merged[key] = value continue if raw_value is not None: merged[key] = raw_value return merged def _action_payload_with_focus_ref(action_payload: dict | None = None, focus_ref: object = None) -> dict: payload = dict(action_payload or {}) merged_focus_ref = _merge_focus_ref(focus_ref, **_normalize_focus_ref(payload.get("focus_ref"))) if merged_focus_ref: payload["focus_ref"] = merged_focus_ref return payload def _driver_launchpad_context(action_payload: dict) -> dict: payload = dict(action_payload or {}) return { "channel": str(payload.get("channel") or "stable").strip() or "stable", "control_plane_base_url": str(payload.get("control_plane_base_url") or "").strip(), } def _release_launchpad_preview(release_launchpad: dict, mode: str) -> dict: if str(mode or "").strip() == "control": return dict(release_launchpad.get("control_rollout_preview") or {}) return dict(release_launchpad.get("worker_rollout_preview") or {}) def _release_launchpad_operational_rows(release_launchpad: dict, *, modes: tuple[str, ...]) -> list[dict]: rows: list[dict] = [] seen: set[tuple[str, str]] = set() for mode in modes: preview = _release_launchpad_preview(release_launchpad, mode) release_gate = dict((dict(preview.get("policy_preview") or {}).get("release_gate") or {})) operational_readiness = dict(release_gate.get("operational_readiness") or {}) for raw_row in list(operational_readiness.get("rows") or []): row = dict(raw_row or {}) node_code = str(row.get("node_code") or "").strip() if not node_code: continue key = (str(mode or "").strip() or "worker", node_code) if key in seen: continue seen.add(key) rows.append( { **row, "mode": str(mode or "").strip() or "worker", } ) return rows def _release_launchpad_gap_rows(release_launchpad: dict) -> list[dict]: rows = _release_launchpad_operational_rows(release_launchpad, modes=("worker", "control")) def _row_execution_ready(row: dict) -> bool: if "execution_ready" in row: return bool(row.get("execution_ready", False)) return bool(row.get("remote_agent_ready", False)) gap_rows = [ row for row in rows if not _row_execution_ready(row) or str(row.get("inspection_status") or "").strip() in {"running", "attention", "missing"} ] gap_rows.sort( key=lambda row: ( 0 if not _row_execution_ready(row) else 1, 0 if str(row.get("inspection_status") or "").strip() in {"attention", "missing"} else 1, 0 if str(row.get("inspection_status") or "").strip() == "running" else 1, str(row.get("node_code") or ""), ) ) return gap_rows def _first_release_launchpad_gap_row(release_launchpad: dict) -> dict: gap_rows = _release_launchpad_gap_rows(release_launchpad) return dict(gap_rows[0] or {}) if gap_rows else {} def _release_launchpad_gap_action_context(release_launchpad: dict) -> dict: first_gap_row = _first_release_launchpad_gap_row(release_launchpad) action_code = str(first_gap_row.get("recovery_action") or "").strip() node_code = str(first_gap_row.get("node_code") or "").strip() summary = str( first_gap_row.get("recovery_summary") or first_gap_row.get("onboarding_summary") or first_gap_row.get("agent_reason") or first_gap_row.get("inspection_reason") or "" ).strip() if action_code not in {"bootstrap_run", "run_acceptance"}: action_code = "" return { "action_code": action_code, "node_code": node_code, "summary": summary, "row": first_gap_row, } def _release_launchpad_target_node_codes(release_launchpad: dict, *, mode: str) -> list[str]: preview = _release_launchpad_preview(release_launchpad, mode) smart_rollout = dict(preview.get("smart_rollout") or {}) target_node_codes = _normalize_driver_node_codes(smart_rollout.get("target_node_codes") or []) if target_node_codes: return target_node_codes return _normalize_driver_node_codes( [str(row.get("node_code") or "").strip() for row in _release_launchpad_operational_rows(release_launchpad, modes=(mode,))] ) def _build_release_launchpad_review_payload(*, release_launchpad: dict, mode: str) -> dict: normalized_mode = "control" if str(mode or "").strip() == "control" else "worker" launchpad = dict(release_launchpad or {}) latest_release = dict(launchpad.get("latest_release") or {}) preview = _release_launchpad_preview(launchpad, normalized_mode) smart_rollout = dict(preview.get("smart_rollout") or {}) launchpad_status = dict(launchpad.get("launchpad_status") or {}) focus_ref = _merge_focus_ref( launchpad_status.get("focus_ref"), release_id=int(latest_release.get("id") or 0), release_version=str(latest_release.get("release_version") or "").strip(), channel=str(latest_release.get("channel") or launchpad.get("channel") or "").strip(), section="release_launchpad", mode=normalized_mode, ) return { "mode": normalized_mode, "focus_ref": focus_ref, "latest_release": latest_release, "launchpad_status": launchpad_status, "preview": preview, "smart_rollout": smart_rollout, "target_node_codes": _release_launchpad_target_node_codes(launchpad, mode=normalized_mode), "gap_rows": _release_launchpad_gap_rows(launchpad), } def _build_release_launchpad_driver_card(*, release_launchpad: dict) -> dict: launchpad = dict(release_launchpad or {}) launchpad_status = dict(launchpad.get("launchpad_status") or {}) action_code = str(launchpad_status.get("recommended_action_code") or "").strip() if not action_code: return {} latest_release = dict(launchpad.get("latest_release") or {}) latest_package = dict(launchpad.get("latest_package") or {}) worker_preview = dict(launchpad.get("worker_rollout_preview") or {}) control_preview = dict(launchpad.get("control_rollout_preview") or {}) worker_plan = dict(worker_preview.get("smart_rollout") or {}) control_plan = dict(control_preview.get("smart_rollout") or {}) first_gap_row = _first_release_launchpad_gap_row(launchpad) gap_node_codes = _normalize_driver_node_codes( [str(row.get("node_code") or "").strip() for row in _release_launchpad_gap_rows(launchpad)] ) worker_node_codes = _release_launchpad_target_node_codes(launchpad, mode="worker") control_node_codes = _release_launchpad_target_node_codes(launchpad, mode="control") latest_release_id = int(latest_release.get("id") or 0) launchpad_focus_ref = _merge_focus_ref( launchpad_status.get("focus_ref"), release_id=latest_release_id, release_version=str(latest_release.get("release_version") or "").strip(), channel=str(latest_release.get("channel") or launchpad.get("channel") or "").strip(), ) release_detail_focus_ref = _merge_focus_ref( latest_release.get("focus_ref"), release_id=latest_release_id, release_version=str(latest_release.get("release_version") or "").strip(), channel=str(latest_release.get("channel") or launchpad.get("channel") or "").strip(), section="release_detail", ) worker_launchpad_focus_ref = _merge_focus_ref(launchpad_focus_ref, section="release_launchpad", mode="worker") control_launchpad_focus_ref = _merge_focus_ref(launchpad_focus_ref, section="release_launchpad", mode="control") rollout_gate_focus_ref = _merge_focus_ref(launchpad_focus_ref, section="default_rollout_gate") release_label = ( str(latest_release.get("release_version") or "").strip() or str(latest_package.get("package_name") or "").strip() or str(launchpad.get("channel") or "stable").strip() or "latest" ) shared_action_payload = { "channel": str(launchpad.get("channel") or "stable").strip() or "stable", "control_plane_base_url": str(launchpad.get("control_plane_base_url") or "").strip(), } if latest_release_id > 0: shared_action_payload["release_id"] = latest_release_id base_card = { "level_label": "推荐", "tag_type": "info", "summary": str(launchpad_status.get("summary") or "").strip(), "reason": "", "node_codes": [], "meta_text": release_label, "focus_ref": launchpad_focus_ref, "primary_focus_ref": {}, "secondary_focus_ref": {}, "primary_type": "primary", "disabled": False, "primary_action_code": action_code, "primary_action_payload": dict(shared_action_payload), "secondary_label": "查看版本区" if latest_release_id > 0 else "", "secondary_action_code": "focus_release_hub" if latest_release_id > 0 else "", "secondary_action_payload": ( { "release_id": latest_release_id, } if latest_release_id > 0 else {} ), } if action_code == "publish_latest_worker": return { **base_card, "key": "release-launchpad-ready", "title": "当前可以发 Worker 灰度", "level_label": "准备就绪", "tag_type": "success", "reason": str(worker_plan.get("summary") or worker_preview.get("message") or "最新发布包与 Worker 智能 Rollout 预检均已收口。").strip(), "node_codes": worker_node_codes, "primary_label": "发 Worker 灰度", "primary_focus_ref": worker_launchpad_focus_ref, "primary_action_payload": _action_payload_with_focus_ref(shared_action_payload, worker_launchpad_focus_ref), "secondary_label": "看 Control 预案" if bool(control_plan.get("available", False)) else base_card["secondary_label"], "secondary_action_code": "review_control_rollout" if bool(control_plan.get("available", False)) else base_card["secondary_action_code"], "secondary_focus_ref": ( control_launchpad_focus_ref if bool(control_plan.get("available", False)) else release_detail_focus_ref ), "secondary_action_payload": ( _action_payload_with_focus_ref(shared_action_payload, control_launchpad_focus_ref) if bool(control_plan.get("available", False)) else _action_payload_with_focus_ref(base_card["secondary_action_payload"], release_detail_focus_ref) ), } if action_code == "review_smart_rollout_preview": return { **base_card, "key": "release-launchpad-review-worker", "title": "先确认 Worker 灰度预案", "level_label": "待确认", "tag_type": "warning", "reason": str(worker_preview.get("message") or worker_plan.get("summary") or "Worker 智能 Rollout 仍需人工确认。").strip(), "node_codes": worker_node_codes, "primary_label": "查看 Worker 预案", "primary_focus_ref": worker_launchpad_focus_ref, "primary_action_payload": _action_payload_with_focus_ref(shared_action_payload, worker_launchpad_focus_ref), "secondary_focus_ref": release_detail_focus_ref, "secondary_action_payload": _action_payload_with_focus_ref(base_card["secondary_action_payload"], release_detail_focus_ref), } if action_code == "review_control_rollout": return { **base_card, "key": "release-launchpad-review-control", "title": "先确认 Control 发布预案", "level_label": "推荐", "tag_type": "warning", "reason": str(control_preview.get("message") or control_plan.get("summary") or "当前只有 Control 侧满足预发条件。").strip(), "node_codes": control_node_codes, "primary_label": "查看 Control 预案", "primary_focus_ref": control_launchpad_focus_ref, "primary_action_payload": _action_payload_with_focus_ref(shared_action_payload, control_launchpad_focus_ref), "secondary_focus_ref": release_detail_focus_ref, "secondary_action_payload": _action_payload_with_focus_ref(base_card["secondary_action_payload"], release_detail_focus_ref), } if action_code == "fix_rollout_blockers": primary_label = "查看 Worker 预案" if str(first_gap_row.get("node_code") or "").strip(): primary_label = "纳管首台节点" if not bool(first_gap_row.get("is_managed", False)) else "补接入首台" if str(first_gap_row.get("recovery_action") or "").strip() == "bootstrap_run": primary_label = "跑接入收口" elif str(first_gap_row.get("recovery_action") or "").strip() == "run_acceptance": primary_label = "跑接管验收" return { **base_card, "key": "release-launchpad-fix-blockers", "title": "先收口发布门禁", "level_label": "最高优先", "tag_type": "danger", "reason": str( first_gap_row.get("agent_reason") or first_gap_row.get("inspection_reason") or worker_preview.get("message") or control_preview.get("message") or "Rollout 预检仍存在阻断条件,需先补接管、在线状态或巡检缺口。" ).strip(), "node_codes": gap_node_codes or worker_node_codes or control_node_codes, "primary_label": primary_label, "primary_focus_ref": rollout_gate_focus_ref, "primary_action_payload": _action_payload_with_focus_ref(shared_action_payload, rollout_gate_focus_ref), "secondary_focus_ref": release_detail_focus_ref, "secondary_action_payload": _action_payload_with_focus_ref(base_card["secondary_action_payload"], release_detail_focus_ref), } if action_code in {"bootstrap_run", "run_acceptance"}: primary_label = "跑接入收口" if action_code == "bootstrap_run" else "跑接管验收" return { **base_card, "key": f"release-launchpad-{action_code}", "title": "先收口执行面接管", "level_label": "最高优先" if action_code == "bootstrap_run" else "推荐", "tag_type": "warning" if action_code == "bootstrap_run" else "success", "reason": str( first_gap_row.get("recovery_summary") or first_gap_row.get("onboarding_summary") or first_gap_row.get("agent_reason") or first_gap_row.get("inspection_reason") or launchpad_status.get("summary") or "当前需要先收口节点接管,再继续发布。" ).strip(), "node_codes": gap_node_codes or ([str(first_gap_row.get("node_code") or "").strip()] if str(first_gap_row.get("node_code") or "").strip() else []), "primary_label": primary_label, "primary_focus_ref": rollout_gate_focus_ref, "primary_action_payload": _action_payload_with_focus_ref( { **shared_action_payload, "node_code": str(first_gap_row.get("node_code") or "").strip(), }, rollout_gate_focus_ref, ), "secondary_focus_ref": release_detail_focus_ref, "secondary_action_payload": _action_payload_with_focus_ref(base_card["secondary_action_payload"], release_detail_focus_ref), } if action_code == "fix_managed_nodes": primary_label = "同步节点" if str(first_gap_row.get("node_code") or "").strip(): primary_label = "纳管首台节点" if not bool(first_gap_row.get("is_managed", False)) else "补接入首台" return { **base_card, "key": "release-launchpad-fix-managed", "title": "先补执行面接管", "level_label": "推荐", "tag_type": "warning", "reason": str( first_gap_row.get("agent_reason") or first_gap_row.get("inspection_reason") or "当前没有可用于智能 Rollout 的在线启用执行节点,需要先补齐托管或接入状态。" ).strip(), "node_codes": gap_node_codes, "primary_label": primary_label, "primary_focus_ref": rollout_gate_focus_ref, "primary_action_payload": _action_payload_with_focus_ref(shared_action_payload, rollout_gate_focus_ref), "secondary_focus_ref": release_detail_focus_ref, "secondary_action_payload": _action_payload_with_focus_ref(base_card["secondary_action_payload"], release_detail_focus_ref), } if action_code == "api-restart": return { **base_card, "key": "release-launchpad-runtime-refresh", "title": "先刷新控制面运行时", "level_label": "最高优先", "tag_type": "warning", "reason": str( launchpad_status.get("summary") or "运行中的控制面 API 还没有刷新到当前仓库能力,建议先执行 runtime-refresh-recover。" ).strip(), "node_codes": [], "primary_label": "查看刷新收口", "primary_focus_ref": worker_launchpad_focus_ref or launchpad_focus_ref, "primary_action_payload": _action_payload_with_focus_ref( shared_action_payload, worker_launchpad_focus_ref or launchpad_focus_ref, ), "secondary_label": "查看版本区" if latest_release_id > 0 else "", "secondary_action_code": "focus_release_hub" if latest_release_id > 0 else "", "secondary_action_payload": {"release_id": latest_release_id} if latest_release_id > 0 else {}, "secondary_focus_ref": release_detail_focus_ref if latest_release_id > 0 else {}, } if action_code == "release_package": return { **base_card, "key": "release-launchpad-package", "title": "先准备发布包", "level_label": "待补齐", "tag_type": "warning", "reason": str(latest_package.get("reason") or "当前还没有可用的最新发布包,请先在控制面本机完成打包。").strip(), "node_codes": [], "primary_label": "打开打包面板", "secondary_label": "", "secondary_action_code": "", "secondary_action_payload": {}, "primary_focus_ref": worker_launchpad_focus_ref or launchpad_focus_ref, "primary_action_payload": _action_payload_with_focus_ref( shared_action_payload, worker_launchpad_focus_ref or launchpad_focus_ref, ), "secondary_focus_ref": {}, } if action_code == "release_prepare": return { **base_card, "key": "release-launchpad-final-signoff", "title": "先完成最终签收", "level_label": "最高优先", "tag_type": "danger", "reason": str( latest_package.get("final_release_report_stale_reason") or launchpad_status.get("summary") or "当前最新发布包还没有完成最终签收,不能直接进入正式发布动作。" ).strip(), "node_codes": [], "primary_label": "执行最终签收", "secondary_label": "查看版本区" if latest_release_id > 0 else "", "secondary_action_code": "focus_release_hub" if latest_release_id > 0 else "", "secondary_action_payload": {"release_id": latest_release_id} if latest_release_id > 0 else {}, "primary_focus_ref": worker_launchpad_focus_ref or launchpad_focus_ref, "primary_action_payload": _action_payload_with_focus_ref( shared_action_payload, worker_launchpad_focus_ref or launchpad_focus_ref, ), "secondary_focus_ref": release_detail_focus_ref if latest_release_id > 0 else {}, } return {} def _runbook_sequence( *, key: str, title: str, summary: str, reason: str, status: str = "ready", status_label: str = "", tag_type: str = "", target_node_codes: list[str] | None = None, target_scope_label: str = "", step_titles: list[str] | None = None, primary_label: str, primary_action_code: str, primary_action_payload: dict | None = None, focus_ref: dict | None = None, primary_focus_ref: dict | None = None, secondary_label: str = "", secondary_action_code: str = "", secondary_action_payload: dict | None = None, secondary_focus_ref: dict | None = None, ) -> dict: return { "key": str(key or "").strip(), "title": str(title or "").strip(), "summary": str(summary or "").strip(), "reason": str(reason or "").strip(), "status": str(status or "").strip() or "ready", "status_label": str(status_label or "").strip() or str(status or "").strip() or "ready", "tag_type": str(tag_type or "").strip() or "info", "target_node_codes": _normalize_driver_node_codes(target_node_codes or []), "target_scope_label": str(target_scope_label or "").strip(), "step_titles": [str(item or "").strip() for item in list(step_titles or []) if str(item or "").strip()], "focus_ref": _normalize_focus_ref(focus_ref), "primary_label": str(primary_label or "").strip() or "执行", "primary_action_code": str(primary_action_code or "").strip(), "primary_action_payload": _action_payload_with_focus_ref(primary_action_payload or {}, primary_focus_ref), "primary_focus_ref": _normalize_focus_ref(primary_focus_ref), "secondary_label": str(secondary_label or "").strip(), "secondary_action_code": str(secondary_action_code or "").strip(), "secondary_action_payload": _action_payload_with_focus_ref(secondary_action_payload or {}, secondary_focus_ref), "secondary_focus_ref": _normalize_focus_ref(secondary_focus_ref), } def _build_release_launchpad_runbook_sequence(*, release_launchpad: dict) -> dict: launchpad = dict(release_launchpad or {}) launchpad_status = dict(launchpad.get("launchpad_status") or {}) action_code = str(launchpad_status.get("recommended_action_code") or "").strip() if not action_code: return {} latest_release = dict(launchpad.get("latest_release") or {}) latest_package = dict(launchpad.get("latest_package") or {}) worker_preview = dict(launchpad.get("worker_rollout_preview") or {}) control_preview = dict(launchpad.get("control_rollout_preview") or {}) worker_plan = dict(worker_preview.get("smart_rollout") or {}) control_plan = dict(control_preview.get("smart_rollout") or {}) first_gap_row = _first_release_launchpad_gap_row(launchpad) latest_release_id = int(latest_release.get("id") or 0) worker_node_codes = _release_launchpad_target_node_codes(launchpad, mode="worker") control_node_codes = _release_launchpad_target_node_codes(launchpad, mode="control") gap_node_codes = _normalize_driver_node_codes( [str(row.get("node_code") or "").strip() for row in _release_launchpad_gap_rows(launchpad)] ) launchpad_focus_ref = _merge_focus_ref( launchpad_status.get("focus_ref"), release_id=latest_release_id, release_version=str(latest_release.get("release_version") or "").strip(), channel=str(latest_release.get("channel") or launchpad.get("channel") or "").strip(), ) release_detail_focus_ref = _merge_focus_ref( latest_release.get("focus_ref"), release_id=latest_release_id, release_version=str(latest_release.get("release_version") or "").strip(), channel=str(latest_release.get("channel") or launchpad.get("channel") or "").strip(), section="release_detail", ) worker_launchpad_focus_ref = _merge_focus_ref(launchpad_focus_ref, section="release_launchpad", mode="worker") control_launchpad_focus_ref = _merge_focus_ref(launchpad_focus_ref, section="release_launchpad", mode="control") rollout_gate_focus_ref = _merge_focus_ref(launchpad_focus_ref, section="default_rollout_gate") release_scope_label = ( str(latest_release.get("release_version") or "").strip() or str(latest_package.get("package_name") or "").strip() or str(launchpad.get("channel") or "stable").strip() or "latest" ) shared_action_payload = { "channel": str(launchpad.get("channel") or "stable").strip() or "stable", "control_plane_base_url": str(launchpad.get("control_plane_base_url") or "").strip(), } if latest_release_id > 0: shared_action_payload["release_id"] = latest_release_id base_kwargs = { "key": "release_progression", "title": "版本发布与放量", "summary": str(launchpad_status.get("summary") or "").strip(), "status": str(launchpad_status.get("status") or "").strip() or "attention", "status_label": str(launchpad_status.get("status_label") or "").strip() or "待处理", "target_scope_label": f"Release / Launchpad / {release_scope_label}", "focus_ref": launchpad_focus_ref, } if action_code == "publish_latest_worker": return _runbook_sequence( **base_kwargs, reason=str(worker_plan.get("summary") or worker_preview.get("message") or "最新发布包与 Worker 智能 Rollout 预检均已收口。").strip(), tag_type="success", target_node_codes=worker_node_codes, step_titles=["确认最新发布包", "执行 Worker 灰度", "观察回执与巡检", "必要时推进 Control"], primary_label="直接发 Worker 灰度", primary_action_code="publish_latest_worker", primary_action_payload=dict(shared_action_payload), primary_focus_ref=worker_launchpad_focus_ref, secondary_label="查看 Control 预案" if bool(control_plan.get("available", False)) else ("查看版本区" if latest_release_id > 0 else ""), secondary_action_code="review_control_rollout" if bool(control_plan.get("available", False)) else ("focus_release_hub" if latest_release_id > 0 else ""), secondary_action_payload=dict(shared_action_payload) if bool(control_plan.get("available", False)) else ({"release_id": latest_release_id} if latest_release_id > 0 else {}), secondary_focus_ref=control_launchpad_focus_ref if bool(control_plan.get("available", False)) else release_detail_focus_ref, ) if action_code == "review_smart_rollout_preview": return _runbook_sequence( **base_kwargs, reason=str(worker_preview.get("message") or worker_plan.get("summary") or "Worker 智能 Rollout 仍需人工确认。").strip(), tag_type="warning", target_node_codes=worker_node_codes, step_titles=["查看 Worker 智能预案", "确认门禁与审批", "必要时继续创建", "回看 Release Hub"], primary_label="查看 Worker 预案", primary_action_code="review_smart_rollout_preview", primary_action_payload=dict(shared_action_payload), primary_focus_ref=worker_launchpad_focus_ref, secondary_label="查看版本区" if latest_release_id > 0 else "", secondary_action_code="focus_release_hub" if latest_release_id > 0 else "", secondary_action_payload={"release_id": latest_release_id} if latest_release_id > 0 else {}, secondary_focus_ref=release_detail_focus_ref, ) if action_code == "review_control_rollout": return _runbook_sequence( **base_kwargs, reason=str(control_preview.get("message") or control_plan.get("summary") or "当前只有 Control 侧满足预发条件。").strip(), tag_type="warning", target_node_codes=control_node_codes, step_titles=["查看 Control 预案", "确认 Control 目标", "确认门禁与审批", "必要时推进发布"], primary_label="查看 Control 预案", primary_action_code="review_control_rollout", primary_action_payload=dict(shared_action_payload), primary_focus_ref=control_launchpad_focus_ref, secondary_label="查看版本区" if latest_release_id > 0 else "", secondary_action_code="focus_release_hub" if latest_release_id > 0 else "", secondary_action_payload={"release_id": latest_release_id} if latest_release_id > 0 else {}, secondary_focus_ref=release_detail_focus_ref, ) if action_code == "fix_rollout_blockers": return _runbook_sequence( **base_kwargs, reason=str( first_gap_row.get("agent_reason") or first_gap_row.get("inspection_reason") or worker_preview.get("message") or control_preview.get("message") or "Rollout 预检仍存在阻断条件。" ).strip(), tag_type="danger", target_node_codes=gap_node_codes or worker_node_codes or control_node_codes, step_titles=["查看 Worker 智能预案", "定位阻断节点", "补接管或巡检缺口", "重新回看预案"], primary_label="查看阻断预案", primary_action_code="fix_rollout_blockers", primary_action_payload=dict(shared_action_payload), primary_focus_ref=rollout_gate_focus_ref, secondary_label="补执行面接管", secondary_action_code="fix_managed_nodes", secondary_action_payload=dict(shared_action_payload), secondary_focus_ref=rollout_gate_focus_ref, ) if action_code == "fix_managed_nodes": step_titles = ["同步集群节点", "纳管首台缺口", "补齐 Agent 接入", "回到 Launchpad 复检"] if str(first_gap_row.get("recovery_action") or "").strip() == "bootstrap_run": step_titles = ["定位首台缺口节点", "执行 onboarding.bootstrap", "等待 register / heartbeat", "回到 Launchpad 复检"] elif str(first_gap_row.get("recovery_action") or "").strip() == "run_acceptance": step_titles = ["定位待验收节点", "执行 onboarding.acceptance", "确认 health / worker / agent", "回到 Launchpad 复检"] return _runbook_sequence( **base_kwargs, reason=str( first_gap_row.get("agent_reason") or first_gap_row.get("inspection_reason") or "当前没有可用于智能 Rollout 的在线启用执行节点,需要先补齐托管或接入状态。" ).strip(), tag_type="warning", target_node_codes=gap_node_codes, step_titles=step_titles, primary_label="补执行面接管", primary_action_code="fix_managed_nodes", primary_action_payload=dict(shared_action_payload), primary_focus_ref=rollout_gate_focus_ref, secondary_label="查看版本区" if latest_release_id > 0 else "", secondary_action_code="focus_release_hub" if latest_release_id > 0 else "", secondary_action_payload={"release_id": latest_release_id} if latest_release_id > 0 else {}, secondary_focus_ref=release_detail_focus_ref, ) if action_code in {"bootstrap_run", "run_acceptance"}: return _runbook_sequence( **base_kwargs, reason=str( first_gap_row.get("recovery_summary") or first_gap_row.get("onboarding_summary") or first_gap_row.get("agent_reason") or "当前需要先收口节点接管,再继续发布。" ).strip(), tag_type="warning" if action_code == "bootstrap_run" else "success", target_node_codes=gap_node_codes or ([str(first_gap_row.get("node_code") or "").strip()] if str(first_gap_row.get("node_code") or "").strip() else []), step_titles=( ["定位首台缺口节点", "执行 onboarding.bootstrap", "等待 register / heartbeat", "回到 Launchpad 复检"] if action_code == "bootstrap_run" else ["定位待验收节点", "执行 onboarding.acceptance", "确认 health / worker / agent", "回到 Launchpad 复检"] ), primary_label="跑接入收口" if action_code == "bootstrap_run" else "跑接管验收", primary_action_code=action_code, primary_action_payload={ **dict(shared_action_payload), "node_code": str(first_gap_row.get("node_code") or "").strip(), }, primary_focus_ref=rollout_gate_focus_ref, secondary_label="查看版本区" if latest_release_id > 0 else "", secondary_action_code="focus_release_hub" if latest_release_id > 0 else "", secondary_action_payload={"release_id": latest_release_id} if latest_release_id > 0 else {}, secondary_focus_ref=release_detail_focus_ref, ) if action_code == "api-restart": return _runbook_sequence( **base_kwargs, reason=str( launchpad_status.get("summary") or "运行中的控制面 API 还没有刷新到当前仓库能力,建议先执行 runtime-refresh-recover。" ).strip(), tag_type="warning", target_node_codes=[], step_titles=["执行 runtime-refresh-recover", "确认 build-info / route surface", "重新拉取 Launchpad", "再决定 Release / Rollout"], primary_label="查看刷新收口", primary_action_code="api-restart", primary_action_payload=dict(shared_action_payload), primary_focus_ref=worker_launchpad_focus_ref or launchpad_focus_ref, secondary_label="查看版本区" if latest_release_id > 0 else "", secondary_action_code="focus_release_hub" if latest_release_id > 0 else "", secondary_action_payload={"release_id": latest_release_id} if latest_release_id > 0 else {}, secondary_focus_ref=release_detail_focus_ref if latest_release_id > 0 else {}, ) if action_code == "release_package": return _runbook_sequence( **base_kwargs, reason=str(latest_package.get("reason") or "当前没有可用的最新发布包,请先在控制面本机完成打包。").strip(), tag_type="warning", target_node_codes=[], step_titles=["打开打包面板", "导入最新发布包", "确认 Release 记录", "重新回看 Launchpad"], primary_label="打开打包面板", primary_action_code="release_package", primary_action_payload=dict(shared_action_payload), primary_focus_ref=worker_launchpad_focus_ref or launchpad_focus_ref, secondary_label="查看版本区" if latest_release_id > 0 else "", secondary_action_code="focus_release_hub" if latest_release_id > 0 else "", secondary_action_payload={"release_id": latest_release_id} if latest_release_id > 0 else {}, secondary_focus_ref=release_detail_focus_ref, ) if action_code == "release_prepare": return _runbook_sequence( **base_kwargs, reason=str( latest_package.get("final_release_report_stale_reason") or launchpad_status.get("summary") or "当前最新发布包还没有完成最终签收,不能直接进入正式发布动作。" ).strip(), tag_type="danger", target_node_codes=[], step_titles=["执行最终签收", "确认 verify / smoke / final report", "刷新最新包状态", "回到 Launchpad 复检"], primary_label="执行最终签收", primary_action_code="release_prepare", primary_action_payload=dict(shared_action_payload), primary_focus_ref=worker_launchpad_focus_ref or launchpad_focus_ref, secondary_label="查看版本区" if latest_release_id > 0 else "", secondary_action_code="focus_release_hub" if latest_release_id > 0 else "", secondary_action_payload={"release_id": latest_release_id} if latest_release_id > 0 else {}, secondary_focus_ref=release_detail_focus_ref, ) return {} def _build_ops_runbook_control_sequences( *, managed_nodes_payload: dict | None = None, release_launchpad: dict | None = None, ) -> list[dict]: nodes = list((managed_nodes_payload or {}).get("nodes") or []) preferred_release = _preferred_release_for_ops() preferred_release_id = int(preferred_release.get("id") or 0) preferred_release_label = ( str(preferred_release.get("release_version") or "").strip() or str(preferred_release.get("channel") or "").strip() or "默认 Release" ) bootstrap_gap_nodes = [ item for item in nodes if bool(item.get("cluster_is_effective_worker", False)) and ( not bool(item.get("is_managed", False)) or not bool(item.get("is_enabled", False)) or not bool(item.get("is_agent_online", False)) ) ] bootstrap_gap_node_codes = _normalize_driver_node_codes(item.get("node_code") for item in bootstrap_gap_nodes) participating_nodes = [ item for item in nodes if bool(item.get("cluster_is_effective_worker", False)) and bool(item.get("is_current_participant", False)) ] participating_node_codes = _normalize_driver_node_codes(item.get("node_code") for item in participating_nodes) inspection_ready_nodes = [ item for item in nodes if _is_inspection_eligible_node(item) ] inspection_ready_node_codes = _normalize_driver_node_codes(item.get("node_code") for item in inspection_ready_nodes) worker_release_node_codes = _normalize_driver_node_codes( item.get("node_code") for item in inspection_ready_nodes if str(item.get("role") or "").strip() == "worker" ) control_release_node_codes = _normalize_driver_node_codes( item.get("node_code") for item in inspection_ready_nodes if str(item.get("role") or "").strip() == "control" ) scene_target_node_codes = participating_node_codes or inspection_ready_node_codes onboarding_status = "warning" if bootstrap_gap_node_codes else "ready" onboarding_status_label = "有待接管节点" if bootstrap_gap_node_codes else "接管链路就绪" onboarding_tag_type = "warning" if bootstrap_gap_node_codes else "success" onboarding_summary = ( f"当前有 {len(bootstrap_gap_node_codes)} 台有效执行节点还没进入 Agent 在线状态,建议先从海外控制面生成接入工单。" if bootstrap_gap_node_codes else "当前没有明显的接管缺口;后续新增大陆机器时,直接从这里生成接入工单。" ) scene_status = "warning" if scene_target_node_codes else "info" scene_status_label = "现场可观察" if scene_target_node_codes else "暂无现场目标" scene_tag_type = "warning" if scene_target_node_codes else "info" scene_summary = ( f"当前有 {len(scene_target_node_codes)} 台节点适合作为执行现场观察目标,优先从关键日志开始,再决定是否升级到全量取证。" if scene_target_node_codes else "当前没有明确的参与节点;后续一旦有节点开始领任务,这里就是海外控制面第一入口。" ) inspection_status = "ready" if inspection_ready_node_codes else "attention" inspection_status_label = "可直接巡检" if inspection_ready_node_codes else "待补接管" inspection_tag_type = "success" if inspection_ready_node_codes else "warning" inspection_summary = ( f"当前有 {len(inspection_ready_node_codes)} 台节点已经满足 已纳管 + 已启用 + Agent 在线,可直接走标准巡检。" if inspection_ready_node_codes else "当前还没有满足标准巡检条件的节点,建议先补齐接管和 Agent 在线状态。" ) release_status = "ready" if preferred_release_id > 0 else "attention" release_status_label = "可进入发布路径" if preferred_release_id > 0 else "待创建 Release" release_tag_type = "success" if preferred_release_id > 0 else "warning" release_summary = ( f"当前默认版本 {preferred_release_label} 已可用于点状灰度或 Rollout;先发 Worker,再逐步推进控制面。" if preferred_release_id > 0 else "当前还没有可用 Release;建议先在海外控制面建立版本入口,再进入发布闭环。" ) release_progression_sequence = _build_release_launchpad_runbook_sequence( release_launchpad=dict(release_launchpad or {}) ) if not release_progression_sequence: release_progression_sequence = _runbook_sequence( key="release_progression", title="版本发布与放量", summary=release_summary, reason="正式运维时,版本更新应该统一落回 Release / Rollout,而不是回退到各机器单独 pull / restart。", status=release_status, status_label=release_status_label, tag_type=release_tag_type, target_node_codes=worker_release_node_codes or control_release_node_codes, target_scope_label="Release / Rollout", step_titles=["点状灰度 Worker", "健康检查", "必要时推进 Rollout", "最后处理控制面"], primary_label="直接创建 Worker Rollout", primary_action_code="create_release_rollout_worker", primary_action_payload={ "release_id": preferred_release_id, "target_node_codes": worker_release_node_codes, }, secondary_label="查看版本区", secondary_action_code="focus_release_hub", secondary_action_payload={ "release_id": preferred_release_id, }, ) sequences = [ _runbook_sequence( key="node_onboarding", title="纳管新节点", summary=onboarding_summary, reason="所有新增大陆 controller / worker,第一步都应该先生成 Node Agent 接入工单,而不是手动 SSH 拼命令。", status=onboarding_status, status_label=onboarding_status_label, tag_type=onboarding_tag_type, target_node_codes=bootstrap_gap_node_codes, target_scope_label="单节点 / 新节点", step_titles=["生成接入工单", "复制 Env/脚本", "节点落地执行", "回到控制面看心跳"], primary_label="跑接入收口", primary_action_code="bootstrap_run", primary_action_payload={ "node_code": bootstrap_gap_node_codes[0] if bootstrap_gap_node_codes else "", }, secondary_label="跑接管验收", secondary_action_code="run_acceptance", secondary_action_payload={ "node_code": inspection_ready_node_codes[0] if inspection_ready_node_codes else "", }, ), _runbook_sequence( key="scene_observe", title="执行现场观察", summary=scene_summary, reason="海外控制面遇到“谁在跑、为什么卡住、日志有没有回来”这类问题时,应优先走统一现场观察路径。", status=scene_status, status_label=scene_status_label, tag_type=scene_tag_type, target_node_codes=scene_target_node_codes, target_scope_label="参与节点 / 现场节点", step_titles=["关键日志", "判断是否升级全量", "补诊断包", "回看联调快照"], primary_label="执行关键现场日志", primary_action_code="run_scene_logs_key", primary_action_payload={ "execution_mode": "remote-agent", "auto_approve": True, }, secondary_label="执行全量现场取证", secondary_action_code="run_scene_logs_full", secondary_action_payload={ "execution_mode": "remote-agent", "auto_approve": True, }, ), _runbook_sequence( key="standard_inspection", title="标准巡检", summary=inspection_summary, reason="现场收口、节点验收和发布前确认,尽量都走同一条标准巡检链,而不是手工拼快照、日志和诊断包。", status=inspection_status, status_label=inspection_status_label, tag_type=inspection_tag_type, target_node_codes=inspection_ready_node_codes, target_scope_label="已接管执行节点", step_titles=["健康快照", "Worker 日志", "诊断包", "统一回看巡检结果"], primary_label="直接执行标准巡检", primary_action_code="run_standard_inspection", primary_action_payload={}, secondary_label="补充诊断包", secondary_action_code="open_diagnostics", secondary_action_payload={ "execution_mode": "remote-agent", "auto_approve": True, }, ), release_progression_sequence, ] return sequences def _activity_time(*candidates: object) -> str: for candidate in candidates: value = str(candidate or "").strip() if value: return value return "" def _truncate_activity_text(value: object, *, limit: int = 240) -> str: text = str(value or "").strip() if len(text) <= limit: return text return f"{text[:limit]}..." def _activity_status_label(status: object, *, kind: str = "") -> str: normalized_status = str(status or "").strip() mapping = { "empty": "无活动", "planned": "待推进", "ready": "就绪", "healthy": "健康", "running": "执行中", "attention": "待处理", "queued": "排队中", "dispatching": "派发中", "awaiting_approval": "待审批", "success": "成功", "failed": "失败", "blocked": "阻断", "cancelled": "已取消", "halted": "已暂停", "completed_with_issues": "带问题完成", "partially_succeeded": "部分成功", } if normalized_status in mapping: return mapping[normalized_status] if normalized_status: return normalized_status if str(kind or "").strip() in {"execution_scene", "log_sync"}: return "待观察" return "未知" def _activity_focus_ref(item: dict) -> dict: normalized_item = dict(item or {}) kind = str(normalized_item.get("kind") or "").strip() ui_intent = dict(normalized_item.get("ui_intent") or {}) if kind == "playbook_run": return { "kind": "playbook_run", "run_code": str(normalized_item.get("run_code") or "").strip(), "focus_step_key": str((ui_intent.get("focus_step_key") or normalized_item.get("focus_step_key") or "")).strip(), "focus_step_title": str((ui_intent.get("focus_step_title") or normalized_item.get("focus_step_title") or "")).strip(), } if kind == "ops_job": return { "kind": "ops_job", "job_id": int(normalized_item.get("job_id") or 0), "job_code": str(normalized_item.get("job_code") or "").strip(), "action": str((ui_intent.get("action") or normalized_item.get("title") or "")).strip(), "target_node_code": str( ( ui_intent.get("target_node_code") or next(iter(normalized_item.get("target_node_codes") or []), "") or "" ) ).strip(), } if kind == "rollout": return { "kind": "rollout", "rollout_id": int(normalized_item.get("rollout_id") or 0), "rollout_code": str(normalized_item.get("title") or "").strip(), "release_id": int((ui_intent.get("release_id") or 0)), } if kind == "runbook_sequence": return { "kind": "runbook_sequence", "sequence_key": str(normalized_item.get("sequence_key") or "").strip(), "driver_action_code": str(ui_intent.get("driver_action_code") or "").strip(), } if kind in {"execution_scene", "log_sync"}: return { "kind": "execution_scene", "scene_key": str(normalized_item.get("activity_key") or kind).strip(), } return { "kind": kind or "activity", "activity_key": str(normalized_item.get("activity_key") or "").strip(), } def _finalize_activity_item(item: dict) -> dict: normalized_item = dict(item or {}) kind = str(normalized_item.get("kind") or "").strip() ui_intent = dict(normalized_item.get("ui_intent") or {}) summary_text = str(normalized_item.get("summary") or "").strip() status_label = str(normalized_item.get("status_label") or "").strip() or _activity_status_label( normalized_item.get("status"), kind=kind, ) target_node_codes = _normalize_driver_node_codes(normalized_item.get("target_node_codes") or []) finalized = { **normalized_item, "summary": summary_text, "summary_text": summary_text, "status_label": status_label, "ui_intent": ui_intent, "ui_intent_kind": str(ui_intent.get("kind") or "").strip(), "target_node_codes": target_node_codes, } existing_focus_ref = dict(normalized_item.get("focus_ref") or {}) existing_source_focus_ref = dict(normalized_item.get("source_focus_ref") or {}) finalized["focus_ref"] = existing_focus_ref if existing_focus_ref else _activity_focus_ref(finalized) finalized["source_focus_ref"] = existing_source_focus_ref contract_keys = _normalize_ops_contract_keys( list(normalized_item.get("contract_keys") or []) + _suggest_ops_contract_keys_for_activity_item(finalized) ) finalized["contract_keys"] = contract_keys finalized["contract_navigation"] = _build_ops_contract_navigation( contract_keys, primary_contract_key=_primary_ops_contract_key_for_activity_item(finalized, contract_keys), registry=get_ops_contract_registry(), ) return finalized def _activity_stream_summary_state(items: list[dict]) -> tuple[str, str, str]: normalized_items = [dict(item or {}) for item in list(items or [])] if not normalized_items: return "empty", "无活动", "当前没有匹配的活动。" hard_attention_statuses = { "attention", "failed", "blocked", "cancelled", "halted", "completed_with_issues", "partially_succeeded", "waiting_sample", "partial_coverage", } soft_attention_statuses = {"awaiting_approval"} running_statuses = {"running", "dispatching"} planned_statuses = {"queued", "planned", "ready"} item_statuses = {str(item.get("status") or "").strip() for item in normalized_items if str(item.get("status") or "").strip()} if item_statuses & hard_attention_statuses: return "attention", "待处理", "最近活动里存在需优先处理的现场、任务或发布推进。" if item_statuses & soft_attention_statuses: return "attention", "待审批", "最近活动里存在待审批或待人工确认的动作。" if item_statuses & running_statuses: return "running", "执行中", "最近活动里存在正在推进的现场、任务、编排或回传。" if item_statuses & planned_statuses: return "planned", "待推进", "最近活动里存在已排队或即将继续推进的动作。" return "ready", "平稳", "最近活动整体平稳,可按需抽样查看。" def _is_playbook_child_job(job: dict) -> bool: return bool(str(((job.get("metadata") or {}).get("playbook") or {}).get("run_code") or "").strip()) def _build_playbook_run_activity(run: dict) -> dict: normalized_run = dict(run or {}) run_code = str(normalized_run.get("run_code") or "").strip() title = ( str(normalized_run.get("playbook_title") or "").strip() or str(normalized_run.get("playbook_key") or "").strip() or "标准编排" ) steps = list(normalized_run.get("steps") or []) step_count = int(normalized_run.get("step_count") or len(steps) or 0) completed_steps = sum(1 for step in steps if str(step.get("status") or "").strip() in {"success", "attention"}) target_node_codes = _normalize_driver_node_codes(normalized_run.get("target_node_codes") or []) focus_summary = str(normalized_run.get("focus_summary") or "").strip() if not focus_summary: focus_summary = f"当前状态 {str(normalized_run.get('status') or '').strip() or 'queued'},建议进入编排详情继续观察。" meta_parts = [run_code] group_title = str(normalized_run.get("group_title") or normalized_run.get("group_key") or "").strip() if group_title: meta_parts.append(group_title) execution_mode_label_text = ( str(normalized_run.get("execution_mode_label") or "").strip() or execution_mode_label(str(normalized_run.get("execution_mode") or "").strip()) ) if execution_mode_label_text: meta_parts.append(f"执行 {execution_mode_label_text}") if step_count > 0: meta_parts.append(f"步骤 {completed_steps}/{step_count}") jobs_total = int(normalized_run.get("jobs_total") or 0) if jobs_total > 0: meta_parts.append(f"任务 {jobs_total}") if target_node_codes: meta_parts.append(f"节点 {len(target_node_codes)}") return { "kind": "playbook_run", "activity_key": f"playbook-run:{run_code or 'latest'}", "title": title, "subtitle": run_code, "summary": _truncate_activity_text(focus_summary), "meta_text": " / ".join(part for part in meta_parts if part), "status": str(normalized_run.get("status") or "").strip(), "execution_mode": str(normalized_run.get("execution_mode") or "").strip(), "execution_mode_label": execution_mode_label_text, "occurred_at": _activity_time(normalized_run.get("updated_at"), normalized_run.get("created_at")), "run_code": run_code, "focus_step_key": str(normalized_run.get("focus_step_key") or "").strip(), "focus_step_title": str(normalized_run.get("focus_step_title") or "").strip(), "target_node_codes": target_node_codes, "ui_intent": _build_driver_ui_intent( "playbook_run_detail", run_code=run_code, focus_step_key=str(normalized_run.get("focus_step_key") or "").strip(), focus_step_title=str(normalized_run.get("focus_step_title") or "").strip(), ), } def _build_bootstrap_job_activity(job: dict, managed_node: dict | None = None) -> dict: normalized_job = dict(job or {}) normalized_node = dict(managed_node or {}) job_id = int(normalized_job.get("id") or 0) job_code = str(normalized_job.get("job_code") or "").strip() status = str(normalized_job.get("status") or "").strip() target_node_code = str(normalized_job.get("target_node_code") or "").strip() error_message = str(normalized_job.get("error_message") or "").strip() result = dict(normalized_job.get("result") or {}) bootstrap_plan = dict(result.get("bootstrap_plan") or {}) script_name = str(bootstrap_plan.get("bootstrap_script_name") or "").strip() agent_state = str(normalized_node.get("agent_state") or "").strip() agent_state_label = str(normalized_node.get("agent_state_label") or "").strip() if error_message and status in {"failed", "blocked", "cancelled"}: summary = error_message elif status == "awaiting_approval": summary = "接入工单等待审批,审批通过后才会生成 bootstrap env、脚本和一键落地命令。" elif status in {"queued", "dispatching", "running"}: summary = "控制面正在生成节点接入工单,完成后可直接复制 bootstrap env、脚本和一键落地命令。" elif status == "success": if agent_state in {"online", "online_busy"}: summary = "接入工单已生成,目标节点已完成 Agent 接入,可继续执行接管后验收。" elif agent_state == "pending_bootstrap": summary = "接入工单已生成,节点尚未回连 Agent,可直接复制脚本或一键落地命令。" elif agent_state == "runtime_only": summary = "接入工单已生成,但当前只有 runtime 心跳,Node Agent 还未真正接管。" elif agent_state == "stale": summary = "接入工单已生成,但 Agent 心跳已过期,建议先看任务详情和 Agent 日志。" else: summary = "接入工单已生成,可直接复制 bootstrap env、脚本和一键落地命令。" else: summary = "节点接入工单最近有新的回执。" execution_mode_label_text = ( str(normalized_job.get("execution_mode_label") or "").strip() or execution_mode_label(str(normalized_job.get("execution_mode") or "control-plane").strip() or "control-plane") ) meta_parts = [ part for part in [ job_code, f"节点 {target_node_code}" if target_node_code else "", f"方式 {execution_mode_label_text}" if execution_mode_label_text else "", script_name, ] if part ] if agent_state_label: meta_parts.append(f"状态 {agent_state_label}") return { "kind": "ops_job", "activity_key": f"ops-job:{job_id or job_code or 'node.bootstrap'}", "action": "node.bootstrap", "title": "节点接入工单", "subtitle": job_code, "summary": _truncate_activity_text(summary), "meta_text": " / ".join(meta_parts), "status": status, "execution_mode": str(normalized_job.get("execution_mode") or "control-plane").strip() or "control-plane", "execution_mode_label": execution_mode_label_text, "occurred_at": _activity_time( normalized_job.get("updated_at"), normalized_job.get("finished_at"), normalized_job.get("started_at"), normalized_job.get("created_at"), ), "job_id": job_id, "job_code": job_code, "target_node_codes": _normalize_driver_node_codes([target_node_code]), "ui_intent": _build_driver_ui_intent( "job_detail", job_id=job_id, job_code=job_code, action="node.bootstrap", target_node_code=target_node_code, ), } def _build_ops_job_activity(job: dict, *, managed_node: dict | None = None) -> dict: normalized_job = dict(job or {}) job_id = int(normalized_job.get("id") or 0) job_code = str(normalized_job.get("job_code") or "").strip() action = str(normalized_job.get("action") or "").strip() or "ops.job" if action == "node.bootstrap": return _build_bootstrap_job_activity(normalized_job, managed_node) target_node_code = str(normalized_job.get("target_node_code") or "").strip() target_type = str(normalized_job.get("target_type") or "").strip() execution_mode = str(normalized_job.get("execution_mode") or "").strip() execution_mode_label_text = ( str(normalized_job.get("execution_mode_label") or "").strip() or execution_mode_label(execution_mode) ) requested_by = str(normalized_job.get("requested_by") or "").strip() status = str(normalized_job.get("status") or "").strip() error_message = str(normalized_job.get("error_message") or "").strip() result = dict(normalized_job.get("result") or {}) start_delivery_state = str( result.get("start_delivery_state") or normalized_job.get("start_delivery_state") or "" ).strip() start_delivery_error = str( result.get("start_delivery_error") or normalized_job.get("start_delivery_error") or "" ).strip() preferred_summary_text = str( normalized_job.get("result_summary_text") or result.get("summary_text") or result.get("summary") or normalized_job.get("summary_text") or normalized_job.get("summary") or "" ).strip() summary_parts: list[str] = [] if not preferred_summary_text: if target_node_code: summary_parts.append(f"目标节点 {target_node_code}") elif target_type: summary_parts.append(f"目标类型 {target_type}") if requested_by: summary_parts.append(f"发起 {requested_by}") if execution_mode_label_text: summary_parts.append(f"方式 {execution_mode_label_text}") summary_candidates: list[str] = [] if preferred_summary_text: summary_candidates.append(preferred_summary_text) if error_message and status in {"failed", "blocked", "cancelled"}: summary_candidates.append(error_message) if start_delivery_state == "failed_local": summary_candidates.append("节点已接单执行,但开始回执未成功送达控制面。") if start_delivery_error: summary_candidates.append(f"开始回执失败:{start_delivery_error}") meta_parts = [job_code] risk_level = str(normalized_job.get("risk_level") or "").strip() if risk_level: meta_parts.append(f"风险 {risk_level}") if execution_mode_label_text: meta_parts.append(f"执行 {execution_mode_label_text}") effective_status = status if start_delivery_state == "failed_local" and status not in {"failed", "blocked", "cancelled"}: effective_status = "attention" if status: meta_parts.append(f"任务 {_activity_status_label(status)}") meta_parts.append("开始回执异常") merged_focus_ref = _merge_focus_ref(normalized_job.get("focus_ref"), **_normalize_focus_ref(result.get("focus_ref"))) source_focus_ref = _normalize_focus_ref(result.get("source_focus_ref")) summary_text = " / ".join( dict.fromkeys(part for part in [*summary_candidates, *summary_parts] if str(part or "").strip()) ) or "标准运维任务最近有新回执。" activity = { "kind": "ops_job", "activity_key": f"ops-job:{job_id or job_code or action}", "action": action, "title": action, "subtitle": job_code, "summary": _truncate_activity_text(summary_text), "meta_text": " / ".join(part for part in meta_parts if part), "status": effective_status, "job_status": status, "execution_mode": execution_mode, "execution_mode_label": execution_mode_label_text, "occurred_at": _activity_time( normalized_job.get("updated_at"), normalized_job.get("finished_at"), normalized_job.get("started_at"), normalized_job.get("created_at"), ), "job_id": job_id, "job_code": job_code, "target_node_codes": _normalize_driver_node_codes([target_node_code]), "focus_ref": merged_focus_ref, "source_focus_ref": source_focus_ref, "ui_intent": _build_driver_ui_intent( "job_events", job_id=job_id, job_code=job_code, action=action, target_node_code=target_node_code, ), } if start_delivery_state: activity["start_delivery_state"] = start_delivery_state if start_delivery_error: activity["start_delivery_error"] = start_delivery_error if start_delivery_state == "failed_local": activity["problem_code"] = "ops_job_start_delivery_failed_local" return activity def _build_rollout_activity(rollout: dict) -> dict: normalized_rollout = dict(rollout or {}) rollout_id = int(normalized_rollout.get("id") or 0) rollout_code = str(normalized_rollout.get("rollout_code") or "").strip() rollout_policy = dict(normalized_rollout.get("policy") or {}) rollout_execution_mode = str(rollout_policy.get("execution_mode") or "remote-agent").strip() or "remote-agent" rollout_execution_mode_label = ( str(normalized_rollout.get("execution_mode_label") or "").strip() or execution_mode_label(rollout_execution_mode) ) rollout_target_node_codes = _normalize_driver_node_codes( [str(item.get("node_code") or "").strip() for item in list(normalized_rollout.get("target_nodes") or [])] ) target_nodes_total = len(rollout_target_node_codes) jobs_total = int(normalized_rollout.get("jobs_total") or 0) jobs_created = int(normalized_rollout.get("jobs_created") or 0) batch_cursor = int(normalized_rollout.get("batch_cursor") or 0) batches_total = int(normalized_rollout.get("batches_total") or 0) summary_parts: list[str] = [] if target_nodes_total > 0: summary_parts.append(f"目标节点 {target_nodes_total}") if jobs_total > 0: summary_parts.append(f"任务 {jobs_created}/{jobs_total}") if batches_total > 0: summary_parts.append(f"批次 {batch_cursor}/{batches_total}") result_summary = dict(normalized_rollout.get("result_summary") or {}) summary_text = str(normalized_rollout.get("summary_text") or normalized_rollout.get("summary") or "").strip() if int(result_summary.get("failed", 0) or 0) > 0: summary_parts.append(f"失败 {int(result_summary.get('failed', 0) or 0)}") elif int(result_summary.get("success", 0) or 0) > 0: summary_parts.append(f"成功 {int(result_summary.get('success', 0) or 0)}") meta_parts = [f"创建人 {str(normalized_rollout.get('created_by') or '').strip() or '-'}"] if rollout_execution_mode_label: meta_parts.append(f"执行 {rollout_execution_mode_label}") return { "kind": "rollout", "activity_key": f"rollout:{rollout_id or rollout_code}", "title": rollout_code or "Release Rollout", "subtitle": f"Release #{int(normalized_rollout.get('release_id') or 0)}" if int(normalized_rollout.get("release_id") or 0) > 0 else "", "summary": _truncate_activity_text(summary_text or " / ".join(part for part in summary_parts if part) or "该 rollout 最近有新的推进或回执。"), "meta_text": " / ".join(part for part in meta_parts if part), "status": str(normalized_rollout.get("status") or "").strip(), "status_label": str(normalized_rollout.get("status_label") or "").strip(), "execution_mode": rollout_execution_mode, "execution_mode_label": rollout_execution_mode_label, "occurred_at": _activity_time(normalized_rollout.get("updated_at"), normalized_rollout.get("created_at")), "rollout_id": rollout_id, "target_node_codes": rollout_target_node_codes, "ui_intent": _build_driver_ui_intent( "rollout_jobs", rollout_id=rollout_id, rollout_code=rollout_code, release_id=int(normalized_rollout.get("release_id") or 0), ), "focus_ref": dict(normalized_rollout.get("focus_ref") or {}), } def _runbook_sequence_activity_status(sequence: dict) -> str: normalized_sequence = dict(sequence or {}) primary_resolution = dict(normalized_sequence.get("primary_resolution") or {}) if primary_resolution and primary_resolution.get("ok") is False: return "attention" normalized_status = str(normalized_sequence.get("status") or "").strip() if normalized_status in {"warning", "attention", "blocking"}: return "attention" if normalized_status in {"ready", "success"}: return "ready" if normalized_status in {"info", "idle"}: return "planned" return normalized_status or "planned" def _build_runbook_sequence_activity(sequence: dict) -> dict: normalized_sequence = dict(sequence or {}) sequence_key = str(normalized_sequence.get("key") or "").strip() if not sequence_key: return {} title = str(normalized_sequence.get("title") or "").strip() or "标准作业路径" status_label = str(normalized_sequence.get("status_label") or normalized_sequence.get("status") or "").strip() reason = str(normalized_sequence.get("reason") or "").strip() target_scope_label = str(normalized_sequence.get("target_scope_label") or "").strip() target_node_codes = _normalize_driver_node_codes(normalized_sequence.get("target_node_codes") or []) primary_resolution = dict(normalized_sequence.get("primary_resolution") or {}) resolved_action_label = ( str(primary_resolution.get("driver_action_label") or "").strip() or str(primary_resolution.get("driver_action_code") or "").strip() ) resolved_action_code = str(primary_resolution.get("driver_action_code") or "").strip() resolved_node_codes = _normalize_driver_node_codes(primary_resolution.get("driver_node_codes") or []) resolution_message = str(primary_resolution.get("message") or "").strip() occurred_at = _activity_time(primary_resolution.get("resolved_at")) summary_parts: list[str] = [] if primary_resolution and primary_resolution.get("ok") is False: summary_parts.append(f"当前解析失败:{resolution_message or '标准作业路径解析失败'}") elif resolved_action_label or resolved_action_code: summary_parts.append( f"当前建议动作:{resolved_action_label or resolved_action_code}" + ( f" ({resolved_action_code})" if resolved_action_code and resolved_action_label and resolved_action_code != resolved_action_label else "" ) ) if resolved_node_codes: summary_parts.append(f"目标节点:{_format_node_code_list(resolved_node_codes)}") elif target_node_codes: summary_parts.append(f"目标节点:{_format_node_code_list(target_node_codes)}") if resolution_message: summary_parts.append(resolution_message) else: summary_parts.append(str(normalized_sequence.get("summary") or "").strip() or "当前标准作业路径已就绪。") meta_parts = [status_label, sequence_key] if target_scope_label: meta_parts.append(target_scope_label) if reason: meta_parts.append(reason) return { "kind": "runbook_sequence", "activity_key": f"runbook-sequence:{sequence_key}", "title": title, "subtitle": sequence_key, "summary": _truncate_activity_text(" / ".join(part for part in summary_parts if part)), "meta_text": _truncate_activity_text(" / ".join(part for part in meta_parts if part), limit=280), "status": _runbook_sequence_activity_status(normalized_sequence), "execution_mode": "", "execution_mode_label": "", "occurred_at": occurred_at, "sequence_key": sequence_key, "target_node_codes": target_node_codes, "focus_ref": _normalize_focus_ref(normalized_sequence.get("focus_ref")), "ui_intent": _build_driver_ui_intent( "focus_runbook_sequence", sequence_key=sequence_key, driver_action_code=resolved_action_code or str(normalized_sequence.get("primary_action_code") or "").strip(), ), } def _matches_activity_query(item: dict, query: str) -> bool: normalized_query = str(query or "").strip().lower() if not normalized_query: return True haystack = " ".join( [ str(item.get("activity_key") or ""), str(item.get("kind") or ""), str(item.get("title") or ""), str(item.get("subtitle") or ""), str(item.get("summary") or ""), str(item.get("meta_text") or ""), str(item.get("status") or ""), str(item.get("execution_mode") or ""), str(item.get("execution_mode_label") or ""), str(item.get("run_code") or ""), str(item.get("job_code") or ""), str(item.get("rollout_id") or ""), str(item.get("sequence_key") or ""), ] ).lower() return normalized_query in haystack def _driver_activity_status_label(status: str) -> str: normalized_status = str(status or "").strip() mapping = { "failed": "失败", "blocked": "阻断", "cancelled": "已取消", "halted": "已暂停", "completed_with_issues": "带问题完成", "partially_succeeded": "部分成功", "awaiting_approval": "待审批", } return mapping.get(normalized_status, normalized_status or "异常") def _pick_driver_activity_focus(items: list[dict]) -> tuple[str, dict]: hard_problem_statuses = {"failed", "blocked", "cancelled", "halted", "completed_with_issues", "partially_succeeded"} soft_attention_statuses = {"awaiting_approval"} candidate_items = [ dict(item or {}) for item in list(items or []) if str(item.get("kind") or "").strip() in {"ops_job", "rollout"} ] for item in candidate_items: if str(item.get("status") or "").strip() in hard_problem_statuses: return "hard", item for item in candidate_items: if str(item.get("status") or "").strip() in soft_attention_statuses: return "soft", item return "", {} def _build_driver_activity_card(level: str, activity_item: dict) -> dict: normalized_item = dict(activity_item or {}) item_kind = str(normalized_item.get("kind") or "").strip() item_status = str(normalized_item.get("status") or "").strip() ui_intent_kind = str(((normalized_item.get("ui_intent") or {}).get("kind") or "")).strip() source_focus_ref = _normalize_focus_ref(normalized_item.get("source_focus_ref")) title = str(normalized_item.get("title") or "").strip() or ("Release Rollout" if item_kind == "rollout" else "标准运维任务") subtitle = str(normalized_item.get("subtitle") or "").strip() summary = str(normalized_item.get("summary") or "").strip() meta_text = str(normalized_item.get("meta_text") or "").strip() status_label = _driver_activity_status_label(item_status) noun_label = "异常 Rollout" if item_kind == "rollout" else "异常任务" is_hard_problem = level == "hard" if item_kind == "rollout": title_text = "先处理异常 Rollout" if is_hard_problem else "先处理待审批 Rollout" primary_label = "查看 Rollout 作业" else: title_text = "先处理异常任务" if is_hard_problem else "先处理待审批任务" primary_label = "查看任务详情" if ui_intent_kind == "job_detail" else "查看任务事件" reason = ( summary or ( "该活动已经进入失败、阻断或带问题完成状态,继续堆动作只会放大噪音。" if is_hard_problem else "该活动正在等待人工审批或决策,先落到对应详情最容易判断下一步。" ) ) detail_summary = f"{title}{' / ' + subtitle if subtitle else ''} 当前状态为 {status_label},建议先进入对应详情。" primary_action_code = "focus_activity_item" action_payload = { "activity_key": str(normalized_item.get("activity_key") or "").strip(), "kind": item_kind, "status": item_status, "ui_intent": dict(normalized_item.get("ui_intent") or {}), "focus_ref": _normalize_focus_ref(normalized_item.get("focus_ref")), } if ui_intent_kind == "job_events" and str(source_focus_ref.get("kind") or "").strip() == "ops_job_event": primary_action_code = "focus_latest_job_events" action_payload["source_focus_ref"] = source_focus_ref return { "key": f"activity-focus-{str(normalized_item.get('activity_key') or item_kind or 'activity').strip()}", "title": title_text, "level_label": "最高优先" if is_hard_problem else "推荐", "tag_type": "danger" if is_hard_problem else "warning", "summary": detail_summary, "reason": reason, "node_codes": [], "meta_text": meta_text or subtitle or str(normalized_item.get("activity_key") or "").strip(), "primary_label": primary_label, "secondary_label": "", "primary_type": "danger" if is_hard_problem else "warning", "disabled": not str(((normalized_item.get("ui_intent") or {}).get("kind") or "")).strip(), "primary_action_code": primary_action_code, "secondary_action_code": "", "action_payload": action_payload, "focus_ref": _normalize_focus_ref(normalized_item.get("focus_ref")), "related_activity": { "kind": item_kind, "title": title, "subtitle": subtitle, "status": item_status, "occurred_at": str(normalized_item.get("occurred_at") or "").strip(), }, } def _build_driver_activity_focus_entry(activity_item: dict) -> dict: normalized_item = dict(activity_item or {}) activity_key = str(normalized_item.get("activity_key") or "").strip() activity_kind = str(normalized_item.get("kind") or "").strip() ui_intent = dict(normalized_item.get("ui_intent") or {}) ui_intent_kind = str(ui_intent.get("kind") or "").strip() if not activity_key or not activity_kind: return {} focus_ref = _normalize_focus_ref(normalized_item.get("focus_ref")) source_focus_ref = _normalize_focus_ref(normalized_item.get("source_focus_ref")) focus_action_code = "focus_activity_item" if ui_intent_kind else "" if ui_intent_kind == "job_events" and str(source_focus_ref.get("kind") or "").strip() == "ops_job_event": focus_action_code = "focus_latest_job_events" focus_action_payload = { "activity_key": activity_key, "kind": activity_kind, "status": str(normalized_item.get("status") or "").strip(), "ui_intent": ui_intent, "focus_ref": focus_ref, } if source_focus_ref: focus_action_payload["source_focus_ref"] = source_focus_ref contract_keys = _normalize_ops_contract_keys( list(normalized_item.get("contract_keys") or []) + _suggest_ops_contract_keys_for_activity_item(normalized_item) ) return { "key": f"activity:{activity_key}", "activity_key": activity_key, "activity_kind": activity_kind, "title": str(normalized_item.get("title") or "").strip() or activity_kind, "subtitle": str(normalized_item.get("subtitle") or "").strip(), "summary": str(normalized_item.get("summary_text") or normalized_item.get("summary") or "").strip(), "status": str(normalized_item.get("status") or "").strip(), "status_label": str(normalized_item.get("status_label") or "").strip(), "meta_text": str(normalized_item.get("meta_text") or "").strip(), "occurred_at": str(normalized_item.get("occurred_at") or "").strip(), "target_node_codes": _normalize_driver_node_codes(normalized_item.get("target_node_codes") or []), "focus_ref": focus_ref, "source_focus_ref": source_focus_ref, "ui_intent": ui_intent, "ui_intent_kind": ui_intent_kind, "focus_action_code": focus_action_code, "focus_action_payload": focus_action_payload if ui_intent_kind else {}, "observation_only": True, "contract_keys": contract_keys, "contract_navigation": _build_ops_contract_navigation( contract_keys, primary_contract_key=_primary_ops_contract_key_for_activity_item(normalized_item, contract_keys), registry=get_ops_contract_registry(), ), } def _build_driver_activity_focus(items: list[dict], *, limit: int = 3) -> list[dict]: allowed_scene_kinds = {"execution_scene", "log_sync"} allowed_run_kinds = {"playbook_run", "ops_job", "rollout"} rows: list[dict] = [] seen_keys: set[str] = set() for raw_item in list(items or []): item = dict(raw_item or {}) item_kind = str(item.get("kind") or "").strip() item_status = str(item.get("status") or "").strip() if item_kind in allowed_scene_kinds: pass elif item_kind == "playbook_run": if item_status in {"success", "completed", "healthy"}: continue elif item_kind in allowed_run_kinds: if item_status in {"success", "completed", "ready", "planned"}: continue else: continue entry = _build_driver_activity_focus_entry(item) entry_key = str(entry.get("key") or "").strip() if not entry or not entry_key or entry_key in seen_keys: continue seen_keys.add(entry_key) rows.append(entry) if len(rows) >= limit: break return rows def _build_execution_scene_log_sync(log_sync: dict, participating_nodes: list[dict]) -> dict: normalized_log_sync = dict(log_sync or {}) enabled = bool(normalized_log_sync.get("enabled", False)) mode = "full" if str(normalized_log_sync.get("mode") or "").strip().lower() == "full" else "key" line_count = int(normalized_log_sync.get("line_count", 0) or 0) source_nodes = [ str(item or "").strip() for item in list(normalized_log_sync.get("source_nodes") or []) if str(item or "").strip() ] source_nodes = list(dict.fromkeys(source_nodes)) preview_lines = [ str(item or "").strip() for item in list(normalized_log_sync.get("preview_lines") or []) if str(item or "").strip() ] preview_lines = preview_lines[-20:] source_node_summaries_map: dict[str, dict] = {} for item in list(normalized_log_sync.get("source_node_summaries") or []): if not isinstance(item, dict): continue node_code = str(item.get("node_code") or "").strip() if not node_code: continue source_node_summaries_map[node_code] = { "node_code": node_code, "line_count": int(item.get("line_count", 0) or 0), "key_line_count": int(item.get("key_line_count", 0) or 0), "full_line_count": int(item.get("full_line_count", 0) or 0), "last_at": str(item.get("last_at") or "").strip(), "last_line": str(item.get("last_line") or "").strip(), } if not source_node_summaries_map and preview_lines: for line in preview_lines: match = _REMOTE_LOG_PREVIEW_LINE_RE.match(line) if not match: continue node_code = str(match.group("node_code") or "").strip() created_at = str(match.group("created_at") or "").strip() if not node_code: continue bucket = source_node_summaries_map.setdefault( node_code, { "node_code": node_code, "line_count": 0, "key_line_count": 0, "full_line_count": 0, "last_at": "", "last_line": "", }, ) bucket["line_count"] += 1 bucket["key_line_count"] += 1 bucket["last_at"] = created_at bucket["last_line"] = line for node_code in source_nodes: source_node_summaries_map.setdefault( node_code, { "node_code": node_code, "line_count": 0, "key_line_count": 0, "full_line_count": 0, "last_at": "", "last_line": "", }, ) source_nodes = list(dict.fromkeys([*source_nodes, *list(source_node_summaries_map.keys())])) source_node_summaries = sorted( source_node_summaries_map.values(), key=lambda item: ( str(item.get("last_at") or ""), str(item.get("node_code") or ""), ), reverse=True, ) source_node_set = set(source_nodes) source_node_set.update(source_node_summaries_map.keys()) participating_node_codes = [ str(item.get("node_code") or "").strip() for item in list(participating_nodes or []) if str(item.get("node_code") or "").strip() ] participating_node_codes = list(dict.fromkeys(participating_node_codes)) participating_node_map = { str(item.get("node_code") or "").strip(): dict(item) for item in list(participating_nodes or []) if str(item.get("node_code") or "").strip() } covered_participating_nodes = [node_code for node_code in participating_node_codes if node_code in source_node_set] missing_participating_nodes = [node_code for node_code in participating_node_codes if node_code not in source_node_set] covered_participating_node_summaries = [] for node_code in covered_participating_nodes: source_summary = dict(source_node_summaries_map.get(node_code) or {}) node_meta = participating_node_map.get(node_code) or {} covered_participating_node_summaries.append( { **source_summary, "region": str(node_meta.get("region") or "").strip(), "role": str(node_meta.get("role") or "").strip(), "status": str(node_meta.get("status") or "").strip(), "participation_state": str(node_meta.get("participation_state") or "").strip(), "participation_bucket": str(node_meta.get("participation_bucket") or "").strip(), "participation_label": str(node_meta.get("participation_label") or "").strip(), "participation_reason": str(node_meta.get("participation_reason") or "").strip(), } ) missing_participating_node_summaries = [] for node_code in missing_participating_nodes: node_meta = participating_node_map.get(node_code) or {} missing_reason_code = "no_sample" missing_reason = "当前还没有收到该参与节点的远端日志样本。" if not enabled: missing_reason_code = "disabled" missing_reason = "远端日志回传当前关闭。" elif line_count <= 0: missing_reason_code = "waiting_sample" missing_reason = "日志回传已开启,但当前现场样本尚未形成。" missing_participating_node_summaries.append( { "node_code": node_code, "region": str(node_meta.get("region") or "").strip(), "role": str(node_meta.get("role") or "").strip(), "status": str(node_meta.get("status") or "").strip(), "participation_state": str(node_meta.get("participation_state") or "").strip(), "participation_bucket": str(node_meta.get("participation_bucket") or "").strip(), "participation_label": str(node_meta.get("participation_label") or "").strip(), "participation_reason": str(node_meta.get("participation_reason") or "").strip(), "missing_reason_code": missing_reason_code, "missing_reason": missing_reason, } ) if not enabled: state = "disabled" status_label = "已关闭" status_type = "info" description = "远端日志回传当前关闭,海外控制面不会持续收到大陆检测过程日志。" recommended_action_label = "建议先开启关键回传" elif not participating_node_codes: state = "idle" status_label = "空闲观察" status_type = "success" description = "当前没有参与检测的节点,日志回传已就绪,待现场出现时会自动开始镜像样本。" recommended_action_label = "当前无需调整" elif line_count <= 0: state = "waiting_sample" status_label = "等待样本" status_type = "warning" description = "日志回传已开启,但当前还没有收到参与节点的现场日志样本。" recommended_action_label = "建议先抓 Worker 日志确认现场输出" elif missing_participating_nodes: state = "partial_coverage" status_label = "部分覆盖" status_type = "warning" description = f"当前已有 {len(covered_participating_nodes)}/{len(participating_node_codes)} 台参与节点回传日志,仍有节点未被覆盖。" recommended_action_label = "建议优先排查未回传节点" elif mode == "full": state = "full_capture" status_label = "全量观察" status_type = "success" description = "参与节点已经全部被覆盖,当前处于全量日志观察模式,适合短时深度排障。" recommended_action_label = "排障结束后可切回关键回传" else: state = "healthy" status_label = "关键覆盖" status_type = "success" description = "参与节点已经全部被覆盖,当前处于关键日志观察模式,适合常态联调。" recommended_action_label = "当前无需调整" return { "enabled": enabled, "mode": mode, "mode_label": "全量回传" if enabled and mode == "full" else ("关键回传" if enabled else "已关闭"), "state": state, "status_label": status_label, "status_type": status_type, "description": description, "recommended_action_label": recommended_action_label, "line_count": line_count, "source_node_count": int(normalized_log_sync.get("source_node_count", 0) or len(source_nodes)), "source_nodes": source_nodes, "source_node_summaries": source_node_summaries, "last_at": str(normalized_log_sync.get("last_at") or ""), "last_line": str(normalized_log_sync.get("last_line") or ""), "preview_lines": preview_lines, "preview_line_count": len(preview_lines), "participating_node_count": len(participating_node_codes), "covered_participating_node_count": len(covered_participating_nodes), "missing_participating_node_count": len(missing_participating_nodes), "covered_participating_nodes": covered_participating_nodes, "missing_participating_nodes": missing_participating_nodes, "covered_participating_node_summaries": covered_participating_node_summaries, "missing_participating_node_summaries": missing_participating_node_summaries, } def _normalize_scene_log_mode(raw_mode: object, *, fallback: str = "key") -> str: normalized_mode = str(raw_mode or "").strip().lower() if normalized_mode == "full": return "full" if normalized_mode == "key": return "key" return "full" if str(fallback or "").strip().lower() == "full" else "key" def _derive_scene_log_source_summary_from_records(node_code: str, records: list[dict]) -> dict: normalized_node_code = str(node_code or "").strip() summary = { "node_code": normalized_node_code, "line_count": 0, "key_line_count": 0, "full_line_count": 0, "last_at": "", "last_line": "", } for record in list(records or []): record_node_code = str(record.get("node_code") or "").strip() if normalized_node_code and record_node_code != normalized_node_code: continue summary["line_count"] += 1 if str(record.get("mode") or "").strip() == "full": summary["full_line_count"] += 1 else: summary["key_line_count"] += 1 summary["last_at"] = str(record.get("created_at") or "").strip() summary["last_line"] = str(record.get("line") or "").strip() return summary def _collect_scene_log_records( active_job: dict | None, *, requested_mode: str, ) -> list[dict]: normalized_active_job = dict(active_job or {}) events = list(normalized_active_job.get("current_cycle_events") or normalized_active_job.get("recent_events") or []) if not events: return [] current_cycle_token = str(normalized_active_job.get("current_cycle_token") or "").strip() normalized_requested_mode = _normalize_scene_log_mode(requested_mode) records: list[dict] = [] for event in reversed(events): event_type = str(event.get("event_type") or "").strip() if event_type != "worker_log": continue payload = event.get("payload") if isinstance(event.get("payload"), dict) else {} event_cycle_token = str(payload.get("cycle_token") or "").strip() if current_cycle_token and event_cycle_token and event_cycle_token != current_cycle_token: continue event_mode = _normalize_scene_log_mode(payload.get("log_mode") or "key") if normalized_requested_mode != "full" and event_mode == "full": continue created_at = str(event.get("created_at") or "").strip() node_code = str(event.get("node_code") or "").strip() or "unknown" message = str(event.get("message") or "").strip() if not message: continue records.append( { "created_at": created_at, "node_code": node_code, "message": message, "mode": event_mode, "cycle_token": event_cycle_token, "line": f"[{created_at}] [{node_code}] {message}", } ) return records def _parse_ops_timestamp(value: object) -> datetime | None: text = str(value or "").strip() if not text: return None try: return datetime.fromisoformat(text) except ValueError: return None def _collect_scene_log_records_from_ops_jobs( *, requested_mode: str, limit: int = 30, max_age_hours: int = 6, ) -> list[dict]: normalized_requested_mode = _normalize_scene_log_mode(requested_mode) cutoff = datetime.now() - timedelta(hours=max(1, int(max_age_hours or 6))) records: list[dict] = [] worker_service_name = str(settings.worker_service_name or "domaincheck-worker").strip() or "domaincheck-worker" jobs = list_ops_jobs(limit=max(1, min(int(limit or 30), 80)), compact=False) for job in reversed(list(jobs or [])): if str(job.get("action") or "").strip() != "logs.collect": continue if str(job.get("status") or "").strip() != "success": continue result = dict(job.get("result") or {}) payload = dict(job.get("payload") or {}) service_name = str(result.get("service_name") or payload.get("service_name") or "").strip() if service_name != worker_service_name: continue node_code = str(job.get("target_node_code") or "").strip() if not node_code: continue created_at = ( str(job.get("finished_at") or "").strip() or str(job.get("updated_at") or "").strip() or str(job.get("created_at") or "").strip() ) created_dt = _parse_ops_timestamp(created_at) if created_dt and created_dt < cutoff: continue output_text = str(result.get("stdout") or result.get("log_output") or "").strip() if not output_text: continue output_lines = [str(item or "").strip() for item in output_text.splitlines() if str(item or "").strip()] if not output_lines: continue sample_limit = 12 if normalized_requested_mode == "full" else 4 for line in output_lines[-sample_limit:]: records.append( { "created_at": created_at, "node_code": node_code, "message": line, "mode": normalized_requested_mode, "cycle_token": "", "line": f"[{created_at}] [{node_code}] {line}", } ) return records def _merge_execution_scene_log_sync_sources(log_sync: dict | None, *, requested_mode: str) -> dict: normalized_log_sync = dict(log_sync or {}) ops_job_records = _collect_scene_log_records_from_ops_jobs( requested_mode=requested_mode, limit=30, max_age_hours=6, ) if not ops_job_records: return normalized_log_sync preview_lines = [ str(item or "").strip() for item in list(normalized_log_sync.get("preview_lines") or []) if str(item or "").strip() ] merged_preview_lines = preview_lines + [str(item.get("line") or "").strip() for item in ops_job_records if str(item.get("line") or "").strip()] merged_preview_lines = merged_preview_lines[-20:] source_nodes = [ str(item or "").strip() for item in list(normalized_log_sync.get("source_nodes") or []) if str(item or "").strip() ] source_nodes.extend( str(item.get("node_code") or "").strip() for item in ops_job_records if str(item.get("node_code") or "").strip() ) source_nodes = list(dict.fromkeys(source_nodes)) existing_summaries = {} for item in list(normalized_log_sync.get("source_node_summaries") or []): if not isinstance(item, dict): continue node_code = str(item.get("node_code") or "").strip() if not node_code: continue existing_summaries[node_code] = dict(item) for node_code in source_nodes: derived_summary = _derive_scene_log_source_summary_from_records( node_code, [record for record in ops_job_records if str(record.get("node_code") or "").strip() == node_code], ) if existing_summaries.get(node_code): current_summary = dict(existing_summaries[node_code]) current_summary["line_count"] = int(current_summary.get("line_count", 0) or 0) + int(derived_summary.get("line_count", 0) or 0) current_summary["key_line_count"] = int(current_summary.get("key_line_count", 0) or 0) + int(derived_summary.get("key_line_count", 0) or 0) current_summary["full_line_count"] = int(current_summary.get("full_line_count", 0) or 0) + int(derived_summary.get("full_line_count", 0) or 0) if str(derived_summary.get("last_at") or "").strip() >= str(current_summary.get("last_at") or "").strip(): current_summary["last_at"] = str(derived_summary.get("last_at") or current_summary.get("last_at") or "").strip() current_summary["last_line"] = str(derived_summary.get("last_line") or current_summary.get("last_line") or "").strip() existing_summaries[node_code] = current_summary elif int(derived_summary.get("line_count", 0) or 0) > 0: existing_summaries[node_code] = derived_summary latest_record = ops_job_records[-1] latest_at = str(latest_record.get("created_at") or "").strip() latest_line = str(latest_record.get("line") or "").strip() current_last_at = str(normalized_log_sync.get("last_at") or "").strip() if current_last_at and current_last_at > latest_at: latest_at = current_last_at latest_line = str(normalized_log_sync.get("last_line") or "").strip() merged_line_count = max( int(normalized_log_sync.get("line_count", 0) or 0), len(merged_preview_lines), sum(int(item.get("line_count", 0) or 0) for item in existing_summaries.values()), ) return { **normalized_log_sync, "preview_lines": merged_preview_lines, "line_count": merged_line_count, "source_nodes": source_nodes, "source_node_count": max(int(normalized_log_sync.get("source_node_count", 0) or 0), len(source_nodes)), "source_node_summaries": list(existing_summaries.values()), "last_at": latest_at, "last_line": latest_line, } def _collect_scene_log_records_from_preview_lines( preview_lines: list[str] | None, *, requested_mode: str, ) -> list[dict]: normalized_requested_mode = _normalize_scene_log_mode(requested_mode) records: list[dict] = [] for raw_line in list(preview_lines or []): line = str(raw_line or "").strip() if not line: continue match = _REMOTE_LOG_PREVIEW_LINE_RE.match(line) if not match: continue created_at = str(match.group("created_at") or "").strip() node_code = str(match.group("node_code") or "").strip() or "unknown" message = str(match.group("message") or "").strip() if not message: continue record_mode = "full" if normalized_requested_mode == "full" else "key" records.append( { "created_at": created_at, "node_code": node_code, "message": message, "mode": record_mode, "cycle_token": "", "line": line, } ) return records def _scene_log_status_meta(status: str, *, log_sync_enabled: bool) -> tuple[str, str]: normalized_status = str(status or "").strip() mapping = { "healthy": ("关键覆盖", "success"), "full_capture": ("全量观察", "success"), "historical_sample": ("历史样本", "success"), "waiting_sample": ("等待样本", "warning"), "missing_sample": ("缺少样本", "warning"), "disabled": ("已关闭", "info"), "standby": ("在线待命", "info"), "unknown": ("未发现", "info"), } if normalized_status in mapping: return mapping[normalized_status] if not log_sync_enabled: return ("已关闭", "info") return ("待确认", "info") def get_ops_node_scene_log(node_code: str, *, limit: int = 80, mode: str = "") -> dict: normalized_node_code = str(node_code or "").strip() if not normalized_node_code: return {} runtime = get_runtime_status() detect = dict(runtime.get("detect") or {}) cluster_nodes = [ dict(item) for item in list((runtime.get("cluster") or {}).get("nodes") or []) if str(item.get("node_code") or "").strip() ] execution_scene = _build_ops_execution_scene(detect) log_sync = dict(execution_scene.get("log_sync") or {}) default_mode = _normalize_scene_log_mode(log_sync.get("mode") or "key") selected_mode = _normalize_scene_log_mode(mode, fallback=default_mode) active_job = dict(detect.get("active_job") or {}) records = _collect_scene_log_records(active_job, requested_mode=selected_mode) if not records: records = _collect_scene_log_records_from_preview_lines( list(log_sync.get("preview_lines") or []), requested_mode=selected_mode, ) node_records = [ dict(record) for record in records if str(record.get("node_code") or "").strip() == normalized_node_code ] normalized_limit = max(1, min(int(limit or 80), 400)) visible_records = node_records[-normalized_limit:] participating_map = { str(item.get("node_code") or "").strip(): dict(item) for item in list(execution_scene.get("participating_nodes") or []) if str(item.get("node_code") or "").strip() } standby_map = { str(item.get("node_code") or "").strip(): dict(item) for item in list(execution_scene.get("standby_nodes") or []) if str(item.get("node_code") or "").strip() } cluster_map = { str(item.get("node_code") or "").strip(): dict(item) for item in cluster_nodes if str(item.get("node_code") or "").strip() } source_summary_map = { str(item.get("node_code") or "").strip(): dict(item) for item in list(log_sync.get("source_node_summaries") or []) if str(item.get("node_code") or "").strip() } missing_summary_map = { str(item.get("node_code") or "").strip(): dict(item) for item in list(log_sync.get("missing_participating_node_summaries") or []) if str(item.get("node_code") or "").strip() } covered_summary_map = { str(item.get("node_code") or "").strip(): dict(item) for item in list(log_sync.get("covered_participating_node_summaries") or []) if str(item.get("node_code") or "").strip() } node_meta = dict(participating_map.get(normalized_node_code) or standby_map.get(normalized_node_code) or {}) cluster_meta = dict(cluster_map.get(normalized_node_code) or {}) source_summary = dict(source_summary_map.get(normalized_node_code) or covered_summary_map.get(normalized_node_code) or {}) derived_source_summary = _derive_scene_log_source_summary_from_records(normalized_node_code, node_records) source_summary = { "node_code": normalized_node_code, "line_count": max( int(source_summary.get("line_count", 0) or 0), int(derived_source_summary.get("line_count", 0) or 0), ), "key_line_count": max( int(source_summary.get("key_line_count", 0) or 0), int(derived_source_summary.get("key_line_count", 0) or 0), ), "full_line_count": max( int(source_summary.get("full_line_count", 0) or 0), int(derived_source_summary.get("full_line_count", 0) or 0), ), "last_at": str( derived_source_summary.get("last_at") or source_summary.get("last_at") or "" ).strip(), "last_line": str( derived_source_summary.get("last_line") or source_summary.get("last_line") or "" ).strip(), } missing_summary = dict(missing_summary_map.get(normalized_node_code) or {}) record_available = bool(source_summary.get("line_count", 0) or node_records) detect_participating = normalized_node_code in participating_map standby_visible = normalized_node_code in standby_map log_sync_enabled = bool(log_sync.get("enabled", False)) if record_available: if detect_participating: status = "full_capture" if selected_mode == "full" else "healthy" elif standby_visible: status = "historical_sample" else: status = "historical_sample" elif missing_summary: missing_reason_code = str(missing_summary.get("missing_reason_code") or "").strip() if missing_reason_code == "disabled": status = "disabled" elif missing_reason_code == "waiting_sample": status = "waiting_sample" else: status = "missing_sample" elif detect_participating: status = "waiting_sample" if log_sync_enabled else "disabled" elif standby_visible: status = "standby" else: status = "unknown" status_label, status_type = _scene_log_status_meta(status, log_sync_enabled=log_sync_enabled) if record_available and detect_participating: summary = f"节点当前正在参与检测,已保留 {int(source_summary.get('line_count', 0) or 0)} 条现场日志样本。" elif record_available and standby_visible: summary = f"节点当前在线待命,仍可查看最近保留的 {int(source_summary.get('line_count', 0) or 0)} 条现场日志样本。" elif record_available: summary = f"节点已保留 {int(source_summary.get('line_count', 0) or 0)} 条现场日志样本。" elif missing_summary: summary = str(missing_summary.get("missing_reason") or "").strip() or "当前还没有收到该节点的现场日志样本。" elif detect_participating and log_sync_enabled: summary = "节点正在参与检测,但当前现场日志样本仍未形成。" elif detect_participating: summary = "节点正在参与检测,但远端日志回传当前关闭。" elif standby_visible: summary = "节点当前在线但未参与本轮检测,暂时没有新的现场日志样本。" else: summary = "当前没有找到该节点的现场日志记录。" latest_record = dict(visible_records[-1] or {}) if visible_records else {} participation_payload = { "detect_participating": detect_participating, "participation_state": str(node_meta.get("participation_state") or "").strip(), "participation_label": str(node_meta.get("participation_label") or "").strip(), "participation_reason": str(node_meta.get("participation_reason") or "").strip(), "participation_bucket": str(node_meta.get("participation_bucket") or "").strip(), "participation_bucket_label": str(node_meta.get("participation_bucket_label") or "").strip(), "is_dispatch_active": bool(node_meta.get("is_dispatch_active", False)), } return { "node_code": normalized_node_code, "available": bool(node_meta or cluster_meta or source_summary.get("line_count", 0) or missing_summary or node_records), "status": status, "status_label": status_label, "status_type": status_type, "summary": summary, "log_sync_enabled": log_sync_enabled, "mode": selected_mode, "mode_label": "全量回传" if selected_mode == "full" else "关键回传", "records_total": len(node_records), "records_visible": len(visible_records), "records_truncated": len(node_records) > len(visible_records), "records": visible_records, "latest_record": latest_record, "source_summary": source_summary, "missing_reason_code": str(missing_summary.get("missing_reason_code") or "").strip(), "missing_reason": str(missing_summary.get("missing_reason") or "").strip(), "node": { "node_code": normalized_node_code, "region": str(node_meta.get("region") or cluster_meta.get("region") or missing_summary.get("region") or "").strip(), "role": str(node_meta.get("role") or cluster_meta.get("role") or missing_summary.get("role") or "").strip(), "status": str(node_meta.get("status") or cluster_meta.get("status") or missing_summary.get("status") or "").strip(), "current_load": int(node_meta.get("current_load", cluster_meta.get("current_load", 0)) or 0), "last_heartbeat_at": str( node_meta.get("last_heartbeat_at") or cluster_meta.get("last_heartbeat_at") or "" ).strip(), }, "participation": participation_payload, "contract_navigation": _build_ops_contract_navigation( ["ops_observability_contract", "ops_stack_diagnosis_contract"], primary_contract_key="ops_observability_contract", registry=get_ops_contract_registry(), ), } def _build_ops_execution_scene(detect: dict) -> dict: normalized_detect = dict(detect or {}) active_job = dict(normalized_detect.get("active_job") or {}) log_sync = _merge_execution_scene_log_sync_sources( normalized_detect.get("log_sync") or {}, requested_mode=str((normalized_detect.get("log_sync") or {}).get("mode") or "key"), ) participation_summary = normalized_detect.get("participation_summary") or {} participating_nodes = [ _slim_execution_scene_node(item) for item in list(normalized_detect.get("participating_nodes") or []) if str(item.get("node_code") or "").strip() ] standby_nodes = [ _slim_execution_scene_node(item) for item in list(normalized_detect.get("non_participating_nodes") or []) if str(item.get("node_code") or "").strip() ] dispatch_active_nodes = [ dict(item) for item in participating_nodes if str(item.get("participation_bucket") or "").strip() == "dispatch_active" ] recent_only_nodes = [ dict(item) for item in participating_nodes if str(item.get("participation_bucket") or "").strip() == "recent_only" ] load_syncing_nodes = [ dict(item) for item in [*participating_nodes, *standby_nodes] if str(item.get("participation_bucket") or "").strip() == "load_syncing" ] counts = { "dispatch_active": int(participation_summary.get("dispatch_active_nodes", len(dispatch_active_nodes)) or len(dispatch_active_nodes)), "recent_only": int(participation_summary.get("recent_only_nodes", len(recent_only_nodes)) or len(recent_only_nodes)), "standby": int(participation_summary.get("standby_nodes", 0) or 0), "load_syncing": int(participation_summary.get("load_syncing_nodes", len(load_syncing_nodes)) or len(load_syncing_nodes)), } latest_event = dict(active_job.get("latest_cycle_event") or active_job.get("latest_event") or {}) latest_at_candidates = [ str(latest_event.get("created_at") or "").strip(), str(active_job.get("started_at") or "").strip(), str(active_job.get("created_at") or "").strip(), str((log_sync or {}).get("last_at") or "").strip(), ] latest_at_candidates.extend( str(item.get("last_heartbeat_at") or "").strip() for item in [*participating_nodes, *standby_nodes] if str(item.get("last_heartbeat_at") or "").strip() ) non_empty_latest_at_candidates = [candidate for candidate in latest_at_candidates if candidate] latest_at = sorted(non_empty_latest_at_candidates)[-1] if non_empty_latest_at_candidates else "" return { "active_job_code": str((normalized_detect.get("active_job") or {}).get("job_code") or ""), "phase_label": str(normalized_detect.get("phase_label") or ""), "phase_detail": str(normalized_detect.get("phase_detail") or ""), "recent_event": str(normalized_detect.get("recent_event") or ""), "recent_warning": str(normalized_detect.get("recent_warning") or ""), "latest_at": latest_at, "summary": str(participation_summary.get("summary") or ""), "counts": counts, "log_sync": _build_execution_scene_log_sync(log_sync, participating_nodes), "participation_summary": { "effective_online_nodes": int(participation_summary.get("effective_online_nodes", 0) or 0), "participating_nodes": int(participation_summary.get("participating_nodes", 0) or 0), "dispatch_active_nodes": int(participation_summary.get("dispatch_active_nodes", 0) or 0), "recent_only_nodes": int(participation_summary.get("recent_only_nodes", 0) or 0), "non_participating_nodes": int(participation_summary.get("non_participating_nodes", 0) or 0), "standby_nodes": int(participation_summary.get("standby_nodes", 0) or 0), "load_syncing_nodes": int(participation_summary.get("load_syncing_nodes", 0) or 0), "dedicated_worker_nodes": int(participation_summary.get("dedicated_worker_nodes", 0) or 0), "controller_worker_nodes": int(participation_summary.get("controller_worker_nodes", 0) or 0), "dispatch_active_node_codes": list(participation_summary.get("dispatch_active_node_codes") or []), "recent_only_node_codes": list(participation_summary.get("recent_only_node_codes") or []), "non_participating_node_codes": list(participation_summary.get("non_participating_node_codes") or []), "standby_node_codes": list(participation_summary.get("standby_node_codes") or []), "load_syncing_node_codes": list(participation_summary.get("load_syncing_node_codes") or []), "summary": str(participation_summary.get("summary") or ""), }, "participating_nodes": participating_nodes, "dispatch_active_nodes": dispatch_active_nodes, "recent_only_nodes": recent_only_nodes, "standby_nodes": standby_nodes, "load_syncing_nodes": load_syncing_nodes, } def _should_emit_execution_scene_activity(execution_scene: dict) -> bool: normalized_scene = dict(execution_scene or {}) summary = dict(normalized_scene.get("participation_summary") or {}) log_sync = dict(normalized_scene.get("log_sync") or {}) return any( [ str(normalized_scene.get("active_job_code") or "").strip(), str(normalized_scene.get("phase_label") or "").strip(), str(normalized_scene.get("phase_detail") or "").strip(), str(normalized_scene.get("recent_event") or "").strip(), str(normalized_scene.get("recent_warning") or "").strip(), str(normalized_scene.get("latest_at") or "").strip(), int(summary.get("effective_online_nodes", 0) or 0) > 0, bool(normalized_scene.get("participating_nodes")), bool(normalized_scene.get("standby_nodes")), bool(log_sync.get("enabled", False)), int(log_sync.get("line_count", 0) or 0) > 0, int(log_sync.get("source_node_count", 0) or 0) > 0, ] ) def _should_emit_log_sync_activity(execution_scene: dict) -> bool: log_sync = dict((execution_scene or {}).get("log_sync") or {}) return any( [ bool(log_sync.get("enabled", False)), int(log_sync.get("participating_node_count", 0) or 0) > 0, int(log_sync.get("line_count", 0) or 0) > 0, int(log_sync.get("source_node_count", 0) or 0) > 0, int(log_sync.get("missing_participating_node_count", 0) or 0) > 0, bool(log_sync.get("preview_lines")), str(log_sync.get("last_at") or "").strip(), ] ) def _build_execution_scene_activity(execution_scene: dict) -> dict: normalized_scene = dict(execution_scene or {}) summary = dict(normalized_scene.get("participation_summary") or {}) participating_nodes = list(normalized_scene.get("participating_nodes") or []) standby_nodes = list(normalized_scene.get("standby_nodes") or []) dispatch_active_nodes = [ item for item in participating_nodes if bool(item.get("is_dispatch_active", False)) ] recent_only_nodes = [ item for item in participating_nodes if str(item.get("participation_state") or "").strip() == "recent_throughput" ] load_syncing_nodes = [ item for item in standby_nodes if str(item.get("participation_state") or "").strip() == "load_syncing" ] pure_standby_nodes = [ item for item in standby_nodes if str(item.get("participation_state") or "").strip() == "standby" ] phase_label = str(normalized_scene.get("phase_label") or "").strip() summary_text = ( str(summary.get("summary") or "").strip() or "当前还没有形成统一的执行现场摘要。" ) status = "ready" if dispatch_active_nodes or recent_only_nodes: status = "running" elif load_syncing_nodes: status = "attention" meta_parts = [ f"执行/已领 {len(dispatch_active_nodes)}", f"近窗 {len(recent_only_nodes)}", f"待命 {len(pure_standby_nodes)}", f"待确认 {len(load_syncing_nodes)}", ] if phase_label: meta_parts.append(f"阶段 {phase_label}") scene_node_codes = _normalize_driver_node_codes( [str(item.get("node_code") or "").strip() for item in [*participating_nodes, *standby_nodes]] ) return { "kind": "execution_scene", "activity_key": "execution-scene:live", "title": "当前执行现场", "subtitle": phase_label, "summary": _truncate_activity_text(summary_text), "meta_text": " / ".join(part for part in meta_parts if part), "status": status, "execution_mode": "", "execution_mode_label": "", "occurred_at": _activity_time( normalized_scene.get("latest_at"), *[item.get("last_heartbeat_at") for item in participating_nodes], *[item.get("last_heartbeat_at") for item in standby_nodes], ), "target_node_codes": scene_node_codes, "ui_intent": _build_driver_ui_intent( "focus_execution_scene", open_log_drawer=False, ), } def _build_log_sync_activity(execution_scene: dict) -> dict: normalized_scene = dict(execution_scene or {}) log_sync = dict(normalized_scene.get("log_sync") or {}) enabled = bool(log_sync.get("enabled", False)) participating_node_count = int(log_sync.get("participating_node_count", 0) or 0) covered_node_count = int(log_sync.get("covered_participating_node_count", 0) or 0) missing_node_count = int(log_sync.get("missing_participating_node_count", 0) or 0) line_count = int(log_sync.get("line_count", 0) or 0) source_node_count = int(log_sync.get("source_node_count", 0) or 0) missing_nodes = [ str(item or "").strip() for item in list(log_sync.get("missing_participating_nodes") or []) if str(item or "").strip() ] status = "ready" if participating_node_count > 0 and not enabled: status = "attention" elif participating_node_count > 0 and (line_count <= 0 or missing_node_count > 0): status = "attention" elif participating_node_count > 0: status = "running" description = str(log_sync.get("description") or "").strip() or "远端日志回传当前暂无补充说明。" if missing_nodes: description = f"{description} 当前仍有未覆盖节点:{'、'.join(missing_nodes)}" observed_node_codes = _normalize_driver_node_codes( [*list(log_sync.get("source_nodes") or []), *missing_nodes] ) return { "kind": "log_sync", "activity_key": "execution-scene:log-sync", "title": "远端日志回传", "subtitle": str(log_sync.get("mode_label") or "").strip(), "summary": _truncate_activity_text(description), "meta_text": " / ".join( part for part in [ f"参与覆盖 {covered_node_count}/{participating_node_count}", f"样本 {line_count}", f"来源 {source_node_count}", f"最近 {str(log_sync.get('last_at') or '').strip()}" if str(log_sync.get("last_at") or "").strip() else "", ] if part ), "status": status, "execution_mode": "control-plane", "execution_mode_label": execution_mode_label("control-plane"), "occurred_at": _activity_time( log_sync.get("last_at"), normalized_scene.get("latest_at"), ), "target_node_codes": observed_node_codes, "ui_intent": _build_driver_ui_intent( "focus_execution_scene", open_log_drawer=bool(list(log_sync.get("preview_lines") or [])), ), } def _default_rollout_target_nodes(cluster_nodes: list[dict]) -> list[dict]: rows: list[dict] = [] for node in list(cluster_nodes or []): node_code = str(node.get("node_code") or "").strip() cluster_status = str(node.get("status") or "").strip() if not node_code: continue if not bool(node.get("is_effective_worker", False)): continue if cluster_status not in {"online", "busy"}: continue rows.append( { "node_code": node_code, "region": str(node.get("region") or ""), "role": str(node.get("role") or ""), "status": cluster_status, "current_load": int(node.get("current_load", 0) or 0), "is_effective_worker": True, } ) return rows def get_ops_activity_stream( *, limit: int = _OPS_ACTIVITY_FETCH_LIMIT, scan_limit: int | None = None, kind: str = "", status: str = "", execution_mode: str = "", query: str = "", ) -> dict: safe_limit = min(max(int(limit or _OPS_ACTIVITY_FETCH_LIMIT), 1), 100) safe_scan_limit = min(max(int(scan_limit or (safe_limit * 4)), safe_limit), 400) normalized_kind = str(kind or "").strip() normalized_status = str(status or "").strip() normalized_execution_mode = str(execution_mode or "").strip() normalized_query = str(query or "").strip() runtime_status = get_runtime_status() execution_scene = _build_ops_execution_scene(runtime_status.get("detect") or {}) managed_nodes_payload = list_managed_nodes_with_agent_state() managed_node_map = { str(item.get("node_code") or "").strip(): dict(item or {}) for item in list(managed_nodes_payload.get("nodes") or []) if str(item.get("node_code") or "").strip() } playbook_runs_payload = get_recent_ops_playbook_runs(limit=safe_scan_limit, scan_limit=max(240, safe_scan_limit * 12)) playbook_runs = list(playbook_runs_payload.get("runs") or []) playbook_items = [_build_playbook_run_activity(run) for run in playbook_runs if str(run.get("run_code") or "").strip()] jobs = list_ops_jobs(limit=safe_scan_limit, compact=False) standalone_jobs = [ job for job in jobs if not _is_playbook_child_job(job) and int(job.get("rollout_id") or 0) <= 0 ] job_items = [ _build_ops_job_activity( job, managed_node=managed_node_map.get(str(job.get("target_node_code") or "").strip(), {}), ) for job in standalone_jobs if int(job.get("id") or 0) > 0 ] rollouts = list_release_rollouts(limit=safe_scan_limit) rollout_items = [_build_rollout_activity(rollout) for rollout in rollouts if int(rollout.get("id") or 0) > 0] runbook = get_ops_runbook() runbook_items = [ _build_runbook_sequence_activity(sequence) for sequence in list(runbook.get("control_sequences") or []) if str((sequence or {}).get("key") or "").strip() ] runbook_items = [item for item in runbook_items if item] scene_items: list[dict] = [] if _should_emit_execution_scene_activity(execution_scene): scene_items.append(_build_execution_scene_activity(execution_scene)) if _should_emit_log_sync_activity(execution_scene): scene_items.append(_build_log_sync_activity(execution_scene)) all_items = [*scene_items, *playbook_items, *job_items, *rollout_items, *runbook_items] all_items = [_finalize_activity_item(item) for item in all_items if item] all_items.sort(key=lambda item: (str(item.get("occurred_at") or ""), str(item.get("activity_key") or "")), reverse=True) all_items.sort( key=lambda item: 1 if str(item.get("kind") or "").strip() == "runbook_sequence" else 0 ) available_kind_counts: dict[str, int] = {} available_status_counts: dict[str, int] = {} available_execution_mode_counts: dict[str, int] = {} for item in all_items: item_kind = str(item.get("kind") or "").strip() or "unknown" item_status = str(item.get("status") or "").strip() or "unknown" item_execution_mode = str(item.get("execution_mode") or "").strip() available_kind_counts[item_kind] = int(available_kind_counts.get(item_kind, 0) or 0) + 1 available_status_counts[item_status] = int(available_status_counts.get(item_status, 0) or 0) + 1 if item_execution_mode: available_execution_mode_counts[item_execution_mode] = ( int(available_execution_mode_counts.get(item_execution_mode, 0) or 0) + 1 ) filtered_items = [ item for item in all_items if (not normalized_kind or str(item.get("kind") or "").strip() == normalized_kind) and (not normalized_status or str(item.get("status") or "").strip() == normalized_status) and (not normalized_execution_mode or str(item.get("execution_mode") or "").strip() == normalized_execution_mode) and _matches_activity_query(item, normalized_query) ] items = filtered_items[:safe_limit] summary_status, summary_status_label, summary_text = _activity_stream_summary_state(filtered_items) kind_counts: dict[str, int] = {} status_counts: dict[str, int] = {} for item in items: kind = str(item.get("kind") or "").strip() or "unknown" status = str(item.get("status") or "").strip() or "unknown" kind_counts[kind] = int(kind_counts.get(kind, 0) or 0) + 1 status_counts[status] = int(status_counts.get(status, 0) or 0) + 1 summary_contract_keys = _normalize_ops_contract_keys( ["ops_observability_contract", "ops_stack_diagnosis_contract"] + [ contract_key for item in filtered_items for contract_key in list((item or {}).get("contract_keys") or []) ] ) return { "items": items, "contract_navigation": _build_ops_contract_navigation( summary_contract_keys, primary_contract_key="ops_observability_contract", registry=get_ops_contract_registry(), ), "summary": { "status": summary_status, "status_label": summary_status_label, "summary_text": summary_text, "total": len(items), "filtered_total": len(filtered_items), "unfiltered_total": len(all_items), "kind_counts": kind_counts, "status_counts": status_counts, "available_kind_counts": available_kind_counts, "available_status_counts": available_status_counts, "available_execution_mode_counts": available_execution_mode_counts, "latest_at": str(items[0].get("occurred_at") or "") if items else "", "limit": safe_limit, "scan_limit": safe_scan_limit, "filters": { "kind": normalized_kind, "status": normalized_status, "execution_mode": normalized_execution_mode, "query": normalized_query, }, }, } def _build_driver_recommendations( *, managed_nodes: list[dict], execution_scene: dict, release_summary: dict, release_gate: dict, release_launchpad: dict, ) -> list[dict]: effective_worker_nodes = [ item for item in list(managed_nodes or []) if bool(item.get("cluster_is_effective_worker", False)) and str(item.get("cluster_status") or "").strip() in {"online", "busy"} ] handover_gap_nodes = [ item for item in effective_worker_nodes if not (bool(item.get("is_managed", False)) and bool(item.get("is_enabled", False)) and bool(item.get("is_agent_online", False))) ] delivery_dead_letter_nodes = [ item for item in list(managed_nodes or []) if str(item.get("delivery_queue_state") or "").strip() == "dead_letter" and bool(item.get("is_managed", False)) and bool(item.get("is_enabled", False)) ] delivery_retry_nodes = [ item for item in list(managed_nodes or []) if str(item.get("delivery_queue_state") or "").strip() == "retrying" and bool(item.get("is_managed", False)) and bool(item.get("is_enabled", False)) ] participating_nodes = [item for item in list(execution_scene.get("participating_nodes") or []) if str(item.get("node_code") or "").strip()] standby_nodes = [item for item in list(execution_scene.get("standby_nodes") or []) if str(item.get("node_code") or "").strip()] participating_node_codes = [str(item.get("node_code") or "").strip() for item in participating_nodes if str(item.get("node_code") or "").strip()] standby_node_codes = [str(item.get("node_code") or "").strip() for item in standby_nodes if str(item.get("node_code") or "").strip()] release_launchpad_card = _build_release_launchpad_driver_card(release_launchpad=release_launchpad) launchpad_gap_action = _release_launchpad_gap_action_context(release_launchpad) log_sync = dict(execution_scene.get("log_sync") or {}) log_sync_enabled = bool(log_sync.get("enabled", False)) remote_log_sample_count = int(log_sync.get("line_count", 0) or 0) remote_log_source_count = int(log_sync.get("source_node_count", 0) or 0) release_label = _preferred_release_label(release_summary) preferred_release = dict(release_gate.get("release") or {}) release_gate_status = str(release_gate.get("status") or "").strip() release_gate_execution_mode = str(release_gate.get("execution_mode") or "remote-agent").strip() or "remote-agent" release_gate_execution_mode_label = ( str(release_gate.get("execution_mode_label") or "").strip() or execution_mode_label(release_gate_execution_mode) ) release_gate_summary = str(release_gate.get("summary") or "").strip() release_gate_blocking_reasons = [str(item).strip() for item in list(release_gate.get("blocking_reasons") or []) if str(item).strip()] release_gate_warning_reasons = [str(item).strip() for item in list(release_gate.get("warning_reasons") or []) if str(item).strip()] release_gate_rows = list(((release_gate.get("operational_readiness") or {}).get("rows") or [])) not_execution_ready_rows = [row for row in release_gate_rows if not bool(row.get("execution_ready", False))] inspection_problem_rows = [ row for row in release_gate_rows if str(row.get("inspection_status") or "").strip() in {"running", "attention", "missing"} ] playbook_runs_payload = get_recent_ops_playbook_runs(limit=6, scan_limit=240) recent_playbook_runs = list(playbook_runs_payload.get("runs") or []) attention_playbook_runs = [item for item in recent_playbook_runs if str(item.get("status") or "").strip() == "attention"] active_playbook_runs = [item for item in recent_playbook_runs if str(item.get("status") or "").strip() == "running"] activity_stream_payload = get_ops_activity_stream(limit=10, scan_limit=80) recent_activity_items = list(activity_stream_payload.get("items") or []) start_delivery_failed_items = [ dict(item or {}) for item in recent_activity_items if str(item.get("start_delivery_state") or "").strip() == "failed_local" ] activity_focus_level, activity_focus_item = _pick_driver_activity_focus(recent_activity_items) cards: list[dict] = [] if attention_playbook_runs: first_attention_run = dict(attention_playbook_runs[0] or {}) run_code = str(first_attention_run.get("run_code") or "").strip() focus_step_title = str(first_attention_run.get("focus_step_title") or "").strip() cards.append( { "key": f"playbook-run-attention-{run_code or 'latest'}", "title": "先处理异常编排", "level_label": "最高优先", "tag_type": "danger", "summary": ( f"{str(first_attention_run.get('playbook_title') or first_attention_run.get('playbook_key') or '标准编排').strip()}" f"{' / ' + focus_step_title if focus_step_title else ''} 当前出现异常,建议先看整轮事件流。" ), "reason": str(first_attention_run.get("focus_summary") or "当前编排存在失败、阻断或取消,先处理这轮编排比继续堆动作更重要。").strip(), "node_codes": _normalize_driver_node_codes(first_attention_run.get("target_node_codes") or []), "meta_text": run_code, "primary_label": "查看编排详情", "secondary_label": "最近事件", "primary_type": "danger", "disabled": not run_code, "primary_action_code": "focus_playbook_run", "secondary_action_code": "open_playbook_run_latest_events", "action_payload": { "run_code": run_code, "focus_step_key": str(first_attention_run.get("focus_step_key") or "").strip(), "focus_step_title": focus_step_title, }, } ) elif active_playbook_runs: first_active_run = dict(active_playbook_runs[0] or {}) run_code = str(first_active_run.get("run_code") or "").strip() focus_step_title = str(first_active_run.get("focus_step_title") or "").strip() cards.append( { "key": f"playbook-run-running-{run_code or 'latest'}", "title": "盯住收口中的编排", "level_label": "推荐", "tag_type": "warning", "summary": ( f"{str(first_active_run.get('playbook_title') or first_active_run.get('playbook_key') or '标准编排').strip()}" f"{' / ' + focus_step_title if focus_step_title else ''} 仍在收口中,建议先看整轮事件流再决定是否补动作。" ), "reason": str(first_active_run.get("focus_summary") or "当前编排还在执行或排队,先看整轮回执和事件流最容易判断是否真卡住。").strip(), "node_codes": _normalize_driver_node_codes(first_active_run.get("target_node_codes") or []), "meta_text": run_code, "primary_label": "查看编排详情", "secondary_label": "最近事件", "primary_type": "warning", "disabled": not run_code, "primary_action_code": "focus_playbook_run", "secondary_action_code": "open_playbook_run_latest_events", "action_payload": { "run_code": run_code, "focus_step_key": str(first_active_run.get("focus_step_key") or "").strip(), "focus_step_title": focus_step_title, }, } ) if activity_focus_level and activity_focus_item: cards.append(_build_driver_activity_card(activity_focus_level, activity_focus_item)) if start_delivery_failed_items: first_delivery_gap_item = dict(start_delivery_failed_items[0] or {}) source_focus_ref = _normalize_focus_ref( first_delivery_gap_item.get("source_focus_ref") or first_delivery_gap_item.get("focus_ref") ) cards.append( { "key": "activity-start-delivery-gap", "title": "先检查开始回执异常", "level_label": "推荐", "tag_type": "warning", "summary": ( f"最近有 {len(start_delivery_failed_items)} 条任务已经在节点侧开始执行,但开始回执没能成功送达控制面。" ), "reason": str(first_delivery_gap_item.get("summary") or "").strip() or "这通常不代表节点没执行,而是控制面和节点现场之间已经出现开始态失联,建议先看任务事件流。", "node_codes": _normalize_driver_node_codes( [ str(source_focus_ref.get("target_node_code") or "").strip(), *list(first_delivery_gap_item.get("target_node_codes") or []), ] ), "meta_text": str(first_delivery_gap_item.get("job_code") or "").strip(), "primary_label": "查看任务事件", "secondary_label": "", "primary_type": "warning", "disabled": False, "primary_action_code": "focus_latest_job_events", "secondary_action_code": "", "primary_action_payload": { "focus_ref": source_focus_ref, }, "focus_ref": source_focus_ref, } ) if handover_gap_nodes: first_gap_node = handover_gap_nodes[0] first_gap_node_code = str(first_gap_node.get("node_code") or "").strip() launchpad_gap_action_code = str(launchpad_gap_action.get("action_code") or "").strip() launchpad_gap_node_code = str(launchpad_gap_action.get("node_code") or "").strip() primary_action_code = "handover_first_gap" primary_label = "为首台补签 Token" if bool(first_gap_node.get("is_managed", False)) else "纳管首台节点" summary = f"当前有 {len(handover_gap_nodes)} 台有效执行节点还没进入标准执行器就绪状态,继续扩机器前要先把接管链路补齐。" reason = "节点虽然在线且属于有效执行面,SSH 已可承担日志、诊断和部分服务控制,但标准巡检编排、正式发布与 Rollout 仍以 Agent 在线为准。" level_label = "最高优先" tag_type = "danger" key = "handover-gap" if launchpad_gap_action_code in {"bootstrap_run", "run_acceptance"} and ( not launchpad_gap_node_code or launchpad_gap_node_code == first_gap_node_code ): primary_action_code = launchpad_gap_action_code primary_label = "跑接入收口" if launchpad_gap_action_code == "bootstrap_run" else "跑接管验收" summary = ( f"当前有 {len(handover_gap_nodes)} 台有效执行节点还没进入标准执行器就绪状态," "并且 Launchpad 已经给出首台节点的标准收口动作。" ) reason = str(launchpad_gap_action.get("summary") or "").strip() or reason level_label = "最高优先" if launchpad_gap_action_code == "bootstrap_run" else "推荐" tag_type = "warning" if launchpad_gap_action_code == "bootstrap_run" else "success" key = f"handover-gap-{launchpad_gap_action_code}" cards.append( { "key": key, "title": "先补接管缺口", "level_label": level_label, "tag_type": tag_type, "summary": summary, "reason": reason, "node_codes": [str(item.get("node_code") or "").strip() for item in handover_gap_nodes if str(item.get("node_code") or "").strip()], "primary_label": primary_label, "secondary_label": "查看目标节点", "primary_type": "danger", "disabled": not first_gap_node_code, "primary_action_code": primary_action_code, "secondary_action_code": "view_first_gap", "primary_action_payload": ( {"node_code": first_gap_node_code} if primary_action_code in {"bootstrap_run", "run_acceptance"} else {} ), } ) if delivery_dead_letter_nodes: first_dead_letter_node = dict(delivery_dead_letter_nodes[0] or {}) affected_node_codes = [ str(item.get("node_code") or "").strip() for item in delivery_dead_letter_nodes if str(item.get("node_code") or "").strip() ] cards.append( { "key": "node-agent-dead-letter", "title": "先处理 Agent 死信", "level_label": "最高优先", "tag_type": "danger", "summary": ( f"当前有 {len(delivery_dead_letter_nodes)} 台托管节点存在 Node Agent 死信记录," "说明部分任务回执或事件已经进入人工介入区。" ), "reason": str(first_dead_letter_node.get("delivery_queue_reason") or "Node Agent 回执队列已经出现死信,继续堆动作会放大控制面与节点现场的不一致。").strip(), "node_codes": affected_node_codes, "meta_text": _format_node_code_list(affected_node_codes), "primary_label": "重放死信队列", "secondary_label": "查看 Worker 日志", "primary_type": "danger", "disabled": not affected_node_codes, "primary_action_code": "replay_delivery_queue", "secondary_action_code": "open_worker_logs", "primary_action_payload": { "template_key": "delivery.queue.replay", "target_node_codes": affected_node_codes, "execution_mode": "remote-agent", "auto_approve": False, "payload": { "limit": min(max(len(affected_node_codes), 1), 20), "flush_after_replay": True, "reason": "", }, }, } ) elif delivery_retry_nodes: affected_node_codes = [ str(item.get("node_code") or "").strip() for item in delivery_retry_nodes if str(item.get("node_code") or "").strip() ] first_retry_node = dict(delivery_retry_nodes[0] or {}) cards.append( { "key": "node-agent-retrying", "title": "关注 Agent 回执积压", "level_label": "推荐", "tag_type": "warning", "summary": ( f"当前有 {len(delivery_retry_nodes)} 台托管节点存在待重试回执," "控制面状态可能会短暂落后于节点现场。" ), "reason": str(first_retry_node.get("delivery_queue_reason") or "Node Agent 正在自动回放积压的完成回执或事件,建议先观察日志再继续堆新的运维动作。").strip(), "node_codes": affected_node_codes, "meta_text": _format_node_code_list(affected_node_codes), "primary_label": "冲刷回执队列", "secondary_label": "查看 Worker 日志", "primary_type": "warning", "disabled": not affected_node_codes, "primary_action_code": "flush_delivery_queue", "secondary_action_code": "open_worker_logs", "primary_action_payload": { "template_key": "delivery.queue.flush", "target_node_codes": affected_node_codes, "execution_mode": "remote-agent", "auto_approve": True, "payload": { "limit": min(max(len(affected_node_codes) * 5, 1), 20), }, }, } ) preferred_release_id = int(preferred_release.get("id") or 0) preferred_release_version = str(preferred_release.get("release_version") or "").strip() or "-" preferred_release_channel = str(preferred_release.get("channel") or "").strip() or "stable" preferred_release_label = f"{preferred_release_version} / {preferred_release_channel}" if release_launchpad_card: cards.append(release_launchpad_card) elif preferred_release_id <= 0: cards.append( { "key": "release-missing", "title": "先建立 Release", "level_label": "待补齐", "tag_type": "info", "summary": "当前还没有默认 Release,正式 Rollout 入口尚未建立。", "reason": "没有 Release 时,节点更新仍然容易回到临时 git pull、手工覆盖或口头版本约定。", "node_codes": [], "meta_text": "", "primary_label": "创建 Release", "secondary_label": "", "primary_type": "primary", "disabled": False, "primary_action_code": "open_release_dialog", "secondary_action_code": "", } ) elif release_gate_status in {"release_not_ready", "artifact_missing"}: cards.append( { "key": "release-finish-metadata", "title": "先收口默认 Release", "level_label": "最高优先" if release_gate_status == "artifact_missing" else "推荐", "tag_type": "danger" if release_gate_status == "artifact_missing" else "warning", "summary": release_gate_summary or f"默认 Release {preferred_release_label} 还没有进入可稳定 Rollout 的状态。", "reason": "先把 Release 状态、制品地址和版本信息补完整,Rollout 才不会重新退回手工发布路径。", "node_codes": [], "meta_text": preferred_release_label, "primary_label": "查看版本区", "secondary_label": "创建 Release", "primary_type": "warning", "disabled": False, "primary_action_code": "focus_release_hub", "secondary_action_code": "open_release_dialog", } ) elif release_gate_status == "blocked": blocker_node_codes = [ str(item.get("node_code") or "").strip() for item in (not_execution_ready_rows or inspection_problem_rows) if str(item.get("node_code") or "").strip() ] first_gap_row = dict(not_execution_ready_rows[0] or {}) first_gap_is_managed = bool(first_gap_row.get("is_managed", False)) first_gap_code = str(first_gap_row.get("node_code") or "").strip() cards.append( { "key": "release-rollout-blocked", "title": "先补 Rollout 前置条件", "level_label": "最高优先", "tag_type": "danger", "summary": release_gate_summary or f"默认 Release {preferred_release_label} 当前仍存在 Rollout 阻断。", "reason": ( release_gate_blocking_reasons[0] if release_gate_blocking_reasons else f"默认目标节点里仍有节点未通过 {release_gate_execution_mode_label}、巡检或接管门禁。" ), "node_codes": blocker_node_codes, "meta_text": preferred_release_label, "primary_label": ( "纳管首台节点" if first_gap_code and not first_gap_is_managed else ("查看首台缺口" if first_gap_code else "执行标准巡检") ), "secondary_label": "查看版本区", "primary_type": "danger", "disabled": not blocker_node_codes, "primary_action_code": ( "handover_first_gap" if first_gap_code and not first_gap_is_managed else ("view_first_gap" if first_gap_code else "run_standard_inspection") ), "secondary_action_code": "focus_release_hub", } ) elif release_gate_status == "attention": warning_node_codes = [ str(item.get("node_code") or "").strip() for item in inspection_problem_rows if str(item.get("node_code") or "").strip() ] cards.append( { "key": "release-rollout-attention", "title": "先补默认 Rollout 巡检", "level_label": "推荐", "tag_type": "warning", "summary": release_gate_summary or f"默认 Release {preferred_release_label} 已接近可发状态,但仍建议先补齐巡检。", "reason": ( release_gate_warning_reasons[0] if release_gate_warning_reasons else "先补健康快照、日志和诊断包,可以显著降低真正发 Rollout 时的误判风险。" ), "node_codes": warning_node_codes, "meta_text": preferred_release_label, "primary_label": "执行标准巡检", "secondary_label": "查看版本区", "primary_type": "warning", "disabled": not warning_node_codes, "primary_action_code": "run_standard_inspection", "secondary_action_code": "focus_release_hub", } ) elif release_gate_status == "no_targets": cards.append( { "key": "release-rollout-no-targets", "title": "先确认默认 Rollout 目标", "level_label": "推荐", "tag_type": "warning", "summary": release_gate_summary or f"默认 Release {preferred_release_label} 当前没有可直接纳入 Rollout 的默认目标节点。", "reason": "通常是当前有效执行面没有在线节点,或节点尚未进入默认 target 集。", "node_codes": [], "meta_text": preferred_release_label, "primary_label": "查看版本区", "secondary_label": "", "primary_type": "warning", "disabled": False, "primary_action_code": "focus_release_hub", "secondary_action_code": "", } ) elif release_gate_status == "ready": cards.append( { "key": "release-rollout-ready", "title": "当前可以发起 Rollout", "level_label": "准备就绪", "tag_type": "success", "summary": release_gate_summary or f"默认 Release {preferred_release_label} 已具备进入 Rollout 的门禁条件。", "reason": "接管、巡检和默认版本都已经进入可推进状态,现在可以把更新真正纳入 Release / Rollout 闭环。", "node_codes": [str(item.get("node_code") or "").strip() for item in list(release_gate.get("default_target_nodes") or []) if str(item.get("node_code") or "").strip()], "meta_text": preferred_release_label, "primary_label": "Worker 灰度", "secondary_label": "Control 发布", "primary_type": "success", "disabled": False, "primary_action_code": "create_release_rollout_worker", "primary_action_payload": { "release_id": preferred_release_id, }, "secondary_action_code": "create_release_rollout_control", "secondary_action_payload": { "release_id": preferred_release_id, }, } ) elif preferred_release_id > 0: cards.append( { "key": "release-rollout-fallback", "title": "查看默认 Release 门禁", "level_label": "推荐", "tag_type": "info", "summary": release_gate_summary or f"默认 Release {preferred_release_label} 的 Rollout 门禁需要进一步确认。", "reason": "当前建议先进入 Release Hub,看默认版本、批次和门禁摘要,再决定下一步。", "node_codes": [], "meta_text": preferred_release_label, "primary_label": "查看版本区", "secondary_label": "", "primary_type": "primary", "disabled": False, "primary_action_code": "focus_release_hub", "secondary_action_code": "", } ) if participating_node_codes and not log_sync_enabled: primary_focus_ref = _build_scene_node_log_focus_ref( participating_node_codes[0], mode="key", limit=80, source="remote_log_sync_disabled", ) secondary_focus_ref = _build_scene_node_log_focus_ref( participating_node_codes[0], mode="full", limit=120, source="remote_log_sync_disabled", ) cards.append( { "key": "enable-scene-log-sync", "title": "先补现场日志回传", "level_label": "最高优先", "tag_type": "danger", "summary": f"当前有 {len(participating_node_codes)} 台节点正在真实参与检测,但海外控制面还看不到过程日志,排障会处于半盲态。", "reason": "先把远端日志回传切到关键模式,才能在不放大噪音的前提下看到阶段、异常、代理与执行过程。", "node_codes": participating_node_codes, "primary_label": "开启关键回传", "secondary_label": "开启全量回传", "primary_type": "danger", "disabled": False, "focus_ref": primary_focus_ref, "primary_focus_ref": primary_focus_ref, "secondary_focus_ref": secondary_focus_ref, "primary_action_code": "enable_log_sync_key", "primary_action_payload": {"focus_ref": primary_focus_ref}, "secondary_action_code": "enable_log_sync_full", "secondary_action_payload": {"focus_ref": secondary_focus_ref}, } ) elif participating_node_codes and remote_log_sample_count <= 0: primary_focus_ref = _build_scene_node_log_focus_ref( participating_node_codes[0], mode=str(log_sync.get("mode") or "key"), limit=120, source="remote_log_sync_waiting_sample", ) secondary_focus_ref = _merge_focus_ref( {}, kind="execution_scene", scene_key="scene.logs", source="remote_log_sync_waiting_sample", ) cards.append( { "key": "inspect-scene-log-gap", "title": "排查现场日志样本缺口", "level_label": "推荐", "tag_type": "warning", "summary": f"当前日志回传已开启,但 {len(participating_node_codes)} 台参与节点还没有回传样本,先确认 Worker 现场输出为什么没回来。", "reason": "这通常意味着现场仍在执行,但日志镜像链路、周期或样本采集还没形成有效观测,适合先抓 Worker 日志再看巡检。", "node_codes": participating_node_codes, "meta_text": f"参与节点 {len(participating_node_codes)} 台 / 已回传来源 {remote_log_source_count} 台", "primary_label": "看 Worker 日志", "secondary_label": "执行标准巡检", "primary_type": "warning", "disabled": not participating_node_codes, "focus_ref": primary_focus_ref, "primary_focus_ref": primary_focus_ref, "secondary_focus_ref": secondary_focus_ref, "primary_action_code": "open_worker_logs_participating", "primary_action_payload": {"focus_ref": primary_focus_ref}, "secondary_action_code": "run_inspection_participating", "secondary_action_payload": {"focus_ref": secondary_focus_ref}, } ) cards.append( { "key": "inspect-participating", "title": "先看执行现场", "level_label": "推荐" if participating_node_codes else "待命", "tag_type": "warning" if participating_node_codes else "info", "summary": ( f"当前有 {len(participating_node_codes)} 台节点正在真实参与检测,优先回收它们的标准巡检结果。" if participating_node_codes else "当前没有节点处于真实执行现场,可先关注待命节点或接管缺口。" ), "reason": "真正参与检测的节点最接近现场,最适合先看健康快照、Worker 日志和诊断包。", "node_codes": participating_node_codes, "primary_label": "执行标准巡检", "secondary_label": "打开诊断模板", "primary_type": "primary", "disabled": not participating_node_codes, "primary_action_code": "run_inspection_participating", "secondary_action_code": "open_diagnostics_participating", } ) cards.append( { "key": "inspect-standby", "title": "排查在线未参与节点", "level_label": "推荐" if standby_node_codes else "待命", "tag_type": "warning" if standby_node_codes else "info", "summary": ( f"当前有 {len(standby_node_codes)} 台节点在线但未参与检测,建议做一轮标准巡检确认是正常待命还是接单异常。" if standby_node_codes else "当前没有在线未参与节点,执行面结构比较干净。" ), "reason": "这组节点最适合定位“在线但不接单”“控制面兼跑未开始执行”“接管完成但还没真正纳入巡检”类问题。", "node_codes": standby_node_codes, "primary_label": "执行标准巡检", "secondary_label": "看 Worker 日志", "primary_type": "primary", "disabled": not standby_node_codes, "primary_action_code": "run_inspection_standby", "secondary_action_code": "open_worker_logs_standby", } ) return cards def _default_ops_priority_recommendation() -> dict: return { "source": "static", "key": "build-node-agent-plane", "priority": "build-node-agent-plane", "title": "build-node-agent-plane", "summary": "下一阶段应把 SSH/脚本式运维升级为海外控制面 + 节点 Agent 的任务编排模式。", "reason": "当前已经具备跨地域心跳、同步、就绪度和日志回传基础,但节点规模继续增加后,人工 SSH 与复制日志的边际成本会快速失控。", "level_label": "长期重点", "tag_type": "info", "primary_label": "", "secondary_label": "", "primary_action_code": "", "secondary_action_code": "", "node_codes": [], "primary_node_codes": [], "secondary_node_codes": [], "primary_action_payload": {}, "secondary_action_payload": {}, "reasons": [ "当前已经具备跨地域心跳、同步、就绪度和日志回传基础。", "节点数继续增加后,人工 SSH 与复制日志的边际成本会快速失控。", "后台按钮化动作需要统一的任务模型、执行回执和日志流通道。", ], } def _build_ops_priority_recommendation(driver_recommendations: list[dict]) -> dict: for raw_item in list(driver_recommendations or []): item = dict(raw_item or {}) if not item: continue title = str(item.get("title") or item.get("key") or "").strip() summary = str(item.get("summary") or item.get("reason") or "").strip() if not title and not summary: continue shared_node_codes = _normalize_driver_node_codes(item.get("node_codes") or []) primary_node_codes = _normalize_driver_node_codes(item.get("primary_node_codes") or shared_node_codes) secondary_node_codes = _normalize_driver_node_codes(item.get("secondary_node_codes") or shared_node_codes) shared_payload = dict(item.get("action_payload") or {}) return { "source": "driver_recommendations", "key": str(item.get("key") or "").strip(), "priority": title or str(item.get("key") or "").strip() or "当前建议", "title": title or str(item.get("key") or "").strip() or "当前建议", "summary": summary or "当前已有建议动作,建议先按驾驶建议卡推进。", "reason": str(item.get("reason") or "").strip(), "level_label": str(item.get("level_label") or "").strip(), "tag_type": str(item.get("tag_type") or "").strip(), "primary_label": str(item.get("primary_label") or "").strip(), "secondary_label": str(item.get("secondary_label") or "").strip(), "primary_action_code": str(item.get("primary_action_code") or "").strip(), "secondary_action_code": str(item.get("secondary_action_code") or "").strip(), "node_codes": shared_node_codes, "primary_node_codes": primary_node_codes, "secondary_node_codes": secondary_node_codes, "primary_action_payload": { **shared_payload, **dict(item.get("primary_action_payload") or {}), }, "secondary_action_payload": { **shared_payload, **dict(item.get("secondary_action_payload") or {}), }, "reasons": [ part for part in [ str(item.get("reason") or "").strip(), str(item.get("summary") or "").strip(), ] if part ], } return _default_ops_priority_recommendation() def _driver_feed_status_from_tag_type(tag_type: str) -> str: normalized_tag_type = str(tag_type or "").strip() if normalized_tag_type in {"danger", "warning"}: return "attention" if normalized_tag_type == "success": return "ready" return "planned" def _build_driver_feed_entry_from_priority_recommendation(recommendation: dict) -> dict: normalized_recommendation = dict(recommendation or {}) key = str(normalized_recommendation.get("key") or "").strip() or "priority-recommendation" shared_node_codes = _normalize_driver_node_codes(normalized_recommendation.get("node_codes") or []) primary_node_codes = _normalize_driver_node_codes( normalized_recommendation.get("primary_node_codes") or shared_node_codes ) secondary_node_codes = _normalize_driver_node_codes( normalized_recommendation.get("secondary_node_codes") or shared_node_codes ) primary_action_payload = dict(normalized_recommendation.get("primary_action_payload") or {}) secondary_action_payload = dict(normalized_recommendation.get("secondary_action_payload") or {}) title = str(normalized_recommendation.get("title") or normalized_recommendation.get("priority") or key).strip() summary = str(normalized_recommendation.get("summary") or "").strip() reason = str(normalized_recommendation.get("reason") or "").strip() detail_lines = _normalize_driver_text_lines( [ *[str(part or "").strip() for part in list(normalized_recommendation.get("reasons") or [])], reason, ], limit=4, ) focus_ref = _normalize_focus_ref(normalized_recommendation.get("focus_ref")) primary_focus_ref = _normalize_focus_ref(normalized_recommendation.get("primary_focus_ref")) secondary_focus_ref = _normalize_focus_ref(normalized_recommendation.get("secondary_focus_ref")) return _attach_driver_feed_contract_navigation({ "key": f"priority:{key}", "kind": "priority_recommendation", "lane": "do_now", "lane_label": "当前优先", "title": title or "当前建议", "summary": summary or reason or "当前已有建议动作,建议先从这里推进。", "reason": reason, "status": _driver_feed_status_from_tag_type(normalized_recommendation.get("tag_type")), "level_label": str(normalized_recommendation.get("level_label") or "").strip() or "推荐", "tag_type": str(normalized_recommendation.get("tag_type") or "").strip() or "info", "meta_text": str(normalized_recommendation.get("source") or "overview.recommendation").strip(), "node_codes": shared_node_codes, "detail_lines": detail_lines, "focus_ref": focus_ref, "executor_kind": "driver_action", "primary_label": str(normalized_recommendation.get("primary_label") or "").strip(), "secondary_label": str(normalized_recommendation.get("secondary_label") or "").strip(), "primary_action_code": str(normalized_recommendation.get("primary_action_code") or "").strip(), "secondary_action_code": str(normalized_recommendation.get("secondary_action_code") or "").strip(), "primary_node_codes": primary_node_codes, "secondary_node_codes": secondary_node_codes, "primary_focus_ref": primary_focus_ref, "secondary_focus_ref": secondary_focus_ref, "primary_action_payload": _action_payload_with_focus_ref(primary_action_payload, primary_focus_ref), "secondary_action_payload": _action_payload_with_focus_ref(secondary_action_payload, secondary_focus_ref), }) def _build_driver_feed_entry_from_driver_recommendation(item: dict) -> dict: normalized_item = dict(item or {}) key = str(normalized_item.get("key") or "").strip() or "driver-recommendation" shared_node_codes = _normalize_driver_node_codes(normalized_item.get("node_codes") or []) primary_node_codes = _normalize_driver_node_codes(normalized_item.get("primary_node_codes") or shared_node_codes) secondary_node_codes = _normalize_driver_node_codes( normalized_item.get("secondary_node_codes") or shared_node_codes ) shared_action_payload = dict(normalized_item.get("action_payload") or {}) primary_action_payload = { **shared_action_payload, **dict(normalized_item.get("primary_action_payload") or {}), } secondary_action_payload = { **shared_action_payload, **dict(normalized_item.get("secondary_action_payload") or {}), } detail_lines = _normalize_driver_text_lines( [ str(normalized_item.get("reason") or "").strip(), str(normalized_item.get("meta_text") or "").strip(), ], limit=3, ) focus_ref = _normalize_focus_ref(normalized_item.get("focus_ref")) primary_focus_ref = _normalize_focus_ref(normalized_item.get("primary_focus_ref")) secondary_focus_ref = _normalize_focus_ref(normalized_item.get("secondary_focus_ref")) return _attach_driver_feed_contract_navigation({ "key": f"recommendation:{key}", "kind": "driver_recommendation", "lane": "do_now", "lane_label": "建议动作", "title": str(normalized_item.get("title") or key).strip() or "驾驶建议", "summary": str(normalized_item.get("summary") or normalized_item.get("reason") or "").strip() or "建议先处理这条驾驶动作。", "reason": str(normalized_item.get("reason") or "").strip(), "status": _driver_feed_status_from_tag_type(normalized_item.get("tag_type")), "level_label": str(normalized_item.get("level_label") or "").strip() or "推荐", "tag_type": str(normalized_item.get("tag_type") or "").strip() or "info", "meta_text": str(normalized_item.get("meta_text") or "").strip(), "node_codes": shared_node_codes, "detail_lines": detail_lines, "focus_ref": focus_ref, "executor_kind": "driver_action", "primary_label": str(normalized_item.get("primary_label") or "").strip(), "secondary_label": str(normalized_item.get("secondary_label") or "").strip(), "primary_action_code": str(normalized_item.get("primary_action_code") or "").strip(), "secondary_action_code": str(normalized_item.get("secondary_action_code") or "").strip(), "primary_node_codes": primary_node_codes, "secondary_node_codes": secondary_node_codes, "primary_focus_ref": primary_focus_ref, "secondary_focus_ref": secondary_focus_ref, "primary_action_payload": _action_payload_with_focus_ref(primary_action_payload, primary_focus_ref), "secondary_action_payload": _action_payload_with_focus_ref(secondary_action_payload, secondary_focus_ref), }) def _build_driver_feed_entry_from_runbook_sequence(sequence: dict) -> dict: normalized_sequence = dict(sequence or {}) sequence_key = str(normalized_sequence.get("key") or "").strip() if not sequence_key: return {} primary_resolution = dict(normalized_sequence.get("primary_resolution") or {}) secondary_resolution = dict(normalized_sequence.get("secondary_resolution") or {}) resolved_action_label = ( str(primary_resolution.get("driver_action_label") or "").strip() or str(primary_resolution.get("driver_action_code") or "").strip() ) resolved_action_code = str(primary_resolution.get("driver_action_code") or "").strip() resolved_node_codes = _normalize_driver_node_codes(primary_resolution.get("driver_node_codes") or []) secondary_resolved_action_label = ( str(secondary_resolution.get("driver_action_label") or "").strip() or str(secondary_resolution.get("driver_action_code") or "").strip() ) target_node_codes = _normalize_driver_node_codes(normalized_sequence.get("target_node_codes") or []) step_titles = [ str(item or "").strip() for item in list(normalized_sequence.get("step_titles") or []) if str(item or "").strip() ] detail_lines: list[str] = [] if primary_resolution and primary_resolution.get("ok") is False: detail_lines.append(f"当前解析失败:{str(primary_resolution.get('message') or '解析失败').strip()}") elif resolved_action_label or resolved_action_code: detail_lines.append( f"当前解析:{resolved_action_label or resolved_action_code}" + ( f" ({resolved_action_code})" if resolved_action_code and resolved_action_label and resolved_action_code != resolved_action_label else "" ) ) detail_lines.append( f"当前目标:{_format_node_code_list(resolved_node_codes or target_node_codes)}" ) if str(normalized_sequence.get("secondary_label") or "").strip() and secondary_resolved_action_label: detail_lines.append(f"备用解析:{secondary_resolved_action_label}") if step_titles: detail_lines.append(f"标准步骤:{' -> '.join(step_titles)}") if str(normalized_sequence.get("reason") or "").strip(): detail_lines.append(str(normalized_sequence.get("reason") or "").strip()) focus_ref = _normalize_focus_ref(normalized_sequence.get("focus_ref")) primary_focus_ref = _normalize_focus_ref(normalized_sequence.get("primary_focus_ref")) secondary_focus_ref = _normalize_focus_ref(normalized_sequence.get("secondary_focus_ref")) return _attach_driver_feed_contract_navigation({ "key": f"runbook:{sequence_key}", "kind": "runbook_sequence", "lane": "standard_path", "lane_label": "标准路径", "title": str(normalized_sequence.get("title") or "").strip() or "标准作业路径", "summary": str(normalized_sequence.get("summary") or "").strip() or "当前标准作业路径已就绪。", "reason": str(normalized_sequence.get("reason") or "").strip(), "status": _runbook_sequence_activity_status(normalized_sequence), "level_label": str(normalized_sequence.get("status_label") or normalized_sequence.get("status") or "").strip() or "ready", "tag_type": str(normalized_sequence.get("tag_type") or "").strip() or "info", "meta_text": str(normalized_sequence.get("target_scope_label") or "").strip(), "node_codes": target_node_codes, "detail_lines": _normalize_driver_text_lines(detail_lines, limit=5), "focus_ref": focus_ref, "executor_kind": "runbook_sequence", "sequence_key": sequence_key, "primary_label": str(normalized_sequence.get("primary_label") or "").strip(), "secondary_label": str(normalized_sequence.get("secondary_label") or "").strip(), "primary_action_code": resolved_action_code or str(normalized_sequence.get("primary_action_code") or "").strip(), "secondary_action_code": ( str(secondary_resolution.get("driver_action_code") or "").strip() or str(normalized_sequence.get("secondary_action_code") or "").strip() ), "primary_node_codes": resolved_node_codes or target_node_codes, "secondary_node_codes": _normalize_driver_node_codes( secondary_resolution.get("driver_node_codes") or target_node_codes ), "primary_focus_ref": primary_focus_ref, "secondary_focus_ref": secondary_focus_ref, "primary_action_payload": _action_payload_with_focus_ref( primary_resolution.get("driver_action_payload") or {}, primary_focus_ref, ), "secondary_action_payload": _action_payload_with_focus_ref( secondary_resolution.get("driver_action_payload") or {}, secondary_focus_ref, ), "occurred_at": _activity_time(primary_resolution.get("resolved_at")), }) def _build_driver_feed_top_recommendation(entry: dict) -> dict: normalized_entry = dict(entry or {}) if not normalized_entry: return {} return { "key": str(normalized_entry.get("key") or "").strip(), "kind": str(normalized_entry.get("kind") or "").strip(), "lane": str(normalized_entry.get("lane") or "").strip(), "title": str(normalized_entry.get("title") or "").strip(), "summary": str(normalized_entry.get("summary") or "").strip(), "reason": str(normalized_entry.get("reason") or "").strip(), "status": str(normalized_entry.get("status") or "").strip(), "level_label": str(normalized_entry.get("level_label") or "").strip(), "tag_type": str(normalized_entry.get("tag_type") or "").strip(), "meta_text": str(normalized_entry.get("meta_text") or "").strip(), "executor_kind": str(normalized_entry.get("executor_kind") or "").strip(), "sequence_key": str(normalized_entry.get("sequence_key") or "").strip(), "focus_ref": _normalize_focus_ref(normalized_entry.get("focus_ref")), "primary_label": str(normalized_entry.get("primary_label") or "").strip(), "secondary_label": str(normalized_entry.get("secondary_label") or "").strip(), "primary_action_code": str(normalized_entry.get("primary_action_code") or "").strip(), "secondary_action_code": str(normalized_entry.get("secondary_action_code") or "").strip(), "node_codes": _normalize_driver_node_codes(normalized_entry.get("node_codes") or []), "primary_node_codes": _normalize_driver_node_codes(normalized_entry.get("primary_node_codes") or []), "secondary_node_codes": _normalize_driver_node_codes(normalized_entry.get("secondary_node_codes") or []), "primary_focus_ref": _normalize_focus_ref(normalized_entry.get("primary_focus_ref")), "secondary_focus_ref": _normalize_focus_ref(normalized_entry.get("secondary_focus_ref")), "primary_action_payload": dict(normalized_entry.get("primary_action_payload") or {}), "secondary_action_payload": dict(normalized_entry.get("secondary_action_payload") or {}), "detail_lines": _normalize_driver_text_lines(normalized_entry.get("detail_lines") or [], limit=5), "contract_keys": list(normalized_entry.get("contract_keys") or []), "contract_navigation": dict(normalized_entry.get("contract_navigation") or {}), } def get_ops_driver_feed() -> dict: overview = get_ops_overview() go_live_summary = get_ops_go_live_summary() runbook = get_ops_runbook() recommendation = dict(overview.get("recommendation") or {}) driver_recommendations = [dict(item or {}) for item in list(overview.get("driver_recommendations") or []) if item] control_sequences = [dict(item or {}) for item in list(runbook.get("control_sequences") or []) if item] activity_stream = dict(overview.get("activity_stream") or {}) activity_focus = _build_driver_activity_focus(list(activity_stream.get("items") or [])) scene_log_observation = _build_scene_log_observation_from_overview(overview) entries: list[dict] = [] seen_keys: set[str] = set() priority_entry = _build_driver_feed_entry_from_priority_recommendation(recommendation) if priority_entry: entries.append(priority_entry) recommendation_key = str(recommendation.get("key") or "").strip() if recommendation_key: seen_keys.add(recommendation_key) for raw_item in driver_recommendations: raw_key = str(raw_item.get("key") or "").strip() if raw_key and raw_key in seen_keys: continue entry = _build_driver_feed_entry_from_driver_recommendation(raw_item) if not entry: continue entries.append(entry) if raw_key: seen_keys.add(raw_key) for raw_sequence in control_sequences: entry = _build_driver_feed_entry_from_runbook_sequence(raw_sequence) if entry: entries.append(entry) lane_counts: dict[str, int] = {} kind_counts: dict[str, int] = {} status_counts: dict[str, int] = {} for item in entries: lane = str(item.get("lane") or "").strip() or "other" kind = str(item.get("kind") or "").strip() or "other" status = str(item.get("status") or "").strip() or "other" lane_counts[lane] = int(lane_counts.get(lane, 0) or 0) + 1 kind_counts[kind] = int(kind_counts.get(kind, 0) or 0) + 1 status_counts[status] = int(status_counts.get(status, 0) or 0) + 1 headline = "" if entries: first_entry = dict(entries[0] or {}) headline = ( f"{str(first_entry.get('title') or '').strip()}:{str(first_entry.get('summary') or '').strip()}" ).strip(":") elif activity_focus: first_focus = dict(activity_focus[0] or {}) headline = ( f"{str(first_focus.get('title') or '').strip()}:{str(first_focus.get('summary') or '').strip()}" ).strip(":") publish_status = str(go_live_summary.get("publish_status") or "").strip() publish_status_label = str(go_live_summary.get("publish_status_label") or "").strip() publish_summary = str(go_live_summary.get("publish_summary") or "").strip() launchpad_recommended_target_node_code = str( go_live_summary.get("launchpad_recommended_target_node_code") or "" ).strip() launchpad_recommended_recovery_label = str( go_live_summary.get("launchpad_recommended_recovery_label") or "" ).strip() launchpad_recommended_recovery_summary = str( go_live_summary.get("launchpad_recommended_recovery_summary") or "" ).strip() launchpad_onboarding_bootstrap_pending_nodes = int( go_live_summary.get("launchpad_onboarding_bootstrap_pending_nodes", 0) or 0 ) launchpad_onboarding_acceptance_ready_nodes = int( go_live_summary.get("launchpad_onboarding_acceptance_ready_nodes", 0) or 0 ) if publish_status and publish_status != "ready": publish_prefix = publish_status_label or publish_status headline = ( f"{headline}|发布闸门:{publish_prefix}" if headline else f"发布闸门:{publish_prefix}{'|' + publish_summary if publish_summary else ''}" ) if launchpad_recommended_target_node_code and launchpad_recommended_recovery_label: launchpad_prefix = f"{launchpad_recommended_recovery_label}:{launchpad_recommended_target_node_code}" headline = f"{headline}|接入缺口:{launchpad_prefix}" if headline else launchpad_prefix summary_contract_keys = _normalize_ops_contract_keys( ["ops_driver_contract"] + [ contract_key for item in entries for contract_key in list((item or {}).get("contract_keys") or []) ] + [ contract_key for item in activity_focus for contract_key in list((item or {}).get("contract_keys") or []) ] + list(scene_log_observation.get("contract_keys") or []) ) top_recommendation = _build_driver_feed_top_recommendation(entries[0] or {}) grouped_driver_recommendations = [ dict(item or {}) for item in entries if str((item or {}).get("kind") or "").strip() in {"priority_recommendation", "driver_recommendation"} ] runbook_sequences = [ dict(item or {}) for item in entries if str((item or {}).get("kind") or "").strip() == "runbook_sequence" ] automation_entries = [ _build_codex_brief_entry(raw_entry) for raw_entry in entries if dict(raw_entry or {}) ] automation_entries = [entry for entry in automation_entries if entry] automation_coverage = _build_ops_automation_coverage_summary( automation_entries, activity_focus_total=len(activity_focus), scene_log_observation=scene_log_observation, go_live_summary=go_live_summary, ) return { "generated_at": datetime.now().isoformat(timespec="seconds"), "headline": headline or "当前还没有可展示的驾驶主线。", "go_live_summary": go_live_summary, "top_recommendation": top_recommendation, "driver_recommendations": grouped_driver_recommendations, "runbook_sequences": runbook_sequences, "activity_focus": activity_focus, "scene_log_observation": scene_log_observation, "entries": entries, "automation_coverage": automation_coverage, "contract_navigation": _build_ops_contract_navigation( summary_contract_keys, primary_contract_key="ops_driver_contract", registry=get_ops_contract_registry(), ), "summary": { "total": len(entries), "lane_counts": lane_counts, "kind_counts": kind_counts, "status_counts": status_counts, "go_live_status": str(go_live_summary.get("go_live_status") or "").strip(), "operator_title": str(go_live_summary.get("operator_title") or "").strip(), "publish_ready": bool(go_live_summary.get("publish_ready")), "publish_status": publish_status, "publish_status_label": publish_status_label, "launchpad_recommended_target_node_code": launchpad_recommended_target_node_code, "launchpad_recommended_recovery_label": launchpad_recommended_recovery_label, "launchpad_recommended_recovery_summary": launchpad_recommended_recovery_summary, "launchpad_onboarding_bootstrap_pending_nodes": launchpad_onboarding_bootstrap_pending_nodes, "launchpad_onboarding_acceptance_ready_nodes": launchpad_onboarding_acceptance_ready_nodes, "do_now_total": int(lane_counts.get("do_now", 0) or 0), "standard_path_total": int(lane_counts.get("standard_path", 0) or 0), "activity_focus_total": len(activity_focus), "scene_log_status": str(scene_log_observation.get("status") or "").strip() or "standby", "launch_status": str(automation_coverage.get("launch_status") or "").strip(), "launch_ready": bool(automation_coverage.get("launch_ready", False)), "preview_only_total": int(automation_coverage.get("preview_only_total", 0) or 0), "backend_handled_total": int(automation_coverage.get("backend_handled_total", 0) or 0), }, } def _build_scene_log_observation_from_overview(overview: dict) -> dict: normalized_overview = dict(overview or {}) execution_scene = dict(normalized_overview.get("execution_scene") or {}) log_sync = dict(execution_scene.get("log_sync") or {}) participation_summary = dict(execution_scene.get("participation_summary") or {}) participating_node_codes = _normalize_ops_node_code_list( execution_scene.get("participating_nodes") or [], participation_summary.get("dispatch_active_node_codes") or [], participation_summary.get("recent_only_node_codes") or [], ) missing_node_codes = _normalize_ops_node_code_list(log_sync.get("missing_participating_nodes") or []) source_node_codes = _normalize_ops_node_code_list( log_sync.get("source_nodes") or [], log_sync.get("source_node_summaries") or [], ) preferred_node_codes = _normalize_ops_node_code_list( missing_node_codes, participating_node_codes, source_node_codes, ) target_node_code = str((preferred_node_codes or [""])[0] or "").strip() enabled = bool(log_sync.get("enabled", False)) mode = _normalize_scene_log_mode(log_sync.get("mode") or "key", fallback="key") participating_total = int( log_sync.get("participating_node_count", participation_summary.get("participating_nodes", len(participating_node_codes))) or participation_summary.get("participating_nodes", len(participating_node_codes)) or len(participating_node_codes) or 0 ) covered_total = int(log_sync.get("covered_participating_node_count", 0) or 0) line_count = int(log_sync.get("line_count", 0) or 0) source_node_count = int(log_sync.get("source_node_count", 0) or 0) missing_total = int(log_sync.get("missing_participating_node_count", len(missing_node_codes)) or len(missing_node_codes)) status = "standby" status_label = "待命" summary = "当前没有参与检测节点,现场日志观察处于待命状态。" source = "driver_scene_log_observation" if participating_total > 0 and not enabled: status = "disabled" status_label = "未开启" summary = ( f"当前有 {participating_total} 台参与节点,但远端日志回传仍关闭," "海外控制面还看不到节点级现场日志。" ) source = "remote_log_sync_disabled" elif participating_total > 0 and ( line_count <= 0 or source_node_count <= 0 or missing_total > 0 ): status = "waiting_sample" status_label = "等待样本" summary = ( f"远端日志回传已经开启,但参与节点只覆盖 {covered_total}/{participating_total}," f"样本 {line_count} 条,仍需继续观察或下钻节点现场日志。" ) source = "remote_log_sync_waiting_sample" elif participating_total > 0: status = "ready" status_label = "可下钻" summary = ( f"远端日志回传已经形成样本,当前覆盖 {covered_total}/{participating_total} 台参与节点," "可以直接下钻节点现场日志。" ) source = "remote_log_sync_ready" focus_ref = ( _build_scene_node_log_focus_ref( target_node_code, mode=mode, limit=120 if status == "waiting_sample" else 80, source=source, ) if target_node_code else {} ) return { "status": status, "status_label": status_label, "summary": summary, "enabled": enabled, "mode": mode, "participating_node_count": participating_total, "covered_participating_node_count": covered_total, "missing_participating_node_count": missing_total, "line_count": line_count, "source_node_count": source_node_count, "target_node_code": target_node_code, "target_node_codes": preferred_node_codes, "missing_node_codes": missing_node_codes, "source_node_codes": source_node_codes, "last_at": str(log_sync.get("last_at") or "").strip(), "focus_ref": focus_ref, "contract_keys": _normalize_ops_contract_keys( ["ops_observability_contract", "ops_stack_diagnosis_contract"] ), "contract_navigation": _build_ops_contract_navigation( ["ops_observability_contract", "ops_stack_diagnosis_contract"], primary_contract_key="ops_observability_contract", registry=get_ops_contract_registry(), ), } def _build_driver_action_request_preview( *, executor_kind: str, action_code: str, action_payload: dict | None = None, node_codes: list[str] | None = None, sequence_key: str = "", secondary: bool = False, ) -> dict: normalized_executor_kind = str(executor_kind or "").strip() normalized_action_code = str(action_code or "").strip() normalized_action_payload = dict(action_payload or {}) normalized_node_codes = _normalize_driver_node_codes(node_codes or []) normalized_sequence_key = str(sequence_key or "").strip() if normalized_executor_kind == "runbook_sequence" and normalized_sequence_key: return { "sequence_key": normalized_sequence_key, "secondary": bool(secondary), "action_payload": normalized_action_payload, } return { "action_code": normalized_action_code, "node_codes": normalized_node_codes, "action_payload": normalized_action_payload, } def _driver_action_missing_fields( action_code: str, *, action_payload: dict | None = None, node_codes: list[str] | None = None, ) -> list[str]: normalized_action_code = str(action_code or "").strip() normalized_action_payload = dict(action_payload or {}) normalized_node_codes = _normalize_driver_node_codes(node_codes or []) missing_fields: list[str] = [] if normalized_action_code == "open_playbook_dialog" and not str(normalized_action_payload.get("playbook_key") or "").strip(): missing_fields.append("playbook_key") if normalized_action_code == "open_action_template_dialog" and not str(normalized_action_payload.get("template_key") or "").strip(): missing_fields.append("template_key") if normalized_action_code in {"replay_delivery_queue", "flush_delivery_queue"}: target_node_codes = _normalize_driver_node_codes(normalized_action_payload.get("target_node_codes") or normalized_node_codes) if not target_node_codes: missing_fields.append("target_node_codes") if normalized_action_code in {"focus_playbook_run", "open_playbook_run_latest_events"} and not str( normalized_action_payload.get("run_code") or "" ).strip(): missing_fields.append("run_code") if normalized_action_code == "focus_activity_item" and not str( (normalized_action_payload.get("ui_intent") or {}).get("kind") or "" ).strip(): missing_fields.append("ui_intent.kind") if normalized_action_code == "focus_latest_job_events": has_ui_intent = bool(str((normalized_action_payload.get("ui_intent") or {}).get("kind") or "").strip()) source_focus_ref = _normalize_focus_ref(normalized_action_payload.get("source_focus_ref")) focus_ref = _normalize_focus_ref(normalized_action_payload.get("focus_ref")) has_event_focus = str(source_focus_ref.get("kind") or "").strip() == "ops_job_event" has_job_focus = str(focus_ref.get("kind") or "").strip() in {"ops_job", "ops_job_event"} if not (has_ui_intent or has_event_focus or has_job_focus): missing_fields.append("ui_intent.kind|focus_ref") if normalized_action_code in {"handover_first_gap", "view_first_gap"} and not str( normalized_action_payload.get("node_code") or (normalized_node_codes[0] if normalized_node_codes else "") ).strip(): missing_fields.append("node_code") if normalized_action_code in { "open_worker_logs_participating", "open_worker_logs_standby", "open_worker_logs", "open_diagnostics_participating", "open_diagnostics", "run_inspection_participating", "run_inspection_standby", "run_standard_inspection", } and not normalized_node_codes: missing_fields.append("node_codes") if normalized_action_code in {"create_release_rollout_worker", "create_release_rollout_control"} and int( normalized_action_payload.get("release_id") or 0 ) <= 0: missing_fields.append("release_id") return missing_fields def _build_driver_action_execution_profile( action_code: str, *, action_payload: dict | None = None, node_codes: list[str] | None = None, ) -> dict: normalized_action_code = str(action_code or "").strip() normalized_action_payload = dict(action_payload or {}) normalized_node_codes = _normalize_driver_node_codes(node_codes or []) missing_fields = _driver_action_missing_fields( normalized_action_code, action_payload=normalized_action_payload, node_codes=normalized_node_codes, ) base_profile = { "action_code": normalized_action_code, "backend_handled": normalized_action_code in _BACKEND_DRIVER_ACTION_CODES, "automation_level": "blocked", "recommendation": "blocked", "executor_mode_hint": "manual-or-ui", "risk_level": "unknown", "confirm_required": False, "ui_only": False, "blocked": False, "requires_fields": [], "missing_fields": missing_fields, "reason": "", } if not normalized_action_code: return { **base_profile, "blocked": True, "reason": "当前条目还没有解析出 driver action,暂时不能自动执行。", } if missing_fields: return { **base_profile, "blocked": True, "requires_fields": list(dict.fromkeys(missing_fields)), "reason": f"当前动作缺少必填上下文:{', '.join(list(dict.fromkeys(missing_fields)))}。", } if normalized_action_code not in _BACKEND_DRIVER_ACTION_CODES: return { **base_profile, "automation_level": "ui_only", "recommendation": "open_ui", "executor_mode_hint": "manual-or-ui", "risk_level": "medium", "ui_only": True, "reason": "当前动作尚未后端化,通常需要页面交互或人工确认后再推进。", } if normalized_action_code in _SAFE_AUTO_DRIVER_ACTION_CODES: executor_mode_hint = "settings" if normalized_action_code.startswith(("enable_", "disable_")) else "ops-playbook" reason = ( "当前动作只会调整运行时日志回传开关,属于低风险设置变更。" if executor_mode_hint == "settings" else "当前动作会走标准 playbook / 巡检链路,属于可自动执行的观测与诊断动作。" ) return { **base_profile, "automation_level": "safe_auto", "recommendation": "auto_execute", "executor_mode_hint": executor_mode_hint, "risk_level": "low", "reason": reason, } if normalized_action_code in _GUARDED_AUTO_DRIVER_ACTION_CODES: return { **base_profile, "automation_level": "guarded_auto", "recommendation": "confirm_then_execute", "executor_mode_hint": "smart-release-rollout", "risk_level": "high", "confirm_required": True, "reason": "当前动作会直接创建 Release / Rollout 或推进发布批次,适合在 Codex 或人工确认后执行。", } if normalized_action_code in _MIXED_DRIVER_ACTION_CODES: return { **base_profile, "automation_level": "mixed", "recommendation": "resolve_first", "executor_mode_hint": "cluster-sync-or-ui", "risk_level": "medium", "reason": "当前动作会先判断接管缺口,可能转为补接入页面,也可能直接同步托管节点,执行前最好先复核解析结果。", } if normalized_action_code in _UI_ONLY_DRIVER_ACTION_CODES: return { **base_profile, "automation_level": "ui_only", "recommendation": "open_ui", "executor_mode_hint": "ui-intent", "risk_level": "low", "ui_only": True, "reason": "当前动作主要用于打开页面、聚焦详情或进入模板,不适合作为无界面自动执行动作。", } return { **base_profile, "automation_level": "ui_only", "recommendation": "open_ui", "executor_mode_hint": "manual-or-ui", "risk_level": "medium", "ui_only": True, "reason": "当前动作暂时只能作为驾驶建议展示,建议先进入对应工作区再决定下一步。", } def _driver_action_executor_kind_label(executor_kind: str) -> str: normalized_executor_kind = str(executor_kind or "").strip() if normalized_executor_kind == "runbook_sequence": return "runbook sequence" return normalized_executor_kind or "driver action" def _driver_action_execution_chain(executor_kind: str) -> str: normalized_executor_kind = str(executor_kind or "").strip() if normalized_executor_kind == "runbook_sequence": return "ops-driver -> runbook-sequence -> resolved driver action / ui-intent" return "ops-driver -> driver-action execute -> playbook / ui-intent / settings" def _driver_action_target_api(*, executor_kind: str, sequence_key: str = "") -> str: normalized_executor_kind = str(executor_kind or "").strip() normalized_sequence_key = str(sequence_key or "").strip() if normalized_executor_kind == "runbook_sequence" and normalized_sequence_key: return f"/api/v1/ops/runbook/sequences/{normalized_sequence_key}/execute" return "/api/v1/ops/driver-actions/execute" def _driver_action_target_method(*, executor_kind: str, sequence_key: str = "") -> str: normalized_executor_kind = str(executor_kind or "").strip() normalized_sequence_key = str(sequence_key or "").strip() if normalized_executor_kind == "runbook_sequence" and normalized_sequence_key: return "POST" return "POST" def _driver_action_recommendation_label(recommendation: str) -> str: normalized_recommendation = str(recommendation or "").strip() return { "auto_execute": "可自动执行", "confirm_then_execute": "建议确认后执行", "resolve_first": "先复核再执行", "open_ui": "需要进入工作区", "blocked": "当前阻断", }.get(normalized_recommendation, normalized_recommendation or "待判断") def _build_driver_action_contract_request_payload( *, executor_kind: str, action_code: str, action_payload: dict | None = None, node_codes: list[str] | None = None, sequence_key: str = "", focus_ref: dict | None = None, secondary: bool = False, requested_by: str = "api/preview", ) -> dict: normalized_executor_kind = str(executor_kind or "").strip() normalized_action_code = str(action_code or "").strip() normalized_action_payload = _action_payload_with_focus_ref(action_payload or {}, focus_ref) normalized_node_codes = _normalize_driver_node_codes(node_codes or []) normalized_sequence_key = str(sequence_key or "").strip() normalized_requested_by = str(requested_by or "api/preview").strip() or "api/preview" if normalized_executor_kind == "runbook_sequence" and normalized_sequence_key: payload = { "secondary": bool(secondary), "requested_by": normalized_requested_by, } if normalized_node_codes: payload["node_codes"] = normalized_node_codes if normalized_action_payload: payload["action_payload"] = normalized_action_payload return payload return { "action_code": normalized_action_code, "node_codes": normalized_node_codes, "action_payload": normalized_action_payload, "requested_by": normalized_requested_by, } def _build_driver_action_contract_preview_section( *, executor_kind: str, action_code: str, label: str = "", action_payload: dict | None = None, node_codes: list[str] | None = None, sequence_key: str = "", focus_ref: dict | None = None, secondary: bool = False, requested_by: str = "api/preview", reason: str = "", ) -> dict: normalized_executor_kind = str(executor_kind or "").strip() or "driver_action" normalized_action_code = str(action_code or "").strip() normalized_label = str(label or "").strip() normalized_action_payload = dict(action_payload or {}) normalized_node_codes = _normalize_driver_node_codes(node_codes or []) normalized_sequence_key = str(sequence_key or "").strip() execution_profile = _build_driver_action_execution_profile( normalized_action_code, action_payload=normalized_action_payload, node_codes=normalized_node_codes, ) return { "label": normalized_label or ("次动作" if secondary else "主动作"), "action_code": normalized_action_code, "node_codes": normalized_node_codes, "focus_ref": _normalize_focus_ref(focus_ref), "target_api": _driver_action_target_api( executor_kind=normalized_executor_kind, sequence_key=normalized_sequence_key, ), "target_method": _driver_action_target_method( executor_kind=normalized_executor_kind, sequence_key=normalized_sequence_key, ), "reason": str(reason or execution_profile.get("reason") or "").strip(), "recommendation_label": _driver_action_recommendation_label(execution_profile.get("recommendation") or ""), "automation_level_label": str(execution_profile.get("automation_level") or "").strip(), "risk_level_label": str(execution_profile.get("risk_level") or "").strip(), "request_payload": _build_driver_action_contract_request_payload( executor_kind=normalized_executor_kind, action_code=normalized_action_code, action_payload=normalized_action_payload, node_codes=normalized_node_codes, sequence_key=normalized_sequence_key, focus_ref=_normalize_focus_ref(focus_ref), secondary=secondary, requested_by=requested_by, ), "execution_profile": execution_profile, } def preview_driver_action(payload: dict) -> tuple[bool, str, dict]: normalized_payload = dict(payload or {}) source_label = str(normalized_payload.get("source_label") or "ops-center").strip() or "ops-center" title = str(normalized_payload.get("title") or "").strip() or "驾驶动作" summary = str(normalized_payload.get("summary") or "").strip() reason = str(normalized_payload.get("reason") or "").strip() meta_text = str(normalized_payload.get("meta_text") or "").strip() executor_kind = str(normalized_payload.get("executor_kind") or "driver_action").strip() or "driver_action" sequence_key = str(normalized_payload.get("sequence_key") or "").strip() requested_by = str(normalized_payload.get("requested_by") or "api/preview").strip() or "api/preview" contract_registry = get_ops_contract_registry() contract = dict((contract_registry.get("contracts_by_key") or {}).get("ops_driver_contract") or {}) if executor_kind == "runbook_sequence": if not sequence_key: return False, "sequence_key 不能为空", {} include_secondary = bool(normalized_payload.get("include_secondary", True)) primary_ok, primary_message, primary_resolved = resolve_ops_runbook_sequence( sequence_key, { "requested_by": requested_by, "secondary": False, "node_codes": _normalize_driver_node_codes(normalized_payload.get("primary_node_codes") or []), "action_payload": dict(normalized_payload.get("primary_action_payload") or {}), }, ) if not primary_ok: return False, primary_message, dict(primary_resolved or {}) primary_label = str(normalized_payload.get("primary_label") or primary_resolved.get("driver_action_label") or "").strip() secondary_label = str(normalized_payload.get("secondary_label") or "").strip() secondary_action_code = str(normalized_payload.get("secondary_action_code") or "").strip() preview_focus_ref = _normalize_focus_ref( normalized_payload.get("focus_ref") or (primary_resolved.get("sequence") or {}).get("focus_ref") ) secondary_preview: dict = {} if include_secondary or secondary_label or secondary_action_code: secondary_ok, secondary_message, secondary_resolved = resolve_ops_runbook_sequence( sequence_key, { "requested_by": requested_by, "secondary": True, "node_codes": _normalize_driver_node_codes(normalized_payload.get("secondary_node_codes") or []), "action_payload": dict(normalized_payload.get("secondary_action_payload") or {}), }, ) if secondary_ok: secondary_preview = _build_driver_action_contract_preview_section( executor_kind=executor_kind, action_code=str(secondary_resolved.get("driver_action_code") or "").strip(), label=str(secondary_label or secondary_resolved.get("driver_action_label") or "").strip(), action_payload=dict(secondary_resolved.get("driver_action_payload") or {}), node_codes=_normalize_driver_node_codes(secondary_resolved.get("driver_node_codes") or []), sequence_key=sequence_key, focus_ref=_normalize_focus_ref( normalized_payload.get("secondary_focus_ref") or (secondary_resolved.get("sequence") or {}).get("action_focus_ref") ), secondary=True, requested_by=requested_by, reason=reason, ) elif secondary_label or secondary_action_code: return False, secondary_message, dict(secondary_resolved or {}) return True, "驾驶动作预览已生成", { "contract_key": "ops_driver_contract", "contract_version": str(contract.get("version") or _OPS_CONTRACT_SCHEMA_VERSION).strip(), "contract_schema_doc_path": str(contract.get("schema_doc_path") or "").strip(), "contract_primary_endpoint": str(contract.get("primary_endpoint") or "").strip(), "preview_endpoint": "/api/v1/ops/driver-actions/preview", "source_label": source_label, "title": title, "summary": summary, "reason": reason, "meta_text": meta_text, "executor_kind": executor_kind, "executor_kind_label": _driver_action_executor_kind_label(executor_kind), "sequence_key": sequence_key, "focus_ref": preview_focus_ref, "primary_focus_ref": _normalize_focus_ref( normalized_payload.get("primary_focus_ref") or (primary_resolved.get("sequence") or {}).get("action_focus_ref") ), "secondary_focus_ref": _normalize_focus_ref( normalized_payload.get("secondary_focus_ref") or (secondary_preview.get("focus_ref") or {}) ), "execution_chain": _driver_action_execution_chain(executor_kind), "primary": _build_driver_action_contract_preview_section( executor_kind=executor_kind, action_code=str(primary_resolved.get("driver_action_code") or "").strip(), label=primary_label, action_payload=dict(primary_resolved.get("driver_action_payload") or {}), node_codes=_normalize_driver_node_codes(primary_resolved.get("driver_node_codes") or []), sequence_key=sequence_key, focus_ref=_normalize_focus_ref( normalized_payload.get("primary_focus_ref") or (primary_resolved.get("sequence") or {}).get("action_focus_ref") ), secondary=False, requested_by=requested_by, reason=reason, ), "secondary": secondary_preview, "resolved_primary": dict(primary_resolved or {}), } primary_action_code = str(normalized_payload.get("primary_action_code") or "").strip() secondary_action_code = str(normalized_payload.get("secondary_action_code") or "").strip() if not primary_action_code and not secondary_action_code: return False, "当前没有可预览的驾驶动作", {} preview_focus_ref = _normalize_focus_ref(normalized_payload.get("focus_ref")) primary_focus_ref = _normalize_focus_ref(normalized_payload.get("primary_focus_ref")) secondary_focus_ref = _normalize_focus_ref(normalized_payload.get("secondary_focus_ref")) return True, "驾驶动作预览已生成", { "contract_key": "ops_driver_contract", "contract_version": str(contract.get("version") or _OPS_CONTRACT_SCHEMA_VERSION).strip(), "contract_schema_doc_path": str(contract.get("schema_doc_path") or "").strip(), "contract_primary_endpoint": str(contract.get("primary_endpoint") or "").strip(), "preview_endpoint": "/api/v1/ops/driver-actions/preview", "source_label": source_label, "title": title, "summary": summary, "reason": reason, "meta_text": meta_text, "executor_kind": executor_kind, "executor_kind_label": _driver_action_executor_kind_label(executor_kind), "sequence_key": sequence_key, "focus_ref": preview_focus_ref, "primary_focus_ref": primary_focus_ref, "secondary_focus_ref": secondary_focus_ref, "execution_chain": _driver_action_execution_chain(executor_kind), "primary": ( _build_driver_action_contract_preview_section( executor_kind=executor_kind, action_code=primary_action_code, label=str(normalized_payload.get("primary_label") or "").strip(), action_payload=dict(normalized_payload.get("primary_action_payload") or {}), node_codes=_normalize_driver_node_codes(normalized_payload.get("primary_node_codes") or normalized_payload.get("node_codes") or []), sequence_key=sequence_key, focus_ref=primary_focus_ref, secondary=False, requested_by=requested_by, reason=reason, ) if primary_action_code else {} ), "secondary": ( _build_driver_action_contract_preview_section( executor_kind=executor_kind, action_code=secondary_action_code, label=str(normalized_payload.get("secondary_label") or "").strip(), action_payload=dict(normalized_payload.get("secondary_action_payload") or {}), node_codes=_normalize_driver_node_codes(normalized_payload.get("secondary_node_codes") or normalized_payload.get("node_codes") or []), sequence_key=sequence_key, focus_ref=secondary_focus_ref, secondary=True, requested_by=requested_by, reason=reason, ) if secondary_action_code else {} ), } def _build_codex_brief_entry(entry: dict) -> dict: normalized_entry = dict(entry or {}) if not normalized_entry: return {} executor_kind = str(normalized_entry.get("executor_kind") or "driver_action").strip() or "driver_action" sequence_key = str(normalized_entry.get("sequence_key") or "").strip() primary_action_code = str(normalized_entry.get("primary_action_code") or "").strip() secondary_action_code = str(normalized_entry.get("secondary_action_code") or "").strip() primary_node_codes = _normalize_driver_node_codes(normalized_entry.get("primary_node_codes") or normalized_entry.get("node_codes") or []) secondary_node_codes = _normalize_driver_node_codes(normalized_entry.get("secondary_node_codes") or normalized_entry.get("node_codes") or []) primary_action_payload = dict(normalized_entry.get("primary_action_payload") or {}) secondary_action_payload = dict(normalized_entry.get("secondary_action_payload") or {}) primary_execution = _build_driver_action_execution_profile( primary_action_code, action_payload=primary_action_payload, node_codes=primary_node_codes, ) secondary_execution = ( _build_driver_action_execution_profile( secondary_action_code, action_payload=secondary_action_payload, node_codes=secondary_node_codes, ) if secondary_action_code else {} ) if executor_kind == "runbook_sequence" and sequence_key: target_api = f"/api/v1/ops/runbook/sequences/{sequence_key}/execute" target_method = "POST" else: target_api = "/api/v1/ops/driver-actions/execute" target_method = "POST" primary_request = _build_driver_action_request_preview( executor_kind=executor_kind, action_code=primary_action_code, action_payload=primary_action_payload, node_codes=primary_node_codes, sequence_key=sequence_key, secondary=False, ) secondary_request = ( _build_driver_action_request_preview( executor_kind=executor_kind, action_code=secondary_action_code, action_payload=secondary_action_payload, node_codes=secondary_node_codes, sequence_key=sequence_key, secondary=True, ) if secondary_action_code else {} ) recommendation = str(primary_execution.get("recommendation") or "blocked").strip() or "blocked" automation_level = str(primary_execution.get("automation_level") or "blocked").strip() or "blocked" executor_mode_hint = str(primary_execution.get("executor_mode_hint") or "").strip() status_text = { "auto_execute": "可自动执行", "confirm_then_execute": "建议确认后执行", "resolve_first": "先复核再执行", "open_ui": "需要进入工作区", "blocked": "当前阻断", }.get(recommendation, "待判断") contract_keys = _suggest_ops_contract_keys_for_codex_entry(normalized_entry) return { "key": str(normalized_entry.get("key") or "").strip(), "kind": str(normalized_entry.get("kind") or "").strip(), "lane": str(normalized_entry.get("lane") or "").strip(), "lane_label": str(normalized_entry.get("lane_label") or "").strip(), "title": str(normalized_entry.get("title") or "").strip(), "summary": str(normalized_entry.get("summary") or "").strip(), "reason": str(normalized_entry.get("reason") or "").strip(), "status": str(normalized_entry.get("status") or "").strip(), "level_label": str(normalized_entry.get("level_label") or "").strip(), "tag_type": str(normalized_entry.get("tag_type") or "").strip(), "meta_text": str(normalized_entry.get("meta_text") or "").strip(), "detail_lines": list(normalized_entry.get("detail_lines") or []), "node_codes": _normalize_driver_node_codes(normalized_entry.get("node_codes") or []), "executor_kind": executor_kind, "sequence_key": sequence_key, "focus_ref": _normalize_focus_ref(normalized_entry.get("focus_ref")), "primary_focus_ref": _normalize_focus_ref(normalized_entry.get("primary_focus_ref")), "secondary_focus_ref": _normalize_focus_ref(normalized_entry.get("secondary_focus_ref")), "occurred_at": str(normalized_entry.get("occurred_at") or "").strip(), "primary_label": str(normalized_entry.get("primary_label") or "").strip(), "secondary_label": str(normalized_entry.get("secondary_label") or "").strip(), "primary_action_code": primary_action_code, "secondary_action_code": secondary_action_code, "contract_keys": contract_keys, "contract_navigation": _build_ops_contract_navigation( contract_keys, primary_contract_key="ops_driver_contract", ), "autopilot": { "status_text": status_text, "automation_level": automation_level, "recommendation": recommendation, "executor_mode_hint": executor_mode_hint, "confirm_required": bool(primary_execution.get("confirm_required", False)), "ui_only": bool(primary_execution.get("ui_only", False)), "blocked": bool(primary_execution.get("blocked", False)), "backend_handled": bool(primary_execution.get("backend_handled", False)), "risk_level": str(primary_execution.get("risk_level") or "unknown").strip() or "unknown", "reason": str(primary_execution.get("reason") or "").strip(), "target_method": target_method, "target_api": target_api, }, "primary_execution": { **primary_execution, "label": str(normalized_entry.get("primary_label") or "").strip() or primary_action_code, "node_codes": primary_node_codes, "action_payload": primary_action_payload, "request_payload_preview": primary_request, }, "secondary_execution": ( { **secondary_execution, "label": str(normalized_entry.get("secondary_label") or "").strip() or secondary_action_code, "node_codes": secondary_node_codes, "action_payload": secondary_action_payload, "request_payload_preview": secondary_request, } if secondary_action_code else {} ), } def _build_codex_activity_focus_entry(entry: dict) -> dict: normalized_entry = dict(entry or {}) if not normalized_entry: return {} contract_keys = _suggest_ops_contract_keys_for_codex_entry(normalized_entry) return { "key": str(normalized_entry.get("key") or "").strip(), "activity_key": str(normalized_entry.get("activity_key") or "").strip(), "activity_kind": str(normalized_entry.get("activity_kind") or "").strip(), "title": str(normalized_entry.get("title") or "").strip(), "subtitle": str(normalized_entry.get("subtitle") or "").strip(), "summary": str(normalized_entry.get("summary") or "").strip(), "status": str(normalized_entry.get("status") or "").strip(), "status_label": str(normalized_entry.get("status_label") or "").strip(), "meta_text": str(normalized_entry.get("meta_text") or "").strip(), "occurred_at": str(normalized_entry.get("occurred_at") or "").strip(), "target_node_codes": _normalize_driver_node_codes(normalized_entry.get("target_node_codes") or []), "focus_ref": _normalize_focus_ref(normalized_entry.get("focus_ref")), "source_focus_ref": _normalize_focus_ref(normalized_entry.get("source_focus_ref")), "ui_intent": dict(normalized_entry.get("ui_intent") or {}), "ui_intent_kind": str(normalized_entry.get("ui_intent_kind") or "").strip(), "focus_action_code": str(normalized_entry.get("focus_action_code") or "").strip(), "focus_action_payload": dict(normalized_entry.get("focus_action_payload") or {}), "contract_keys": contract_keys, "contract_navigation": _build_ops_contract_navigation( contract_keys, primary_contract_key="ops_driver_contract", ), "observation_only": True, } def _build_codex_action_entry_from_activity_focus(entry: dict) -> dict: normalized_entry = dict(entry or {}) if not normalized_entry: return {} key = str(normalized_entry.get("key") or "").strip() focus_action_code = str(normalized_entry.get("focus_action_code") or "").strip() focus_ref = _normalize_focus_ref(normalized_entry.get("focus_ref")) target_node_codes = _normalize_driver_node_codes(normalized_entry.get("target_node_codes") or []) focus_action_payload = dict(normalized_entry.get("focus_action_payload") or {}) status_text = str(normalized_entry.get("status_label") or normalized_entry.get("status") or "").strip() summary = str(normalized_entry.get("summary") or "").strip() if not key or not focus_action_code: return {} reason = summary or "该焦点属于运行态观察项,建议先进入对应工作区查看上下文。" return { "key": key, "title": str(normalized_entry.get("title") or "").strip() or key, "summary": summary, "reason": reason, "meta_text": str(normalized_entry.get("meta_text") or "").strip(), "executor_kind": "driver_action", "sequence_key": "", "focus_ref": focus_ref, "primary_focus_ref": focus_ref, "secondary_focus_ref": {}, "primary_label": "进入工作区", "secondary_label": "", "primary_action_code": focus_action_code, "secondary_action_code": "", "autopilot": { "status_text": status_text, "automation_level": "ui_only", "recommendation": "open_ui", "executor_mode_hint": "ui-intent", "confirm_required": False, "ui_only": True, "blocked": False, "backend_handled": True, "risk_level": "low", "reason": reason, "target_method": "POST", "target_api": "/api/v1/ops/driver-actions/execute", }, "primary_execution": { "label": "进入工作区", "node_codes": target_node_codes, "action_payload": focus_action_payload, "request_payload_preview": { "action_code": focus_action_code, "node_codes": target_node_codes, "action_payload": focus_action_payload, }, }, "secondary_execution": {}, "activity_key": str(normalized_entry.get("activity_key") or "").strip(), "activity_kind": str(normalized_entry.get("activity_kind") or "").strip(), "status": str(normalized_entry.get("status") or "").strip(), "status_label": status_text, "occurred_at": str(normalized_entry.get("occurred_at") or "").strip(), "target_node_codes": target_node_codes, "source_focus_ref": _normalize_focus_ref(normalized_entry.get("source_focus_ref")), "ui_intent": dict(normalized_entry.get("ui_intent") or {}), "ui_intent_kind": str(normalized_entry.get("ui_intent_kind") or "").strip(), "focus_action_code": focus_action_code, "focus_action_payload": focus_action_payload, "observation_only": True, } def _is_preview_only_driver_action(action_code: str) -> bool: normalized_action_code = str(action_code or "").strip() if not normalized_action_code: return False if normalized_action_code.startswith(("open_", "focus_", "view_")): return True return normalized_action_code in { "release_package", "handover_first_gap", "review_smart_rollout_preview", "review_control_rollout", "fix_rollout_blockers", } def _build_ops_automation_coverage_summary( entries: list[dict] | None = None, *, activity_focus_total: int = 0, scene_log_observation: dict | None = None, go_live_summary: dict | None = None, ) -> dict: normalized_entries = [dict(item or {}) for item in list(entries or []) if dict(item or {})] normalized_scene_log_observation = dict(scene_log_observation or {}) normalized_go_live_summary = dict(go_live_summary or {}) automation_level_counts: dict[str, int] = {} recommendation_counts: dict[str, int] = {} executor_kind_counts: dict[str, int] = {} backend_handled_total = 0 preview_only_total = 0 for item in normalized_entries: autopilot = dict(item.get("autopilot") or {}) automation_level = str(autopilot.get("automation_level") or "unknown").strip() or "unknown" recommendation = str(autopilot.get("recommendation") or "unknown").strip() or "unknown" executor_kind = str(item.get("executor_kind") or "unknown").strip() or "unknown" action_code = str(item.get("primary_action_code") or "").strip() if bool(autopilot.get("backend_handled", False)): backend_handled_total += 1 if _is_preview_only_driver_action(action_code): preview_only_total += 1 automation_level_counts[automation_level] = int(automation_level_counts.get(automation_level, 0) or 0) + 1 recommendation_counts[recommendation] = int(recommendation_counts.get(recommendation, 0) or 0) + 1 executor_kind_counts[executor_kind] = int(executor_kind_counts.get(executor_kind, 0) or 0) + 1 total = len(normalized_entries) safe_auto_total = int(automation_level_counts.get("safe_auto", 0) or 0) guarded_auto_total = int(automation_level_counts.get("guarded_auto", 0) or 0) mixed_total = int(automation_level_counts.get("mixed", 0) or 0) ui_only_total = int(automation_level_counts.get("ui_only", 0) or 0) blocked_total = int(automation_level_counts.get("blocked", 0) or 0) unknown_total = max(total - safe_auto_total - guarded_auto_total - mixed_total - ui_only_total - blocked_total, 0) auto_execute_total = int(recommendation_counts.get("auto_execute", 0) or 0) confirm_then_execute_total = int(recommendation_counts.get("confirm_then_execute", 0) or 0) resolve_first_total = int(recommendation_counts.get("resolve_first", 0) or 0) open_ui_total = int(recommendation_counts.get("open_ui", 0) or 0) blocked_recommendation_total = int(recommendation_counts.get("blocked", 0) or 0) execution_ready_total = auto_execute_total + confirm_then_execute_total human_dependency_total = resolve_first_total + open_ui_total + blocked_recommendation_total coverage_ratio = round((backend_handled_total / total), 4) if total > 0 else 1.0 launch_status = "ready" go_live_status = str(normalized_go_live_summary.get("go_live_status") or "").strip() publish_ready = bool(normalized_go_live_summary.get("publish_ready", False)) scene_log_status = str(normalized_scene_log_observation.get("status") or "").strip() or "standby" if blocked_total > 0 or blocked_recommendation_total > 0 or go_live_status == "blocked": launch_status = "blocked" elif ( ui_only_total > 0 or resolve_first_total > 0 or go_live_status not in {"", "ready"} or not publish_ready ): launch_status = "attention" return { "total": total, "automation_level_counts": automation_level_counts, "recommendation_counts": recommendation_counts, "executor_kind_counts": executor_kind_counts, "safe_auto_total": safe_auto_total, "guarded_auto_total": guarded_auto_total, "mixed_total": mixed_total, "ui_only_total": ui_only_total, "blocked_total": blocked_total, "unknown_total": unknown_total, "auto_execute_total": auto_execute_total, "confirm_then_execute_total": confirm_then_execute_total, "resolve_first_total": resolve_first_total, "open_ui_total": open_ui_total, "blocked_recommendation_total": blocked_recommendation_total, "backend_handled_total": backend_handled_total, "preview_only_total": preview_only_total, "execution_ready_total": execution_ready_total, "human_dependency_total": human_dependency_total, "activity_focus_total": int(activity_focus_total or 0), "scene_log_status": scene_log_status, "launch_status": launch_status, "launch_ready": launch_status == "ready", "backend_coverage_ratio": coverage_ratio, } def get_ops_codex_brief() -> dict: driver_feed = get_ops_driver_feed() go_live_summary = dict(driver_feed.get("go_live_summary") or {}) contract_registry = get_ops_contract_registry() scene_log_observation = dict(driver_feed.get("scene_log_observation") or {}) activity_focus_entries = [ _build_codex_activity_focus_entry(raw_entry) for raw_entry in list(driver_feed.get("activity_focus") or []) if dict(raw_entry or {}) ] activity_focus_entries = [entry for entry in activity_focus_entries if entry] entries = [ _build_codex_brief_entry(raw_entry) for raw_entry in list(driver_feed.get("entries") or []) if dict(raw_entry or {}) ] entries = [entry for entry in entries if entry] automation_coverage = _build_ops_automation_coverage_summary( entries, activity_focus_total=len(activity_focus_entries), scene_log_observation=scene_log_observation, go_live_summary=go_live_summary, ) focus_entry = dict(entries[0] or {}) if entries else {} focus_autopilot = dict(focus_entry.get("autopilot") or {}) focus_payload = { "entry_key": str(focus_entry.get("key") or "").strip(), "title": str(focus_entry.get("title") or "").strip(), "recommendation": str(focus_autopilot.get("recommendation") or "").strip(), "status_text": str(focus_autopilot.get("status_text") or "").strip(), "target_api": str(focus_autopilot.get("target_api") or "").strip(), "target_method": str(focus_autopilot.get("target_method") or "").strip(), "focus_ref": _normalize_focus_ref(focus_entry.get("focus_ref")), "primary_focus_ref": _normalize_focus_ref(focus_entry.get("primary_focus_ref")), "secondary_focus_ref": _normalize_focus_ref(focus_entry.get("secondary_focus_ref")), } if not focus_payload["entry_key"]: first_activity_focus = next( ( dict(item or {}) for item in activity_focus_entries if str((item or {}).get("focus_action_code") or "").strip() ), {}, ) else: first_activity_focus = {} if not focus_payload["entry_key"] and first_activity_focus: activity_focus_contract_keys = _suggest_ops_contract_keys_for_codex_entry(first_activity_focus) focus_payload = { "entry_key": str(first_activity_focus.get("key") or "").strip(), "title": str(first_activity_focus.get("title") or "").strip(), "recommendation": "open_ui" if str(first_activity_focus.get("ui_intent_kind") or "").strip() else "", "status_text": str(first_activity_focus.get("status_label") or first_activity_focus.get("status") or "").strip(), "target_api": "", "target_method": "", "focus_ref": _normalize_focus_ref(first_activity_focus.get("focus_ref")), "primary_focus_ref": _normalize_focus_ref(first_activity_focus.get("focus_ref")), "secondary_focus_ref": {}, "contract_keys": activity_focus_contract_keys, "contract_navigation": _build_ops_contract_navigation( activity_focus_contract_keys, primary_contract_key="ops_driver_contract", registry=contract_registry, ), } elif ( not focus_payload["entry_key"] and _normalize_focus_ref(scene_log_observation.get("focus_ref")) ): scene_log_contract_keys = _normalize_ops_contract_keys( list(scene_log_observation.get("contract_keys") or []) + ["ops_driver_contract"] ) focus_payload = { "entry_key": "scene-log-observation", "title": "现场日志观察", "recommendation": "open_ui", "status_text": str(scene_log_observation.get("status_label") or scene_log_observation.get("status") or "").strip(), "target_api": "", "target_method": "", "focus_ref": _normalize_focus_ref(scene_log_observation.get("focus_ref")), "primary_focus_ref": _normalize_focus_ref(scene_log_observation.get("focus_ref")), "secondary_focus_ref": {}, "contract_keys": scene_log_contract_keys, "contract_navigation": _build_ops_contract_navigation( scene_log_contract_keys, primary_contract_key="ops_driver_contract", registry=contract_registry, ), } elif focus_payload["entry_key"]: focus_contract_keys = _normalize_ops_contract_keys( (focus_entry.get("contract_keys") or []) + _suggest_ops_contract_keys_for_codex_entry(focus_entry) ) focus_payload["contract_keys"] = focus_contract_keys focus_payload["contract_navigation"] = _build_ops_contract_navigation( focus_contract_keys, primary_contract_key="ops_driver_contract", registry=contract_registry, ) else: focus_payload["contract_keys"] = ["ops_driver_contract"] focus_payload["contract_navigation"] = _build_ops_contract_navigation( ["ops_driver_contract"], primary_contract_key="ops_driver_contract", registry=contract_registry, ) summary_contract_keys = _normalize_ops_contract_keys( ["ops_driver_contract", "ops_stack_diagnosis_contract"] + [ contract_key for item in entries for contract_key in list(item.get("contract_keys") or []) ] + [ contract_key for contract_key in list(focus_payload.get("contract_keys") or []) ] ) return { "generated_at": datetime.now().isoformat(timespec="seconds"), "headline": str(driver_feed.get("headline") or "").strip() or "当前还没有可执行的驾驶主线。", "driver_feed": driver_feed, "go_live_summary": go_live_summary, "scene_log_observation": scene_log_observation, "focus": focus_payload, "entries": entries, "activity_focus": activity_focus_entries, "automation_coverage": automation_coverage, "contract_navigation": _build_ops_contract_navigation( summary_contract_keys, primary_contract_key="ops_driver_contract", registry=contract_registry, ), "summary": { "total": len(entries), "automation_level_counts": dict(automation_coverage.get("automation_level_counts") or {}), "recommendation_counts": dict(automation_coverage.get("recommendation_counts") or {}), "executor_kind_counts": dict(automation_coverage.get("executor_kind_counts") or {}), "go_live_status": str(go_live_summary.get("go_live_status") or "").strip(), "operator_title": str(go_live_summary.get("operator_title") or "").strip(), "publish_ready": bool(go_live_summary.get("publish_ready")), "publish_status": str(go_live_summary.get("publish_status") or "").strip(), "publish_status_label": str(go_live_summary.get("publish_status_label") or "").strip(), "launchpad_recommended_target_node_code": str( go_live_summary.get("launchpad_recommended_target_node_code") or "" ).strip(), "launchpad_recommended_recovery_label": str( go_live_summary.get("launchpad_recommended_recovery_label") or "" ).strip(), "launchpad_recommended_recovery_summary": str( go_live_summary.get("launchpad_recommended_recovery_summary") or "" ).strip(), "launchpad_onboarding_bootstrap_pending_nodes": int( go_live_summary.get("launchpad_onboarding_bootstrap_pending_nodes", 0) or 0 ), "launchpad_onboarding_acceptance_ready_nodes": int( go_live_summary.get("launchpad_onboarding_acceptance_ready_nodes", 0) or 0 ), "auto_execute_total": int(automation_coverage.get("auto_execute_total", 0) or 0), "confirm_then_execute_total": int(automation_coverage.get("confirm_then_execute_total", 0) or 0), "resolve_first_total": int(automation_coverage.get("resolve_first_total", 0) or 0), "open_ui_total": int(automation_coverage.get("open_ui_total", 0) or 0), "blocked_total": int(automation_coverage.get("blocked_recommendation_total", 0) or 0), "activity_focus_total": int(automation_coverage.get("activity_focus_total", 0) or 0), "preview_only_total": int(automation_coverage.get("preview_only_total", 0) or 0), "backend_handled_total": int(automation_coverage.get("backend_handled_total", 0) or 0), "launch_status": str(automation_coverage.get("launch_status") or "").strip(), "launch_ready": bool(automation_coverage.get("launch_ready", False)), }, } def _build_resolved_driver_action_gate( preview_data: dict, *, secondary: bool = False, confirm: bool = False, ) -> dict: normalized_preview = dict(preview_data or {}) preview_section = dict((normalized_preview.get("secondary") if secondary else normalized_preview.get("primary")) or {}) preview_execution = dict(preview_section.get("execution_profile") or {}) recommendation = str(preview_execution.get("recommendation") or "blocked").strip() or "blocked" automation_level = str(preview_execution.get("automation_level") or "blocked").strip() or "blocked" executor_mode_hint = str(preview_execution.get("executor_mode_hint") or "").strip() risk_level = str(preview_execution.get("risk_level") or "unknown").strip() or "unknown" confirm_required = bool(preview_execution.get("confirm_required", False)) ui_only = bool(preview_execution.get("ui_only", False)) blocked = bool(preview_execution.get("blocked", False)) reason = str(preview_execution.get("reason") or "").strip() target_api = str(preview_section.get("target_api") or "").strip() target_method = str(preview_section.get("target_method") or "POST").strip() or "POST" request_payload = dict(preview_section.get("request_payload") or {}) confirm_received = bool(confirm) decision = "blocked" decision_reason = reason or "当前动作被标记为阻断,不能自动执行。" will_execute = False if recommendation == "auto_execute": decision = "execute_now" will_execute = True decision_reason = reason or "当前动作属于 safe_auto,可直接执行。" elif recommendation == "confirm_then_execute": if confirm_received: decision = "execute_confirmed" will_execute = True decision_reason = reason or "当前动作属于 guarded_auto,已收到确认,允许执行。" else: decision = "confirmation_required" decision_reason = "当前动作属于 guarded_auto,需要显式确认后才能执行。" if reason: decision_reason = f"{decision_reason} {reason}" elif recommendation == "resolve_first": decision = "resolve_first" decision_reason = reason or "当前动作要求先复核解析结果,不应直接执行。" elif recommendation == "open_ui": decision = "open_ui" decision_reason = reason or "当前动作属于 UI-only,建议进入工作区处理。" decision_label = { "execute_now": "直接执行", "execute_confirmed": "确认后执行", "confirmation_required": "等待确认", "resolve_first": "先复核", "open_ui": "进入工作区", "blocked": "当前阻断", }.get(decision, decision) return { "decision": decision, "decision_label": decision_label, "decision_reason": decision_reason, "will_execute": will_execute, "recommendation": recommendation, "recommendation_label": _driver_action_recommendation_label(recommendation), "automation_level": automation_level, "status_text": "", "executor_mode_hint": executor_mode_hint, "risk_level": risk_level, "confirm_required": confirm_required, "confirm_received": confirm_received, "ui_only": ui_only, "blocked": blocked, "secondary": bool(secondary), "backend_handled": bool(preview_execution.get("backend_handled", False)), "target_api": target_api, "target_method": target_method, "request_payload": request_payload, "focus_ref": _normalize_focus_ref( preview_section.get("focus_ref") or normalized_preview.get("secondary_focus_ref" if secondary else "primary_focus_ref") or normalized_preview.get("focus_ref") ), "action_code": str(preview_section.get("action_code") or "").strip(), "sequence_key": str(normalized_preview.get("sequence_key") or "").strip(), "label": str(preview_section.get("label") or "").strip(), "node_codes": _normalize_driver_node_codes(preview_section.get("node_codes") or []), "requires_fields": list(preview_execution.get("requires_fields") or []), "missing_fields": list(preview_execution.get("missing_fields") or []), } def resolve_driver_action_request(payload: dict | None = None) -> tuple[bool, str, dict]: normalized_payload = dict(payload or {}) secondary = bool(normalized_payload.get("secondary", False)) confirm = bool(normalized_payload.get("confirm", False)) preview_request = { "source_label": str(normalized_payload.get("source_label") or "ops/driver-action").strip() or "ops/driver-action", "title": str(normalized_payload.get("title") or "").strip(), "summary": str(normalized_payload.get("summary") or "").strip(), "reason": str(normalized_payload.get("reason") or "").strip(), "meta_text": str(normalized_payload.get("meta_text") or "").strip(), "executor_kind": str(normalized_payload.get("executor_kind") or "driver_action").strip() or "driver_action", "sequence_key": str(normalized_payload.get("sequence_key") or "").strip(), "focus_ref": _normalize_focus_ref(normalized_payload.get("focus_ref")), "primary_focus_ref": _normalize_focus_ref(normalized_payload.get("primary_focus_ref")), "secondary_focus_ref": _normalize_focus_ref(normalized_payload.get("secondary_focus_ref")), "primary_label": str(normalized_payload.get("primary_label") or "").strip(), "secondary_label": str(normalized_payload.get("secondary_label") or "").strip(), "primary_action_code": str(normalized_payload.get("primary_action_code") or "").strip(), "secondary_action_code": str(normalized_payload.get("secondary_action_code") or "").strip(), "primary_node_codes": _normalize_driver_node_codes(normalized_payload.get("primary_node_codes") or normalized_payload.get("node_codes") or []), "secondary_node_codes": _normalize_driver_node_codes(normalized_payload.get("secondary_node_codes") or normalized_payload.get("node_codes") or []), "primary_action_payload": dict(normalized_payload.get("primary_action_payload") or normalized_payload.get("action_payload") or {}), "secondary_action_payload": dict(normalized_payload.get("secondary_action_payload") or normalized_payload.get("action_payload") or {}), "include_secondary": bool(normalized_payload.get("include_secondary", True)), "requested_by": str(normalized_payload.get("requested_by") or "api/driver-action").strip() or "api/driver-action", } preview_ok, preview_message, preview_data = preview_driver_action(preview_request) if not preview_ok: return False, preview_message, { "preview_request": preview_request, "preview": dict(preview_data or {}), "selected_section": "secondary" if secondary else "primary", } gate = _build_resolved_driver_action_gate(preview_data, secondary=secondary, confirm=confirm) return True, "驾驶动作解析完成", { "preview_request": preview_request, "preview": dict(preview_data or {}), "gate": gate, "selected_section": "secondary" if secondary else "primary", "focus_ref": _normalize_focus_ref( gate.get("focus_ref") or preview_data.get("secondary_focus_ref" if secondary else "primary_focus_ref") or preview_data.get("focus_ref") ), } def execute_resolved_driver_action(payload: dict | None = None) -> tuple[bool, str, dict]: resolve_ok, resolve_message, resolved = resolve_driver_action_request(payload or {}) if not resolve_ok: return False, resolve_message, dict(resolved or {}) gate = dict((resolved or {}).get("gate") or {}) if not bool(gate.get("will_execute", False)): return False, str(gate.get("decision_reason") or "当前不允许自动执行").strip() or "当前不允许自动执行", { **dict(resolved or {}), "executed": False, } preview = dict((resolved or {}).get("preview") or {}) executor_kind = str(preview.get("executor_kind") or "driver_action").strip() or "driver_action" request_payload = dict(gate.get("request_payload") or {}) if executor_kind == "runbook_sequence": sequence_key = str(gate.get("sequence_key") or preview.get("sequence_key") or "").strip() if not sequence_key: return False, "缺少 sequence_key,无法执行 runbook sequence", { **dict(resolved or {}), "executed": False, } execute_ok, execute_message, execute_data = execute_ops_runbook_sequence(sequence_key, request_payload) else: execute_ok, execute_message, execute_data = execute_driver_action(request_payload) return execute_ok, execute_message, { **dict(resolved or {}), "executed": bool(execute_ok), "execution_result": dict(execute_data or {}), } def _select_codex_brief_entry(codex_brief: dict, entry_key: str = "") -> dict: normalized_entry_key = str(entry_key or "").strip() entries = [dict(item or {}) for item in list((codex_brief or {}).get("entries") or []) if dict(item or {})] activity_focus_entries = [ _build_codex_action_entry_from_activity_focus(item) for item in list((codex_brief or {}).get("activity_focus") or []) if dict(item or {}) ] activity_focus_entries = [item for item in activity_focus_entries if item] focus = dict((codex_brief or {}).get("focus") or {}) target_entry_key = normalized_entry_key or str(focus.get("entry_key") or "").strip() if target_entry_key: for item in entries + activity_focus_entries: if str(item.get("key") or "").strip() == target_entry_key: return item searchable_entries = entries + activity_focus_entries return dict(searchable_entries[0] or {}) if searchable_entries else {} def _build_codex_action_preview_request( selected_entry: dict, *, requested_by: str, source_label: str, include_secondary: bool = True, ) -> dict: normalized_entry = dict(selected_entry or {}) autopilot = dict(normalized_entry.get("autopilot") or {}) primary_execution = dict(normalized_entry.get("primary_execution") or {}) secondary_execution = dict(normalized_entry.get("secondary_execution") or {}) requested_by_value = str(requested_by or "api/codex-action").strip() or "api/codex-action" source_label_value = str(source_label or "ops/codex-action").strip() or "ops/codex-action" return { "source_label": source_label_value, "title": str(normalized_entry.get("title") or "").strip() or str(normalized_entry.get("key") or "").strip() or "codex-action", "summary": str(normalized_entry.get("summary") or "").strip(), "reason": str(normalized_entry.get("reason") or autopilot.get("reason") or "").strip(), "meta_text": str(normalized_entry.get("meta_text") or "").strip(), "executor_kind": str(normalized_entry.get("executor_kind") or "driver_action").strip() or "driver_action", "sequence_key": str(normalized_entry.get("sequence_key") or "").strip(), "focus_ref": _normalize_focus_ref(normalized_entry.get("focus_ref")), "primary_focus_ref": _normalize_focus_ref(normalized_entry.get("primary_focus_ref")), "secondary_focus_ref": _normalize_focus_ref(normalized_entry.get("secondary_focus_ref")), "primary_label": str(primary_execution.get("label") or normalized_entry.get("primary_label") or "").strip(), "secondary_label": str(secondary_execution.get("label") or normalized_entry.get("secondary_label") or "").strip(), "primary_action_code": str(normalized_entry.get("primary_action_code") or "").strip(), "secondary_action_code": str(normalized_entry.get("secondary_action_code") or "").strip(), "primary_node_codes": _normalize_driver_node_codes( primary_execution.get("node_codes") or normalized_entry.get("node_codes") or [] ), "secondary_node_codes": _normalize_driver_node_codes( secondary_execution.get("node_codes") or normalized_entry.get("node_codes") or [] ), "primary_action_payload": dict(primary_execution.get("action_payload") or {}), "secondary_action_payload": dict(secondary_execution.get("action_payload") or {}), "include_secondary": bool(include_secondary), "requested_by": requested_by_value, } def _build_codex_action_gate( selected_entry: dict, preview_data: dict, *, secondary: bool = False, confirm: bool = False, ) -> dict: normalized_entry = dict(selected_entry or {}) normalized_preview = dict(preview_data or {}) autopilot = dict(normalized_entry.get("autopilot") or {}) preview_section = dict((normalized_preview.get("secondary") if secondary else normalized_preview.get("primary")) or {}) execution_key = "secondary_execution" if secondary else "primary_execution" entry_execution = dict(normalized_entry.get(execution_key) or {}) preview_execution = dict(preview_section.get("execution_profile") or {}) recommendation = str( preview_execution.get("recommendation") or autopilot.get("recommendation") or "blocked" ).strip() or "blocked" automation_level = str( preview_execution.get("automation_level") or autopilot.get("automation_level") or "blocked" ).strip() or "blocked" executor_mode_hint = str( preview_execution.get("executor_mode_hint") or autopilot.get("executor_mode_hint") or "" ).strip() risk_level = str(preview_execution.get("risk_level") or autopilot.get("risk_level") or "unknown").strip() or "unknown" confirm_required = bool(preview_execution.get("confirm_required", False) or autopilot.get("confirm_required", False)) ui_only = bool(preview_execution.get("ui_only", False) or autopilot.get("ui_only", False)) blocked = bool(preview_execution.get("blocked", False) or autopilot.get("blocked", False)) reason = str(preview_execution.get("reason") or autopilot.get("reason") or "").strip() target_api = str(preview_section.get("target_api") or autopilot.get("target_api") or "").strip() target_method = str(preview_section.get("target_method") or autopilot.get("target_method") or "POST").strip() or "POST" request_payload = dict(preview_section.get("request_payload") or {}) confirm_received = bool(confirm) decision = "blocked" decision_reason = reason or "当前焦点被标记为阻断,不能自动执行。" will_execute = False if recommendation == "auto_execute": decision = "execute_now" will_execute = True decision_reason = reason or "当前焦点属于 safe_auto,可直接执行。" elif recommendation == "confirm_then_execute": if confirm_received: decision = "execute_confirmed" will_execute = True decision_reason = reason or "当前焦点属于 guarded_auto,已收到确认,允许执行。" else: decision = "confirmation_required" decision_reason = "当前焦点属于 guarded_auto,需要显式确认后才能执行。" if reason: decision_reason = f"{decision_reason} {reason}" elif recommendation == "resolve_first": decision = "resolve_first" decision_reason = reason or "当前焦点要求先复核解析结果,不应直接执行。" elif recommendation == "open_ui": decision = "open_ui" decision_reason = reason or "当前焦点属于 UI-only,建议进入工作区处理。" decision_label = { "execute_now": "直接执行", "execute_confirmed": "确认后执行", "confirmation_required": "等待确认", "resolve_first": "先复核", "open_ui": "进入工作区", "blocked": "当前阻断", }.get(decision, decision) return { "decision": decision, "decision_label": decision_label, "decision_reason": decision_reason, "will_execute": will_execute, "recommendation": recommendation, "recommendation_label": _driver_action_recommendation_label(recommendation), "automation_level": automation_level, "status_text": str(autopilot.get("status_text") or "").strip(), "executor_mode_hint": executor_mode_hint, "risk_level": risk_level, "confirm_required": confirm_required, "confirm_received": confirm_received, "ui_only": ui_only, "blocked": blocked, "secondary": bool(secondary), "backend_handled": bool(preview_execution.get("backend_handled", False) or autopilot.get("backend_handled", False)), "target_api": target_api, "target_method": target_method, "request_payload": request_payload, "focus_ref": _normalize_focus_ref( preview_section.get("focus_ref") or normalized_entry.get("secondary_focus_ref" if secondary else "primary_focus_ref") or normalized_entry.get("focus_ref") ), "action_code": str( preview_section.get("action_code") or normalized_entry.get("secondary_action_code" if secondary else "primary_action_code") or "" ).strip(), "sequence_key": str(normalized_preview.get("sequence_key") or normalized_entry.get("sequence_key") or "").strip(), "label": str( preview_section.get("label") or entry_execution.get("label") or normalized_entry.get("secondary_label" if secondary else "primary_label") or "" ).strip(), "requires_fields": list(preview_execution.get("requires_fields") or []), "missing_fields": list(preview_execution.get("missing_fields") or []), } def resolve_codex_action(payload: dict | None = None) -> tuple[bool, str, dict]: normalized_payload = dict(payload or {}) entry_key = str(normalized_payload.get("entry_key") or "").strip() requested_by = str(normalized_payload.get("requested_by") or "api/codex-action").strip() or "api/codex-action" source_label = str(normalized_payload.get("source_label") or "ops/codex-action").strip() or "ops/codex-action" include_secondary = bool(normalized_payload.get("include_secondary", True)) secondary = bool(normalized_payload.get("secondary", False)) confirm = bool(normalized_payload.get("confirm", False)) codex_brief = get_ops_codex_brief() selected_entry = _select_codex_brief_entry(codex_brief, entry_key=entry_key) if not selected_entry: return False, "codex brief 当前没有可执行条目", {"entry_key": entry_key} preview_request = _build_codex_action_preview_request( selected_entry, requested_by=requested_by, source_label=source_label, include_secondary=include_secondary, ) preview_ok, preview_message, preview_data = preview_driver_action(preview_request) if not preview_ok: return False, preview_message, { "entry_key": str(selected_entry.get("key") or "").strip(), "selected_entry": selected_entry, "preview_request": preview_request, "preview": dict(preview_data or {}), } gate = _build_codex_action_gate(selected_entry, preview_data, secondary=secondary, confirm=confirm) return True, "Codex 驾驶动作解析完成", { "entry_key": str(selected_entry.get("key") or "").strip(), "selected_entry": selected_entry, "preview_request": preview_request, "preview": dict(preview_data or {}), "gate": gate, "selected_section": "secondary" if secondary else "primary", "focus": dict(codex_brief.get("focus") or {}), "summary": dict(codex_brief.get("summary") or {}), "focus_ref": _normalize_focus_ref( gate.get("focus_ref") or selected_entry.get("secondary_focus_ref" if secondary else "primary_focus_ref") or selected_entry.get("focus_ref") ), } def execute_codex_action(payload: dict | None = None) -> tuple[bool, str, dict]: resolve_ok, resolve_message, resolved = resolve_codex_action(payload or {}) if not resolve_ok: return False, resolve_message, dict(resolved or {}) gate = dict((resolved or {}).get("gate") or {}) if not bool(gate.get("will_execute", False)): selected_entry = dict((resolved or {}).get("selected_entry") or {}) blocked_message = str(gate.get("decision_reason") or "").strip() if str(gate.get("decision") or "").strip() == "open_ui": blocked_message = blocked_message or "当前焦点属于 UI-only,建议进入工作区处理。" if bool(selected_entry.get("observation_only", False)) and "工作区" not in blocked_message: blocked_message = f"{blocked_message} 请进入对应工作区查看当前观察项。".strip() return False, blocked_message or "当前不允许自动执行", { **dict(resolved or {}), "executed": False, } selected_entry = dict((resolved or {}).get("selected_entry") or {}) preview = dict((resolved or {}).get("preview") or {}) executor_kind = str(preview.get("executor_kind") or selected_entry.get("executor_kind") or "driver_action").strip() or "driver_action" request_payload = dict(gate.get("request_payload") or {}) if executor_kind == "runbook_sequence": sequence_key = str(gate.get("sequence_key") or preview.get("sequence_key") or selected_entry.get("sequence_key") or "").strip() if not sequence_key: return False, "缺少 sequence_key,无法执行 runbook sequence", { **dict(resolved or {}), "executed": False, } execute_ok, execute_message, execute_data = execute_ops_runbook_sequence(sequence_key, request_payload) else: execute_ok, execute_message, execute_data = execute_driver_action(request_payload) return execute_ok, execute_message, { **dict(resolved or {}), "executed": bool(execute_ok), "execution_result": dict(execute_data or {}), } def execute_driver_action(payload: dict) -> tuple[bool, str, dict]: action_code = str((payload or {}).get("action_code") or "").strip() requested_by = str((payload or {}).get("requested_by") or "api").strip() or "api" node_codes = _normalize_driver_node_codes((payload or {}).get("node_codes") or []) action_payload = dict((payload or {}).get("action_payload") or {}) if not action_code: return False, "action_code 不能为空", {} if action_code not in _BACKEND_DRIVER_ACTION_CODES: return True, "该驾驶动作暂需前端交互或尚未后端化", { "handled": False, "manual_required": True, "action_code": action_code, "node_codes": node_codes, } if action_code == "enable_log_sync_key": updated = update_runtime_settings( { **get_runtime_settings(), "worker_log_sync_enabled": True, "worker_log_sync_mode": "key", } ) return True, "远端日志回传已切到关键模式", { "handled": True, "action_code": action_code, "mode": "settings", "runtime_settings": updated, } if action_code == "disable_log_sync": updated = update_runtime_settings( { **get_runtime_settings(), "worker_log_sync_enabled": False, "worker_log_sync_mode": "key", } ) return True, "远端日志回传已关闭", { "handled": True, "action_code": action_code, "mode": "settings", "runtime_settings": updated, } if action_code == "enable_log_sync_full": updated = update_runtime_settings( { **get_runtime_settings(), "worker_log_sync_enabled": True, "worker_log_sync_mode": "full", } ) return True, "远端日志回传已切到全量模式", { "handled": True, "action_code": action_code, "mode": "settings", "runtime_settings": updated, } if action_code == "open_playbook_dialog": playbook_key = str(action_payload.get("playbook_key") or "").strip() if not playbook_key: return False, "该驾驶动作需要 playbook_key", {"handled": False, "action_code": action_code} target_node_codes = _normalize_driver_node_codes(action_payload.get("target_node_codes") or node_codes) requested_execution_mode = str(action_payload.get("execution_mode") or "").strip() if not requested_execution_mode: requested_execution_mode = _recommend_driver_execution_mode_for_playbook( playbook_key, target_node_codes, ).get("execution_mode", "") preview_ok, preview_message, playbook_preview = preview_ops_playbook( { "playbook_key": playbook_key, "node_codes": target_node_codes, "execution_mode": requested_execution_mode, "auto_approve": bool(action_payload.get("auto_approve", False)), "requested_by": requested_by, } ) if not preview_ok: return False, preview_message, {"handled": False, "action_code": action_code} return True, "", { "handled": True, "action_code": action_code, "mode": "playbook-preview", "playbook_preview": playbook_preview, "ui_intent": _build_driver_ui_intent( "open_playbook_dialog", playbook_key=playbook_key, target_node_codes=target_node_codes, execution_mode=requested_execution_mode, auto_approve=bool(action_payload.get("auto_approve", False)), ), } if action_code == "open_action_template_dialog": template_key = str(action_payload.get("template_key") or "").strip() if not template_key: return False, "该驾驶动作需要 template_key", {"handled": False, "action_code": action_code} template = dict(get_ops_action_template(template_key) or {}) target_node_codes = _normalize_driver_node_codes(action_payload.get("target_node_codes") or node_codes) requested_execution_mode = str(action_payload.get("execution_mode") or "").strip() if not requested_execution_mode: requested_execution_mode = _recommend_driver_execution_mode_for_action_template( template_key, target_node_codes, ).get("execution_mode", "") return True, "", { "handled": True, "action_code": action_code, "mode": "action-template-preview", "template_preview": { "template": template, "target_node_codes": target_node_codes, "execution_mode": requested_execution_mode, "payload": dict(action_payload.get("payload") or {}), }, "ui_intent": _build_driver_ui_intent( "open_action_template_dialog", template_key=template_key, target_node_codes=target_node_codes, execution_mode=requested_execution_mode, auto_approve=bool(action_payload.get("auto_approve", False)), payload=dict(action_payload.get("payload") or {}), ), } if action_code in {"replay_delivery_queue", "flush_delivery_queue"}: template_key = "delivery.queue.replay" if action_code == "replay_delivery_queue" else "delivery.queue.flush" target_node_codes = _normalize_driver_node_codes(action_payload.get("target_node_codes") or node_codes) if not target_node_codes: return False, "该驾驶动作需要 target_node_codes", {"handled": False, "action_code": action_code} requested_execution_mode = str(action_payload.get("execution_mode") or "").strip() if not requested_execution_mode: requested_execution_mode = _recommend_driver_execution_mode_for_action_template( template_key, target_node_codes, ).get("execution_mode", "") batch_payload = { "template_key": template_key, "target_node_codes": target_node_codes, "requested_by": f"{requested_by}/{action_code}", "execution_mode": requested_execution_mode or "remote-agent", "auto_approve": bool(action_payload.get("auto_approve", False)), "payload": dict(action_payload.get("payload") or {}), "metadata": { "source": "ops-driver", "driver_action_code": action_code, }, } ok, message, data = create_ops_job_batch(batch_payload) return ok, (message or ("队列修复任务已创建" if ok else "队列修复任务创建失败")), { "handled": ok, "action_code": action_code, "mode": "ops-job-batch", "template_key": template_key, "execution_mode": batch_payload["execution_mode"], "target_node_codes": target_node_codes, "job_batch": data, } if action_code == "focus_playbook_run": run_code = str(action_payload.get("run_code") or "").strip() if not run_code: return False, "该驾驶动作需要 run_code", {"handled": False, "action_code": action_code} run_detail = get_ops_playbook_run(run_code) run_events = list_ops_playbook_run_events( run_code, limit=40, step_key=str(action_payload.get("focus_step_key") or "").strip(), node_code=str(action_payload.get("node_code") or "").strip(), ) return True, "", { "handled": True, "action_code": action_code, "mode": "playbook-run-focus", "playbook_run_focus": { "run": run_detail, "events": run_events, }, "ui_intent": _build_driver_ui_intent( "playbook_run_detail", run_code=run_code, focus_step_key=str(action_payload.get("focus_step_key") or "").strip(), focus_step_title=str(action_payload.get("focus_step_title") or "").strip(), ), } if action_code == "focus_activity_item": ui_intent = dict(action_payload.get("ui_intent") or {}) if not str(ui_intent.get("kind") or "").strip(): return False, "该驾驶动作需要 ui_intent", {"handled": False, "action_code": action_code} activity_key = str(action_payload.get("activity_key") or "").strip() activity_kind = str(action_payload.get("kind") or "").strip() activity_item = _find_activity_stream_item(activity_key, kind=activity_kind) if activity_key else {} if not activity_item: activity_item = { "activity_key": activity_key, "kind": activity_kind, "status": str(action_payload.get("status") or "").strip(), "summary": str(action_payload.get("summary") or "").strip(), "focus_ref": _normalize_focus_ref(action_payload.get("focus_ref")), "ui_intent": ui_intent, } return True, "", { "handled": True, "action_code": action_code, "mode": "activity-focus", "activity_focus_preview": { "activity": activity_item, }, "ui_intent": ui_intent, } if action_code == "focus_latest_job_events": ui_intent = dict(action_payload.get("ui_intent") or {}) if not str(ui_intent.get("kind") or "").strip(): source_focus_ref = _normalize_focus_ref(action_payload.get("source_focus_ref")) focus_ref = _normalize_focus_ref(action_payload.get("focus_ref")) resolved_focus_ref = source_focus_ref or focus_ref if not resolved_focus_ref: return False, "该驾驶动作需要 ui_intent 或 focus_ref", {"handled": False, "action_code": action_code} ui_intent = _build_driver_ui_intent( "job_events", job_id=int(resolved_focus_ref.get("job_id") or focus_ref.get("job_id") or 0), job_code=str(resolved_focus_ref.get("job_code") or focus_ref.get("job_code") or "").strip(), event_key=str(resolved_focus_ref.get("event_key") or "").strip(), target_node_code=str( resolved_focus_ref.get("target_node_code") or focus_ref.get("target_node_code") or "" ).strip(), ) job_id = int(ui_intent.get("job_id") or action_payload.get("job_id") or 0) job_events = list_ops_job_events_for_jobs([job_id], limit=60) if job_id > 0 else [] return True, "", { "handled": True, "action_code": action_code, "mode": "job-events-focus", "job_events_preview": { "job_id": job_id, "job_code": str(ui_intent.get("job_code") or action_payload.get("job_code") or "").strip(), "events": job_events, }, "ui_intent": ui_intent, } if action_code == "open_playbook_run_latest_events": run_code = str(action_payload.get("run_code") or "").strip() if not run_code: return False, "该驾驶动作需要 run_code", {"handled": False, "action_code": action_code} run_events = list_ops_playbook_run_events( run_code, limit=60, step_key=str(action_payload.get("focus_step_key") or "").strip(), node_code=str(action_payload.get("node_code") or "").strip(), ) return True, "", { "handled": True, "action_code": action_code, "mode": "playbook-run-events", "playbook_run_events_preview": run_events, "ui_intent": _build_driver_ui_intent( "playbook_run_latest_events", run_code=run_code, focus_step_key=str(action_payload.get("focus_step_key") or "").strip(), focus_step_title=str(action_payload.get("focus_step_title") or "").strip(), ), } if action_code == "open_release_dialog": release_package_preview = get_latest_release_package_metadata() release_summary = get_release_summary() return True, "", { "handled": True, "action_code": action_code, "mode": "release-dialog-preview", "release_dialog_preview": { "summary": release_summary, "release_package_preview": release_package_preview, }, "ui_intent": _build_driver_ui_intent("open_release_dialog"), } if action_code == "release_package": release_package_preview = get_latest_release_package_metadata() return True, "", { "handled": True, "action_code": action_code, "mode": "release-package-preview", "release_package_preview": release_package_preview, "ui_intent": _build_driver_ui_intent("open_release_dialog"), } if action_code == "release_prepare": ok, message, data = prepare_latest_release_package( requested_by=str(action_payload.get("requested_by") or f"{requested_by}/{action_code}").strip() or f"{requested_by}/{action_code}" ) return ok, message, { "handled": ok, "action_code": action_code, "mode": "release-prepare", "release_prepare_result": data or {}, } if action_code == "api-restart": launchpad_context = _driver_launchpad_context(action_payload) runtime_build_info = get_runtime_build_info() recommended_command = build_bash_command( "drive_ops_center.sh", "runtime-refresh-recover", launchpad_context["control_plane_base_url"], ) return True, "建议先刷新控制面运行时,再继续 Launchpad 评估。", { "handled": True, "action_code": action_code, "mode": "runtime-refresh-preview", "runtime_build_info": runtime_build_info, "recommended_command": recommended_command, "ui_intent": _build_driver_ui_intent("open_stack_diagnosis"), } if action_code == "publish_latest_worker": launchpad_context = _driver_launchpad_context(action_payload) ok, message, data = create_release_and_smart_rollout_from_latest_package( { **action_payload, "channel": launchpad_context["channel"], "mode": "worker", "created_by": str(action_payload.get("created_by") or f"{requested_by}/{action_code}").strip() or f"{requested_by}/{action_code}", "rollout_created_by": str( action_payload.get("rollout_created_by") or f"{requested_by}/{action_code}/rollout" ).strip() or f"{requested_by}/{action_code}/rollout", }, control_plane_base_url=launchpad_context["control_plane_base_url"], ) return ok, message, { "handled": ok, "action_code": action_code, "mode": "smart-release-rollout", "smart_release_result": data or {}, } if action_code in {"review_smart_rollout_preview", "review_control_rollout", "fix_rollout_blockers"}: launchpad_context = _driver_launchpad_context(action_payload) release_launchpad = get_release_launchpad( control_plane_base_url=launchpad_context["control_plane_base_url"], channel=launchpad_context["channel"], ) review_mode = "control" if action_code == "review_control_rollout" else "worker" review_payload = _build_release_launchpad_review_payload( release_launchpad=release_launchpad, mode=review_mode, ) latest_release = dict(review_payload.get("latest_release") or {}) return True, "", { "handled": True, "action_code": action_code, "mode": "release-launchpad-preview", "review_mode": review_mode, "release_launchpad_review": review_payload, "ui_intent": _build_driver_ui_intent( "release_launchpad_review", mode=review_mode, release_id=int(latest_release.get("id") or 0), channel=launchpad_context["channel"], ), } if action_code == "fix_managed_nodes": launchpad_context = _driver_launchpad_context(action_payload) release_launchpad = get_release_launchpad( control_plane_base_url=launchpad_context["control_plane_base_url"], channel=launchpad_context["channel"], ) first_gap_row = _first_release_launchpad_gap_row(release_launchpad) first_gap_node_code = str(first_gap_row.get("node_code") or "").strip() if first_gap_node_code: handover = get_managed_node_handover( first_gap_node_code, control_plane_base_url=launchpad_context["control_plane_base_url"], ) return True, "", { "handled": True, "action_code": action_code, "mode": "ui-intent", "ui_intent": _build_driver_ui_intent( "managed_node_handover", node_code=first_gap_node_code, ), "handover": handover, } sync_result = sync_managed_nodes_from_cluster(dry_run=False) return True, "已按集群快照同步托管节点,请继续检查接管状态。", { "handled": True, "action_code": action_code, "mode": "cluster-sync", "sync_result": sync_result, } if action_code in {"bootstrap_run", "run_acceptance"}: target_node_code = str(action_payload.get("node_code") or (node_codes[0] if node_codes else "")).strip() if not target_node_code: launchpad_context = _driver_launchpad_context(action_payload) release_launchpad = get_release_launchpad( control_plane_base_url=launchpad_context["control_plane_base_url"], channel=launchpad_context["channel"], ) first_gap_row = _first_release_launchpad_gap_row(release_launchpad) target_node_code = str(first_gap_row.get("node_code") or "").strip() if not target_node_code: target_node_code = str( (dict(release_launchpad.get("launchpad_status") or {}).get("recommended_target_node_code") or "") ).strip() if not target_node_code: return False, "该驾驶动作需要 node_code", {"handled": False, "action_code": action_code} ok, message, data = execute_managed_node_onboarding_recovery( node_code=target_node_code, requested_by=f"{requested_by}/{action_code}", control_plane_base_url=str(action_payload.get("control_plane_base_url") or "").strip(), root_dir=str(action_payload.get("root_dir") or "").strip(), ) return ok, message, { "handled": ok, "action_code": action_code, "mode": "node-onboarding-recovery", "node_code": target_node_code, "recovery_result": data or {}, } if action_code in { "create_smart_release_rollout_worker", "create_smart_release_rollout_control", "create_release_rollout_worker", "create_release_rollout_control", }: rollout_mode = "control" if action_code.endswith("_control") else "worker" normalized_payload = { **action_payload, "mode": rollout_mode, "created_by": str(action_payload.get("created_by") or f"{requested_by}/{action_code}").strip() or f"{requested_by}/{action_code}", "rollout_created_by": str( action_payload.get("rollout_created_by") or f"{requested_by}/{action_code}/rollout" ).strip() or f"{requested_by}/{action_code}/rollout", "confirm_risky": bool(action_payload.get("confirm_risky", False)), } if action_code.startswith("create_release_rollout_"): release_id = int(action_payload.get("release_id") or (_preferred_release_for_ops().get("id") or 0) or 0) if release_id <= 0: return False, "该驾驶动作需要 release_id", {"handled": False, "action_code": action_code} ok, message, data = create_smart_release_rollout(release_id, normalized_payload) else: ok, message, data = create_release_and_smart_rollout_from_latest_package(normalized_payload) return ok, message, { "handled": ok, "action_code": action_code, "mode": "smart-release-rollout", "smart_release_result": data or {}, } if action_code == "open_rollout_dialog": release_id = int(action_payload.get("release_id") or (_preferred_release_for_ops().get("id") or 0) or 0) release_summary = get_release_summary() release_launchpad = get_release_launchpad() return True, "", { "handled": True, "action_code": action_code, "mode": "rollout-dialog-preview", "rollout_dialog_preview": { "summary": release_summary, "launchpad": release_launchpad, }, "ui_intent": _build_driver_ui_intent("open_rollout_dialog", release_id=release_id), } if action_code == "focus_release_hub": release_id = int(action_payload.get("release_id") or (_preferred_release_for_ops().get("id") or 0) or 0) launchpad_context = _driver_launchpad_context(action_payload) release_summary = get_release_summary() release_launchpad = get_release_launchpad( control_plane_base_url=launchpad_context["control_plane_base_url"], channel=launchpad_context["channel"], ) return True, "", { "handled": True, "action_code": action_code, "mode": "release-hub-preview", "release_hub_preview": { "summary": release_summary, "launchpad": release_launchpad, }, "ui_intent": _build_driver_ui_intent("focus_release_hub", release_id=release_id), } if action_code in {"open_release_deploy_control", "open_release_deploy_worker", "open_release_deploy_custom"}: template_key_map = { "open_release_deploy_control": "deploy.release.control", "open_release_deploy_worker": "deploy.release.worker", "open_release_deploy_custom": "deploy.release.custom", } template_key = template_key_map.get(action_code, "deploy.release.custom") template = dict(get_ops_action_template(template_key) or {}) target_node_codes = _normalize_driver_node_codes(action_payload.get("target_node_codes") or node_codes) release_id = int(action_payload.get("release_id") or (_preferred_release_for_ops().get("id") or 0) or 0) execution_mode = str(action_payload.get("execution_mode") or "").strip() if not execution_mode: execution_mode = _recommend_driver_execution_mode(target_node_codes).get("execution_mode", "remote-agent") release_summary = get_release_summary() release_launchpad = get_release_launchpad() return True, "", { "handled": True, "action_code": action_code, "mode": "release-deploy-preview", "release_deploy_preview": { "template": template, "summary": release_summary, "launchpad": release_launchpad, "target_node_codes": target_node_codes, "execution_mode": execution_mode, "release_id": release_id, }, "ui_intent": _build_driver_ui_intent( "open_release_deploy_template", template_key=template_key, release_id=release_id, target_node_codes=target_node_codes, execution_mode=execution_mode, ), } if action_code in {"handover_first_gap", "view_first_gap"}: target_node_code = str(action_payload.get("node_code") or (node_codes[0] if node_codes else "")).strip() if not target_node_code: return False, "该驾驶动作需要 node_code", {"handled": False, "action_code": action_code} handover = get_managed_node_handover( target_node_code, control_plane_base_url=str(action_payload.get("control_plane_base_url") or "").strip(), ) return True, "", { "handled": True, "action_code": action_code, "mode": "handover-preview", "handover_preview": handover, "ui_intent": _build_driver_ui_intent( "managed_node_handover" if action_code == "handover_first_gap" else "managed_node_edit", node_code=target_node_code, ), "handover": handover, } if not node_codes: return False, "该驾驶动作需要 node_codes", { "handled": False, "action_code": action_code, } if action_code in {"open_worker_logs_participating", "open_worker_logs_standby", "open_worker_logs"}: playbook_key = "scene.logs.key" execution_mode = str(action_payload.get("execution_mode") or "").strip() if not execution_mode: execution_mode = _recommend_driver_execution_mode(node_codes).get("execution_mode", "remote-agent") ok, message, data = execute_ops_playbook( { "playbook_key": playbook_key, "node_codes": node_codes, "execution_mode": execution_mode, "auto_approve": True, "requested_by": f"{requested_by}/{action_code}", } ) return ok, message, { "handled": ok, "action_code": action_code, "mode": "ops-playbook", "playbook_key": playbook_key, "node_codes": node_codes, "execution_mode": execution_mode, **(data or {}), } if action_code in {"run_scene_logs_key", "run_scene_logs_full"}: playbook_key = "scene.logs.full" if action_code == "run_scene_logs_full" else "scene.logs.key" execution_mode = str(action_payload.get("execution_mode") or "").strip() if not execution_mode: execution_mode = _recommend_driver_execution_mode(node_codes).get("execution_mode", "remote-agent") ok, message, data = execute_ops_playbook( { "playbook_key": playbook_key, "node_codes": node_codes, "execution_mode": execution_mode, "auto_approve": bool(action_payload.get("auto_approve", True)), "requested_by": f"{requested_by}/{action_code}", } ) return ok, message, { "handled": ok, "action_code": action_code, "mode": "ops-playbook", "playbook_key": playbook_key, "node_codes": node_codes, "execution_mode": execution_mode, **(data or {}), } if action_code in {"open_diagnostics_participating", "open_diagnostics"}: playbook_key = "scene.diagnostics" execution_mode = str(action_payload.get("execution_mode") or "").strip() if not execution_mode: execution_mode = _recommend_driver_execution_mode(node_codes).get("execution_mode", "remote-agent") ok, message, data = execute_ops_playbook( { "playbook_key": playbook_key, "node_codes": node_codes, "execution_mode": execution_mode, "auto_approve": True, "requested_by": f"{requested_by}/{action_code}", } ) return ok, message, { "handled": ok, "action_code": action_code, "mode": "ops-playbook", "playbook_key": playbook_key, "node_codes": node_codes, "execution_mode": execution_mode, **(data or {}), } if action_code in {"run_inspection_participating", "run_inspection_standby", "run_standard_inspection"}: playbook_key = "inspection.standard" execution_mode = str(action_payload.get("execution_mode") or "").strip() if not execution_mode: execution_mode = _recommend_driver_execution_mode(node_codes).get("execution_mode", "remote-agent") ok, message, data = execute_ops_playbook( { "playbook_key": playbook_key, "node_codes": node_codes, "execution_mode": execution_mode, "auto_approve": True, "requested_by": f"{requested_by}/{action_code}", } ) return ok, message, { "handled": ok, "action_code": action_code, "mode": "ops-playbook", "playbook_key": playbook_key, "node_codes": node_codes, "execution_mode": execution_mode, **(data or {}), } return True, "该驾驶动作暂需前端交互或尚未后端化", { "handled": False, "manual_required": True, "action_code": action_code, "node_codes": node_codes, } def _resolve_ops_runbook_sequence_entry( sequence: dict, *, secondary: bool = False, payload: dict | None = None, requested_by: str = "api", ) -> tuple[bool, str, dict]: normalized_sequence = dict(sequence or {}) normalized_payload = dict(payload or {}) normalized_sequence_key = str(normalized_sequence.get("key") or "").strip() if not normalized_sequence_key: return False, "sequence_key 不能为空", {} action_field = "secondary_action_code" if secondary else "primary_action_code" payload_field = "secondary_action_payload" if secondary else "primary_action_payload" label_field = "secondary_label" if secondary else "primary_label" action_code = str(normalized_sequence.get(action_field) or "").strip() if not action_code: return False, "当前标准作业路径没有可执行动作", { "sequence_key": normalized_sequence_key, "secondary": secondary, "sequence": normalized_sequence, } sequence_node_codes = _normalize_driver_node_codes(normalized_sequence.get("target_node_codes") or []) override_node_codes = _normalize_driver_node_codes(normalized_payload.get("node_codes") or []) node_codes = override_node_codes or sequence_node_codes action_payload = { **dict(normalized_sequence.get(payload_field) or {}), **dict(normalized_payload.get("action_payload") or {}), } sequence_focus_ref = _normalize_focus_ref(normalized_sequence.get("focus_ref")) action_focus_ref = _normalize_focus_ref( normalized_sequence.get("secondary_focus_ref" if secondary else "primary_focus_ref") ) if sequence_focus_ref and not action_payload.get("focus_ref"): action_payload["focus_ref"] = action_focus_ref or sequence_focus_ref normalized_requested_by = str(requested_by or "api").strip().rstrip("/") or "api" driver_requested_by = ( f"{normalized_requested_by}/{normalized_sequence_key}" if normalized_requested_by.endswith("/runbook") else f"{normalized_requested_by}/runbook/{normalized_sequence_key}" ) sequence_summary = { "key": normalized_sequence_key, "title": str(normalized_sequence.get("title") or "").strip(), "status": str(normalized_sequence.get("status") or "").strip(), "status_label": str(normalized_sequence.get("status_label") or "").strip(), "summary": str(normalized_sequence.get("summary") or "").strip(), "reason": str(normalized_sequence.get("reason") or "").strip(), "secondary": secondary, "action_label": str(normalized_sequence.get(label_field) or "").strip(), "target_scope_label": str(normalized_sequence.get("target_scope_label") or "").strip(), "step_titles": [str(item or "").strip() for item in list(normalized_sequence.get("step_titles") or []) if str(item or "").strip()], "focus_ref": sequence_focus_ref, "action_focus_ref": action_focus_ref, } return True, "标准作业路径解析成功", { "sequence_key": normalized_sequence_key, "sequence": sequence_summary, "driver_action_code": action_code, "driver_action_label": str(normalized_sequence.get(label_field) or "").strip() or action_code, "driver_node_codes": node_codes, "driver_action_payload": action_payload, "driver_requested_by": driver_requested_by, "resolved_at": datetime.now().isoformat(timespec="seconds"), } def _attach_ops_runbook_sequence_resolutions( sequences: list[dict], *, requested_by: str = "api/runbook-snapshot", ) -> list[dict]: annotated_sequences: list[dict] = [] for raw_sequence in list(sequences or []): sequence = dict(raw_sequence or {}) if not str(sequence.get("key") or "").strip(): continue primary_ok, primary_message, primary_data = _resolve_ops_runbook_sequence_entry( sequence, secondary=False, requested_by=requested_by, ) sequence["primary_resolution"] = { "ok": primary_ok, "message": primary_message, **dict(primary_data or {}), } if str(sequence.get("secondary_action_code") or "").strip(): secondary_ok, secondary_message, secondary_data = _resolve_ops_runbook_sequence_entry( sequence, secondary=True, requested_by=requested_by, ) sequence["secondary_resolution"] = { "ok": secondary_ok, "message": secondary_message, **dict(secondary_data or {}), } else: sequence["secondary_resolution"] = {} annotated_sequences.append(sequence) return annotated_sequences def resolve_ops_runbook_sequence(sequence_key: str, payload: dict | None = None) -> tuple[bool, str, dict]: normalized_sequence_key = str(sequence_key or "").strip() normalized_payload = dict(payload or {}) if not normalized_sequence_key: return False, "sequence_key 不能为空", {} requested_by = str(normalized_payload.get("requested_by") or "api").strip() or "api" secondary = bool(normalized_payload.get("secondary", False)) runbook = get_ops_runbook() sequences = [ dict(item or {}) for item in list(runbook.get("control_sequences") or []) if str((item or {}).get("key") or "").strip() ] sequence = next( (item for item in sequences if str(item.get("key") or "").strip() == normalized_sequence_key), {}, ) if not sequence: return False, "标准作业路径不存在", {"sequence_key": normalized_sequence_key} return _resolve_ops_runbook_sequence_entry( sequence, secondary=secondary, payload=normalized_payload, requested_by=requested_by, ) def execute_ops_runbook_sequence(sequence_key: str, payload: dict | None = None) -> tuple[bool, str, dict]: ok, message, resolved = resolve_ops_runbook_sequence(sequence_key, payload or {}) if not ok: return ok, message, resolved action_code = str(resolved.get("driver_action_code") or "").strip() node_codes = _normalize_driver_node_codes(resolved.get("driver_node_codes") or []) action_payload = dict(resolved.get("driver_action_payload") or {}) ok, message, data = execute_driver_action( { "action_code": action_code, "node_codes": node_codes, "action_payload": action_payload, "requested_by": str(resolved.get("driver_requested_by") or "api/runbook").strip() or "api/runbook", } ) return ok, message, { **dict(resolved or {}), **dict(data or {}), } def get_ops_inspection_overview( *, managed_nodes: list[dict] | None = None, fetch_limit: int = _OPS_INSPECTION_FETCH_LIMIT, status: str = "", problem_kind: str = "", query: str = "", only_problem: bool = False, only_participating: bool = False, ) -> dict: nodes = list(managed_nodes or []) if managed_nodes is None: managed_nodes_payload = list_managed_nodes_with_agent_state() nodes = list(managed_nodes_payload.get("nodes") or []) jobs = list_ops_jobs(limit=fetch_limit, compact=False) inspection_jobs = [job for job in jobs if _inspection_job_bucket(job) and str(job.get("target_node_code") or "").strip()] node_map = { str(item.get("node_code") or "").strip(): item for item in nodes if str(item.get("node_code") or "").strip() } row_map: dict[str, dict] = {} def ensure_row(node_code: str) -> dict: if node_code not in row_map: node = node_map.get(node_code) or {} row_map[node_code] = { "node_code": node_code, "region": str(node.get("region") or ""), "role": str(node.get("role") or ""), "is_managed": bool(node.get("is_managed", False)), "is_enabled": bool(node.get("is_enabled", False)), "is_agent_online": bool(node.get("is_agent_online", False)), "has_ssh_access": bool(node.get("has_ssh_access", False)), "agent_state": str(node.get("agent_state") or ""), "agent_state_label": str(node.get("agent_state_label") or ""), "agent_state_reason": str(node.get("agent_state_reason") or ""), "remote_access_state": str(node.get("remote_access_state") or ""), "remote_access_label": str(node.get("remote_access_label") or ""), "remote_access_reason": str(node.get("remote_access_reason") or ""), "is_remote_access_ready": bool(node.get("is_remote_access_ready", False)), "cluster_status": str(node.get("cluster_status") or ""), "cluster_ip": str(node.get("cluster_ip") or ""), "cluster_hostname": str(node.get("cluster_hostname") or ""), "cluster_is_effective_worker": bool(node.get("cluster_is_effective_worker", False)), "cluster_detect_participating": bool(node.get("cluster_detect_participating", False)), "participation_state": str(node.get("participation_state") or ""), "participation_label": str(node.get("participation_label") or ""), "participation_reason": str(node.get("participation_reason") or ""), "is_inspection_visible": _is_inspection_visible_node(node), "is_inspection_eligible": _is_inspection_eligible_node(node), "job_map": {}, "latest_job": {}, "last_updated_at": "", "success_count": 0, "overall_status": "missing", "issue_summary": "", "problem_kind": "", "problem_label": "", "problem_title": "", "problem_level": "", "problem_keys": [], "recommended_action": "", "recommended_action_code": "", } return row_map[node_code] for job in inspection_jobs: node_code = str(job.get("target_node_code") or "").strip() bucket = _inspection_job_bucket(job) if not node_code or not bucket: continue row = ensure_row(node_code) if bucket not in row["job_map"]: row["job_map"][bucket] = _slim_inspection_job(job) candidate_time = str(job.get("updated_at") or job.get("finished_at") or job.get("started_at") or job.get("created_at") or "").strip() current_latest_time = str(row.get("last_updated_at") or "").strip() if not current_latest_time or candidate_time > current_latest_time: row["last_updated_at"] = candidate_time row["latest_job"] = _slim_inspection_job(job) for node in nodes: node_code = str(node.get("node_code") or "").strip() if node_code and _is_inspection_visible_node(node): ensure_row(node_code) def derive_overall_status(row: dict, job_map: dict[str, dict]) -> str: statuses = [str((job_map.get(key) or {}).get("status") or "").strip() for key in _OPS_INSPECTION_ACTION_KEYS] present_statuses = [status for status in statuses if status] if not present_statuses: if bool(row.get("is_inspection_visible", False)): if not bool(row.get("is_inspection_eligible", False)): return "attention" if bool(row.get("cluster_detect_participating", False)): return "attention" return "missing" if any(status in {"failed", "blocked", "cancelled"} for status in present_statuses): return "attention" if any(status in {"queued", "dispatching", "running", "awaiting_approval"} for status in present_statuses): return "running" if all(str((job_map.get(key) or {}).get("status") or "").strip() == "success" for key in _OPS_INSPECTION_ACTION_KEYS): return "healthy" return "attention" rows: list[dict] = [] for row in row_map.values(): job_map = dict(row.get("job_map") or {}) success_count = sum(1 for key in _OPS_INSPECTION_ACTION_KEYS if str((job_map.get(key) or {}).get("status") or "").strip() == "success") issue_summary = _build_inspection_issue_summary( job_map, node_code=str(row.get("node_code") or ""), latest_job=dict(row.get("latest_job") or {}), node=row, ) rows.append( { **row, "job_map": job_map, "success_count": success_count, "overall_status": derive_overall_status(row, job_map), "issue_summary": str(issue_summary.get("summary") or ""), "problem_kind": str(issue_summary.get("problem_kind") or ""), "problem_label": str(issue_summary.get("problem_label") or ""), "problem_title": str(issue_summary.get("problem_title") or ""), "problem_level": str(issue_summary.get("problem_level") or ""), "problem_keys": list(issue_summary.get("problem_keys") or []), "recommended_action": str(issue_summary.get("recommended_action") or ""), "recommended_action_code": str(issue_summary.get("recommended_action_code") or ""), "ui_intent": dict(issue_summary.get("ui_intent") or {}), "latest_health_snapshot": _build_inspection_contract_result( job_map.get("health.snapshot"), action_label="健康快照", ), "latest_worker_logs": _build_inspection_contract_result( job_map.get("logs.collect"), action_label="Worker 日志", ), "latest_diagnostics": _build_inspection_contract_result( job_map.get("diagnostics.collect"), action_label="诊断包", ), } ) status_weight = {"attention": 0, "running": 1, "missing": 2, "healthy": 3} # Stable-sort so the primary order is status bucket, then newest update, then node code. rows.sort(key=lambda item: str(item.get("node_code") or "")) rows.sort(key=lambda item: str(item.get("last_updated_at") or ""), reverse=True) rows.sort(key=lambda item: int(status_weight.get(str(item.get("overall_status") or ""), 99))) summary = { "healthy": 0, "running": 0, "attention": 0, "missing": 0, } for row in rows: row_overall_status = str(row.get("overall_status") or "missing").strip() if row_overall_status in summary: summary[row_overall_status] += 1 priority_weight = {"attention": 0, "running": 1, "missing": 2, "healthy": 9} normalized_status = str(status or "").strip().lower() normalized_problem_kind = str(problem_kind or "").strip().lower() normalized_query = str(query or "").strip().lower() filtered_rows = list(rows) if normalized_status: filtered_rows = [ item for item in filtered_rows if str(item.get("overall_status") or "").strip().lower() == normalized_status ] if normalized_problem_kind: filtered_rows = [ item for item in filtered_rows if str(item.get("problem_kind") or "").strip().lower() == normalized_problem_kind ] if only_problem: filtered_rows = [item for item in filtered_rows if str(item.get("overall_status") or "").strip() != "healthy"] if only_participating: filtered_rows = [item for item in filtered_rows if bool(item.get("cluster_detect_participating", False))] if normalized_query: filtered_rows = [ item for item in filtered_rows if normalized_query in " ".join( [ str(item.get("node_code") or ""), str(item.get("region") or ""), str(item.get("role") or ""), str(item.get("overall_status") or ""), str(item.get("problem_kind") or ""), str(item.get("problem_label") or ""), str(item.get("problem_title") or ""), str(item.get("issue_summary") or ""), str(item.get("recommended_action") or ""), ] ).lower() ] filtered_summary = { "healthy": 0, "running": 0, "attention": 0, "missing": 0, } for row in filtered_rows: row_status = str(row.get("overall_status") or "missing").strip() if row_status in filtered_summary: filtered_summary[row_status] += 1 if not filtered_rows: overview_status = "empty" overview_status_label = "无匹配结果" overview_summary_text = "当前筛选条件下没有匹配节点。" elif int(filtered_summary.get("attention", 0) or 0) > 0: overview_status = "attention" overview_status_label = "待处理" overview_summary_text = f"存在 {int(filtered_summary.get('attention', 0) or 0)} 台节点需要优先处理。" elif int(filtered_summary.get("running", 0) or 0) > 0: overview_status = "running" overview_status_label = "执行中" overview_summary_text = f"存在 {int(filtered_summary.get('running', 0) or 0)} 台节点的标准巡检仍在执行。" elif int(filtered_summary.get("missing", 0) or 0) > 0: overview_status = "attention" overview_status_label = "待补齐" overview_summary_text = f"存在 {int(filtered_summary.get('missing', 0) or 0)} 台节点尚未形成完整巡检收口。" else: overview_status = "healthy" overview_status_label = "健康" overview_summary_text = "当前可见节点最近巡检收口正常。" priority_queue = [ { **row, "priority_label": "P1" if row.get("overall_status") == "attention" else ("P2" if row.get("overall_status") == "running" else "P3"), "priority_type": "danger" if row.get("overall_status") == "attention" else ("warning" if row.get("overall_status") == "running" else "info"), "priority_weight": int(priority_weight.get(str(row.get("overall_status") or ""), 99)), } for row in filtered_rows if str(row.get("overall_status") or "") != "healthy" ] priority_queue.sort(key=lambda item: str(item.get("node_code") or "")) priority_queue.sort(key=lambda item: str(item.get("last_updated_at") or ""), reverse=True) priority_queue.sort(key=lambda item: int(item.get("priority_weight", 99))) available_status_counts = { "healthy": int(summary.get("healthy", 0) or 0), "running": int(summary.get("running", 0) or 0), "attention": int(summary.get("attention", 0) or 0), "missing": int(summary.get("missing", 0) or 0), } available_problem_kind_counts: dict[str, int] = {} for row in rows: row_problem_kind = str(row.get("problem_kind") or "").strip() if not row_problem_kind: continue available_problem_kind_counts[row_problem_kind] = int(available_problem_kind_counts.get(row_problem_kind, 0) or 0) + 1 return { "status": overview_status, "status_label": overview_status_label, "summary_text": overview_summary_text, "fetch_limit": int(fetch_limit), "jobs_total": len(inspection_jobs), "visible_nodes_total": sum(1 for item in nodes if _is_inspection_visible_node(item)), "eligible_nodes_total": sum(1 for item in nodes if _is_inspection_eligible_node(item)), "handover_gap_nodes_total": sum( 1 for item in nodes if _is_inspection_visible_node(item) and not _is_inspection_eligible_node(item) ), "participating_nodes_total": sum(1 for item in rows if bool(item.get("cluster_detect_participating", False))), "standby_nodes_total": sum( 1 for item in rows if bool(item.get("is_inspection_visible", False)) and not bool(item.get("cluster_detect_participating", False)) ), "rows_total": len(rows), "filtered_rows_total": len(filtered_rows), "filters": { "status": normalized_status, "problem_kind": normalized_problem_kind, "query": str(query or "").strip(), "only_problem": bool(only_problem), "only_participating": bool(only_participating), }, "available_status_counts": available_status_counts, "available_problem_kind_counts": available_problem_kind_counts, "rows": filtered_rows, "summary": filtered_summary, "counts": { "healthy_nodes": int(filtered_summary.get("healthy", 0) or 0), "running_nodes": int(filtered_summary.get("running", 0) or 0), "attention_nodes": int(filtered_summary.get("attention", 0) or 0), "missing_nodes": int(filtered_summary.get("missing", 0) or 0), }, "unfiltered_summary": summary, "priority_queue": priority_queue, } def get_ops_overview() -> dict: runtime = get_runtime_status() readiness = runtime.get("readiness") or {} detect = runtime.get("detect") or {} cluster = runtime.get("cluster") or {} sync_summary = get_sync_summary() job_summary = get_ops_job_summary() managed_nodes_payload = list_managed_nodes_with_agent_state(participation_payload=detect) managed_nodes = list(managed_nodes_payload.get("nodes") or []) managed_nodes_summary = managed_nodes_payload.get("summary") or {} inspection_overview = get_ops_inspection_overview(managed_nodes=managed_nodes) activity_stream = get_ops_activity_stream(limit=8) release_summary = get_release_summary() preferred_release = _preferred_release_for_ops() release_launchpad = get_release_launchpad() sync_batches = (sync_summary.get("detect_result_batches") or {}).get("state_counts") or {} online_worker_nodes = int((cluster.get("summary") or {}).get("online_worker_nodes", 0) or 0) online_control_nodes = int((cluster.get("summary") or {}).get("online_control_nodes", 0) or 0) projected_batches = int(sync_batches.get("projected", 0) or 0) failed_batches = int(sync_batches.get("failed", 0) or 0) execution_scene = _build_ops_execution_scene(detect) default_rollout_targets = _default_rollout_target_nodes(list(cluster.get("nodes") or [])) release_execution_mode = build_release_execution_mode_recommendation(preferred_release, default_rollout_targets) release_gate = dict(release_execution_mode.get("recommended_gate") or {}) release_gate = { **release_gate, "default_target_nodes": list(release_gate.get("target_nodes") or []), "default_target_node_codes": list(release_gate.get("target_node_codes") or []), } driver_recommendations = _build_driver_recommendations( managed_nodes=managed_nodes, execution_scene=execution_scene, release_summary=release_summary, release_gate=release_gate, release_launchpad=release_launchpad, ) recommendation = _build_ops_priority_recommendation(driver_recommendations) return { "mode": "centralized-ops", "vision": "海外控制面统一接管大陆节点安装、更新、巡检、日志回传与故障诊断。", "current_topology": { "node_code": settings.node_code, "node_region": settings.node_region, "node_role": settings.node_role, "worker_mode": settings.worker_mode, "sync_push_enabled": bool(settings.sync_push_enabled), "sync_target_api_base_url": str(settings.sync_target_api_base_url or "").strip(), }, "cluster_summary": { "online_control_nodes": online_control_nodes, "online_worker_nodes": online_worker_nodes, "dedicated_online_worker_nodes": int((cluster.get("summary") or {}).get("dedicated_online_worker_nodes", 0) or 0), "busy_nodes": list((cluster.get("summary") or {}).get("busy_nodes") or []), }, "cluster_nodes": [ { "node_code": str(item.get("node_code") or ""), "region": str(item.get("region") or ""), "role": str(item.get("role") or ""), "status": str(item.get("status") or ""), "current_load": int(item.get("current_load", 0) or 0), "hostname": str(item.get("hostname") or ""), "last_heartbeat_at": str(item.get("last_heartbeat_at") or ""), "is_effective_worker": bool(item.get("is_effective_worker", False)), "detect_participating": bool(item.get("detect_participating", False)), } for item in list(cluster.get("nodes") or []) if str(item.get("node_code") or "").strip() ], "runtime_summary": { "readiness_status": str(readiness.get("status") or ""), "readiness_summary": str(readiness.get("summary") or ""), "phase_label": str(detect.get("phase_label") or ""), "phase_detail": str(detect.get("phase_detail") or ""), "active_job_code": str((detect.get("active_job") or {}).get("job_code") or ""), "progress_percent": float(detect.get("progress_percent", 0) or 0), }, "execution_scene": execution_scene, "sync_summary": { "enabled": bool(sync_summary.get("enabled", False)), "source_region": str(sync_summary.get("source_region") or ""), "target_region": str(sync_summary.get("target_region") or ""), "projected_batches": projected_batches, "failed_batches": failed_batches, "records_total": int(sync_summary.get("records_total", 0) or 0), }, "ops_jobs": job_summary, "managed_nodes": { "total": int(managed_nodes_summary.get("total", len(managed_nodes)) or 0), "managed_total": int(managed_nodes_summary.get("managed_total", 0) or 0), "enabled": int(managed_nodes_summary.get("managed_enabled", 0) or 0), "online": int(managed_nodes_summary.get("online", 0) or 0), "stale": int(managed_nodes_summary.get("stale", 0) or 0), "pending_bootstrap": int(managed_nodes_summary.get("pending_bootstrap", 0) or 0), "runtime_only": int(managed_nodes_summary.get("runtime_only", 0) or 0), "unmanaged": int(managed_nodes_summary.get("unmanaged", 0) or 0), "token_issue": int(managed_nodes_summary.get("token_issue", 0) or 0), "queue_retrying_nodes": int(managed_nodes_summary.get("queue_retrying_nodes", 0) or 0), "queue_dead_letter_nodes": int(managed_nodes_summary.get("queue_dead_letter_nodes", 0) or 0), "queue_pending_records": int(managed_nodes_summary.get("queue_pending_records", 0) or 0), "queue_dead_letter_records": int(managed_nodes_summary.get("queue_dead_letter_records", 0) or 0), "agent_state_counts": dict(managed_nodes_summary.get("status_counts") or {}), "delivery_queue_state_counts": dict(managed_nodes_summary.get("delivery_queue_state_counts") or {}), }, "inspection": inspection_overview, "activity_stream": activity_stream, "driver_recommendations": driver_recommendations, "release_hub": { **release_summary, "preferred_release": preferred_release, "default_rollout_execution": release_execution_mode, "default_rollout_gate": release_gate, "default_rollout_gate_options": dict(release_execution_mode.get("gates") or {}), "launchpad": release_launchpad, }, "recommendation": recommendation, } def _build_ops_link_snapshot_from_overview(overview: dict) -> dict: overview = dict(overview or {}) topology = dict(overview.get("current_topology") or {}) cluster_summary = dict(overview.get("cluster_summary") or {}) runtime_summary = dict(overview.get("runtime_summary") or {}) execution_scene = dict(overview.get("execution_scene") or {}) participation_summary = dict(execution_scene.get("participation_summary") or {}) log_sync = dict(execution_scene.get("log_sync") or {}) sync_summary = dict(overview.get("sync_summary") or {}) recommendation = dict(overview.get("recommendation") or {}) readiness_status = str(runtime_summary.get("readiness_status") or "").strip() or "attention" readiness_summary = str(runtime_summary.get("readiness_summary") or "").strip() node_code = str(topology.get("node_code") or "").strip() node_region = str(topology.get("node_region") or "").strip() node_role = str(topology.get("node_role") or "").strip() worker_mode = str(topology.get("worker_mode") or "").strip() active_job_code = str(runtime_summary.get("active_job_code") or "").strip() phase_label = str(runtime_summary.get("phase_label") or "").strip() phase_detail = str(runtime_summary.get("phase_detail") or "").strip() progress_percent = float(runtime_summary.get("progress_percent", 0) or 0) online_control_nodes = int(cluster_summary.get("online_control_nodes", 0) or 0) online_worker_nodes = int(cluster_summary.get("online_worker_nodes", 0) or 0) dedicated_online_worker_nodes = int(cluster_summary.get("dedicated_online_worker_nodes", 0) or 0) busy_nodes = list(cluster_summary.get("busy_nodes") or []) effective_online_nodes = int( participation_summary.get("effective_online_nodes", online_worker_nodes) or online_worker_nodes or 0 ) participating_nodes = int(participation_summary.get("participating_nodes", 0) or 0) dispatch_active_nodes = int(participation_summary.get("dispatch_active_nodes", 0) or 0) recent_only_nodes = int(participation_summary.get("recent_only_nodes", 0) or 0) non_participating_nodes = int(participation_summary.get("non_participating_nodes", 0) or 0) standby_nodes = int(participation_summary.get("standby_nodes", 0) or 0) load_syncing_nodes = int(participation_summary.get("load_syncing_nodes", 0) or 0) dispatch_active_node_codes = list(participation_summary.get("dispatch_active_node_codes") or []) non_participating_node_codes = list(participation_summary.get("non_participating_node_codes") or []) log_sync_enabled = bool(log_sync.get("enabled", False)) log_sync_mode = str(log_sync.get("mode") or "").strip() or "key" log_sync_samples = int(log_sync.get("line_count", 0) or 0) log_sync_sources = int(log_sync.get("source_node_count", 0) or 0) missing_log_nodes = list(log_sync.get("missing_participating_nodes") or []) covered_participating_nodes = int(log_sync.get("covered_participating_node_count", 0) or 0) participating_node_count = int(log_sync.get("participating_node_count", participating_nodes) or participating_nodes or 0) log_sync_last_at = str(log_sync.get("last_at") or "").strip() log_sync_preview_lines = [str(item or "").strip() for item in list(log_sync.get("preview_lines") or []) if str(item or "").strip()] sync_enabled = bool(sync_summary.get("enabled", False)) sync_source_region = str(sync_summary.get("source_region") or "").strip() sync_target_region = str(sync_summary.get("target_region") or "").strip() projected_batches = int(sync_summary.get("projected_batches", 0) or 0) failed_batches = int(sync_summary.get("failed_batches", 0) or 0) sync_records_total = int(sync_summary.get("records_total", 0) or 0) if effective_online_nodes <= 0: status = "blocking" headline = "当前没有有效执行节点,海外控制面暂时无法继续推进联调、巡检或检测任务。" elif dispatch_active_nodes > 0 and not log_sync_enabled: status = "attention" headline = "当前已有执行节点,但远端日志回传仍关闭,海外控制面对现场仍处于半盲态。" elif dispatch_active_nodes > 0 and missing_log_nodes: status = "attention" headline = "当前已有执行节点,但远端日志回传仍未覆盖全部参与节点,建议先补齐现场观测。" elif failed_batches > 0: status = "attention" headline = "当前同步链路存在失败批次,建议先收口跨地域同步状态,再继续放量联调。" elif readiness_status != "ready": status = readiness_status headline = readiness_summary or "当前多机现场仍有待处理事项,建议先按运行中心提示逐项收口。" elif active_job_code: status = "ready" headline = "当前多机现场已进入可观测状态,可以直接从海外控制面继续盯检测任务和节点执行过程。" else: status = "ready" headline = "当前多机骨架已就绪,可以继续拉取任务、做巡检验收或推进发布演练。" summary_lines = [ f"控制面节点 {node_code or '-'} / 区域 {node_region or '-'} / 角色 {node_role or '-'} / 托管方式 {worker_mode or '-'}", f"集群在线:控制面 {online_control_nodes} 台 / 有效执行节点 {effective_online_nodes} 台 / 独立 Worker {dedicated_online_worker_nodes} 台 / 忙碌节点 {_format_node_code_list(busy_nodes)}", ( f"活跃任务 {active_job_code} / 阶段 {phase_label or '-'} / 进度 {progress_percent:.1f}%" if active_job_code else f"当前无活跃任务 / 就绪度 {readiness_status} / 摘要 {readiness_summary or '-'}" ), ( f"执行现场:参与 {participating_nodes} 台 / 执行中或已领 {dispatch_active_nodes} 台 / 近窗吞吐 {recent_only_nodes} 台 / 在线未参与 {non_participating_nodes} 台" ), ( f"远端日志:{'开启' if log_sync_enabled else '关闭'} / 模式 {log_sync_mode} / 覆盖 {covered_participating_nodes}/{participating_node_count} / 样本 {log_sync_samples} / 缺口 {_format_node_code_list(missing_log_nodes)}" ), ( f"跨地域同步:{'启用' if sync_enabled else '未启用'} / {sync_source_region or '-'} -> {sync_target_region or '-'} / projected {projected_batches} / failed {failed_batches} / records {sync_records_total}" ), ] if phase_detail: summary_lines.append(f"现场阶段说明:{phase_detail}") if log_sync_last_at: summary_lines.append(f"最近日志回传:{log_sync_last_at}") next_actions: list[dict] = [] if dispatch_active_nodes > 0 and not log_sync_enabled: target_node_codes = list(dispatch_active_node_codes or []) next_actions = [ { "label": "开启关键回传", "action_code": "enable_log_sync_key", "node_codes": target_node_codes, "payload": {}, }, { "label": "开启全量回传", "action_code": "enable_log_sync_full", "node_codes": target_node_codes, "payload": {}, }, ] elif dispatch_active_nodes > 0 and missing_log_nodes: target_node_codes = list(dispatch_active_node_codes or missing_log_nodes or []) next_actions = [ { "label": "看参与节点日志", "action_code": "open_worker_logs_participating", "node_codes": target_node_codes, "payload": {}, }, { "label": "执行标准巡检", "action_code": "run_inspection_participating", "node_codes": target_node_codes, "payload": {}, }, ] else: primary_label = str(recommendation.get("primary_label") or "").strip() primary_action_code = str(recommendation.get("primary_action_code") or "").strip() if primary_label or primary_action_code: next_actions.append( { "label": primary_label or "执行建议动作", "action_code": primary_action_code, "node_codes": list(recommendation.get("primary_node_codes") or recommendation.get("node_codes") or []), "payload": dict(recommendation.get("primary_action_payload") or {}), } ) secondary_label = str(recommendation.get("secondary_label") or "").strip() secondary_action_code = str(recommendation.get("secondary_action_code") or "").strip() if secondary_label or secondary_action_code: next_actions.append( { "label": secondary_label or "执行次要动作", "action_code": secondary_action_code, "node_codes": list(recommendation.get("secondary_node_codes") or recommendation.get("node_codes") or []), "payload": dict(recommendation.get("secondary_action_payload") or {}), } ) return { "generated_at": datetime.now().strftime("%Y-%m-%d %H:%M:%S"), "status": status, "headline": headline, "summary_lines": summary_lines, "control_plane": { "node_code": node_code, "region": node_region, "role": node_role, "worker_mode": worker_mode, "sync_push_enabled": bool(topology.get("sync_push_enabled", False)), "sync_target_api_base_url": str(topology.get("sync_target_api_base_url") or "").strip(), }, "cluster": { "online_control_nodes": online_control_nodes, "online_worker_nodes": online_worker_nodes, "dedicated_online_worker_nodes": dedicated_online_worker_nodes, "effective_online_nodes": effective_online_nodes, "busy_nodes": list(busy_nodes), }, "execution": { "active_job_code": active_job_code, "phase_label": phase_label, "phase_detail": phase_detail, "progress_percent": progress_percent, "participating_nodes": participating_nodes, "dispatch_active_nodes": dispatch_active_nodes, "recent_only_nodes": recent_only_nodes, "non_participating_nodes": non_participating_nodes, "standby_nodes": standby_nodes, "load_syncing_nodes": load_syncing_nodes, "dispatch_active_node_codes": list(dispatch_active_node_codes), "non_participating_node_codes": list(non_participating_node_codes), }, "log_sync": { "enabled": log_sync_enabled, "mode": log_sync_mode, "line_count": log_sync_samples, "source_node_count": log_sync_sources, "covered_participating_node_count": covered_participating_nodes, "participating_node_count": participating_node_count, "missing_participating_nodes": list(missing_log_nodes), "last_at": log_sync_last_at, "preview_lines": log_sync_preview_lines, }, "sync": { "enabled": sync_enabled, "source_region": sync_source_region, "target_region": sync_target_region, "projected_batches": projected_batches, "failed_batches": failed_batches, "records_total": sync_records_total, }, "recommendation": { "key": str(recommendation.get("key") or "").strip(), "priority": str(recommendation.get("priority") or "").strip(), "summary": str(recommendation.get("summary") or "").strip(), "reason": str(recommendation.get("reason") or "").strip(), }, "next_actions": next_actions, } def get_ops_link_snapshot(overview: dict | None = None) -> dict: normalized_overview = dict(overview or {}) if not normalized_overview: normalized_overview = get_ops_overview() return _build_ops_link_snapshot_from_overview(normalized_overview) def _normalize_ops_node_code_list(*groups: object) -> list[str]: normalized: list[str] = [] seen: set[str] = set() for group in groups: if isinstance(group, dict): group = [group] if not isinstance(group, (list, tuple, set)): continue for item in group: if isinstance(item, dict): node_code = str(item.get("node_code") or "").strip() else: node_code = str(item or "").strip() if not node_code or node_code in seen: continue seen.add(node_code) normalized.append(node_code) return normalized def _build_scene_node_log_command(base_url: str, node_code: str, *, limit: int = 80, mode: str = "key") -> str: normalized_node_code = str(node_code or "").strip() if not normalized_node_code: return "" normalized_base_url = _ops_stack_api_base_url(base_url) normalized_mode = _normalize_scene_log_mode(mode, fallback="key") normalized_limit = max(20, min(int(limit or 80), 300)) return build_bash_command( "drive_ops_center.sh", "scene-node-log", normalized_base_url, normalized_node_code, normalized_limit, normalized_mode, ) def _build_scene_node_log_focus_ref( node_code: str, *, mode: str = "key", limit: int = 80, source: str = "stack_diagnosis", ) -> dict: normalized_node_code = str(node_code or "").strip() if not normalized_node_code: return {} return _merge_focus_ref( {}, kind="node_scene_log", node_code=normalized_node_code, mode=_normalize_scene_log_mode(mode, fallback="key"), limit=max(20, min(int(limit or 80), 300)), source=source, ) def _build_scene_node_log_commands( base_url: str, node_codes: list[str] | tuple[str, ...] | set[str], *, limit: int = 80, mode: str = "key", max_commands: int = 3, ) -> list[str]: commands: list[str] = [] for node_code in _normalize_ops_node_code_list(list(node_codes or []))[: max(1, int(max_commands or 3))]: command = _build_scene_node_log_command(base_url, node_code, limit=limit, mode=mode) if command: commands.append(command) return commands def _ops_stack_api_base_url(base_url: str = "") -> str: normalized_base_url = str(base_url or "").strip() if normalized_base_url: return normalized_base_url.rstrip("/") return f"http://127.0.0.1:{settings.api_port}" def _ops_stack_capture(section_key: str, callback) -> dict: started_at = time.perf_counter() try: payload = callback() or {} except Exception as exc: elapsed_ms = int((time.perf_counter() - started_at) * 1000) return { "section_key": str(section_key or "").strip(), "http_status": "500", "available": False, "data": {}, "error": f"{exc.__class__.__name__}: {exc}", "elapsed_ms": elapsed_ms, } if isinstance(payload, dict): data = dict(payload or {}) else: data = {"value": payload} elapsed_ms = int((time.perf_counter() - started_at) * 1000) return { "section_key": str(section_key or "").strip(), "http_status": "200", "available": True, "data": data, "error": "", "elapsed_ms": elapsed_ms, } def _ops_stack_issue_rank(severity: str) -> int: return {"blocked": 3, "warning": 2, "info": 1}.get(str(severity or "").strip(), 0) def _suggest_ops_contract_keys_for_stack_issue(issue: dict) -> list[str]: normalized_issue = dict(issue or {}) layer = str(normalized_issue.get("layer") or "").strip() contract_keys: list[str] = ["ops_stack_diagnosis_contract"] if layer.startswith("managed_nodes"): contract_keys.append("ops_agent_protocol") elif layer == "release_hub": contract_keys.append("release_hub_contract") elif layer == "playbook_runs": contract_keys.append("ops_playbook_contract") elif layer in {"overview.log_sync", "activity_stream", "link_snapshot"}: contract_keys.append("ops_observability_contract") elif layer == "contracts": contract_keys.append("ops_driver_contract") contract_keys.extend(_ops_contract_keys_from_focus_ref(normalized_issue.get("focus_ref"))) contract_keys.extend(_ops_contract_keys_from_action_code(str(normalized_issue.get("action_code") or "").strip())) return _normalize_ops_contract_keys(contract_keys) def _build_stack_diagnosis_recommended_commands( *, stack_base_url: str, recommended_actions: list[dict], first_handover_gap_node_code: str = "", launchpad_status: dict | None = None, default_rollout_gate: dict | None = None, next_step: dict | None = None, ) -> dict: commands: dict[str, str] = {} commands["stack-diagnosis"] = build_bash_command("drive_ops_center.sh", "stack-diagnosis", stack_base_url, "summary") commands["stack-check"] = commands["stack-diagnosis"] commands["doctor"] = build_bash_command("drive_ops_center.sh", "doctor", stack_base_url) commands["contracts"] = build_bash_command("drive_ops_center.sh", "contracts", stack_base_url) commands["runtime-refresh-recover"] = build_bash_command( "drive_ops_center.sh", "runtime-refresh-recover", stack_base_url, ) commands["api-restart"] = commands["runtime-refresh-recover"] for action in list(recommended_actions or []): action_code = str(action.get("action_code") or "").strip() command = str(action.get("command") or "").strip() if action_code and command and action_code not in commands: commands[action_code] = command normalized_gap_node_code = str(first_handover_gap_node_code or "").strip() if normalized_gap_node_code: commands["managed-node-handover"] = build_bash_command("drive_ops_center.sh", "node-handover", stack_base_url, normalized_gap_node_code) commands["managed-node-recover"] = build_bash_command("drive_ops_center.sh", "node-recover", stack_base_url, normalized_gap_node_code) commands["managed-node-bootstrap-plan"] = build_bash_command("drive_ops_center.sh", "node-bootstrap-plan", stack_base_url, normalized_gap_node_code) normalized_next_step = dict(next_step or {}) next_step_focus_ref = dict(normalized_next_step.get("focus_ref") or {}) if str(next_step_focus_ref.get("kind") or "").strip() == "node_scene_log": node_code = str(next_step_focus_ref.get("node_code") or "").strip() if node_code: commands["scene-node-log"] = _build_scene_node_log_command( stack_base_url, node_code, limit=int(next_step_focus_ref.get("limit", 120) or 120), mode=str(next_step_focus_ref.get("mode") or "key"), ) normalized_launchpad_status = dict(launchpad_status or {}) normalized_default_rollout_gate = dict(default_rollout_gate or {}) launchpad_focus_ref = dict( normalized_launchpad_status.get("focus_ref") or normalized_default_rollout_gate.get("focus_ref") or {} ) if str(launchpad_focus_ref.get("section") or "").strip() == "release_launchpad": commands["focus-release-hub"] = build_bash_command("drive_ops_center.sh", "release-launchpad", stack_base_url) mode = str(launchpad_focus_ref.get("mode") or "").strip() if mode == "control": commands["release-launchpad-control"] = build_bash_command( "drive_ops_center.sh", "release-launchpad", stack_base_url, "control", ) else: commands["release-launchpad-worker"] = build_bash_command( "drive_ops_center.sh", "release-launchpad", stack_base_url, "worker", ) launchpad_recommended_action_code = str(normalized_launchpad_status.get("recommended_action_code") or "").strip() launchpad_recommended_target_node_code = str( normalized_launchpad_status.get("recommended_target_node_code") or "" ).strip() if ( launchpad_recommended_action_code in {"bootstrap_run", "run_acceptance"} and launchpad_recommended_target_node_code ): commands[launchpad_recommended_action_code] = build_bash_command( "drive_ops_center.sh", "node-bootstrap-run" if launchpad_recommended_action_code == "bootstrap_run" else "node-acceptance-run", stack_base_url, launchpad_recommended_target_node_code, "cli", ) return {key: value for key, value in commands.items() if str(value or "").strip()} def _resolve_first_handover_gap_recovery( *, first_handover_gap: dict | None = None, launchpad_gap_action: dict | None = None, launchpad_status: dict | None = None, fallback_node_code: str = "", ) -> dict: normalized_handover = dict(first_handover_gap or {}) stage = dict(normalized_handover.get("stage") or {}) stage_next_step = dict(stage.get("next_step") or {}) normalized_launchpad_gap_action = dict(launchpad_gap_action or {}) normalized_launchpad_status = dict(launchpad_status or {}) node_code = str( normalized_handover.get("node_code") or normalized_launchpad_gap_action.get("node_code") or normalized_launchpad_status.get("recommended_target_node_code") or fallback_node_code or "" ).strip() summary = str( normalized_handover.get("summary") or stage.get("summary") or normalized_launchpad_gap_action.get("summary") or normalized_launchpad_status.get("recommended_recovery_summary") or "" ).strip() stage_code = str(stage.get("code") or "").strip() stage_next_step_code = str(stage_next_step.get("code") or "").strip() has_gap_signal = bool(node_code or stage_code or stage_next_step_code or summary) action_code = str(normalized_launchpad_gap_action.get("action_code") or "").strip() if not has_gap_signal: action_code = "" elif action_code not in {"bootstrap_run", "run_acceptance"}: if stage_next_step_code in {"run_acceptance", "acceptance", "ready"} or stage_code in {"acceptance_ready"}: action_code = "run_acceptance" elif stage_next_step_code in {"execute_bootstrap_plan", "bootstrap_run"} or stage_code in { "pending_bootstrap", "runtime_only", "profile_incomplete", "ssh_ready", "token_issue", "stale", "agent_pending", "unknown", "unmanaged", }: action_code = "bootstrap_run" else: action_code = "" recovery_label = str(normalized_launchpad_status.get("recommended_recovery_label") or "").strip() if not recovery_label and action_code == "bootstrap_run": recovery_label = "签发接入工单" elif not recovery_label and action_code == "run_acceptance": recovery_label = "执行接管验收" if not recovery_label: recovery_label = str(stage_next_step.get("label") or stage.get("label") or "").strip() focus_ref = dict( (dict(normalized_launchpad_gap_action.get("row") or {}).get("focus_ref") or {}) or normalized_launchpad_status.get("focus_ref") or {} ) if node_code and not str(focus_ref.get("node_code") or "").strip(): focus_ref["node_code"] = node_code if node_code and not str(focus_ref.get("kind") or "").strip(): focus_ref["kind"] = "managed_node" return { "action_code": action_code if action_code in {"bootstrap_run", "run_acceptance"} else "", "node_code": node_code, "summary": summary, "recovery_label": recovery_label, "focus_ref": focus_ref, "stage_code": stage_code, "stage_label": str(stage.get("label") or "").strip(), "stage_next_step_code": stage_next_step_code, } def _build_stack_diagnosis_operator_decision( *, issues: list[dict], next_step: dict, recommended_actions: list[dict], recommended_commands: dict, first_handover_gap_node_code: str = "", launchpad_status: dict | None = None, default_rollout_gate: dict | None = None, ) -> tuple[dict, dict]: top_issue = ( sorted( list(issues or []), key=lambda item: ( -_ops_stack_issue_rank(str(item.get("severity") or "").strip()), str(item.get("code") or "").strip(), ), )[0] if issues else {} ) normalized_next_step = dict(next_step or {}) primary_action_code = str(normalized_next_step.get("action_code") or "").strip() next_focus_ref = dict(normalized_next_step.get("focus_ref") or {}) normalized_launchpad_status = dict(launchpad_status or {}) normalized_default_rollout_gate = dict(default_rollout_gate or {}) operator_decision = { "lane": "steady", "priority": "low", "reason_code": "steady_state", "title": "当前总检已进入稳定观察态", "summary": "控制面总检、执行现场、纳管与发布骨架当前没有阻断项,可以按既定节奏继续联调或推进发布。", "next_focus": next_focus_ref, "primary_command_key": primary_action_code, "secondary_command_key": "", } if str(next_focus_ref.get("kind") or "").strip() == "node_scene_log": operator_decision = { "lane": "observability", "priority": "high" if top_issue else "medium", "reason_code": str(top_issue.get("code") or "scene_log_observation").strip(), "title": "优先补齐执行现场日志观察", "summary": str(top_issue.get("summary") or normalized_next_step.get("reason") or "").strip() or "当前已经定位到参与节点现场日志焦点,建议先看现场,再决定是否继续接管或推进发布。", "next_focus": next_focus_ref, "primary_command_key": "scene-node-log", "secondary_command_key": primary_action_code or "open_worker_logs_participating", } elif str(top_issue.get("code") or "").strip() == "runtime_build_schema_stale": secondary_key = "stack-diagnosis" if "stack-diagnosis" in recommended_commands else "" if not secondary_key and "stack-check" in recommended_commands: secondary_key = "stack-check" if "managed-node-bootstrap-plan" in recommended_commands: secondary_key = "managed-node-bootstrap-plan" operator_decision = { "lane": "runtime_recovery", "priority": "high", "reason_code": "runtime_build_schema_stale", "title": "优先重启控制面 API 并复检运行时版本", "summary": str(top_issue.get("summary") or normalized_next_step.get("reason") or "").strip() or "仓库代码已经具备新能力,但运行中的 API 仍停留在旧 schema,当前第一动作应先让服务吃到最新代码。", "next_focus": dict(top_issue.get("focus_ref") or normalized_next_step.get("focus_ref") or {}), "primary_command_key": "api-restart", "secondary_command_key": secondary_key, } elif str(top_issue.get("code") or "").strip() == "managed_nodes_agent_pending" or ( str(top_issue.get("layer") or "").strip().startswith("managed_nodes") and str(first_handover_gap_node_code or "").strip() ): managed_primary_key = primary_action_code if managed_primary_key not in {"bootstrap_run", "run_acceptance"}: managed_primary_key = ( "managed-node-recover" if "managed-node-recover" in recommended_commands else "managed-node-handover" ) managed_secondary_key = "" if managed_primary_key in {"bootstrap_run", "run_acceptance"}: if "managed-node-bootstrap-plan" in recommended_commands: managed_secondary_key = "managed-node-bootstrap-plan" elif "managed-node-handover" in recommended_commands: managed_secondary_key = "managed-node-handover" elif "fix_managed_nodes" in recommended_commands: managed_secondary_key = "fix_managed_nodes" elif primary_action_code and primary_action_code != managed_primary_key: managed_secondary_key = primary_action_code operator_decision = { "lane": "node_handover", "priority": "high", "reason_code": str(top_issue.get("code") or "managed_nodes_agent_pending").strip(), "title": "优先执行节点接管收口" if managed_primary_key in {"bootstrap_run", "run_acceptance"} else "优先打通节点接管链路", "summary": ( str(normalized_next_step.get("reason") or "").strip() if managed_primary_key in {"bootstrap_run", "run_acceptance"} else "" ) or str(top_issue.get("summary") or "").strip() or "托管节点还没有进入 remote-agent ready,海外控制面暂时不能把执行统一收口到标准动作链。", "next_focus": dict(top_issue.get("focus_ref") or normalized_next_step.get("focus_ref") or {}), "primary_command_key": managed_primary_key, "secondary_command_key": managed_secondary_key, } elif ( str(top_issue.get("code") or "").strip() == "release_rollout_gate_blocked" or str(normalized_default_rollout_gate.get("status") or "").strip() == "blocked" or str(normalized_launchpad_status.get("recommended_action_code") or "").strip() in { "review_smart_rollout_preview", "review_control_rollout", "fix_rollout_blockers", } ): launchpad_focus_ref = dict( normalized_default_rollout_gate.get("focus_ref") or normalized_launchpad_status.get("focus_ref") or normalized_next_step.get("focus_ref") or {} ) launchpad_mode = str(launchpad_focus_ref.get("mode") or "").strip() primary_key = "release-launchpad-control" if launchpad_mode == "control" else "release-launchpad-worker" if primary_key not in recommended_commands: primary_key = "focus-release-hub" operator_decision = { "lane": "release", "priority": "high" if str(normalized_default_rollout_gate.get("status") or "").strip() == "blocked" else "medium", "reason_code": str(top_issue.get("code") or "release_launchpad_attention").strip(), "title": "优先处理发布门禁与 Launchpad 决策", "summary": str(top_issue.get("summary") or normalized_default_rollout_gate.get("summary_text") or "").strip() or "发布包和 Rollout 入口已经就位,但默认门禁还没完全放行,建议先看 Release Hub 再推进下一批。", "next_focus": launchpad_focus_ref, "primary_command_key": primary_key, "secondary_command_key": primary_action_code or str(normalized_launchpad_status.get("recommended_action_code") or "").strip(), } elif str(normalized_launchpad_status.get("recommended_action_code") or "").strip() in {"bootstrap_run", "run_acceptance"}: launchpad_action_code = str(normalized_launchpad_status.get("recommended_action_code") or "").strip() launchpad_target_node_code = str(normalized_launchpad_status.get("recommended_target_node_code") or "").strip() launchpad_focus_ref = dict( normalized_launchpad_status.get("focus_ref") or normalized_default_rollout_gate.get("focus_ref") or normalized_next_step.get("focus_ref") or {} ) operator_decision = { "lane": "node_handover", "priority": "high" if launchpad_action_code == "bootstrap_run" else "medium", "reason_code": f"release_launchpad_{launchpad_action_code}", "title": "优先执行节点接入收口" if launchpad_action_code == "bootstrap_run" else "优先执行节点接管验收", "summary": str(normalized_launchpad_status.get("recommended_recovery_summary") or "").strip() or str(top_issue.get("summary") or normalized_next_step.get("reason") or "").strip() or "Release Launchpad 已经明确给出节点接入缺口,建议先完成该节点的接入收口,再回到发布门禁。", "next_focus": { **launchpad_focus_ref, **({"node_code": launchpad_target_node_code} if launchpad_target_node_code else {}), }, "primary_command_key": launchpad_action_code, "secondary_command_key": "focus-release-hub" if "focus-release-hub" in recommended_commands else "", } elif primary_action_code: operator_decision = { "lane": "ops_jobs" if primary_action_code.startswith("focus_") else "steady", "priority": "medium" if top_issue else "low", "reason_code": str(top_issue.get("code") or primary_action_code).strip(), "title": "按总检默认下一步继续处理", "summary": str(normalized_next_step.get("reason") or "").strip() or "当前总检已经给出默认下一步,可直接按统一 driver action 继续推进。", "next_focus": next_focus_ref, "primary_command_key": primary_action_code, "secondary_command_key": "", } next_actions = { "primary_command_key": str(operator_decision.get("primary_command_key") or "").strip(), "primary_command": str( recommended_commands.get(str(operator_decision.get("primary_command_key") or "").strip()) or "" ).strip(), "secondary_command_key": str(operator_decision.get("secondary_command_key") or "").strip(), "secondary_command": str( recommended_commands.get(str(operator_decision.get("secondary_command_key") or "").strip()) or "" ).strip(), } return operator_decision, next_actions def get_ops_stack_diagnosis(*, base_url: str = "") -> dict: stack_base_url = _ops_stack_api_base_url(base_url) api_info = { "service": "domain-api", "version": "0.1.0", "api_prefix": settings.api_prefix, "worker_mode": settings.worker_mode, } contracts_section = _ops_stack_capture("contracts", get_ops_contract_registry) overview_section = _ops_stack_capture("overview", get_ops_overview) overview_data = dict(overview_section.get("data") or {}) if bool(overview_section.get("available", False)) and overview_data: link_snapshot_section = _ops_stack_capture( "link_snapshot", lambda: get_ops_link_snapshot(overview=overview_data), ) else: link_snapshot_section = _ops_stack_capture("link_snapshot", get_ops_link_snapshot) nodes_section = _ops_stack_capture("managed_nodes", list_managed_nodes_with_agent_state) def _stack_release_hub_payload() -> dict: overview_launchpad = dict(((overview_data.get("release_hub") or {}).get("launchpad") or {})) if overview_launchpad: return overview_launchpad return get_release_launchpad() release_hub_section = _ops_stack_capture("release_hub", _stack_release_hub_payload) build_info_section = _ops_stack_capture("runtime_build_info", get_runtime_build_info) playbook_runs_section = _ops_stack_capture( "playbook_runs", lambda: get_recent_ops_playbook_runs(limit=6, scan_limit=240), ) def _stack_activity_stream_payload() -> dict: overview_activity_stream = dict(overview_data.get("activity_stream") or {}) if overview_activity_stream: return overview_activity_stream return get_ops_activity_stream(limit=8, scan_limit=80) activity_stream_section = _ops_stack_capture("activity_stream", _stack_activity_stream_payload) contracts = dict(contracts_section.get("data") or {}) contract_registry = contracts if contracts else get_ops_contract_registry() link_snapshot = dict(link_snapshot_section.get("data") or {}) overview = overview_data nodes_payload = dict(nodes_section.get("data") or {}) launchpad = dict(release_hub_section.get("data") or {}) build_info = dict(build_info_section.get("data") or {}) playbook_runs_payload = dict(playbook_runs_section.get("data") or {}) activity_payload = dict(activity_stream_section.get("data") or {}) contracts_rows = list(contracts.get("contracts") or []) nodes = list(nodes_payload.get("nodes") or []) nodes_summary = dict(nodes_payload.get("summary") or {}) playbook_runs = list(playbook_runs_payload.get("runs") or playbook_runs_payload.get("playbook_runs") or []) activities = list(activity_payload.get("items") or activity_payload.get("activities") or []) recommendation = dict(overview.get("recommendation") or {}) execution_scene = dict(overview.get("execution_scene") or {}) log_sync = dict(execution_scene.get("log_sync") or overview.get("log_sync") or {}) release_hub = dict(overview.get("release_hub") or {}) default_rollout_gate = dict(release_hub.get("default_rollout_gate") or {}) launchpad_status = dict(launchpad.get("launchpad_status") or {}) launchpad_gap_action = _release_launchpad_gap_action_context(launchpad) latest_release = dict(launchpad.get("latest_release") or release_hub.get("latest_release") or {}) route_surface = dict(build_info.get("route_surface") or {}) repository_capabilities = dict(build_info.get("repository_capabilities") or {}) route_surface_missing_keys = [ str(item).strip() for item in list(route_surface.get("missing_keys") or []) if str(item).strip() ] route_surface_expected_paths = { str(key).strip(): str(value).strip() for key, value in dict(route_surface.get("expected_paths") or {}).items() if str(key).strip() } repo_supports_install_command_block = bool(repository_capabilities.get("supports_install_command_block", False)) repo_supports_multi_layout_bootstrap = bool(repository_capabilities.get("supports_multi_layout_bootstrap", False)) route_surface_declares_bootstrap_plan = "ops_node_handover_bootstrap_plan" in route_surface_expected_paths runtime_schema_stale = bool(repo_supports_install_command_block and not route_surface_declares_bootstrap_plan) problem_nodes: list[dict] = [] for node in nodes: node_code = str(node.get("node_code") or "").strip() if not node_code: continue queue_state = str(node.get("delivery_queue_state") or "").strip() agent_state = str(node.get("agent_state") or "").strip() remote_access_state = str(node.get("remote_access_state") or "").strip() if ( queue_state == "dead_letter" or agent_state not in {"online", "online_busy"} or remote_access_state in {"agent_pending", "disabled", "unmanaged"} ): problem_nodes.append( { "node_code": node_code, "agent_state": agent_state, "remote_access_state": remote_access_state, "delivery_queue_state": queue_state, "participation_state": str(node.get("participation_state") or "").strip(), } ) first_handover_gap_node = next( ( dict(node or {}) for node in nodes if str(node.get("node_code") or "").strip() and ( str(node.get("agent_state") or "").strip() not in {"online", "online_busy"} or str(node.get("remote_access_state") or "").strip() in {"agent_pending", "disabled", "unmanaged"} ) ), {}, ) first_handover_gap_node_code = str(first_handover_gap_node.get("node_code") or "").strip() first_handover_gap = ( get_managed_node_handover( first_handover_gap_node_code, control_plane_base_url=stack_base_url, nodes_payload=nodes_payload, ) if first_handover_gap_node_code else {} ) first_handover_gap_recovery = _resolve_first_handover_gap_recovery( first_handover_gap=first_handover_gap, launchpad_gap_action=launchpad_gap_action, launchpad_status=launchpad_status, fallback_node_code=first_handover_gap_node_code, ) problem_runs = [ { "run_code": str(item.get("run_code") or "").strip(), "status": str(item.get("status") or "").strip(), "status_label": str(item.get("status_label") or "").strip(), "focus_step_key": str(item.get("focus_step_key") or "").strip(), "focus_summary": str(item.get("focus_summary") or item.get("summary_text") or "").strip(), "focus_ref": dict(item.get("focus_ref") or {}), } for item in playbook_runs if str(item.get("status") or "").strip() not in {"success", "completed", "healthy"} ] activity_counts: dict[str, int] = {} for item in activities: status_key = str(item.get("status") or item.get("job_status") or "unknown").strip() or "unknown" activity_counts[status_key] = int(activity_counts.get(status_key, 0) or 0) + 1 start_delivery_failed_items = [ dict(item or {}) for item in activities if str(item.get("start_delivery_state") or "").strip() == "failed_local" ] issues: list[dict] = [] def add_issue( *, code: str, severity: str, layer: str, summary: str, detail: str = "", action_code: str = "", focus_ref: dict | None = None, commands: list[str] | None = None, ) -> None: contract_keys = _suggest_ops_contract_keys_for_stack_issue( { "layer": layer, "focus_ref": focus_ref or {}, "action_code": action_code, } ) issues.append( { "code": str(code or "").strip(), "severity": str(severity or "").strip() or "warning", "layer": str(layer or "").strip(), "summary": str(summary or "").strip(), "detail": str(detail or "").strip(), "action_code": str(action_code or "").strip(), "focus_ref": dict(focus_ref or {}), "commands": [str(item).strip() for item in list(commands or []) if str(item).strip()], "contract_keys": contract_keys, "contract_navigation": _build_ops_contract_navigation( contract_keys, primary_contract_key="ops_stack_diagnosis_contract", registry=contract_registry, ), } ) surface_matrix = [ ("api", True), ("contracts", bool(contracts_section.get("available", False))), ("link_snapshot", bool(link_snapshot_section.get("available", False))), ("overview", bool(overview_section.get("available", False))), ("managed_nodes", bool(nodes_section.get("available", False))), ("release_hub", bool(release_hub_section.get("available", False))), ("runtime_build_info", bool(build_info_section.get("available", False))), ("playbook_runs", bool(playbook_runs_section.get("available", False))), ("activity_stream", bool(activity_stream_section.get("available", False))), ] surface_status_counts = { "total": len(surface_matrix), "available": sum(1 for _, available in surface_matrix if available), "missing": sum(1 for _, available in surface_matrix if not available), } missing_surfaces = [name for name, available in surface_matrix if not available] if not bool(contracts_section.get("available", False)): add_issue( code="ops_contracts_unavailable", severity="warning", layer="contracts", summary="控制面契约注册表当前未能正常生成,总检无法确认当前运行的是哪一版正式 contract。", detail=str(contracts_section.get("error") or "ops contract registry unavailable").strip(), commands=[ build_bash_command("drive_ops_center.sh", "contracts", stack_base_url), build_bash_command("drive_ops_center.sh", "doctor", stack_base_url), ], ) if not bool(build_info_section.get("available", False)): add_issue( code="runtime_build_info_unavailable", severity="warning", layer="runtime_build_info", summary="当前无法读取运行中 API 的 build-info,总检暂时无法确认服务是否已经吃到最新路由面。", detail=str(build_info_section.get("error") or "runtime build-info unavailable").strip(), action_code="api-restart", commands=[ build_bash_command("drive_ops_center.sh", "runtime-refresh-recover", stack_base_url), build_bash_command("drive_ops_center.sh", "doctor", stack_base_url), ], ) elif route_surface and not bool(route_surface.get("surface_complete", False)): add_issue( code="runtime_route_surface_incomplete", severity="warning", layer="runtime_build_info.route_surface", summary="当前运行中的 API 路由面不完整,说明服务可能还没重启到最新代码,或当前发布包缺少关键运维入口。", detail=( f"missing_keys={', '.join(route_surface_missing_keys) or '-'};" f"mode={str(route_surface.get('mode') or '').strip() or 'unknown'};" f"registered_paths_total={int(route_surface.get('registered_paths_total', 0) or 0)}" ), action_code="api-restart", focus_ref={ "kind": "runtime_build_info", "section": "route_surface", "missing_keys": route_surface_missing_keys, }, commands=[ build_bash_command("drive_ops_center.sh", "runtime-refresh-recover", stack_base_url), build_bash_command("drive_ops_center.sh", "stack-diagnosis", stack_base_url, "summary"), build_bash_command("drive_ops_center.sh", "doctor", stack_base_url), ], ) elif runtime_schema_stale: add_issue( code="runtime_build_schema_stale", severity="warning", layer="runtime_build_info.schema", summary="仓库已经具备新的节点接管能力,但运行中的 build-info 仍未声明对应路由键,当前更像 API 还没重启到最新代码。", detail=( f"supports_install_command_block={repo_supports_install_command_block};" f"supports_multi_layout_bootstrap={repo_supports_multi_layout_bootstrap};" f"route_surface_declares_bootstrap_plan={route_surface_declares_bootstrap_plan}" ), action_code="api-restart", focus_ref={ "kind": "runtime_build_info", "section": "schema", "expected_route_key": "ops_node_handover_bootstrap_plan", }, commands=[ build_bash_command("drive_ops_center.sh", "runtime-refresh-recover", stack_base_url), build_bash_command("drive_ops_center.sh", "stack-diagnosis", stack_base_url, "summary"), build_bash_command("drive_ops_center.sh", "node-bootstrap-plan", stack_base_url, first_handover_gap_node_code or "mainland-worker-01"), ], ) if missing_surfaces and any( name in missing_surfaces for name in ("link_snapshot", "overview", "managed_nodes", "release_hub") ): add_issue( code="ops_surface_partial", severity="warning", layer="stack", summary="海外单脑控制面的关键观察层未全部联通,总检目前只能给出部分收口。", detail=f"当前缺失层: {', '.join(missing_surfaces)}", commands=[ build_bash_command("drive_ops_center.sh", "stack-diagnosis", stack_base_url, "full"), build_bash_command("drive_ops_center.sh", "doctor", stack_base_url), ], ) participating_nodes_total = int( log_sync.get("participating_nodes_total", log_sync.get("participating_node_count", 0)) or 0 ) participating_node_codes = _normalize_ops_node_code_list( execution_scene.get("participating_nodes") or [], (execution_scene.get("participation_summary") or {}).get("dispatch_active_node_codes") or [], (execution_scene.get("participation_summary") or {}).get("recent_only_node_codes") or [], ) missing_log_node_codes = _normalize_ops_node_code_list(log_sync.get("missing_participating_nodes") or []) scene_log_source_node_codes = _normalize_ops_node_code_list( log_sync.get("source_nodes") or [], log_sync.get("source_node_summaries") or [], ) preferred_scene_log_target_codes = _normalize_ops_node_code_list( missing_log_node_codes, participating_node_codes, scene_log_source_node_codes, ) if bool(log_sync.get("enabled", False)) is False and participating_nodes_total > 0: log_sync_action = "" for item in list(link_snapshot.get("next_actions") or []): action_code = str(item.get("action_code") or "").strip() if action_code.startswith("enable_log_sync"): log_sync_action = action_code break if not log_sync_action: log_sync_action = "enable_log_sync_key" add_issue( code="remote_log_sync_disabled", severity="warning", layer="overview.log_sync", summary="现场已有参与检测节点,但远端日志回传仍关闭,海外控制面对执行细节仍处于半盲态。", detail=( f"参与检测节点 {participating_nodes_total} 台," f"已覆盖 {int(log_sync.get('covered_participating_node_count', 0) or 0)} 台。" ), action_code=log_sync_action, focus_ref=_build_scene_node_log_focus_ref( (preferred_scene_log_target_codes or participating_node_codes or [""])[0], mode="key", limit=80, source="remote_log_sync_disabled", ), commands=[ build_bash_command("drive_ops_center.sh", "driver-resolve", stack_base_url, log_sync_action), *_build_scene_node_log_commands( stack_base_url, preferred_scene_log_target_codes, limit=80, mode="key", max_commands=2, ), ], ) elif bool(log_sync.get("enabled", False)) and participating_nodes_total > 0 and ( missing_log_node_codes or int(log_sync.get("line_count", 0) or 0) <= 0 or int(log_sync.get("source_node_count", 0) or 0) <= 0 ): waiting_node_codes = preferred_scene_log_target_codes or participating_node_codes add_issue( code="remote_log_sync_waiting_sample", severity="warning", layer="overview.log_sync", summary="远端日志回传虽然已经开启,但参与检测节点的现场样本仍未完全形成。", detail=( f"参与检测节点 {participating_nodes_total} 台," f"已覆盖 {int(log_sync.get('covered_participating_node_count', 0) or 0)} 台," f"缺口节点 {_format_node_code_list(missing_log_node_codes)}。" ), action_code="open_worker_logs_participating", focus_ref=_build_scene_node_log_focus_ref( (waiting_node_codes or [""])[0], mode=str(log_sync.get("mode") or "key"), limit=120, source="remote_log_sync_waiting_sample", ), commands=[ *_build_scene_node_log_commands( stack_base_url, waiting_node_codes, limit=120, mode=str(log_sync.get("mode") or "key"), max_commands=3, ), build_bash_command("drive_ops_center.sh", "driver-resolve", stack_base_url, "open_worker_logs_participating"), build_bash_command("drive_ops_center.sh", "driver-resolve", stack_base_url, "run_inspection_participating"), ], ) managed_enabled = int(nodes_summary.get("managed_enabled", 0) or 0) remote_access_ready = int(nodes_summary.get("remote_access_ready", 0) or 0) if managed_enabled > 0 and remote_access_ready == 0: first_handover_gap_stage = dict(first_handover_gap.get("stage") or {}) first_handover_gap_next_step = dict(first_handover_gap_stage.get("next_step") or {}) first_handover_gap_summary = str(first_handover_gap.get("summary") or "").strip() managed_nodes_action_code = str( first_handover_gap_recovery.get("action_code") or launchpad_status.get("recommended_action_code") or "fix_managed_nodes" ).strip() add_issue( code="managed_nodes_agent_pending", severity="blocked", layer="managed_nodes", summary="托管节点虽然已经纳入 Ops Center,但 0 台 remote-agent 就绪,海外主控还不能统一下发标准执行动作。", detail=( f"managed_enabled={managed_enabled},remote_access_ready={remote_access_ready};" "当前节点多数仍处于 agent_pending / runtime_only / pending_bootstrap。" + ( f" 首个缺口节点 {first_handover_gap_node_code}:" f"{str(first_handover_gap_stage.get('label') or '').strip() or '待处理'}," f"{first_handover_gap_summary}" if first_handover_gap_node_code and first_handover_gap_summary else "" ) ), action_code=managed_nodes_action_code, focus_ref=dict(first_handover_gap_recovery.get("focus_ref") or {}), commands=[ build_bash_command("drive_ops_center.sh", "driver-resolve", stack_base_url, managed_nodes_action_code), *( [build_bash_command("drive_ops_center.sh", "node-handover", stack_base_url, first_handover_gap_node_code)] if first_handover_gap_node_code else [] ), *( [ build_bash_command("drive_ops_center.sh", "node-bootstrap-plan", stack_base_url, first_handover_gap_node_code) ] if first_handover_gap_node_code and str(first_handover_gap_next_step.get("code") or "").strip() not in {"ready", "run_acceptance"} else [] ), *( [ build_bash_command( "drive_ops_center.sh", "node-bootstrap-run" if managed_nodes_action_code == "bootstrap_run" else "node-acceptance-run", stack_base_url, first_handover_gap_node_code, "cli", ) ] if first_handover_gap_node_code and managed_nodes_action_code in {"bootstrap_run", "run_acceptance"} else [] ), build_bash_command("drive_ops_center.sh", "doctor", stack_base_url), ], ) queue_dead_letter_nodes = int(nodes_summary.get("queue_dead_letter_nodes", 0) or 0) queue_dead_letter_records = int(nodes_summary.get("queue_dead_letter_records", 0) or 0) if queue_dead_letter_nodes > 0 or queue_dead_letter_records > 0: add_issue( code="delivery_queue_dead_letter", severity="warning", layer="managed_nodes.queue", summary="存在 Node Agent 回执死信,后续自动化动作可能反复失败或无法收敛。", detail=( f"queue_dead_letter_nodes={queue_dead_letter_nodes}," f"queue_dead_letter_records={queue_dead_letter_records}" ), commands=[ build_bash_command("drive_ops_center.sh", "doctor", stack_base_url), ], ) if str(default_rollout_gate.get("status") or "").strip() == "blocked": add_issue( code="release_rollout_gate_blocked", severity="warning", layer="release_hub", summary="ReleaseHub 默认门禁当前阻断,说明虽然发布包已准备好,但正式 Rollout 还不具备条件。", detail=str(default_rollout_gate.get("summary_text") or default_rollout_gate.get("summary") or "").strip(), action_code=str(launchpad_status.get("recommended_action_code") or "").strip(), focus_ref=dict(default_rollout_gate.get("focus_ref") or launchpad_status.get("focus_ref") or {}), commands=[ build_bash_command("drive_ops_center.sh", "release-launchpad", stack_base_url), build_bash_command( "drive_ops_center.sh", "driver-resolve", stack_base_url, str(launchpad_status.get("recommended_action_code") or "fix_managed_nodes").strip(), ), ], ) if problem_runs: add_issue( code="playbook_runs_need_attention", severity="warning", layer="playbook_runs", summary="最近存在未完全收口的 playbook run,建议先确认执行序列是否停在中间步骤。", detail=f"problem_runs_total={len(problem_runs)}", commands=[ build_bash_command("drive_ops_center.sh", "doctor", stack_base_url), ], ) if start_delivery_failed_items: first_delivery_gap_item = dict(start_delivery_failed_items[0] or {}) first_delivery_focus_ref = dict( first_delivery_gap_item.get("source_focus_ref") or first_delivery_gap_item.get("focus_ref") or {} ) first_delivery_job_code = str(first_delivery_gap_item.get("job_code") or "").strip() first_delivery_node_code = str(first_delivery_focus_ref.get("target_node_code") or "").strip() add_issue( code="ops_job_start_delivery_failed_local", severity="warning", layer="activity_stream.ops_job", summary=( f"最近有 {len(start_delivery_failed_items)} 条标准运维任务在节点侧已开始执行,但开始回执没有成功送达控制面。" ), detail=" / ".join( part for part in [ f"首条任务 {first_delivery_job_code}" if first_delivery_job_code else "", f"目标节点 {first_delivery_node_code}" if first_delivery_node_code else "", str(first_delivery_gap_item.get("start_delivery_error") or "").strip(), str(first_delivery_gap_item.get("summary_text") or first_delivery_gap_item.get("summary") or "").strip(), ] if part ), action_code="focus_latest_job_events", focus_ref=first_delivery_focus_ref, commands=[ build_bash_command("drive_ops_center.sh", "driver-resolve", stack_base_url, "focus_latest_job_events"), build_bash_command("drive_ops_center.sh", "doctor", stack_base_url), ], ) ordered_action_candidates = [ { "source": "overview.primary", "action_code": str(recommendation.get("primary_action_code") or "").strip(), "title": str(recommendation.get("title") or "").strip(), "reason": "来自 overview.recommendation.primary_action_code", "focus_ref": dict(recommendation.get("focus_ref") or {}), }, { "source": "release_hub.launchpad", "action_code": str(launchpad_status.get("recommended_action_code") or "").strip(), "title": str(launchpad_status.get("status_label") or "").strip(), "reason": "来自 release_launchpad.launchpad_status.recommended_action_code", "focus_ref": dict(launchpad_status.get("focus_ref") or {}), }, { "source": "overview.secondary", "action_code": str(recommendation.get("secondary_action_code") or "").strip(), "title": str(recommendation.get("title") or "").strip(), "reason": "来自 overview.recommendation.secondary_action_code", "focus_ref": dict(recommendation.get("focus_ref") or {}), }, ] if str(first_handover_gap_recovery.get("action_code") or "").strip() in {"bootstrap_run", "run_acceptance"}: ordered_action_candidates.insert( 0, { "source": "managed_nodes.first_handover_gap", "action_code": str(first_handover_gap_recovery.get("action_code") or "").strip(), "title": str(first_handover_gap_recovery.get("recovery_label") or "").strip(), "reason": str(first_handover_gap_recovery.get("summary") or "").strip() or "来自首个 handover gap 的标准恢复动作。", "focus_ref": dict(first_handover_gap_recovery.get("focus_ref") or {}), }, ) for issue in issues: if str(issue.get("code") or "").strip() == "runtime_build_schema_stale": ordered_action_candidates.insert( 0, { "source": f"issue:{issue.get('code')}", "action_code": "api-restart", "title": "优先重启控制面 API", "reason": str(issue.get("detail") or issue.get("summary") or "").strip(), "focus_ref": dict(issue.get("focus_ref") or {}), }, ) break for issue in issues: issue_action_code = str(issue.get("action_code") or "").strip() if issue_action_code: ordered_action_candidates.append( { "source": f"issue:{issue.get('code')}", "action_code": issue_action_code, "title": str(issue.get("summary") or "").strip(), "reason": str(issue.get("detail") or issue.get("summary") or "").strip(), "focus_ref": dict(issue.get("focus_ref") or {}), } ) recommended_actions: list[dict] = [] seen_action_codes: set[str] = set() for item in ordered_action_candidates: action_code = str(item.get("action_code") or "").strip() if not action_code or action_code in seen_action_codes: continue seen_action_codes.add(action_code) recommended_action_contract_keys = _normalize_ops_contract_keys( ["ops_stack_diagnosis_contract"] + _ops_contract_keys_from_action_code(action_code) + _ops_contract_keys_from_focus_ref(item.get("focus_ref")) ) recommended_actions.append( { "action_code": action_code, "source": str(item.get("source") or "").strip(), "title": str(item.get("title") or "").strip(), "reason": str(item.get("reason") or "").strip(), "focus_ref": dict(item.get("focus_ref") or {}), "command": build_bash_command("drive_ops_center.sh", "driver-resolve", stack_base_url, action_code), "contract_keys": recommended_action_contract_keys, "contract_navigation": _build_ops_contract_navigation( recommended_action_contract_keys, primary_contract_key="ops_stack_diagnosis_contract", registry=contract_registry, ), } ) blocking_issue_total = sum(1 for item in issues if str(item.get("severity") or "").strip() == "blocked") warning_issue_total = sum(1 for item in issues if str(item.get("severity") or "").strip() == "warning") info_issue_total = sum(1 for item in issues if str(item.get("severity") or "").strip() == "info") surface_status = "healthy" if missing_surfaces: surface_status = "partial" automation_status = "ready" if blocking_issue_total > 0: automation_status = "blocked" elif warning_issue_total > 0: automation_status = "attention" stack_status = "ready" if automation_status == "blocked": stack_status = "blocked" elif surface_status != "healthy" or automation_status != "ready": stack_status = "attention" top_issue = None if issues: top_issue = sorted( issues, key=lambda item: ( -_ops_stack_issue_rank(str(item.get("severity") or "").strip()), str(item.get("code") or "").strip(), ), )[0] next_step = {} if recommended_actions: first_action = dict(recommended_actions[0] or {}) next_step_contract_keys = _normalize_ops_contract_keys( ["ops_stack_diagnosis_contract"] + _ops_contract_keys_from_action_code(str(first_action.get("action_code") or "").strip()) + _ops_contract_keys_from_focus_ref(first_action.get("focus_ref")) ) next_step = { "action_code": str(first_action.get("action_code") or "").strip(), "source": str(first_action.get("source") or "").strip(), "reason": str(first_action.get("reason") or "").strip(), "command": str(first_action.get("command") or "").strip(), "focus_ref": dict(first_action.get("focus_ref") or {}), "contract_keys": next_step_contract_keys, "contract_navigation": _build_ops_contract_navigation( next_step_contract_keys, primary_contract_key="ops_stack_diagnosis_contract", registry=contract_registry, ), } elif top_issue: next_step_contract_keys = _normalize_ops_contract_keys(top_issue.get("contract_keys") or ["ops_stack_diagnosis_contract"]) next_step = { "action_code": "", "source": f"issue:{str(top_issue.get('code') or '').strip()}", "reason": str(top_issue.get("summary") or "").strip(), "command": "", "focus_ref": dict(top_issue.get("focus_ref") or {}), "contract_keys": next_step_contract_keys, "contract_navigation": _build_ops_contract_navigation( next_step_contract_keys, primary_contract_key="ops_stack_diagnosis_contract", registry=contract_registry, ), } quick_commands = [ build_bash_command("drive_ops_center.sh", "stack-diagnosis", stack_base_url, "summary"), build_bash_command("drive_ops_center.sh", "doctor", stack_base_url), ] for action in recommended_actions[:3]: command = str(action.get("command") or "").strip() if command and command not in quick_commands: quick_commands.append(command) for issue in issues[:5]: for command in list(issue.get("commands") or []): normalized_command = str(command or "").strip() if normalized_command and normalized_command not in quick_commands: quick_commands.append(normalized_command) if len(quick_commands) >= 8: break if len(quick_commands) >= 8: break if not bool(contracts_section.get("available", False)): contract_check_command = build_bash_command("drive_ops_center.sh", "contracts", stack_base_url) if contract_check_command not in quick_commands: quick_commands.append(contract_check_command) recommended_commands = _build_stack_diagnosis_recommended_commands( stack_base_url=stack_base_url, recommended_actions=recommended_actions, first_handover_gap_node_code=first_handover_gap_node_code, launchpad_status=launchpad_status, default_rollout_gate=default_rollout_gate, next_step=next_step, ) operator_decision, next_actions = _build_stack_diagnosis_operator_decision( issues=issues, next_step=next_step, recommended_actions=recommended_actions, recommended_commands=recommended_commands, first_handover_gap_node_code=first_handover_gap_node_code, launchpad_status=launchpad_status, default_rollout_gate=default_rollout_gate, ) operator_hints: list[str] = [] if stack_status == "blocked": operator_hints.append("当前总检已经能看到现场,但海外单脑自动化执行链路仍未正式打通,先处理阻断问题再继续 Rollout 或批量运维。") elif stack_status == "attention": operator_hints.append("当前总检可用,但仍有缺口;建议先按 next_step 收口,再进入更细粒度脚本。") else: operator_hints.append("当前海外单脑控制面总检已基本就绪,可以把它作为后续所有联调和发布的固定起手式。") if not bool(contracts_section.get("available", False)): operator_hints.append("contracts 当前未正常可用,优先先恢复 contract registry,再让页面、CLI、Codex 共享同一份正式契约。") if route_surface_missing_keys: operator_hints.append("build-info 显示当前运行中的 API 路由面还不完整,先重启 API 并确认最新运维入口已经注册,再继续做节点接管或发布判断。") if any(str(item.get("code") or "").strip() == "runtime_build_schema_stale" for item in issues): operator_hints.append("当前属于典型的“仓库代码已更新,但运行中 API 还是旧 schema”场景,先重启 domaincheck-api,再重新看 stack-diagnosis 与 node-bootstrap-plan。") if bool(log_sync.get("enabled", False)) is False and participating_nodes_total > 0: operator_hints.append("远端日志回传当前关闭,后续如果继续多机联调,海外控制面会继续处于半盲态。") next_step_action_code = str(next_step.get("action_code") or "").strip() next_step_focus_ref = dict(next_step.get("focus_ref") or {}) if next_step_action_code in {"bootstrap_run", "run_acceptance"}: next_step_node_code = str(next_step_focus_ref.get("node_code") or first_handover_gap_node_code or "").strip() operator_hints.append( ( f"当前第一动作已经收敛为 {'接入收口' if next_step_action_code == 'bootstrap_run' else '接管验收'}" + (f":{next_step_node_code}" if next_step_node_code else "") + ",页面、CLI、Codex 都应优先围绕这一个节点推进,不再回退到泛化的托管节点修复。" ) ) resolved_launchpad_target_node_code = str(launchpad_status.get("recommended_target_node_code") or "").strip() resolved_launchpad_recovery_label = str(launchpad_status.get("recommended_recovery_label") or "").strip() resolved_launchpad_recovery_summary = str(launchpad_status.get("recommended_recovery_summary") or "").strip() if ( str(launchpad_status.get("recommended_action_code") or "").strip() not in {"bootstrap_run", "run_acceptance"} and str(first_handover_gap_recovery.get("action_code") or "").strip() in {"bootstrap_run", "run_acceptance"} ): resolved_launchpad_target_node_code = ( resolved_launchpad_target_node_code or str(next_step_focus_ref.get("node_code") or "").strip() or str(first_handover_gap_recovery.get("node_code") or "").strip() ) resolved_launchpad_recovery_label = ( resolved_launchpad_recovery_label or str(first_handover_gap_recovery.get("recovery_label") or "").strip() ) resolved_launchpad_recovery_summary = ( resolved_launchpad_recovery_summary or str(first_handover_gap_recovery.get("summary") or "").strip() ) diagnosis_contract_keys = _normalize_ops_contract_keys( ["ops_stack_diagnosis_contract", "ops_driver_contract"] + [contract_key for issue in issues for contract_key in list(issue.get("contract_keys") or [])] + list(next_step.get("contract_keys") or []) ) return { "generated_at": datetime.now().strftime("%Y-%m-%d %H:%M:%S"), "diagnosis": { "contract_key": "ops_stack_diagnosis_contract", "contract_version": _OPS_CONTRACT_SCHEMA_VERSION, "registry_version": _OPS_CONTRACT_REGISTRY_VERSION, "base_url": stack_base_url, "contract_navigation": _build_ops_contract_navigation( diagnosis_contract_keys, primary_contract_key="ops_stack_diagnosis_contract", registry=contract_registry, ), "surface_status": surface_status, "automation_status": automation_status, "stack_status": stack_status, "issue_total": len(issues), "blocking_issue_total": blocking_issue_total, "warning_issue_total": warning_issue_total, "info_issue_total": info_issue_total, "missing_surfaces": missing_surfaces, "launchpad_recommended_target_node_code": resolved_launchpad_target_node_code, "launchpad_recommended_recovery_label": resolved_launchpad_recovery_label, "launchpad_recommended_recovery_summary": resolved_launchpad_recovery_summary, "launchpad_onboarding_bootstrap_pending_nodes": int( launchpad_status.get("onboarding_bootstrap_pending_nodes", 0) or 0 ), "launchpad_onboarding_acceptance_ready_nodes": int( launchpad_status.get("onboarding_acceptance_ready_nodes", 0) or 0 ), "next_step": next_step, "recommended_actions": recommended_actions[:5], "operator_decision": operator_decision, "next_actions": next_actions, "recommended_commands": recommended_commands, "issues": issues[:8], "operator_hints": operator_hints, "quick_commands": quick_commands, }, "api": { "http_status": "200", "available": True, **api_info, }, "surface_matrix": { "status_counts": surface_status_counts, "items": [ { "name": name, "available": bool(available), "elapsed_ms": int( { "contracts": contracts_section, "link_snapshot": link_snapshot_section, "overview": overview_section, "managed_nodes": nodes_section, "release_hub": release_hub_section, "runtime_build_info": build_info_section, "playbook_runs": playbook_runs_section, "activity_stream": activity_stream_section, }.get(name, {}).get("elapsed_ms", 0) or 0 ), } for name, available in surface_matrix ], }, "contracts": { "http_status": str(contracts_section.get("http_status") or "500"), "available": bool(contracts_section.get("available", False)), "elapsed_ms": int(contracts_section.get("elapsed_ms", 0) or 0), "registry_version": str(contracts.get("registry_version") or ""), "schema_version": str(contracts.get("schema_version") or ""), "contracts_total": len(contracts_rows), "contract_keys": [str(item.get("key") or "").strip() for item in contracts_rows if str(item.get("key") or "").strip()], "error": str(contracts_section.get("error") or ""), }, "link_snapshot": { "http_status": str(link_snapshot_section.get("http_status") or "500"), "available": bool(link_snapshot_section.get("available", False)), "elapsed_ms": int(link_snapshot_section.get("elapsed_ms", 0) or 0), "status": str(link_snapshot.get("status") or ""), "headline": str(link_snapshot.get("headline") or ""), "next_action_codes": [ str(item.get("action_code") or "").strip() for item in list(link_snapshot.get("next_actions") or []) if str(item.get("action_code") or "").strip() ], "error": str(link_snapshot_section.get("error") or ""), }, "overview": { "http_status": str(overview_section.get("http_status") or "500"), "available": bool(overview_section.get("available", False)), "elapsed_ms": int(overview_section.get("elapsed_ms", 0) or 0), "recommendation_key": str(recommendation.get("key") or ""), "recommendation_title": str(recommendation.get("title") or ""), "primary_action_code": str(recommendation.get("primary_action_code") or ""), "secondary_action_code": str(recommendation.get("secondary_action_code") or ""), "focus_ref": dict(recommendation.get("focus_ref") or {}), "execution_scene_summary": str( execution_scene.get("summary") or (execution_scene.get("participation_summary") or {}).get("summary") or execution_scene.get("phase_detail") or "" ), "execution_scene_counts": dict(execution_scene.get("counts") or {}), "log_sync": { "status": str(log_sync.get("status") or log_sync.get("state") or ""), "status_label": str(log_sync.get("status_label") or ""), "mode": str(log_sync.get("mode") or ""), "mode_label": str(log_sync.get("mode_label") or ""), "enabled": bool(log_sync.get("enabled", False)), "covered_participating_nodes": int(log_sync.get("covered_participating_node_count", 0) or 0), "participating_nodes_total": int( log_sync.get("participating_nodes_total", log_sync.get("participating_node_count", 0)) or 0 ), "missing_sample_node_codes": list( log_sync.get("missing_sample_node_codes") or log_sync.get("missing_participating_nodes") or [] ), }, "error": str(overview_section.get("error") or ""), }, "managed_nodes": { "http_status": str(nodes_section.get("http_status") or "500"), "available": bool(nodes_section.get("available", False)), "elapsed_ms": int(nodes_section.get("elapsed_ms", 0) or 0), "total": int(nodes_summary.get("total", len(nodes)) or 0), "managed_total": int(nodes_summary.get("managed_total", 0) or 0), "managed_enabled": int(nodes_summary.get("managed_enabled", 0) or 0), "online": int(nodes_summary.get("online", 0) or 0), "agent_ready": int(nodes_summary.get("agent_ready", 0) or 0), "ssh_ready": int(nodes_summary.get("ssh_ready", 0) or 0), "remote_access_ready": int(nodes_summary.get("remote_access_ready", 0) or 0), "participating": int(nodes_summary.get("participating", 0) or 0), "dispatch_active": int(nodes_summary.get("dispatch_active", 0) or 0), "standby": int(nodes_summary.get("standby", 0) or 0), "load_syncing": int(nodes_summary.get("load_syncing", 0) or 0), "queue_retrying_nodes": int(nodes_summary.get("queue_retrying_nodes", 0) or 0), "queue_dead_letter_nodes": int(nodes_summary.get("queue_dead_letter_nodes", 0) or 0), "queue_pending_records": int(nodes_summary.get("queue_pending_records", 0) or 0), "queue_dead_letter_records": int(nodes_summary.get("queue_dead_letter_records", 0) or 0), "agent_state_counts": dict(nodes_summary.get("status_counts") or {}), "remote_access_state_counts": dict(nodes_summary.get("remote_access_state_counts") or {}), "delivery_queue_state_counts": dict(nodes_summary.get("delivery_queue_state_counts") or {}), "problem_nodes": problem_nodes[:8], "first_handover_gap_node_code": first_handover_gap_node_code, "first_handover_gap": first_handover_gap, "error": str(nodes_section.get("error") or ""), }, "release_hub": { "http_status": str(release_hub_section.get("http_status") or "500"), "available": bool(release_hub_section.get("available", False)), "elapsed_ms": int(release_hub_section.get("elapsed_ms", 0) or 0), "latest_release_version": str(latest_release.get("release_version") or ""), "latest_release_status": str(latest_release.get("status") or ""), "default_rollout_gate_status": str(default_rollout_gate.get("status") or ""), "default_rollout_gate_label": str(default_rollout_gate.get("status_label") or ""), "default_rollout_gate_summary": str(default_rollout_gate.get("summary_text") or default_rollout_gate.get("summary") or ""), "default_rollout_gate_focus_ref": dict(default_rollout_gate.get("focus_ref") or {}), "launchpad_status": str(launchpad_status.get("status") or ""), "launchpad_status_label": str(launchpad_status.get("status_label") or ""), "launchpad_summary": str(launchpad_status.get("summary_text") or launchpad_status.get("summary") or ""), "recommended_action_code": str(launchpad_status.get("recommended_action_code") or ""), "recommended_execution_mode": str(launchpad_status.get("recommended_execution_mode") or ""), "focus_ref": dict(launchpad_status.get("focus_ref") or {}), "error": str(release_hub_section.get("error") or ""), }, "runtime_build_info": { "http_status": str(build_info_section.get("http_status") or "500"), "available": bool(build_info_section.get("available", False)), "elapsed_ms": int(build_info_section.get("elapsed_ms", 0) or 0), "source": str(build_info.get("source") or ""), "package_name": str(build_info.get("package_name") or ""), "generated_at": str(build_info.get("generated_at") or ""), "commit_sha": str(build_info.get("commit_sha") or ""), "commit_ref": str(build_info.get("commit_ref") or ""), "checksum": str(build_info.get("checksum") or ""), "manifest_path": str(build_info.get("manifest_path") or ""), "repository_capabilities": repository_capabilities, "route_surface": route_surface, "route_surface_missing_keys": route_surface_missing_keys, "route_surface_declares_bootstrap_plan": route_surface_declares_bootstrap_plan, "runtime_schema_stale": runtime_schema_stale, "error": str(build_info_section.get("error") or ""), }, "playbook_runs": { "http_status": str(playbook_runs_section.get("http_status") or "500"), "available": bool(playbook_runs_section.get("available", False)), "elapsed_ms": int(playbook_runs_section.get("elapsed_ms", 0) or 0), "recent_total": len(playbook_runs), "problem_runs_total": len(problem_runs), "problem_runs": problem_runs[:5], "error": str(playbook_runs_section.get("error") or ""), }, "activity_stream": { "http_status": str(activity_stream_section.get("http_status") or "500"), "available": bool(activity_stream_section.get("available", False)), "elapsed_ms": int(activity_stream_section.get("elapsed_ms", 0) or 0), "recent_total": len(activities), "status_counts": activity_counts, "top_items": [ { "kind": str(item.get("kind") or "").strip(), "status": str(item.get("status") or item.get("job_status") or "").strip(), "summary": str(item.get("summary_text") or item.get("summary") or "").strip(), "occurred_at": str(item.get("occurred_at") or "").strip(), "focus_ref": dict(item.get("focus_ref") or {}), "source_focus_ref": dict(item.get("source_focus_ref") or {}), "ui_intent_kind": str(((item.get("ui_intent") or {}).get("kind") or "")).strip(), } for item in activities[:5] ], "error": str(activity_stream_section.get("error") or ""), }, } def get_ops_capabilities() -> dict: contract_registry = get_ops_contract_registry() return { "control_plane": { "deployment_mode": "overseas-control-plane", "codex_driver_ready": True, "ui_button_ready": True, "daily_entrypoint": "overseas dashboard / ops api", }, "contract_registry": { "endpoint": "/api/v1/ops/contracts", "detail_endpoint_pattern": "/api/v1/ops/contracts/{contract_key}", "registry_version": str(contract_registry.get("registry_version") or _OPS_CONTRACT_REGISTRY_VERSION), "schema_version": str(contract_registry.get("schema_version") or _OPS_CONTRACT_SCHEMA_VERSION), "contracts_total": int(contract_registry.get("contracts_total", 0) or 0), "contract_keys": [ str(item.get("key") or "").strip() for item in list(contract_registry.get("contracts") or []) if str(item.get("key") or "").strip() ], "schema_docs": { str(item.get("key") or "").strip(): str(item.get("schema_doc_path") or "").strip() for item in list(contract_registry.get("contracts") or []) if str(item.get("key") or "").strip() }, }, "capabilities": [ { "key": "cluster_visibility", "title": "集群可观测", "current_state": "ready", "today": "已具备 runtime/cluster/readiness/sync-summary 聚合能力。", "target": "继续增强节点参与度、诊断报告和远端日志回传。", }, { "key": "remote_log_sync", "title": "远端检测日志回传", "current_state": "ready", "today": "已具备日志回传开关、关键/全量模式、聚合样本面板与按节点现场日志钻取。", "target": "后续如规模继续扩大,再补游标续传、流式 tail 与更长窗口归档。", }, { "key": "button_driven_ops", "title": "后台按钮化运维", "current_state": "partial", "today": "已有 runtime actions / ops jobs / ops playbooks,开始收口为统一按钮化编排。", "target": "继续扩大安装、更新、重启、巡检、诊断包收集与发布回滚 playbook。", }, { "key": "node_agent", "title": "节点 Agent", "current_state": "in_progress", "today": "已具备 agent token / register / heartbeat / pull / complete 协议与本地执行器骨架。", "target": "继续增强 stdout/stderr 游标回传、流式日志、诊断包与更多动作模板。", }, { "key": "release_management", "title": "发布包管理", "current_state": "partial", "today": "已具备 release / rollout / deploy.release 能力,支持 checksum、切换 current、健康检查、失败回滚,以及 Agent / SSH 双通道单节点发布。", "target": "继续增强批次策略、灰度放量、发布观测面板与一键回滚编排。", }, ], "action_catalog": [ {"action": "node.bootstrap", "title": "纳管新节点", "transport": "ops-job -> control-plane -> bootstrap-plan"}, {"action": "deploy.release", "title": "发布新版本", "transport": "ops-job -> node-agent / ssh-executor"}, { "action": "deploy.release.smart_worker", "title": "按最新包一键 Worker 灰度", "transport": "ops api -> package release builder -> rollout engine", }, { "action": "deploy.release.smart_control", "title": "按最新包一键 Control 发布", "transport": "ops api -> package release builder -> rollout engine", }, { "action": "deploy.rollout.smart_worker", "title": "对已有 Release 一键 Worker 灰度", "transport": "ops api -> rollout engine", }, { "action": "deploy.rollout.smart_control", "title": "对已有 Release 一键 Control 发布", "transport": "ops api -> rollout engine", }, {"action": "service.restart", "title": "重启 API / Worker / Sync-Agent", "transport": "ops-job -> node-agent / ssh-executor"}, {"action": "logs.collect", "title": "收集远端日志", "transport": "ops-job -> node-agent / ssh-executor"}, {"action": "diagnostics.collect", "title": "收集诊断包", "transport": "ops-job -> node-agent / ssh-executor"}, {"action": "ops.playbook.execute", "title": "执行标准 playbook", "transport": "ops api -> playbook expander -> ops-jobs"}, {"action": "health.check", "title": "执行巡检", "transport": "ops-job -> node-agent"}, {"action": "deploy.rollout.advance", "title": "推进下一批发布", "transport": "ops api -> rollout engine"}, {"action": "runtime.restart_api", "title": "本机重启 API", "transport": "ops-job -> local-runtime / ssh-executor"}, {"action": "runtime.start_worker", "title": "本机启动 Worker", "transport": "ops-job -> local-runtime / ssh-executor"}, {"action": "runtime.stop_worker", "title": "本机停止 Worker", "transport": "ops-job -> local-runtime / ssh-executor"}, {"action": "runtime.start_sync_agent", "title": "本机启动 Sync Agent", "transport": "ops-job -> local-runtime / ssh-executor"}, {"action": "runtime.stop_sync_agent", "title": "本机停止 Sync Agent", "transport": "ops-job -> local-runtime / ssh-executor"}, {"action": "runtime.pull_tasks", "title": "本机立即拉取任务", "transport": "ops-job -> local-runtime"}, {"action": "runtime.push_sync", "title": "本机立即推送同步", "transport": "ops-job -> local-runtime"}, {"action": "health.snapshot", "title": "采集本机运行时快照", "transport": "ops-job -> local-runtime / ssh-executor"}, ], } def get_ops_go_live_summary(*, base_url: str = "") -> dict: normalized_base_url = str(base_url or "").strip() or "http://127.0.0.1:8100" stack_payload = get_ops_stack_diagnosis(base_url=normalized_base_url) contracts_payload = get_ops_contract_registry() launchpad_payload = get_release_launchpad() build_info_payload = get_runtime_build_info() nodes_payload = list_managed_nodes_with_agent_state() overview_payload = get_ops_overview() diagnosis = dict(stack_payload.get("diagnosis") or {}) diagnosis_issues = [dict(item or {}) for item in list(diagnosis.get("issues") or []) if item] route_surface = dict(build_info_payload.get("route_surface") or {}) repository_capabilities = dict(build_info_payload.get("repository_capabilities") or {}) nodes_summary = dict(nodes_payload.get("summary") or {}) execution_scene = dict(overview_payload.get("execution_scene") or {}) log_sync = dict(execution_scene.get("log_sync") or overview_payload.get("log_sync") or {}) recommendation = dict(overview_payload.get("recommendation") or {}) launchpad_status = dict(launchpad_payload.get("launchpad_status") or {}) next_step = dict(diagnosis.get("next_step") or {}) operator_decision = dict(diagnosis.get("operator_decision") or {}) contracts = [dict(item or {}) for item in list(contracts_payload.get("contracts") or []) if item] stack_status = str(diagnosis.get("stack_status") or "").strip() or "unknown" launchpad_status_code = ( str(launchpad_status.get("status") or "").strip() or str((stack_payload.get("release_hub") or {}).get("launchpad_status") or "").strip() ) managed_enabled = int(nodes_summary.get("managed_enabled", 0) or 0) remote_access_ready = int(nodes_summary.get("remote_access_ready", 0) or 0) queue_dead_letter_nodes = int(nodes_summary.get("queue_dead_letter_nodes", 0) or 0) participating_nodes_total = int(log_sync.get("participating_node_count", 0) or 0) log_sync_covered_nodes = int(log_sync.get("covered_participating_node_count", 0) or 0) launchpad_onboarding_bootstrap_pending_nodes = int( launchpad_status.get("onboarding_bootstrap_pending_nodes", 0) or 0 ) launchpad_onboarding_acceptance_ready_nodes = int( launchpad_status.get("onboarding_acceptance_ready_nodes", 0) or 0 ) start_delivery_issue_total = sum( 1 for item in diagnosis_issues if str(item.get("code") or "").strip() == "ops_job_start_delivery_failed_local" ) blocking_reasons: list[str] = [] warnings: list[str] = [] if stack_status == "blocked": blocking_reasons.append("stack_diagnosis=blocked") elif stack_status == "attention": warnings.append("stack_diagnosis=attention") route_surface_complete = bool(route_surface.get("surface_complete", False)) route_surface_missing_keys = [ str(item or "").strip() for item in list(route_surface.get("missing_keys") or []) if str(item or "").strip() ] route_surface_expected_paths = { str(key).strip(): str(value).strip() for key, value in dict(route_surface.get("expected_paths") or {}).items() if str(key).strip() } route_surface_declares_bootstrap_plan = "ops_node_handover_bootstrap_plan" in route_surface_expected_paths runtime_schema_stale = bool( repository_capabilities.get("supports_install_command_block", False) and not route_surface_declares_bootstrap_plan ) if not route_surface_complete: blocking_reasons.append( "route_surface_incomplete" + (f":{','.join(route_surface_missing_keys)}" if route_surface_missing_keys else "") ) elif runtime_schema_stale: warnings.append("runtime_build_schema_stale") if not contracts: blocking_reasons.append("contracts_unavailable") if launchpad_status_code == "blocked": blocking_reasons.append("release_launchpad=blocked") elif launchpad_status_code == "attention": warnings.append("release_launchpad=attention") if launchpad_onboarding_bootstrap_pending_nodes > 0: warnings.append(f"launchpad_onboarding_bootstrap_pending={launchpad_onboarding_bootstrap_pending_nodes}") if launchpad_onboarding_acceptance_ready_nodes > 0: warnings.append(f"launchpad_onboarding_acceptance_ready={launchpad_onboarding_acceptance_ready_nodes}") if managed_enabled > 0 and remote_access_ready == 0: blocking_reasons.append("remote_agent_ready=0") elif managed_enabled > 0 and remote_access_ready < managed_enabled: warnings.append(f"remote_agent_ready={remote_access_ready}/{managed_enabled}") if queue_dead_letter_nodes > 0: warnings.append(f"queue_dead_letter_nodes={queue_dead_letter_nodes}") if start_delivery_issue_total > 0: warnings.append(f"ops_job_start_delivery_failed_local={start_delivery_issue_total}") if bool(log_sync.get("enabled")) and participating_nodes_total > 0: if log_sync_covered_nodes == 0: warnings.append(f"log_sync_waiting_sample=0/{participating_nodes_total}") elif log_sync_covered_nodes < participating_nodes_total: warnings.append(f"log_sync_partial={log_sync_covered_nodes}/{participating_nodes_total}") publish_blocking_reasons: list[str] = [] publish_warnings: list[str] = [] if stack_status == "blocked": publish_blocking_reasons.append("stack_diagnosis=blocked") elif stack_status == "attention": publish_warnings.append("stack_diagnosis=attention") if not route_surface_complete: publish_blocking_reasons.append( "route_surface_incomplete" + (f":{','.join(route_surface_missing_keys)}" if route_surface_missing_keys else "") ) elif runtime_schema_stale: publish_warnings.append("runtime_build_schema_stale") if not contracts: publish_blocking_reasons.append("contracts_unavailable") if launchpad_status_code == "blocked": publish_blocking_reasons.append("release_launchpad=blocked") elif launchpad_status_code == "attention": publish_warnings.append("release_launchpad=attention") if launchpad_onboarding_bootstrap_pending_nodes > 0: publish_warnings.append(f"launchpad_onboarding_bootstrap_pending={launchpad_onboarding_bootstrap_pending_nodes}") if launchpad_onboarding_acceptance_ready_nodes > 0: publish_warnings.append(f"launchpad_onboarding_acceptance_ready={launchpad_onboarding_acceptance_ready_nodes}") if managed_enabled > 0 and remote_access_ready == 0: publish_blocking_reasons.append("remote_agent_ready=0") elif managed_enabled > 0 and remote_access_ready < managed_enabled: publish_warnings.append(f"remote_agent_ready={remote_access_ready}/{managed_enabled}") if queue_dead_letter_nodes > 0: publish_blocking_reasons.append(f"queue_dead_letter_nodes={queue_dead_letter_nodes}") if start_delivery_issue_total > 0: publish_warnings.append(f"ops_job_start_delivery_failed_local={start_delivery_issue_total}") if bool(log_sync.get("enabled")) and participating_nodes_total > 0: if log_sync_covered_nodes == 0: publish_warnings.append(f"log_sync_waiting_sample=0/{participating_nodes_total}") elif log_sync_covered_nodes < participating_nodes_total: publish_warnings.append(f"log_sync_partial={log_sync_covered_nodes}/{participating_nodes_total}") publish_status = "ready" if publish_blocking_reasons: publish_status = "blocked" elif publish_warnings: publish_status = "attention" publish_ready = publish_status == "ready" if publish_status == "ready": publish_status_label = "可发布" publish_summary = "当前执行面、路由面、合同面与发布门禁已收口,可以进入正式发版。" elif publish_status == "attention": publish_status_label = "可发布但建议先复核" publish_summary = "当前没有硬阻断,但仍有上线前关注项,建议先完成复核再正式发版。" else: publish_status_label = "暂不可发布" publish_summary = "当前仍有硬阻断项,不能直接进入正式发版。" go_live_status = "ready" if blocking_reasons: go_live_status = "blocked" elif warnings: go_live_status = "attention" next_step_action_code = ( str(next_step.get("action_code") or "").strip() or str(recommendation.get("primary_action_code") or "").strip() or str(launchpad_status.get("recommended_action_code") or "").strip() ) next_step_reason = ( str(next_step.get("reason") or "").strip() or str(recommendation.get("reason") or "").strip() or "来自 overview / launchpad fallback" ) stack_launchpad_target_node_code = str( diagnosis.get("launchpad_recommended_target_node_code") or (dict(next_step.get("focus_ref") or {}).get("node_code") if next_step_action_code in {"bootstrap_run", "run_acceptance"} else "") or "" ).strip() stack_launchpad_recovery_label = str( diagnosis.get("launchpad_recommended_recovery_label") or ("签发接入工单" if next_step_action_code == "bootstrap_run" else "执行接管验收" if next_step_action_code == "run_acceptance" else "") or "" ).strip() stack_launchpad_recovery_summary = str( diagnosis.get("launchpad_recommended_recovery_summary") or next_step_reason or "" ).strip() launchpad_recommended_target_node_code = str( launchpad_status.get("recommended_target_node_code") or stack_launchpad_target_node_code or "" ).strip() launchpad_recommended_recovery_label = str( launchpad_status.get("recommended_recovery_label") or stack_launchpad_recovery_label or "" ).strip() launchpad_recommended_recovery_summary = str( launchpad_status.get("recommended_recovery_summary") or stack_launchpad_recovery_summary or "" ).strip() operator_lane = str(operator_decision.get("lane") or "").strip() if not operator_lane: operator_lane = "recovery" if blocking_reasons else ("verification" if warnings else "go-live") operator_title = ( str(operator_decision.get("title") or "").strip() or str(recommendation.get("title") or "").strip() or str(launchpad_status.get("status_label") or "").strip() or ("先处理阻塞项" if blocking_reasons else "可以进入上线复核") ) operator_primary_command_key = str(operator_decision.get("primary_command_key") or "").strip() or next_step_action_code recommended_commands = { "stack_summary": build_bash_command("check_ops_center_stack.sh", normalized_base_url, "summary"), "contracts": build_bash_command("check_ops_contracts.sh", normalized_base_url), "ops_plane": build_bash_command("check_ops_plane.sh", normalized_base_url), "release_hub": build_bash_command("check_release_hub.sh", normalized_base_url), "inspection": build_bash_command("check_ops_inspection.sh", normalized_base_url), "overview": build_bash_command("drive_ops_center.sh", "overview", normalized_base_url), "go_live_recover": build_bash_command("drive_ops_center.sh", "go-live-recover", normalized_base_url), "doctor_export": build_bash_command("drive_ops_center.sh", "doctor-export", "/tmp/domaincheck-go-live", normalized_base_url), } if next_step_action_code: recommended_commands["next_step"] = build_bash_command("drive_ops_center.sh", "driver-resolve", normalized_base_url, next_step_action_code) if runtime_schema_stale: recommended_commands["runtime_refresh_recover"] = build_bash_command( "drive_ops_center.sh", "runtime-refresh-recover", normalized_base_url, ) recommended_commands["api_restart"] = recommended_commands["runtime_refresh_recover"] recommended_commands["bootstrap_plan_recheck"] = build_bash_command( "drive_ops_center.sh", "node-bootstrap-plan", normalized_base_url, "mainland-worker-01", ) if bool(log_sync.get("enabled")): recommended_commands["log_sync_logs"] = build_bash_command("drive_ops_center.sh", "driver-resolve", normalized_base_url, "open_worker_logs_participating") recommended_commands["log_sync_inspection"] = build_bash_command("drive_ops_center.sh", "driver-resolve", normalized_base_url, "run_inspection_participating") return { "base_url": normalized_base_url, "generated_at": datetime.now().strftime("%Y-%m-%d %H:%M:%S"), "go_live_status": go_live_status, "publish_ready": publish_ready, "publish_status": publish_status, "publish_status_label": publish_status_label, "publish_summary": publish_summary, "stack_status": stack_status, "contracts_ready": bool(contracts), "contracts_total": len(contracts), "launchpad_status": launchpad_status_code, "launchpad_status_label": str(launchpad_status.get("status_label") or "").strip(), "launchpad_recommended_action_code": str(launchpad_status.get("recommended_action_code") or "").strip(), "launchpad_recommended_target_node_code": launchpad_recommended_target_node_code, "launchpad_recommended_recovery_label": launchpad_recommended_recovery_label, "launchpad_recommended_recovery_summary": launchpad_recommended_recovery_summary, "launchpad_onboarding_bootstrap_pending_nodes": launchpad_onboarding_bootstrap_pending_nodes, "launchpad_onboarding_acceptance_ready_nodes": launchpad_onboarding_acceptance_ready_nodes, "route_surface_complete": route_surface_complete, "route_surface_missing_keys": route_surface_missing_keys, "route_surface_declares_bootstrap_plan": route_surface_declares_bootstrap_plan, "runtime_schema_stale": runtime_schema_stale, "repository_capabilities": repository_capabilities, "managed_enabled": managed_enabled, "remote_access_ready": remote_access_ready, "queue_dead_letter_nodes": queue_dead_letter_nodes, "activity_start_delivery_issue_total": start_delivery_issue_total, "participating_nodes_total": participating_nodes_total, "log_sync_enabled": bool(log_sync.get("enabled")), "log_sync_state": str(log_sync.get("state") or "").strip(), "log_sync_mode": str(log_sync.get("mode") or "").strip(), "log_sync_covered_nodes": log_sync_covered_nodes, "log_sync_missing_node_codes": [ str(item or "").strip() for item in list(log_sync.get("missing_participating_nodes") or []) if str(item or "").strip() ], "next_step_action_code": next_step_action_code, "next_step_reason": next_step_reason, "operator_lane": operator_lane, "operator_title": operator_title, "operator_primary_command_key": operator_primary_command_key, "publish_blocking_reasons": publish_blocking_reasons, "publish_warnings": publish_warnings, "blocking_reasons": blocking_reasons, "warnings": warnings, "recommended_commands": recommended_commands, "source_refs": { "stack_diagnosis_contract_key": str(diagnosis.get("contract_key") or "").strip(), "contracts_registry_version": str(contracts_payload.get("registry_version") or "").strip(), "runtime_build_commit_sha": str(build_info_payload.get("commit_sha") or "").strip(), "release_focus_ref": dict(launchpad_status.get("focus_ref") or {}), }, } def _build_doctor_preferred_surface(operator_decision: dict) -> str: next_focus = dict(operator_decision.get("next_focus") or {}) focus_kind = str(next_focus.get("kind") or "").strip() lane = str(operator_decision.get("lane") or "").strip() if focus_kind == "node_scene_log": return "execution-scene" if lane == "node_handover": return "managed-nodes" if lane == "release": return "release-hub" if lane == "observability": return "execution-scene" if lane in {"ops_jobs", "steady"}: return "driver-feed" return "stack-summary" def _dedupe_command_list(command_candidates: list[object]) -> list[str]: normalized_commands: list[str] = [] for candidate in command_candidates: if isinstance(candidate, (list, tuple)): for nested in candidate: nested_command = str(nested or "").strip() if nested_command and nested_command not in normalized_commands: normalized_commands.append(nested_command) continue command = str(candidate or "").strip() if command and command not in normalized_commands: normalized_commands.append(command) return normalized_commands def _doctor_recommended_command_map( *, base_url: str, manifest_target: str, raw_recommended_commands: list[str], ) -> dict[str, str]: target = str(manifest_target or "").strip() or str(base_url or "").strip() or "http://127.0.0.1:8100" mapped = { "doctor_decision": build_bash_command("drive_ops_center.sh", "doctor-decision", target), } if str(manifest_target or "").strip(): mapped["go_live_review"] = build_bash_command("drive_ops_center.sh", "go-live-review", manifest_target) mapped["go_live_signoff"] = build_bash_command("drive_ops_center.sh", "go-live-signoff", manifest_target) else: mapped["stack_check"] = build_bash_command("drive_ops_center.sh", "stack-diagnosis", base_url, "summary") mapped["stack_diagnosis"] = mapped["stack_check"] mapped["doctor"] = build_bash_command("drive_ops_center.sh", "doctor", base_url) for index, command in enumerate(_dedupe_command_list(list(raw_recommended_commands or []))[:4]): key = "primary_recovery" if index == 0 else f"follow_up_{index}" mapped[key] = command return mapped def _doctor_scene_log_problem_node_codes(scene_log_reports: list[dict]) -> list[str]: problem_nodes: list[str] = [] for item in scene_log_reports: if not isinstance(item, dict): continue node_code = str(item.get("node_code") or "").strip() if not node_code: continue if item.get("ok") is False: if node_code not in problem_nodes: problem_nodes.append(node_code) continue status = str(item.get("status") or "").strip().lower() if status and status not in {"ok", "ready", "success"} and node_code not in problem_nodes: problem_nodes.append(node_code) return problem_nodes def _build_live_doctor_payload(base_url: str) -> tuple[dict, dict]: go_live_summary = get_ops_go_live_summary(base_url=base_url) stack_payload = get_ops_stack_diagnosis(base_url=base_url) diagnosis = dict(stack_payload.get("diagnosis") or {}) operator_decision = dict(diagnosis.get("operator_decision") or {}) next_step = dict(diagnosis.get("next_step") or {}) next_actions = dict(diagnosis.get("next_actions") or {}) recommended_commands = dict(diagnosis.get("recommended_commands") or {}) stack_status = str(diagnosis.get("stack_status") or "").strip() go_live_status = str(go_live_summary.get("go_live_status") or "").strip() publish_status = str(go_live_summary.get("publish_status") or "").strip() status = "ready" if "blocked" in {stack_status, go_live_status, publish_status}: status = "blocked" elif "attention" in {stack_status, go_live_status, publish_status}: status = "attention" preferred_surface = _build_doctor_preferred_surface(operator_decision) headline = ( str(operator_decision.get("title") or "").strip() or str(diagnosis.get("headline") or "").strip() or str(go_live_summary.get("headline") or "").strip() or "当前总检主决策还没有形成明确 headline。" ) detail = ( str(operator_decision.get("summary") or "").strip() or str(next_step.get("reason") or "").strip() or str(go_live_summary.get("next_step_reason") or "").strip() or str(diagnosis.get("blocked_issue_detail") or "").strip() ) next_action_code = ( str(operator_decision.get("primary_command_key") or "").strip() or str(next_step.get("action_code") or "").strip() or str(go_live_summary.get("next_step_action_code") or "").strip() ) raw_commands = _dedupe_command_list( [ next_actions.get("primary_command"), next_actions.get("secondary_command"), recommended_commands.get(next_action_code), list(recommended_commands.values())[:3], ] ) decision = { "status": status, "reason_code": str( operator_decision.get("reason_code") or next_step.get("source") or stack_status or "live_fallback" ).strip(), "headline": headline, "detail": detail, "preferred_surface": preferred_surface, "next_action_code": next_action_code, "recommended_commands": raw_commands, "evidence": { "blocking_issue_total": int(diagnosis.get("blocking_issue_total", 0) or 0), "warning_issue_total": int(diagnosis.get("warning_issue_total", 0) or 0), "missing_surfaces": list(diagnosis.get("missing_surfaces") or []), }, } summary = { "ok": status == "ready", "required_failures": [], "optional_unavailable": [], "scene_log_reports_total": 0, "scene_log_reports_ok": 0, "scene_log_status_counts": {}, "scene_log_reports": [], "contract_surface_gaps": [], "launchpad_recommended_target_node_code": str( go_live_summary.get("launchpad_recommended_target_node_code") or "" ).strip(), "launchpad_recommended_recovery_label": str( go_live_summary.get("launchpad_recommended_recovery_label") or "" ).strip(), "launchpad_recommended_recovery_summary": str( go_live_summary.get("launchpad_recommended_recovery_summary") or "" ).strip(), "launchpad_onboarding_bootstrap_pending_nodes": int( go_live_summary.get("launchpad_onboarding_bootstrap_pending_nodes", 0) or 0 ), "launchpad_onboarding_acceptance_ready_nodes": int( go_live_summary.get("launchpad_onboarding_acceptance_ready_nodes", 0) or 0 ), } return decision, summary def get_ops_doctor_decision(*, base_url: str = "", report_dir: str = "") -> dict: normalized_base_url = str(base_url or "").strip() or "http://127.0.0.1:8100" normalized_report_dir = str(report_dir or "").strip() contract_registry = get_ops_contract_registry() manifest_path: Path | None = None resolved_report_dir: Path | None = None if normalized_report_dir: candidate_path = Path(normalized_report_dir) if candidate_path.is_dir(): resolved_report_dir = candidate_path nested_manifest = candidate_path / "manifest.json" if nested_manifest.is_file(): manifest_path = nested_manifest elif candidate_path.is_file(): manifest_path = candidate_path resolved_report_dir = candidate_path.parent else: manifest_path, resolved_report_dir = _find_latest_go_live_bundle_manifest() raw_decision: dict = {} raw_summary: dict = {} source = "live_fallback" if manifest_path is not None and manifest_path.is_file(): manifest = _load_json_path(manifest_path) artifacts = list(manifest.get("artifacts") or []) artifact_by_key = { str(item.get("key") or "").strip(): dict(item) for item in artifacts if isinstance(item, dict) and str(item.get("key") or "").strip() } doctor_artifact = artifact_by_key.get("doctor_decision") or {} doctor_artifact_path = Path(str(doctor_artifact.get("path") or "").strip()) if doctor_artifact else None doctor_payload = _load_json_path(doctor_artifact_path) if doctor_artifact_path else {} raw_decision = dict(doctor_payload.get("decision") or manifest.get("decision") or {}) raw_summary = dict(doctor_payload.get("summary") or manifest.get("summary") or {}) if raw_decision: source = "bundle_manifest" if doctor_payload else "doctor_manifest" if not raw_decision: raw_decision, raw_summary = _build_live_doctor_payload(normalized_base_url) source = "live_fallback" status = str(raw_decision.get("status") or "").strip() or ("ready" if bool(raw_summary.get("ok", False)) else "attention") status_label = { "ready": "主决策已就绪", "attention": "主决策待复核", "blocked": "主决策阻断", "missing": "待生成", }.get(status, status or "主决策未知") summary_scene_log_reports = [ dict(item) for item in list(raw_summary.get("scene_log_reports") or []) if isinstance(item, dict) ] scene_log_problem_node_codes = _doctor_scene_log_problem_node_codes(summary_scene_log_reports) manifest_target = str(manifest_path or normalized_report_dir or "").strip() recommended_commands = _doctor_recommended_command_map( base_url=normalized_base_url, manifest_target=manifest_target, raw_recommended_commands=[ str(item).strip() for item in list(raw_decision.get("recommended_commands") or []) if str(item).strip() ], ) contract_keys = _normalize_ops_contract_keys( [ "ops_doctor_decision_contract", "ops_stack_diagnosis_contract", "ops_driver_contract", "ops_go_live_bundle_contract", "ops_go_live_signoff_contract", ] ) return { "generated_at": datetime.now().isoformat(timespec="seconds"), "base_url": normalized_base_url, "contract_key": "ops_doctor_decision_contract", "contract_version": str( (contract_registry.get("contracts_by_key") or {}).get("ops_doctor_decision_contract", {}).get("version") or _OPS_CONTRACT_SCHEMA_VERSION ).strip(), "status": status, "status_label": status_label, "headline": str(raw_decision.get("headline") or "").strip() or "当前还没有生成总检主决策。", "detail": str(raw_decision.get("detail") or "").strip(), "source": source, "report_dir": str((resolved_report_dir or manifest_path.parent) if manifest_path else normalized_report_dir), "manifest_path": str(manifest_path or ""), "decision_available": bool(raw_decision), "decision": { "status": status, "reason_code": str(raw_decision.get("reason_code") or "").strip(), "headline": str(raw_decision.get("headline") or "").strip(), "detail": str(raw_decision.get("detail") or "").strip(), "preferred_surface": str(raw_decision.get("preferred_surface") or "").strip(), "next_action_code": str(raw_decision.get("next_action_code") or "").strip(), "recommended_commands": [ str(item).strip() for item in list(raw_decision.get("recommended_commands") or []) if str(item).strip() ], "evidence": dict(raw_decision.get("evidence") or {}), }, "summary": { "ok": bool(raw_summary.get("ok", False)), "required_failures": [ str(item).strip() for item in list(raw_summary.get("required_failures") or []) if str(item).strip() ], "optional_unavailable": [ str(item).strip() for item in list(raw_summary.get("optional_unavailable") or []) if str(item).strip() ], "scene_log_reports_total": int(raw_summary.get("scene_log_reports_total", len(summary_scene_log_reports)) or 0), "scene_log_reports_ok": int(raw_summary.get("scene_log_reports_ok", 0) or 0), "scene_log_status_counts": dict(raw_summary.get("scene_log_status_counts") or {}), "scene_log_reports": summary_scene_log_reports, "scene_log_problem_node_codes": scene_log_problem_node_codes, "contract_surface_gaps": [ str(item).strip() for item in list(raw_summary.get("contract_surface_gaps") or []) if str(item).strip() ], "launchpad_recommended_target_node_code": str( raw_summary.get("launchpad_recommended_target_node_code") or "" ).strip(), "launchpad_recommended_recovery_label": str( raw_summary.get("launchpad_recommended_recovery_label") or "" ).strip(), "launchpad_recommended_recovery_summary": str( raw_summary.get("launchpad_recommended_recovery_summary") or "" ).strip(), "launchpad_onboarding_bootstrap_pending_nodes": int( raw_summary.get("launchpad_onboarding_bootstrap_pending_nodes", 0) or 0 ), "launchpad_onboarding_acceptance_ready_nodes": int( raw_summary.get("launchpad_onboarding_acceptance_ready_nodes", 0) or 0 ), }, "recommended_commands": recommended_commands, "contract_navigation": _build_ops_contract_navigation( contract_keys, primary_contract_key="ops_doctor_decision_contract", registry=contract_registry, ), } def get_ops_go_live_signoff(*, base_url: str = "", report_dir: str = "") -> dict: normalized_base_url = str(base_url or "").strip() or "http://127.0.0.1:8100" normalized_report_dir = str(report_dir or "").strip() go_live_summary = get_ops_go_live_summary(base_url=normalized_base_url) stack_payload = get_ops_stack_diagnosis(base_url=normalized_base_url) driver_feed = get_ops_driver_feed() codex_brief = get_ops_codex_brief() release_launchpad = get_release_launchpad() go_live_review = get_ops_go_live_review(base_url=normalized_base_url, report_dir=normalized_report_dir) doctor_decision_payload = get_ops_doctor_decision(base_url=normalized_base_url, report_dir=normalized_report_dir) contract_registry = get_ops_contract_registry() stack_diagnosis = dict(stack_payload.get("diagnosis") or {}) driver_summary = dict(driver_feed.get("summary") or {}) codex_summary = dict(codex_brief.get("summary") or {}) launchpad_status = dict(release_launchpad.get("launchpad_status") or {}) operator_decision = dict(stack_diagnosis.get("operator_decision") or {}) next_step = dict(stack_diagnosis.get("next_step") or {}) doctor_decision = dict(doctor_decision_payload.get("decision") or {}) launchpad_target_node_codes = { "go_live_summary": str(go_live_summary.get("launchpad_recommended_target_node_code") or "").strip(), "driver_feed": str(driver_summary.get("launchpad_recommended_target_node_code") or "").strip(), "codex_brief": str(codex_summary.get("launchpad_recommended_target_node_code") or "").strip(), "release_launchpad": str(launchpad_status.get("recommended_target_node_code") or "").strip(), } launchpad_recovery_labels = { "go_live_summary": str(go_live_summary.get("launchpad_recommended_recovery_label") or "").strip(), "driver_feed": str(driver_summary.get("launchpad_recommended_recovery_label") or "").strip(), "codex_brief": str(codex_summary.get("launchpad_recommended_recovery_label") or "").strip(), "release_launchpad": str(launchpad_status.get("recommended_recovery_label") or "").strip(), } launchpad_recovery_summaries = { "go_live_summary": str(go_live_summary.get("launchpad_recommended_recovery_summary") or "").strip(), "driver_feed": str(driver_summary.get("launchpad_recommended_recovery_summary") or "").strip(), "codex_brief": str(codex_summary.get("launchpad_recommended_recovery_summary") or "").strip(), "release_launchpad": str(launchpad_status.get("recommended_recovery_summary") or "").strip(), } launchpad_bootstrap_pending_nodes = { "go_live_summary": int(go_live_summary.get("launchpad_onboarding_bootstrap_pending_nodes", 0) or 0), "driver_feed": int(driver_summary.get("launchpad_onboarding_bootstrap_pending_nodes", 0) or 0), "codex_brief": int(codex_summary.get("launchpad_onboarding_bootstrap_pending_nodes", 0) or 0), "release_launchpad": int(launchpad_status.get("onboarding_bootstrap_pending_nodes", 0) or 0), } launchpad_acceptance_ready_nodes = { "go_live_summary": int(go_live_summary.get("launchpad_onboarding_acceptance_ready_nodes", 0) or 0), "driver_feed": int(driver_summary.get("launchpad_onboarding_acceptance_ready_nodes", 0) or 0), "codex_brief": int(codex_summary.get("launchpad_onboarding_acceptance_ready_nodes", 0) or 0), "release_launchpad": int(launchpad_status.get("onboarding_acceptance_ready_nodes", 0) or 0), } def _nonempty_distinct(values: dict[str, str]) -> list[str]: return sorted({value for value in values.values() if value}) def _distinct_ints(values: dict[str, int]) -> list[int]: return sorted({int(value) for value in values.values()}) launchpad_alignment = { "consistent": True, "target_node_code_consistent": len(_nonempty_distinct(launchpad_target_node_codes)) <= 1, "recovery_label_consistent": len(_nonempty_distinct(launchpad_recovery_labels)) <= 1, "recovery_summary_consistent": len(_nonempty_distinct(launchpad_recovery_summaries)) <= 1, "bootstrap_pending_consistent": len(_distinct_ints(launchpad_bootstrap_pending_nodes)) <= 1, "acceptance_ready_consistent": len(_distinct_ints(launchpad_acceptance_ready_nodes)) <= 1, "available_sources": sorted(launchpad_target_node_codes.keys()), "target_node_codes": launchpad_target_node_codes, "recovery_labels": launchpad_recovery_labels, "recovery_summaries": launchpad_recovery_summaries, "bootstrap_pending_nodes": launchpad_bootstrap_pending_nodes, "acceptance_ready_nodes": launchpad_acceptance_ready_nodes, } launchpad_alignment["consistent"] = all( bool(launchpad_alignment.get(key)) for key in ( "target_node_code_consistent", "recovery_label_consistent", "recovery_summary_consistent", "bootstrap_pending_consistent", "acceptance_ready_consistent", ) ) blocked_reasons: list[str] = [] attention_reasons: list[str] = [] go_live_status = str(go_live_summary.get("go_live_status") or "").strip() publish_status = str(go_live_summary.get("publish_status") or "").strip() stack_status = str(stack_diagnosis.get("stack_status") or "").strip() driver_launch_status = str(driver_summary.get("launch_status") or "").strip() codex_launch_status = str(codex_summary.get("launch_status") or "").strip() release_launchpad_status = str(launchpad_status.get("status") or "").strip() review_status = str(go_live_review.get("status") or "").strip() doctor_status = str(doctor_decision_payload.get("status") or "").strip() if go_live_status == "blocked": blocked_reasons.append("go_live_summary=blocked") elif go_live_status == "attention": attention_reasons.append("go_live_summary=attention") if publish_status == "blocked": blocked_reasons.append("publish_status=blocked") elif publish_status == "attention": attention_reasons.append("publish_status=attention") if stack_status == "blocked": blocked_reasons.append("stack_diagnosis=blocked") elif stack_status == "attention": attention_reasons.append("stack_diagnosis=attention") if driver_launch_status == "blocked": blocked_reasons.append("driver_feed_launch=blocked") elif driver_launch_status == "attention": attention_reasons.append("driver_feed_launch=attention") if codex_launch_status == "blocked": blocked_reasons.append("codex_brief_launch=blocked") elif codex_launch_status == "attention": attention_reasons.append("codex_brief_launch=attention") if release_launchpad_status == "blocked": blocked_reasons.append("release_launchpad=blocked") elif release_launchpad_status == "attention": attention_reasons.append("release_launchpad=attention") if review_status == "blocked": blocked_reasons.append("go_live_review=blocked") elif review_status in {"attention", "missing"}: attention_reasons.append(f"go_live_review={review_status}") if doctor_status == "blocked": blocked_reasons.append("doctor_decision=blocked") elif doctor_status in {"attention", "missing"}: attention_reasons.append(f"doctor_decision={doctor_status}") if not bool(launchpad_alignment.get("consistent", False)): attention_reasons.append("launchpad_alignment=inconsistent") signoff_status = "ready" if blocked_reasons: signoff_status = "blocked" elif attention_reasons: signoff_status = "attention" status_label = { "ready": "可签字上线", "attention": "待签字复核", "blocked": "暂不可签字", }.get(signoff_status, signoff_status or "未知") if signoff_status == "blocked": primary_blocked_surface = ( "正式复核" if review_status == "blocked" else "总检主决策" if doctor_status == "blocked" else "发布门禁" ) headline = ( f"当前仍存在{primary_blocked_surface}阻断,建议优先处理 " f"{str(go_live_summary.get('operator_title') or '').strip() or str(operator_decision.get('title') or '').strip() or str(doctor_decision.get('next_action_code') or '').strip() or '阻断项'}。" ) elif signoff_status == "attention": if review_status in {"attention", "missing"}: headline = "当前已进入最终收口阶段,但正式复核仍待补口,建议先确认 bundle manifest 结论再签字上线。" elif doctor_status in {"attention", "missing"}: headline = "当前已进入最终收口阶段,但总检主决策仍待补口,建议先按主建议完成复核再签字上线。" else: headline = ( f"当前已进入最终收口阶段,但仍建议先完成 {str(go_live_summary.get('operator_title') or '').strip() or str(operator_decision.get('title') or '').strip() or 'attention 项'} 再签字上线。" ) else: headline = "当前收口、正式复核、主决策、发布门禁与 launchpad 摘要一致,可以进入最终人工签字或正式发布。" contract_keys = _normalize_ops_contract_keys( [ "ops_go_live_signoff_contract", "ops_go_live_review_contract", "ops_doctor_decision_contract", "ops_stack_diagnosis_contract", "ops_driver_contract", "release_hub_contract", "ops_observability_contract", ] ) recommended_commands = { **dict(go_live_summary.get("recommended_commands") or {}), **dict(go_live_review.get("recommended_commands") or {}), **dict(doctor_decision_payload.get("recommended_commands") or {}), "release_launchpad": build_bash_command("drive_ops_center.sh", "release-launchpad", normalized_base_url), "driver_feed": build_bash_command("drive_ops_center.sh", "driver-feed", normalized_base_url), "codex_brief": build_bash_command("drive_ops_center.sh", "codex-brief", normalized_base_url), "go_live_signoff": build_bash_command( "drive_ops_center.sh", "go-live-signoff", normalized_report_dir or normalized_base_url, ), } return { "generated_at": datetime.now().isoformat(timespec="seconds"), "base_url": normalized_base_url, "contract_key": "ops_go_live_signoff_contract", "contract_version": str( (contract_registry.get("contracts_by_key") or {}).get("ops_go_live_signoff_contract", {}).get("version") or _OPS_CONTRACT_SCHEMA_VERSION ).strip(), "signoff_status": signoff_status, "status_label": status_label, "headline": headline, "release_gate": { "go_live_status": go_live_status, "publish_status": publish_status, "stack_status": stack_status, "driver_launch_status": driver_launch_status, "codex_launch_status": codex_launch_status, "release_launchpad_status": release_launchpad_status, "go_live_review_status": review_status, "doctor_decision_status": doctor_status, }, "blocked_reasons": blocked_reasons, "attention_reasons": attention_reasons, "report_dir": str(go_live_review.get("report_dir") or doctor_decision_payload.get("report_dir") or "").strip(), "manifest_path": str(go_live_review.get("manifest_path") or doctor_decision_payload.get("manifest_path") or "").strip(), "decision": { "operator_title": str(go_live_summary.get("operator_title") or "").strip(), "next_step_action_code": str(go_live_summary.get("next_step_action_code") or "").strip(), "next_step_reason": str(go_live_summary.get("next_step_reason") or "").strip(), "preferred_surface": ( "go-live-review" if review_status in {"blocked", "attention", "missing"} else str(doctor_decision.get("preferred_surface") or "").strip() if doctor_status in {"blocked", "attention", "missing"} and str(doctor_decision.get("preferred_surface") or "").strip() else "release-launchpad" if release_launchpad_status in {"blocked", "attention"} else "codex-brief" if codex_launch_status in {"blocked", "attention"} else "go-live-summary" ), "doctor_reason_code": str(doctor_decision.get("reason_code") or "").strip(), "doctor_preferred_surface": str(doctor_decision.get("preferred_surface") or "").strip(), "doctor_next_action_code": str(doctor_decision.get("next_action_code") or "").strip(), "review_status": review_status, "review_headline": str(go_live_review.get("headline") or "").strip(), "review_report_dir": str(go_live_review.get("report_dir") or "").strip(), "review_manifest_path": str(go_live_review.get("manifest_path") or "").strip(), "stack_operator_title": str(operator_decision.get("title") or "").strip(), "stack_next_action_code": str(next_step.get("action_code") or "").strip(), "launchpad_recommended_target_node_code": str( go_live_summary.get("launchpad_recommended_target_node_code") or "" ).strip(), "launchpad_recommended_recovery_label": str( go_live_summary.get("launchpad_recommended_recovery_label") or "" ).strip(), "launchpad_recommended_recovery_summary": str( go_live_summary.get("launchpad_recommended_recovery_summary") or "" ).strip(), }, "launchpad_alignment": launchpad_alignment, "recommended_commands": recommended_commands, "contract_navigation": _build_ops_contract_navigation( contract_keys, primary_contract_key="ops_go_live_signoff_contract", registry=contract_registry, ), "go_live_summary": go_live_summary, "go_live_review": { "status": review_status, "status_label": str(go_live_review.get("status_label") or "").strip(), "headline": str(go_live_review.get("headline") or "").strip(), "report_dir": str(go_live_review.get("report_dir") or "").strip(), "manifest_path": str(go_live_review.get("manifest_path") or "").strip(), "critical_failures": list(go_live_review.get("critical_failures") or []), "noncritical_failures": list(go_live_review.get("noncritical_failures") or []), }, "doctor_decision": { "status": doctor_status, "headline": str(doctor_decision_payload.get("headline") or "").strip(), "detail": str(doctor_decision_payload.get("detail") or "").strip(), "summary": dict(doctor_decision_payload.get("summary") or {}), "decision": doctor_decision, "report_dir": str(doctor_decision_payload.get("report_dir") or "").strip(), "manifest_path": str(doctor_decision_payload.get("manifest_path") or "").strip(), }, "stack_diagnosis": { "stack_status": stack_status, "operator_decision": operator_decision, "next_step": next_step, }, "driver_feed": { "headline": str(driver_feed.get("headline") or "").strip(), "summary": driver_summary, }, "codex_brief": { "headline": str(codex_brief.get("headline") or "").strip(), "summary": codex_summary, "focus": dict(codex_brief.get("focus") or {}), }, "release_launchpad": { "status": release_launchpad_status, "status_label": str(launchpad_status.get("status_label") or "").strip(), "recommended_action_code": str(launchpad_status.get("recommended_action_code") or "").strip(), "recommended_execution_mode": str(launchpad_status.get("recommended_execution_mode") or "").strip(), }, } def _ops_go_live_bundle_root() -> Path: return runtime_root() / "ops-center-reports" / "go-live-bundles" def _load_json_path(path: Path) -> dict: if not path.is_file(): return {} try: payload = json.loads(path.read_text(encoding="utf-8")) except Exception: return {} return payload if isinstance(payload, dict) else {} def _find_latest_go_live_bundle_manifest() -> tuple[Path | None, Path | None]: bundle_root = _ops_go_live_bundle_root() if not bundle_root.exists(): return None, bundle_root manifest_paths = sorted(bundle_root.glob("*/manifest.json"), key=lambda item: item.stat().st_mtime, reverse=True) if not manifest_paths: return None, bundle_root manifest_path = manifest_paths[0] return manifest_path, manifest_path.parent def _resolve_go_live_bundle_manifest(report_dir: str) -> tuple[Path | None, Path | None]: normalized_report_dir = str(report_dir or "").strip() if normalized_report_dir: candidate_path = Path(normalized_report_dir) if candidate_path.is_dir(): nested_manifest = candidate_path / "manifest.json" return (nested_manifest if nested_manifest.is_file() else None), candidate_path if candidate_path.is_file(): return candidate_path, candidate_path.parent return None, candidate_path return _find_latest_go_live_bundle_manifest() def _load_bundle_json_artifact(artifact_by_key: dict[str, dict], key: str) -> dict: artifact = artifact_by_key.get(str(key or "").strip()) or {} artifact_path = Path(str(artifact.get("path") or "").strip()) return _load_json_path(artifact_path) if artifact_path else {} def _load_bundle_env_audit_summary(artifact_by_key: dict[str, dict]) -> dict: env_audit_artifact = artifact_by_key.get("env_audit") or {} env_audit_path = Path(str(env_audit_artifact.get("path") or "").strip()) if not env_audit_path.is_file(): return {} raw_text = env_audit_path.read_text(encoding="utf-8", errors="replace") marker = "[8/8] condensed env audit summary" candidate_text = raw_text.split(marker, 1)[1].strip() if marker in raw_text else raw_text.strip() if not candidate_text: return {} try: loaded_env_audit = json.loads(candidate_text) except Exception: loaded_env_audit = {} return loaded_env_audit if isinstance(loaded_env_audit, dict) else {} def get_ops_go_live_bundle(*, base_url: str = "", report_dir: str = "") -> dict: normalized_base_url = str(base_url or "").strip() or "http://127.0.0.1:8100" contract_registry = get_ops_contract_registry() manifest_path, resolved_report_dir = _resolve_go_live_bundle_manifest(report_dir) recommended_commands = { "go_live_export": build_bash_command("drive_ops_center.sh", "go-live-export"), "go_live_review": build_bash_command("drive_ops_center.sh", "go-live-review", str(report_dir or "").strip() or "/path/to/go-live-bundle"), "go_live_signoff": build_bash_command("drive_ops_center.sh", "go-live-signoff", str(report_dir or "").strip() or "/path/to/go-live-bundle"), "doctor_export": build_bash_command("drive_ops_center.sh", "doctor-export", "/tmp/domaincheck-go-live", normalized_base_url), } contract_keys = _normalize_ops_contract_keys( [ "ops_go_live_bundle_contract", "ops_go_live_signoff_contract", "ops_stack_diagnosis_contract", "ops_driver_contract", ] ) if manifest_path is None or not manifest_path.is_file(): bundle_root = resolved_report_dir or _ops_go_live_bundle_root() headline = ( "当前还没有可复核的 go-live bundle,建议先导出证据包,再进入 manifest 与最终签收复核。" ) return { "generated_at": datetime.now().isoformat(timespec="seconds"), "base_url": normalized_base_url, "contract_key": "ops_go_live_bundle_contract", "contract_version": str( (contract_registry.get("contracts_by_key") or {}).get("ops_go_live_bundle_contract", {}).get("version") or _OPS_CONTRACT_SCHEMA_VERSION ).strip(), "status": "missing", "status_label": "待导出", "headline": headline, "bundle_available": False, "report_dir": str(bundle_root), "manifest_path": "", "artifact_total": 0, "failure_total": 0, "critical_failures": [], "noncritical_failures": [], "env_audit": {}, "summary": { "ok": False, "review_status_hint": "missing", "review_headline": headline, "launchpad_alignment": {"consistent": False}, }, "recommended_commands": recommended_commands, "contract_navigation": _build_ops_contract_navigation( contract_keys, primary_contract_key="ops_go_live_bundle_contract", registry=contract_registry, ), } manifest = _load_json_path(manifest_path) artifacts = list(manifest.get("artifacts") or []) failures = [str(item or "").strip() for item in list(manifest.get("failures") or []) if str(item or "").strip()] artifact_by_key = { str(item.get("key") or "").strip(): dict(item) for item in artifacts if isinstance(item, dict) and str(item.get("key") or "").strip() } summary = dict(manifest.get("summary") or {}) env_audit = _load_bundle_env_audit_summary(artifact_by_key) review_status_hint = str(summary.get("review_status_hint") or "").strip() status = review_status_hint or ("ready" if bool(summary.get("ok", False)) else "attention") status_label = { "ready": "证据可交付", "attention": "证据待复核", "blocked": "证据阻断", "missing": "待导出", }.get(status, status or "未知") headline = ( str(summary.get("review_headline") or "").strip() or "当前已读取最新 go-live bundle,可继续做 manifest 与签收复核。" ) critical_keys = {"go_live_summary", "stack_diagnosis", "driver_feed", "codex_brief", "release_launchpad"} critical_failures = [item for item in failures if item in critical_keys] noncritical_failures = [item for item in failures if item not in critical_keys] launchpad_alignment = dict(summary.get("launchpad_alignment") or {}) return { "generated_at": datetime.now().isoformat(timespec="seconds"), "base_url": normalized_base_url, "contract_key": "ops_go_live_bundle_contract", "contract_version": str( (contract_registry.get("contracts_by_key") or {}).get("ops_go_live_bundle_contract", {}).get("version") or _OPS_CONTRACT_SCHEMA_VERSION ).strip(), "status": status, "status_label": status_label, "headline": headline, "bundle_available": True, "report_dir": str(manifest.get("report_dir") or resolved_report_dir or manifest_path.parent), "manifest_path": str(manifest_path), "artifact_total": int(summary.get("artifact_total", len(artifacts)) or len(artifacts)), "failure_total": int(summary.get("failure_total", len(failures)) or len(failures)), "critical_failures": critical_failures, "noncritical_failures": noncritical_failures, "env_audit": { "status": str(env_audit.get("status") or "").strip(), "headline": str(env_audit.get("headline") or "").strip(), "missing_items": list(env_audit.get("missing_items") or []), "runtime_may_need_restart": bool(((env_audit.get("runtime") or {}).get("runtime_may_need_restart", False))), "recommended_actions": [ str(item).strip() for item in list(env_audit.get("recommended_actions") or []) if str(item).strip() ], }, "summary": { "ok": bool(summary.get("ok", False)), "review_status_hint": review_status_hint or status, "review_headline": headline, "launchpad_recommended_target_node_code": str( summary.get("launchpad_recommended_target_node_code") or "" ).strip(), "launchpad_recommended_recovery_label": str( summary.get("launchpad_recommended_recovery_label") or "" ).strip(), "launchpad_recommended_recovery_summary": str( summary.get("launchpad_recommended_recovery_summary") or "" ).strip(), "launchpad_onboarding_bootstrap_pending_nodes": int( summary.get("launchpad_onboarding_bootstrap_pending_nodes", 0) or 0 ), "launchpad_onboarding_acceptance_ready_nodes": int( summary.get("launchpad_onboarding_acceptance_ready_nodes", 0) or 0 ), "launchpad_alignment": launchpad_alignment, "recommended_reading_order": list(summary.get("recommended_reading_order") or []), "contract_surface_gaps": list(summary.get("contract_surface_gaps") or []), "discovered_contract_keys": list(summary.get("discovered_contract_keys") or []), }, "artifacts": artifacts, "failures": failures, "recommended_commands": { **recommended_commands, "go_live_review": build_bash_command("drive_ops_center.sh", "go-live-review", str(manifest_path)), "go_live_signoff": build_bash_command("drive_ops_center.sh", "go-live-signoff", str(manifest_path)), }, "contract_navigation": _build_ops_contract_navigation( contract_keys, primary_contract_key="ops_go_live_bundle_contract", registry=contract_registry, ), } def get_ops_go_live_review(*, base_url: str = "", report_dir: str = "") -> dict: normalized_base_url = str(base_url or "").strip() or "http://127.0.0.1:8100" normalized_report_dir = str(report_dir or "").strip() contract_registry = get_ops_contract_registry() manifest_path, resolved_report_dir = _resolve_go_live_bundle_manifest(report_dir) contract_keys = _normalize_ops_contract_keys( [ "ops_go_live_review_contract", "ops_go_live_bundle_contract", "ops_doctor_decision_contract", "ops_go_live_signoff_contract", "ops_stack_diagnosis_contract", "ops_driver_contract", "release_hub_contract", ] ) recommended_commands = { "go_live_export": build_bash_command("drive_ops_center.sh", "go-live-export"), "go_live_review": build_bash_command("drive_ops_center.sh", "go-live-review", normalized_report_dir or "/path/to/go-live-bundle"), "go_live_signoff": build_bash_command("drive_ops_center.sh", "go-live-signoff", normalized_report_dir or "/path/to/go-live-bundle"), "doctor_decision": build_bash_command("drive_ops_center.sh", "doctor-decision", normalized_report_dir or normalized_base_url), } if manifest_path is None or not manifest_path.is_file(): headline = "当前还没有可复核的 go-live bundle,建议先导出证据包,再进入正式复核。" return { "generated_at": datetime.now().isoformat(timespec="seconds"), "base_url": normalized_base_url, "contract_key": "ops_go_live_review_contract", "contract_version": str( (contract_registry.get("contracts_by_key") or {}).get("ops_go_live_review_contract", {}).get("version") or _OPS_CONTRACT_SCHEMA_VERSION ).strip(), "status": "missing", "status_label": "待导出", "headline": headline, "report_dir": str(resolved_report_dir or _ops_go_live_bundle_root()), "manifest_path": "", "artifact_total": 0, "failure_total": 0, "critical_failures": [], "noncritical_failures": [], "env_audit": {}, "go_live_summary": {}, "stack_diagnosis": {}, "driver_feed": {}, "codex_brief": {}, "launchpad_alignment": {"consistent": False, "available_sources": []}, "recommended_reading_order": [], "recommended_next_steps": [ "先执行 go-live-export,导出完整上线证据包。", "导出完成后再执行 go-live-review 或直接打开页面里的证据复核详情。", ], "recommended_commands": recommended_commands, "contract_navigation": _build_ops_contract_navigation( contract_keys, primary_contract_key="ops_go_live_review_contract", registry=contract_registry, ), } manifest = _load_json_path(manifest_path) artifacts = list(manifest.get("artifacts") or []) failures = [str(item or "").strip() for item in list(manifest.get("failures") or []) if str(item or "").strip()] summary = dict(manifest.get("summary") or {}) artifact_by_key = { str(item.get("key") or "").strip(): dict(item) for item in artifacts if isinstance(item, dict) and str(item.get("key") or "").strip() } env_audit = _load_bundle_env_audit_summary(artifact_by_key) go_live_summary_payload = _load_bundle_json_artifact(artifact_by_key, "go_live_summary") stack_diagnosis_payload = _load_bundle_json_artifact(artifact_by_key, "stack_diagnosis") driver_feed_payload = _load_bundle_json_artifact(artifact_by_key, "driver_feed") codex_brief_payload = _load_bundle_json_artifact(artifact_by_key, "codex_brief") critical_keys = {"go_live_summary", "stack_diagnosis", "driver_feed", "codex_brief", "release_launchpad"} critical_failures = sorted( { key for key in critical_keys if key in failures or not bool((artifact_by_key.get(key) or {}).get("ok", False)) } ) noncritical_failures = [key for key in failures if key not in critical_keys] go_live_launchpad_target_node_code = str(go_live_summary_payload.get("launchpad_recommended_target_node_code") or "").strip() go_live_launchpad_recovery_label = str(go_live_summary_payload.get("launchpad_recommended_recovery_label") or "").strip() go_live_launchpad_recovery_summary = str(go_live_summary_payload.get("launchpad_recommended_recovery_summary") or "").strip() go_live_launchpad_bootstrap_pending_nodes = int(go_live_summary_payload.get("launchpad_onboarding_bootstrap_pending_nodes", 0) or 0) go_live_launchpad_acceptance_ready_nodes = int(go_live_summary_payload.get("launchpad_onboarding_acceptance_ready_nodes", 0) or 0) stack_diagnosis = dict(stack_diagnosis_payload.get("diagnosis") or {}) driver_feed_summary = dict(driver_feed_payload.get("summary") or {}) codex_brief_summary = dict(codex_brief_payload.get("summary") or {}) launchpad_target_node_codes = { "go_live_summary": go_live_launchpad_target_node_code, "stack_diagnosis": str(stack_diagnosis.get("launchpad_recommended_target_node_code") or "").strip(), "driver_feed": str(driver_feed_summary.get("launchpad_recommended_target_node_code") or "").strip(), "codex_brief": str(codex_brief_summary.get("launchpad_recommended_target_node_code") or "").strip(), } launchpad_recovery_labels = { "go_live_summary": go_live_launchpad_recovery_label, "stack_diagnosis": str(stack_diagnosis.get("launchpad_recommended_recovery_label") or "").strip(), "driver_feed": str(driver_feed_summary.get("launchpad_recommended_recovery_label") or "").strip(), "codex_brief": str(codex_brief_summary.get("launchpad_recommended_recovery_label") or "").strip(), } launchpad_recovery_summaries = { "go_live_summary": go_live_launchpad_recovery_summary, "stack_diagnosis": str(stack_diagnosis.get("launchpad_recommended_recovery_summary") or "").strip(), "driver_feed": str(driver_feed_summary.get("launchpad_recommended_recovery_summary") or "").strip(), "codex_brief": str(codex_brief_summary.get("launchpad_recommended_recovery_summary") or "").strip(), } launchpad_bootstrap_pending_nodes = { "go_live_summary": go_live_launchpad_bootstrap_pending_nodes, "stack_diagnosis": int(stack_diagnosis.get("launchpad_onboarding_bootstrap_pending_nodes", 0) or 0), "driver_feed": int(driver_feed_summary.get("launchpad_onboarding_bootstrap_pending_nodes", 0) or 0), "codex_brief": int(codex_brief_summary.get("launchpad_onboarding_bootstrap_pending_nodes", 0) or 0), } launchpad_acceptance_ready_nodes = { "go_live_summary": go_live_launchpad_acceptance_ready_nodes, "stack_diagnosis": int(stack_diagnosis.get("launchpad_onboarding_acceptance_ready_nodes", 0) or 0), "driver_feed": int(driver_feed_summary.get("launchpad_onboarding_acceptance_ready_nodes", 0) or 0), "codex_brief": int(codex_brief_summary.get("launchpad_onboarding_acceptance_ready_nodes", 0) or 0), } available_alignment_sources = { source for source, payload_value in { "go_live_summary": go_live_summary_payload, "stack_diagnosis": stack_diagnosis_payload, "driver_feed": driver_feed_payload, "codex_brief": codex_brief_payload, }.items() if isinstance(payload_value, dict) and payload_value } def _nonempty_distinct(values: dict[str, str]) -> list[str]: return sorted({str(value or "").strip() for key, value in values.items() if key in available_alignment_sources and str(value or "").strip()}) def _distinct_ints(values: dict[str, int]) -> list[int]: return sorted({int(value) for key, value in values.items() if key in available_alignment_sources}) launchpad_alignment = { "consistent": True, "target_node_code_consistent": len(_nonempty_distinct(launchpad_target_node_codes)) <= 1, "recovery_label_consistent": len(_nonempty_distinct(launchpad_recovery_labels)) <= 1, "recovery_summary_consistent": len(_nonempty_distinct(launchpad_recovery_summaries)) <= 1, "bootstrap_pending_consistent": len(_distinct_ints(launchpad_bootstrap_pending_nodes)) <= 1, "acceptance_ready_consistent": len(_distinct_ints(launchpad_acceptance_ready_nodes)) <= 1, "available_sources": sorted(available_alignment_sources), "target_node_codes": launchpad_target_node_codes, "recovery_labels": launchpad_recovery_labels, "recovery_summaries": launchpad_recovery_summaries, "bootstrap_pending_nodes": launchpad_bootstrap_pending_nodes, "acceptance_ready_nodes": launchpad_acceptance_ready_nodes, } launchpad_alignment["consistent"] = all( bool(launchpad_alignment.get(key)) for key in ( "target_node_code_consistent", "recovery_label_consistent", "recovery_summary_consistent", "bootstrap_pending_consistent", "acceptance_ready_consistent", ) ) env_audit_status = str(env_audit.get("status") or "").strip() env_audit_blocking = env_audit_status == "blocked" env_audit_attention = env_audit_status == "attention" env_audit_missing_items = [ str(item).strip() for item in list(env_audit.get("missing_items") or []) if str(item).strip() ] env_audit_recommended_actions = [ str(item).strip() for item in list(env_audit.get("recommended_actions") or []) if str(item).strip() ] env_audit_runtime_may_need_restart = bool(((env_audit.get("runtime") or {}).get("runtime_may_need_restart", False))) node_agent_env_missing = any(item == "missing_env:/etc/default/domaincheck-node-agent" for item in env_audit_missing_items) if critical_failures or env_audit_blocking: status = "blocked" if critical_failures and env_audit_blocking: headline = "关键报告存在缺失,且环境审计已给出阻断结论,当前不适合上线。" elif critical_failures: headline = "上线证据包仍缺关键报告,暂不建议直接宣称收口完成。" else: headline = "环境审计已给出阻断结论,需先修复现场环境后再进入发布门禁。" elif noncritical_failures or env_audit_attention or not bool(launchpad_alignment.get("consistent")): status = "attention" if not bool(launchpad_alignment.get("consistent")): headline = "核心报告已齐,但 launchpad 摘要在不同 surface 之间存在漂移,建议先复核再发版。" elif noncritical_failures and env_audit_attention: headline = "核心报告已齐,但补充报告和环境审计都提示仍有缺口,建议先复核再发版。" elif noncritical_failures: headline = "核心报告已齐,但仍有补充报告超时或失败,建议先复核再发版。" else: headline = "核心报告已齐,但环境审计仍提示基础缺口,建议补齐后再发版。" else: status = "ready" headline = "核心上线证据已经齐备,可进入最终人工复核或正式发布门禁。" recommended_next_steps = [ "先看 00_env_audit.txt,确认当前机器环境是否存在阻断或注意项", "先看 02_go_live_summary.json 和 03_stack_diagnosis.json", "再看 04_driver_feed.json 和 05_codex_brief.json", "若涉及发布门禁,再看 06_release_launchpad.json", "若 manifest 里仍有失败项,再看对应 artifact 原文件", ] if node_agent_env_missing: recommended_next_steps.extend( [ "当前缺口指向 Node Agent 未接管:先执行 agent-gap-export,拿到首个缺口节点的接管包", "随后执行 node-bootstrap-plan,为缺口节点生成可直接落地的接管脚本", "如果仓库代码已更新但 bootstrap-plan 输出仍偏旧,先重启控制面 API 再重新导出接管计划", ] ) if env_audit_runtime_may_need_restart: recommended_next_steps.extend( [ "env-audit 已识别运行中 API 可能仍是旧代码:先执行 drive_ops_center.sh runtime-refresh-recover,按固定链路完成 API 重启与复检", "若仍需人工逐条确认,再执行 recommended_actions 里的 API 重启与复检命令", ] ) if go_live_launchpad_target_node_code: recommended_next_steps.extend( [ f"当前 launchpad 缺口已经收敛到节点 {go_live_launchpad_target_node_code},建议优先执行:{go_live_launchpad_recovery_label or '节点恢复'}", "先对照 02_go_live_summary.json、04_driver_feed.json、05_codex_brief.json 中的 launchpad 字段,确认三处摘要是否一致", go_live_launchpad_recovery_summary or "如果 recovery_summary 仍为空,再回到 06_release_launchpad.json 查看 gap rows 详情", ] ) if not bool(launchpad_alignment.get("consistent")): recommended_next_steps.extend( [ "当前 bundle 内的 launchpad 摘要在 go_live_summary / stack_diagnosis / driver_feed / codex_brief 之间存在不一致,先不要直接交付。", "优先复核 02_go_live_summary.json 与 03_stack_diagnosis.json,再检查 04_driver_feed.json 与 05_codex_brief.json 是否使用了同一版控制面数据。", "若只有局部摘要偏旧,先执行 drive_ops_center.sh runtime-refresh-recover,并重新导出 go-live bundle。", ] ) return { "generated_at": datetime.now().isoformat(timespec="seconds"), "base_url": normalized_base_url, "contract_key": "ops_go_live_review_contract", "contract_version": str( (contract_registry.get("contracts_by_key") or {}).get("ops_go_live_review_contract", {}).get("version") or _OPS_CONTRACT_SCHEMA_VERSION ).strip(), "status": status, "status_label": { "ready": "可复核发布", "attention": "待复核补口", "blocked": "暂不可发布", "missing": "待导出", }.get(status, status or "未知"), "headline": headline, "report_dir": str(manifest.get("report_dir") or resolved_report_dir or manifest_path.parent), "manifest_path": str(manifest_path), "artifact_total": int(summary.get("artifact_total", len(artifacts)) or len(artifacts)), "failure_total": int(summary.get("failure_total", len(failures)) or len(failures)), "critical_failures": critical_failures, "noncritical_failures": noncritical_failures, "env_audit": { "ok": bool((artifact_by_key.get("env_audit") or {}).get("ok", False)), "status": env_audit_status, "headline": str(env_audit.get("headline") or "").strip(), "missing_items": env_audit_missing_items, "runtime_may_need_restart": env_audit_runtime_may_need_restart, "recommended_actions": env_audit_recommended_actions, }, "go_live_summary": { "go_live_status": str(go_live_summary_payload.get("go_live_status") or "").strip(), "publish_status": str(go_live_summary_payload.get("publish_status") or "").strip(), "operator_title": str(go_live_summary_payload.get("operator_title") or "").strip(), "next_step_action_code": str(go_live_summary_payload.get("next_step_action_code") or "").strip(), "launchpad_recommended_target_node_code": go_live_launchpad_target_node_code, "launchpad_recommended_recovery_label": go_live_launchpad_recovery_label, "launchpad_recommended_recovery_summary": go_live_launchpad_recovery_summary, "launchpad_onboarding_bootstrap_pending_nodes": go_live_launchpad_bootstrap_pending_nodes, "launchpad_onboarding_acceptance_ready_nodes": go_live_launchpad_acceptance_ready_nodes, }, "stack_diagnosis": { "stack_status": str(stack_diagnosis.get("stack_status") or "").strip(), "operator_title": str((dict(stack_diagnosis.get("operator_decision") or {})).get("title") or "").strip(), "next_step_action_code": str((dict(stack_diagnosis.get("next_step") or {})).get("action_code") or "").strip(), "launchpad_recommended_target_node_code": str(stack_diagnosis.get("launchpad_recommended_target_node_code") or "").strip(), "launchpad_recommended_recovery_label": str(stack_diagnosis.get("launchpad_recommended_recovery_label") or "").strip(), "launchpad_recommended_recovery_summary": str(stack_diagnosis.get("launchpad_recommended_recovery_summary") or "").strip(), "launchpad_onboarding_bootstrap_pending_nodes": int(stack_diagnosis.get("launchpad_onboarding_bootstrap_pending_nodes", 0) or 0), "launchpad_onboarding_acceptance_ready_nodes": int(stack_diagnosis.get("launchpad_onboarding_acceptance_ready_nodes", 0) or 0), }, "driver_feed": { "headline": str(driver_feed_payload.get("headline") or "").strip(), "launch_status": str((dict(driver_feed_payload.get("automation_coverage") or {})).get("launch_status") or "").strip(), "launch_ready": bool((dict(driver_feed_payload.get("automation_coverage") or {})).get("launch_ready", False)), "launchpad_recommended_target_node_code": str(driver_feed_summary.get("launchpad_recommended_target_node_code") or "").strip(), "launchpad_recommended_recovery_label": str(driver_feed_summary.get("launchpad_recommended_recovery_label") or "").strip(), }, "codex_brief": { "headline": str(codex_brief_payload.get("headline") or "").strip(), "focus_entry_key": str((dict(codex_brief_payload.get("focus") or {})).get("entry_key") or "").strip(), "launch_status": str((dict(codex_brief_payload.get("automation_coverage") or {})).get("launch_status") or "").strip(), "launch_ready": bool((dict(codex_brief_payload.get("automation_coverage") or {})).get("launch_ready", False)), "launchpad_recommended_target_node_code": str(codex_brief_summary.get("launchpad_recommended_target_node_code") or "").strip(), "launchpad_recommended_recovery_label": str(codex_brief_summary.get("launchpad_recommended_recovery_label") or "").strip(), }, "launchpad_alignment": launchpad_alignment, "recommended_reading_order": list(summary.get("recommended_reading_order") or []), "recommended_next_steps": _dedupe_command_list(recommended_next_steps), "recommended_commands": { **recommended_commands, "go_live_review": build_bash_command("drive_ops_center.sh", "go-live-review", str(manifest_path)), "go_live_signoff": build_bash_command("drive_ops_center.sh", "go-live-signoff", str(manifest_path)), "doctor_decision": build_bash_command("drive_ops_center.sh", "doctor-decision", str(manifest_path)), }, "contract_navigation": _build_ops_contract_navigation( contract_keys, primary_contract_key="ops_go_live_review_contract", registry=contract_registry, ), } def get_ops_blueprint() -> dict: return { "contract_registry": get_ops_contract_registry(), "architecture": { "control_plane": { "region": "overseas", "responsibility": [ "web backend", "ops api", "release registry", "task scheduler", "log aggregation entry", ], }, "managed_nodes": { "region": "mainland", "responsibility": [ "worker/controller runtime", "node agent", "systemd execution", "diagnostics collection", ], }, }, "execution_model": { "default_transport": "https-polling", "future_transport": "websocket", "fallback_transport": "ssh", "job_result_contract": { "required_fields": [ "job_id", "node_code", "step", "started_at", "finished_at", "exit_code", "stdout", "stderr", "structured_result", ] }, }, "mvp_scope": { "phase_1": [ "ops overview api", "action catalog api", "worker participation / standby visibility", "remote log sync switch", ], "phase_2": [ "ops jobs table", "node registration token", "node heartbeat and pull-task api", "service restart / health check actions", ], "phase_3": [ "release package distribution", "one-click bootstrap", "rollback", "streaming logs", ], }, "operator_model": { "with_codex": "海外 Codex 作为智能驾驶员,自动分析现网状态并选择合适动作。", "without_codex": "后台按钮直接创建 ops job,由控制面和 agent 自动完成执行与回执。", }, } def get_ops_runbook() -> dict: runtime = get_runtime_status() readiness = runtime.get("readiness") or {} worker_runtime = runtime.get("worker") or {} sync_agent_runtime = runtime.get("sync_agent") or {} managed_nodes_payload = list_managed_nodes_with_agent_state(participation_payload=runtime.get("detect") or {}) release_launchpad = get_release_launchpad() control_sequences = _attach_ops_runbook_sequence_resolutions( _build_ops_runbook_control_sequences( managed_nodes_payload=managed_nodes_payload, release_launchpad=release_launchpad, ), requested_by="api/runbook", ) return { "entrypoint": { "preferred_host": str(settings.sync_target_api_base_url or "").strip() or f"http://127.0.0.1:{settings.api_port}", "api_prefix": settings.api_prefix, }, "services": { "api": { "service_name": settings.api_service_name, "status": "running", }, "worker": { "service_name": settings.worker_service_name, "status": _bool_label(bool(worker_runtime.get("running", False))), }, "sync_agent": { "service_name": settings.sync_agent_service_name, "status": _bool_label(bool(sync_agent_runtime.get("running", False))), }, }, "fast_checks": [ "GET /health", "GET /api/v1/runtime/status", "GET /api/v1/runtime/cluster", "GET /api/v1/runtime/readiness", "GET /api/v1/ops/stack-diagnosis", "GET /api/v1/ops/overview", "GET /api/v1/ops/contracts", "GET /api/v1/ops/contracts/ops_stack_diagnosis_contract", "GET /api/v1/ops/releases/launchpad", "GET /api/v1/ops/activity-stream", "GET /api/v1/ops/capabilities", ], "current_readiness": { "status": str(readiness.get("status") or ""), "summary": str(readiness.get("summary") or ""), }, "release_launchpad": release_launchpad, "control_sequences": control_sequences, }