Files
getDomain/domain-api/app/services/ops_service.py
2026-04-18 23:52:51 +08:00

11612 lines
549 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
from __future__ import annotations
import json
from datetime import datetime, timedelta
from pathlib import Path
import re
import time
from app.core.config import settings
from app.core.files import runtime_root
from app.services.ops_command_service import build_bash_command
from app.services.ops_agent_service import (
execute_managed_node_onboarding_recovery,
get_managed_node_handover,
list_ops_job_events_for_jobs,
list_managed_nodes_with_agent_state,
)
from app.services.ops_execution_mode_service import execution_mode_label
from app.services.ops_job_service import create_ops_job_batch, get_ops_job_summary, list_ops_jobs, sync_managed_nodes_from_cluster
from app.services.ops_playbook_service import (
execute_ops_playbook,
get_ops_playbook,
get_ops_playbook_run,
get_recent_ops_playbook_runs,
list_ops_playbook_run_events,
preview_ops_playbook,
)
from app.services.ops_release_service import (
build_release_execution_mode_recommendation,
create_release_and_smart_rollout_from_latest_package,
get_latest_release_package_metadata,
create_smart_release_rollout,
get_latest_release,
get_release_launchpad,
get_release_summary,
list_release_rollouts,
prepare_latest_release_package,
)
from app.services.ops_template_service import get_ops_action_template
from app.services.build_info_service import get_runtime_build_info
from app.services.runtime_status_service import get_runtime_status
from app.services.runtime_settings_service import get_runtime_settings, update_runtime_settings
from app.services.sync_record_service import get_sync_summary
_OPS_INSPECTION_FETCH_LIMIT = 80
_OPS_ACTIVITY_FETCH_LIMIT = 18
_OPS_INSPECTION_ACTION_KEYS = ("health.snapshot", "logs.collect", "diagnostics.collect")
_OPS_CONTRACT_REGISTRY_VERSION = "2026-04-18"
_OPS_CONTRACT_SCHEMA_VERSION = "v1"
_REMOTE_LOG_PREVIEW_LINE_RE = re.compile(r"^\[(?P<created_at>[^\]]+)\]\s+\[(?P<node_code>[^\]]+)\]\s+(?P<message>.*)$")
_BACKEND_DRIVER_ACTION_CODES = {
"disable_log_sync",
"enable_log_sync_key",
"enable_log_sync_full",
"replay_delivery_queue",
"flush_delivery_queue",
"run_scene_logs_key",
"run_scene_logs_full",
"open_playbook_dialog",
"open_action_template_dialog",
"focus_playbook_run",
"focus_activity_item",
"focus_latest_job_events",
"open_playbook_run_latest_events",
"handover_first_gap",
"view_first_gap",
"open_rollout_dialog",
"open_release_dialog",
"open_release_deploy_control",
"open_release_deploy_worker",
"open_release_deploy_custom",
"create_smart_release_rollout_worker",
"create_smart_release_rollout_control",
"create_release_rollout_worker",
"create_release_rollout_control",
"focus_release_hub",
"open_worker_logs_participating",
"open_worker_logs_standby",
"open_worker_logs",
"run_inspection_participating",
"run_inspection_standby",
"run_standard_inspection",
"open_diagnostics_participating",
"open_diagnostics",
"publish_latest_worker",
"review_smart_rollout_preview",
"review_control_rollout",
"fix_rollout_blockers",
"fix_managed_nodes",
"bootstrap_run",
"run_acceptance",
"release_package",
"release_prepare",
"api-restart",
}
_SAFE_AUTO_DRIVER_ACTION_CODES = {
"disable_log_sync",
"enable_log_sync_key",
"enable_log_sync_full",
"run_scene_logs_key",
"run_scene_logs_full",
"open_playbook_dialog",
"open_action_template_dialog",
"focus_playbook_run",
"focus_activity_item",
"focus_latest_job_events",
"open_playbook_run_latest_events",
"handover_first_gap",
"view_first_gap",
"open_rollout_dialog",
"open_release_dialog",
"open_release_deploy_control",
"open_release_deploy_worker",
"open_release_deploy_custom",
"focus_release_hub",
"release_package",
"release_prepare",
"api-restart",
"review_smart_rollout_preview",
"review_control_rollout",
"fix_rollout_blockers",
"open_worker_logs_participating",
"open_worker_logs_standby",
"open_worker_logs",
"open_diagnostics_participating",
"open_diagnostics",
"run_inspection_participating",
"run_inspection_standby",
"run_standard_inspection",
}
_GUARDED_AUTO_DRIVER_ACTION_CODES = {
"replay_delivery_queue",
"flush_delivery_queue",
"publish_latest_worker",
"create_smart_release_rollout_worker",
"create_smart_release_rollout_control",
"create_release_rollout_worker",
"create_release_rollout_control",
}
_MIXED_DRIVER_ACTION_CODES = {
"fix_managed_nodes",
"bootstrap_run",
"run_acceptance",
}
_UI_ONLY_DRIVER_ACTION_CODES = set()
_OPS_CONTRACT_DEFINITIONS = (
{
"key": "ops_job_contract",
"title": "Ops Job Contract",
"version": _OPS_CONTRACT_SCHEMA_VERSION,
"status": "active",
"summary": "冻结 ops job / step / event / policy / approve / cancel / dispatch 的正式执行对象 contract。",
"schema_doc_path": "docs/schemas/ops_job_contract.md",
"primary_endpoint": "/api/v1/ops/jobs",
"discovery_endpoints": [
"/api/v1/ops/jobs",
"/api/v1/ops/jobs/{job_id}",
"/api/v1/ops/jobs/{job_id}/events",
"/api/v1/ops/jobs",
"/api/v1/ops/jobs/batch",
"/api/v1/ops/policy/preview",
"/api/v1/ops/jobs/{job_id}/approve",
"/api/v1/ops/jobs/{job_id}/cancel",
"/api/v1/ops/jobs/{job_id}/dispatch",
],
"service_keys": [
"app.services.ops_job_service",
"app.services.ops_agent_service",
],
"consumers": [
"ops-center",
"codex-driver",
"cli",
"node-agent",
"release-hub",
"playbook-runner",
],
"related_contract_keys": [
"ops_agent_protocol",
"release_hub_contract",
"ops_playbook_contract",
"ops_observability_contract",
"ops_stack_diagnosis_contract",
],
"notes": [
"ops job 是正式执行颗粒度对象playbook run 与 rollout 最终都应收敛到它。",
"approval / cancel / dispatch 不只是按钮动作,而是正式状态迁移。",
"页面列表默认消费 compact job详情页再读取 full job + steps + events。",
],
},
{
"key": "ops_agent_protocol",
"title": "Node Agent Protocol",
"version": _OPS_CONTRACT_SCHEMA_VERSION,
"status": "active",
"summary": "冻结海外控制面与大陆 Node Agent 之间的认证、注册、心跳、拉任务、完成回执与事件回放 contract。",
"schema_doc_path": "docs/schemas/ops_agent_protocol.md",
"primary_endpoint": "/api/v1/ops/agent/bootstrap-plan",
"discovery_endpoints": [
"/api/v1/ops/agent/tokens",
"/api/v1/ops/agent/bootstrap-plan",
"/api/v1/ops/agent/register",
"/api/v1/ops/agent/heartbeat",
"/api/v1/ops/agent/pull",
"/api/v1/ops/agent/jobs/{job_id}/start",
"/api/v1/ops/agent/jobs/{job_id}/complete",
"/api/v1/ops/agent/jobs/{job_id}/events",
"/api/v1/ops/nodes/{node_code}/delivery-queue",
"/api/v1/ops/nodes/{node_code}/delivery-queue/records",
"/api/v1/ops/nodes/{node_code}/delivery-queue/flush",
"/api/v1/ops/nodes/{node_code}/delivery-queue/replay",
"/api/v1/ops/nodes/{node_code}/delivery-queue/records/{record_id}/replay",
"/api/v1/ops/nodes/{node_code}/delivery-queue/records/{record_id}/discard",
],
"service_keys": [
"app.services.ops_agent_service",
"app.node_agent",
],
"consumers": [
"ops-center",
"codex-driver",
"cli",
"node-agent",
],
"related_contract_keys": [
"ops_job_contract",
"ops_observability_contract",
"ops_stack_diagnosis_contract",
],
"notes": [
"Agent 只执行结构化动作,不长期接受任意 shell。",
"complete / events 回执已经具备 client_request_id / client_event_id 幂等语义。",
"heartbeat 已纳入 delivery_queue 快照,可暴露 retrying / dead_letter 现场。",
],
},
{
"key": "release_hub_contract",
"title": "Release Hub Contract",
"version": _OPS_CONTRACT_SCHEMA_VERSION,
"status": "active",
"summary": "冻结 Release / Rollout / Launchpad / Default Gate 的对象模型与门禁口径。",
"schema_doc_path": "docs/schemas/release_hub_contract.md",
"primary_endpoint": "/api/v1/ops/releases/launchpad",
"discovery_endpoints": [
"/api/v1/ops/overview",
"/api/v1/ops/releases",
"/api/v1/ops/releases/latest",
"/api/v1/ops/releases/launchpad",
"/api/v1/ops/releases/{release_id}",
"/api/v1/ops/releases/{release_id}/rollouts",
"/api/v1/ops/rollouts/{rollout_id}",
"/api/v1/ops/rollouts/{rollout_id}/jobs",
"/api/v1/ops/rollouts/{rollout_id}/advance",
],
"service_keys": [
"app.services.ops_release_service",
"app.services.ops_service",
],
"consumers": [
"ops-center",
"codex-driver",
"cli",
"release-hub",
],
"related_contract_keys": [
"ops_job_contract",
"ops_driver_contract",
"ops_playbook_contract",
"ops_stack_diagnosis_contract",
],
"notes": [
"发布必须先有 Release再有 Rollout不回退到节点 git pull。",
"default_rollout_gate 是首页、版本区和自动驾驶共用的统一门禁。",
"Rollout 状态机应继续固定为 planned/running/awaiting_approval/ready_for_next_batch/halted/completed/completed_with_issues。",
],
},
{
"key": "ops_driver_contract",
"title": "Ops Driver Contract",
"version": _OPS_CONTRACT_SCHEMA_VERSION,
"status": "active",
"summary": "冻结 overview / driver-feed / codex-brief / driver-actions / runbook sequence 的统一驾驶 contract。",
"schema_doc_path": "docs/schemas/ops_driver_contract.md",
"primary_endpoint": "/api/v1/ops/driver-feed",
"discovery_endpoints": [
"/api/v1/ops/overview",
"/api/v1/ops/driver-feed",
"/api/v1/ops/codex-brief",
"/api/v1/ops/driver-actions/resolve",
"/api/v1/ops/driver-actions/execute-resolved",
"/api/v1/ops/codex-actions/resolve",
"/api/v1/ops/codex-actions/execute",
"/api/v1/ops/activity-stream",
"/api/v1/ops/runbook",
"/api/v1/ops/runbook/sequences/{sequence_key}/resolve",
"/api/v1/ops/runbook/sequences/{sequence_key}/execute",
"/api/v1/ops/driver-actions/preview",
"/api/v1/ops/driver-actions/execute",
],
"service_keys": [
"app.services.ops_service",
"app.services.ops_playbook_service",
],
"consumers": [
"ops-center",
"codex-driver",
"cli",
],
"related_contract_keys": [
"release_hub_contract",
"ops_playbook_contract",
"ops_observability_contract",
"ops_stack_diagnosis_contract",
],
"notes": [
"后端负责推荐优先级和 ui_intent页面不再自算。",
"driver recommendation / runbook sequence / activity stream 是三条不同 contract。",
"Codex 通过 codex-brief 判断 safe_auto / guarded_auto / ui_only / blocked。",
"driver-actions/preview 负责统一生成执行链、请求体预览和主次动作 contract 视图。",
],
},
{
"key": "ops_playbook_contract",
"title": "Ops Playbook Contract",
"version": _OPS_CONTRACT_SCHEMA_VERSION,
"status": "active",
"summary": "冻结 playbook catalog / preview / run / events 的正式编排 contract。",
"schema_doc_path": "docs/schemas/ops_playbook_contract.md",
"primary_endpoint": "/api/v1/ops/playbooks",
"discovery_endpoints": [
"/api/v1/ops/playbooks",
"/api/v1/ops/playbooks/preview",
"/api/v1/ops/playbooks/execute",
"/api/v1/ops/playbook-runs",
"/api/v1/ops/playbook-runs/{run_code}",
"/api/v1/ops/playbook-runs/{run_code}/events",
"/api/v1/ops/playbook-runs/{run_code}/rerun",
"/api/v1/ops/playbook-runs/{run_code}/cancel",
],
"service_keys": [
"app.services.ops_playbook_service",
"app.services.ops_service",
],
"consumers": [
"ops-center",
"codex-driver",
"cli",
],
"related_contract_keys": [
"ops_job_contract",
"release_hub_contract",
"ops_driver_contract",
"ops_observability_contract",
"ops_stack_diagnosis_contract",
],
"notes": [
"接管验收、标准巡检、现场观察应优先收口为 playbook run。",
"playbook run 是 ops job 的编排聚合对象,不是单条任务。",
"页面、CLI、Codex 应围绕同一份 preview / run / events contract 观察与执行。",
],
},
{
"key": "ops_observability_contract",
"title": "Ops Observability Contract",
"version": _OPS_CONTRACT_SCHEMA_VERSION,
"status": "active",
"summary": "冻结 execution scene / inspection overview / activity stream / delivery queue 的正式观察面 contract。",
"schema_doc_path": "docs/schemas/ops_observability_contract.md",
"primary_endpoint": "/api/v1/ops/overview",
"discovery_endpoints": [
"/api/v1/ops/overview",
"/api/v1/ops/inspection-overview",
"/api/v1/ops/activity-stream",
"/api/v1/ops/nodes/{node_code}/scene-log",
"/api/v1/ops/nodes/{node_code}/delivery-queue",
"/api/v1/ops/nodes/{node_code}/delivery-queue/records",
"/api/v1/ops/nodes/{node_code}/delivery-queue/flush",
"/api/v1/ops/nodes/{node_code}/delivery-queue/replay",
"/api/v1/ops/nodes/{node_code}/delivery-queue/records/{record_id}/replay",
"/api/v1/ops/nodes/{node_code}/delivery-queue/records/{record_id}/discard",
],
"service_keys": [
"app.services.ops_service",
"app.services.ops_agent_service",
],
"consumers": [
"ops-center",
"codex-driver",
"cli",
],
"related_contract_keys": [
"ops_job_contract",
"ops_agent_protocol",
"ops_driver_contract",
"ops_playbook_contract",
"ops_stack_diagnosis_contract",
],
"notes": [
"execution scene 必须明确区分 dispatch_active / recent_only / standby / load_syncing。",
"inspection overview 必须按节点收口最近 health / worker logs / diagnostics。",
"delivery queue 当前允许 head_only但不允许绕过 ops job 直接改远端记录。",
],
},
{
"key": "ops_stack_diagnosis_contract",
"title": "Ops Stack Diagnosis Contract",
"version": _OPS_CONTRACT_SCHEMA_VERSION,
"status": "active",
"summary": "冻结海外单脑总检入口的统一诊断 contract供页面、CLI、Codex、按钮共享同一份第一现场判断。",
"schema_doc_path": "docs/schemas/ops_stack_diagnosis_contract.md",
"primary_endpoint": "/api/v1/ops/stack-diagnosis",
"discovery_endpoints": [
"/api/v1/ops/go-live-summary",
"/api/v1/ops/stack-diagnosis",
"/api/v1/ops/contracts",
"/api/v1/ops/link-snapshot",
"/api/v1/ops/overview",
"/api/v1/ops/nodes",
"/api/v1/ops/releases/launchpad",
"/api/v1/ops/playbook-runs",
"/api/v1/ops/activity-stream",
],
"service_keys": [
"app.services.ops_service",
],
"consumers": [
"ops-center",
"codex-driver",
"cli",
],
"related_contract_keys": [
"ops_job_contract",
"ops_agent_protocol",
"release_hub_contract",
"ops_driver_contract",
"ops_playbook_contract",
"ops_observability_contract",
],
"notes": [
"这是海外单脑控制面的固定起手式不允许页面、CLI、Codex 再各自拼一份第一现场诊断。",
"总检必须同时给出 stack_status、issues、next_step、quick_commands不能只回一段说明文字。",
"go-live-summary 是给页面 / CLI / Codex 直接消费的收口摘要层stack-diagnosis 则保留更细的可解释结构。",
"当总检 contract 可用后check_ops_center_stack.sh 应优先直接消费它,而不是继续各算各的。",
],
},
{
"key": "ops_doctor_decision_contract",
"title": "Ops Doctor Decision Contract",
"version": _OPS_CONTRACT_SCHEMA_VERSION,
"status": "active",
"summary": "冻结 doctor-decision 主决策 contract把总检主判断、建议处理面、下一步动作和证据摘要统一暴露给页面、CLI 和 Codex。",
"schema_doc_path": "docs/schemas/ops_doctor_decision_contract.md",
"primary_endpoint": "/api/v1/ops/doctor-decision",
"discovery_endpoints": [
"/api/v1/ops/doctor-decision",
"/api/v1/ops/stack-diagnosis",
"/api/v1/ops/go-live-summary",
"/api/v1/ops/go-live-bundle",
"/api/v1/ops/contracts",
],
"service_keys": [
"app.services.ops_service",
],
"consumers": [
"ops-center",
"codex-driver",
"cli",
],
"related_contract_keys": [
"ops_stack_diagnosis_contract",
"ops_driver_contract",
"ops_go_live_bundle_contract",
"ops_go_live_signoff_contract",
"release_hub_contract",
],
"notes": [
"doctor-decision 是页面、CLI、Codex 的统一主决策入口,不应再让不同消费方各算各的“第一处理面”。",
"优先读取最新导出的 doctor manifest / go-live bundle 中的 doctor_decision再在缺 bundle 时回落到 live 总检兜底。",
"返回值必须同时给出 status、preferred_surface、next_action_code 与推荐命令,不能只给一句提示文字。",
],
},
{
"key": "ops_go_live_signoff_contract",
"title": "Ops Go-Live Signoff Contract",
"version": _OPS_CONTRACT_SCHEMA_VERSION,
"status": "active",
"summary": "冻结发布前最终签收结论 contract把收口、发布、自动化、launchpad 对齐状态压成同一份签字判断。",
"schema_doc_path": "docs/schemas/ops_go_live_signoff_contract.md",
"primary_endpoint": "/api/v1/ops/go-live-signoff",
"discovery_endpoints": [
"/api/v1/ops/go-live-signoff",
"/api/v1/ops/go-live-summary",
"/api/v1/ops/stack-diagnosis",
"/api/v1/ops/driver-feed",
"/api/v1/ops/codex-brief",
"/api/v1/ops/releases/launchpad",
"/api/v1/ops/contracts",
],
"service_keys": [
"app.services.ops_service",
],
"consumers": [
"ops-center",
"codex-driver",
"cli",
],
"related_contract_keys": [
"ops_doctor_decision_contract",
"ops_stack_diagnosis_contract",
"ops_driver_contract",
"release_hub_contract",
"ops_observability_contract",
],
"notes": [
"signoff 不是替代 go-live-summary而是在其基础上给出最终可否签字上线的统一结论。",
"当 signoff_status=blocked 时,不允许页面再显示‘基本可上线’这类模糊表述。",
"launchpad 摘要在 go-live-summary / driver-feed / codex-brief 之间若不一致,必须至少提升为 attention。",
],
},
{
"key": "ops_go_live_bundle_contract",
"title": "Ops Go-Live Bundle Contract",
"version": _OPS_CONTRACT_SCHEMA_VERSION,
"status": "active",
"summary": "冻结发布前证据包与 manifest 复核 contract把 bundle 是否存在、manifest 状态、失败项和交付建议统一暴露给页面、CLI 和 Codex。",
"schema_doc_path": "docs/schemas/ops_go_live_bundle_contract.md",
"primary_endpoint": "/api/v1/ops/go-live-bundle",
"discovery_endpoints": [
"/api/v1/ops/go-live-bundle",
"/api/v1/ops/go-live-signoff",
"/api/v1/ops/go-live-summary",
"/api/v1/ops/contracts",
],
"service_keys": [
"app.services.ops_service",
],
"consumers": [
"ops-center",
"codex-driver",
"cli",
],
"related_contract_keys": [
"ops_doctor_decision_contract",
"ops_go_live_signoff_contract",
"ops_stack_diagnosis_contract",
"ops_driver_contract",
"release_hub_contract",
],
"notes": [
"页面不应再假设证据包已经导出,而是必须明确显示当前有没有 bundle、manifest 是否可复核。",
"当 bundle 不存在时,应明确回落到 go-live-export / go-live-review / go-live-signoff 等固定命令,而不是提示人工自行找目录。",
"这层只负责读取最新 bundle / manifest 与复核摘要,不直接在后端发起 shell 导出。",
],
},
{
"key": "ops_go_live_review_contract",
"title": "Ops Go-Live Review Contract",
"version": _OPS_CONTRACT_SCHEMA_VERSION,
"status": "active",
"summary": "冻结 go-live-review 复核结论 contract把 bundle manifest 的正式复核结果、launchpad 对齐与下一步建议统一暴露给页面、CLI 和 Codex。",
"schema_doc_path": "docs/schemas/ops_go_live_review_contract.md",
"primary_endpoint": "/api/v1/ops/go-live-review",
"discovery_endpoints": [
"/api/v1/ops/go-live-review",
"/api/v1/ops/go-live-bundle",
"/api/v1/ops/doctor-decision",
"/api/v1/ops/go-live-signoff",
"/api/v1/ops/contracts",
],
"service_keys": [
"app.services.ops_service",
],
"consumers": [
"ops-center",
"codex-driver",
"cli",
],
"related_contract_keys": [
"ops_go_live_bundle_contract",
"ops_doctor_decision_contract",
"ops_go_live_signoff_contract",
"ops_stack_diagnosis_contract",
"ops_driver_contract",
"release_hub_contract",
],
"notes": [
"go-live-review 不是简单重复 bundle 状态,而是 bundle manifest 的正式复核结论层。",
"当 launchpad 摘要在 go_live_summary / stack_diagnosis / driver_feed / codex_brief 之间漂移时,即使关键文件都存在,也必须至少回到 attention。",
"页面和 Codex 应优先消费这层的 headline / recommended_next_steps而不是自行拼 review 说明。",
],
},
)
def _bool_label(value: bool) -> str:
return "enabled" if value else "disabled"
def get_ops_contract_registry() -> dict:
contracts: list[dict] = []
contracts_by_key: dict[str, dict] = {}
for raw_item in _OPS_CONTRACT_DEFINITIONS:
contract = {
"key": str(raw_item.get("key") or "").strip(),
"title": str(raw_item.get("title") or "").strip(),
"version": str(raw_item.get("version") or _OPS_CONTRACT_SCHEMA_VERSION).strip(),
"status": str(raw_item.get("status") or "active").strip() or "active",
"summary": str(raw_item.get("summary") or "").strip(),
"schema_doc_path": str(raw_item.get("schema_doc_path") or "").strip(),
"primary_endpoint": str(raw_item.get("primary_endpoint") or "").strip(),
"discovery_endpoints": [str(item or "").strip() for item in list(raw_item.get("discovery_endpoints") or []) if str(item or "").strip()],
"service_keys": [str(item or "").strip() for item in list(raw_item.get("service_keys") or []) if str(item or "").strip()],
"consumers": [str(item or "").strip() for item in list(raw_item.get("consumers") or []) if str(item or "").strip()],
"related_contract_keys": [
str(item or "").strip()
for item in list(raw_item.get("related_contract_keys") or [])
if str(item or "").strip()
],
"notes": [str(item or "").strip() for item in list(raw_item.get("notes") or []) if str(item or "").strip()],
}
if not contract["key"]:
continue
contracts.append(contract)
contracts_by_key[contract["key"]] = contract
return {
"registry_version": _OPS_CONTRACT_REGISTRY_VERSION,
"schema_version": _OPS_CONTRACT_SCHEMA_VERSION,
"contracts_total": len(contracts),
"contracts": contracts,
"contracts_by_key": contracts_by_key,
"docs_root": "docs/schemas",
"discovery_entrypoints": [
"/api/v1/ops/contracts",
"/api/v1/ops/contracts/{contract_key}",
"/api/v1/ops/stack-diagnosis",
"/api/v1/ops/capabilities",
"/api/v1/ops/overview",
"/api/v1/ops/driver-feed",
"/api/v1/ops/codex-brief",
"/api/v1/ops/releases/launchpad",
],
}
def get_ops_contract_detail(contract_key: str) -> dict:
normalized_contract_key = str(contract_key or "").strip()
if not normalized_contract_key:
return {}
registry = get_ops_contract_registry()
contracts_by_key = dict(registry.get("contracts_by_key") or {})
selected_contract = dict(contracts_by_key.get(normalized_contract_key) or {})
if not selected_contract:
return {}
related_contracts = [
dict(contracts_by_key.get(related_key) or {})
for related_key in list(selected_contract.get("related_contract_keys") or [])
if str(related_key or "").strip() and dict(contracts_by_key.get(related_key) or {})
]
related_contracts = [
{
"key": str(item.get("key") or "").strip(),
"title": str(item.get("title") or "").strip(),
"summary": str(item.get("summary") or "").strip(),
"schema_doc_path": str(item.get("schema_doc_path") or "").strip(),
"primary_endpoint": str(item.get("primary_endpoint") or "").strip(),
}
for item in related_contracts
]
return {
"registry_version": str(registry.get("registry_version") or _OPS_CONTRACT_REGISTRY_VERSION),
"schema_version": str(registry.get("schema_version") or _OPS_CONTRACT_SCHEMA_VERSION),
"docs_root": str(registry.get("docs_root") or "docs/schemas"),
"detail_endpoint": f"/api/v1/ops/contracts/{normalized_contract_key}",
"contract": selected_contract,
"related_contracts": related_contracts,
"related_contract_keys": [
str(item.get("key") or "").strip()
for item in related_contracts
if str(item.get("key") or "").strip()
],
}
def _normalize_ops_contract_keys(raw_values: list | tuple | set | None) -> list[str]:
normalized_values: list[str] = []
seen_values: set[str] = set()
for raw_value in list(raw_values or []):
normalized_value = str(raw_value or "").strip()
if not normalized_value or normalized_value in seen_values:
continue
seen_values.add(normalized_value)
normalized_values.append(normalized_value)
return normalized_values
def _build_ops_contract_navigation(
contract_keys: list | tuple | set | None,
*,
primary_contract_key: str = "",
registry: dict | None = None,
) -> dict:
registry_payload = dict(registry or get_ops_contract_registry())
contracts_by_key = dict(registry_payload.get("contracts_by_key") or {})
normalized_primary_contract_key = str(primary_contract_key or "").strip()
ordered_contract_keys = _normalize_ops_contract_keys(contract_keys)
if normalized_primary_contract_key:
ordered_contract_keys = [normalized_primary_contract_key] + [
item for item in ordered_contract_keys if item != normalized_primary_contract_key
]
contracts: list[dict] = []
for contract_key in ordered_contract_keys:
raw_contract = dict(contracts_by_key.get(contract_key) or {})
if not raw_contract:
continue
related_contract_keys = _normalize_ops_contract_keys(raw_contract.get("related_contract_keys") or [])
contracts.append(
{
"key": contract_key,
"title": str(raw_contract.get("title") or "").strip(),
"status": str(raw_contract.get("status") or "").strip(),
"version": str(raw_contract.get("version") or _OPS_CONTRACT_SCHEMA_VERSION).strip(),
"summary": str(raw_contract.get("summary") or "").strip(),
"primary_endpoint": str(raw_contract.get("primary_endpoint") or "").strip(),
"schema_doc_path": str(raw_contract.get("schema_doc_path") or "").strip(),
"detail_endpoint": f"/api/v1/ops/contracts/{contract_key}",
"discovery_endpoints": [
str(item).strip()
for item in list(raw_contract.get("discovery_endpoints") or [])
if str(item).strip()
],
"related_contract_keys": related_contract_keys,
}
)
return {
"detail_endpoint_pattern": "/api/v1/ops/contracts/{contract_key}",
"primary_contract_key": (
normalized_primary_contract_key
if normalized_primary_contract_key
else str((contracts[0] or {}).get("key") or "").strip()
),
"contract_keys": (
[str(item.get("key") or "").strip() for item in contracts if str(item.get("key") or "").strip()]
if contracts
else ordered_contract_keys
),
"contracts": contracts,
}
def _ops_contract_keys_from_focus_ref(focus_ref: dict | None) -> list[str]:
normalized_focus_ref = dict(focus_ref or {})
kind = str(normalized_focus_ref.get("kind") or "").strip()
section = str(normalized_focus_ref.get("section") or "").strip()
if kind in {"release_hub", "release", "rollout"} or section in {
"release_launchpad",
"release_detail",
"default_rollout_gate",
}:
return ["release_hub_contract"]
if kind in {"execution_scene", "ops_job", "ops_job_event", "activity_stream", "node_scene_log"}:
return ["ops_observability_contract", "ops_stack_diagnosis_contract"]
if kind in {"playbook_run", "playbook", "playbook_step"}:
return ["ops_playbook_contract"]
if kind in {"managed_node", "node_agent", "delivery_queue"}:
return ["ops_agent_protocol"]
return []
def _ops_contract_keys_from_action_code(action_code: str) -> list[str]:
normalized_action_code = str(action_code or "").strip()
if not normalized_action_code:
return []
if normalized_action_code.startswith("enable_log_sync") or normalized_action_code in {
"run_inspection_participating",
"run_inspection_all",
"focus_execution_scene",
"focus_activity_item",
}:
return ["ops_observability_contract", "ops_stack_diagnosis_contract"]
if normalized_action_code in {
"fix_managed_nodes",
"issue_agent_token",
"create_bootstrap_plan",
"open_agent_onboarding",
"bootstrap_run",
"run_acceptance",
}:
return ["ops_agent_protocol"]
if normalized_action_code in {
"open_release_dialog",
"focus_release_hub",
"create_release_rollout_worker",
"create_release_rollout_control",
"publish_latest_worker",
"publish_latest_control",
}:
return ["release_hub_contract"]
return []
def _ops_contract_keys_from_job_action(job_action: str) -> list[str]:
normalized_job_action = str(job_action or "").strip()
if not normalized_job_action:
return []
if normalized_job_action in {"health.snapshot", "logs.collect", "diagnostics.collect", "service.status"}:
return ["ops_observability_contract", "ops_stack_diagnosis_contract"]
if normalized_job_action == "node.bootstrap" or normalized_job_action.startswith(("runtime.", "service.", "delivery.queue.")):
return ["ops_agent_protocol"]
if normalized_job_action == "deploy.release":
return ["release_hub_contract"]
return []
def _suggest_ops_contract_keys_for_codex_entry(entry: dict) -> list[str]:
normalized_entry = dict(entry or {})
executor_kind = str(normalized_entry.get("executor_kind") or "driver_action").strip() or "driver_action"
contract_keys: list[str] = ["ops_driver_contract"]
if executor_kind == "runbook_sequence":
contract_keys.append("ops_playbook_contract")
for focus_ref in (
normalized_entry.get("focus_ref"),
normalized_entry.get("primary_focus_ref"),
normalized_entry.get("secondary_focus_ref"),
):
contract_keys.extend(_ops_contract_keys_from_focus_ref(focus_ref))
for action_code in (
normalized_entry.get("primary_action_code"),
normalized_entry.get("secondary_action_code"),
normalized_entry.get("focus_action_code"),
):
contract_keys.extend(_ops_contract_keys_from_action_code(str(action_code or "").strip()))
return _normalize_ops_contract_keys(contract_keys)
def _suggest_ops_contract_keys_for_activity_item(item: dict) -> list[str]:
normalized_item = dict(item or {})
kind = str(normalized_item.get("kind") or "").strip()
ui_intent = dict(normalized_item.get("ui_intent") or {})
contract_keys: list[str] = []
if kind in {"execution_scene", "log_sync"}:
contract_keys.extend(["ops_observability_contract", "ops_stack_diagnosis_contract"])
elif kind in {"playbook_run", "runbook_sequence"}:
contract_keys.append("ops_playbook_contract")
if kind == "runbook_sequence":
contract_keys.append("ops_driver_contract")
elif kind == "rollout":
contract_keys.append("release_hub_contract")
elif kind == "ops_job":
contract_keys.extend(
_ops_contract_keys_from_job_action(
str(normalized_item.get("action") or normalized_item.get("title") or "").strip()
)
)
else:
contract_keys.append("ops_observability_contract")
for focus_ref in (
normalized_item.get("focus_ref"),
normalized_item.get("source_focus_ref"),
):
contract_keys.extend(_ops_contract_keys_from_focus_ref(focus_ref))
for action_code in (
ui_intent.get("driver_action_code"),
normalized_item.get("primary_action_code"),
normalized_item.get("secondary_action_code"),
normalized_item.get("focus_action_code"),
):
contract_keys.extend(_ops_contract_keys_from_action_code(str(action_code or "").strip()))
if str(ui_intent.get("kind") or "").strip() == "focus_execution_scene":
contract_keys.extend(["ops_observability_contract", "ops_stack_diagnosis_contract"])
return _normalize_ops_contract_keys(contract_keys)
def _primary_ops_contract_key_for_activity_item(item: dict, contract_keys: list[str]) -> str:
normalized_item = dict(item or {})
normalized_contract_keys = _normalize_ops_contract_keys(contract_keys)
kind = str(normalized_item.get("kind") or "").strip()
job_action = str(normalized_item.get("action") or normalized_item.get("title") or "").strip()
if kind in {"playbook_run", "runbook_sequence"} and "ops_playbook_contract" in normalized_contract_keys:
return "ops_playbook_contract"
if kind == "rollout" and "release_hub_contract" in normalized_contract_keys:
return "release_hub_contract"
if kind == "ops_job":
if job_action == "node.bootstrap" and "ops_agent_protocol" in normalized_contract_keys:
return "ops_agent_protocol"
if job_action == "deploy.release" and "release_hub_contract" in normalized_contract_keys:
return "release_hub_contract"
if "ops_observability_contract" in normalized_contract_keys:
return "ops_observability_contract"
if "ops_driver_contract" in normalized_contract_keys:
return "ops_driver_contract"
return normalized_contract_keys[0] if normalized_contract_keys else ""
def _attach_driver_feed_contract_navigation(entry: dict) -> dict:
normalized_entry = dict(entry or {})
contract_keys = _normalize_ops_contract_keys(
list(normalized_entry.get("contract_keys") or [])
+ _suggest_ops_contract_keys_for_codex_entry(normalized_entry)
)
return {
**normalized_entry,
"contract_keys": contract_keys,
"contract_navigation": _build_ops_contract_navigation(
contract_keys,
primary_contract_key="ops_driver_contract",
registry=get_ops_contract_registry(),
),
}
def _format_node_code_list(raw_values: object, *, limit: int = 5) -> str:
values: list[str] = []
seen: set[str] = set()
for item in list(raw_values or []):
value = str(item or "").strip()
if not value or value in seen:
continue
seen.add(value)
values.append(value)
if not values:
return "-"
visible = values[: max(1, int(limit or 5))]
suffix = f"{len(values)}" if len(values) > len(visible) else ""
return "".join(visible) + suffix
def _normalize_driver_text_lines(raw_values: object, *, limit: int = 5) -> list[str]:
normalized: list[str] = []
seen: set[str] = set()
for item in list(raw_values or []):
value = str(item or "").strip()
if not value or value in seen:
continue
seen.add(value)
normalized.append(value)
return normalized[: max(1, int(limit or 5))]
def _is_cluster_online_node(node: dict) -> bool:
return str(node.get("cluster_status") or "").strip() in {"online", "busy"}
def _is_inspection_visible_node(node: dict) -> bool:
return bool(node.get("cluster_is_effective_worker", False)) and _is_cluster_online_node(node)
def _is_inspection_eligible_node(node: dict) -> bool:
return (
bool(node.get("is_managed", False))
and bool(node.get("is_enabled", False))
and bool(node.get("is_agent_online", False))
and _is_inspection_visible_node(node)
)
def _inspection_job_bucket(job: dict) -> str:
action = str(job.get("action") or "").strip()
if action in {"health.snapshot", "diagnostics.collect"}:
return action
if action != "logs.collect":
return ""
service_name = str((job.get("payload") or {}).get("service_name") or "").strip().lower()
if "worker" not in service_name:
return ""
return "logs.collect"
def _slim_inspection_job(job: dict) -> dict:
payload = dict(job.get("payload") or {})
return {
"id": int(job.get("id") or 0),
"job_code": str(job.get("job_code") or ""),
"action": str(job.get("action") or ""),
"status": str(job.get("status") or ""),
"target_node_code": str(job.get("target_node_code") or ""),
"created_at": str(job.get("created_at") or ""),
"started_at": str(job.get("started_at") or ""),
"finished_at": str(job.get("finished_at") or ""),
"updated_at": str(job.get("updated_at") or ""),
"payload": {
"service_name": str(payload.get("service_name") or ""),
"lines": int(payload.get("lines") or 0) if payload.get("lines") not in (None, "") else 0,
},
"error_message": str(job.get("error_message") or ""),
}
def _inspection_status_label(status: str) -> str:
normalized_status = str(status or "").strip()
if not normalized_status:
return "未收口"
mapping = {
"success": "成功",
"failed": "失败",
"blocked": "阻断",
"cancelled": "已取消",
"awaiting_approval": "待审批",
"dispatching": "待派发",
"running": "执行中",
"queued": "排队中",
}
return mapping.get(normalized_status, normalized_status)
def _build_inspection_contract_result(job: dict | None, *, action_label: str) -> dict:
normalized_job = dict(job or {})
if not normalized_job:
return {
"status": "",
"status_label": "未收口",
"occurred_at": "",
"job_code": "",
"summary": f"最近没有 {action_label} 收口记录。",
"error_message": "",
}
status = str(normalized_job.get("status") or "").strip()
occurred_at = (
str(normalized_job.get("finished_at") or "").strip()
or str(normalized_job.get("updated_at") or "").strip()
or str(normalized_job.get("started_at") or "").strip()
or str(normalized_job.get("created_at") or "").strip()
)
job_code = str(normalized_job.get("job_code") or "").strip()
error_message = str(normalized_job.get("error_message") or "").strip()
if status == "success":
summary = f"最近一次{action_label}收口成功。"
elif status:
summary = f"最近一次{action_label}状态为 {_inspection_status_label(status)}"
else:
summary = f"最近没有 {action_label} 收口记录。"
if error_message:
summary = f"{summary} 错误:{error_message}"
return {
"status": status,
"status_label": _inspection_status_label(status) if status else "未收口",
"occurred_at": occurred_at,
"job_code": job_code,
"summary": summary,
"error_message": error_message,
}
def _build_inspection_issue_summary(
job_map: dict[str, dict],
*,
node_code: str = "",
latest_job: dict | None = None,
node: dict | None = None,
) -> dict:
action_labels = {
"health.snapshot": "健康快照",
"logs.collect": "Worker 日志",
"diagnostics.collect": "诊断包",
}
node_context = dict(node or {})
node_is_visible = _is_inspection_visible_node(node_context)
node_is_eligible = bool(node_context.get("is_inspection_eligible", False)) or _is_inspection_eligible_node(node_context)
node_is_participating = bool(node_context.get("cluster_detect_participating", False))
node_is_managed = bool(node_context.get("is_managed", False))
node_is_enabled = bool(node_context.get("is_enabled", False))
node_agent_online = bool(node_context.get("is_agent_online", False))
node_has_ssh_access = bool(node_context.get("has_ssh_access", False))
agent_state_label = str(node_context.get("agent_state_label") or "").strip()
agent_state_reason = str(node_context.get("agent_state_reason") or "").strip()
preferred_job = (
job_map.get("diagnostics.collect")
or job_map.get("logs.collect")
or job_map.get("health.snapshot")
or dict(latest_job or {})
)
preferred_job_id = int(preferred_job.get("id") or 0)
preferred_job_code = str(preferred_job.get("job_code") or "").strip()
preferred_action = str(preferred_job.get("action") or "").strip()
preferred_target_node_code = str(preferred_job.get("target_node_code") or node_code or "").strip()
preferred_ui_intent = (
_build_driver_ui_intent(
"job_events",
job_id=preferred_job_id,
job_code=preferred_job_code,
action=preferred_action,
target_node_code=preferred_target_node_code,
)
if preferred_job_id > 0
else {}
)
present_statuses = [
str((job_map.get(key) or {}).get("status") or "").strip()
for key in _OPS_INSPECTION_ACTION_KEYS
if str((job_map.get(key) or {}).get("status") or "").strip()
]
if node_is_visible and not node_is_eligible:
if not node_is_managed:
return {
"problem_kind": "handover_gap",
"problem_label": "未接管",
"problem_title": "执行节点未接管",
"problem_level": "danger",
"problem_keys": [],
"summary": "节点在线且属于有效执行面,但还没纳入 Node Agent 接管,当前不能稳定下发标准巡检、发布或诊断动作。",
"recommended_action": "纳管节点",
"recommended_action_code": "handover_first_gap",
"ui_intent": {},
}
if not node_is_enabled:
return {
"problem_kind": "handover_gap",
"problem_label": "已停用",
"problem_title": "节点已停用",
"problem_level": "warning",
"problem_keys": [],
"summary": "节点虽然仍在有效执行面中,但当前被标记为停用,先确认是否需要重新启用后再下发巡检或发布动作。",
"recommended_action": "查看节点",
"recommended_action_code": "view_first_gap",
"ui_intent": {},
}
if not node_agent_online:
return {
"problem_kind": "handover_gap",
"problem_label": "SSH 已备好" if node_has_ssh_access else (agent_state_label or "待接入"),
"problem_title": "Agent 未就绪",
"problem_level": "danger",
"problem_keys": [],
"summary": (
f"节点已纳管SSH 入口已备好,虽然仍未进入 Agent 在线状态,但已可通过 SSH 执行日志、诊断与部分服务控制。{agent_state_reason}"
if node_has_ssh_access and agent_state_reason
else (
"节点已纳管SSH 入口已备好,当前已可通过 SSH 执行日志、诊断与部分服务控制;标准巡检编排、正式发布与 Rollout 仍优先依赖 Node Agent 在线。"
if node_has_ssh_access
else (
f"节点已纳管,但仍未进入 Agent 在线状态。{agent_state_reason}"
if agent_state_reason
else "节点已纳管,但仍未进入 Agent 在线状态,当前不能稳定执行标准巡检。"
)
)
),
"recommended_action": "补接入",
"recommended_action_code": "handover_first_gap",
"ui_intent": {},
}
if node_is_visible and node_is_participating and not present_statuses:
return {
"problem_kind": "participating_no_inspection",
"problem_label": "现场未巡检",
"problem_title": "参与检测但未形成巡检",
"problem_level": "warning",
"problem_keys": [],
"summary": "节点当前正在真实参与检测,但还没有形成标准巡检记录,建议先看现场日志或补一轮标准巡检。",
"recommended_action": "看现场日志",
"recommended_action_code": "open_worker_logs_participating",
"ui_intent": {},
}
if node_is_visible and node_is_eligible and not node_is_participating and not present_statuses:
return {
"problem_kind": "standby_no_inspection",
"problem_label": "待命未巡检",
"problem_title": "在线待命但未巡检",
"problem_level": "info",
"problem_keys": [],
"summary": "节点当前在线可执行,但尚未参与检测,也还没有形成标准巡检记录,适合先做一轮巡检确认是否正常待命。",
"recommended_action": "补齐标准巡检",
"recommended_action_code": "run_standard_inspection",
"ui_intent": {},
}
terminal_problem_actions = [
key
for key in _OPS_INSPECTION_ACTION_KEYS
if str((job_map.get(key) or {}).get("status") or "").strip() in {"failed", "blocked", "cancelled"}
]
if terminal_problem_actions:
key = terminal_problem_actions[0]
job = job_map.get(key) or {}
return {
"problem_kind": "terminal_failure",
"problem_label": "失败/阻断",
"problem_title": f"{action_labels[key]}失败",
"problem_level": "error",
"problem_keys": terminal_problem_actions,
"summary": f"{action_labels[key]}当前状态为 {_inspection_status_label(str(job.get('status') or ''))},优先查看事件并重跑标准巡检。",
"recommended_action": "先看事件",
"recommended_action_code": "focus_latest_job_events",
"ui_intent": preferred_ui_intent,
}
running_actions = [
key
for key in _OPS_INSPECTION_ACTION_KEYS
if str((job_map.get(key) or {}).get("status") or "").strip() in {"queued", "dispatching", "running", "awaiting_approval"}
]
if running_actions:
key = running_actions[0]
job = job_map.get(key) or {}
return {
"problem_kind": "inflight_execution",
"problem_label": "执行中",
"problem_title": f"{action_labels[key]}执行中",
"problem_level": "warning",
"problem_keys": running_actions,
"summary": f"{action_labels[key]}仍在{_inspection_status_label(str(job.get('status') or ''))},先等待回执或检查对应节点事件流。",
"recommended_action": "查看事件",
"recommended_action_code": "focus_latest_job_events",
"ui_intent": preferred_ui_intent,
}
missing_actions = [key for key in _OPS_INSPECTION_ACTION_KEYS if not str((job_map.get(key) or {}).get("status") or "").strip()]
if missing_actions:
missing_problem_map = {
"health.snapshot": ("missing_health_snapshot", "缺健康快照", "健康快照缺口"),
"logs.collect": ("missing_worker_logs", "缺 Worker 日志", "Worker 日志缺口"),
"diagnostics.collect": ("missing_diagnostics", "缺诊断包", "诊断包缺口"),
}
if len(missing_actions) == 1:
problem_kind, problem_label, problem_title = missing_problem_map.get(
missing_actions[0],
("missing_inspection_steps", "巡检缺口", "巡检缺口"),
)
else:
problem_kind, problem_label, problem_title = (
"missing_inspection_steps",
"巡检缺口",
"巡检缺口",
)
return {
"problem_kind": problem_kind,
"problem_label": problem_label,
"problem_title": problem_title,
"problem_level": "info",
"problem_keys": missing_actions,
"summary": f"当前仍缺少 {''.join(action_labels[key] for key in missing_actions)} 的巡检记录。",
"recommended_action": "补齐标准巡检",
"recommended_action_code": "run_standard_inspection",
"ui_intent": {},
}
return {
"problem_kind": "healthy",
"problem_label": "已收口",
"problem_title": "巡检健康",
"problem_level": "success",
"problem_keys": [],
"summary": "当前三类巡检均已成功回执。",
"recommended_action": "当前无需处理",
"recommended_action_code": "no_action",
"ui_intent": {},
}
def _slim_execution_scene_node(node: dict) -> dict:
raw_participation_state = str(node.get("participation_state") or "").strip()
is_dispatch_active = bool(node.get("is_dispatch_active", False))
if is_dispatch_active or raw_participation_state in {"running", "claimed", "dispatch_active"}:
participation_bucket = "dispatch_active"
participation_bucket_label = "执行/已领"
elif raw_participation_state in {"recent_throughput", "recent_only"}:
participation_bucket = "recent_only"
participation_bucket_label = "近窗有吞吐"
elif raw_participation_state == "load_syncing":
participation_bucket = "load_syncing"
participation_bucket_label = "负载待确认"
elif raw_participation_state in {"standby", "idle"}:
participation_bucket = "standby"
participation_bucket_label = "在线待命"
else:
participation_bucket = raw_participation_state
participation_bucket_label = str(node.get("participation_label") or "").strip()
return {
"node_code": str(node.get("node_code") or ""),
"region": str(node.get("region") or ""),
"role": str(node.get("role") or ""),
"status": str(node.get("status") or ""),
"current_load": int(node.get("current_load", 0) or 0),
"last_heartbeat_at": str(node.get("last_heartbeat_at") or ""),
"participation_state": raw_participation_state,
"participation_state_raw": raw_participation_state,
"participation_label": str(node.get("participation_label") or ""),
"participation_reason": str(node.get("participation_reason") or node.get("standby_reason") or ""),
"participation_bucket": participation_bucket,
"participation_bucket_label": participation_bucket_label,
"is_dispatch_active": is_dispatch_active,
"items_total": int(node.get("items_total", 0) or 0),
"items_claimed": int(node.get("items_claimed", 0) or 0),
"items_running": int(node.get("items_running", 0) or 0),
"items_completed": int(node.get("items_completed", 0) or 0),
"items_failed": int(node.get("items_failed", 0) or 0),
"processed_recent": int(node.get("processed_recent", 0) or 0),
"processed_per_minute": float(node.get("processed_per_minute", 0) or 0),
"detail": str(node.get("detail") or ""),
"phase": str(node.get("phase") or ""),
}
def _preferred_release_label(release_summary: dict) -> str:
active_by_channel = dict(release_summary.get("active_by_channel") or {})
for channel in ("stable", "beta", "canary"):
version = str(active_by_channel.get(channel) or "").strip()
if version:
return f"{version} / {channel}"
return ""
def _preferred_release_for_ops() -> dict:
for channel in ("stable", "beta", "canary"):
release = get_latest_release(channel=channel)
if int(release.get("id") or 0) > 0:
return release
return {}
def _normalize_driver_node_codes(raw_node_codes: object) -> list[str]:
normalized: list[str] = []
seen: set[str] = set()
for item in list(raw_node_codes or []):
node_code = str(item or "").strip()
if not node_code or node_code in seen:
continue
seen.add(node_code)
normalized.append(node_code)
return normalized
def _recommend_driver_execution_mode(node_codes: list[str]) -> dict:
return _recommend_driver_execution_mode_for_supported_modes(
node_codes,
supported_modes=["remote-agent", "ssh"],
default_mode="remote-agent",
)
def _normalize_driver_supported_execution_modes(
supported_modes: list[str] | tuple[str, ...] | None = None,
*,
default_mode: str = "remote-agent",
) -> list[str]:
normalized_default_mode = str(default_mode or "").strip() or "remote-agent"
normalized_modes: list[str] = []
seen: set[str] = set()
for item in list(supported_modes or []):
mode = str(item or "").strip()
if not mode or mode in seen:
continue
seen.add(mode)
normalized_modes.append(mode)
if normalized_modes:
return normalized_modes
return [normalized_default_mode]
def _recommend_driver_execution_mode_for_supported_modes(
node_codes: list[str],
*,
supported_modes: list[str] | tuple[str, ...] | None = None,
default_mode: str = "remote-agent",
) -> dict:
normalized_node_codes = _normalize_driver_node_codes(node_codes)
normalized_supported_modes = _normalize_driver_supported_execution_modes(
supported_modes,
default_mode=default_mode,
)
normalized_default_mode = (
str(default_mode or "").strip()
if str(default_mode or "").strip() in normalized_supported_modes
else normalized_supported_modes[0]
)
if len(normalized_supported_modes) == 1:
only_mode = normalized_supported_modes[0]
return {
"execution_mode": only_mode,
"reason": f"当前动作仅支持 {only_mode},保持该执行路径。",
"resolved_nodes": [],
"missing_node_codes": [],
"supported_modes": normalized_supported_modes,
}
if not normalized_node_codes:
return {
"execution_mode": normalized_default_mode,
"reason": f"当前未指定目标节点,默认继续使用 {normalized_default_mode}",
"resolved_nodes": [],
"missing_node_codes": [],
"supported_modes": normalized_supported_modes,
}
if (
"local-runtime" in normalized_supported_modes
and normalized_node_codes
and all(node_code == settings.node_code for node_code in normalized_node_codes)
):
return {
"execution_mode": "local-runtime",
"reason": "当前目标节点就是本机控制面节点,推荐直接走 local-runtime。",
"resolved_nodes": [],
"missing_node_codes": [],
"supported_modes": normalized_supported_modes,
}
managed_nodes_payload = list_managed_nodes_with_agent_state()
managed_nodes = list(managed_nodes_payload.get("nodes") or [])
managed_map = {
str(item.get("node_code") or "").strip(): dict(item or {})
for item in managed_nodes
if str(item.get("node_code") or "").strip()
}
resolved_nodes = [managed_map[node_code] for node_code in normalized_node_codes if node_code in managed_map]
missing_node_codes = [node_code for node_code in normalized_node_codes if node_code not in managed_map]
def _agent_ready(node: dict) -> bool:
return (
bool(node.get("is_managed", False))
and bool(node.get("is_enabled", False))
and bool(node.get("is_agent_online", False))
)
def _ssh_ready(node: dict) -> bool:
return (
bool(node.get("is_managed", False))
and bool(node.get("is_enabled", False))
and bool(node.get("has_ssh_access", False))
)
if (
"remote-agent" in normalized_supported_modes
and resolved_nodes
and len(resolved_nodes) == len(normalized_node_codes)
and all(_agent_ready(node) for node in resolved_nodes)
):
return {
"execution_mode": "remote-agent",
"reason": "当前目标节点都已进入 Agent 在线状态,默认沿用 remote-agent。",
"resolved_nodes": resolved_nodes,
"missing_node_codes": missing_node_codes,
"supported_modes": normalized_supported_modes,
}
if (
"ssh" in normalized_supported_modes
and resolved_nodes
and len(resolved_nodes) == len(normalized_node_codes)
and all(_ssh_ready(node) for node in resolved_nodes)
):
return {
"execution_mode": "ssh",
"reason": "当前目标节点 SSH 已全部备好,自动切到 SSH 执行路径。",
"resolved_nodes": resolved_nodes,
"missing_node_codes": missing_node_codes,
"supported_modes": normalized_supported_modes,
}
return {
"execution_mode": normalized_default_mode,
"reason": f"当前目标节点执行能力不一致,默认保守回退到 {normalized_default_mode}",
"resolved_nodes": resolved_nodes,
"missing_node_codes": missing_node_codes,
"supported_modes": normalized_supported_modes,
}
def _recommend_driver_execution_mode_for_playbook(playbook_key: str, node_codes: list[str]) -> dict:
playbook = dict(get_ops_playbook(playbook_key) or {})
supported_modes = list(playbook.get("execution_modes") or [])
default_mode = str(playbook.get("default_execution_mode") or "remote-agent").strip() or "remote-agent"
return _recommend_driver_execution_mode_for_supported_modes(
node_codes,
supported_modes=supported_modes,
default_mode=default_mode,
)
def _recommend_driver_execution_mode_for_action_template(template_key: str, node_codes: list[str]) -> dict:
template = dict(get_ops_action_template(template_key) or {})
supported_modes = list(template.get("execution_modes") or [])
default_mode = str(template.get("default_execution_mode") or "remote-agent").strip() or "remote-agent"
return _recommend_driver_execution_mode_for_supported_modes(
node_codes,
supported_modes=supported_modes,
default_mode=default_mode,
)
def _build_driver_ui_intent(kind: str, **payload: object) -> dict:
return {
"kind": str(kind or "").strip(),
**payload,
}
def _normalize_focus_ref(focus_ref: object) -> dict:
if not isinstance(focus_ref, dict):
return {}
normalized: dict[str, object] = {}
for raw_key, raw_value in focus_ref.items():
key = str(raw_key or "").strip()
if not key:
continue
if isinstance(raw_value, bool):
normalized[key] = raw_value
continue
if isinstance(raw_value, int):
if raw_value:
normalized[key] = raw_value
continue
if isinstance(raw_value, float):
if raw_value:
normalized[key] = raw_value
continue
if isinstance(raw_value, str):
value = raw_value.strip()
if value:
normalized[key] = value
continue
if raw_value is not None:
normalized[key] = raw_value
return normalized
def _find_activity_stream_item(activity_key: str, *, kind: str = "", scan_limit: int = 200) -> dict:
normalized_activity_key = str(activity_key or "").strip()
normalized_kind = str(kind or "").strip()
if not normalized_activity_key:
return {}
activity_stream = get_ops_activity_stream(limit=min(max(scan_limit, 20), 100), scan_limit=max(scan_limit, 80))
for item in list(activity_stream.get("items") or []):
if str(item.get("activity_key") or "").strip() != normalized_activity_key:
continue
if normalized_kind and str(item.get("kind") or "").strip() != normalized_kind:
continue
return dict(item or {})
return {}
def _merge_focus_ref(focus_ref: object, **extra: object) -> dict:
merged = _normalize_focus_ref(focus_ref)
for raw_key, raw_value in extra.items():
key = str(raw_key or "").strip()
if not key:
continue
if isinstance(raw_value, bool):
merged[key] = raw_value
continue
if isinstance(raw_value, int):
if raw_value:
merged[key] = raw_value
continue
if isinstance(raw_value, float):
if raw_value:
merged[key] = raw_value
continue
if isinstance(raw_value, str):
value = raw_value.strip()
if value:
merged[key] = value
continue
if raw_value is not None:
merged[key] = raw_value
return merged
def _action_payload_with_focus_ref(action_payload: dict | None = None, focus_ref: object = None) -> dict:
payload = dict(action_payload or {})
merged_focus_ref = _merge_focus_ref(focus_ref, **_normalize_focus_ref(payload.get("focus_ref")))
if merged_focus_ref:
payload["focus_ref"] = merged_focus_ref
return payload
def _driver_launchpad_context(action_payload: dict) -> dict:
payload = dict(action_payload or {})
return {
"channel": str(payload.get("channel") or "stable").strip() or "stable",
"control_plane_base_url": str(payload.get("control_plane_base_url") or "").strip(),
}
def _release_launchpad_preview(release_launchpad: dict, mode: str) -> dict:
if str(mode or "").strip() == "control":
return dict(release_launchpad.get("control_rollout_preview") or {})
return dict(release_launchpad.get("worker_rollout_preview") or {})
def _release_launchpad_operational_rows(release_launchpad: dict, *, modes: tuple[str, ...]) -> list[dict]:
rows: list[dict] = []
seen: set[tuple[str, str]] = set()
for mode in modes:
preview = _release_launchpad_preview(release_launchpad, mode)
release_gate = dict((dict(preview.get("policy_preview") or {}).get("release_gate") or {}))
operational_readiness = dict(release_gate.get("operational_readiness") or {})
for raw_row in list(operational_readiness.get("rows") or []):
row = dict(raw_row or {})
node_code = str(row.get("node_code") or "").strip()
if not node_code:
continue
key = (str(mode or "").strip() or "worker", node_code)
if key in seen:
continue
seen.add(key)
rows.append(
{
**row,
"mode": str(mode or "").strip() or "worker",
}
)
return rows
def _release_launchpad_gap_rows(release_launchpad: dict) -> list[dict]:
rows = _release_launchpad_operational_rows(release_launchpad, modes=("worker", "control"))
def _row_execution_ready(row: dict) -> bool:
if "execution_ready" in row:
return bool(row.get("execution_ready", False))
return bool(row.get("remote_agent_ready", False))
gap_rows = [
row
for row in rows
if not _row_execution_ready(row)
or str(row.get("inspection_status") or "").strip() in {"running", "attention", "missing"}
]
gap_rows.sort(
key=lambda row: (
0 if not _row_execution_ready(row) else 1,
0 if str(row.get("inspection_status") or "").strip() in {"attention", "missing"} else 1,
0 if str(row.get("inspection_status") or "").strip() == "running" else 1,
str(row.get("node_code") or ""),
)
)
return gap_rows
def _first_release_launchpad_gap_row(release_launchpad: dict) -> dict:
gap_rows = _release_launchpad_gap_rows(release_launchpad)
return dict(gap_rows[0] or {}) if gap_rows else {}
def _release_launchpad_gap_action_context(release_launchpad: dict) -> dict:
first_gap_row = _first_release_launchpad_gap_row(release_launchpad)
action_code = str(first_gap_row.get("recovery_action") or "").strip()
node_code = str(first_gap_row.get("node_code") or "").strip()
summary = str(
first_gap_row.get("recovery_summary")
or first_gap_row.get("onboarding_summary")
or first_gap_row.get("agent_reason")
or first_gap_row.get("inspection_reason")
or ""
).strip()
if action_code not in {"bootstrap_run", "run_acceptance"}:
action_code = ""
return {
"action_code": action_code,
"node_code": node_code,
"summary": summary,
"row": first_gap_row,
}
def _release_launchpad_target_node_codes(release_launchpad: dict, *, mode: str) -> list[str]:
preview = _release_launchpad_preview(release_launchpad, mode)
smart_rollout = dict(preview.get("smart_rollout") or {})
target_node_codes = _normalize_driver_node_codes(smart_rollout.get("target_node_codes") or [])
if target_node_codes:
return target_node_codes
return _normalize_driver_node_codes(
[str(row.get("node_code") or "").strip() for row in _release_launchpad_operational_rows(release_launchpad, modes=(mode,))]
)
def _build_release_launchpad_review_payload(*, release_launchpad: dict, mode: str) -> dict:
normalized_mode = "control" if str(mode or "").strip() == "control" else "worker"
launchpad = dict(release_launchpad or {})
latest_release = dict(launchpad.get("latest_release") or {})
preview = _release_launchpad_preview(launchpad, normalized_mode)
smart_rollout = dict(preview.get("smart_rollout") or {})
launchpad_status = dict(launchpad.get("launchpad_status") or {})
focus_ref = _merge_focus_ref(
launchpad_status.get("focus_ref"),
release_id=int(latest_release.get("id") or 0),
release_version=str(latest_release.get("release_version") or "").strip(),
channel=str(latest_release.get("channel") or launchpad.get("channel") or "").strip(),
section="release_launchpad",
mode=normalized_mode,
)
return {
"mode": normalized_mode,
"focus_ref": focus_ref,
"latest_release": latest_release,
"launchpad_status": launchpad_status,
"preview": preview,
"smart_rollout": smart_rollout,
"target_node_codes": _release_launchpad_target_node_codes(launchpad, mode=normalized_mode),
"gap_rows": _release_launchpad_gap_rows(launchpad),
}
def _build_release_launchpad_driver_card(*, release_launchpad: dict) -> dict:
launchpad = dict(release_launchpad or {})
launchpad_status = dict(launchpad.get("launchpad_status") or {})
action_code = str(launchpad_status.get("recommended_action_code") or "").strip()
if not action_code:
return {}
latest_release = dict(launchpad.get("latest_release") or {})
latest_package = dict(launchpad.get("latest_package") or {})
worker_preview = dict(launchpad.get("worker_rollout_preview") or {})
control_preview = dict(launchpad.get("control_rollout_preview") or {})
worker_plan = dict(worker_preview.get("smart_rollout") or {})
control_plan = dict(control_preview.get("smart_rollout") or {})
first_gap_row = _first_release_launchpad_gap_row(launchpad)
gap_node_codes = _normalize_driver_node_codes(
[str(row.get("node_code") or "").strip() for row in _release_launchpad_gap_rows(launchpad)]
)
worker_node_codes = _release_launchpad_target_node_codes(launchpad, mode="worker")
control_node_codes = _release_launchpad_target_node_codes(launchpad, mode="control")
latest_release_id = int(latest_release.get("id") or 0)
launchpad_focus_ref = _merge_focus_ref(
launchpad_status.get("focus_ref"),
release_id=latest_release_id,
release_version=str(latest_release.get("release_version") or "").strip(),
channel=str(latest_release.get("channel") or launchpad.get("channel") or "").strip(),
)
release_detail_focus_ref = _merge_focus_ref(
latest_release.get("focus_ref"),
release_id=latest_release_id,
release_version=str(latest_release.get("release_version") or "").strip(),
channel=str(latest_release.get("channel") or launchpad.get("channel") or "").strip(),
section="release_detail",
)
worker_launchpad_focus_ref = _merge_focus_ref(launchpad_focus_ref, section="release_launchpad", mode="worker")
control_launchpad_focus_ref = _merge_focus_ref(launchpad_focus_ref, section="release_launchpad", mode="control")
rollout_gate_focus_ref = _merge_focus_ref(launchpad_focus_ref, section="default_rollout_gate")
release_label = (
str(latest_release.get("release_version") or "").strip()
or str(latest_package.get("package_name") or "").strip()
or str(launchpad.get("channel") or "stable").strip()
or "latest"
)
shared_action_payload = {
"channel": str(launchpad.get("channel") or "stable").strip() or "stable",
"control_plane_base_url": str(launchpad.get("control_plane_base_url") or "").strip(),
}
if latest_release_id > 0:
shared_action_payload["release_id"] = latest_release_id
base_card = {
"level_label": "推荐",
"tag_type": "info",
"summary": str(launchpad_status.get("summary") or "").strip(),
"reason": "",
"node_codes": [],
"meta_text": release_label,
"focus_ref": launchpad_focus_ref,
"primary_focus_ref": {},
"secondary_focus_ref": {},
"primary_type": "primary",
"disabled": False,
"primary_action_code": action_code,
"primary_action_payload": dict(shared_action_payload),
"secondary_label": "查看版本区" if latest_release_id > 0 else "",
"secondary_action_code": "focus_release_hub" if latest_release_id > 0 else "",
"secondary_action_payload": (
{
"release_id": latest_release_id,
}
if latest_release_id > 0
else {}
),
}
if action_code == "publish_latest_worker":
return {
**base_card,
"key": "release-launchpad-ready",
"title": "当前可以发 Worker 灰度",
"level_label": "准备就绪",
"tag_type": "success",
"reason": str(worker_plan.get("summary") or worker_preview.get("message") or "最新发布包与 Worker 智能 Rollout 预检均已收口。").strip(),
"node_codes": worker_node_codes,
"primary_label": "发 Worker 灰度",
"primary_focus_ref": worker_launchpad_focus_ref,
"primary_action_payload": _action_payload_with_focus_ref(shared_action_payload, worker_launchpad_focus_ref),
"secondary_label": "看 Control 预案" if bool(control_plan.get("available", False)) else base_card["secondary_label"],
"secondary_action_code": "review_control_rollout" if bool(control_plan.get("available", False)) else base_card["secondary_action_code"],
"secondary_focus_ref": (
control_launchpad_focus_ref if bool(control_plan.get("available", False)) else release_detail_focus_ref
),
"secondary_action_payload": (
_action_payload_with_focus_ref(shared_action_payload, control_launchpad_focus_ref)
if bool(control_plan.get("available", False))
else _action_payload_with_focus_ref(base_card["secondary_action_payload"], release_detail_focus_ref)
),
}
if action_code == "review_smart_rollout_preview":
return {
**base_card,
"key": "release-launchpad-review-worker",
"title": "先确认 Worker 灰度预案",
"level_label": "待确认",
"tag_type": "warning",
"reason": str(worker_preview.get("message") or worker_plan.get("summary") or "Worker 智能 Rollout 仍需人工确认。").strip(),
"node_codes": worker_node_codes,
"primary_label": "查看 Worker 预案",
"primary_focus_ref": worker_launchpad_focus_ref,
"primary_action_payload": _action_payload_with_focus_ref(shared_action_payload, worker_launchpad_focus_ref),
"secondary_focus_ref": release_detail_focus_ref,
"secondary_action_payload": _action_payload_with_focus_ref(base_card["secondary_action_payload"], release_detail_focus_ref),
}
if action_code == "review_control_rollout":
return {
**base_card,
"key": "release-launchpad-review-control",
"title": "先确认 Control 发布预案",
"level_label": "推荐",
"tag_type": "warning",
"reason": str(control_preview.get("message") or control_plan.get("summary") or "当前只有 Control 侧满足预发条件。").strip(),
"node_codes": control_node_codes,
"primary_label": "查看 Control 预案",
"primary_focus_ref": control_launchpad_focus_ref,
"primary_action_payload": _action_payload_with_focus_ref(shared_action_payload, control_launchpad_focus_ref),
"secondary_focus_ref": release_detail_focus_ref,
"secondary_action_payload": _action_payload_with_focus_ref(base_card["secondary_action_payload"], release_detail_focus_ref),
}
if action_code == "fix_rollout_blockers":
primary_label = "查看 Worker 预案"
if str(first_gap_row.get("node_code") or "").strip():
primary_label = "纳管首台节点" if not bool(first_gap_row.get("is_managed", False)) else "补接入首台"
if str(first_gap_row.get("recovery_action") or "").strip() == "bootstrap_run":
primary_label = "跑接入收口"
elif str(first_gap_row.get("recovery_action") or "").strip() == "run_acceptance":
primary_label = "跑接管验收"
return {
**base_card,
"key": "release-launchpad-fix-blockers",
"title": "先收口发布门禁",
"level_label": "最高优先",
"tag_type": "danger",
"reason": str(
first_gap_row.get("agent_reason")
or first_gap_row.get("inspection_reason")
or worker_preview.get("message")
or control_preview.get("message")
or "Rollout 预检仍存在阻断条件,需先补接管、在线状态或巡检缺口。"
).strip(),
"node_codes": gap_node_codes or worker_node_codes or control_node_codes,
"primary_label": primary_label,
"primary_focus_ref": rollout_gate_focus_ref,
"primary_action_payload": _action_payload_with_focus_ref(shared_action_payload, rollout_gate_focus_ref),
"secondary_focus_ref": release_detail_focus_ref,
"secondary_action_payload": _action_payload_with_focus_ref(base_card["secondary_action_payload"], release_detail_focus_ref),
}
if action_code in {"bootstrap_run", "run_acceptance"}:
primary_label = "跑接入收口" if action_code == "bootstrap_run" else "跑接管验收"
return {
**base_card,
"key": f"release-launchpad-{action_code}",
"title": "先收口执行面接管",
"level_label": "最高优先" if action_code == "bootstrap_run" else "推荐",
"tag_type": "warning" if action_code == "bootstrap_run" else "success",
"reason": str(
first_gap_row.get("recovery_summary")
or first_gap_row.get("onboarding_summary")
or first_gap_row.get("agent_reason")
or first_gap_row.get("inspection_reason")
or launchpad_status.get("summary")
or "当前需要先收口节点接管,再继续发布。"
).strip(),
"node_codes": gap_node_codes or ([str(first_gap_row.get("node_code") or "").strip()] if str(first_gap_row.get("node_code") or "").strip() else []),
"primary_label": primary_label,
"primary_focus_ref": rollout_gate_focus_ref,
"primary_action_payload": _action_payload_with_focus_ref(
{
**shared_action_payload,
"node_code": str(first_gap_row.get("node_code") or "").strip(),
},
rollout_gate_focus_ref,
),
"secondary_focus_ref": release_detail_focus_ref,
"secondary_action_payload": _action_payload_with_focus_ref(base_card["secondary_action_payload"], release_detail_focus_ref),
}
if action_code == "fix_managed_nodes":
primary_label = "同步节点"
if str(first_gap_row.get("node_code") or "").strip():
primary_label = "纳管首台节点" if not bool(first_gap_row.get("is_managed", False)) else "补接入首台"
return {
**base_card,
"key": "release-launchpad-fix-managed",
"title": "先补执行面接管",
"level_label": "推荐",
"tag_type": "warning",
"reason": str(
first_gap_row.get("agent_reason")
or first_gap_row.get("inspection_reason")
or "当前没有可用于智能 Rollout 的在线启用执行节点,需要先补齐托管或接入状态。"
).strip(),
"node_codes": gap_node_codes,
"primary_label": primary_label,
"primary_focus_ref": rollout_gate_focus_ref,
"primary_action_payload": _action_payload_with_focus_ref(shared_action_payload, rollout_gate_focus_ref),
"secondary_focus_ref": release_detail_focus_ref,
"secondary_action_payload": _action_payload_with_focus_ref(base_card["secondary_action_payload"], release_detail_focus_ref),
}
if action_code == "api-restart":
return {
**base_card,
"key": "release-launchpad-runtime-refresh",
"title": "先刷新控制面运行时",
"level_label": "最高优先",
"tag_type": "warning",
"reason": str(
launchpad_status.get("summary")
or "运行中的控制面 API 还没有刷新到当前仓库能力,建议先执行 runtime-refresh-recover。"
).strip(),
"node_codes": [],
"primary_label": "查看刷新收口",
"primary_focus_ref": worker_launchpad_focus_ref or launchpad_focus_ref,
"primary_action_payload": _action_payload_with_focus_ref(
shared_action_payload,
worker_launchpad_focus_ref or launchpad_focus_ref,
),
"secondary_label": "查看版本区" if latest_release_id > 0 else "",
"secondary_action_code": "focus_release_hub" if latest_release_id > 0 else "",
"secondary_action_payload": {"release_id": latest_release_id} if latest_release_id > 0 else {},
"secondary_focus_ref": release_detail_focus_ref if latest_release_id > 0 else {},
}
if action_code == "release_package":
return {
**base_card,
"key": "release-launchpad-package",
"title": "先准备发布包",
"level_label": "待补齐",
"tag_type": "warning",
"reason": str(latest_package.get("reason") or "当前还没有可用的最新发布包,请先在控制面本机完成打包。").strip(),
"node_codes": [],
"primary_label": "打开打包面板",
"secondary_label": "",
"secondary_action_code": "",
"secondary_action_payload": {},
"primary_focus_ref": worker_launchpad_focus_ref or launchpad_focus_ref,
"primary_action_payload": _action_payload_with_focus_ref(
shared_action_payload,
worker_launchpad_focus_ref or launchpad_focus_ref,
),
"secondary_focus_ref": {},
}
if action_code == "release_prepare":
return {
**base_card,
"key": "release-launchpad-final-signoff",
"title": "先完成最终签收",
"level_label": "最高优先",
"tag_type": "danger",
"reason": str(
latest_package.get("final_release_report_stale_reason")
or launchpad_status.get("summary")
or "当前最新发布包还没有完成最终签收,不能直接进入正式发布动作。"
).strip(),
"node_codes": [],
"primary_label": "执行最终签收",
"secondary_label": "查看版本区" if latest_release_id > 0 else "",
"secondary_action_code": "focus_release_hub" if latest_release_id > 0 else "",
"secondary_action_payload": {"release_id": latest_release_id} if latest_release_id > 0 else {},
"primary_focus_ref": worker_launchpad_focus_ref or launchpad_focus_ref,
"primary_action_payload": _action_payload_with_focus_ref(
shared_action_payload,
worker_launchpad_focus_ref or launchpad_focus_ref,
),
"secondary_focus_ref": release_detail_focus_ref if latest_release_id > 0 else {},
}
return {}
def _runbook_sequence(
*,
key: str,
title: str,
summary: str,
reason: str,
status: str = "ready",
status_label: str = "",
tag_type: str = "",
target_node_codes: list[str] | None = None,
target_scope_label: str = "",
step_titles: list[str] | None = None,
primary_label: str,
primary_action_code: str,
primary_action_payload: dict | None = None,
focus_ref: dict | None = None,
primary_focus_ref: dict | None = None,
secondary_label: str = "",
secondary_action_code: str = "",
secondary_action_payload: dict | None = None,
secondary_focus_ref: dict | None = None,
) -> dict:
return {
"key": str(key or "").strip(),
"title": str(title or "").strip(),
"summary": str(summary or "").strip(),
"reason": str(reason or "").strip(),
"status": str(status or "").strip() or "ready",
"status_label": str(status_label or "").strip() or str(status or "").strip() or "ready",
"tag_type": str(tag_type or "").strip() or "info",
"target_node_codes": _normalize_driver_node_codes(target_node_codes or []),
"target_scope_label": str(target_scope_label or "").strip(),
"step_titles": [str(item or "").strip() for item in list(step_titles or []) if str(item or "").strip()],
"focus_ref": _normalize_focus_ref(focus_ref),
"primary_label": str(primary_label or "").strip() or "执行",
"primary_action_code": str(primary_action_code or "").strip(),
"primary_action_payload": _action_payload_with_focus_ref(primary_action_payload or {}, primary_focus_ref),
"primary_focus_ref": _normalize_focus_ref(primary_focus_ref),
"secondary_label": str(secondary_label or "").strip(),
"secondary_action_code": str(secondary_action_code or "").strip(),
"secondary_action_payload": _action_payload_with_focus_ref(secondary_action_payload or {}, secondary_focus_ref),
"secondary_focus_ref": _normalize_focus_ref(secondary_focus_ref),
}
def _build_release_launchpad_runbook_sequence(*, release_launchpad: dict) -> dict:
launchpad = dict(release_launchpad or {})
launchpad_status = dict(launchpad.get("launchpad_status") or {})
action_code = str(launchpad_status.get("recommended_action_code") or "").strip()
if not action_code:
return {}
latest_release = dict(launchpad.get("latest_release") or {})
latest_package = dict(launchpad.get("latest_package") or {})
worker_preview = dict(launchpad.get("worker_rollout_preview") or {})
control_preview = dict(launchpad.get("control_rollout_preview") or {})
worker_plan = dict(worker_preview.get("smart_rollout") or {})
control_plan = dict(control_preview.get("smart_rollout") or {})
first_gap_row = _first_release_launchpad_gap_row(launchpad)
latest_release_id = int(latest_release.get("id") or 0)
worker_node_codes = _release_launchpad_target_node_codes(launchpad, mode="worker")
control_node_codes = _release_launchpad_target_node_codes(launchpad, mode="control")
gap_node_codes = _normalize_driver_node_codes(
[str(row.get("node_code") or "").strip() for row in _release_launchpad_gap_rows(launchpad)]
)
launchpad_focus_ref = _merge_focus_ref(
launchpad_status.get("focus_ref"),
release_id=latest_release_id,
release_version=str(latest_release.get("release_version") or "").strip(),
channel=str(latest_release.get("channel") or launchpad.get("channel") or "").strip(),
)
release_detail_focus_ref = _merge_focus_ref(
latest_release.get("focus_ref"),
release_id=latest_release_id,
release_version=str(latest_release.get("release_version") or "").strip(),
channel=str(latest_release.get("channel") or launchpad.get("channel") or "").strip(),
section="release_detail",
)
worker_launchpad_focus_ref = _merge_focus_ref(launchpad_focus_ref, section="release_launchpad", mode="worker")
control_launchpad_focus_ref = _merge_focus_ref(launchpad_focus_ref, section="release_launchpad", mode="control")
rollout_gate_focus_ref = _merge_focus_ref(launchpad_focus_ref, section="default_rollout_gate")
release_scope_label = (
str(latest_release.get("release_version") or "").strip()
or str(latest_package.get("package_name") or "").strip()
or str(launchpad.get("channel") or "stable").strip()
or "latest"
)
shared_action_payload = {
"channel": str(launchpad.get("channel") or "stable").strip() or "stable",
"control_plane_base_url": str(launchpad.get("control_plane_base_url") or "").strip(),
}
if latest_release_id > 0:
shared_action_payload["release_id"] = latest_release_id
base_kwargs = {
"key": "release_progression",
"title": "版本发布与放量",
"summary": str(launchpad_status.get("summary") or "").strip(),
"status": str(launchpad_status.get("status") or "").strip() or "attention",
"status_label": str(launchpad_status.get("status_label") or "").strip() or "待处理",
"target_scope_label": f"Release / Launchpad / {release_scope_label}",
"focus_ref": launchpad_focus_ref,
}
if action_code == "publish_latest_worker":
return _runbook_sequence(
**base_kwargs,
reason=str(worker_plan.get("summary") or worker_preview.get("message") or "最新发布包与 Worker 智能 Rollout 预检均已收口。").strip(),
tag_type="success",
target_node_codes=worker_node_codes,
step_titles=["确认最新发布包", "执行 Worker 灰度", "观察回执与巡检", "必要时推进 Control"],
primary_label="直接发 Worker 灰度",
primary_action_code="publish_latest_worker",
primary_action_payload=dict(shared_action_payload),
primary_focus_ref=worker_launchpad_focus_ref,
secondary_label="查看 Control 预案" if bool(control_plan.get("available", False)) else ("查看版本区" if latest_release_id > 0 else ""),
secondary_action_code="review_control_rollout" if bool(control_plan.get("available", False)) else ("focus_release_hub" if latest_release_id > 0 else ""),
secondary_action_payload=dict(shared_action_payload) if bool(control_plan.get("available", False)) else ({"release_id": latest_release_id} if latest_release_id > 0 else {}),
secondary_focus_ref=control_launchpad_focus_ref if bool(control_plan.get("available", False)) else release_detail_focus_ref,
)
if action_code == "review_smart_rollout_preview":
return _runbook_sequence(
**base_kwargs,
reason=str(worker_preview.get("message") or worker_plan.get("summary") or "Worker 智能 Rollout 仍需人工确认。").strip(),
tag_type="warning",
target_node_codes=worker_node_codes,
step_titles=["查看 Worker 智能预案", "确认门禁与审批", "必要时继续创建", "回看 Release Hub"],
primary_label="查看 Worker 预案",
primary_action_code="review_smart_rollout_preview",
primary_action_payload=dict(shared_action_payload),
primary_focus_ref=worker_launchpad_focus_ref,
secondary_label="查看版本区" if latest_release_id > 0 else "",
secondary_action_code="focus_release_hub" if latest_release_id > 0 else "",
secondary_action_payload={"release_id": latest_release_id} if latest_release_id > 0 else {},
secondary_focus_ref=release_detail_focus_ref,
)
if action_code == "review_control_rollout":
return _runbook_sequence(
**base_kwargs,
reason=str(control_preview.get("message") or control_plan.get("summary") or "当前只有 Control 侧满足预发条件。").strip(),
tag_type="warning",
target_node_codes=control_node_codes,
step_titles=["查看 Control 预案", "确认 Control 目标", "确认门禁与审批", "必要时推进发布"],
primary_label="查看 Control 预案",
primary_action_code="review_control_rollout",
primary_action_payload=dict(shared_action_payload),
primary_focus_ref=control_launchpad_focus_ref,
secondary_label="查看版本区" if latest_release_id > 0 else "",
secondary_action_code="focus_release_hub" if latest_release_id > 0 else "",
secondary_action_payload={"release_id": latest_release_id} if latest_release_id > 0 else {},
secondary_focus_ref=release_detail_focus_ref,
)
if action_code == "fix_rollout_blockers":
return _runbook_sequence(
**base_kwargs,
reason=str(
first_gap_row.get("agent_reason")
or first_gap_row.get("inspection_reason")
or worker_preview.get("message")
or control_preview.get("message")
or "Rollout 预检仍存在阻断条件。"
).strip(),
tag_type="danger",
target_node_codes=gap_node_codes or worker_node_codes or control_node_codes,
step_titles=["查看 Worker 智能预案", "定位阻断节点", "补接管或巡检缺口", "重新回看预案"],
primary_label="查看阻断预案",
primary_action_code="fix_rollout_blockers",
primary_action_payload=dict(shared_action_payload),
primary_focus_ref=rollout_gate_focus_ref,
secondary_label="补执行面接管",
secondary_action_code="fix_managed_nodes",
secondary_action_payload=dict(shared_action_payload),
secondary_focus_ref=rollout_gate_focus_ref,
)
if action_code == "fix_managed_nodes":
step_titles = ["同步集群节点", "纳管首台缺口", "补齐 Agent 接入", "回到 Launchpad 复检"]
if str(first_gap_row.get("recovery_action") or "").strip() == "bootstrap_run":
step_titles = ["定位首台缺口节点", "执行 onboarding.bootstrap", "等待 register / heartbeat", "回到 Launchpad 复检"]
elif str(first_gap_row.get("recovery_action") or "").strip() == "run_acceptance":
step_titles = ["定位待验收节点", "执行 onboarding.acceptance", "确认 health / worker / agent", "回到 Launchpad 复检"]
return _runbook_sequence(
**base_kwargs,
reason=str(
first_gap_row.get("agent_reason")
or first_gap_row.get("inspection_reason")
or "当前没有可用于智能 Rollout 的在线启用执行节点,需要先补齐托管或接入状态。"
).strip(),
tag_type="warning",
target_node_codes=gap_node_codes,
step_titles=step_titles,
primary_label="补执行面接管",
primary_action_code="fix_managed_nodes",
primary_action_payload=dict(shared_action_payload),
primary_focus_ref=rollout_gate_focus_ref,
secondary_label="查看版本区" if latest_release_id > 0 else "",
secondary_action_code="focus_release_hub" if latest_release_id > 0 else "",
secondary_action_payload={"release_id": latest_release_id} if latest_release_id > 0 else {},
secondary_focus_ref=release_detail_focus_ref,
)
if action_code in {"bootstrap_run", "run_acceptance"}:
return _runbook_sequence(
**base_kwargs,
reason=str(
first_gap_row.get("recovery_summary")
or first_gap_row.get("onboarding_summary")
or first_gap_row.get("agent_reason")
or "当前需要先收口节点接管,再继续发布。"
).strip(),
tag_type="warning" if action_code == "bootstrap_run" else "success",
target_node_codes=gap_node_codes or ([str(first_gap_row.get("node_code") or "").strip()] if str(first_gap_row.get("node_code") or "").strip() else []),
step_titles=(
["定位首台缺口节点", "执行 onboarding.bootstrap", "等待 register / heartbeat", "回到 Launchpad 复检"]
if action_code == "bootstrap_run"
else ["定位待验收节点", "执行 onboarding.acceptance", "确认 health / worker / agent", "回到 Launchpad 复检"]
),
primary_label="跑接入收口" if action_code == "bootstrap_run" else "跑接管验收",
primary_action_code=action_code,
primary_action_payload={
**dict(shared_action_payload),
"node_code": str(first_gap_row.get("node_code") or "").strip(),
},
primary_focus_ref=rollout_gate_focus_ref,
secondary_label="查看版本区" if latest_release_id > 0 else "",
secondary_action_code="focus_release_hub" if latest_release_id > 0 else "",
secondary_action_payload={"release_id": latest_release_id} if latest_release_id > 0 else {},
secondary_focus_ref=release_detail_focus_ref,
)
if action_code == "api-restart":
return _runbook_sequence(
**base_kwargs,
reason=str(
launchpad_status.get("summary")
or "运行中的控制面 API 还没有刷新到当前仓库能力,建议先执行 runtime-refresh-recover。"
).strip(),
tag_type="warning",
target_node_codes=[],
step_titles=["执行 runtime-refresh-recover", "确认 build-info / route surface", "重新拉取 Launchpad", "再决定 Release / Rollout"],
primary_label="查看刷新收口",
primary_action_code="api-restart",
primary_action_payload=dict(shared_action_payload),
primary_focus_ref=worker_launchpad_focus_ref or launchpad_focus_ref,
secondary_label="查看版本区" if latest_release_id > 0 else "",
secondary_action_code="focus_release_hub" if latest_release_id > 0 else "",
secondary_action_payload={"release_id": latest_release_id} if latest_release_id > 0 else {},
secondary_focus_ref=release_detail_focus_ref if latest_release_id > 0 else {},
)
if action_code == "release_package":
return _runbook_sequence(
**base_kwargs,
reason=str(latest_package.get("reason") or "当前没有可用的最新发布包,请先在控制面本机完成打包。").strip(),
tag_type="warning",
target_node_codes=[],
step_titles=["打开打包面板", "导入最新发布包", "确认 Release 记录", "重新回看 Launchpad"],
primary_label="打开打包面板",
primary_action_code="release_package",
primary_action_payload=dict(shared_action_payload),
primary_focus_ref=worker_launchpad_focus_ref or launchpad_focus_ref,
secondary_label="查看版本区" if latest_release_id > 0 else "",
secondary_action_code="focus_release_hub" if latest_release_id > 0 else "",
secondary_action_payload={"release_id": latest_release_id} if latest_release_id > 0 else {},
secondary_focus_ref=release_detail_focus_ref,
)
if action_code == "release_prepare":
return _runbook_sequence(
**base_kwargs,
reason=str(
latest_package.get("final_release_report_stale_reason")
or launchpad_status.get("summary")
or "当前最新发布包还没有完成最终签收,不能直接进入正式发布动作。"
).strip(),
tag_type="danger",
target_node_codes=[],
step_titles=["执行最终签收", "确认 verify / smoke / final report", "刷新最新包状态", "回到 Launchpad 复检"],
primary_label="执行最终签收",
primary_action_code="release_prepare",
primary_action_payload=dict(shared_action_payload),
primary_focus_ref=worker_launchpad_focus_ref or launchpad_focus_ref,
secondary_label="查看版本区" if latest_release_id > 0 else "",
secondary_action_code="focus_release_hub" if latest_release_id > 0 else "",
secondary_action_payload={"release_id": latest_release_id} if latest_release_id > 0 else {},
secondary_focus_ref=release_detail_focus_ref,
)
return {}
def _build_ops_runbook_control_sequences(
*,
managed_nodes_payload: dict | None = None,
release_launchpad: dict | None = None,
) -> list[dict]:
nodes = list((managed_nodes_payload or {}).get("nodes") or [])
preferred_release = _preferred_release_for_ops()
preferred_release_id = int(preferred_release.get("id") or 0)
preferred_release_label = (
str(preferred_release.get("release_version") or "").strip()
or str(preferred_release.get("channel") or "").strip()
or "默认 Release"
)
bootstrap_gap_nodes = [
item
for item in nodes
if bool(item.get("cluster_is_effective_worker", False))
and (
not bool(item.get("is_managed", False))
or not bool(item.get("is_enabled", False))
or not bool(item.get("is_agent_online", False))
)
]
bootstrap_gap_node_codes = _normalize_driver_node_codes(item.get("node_code") for item in bootstrap_gap_nodes)
participating_nodes = [
item
for item in nodes
if bool(item.get("cluster_is_effective_worker", False)) and bool(item.get("is_current_participant", False))
]
participating_node_codes = _normalize_driver_node_codes(item.get("node_code") for item in participating_nodes)
inspection_ready_nodes = [
item
for item in nodes
if _is_inspection_eligible_node(item)
]
inspection_ready_node_codes = _normalize_driver_node_codes(item.get("node_code") for item in inspection_ready_nodes)
worker_release_node_codes = _normalize_driver_node_codes(
item.get("node_code")
for item in inspection_ready_nodes
if str(item.get("role") or "").strip() == "worker"
)
control_release_node_codes = _normalize_driver_node_codes(
item.get("node_code")
for item in inspection_ready_nodes
if str(item.get("role") or "").strip() == "control"
)
scene_target_node_codes = participating_node_codes or inspection_ready_node_codes
onboarding_status = "warning" if bootstrap_gap_node_codes else "ready"
onboarding_status_label = "有待接管节点" if bootstrap_gap_node_codes else "接管链路就绪"
onboarding_tag_type = "warning" if bootstrap_gap_node_codes else "success"
onboarding_summary = (
f"当前有 {len(bootstrap_gap_node_codes)} 台有效执行节点还没进入 Agent 在线状态,建议先从海外控制面生成接入工单。"
if bootstrap_gap_node_codes
else "当前没有明显的接管缺口;后续新增大陆机器时,直接从这里生成接入工单。"
)
scene_status = "warning" if scene_target_node_codes else "info"
scene_status_label = "现场可观察" if scene_target_node_codes else "暂无现场目标"
scene_tag_type = "warning" if scene_target_node_codes else "info"
scene_summary = (
f"当前有 {len(scene_target_node_codes)} 台节点适合作为执行现场观察目标,优先从关键日志开始,再决定是否升级到全量取证。"
if scene_target_node_codes
else "当前没有明确的参与节点;后续一旦有节点开始领任务,这里就是海外控制面第一入口。"
)
inspection_status = "ready" if inspection_ready_node_codes else "attention"
inspection_status_label = "可直接巡检" if inspection_ready_node_codes else "待补接管"
inspection_tag_type = "success" if inspection_ready_node_codes else "warning"
inspection_summary = (
f"当前有 {len(inspection_ready_node_codes)} 台节点已经满足 已纳管 + 已启用 + Agent 在线,可直接走标准巡检。"
if inspection_ready_node_codes
else "当前还没有满足标准巡检条件的节点,建议先补齐接管和 Agent 在线状态。"
)
release_status = "ready" if preferred_release_id > 0 else "attention"
release_status_label = "可进入发布路径" if preferred_release_id > 0 else "待创建 Release"
release_tag_type = "success" if preferred_release_id > 0 else "warning"
release_summary = (
f"当前默认版本 {preferred_release_label} 已可用于点状灰度或 Rollout先发 Worker再逐步推进控制面。"
if preferred_release_id > 0
else "当前还没有可用 Release建议先在海外控制面建立版本入口再进入发布闭环。"
)
release_progression_sequence = _build_release_launchpad_runbook_sequence(
release_launchpad=dict(release_launchpad or {})
)
if not release_progression_sequence:
release_progression_sequence = _runbook_sequence(
key="release_progression",
title="版本发布与放量",
summary=release_summary,
reason="正式运维时,版本更新应该统一落回 Release / Rollout而不是回退到各机器单独 pull / restart。",
status=release_status,
status_label=release_status_label,
tag_type=release_tag_type,
target_node_codes=worker_release_node_codes or control_release_node_codes,
target_scope_label="Release / Rollout",
step_titles=["点状灰度 Worker", "健康检查", "必要时推进 Rollout", "最后处理控制面"],
primary_label="直接创建 Worker Rollout",
primary_action_code="create_release_rollout_worker",
primary_action_payload={
"release_id": preferred_release_id,
"target_node_codes": worker_release_node_codes,
},
secondary_label="查看版本区",
secondary_action_code="focus_release_hub",
secondary_action_payload={
"release_id": preferred_release_id,
},
)
sequences = [
_runbook_sequence(
key="node_onboarding",
title="纳管新节点",
summary=onboarding_summary,
reason="所有新增大陆 controller / worker第一步都应该先生成 Node Agent 接入工单,而不是手动 SSH 拼命令。",
status=onboarding_status,
status_label=onboarding_status_label,
tag_type=onboarding_tag_type,
target_node_codes=bootstrap_gap_node_codes,
target_scope_label="单节点 / 新节点",
step_titles=["生成接入工单", "复制 Env/脚本", "节点落地执行", "回到控制面看心跳"],
primary_label="跑接入收口",
primary_action_code="bootstrap_run",
primary_action_payload={
"node_code": bootstrap_gap_node_codes[0] if bootstrap_gap_node_codes else "",
},
secondary_label="跑接管验收",
secondary_action_code="run_acceptance",
secondary_action_payload={
"node_code": inspection_ready_node_codes[0] if inspection_ready_node_codes else "",
},
),
_runbook_sequence(
key="scene_observe",
title="执行现场观察",
summary=scene_summary,
reason="海外控制面遇到“谁在跑、为什么卡住、日志有没有回来”这类问题时,应优先走统一现场观察路径。",
status=scene_status,
status_label=scene_status_label,
tag_type=scene_tag_type,
target_node_codes=scene_target_node_codes,
target_scope_label="参与节点 / 现场节点",
step_titles=["关键日志", "判断是否升级全量", "补诊断包", "回看联调快照"],
primary_label="执行关键现场日志",
primary_action_code="run_scene_logs_key",
primary_action_payload={
"execution_mode": "remote-agent",
"auto_approve": True,
},
secondary_label="执行全量现场取证",
secondary_action_code="run_scene_logs_full",
secondary_action_payload={
"execution_mode": "remote-agent",
"auto_approve": True,
},
),
_runbook_sequence(
key="standard_inspection",
title="标准巡检",
summary=inspection_summary,
reason="现场收口、节点验收和发布前确认,尽量都走同一条标准巡检链,而不是手工拼快照、日志和诊断包。",
status=inspection_status,
status_label=inspection_status_label,
tag_type=inspection_tag_type,
target_node_codes=inspection_ready_node_codes,
target_scope_label="已接管执行节点",
step_titles=["健康快照", "Worker 日志", "诊断包", "统一回看巡检结果"],
primary_label="直接执行标准巡检",
primary_action_code="run_standard_inspection",
primary_action_payload={},
secondary_label="补充诊断包",
secondary_action_code="open_diagnostics",
secondary_action_payload={
"execution_mode": "remote-agent",
"auto_approve": True,
},
),
release_progression_sequence,
]
return sequences
def _activity_time(*candidates: object) -> str:
for candidate in candidates:
value = str(candidate or "").strip()
if value:
return value
return ""
def _truncate_activity_text(value: object, *, limit: int = 240) -> str:
text = str(value or "").strip()
if len(text) <= limit:
return text
return f"{text[:limit]}..."
def _activity_status_label(status: object, *, kind: str = "") -> str:
normalized_status = str(status or "").strip()
mapping = {
"empty": "无活动",
"planned": "待推进",
"ready": "就绪",
"healthy": "健康",
"running": "执行中",
"attention": "待处理",
"queued": "排队中",
"dispatching": "派发中",
"awaiting_approval": "待审批",
"success": "成功",
"failed": "失败",
"blocked": "阻断",
"cancelled": "已取消",
"halted": "已暂停",
"completed_with_issues": "带问题完成",
"partially_succeeded": "部分成功",
}
if normalized_status in mapping:
return mapping[normalized_status]
if normalized_status:
return normalized_status
if str(kind or "").strip() in {"execution_scene", "log_sync"}:
return "待观察"
return "未知"
def _activity_focus_ref(item: dict) -> dict:
normalized_item = dict(item or {})
kind = str(normalized_item.get("kind") or "").strip()
ui_intent = dict(normalized_item.get("ui_intent") or {})
if kind == "playbook_run":
return {
"kind": "playbook_run",
"run_code": str(normalized_item.get("run_code") or "").strip(),
"focus_step_key": str((ui_intent.get("focus_step_key") or normalized_item.get("focus_step_key") or "")).strip(),
"focus_step_title": str((ui_intent.get("focus_step_title") or normalized_item.get("focus_step_title") or "")).strip(),
}
if kind == "ops_job":
return {
"kind": "ops_job",
"job_id": int(normalized_item.get("job_id") or 0),
"job_code": str(normalized_item.get("job_code") or "").strip(),
"action": str((ui_intent.get("action") or normalized_item.get("title") or "")).strip(),
"target_node_code": str(
(
ui_intent.get("target_node_code")
or next(iter(normalized_item.get("target_node_codes") or []), "")
or ""
)
).strip(),
}
if kind == "rollout":
return {
"kind": "rollout",
"rollout_id": int(normalized_item.get("rollout_id") or 0),
"rollout_code": str(normalized_item.get("title") or "").strip(),
"release_id": int((ui_intent.get("release_id") or 0)),
}
if kind == "runbook_sequence":
return {
"kind": "runbook_sequence",
"sequence_key": str(normalized_item.get("sequence_key") or "").strip(),
"driver_action_code": str(ui_intent.get("driver_action_code") or "").strip(),
}
if kind in {"execution_scene", "log_sync"}:
return {
"kind": "execution_scene",
"scene_key": str(normalized_item.get("activity_key") or kind).strip(),
}
return {
"kind": kind or "activity",
"activity_key": str(normalized_item.get("activity_key") or "").strip(),
}
def _finalize_activity_item(item: dict) -> dict:
normalized_item = dict(item or {})
kind = str(normalized_item.get("kind") or "").strip()
ui_intent = dict(normalized_item.get("ui_intent") or {})
summary_text = str(normalized_item.get("summary") or "").strip()
status_label = str(normalized_item.get("status_label") or "").strip() or _activity_status_label(
normalized_item.get("status"),
kind=kind,
)
target_node_codes = _normalize_driver_node_codes(normalized_item.get("target_node_codes") or [])
finalized = {
**normalized_item,
"summary": summary_text,
"summary_text": summary_text,
"status_label": status_label,
"ui_intent": ui_intent,
"ui_intent_kind": str(ui_intent.get("kind") or "").strip(),
"target_node_codes": target_node_codes,
}
existing_focus_ref = dict(normalized_item.get("focus_ref") or {})
existing_source_focus_ref = dict(normalized_item.get("source_focus_ref") or {})
finalized["focus_ref"] = existing_focus_ref if existing_focus_ref else _activity_focus_ref(finalized)
finalized["source_focus_ref"] = existing_source_focus_ref
contract_keys = _normalize_ops_contract_keys(
list(normalized_item.get("contract_keys") or [])
+ _suggest_ops_contract_keys_for_activity_item(finalized)
)
finalized["contract_keys"] = contract_keys
finalized["contract_navigation"] = _build_ops_contract_navigation(
contract_keys,
primary_contract_key=_primary_ops_contract_key_for_activity_item(finalized, contract_keys),
registry=get_ops_contract_registry(),
)
return finalized
def _activity_stream_summary_state(items: list[dict]) -> tuple[str, str, str]:
normalized_items = [dict(item or {}) for item in list(items or [])]
if not normalized_items:
return "empty", "无活动", "当前没有匹配的活动。"
hard_attention_statuses = {
"attention",
"failed",
"blocked",
"cancelled",
"halted",
"completed_with_issues",
"partially_succeeded",
"waiting_sample",
"partial_coverage",
}
soft_attention_statuses = {"awaiting_approval"}
running_statuses = {"running", "dispatching"}
planned_statuses = {"queued", "planned", "ready"}
item_statuses = {str(item.get("status") or "").strip() for item in normalized_items if str(item.get("status") or "").strip()}
if item_statuses & hard_attention_statuses:
return "attention", "待处理", "最近活动里存在需优先处理的现场、任务或发布推进。"
if item_statuses & soft_attention_statuses:
return "attention", "待审批", "最近活动里存在待审批或待人工确认的动作。"
if item_statuses & running_statuses:
return "running", "执行中", "最近活动里存在正在推进的现场、任务、编排或回传。"
if item_statuses & planned_statuses:
return "planned", "待推进", "最近活动里存在已排队或即将继续推进的动作。"
return "ready", "平稳", "最近活动整体平稳,可按需抽样查看。"
def _is_playbook_child_job(job: dict) -> bool:
return bool(str(((job.get("metadata") or {}).get("playbook") or {}).get("run_code") or "").strip())
def _build_playbook_run_activity(run: dict) -> dict:
normalized_run = dict(run or {})
run_code = str(normalized_run.get("run_code") or "").strip()
title = (
str(normalized_run.get("playbook_title") or "").strip()
or str(normalized_run.get("playbook_key") or "").strip()
or "标准编排"
)
steps = list(normalized_run.get("steps") or [])
step_count = int(normalized_run.get("step_count") or len(steps) or 0)
completed_steps = sum(1 for step in steps if str(step.get("status") or "").strip() in {"success", "attention"})
target_node_codes = _normalize_driver_node_codes(normalized_run.get("target_node_codes") or [])
focus_summary = str(normalized_run.get("focus_summary") or "").strip()
if not focus_summary:
focus_summary = f"当前状态 {str(normalized_run.get('status') or '').strip() or 'queued'},建议进入编排详情继续观察。"
meta_parts = [run_code]
group_title = str(normalized_run.get("group_title") or normalized_run.get("group_key") or "").strip()
if group_title:
meta_parts.append(group_title)
execution_mode_label_text = (
str(normalized_run.get("execution_mode_label") or "").strip()
or execution_mode_label(str(normalized_run.get("execution_mode") or "").strip())
)
if execution_mode_label_text:
meta_parts.append(f"执行 {execution_mode_label_text}")
if step_count > 0:
meta_parts.append(f"步骤 {completed_steps}/{step_count}")
jobs_total = int(normalized_run.get("jobs_total") or 0)
if jobs_total > 0:
meta_parts.append(f"任务 {jobs_total}")
if target_node_codes:
meta_parts.append(f"节点 {len(target_node_codes)}")
return {
"kind": "playbook_run",
"activity_key": f"playbook-run:{run_code or 'latest'}",
"title": title,
"subtitle": run_code,
"summary": _truncate_activity_text(focus_summary),
"meta_text": " / ".join(part for part in meta_parts if part),
"status": str(normalized_run.get("status") or "").strip(),
"execution_mode": str(normalized_run.get("execution_mode") or "").strip(),
"execution_mode_label": execution_mode_label_text,
"occurred_at": _activity_time(normalized_run.get("updated_at"), normalized_run.get("created_at")),
"run_code": run_code,
"focus_step_key": str(normalized_run.get("focus_step_key") or "").strip(),
"focus_step_title": str(normalized_run.get("focus_step_title") or "").strip(),
"target_node_codes": target_node_codes,
"ui_intent": _build_driver_ui_intent(
"playbook_run_detail",
run_code=run_code,
focus_step_key=str(normalized_run.get("focus_step_key") or "").strip(),
focus_step_title=str(normalized_run.get("focus_step_title") or "").strip(),
),
}
def _build_bootstrap_job_activity(job: dict, managed_node: dict | None = None) -> dict:
normalized_job = dict(job or {})
normalized_node = dict(managed_node or {})
job_id = int(normalized_job.get("id") or 0)
job_code = str(normalized_job.get("job_code") or "").strip()
status = str(normalized_job.get("status") or "").strip()
target_node_code = str(normalized_job.get("target_node_code") or "").strip()
error_message = str(normalized_job.get("error_message") or "").strip()
result = dict(normalized_job.get("result") or {})
bootstrap_plan = dict(result.get("bootstrap_plan") or {})
script_name = str(bootstrap_plan.get("bootstrap_script_name") or "").strip()
agent_state = str(normalized_node.get("agent_state") or "").strip()
agent_state_label = str(normalized_node.get("agent_state_label") or "").strip()
if error_message and status in {"failed", "blocked", "cancelled"}:
summary = error_message
elif status == "awaiting_approval":
summary = "接入工单等待审批,审批通过后才会生成 bootstrap env、脚本和一键落地命令。"
elif status in {"queued", "dispatching", "running"}:
summary = "控制面正在生成节点接入工单,完成后可直接复制 bootstrap env、脚本和一键落地命令。"
elif status == "success":
if agent_state in {"online", "online_busy"}:
summary = "接入工单已生成,目标节点已完成 Agent 接入,可继续执行接管后验收。"
elif agent_state == "pending_bootstrap":
summary = "接入工单已生成,节点尚未回连 Agent可直接复制脚本或一键落地命令。"
elif agent_state == "runtime_only":
summary = "接入工单已生成,但当前只有 runtime 心跳Node Agent 还未真正接管。"
elif agent_state == "stale":
summary = "接入工单已生成,但 Agent 心跳已过期,建议先看任务详情和 Agent 日志。"
else:
summary = "接入工单已生成,可直接复制 bootstrap env、脚本和一键落地命令。"
else:
summary = "节点接入工单最近有新的回执。"
execution_mode_label_text = (
str(normalized_job.get("execution_mode_label") or "").strip()
or execution_mode_label(str(normalized_job.get("execution_mode") or "control-plane").strip() or "control-plane")
)
meta_parts = [
part
for part in [
job_code,
f"节点 {target_node_code}" if target_node_code else "",
f"方式 {execution_mode_label_text}" if execution_mode_label_text else "",
script_name,
]
if part
]
if agent_state_label:
meta_parts.append(f"状态 {agent_state_label}")
return {
"kind": "ops_job",
"activity_key": f"ops-job:{job_id or job_code or 'node.bootstrap'}",
"action": "node.bootstrap",
"title": "节点接入工单",
"subtitle": job_code,
"summary": _truncate_activity_text(summary),
"meta_text": " / ".join(meta_parts),
"status": status,
"execution_mode": str(normalized_job.get("execution_mode") or "control-plane").strip() or "control-plane",
"execution_mode_label": execution_mode_label_text,
"occurred_at": _activity_time(
normalized_job.get("updated_at"),
normalized_job.get("finished_at"),
normalized_job.get("started_at"),
normalized_job.get("created_at"),
),
"job_id": job_id,
"job_code": job_code,
"target_node_codes": _normalize_driver_node_codes([target_node_code]),
"ui_intent": _build_driver_ui_intent(
"job_detail",
job_id=job_id,
job_code=job_code,
action="node.bootstrap",
target_node_code=target_node_code,
),
}
def _build_ops_job_activity(job: dict, *, managed_node: dict | None = None) -> dict:
normalized_job = dict(job or {})
job_id = int(normalized_job.get("id") or 0)
job_code = str(normalized_job.get("job_code") or "").strip()
action = str(normalized_job.get("action") or "").strip() or "ops.job"
if action == "node.bootstrap":
return _build_bootstrap_job_activity(normalized_job, managed_node)
target_node_code = str(normalized_job.get("target_node_code") or "").strip()
target_type = str(normalized_job.get("target_type") or "").strip()
execution_mode = str(normalized_job.get("execution_mode") or "").strip()
execution_mode_label_text = (
str(normalized_job.get("execution_mode_label") or "").strip()
or execution_mode_label(execution_mode)
)
requested_by = str(normalized_job.get("requested_by") or "").strip()
status = str(normalized_job.get("status") or "").strip()
error_message = str(normalized_job.get("error_message") or "").strip()
result = dict(normalized_job.get("result") or {})
start_delivery_state = str(
result.get("start_delivery_state")
or normalized_job.get("start_delivery_state")
or ""
).strip()
start_delivery_error = str(
result.get("start_delivery_error")
or normalized_job.get("start_delivery_error")
or ""
).strip()
preferred_summary_text = str(
normalized_job.get("result_summary_text")
or result.get("summary_text")
or result.get("summary")
or normalized_job.get("summary_text")
or normalized_job.get("summary")
or ""
).strip()
summary_parts: list[str] = []
if not preferred_summary_text:
if target_node_code:
summary_parts.append(f"目标节点 {target_node_code}")
elif target_type:
summary_parts.append(f"目标类型 {target_type}")
if requested_by:
summary_parts.append(f"发起 {requested_by}")
if execution_mode_label_text:
summary_parts.append(f"方式 {execution_mode_label_text}")
summary_candidates: list[str] = []
if preferred_summary_text:
summary_candidates.append(preferred_summary_text)
if error_message and status in {"failed", "blocked", "cancelled"}:
summary_candidates.append(error_message)
if start_delivery_state == "failed_local":
summary_candidates.append("节点已接单执行,但开始回执未成功送达控制面。")
if start_delivery_error:
summary_candidates.append(f"开始回执失败:{start_delivery_error}")
meta_parts = [job_code]
risk_level = str(normalized_job.get("risk_level") or "").strip()
if risk_level:
meta_parts.append(f"风险 {risk_level}")
if execution_mode_label_text:
meta_parts.append(f"执行 {execution_mode_label_text}")
effective_status = status
if start_delivery_state == "failed_local" and status not in {"failed", "blocked", "cancelled"}:
effective_status = "attention"
if status:
meta_parts.append(f"任务 {_activity_status_label(status)}")
meta_parts.append("开始回执异常")
merged_focus_ref = _merge_focus_ref(normalized_job.get("focus_ref"), **_normalize_focus_ref(result.get("focus_ref")))
source_focus_ref = _normalize_focus_ref(result.get("source_focus_ref"))
summary_text = " / ".join(
dict.fromkeys(part for part in [*summary_candidates, *summary_parts] if str(part or "").strip())
) or "标准运维任务最近有新回执。"
activity = {
"kind": "ops_job",
"activity_key": f"ops-job:{job_id or job_code or action}",
"action": action,
"title": action,
"subtitle": job_code,
"summary": _truncate_activity_text(summary_text),
"meta_text": " / ".join(part for part in meta_parts if part),
"status": effective_status,
"job_status": status,
"execution_mode": execution_mode,
"execution_mode_label": execution_mode_label_text,
"occurred_at": _activity_time(
normalized_job.get("updated_at"),
normalized_job.get("finished_at"),
normalized_job.get("started_at"),
normalized_job.get("created_at"),
),
"job_id": job_id,
"job_code": job_code,
"target_node_codes": _normalize_driver_node_codes([target_node_code]),
"focus_ref": merged_focus_ref,
"source_focus_ref": source_focus_ref,
"ui_intent": _build_driver_ui_intent(
"job_events",
job_id=job_id,
job_code=job_code,
action=action,
target_node_code=target_node_code,
),
}
if start_delivery_state:
activity["start_delivery_state"] = start_delivery_state
if start_delivery_error:
activity["start_delivery_error"] = start_delivery_error
if start_delivery_state == "failed_local":
activity["problem_code"] = "ops_job_start_delivery_failed_local"
return activity
def _build_rollout_activity(rollout: dict) -> dict:
normalized_rollout = dict(rollout or {})
rollout_id = int(normalized_rollout.get("id") or 0)
rollout_code = str(normalized_rollout.get("rollout_code") or "").strip()
rollout_policy = dict(normalized_rollout.get("policy") or {})
rollout_execution_mode = str(rollout_policy.get("execution_mode") or "remote-agent").strip() or "remote-agent"
rollout_execution_mode_label = (
str(normalized_rollout.get("execution_mode_label") or "").strip()
or execution_mode_label(rollout_execution_mode)
)
rollout_target_node_codes = _normalize_driver_node_codes(
[str(item.get("node_code") or "").strip() for item in list(normalized_rollout.get("target_nodes") or [])]
)
target_nodes_total = len(rollout_target_node_codes)
jobs_total = int(normalized_rollout.get("jobs_total") or 0)
jobs_created = int(normalized_rollout.get("jobs_created") or 0)
batch_cursor = int(normalized_rollout.get("batch_cursor") or 0)
batches_total = int(normalized_rollout.get("batches_total") or 0)
summary_parts: list[str] = []
if target_nodes_total > 0:
summary_parts.append(f"目标节点 {target_nodes_total}")
if jobs_total > 0:
summary_parts.append(f"任务 {jobs_created}/{jobs_total}")
if batches_total > 0:
summary_parts.append(f"批次 {batch_cursor}/{batches_total}")
result_summary = dict(normalized_rollout.get("result_summary") or {})
summary_text = str(normalized_rollout.get("summary_text") or normalized_rollout.get("summary") or "").strip()
if int(result_summary.get("failed", 0) or 0) > 0:
summary_parts.append(f"失败 {int(result_summary.get('failed', 0) or 0)}")
elif int(result_summary.get("success", 0) or 0) > 0:
summary_parts.append(f"成功 {int(result_summary.get('success', 0) or 0)}")
meta_parts = [f"创建人 {str(normalized_rollout.get('created_by') or '').strip() or '-'}"]
if rollout_execution_mode_label:
meta_parts.append(f"执行 {rollout_execution_mode_label}")
return {
"kind": "rollout",
"activity_key": f"rollout:{rollout_id or rollout_code}",
"title": rollout_code or "Release Rollout",
"subtitle": f"Release #{int(normalized_rollout.get('release_id') or 0)}" if int(normalized_rollout.get("release_id") or 0) > 0 else "",
"summary": _truncate_activity_text(summary_text or " / ".join(part for part in summary_parts if part) or "该 rollout 最近有新的推进或回执。"),
"meta_text": " / ".join(part for part in meta_parts if part),
"status": str(normalized_rollout.get("status") or "").strip(),
"status_label": str(normalized_rollout.get("status_label") or "").strip(),
"execution_mode": rollout_execution_mode,
"execution_mode_label": rollout_execution_mode_label,
"occurred_at": _activity_time(normalized_rollout.get("updated_at"), normalized_rollout.get("created_at")),
"rollout_id": rollout_id,
"target_node_codes": rollout_target_node_codes,
"ui_intent": _build_driver_ui_intent(
"rollout_jobs",
rollout_id=rollout_id,
rollout_code=rollout_code,
release_id=int(normalized_rollout.get("release_id") or 0),
),
"focus_ref": dict(normalized_rollout.get("focus_ref") or {}),
}
def _runbook_sequence_activity_status(sequence: dict) -> str:
normalized_sequence = dict(sequence or {})
primary_resolution = dict(normalized_sequence.get("primary_resolution") or {})
if primary_resolution and primary_resolution.get("ok") is False:
return "attention"
normalized_status = str(normalized_sequence.get("status") or "").strip()
if normalized_status in {"warning", "attention", "blocking"}:
return "attention"
if normalized_status in {"ready", "success"}:
return "ready"
if normalized_status in {"info", "idle"}:
return "planned"
return normalized_status or "planned"
def _build_runbook_sequence_activity(sequence: dict) -> dict:
normalized_sequence = dict(sequence or {})
sequence_key = str(normalized_sequence.get("key") or "").strip()
if not sequence_key:
return {}
title = str(normalized_sequence.get("title") or "").strip() or "标准作业路径"
status_label = str(normalized_sequence.get("status_label") or normalized_sequence.get("status") or "").strip()
reason = str(normalized_sequence.get("reason") or "").strip()
target_scope_label = str(normalized_sequence.get("target_scope_label") or "").strip()
target_node_codes = _normalize_driver_node_codes(normalized_sequence.get("target_node_codes") or [])
primary_resolution = dict(normalized_sequence.get("primary_resolution") or {})
resolved_action_label = (
str(primary_resolution.get("driver_action_label") or "").strip()
or str(primary_resolution.get("driver_action_code") or "").strip()
)
resolved_action_code = str(primary_resolution.get("driver_action_code") or "").strip()
resolved_node_codes = _normalize_driver_node_codes(primary_resolution.get("driver_node_codes") or [])
resolution_message = str(primary_resolution.get("message") or "").strip()
occurred_at = _activity_time(primary_resolution.get("resolved_at"))
summary_parts: list[str] = []
if primary_resolution and primary_resolution.get("ok") is False:
summary_parts.append(f"当前解析失败:{resolution_message or '标准作业路径解析失败'}")
elif resolved_action_label or resolved_action_code:
summary_parts.append(
f"当前建议动作:{resolved_action_label or resolved_action_code}"
+ (
f" ({resolved_action_code})"
if resolved_action_code and resolved_action_label and resolved_action_code != resolved_action_label
else ""
)
)
if resolved_node_codes:
summary_parts.append(f"目标节点:{_format_node_code_list(resolved_node_codes)}")
elif target_node_codes:
summary_parts.append(f"目标节点:{_format_node_code_list(target_node_codes)}")
if resolution_message:
summary_parts.append(resolution_message)
else:
summary_parts.append(str(normalized_sequence.get("summary") or "").strip() or "当前标准作业路径已就绪。")
meta_parts = [status_label, sequence_key]
if target_scope_label:
meta_parts.append(target_scope_label)
if reason:
meta_parts.append(reason)
return {
"kind": "runbook_sequence",
"activity_key": f"runbook-sequence:{sequence_key}",
"title": title,
"subtitle": sequence_key,
"summary": _truncate_activity_text(" / ".join(part for part in summary_parts if part)),
"meta_text": _truncate_activity_text(" / ".join(part for part in meta_parts if part), limit=280),
"status": _runbook_sequence_activity_status(normalized_sequence),
"execution_mode": "",
"execution_mode_label": "",
"occurred_at": occurred_at,
"sequence_key": sequence_key,
"target_node_codes": target_node_codes,
"focus_ref": _normalize_focus_ref(normalized_sequence.get("focus_ref")),
"ui_intent": _build_driver_ui_intent(
"focus_runbook_sequence",
sequence_key=sequence_key,
driver_action_code=resolved_action_code or str(normalized_sequence.get("primary_action_code") or "").strip(),
),
}
def _matches_activity_query(item: dict, query: str) -> bool:
normalized_query = str(query or "").strip().lower()
if not normalized_query:
return True
haystack = " ".join(
[
str(item.get("activity_key") or ""),
str(item.get("kind") or ""),
str(item.get("title") or ""),
str(item.get("subtitle") or ""),
str(item.get("summary") or ""),
str(item.get("meta_text") or ""),
str(item.get("status") or ""),
str(item.get("execution_mode") or ""),
str(item.get("execution_mode_label") or ""),
str(item.get("run_code") or ""),
str(item.get("job_code") or ""),
str(item.get("rollout_id") or ""),
str(item.get("sequence_key") or ""),
]
).lower()
return normalized_query in haystack
def _driver_activity_status_label(status: str) -> str:
normalized_status = str(status or "").strip()
mapping = {
"failed": "失败",
"blocked": "阻断",
"cancelled": "已取消",
"halted": "已暂停",
"completed_with_issues": "带问题完成",
"partially_succeeded": "部分成功",
"awaiting_approval": "待审批",
}
return mapping.get(normalized_status, normalized_status or "异常")
def _pick_driver_activity_focus(items: list[dict]) -> tuple[str, dict]:
hard_problem_statuses = {"failed", "blocked", "cancelled", "halted", "completed_with_issues", "partially_succeeded"}
soft_attention_statuses = {"awaiting_approval"}
candidate_items = [
dict(item or {})
for item in list(items or [])
if str(item.get("kind") or "").strip() in {"ops_job", "rollout"}
]
for item in candidate_items:
if str(item.get("status") or "").strip() in hard_problem_statuses:
return "hard", item
for item in candidate_items:
if str(item.get("status") or "").strip() in soft_attention_statuses:
return "soft", item
return "", {}
def _build_driver_activity_card(level: str, activity_item: dict) -> dict:
normalized_item = dict(activity_item or {})
item_kind = str(normalized_item.get("kind") or "").strip()
item_status = str(normalized_item.get("status") or "").strip()
ui_intent_kind = str(((normalized_item.get("ui_intent") or {}).get("kind") or "")).strip()
source_focus_ref = _normalize_focus_ref(normalized_item.get("source_focus_ref"))
title = str(normalized_item.get("title") or "").strip() or ("Release Rollout" if item_kind == "rollout" else "标准运维任务")
subtitle = str(normalized_item.get("subtitle") or "").strip()
summary = str(normalized_item.get("summary") or "").strip()
meta_text = str(normalized_item.get("meta_text") or "").strip()
status_label = _driver_activity_status_label(item_status)
noun_label = "异常 Rollout" if item_kind == "rollout" else "异常任务"
is_hard_problem = level == "hard"
if item_kind == "rollout":
title_text = "先处理异常 Rollout" if is_hard_problem else "先处理待审批 Rollout"
primary_label = "查看 Rollout 作业"
else:
title_text = "先处理异常任务" if is_hard_problem else "先处理待审批任务"
primary_label = "查看任务详情" if ui_intent_kind == "job_detail" else "查看任务事件"
reason = (
summary
or (
"该活动已经进入失败、阻断或带问题完成状态,继续堆动作只会放大噪音。"
if is_hard_problem
else "该活动正在等待人工审批或决策,先落到对应详情最容易判断下一步。"
)
)
detail_summary = f"{title}{' / ' + subtitle if subtitle else ''} 当前状态为 {status_label},建议先进入对应详情。"
primary_action_code = "focus_activity_item"
action_payload = {
"activity_key": str(normalized_item.get("activity_key") or "").strip(),
"kind": item_kind,
"status": item_status,
"ui_intent": dict(normalized_item.get("ui_intent") or {}),
"focus_ref": _normalize_focus_ref(normalized_item.get("focus_ref")),
}
if ui_intent_kind == "job_events" and str(source_focus_ref.get("kind") or "").strip() == "ops_job_event":
primary_action_code = "focus_latest_job_events"
action_payload["source_focus_ref"] = source_focus_ref
return {
"key": f"activity-focus-{str(normalized_item.get('activity_key') or item_kind or 'activity').strip()}",
"title": title_text,
"level_label": "最高优先" if is_hard_problem else "推荐",
"tag_type": "danger" if is_hard_problem else "warning",
"summary": detail_summary,
"reason": reason,
"node_codes": [],
"meta_text": meta_text or subtitle or str(normalized_item.get("activity_key") or "").strip(),
"primary_label": primary_label,
"secondary_label": "",
"primary_type": "danger" if is_hard_problem else "warning",
"disabled": not str(((normalized_item.get("ui_intent") or {}).get("kind") or "")).strip(),
"primary_action_code": primary_action_code,
"secondary_action_code": "",
"action_payload": action_payload,
"focus_ref": _normalize_focus_ref(normalized_item.get("focus_ref")),
"related_activity": {
"kind": item_kind,
"title": title,
"subtitle": subtitle,
"status": item_status,
"occurred_at": str(normalized_item.get("occurred_at") or "").strip(),
},
}
def _build_driver_activity_focus_entry(activity_item: dict) -> dict:
normalized_item = dict(activity_item or {})
activity_key = str(normalized_item.get("activity_key") or "").strip()
activity_kind = str(normalized_item.get("kind") or "").strip()
ui_intent = dict(normalized_item.get("ui_intent") or {})
ui_intent_kind = str(ui_intent.get("kind") or "").strip()
if not activity_key or not activity_kind:
return {}
focus_ref = _normalize_focus_ref(normalized_item.get("focus_ref"))
source_focus_ref = _normalize_focus_ref(normalized_item.get("source_focus_ref"))
focus_action_code = "focus_activity_item" if ui_intent_kind else ""
if ui_intent_kind == "job_events" and str(source_focus_ref.get("kind") or "").strip() == "ops_job_event":
focus_action_code = "focus_latest_job_events"
focus_action_payload = {
"activity_key": activity_key,
"kind": activity_kind,
"status": str(normalized_item.get("status") or "").strip(),
"ui_intent": ui_intent,
"focus_ref": focus_ref,
}
if source_focus_ref:
focus_action_payload["source_focus_ref"] = source_focus_ref
contract_keys = _normalize_ops_contract_keys(
list(normalized_item.get("contract_keys") or [])
+ _suggest_ops_contract_keys_for_activity_item(normalized_item)
)
return {
"key": f"activity:{activity_key}",
"activity_key": activity_key,
"activity_kind": activity_kind,
"title": str(normalized_item.get("title") or "").strip() or activity_kind,
"subtitle": str(normalized_item.get("subtitle") or "").strip(),
"summary": str(normalized_item.get("summary_text") or normalized_item.get("summary") or "").strip(),
"status": str(normalized_item.get("status") or "").strip(),
"status_label": str(normalized_item.get("status_label") or "").strip(),
"meta_text": str(normalized_item.get("meta_text") or "").strip(),
"occurred_at": str(normalized_item.get("occurred_at") or "").strip(),
"target_node_codes": _normalize_driver_node_codes(normalized_item.get("target_node_codes") or []),
"focus_ref": focus_ref,
"source_focus_ref": source_focus_ref,
"ui_intent": ui_intent,
"ui_intent_kind": ui_intent_kind,
"focus_action_code": focus_action_code,
"focus_action_payload": focus_action_payload if ui_intent_kind else {},
"observation_only": True,
"contract_keys": contract_keys,
"contract_navigation": _build_ops_contract_navigation(
contract_keys,
primary_contract_key=_primary_ops_contract_key_for_activity_item(normalized_item, contract_keys),
registry=get_ops_contract_registry(),
),
}
def _build_driver_activity_focus(items: list[dict], *, limit: int = 3) -> list[dict]:
allowed_scene_kinds = {"execution_scene", "log_sync"}
allowed_run_kinds = {"playbook_run", "ops_job", "rollout"}
rows: list[dict] = []
seen_keys: set[str] = set()
for raw_item in list(items or []):
item = dict(raw_item or {})
item_kind = str(item.get("kind") or "").strip()
item_status = str(item.get("status") or "").strip()
if item_kind in allowed_scene_kinds:
pass
elif item_kind == "playbook_run":
if item_status in {"success", "completed", "healthy"}:
continue
elif item_kind in allowed_run_kinds:
if item_status in {"success", "completed", "ready", "planned"}:
continue
else:
continue
entry = _build_driver_activity_focus_entry(item)
entry_key = str(entry.get("key") or "").strip()
if not entry or not entry_key or entry_key in seen_keys:
continue
seen_keys.add(entry_key)
rows.append(entry)
if len(rows) >= limit:
break
return rows
def _build_execution_scene_log_sync(log_sync: dict, participating_nodes: list[dict]) -> dict:
normalized_log_sync = dict(log_sync or {})
enabled = bool(normalized_log_sync.get("enabled", False))
mode = "full" if str(normalized_log_sync.get("mode") or "").strip().lower() == "full" else "key"
line_count = int(normalized_log_sync.get("line_count", 0) or 0)
source_nodes = [
str(item or "").strip()
for item in list(normalized_log_sync.get("source_nodes") or [])
if str(item or "").strip()
]
source_nodes = list(dict.fromkeys(source_nodes))
preview_lines = [
str(item or "").strip()
for item in list(normalized_log_sync.get("preview_lines") or [])
if str(item or "").strip()
]
preview_lines = preview_lines[-20:]
source_node_summaries_map: dict[str, dict] = {}
for item in list(normalized_log_sync.get("source_node_summaries") or []):
if not isinstance(item, dict):
continue
node_code = str(item.get("node_code") or "").strip()
if not node_code:
continue
source_node_summaries_map[node_code] = {
"node_code": node_code,
"line_count": int(item.get("line_count", 0) or 0),
"key_line_count": int(item.get("key_line_count", 0) or 0),
"full_line_count": int(item.get("full_line_count", 0) or 0),
"last_at": str(item.get("last_at") or "").strip(),
"last_line": str(item.get("last_line") or "").strip(),
}
if not source_node_summaries_map and preview_lines:
for line in preview_lines:
match = _REMOTE_LOG_PREVIEW_LINE_RE.match(line)
if not match:
continue
node_code = str(match.group("node_code") or "").strip()
created_at = str(match.group("created_at") or "").strip()
if not node_code:
continue
bucket = source_node_summaries_map.setdefault(
node_code,
{
"node_code": node_code,
"line_count": 0,
"key_line_count": 0,
"full_line_count": 0,
"last_at": "",
"last_line": "",
},
)
bucket["line_count"] += 1
bucket["key_line_count"] += 1
bucket["last_at"] = created_at
bucket["last_line"] = line
for node_code in source_nodes:
source_node_summaries_map.setdefault(
node_code,
{
"node_code": node_code,
"line_count": 0,
"key_line_count": 0,
"full_line_count": 0,
"last_at": "",
"last_line": "",
},
)
source_nodes = list(dict.fromkeys([*source_nodes, *list(source_node_summaries_map.keys())]))
source_node_summaries = sorted(
source_node_summaries_map.values(),
key=lambda item: (
str(item.get("last_at") or ""),
str(item.get("node_code") or ""),
),
reverse=True,
)
source_node_set = set(source_nodes)
source_node_set.update(source_node_summaries_map.keys())
participating_node_codes = [
str(item.get("node_code") or "").strip()
for item in list(participating_nodes or [])
if str(item.get("node_code") or "").strip()
]
participating_node_codes = list(dict.fromkeys(participating_node_codes))
participating_node_map = {
str(item.get("node_code") or "").strip(): dict(item)
for item in list(participating_nodes or [])
if str(item.get("node_code") or "").strip()
}
covered_participating_nodes = [node_code for node_code in participating_node_codes if node_code in source_node_set]
missing_participating_nodes = [node_code for node_code in participating_node_codes if node_code not in source_node_set]
covered_participating_node_summaries = []
for node_code in covered_participating_nodes:
source_summary = dict(source_node_summaries_map.get(node_code) or {})
node_meta = participating_node_map.get(node_code) or {}
covered_participating_node_summaries.append(
{
**source_summary,
"region": str(node_meta.get("region") or "").strip(),
"role": str(node_meta.get("role") or "").strip(),
"status": str(node_meta.get("status") or "").strip(),
"participation_state": str(node_meta.get("participation_state") or "").strip(),
"participation_bucket": str(node_meta.get("participation_bucket") or "").strip(),
"participation_label": str(node_meta.get("participation_label") or "").strip(),
"participation_reason": str(node_meta.get("participation_reason") or "").strip(),
}
)
missing_participating_node_summaries = []
for node_code in missing_participating_nodes:
node_meta = participating_node_map.get(node_code) or {}
missing_reason_code = "no_sample"
missing_reason = "当前还没有收到该参与节点的远端日志样本。"
if not enabled:
missing_reason_code = "disabled"
missing_reason = "远端日志回传当前关闭。"
elif line_count <= 0:
missing_reason_code = "waiting_sample"
missing_reason = "日志回传已开启,但当前现场样本尚未形成。"
missing_participating_node_summaries.append(
{
"node_code": node_code,
"region": str(node_meta.get("region") or "").strip(),
"role": str(node_meta.get("role") or "").strip(),
"status": str(node_meta.get("status") or "").strip(),
"participation_state": str(node_meta.get("participation_state") or "").strip(),
"participation_bucket": str(node_meta.get("participation_bucket") or "").strip(),
"participation_label": str(node_meta.get("participation_label") or "").strip(),
"participation_reason": str(node_meta.get("participation_reason") or "").strip(),
"missing_reason_code": missing_reason_code,
"missing_reason": missing_reason,
}
)
if not enabled:
state = "disabled"
status_label = "已关闭"
status_type = "info"
description = "远端日志回传当前关闭,海外控制面不会持续收到大陆检测过程日志。"
recommended_action_label = "建议先开启关键回传"
elif not participating_node_codes:
state = "idle"
status_label = "空闲观察"
status_type = "success"
description = "当前没有参与检测的节点,日志回传已就绪,待现场出现时会自动开始镜像样本。"
recommended_action_label = "当前无需调整"
elif line_count <= 0:
state = "waiting_sample"
status_label = "等待样本"
status_type = "warning"
description = "日志回传已开启,但当前还没有收到参与节点的现场日志样本。"
recommended_action_label = "建议先抓 Worker 日志确认现场输出"
elif missing_participating_nodes:
state = "partial_coverage"
status_label = "部分覆盖"
status_type = "warning"
description = f"当前已有 {len(covered_participating_nodes)}/{len(participating_node_codes)} 台参与节点回传日志,仍有节点未被覆盖。"
recommended_action_label = "建议优先排查未回传节点"
elif mode == "full":
state = "full_capture"
status_label = "全量观察"
status_type = "success"
description = "参与节点已经全部被覆盖,当前处于全量日志观察模式,适合短时深度排障。"
recommended_action_label = "排障结束后可切回关键回传"
else:
state = "healthy"
status_label = "关键覆盖"
status_type = "success"
description = "参与节点已经全部被覆盖,当前处于关键日志观察模式,适合常态联调。"
recommended_action_label = "当前无需调整"
return {
"enabled": enabled,
"mode": mode,
"mode_label": "全量回传" if enabled and mode == "full" else ("关键回传" if enabled else "已关闭"),
"state": state,
"status_label": status_label,
"status_type": status_type,
"description": description,
"recommended_action_label": recommended_action_label,
"line_count": line_count,
"source_node_count": int(normalized_log_sync.get("source_node_count", 0) or len(source_nodes)),
"source_nodes": source_nodes,
"source_node_summaries": source_node_summaries,
"last_at": str(normalized_log_sync.get("last_at") or ""),
"last_line": str(normalized_log_sync.get("last_line") or ""),
"preview_lines": preview_lines,
"preview_line_count": len(preview_lines),
"participating_node_count": len(participating_node_codes),
"covered_participating_node_count": len(covered_participating_nodes),
"missing_participating_node_count": len(missing_participating_nodes),
"covered_participating_nodes": covered_participating_nodes,
"missing_participating_nodes": missing_participating_nodes,
"covered_participating_node_summaries": covered_participating_node_summaries,
"missing_participating_node_summaries": missing_participating_node_summaries,
}
def _normalize_scene_log_mode(raw_mode: object, *, fallback: str = "key") -> str:
normalized_mode = str(raw_mode or "").strip().lower()
if normalized_mode == "full":
return "full"
if normalized_mode == "key":
return "key"
return "full" if str(fallback or "").strip().lower() == "full" else "key"
def _derive_scene_log_source_summary_from_records(node_code: str, records: list[dict]) -> dict:
normalized_node_code = str(node_code or "").strip()
summary = {
"node_code": normalized_node_code,
"line_count": 0,
"key_line_count": 0,
"full_line_count": 0,
"last_at": "",
"last_line": "",
}
for record in list(records or []):
record_node_code = str(record.get("node_code") or "").strip()
if normalized_node_code and record_node_code != normalized_node_code:
continue
summary["line_count"] += 1
if str(record.get("mode") or "").strip() == "full":
summary["full_line_count"] += 1
else:
summary["key_line_count"] += 1
summary["last_at"] = str(record.get("created_at") or "").strip()
summary["last_line"] = str(record.get("line") or "").strip()
return summary
def _collect_scene_log_records(
active_job: dict | None,
*,
requested_mode: str,
) -> list[dict]:
normalized_active_job = dict(active_job or {})
events = list(normalized_active_job.get("current_cycle_events") or normalized_active_job.get("recent_events") or [])
if not events:
return []
current_cycle_token = str(normalized_active_job.get("current_cycle_token") or "").strip()
normalized_requested_mode = _normalize_scene_log_mode(requested_mode)
records: list[dict] = []
for event in reversed(events):
event_type = str(event.get("event_type") or "").strip()
if event_type != "worker_log":
continue
payload = event.get("payload") if isinstance(event.get("payload"), dict) else {}
event_cycle_token = str(payload.get("cycle_token") or "").strip()
if current_cycle_token and event_cycle_token and event_cycle_token != current_cycle_token:
continue
event_mode = _normalize_scene_log_mode(payload.get("log_mode") or "key")
if normalized_requested_mode != "full" and event_mode == "full":
continue
created_at = str(event.get("created_at") or "").strip()
node_code = str(event.get("node_code") or "").strip() or "unknown"
message = str(event.get("message") or "").strip()
if not message:
continue
records.append(
{
"created_at": created_at,
"node_code": node_code,
"message": message,
"mode": event_mode,
"cycle_token": event_cycle_token,
"line": f"[{created_at}] [{node_code}] {message}",
}
)
return records
def _parse_ops_timestamp(value: object) -> datetime | None:
text = str(value or "").strip()
if not text:
return None
try:
return datetime.fromisoformat(text)
except ValueError:
return None
def _collect_scene_log_records_from_ops_jobs(
*,
requested_mode: str,
limit: int = 30,
max_age_hours: int = 6,
) -> list[dict]:
normalized_requested_mode = _normalize_scene_log_mode(requested_mode)
cutoff = datetime.now() - timedelta(hours=max(1, int(max_age_hours or 6)))
records: list[dict] = []
worker_service_name = str(settings.worker_service_name or "domaincheck-worker").strip() or "domaincheck-worker"
jobs = list_ops_jobs(limit=max(1, min(int(limit or 30), 80)), compact=False)
for job in reversed(list(jobs or [])):
if str(job.get("action") or "").strip() != "logs.collect":
continue
if str(job.get("status") or "").strip() != "success":
continue
result = dict(job.get("result") or {})
payload = dict(job.get("payload") or {})
service_name = str(result.get("service_name") or payload.get("service_name") or "").strip()
if service_name != worker_service_name:
continue
node_code = str(job.get("target_node_code") or "").strip()
if not node_code:
continue
created_at = (
str(job.get("finished_at") or "").strip()
or str(job.get("updated_at") or "").strip()
or str(job.get("created_at") or "").strip()
)
created_dt = _parse_ops_timestamp(created_at)
if created_dt and created_dt < cutoff:
continue
output_text = str(result.get("stdout") or result.get("log_output") or "").strip()
if not output_text:
continue
output_lines = [str(item or "").strip() for item in output_text.splitlines() if str(item or "").strip()]
if not output_lines:
continue
sample_limit = 12 if normalized_requested_mode == "full" else 4
for line in output_lines[-sample_limit:]:
records.append(
{
"created_at": created_at,
"node_code": node_code,
"message": line,
"mode": normalized_requested_mode,
"cycle_token": "",
"line": f"[{created_at}] [{node_code}] {line}",
}
)
return records
def _merge_execution_scene_log_sync_sources(log_sync: dict | None, *, requested_mode: str) -> dict:
normalized_log_sync = dict(log_sync or {})
ops_job_records = _collect_scene_log_records_from_ops_jobs(
requested_mode=requested_mode,
limit=30,
max_age_hours=6,
)
if not ops_job_records:
return normalized_log_sync
preview_lines = [
str(item or "").strip()
for item in list(normalized_log_sync.get("preview_lines") or [])
if str(item or "").strip()
]
merged_preview_lines = preview_lines + [str(item.get("line") or "").strip() for item in ops_job_records if str(item.get("line") or "").strip()]
merged_preview_lines = merged_preview_lines[-20:]
source_nodes = [
str(item or "").strip()
for item in list(normalized_log_sync.get("source_nodes") or [])
if str(item or "").strip()
]
source_nodes.extend(
str(item.get("node_code") or "").strip()
for item in ops_job_records
if str(item.get("node_code") or "").strip()
)
source_nodes = list(dict.fromkeys(source_nodes))
existing_summaries = {}
for item in list(normalized_log_sync.get("source_node_summaries") or []):
if not isinstance(item, dict):
continue
node_code = str(item.get("node_code") or "").strip()
if not node_code:
continue
existing_summaries[node_code] = dict(item)
for node_code in source_nodes:
derived_summary = _derive_scene_log_source_summary_from_records(
node_code,
[record for record in ops_job_records if str(record.get("node_code") or "").strip() == node_code],
)
if existing_summaries.get(node_code):
current_summary = dict(existing_summaries[node_code])
current_summary["line_count"] = int(current_summary.get("line_count", 0) or 0) + int(derived_summary.get("line_count", 0) or 0)
current_summary["key_line_count"] = int(current_summary.get("key_line_count", 0) or 0) + int(derived_summary.get("key_line_count", 0) or 0)
current_summary["full_line_count"] = int(current_summary.get("full_line_count", 0) or 0) + int(derived_summary.get("full_line_count", 0) or 0)
if str(derived_summary.get("last_at") or "").strip() >= str(current_summary.get("last_at") or "").strip():
current_summary["last_at"] = str(derived_summary.get("last_at") or current_summary.get("last_at") or "").strip()
current_summary["last_line"] = str(derived_summary.get("last_line") or current_summary.get("last_line") or "").strip()
existing_summaries[node_code] = current_summary
elif int(derived_summary.get("line_count", 0) or 0) > 0:
existing_summaries[node_code] = derived_summary
latest_record = ops_job_records[-1]
latest_at = str(latest_record.get("created_at") or "").strip()
latest_line = str(latest_record.get("line") or "").strip()
current_last_at = str(normalized_log_sync.get("last_at") or "").strip()
if current_last_at and current_last_at > latest_at:
latest_at = current_last_at
latest_line = str(normalized_log_sync.get("last_line") or "").strip()
merged_line_count = max(
int(normalized_log_sync.get("line_count", 0) or 0),
len(merged_preview_lines),
sum(int(item.get("line_count", 0) or 0) for item in existing_summaries.values()),
)
return {
**normalized_log_sync,
"preview_lines": merged_preview_lines,
"line_count": merged_line_count,
"source_nodes": source_nodes,
"source_node_count": max(int(normalized_log_sync.get("source_node_count", 0) or 0), len(source_nodes)),
"source_node_summaries": list(existing_summaries.values()),
"last_at": latest_at,
"last_line": latest_line,
}
def _collect_scene_log_records_from_preview_lines(
preview_lines: list[str] | None,
*,
requested_mode: str,
) -> list[dict]:
normalized_requested_mode = _normalize_scene_log_mode(requested_mode)
records: list[dict] = []
for raw_line in list(preview_lines or []):
line = str(raw_line or "").strip()
if not line:
continue
match = _REMOTE_LOG_PREVIEW_LINE_RE.match(line)
if not match:
continue
created_at = str(match.group("created_at") or "").strip()
node_code = str(match.group("node_code") or "").strip() or "unknown"
message = str(match.group("message") or "").strip()
if not message:
continue
record_mode = "full" if normalized_requested_mode == "full" else "key"
records.append(
{
"created_at": created_at,
"node_code": node_code,
"message": message,
"mode": record_mode,
"cycle_token": "",
"line": line,
}
)
return records
def _scene_log_status_meta(status: str, *, log_sync_enabled: bool) -> tuple[str, str]:
normalized_status = str(status or "").strip()
mapping = {
"healthy": ("关键覆盖", "success"),
"full_capture": ("全量观察", "success"),
"historical_sample": ("历史样本", "success"),
"waiting_sample": ("等待样本", "warning"),
"missing_sample": ("缺少样本", "warning"),
"disabled": ("已关闭", "info"),
"standby": ("在线待命", "info"),
"unknown": ("未发现", "info"),
}
if normalized_status in mapping:
return mapping[normalized_status]
if not log_sync_enabled:
return ("已关闭", "info")
return ("待确认", "info")
def get_ops_node_scene_log(node_code: str, *, limit: int = 80, mode: str = "") -> dict:
normalized_node_code = str(node_code or "").strip()
if not normalized_node_code:
return {}
runtime = get_runtime_status()
detect = dict(runtime.get("detect") or {})
cluster_nodes = [
dict(item)
for item in list((runtime.get("cluster") or {}).get("nodes") or [])
if str(item.get("node_code") or "").strip()
]
execution_scene = _build_ops_execution_scene(detect)
log_sync = dict(execution_scene.get("log_sync") or {})
default_mode = _normalize_scene_log_mode(log_sync.get("mode") or "key")
selected_mode = _normalize_scene_log_mode(mode, fallback=default_mode)
active_job = dict(detect.get("active_job") or {})
records = _collect_scene_log_records(active_job, requested_mode=selected_mode)
if not records:
records = _collect_scene_log_records_from_preview_lines(
list(log_sync.get("preview_lines") or []),
requested_mode=selected_mode,
)
node_records = [
dict(record)
for record in records
if str(record.get("node_code") or "").strip() == normalized_node_code
]
normalized_limit = max(1, min(int(limit or 80), 400))
visible_records = node_records[-normalized_limit:]
participating_map = {
str(item.get("node_code") or "").strip(): dict(item)
for item in list(execution_scene.get("participating_nodes") or [])
if str(item.get("node_code") or "").strip()
}
standby_map = {
str(item.get("node_code") or "").strip(): dict(item)
for item in list(execution_scene.get("standby_nodes") or [])
if str(item.get("node_code") or "").strip()
}
cluster_map = {
str(item.get("node_code") or "").strip(): dict(item)
for item in cluster_nodes
if str(item.get("node_code") or "").strip()
}
source_summary_map = {
str(item.get("node_code") or "").strip(): dict(item)
for item in list(log_sync.get("source_node_summaries") or [])
if str(item.get("node_code") or "").strip()
}
missing_summary_map = {
str(item.get("node_code") or "").strip(): dict(item)
for item in list(log_sync.get("missing_participating_node_summaries") or [])
if str(item.get("node_code") or "").strip()
}
covered_summary_map = {
str(item.get("node_code") or "").strip(): dict(item)
for item in list(log_sync.get("covered_participating_node_summaries") or [])
if str(item.get("node_code") or "").strip()
}
node_meta = dict(participating_map.get(normalized_node_code) or standby_map.get(normalized_node_code) or {})
cluster_meta = dict(cluster_map.get(normalized_node_code) or {})
source_summary = dict(source_summary_map.get(normalized_node_code) or covered_summary_map.get(normalized_node_code) or {})
derived_source_summary = _derive_scene_log_source_summary_from_records(normalized_node_code, node_records)
source_summary = {
"node_code": normalized_node_code,
"line_count": max(
int(source_summary.get("line_count", 0) or 0),
int(derived_source_summary.get("line_count", 0) or 0),
),
"key_line_count": max(
int(source_summary.get("key_line_count", 0) or 0),
int(derived_source_summary.get("key_line_count", 0) or 0),
),
"full_line_count": max(
int(source_summary.get("full_line_count", 0) or 0),
int(derived_source_summary.get("full_line_count", 0) or 0),
),
"last_at": str(
derived_source_summary.get("last_at")
or source_summary.get("last_at")
or ""
).strip(),
"last_line": str(
derived_source_summary.get("last_line")
or source_summary.get("last_line")
or ""
).strip(),
}
missing_summary = dict(missing_summary_map.get(normalized_node_code) or {})
record_available = bool(source_summary.get("line_count", 0) or node_records)
detect_participating = normalized_node_code in participating_map
standby_visible = normalized_node_code in standby_map
log_sync_enabled = bool(log_sync.get("enabled", False))
if record_available:
if detect_participating:
status = "full_capture" if selected_mode == "full" else "healthy"
elif standby_visible:
status = "historical_sample"
else:
status = "historical_sample"
elif missing_summary:
missing_reason_code = str(missing_summary.get("missing_reason_code") or "").strip()
if missing_reason_code == "disabled":
status = "disabled"
elif missing_reason_code == "waiting_sample":
status = "waiting_sample"
else:
status = "missing_sample"
elif detect_participating:
status = "waiting_sample" if log_sync_enabled else "disabled"
elif standby_visible:
status = "standby"
else:
status = "unknown"
status_label, status_type = _scene_log_status_meta(status, log_sync_enabled=log_sync_enabled)
if record_available and detect_participating:
summary = f"节点当前正在参与检测,已保留 {int(source_summary.get('line_count', 0) or 0)} 条现场日志样本。"
elif record_available and standby_visible:
summary = f"节点当前在线待命,仍可查看最近保留的 {int(source_summary.get('line_count', 0) or 0)} 条现场日志样本。"
elif record_available:
summary = f"节点已保留 {int(source_summary.get('line_count', 0) or 0)} 条现场日志样本。"
elif missing_summary:
summary = str(missing_summary.get("missing_reason") or "").strip() or "当前还没有收到该节点的现场日志样本。"
elif detect_participating and log_sync_enabled:
summary = "节点正在参与检测,但当前现场日志样本仍未形成。"
elif detect_participating:
summary = "节点正在参与检测,但远端日志回传当前关闭。"
elif standby_visible:
summary = "节点当前在线但未参与本轮检测,暂时没有新的现场日志样本。"
else:
summary = "当前没有找到该节点的现场日志记录。"
latest_record = dict(visible_records[-1] or {}) if visible_records else {}
participation_payload = {
"detect_participating": detect_participating,
"participation_state": str(node_meta.get("participation_state") or "").strip(),
"participation_label": str(node_meta.get("participation_label") or "").strip(),
"participation_reason": str(node_meta.get("participation_reason") or "").strip(),
"participation_bucket": str(node_meta.get("participation_bucket") or "").strip(),
"participation_bucket_label": str(node_meta.get("participation_bucket_label") or "").strip(),
"is_dispatch_active": bool(node_meta.get("is_dispatch_active", False)),
}
return {
"node_code": normalized_node_code,
"available": bool(node_meta or cluster_meta or source_summary.get("line_count", 0) or missing_summary or node_records),
"status": status,
"status_label": status_label,
"status_type": status_type,
"summary": summary,
"log_sync_enabled": log_sync_enabled,
"mode": selected_mode,
"mode_label": "全量回传" if selected_mode == "full" else "关键回传",
"records_total": len(node_records),
"records_visible": len(visible_records),
"records_truncated": len(node_records) > len(visible_records),
"records": visible_records,
"latest_record": latest_record,
"source_summary": source_summary,
"missing_reason_code": str(missing_summary.get("missing_reason_code") or "").strip(),
"missing_reason": str(missing_summary.get("missing_reason") or "").strip(),
"node": {
"node_code": normalized_node_code,
"region": str(node_meta.get("region") or cluster_meta.get("region") or missing_summary.get("region") or "").strip(),
"role": str(node_meta.get("role") or cluster_meta.get("role") or missing_summary.get("role") or "").strip(),
"status": str(node_meta.get("status") or cluster_meta.get("status") or missing_summary.get("status") or "").strip(),
"current_load": int(node_meta.get("current_load", cluster_meta.get("current_load", 0)) or 0),
"last_heartbeat_at": str(
node_meta.get("last_heartbeat_at") or cluster_meta.get("last_heartbeat_at") or ""
).strip(),
},
"participation": participation_payload,
"contract_navigation": _build_ops_contract_navigation(
["ops_observability_contract", "ops_stack_diagnosis_contract"],
primary_contract_key="ops_observability_contract",
registry=get_ops_contract_registry(),
),
}
def _build_ops_execution_scene(detect: dict) -> dict:
normalized_detect = dict(detect or {})
active_job = dict(normalized_detect.get("active_job") or {})
log_sync = _merge_execution_scene_log_sync_sources(
normalized_detect.get("log_sync") or {},
requested_mode=str((normalized_detect.get("log_sync") or {}).get("mode") or "key"),
)
participation_summary = normalized_detect.get("participation_summary") or {}
participating_nodes = [
_slim_execution_scene_node(item)
for item in list(normalized_detect.get("participating_nodes") or [])
if str(item.get("node_code") or "").strip()
]
standby_nodes = [
_slim_execution_scene_node(item)
for item in list(normalized_detect.get("non_participating_nodes") or [])
if str(item.get("node_code") or "").strip()
]
dispatch_active_nodes = [
dict(item)
for item in participating_nodes
if str(item.get("participation_bucket") or "").strip() == "dispatch_active"
]
recent_only_nodes = [
dict(item)
for item in participating_nodes
if str(item.get("participation_bucket") or "").strip() == "recent_only"
]
load_syncing_nodes = [
dict(item)
for item in [*participating_nodes, *standby_nodes]
if str(item.get("participation_bucket") or "").strip() == "load_syncing"
]
counts = {
"dispatch_active": int(participation_summary.get("dispatch_active_nodes", len(dispatch_active_nodes)) or len(dispatch_active_nodes)),
"recent_only": int(participation_summary.get("recent_only_nodes", len(recent_only_nodes)) or len(recent_only_nodes)),
"standby": int(participation_summary.get("standby_nodes", 0) or 0),
"load_syncing": int(participation_summary.get("load_syncing_nodes", len(load_syncing_nodes)) or len(load_syncing_nodes)),
}
latest_event = dict(active_job.get("latest_cycle_event") or active_job.get("latest_event") or {})
latest_at_candidates = [
str(latest_event.get("created_at") or "").strip(),
str(active_job.get("started_at") or "").strip(),
str(active_job.get("created_at") or "").strip(),
str((log_sync or {}).get("last_at") or "").strip(),
]
latest_at_candidates.extend(
str(item.get("last_heartbeat_at") or "").strip()
for item in [*participating_nodes, *standby_nodes]
if str(item.get("last_heartbeat_at") or "").strip()
)
non_empty_latest_at_candidates = [candidate for candidate in latest_at_candidates if candidate]
latest_at = sorted(non_empty_latest_at_candidates)[-1] if non_empty_latest_at_candidates else ""
return {
"active_job_code": str((normalized_detect.get("active_job") or {}).get("job_code") or ""),
"phase_label": str(normalized_detect.get("phase_label") or ""),
"phase_detail": str(normalized_detect.get("phase_detail") or ""),
"recent_event": str(normalized_detect.get("recent_event") or ""),
"recent_warning": str(normalized_detect.get("recent_warning") or ""),
"latest_at": latest_at,
"summary": str(participation_summary.get("summary") or ""),
"counts": counts,
"log_sync": _build_execution_scene_log_sync(log_sync, participating_nodes),
"participation_summary": {
"effective_online_nodes": int(participation_summary.get("effective_online_nodes", 0) or 0),
"participating_nodes": int(participation_summary.get("participating_nodes", 0) or 0),
"dispatch_active_nodes": int(participation_summary.get("dispatch_active_nodes", 0) or 0),
"recent_only_nodes": int(participation_summary.get("recent_only_nodes", 0) or 0),
"non_participating_nodes": int(participation_summary.get("non_participating_nodes", 0) or 0),
"standby_nodes": int(participation_summary.get("standby_nodes", 0) or 0),
"load_syncing_nodes": int(participation_summary.get("load_syncing_nodes", 0) or 0),
"dedicated_worker_nodes": int(participation_summary.get("dedicated_worker_nodes", 0) or 0),
"controller_worker_nodes": int(participation_summary.get("controller_worker_nodes", 0) or 0),
"dispatch_active_node_codes": list(participation_summary.get("dispatch_active_node_codes") or []),
"recent_only_node_codes": list(participation_summary.get("recent_only_node_codes") or []),
"non_participating_node_codes": list(participation_summary.get("non_participating_node_codes") or []),
"standby_node_codes": list(participation_summary.get("standby_node_codes") or []),
"load_syncing_node_codes": list(participation_summary.get("load_syncing_node_codes") or []),
"summary": str(participation_summary.get("summary") or ""),
},
"participating_nodes": participating_nodes,
"dispatch_active_nodes": dispatch_active_nodes,
"recent_only_nodes": recent_only_nodes,
"standby_nodes": standby_nodes,
"load_syncing_nodes": load_syncing_nodes,
}
def _should_emit_execution_scene_activity(execution_scene: dict) -> bool:
normalized_scene = dict(execution_scene or {})
summary = dict(normalized_scene.get("participation_summary") or {})
log_sync = dict(normalized_scene.get("log_sync") or {})
return any(
[
str(normalized_scene.get("active_job_code") or "").strip(),
str(normalized_scene.get("phase_label") or "").strip(),
str(normalized_scene.get("phase_detail") or "").strip(),
str(normalized_scene.get("recent_event") or "").strip(),
str(normalized_scene.get("recent_warning") or "").strip(),
str(normalized_scene.get("latest_at") or "").strip(),
int(summary.get("effective_online_nodes", 0) or 0) > 0,
bool(normalized_scene.get("participating_nodes")),
bool(normalized_scene.get("standby_nodes")),
bool(log_sync.get("enabled", False)),
int(log_sync.get("line_count", 0) or 0) > 0,
int(log_sync.get("source_node_count", 0) or 0) > 0,
]
)
def _should_emit_log_sync_activity(execution_scene: dict) -> bool:
log_sync = dict((execution_scene or {}).get("log_sync") or {})
return any(
[
bool(log_sync.get("enabled", False)),
int(log_sync.get("participating_node_count", 0) or 0) > 0,
int(log_sync.get("line_count", 0) or 0) > 0,
int(log_sync.get("source_node_count", 0) or 0) > 0,
int(log_sync.get("missing_participating_node_count", 0) or 0) > 0,
bool(log_sync.get("preview_lines")),
str(log_sync.get("last_at") or "").strip(),
]
)
def _build_execution_scene_activity(execution_scene: dict) -> dict:
normalized_scene = dict(execution_scene or {})
summary = dict(normalized_scene.get("participation_summary") or {})
participating_nodes = list(normalized_scene.get("participating_nodes") or [])
standby_nodes = list(normalized_scene.get("standby_nodes") or [])
dispatch_active_nodes = [
item for item in participating_nodes
if bool(item.get("is_dispatch_active", False))
]
recent_only_nodes = [
item for item in participating_nodes
if str(item.get("participation_state") or "").strip() == "recent_throughput"
]
load_syncing_nodes = [
item for item in standby_nodes
if str(item.get("participation_state") or "").strip() == "load_syncing"
]
pure_standby_nodes = [
item for item in standby_nodes
if str(item.get("participation_state") or "").strip() == "standby"
]
phase_label = str(normalized_scene.get("phase_label") or "").strip()
summary_text = (
str(summary.get("summary") or "").strip()
or "当前还没有形成统一的执行现场摘要。"
)
status = "ready"
if dispatch_active_nodes or recent_only_nodes:
status = "running"
elif load_syncing_nodes:
status = "attention"
meta_parts = [
f"执行/已领 {len(dispatch_active_nodes)}",
f"近窗 {len(recent_only_nodes)}",
f"待命 {len(pure_standby_nodes)}",
f"待确认 {len(load_syncing_nodes)}",
]
if phase_label:
meta_parts.append(f"阶段 {phase_label}")
scene_node_codes = _normalize_driver_node_codes(
[str(item.get("node_code") or "").strip() for item in [*participating_nodes, *standby_nodes]]
)
return {
"kind": "execution_scene",
"activity_key": "execution-scene:live",
"title": "当前执行现场",
"subtitle": phase_label,
"summary": _truncate_activity_text(summary_text),
"meta_text": " / ".join(part for part in meta_parts if part),
"status": status,
"execution_mode": "",
"execution_mode_label": "",
"occurred_at": _activity_time(
normalized_scene.get("latest_at"),
*[item.get("last_heartbeat_at") for item in participating_nodes],
*[item.get("last_heartbeat_at") for item in standby_nodes],
),
"target_node_codes": scene_node_codes,
"ui_intent": _build_driver_ui_intent(
"focus_execution_scene",
open_log_drawer=False,
),
}
def _build_log_sync_activity(execution_scene: dict) -> dict:
normalized_scene = dict(execution_scene or {})
log_sync = dict(normalized_scene.get("log_sync") or {})
enabled = bool(log_sync.get("enabled", False))
participating_node_count = int(log_sync.get("participating_node_count", 0) or 0)
covered_node_count = int(log_sync.get("covered_participating_node_count", 0) or 0)
missing_node_count = int(log_sync.get("missing_participating_node_count", 0) or 0)
line_count = int(log_sync.get("line_count", 0) or 0)
source_node_count = int(log_sync.get("source_node_count", 0) or 0)
missing_nodes = [
str(item or "").strip()
for item in list(log_sync.get("missing_participating_nodes") or [])
if str(item or "").strip()
]
status = "ready"
if participating_node_count > 0 and not enabled:
status = "attention"
elif participating_node_count > 0 and (line_count <= 0 or missing_node_count > 0):
status = "attention"
elif participating_node_count > 0:
status = "running"
description = str(log_sync.get("description") or "").strip() or "远端日志回传当前暂无补充说明。"
if missing_nodes:
description = f"{description} 当前仍有未覆盖节点:{''.join(missing_nodes)}"
observed_node_codes = _normalize_driver_node_codes(
[*list(log_sync.get("source_nodes") or []), *missing_nodes]
)
return {
"kind": "log_sync",
"activity_key": "execution-scene:log-sync",
"title": "远端日志回传",
"subtitle": str(log_sync.get("mode_label") or "").strip(),
"summary": _truncate_activity_text(description),
"meta_text": " / ".join(
part
for part in [
f"参与覆盖 {covered_node_count}/{participating_node_count}",
f"样本 {line_count}",
f"来源 {source_node_count}",
f"最近 {str(log_sync.get('last_at') or '').strip()}" if str(log_sync.get("last_at") or "").strip() else "",
]
if part
),
"status": status,
"execution_mode": "control-plane",
"execution_mode_label": execution_mode_label("control-plane"),
"occurred_at": _activity_time(
log_sync.get("last_at"),
normalized_scene.get("latest_at"),
),
"target_node_codes": observed_node_codes,
"ui_intent": _build_driver_ui_intent(
"focus_execution_scene",
open_log_drawer=bool(list(log_sync.get("preview_lines") or [])),
),
}
def _default_rollout_target_nodes(cluster_nodes: list[dict]) -> list[dict]:
rows: list[dict] = []
for node in list(cluster_nodes or []):
node_code = str(node.get("node_code") or "").strip()
cluster_status = str(node.get("status") or "").strip()
if not node_code:
continue
if not bool(node.get("is_effective_worker", False)):
continue
if cluster_status not in {"online", "busy"}:
continue
rows.append(
{
"node_code": node_code,
"region": str(node.get("region") or ""),
"role": str(node.get("role") or ""),
"status": cluster_status,
"current_load": int(node.get("current_load", 0) or 0),
"is_effective_worker": True,
}
)
return rows
def get_ops_activity_stream(
*,
limit: int = _OPS_ACTIVITY_FETCH_LIMIT,
scan_limit: int | None = None,
kind: str = "",
status: str = "",
execution_mode: str = "",
query: str = "",
) -> dict:
safe_limit = min(max(int(limit or _OPS_ACTIVITY_FETCH_LIMIT), 1), 100)
safe_scan_limit = min(max(int(scan_limit or (safe_limit * 4)), safe_limit), 400)
normalized_kind = str(kind or "").strip()
normalized_status = str(status or "").strip()
normalized_execution_mode = str(execution_mode or "").strip()
normalized_query = str(query or "").strip()
runtime_status = get_runtime_status()
execution_scene = _build_ops_execution_scene(runtime_status.get("detect") or {})
managed_nodes_payload = list_managed_nodes_with_agent_state()
managed_node_map = {
str(item.get("node_code") or "").strip(): dict(item or {})
for item in list(managed_nodes_payload.get("nodes") or [])
if str(item.get("node_code") or "").strip()
}
playbook_runs_payload = get_recent_ops_playbook_runs(limit=safe_scan_limit, scan_limit=max(240, safe_scan_limit * 12))
playbook_runs = list(playbook_runs_payload.get("runs") or [])
playbook_items = [_build_playbook_run_activity(run) for run in playbook_runs if str(run.get("run_code") or "").strip()]
jobs = list_ops_jobs(limit=safe_scan_limit, compact=False)
standalone_jobs = [
job
for job in jobs
if not _is_playbook_child_job(job)
and int(job.get("rollout_id") or 0) <= 0
]
job_items = [
_build_ops_job_activity(
job,
managed_node=managed_node_map.get(str(job.get("target_node_code") or "").strip(), {}),
)
for job in standalone_jobs
if int(job.get("id") or 0) > 0
]
rollouts = list_release_rollouts(limit=safe_scan_limit)
rollout_items = [_build_rollout_activity(rollout) for rollout in rollouts if int(rollout.get("id") or 0) > 0]
runbook = get_ops_runbook()
runbook_items = [
_build_runbook_sequence_activity(sequence)
for sequence in list(runbook.get("control_sequences") or [])
if str((sequence or {}).get("key") or "").strip()
]
runbook_items = [item for item in runbook_items if item]
scene_items: list[dict] = []
if _should_emit_execution_scene_activity(execution_scene):
scene_items.append(_build_execution_scene_activity(execution_scene))
if _should_emit_log_sync_activity(execution_scene):
scene_items.append(_build_log_sync_activity(execution_scene))
all_items = [*scene_items, *playbook_items, *job_items, *rollout_items, *runbook_items]
all_items = [_finalize_activity_item(item) for item in all_items if item]
all_items.sort(key=lambda item: (str(item.get("occurred_at") or ""), str(item.get("activity_key") or "")), reverse=True)
all_items.sort(
key=lambda item: 1 if str(item.get("kind") or "").strip() == "runbook_sequence" else 0
)
available_kind_counts: dict[str, int] = {}
available_status_counts: dict[str, int] = {}
available_execution_mode_counts: dict[str, int] = {}
for item in all_items:
item_kind = str(item.get("kind") or "").strip() or "unknown"
item_status = str(item.get("status") or "").strip() or "unknown"
item_execution_mode = str(item.get("execution_mode") or "").strip()
available_kind_counts[item_kind] = int(available_kind_counts.get(item_kind, 0) or 0) + 1
available_status_counts[item_status] = int(available_status_counts.get(item_status, 0) or 0) + 1
if item_execution_mode:
available_execution_mode_counts[item_execution_mode] = (
int(available_execution_mode_counts.get(item_execution_mode, 0) or 0) + 1
)
filtered_items = [
item
for item in all_items
if (not normalized_kind or str(item.get("kind") or "").strip() == normalized_kind)
and (not normalized_status or str(item.get("status") or "").strip() == normalized_status)
and (not normalized_execution_mode or str(item.get("execution_mode") or "").strip() == normalized_execution_mode)
and _matches_activity_query(item, normalized_query)
]
items = filtered_items[:safe_limit]
summary_status, summary_status_label, summary_text = _activity_stream_summary_state(filtered_items)
kind_counts: dict[str, int] = {}
status_counts: dict[str, int] = {}
for item in items:
kind = str(item.get("kind") or "").strip() or "unknown"
status = str(item.get("status") or "").strip() or "unknown"
kind_counts[kind] = int(kind_counts.get(kind, 0) or 0) + 1
status_counts[status] = int(status_counts.get(status, 0) or 0) + 1
summary_contract_keys = _normalize_ops_contract_keys(
["ops_observability_contract", "ops_stack_diagnosis_contract"]
+ [
contract_key
for item in filtered_items
for contract_key in list((item or {}).get("contract_keys") or [])
]
)
return {
"items": items,
"contract_navigation": _build_ops_contract_navigation(
summary_contract_keys,
primary_contract_key="ops_observability_contract",
registry=get_ops_contract_registry(),
),
"summary": {
"status": summary_status,
"status_label": summary_status_label,
"summary_text": summary_text,
"total": len(items),
"filtered_total": len(filtered_items),
"unfiltered_total": len(all_items),
"kind_counts": kind_counts,
"status_counts": status_counts,
"available_kind_counts": available_kind_counts,
"available_status_counts": available_status_counts,
"available_execution_mode_counts": available_execution_mode_counts,
"latest_at": str(items[0].get("occurred_at") or "") if items else "",
"limit": safe_limit,
"scan_limit": safe_scan_limit,
"filters": {
"kind": normalized_kind,
"status": normalized_status,
"execution_mode": normalized_execution_mode,
"query": normalized_query,
},
},
}
def _build_driver_recommendations(
*,
managed_nodes: list[dict],
execution_scene: dict,
release_summary: dict,
release_gate: dict,
release_launchpad: dict,
) -> list[dict]:
effective_worker_nodes = [
item
for item in list(managed_nodes or [])
if bool(item.get("cluster_is_effective_worker", False))
and str(item.get("cluster_status") or "").strip() in {"online", "busy"}
]
handover_gap_nodes = [
item
for item in effective_worker_nodes
if not (bool(item.get("is_managed", False)) and bool(item.get("is_enabled", False)) and bool(item.get("is_agent_online", False)))
]
delivery_dead_letter_nodes = [
item
for item in list(managed_nodes or [])
if str(item.get("delivery_queue_state") or "").strip() == "dead_letter"
and bool(item.get("is_managed", False))
and bool(item.get("is_enabled", False))
]
delivery_retry_nodes = [
item
for item in list(managed_nodes or [])
if str(item.get("delivery_queue_state") or "").strip() == "retrying"
and bool(item.get("is_managed", False))
and bool(item.get("is_enabled", False))
]
participating_nodes = [item for item in list(execution_scene.get("participating_nodes") or []) if str(item.get("node_code") or "").strip()]
standby_nodes = [item for item in list(execution_scene.get("standby_nodes") or []) if str(item.get("node_code") or "").strip()]
participating_node_codes = [str(item.get("node_code") or "").strip() for item in participating_nodes if str(item.get("node_code") or "").strip()]
standby_node_codes = [str(item.get("node_code") or "").strip() for item in standby_nodes if str(item.get("node_code") or "").strip()]
release_launchpad_card = _build_release_launchpad_driver_card(release_launchpad=release_launchpad)
launchpad_gap_action = _release_launchpad_gap_action_context(release_launchpad)
log_sync = dict(execution_scene.get("log_sync") or {})
log_sync_enabled = bool(log_sync.get("enabled", False))
remote_log_sample_count = int(log_sync.get("line_count", 0) or 0)
remote_log_source_count = int(log_sync.get("source_node_count", 0) or 0)
release_label = _preferred_release_label(release_summary)
preferred_release = dict(release_gate.get("release") or {})
release_gate_status = str(release_gate.get("status") or "").strip()
release_gate_execution_mode = str(release_gate.get("execution_mode") or "remote-agent").strip() or "remote-agent"
release_gate_execution_mode_label = (
str(release_gate.get("execution_mode_label") or "").strip() or execution_mode_label(release_gate_execution_mode)
)
release_gate_summary = str(release_gate.get("summary") or "").strip()
release_gate_blocking_reasons = [str(item).strip() for item in list(release_gate.get("blocking_reasons") or []) if str(item).strip()]
release_gate_warning_reasons = [str(item).strip() for item in list(release_gate.get("warning_reasons") or []) if str(item).strip()]
release_gate_rows = list(((release_gate.get("operational_readiness") or {}).get("rows") or []))
not_execution_ready_rows = [row for row in release_gate_rows if not bool(row.get("execution_ready", False))]
inspection_problem_rows = [
row
for row in release_gate_rows
if str(row.get("inspection_status") or "").strip() in {"running", "attention", "missing"}
]
playbook_runs_payload = get_recent_ops_playbook_runs(limit=6, scan_limit=240)
recent_playbook_runs = list(playbook_runs_payload.get("runs") or [])
attention_playbook_runs = [item for item in recent_playbook_runs if str(item.get("status") or "").strip() == "attention"]
active_playbook_runs = [item for item in recent_playbook_runs if str(item.get("status") or "").strip() == "running"]
activity_stream_payload = get_ops_activity_stream(limit=10, scan_limit=80)
recent_activity_items = list(activity_stream_payload.get("items") or [])
start_delivery_failed_items = [
dict(item or {})
for item in recent_activity_items
if str(item.get("start_delivery_state") or "").strip() == "failed_local"
]
activity_focus_level, activity_focus_item = _pick_driver_activity_focus(recent_activity_items)
cards: list[dict] = []
if attention_playbook_runs:
first_attention_run = dict(attention_playbook_runs[0] or {})
run_code = str(first_attention_run.get("run_code") or "").strip()
focus_step_title = str(first_attention_run.get("focus_step_title") or "").strip()
cards.append(
{
"key": f"playbook-run-attention-{run_code or 'latest'}",
"title": "先处理异常编排",
"level_label": "最高优先",
"tag_type": "danger",
"summary": (
f"{str(first_attention_run.get('playbook_title') or first_attention_run.get('playbook_key') or '标准编排').strip()}"
f"{' / ' + focus_step_title if focus_step_title else ''} 当前出现异常,建议先看整轮事件流。"
),
"reason": str(first_attention_run.get("focus_summary") or "当前编排存在失败、阻断或取消,先处理这轮编排比继续堆动作更重要。").strip(),
"node_codes": _normalize_driver_node_codes(first_attention_run.get("target_node_codes") or []),
"meta_text": run_code,
"primary_label": "查看编排详情",
"secondary_label": "最近事件",
"primary_type": "danger",
"disabled": not run_code,
"primary_action_code": "focus_playbook_run",
"secondary_action_code": "open_playbook_run_latest_events",
"action_payload": {
"run_code": run_code,
"focus_step_key": str(first_attention_run.get("focus_step_key") or "").strip(),
"focus_step_title": focus_step_title,
},
}
)
elif active_playbook_runs:
first_active_run = dict(active_playbook_runs[0] or {})
run_code = str(first_active_run.get("run_code") or "").strip()
focus_step_title = str(first_active_run.get("focus_step_title") or "").strip()
cards.append(
{
"key": f"playbook-run-running-{run_code or 'latest'}",
"title": "盯住收口中的编排",
"level_label": "推荐",
"tag_type": "warning",
"summary": (
f"{str(first_active_run.get('playbook_title') or first_active_run.get('playbook_key') or '标准编排').strip()}"
f"{' / ' + focus_step_title if focus_step_title else ''} 仍在收口中,建议先看整轮事件流再决定是否补动作。"
),
"reason": str(first_active_run.get("focus_summary") or "当前编排还在执行或排队,先看整轮回执和事件流最容易判断是否真卡住。").strip(),
"node_codes": _normalize_driver_node_codes(first_active_run.get("target_node_codes") or []),
"meta_text": run_code,
"primary_label": "查看编排详情",
"secondary_label": "最近事件",
"primary_type": "warning",
"disabled": not run_code,
"primary_action_code": "focus_playbook_run",
"secondary_action_code": "open_playbook_run_latest_events",
"action_payload": {
"run_code": run_code,
"focus_step_key": str(first_active_run.get("focus_step_key") or "").strip(),
"focus_step_title": focus_step_title,
},
}
)
if activity_focus_level and activity_focus_item:
cards.append(_build_driver_activity_card(activity_focus_level, activity_focus_item))
if start_delivery_failed_items:
first_delivery_gap_item = dict(start_delivery_failed_items[0] or {})
source_focus_ref = _normalize_focus_ref(
first_delivery_gap_item.get("source_focus_ref") or first_delivery_gap_item.get("focus_ref")
)
cards.append(
{
"key": "activity-start-delivery-gap",
"title": "先检查开始回执异常",
"level_label": "推荐",
"tag_type": "warning",
"summary": (
f"最近有 {len(start_delivery_failed_items)} 条任务已经在节点侧开始执行,但开始回执没能成功送达控制面。"
),
"reason": str(first_delivery_gap_item.get("summary") or "").strip()
or "这通常不代表节点没执行,而是控制面和节点现场之间已经出现开始态失联,建议先看任务事件流。",
"node_codes": _normalize_driver_node_codes(
[
str(source_focus_ref.get("target_node_code") or "").strip(),
*list(first_delivery_gap_item.get("target_node_codes") or []),
]
),
"meta_text": str(first_delivery_gap_item.get("job_code") or "").strip(),
"primary_label": "查看任务事件",
"secondary_label": "",
"primary_type": "warning",
"disabled": False,
"primary_action_code": "focus_latest_job_events",
"secondary_action_code": "",
"primary_action_payload": {
"focus_ref": source_focus_ref,
},
"focus_ref": source_focus_ref,
}
)
if handover_gap_nodes:
first_gap_node = handover_gap_nodes[0]
first_gap_node_code = str(first_gap_node.get("node_code") or "").strip()
launchpad_gap_action_code = str(launchpad_gap_action.get("action_code") or "").strip()
launchpad_gap_node_code = str(launchpad_gap_action.get("node_code") or "").strip()
primary_action_code = "handover_first_gap"
primary_label = "为首台补签 Token" if bool(first_gap_node.get("is_managed", False)) else "纳管首台节点"
summary = f"当前有 {len(handover_gap_nodes)} 台有效执行节点还没进入标准执行器就绪状态,继续扩机器前要先把接管链路补齐。"
reason = "节点虽然在线且属于有效执行面SSH 已可承担日志、诊断和部分服务控制,但标准巡检编排、正式发布与 Rollout 仍以 Agent 在线为准。"
level_label = "最高优先"
tag_type = "danger"
key = "handover-gap"
if launchpad_gap_action_code in {"bootstrap_run", "run_acceptance"} and (
not launchpad_gap_node_code or launchpad_gap_node_code == first_gap_node_code
):
primary_action_code = launchpad_gap_action_code
primary_label = "跑接入收口" if launchpad_gap_action_code == "bootstrap_run" else "跑接管验收"
summary = (
f"当前有 {len(handover_gap_nodes)} 台有效执行节点还没进入标准执行器就绪状态,"
"并且 Launchpad 已经给出首台节点的标准收口动作。"
)
reason = str(launchpad_gap_action.get("summary") or "").strip() or reason
level_label = "最高优先" if launchpad_gap_action_code == "bootstrap_run" else "推荐"
tag_type = "warning" if launchpad_gap_action_code == "bootstrap_run" else "success"
key = f"handover-gap-{launchpad_gap_action_code}"
cards.append(
{
"key": key,
"title": "先补接管缺口",
"level_label": level_label,
"tag_type": tag_type,
"summary": summary,
"reason": reason,
"node_codes": [str(item.get("node_code") or "").strip() for item in handover_gap_nodes if str(item.get("node_code") or "").strip()],
"primary_label": primary_label,
"secondary_label": "查看目标节点",
"primary_type": "danger",
"disabled": not first_gap_node_code,
"primary_action_code": primary_action_code,
"secondary_action_code": "view_first_gap",
"primary_action_payload": (
{"node_code": first_gap_node_code}
if primary_action_code in {"bootstrap_run", "run_acceptance"}
else {}
),
}
)
if delivery_dead_letter_nodes:
first_dead_letter_node = dict(delivery_dead_letter_nodes[0] or {})
affected_node_codes = [
str(item.get("node_code") or "").strip()
for item in delivery_dead_letter_nodes
if str(item.get("node_code") or "").strip()
]
cards.append(
{
"key": "node-agent-dead-letter",
"title": "先处理 Agent 死信",
"level_label": "最高优先",
"tag_type": "danger",
"summary": (
f"当前有 {len(delivery_dead_letter_nodes)} 台托管节点存在 Node Agent 死信记录,"
"说明部分任务回执或事件已经进入人工介入区。"
),
"reason": str(first_dead_letter_node.get("delivery_queue_reason") or "Node Agent 回执队列已经出现死信,继续堆动作会放大控制面与节点现场的不一致。").strip(),
"node_codes": affected_node_codes,
"meta_text": _format_node_code_list(affected_node_codes),
"primary_label": "重放死信队列",
"secondary_label": "查看 Worker 日志",
"primary_type": "danger",
"disabled": not affected_node_codes,
"primary_action_code": "replay_delivery_queue",
"secondary_action_code": "open_worker_logs",
"primary_action_payload": {
"template_key": "delivery.queue.replay",
"target_node_codes": affected_node_codes,
"execution_mode": "remote-agent",
"auto_approve": False,
"payload": {
"limit": min(max(len(affected_node_codes), 1), 20),
"flush_after_replay": True,
"reason": "",
},
},
}
)
elif delivery_retry_nodes:
affected_node_codes = [
str(item.get("node_code") or "").strip()
for item in delivery_retry_nodes
if str(item.get("node_code") or "").strip()
]
first_retry_node = dict(delivery_retry_nodes[0] or {})
cards.append(
{
"key": "node-agent-retrying",
"title": "关注 Agent 回执积压",
"level_label": "推荐",
"tag_type": "warning",
"summary": (
f"当前有 {len(delivery_retry_nodes)} 台托管节点存在待重试回执,"
"控制面状态可能会短暂落后于节点现场。"
),
"reason": str(first_retry_node.get("delivery_queue_reason") or "Node Agent 正在自动回放积压的完成回执或事件,建议先观察日志再继续堆新的运维动作。").strip(),
"node_codes": affected_node_codes,
"meta_text": _format_node_code_list(affected_node_codes),
"primary_label": "冲刷回执队列",
"secondary_label": "查看 Worker 日志",
"primary_type": "warning",
"disabled": not affected_node_codes,
"primary_action_code": "flush_delivery_queue",
"secondary_action_code": "open_worker_logs",
"primary_action_payload": {
"template_key": "delivery.queue.flush",
"target_node_codes": affected_node_codes,
"execution_mode": "remote-agent",
"auto_approve": True,
"payload": {
"limit": min(max(len(affected_node_codes) * 5, 1), 20),
},
},
}
)
preferred_release_id = int(preferred_release.get("id") or 0)
preferred_release_version = str(preferred_release.get("release_version") or "").strip() or "-"
preferred_release_channel = str(preferred_release.get("channel") or "").strip() or "stable"
preferred_release_label = f"{preferred_release_version} / {preferred_release_channel}"
if release_launchpad_card:
cards.append(release_launchpad_card)
elif preferred_release_id <= 0:
cards.append(
{
"key": "release-missing",
"title": "先建立 Release",
"level_label": "待补齐",
"tag_type": "info",
"summary": "当前还没有默认 Release正式 Rollout 入口尚未建立。",
"reason": "没有 Release 时,节点更新仍然容易回到临时 git pull、手工覆盖或口头版本约定。",
"node_codes": [],
"meta_text": "",
"primary_label": "创建 Release",
"secondary_label": "",
"primary_type": "primary",
"disabled": False,
"primary_action_code": "open_release_dialog",
"secondary_action_code": "",
}
)
elif release_gate_status in {"release_not_ready", "artifact_missing"}:
cards.append(
{
"key": "release-finish-metadata",
"title": "先收口默认 Release",
"level_label": "最高优先" if release_gate_status == "artifact_missing" else "推荐",
"tag_type": "danger" if release_gate_status == "artifact_missing" else "warning",
"summary": release_gate_summary or f"默认 Release {preferred_release_label} 还没有进入可稳定 Rollout 的状态。",
"reason": "先把 Release 状态、制品地址和版本信息补完整Rollout 才不会重新退回手工发布路径。",
"node_codes": [],
"meta_text": preferred_release_label,
"primary_label": "查看版本区",
"secondary_label": "创建 Release",
"primary_type": "warning",
"disabled": False,
"primary_action_code": "focus_release_hub",
"secondary_action_code": "open_release_dialog",
}
)
elif release_gate_status == "blocked":
blocker_node_codes = [
str(item.get("node_code") or "").strip()
for item in (not_execution_ready_rows or inspection_problem_rows)
if str(item.get("node_code") or "").strip()
]
first_gap_row = dict(not_execution_ready_rows[0] or {})
first_gap_is_managed = bool(first_gap_row.get("is_managed", False))
first_gap_code = str(first_gap_row.get("node_code") or "").strip()
cards.append(
{
"key": "release-rollout-blocked",
"title": "先补 Rollout 前置条件",
"level_label": "最高优先",
"tag_type": "danger",
"summary": release_gate_summary or f"默认 Release {preferred_release_label} 当前仍存在 Rollout 阻断。",
"reason": (
release_gate_blocking_reasons[0]
if release_gate_blocking_reasons
else f"默认目标节点里仍有节点未通过 {release_gate_execution_mode_label}、巡检或接管门禁。"
),
"node_codes": blocker_node_codes,
"meta_text": preferred_release_label,
"primary_label": (
"纳管首台节点"
if first_gap_code and not first_gap_is_managed
else ("查看首台缺口" if first_gap_code else "执行标准巡检")
),
"secondary_label": "查看版本区",
"primary_type": "danger",
"disabled": not blocker_node_codes,
"primary_action_code": (
"handover_first_gap"
if first_gap_code and not first_gap_is_managed
else ("view_first_gap" if first_gap_code else "run_standard_inspection")
),
"secondary_action_code": "focus_release_hub",
}
)
elif release_gate_status == "attention":
warning_node_codes = [
str(item.get("node_code") or "").strip()
for item in inspection_problem_rows
if str(item.get("node_code") or "").strip()
]
cards.append(
{
"key": "release-rollout-attention",
"title": "先补默认 Rollout 巡检",
"level_label": "推荐",
"tag_type": "warning",
"summary": release_gate_summary or f"默认 Release {preferred_release_label} 已接近可发状态,但仍建议先补齐巡检。",
"reason": (
release_gate_warning_reasons[0]
if release_gate_warning_reasons
else "先补健康快照、日志和诊断包,可以显著降低真正发 Rollout 时的误判风险。"
),
"node_codes": warning_node_codes,
"meta_text": preferred_release_label,
"primary_label": "执行标准巡检",
"secondary_label": "查看版本区",
"primary_type": "warning",
"disabled": not warning_node_codes,
"primary_action_code": "run_standard_inspection",
"secondary_action_code": "focus_release_hub",
}
)
elif release_gate_status == "no_targets":
cards.append(
{
"key": "release-rollout-no-targets",
"title": "先确认默认 Rollout 目标",
"level_label": "推荐",
"tag_type": "warning",
"summary": release_gate_summary or f"默认 Release {preferred_release_label} 当前没有可直接纳入 Rollout 的默认目标节点。",
"reason": "通常是当前有效执行面没有在线节点,或节点尚未进入默认 target 集。",
"node_codes": [],
"meta_text": preferred_release_label,
"primary_label": "查看版本区",
"secondary_label": "",
"primary_type": "warning",
"disabled": False,
"primary_action_code": "focus_release_hub",
"secondary_action_code": "",
}
)
elif release_gate_status == "ready":
cards.append(
{
"key": "release-rollout-ready",
"title": "当前可以发起 Rollout",
"level_label": "准备就绪",
"tag_type": "success",
"summary": release_gate_summary or f"默认 Release {preferred_release_label} 已具备进入 Rollout 的门禁条件。",
"reason": "接管、巡检和默认版本都已经进入可推进状态,现在可以把更新真正纳入 Release / Rollout 闭环。",
"node_codes": [str(item.get("node_code") or "").strip() for item in list(release_gate.get("default_target_nodes") or []) if str(item.get("node_code") or "").strip()],
"meta_text": preferred_release_label,
"primary_label": "Worker 灰度",
"secondary_label": "Control 发布",
"primary_type": "success",
"disabled": False,
"primary_action_code": "create_release_rollout_worker",
"primary_action_payload": {
"release_id": preferred_release_id,
},
"secondary_action_code": "create_release_rollout_control",
"secondary_action_payload": {
"release_id": preferred_release_id,
},
}
)
elif preferred_release_id > 0:
cards.append(
{
"key": "release-rollout-fallback",
"title": "查看默认 Release 门禁",
"level_label": "推荐",
"tag_type": "info",
"summary": release_gate_summary or f"默认 Release {preferred_release_label} 的 Rollout 门禁需要进一步确认。",
"reason": "当前建议先进入 Release Hub看默认版本、批次和门禁摘要再决定下一步。",
"node_codes": [],
"meta_text": preferred_release_label,
"primary_label": "查看版本区",
"secondary_label": "",
"primary_type": "primary",
"disabled": False,
"primary_action_code": "focus_release_hub",
"secondary_action_code": "",
}
)
if participating_node_codes and not log_sync_enabled:
primary_focus_ref = _build_scene_node_log_focus_ref(
participating_node_codes[0],
mode="key",
limit=80,
source="remote_log_sync_disabled",
)
secondary_focus_ref = _build_scene_node_log_focus_ref(
participating_node_codes[0],
mode="full",
limit=120,
source="remote_log_sync_disabled",
)
cards.append(
{
"key": "enable-scene-log-sync",
"title": "先补现场日志回传",
"level_label": "最高优先",
"tag_type": "danger",
"summary": f"当前有 {len(participating_node_codes)} 台节点正在真实参与检测,但海外控制面还看不到过程日志,排障会处于半盲态。",
"reason": "先把远端日志回传切到关键模式,才能在不放大噪音的前提下看到阶段、异常、代理与执行过程。",
"node_codes": participating_node_codes,
"primary_label": "开启关键回传",
"secondary_label": "开启全量回传",
"primary_type": "danger",
"disabled": False,
"focus_ref": primary_focus_ref,
"primary_focus_ref": primary_focus_ref,
"secondary_focus_ref": secondary_focus_ref,
"primary_action_code": "enable_log_sync_key",
"primary_action_payload": {"focus_ref": primary_focus_ref},
"secondary_action_code": "enable_log_sync_full",
"secondary_action_payload": {"focus_ref": secondary_focus_ref},
}
)
elif participating_node_codes and remote_log_sample_count <= 0:
primary_focus_ref = _build_scene_node_log_focus_ref(
participating_node_codes[0],
mode=str(log_sync.get("mode") or "key"),
limit=120,
source="remote_log_sync_waiting_sample",
)
secondary_focus_ref = _merge_focus_ref(
{},
kind="execution_scene",
scene_key="scene.logs",
source="remote_log_sync_waiting_sample",
)
cards.append(
{
"key": "inspect-scene-log-gap",
"title": "排查现场日志样本缺口",
"level_label": "推荐",
"tag_type": "warning",
"summary": f"当前日志回传已开启,但 {len(participating_node_codes)} 台参与节点还没有回传样本,先确认 Worker 现场输出为什么没回来。",
"reason": "这通常意味着现场仍在执行,但日志镜像链路、周期或样本采集还没形成有效观测,适合先抓 Worker 日志再看巡检。",
"node_codes": participating_node_codes,
"meta_text": f"参与节点 {len(participating_node_codes)} 台 / 已回传来源 {remote_log_source_count}",
"primary_label": "看 Worker 日志",
"secondary_label": "执行标准巡检",
"primary_type": "warning",
"disabled": not participating_node_codes,
"focus_ref": primary_focus_ref,
"primary_focus_ref": primary_focus_ref,
"secondary_focus_ref": secondary_focus_ref,
"primary_action_code": "open_worker_logs_participating",
"primary_action_payload": {"focus_ref": primary_focus_ref},
"secondary_action_code": "run_inspection_participating",
"secondary_action_payload": {"focus_ref": secondary_focus_ref},
}
)
cards.append(
{
"key": "inspect-participating",
"title": "先看执行现场",
"level_label": "推荐" if participating_node_codes else "待命",
"tag_type": "warning" if participating_node_codes else "info",
"summary": (
f"当前有 {len(participating_node_codes)} 台节点正在真实参与检测,优先回收它们的标准巡检结果。"
if participating_node_codes
else "当前没有节点处于真实执行现场,可先关注待命节点或接管缺口。"
),
"reason": "真正参与检测的节点最接近现场最适合先看健康快照、Worker 日志和诊断包。",
"node_codes": participating_node_codes,
"primary_label": "执行标准巡检",
"secondary_label": "打开诊断模板",
"primary_type": "primary",
"disabled": not participating_node_codes,
"primary_action_code": "run_inspection_participating",
"secondary_action_code": "open_diagnostics_participating",
}
)
cards.append(
{
"key": "inspect-standby",
"title": "排查在线未参与节点",
"level_label": "推荐" if standby_node_codes else "待命",
"tag_type": "warning" if standby_node_codes else "info",
"summary": (
f"当前有 {len(standby_node_codes)} 台节点在线但未参与检测,建议做一轮标准巡检确认是正常待命还是接单异常。"
if standby_node_codes
else "当前没有在线未参与节点,执行面结构比较干净。"
),
"reason": "这组节点最适合定位“在线但不接单”“控制面兼跑未开始执行”“接管完成但还没真正纳入巡检”类问题。",
"node_codes": standby_node_codes,
"primary_label": "执行标准巡检",
"secondary_label": "看 Worker 日志",
"primary_type": "primary",
"disabled": not standby_node_codes,
"primary_action_code": "run_inspection_standby",
"secondary_action_code": "open_worker_logs_standby",
}
)
return cards
def _default_ops_priority_recommendation() -> dict:
return {
"source": "static",
"key": "build-node-agent-plane",
"priority": "build-node-agent-plane",
"title": "build-node-agent-plane",
"summary": "下一阶段应把 SSH/脚本式运维升级为海外控制面 + 节点 Agent 的任务编排模式。",
"reason": "当前已经具备跨地域心跳、同步、就绪度和日志回传基础,但节点规模继续增加后,人工 SSH 与复制日志的边际成本会快速失控。",
"level_label": "长期重点",
"tag_type": "info",
"primary_label": "",
"secondary_label": "",
"primary_action_code": "",
"secondary_action_code": "",
"node_codes": [],
"primary_node_codes": [],
"secondary_node_codes": [],
"primary_action_payload": {},
"secondary_action_payload": {},
"reasons": [
"当前已经具备跨地域心跳、同步、就绪度和日志回传基础。",
"节点数继续增加后,人工 SSH 与复制日志的边际成本会快速失控。",
"后台按钮化动作需要统一的任务模型、执行回执和日志流通道。",
],
}
def _build_ops_priority_recommendation(driver_recommendations: list[dict]) -> dict:
for raw_item in list(driver_recommendations or []):
item = dict(raw_item or {})
if not item:
continue
title = str(item.get("title") or item.get("key") or "").strip()
summary = str(item.get("summary") or item.get("reason") or "").strip()
if not title and not summary:
continue
shared_node_codes = _normalize_driver_node_codes(item.get("node_codes") or [])
primary_node_codes = _normalize_driver_node_codes(item.get("primary_node_codes") or shared_node_codes)
secondary_node_codes = _normalize_driver_node_codes(item.get("secondary_node_codes") or shared_node_codes)
shared_payload = dict(item.get("action_payload") or {})
return {
"source": "driver_recommendations",
"key": str(item.get("key") or "").strip(),
"priority": title or str(item.get("key") or "").strip() or "当前建议",
"title": title or str(item.get("key") or "").strip() or "当前建议",
"summary": summary or "当前已有建议动作,建议先按驾驶建议卡推进。",
"reason": str(item.get("reason") or "").strip(),
"level_label": str(item.get("level_label") or "").strip(),
"tag_type": str(item.get("tag_type") or "").strip(),
"primary_label": str(item.get("primary_label") or "").strip(),
"secondary_label": str(item.get("secondary_label") or "").strip(),
"primary_action_code": str(item.get("primary_action_code") or "").strip(),
"secondary_action_code": str(item.get("secondary_action_code") or "").strip(),
"node_codes": shared_node_codes,
"primary_node_codes": primary_node_codes,
"secondary_node_codes": secondary_node_codes,
"primary_action_payload": {
**shared_payload,
**dict(item.get("primary_action_payload") or {}),
},
"secondary_action_payload": {
**shared_payload,
**dict(item.get("secondary_action_payload") or {}),
},
"reasons": [
part
for part in [
str(item.get("reason") or "").strip(),
str(item.get("summary") or "").strip(),
]
if part
],
}
return _default_ops_priority_recommendation()
def _driver_feed_status_from_tag_type(tag_type: str) -> str:
normalized_tag_type = str(tag_type or "").strip()
if normalized_tag_type in {"danger", "warning"}:
return "attention"
if normalized_tag_type == "success":
return "ready"
return "planned"
def _build_driver_feed_entry_from_priority_recommendation(recommendation: dict) -> dict:
normalized_recommendation = dict(recommendation or {})
key = str(normalized_recommendation.get("key") or "").strip() or "priority-recommendation"
shared_node_codes = _normalize_driver_node_codes(normalized_recommendation.get("node_codes") or [])
primary_node_codes = _normalize_driver_node_codes(
normalized_recommendation.get("primary_node_codes") or shared_node_codes
)
secondary_node_codes = _normalize_driver_node_codes(
normalized_recommendation.get("secondary_node_codes") or shared_node_codes
)
primary_action_payload = dict(normalized_recommendation.get("primary_action_payload") or {})
secondary_action_payload = dict(normalized_recommendation.get("secondary_action_payload") or {})
title = str(normalized_recommendation.get("title") or normalized_recommendation.get("priority") or key).strip()
summary = str(normalized_recommendation.get("summary") or "").strip()
reason = str(normalized_recommendation.get("reason") or "").strip()
detail_lines = _normalize_driver_text_lines(
[
*[str(part or "").strip() for part in list(normalized_recommendation.get("reasons") or [])],
reason,
],
limit=4,
)
focus_ref = _normalize_focus_ref(normalized_recommendation.get("focus_ref"))
primary_focus_ref = _normalize_focus_ref(normalized_recommendation.get("primary_focus_ref"))
secondary_focus_ref = _normalize_focus_ref(normalized_recommendation.get("secondary_focus_ref"))
return _attach_driver_feed_contract_navigation({
"key": f"priority:{key}",
"kind": "priority_recommendation",
"lane": "do_now",
"lane_label": "当前优先",
"title": title or "当前建议",
"summary": summary or reason or "当前已有建议动作,建议先从这里推进。",
"reason": reason,
"status": _driver_feed_status_from_tag_type(normalized_recommendation.get("tag_type")),
"level_label": str(normalized_recommendation.get("level_label") or "").strip() or "推荐",
"tag_type": str(normalized_recommendation.get("tag_type") or "").strip() or "info",
"meta_text": str(normalized_recommendation.get("source") or "overview.recommendation").strip(),
"node_codes": shared_node_codes,
"detail_lines": detail_lines,
"focus_ref": focus_ref,
"executor_kind": "driver_action",
"primary_label": str(normalized_recommendation.get("primary_label") or "").strip(),
"secondary_label": str(normalized_recommendation.get("secondary_label") or "").strip(),
"primary_action_code": str(normalized_recommendation.get("primary_action_code") or "").strip(),
"secondary_action_code": str(normalized_recommendation.get("secondary_action_code") or "").strip(),
"primary_node_codes": primary_node_codes,
"secondary_node_codes": secondary_node_codes,
"primary_focus_ref": primary_focus_ref,
"secondary_focus_ref": secondary_focus_ref,
"primary_action_payload": _action_payload_with_focus_ref(primary_action_payload, primary_focus_ref),
"secondary_action_payload": _action_payload_with_focus_ref(secondary_action_payload, secondary_focus_ref),
})
def _build_driver_feed_entry_from_driver_recommendation(item: dict) -> dict:
normalized_item = dict(item or {})
key = str(normalized_item.get("key") or "").strip() or "driver-recommendation"
shared_node_codes = _normalize_driver_node_codes(normalized_item.get("node_codes") or [])
primary_node_codes = _normalize_driver_node_codes(normalized_item.get("primary_node_codes") or shared_node_codes)
secondary_node_codes = _normalize_driver_node_codes(
normalized_item.get("secondary_node_codes") or shared_node_codes
)
shared_action_payload = dict(normalized_item.get("action_payload") or {})
primary_action_payload = {
**shared_action_payload,
**dict(normalized_item.get("primary_action_payload") or {}),
}
secondary_action_payload = {
**shared_action_payload,
**dict(normalized_item.get("secondary_action_payload") or {}),
}
detail_lines = _normalize_driver_text_lines(
[
str(normalized_item.get("reason") or "").strip(),
str(normalized_item.get("meta_text") or "").strip(),
],
limit=3,
)
focus_ref = _normalize_focus_ref(normalized_item.get("focus_ref"))
primary_focus_ref = _normalize_focus_ref(normalized_item.get("primary_focus_ref"))
secondary_focus_ref = _normalize_focus_ref(normalized_item.get("secondary_focus_ref"))
return _attach_driver_feed_contract_navigation({
"key": f"recommendation:{key}",
"kind": "driver_recommendation",
"lane": "do_now",
"lane_label": "建议动作",
"title": str(normalized_item.get("title") or key).strip() or "驾驶建议",
"summary": str(normalized_item.get("summary") or normalized_item.get("reason") or "").strip() or "建议先处理这条驾驶动作。",
"reason": str(normalized_item.get("reason") or "").strip(),
"status": _driver_feed_status_from_tag_type(normalized_item.get("tag_type")),
"level_label": str(normalized_item.get("level_label") or "").strip() or "推荐",
"tag_type": str(normalized_item.get("tag_type") or "").strip() or "info",
"meta_text": str(normalized_item.get("meta_text") or "").strip(),
"node_codes": shared_node_codes,
"detail_lines": detail_lines,
"focus_ref": focus_ref,
"executor_kind": "driver_action",
"primary_label": str(normalized_item.get("primary_label") or "").strip(),
"secondary_label": str(normalized_item.get("secondary_label") or "").strip(),
"primary_action_code": str(normalized_item.get("primary_action_code") or "").strip(),
"secondary_action_code": str(normalized_item.get("secondary_action_code") or "").strip(),
"primary_node_codes": primary_node_codes,
"secondary_node_codes": secondary_node_codes,
"primary_focus_ref": primary_focus_ref,
"secondary_focus_ref": secondary_focus_ref,
"primary_action_payload": _action_payload_with_focus_ref(primary_action_payload, primary_focus_ref),
"secondary_action_payload": _action_payload_with_focus_ref(secondary_action_payload, secondary_focus_ref),
})
def _build_driver_feed_entry_from_runbook_sequence(sequence: dict) -> dict:
normalized_sequence = dict(sequence or {})
sequence_key = str(normalized_sequence.get("key") or "").strip()
if not sequence_key:
return {}
primary_resolution = dict(normalized_sequence.get("primary_resolution") or {})
secondary_resolution = dict(normalized_sequence.get("secondary_resolution") or {})
resolved_action_label = (
str(primary_resolution.get("driver_action_label") or "").strip()
or str(primary_resolution.get("driver_action_code") or "").strip()
)
resolved_action_code = str(primary_resolution.get("driver_action_code") or "").strip()
resolved_node_codes = _normalize_driver_node_codes(primary_resolution.get("driver_node_codes") or [])
secondary_resolved_action_label = (
str(secondary_resolution.get("driver_action_label") or "").strip()
or str(secondary_resolution.get("driver_action_code") or "").strip()
)
target_node_codes = _normalize_driver_node_codes(normalized_sequence.get("target_node_codes") or [])
step_titles = [
str(item or "").strip()
for item in list(normalized_sequence.get("step_titles") or [])
if str(item or "").strip()
]
detail_lines: list[str] = []
if primary_resolution and primary_resolution.get("ok") is False:
detail_lines.append(f"当前解析失败:{str(primary_resolution.get('message') or '解析失败').strip()}")
elif resolved_action_label or resolved_action_code:
detail_lines.append(
f"当前解析:{resolved_action_label or resolved_action_code}"
+ (
f" ({resolved_action_code})"
if resolved_action_code and resolved_action_label and resolved_action_code != resolved_action_label
else ""
)
)
detail_lines.append(
f"当前目标:{_format_node_code_list(resolved_node_codes or target_node_codes)}"
)
if str(normalized_sequence.get("secondary_label") or "").strip() and secondary_resolved_action_label:
detail_lines.append(f"备用解析:{secondary_resolved_action_label}")
if step_titles:
detail_lines.append(f"标准步骤:{' -> '.join(step_titles)}")
if str(normalized_sequence.get("reason") or "").strip():
detail_lines.append(str(normalized_sequence.get("reason") or "").strip())
focus_ref = _normalize_focus_ref(normalized_sequence.get("focus_ref"))
primary_focus_ref = _normalize_focus_ref(normalized_sequence.get("primary_focus_ref"))
secondary_focus_ref = _normalize_focus_ref(normalized_sequence.get("secondary_focus_ref"))
return _attach_driver_feed_contract_navigation({
"key": f"runbook:{sequence_key}",
"kind": "runbook_sequence",
"lane": "standard_path",
"lane_label": "标准路径",
"title": str(normalized_sequence.get("title") or "").strip() or "标准作业路径",
"summary": str(normalized_sequence.get("summary") or "").strip() or "当前标准作业路径已就绪。",
"reason": str(normalized_sequence.get("reason") or "").strip(),
"status": _runbook_sequence_activity_status(normalized_sequence),
"level_label": str(normalized_sequence.get("status_label") or normalized_sequence.get("status") or "").strip() or "ready",
"tag_type": str(normalized_sequence.get("tag_type") or "").strip() or "info",
"meta_text": str(normalized_sequence.get("target_scope_label") or "").strip(),
"node_codes": target_node_codes,
"detail_lines": _normalize_driver_text_lines(detail_lines, limit=5),
"focus_ref": focus_ref,
"executor_kind": "runbook_sequence",
"sequence_key": sequence_key,
"primary_label": str(normalized_sequence.get("primary_label") or "").strip(),
"secondary_label": str(normalized_sequence.get("secondary_label") or "").strip(),
"primary_action_code": resolved_action_code or str(normalized_sequence.get("primary_action_code") or "").strip(),
"secondary_action_code": (
str(secondary_resolution.get("driver_action_code") or "").strip()
or str(normalized_sequence.get("secondary_action_code") or "").strip()
),
"primary_node_codes": resolved_node_codes or target_node_codes,
"secondary_node_codes": _normalize_driver_node_codes(
secondary_resolution.get("driver_node_codes") or target_node_codes
),
"primary_focus_ref": primary_focus_ref,
"secondary_focus_ref": secondary_focus_ref,
"primary_action_payload": _action_payload_with_focus_ref(
primary_resolution.get("driver_action_payload") or {},
primary_focus_ref,
),
"secondary_action_payload": _action_payload_with_focus_ref(
secondary_resolution.get("driver_action_payload") or {},
secondary_focus_ref,
),
"occurred_at": _activity_time(primary_resolution.get("resolved_at")),
})
def _build_driver_feed_top_recommendation(entry: dict) -> dict:
normalized_entry = dict(entry or {})
if not normalized_entry:
return {}
return {
"key": str(normalized_entry.get("key") or "").strip(),
"kind": str(normalized_entry.get("kind") or "").strip(),
"lane": str(normalized_entry.get("lane") or "").strip(),
"title": str(normalized_entry.get("title") or "").strip(),
"summary": str(normalized_entry.get("summary") or "").strip(),
"reason": str(normalized_entry.get("reason") or "").strip(),
"status": str(normalized_entry.get("status") or "").strip(),
"level_label": str(normalized_entry.get("level_label") or "").strip(),
"tag_type": str(normalized_entry.get("tag_type") or "").strip(),
"meta_text": str(normalized_entry.get("meta_text") or "").strip(),
"executor_kind": str(normalized_entry.get("executor_kind") or "").strip(),
"sequence_key": str(normalized_entry.get("sequence_key") or "").strip(),
"focus_ref": _normalize_focus_ref(normalized_entry.get("focus_ref")),
"primary_label": str(normalized_entry.get("primary_label") or "").strip(),
"secondary_label": str(normalized_entry.get("secondary_label") or "").strip(),
"primary_action_code": str(normalized_entry.get("primary_action_code") or "").strip(),
"secondary_action_code": str(normalized_entry.get("secondary_action_code") or "").strip(),
"node_codes": _normalize_driver_node_codes(normalized_entry.get("node_codes") or []),
"primary_node_codes": _normalize_driver_node_codes(normalized_entry.get("primary_node_codes") or []),
"secondary_node_codes": _normalize_driver_node_codes(normalized_entry.get("secondary_node_codes") or []),
"primary_focus_ref": _normalize_focus_ref(normalized_entry.get("primary_focus_ref")),
"secondary_focus_ref": _normalize_focus_ref(normalized_entry.get("secondary_focus_ref")),
"primary_action_payload": dict(normalized_entry.get("primary_action_payload") or {}),
"secondary_action_payload": dict(normalized_entry.get("secondary_action_payload") or {}),
"detail_lines": _normalize_driver_text_lines(normalized_entry.get("detail_lines") or [], limit=5),
"contract_keys": list(normalized_entry.get("contract_keys") or []),
"contract_navigation": dict(normalized_entry.get("contract_navigation") or {}),
}
def get_ops_driver_feed() -> dict:
overview = get_ops_overview()
go_live_summary = get_ops_go_live_summary()
runbook = get_ops_runbook()
recommendation = dict(overview.get("recommendation") or {})
driver_recommendations = [dict(item or {}) for item in list(overview.get("driver_recommendations") or []) if item]
control_sequences = [dict(item or {}) for item in list(runbook.get("control_sequences") or []) if item]
activity_stream = dict(overview.get("activity_stream") or {})
activity_focus = _build_driver_activity_focus(list(activity_stream.get("items") or []))
scene_log_observation = _build_scene_log_observation_from_overview(overview)
entries: list[dict] = []
seen_keys: set[str] = set()
priority_entry = _build_driver_feed_entry_from_priority_recommendation(recommendation)
if priority_entry:
entries.append(priority_entry)
recommendation_key = str(recommendation.get("key") or "").strip()
if recommendation_key:
seen_keys.add(recommendation_key)
for raw_item in driver_recommendations:
raw_key = str(raw_item.get("key") or "").strip()
if raw_key and raw_key in seen_keys:
continue
entry = _build_driver_feed_entry_from_driver_recommendation(raw_item)
if not entry:
continue
entries.append(entry)
if raw_key:
seen_keys.add(raw_key)
for raw_sequence in control_sequences:
entry = _build_driver_feed_entry_from_runbook_sequence(raw_sequence)
if entry:
entries.append(entry)
lane_counts: dict[str, int] = {}
kind_counts: dict[str, int] = {}
status_counts: dict[str, int] = {}
for item in entries:
lane = str(item.get("lane") or "").strip() or "other"
kind = str(item.get("kind") or "").strip() or "other"
status = str(item.get("status") or "").strip() or "other"
lane_counts[lane] = int(lane_counts.get(lane, 0) or 0) + 1
kind_counts[kind] = int(kind_counts.get(kind, 0) or 0) + 1
status_counts[status] = int(status_counts.get(status, 0) or 0) + 1
headline = ""
if entries:
first_entry = dict(entries[0] or {})
headline = (
f"{str(first_entry.get('title') or '').strip()}{str(first_entry.get('summary') or '').strip()}"
).strip("")
elif activity_focus:
first_focus = dict(activity_focus[0] or {})
headline = (
f"{str(first_focus.get('title') or '').strip()}{str(first_focus.get('summary') or '').strip()}"
).strip("")
publish_status = str(go_live_summary.get("publish_status") or "").strip()
publish_status_label = str(go_live_summary.get("publish_status_label") or "").strip()
publish_summary = str(go_live_summary.get("publish_summary") or "").strip()
launchpad_recommended_target_node_code = str(
go_live_summary.get("launchpad_recommended_target_node_code") or ""
).strip()
launchpad_recommended_recovery_label = str(
go_live_summary.get("launchpad_recommended_recovery_label") or ""
).strip()
launchpad_recommended_recovery_summary = str(
go_live_summary.get("launchpad_recommended_recovery_summary") or ""
).strip()
launchpad_onboarding_bootstrap_pending_nodes = int(
go_live_summary.get("launchpad_onboarding_bootstrap_pending_nodes", 0) or 0
)
launchpad_onboarding_acceptance_ready_nodes = int(
go_live_summary.get("launchpad_onboarding_acceptance_ready_nodes", 0) or 0
)
if publish_status and publish_status != "ready":
publish_prefix = publish_status_label or publish_status
headline = (
f"{headline}|发布闸门:{publish_prefix}"
if headline
else f"发布闸门:{publish_prefix}{'' + publish_summary if publish_summary else ''}"
)
if launchpad_recommended_target_node_code and launchpad_recommended_recovery_label:
launchpad_prefix = f"{launchpad_recommended_recovery_label}{launchpad_recommended_target_node_code}"
headline = f"{headline}|接入缺口:{launchpad_prefix}" if headline else launchpad_prefix
summary_contract_keys = _normalize_ops_contract_keys(
["ops_driver_contract"]
+ [
contract_key
for item in entries
for contract_key in list((item or {}).get("contract_keys") or [])
]
+ [
contract_key
for item in activity_focus
for contract_key in list((item or {}).get("contract_keys") or [])
]
+ list(scene_log_observation.get("contract_keys") or [])
)
top_recommendation = _build_driver_feed_top_recommendation(entries[0] or {})
grouped_driver_recommendations = [
dict(item or {})
for item in entries
if str((item or {}).get("kind") or "").strip() in {"priority_recommendation", "driver_recommendation"}
]
runbook_sequences = [
dict(item or {})
for item in entries
if str((item or {}).get("kind") or "").strip() == "runbook_sequence"
]
automation_entries = [
_build_codex_brief_entry(raw_entry)
for raw_entry in entries
if dict(raw_entry or {})
]
automation_entries = [entry for entry in automation_entries if entry]
automation_coverage = _build_ops_automation_coverage_summary(
automation_entries,
activity_focus_total=len(activity_focus),
scene_log_observation=scene_log_observation,
go_live_summary=go_live_summary,
)
return {
"generated_at": datetime.now().isoformat(timespec="seconds"),
"headline": headline or "当前还没有可展示的驾驶主线。",
"go_live_summary": go_live_summary,
"top_recommendation": top_recommendation,
"driver_recommendations": grouped_driver_recommendations,
"runbook_sequences": runbook_sequences,
"activity_focus": activity_focus,
"scene_log_observation": scene_log_observation,
"entries": entries,
"automation_coverage": automation_coverage,
"contract_navigation": _build_ops_contract_navigation(
summary_contract_keys,
primary_contract_key="ops_driver_contract",
registry=get_ops_contract_registry(),
),
"summary": {
"total": len(entries),
"lane_counts": lane_counts,
"kind_counts": kind_counts,
"status_counts": status_counts,
"go_live_status": str(go_live_summary.get("go_live_status") or "").strip(),
"operator_title": str(go_live_summary.get("operator_title") or "").strip(),
"publish_ready": bool(go_live_summary.get("publish_ready")),
"publish_status": publish_status,
"publish_status_label": publish_status_label,
"launchpad_recommended_target_node_code": launchpad_recommended_target_node_code,
"launchpad_recommended_recovery_label": launchpad_recommended_recovery_label,
"launchpad_recommended_recovery_summary": launchpad_recommended_recovery_summary,
"launchpad_onboarding_bootstrap_pending_nodes": launchpad_onboarding_bootstrap_pending_nodes,
"launchpad_onboarding_acceptance_ready_nodes": launchpad_onboarding_acceptance_ready_nodes,
"do_now_total": int(lane_counts.get("do_now", 0) or 0),
"standard_path_total": int(lane_counts.get("standard_path", 0) or 0),
"activity_focus_total": len(activity_focus),
"scene_log_status": str(scene_log_observation.get("status") or "").strip() or "standby",
"launch_status": str(automation_coverage.get("launch_status") or "").strip(),
"launch_ready": bool(automation_coverage.get("launch_ready", False)),
"preview_only_total": int(automation_coverage.get("preview_only_total", 0) or 0),
"backend_handled_total": int(automation_coverage.get("backend_handled_total", 0) or 0),
},
}
def _build_scene_log_observation_from_overview(overview: dict) -> dict:
normalized_overview = dict(overview or {})
execution_scene = dict(normalized_overview.get("execution_scene") or {})
log_sync = dict(execution_scene.get("log_sync") or {})
participation_summary = dict(execution_scene.get("participation_summary") or {})
participating_node_codes = _normalize_ops_node_code_list(
execution_scene.get("participating_nodes") or [],
participation_summary.get("dispatch_active_node_codes") or [],
participation_summary.get("recent_only_node_codes") or [],
)
missing_node_codes = _normalize_ops_node_code_list(log_sync.get("missing_participating_nodes") or [])
source_node_codes = _normalize_ops_node_code_list(
log_sync.get("source_nodes") or [],
log_sync.get("source_node_summaries") or [],
)
preferred_node_codes = _normalize_ops_node_code_list(
missing_node_codes,
participating_node_codes,
source_node_codes,
)
target_node_code = str((preferred_node_codes or [""])[0] or "").strip()
enabled = bool(log_sync.get("enabled", False))
mode = _normalize_scene_log_mode(log_sync.get("mode") or "key", fallback="key")
participating_total = int(
log_sync.get("participating_node_count", participation_summary.get("participating_nodes", len(participating_node_codes)))
or participation_summary.get("participating_nodes", len(participating_node_codes))
or len(participating_node_codes)
or 0
)
covered_total = int(log_sync.get("covered_participating_node_count", 0) or 0)
line_count = int(log_sync.get("line_count", 0) or 0)
source_node_count = int(log_sync.get("source_node_count", 0) or 0)
missing_total = int(log_sync.get("missing_participating_node_count", len(missing_node_codes)) or len(missing_node_codes))
status = "standby"
status_label = "待命"
summary = "当前没有参与检测节点,现场日志观察处于待命状态。"
source = "driver_scene_log_observation"
if participating_total > 0 and not enabled:
status = "disabled"
status_label = "未开启"
summary = (
f"当前有 {participating_total} 台参与节点,但远端日志回传仍关闭,"
"海外控制面还看不到节点级现场日志。"
)
source = "remote_log_sync_disabled"
elif participating_total > 0 and (
line_count <= 0 or source_node_count <= 0 or missing_total > 0
):
status = "waiting_sample"
status_label = "等待样本"
summary = (
f"远端日志回传已经开启,但参与节点只覆盖 {covered_total}/{participating_total}"
f"样本 {line_count} 条,仍需继续观察或下钻节点现场日志。"
)
source = "remote_log_sync_waiting_sample"
elif participating_total > 0:
status = "ready"
status_label = "可下钻"
summary = (
f"远端日志回传已经形成样本,当前覆盖 {covered_total}/{participating_total} 台参与节点,"
"可以直接下钻节点现场日志。"
)
source = "remote_log_sync_ready"
focus_ref = (
_build_scene_node_log_focus_ref(
target_node_code,
mode=mode,
limit=120 if status == "waiting_sample" else 80,
source=source,
)
if target_node_code
else {}
)
return {
"status": status,
"status_label": status_label,
"summary": summary,
"enabled": enabled,
"mode": mode,
"participating_node_count": participating_total,
"covered_participating_node_count": covered_total,
"missing_participating_node_count": missing_total,
"line_count": line_count,
"source_node_count": source_node_count,
"target_node_code": target_node_code,
"target_node_codes": preferred_node_codes,
"missing_node_codes": missing_node_codes,
"source_node_codes": source_node_codes,
"last_at": str(log_sync.get("last_at") or "").strip(),
"focus_ref": focus_ref,
"contract_keys": _normalize_ops_contract_keys(
["ops_observability_contract", "ops_stack_diagnosis_contract"]
),
"contract_navigation": _build_ops_contract_navigation(
["ops_observability_contract", "ops_stack_diagnosis_contract"],
primary_contract_key="ops_observability_contract",
registry=get_ops_contract_registry(),
),
}
def _build_driver_action_request_preview(
*,
executor_kind: str,
action_code: str,
action_payload: dict | None = None,
node_codes: list[str] | None = None,
sequence_key: str = "",
secondary: bool = False,
) -> dict:
normalized_executor_kind = str(executor_kind or "").strip()
normalized_action_code = str(action_code or "").strip()
normalized_action_payload = dict(action_payload or {})
normalized_node_codes = _normalize_driver_node_codes(node_codes or [])
normalized_sequence_key = str(sequence_key or "").strip()
if normalized_executor_kind == "runbook_sequence" and normalized_sequence_key:
return {
"sequence_key": normalized_sequence_key,
"secondary": bool(secondary),
"action_payload": normalized_action_payload,
}
return {
"action_code": normalized_action_code,
"node_codes": normalized_node_codes,
"action_payload": normalized_action_payload,
}
def _driver_action_missing_fields(
action_code: str,
*,
action_payload: dict | None = None,
node_codes: list[str] | None = None,
) -> list[str]:
normalized_action_code = str(action_code or "").strip()
normalized_action_payload = dict(action_payload or {})
normalized_node_codes = _normalize_driver_node_codes(node_codes or [])
missing_fields: list[str] = []
if normalized_action_code == "open_playbook_dialog" and not str(normalized_action_payload.get("playbook_key") or "").strip():
missing_fields.append("playbook_key")
if normalized_action_code == "open_action_template_dialog" and not str(normalized_action_payload.get("template_key") or "").strip():
missing_fields.append("template_key")
if normalized_action_code in {"replay_delivery_queue", "flush_delivery_queue"}:
target_node_codes = _normalize_driver_node_codes(normalized_action_payload.get("target_node_codes") or normalized_node_codes)
if not target_node_codes:
missing_fields.append("target_node_codes")
if normalized_action_code in {"focus_playbook_run", "open_playbook_run_latest_events"} and not str(
normalized_action_payload.get("run_code") or ""
).strip():
missing_fields.append("run_code")
if normalized_action_code == "focus_activity_item" and not str(
(normalized_action_payload.get("ui_intent") or {}).get("kind") or ""
).strip():
missing_fields.append("ui_intent.kind")
if normalized_action_code == "focus_latest_job_events":
has_ui_intent = bool(str((normalized_action_payload.get("ui_intent") or {}).get("kind") or "").strip())
source_focus_ref = _normalize_focus_ref(normalized_action_payload.get("source_focus_ref"))
focus_ref = _normalize_focus_ref(normalized_action_payload.get("focus_ref"))
has_event_focus = str(source_focus_ref.get("kind") or "").strip() == "ops_job_event"
has_job_focus = str(focus_ref.get("kind") or "").strip() in {"ops_job", "ops_job_event"}
if not (has_ui_intent or has_event_focus or has_job_focus):
missing_fields.append("ui_intent.kind|focus_ref")
if normalized_action_code in {"handover_first_gap", "view_first_gap"} and not str(
normalized_action_payload.get("node_code") or (normalized_node_codes[0] if normalized_node_codes else "")
).strip():
missing_fields.append("node_code")
if normalized_action_code in {
"open_worker_logs_participating",
"open_worker_logs_standby",
"open_worker_logs",
"open_diagnostics_participating",
"open_diagnostics",
"run_inspection_participating",
"run_inspection_standby",
"run_standard_inspection",
} and not normalized_node_codes:
missing_fields.append("node_codes")
if normalized_action_code in {"create_release_rollout_worker", "create_release_rollout_control"} and int(
normalized_action_payload.get("release_id") or 0
) <= 0:
missing_fields.append("release_id")
return missing_fields
def _build_driver_action_execution_profile(
action_code: str,
*,
action_payload: dict | None = None,
node_codes: list[str] | None = None,
) -> dict:
normalized_action_code = str(action_code or "").strip()
normalized_action_payload = dict(action_payload or {})
normalized_node_codes = _normalize_driver_node_codes(node_codes or [])
missing_fields = _driver_action_missing_fields(
normalized_action_code,
action_payload=normalized_action_payload,
node_codes=normalized_node_codes,
)
base_profile = {
"action_code": normalized_action_code,
"backend_handled": normalized_action_code in _BACKEND_DRIVER_ACTION_CODES,
"automation_level": "blocked",
"recommendation": "blocked",
"executor_mode_hint": "manual-or-ui",
"risk_level": "unknown",
"confirm_required": False,
"ui_only": False,
"blocked": False,
"requires_fields": [],
"missing_fields": missing_fields,
"reason": "",
}
if not normalized_action_code:
return {
**base_profile,
"blocked": True,
"reason": "当前条目还没有解析出 driver action暂时不能自动执行。",
}
if missing_fields:
return {
**base_profile,
"blocked": True,
"requires_fields": list(dict.fromkeys(missing_fields)),
"reason": f"当前动作缺少必填上下文:{', '.join(list(dict.fromkeys(missing_fields)))}",
}
if normalized_action_code not in _BACKEND_DRIVER_ACTION_CODES:
return {
**base_profile,
"automation_level": "ui_only",
"recommendation": "open_ui",
"executor_mode_hint": "manual-or-ui",
"risk_level": "medium",
"ui_only": True,
"reason": "当前动作尚未后端化,通常需要页面交互或人工确认后再推进。",
}
if normalized_action_code in _SAFE_AUTO_DRIVER_ACTION_CODES:
executor_mode_hint = "settings" if normalized_action_code.startswith(("enable_", "disable_")) else "ops-playbook"
reason = (
"当前动作只会调整运行时日志回传开关,属于低风险设置变更。"
if executor_mode_hint == "settings"
else "当前动作会走标准 playbook / 巡检链路,属于可自动执行的观测与诊断动作。"
)
return {
**base_profile,
"automation_level": "safe_auto",
"recommendation": "auto_execute",
"executor_mode_hint": executor_mode_hint,
"risk_level": "low",
"reason": reason,
}
if normalized_action_code in _GUARDED_AUTO_DRIVER_ACTION_CODES:
return {
**base_profile,
"automation_level": "guarded_auto",
"recommendation": "confirm_then_execute",
"executor_mode_hint": "smart-release-rollout",
"risk_level": "high",
"confirm_required": True,
"reason": "当前动作会直接创建 Release / Rollout 或推进发布批次,适合在 Codex 或人工确认后执行。",
}
if normalized_action_code in _MIXED_DRIVER_ACTION_CODES:
return {
**base_profile,
"automation_level": "mixed",
"recommendation": "resolve_first",
"executor_mode_hint": "cluster-sync-or-ui",
"risk_level": "medium",
"reason": "当前动作会先判断接管缺口,可能转为补接入页面,也可能直接同步托管节点,执行前最好先复核解析结果。",
}
if normalized_action_code in _UI_ONLY_DRIVER_ACTION_CODES:
return {
**base_profile,
"automation_level": "ui_only",
"recommendation": "open_ui",
"executor_mode_hint": "ui-intent",
"risk_level": "low",
"ui_only": True,
"reason": "当前动作主要用于打开页面、聚焦详情或进入模板,不适合作为无界面自动执行动作。",
}
return {
**base_profile,
"automation_level": "ui_only",
"recommendation": "open_ui",
"executor_mode_hint": "manual-or-ui",
"risk_level": "medium",
"ui_only": True,
"reason": "当前动作暂时只能作为驾驶建议展示,建议先进入对应工作区再决定下一步。",
}
def _driver_action_executor_kind_label(executor_kind: str) -> str:
normalized_executor_kind = str(executor_kind or "").strip()
if normalized_executor_kind == "runbook_sequence":
return "runbook sequence"
return normalized_executor_kind or "driver action"
def _driver_action_execution_chain(executor_kind: str) -> str:
normalized_executor_kind = str(executor_kind or "").strip()
if normalized_executor_kind == "runbook_sequence":
return "ops-driver -> runbook-sequence -> resolved driver action / ui-intent"
return "ops-driver -> driver-action execute -> playbook / ui-intent / settings"
def _driver_action_target_api(*, executor_kind: str, sequence_key: str = "") -> str:
normalized_executor_kind = str(executor_kind or "").strip()
normalized_sequence_key = str(sequence_key or "").strip()
if normalized_executor_kind == "runbook_sequence" and normalized_sequence_key:
return f"/api/v1/ops/runbook/sequences/{normalized_sequence_key}/execute"
return "/api/v1/ops/driver-actions/execute"
def _driver_action_target_method(*, executor_kind: str, sequence_key: str = "") -> str:
normalized_executor_kind = str(executor_kind or "").strip()
normalized_sequence_key = str(sequence_key or "").strip()
if normalized_executor_kind == "runbook_sequence" and normalized_sequence_key:
return "POST"
return "POST"
def _driver_action_recommendation_label(recommendation: str) -> str:
normalized_recommendation = str(recommendation or "").strip()
return {
"auto_execute": "可自动执行",
"confirm_then_execute": "建议确认后执行",
"resolve_first": "先复核再执行",
"open_ui": "需要进入工作区",
"blocked": "当前阻断",
}.get(normalized_recommendation, normalized_recommendation or "待判断")
def _build_driver_action_contract_request_payload(
*,
executor_kind: str,
action_code: str,
action_payload: dict | None = None,
node_codes: list[str] | None = None,
sequence_key: str = "",
focus_ref: dict | None = None,
secondary: bool = False,
requested_by: str = "api/preview",
) -> dict:
normalized_executor_kind = str(executor_kind or "").strip()
normalized_action_code = str(action_code or "").strip()
normalized_action_payload = _action_payload_with_focus_ref(action_payload or {}, focus_ref)
normalized_node_codes = _normalize_driver_node_codes(node_codes or [])
normalized_sequence_key = str(sequence_key or "").strip()
normalized_requested_by = str(requested_by or "api/preview").strip() or "api/preview"
if normalized_executor_kind == "runbook_sequence" and normalized_sequence_key:
payload = {
"secondary": bool(secondary),
"requested_by": normalized_requested_by,
}
if normalized_node_codes:
payload["node_codes"] = normalized_node_codes
if normalized_action_payload:
payload["action_payload"] = normalized_action_payload
return payload
return {
"action_code": normalized_action_code,
"node_codes": normalized_node_codes,
"action_payload": normalized_action_payload,
"requested_by": normalized_requested_by,
}
def _build_driver_action_contract_preview_section(
*,
executor_kind: str,
action_code: str,
label: str = "",
action_payload: dict | None = None,
node_codes: list[str] | None = None,
sequence_key: str = "",
focus_ref: dict | None = None,
secondary: bool = False,
requested_by: str = "api/preview",
reason: str = "",
) -> dict:
normalized_executor_kind = str(executor_kind or "").strip() or "driver_action"
normalized_action_code = str(action_code or "").strip()
normalized_label = str(label or "").strip()
normalized_action_payload = dict(action_payload or {})
normalized_node_codes = _normalize_driver_node_codes(node_codes or [])
normalized_sequence_key = str(sequence_key or "").strip()
execution_profile = _build_driver_action_execution_profile(
normalized_action_code,
action_payload=normalized_action_payload,
node_codes=normalized_node_codes,
)
return {
"label": normalized_label or ("次动作" if secondary else "主动作"),
"action_code": normalized_action_code,
"node_codes": normalized_node_codes,
"focus_ref": _normalize_focus_ref(focus_ref),
"target_api": _driver_action_target_api(
executor_kind=normalized_executor_kind,
sequence_key=normalized_sequence_key,
),
"target_method": _driver_action_target_method(
executor_kind=normalized_executor_kind,
sequence_key=normalized_sequence_key,
),
"reason": str(reason or execution_profile.get("reason") or "").strip(),
"recommendation_label": _driver_action_recommendation_label(execution_profile.get("recommendation") or ""),
"automation_level_label": str(execution_profile.get("automation_level") or "").strip(),
"risk_level_label": str(execution_profile.get("risk_level") or "").strip(),
"request_payload": _build_driver_action_contract_request_payload(
executor_kind=normalized_executor_kind,
action_code=normalized_action_code,
action_payload=normalized_action_payload,
node_codes=normalized_node_codes,
sequence_key=normalized_sequence_key,
focus_ref=_normalize_focus_ref(focus_ref),
secondary=secondary,
requested_by=requested_by,
),
"execution_profile": execution_profile,
}
def preview_driver_action(payload: dict) -> tuple[bool, str, dict]:
normalized_payload = dict(payload or {})
source_label = str(normalized_payload.get("source_label") or "ops-center").strip() or "ops-center"
title = str(normalized_payload.get("title") or "").strip() or "驾驶动作"
summary = str(normalized_payload.get("summary") or "").strip()
reason = str(normalized_payload.get("reason") or "").strip()
meta_text = str(normalized_payload.get("meta_text") or "").strip()
executor_kind = str(normalized_payload.get("executor_kind") or "driver_action").strip() or "driver_action"
sequence_key = str(normalized_payload.get("sequence_key") or "").strip()
requested_by = str(normalized_payload.get("requested_by") or "api/preview").strip() or "api/preview"
contract_registry = get_ops_contract_registry()
contract = dict((contract_registry.get("contracts_by_key") or {}).get("ops_driver_contract") or {})
if executor_kind == "runbook_sequence":
if not sequence_key:
return False, "sequence_key 不能为空", {}
include_secondary = bool(normalized_payload.get("include_secondary", True))
primary_ok, primary_message, primary_resolved = resolve_ops_runbook_sequence(
sequence_key,
{
"requested_by": requested_by,
"secondary": False,
"node_codes": _normalize_driver_node_codes(normalized_payload.get("primary_node_codes") or []),
"action_payload": dict(normalized_payload.get("primary_action_payload") or {}),
},
)
if not primary_ok:
return False, primary_message, dict(primary_resolved or {})
primary_label = str(normalized_payload.get("primary_label") or primary_resolved.get("driver_action_label") or "").strip()
secondary_label = str(normalized_payload.get("secondary_label") or "").strip()
secondary_action_code = str(normalized_payload.get("secondary_action_code") or "").strip()
preview_focus_ref = _normalize_focus_ref(
normalized_payload.get("focus_ref") or (primary_resolved.get("sequence") or {}).get("focus_ref")
)
secondary_preview: dict = {}
if include_secondary or secondary_label or secondary_action_code:
secondary_ok, secondary_message, secondary_resolved = resolve_ops_runbook_sequence(
sequence_key,
{
"requested_by": requested_by,
"secondary": True,
"node_codes": _normalize_driver_node_codes(normalized_payload.get("secondary_node_codes") or []),
"action_payload": dict(normalized_payload.get("secondary_action_payload") or {}),
},
)
if secondary_ok:
secondary_preview = _build_driver_action_contract_preview_section(
executor_kind=executor_kind,
action_code=str(secondary_resolved.get("driver_action_code") or "").strip(),
label=str(secondary_label or secondary_resolved.get("driver_action_label") or "").strip(),
action_payload=dict(secondary_resolved.get("driver_action_payload") or {}),
node_codes=_normalize_driver_node_codes(secondary_resolved.get("driver_node_codes") or []),
sequence_key=sequence_key,
focus_ref=_normalize_focus_ref(
normalized_payload.get("secondary_focus_ref")
or (secondary_resolved.get("sequence") or {}).get("action_focus_ref")
),
secondary=True,
requested_by=requested_by,
reason=reason,
)
elif secondary_label or secondary_action_code:
return False, secondary_message, dict(secondary_resolved or {})
return True, "驾驶动作预览已生成", {
"contract_key": "ops_driver_contract",
"contract_version": str(contract.get("version") or _OPS_CONTRACT_SCHEMA_VERSION).strip(),
"contract_schema_doc_path": str(contract.get("schema_doc_path") or "").strip(),
"contract_primary_endpoint": str(contract.get("primary_endpoint") or "").strip(),
"preview_endpoint": "/api/v1/ops/driver-actions/preview",
"source_label": source_label,
"title": title,
"summary": summary,
"reason": reason,
"meta_text": meta_text,
"executor_kind": executor_kind,
"executor_kind_label": _driver_action_executor_kind_label(executor_kind),
"sequence_key": sequence_key,
"focus_ref": preview_focus_ref,
"primary_focus_ref": _normalize_focus_ref(
normalized_payload.get("primary_focus_ref")
or (primary_resolved.get("sequence") or {}).get("action_focus_ref")
),
"secondary_focus_ref": _normalize_focus_ref(
normalized_payload.get("secondary_focus_ref")
or (secondary_preview.get("focus_ref") or {})
),
"execution_chain": _driver_action_execution_chain(executor_kind),
"primary": _build_driver_action_contract_preview_section(
executor_kind=executor_kind,
action_code=str(primary_resolved.get("driver_action_code") or "").strip(),
label=primary_label,
action_payload=dict(primary_resolved.get("driver_action_payload") or {}),
node_codes=_normalize_driver_node_codes(primary_resolved.get("driver_node_codes") or []),
sequence_key=sequence_key,
focus_ref=_normalize_focus_ref(
normalized_payload.get("primary_focus_ref")
or (primary_resolved.get("sequence") or {}).get("action_focus_ref")
),
secondary=False,
requested_by=requested_by,
reason=reason,
),
"secondary": secondary_preview,
"resolved_primary": dict(primary_resolved or {}),
}
primary_action_code = str(normalized_payload.get("primary_action_code") or "").strip()
secondary_action_code = str(normalized_payload.get("secondary_action_code") or "").strip()
if not primary_action_code and not secondary_action_code:
return False, "当前没有可预览的驾驶动作", {}
preview_focus_ref = _normalize_focus_ref(normalized_payload.get("focus_ref"))
primary_focus_ref = _normalize_focus_ref(normalized_payload.get("primary_focus_ref"))
secondary_focus_ref = _normalize_focus_ref(normalized_payload.get("secondary_focus_ref"))
return True, "驾驶动作预览已生成", {
"contract_key": "ops_driver_contract",
"contract_version": str(contract.get("version") or _OPS_CONTRACT_SCHEMA_VERSION).strip(),
"contract_schema_doc_path": str(contract.get("schema_doc_path") or "").strip(),
"contract_primary_endpoint": str(contract.get("primary_endpoint") or "").strip(),
"preview_endpoint": "/api/v1/ops/driver-actions/preview",
"source_label": source_label,
"title": title,
"summary": summary,
"reason": reason,
"meta_text": meta_text,
"executor_kind": executor_kind,
"executor_kind_label": _driver_action_executor_kind_label(executor_kind),
"sequence_key": sequence_key,
"focus_ref": preview_focus_ref,
"primary_focus_ref": primary_focus_ref,
"secondary_focus_ref": secondary_focus_ref,
"execution_chain": _driver_action_execution_chain(executor_kind),
"primary": (
_build_driver_action_contract_preview_section(
executor_kind=executor_kind,
action_code=primary_action_code,
label=str(normalized_payload.get("primary_label") or "").strip(),
action_payload=dict(normalized_payload.get("primary_action_payload") or {}),
node_codes=_normalize_driver_node_codes(normalized_payload.get("primary_node_codes") or normalized_payload.get("node_codes") or []),
sequence_key=sequence_key,
focus_ref=primary_focus_ref,
secondary=False,
requested_by=requested_by,
reason=reason,
)
if primary_action_code
else {}
),
"secondary": (
_build_driver_action_contract_preview_section(
executor_kind=executor_kind,
action_code=secondary_action_code,
label=str(normalized_payload.get("secondary_label") or "").strip(),
action_payload=dict(normalized_payload.get("secondary_action_payload") or {}),
node_codes=_normalize_driver_node_codes(normalized_payload.get("secondary_node_codes") or normalized_payload.get("node_codes") or []),
sequence_key=sequence_key,
focus_ref=secondary_focus_ref,
secondary=True,
requested_by=requested_by,
reason=reason,
)
if secondary_action_code
else {}
),
}
def _build_codex_brief_entry(entry: dict) -> dict:
normalized_entry = dict(entry or {})
if not normalized_entry:
return {}
executor_kind = str(normalized_entry.get("executor_kind") or "driver_action").strip() or "driver_action"
sequence_key = str(normalized_entry.get("sequence_key") or "").strip()
primary_action_code = str(normalized_entry.get("primary_action_code") or "").strip()
secondary_action_code = str(normalized_entry.get("secondary_action_code") or "").strip()
primary_node_codes = _normalize_driver_node_codes(normalized_entry.get("primary_node_codes") or normalized_entry.get("node_codes") or [])
secondary_node_codes = _normalize_driver_node_codes(normalized_entry.get("secondary_node_codes") or normalized_entry.get("node_codes") or [])
primary_action_payload = dict(normalized_entry.get("primary_action_payload") or {})
secondary_action_payload = dict(normalized_entry.get("secondary_action_payload") or {})
primary_execution = _build_driver_action_execution_profile(
primary_action_code,
action_payload=primary_action_payload,
node_codes=primary_node_codes,
)
secondary_execution = (
_build_driver_action_execution_profile(
secondary_action_code,
action_payload=secondary_action_payload,
node_codes=secondary_node_codes,
)
if secondary_action_code
else {}
)
if executor_kind == "runbook_sequence" and sequence_key:
target_api = f"/api/v1/ops/runbook/sequences/{sequence_key}/execute"
target_method = "POST"
else:
target_api = "/api/v1/ops/driver-actions/execute"
target_method = "POST"
primary_request = _build_driver_action_request_preview(
executor_kind=executor_kind,
action_code=primary_action_code,
action_payload=primary_action_payload,
node_codes=primary_node_codes,
sequence_key=sequence_key,
secondary=False,
)
secondary_request = (
_build_driver_action_request_preview(
executor_kind=executor_kind,
action_code=secondary_action_code,
action_payload=secondary_action_payload,
node_codes=secondary_node_codes,
sequence_key=sequence_key,
secondary=True,
)
if secondary_action_code
else {}
)
recommendation = str(primary_execution.get("recommendation") or "blocked").strip() or "blocked"
automation_level = str(primary_execution.get("automation_level") or "blocked").strip() or "blocked"
executor_mode_hint = str(primary_execution.get("executor_mode_hint") or "").strip()
status_text = {
"auto_execute": "可自动执行",
"confirm_then_execute": "建议确认后执行",
"resolve_first": "先复核再执行",
"open_ui": "需要进入工作区",
"blocked": "当前阻断",
}.get(recommendation, "待判断")
contract_keys = _suggest_ops_contract_keys_for_codex_entry(normalized_entry)
return {
"key": str(normalized_entry.get("key") or "").strip(),
"kind": str(normalized_entry.get("kind") or "").strip(),
"lane": str(normalized_entry.get("lane") or "").strip(),
"lane_label": str(normalized_entry.get("lane_label") or "").strip(),
"title": str(normalized_entry.get("title") or "").strip(),
"summary": str(normalized_entry.get("summary") or "").strip(),
"reason": str(normalized_entry.get("reason") or "").strip(),
"status": str(normalized_entry.get("status") or "").strip(),
"level_label": str(normalized_entry.get("level_label") or "").strip(),
"tag_type": str(normalized_entry.get("tag_type") or "").strip(),
"meta_text": str(normalized_entry.get("meta_text") or "").strip(),
"detail_lines": list(normalized_entry.get("detail_lines") or []),
"node_codes": _normalize_driver_node_codes(normalized_entry.get("node_codes") or []),
"executor_kind": executor_kind,
"sequence_key": sequence_key,
"focus_ref": _normalize_focus_ref(normalized_entry.get("focus_ref")),
"primary_focus_ref": _normalize_focus_ref(normalized_entry.get("primary_focus_ref")),
"secondary_focus_ref": _normalize_focus_ref(normalized_entry.get("secondary_focus_ref")),
"occurred_at": str(normalized_entry.get("occurred_at") or "").strip(),
"primary_label": str(normalized_entry.get("primary_label") or "").strip(),
"secondary_label": str(normalized_entry.get("secondary_label") or "").strip(),
"primary_action_code": primary_action_code,
"secondary_action_code": secondary_action_code,
"contract_keys": contract_keys,
"contract_navigation": _build_ops_contract_navigation(
contract_keys,
primary_contract_key="ops_driver_contract",
),
"autopilot": {
"status_text": status_text,
"automation_level": automation_level,
"recommendation": recommendation,
"executor_mode_hint": executor_mode_hint,
"confirm_required": bool(primary_execution.get("confirm_required", False)),
"ui_only": bool(primary_execution.get("ui_only", False)),
"blocked": bool(primary_execution.get("blocked", False)),
"backend_handled": bool(primary_execution.get("backend_handled", False)),
"risk_level": str(primary_execution.get("risk_level") or "unknown").strip() or "unknown",
"reason": str(primary_execution.get("reason") or "").strip(),
"target_method": target_method,
"target_api": target_api,
},
"primary_execution": {
**primary_execution,
"label": str(normalized_entry.get("primary_label") or "").strip() or primary_action_code,
"node_codes": primary_node_codes,
"action_payload": primary_action_payload,
"request_payload_preview": primary_request,
},
"secondary_execution": (
{
**secondary_execution,
"label": str(normalized_entry.get("secondary_label") or "").strip() or secondary_action_code,
"node_codes": secondary_node_codes,
"action_payload": secondary_action_payload,
"request_payload_preview": secondary_request,
}
if secondary_action_code
else {}
),
}
def _build_codex_activity_focus_entry(entry: dict) -> dict:
normalized_entry = dict(entry or {})
if not normalized_entry:
return {}
contract_keys = _suggest_ops_contract_keys_for_codex_entry(normalized_entry)
return {
"key": str(normalized_entry.get("key") or "").strip(),
"activity_key": str(normalized_entry.get("activity_key") or "").strip(),
"activity_kind": str(normalized_entry.get("activity_kind") or "").strip(),
"title": str(normalized_entry.get("title") or "").strip(),
"subtitle": str(normalized_entry.get("subtitle") or "").strip(),
"summary": str(normalized_entry.get("summary") or "").strip(),
"status": str(normalized_entry.get("status") or "").strip(),
"status_label": str(normalized_entry.get("status_label") or "").strip(),
"meta_text": str(normalized_entry.get("meta_text") or "").strip(),
"occurred_at": str(normalized_entry.get("occurred_at") or "").strip(),
"target_node_codes": _normalize_driver_node_codes(normalized_entry.get("target_node_codes") or []),
"focus_ref": _normalize_focus_ref(normalized_entry.get("focus_ref")),
"source_focus_ref": _normalize_focus_ref(normalized_entry.get("source_focus_ref")),
"ui_intent": dict(normalized_entry.get("ui_intent") or {}),
"ui_intent_kind": str(normalized_entry.get("ui_intent_kind") or "").strip(),
"focus_action_code": str(normalized_entry.get("focus_action_code") or "").strip(),
"focus_action_payload": dict(normalized_entry.get("focus_action_payload") or {}),
"contract_keys": contract_keys,
"contract_navigation": _build_ops_contract_navigation(
contract_keys,
primary_contract_key="ops_driver_contract",
),
"observation_only": True,
}
def _build_codex_action_entry_from_activity_focus(entry: dict) -> dict:
normalized_entry = dict(entry or {})
if not normalized_entry:
return {}
key = str(normalized_entry.get("key") or "").strip()
focus_action_code = str(normalized_entry.get("focus_action_code") or "").strip()
focus_ref = _normalize_focus_ref(normalized_entry.get("focus_ref"))
target_node_codes = _normalize_driver_node_codes(normalized_entry.get("target_node_codes") or [])
focus_action_payload = dict(normalized_entry.get("focus_action_payload") or {})
status_text = str(normalized_entry.get("status_label") or normalized_entry.get("status") or "").strip()
summary = str(normalized_entry.get("summary") or "").strip()
if not key or not focus_action_code:
return {}
reason = summary or "该焦点属于运行态观察项,建议先进入对应工作区查看上下文。"
return {
"key": key,
"title": str(normalized_entry.get("title") or "").strip() or key,
"summary": summary,
"reason": reason,
"meta_text": str(normalized_entry.get("meta_text") or "").strip(),
"executor_kind": "driver_action",
"sequence_key": "",
"focus_ref": focus_ref,
"primary_focus_ref": focus_ref,
"secondary_focus_ref": {},
"primary_label": "进入工作区",
"secondary_label": "",
"primary_action_code": focus_action_code,
"secondary_action_code": "",
"autopilot": {
"status_text": status_text,
"automation_level": "ui_only",
"recommendation": "open_ui",
"executor_mode_hint": "ui-intent",
"confirm_required": False,
"ui_only": True,
"blocked": False,
"backend_handled": True,
"risk_level": "low",
"reason": reason,
"target_method": "POST",
"target_api": "/api/v1/ops/driver-actions/execute",
},
"primary_execution": {
"label": "进入工作区",
"node_codes": target_node_codes,
"action_payload": focus_action_payload,
"request_payload_preview": {
"action_code": focus_action_code,
"node_codes": target_node_codes,
"action_payload": focus_action_payload,
},
},
"secondary_execution": {},
"activity_key": str(normalized_entry.get("activity_key") or "").strip(),
"activity_kind": str(normalized_entry.get("activity_kind") or "").strip(),
"status": str(normalized_entry.get("status") or "").strip(),
"status_label": status_text,
"occurred_at": str(normalized_entry.get("occurred_at") or "").strip(),
"target_node_codes": target_node_codes,
"source_focus_ref": _normalize_focus_ref(normalized_entry.get("source_focus_ref")),
"ui_intent": dict(normalized_entry.get("ui_intent") or {}),
"ui_intent_kind": str(normalized_entry.get("ui_intent_kind") or "").strip(),
"focus_action_code": focus_action_code,
"focus_action_payload": focus_action_payload,
"observation_only": True,
}
def _is_preview_only_driver_action(action_code: str) -> bool:
normalized_action_code = str(action_code or "").strip()
if not normalized_action_code:
return False
if normalized_action_code.startswith(("open_", "focus_", "view_")):
return True
return normalized_action_code in {
"release_package",
"handover_first_gap",
"review_smart_rollout_preview",
"review_control_rollout",
"fix_rollout_blockers",
}
def _build_ops_automation_coverage_summary(
entries: list[dict] | None = None,
*,
activity_focus_total: int = 0,
scene_log_observation: dict | None = None,
go_live_summary: dict | None = None,
) -> dict:
normalized_entries = [dict(item or {}) for item in list(entries or []) if dict(item or {})]
normalized_scene_log_observation = dict(scene_log_observation or {})
normalized_go_live_summary = dict(go_live_summary or {})
automation_level_counts: dict[str, int] = {}
recommendation_counts: dict[str, int] = {}
executor_kind_counts: dict[str, int] = {}
backend_handled_total = 0
preview_only_total = 0
for item in normalized_entries:
autopilot = dict(item.get("autopilot") or {})
automation_level = str(autopilot.get("automation_level") or "unknown").strip() or "unknown"
recommendation = str(autopilot.get("recommendation") or "unknown").strip() or "unknown"
executor_kind = str(item.get("executor_kind") or "unknown").strip() or "unknown"
action_code = str(item.get("primary_action_code") or "").strip()
if bool(autopilot.get("backend_handled", False)):
backend_handled_total += 1
if _is_preview_only_driver_action(action_code):
preview_only_total += 1
automation_level_counts[automation_level] = int(automation_level_counts.get(automation_level, 0) or 0) + 1
recommendation_counts[recommendation] = int(recommendation_counts.get(recommendation, 0) or 0) + 1
executor_kind_counts[executor_kind] = int(executor_kind_counts.get(executor_kind, 0) or 0) + 1
total = len(normalized_entries)
safe_auto_total = int(automation_level_counts.get("safe_auto", 0) or 0)
guarded_auto_total = int(automation_level_counts.get("guarded_auto", 0) or 0)
mixed_total = int(automation_level_counts.get("mixed", 0) or 0)
ui_only_total = int(automation_level_counts.get("ui_only", 0) or 0)
blocked_total = int(automation_level_counts.get("blocked", 0) or 0)
unknown_total = max(total - safe_auto_total - guarded_auto_total - mixed_total - ui_only_total - blocked_total, 0)
auto_execute_total = int(recommendation_counts.get("auto_execute", 0) or 0)
confirm_then_execute_total = int(recommendation_counts.get("confirm_then_execute", 0) or 0)
resolve_first_total = int(recommendation_counts.get("resolve_first", 0) or 0)
open_ui_total = int(recommendation_counts.get("open_ui", 0) or 0)
blocked_recommendation_total = int(recommendation_counts.get("blocked", 0) or 0)
execution_ready_total = auto_execute_total + confirm_then_execute_total
human_dependency_total = resolve_first_total + open_ui_total + blocked_recommendation_total
coverage_ratio = round((backend_handled_total / total), 4) if total > 0 else 1.0
launch_status = "ready"
go_live_status = str(normalized_go_live_summary.get("go_live_status") or "").strip()
publish_ready = bool(normalized_go_live_summary.get("publish_ready", False))
scene_log_status = str(normalized_scene_log_observation.get("status") or "").strip() or "standby"
if blocked_total > 0 or blocked_recommendation_total > 0 or go_live_status == "blocked":
launch_status = "blocked"
elif (
ui_only_total > 0
or resolve_first_total > 0
or go_live_status not in {"", "ready"}
or not publish_ready
):
launch_status = "attention"
return {
"total": total,
"automation_level_counts": automation_level_counts,
"recommendation_counts": recommendation_counts,
"executor_kind_counts": executor_kind_counts,
"safe_auto_total": safe_auto_total,
"guarded_auto_total": guarded_auto_total,
"mixed_total": mixed_total,
"ui_only_total": ui_only_total,
"blocked_total": blocked_total,
"unknown_total": unknown_total,
"auto_execute_total": auto_execute_total,
"confirm_then_execute_total": confirm_then_execute_total,
"resolve_first_total": resolve_first_total,
"open_ui_total": open_ui_total,
"blocked_recommendation_total": blocked_recommendation_total,
"backend_handled_total": backend_handled_total,
"preview_only_total": preview_only_total,
"execution_ready_total": execution_ready_total,
"human_dependency_total": human_dependency_total,
"activity_focus_total": int(activity_focus_total or 0),
"scene_log_status": scene_log_status,
"launch_status": launch_status,
"launch_ready": launch_status == "ready",
"backend_coverage_ratio": coverage_ratio,
}
def get_ops_codex_brief() -> dict:
driver_feed = get_ops_driver_feed()
go_live_summary = dict(driver_feed.get("go_live_summary") or {})
contract_registry = get_ops_contract_registry()
scene_log_observation = dict(driver_feed.get("scene_log_observation") or {})
activity_focus_entries = [
_build_codex_activity_focus_entry(raw_entry)
for raw_entry in list(driver_feed.get("activity_focus") or [])
if dict(raw_entry or {})
]
activity_focus_entries = [entry for entry in activity_focus_entries if entry]
entries = [
_build_codex_brief_entry(raw_entry)
for raw_entry in list(driver_feed.get("entries") or [])
if dict(raw_entry or {})
]
entries = [entry for entry in entries if entry]
automation_coverage = _build_ops_automation_coverage_summary(
entries,
activity_focus_total=len(activity_focus_entries),
scene_log_observation=scene_log_observation,
go_live_summary=go_live_summary,
)
focus_entry = dict(entries[0] or {}) if entries else {}
focus_autopilot = dict(focus_entry.get("autopilot") or {})
focus_payload = {
"entry_key": str(focus_entry.get("key") or "").strip(),
"title": str(focus_entry.get("title") or "").strip(),
"recommendation": str(focus_autopilot.get("recommendation") or "").strip(),
"status_text": str(focus_autopilot.get("status_text") or "").strip(),
"target_api": str(focus_autopilot.get("target_api") or "").strip(),
"target_method": str(focus_autopilot.get("target_method") or "").strip(),
"focus_ref": _normalize_focus_ref(focus_entry.get("focus_ref")),
"primary_focus_ref": _normalize_focus_ref(focus_entry.get("primary_focus_ref")),
"secondary_focus_ref": _normalize_focus_ref(focus_entry.get("secondary_focus_ref")),
}
if not focus_payload["entry_key"]:
first_activity_focus = next(
(
dict(item or {})
for item in activity_focus_entries
if str((item or {}).get("focus_action_code") or "").strip()
),
{},
)
else:
first_activity_focus = {}
if not focus_payload["entry_key"] and first_activity_focus:
activity_focus_contract_keys = _suggest_ops_contract_keys_for_codex_entry(first_activity_focus)
focus_payload = {
"entry_key": str(first_activity_focus.get("key") or "").strip(),
"title": str(first_activity_focus.get("title") or "").strip(),
"recommendation": "open_ui" if str(first_activity_focus.get("ui_intent_kind") or "").strip() else "",
"status_text": str(first_activity_focus.get("status_label") or first_activity_focus.get("status") or "").strip(),
"target_api": "",
"target_method": "",
"focus_ref": _normalize_focus_ref(first_activity_focus.get("focus_ref")),
"primary_focus_ref": _normalize_focus_ref(first_activity_focus.get("focus_ref")),
"secondary_focus_ref": {},
"contract_keys": activity_focus_contract_keys,
"contract_navigation": _build_ops_contract_navigation(
activity_focus_contract_keys,
primary_contract_key="ops_driver_contract",
registry=contract_registry,
),
}
elif (
not focus_payload["entry_key"]
and _normalize_focus_ref(scene_log_observation.get("focus_ref"))
):
scene_log_contract_keys = _normalize_ops_contract_keys(
list(scene_log_observation.get("contract_keys") or [])
+ ["ops_driver_contract"]
)
focus_payload = {
"entry_key": "scene-log-observation",
"title": "现场日志观察",
"recommendation": "open_ui",
"status_text": str(scene_log_observation.get("status_label") or scene_log_observation.get("status") or "").strip(),
"target_api": "",
"target_method": "",
"focus_ref": _normalize_focus_ref(scene_log_observation.get("focus_ref")),
"primary_focus_ref": _normalize_focus_ref(scene_log_observation.get("focus_ref")),
"secondary_focus_ref": {},
"contract_keys": scene_log_contract_keys,
"contract_navigation": _build_ops_contract_navigation(
scene_log_contract_keys,
primary_contract_key="ops_driver_contract",
registry=contract_registry,
),
}
elif focus_payload["entry_key"]:
focus_contract_keys = _normalize_ops_contract_keys(
(focus_entry.get("contract_keys") or [])
+ _suggest_ops_contract_keys_for_codex_entry(focus_entry)
)
focus_payload["contract_keys"] = focus_contract_keys
focus_payload["contract_navigation"] = _build_ops_contract_navigation(
focus_contract_keys,
primary_contract_key="ops_driver_contract",
registry=contract_registry,
)
else:
focus_payload["contract_keys"] = ["ops_driver_contract"]
focus_payload["contract_navigation"] = _build_ops_contract_navigation(
["ops_driver_contract"],
primary_contract_key="ops_driver_contract",
registry=contract_registry,
)
summary_contract_keys = _normalize_ops_contract_keys(
["ops_driver_contract", "ops_stack_diagnosis_contract"]
+ [
contract_key
for item in entries
for contract_key in list(item.get("contract_keys") or [])
]
+ [
contract_key
for contract_key in list(focus_payload.get("contract_keys") or [])
]
)
return {
"generated_at": datetime.now().isoformat(timespec="seconds"),
"headline": str(driver_feed.get("headline") or "").strip() or "当前还没有可执行的驾驶主线。",
"driver_feed": driver_feed,
"go_live_summary": go_live_summary,
"scene_log_observation": scene_log_observation,
"focus": focus_payload,
"entries": entries,
"activity_focus": activity_focus_entries,
"automation_coverage": automation_coverage,
"contract_navigation": _build_ops_contract_navigation(
summary_contract_keys,
primary_contract_key="ops_driver_contract",
registry=contract_registry,
),
"summary": {
"total": len(entries),
"automation_level_counts": dict(automation_coverage.get("automation_level_counts") or {}),
"recommendation_counts": dict(automation_coverage.get("recommendation_counts") or {}),
"executor_kind_counts": dict(automation_coverage.get("executor_kind_counts") or {}),
"go_live_status": str(go_live_summary.get("go_live_status") or "").strip(),
"operator_title": str(go_live_summary.get("operator_title") or "").strip(),
"publish_ready": bool(go_live_summary.get("publish_ready")),
"publish_status": str(go_live_summary.get("publish_status") or "").strip(),
"publish_status_label": str(go_live_summary.get("publish_status_label") or "").strip(),
"launchpad_recommended_target_node_code": str(
go_live_summary.get("launchpad_recommended_target_node_code") or ""
).strip(),
"launchpad_recommended_recovery_label": str(
go_live_summary.get("launchpad_recommended_recovery_label") or ""
).strip(),
"launchpad_recommended_recovery_summary": str(
go_live_summary.get("launchpad_recommended_recovery_summary") or ""
).strip(),
"launchpad_onboarding_bootstrap_pending_nodes": int(
go_live_summary.get("launchpad_onboarding_bootstrap_pending_nodes", 0) or 0
),
"launchpad_onboarding_acceptance_ready_nodes": int(
go_live_summary.get("launchpad_onboarding_acceptance_ready_nodes", 0) or 0
),
"auto_execute_total": int(automation_coverage.get("auto_execute_total", 0) or 0),
"confirm_then_execute_total": int(automation_coverage.get("confirm_then_execute_total", 0) or 0),
"resolve_first_total": int(automation_coverage.get("resolve_first_total", 0) or 0),
"open_ui_total": int(automation_coverage.get("open_ui_total", 0) or 0),
"blocked_total": int(automation_coverage.get("blocked_recommendation_total", 0) or 0),
"activity_focus_total": int(automation_coverage.get("activity_focus_total", 0) or 0),
"preview_only_total": int(automation_coverage.get("preview_only_total", 0) or 0),
"backend_handled_total": int(automation_coverage.get("backend_handled_total", 0) or 0),
"launch_status": str(automation_coverage.get("launch_status") or "").strip(),
"launch_ready": bool(automation_coverage.get("launch_ready", False)),
},
}
def _build_resolved_driver_action_gate(
preview_data: dict,
*,
secondary: bool = False,
confirm: bool = False,
) -> dict:
normalized_preview = dict(preview_data or {})
preview_section = dict((normalized_preview.get("secondary") if secondary else normalized_preview.get("primary")) or {})
preview_execution = dict(preview_section.get("execution_profile") or {})
recommendation = str(preview_execution.get("recommendation") or "blocked").strip() or "blocked"
automation_level = str(preview_execution.get("automation_level") or "blocked").strip() or "blocked"
executor_mode_hint = str(preview_execution.get("executor_mode_hint") or "").strip()
risk_level = str(preview_execution.get("risk_level") or "unknown").strip() or "unknown"
confirm_required = bool(preview_execution.get("confirm_required", False))
ui_only = bool(preview_execution.get("ui_only", False))
blocked = bool(preview_execution.get("blocked", False))
reason = str(preview_execution.get("reason") or "").strip()
target_api = str(preview_section.get("target_api") or "").strip()
target_method = str(preview_section.get("target_method") or "POST").strip() or "POST"
request_payload = dict(preview_section.get("request_payload") or {})
confirm_received = bool(confirm)
decision = "blocked"
decision_reason = reason or "当前动作被标记为阻断,不能自动执行。"
will_execute = False
if recommendation == "auto_execute":
decision = "execute_now"
will_execute = True
decision_reason = reason or "当前动作属于 safe_auto可直接执行。"
elif recommendation == "confirm_then_execute":
if confirm_received:
decision = "execute_confirmed"
will_execute = True
decision_reason = reason or "当前动作属于 guarded_auto已收到确认允许执行。"
else:
decision = "confirmation_required"
decision_reason = "当前动作属于 guarded_auto需要显式确认后才能执行。"
if reason:
decision_reason = f"{decision_reason} {reason}"
elif recommendation == "resolve_first":
decision = "resolve_first"
decision_reason = reason or "当前动作要求先复核解析结果,不应直接执行。"
elif recommendation == "open_ui":
decision = "open_ui"
decision_reason = reason or "当前动作属于 UI-only建议进入工作区处理。"
decision_label = {
"execute_now": "直接执行",
"execute_confirmed": "确认后执行",
"confirmation_required": "等待确认",
"resolve_first": "先复核",
"open_ui": "进入工作区",
"blocked": "当前阻断",
}.get(decision, decision)
return {
"decision": decision,
"decision_label": decision_label,
"decision_reason": decision_reason,
"will_execute": will_execute,
"recommendation": recommendation,
"recommendation_label": _driver_action_recommendation_label(recommendation),
"automation_level": automation_level,
"status_text": "",
"executor_mode_hint": executor_mode_hint,
"risk_level": risk_level,
"confirm_required": confirm_required,
"confirm_received": confirm_received,
"ui_only": ui_only,
"blocked": blocked,
"secondary": bool(secondary),
"backend_handled": bool(preview_execution.get("backend_handled", False)),
"target_api": target_api,
"target_method": target_method,
"request_payload": request_payload,
"focus_ref": _normalize_focus_ref(
preview_section.get("focus_ref")
or normalized_preview.get("secondary_focus_ref" if secondary else "primary_focus_ref")
or normalized_preview.get("focus_ref")
),
"action_code": str(preview_section.get("action_code") or "").strip(),
"sequence_key": str(normalized_preview.get("sequence_key") or "").strip(),
"label": str(preview_section.get("label") or "").strip(),
"node_codes": _normalize_driver_node_codes(preview_section.get("node_codes") or []),
"requires_fields": list(preview_execution.get("requires_fields") or []),
"missing_fields": list(preview_execution.get("missing_fields") or []),
}
def resolve_driver_action_request(payload: dict | None = None) -> tuple[bool, str, dict]:
normalized_payload = dict(payload or {})
secondary = bool(normalized_payload.get("secondary", False))
confirm = bool(normalized_payload.get("confirm", False))
preview_request = {
"source_label": str(normalized_payload.get("source_label") or "ops/driver-action").strip() or "ops/driver-action",
"title": str(normalized_payload.get("title") or "").strip(),
"summary": str(normalized_payload.get("summary") or "").strip(),
"reason": str(normalized_payload.get("reason") or "").strip(),
"meta_text": str(normalized_payload.get("meta_text") or "").strip(),
"executor_kind": str(normalized_payload.get("executor_kind") or "driver_action").strip() or "driver_action",
"sequence_key": str(normalized_payload.get("sequence_key") or "").strip(),
"focus_ref": _normalize_focus_ref(normalized_payload.get("focus_ref")),
"primary_focus_ref": _normalize_focus_ref(normalized_payload.get("primary_focus_ref")),
"secondary_focus_ref": _normalize_focus_ref(normalized_payload.get("secondary_focus_ref")),
"primary_label": str(normalized_payload.get("primary_label") or "").strip(),
"secondary_label": str(normalized_payload.get("secondary_label") or "").strip(),
"primary_action_code": str(normalized_payload.get("primary_action_code") or "").strip(),
"secondary_action_code": str(normalized_payload.get("secondary_action_code") or "").strip(),
"primary_node_codes": _normalize_driver_node_codes(normalized_payload.get("primary_node_codes") or normalized_payload.get("node_codes") or []),
"secondary_node_codes": _normalize_driver_node_codes(normalized_payload.get("secondary_node_codes") or normalized_payload.get("node_codes") or []),
"primary_action_payload": dict(normalized_payload.get("primary_action_payload") or normalized_payload.get("action_payload") or {}),
"secondary_action_payload": dict(normalized_payload.get("secondary_action_payload") or normalized_payload.get("action_payload") or {}),
"include_secondary": bool(normalized_payload.get("include_secondary", True)),
"requested_by": str(normalized_payload.get("requested_by") or "api/driver-action").strip() or "api/driver-action",
}
preview_ok, preview_message, preview_data = preview_driver_action(preview_request)
if not preview_ok:
return False, preview_message, {
"preview_request": preview_request,
"preview": dict(preview_data or {}),
"selected_section": "secondary" if secondary else "primary",
}
gate = _build_resolved_driver_action_gate(preview_data, secondary=secondary, confirm=confirm)
return True, "驾驶动作解析完成", {
"preview_request": preview_request,
"preview": dict(preview_data or {}),
"gate": gate,
"selected_section": "secondary" if secondary else "primary",
"focus_ref": _normalize_focus_ref(
gate.get("focus_ref")
or preview_data.get("secondary_focus_ref" if secondary else "primary_focus_ref")
or preview_data.get("focus_ref")
),
}
def execute_resolved_driver_action(payload: dict | None = None) -> tuple[bool, str, dict]:
resolve_ok, resolve_message, resolved = resolve_driver_action_request(payload or {})
if not resolve_ok:
return False, resolve_message, dict(resolved or {})
gate = dict((resolved or {}).get("gate") or {})
if not bool(gate.get("will_execute", False)):
return False, str(gate.get("decision_reason") or "当前不允许自动执行").strip() or "当前不允许自动执行", {
**dict(resolved or {}),
"executed": False,
}
preview = dict((resolved or {}).get("preview") or {})
executor_kind = str(preview.get("executor_kind") or "driver_action").strip() or "driver_action"
request_payload = dict(gate.get("request_payload") or {})
if executor_kind == "runbook_sequence":
sequence_key = str(gate.get("sequence_key") or preview.get("sequence_key") or "").strip()
if not sequence_key:
return False, "缺少 sequence_key无法执行 runbook sequence", {
**dict(resolved or {}),
"executed": False,
}
execute_ok, execute_message, execute_data = execute_ops_runbook_sequence(sequence_key, request_payload)
else:
execute_ok, execute_message, execute_data = execute_driver_action(request_payload)
return execute_ok, execute_message, {
**dict(resolved or {}),
"executed": bool(execute_ok),
"execution_result": dict(execute_data or {}),
}
def _select_codex_brief_entry(codex_brief: dict, entry_key: str = "") -> dict:
normalized_entry_key = str(entry_key or "").strip()
entries = [dict(item or {}) for item in list((codex_brief or {}).get("entries") or []) if dict(item or {})]
activity_focus_entries = [
_build_codex_action_entry_from_activity_focus(item)
for item in list((codex_brief or {}).get("activity_focus") or [])
if dict(item or {})
]
activity_focus_entries = [item for item in activity_focus_entries if item]
focus = dict((codex_brief or {}).get("focus") or {})
target_entry_key = normalized_entry_key or str(focus.get("entry_key") or "").strip()
if target_entry_key:
for item in entries + activity_focus_entries:
if str(item.get("key") or "").strip() == target_entry_key:
return item
searchable_entries = entries + activity_focus_entries
return dict(searchable_entries[0] or {}) if searchable_entries else {}
def _build_codex_action_preview_request(
selected_entry: dict,
*,
requested_by: str,
source_label: str,
include_secondary: bool = True,
) -> dict:
normalized_entry = dict(selected_entry or {})
autopilot = dict(normalized_entry.get("autopilot") or {})
primary_execution = dict(normalized_entry.get("primary_execution") or {})
secondary_execution = dict(normalized_entry.get("secondary_execution") or {})
requested_by_value = str(requested_by or "api/codex-action").strip() or "api/codex-action"
source_label_value = str(source_label or "ops/codex-action").strip() or "ops/codex-action"
return {
"source_label": source_label_value,
"title": str(normalized_entry.get("title") or "").strip() or str(normalized_entry.get("key") or "").strip() or "codex-action",
"summary": str(normalized_entry.get("summary") or "").strip(),
"reason": str(normalized_entry.get("reason") or autopilot.get("reason") or "").strip(),
"meta_text": str(normalized_entry.get("meta_text") or "").strip(),
"executor_kind": str(normalized_entry.get("executor_kind") or "driver_action").strip() or "driver_action",
"sequence_key": str(normalized_entry.get("sequence_key") or "").strip(),
"focus_ref": _normalize_focus_ref(normalized_entry.get("focus_ref")),
"primary_focus_ref": _normalize_focus_ref(normalized_entry.get("primary_focus_ref")),
"secondary_focus_ref": _normalize_focus_ref(normalized_entry.get("secondary_focus_ref")),
"primary_label": str(primary_execution.get("label") or normalized_entry.get("primary_label") or "").strip(),
"secondary_label": str(secondary_execution.get("label") or normalized_entry.get("secondary_label") or "").strip(),
"primary_action_code": str(normalized_entry.get("primary_action_code") or "").strip(),
"secondary_action_code": str(normalized_entry.get("secondary_action_code") or "").strip(),
"primary_node_codes": _normalize_driver_node_codes(
primary_execution.get("node_codes") or normalized_entry.get("node_codes") or []
),
"secondary_node_codes": _normalize_driver_node_codes(
secondary_execution.get("node_codes") or normalized_entry.get("node_codes") or []
),
"primary_action_payload": dict(primary_execution.get("action_payload") or {}),
"secondary_action_payload": dict(secondary_execution.get("action_payload") or {}),
"include_secondary": bool(include_secondary),
"requested_by": requested_by_value,
}
def _build_codex_action_gate(
selected_entry: dict,
preview_data: dict,
*,
secondary: bool = False,
confirm: bool = False,
) -> dict:
normalized_entry = dict(selected_entry or {})
normalized_preview = dict(preview_data or {})
autopilot = dict(normalized_entry.get("autopilot") or {})
preview_section = dict((normalized_preview.get("secondary") if secondary else normalized_preview.get("primary")) or {})
execution_key = "secondary_execution" if secondary else "primary_execution"
entry_execution = dict(normalized_entry.get(execution_key) or {})
preview_execution = dict(preview_section.get("execution_profile") or {})
recommendation = str(
preview_execution.get("recommendation")
or autopilot.get("recommendation")
or "blocked"
).strip() or "blocked"
automation_level = str(
preview_execution.get("automation_level")
or autopilot.get("automation_level")
or "blocked"
).strip() or "blocked"
executor_mode_hint = str(
preview_execution.get("executor_mode_hint")
or autopilot.get("executor_mode_hint")
or ""
).strip()
risk_level = str(preview_execution.get("risk_level") or autopilot.get("risk_level") or "unknown").strip() or "unknown"
confirm_required = bool(preview_execution.get("confirm_required", False) or autopilot.get("confirm_required", False))
ui_only = bool(preview_execution.get("ui_only", False) or autopilot.get("ui_only", False))
blocked = bool(preview_execution.get("blocked", False) or autopilot.get("blocked", False))
reason = str(preview_execution.get("reason") or autopilot.get("reason") or "").strip()
target_api = str(preview_section.get("target_api") or autopilot.get("target_api") or "").strip()
target_method = str(preview_section.get("target_method") or autopilot.get("target_method") or "POST").strip() or "POST"
request_payload = dict(preview_section.get("request_payload") or {})
confirm_received = bool(confirm)
decision = "blocked"
decision_reason = reason or "当前焦点被标记为阻断,不能自动执行。"
will_execute = False
if recommendation == "auto_execute":
decision = "execute_now"
will_execute = True
decision_reason = reason or "当前焦点属于 safe_auto可直接执行。"
elif recommendation == "confirm_then_execute":
if confirm_received:
decision = "execute_confirmed"
will_execute = True
decision_reason = reason or "当前焦点属于 guarded_auto已收到确认允许执行。"
else:
decision = "confirmation_required"
decision_reason = "当前焦点属于 guarded_auto需要显式确认后才能执行。"
if reason:
decision_reason = f"{decision_reason} {reason}"
elif recommendation == "resolve_first":
decision = "resolve_first"
decision_reason = reason or "当前焦点要求先复核解析结果,不应直接执行。"
elif recommendation == "open_ui":
decision = "open_ui"
decision_reason = reason or "当前焦点属于 UI-only建议进入工作区处理。"
decision_label = {
"execute_now": "直接执行",
"execute_confirmed": "确认后执行",
"confirmation_required": "等待确认",
"resolve_first": "先复核",
"open_ui": "进入工作区",
"blocked": "当前阻断",
}.get(decision, decision)
return {
"decision": decision,
"decision_label": decision_label,
"decision_reason": decision_reason,
"will_execute": will_execute,
"recommendation": recommendation,
"recommendation_label": _driver_action_recommendation_label(recommendation),
"automation_level": automation_level,
"status_text": str(autopilot.get("status_text") or "").strip(),
"executor_mode_hint": executor_mode_hint,
"risk_level": risk_level,
"confirm_required": confirm_required,
"confirm_received": confirm_received,
"ui_only": ui_only,
"blocked": blocked,
"secondary": bool(secondary),
"backend_handled": bool(preview_execution.get("backend_handled", False) or autopilot.get("backend_handled", False)),
"target_api": target_api,
"target_method": target_method,
"request_payload": request_payload,
"focus_ref": _normalize_focus_ref(
preview_section.get("focus_ref")
or normalized_entry.get("secondary_focus_ref" if secondary else "primary_focus_ref")
or normalized_entry.get("focus_ref")
),
"action_code": str(
preview_section.get("action_code")
or normalized_entry.get("secondary_action_code" if secondary else "primary_action_code")
or ""
).strip(),
"sequence_key": str(normalized_preview.get("sequence_key") or normalized_entry.get("sequence_key") or "").strip(),
"label": str(
preview_section.get("label")
or entry_execution.get("label")
or normalized_entry.get("secondary_label" if secondary else "primary_label")
or ""
).strip(),
"requires_fields": list(preview_execution.get("requires_fields") or []),
"missing_fields": list(preview_execution.get("missing_fields") or []),
}
def resolve_codex_action(payload: dict | None = None) -> tuple[bool, str, dict]:
normalized_payload = dict(payload or {})
entry_key = str(normalized_payload.get("entry_key") or "").strip()
requested_by = str(normalized_payload.get("requested_by") or "api/codex-action").strip() or "api/codex-action"
source_label = str(normalized_payload.get("source_label") or "ops/codex-action").strip() or "ops/codex-action"
include_secondary = bool(normalized_payload.get("include_secondary", True))
secondary = bool(normalized_payload.get("secondary", False))
confirm = bool(normalized_payload.get("confirm", False))
codex_brief = get_ops_codex_brief()
selected_entry = _select_codex_brief_entry(codex_brief, entry_key=entry_key)
if not selected_entry:
return False, "codex brief 当前没有可执行条目", {"entry_key": entry_key}
preview_request = _build_codex_action_preview_request(
selected_entry,
requested_by=requested_by,
source_label=source_label,
include_secondary=include_secondary,
)
preview_ok, preview_message, preview_data = preview_driver_action(preview_request)
if not preview_ok:
return False, preview_message, {
"entry_key": str(selected_entry.get("key") or "").strip(),
"selected_entry": selected_entry,
"preview_request": preview_request,
"preview": dict(preview_data or {}),
}
gate = _build_codex_action_gate(selected_entry, preview_data, secondary=secondary, confirm=confirm)
return True, "Codex 驾驶动作解析完成", {
"entry_key": str(selected_entry.get("key") or "").strip(),
"selected_entry": selected_entry,
"preview_request": preview_request,
"preview": dict(preview_data or {}),
"gate": gate,
"selected_section": "secondary" if secondary else "primary",
"focus": dict(codex_brief.get("focus") or {}),
"summary": dict(codex_brief.get("summary") or {}),
"focus_ref": _normalize_focus_ref(
gate.get("focus_ref")
or selected_entry.get("secondary_focus_ref" if secondary else "primary_focus_ref")
or selected_entry.get("focus_ref")
),
}
def execute_codex_action(payload: dict | None = None) -> tuple[bool, str, dict]:
resolve_ok, resolve_message, resolved = resolve_codex_action(payload or {})
if not resolve_ok:
return False, resolve_message, dict(resolved or {})
gate = dict((resolved or {}).get("gate") or {})
if not bool(gate.get("will_execute", False)):
selected_entry = dict((resolved or {}).get("selected_entry") or {})
blocked_message = str(gate.get("decision_reason") or "").strip()
if str(gate.get("decision") or "").strip() == "open_ui":
blocked_message = blocked_message or "当前焦点属于 UI-only建议进入工作区处理。"
if bool(selected_entry.get("observation_only", False)) and "工作区" not in blocked_message:
blocked_message = f"{blocked_message} 请进入对应工作区查看当前观察项。".strip()
return False, blocked_message or "当前不允许自动执行", {
**dict(resolved or {}),
"executed": False,
}
selected_entry = dict((resolved or {}).get("selected_entry") or {})
preview = dict((resolved or {}).get("preview") or {})
executor_kind = str(preview.get("executor_kind") or selected_entry.get("executor_kind") or "driver_action").strip() or "driver_action"
request_payload = dict(gate.get("request_payload") or {})
if executor_kind == "runbook_sequence":
sequence_key = str(gate.get("sequence_key") or preview.get("sequence_key") or selected_entry.get("sequence_key") or "").strip()
if not sequence_key:
return False, "缺少 sequence_key无法执行 runbook sequence", {
**dict(resolved or {}),
"executed": False,
}
execute_ok, execute_message, execute_data = execute_ops_runbook_sequence(sequence_key, request_payload)
else:
execute_ok, execute_message, execute_data = execute_driver_action(request_payload)
return execute_ok, execute_message, {
**dict(resolved or {}),
"executed": bool(execute_ok),
"execution_result": dict(execute_data or {}),
}
def execute_driver_action(payload: dict) -> tuple[bool, str, dict]:
action_code = str((payload or {}).get("action_code") or "").strip()
requested_by = str((payload or {}).get("requested_by") or "api").strip() or "api"
node_codes = _normalize_driver_node_codes((payload or {}).get("node_codes") or [])
action_payload = dict((payload or {}).get("action_payload") or {})
if not action_code:
return False, "action_code 不能为空", {}
if action_code not in _BACKEND_DRIVER_ACTION_CODES:
return True, "该驾驶动作暂需前端交互或尚未后端化", {
"handled": False,
"manual_required": True,
"action_code": action_code,
"node_codes": node_codes,
}
if action_code == "enable_log_sync_key":
updated = update_runtime_settings(
{
**get_runtime_settings(),
"worker_log_sync_enabled": True,
"worker_log_sync_mode": "key",
}
)
return True, "远端日志回传已切到关键模式", {
"handled": True,
"action_code": action_code,
"mode": "settings",
"runtime_settings": updated,
}
if action_code == "disable_log_sync":
updated = update_runtime_settings(
{
**get_runtime_settings(),
"worker_log_sync_enabled": False,
"worker_log_sync_mode": "key",
}
)
return True, "远端日志回传已关闭", {
"handled": True,
"action_code": action_code,
"mode": "settings",
"runtime_settings": updated,
}
if action_code == "enable_log_sync_full":
updated = update_runtime_settings(
{
**get_runtime_settings(),
"worker_log_sync_enabled": True,
"worker_log_sync_mode": "full",
}
)
return True, "远端日志回传已切到全量模式", {
"handled": True,
"action_code": action_code,
"mode": "settings",
"runtime_settings": updated,
}
if action_code == "open_playbook_dialog":
playbook_key = str(action_payload.get("playbook_key") or "").strip()
if not playbook_key:
return False, "该驾驶动作需要 playbook_key", {"handled": False, "action_code": action_code}
target_node_codes = _normalize_driver_node_codes(action_payload.get("target_node_codes") or node_codes)
requested_execution_mode = str(action_payload.get("execution_mode") or "").strip()
if not requested_execution_mode:
requested_execution_mode = _recommend_driver_execution_mode_for_playbook(
playbook_key,
target_node_codes,
).get("execution_mode", "")
preview_ok, preview_message, playbook_preview = preview_ops_playbook(
{
"playbook_key": playbook_key,
"node_codes": target_node_codes,
"execution_mode": requested_execution_mode,
"auto_approve": bool(action_payload.get("auto_approve", False)),
"requested_by": requested_by,
}
)
if not preview_ok:
return False, preview_message, {"handled": False, "action_code": action_code}
return True, "", {
"handled": True,
"action_code": action_code,
"mode": "playbook-preview",
"playbook_preview": playbook_preview,
"ui_intent": _build_driver_ui_intent(
"open_playbook_dialog",
playbook_key=playbook_key,
target_node_codes=target_node_codes,
execution_mode=requested_execution_mode,
auto_approve=bool(action_payload.get("auto_approve", False)),
),
}
if action_code == "open_action_template_dialog":
template_key = str(action_payload.get("template_key") or "").strip()
if not template_key:
return False, "该驾驶动作需要 template_key", {"handled": False, "action_code": action_code}
template = dict(get_ops_action_template(template_key) or {})
target_node_codes = _normalize_driver_node_codes(action_payload.get("target_node_codes") or node_codes)
requested_execution_mode = str(action_payload.get("execution_mode") or "").strip()
if not requested_execution_mode:
requested_execution_mode = _recommend_driver_execution_mode_for_action_template(
template_key,
target_node_codes,
).get("execution_mode", "")
return True, "", {
"handled": True,
"action_code": action_code,
"mode": "action-template-preview",
"template_preview": {
"template": template,
"target_node_codes": target_node_codes,
"execution_mode": requested_execution_mode,
"payload": dict(action_payload.get("payload") or {}),
},
"ui_intent": _build_driver_ui_intent(
"open_action_template_dialog",
template_key=template_key,
target_node_codes=target_node_codes,
execution_mode=requested_execution_mode,
auto_approve=bool(action_payload.get("auto_approve", False)),
payload=dict(action_payload.get("payload") or {}),
),
}
if action_code in {"replay_delivery_queue", "flush_delivery_queue"}:
template_key = "delivery.queue.replay" if action_code == "replay_delivery_queue" else "delivery.queue.flush"
target_node_codes = _normalize_driver_node_codes(action_payload.get("target_node_codes") or node_codes)
if not target_node_codes:
return False, "该驾驶动作需要 target_node_codes", {"handled": False, "action_code": action_code}
requested_execution_mode = str(action_payload.get("execution_mode") or "").strip()
if not requested_execution_mode:
requested_execution_mode = _recommend_driver_execution_mode_for_action_template(
template_key,
target_node_codes,
).get("execution_mode", "")
batch_payload = {
"template_key": template_key,
"target_node_codes": target_node_codes,
"requested_by": f"{requested_by}/{action_code}",
"execution_mode": requested_execution_mode or "remote-agent",
"auto_approve": bool(action_payload.get("auto_approve", False)),
"payload": dict(action_payload.get("payload") or {}),
"metadata": {
"source": "ops-driver",
"driver_action_code": action_code,
},
}
ok, message, data = create_ops_job_batch(batch_payload)
return ok, (message or ("队列修复任务已创建" if ok else "队列修复任务创建失败")), {
"handled": ok,
"action_code": action_code,
"mode": "ops-job-batch",
"template_key": template_key,
"execution_mode": batch_payload["execution_mode"],
"target_node_codes": target_node_codes,
"job_batch": data,
}
if action_code == "focus_playbook_run":
run_code = str(action_payload.get("run_code") or "").strip()
if not run_code:
return False, "该驾驶动作需要 run_code", {"handled": False, "action_code": action_code}
run_detail = get_ops_playbook_run(run_code)
run_events = list_ops_playbook_run_events(
run_code,
limit=40,
step_key=str(action_payload.get("focus_step_key") or "").strip(),
node_code=str(action_payload.get("node_code") or "").strip(),
)
return True, "", {
"handled": True,
"action_code": action_code,
"mode": "playbook-run-focus",
"playbook_run_focus": {
"run": run_detail,
"events": run_events,
},
"ui_intent": _build_driver_ui_intent(
"playbook_run_detail",
run_code=run_code,
focus_step_key=str(action_payload.get("focus_step_key") or "").strip(),
focus_step_title=str(action_payload.get("focus_step_title") or "").strip(),
),
}
if action_code == "focus_activity_item":
ui_intent = dict(action_payload.get("ui_intent") or {})
if not str(ui_intent.get("kind") or "").strip():
return False, "该驾驶动作需要 ui_intent", {"handled": False, "action_code": action_code}
activity_key = str(action_payload.get("activity_key") or "").strip()
activity_kind = str(action_payload.get("kind") or "").strip()
activity_item = _find_activity_stream_item(activity_key, kind=activity_kind) if activity_key else {}
if not activity_item:
activity_item = {
"activity_key": activity_key,
"kind": activity_kind,
"status": str(action_payload.get("status") or "").strip(),
"summary": str(action_payload.get("summary") or "").strip(),
"focus_ref": _normalize_focus_ref(action_payload.get("focus_ref")),
"ui_intent": ui_intent,
}
return True, "", {
"handled": True,
"action_code": action_code,
"mode": "activity-focus",
"activity_focus_preview": {
"activity": activity_item,
},
"ui_intent": ui_intent,
}
if action_code == "focus_latest_job_events":
ui_intent = dict(action_payload.get("ui_intent") or {})
if not str(ui_intent.get("kind") or "").strip():
source_focus_ref = _normalize_focus_ref(action_payload.get("source_focus_ref"))
focus_ref = _normalize_focus_ref(action_payload.get("focus_ref"))
resolved_focus_ref = source_focus_ref or focus_ref
if not resolved_focus_ref:
return False, "该驾驶动作需要 ui_intent 或 focus_ref", {"handled": False, "action_code": action_code}
ui_intent = _build_driver_ui_intent(
"job_events",
job_id=int(resolved_focus_ref.get("job_id") or focus_ref.get("job_id") or 0),
job_code=str(resolved_focus_ref.get("job_code") or focus_ref.get("job_code") or "").strip(),
event_key=str(resolved_focus_ref.get("event_key") or "").strip(),
target_node_code=str(
resolved_focus_ref.get("target_node_code") or focus_ref.get("target_node_code") or ""
).strip(),
)
job_id = int(ui_intent.get("job_id") or action_payload.get("job_id") or 0)
job_events = list_ops_job_events_for_jobs([job_id], limit=60) if job_id > 0 else []
return True, "", {
"handled": True,
"action_code": action_code,
"mode": "job-events-focus",
"job_events_preview": {
"job_id": job_id,
"job_code": str(ui_intent.get("job_code") or action_payload.get("job_code") or "").strip(),
"events": job_events,
},
"ui_intent": ui_intent,
}
if action_code == "open_playbook_run_latest_events":
run_code = str(action_payload.get("run_code") or "").strip()
if not run_code:
return False, "该驾驶动作需要 run_code", {"handled": False, "action_code": action_code}
run_events = list_ops_playbook_run_events(
run_code,
limit=60,
step_key=str(action_payload.get("focus_step_key") or "").strip(),
node_code=str(action_payload.get("node_code") or "").strip(),
)
return True, "", {
"handled": True,
"action_code": action_code,
"mode": "playbook-run-events",
"playbook_run_events_preview": run_events,
"ui_intent": _build_driver_ui_intent(
"playbook_run_latest_events",
run_code=run_code,
focus_step_key=str(action_payload.get("focus_step_key") or "").strip(),
focus_step_title=str(action_payload.get("focus_step_title") or "").strip(),
),
}
if action_code == "open_release_dialog":
release_package_preview = get_latest_release_package_metadata()
release_summary = get_release_summary()
return True, "", {
"handled": True,
"action_code": action_code,
"mode": "release-dialog-preview",
"release_dialog_preview": {
"summary": release_summary,
"release_package_preview": release_package_preview,
},
"ui_intent": _build_driver_ui_intent("open_release_dialog"),
}
if action_code == "release_package":
release_package_preview = get_latest_release_package_metadata()
return True, "", {
"handled": True,
"action_code": action_code,
"mode": "release-package-preview",
"release_package_preview": release_package_preview,
"ui_intent": _build_driver_ui_intent("open_release_dialog"),
}
if action_code == "release_prepare":
ok, message, data = prepare_latest_release_package(
requested_by=str(action_payload.get("requested_by") or f"{requested_by}/{action_code}").strip()
or f"{requested_by}/{action_code}"
)
return ok, message, {
"handled": ok,
"action_code": action_code,
"mode": "release-prepare",
"release_prepare_result": data or {},
}
if action_code == "api-restart":
launchpad_context = _driver_launchpad_context(action_payload)
runtime_build_info = get_runtime_build_info()
recommended_command = build_bash_command(
"drive_ops_center.sh",
"runtime-refresh-recover",
launchpad_context["control_plane_base_url"],
)
return True, "建议先刷新控制面运行时,再继续 Launchpad 评估。", {
"handled": True,
"action_code": action_code,
"mode": "runtime-refresh-preview",
"runtime_build_info": runtime_build_info,
"recommended_command": recommended_command,
"ui_intent": _build_driver_ui_intent("open_stack_diagnosis"),
}
if action_code == "publish_latest_worker":
launchpad_context = _driver_launchpad_context(action_payload)
ok, message, data = create_release_and_smart_rollout_from_latest_package(
{
**action_payload,
"channel": launchpad_context["channel"],
"mode": "worker",
"created_by": str(action_payload.get("created_by") or f"{requested_by}/{action_code}").strip()
or f"{requested_by}/{action_code}",
"rollout_created_by": str(
action_payload.get("rollout_created_by") or f"{requested_by}/{action_code}/rollout"
).strip()
or f"{requested_by}/{action_code}/rollout",
},
control_plane_base_url=launchpad_context["control_plane_base_url"],
)
return ok, message, {
"handled": ok,
"action_code": action_code,
"mode": "smart-release-rollout",
"smart_release_result": data or {},
}
if action_code in {"review_smart_rollout_preview", "review_control_rollout", "fix_rollout_blockers"}:
launchpad_context = _driver_launchpad_context(action_payload)
release_launchpad = get_release_launchpad(
control_plane_base_url=launchpad_context["control_plane_base_url"],
channel=launchpad_context["channel"],
)
review_mode = "control" if action_code == "review_control_rollout" else "worker"
review_payload = _build_release_launchpad_review_payload(
release_launchpad=release_launchpad,
mode=review_mode,
)
latest_release = dict(review_payload.get("latest_release") or {})
return True, "", {
"handled": True,
"action_code": action_code,
"mode": "release-launchpad-preview",
"review_mode": review_mode,
"release_launchpad_review": review_payload,
"ui_intent": _build_driver_ui_intent(
"release_launchpad_review",
mode=review_mode,
release_id=int(latest_release.get("id") or 0),
channel=launchpad_context["channel"],
),
}
if action_code == "fix_managed_nodes":
launchpad_context = _driver_launchpad_context(action_payload)
release_launchpad = get_release_launchpad(
control_plane_base_url=launchpad_context["control_plane_base_url"],
channel=launchpad_context["channel"],
)
first_gap_row = _first_release_launchpad_gap_row(release_launchpad)
first_gap_node_code = str(first_gap_row.get("node_code") or "").strip()
if first_gap_node_code:
handover = get_managed_node_handover(
first_gap_node_code,
control_plane_base_url=launchpad_context["control_plane_base_url"],
)
return True, "", {
"handled": True,
"action_code": action_code,
"mode": "ui-intent",
"ui_intent": _build_driver_ui_intent(
"managed_node_handover",
node_code=first_gap_node_code,
),
"handover": handover,
}
sync_result = sync_managed_nodes_from_cluster(dry_run=False)
return True, "已按集群快照同步托管节点,请继续检查接管状态。", {
"handled": True,
"action_code": action_code,
"mode": "cluster-sync",
"sync_result": sync_result,
}
if action_code in {"bootstrap_run", "run_acceptance"}:
target_node_code = str(action_payload.get("node_code") or (node_codes[0] if node_codes else "")).strip()
if not target_node_code:
launchpad_context = _driver_launchpad_context(action_payload)
release_launchpad = get_release_launchpad(
control_plane_base_url=launchpad_context["control_plane_base_url"],
channel=launchpad_context["channel"],
)
first_gap_row = _first_release_launchpad_gap_row(release_launchpad)
target_node_code = str(first_gap_row.get("node_code") or "").strip()
if not target_node_code:
target_node_code = str(
(dict(release_launchpad.get("launchpad_status") or {}).get("recommended_target_node_code") or "")
).strip()
if not target_node_code:
return False, "该驾驶动作需要 node_code", {"handled": False, "action_code": action_code}
ok, message, data = execute_managed_node_onboarding_recovery(
node_code=target_node_code,
requested_by=f"{requested_by}/{action_code}",
control_plane_base_url=str(action_payload.get("control_plane_base_url") or "").strip(),
root_dir=str(action_payload.get("root_dir") or "").strip(),
)
return ok, message, {
"handled": ok,
"action_code": action_code,
"mode": "node-onboarding-recovery",
"node_code": target_node_code,
"recovery_result": data or {},
}
if action_code in {
"create_smart_release_rollout_worker",
"create_smart_release_rollout_control",
"create_release_rollout_worker",
"create_release_rollout_control",
}:
rollout_mode = "control" if action_code.endswith("_control") else "worker"
normalized_payload = {
**action_payload,
"mode": rollout_mode,
"created_by": str(action_payload.get("created_by") or f"{requested_by}/{action_code}").strip()
or f"{requested_by}/{action_code}",
"rollout_created_by": str(
action_payload.get("rollout_created_by") or f"{requested_by}/{action_code}/rollout"
).strip()
or f"{requested_by}/{action_code}/rollout",
"confirm_risky": bool(action_payload.get("confirm_risky", False)),
}
if action_code.startswith("create_release_rollout_"):
release_id = int(action_payload.get("release_id") or (_preferred_release_for_ops().get("id") or 0) or 0)
if release_id <= 0:
return False, "该驾驶动作需要 release_id", {"handled": False, "action_code": action_code}
ok, message, data = create_smart_release_rollout(release_id, normalized_payload)
else:
ok, message, data = create_release_and_smart_rollout_from_latest_package(normalized_payload)
return ok, message, {
"handled": ok,
"action_code": action_code,
"mode": "smart-release-rollout",
"smart_release_result": data or {},
}
if action_code == "open_rollout_dialog":
release_id = int(action_payload.get("release_id") or (_preferred_release_for_ops().get("id") or 0) or 0)
release_summary = get_release_summary()
release_launchpad = get_release_launchpad()
return True, "", {
"handled": True,
"action_code": action_code,
"mode": "rollout-dialog-preview",
"rollout_dialog_preview": {
"summary": release_summary,
"launchpad": release_launchpad,
},
"ui_intent": _build_driver_ui_intent("open_rollout_dialog", release_id=release_id),
}
if action_code == "focus_release_hub":
release_id = int(action_payload.get("release_id") or (_preferred_release_for_ops().get("id") or 0) or 0)
launchpad_context = _driver_launchpad_context(action_payload)
release_summary = get_release_summary()
release_launchpad = get_release_launchpad(
control_plane_base_url=launchpad_context["control_plane_base_url"],
channel=launchpad_context["channel"],
)
return True, "", {
"handled": True,
"action_code": action_code,
"mode": "release-hub-preview",
"release_hub_preview": {
"summary": release_summary,
"launchpad": release_launchpad,
},
"ui_intent": _build_driver_ui_intent("focus_release_hub", release_id=release_id),
}
if action_code in {"open_release_deploy_control", "open_release_deploy_worker", "open_release_deploy_custom"}:
template_key_map = {
"open_release_deploy_control": "deploy.release.control",
"open_release_deploy_worker": "deploy.release.worker",
"open_release_deploy_custom": "deploy.release.custom",
}
template_key = template_key_map.get(action_code, "deploy.release.custom")
template = dict(get_ops_action_template(template_key) or {})
target_node_codes = _normalize_driver_node_codes(action_payload.get("target_node_codes") or node_codes)
release_id = int(action_payload.get("release_id") or (_preferred_release_for_ops().get("id") or 0) or 0)
execution_mode = str(action_payload.get("execution_mode") or "").strip()
if not execution_mode:
execution_mode = _recommend_driver_execution_mode(target_node_codes).get("execution_mode", "remote-agent")
release_summary = get_release_summary()
release_launchpad = get_release_launchpad()
return True, "", {
"handled": True,
"action_code": action_code,
"mode": "release-deploy-preview",
"release_deploy_preview": {
"template": template,
"summary": release_summary,
"launchpad": release_launchpad,
"target_node_codes": target_node_codes,
"execution_mode": execution_mode,
"release_id": release_id,
},
"ui_intent": _build_driver_ui_intent(
"open_release_deploy_template",
template_key=template_key,
release_id=release_id,
target_node_codes=target_node_codes,
execution_mode=execution_mode,
),
}
if action_code in {"handover_first_gap", "view_first_gap"}:
target_node_code = str(action_payload.get("node_code") or (node_codes[0] if node_codes else "")).strip()
if not target_node_code:
return False, "该驾驶动作需要 node_code", {"handled": False, "action_code": action_code}
handover = get_managed_node_handover(
target_node_code,
control_plane_base_url=str(action_payload.get("control_plane_base_url") or "").strip(),
)
return True, "", {
"handled": True,
"action_code": action_code,
"mode": "handover-preview",
"handover_preview": handover,
"ui_intent": _build_driver_ui_intent(
"managed_node_handover" if action_code == "handover_first_gap" else "managed_node_edit",
node_code=target_node_code,
),
"handover": handover,
}
if not node_codes:
return False, "该驾驶动作需要 node_codes", {
"handled": False,
"action_code": action_code,
}
if action_code in {"open_worker_logs_participating", "open_worker_logs_standby", "open_worker_logs"}:
playbook_key = "scene.logs.key"
execution_mode = str(action_payload.get("execution_mode") or "").strip()
if not execution_mode:
execution_mode = _recommend_driver_execution_mode(node_codes).get("execution_mode", "remote-agent")
ok, message, data = execute_ops_playbook(
{
"playbook_key": playbook_key,
"node_codes": node_codes,
"execution_mode": execution_mode,
"auto_approve": True,
"requested_by": f"{requested_by}/{action_code}",
}
)
return ok, message, {
"handled": ok,
"action_code": action_code,
"mode": "ops-playbook",
"playbook_key": playbook_key,
"node_codes": node_codes,
"execution_mode": execution_mode,
**(data or {}),
}
if action_code in {"run_scene_logs_key", "run_scene_logs_full"}:
playbook_key = "scene.logs.full" if action_code == "run_scene_logs_full" else "scene.logs.key"
execution_mode = str(action_payload.get("execution_mode") or "").strip()
if not execution_mode:
execution_mode = _recommend_driver_execution_mode(node_codes).get("execution_mode", "remote-agent")
ok, message, data = execute_ops_playbook(
{
"playbook_key": playbook_key,
"node_codes": node_codes,
"execution_mode": execution_mode,
"auto_approve": bool(action_payload.get("auto_approve", True)),
"requested_by": f"{requested_by}/{action_code}",
}
)
return ok, message, {
"handled": ok,
"action_code": action_code,
"mode": "ops-playbook",
"playbook_key": playbook_key,
"node_codes": node_codes,
"execution_mode": execution_mode,
**(data or {}),
}
if action_code in {"open_diagnostics_participating", "open_diagnostics"}:
playbook_key = "scene.diagnostics"
execution_mode = str(action_payload.get("execution_mode") or "").strip()
if not execution_mode:
execution_mode = _recommend_driver_execution_mode(node_codes).get("execution_mode", "remote-agent")
ok, message, data = execute_ops_playbook(
{
"playbook_key": playbook_key,
"node_codes": node_codes,
"execution_mode": execution_mode,
"auto_approve": True,
"requested_by": f"{requested_by}/{action_code}",
}
)
return ok, message, {
"handled": ok,
"action_code": action_code,
"mode": "ops-playbook",
"playbook_key": playbook_key,
"node_codes": node_codes,
"execution_mode": execution_mode,
**(data or {}),
}
if action_code in {"run_inspection_participating", "run_inspection_standby", "run_standard_inspection"}:
playbook_key = "inspection.standard"
execution_mode = str(action_payload.get("execution_mode") or "").strip()
if not execution_mode:
execution_mode = _recommend_driver_execution_mode(node_codes).get("execution_mode", "remote-agent")
ok, message, data = execute_ops_playbook(
{
"playbook_key": playbook_key,
"node_codes": node_codes,
"execution_mode": execution_mode,
"auto_approve": True,
"requested_by": f"{requested_by}/{action_code}",
}
)
return ok, message, {
"handled": ok,
"action_code": action_code,
"mode": "ops-playbook",
"playbook_key": playbook_key,
"node_codes": node_codes,
"execution_mode": execution_mode,
**(data or {}),
}
return True, "该驾驶动作暂需前端交互或尚未后端化", {
"handled": False,
"manual_required": True,
"action_code": action_code,
"node_codes": node_codes,
}
def _resolve_ops_runbook_sequence_entry(
sequence: dict,
*,
secondary: bool = False,
payload: dict | None = None,
requested_by: str = "api",
) -> tuple[bool, str, dict]:
normalized_sequence = dict(sequence or {})
normalized_payload = dict(payload or {})
normalized_sequence_key = str(normalized_sequence.get("key") or "").strip()
if not normalized_sequence_key:
return False, "sequence_key 不能为空", {}
action_field = "secondary_action_code" if secondary else "primary_action_code"
payload_field = "secondary_action_payload" if secondary else "primary_action_payload"
label_field = "secondary_label" if secondary else "primary_label"
action_code = str(normalized_sequence.get(action_field) or "").strip()
if not action_code:
return False, "当前标准作业路径没有可执行动作", {
"sequence_key": normalized_sequence_key,
"secondary": secondary,
"sequence": normalized_sequence,
}
sequence_node_codes = _normalize_driver_node_codes(normalized_sequence.get("target_node_codes") or [])
override_node_codes = _normalize_driver_node_codes(normalized_payload.get("node_codes") or [])
node_codes = override_node_codes or sequence_node_codes
action_payload = {
**dict(normalized_sequence.get(payload_field) or {}),
**dict(normalized_payload.get("action_payload") or {}),
}
sequence_focus_ref = _normalize_focus_ref(normalized_sequence.get("focus_ref"))
action_focus_ref = _normalize_focus_ref(
normalized_sequence.get("secondary_focus_ref" if secondary else "primary_focus_ref")
)
if sequence_focus_ref and not action_payload.get("focus_ref"):
action_payload["focus_ref"] = action_focus_ref or sequence_focus_ref
normalized_requested_by = str(requested_by or "api").strip().rstrip("/") or "api"
driver_requested_by = (
f"{normalized_requested_by}/{normalized_sequence_key}"
if normalized_requested_by.endswith("/runbook")
else f"{normalized_requested_by}/runbook/{normalized_sequence_key}"
)
sequence_summary = {
"key": normalized_sequence_key,
"title": str(normalized_sequence.get("title") or "").strip(),
"status": str(normalized_sequence.get("status") or "").strip(),
"status_label": str(normalized_sequence.get("status_label") or "").strip(),
"summary": str(normalized_sequence.get("summary") or "").strip(),
"reason": str(normalized_sequence.get("reason") or "").strip(),
"secondary": secondary,
"action_label": str(normalized_sequence.get(label_field) or "").strip(),
"target_scope_label": str(normalized_sequence.get("target_scope_label") or "").strip(),
"step_titles": [str(item or "").strip() for item in list(normalized_sequence.get("step_titles") or []) if str(item or "").strip()],
"focus_ref": sequence_focus_ref,
"action_focus_ref": action_focus_ref,
}
return True, "标准作业路径解析成功", {
"sequence_key": normalized_sequence_key,
"sequence": sequence_summary,
"driver_action_code": action_code,
"driver_action_label": str(normalized_sequence.get(label_field) or "").strip() or action_code,
"driver_node_codes": node_codes,
"driver_action_payload": action_payload,
"driver_requested_by": driver_requested_by,
"resolved_at": datetime.now().isoformat(timespec="seconds"),
}
def _attach_ops_runbook_sequence_resolutions(
sequences: list[dict],
*,
requested_by: str = "api/runbook-snapshot",
) -> list[dict]:
annotated_sequences: list[dict] = []
for raw_sequence in list(sequences or []):
sequence = dict(raw_sequence or {})
if not str(sequence.get("key") or "").strip():
continue
primary_ok, primary_message, primary_data = _resolve_ops_runbook_sequence_entry(
sequence,
secondary=False,
requested_by=requested_by,
)
sequence["primary_resolution"] = {
"ok": primary_ok,
"message": primary_message,
**dict(primary_data or {}),
}
if str(sequence.get("secondary_action_code") or "").strip():
secondary_ok, secondary_message, secondary_data = _resolve_ops_runbook_sequence_entry(
sequence,
secondary=True,
requested_by=requested_by,
)
sequence["secondary_resolution"] = {
"ok": secondary_ok,
"message": secondary_message,
**dict(secondary_data or {}),
}
else:
sequence["secondary_resolution"] = {}
annotated_sequences.append(sequence)
return annotated_sequences
def resolve_ops_runbook_sequence(sequence_key: str, payload: dict | None = None) -> tuple[bool, str, dict]:
normalized_sequence_key = str(sequence_key or "").strip()
normalized_payload = dict(payload or {})
if not normalized_sequence_key:
return False, "sequence_key 不能为空", {}
requested_by = str(normalized_payload.get("requested_by") or "api").strip() or "api"
secondary = bool(normalized_payload.get("secondary", False))
runbook = get_ops_runbook()
sequences = [
dict(item or {})
for item in list(runbook.get("control_sequences") or [])
if str((item or {}).get("key") or "").strip()
]
sequence = next(
(item for item in sequences if str(item.get("key") or "").strip() == normalized_sequence_key),
{},
)
if not sequence:
return False, "标准作业路径不存在", {"sequence_key": normalized_sequence_key}
return _resolve_ops_runbook_sequence_entry(
sequence,
secondary=secondary,
payload=normalized_payload,
requested_by=requested_by,
)
def execute_ops_runbook_sequence(sequence_key: str, payload: dict | None = None) -> tuple[bool, str, dict]:
ok, message, resolved = resolve_ops_runbook_sequence(sequence_key, payload or {})
if not ok:
return ok, message, resolved
action_code = str(resolved.get("driver_action_code") or "").strip()
node_codes = _normalize_driver_node_codes(resolved.get("driver_node_codes") or [])
action_payload = dict(resolved.get("driver_action_payload") or {})
ok, message, data = execute_driver_action(
{
"action_code": action_code,
"node_codes": node_codes,
"action_payload": action_payload,
"requested_by": str(resolved.get("driver_requested_by") or "api/runbook").strip() or "api/runbook",
}
)
return ok, message, {
**dict(resolved or {}),
**dict(data or {}),
}
def get_ops_inspection_overview(
*,
managed_nodes: list[dict] | None = None,
fetch_limit: int = _OPS_INSPECTION_FETCH_LIMIT,
status: str = "",
problem_kind: str = "",
query: str = "",
only_problem: bool = False,
only_participating: bool = False,
) -> dict:
nodes = list(managed_nodes or [])
if managed_nodes is None:
managed_nodes_payload = list_managed_nodes_with_agent_state()
nodes = list(managed_nodes_payload.get("nodes") or [])
jobs = list_ops_jobs(limit=fetch_limit, compact=False)
inspection_jobs = [job for job in jobs if _inspection_job_bucket(job) and str(job.get("target_node_code") or "").strip()]
node_map = {
str(item.get("node_code") or "").strip(): item
for item in nodes
if str(item.get("node_code") or "").strip()
}
row_map: dict[str, dict] = {}
def ensure_row(node_code: str) -> dict:
if node_code not in row_map:
node = node_map.get(node_code) or {}
row_map[node_code] = {
"node_code": node_code,
"region": str(node.get("region") or ""),
"role": str(node.get("role") or ""),
"is_managed": bool(node.get("is_managed", False)),
"is_enabled": bool(node.get("is_enabled", False)),
"is_agent_online": bool(node.get("is_agent_online", False)),
"has_ssh_access": bool(node.get("has_ssh_access", False)),
"agent_state": str(node.get("agent_state") or ""),
"agent_state_label": str(node.get("agent_state_label") or ""),
"agent_state_reason": str(node.get("agent_state_reason") or ""),
"remote_access_state": str(node.get("remote_access_state") or ""),
"remote_access_label": str(node.get("remote_access_label") or ""),
"remote_access_reason": str(node.get("remote_access_reason") or ""),
"is_remote_access_ready": bool(node.get("is_remote_access_ready", False)),
"cluster_status": str(node.get("cluster_status") or ""),
"cluster_ip": str(node.get("cluster_ip") or ""),
"cluster_hostname": str(node.get("cluster_hostname") or ""),
"cluster_is_effective_worker": bool(node.get("cluster_is_effective_worker", False)),
"cluster_detect_participating": bool(node.get("cluster_detect_participating", False)),
"participation_state": str(node.get("participation_state") or ""),
"participation_label": str(node.get("participation_label") or ""),
"participation_reason": str(node.get("participation_reason") or ""),
"is_inspection_visible": _is_inspection_visible_node(node),
"is_inspection_eligible": _is_inspection_eligible_node(node),
"job_map": {},
"latest_job": {},
"last_updated_at": "",
"success_count": 0,
"overall_status": "missing",
"issue_summary": "",
"problem_kind": "",
"problem_label": "",
"problem_title": "",
"problem_level": "",
"problem_keys": [],
"recommended_action": "",
"recommended_action_code": "",
}
return row_map[node_code]
for job in inspection_jobs:
node_code = str(job.get("target_node_code") or "").strip()
bucket = _inspection_job_bucket(job)
if not node_code or not bucket:
continue
row = ensure_row(node_code)
if bucket not in row["job_map"]:
row["job_map"][bucket] = _slim_inspection_job(job)
candidate_time = str(job.get("updated_at") or job.get("finished_at") or job.get("started_at") or job.get("created_at") or "").strip()
current_latest_time = str(row.get("last_updated_at") or "").strip()
if not current_latest_time or candidate_time > current_latest_time:
row["last_updated_at"] = candidate_time
row["latest_job"] = _slim_inspection_job(job)
for node in nodes:
node_code = str(node.get("node_code") or "").strip()
if node_code and _is_inspection_visible_node(node):
ensure_row(node_code)
def derive_overall_status(row: dict, job_map: dict[str, dict]) -> str:
statuses = [str((job_map.get(key) or {}).get("status") or "").strip() for key in _OPS_INSPECTION_ACTION_KEYS]
present_statuses = [status for status in statuses if status]
if not present_statuses:
if bool(row.get("is_inspection_visible", False)):
if not bool(row.get("is_inspection_eligible", False)):
return "attention"
if bool(row.get("cluster_detect_participating", False)):
return "attention"
return "missing"
if any(status in {"failed", "blocked", "cancelled"} for status in present_statuses):
return "attention"
if any(status in {"queued", "dispatching", "running", "awaiting_approval"} for status in present_statuses):
return "running"
if all(str((job_map.get(key) or {}).get("status") or "").strip() == "success" for key in _OPS_INSPECTION_ACTION_KEYS):
return "healthy"
return "attention"
rows: list[dict] = []
for row in row_map.values():
job_map = dict(row.get("job_map") or {})
success_count = sum(1 for key in _OPS_INSPECTION_ACTION_KEYS if str((job_map.get(key) or {}).get("status") or "").strip() == "success")
issue_summary = _build_inspection_issue_summary(
job_map,
node_code=str(row.get("node_code") or ""),
latest_job=dict(row.get("latest_job") or {}),
node=row,
)
rows.append(
{
**row,
"job_map": job_map,
"success_count": success_count,
"overall_status": derive_overall_status(row, job_map),
"issue_summary": str(issue_summary.get("summary") or ""),
"problem_kind": str(issue_summary.get("problem_kind") or ""),
"problem_label": str(issue_summary.get("problem_label") or ""),
"problem_title": str(issue_summary.get("problem_title") or ""),
"problem_level": str(issue_summary.get("problem_level") or ""),
"problem_keys": list(issue_summary.get("problem_keys") or []),
"recommended_action": str(issue_summary.get("recommended_action") or ""),
"recommended_action_code": str(issue_summary.get("recommended_action_code") or ""),
"ui_intent": dict(issue_summary.get("ui_intent") or {}),
"latest_health_snapshot": _build_inspection_contract_result(
job_map.get("health.snapshot"),
action_label="健康快照",
),
"latest_worker_logs": _build_inspection_contract_result(
job_map.get("logs.collect"),
action_label="Worker 日志",
),
"latest_diagnostics": _build_inspection_contract_result(
job_map.get("diagnostics.collect"),
action_label="诊断包",
),
}
)
status_weight = {"attention": 0, "running": 1, "missing": 2, "healthy": 3}
# Stable-sort so the primary order is status bucket, then newest update, then node code.
rows.sort(key=lambda item: str(item.get("node_code") or ""))
rows.sort(key=lambda item: str(item.get("last_updated_at") or ""), reverse=True)
rows.sort(key=lambda item: int(status_weight.get(str(item.get("overall_status") or ""), 99)))
summary = {
"healthy": 0,
"running": 0,
"attention": 0,
"missing": 0,
}
for row in rows:
row_overall_status = str(row.get("overall_status") or "missing").strip()
if row_overall_status in summary:
summary[row_overall_status] += 1
priority_weight = {"attention": 0, "running": 1, "missing": 2, "healthy": 9}
normalized_status = str(status or "").strip().lower()
normalized_problem_kind = str(problem_kind or "").strip().lower()
normalized_query = str(query or "").strip().lower()
filtered_rows = list(rows)
if normalized_status:
filtered_rows = [
item for item in filtered_rows if str(item.get("overall_status") or "").strip().lower() == normalized_status
]
if normalized_problem_kind:
filtered_rows = [
item for item in filtered_rows if str(item.get("problem_kind") or "").strip().lower() == normalized_problem_kind
]
if only_problem:
filtered_rows = [item for item in filtered_rows if str(item.get("overall_status") or "").strip() != "healthy"]
if only_participating:
filtered_rows = [item for item in filtered_rows if bool(item.get("cluster_detect_participating", False))]
if normalized_query:
filtered_rows = [
item
for item in filtered_rows
if normalized_query in " ".join(
[
str(item.get("node_code") or ""),
str(item.get("region") or ""),
str(item.get("role") or ""),
str(item.get("overall_status") or ""),
str(item.get("problem_kind") or ""),
str(item.get("problem_label") or ""),
str(item.get("problem_title") or ""),
str(item.get("issue_summary") or ""),
str(item.get("recommended_action") or ""),
]
).lower()
]
filtered_summary = {
"healthy": 0,
"running": 0,
"attention": 0,
"missing": 0,
}
for row in filtered_rows:
row_status = str(row.get("overall_status") or "missing").strip()
if row_status in filtered_summary:
filtered_summary[row_status] += 1
if not filtered_rows:
overview_status = "empty"
overview_status_label = "无匹配结果"
overview_summary_text = "当前筛选条件下没有匹配节点。"
elif int(filtered_summary.get("attention", 0) or 0) > 0:
overview_status = "attention"
overview_status_label = "待处理"
overview_summary_text = f"存在 {int(filtered_summary.get('attention', 0) or 0)} 台节点需要优先处理。"
elif int(filtered_summary.get("running", 0) or 0) > 0:
overview_status = "running"
overview_status_label = "执行中"
overview_summary_text = f"存在 {int(filtered_summary.get('running', 0) or 0)} 台节点的标准巡检仍在执行。"
elif int(filtered_summary.get("missing", 0) or 0) > 0:
overview_status = "attention"
overview_status_label = "待补齐"
overview_summary_text = f"存在 {int(filtered_summary.get('missing', 0) or 0)} 台节点尚未形成完整巡检收口。"
else:
overview_status = "healthy"
overview_status_label = "健康"
overview_summary_text = "当前可见节点最近巡检收口正常。"
priority_queue = [
{
**row,
"priority_label": "P1" if row.get("overall_status") == "attention" else ("P2" if row.get("overall_status") == "running" else "P3"),
"priority_type": "danger" if row.get("overall_status") == "attention" else ("warning" if row.get("overall_status") == "running" else "info"),
"priority_weight": int(priority_weight.get(str(row.get("overall_status") or ""), 99)),
}
for row in filtered_rows
if str(row.get("overall_status") or "") != "healthy"
]
priority_queue.sort(key=lambda item: str(item.get("node_code") or ""))
priority_queue.sort(key=lambda item: str(item.get("last_updated_at") or ""), reverse=True)
priority_queue.sort(key=lambda item: int(item.get("priority_weight", 99)))
available_status_counts = {
"healthy": int(summary.get("healthy", 0) or 0),
"running": int(summary.get("running", 0) or 0),
"attention": int(summary.get("attention", 0) or 0),
"missing": int(summary.get("missing", 0) or 0),
}
available_problem_kind_counts: dict[str, int] = {}
for row in rows:
row_problem_kind = str(row.get("problem_kind") or "").strip()
if not row_problem_kind:
continue
available_problem_kind_counts[row_problem_kind] = int(available_problem_kind_counts.get(row_problem_kind, 0) or 0) + 1
return {
"status": overview_status,
"status_label": overview_status_label,
"summary_text": overview_summary_text,
"fetch_limit": int(fetch_limit),
"jobs_total": len(inspection_jobs),
"visible_nodes_total": sum(1 for item in nodes if _is_inspection_visible_node(item)),
"eligible_nodes_total": sum(1 for item in nodes if _is_inspection_eligible_node(item)),
"handover_gap_nodes_total": sum(
1 for item in nodes if _is_inspection_visible_node(item) and not _is_inspection_eligible_node(item)
),
"participating_nodes_total": sum(1 for item in rows if bool(item.get("cluster_detect_participating", False))),
"standby_nodes_total": sum(
1
for item in rows
if bool(item.get("is_inspection_visible", False)) and not bool(item.get("cluster_detect_participating", False))
),
"rows_total": len(rows),
"filtered_rows_total": len(filtered_rows),
"filters": {
"status": normalized_status,
"problem_kind": normalized_problem_kind,
"query": str(query or "").strip(),
"only_problem": bool(only_problem),
"only_participating": bool(only_participating),
},
"available_status_counts": available_status_counts,
"available_problem_kind_counts": available_problem_kind_counts,
"rows": filtered_rows,
"summary": filtered_summary,
"counts": {
"healthy_nodes": int(filtered_summary.get("healthy", 0) or 0),
"running_nodes": int(filtered_summary.get("running", 0) or 0),
"attention_nodes": int(filtered_summary.get("attention", 0) or 0),
"missing_nodes": int(filtered_summary.get("missing", 0) or 0),
},
"unfiltered_summary": summary,
"priority_queue": priority_queue,
}
def get_ops_overview() -> dict:
runtime = get_runtime_status()
readiness = runtime.get("readiness") or {}
detect = runtime.get("detect") or {}
cluster = runtime.get("cluster") or {}
sync_summary = get_sync_summary()
job_summary = get_ops_job_summary()
managed_nodes_payload = list_managed_nodes_with_agent_state(participation_payload=detect)
managed_nodes = list(managed_nodes_payload.get("nodes") or [])
managed_nodes_summary = managed_nodes_payload.get("summary") or {}
inspection_overview = get_ops_inspection_overview(managed_nodes=managed_nodes)
activity_stream = get_ops_activity_stream(limit=8)
release_summary = get_release_summary()
preferred_release = _preferred_release_for_ops()
release_launchpad = get_release_launchpad()
sync_batches = (sync_summary.get("detect_result_batches") or {}).get("state_counts") or {}
online_worker_nodes = int((cluster.get("summary") or {}).get("online_worker_nodes", 0) or 0)
online_control_nodes = int((cluster.get("summary") or {}).get("online_control_nodes", 0) or 0)
projected_batches = int(sync_batches.get("projected", 0) or 0)
failed_batches = int(sync_batches.get("failed", 0) or 0)
execution_scene = _build_ops_execution_scene(detect)
default_rollout_targets = _default_rollout_target_nodes(list(cluster.get("nodes") or []))
release_execution_mode = build_release_execution_mode_recommendation(preferred_release, default_rollout_targets)
release_gate = dict(release_execution_mode.get("recommended_gate") or {})
release_gate = {
**release_gate,
"default_target_nodes": list(release_gate.get("target_nodes") or []),
"default_target_node_codes": list(release_gate.get("target_node_codes") or []),
}
driver_recommendations = _build_driver_recommendations(
managed_nodes=managed_nodes,
execution_scene=execution_scene,
release_summary=release_summary,
release_gate=release_gate,
release_launchpad=release_launchpad,
)
recommendation = _build_ops_priority_recommendation(driver_recommendations)
return {
"mode": "centralized-ops",
"vision": "海外控制面统一接管大陆节点安装、更新、巡检、日志回传与故障诊断。",
"current_topology": {
"node_code": settings.node_code,
"node_region": settings.node_region,
"node_role": settings.node_role,
"worker_mode": settings.worker_mode,
"sync_push_enabled": bool(settings.sync_push_enabled),
"sync_target_api_base_url": str(settings.sync_target_api_base_url or "").strip(),
},
"cluster_summary": {
"online_control_nodes": online_control_nodes,
"online_worker_nodes": online_worker_nodes,
"dedicated_online_worker_nodes": int((cluster.get("summary") or {}).get("dedicated_online_worker_nodes", 0) or 0),
"busy_nodes": list((cluster.get("summary") or {}).get("busy_nodes") or []),
},
"cluster_nodes": [
{
"node_code": str(item.get("node_code") or ""),
"region": str(item.get("region") or ""),
"role": str(item.get("role") or ""),
"status": str(item.get("status") or ""),
"current_load": int(item.get("current_load", 0) or 0),
"hostname": str(item.get("hostname") or ""),
"last_heartbeat_at": str(item.get("last_heartbeat_at") or ""),
"is_effective_worker": bool(item.get("is_effective_worker", False)),
"detect_participating": bool(item.get("detect_participating", False)),
}
for item in list(cluster.get("nodes") or [])
if str(item.get("node_code") or "").strip()
],
"runtime_summary": {
"readiness_status": str(readiness.get("status") or ""),
"readiness_summary": str(readiness.get("summary") or ""),
"phase_label": str(detect.get("phase_label") or ""),
"phase_detail": str(detect.get("phase_detail") or ""),
"active_job_code": str((detect.get("active_job") or {}).get("job_code") or ""),
"progress_percent": float(detect.get("progress_percent", 0) or 0),
},
"execution_scene": execution_scene,
"sync_summary": {
"enabled": bool(sync_summary.get("enabled", False)),
"source_region": str(sync_summary.get("source_region") or ""),
"target_region": str(sync_summary.get("target_region") or ""),
"projected_batches": projected_batches,
"failed_batches": failed_batches,
"records_total": int(sync_summary.get("records_total", 0) or 0),
},
"ops_jobs": job_summary,
"managed_nodes": {
"total": int(managed_nodes_summary.get("total", len(managed_nodes)) or 0),
"managed_total": int(managed_nodes_summary.get("managed_total", 0) or 0),
"enabled": int(managed_nodes_summary.get("managed_enabled", 0) or 0),
"online": int(managed_nodes_summary.get("online", 0) or 0),
"stale": int(managed_nodes_summary.get("stale", 0) or 0),
"pending_bootstrap": int(managed_nodes_summary.get("pending_bootstrap", 0) or 0),
"runtime_only": int(managed_nodes_summary.get("runtime_only", 0) or 0),
"unmanaged": int(managed_nodes_summary.get("unmanaged", 0) or 0),
"token_issue": int(managed_nodes_summary.get("token_issue", 0) or 0),
"queue_retrying_nodes": int(managed_nodes_summary.get("queue_retrying_nodes", 0) or 0),
"queue_dead_letter_nodes": int(managed_nodes_summary.get("queue_dead_letter_nodes", 0) or 0),
"queue_pending_records": int(managed_nodes_summary.get("queue_pending_records", 0) or 0),
"queue_dead_letter_records": int(managed_nodes_summary.get("queue_dead_letter_records", 0) or 0),
"agent_state_counts": dict(managed_nodes_summary.get("status_counts") or {}),
"delivery_queue_state_counts": dict(managed_nodes_summary.get("delivery_queue_state_counts") or {}),
},
"inspection": inspection_overview,
"activity_stream": activity_stream,
"driver_recommendations": driver_recommendations,
"release_hub": {
**release_summary,
"preferred_release": preferred_release,
"default_rollout_execution": release_execution_mode,
"default_rollout_gate": release_gate,
"default_rollout_gate_options": dict(release_execution_mode.get("gates") or {}),
"launchpad": release_launchpad,
},
"recommendation": recommendation,
}
def _build_ops_link_snapshot_from_overview(overview: dict) -> dict:
overview = dict(overview or {})
topology = dict(overview.get("current_topology") or {})
cluster_summary = dict(overview.get("cluster_summary") or {})
runtime_summary = dict(overview.get("runtime_summary") or {})
execution_scene = dict(overview.get("execution_scene") or {})
participation_summary = dict(execution_scene.get("participation_summary") or {})
log_sync = dict(execution_scene.get("log_sync") or {})
sync_summary = dict(overview.get("sync_summary") or {})
recommendation = dict(overview.get("recommendation") or {})
readiness_status = str(runtime_summary.get("readiness_status") or "").strip() or "attention"
readiness_summary = str(runtime_summary.get("readiness_summary") or "").strip()
node_code = str(topology.get("node_code") or "").strip()
node_region = str(topology.get("node_region") or "").strip()
node_role = str(topology.get("node_role") or "").strip()
worker_mode = str(topology.get("worker_mode") or "").strip()
active_job_code = str(runtime_summary.get("active_job_code") or "").strip()
phase_label = str(runtime_summary.get("phase_label") or "").strip()
phase_detail = str(runtime_summary.get("phase_detail") or "").strip()
progress_percent = float(runtime_summary.get("progress_percent", 0) or 0)
online_control_nodes = int(cluster_summary.get("online_control_nodes", 0) or 0)
online_worker_nodes = int(cluster_summary.get("online_worker_nodes", 0) or 0)
dedicated_online_worker_nodes = int(cluster_summary.get("dedicated_online_worker_nodes", 0) or 0)
busy_nodes = list(cluster_summary.get("busy_nodes") or [])
effective_online_nodes = int(
participation_summary.get("effective_online_nodes", online_worker_nodes) or online_worker_nodes or 0
)
participating_nodes = int(participation_summary.get("participating_nodes", 0) or 0)
dispatch_active_nodes = int(participation_summary.get("dispatch_active_nodes", 0) or 0)
recent_only_nodes = int(participation_summary.get("recent_only_nodes", 0) or 0)
non_participating_nodes = int(participation_summary.get("non_participating_nodes", 0) or 0)
standby_nodes = int(participation_summary.get("standby_nodes", 0) or 0)
load_syncing_nodes = int(participation_summary.get("load_syncing_nodes", 0) or 0)
dispatch_active_node_codes = list(participation_summary.get("dispatch_active_node_codes") or [])
non_participating_node_codes = list(participation_summary.get("non_participating_node_codes") or [])
log_sync_enabled = bool(log_sync.get("enabled", False))
log_sync_mode = str(log_sync.get("mode") or "").strip() or "key"
log_sync_samples = int(log_sync.get("line_count", 0) or 0)
log_sync_sources = int(log_sync.get("source_node_count", 0) or 0)
missing_log_nodes = list(log_sync.get("missing_participating_nodes") or [])
covered_participating_nodes = int(log_sync.get("covered_participating_node_count", 0) or 0)
participating_node_count = int(log_sync.get("participating_node_count", participating_nodes) or participating_nodes or 0)
log_sync_last_at = str(log_sync.get("last_at") or "").strip()
log_sync_preview_lines = [str(item or "").strip() for item in list(log_sync.get("preview_lines") or []) if str(item or "").strip()]
sync_enabled = bool(sync_summary.get("enabled", False))
sync_source_region = str(sync_summary.get("source_region") or "").strip()
sync_target_region = str(sync_summary.get("target_region") or "").strip()
projected_batches = int(sync_summary.get("projected_batches", 0) or 0)
failed_batches = int(sync_summary.get("failed_batches", 0) or 0)
sync_records_total = int(sync_summary.get("records_total", 0) or 0)
if effective_online_nodes <= 0:
status = "blocking"
headline = "当前没有有效执行节点,海外控制面暂时无法继续推进联调、巡检或检测任务。"
elif dispatch_active_nodes > 0 and not log_sync_enabled:
status = "attention"
headline = "当前已有执行节点,但远端日志回传仍关闭,海外控制面对现场仍处于半盲态。"
elif dispatch_active_nodes > 0 and missing_log_nodes:
status = "attention"
headline = "当前已有执行节点,但远端日志回传仍未覆盖全部参与节点,建议先补齐现场观测。"
elif failed_batches > 0:
status = "attention"
headline = "当前同步链路存在失败批次,建议先收口跨地域同步状态,再继续放量联调。"
elif readiness_status != "ready":
status = readiness_status
headline = readiness_summary or "当前多机现场仍有待处理事项,建议先按运行中心提示逐项收口。"
elif active_job_code:
status = "ready"
headline = "当前多机现场已进入可观测状态,可以直接从海外控制面继续盯检测任务和节点执行过程。"
else:
status = "ready"
headline = "当前多机骨架已就绪,可以继续拉取任务、做巡检验收或推进发布演练。"
summary_lines = [
f"控制面节点 {node_code or '-'} / 区域 {node_region or '-'} / 角色 {node_role or '-'} / 托管方式 {worker_mode or '-'}",
f"集群在线:控制面 {online_control_nodes} 台 / 有效执行节点 {effective_online_nodes} 台 / 独立 Worker {dedicated_online_worker_nodes} 台 / 忙碌节点 {_format_node_code_list(busy_nodes)}",
(
f"活跃任务 {active_job_code} / 阶段 {phase_label or '-'} / 进度 {progress_percent:.1f}%"
if active_job_code
else f"当前无活跃任务 / 就绪度 {readiness_status} / 摘要 {readiness_summary or '-'}"
),
(
f"执行现场:参与 {participating_nodes} 台 / 执行中或已领 {dispatch_active_nodes} 台 / 近窗吞吐 {recent_only_nodes} 台 / 在线未参与 {non_participating_nodes}"
),
(
f"远端日志:{'开启' if log_sync_enabled else '关闭'} / 模式 {log_sync_mode} / 覆盖 {covered_participating_nodes}/{participating_node_count} / 样本 {log_sync_samples} / 缺口 {_format_node_code_list(missing_log_nodes)}"
),
(
f"跨地域同步:{'启用' if sync_enabled else '未启用'} / {sync_source_region or '-'} -> {sync_target_region or '-'} / projected {projected_batches} / failed {failed_batches} / records {sync_records_total}"
),
]
if phase_detail:
summary_lines.append(f"现场阶段说明:{phase_detail}")
if log_sync_last_at:
summary_lines.append(f"最近日志回传:{log_sync_last_at}")
next_actions: list[dict] = []
if dispatch_active_nodes > 0 and not log_sync_enabled:
target_node_codes = list(dispatch_active_node_codes or [])
next_actions = [
{
"label": "开启关键回传",
"action_code": "enable_log_sync_key",
"node_codes": target_node_codes,
"payload": {},
},
{
"label": "开启全量回传",
"action_code": "enable_log_sync_full",
"node_codes": target_node_codes,
"payload": {},
},
]
elif dispatch_active_nodes > 0 and missing_log_nodes:
target_node_codes = list(dispatch_active_node_codes or missing_log_nodes or [])
next_actions = [
{
"label": "看参与节点日志",
"action_code": "open_worker_logs_participating",
"node_codes": target_node_codes,
"payload": {},
},
{
"label": "执行标准巡检",
"action_code": "run_inspection_participating",
"node_codes": target_node_codes,
"payload": {},
},
]
else:
primary_label = str(recommendation.get("primary_label") or "").strip()
primary_action_code = str(recommendation.get("primary_action_code") or "").strip()
if primary_label or primary_action_code:
next_actions.append(
{
"label": primary_label or "执行建议动作",
"action_code": primary_action_code,
"node_codes": list(recommendation.get("primary_node_codes") or recommendation.get("node_codes") or []),
"payload": dict(recommendation.get("primary_action_payload") or {}),
}
)
secondary_label = str(recommendation.get("secondary_label") or "").strip()
secondary_action_code = str(recommendation.get("secondary_action_code") or "").strip()
if secondary_label or secondary_action_code:
next_actions.append(
{
"label": secondary_label or "执行次要动作",
"action_code": secondary_action_code,
"node_codes": list(recommendation.get("secondary_node_codes") or recommendation.get("node_codes") or []),
"payload": dict(recommendation.get("secondary_action_payload") or {}),
}
)
return {
"generated_at": datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
"status": status,
"headline": headline,
"summary_lines": summary_lines,
"control_plane": {
"node_code": node_code,
"region": node_region,
"role": node_role,
"worker_mode": worker_mode,
"sync_push_enabled": bool(topology.get("sync_push_enabled", False)),
"sync_target_api_base_url": str(topology.get("sync_target_api_base_url") or "").strip(),
},
"cluster": {
"online_control_nodes": online_control_nodes,
"online_worker_nodes": online_worker_nodes,
"dedicated_online_worker_nodes": dedicated_online_worker_nodes,
"effective_online_nodes": effective_online_nodes,
"busy_nodes": list(busy_nodes),
},
"execution": {
"active_job_code": active_job_code,
"phase_label": phase_label,
"phase_detail": phase_detail,
"progress_percent": progress_percent,
"participating_nodes": participating_nodes,
"dispatch_active_nodes": dispatch_active_nodes,
"recent_only_nodes": recent_only_nodes,
"non_participating_nodes": non_participating_nodes,
"standby_nodes": standby_nodes,
"load_syncing_nodes": load_syncing_nodes,
"dispatch_active_node_codes": list(dispatch_active_node_codes),
"non_participating_node_codes": list(non_participating_node_codes),
},
"log_sync": {
"enabled": log_sync_enabled,
"mode": log_sync_mode,
"line_count": log_sync_samples,
"source_node_count": log_sync_sources,
"covered_participating_node_count": covered_participating_nodes,
"participating_node_count": participating_node_count,
"missing_participating_nodes": list(missing_log_nodes),
"last_at": log_sync_last_at,
"preview_lines": log_sync_preview_lines,
},
"sync": {
"enabled": sync_enabled,
"source_region": sync_source_region,
"target_region": sync_target_region,
"projected_batches": projected_batches,
"failed_batches": failed_batches,
"records_total": sync_records_total,
},
"recommendation": {
"key": str(recommendation.get("key") or "").strip(),
"priority": str(recommendation.get("priority") or "").strip(),
"summary": str(recommendation.get("summary") or "").strip(),
"reason": str(recommendation.get("reason") or "").strip(),
},
"next_actions": next_actions,
}
def get_ops_link_snapshot(overview: dict | None = None) -> dict:
normalized_overview = dict(overview or {})
if not normalized_overview:
normalized_overview = get_ops_overview()
return _build_ops_link_snapshot_from_overview(normalized_overview)
def _normalize_ops_node_code_list(*groups: object) -> list[str]:
normalized: list[str] = []
seen: set[str] = set()
for group in groups:
if isinstance(group, dict):
group = [group]
if not isinstance(group, (list, tuple, set)):
continue
for item in group:
if isinstance(item, dict):
node_code = str(item.get("node_code") or "").strip()
else:
node_code = str(item or "").strip()
if not node_code or node_code in seen:
continue
seen.add(node_code)
normalized.append(node_code)
return normalized
def _build_scene_node_log_command(base_url: str, node_code: str, *, limit: int = 80, mode: str = "key") -> str:
normalized_node_code = str(node_code or "").strip()
if not normalized_node_code:
return ""
normalized_base_url = _ops_stack_api_base_url(base_url)
normalized_mode = _normalize_scene_log_mode(mode, fallback="key")
normalized_limit = max(20, min(int(limit or 80), 300))
return build_bash_command(
"drive_ops_center.sh",
"scene-node-log",
normalized_base_url,
normalized_node_code,
normalized_limit,
normalized_mode,
)
def _build_scene_node_log_focus_ref(
node_code: str,
*,
mode: str = "key",
limit: int = 80,
source: str = "stack_diagnosis",
) -> dict:
normalized_node_code = str(node_code or "").strip()
if not normalized_node_code:
return {}
return _merge_focus_ref(
{},
kind="node_scene_log",
node_code=normalized_node_code,
mode=_normalize_scene_log_mode(mode, fallback="key"),
limit=max(20, min(int(limit or 80), 300)),
source=source,
)
def _build_scene_node_log_commands(
base_url: str,
node_codes: list[str] | tuple[str, ...] | set[str],
*,
limit: int = 80,
mode: str = "key",
max_commands: int = 3,
) -> list[str]:
commands: list[str] = []
for node_code in _normalize_ops_node_code_list(list(node_codes or []))[: max(1, int(max_commands or 3))]:
command = _build_scene_node_log_command(base_url, node_code, limit=limit, mode=mode)
if command:
commands.append(command)
return commands
def _ops_stack_api_base_url(base_url: str = "") -> str:
normalized_base_url = str(base_url or "").strip()
if normalized_base_url:
return normalized_base_url.rstrip("/")
return f"http://127.0.0.1:{settings.api_port}"
def _ops_stack_capture(section_key: str, callback) -> dict:
started_at = time.perf_counter()
try:
payload = callback() or {}
except Exception as exc:
elapsed_ms = int((time.perf_counter() - started_at) * 1000)
return {
"section_key": str(section_key or "").strip(),
"http_status": "500",
"available": False,
"data": {},
"error": f"{exc.__class__.__name__}: {exc}",
"elapsed_ms": elapsed_ms,
}
if isinstance(payload, dict):
data = dict(payload or {})
else:
data = {"value": payload}
elapsed_ms = int((time.perf_counter() - started_at) * 1000)
return {
"section_key": str(section_key or "").strip(),
"http_status": "200",
"available": True,
"data": data,
"error": "",
"elapsed_ms": elapsed_ms,
}
def _ops_stack_issue_rank(severity: str) -> int:
return {"blocked": 3, "warning": 2, "info": 1}.get(str(severity or "").strip(), 0)
def _suggest_ops_contract_keys_for_stack_issue(issue: dict) -> list[str]:
normalized_issue = dict(issue or {})
layer = str(normalized_issue.get("layer") or "").strip()
contract_keys: list[str] = ["ops_stack_diagnosis_contract"]
if layer.startswith("managed_nodes"):
contract_keys.append("ops_agent_protocol")
elif layer == "release_hub":
contract_keys.append("release_hub_contract")
elif layer == "playbook_runs":
contract_keys.append("ops_playbook_contract")
elif layer in {"overview.log_sync", "activity_stream", "link_snapshot"}:
contract_keys.append("ops_observability_contract")
elif layer == "contracts":
contract_keys.append("ops_driver_contract")
contract_keys.extend(_ops_contract_keys_from_focus_ref(normalized_issue.get("focus_ref")))
contract_keys.extend(_ops_contract_keys_from_action_code(str(normalized_issue.get("action_code") or "").strip()))
return _normalize_ops_contract_keys(contract_keys)
def _build_stack_diagnosis_recommended_commands(
*,
stack_base_url: str,
recommended_actions: list[dict],
first_handover_gap_node_code: str = "",
launchpad_status: dict | None = None,
default_rollout_gate: dict | None = None,
next_step: dict | None = None,
) -> dict:
commands: dict[str, str] = {}
commands["stack-diagnosis"] = build_bash_command("drive_ops_center.sh", "stack-diagnosis", stack_base_url, "summary")
commands["stack-check"] = commands["stack-diagnosis"]
commands["doctor"] = build_bash_command("drive_ops_center.sh", "doctor", stack_base_url)
commands["contracts"] = build_bash_command("drive_ops_center.sh", "contracts", stack_base_url)
commands["runtime-refresh-recover"] = build_bash_command(
"drive_ops_center.sh",
"runtime-refresh-recover",
stack_base_url,
)
commands["api-restart"] = commands["runtime-refresh-recover"]
for action in list(recommended_actions or []):
action_code = str(action.get("action_code") or "").strip()
command = str(action.get("command") or "").strip()
if action_code and command and action_code not in commands:
commands[action_code] = command
normalized_gap_node_code = str(first_handover_gap_node_code or "").strip()
if normalized_gap_node_code:
commands["managed-node-handover"] = build_bash_command("drive_ops_center.sh", "node-handover", stack_base_url, normalized_gap_node_code)
commands["managed-node-recover"] = build_bash_command("drive_ops_center.sh", "node-recover", stack_base_url, normalized_gap_node_code)
commands["managed-node-bootstrap-plan"] = build_bash_command("drive_ops_center.sh", "node-bootstrap-plan", stack_base_url, normalized_gap_node_code)
normalized_next_step = dict(next_step or {})
next_step_focus_ref = dict(normalized_next_step.get("focus_ref") or {})
if str(next_step_focus_ref.get("kind") or "").strip() == "node_scene_log":
node_code = str(next_step_focus_ref.get("node_code") or "").strip()
if node_code:
commands["scene-node-log"] = _build_scene_node_log_command(
stack_base_url,
node_code,
limit=int(next_step_focus_ref.get("limit", 120) or 120),
mode=str(next_step_focus_ref.get("mode") or "key"),
)
normalized_launchpad_status = dict(launchpad_status or {})
normalized_default_rollout_gate = dict(default_rollout_gate or {})
launchpad_focus_ref = dict(
normalized_launchpad_status.get("focus_ref")
or normalized_default_rollout_gate.get("focus_ref")
or {}
)
if str(launchpad_focus_ref.get("section") or "").strip() == "release_launchpad":
commands["focus-release-hub"] = build_bash_command("drive_ops_center.sh", "release-launchpad", stack_base_url)
mode = str(launchpad_focus_ref.get("mode") or "").strip()
if mode == "control":
commands["release-launchpad-control"] = build_bash_command(
"drive_ops_center.sh",
"release-launchpad",
stack_base_url,
"control",
)
else:
commands["release-launchpad-worker"] = build_bash_command(
"drive_ops_center.sh",
"release-launchpad",
stack_base_url,
"worker",
)
launchpad_recommended_action_code = str(normalized_launchpad_status.get("recommended_action_code") or "").strip()
launchpad_recommended_target_node_code = str(
normalized_launchpad_status.get("recommended_target_node_code") or ""
).strip()
if (
launchpad_recommended_action_code in {"bootstrap_run", "run_acceptance"}
and launchpad_recommended_target_node_code
):
commands[launchpad_recommended_action_code] = build_bash_command(
"drive_ops_center.sh",
"node-bootstrap-run" if launchpad_recommended_action_code == "bootstrap_run" else "node-acceptance-run",
stack_base_url,
launchpad_recommended_target_node_code,
"cli",
)
return {key: value for key, value in commands.items() if str(value or "").strip()}
def _resolve_first_handover_gap_recovery(
*,
first_handover_gap: dict | None = None,
launchpad_gap_action: dict | None = None,
launchpad_status: dict | None = None,
fallback_node_code: str = "",
) -> dict:
normalized_handover = dict(first_handover_gap or {})
stage = dict(normalized_handover.get("stage") or {})
stage_next_step = dict(stage.get("next_step") or {})
normalized_launchpad_gap_action = dict(launchpad_gap_action or {})
normalized_launchpad_status = dict(launchpad_status or {})
node_code = str(
normalized_handover.get("node_code")
or normalized_launchpad_gap_action.get("node_code")
or normalized_launchpad_status.get("recommended_target_node_code")
or fallback_node_code
or ""
).strip()
summary = str(
normalized_handover.get("summary")
or stage.get("summary")
or normalized_launchpad_gap_action.get("summary")
or normalized_launchpad_status.get("recommended_recovery_summary")
or ""
).strip()
stage_code = str(stage.get("code") or "").strip()
stage_next_step_code = str(stage_next_step.get("code") or "").strip()
has_gap_signal = bool(node_code or stage_code or stage_next_step_code or summary)
action_code = str(normalized_launchpad_gap_action.get("action_code") or "").strip()
if not has_gap_signal:
action_code = ""
elif action_code not in {"bootstrap_run", "run_acceptance"}:
if stage_next_step_code in {"run_acceptance", "acceptance", "ready"} or stage_code in {"acceptance_ready"}:
action_code = "run_acceptance"
elif stage_next_step_code in {"execute_bootstrap_plan", "bootstrap_run"} or stage_code in {
"pending_bootstrap",
"runtime_only",
"profile_incomplete",
"ssh_ready",
"token_issue",
"stale",
"agent_pending",
"unknown",
"unmanaged",
}:
action_code = "bootstrap_run"
else:
action_code = ""
recovery_label = str(normalized_launchpad_status.get("recommended_recovery_label") or "").strip()
if not recovery_label and action_code == "bootstrap_run":
recovery_label = "签发接入工单"
elif not recovery_label and action_code == "run_acceptance":
recovery_label = "执行接管验收"
if not recovery_label:
recovery_label = str(stage_next_step.get("label") or stage.get("label") or "").strip()
focus_ref = dict(
(dict(normalized_launchpad_gap_action.get("row") or {}).get("focus_ref") or {})
or normalized_launchpad_status.get("focus_ref")
or {}
)
if node_code and not str(focus_ref.get("node_code") or "").strip():
focus_ref["node_code"] = node_code
if node_code and not str(focus_ref.get("kind") or "").strip():
focus_ref["kind"] = "managed_node"
return {
"action_code": action_code if action_code in {"bootstrap_run", "run_acceptance"} else "",
"node_code": node_code,
"summary": summary,
"recovery_label": recovery_label,
"focus_ref": focus_ref,
"stage_code": stage_code,
"stage_label": str(stage.get("label") or "").strip(),
"stage_next_step_code": stage_next_step_code,
}
def _build_stack_diagnosis_operator_decision(
*,
issues: list[dict],
next_step: dict,
recommended_actions: list[dict],
recommended_commands: dict,
first_handover_gap_node_code: str = "",
launchpad_status: dict | None = None,
default_rollout_gate: dict | None = None,
) -> tuple[dict, dict]:
top_issue = (
sorted(
list(issues or []),
key=lambda item: (
-_ops_stack_issue_rank(str(item.get("severity") or "").strip()),
str(item.get("code") or "").strip(),
),
)[0]
if issues
else {}
)
normalized_next_step = dict(next_step or {})
primary_action_code = str(normalized_next_step.get("action_code") or "").strip()
next_focus_ref = dict(normalized_next_step.get("focus_ref") or {})
normalized_launchpad_status = dict(launchpad_status or {})
normalized_default_rollout_gate = dict(default_rollout_gate or {})
operator_decision = {
"lane": "steady",
"priority": "low",
"reason_code": "steady_state",
"title": "当前总检已进入稳定观察态",
"summary": "控制面总检、执行现场、纳管与发布骨架当前没有阻断项,可以按既定节奏继续联调或推进发布。",
"next_focus": next_focus_ref,
"primary_command_key": primary_action_code,
"secondary_command_key": "",
}
if str(next_focus_ref.get("kind") or "").strip() == "node_scene_log":
operator_decision = {
"lane": "observability",
"priority": "high" if top_issue else "medium",
"reason_code": str(top_issue.get("code") or "scene_log_observation").strip(),
"title": "优先补齐执行现场日志观察",
"summary": str(top_issue.get("summary") or normalized_next_step.get("reason") or "").strip()
or "当前已经定位到参与节点现场日志焦点,建议先看现场,再决定是否继续接管或推进发布。",
"next_focus": next_focus_ref,
"primary_command_key": "scene-node-log",
"secondary_command_key": primary_action_code or "open_worker_logs_participating",
}
elif str(top_issue.get("code") or "").strip() == "runtime_build_schema_stale":
secondary_key = "stack-diagnosis" if "stack-diagnosis" in recommended_commands else ""
if not secondary_key and "stack-check" in recommended_commands:
secondary_key = "stack-check"
if "managed-node-bootstrap-plan" in recommended_commands:
secondary_key = "managed-node-bootstrap-plan"
operator_decision = {
"lane": "runtime_recovery",
"priority": "high",
"reason_code": "runtime_build_schema_stale",
"title": "优先重启控制面 API 并复检运行时版本",
"summary": str(top_issue.get("summary") or normalized_next_step.get("reason") or "").strip()
or "仓库代码已经具备新能力,但运行中的 API 仍停留在旧 schema当前第一动作应先让服务吃到最新代码。",
"next_focus": dict(top_issue.get("focus_ref") or normalized_next_step.get("focus_ref") or {}),
"primary_command_key": "api-restart",
"secondary_command_key": secondary_key,
}
elif str(top_issue.get("code") or "").strip() == "managed_nodes_agent_pending" or (
str(top_issue.get("layer") or "").strip().startswith("managed_nodes")
and str(first_handover_gap_node_code or "").strip()
):
managed_primary_key = primary_action_code
if managed_primary_key not in {"bootstrap_run", "run_acceptance"}:
managed_primary_key = (
"managed-node-recover" if "managed-node-recover" in recommended_commands else "managed-node-handover"
)
managed_secondary_key = ""
if managed_primary_key in {"bootstrap_run", "run_acceptance"}:
if "managed-node-bootstrap-plan" in recommended_commands:
managed_secondary_key = "managed-node-bootstrap-plan"
elif "managed-node-handover" in recommended_commands:
managed_secondary_key = "managed-node-handover"
elif "fix_managed_nodes" in recommended_commands:
managed_secondary_key = "fix_managed_nodes"
elif primary_action_code and primary_action_code != managed_primary_key:
managed_secondary_key = primary_action_code
operator_decision = {
"lane": "node_handover",
"priority": "high",
"reason_code": str(top_issue.get("code") or "managed_nodes_agent_pending").strip(),
"title": "优先执行节点接管收口" if managed_primary_key in {"bootstrap_run", "run_acceptance"} else "优先打通节点接管链路",
"summary": (
str(normalized_next_step.get("reason") or "").strip()
if managed_primary_key in {"bootstrap_run", "run_acceptance"}
else ""
)
or str(top_issue.get("summary") or "").strip()
or "托管节点还没有进入 remote-agent ready海外控制面暂时不能把执行统一收口到标准动作链。",
"next_focus": dict(top_issue.get("focus_ref") or normalized_next_step.get("focus_ref") or {}),
"primary_command_key": managed_primary_key,
"secondary_command_key": managed_secondary_key,
}
elif (
str(top_issue.get("code") or "").strip() == "release_rollout_gate_blocked"
or str(normalized_default_rollout_gate.get("status") or "").strip() == "blocked"
or str(normalized_launchpad_status.get("recommended_action_code") or "").strip() in {
"review_smart_rollout_preview",
"review_control_rollout",
"fix_rollout_blockers",
}
):
launchpad_focus_ref = dict(
normalized_default_rollout_gate.get("focus_ref")
or normalized_launchpad_status.get("focus_ref")
or normalized_next_step.get("focus_ref")
or {}
)
launchpad_mode = str(launchpad_focus_ref.get("mode") or "").strip()
primary_key = "release-launchpad-control" if launchpad_mode == "control" else "release-launchpad-worker"
if primary_key not in recommended_commands:
primary_key = "focus-release-hub"
operator_decision = {
"lane": "release",
"priority": "high" if str(normalized_default_rollout_gate.get("status") or "").strip() == "blocked" else "medium",
"reason_code": str(top_issue.get("code") or "release_launchpad_attention").strip(),
"title": "优先处理发布门禁与 Launchpad 决策",
"summary": str(top_issue.get("summary") or normalized_default_rollout_gate.get("summary_text") or "").strip()
or "发布包和 Rollout 入口已经就位,但默认门禁还没完全放行,建议先看 Release Hub 再推进下一批。",
"next_focus": launchpad_focus_ref,
"primary_command_key": primary_key,
"secondary_command_key": primary_action_code or str(normalized_launchpad_status.get("recommended_action_code") or "").strip(),
}
elif str(normalized_launchpad_status.get("recommended_action_code") or "").strip() in {"bootstrap_run", "run_acceptance"}:
launchpad_action_code = str(normalized_launchpad_status.get("recommended_action_code") or "").strip()
launchpad_target_node_code = str(normalized_launchpad_status.get("recommended_target_node_code") or "").strip()
launchpad_focus_ref = dict(
normalized_launchpad_status.get("focus_ref")
or normalized_default_rollout_gate.get("focus_ref")
or normalized_next_step.get("focus_ref")
or {}
)
operator_decision = {
"lane": "node_handover",
"priority": "high" if launchpad_action_code == "bootstrap_run" else "medium",
"reason_code": f"release_launchpad_{launchpad_action_code}",
"title": "优先执行节点接入收口" if launchpad_action_code == "bootstrap_run" else "优先执行节点接管验收",
"summary": str(normalized_launchpad_status.get("recommended_recovery_summary") or "").strip()
or str(top_issue.get("summary") or normalized_next_step.get("reason") or "").strip()
or "Release Launchpad 已经明确给出节点接入缺口,建议先完成该节点的接入收口,再回到发布门禁。",
"next_focus": {
**launchpad_focus_ref,
**({"node_code": launchpad_target_node_code} if launchpad_target_node_code else {}),
},
"primary_command_key": launchpad_action_code,
"secondary_command_key": "focus-release-hub" if "focus-release-hub" in recommended_commands else "",
}
elif primary_action_code:
operator_decision = {
"lane": "ops_jobs" if primary_action_code.startswith("focus_") else "steady",
"priority": "medium" if top_issue else "low",
"reason_code": str(top_issue.get("code") or primary_action_code).strip(),
"title": "按总检默认下一步继续处理",
"summary": str(normalized_next_step.get("reason") or "").strip()
or "当前总检已经给出默认下一步,可直接按统一 driver action 继续推进。",
"next_focus": next_focus_ref,
"primary_command_key": primary_action_code,
"secondary_command_key": "",
}
next_actions = {
"primary_command_key": str(operator_decision.get("primary_command_key") or "").strip(),
"primary_command": str(
recommended_commands.get(str(operator_decision.get("primary_command_key") or "").strip()) or ""
).strip(),
"secondary_command_key": str(operator_decision.get("secondary_command_key") or "").strip(),
"secondary_command": str(
recommended_commands.get(str(operator_decision.get("secondary_command_key") or "").strip()) or ""
).strip(),
}
return operator_decision, next_actions
def get_ops_stack_diagnosis(*, base_url: str = "") -> dict:
stack_base_url = _ops_stack_api_base_url(base_url)
api_info = {
"service": "domain-api",
"version": "0.1.0",
"api_prefix": settings.api_prefix,
"worker_mode": settings.worker_mode,
}
contracts_section = _ops_stack_capture("contracts", get_ops_contract_registry)
overview_section = _ops_stack_capture("overview", get_ops_overview)
overview_data = dict(overview_section.get("data") or {})
if bool(overview_section.get("available", False)) and overview_data:
link_snapshot_section = _ops_stack_capture(
"link_snapshot",
lambda: get_ops_link_snapshot(overview=overview_data),
)
else:
link_snapshot_section = _ops_stack_capture("link_snapshot", get_ops_link_snapshot)
nodes_section = _ops_stack_capture("managed_nodes", list_managed_nodes_with_agent_state)
def _stack_release_hub_payload() -> dict:
overview_launchpad = dict(((overview_data.get("release_hub") or {}).get("launchpad") or {}))
if overview_launchpad:
return overview_launchpad
return get_release_launchpad()
release_hub_section = _ops_stack_capture("release_hub", _stack_release_hub_payload)
build_info_section = _ops_stack_capture("runtime_build_info", get_runtime_build_info)
playbook_runs_section = _ops_stack_capture(
"playbook_runs",
lambda: get_recent_ops_playbook_runs(limit=6, scan_limit=240),
)
def _stack_activity_stream_payload() -> dict:
overview_activity_stream = dict(overview_data.get("activity_stream") or {})
if overview_activity_stream:
return overview_activity_stream
return get_ops_activity_stream(limit=8, scan_limit=80)
activity_stream_section = _ops_stack_capture("activity_stream", _stack_activity_stream_payload)
contracts = dict(contracts_section.get("data") or {})
contract_registry = contracts if contracts else get_ops_contract_registry()
link_snapshot = dict(link_snapshot_section.get("data") or {})
overview = overview_data
nodes_payload = dict(nodes_section.get("data") or {})
launchpad = dict(release_hub_section.get("data") or {})
build_info = dict(build_info_section.get("data") or {})
playbook_runs_payload = dict(playbook_runs_section.get("data") or {})
activity_payload = dict(activity_stream_section.get("data") or {})
contracts_rows = list(contracts.get("contracts") or [])
nodes = list(nodes_payload.get("nodes") or [])
nodes_summary = dict(nodes_payload.get("summary") or {})
playbook_runs = list(playbook_runs_payload.get("runs") or playbook_runs_payload.get("playbook_runs") or [])
activities = list(activity_payload.get("items") or activity_payload.get("activities") or [])
recommendation = dict(overview.get("recommendation") or {})
execution_scene = dict(overview.get("execution_scene") or {})
log_sync = dict(execution_scene.get("log_sync") or overview.get("log_sync") or {})
release_hub = dict(overview.get("release_hub") or {})
default_rollout_gate = dict(release_hub.get("default_rollout_gate") or {})
launchpad_status = dict(launchpad.get("launchpad_status") or {})
launchpad_gap_action = _release_launchpad_gap_action_context(launchpad)
latest_release = dict(launchpad.get("latest_release") or release_hub.get("latest_release") or {})
route_surface = dict(build_info.get("route_surface") or {})
repository_capabilities = dict(build_info.get("repository_capabilities") or {})
route_surface_missing_keys = [
str(item).strip()
for item in list(route_surface.get("missing_keys") or [])
if str(item).strip()
]
route_surface_expected_paths = {
str(key).strip(): str(value).strip()
for key, value in dict(route_surface.get("expected_paths") or {}).items()
if str(key).strip()
}
repo_supports_install_command_block = bool(repository_capabilities.get("supports_install_command_block", False))
repo_supports_multi_layout_bootstrap = bool(repository_capabilities.get("supports_multi_layout_bootstrap", False))
route_surface_declares_bootstrap_plan = "ops_node_handover_bootstrap_plan" in route_surface_expected_paths
runtime_schema_stale = bool(repo_supports_install_command_block and not route_surface_declares_bootstrap_plan)
problem_nodes: list[dict] = []
for node in nodes:
node_code = str(node.get("node_code") or "").strip()
if not node_code:
continue
queue_state = str(node.get("delivery_queue_state") or "").strip()
agent_state = str(node.get("agent_state") or "").strip()
remote_access_state = str(node.get("remote_access_state") or "").strip()
if (
queue_state == "dead_letter"
or agent_state not in {"online", "online_busy"}
or remote_access_state in {"agent_pending", "disabled", "unmanaged"}
):
problem_nodes.append(
{
"node_code": node_code,
"agent_state": agent_state,
"remote_access_state": remote_access_state,
"delivery_queue_state": queue_state,
"participation_state": str(node.get("participation_state") or "").strip(),
}
)
first_handover_gap_node = next(
(
dict(node or {})
for node in nodes
if str(node.get("node_code") or "").strip()
and (
str(node.get("agent_state") or "").strip() not in {"online", "online_busy"}
or str(node.get("remote_access_state") or "").strip() in {"agent_pending", "disabled", "unmanaged"}
)
),
{},
)
first_handover_gap_node_code = str(first_handover_gap_node.get("node_code") or "").strip()
first_handover_gap = (
get_managed_node_handover(
first_handover_gap_node_code,
control_plane_base_url=stack_base_url,
nodes_payload=nodes_payload,
)
if first_handover_gap_node_code
else {}
)
first_handover_gap_recovery = _resolve_first_handover_gap_recovery(
first_handover_gap=first_handover_gap,
launchpad_gap_action=launchpad_gap_action,
launchpad_status=launchpad_status,
fallback_node_code=first_handover_gap_node_code,
)
problem_runs = [
{
"run_code": str(item.get("run_code") or "").strip(),
"status": str(item.get("status") or "").strip(),
"status_label": str(item.get("status_label") or "").strip(),
"focus_step_key": str(item.get("focus_step_key") or "").strip(),
"focus_summary": str(item.get("focus_summary") or item.get("summary_text") or "").strip(),
"focus_ref": dict(item.get("focus_ref") or {}),
}
for item in playbook_runs
if str(item.get("status") or "").strip() not in {"success", "completed", "healthy"}
]
activity_counts: dict[str, int] = {}
for item in activities:
status_key = str(item.get("status") or item.get("job_status") or "unknown").strip() or "unknown"
activity_counts[status_key] = int(activity_counts.get(status_key, 0) or 0) + 1
start_delivery_failed_items = [
dict(item or {})
for item in activities
if str(item.get("start_delivery_state") or "").strip() == "failed_local"
]
issues: list[dict] = []
def add_issue(
*,
code: str,
severity: str,
layer: str,
summary: str,
detail: str = "",
action_code: str = "",
focus_ref: dict | None = None,
commands: list[str] | None = None,
) -> None:
contract_keys = _suggest_ops_contract_keys_for_stack_issue(
{
"layer": layer,
"focus_ref": focus_ref or {},
"action_code": action_code,
}
)
issues.append(
{
"code": str(code or "").strip(),
"severity": str(severity or "").strip() or "warning",
"layer": str(layer or "").strip(),
"summary": str(summary or "").strip(),
"detail": str(detail or "").strip(),
"action_code": str(action_code or "").strip(),
"focus_ref": dict(focus_ref or {}),
"commands": [str(item).strip() for item in list(commands or []) if str(item).strip()],
"contract_keys": contract_keys,
"contract_navigation": _build_ops_contract_navigation(
contract_keys,
primary_contract_key="ops_stack_diagnosis_contract",
registry=contract_registry,
),
}
)
surface_matrix = [
("api", True),
("contracts", bool(contracts_section.get("available", False))),
("link_snapshot", bool(link_snapshot_section.get("available", False))),
("overview", bool(overview_section.get("available", False))),
("managed_nodes", bool(nodes_section.get("available", False))),
("release_hub", bool(release_hub_section.get("available", False))),
("runtime_build_info", bool(build_info_section.get("available", False))),
("playbook_runs", bool(playbook_runs_section.get("available", False))),
("activity_stream", bool(activity_stream_section.get("available", False))),
]
surface_status_counts = {
"total": len(surface_matrix),
"available": sum(1 for _, available in surface_matrix if available),
"missing": sum(1 for _, available in surface_matrix if not available),
}
missing_surfaces = [name for name, available in surface_matrix if not available]
if not bool(contracts_section.get("available", False)):
add_issue(
code="ops_contracts_unavailable",
severity="warning",
layer="contracts",
summary="控制面契约注册表当前未能正常生成,总检无法确认当前运行的是哪一版正式 contract。",
detail=str(contracts_section.get("error") or "ops contract registry unavailable").strip(),
commands=[
build_bash_command("drive_ops_center.sh", "contracts", stack_base_url),
build_bash_command("drive_ops_center.sh", "doctor", stack_base_url),
],
)
if not bool(build_info_section.get("available", False)):
add_issue(
code="runtime_build_info_unavailable",
severity="warning",
layer="runtime_build_info",
summary="当前无法读取运行中 API 的 build-info总检暂时无法确认服务是否已经吃到最新路由面。",
detail=str(build_info_section.get("error") or "runtime build-info unavailable").strip(),
action_code="api-restart",
commands=[
build_bash_command("drive_ops_center.sh", "runtime-refresh-recover", stack_base_url),
build_bash_command("drive_ops_center.sh", "doctor", stack_base_url),
],
)
elif route_surface and not bool(route_surface.get("surface_complete", False)):
add_issue(
code="runtime_route_surface_incomplete",
severity="warning",
layer="runtime_build_info.route_surface",
summary="当前运行中的 API 路由面不完整,说明服务可能还没重启到最新代码,或当前发布包缺少关键运维入口。",
detail=(
f"missing_keys={', '.join(route_surface_missing_keys) or '-'}"
f"mode={str(route_surface.get('mode') or '').strip() or 'unknown'}"
f"registered_paths_total={int(route_surface.get('registered_paths_total', 0) or 0)}"
),
action_code="api-restart",
focus_ref={
"kind": "runtime_build_info",
"section": "route_surface",
"missing_keys": route_surface_missing_keys,
},
commands=[
build_bash_command("drive_ops_center.sh", "runtime-refresh-recover", stack_base_url),
build_bash_command("drive_ops_center.sh", "stack-diagnosis", stack_base_url, "summary"),
build_bash_command("drive_ops_center.sh", "doctor", stack_base_url),
],
)
elif runtime_schema_stale:
add_issue(
code="runtime_build_schema_stale",
severity="warning",
layer="runtime_build_info.schema",
summary="仓库已经具备新的节点接管能力,但运行中的 build-info 仍未声明对应路由键,当前更像 API 还没重启到最新代码。",
detail=(
f"supports_install_command_block={repo_supports_install_command_block}"
f"supports_multi_layout_bootstrap={repo_supports_multi_layout_bootstrap}"
f"route_surface_declares_bootstrap_plan={route_surface_declares_bootstrap_plan}"
),
action_code="api-restart",
focus_ref={
"kind": "runtime_build_info",
"section": "schema",
"expected_route_key": "ops_node_handover_bootstrap_plan",
},
commands=[
build_bash_command("drive_ops_center.sh", "runtime-refresh-recover", stack_base_url),
build_bash_command("drive_ops_center.sh", "stack-diagnosis", stack_base_url, "summary"),
build_bash_command("drive_ops_center.sh", "node-bootstrap-plan", stack_base_url, first_handover_gap_node_code or "mainland-worker-01"),
],
)
if missing_surfaces and any(
name in missing_surfaces for name in ("link_snapshot", "overview", "managed_nodes", "release_hub")
):
add_issue(
code="ops_surface_partial",
severity="warning",
layer="stack",
summary="海外单脑控制面的关键观察层未全部联通,总检目前只能给出部分收口。",
detail=f"当前缺失层: {', '.join(missing_surfaces)}",
commands=[
build_bash_command("drive_ops_center.sh", "stack-diagnosis", stack_base_url, "full"),
build_bash_command("drive_ops_center.sh", "doctor", stack_base_url),
],
)
participating_nodes_total = int(
log_sync.get("participating_nodes_total", log_sync.get("participating_node_count", 0)) or 0
)
participating_node_codes = _normalize_ops_node_code_list(
execution_scene.get("participating_nodes") or [],
(execution_scene.get("participation_summary") or {}).get("dispatch_active_node_codes") or [],
(execution_scene.get("participation_summary") or {}).get("recent_only_node_codes") or [],
)
missing_log_node_codes = _normalize_ops_node_code_list(log_sync.get("missing_participating_nodes") or [])
scene_log_source_node_codes = _normalize_ops_node_code_list(
log_sync.get("source_nodes") or [],
log_sync.get("source_node_summaries") or [],
)
preferred_scene_log_target_codes = _normalize_ops_node_code_list(
missing_log_node_codes,
participating_node_codes,
scene_log_source_node_codes,
)
if bool(log_sync.get("enabled", False)) is False and participating_nodes_total > 0:
log_sync_action = ""
for item in list(link_snapshot.get("next_actions") or []):
action_code = str(item.get("action_code") or "").strip()
if action_code.startswith("enable_log_sync"):
log_sync_action = action_code
break
if not log_sync_action:
log_sync_action = "enable_log_sync_key"
add_issue(
code="remote_log_sync_disabled",
severity="warning",
layer="overview.log_sync",
summary="现场已有参与检测节点,但远端日志回传仍关闭,海外控制面对执行细节仍处于半盲态。",
detail=(
f"参与检测节点 {participating_nodes_total} 台,"
f"已覆盖 {int(log_sync.get('covered_participating_node_count', 0) or 0)} 台。"
),
action_code=log_sync_action,
focus_ref=_build_scene_node_log_focus_ref(
(preferred_scene_log_target_codes or participating_node_codes or [""])[0],
mode="key",
limit=80,
source="remote_log_sync_disabled",
),
commands=[
build_bash_command("drive_ops_center.sh", "driver-resolve", stack_base_url, log_sync_action),
*_build_scene_node_log_commands(
stack_base_url,
preferred_scene_log_target_codes,
limit=80,
mode="key",
max_commands=2,
),
],
)
elif bool(log_sync.get("enabled", False)) and participating_nodes_total > 0 and (
missing_log_node_codes
or int(log_sync.get("line_count", 0) or 0) <= 0
or int(log_sync.get("source_node_count", 0) or 0) <= 0
):
waiting_node_codes = preferred_scene_log_target_codes or participating_node_codes
add_issue(
code="remote_log_sync_waiting_sample",
severity="warning",
layer="overview.log_sync",
summary="远端日志回传虽然已经开启,但参与检测节点的现场样本仍未完全形成。",
detail=(
f"参与检测节点 {participating_nodes_total} 台,"
f"已覆盖 {int(log_sync.get('covered_participating_node_count', 0) or 0)} 台,"
f"缺口节点 {_format_node_code_list(missing_log_node_codes)}"
),
action_code="open_worker_logs_participating",
focus_ref=_build_scene_node_log_focus_ref(
(waiting_node_codes or [""])[0],
mode=str(log_sync.get("mode") or "key"),
limit=120,
source="remote_log_sync_waiting_sample",
),
commands=[
*_build_scene_node_log_commands(
stack_base_url,
waiting_node_codes,
limit=120,
mode=str(log_sync.get("mode") or "key"),
max_commands=3,
),
build_bash_command("drive_ops_center.sh", "driver-resolve", stack_base_url, "open_worker_logs_participating"),
build_bash_command("drive_ops_center.sh", "driver-resolve", stack_base_url, "run_inspection_participating"),
],
)
managed_enabled = int(nodes_summary.get("managed_enabled", 0) or 0)
remote_access_ready = int(nodes_summary.get("remote_access_ready", 0) or 0)
if managed_enabled > 0 and remote_access_ready == 0:
first_handover_gap_stage = dict(first_handover_gap.get("stage") or {})
first_handover_gap_next_step = dict(first_handover_gap_stage.get("next_step") or {})
first_handover_gap_summary = str(first_handover_gap.get("summary") or "").strip()
managed_nodes_action_code = str(
first_handover_gap_recovery.get("action_code")
or launchpad_status.get("recommended_action_code")
or "fix_managed_nodes"
).strip()
add_issue(
code="managed_nodes_agent_pending",
severity="blocked",
layer="managed_nodes",
summary="托管节点虽然已经纳入 Ops Center但 0 台 remote-agent 就绪,海外主控还不能统一下发标准执行动作。",
detail=(
f"managed_enabled={managed_enabled}remote_access_ready={remote_access_ready}"
"当前节点多数仍处于 agent_pending / runtime_only / pending_bootstrap。"
+ (
f" 首个缺口节点 {first_handover_gap_node_code}"
f"{str(first_handover_gap_stage.get('label') or '').strip() or '待处理'}"
f"{first_handover_gap_summary}"
if first_handover_gap_node_code and first_handover_gap_summary
else ""
)
),
action_code=managed_nodes_action_code,
focus_ref=dict(first_handover_gap_recovery.get("focus_ref") or {}),
commands=[
build_bash_command("drive_ops_center.sh", "driver-resolve", stack_base_url, managed_nodes_action_code),
*(
[build_bash_command("drive_ops_center.sh", "node-handover", stack_base_url, first_handover_gap_node_code)]
if first_handover_gap_node_code
else []
),
*(
[
build_bash_command("drive_ops_center.sh", "node-bootstrap-plan", stack_base_url, first_handover_gap_node_code)
]
if first_handover_gap_node_code
and str(first_handover_gap_next_step.get("code") or "").strip()
not in {"ready", "run_acceptance"}
else []
),
*(
[
build_bash_command(
"drive_ops_center.sh",
"node-bootstrap-run" if managed_nodes_action_code == "bootstrap_run" else "node-acceptance-run",
stack_base_url,
first_handover_gap_node_code,
"cli",
)
]
if first_handover_gap_node_code and managed_nodes_action_code in {"bootstrap_run", "run_acceptance"}
else []
),
build_bash_command("drive_ops_center.sh", "doctor", stack_base_url),
],
)
queue_dead_letter_nodes = int(nodes_summary.get("queue_dead_letter_nodes", 0) or 0)
queue_dead_letter_records = int(nodes_summary.get("queue_dead_letter_records", 0) or 0)
if queue_dead_letter_nodes > 0 or queue_dead_letter_records > 0:
add_issue(
code="delivery_queue_dead_letter",
severity="warning",
layer="managed_nodes.queue",
summary="存在 Node Agent 回执死信,后续自动化动作可能反复失败或无法收敛。",
detail=(
f"queue_dead_letter_nodes={queue_dead_letter_nodes}"
f"queue_dead_letter_records={queue_dead_letter_records}"
),
commands=[
build_bash_command("drive_ops_center.sh", "doctor", stack_base_url),
],
)
if str(default_rollout_gate.get("status") or "").strip() == "blocked":
add_issue(
code="release_rollout_gate_blocked",
severity="warning",
layer="release_hub",
summary="ReleaseHub 默认门禁当前阻断,说明虽然发布包已准备好,但正式 Rollout 还不具备条件。",
detail=str(default_rollout_gate.get("summary_text") or default_rollout_gate.get("summary") or "").strip(),
action_code=str(launchpad_status.get("recommended_action_code") or "").strip(),
focus_ref=dict(default_rollout_gate.get("focus_ref") or launchpad_status.get("focus_ref") or {}),
commands=[
build_bash_command("drive_ops_center.sh", "release-launchpad", stack_base_url),
build_bash_command(
"drive_ops_center.sh",
"driver-resolve",
stack_base_url,
str(launchpad_status.get("recommended_action_code") or "fix_managed_nodes").strip(),
),
],
)
if problem_runs:
add_issue(
code="playbook_runs_need_attention",
severity="warning",
layer="playbook_runs",
summary="最近存在未完全收口的 playbook run建议先确认执行序列是否停在中间步骤。",
detail=f"problem_runs_total={len(problem_runs)}",
commands=[
build_bash_command("drive_ops_center.sh", "doctor", stack_base_url),
],
)
if start_delivery_failed_items:
first_delivery_gap_item = dict(start_delivery_failed_items[0] or {})
first_delivery_focus_ref = dict(
first_delivery_gap_item.get("source_focus_ref")
or first_delivery_gap_item.get("focus_ref")
or {}
)
first_delivery_job_code = str(first_delivery_gap_item.get("job_code") or "").strip()
first_delivery_node_code = str(first_delivery_focus_ref.get("target_node_code") or "").strip()
add_issue(
code="ops_job_start_delivery_failed_local",
severity="warning",
layer="activity_stream.ops_job",
summary=(
f"最近有 {len(start_delivery_failed_items)} 条标准运维任务在节点侧已开始执行,但开始回执没有成功送达控制面。"
),
detail=" / ".join(
part
for part in [
f"首条任务 {first_delivery_job_code}" if first_delivery_job_code else "",
f"目标节点 {first_delivery_node_code}" if first_delivery_node_code else "",
str(first_delivery_gap_item.get("start_delivery_error") or "").strip(),
str(first_delivery_gap_item.get("summary_text") or first_delivery_gap_item.get("summary") or "").strip(),
]
if part
),
action_code="focus_latest_job_events",
focus_ref=first_delivery_focus_ref,
commands=[
build_bash_command("drive_ops_center.sh", "driver-resolve", stack_base_url, "focus_latest_job_events"),
build_bash_command("drive_ops_center.sh", "doctor", stack_base_url),
],
)
ordered_action_candidates = [
{
"source": "overview.primary",
"action_code": str(recommendation.get("primary_action_code") or "").strip(),
"title": str(recommendation.get("title") or "").strip(),
"reason": "来自 overview.recommendation.primary_action_code",
"focus_ref": dict(recommendation.get("focus_ref") or {}),
},
{
"source": "release_hub.launchpad",
"action_code": str(launchpad_status.get("recommended_action_code") or "").strip(),
"title": str(launchpad_status.get("status_label") or "").strip(),
"reason": "来自 release_launchpad.launchpad_status.recommended_action_code",
"focus_ref": dict(launchpad_status.get("focus_ref") or {}),
},
{
"source": "overview.secondary",
"action_code": str(recommendation.get("secondary_action_code") or "").strip(),
"title": str(recommendation.get("title") or "").strip(),
"reason": "来自 overview.recommendation.secondary_action_code",
"focus_ref": dict(recommendation.get("focus_ref") or {}),
},
]
if str(first_handover_gap_recovery.get("action_code") or "").strip() in {"bootstrap_run", "run_acceptance"}:
ordered_action_candidates.insert(
0,
{
"source": "managed_nodes.first_handover_gap",
"action_code": str(first_handover_gap_recovery.get("action_code") or "").strip(),
"title": str(first_handover_gap_recovery.get("recovery_label") or "").strip(),
"reason": str(first_handover_gap_recovery.get("summary") or "").strip()
or "来自首个 handover gap 的标准恢复动作。",
"focus_ref": dict(first_handover_gap_recovery.get("focus_ref") or {}),
},
)
for issue in issues:
if str(issue.get("code") or "").strip() == "runtime_build_schema_stale":
ordered_action_candidates.insert(
0,
{
"source": f"issue:{issue.get('code')}",
"action_code": "api-restart",
"title": "优先重启控制面 API",
"reason": str(issue.get("detail") or issue.get("summary") or "").strip(),
"focus_ref": dict(issue.get("focus_ref") or {}),
},
)
break
for issue in issues:
issue_action_code = str(issue.get("action_code") or "").strip()
if issue_action_code:
ordered_action_candidates.append(
{
"source": f"issue:{issue.get('code')}",
"action_code": issue_action_code,
"title": str(issue.get("summary") or "").strip(),
"reason": str(issue.get("detail") or issue.get("summary") or "").strip(),
"focus_ref": dict(issue.get("focus_ref") or {}),
}
)
recommended_actions: list[dict] = []
seen_action_codes: set[str] = set()
for item in ordered_action_candidates:
action_code = str(item.get("action_code") or "").strip()
if not action_code or action_code in seen_action_codes:
continue
seen_action_codes.add(action_code)
recommended_action_contract_keys = _normalize_ops_contract_keys(
["ops_stack_diagnosis_contract"]
+ _ops_contract_keys_from_action_code(action_code)
+ _ops_contract_keys_from_focus_ref(item.get("focus_ref"))
)
recommended_actions.append(
{
"action_code": action_code,
"source": str(item.get("source") or "").strip(),
"title": str(item.get("title") or "").strip(),
"reason": str(item.get("reason") or "").strip(),
"focus_ref": dict(item.get("focus_ref") or {}),
"command": build_bash_command("drive_ops_center.sh", "driver-resolve", stack_base_url, action_code),
"contract_keys": recommended_action_contract_keys,
"contract_navigation": _build_ops_contract_navigation(
recommended_action_contract_keys,
primary_contract_key="ops_stack_diagnosis_contract",
registry=contract_registry,
),
}
)
blocking_issue_total = sum(1 for item in issues if str(item.get("severity") or "").strip() == "blocked")
warning_issue_total = sum(1 for item in issues if str(item.get("severity") or "").strip() == "warning")
info_issue_total = sum(1 for item in issues if str(item.get("severity") or "").strip() == "info")
surface_status = "healthy"
if missing_surfaces:
surface_status = "partial"
automation_status = "ready"
if blocking_issue_total > 0:
automation_status = "blocked"
elif warning_issue_total > 0:
automation_status = "attention"
stack_status = "ready"
if automation_status == "blocked":
stack_status = "blocked"
elif surface_status != "healthy" or automation_status != "ready":
stack_status = "attention"
top_issue = None
if issues:
top_issue = sorted(
issues,
key=lambda item: (
-_ops_stack_issue_rank(str(item.get("severity") or "").strip()),
str(item.get("code") or "").strip(),
),
)[0]
next_step = {}
if recommended_actions:
first_action = dict(recommended_actions[0] or {})
next_step_contract_keys = _normalize_ops_contract_keys(
["ops_stack_diagnosis_contract"]
+ _ops_contract_keys_from_action_code(str(first_action.get("action_code") or "").strip())
+ _ops_contract_keys_from_focus_ref(first_action.get("focus_ref"))
)
next_step = {
"action_code": str(first_action.get("action_code") or "").strip(),
"source": str(first_action.get("source") or "").strip(),
"reason": str(first_action.get("reason") or "").strip(),
"command": str(first_action.get("command") or "").strip(),
"focus_ref": dict(first_action.get("focus_ref") or {}),
"contract_keys": next_step_contract_keys,
"contract_navigation": _build_ops_contract_navigation(
next_step_contract_keys,
primary_contract_key="ops_stack_diagnosis_contract",
registry=contract_registry,
),
}
elif top_issue:
next_step_contract_keys = _normalize_ops_contract_keys(top_issue.get("contract_keys") or ["ops_stack_diagnosis_contract"])
next_step = {
"action_code": "",
"source": f"issue:{str(top_issue.get('code') or '').strip()}",
"reason": str(top_issue.get("summary") or "").strip(),
"command": "",
"focus_ref": dict(top_issue.get("focus_ref") or {}),
"contract_keys": next_step_contract_keys,
"contract_navigation": _build_ops_contract_navigation(
next_step_contract_keys,
primary_contract_key="ops_stack_diagnosis_contract",
registry=contract_registry,
),
}
quick_commands = [
build_bash_command("drive_ops_center.sh", "stack-diagnosis", stack_base_url, "summary"),
build_bash_command("drive_ops_center.sh", "doctor", stack_base_url),
]
for action in recommended_actions[:3]:
command = str(action.get("command") or "").strip()
if command and command not in quick_commands:
quick_commands.append(command)
for issue in issues[:5]:
for command in list(issue.get("commands") or []):
normalized_command = str(command or "").strip()
if normalized_command and normalized_command not in quick_commands:
quick_commands.append(normalized_command)
if len(quick_commands) >= 8:
break
if len(quick_commands) >= 8:
break
if not bool(contracts_section.get("available", False)):
contract_check_command = build_bash_command("drive_ops_center.sh", "contracts", stack_base_url)
if contract_check_command not in quick_commands:
quick_commands.append(contract_check_command)
recommended_commands = _build_stack_diagnosis_recommended_commands(
stack_base_url=stack_base_url,
recommended_actions=recommended_actions,
first_handover_gap_node_code=first_handover_gap_node_code,
launchpad_status=launchpad_status,
default_rollout_gate=default_rollout_gate,
next_step=next_step,
)
operator_decision, next_actions = _build_stack_diagnosis_operator_decision(
issues=issues,
next_step=next_step,
recommended_actions=recommended_actions,
recommended_commands=recommended_commands,
first_handover_gap_node_code=first_handover_gap_node_code,
launchpad_status=launchpad_status,
default_rollout_gate=default_rollout_gate,
)
operator_hints: list[str] = []
if stack_status == "blocked":
operator_hints.append("当前总检已经能看到现场,但海外单脑自动化执行链路仍未正式打通,先处理阻断问题再继续 Rollout 或批量运维。")
elif stack_status == "attention":
operator_hints.append("当前总检可用,但仍有缺口;建议先按 next_step 收口,再进入更细粒度脚本。")
else:
operator_hints.append("当前海外单脑控制面总检已基本就绪,可以把它作为后续所有联调和发布的固定起手式。")
if not bool(contracts_section.get("available", False)):
operator_hints.append("contracts 当前未正常可用,优先先恢复 contract registry再让页面、CLI、Codex 共享同一份正式契约。")
if route_surface_missing_keys:
operator_hints.append("build-info 显示当前运行中的 API 路由面还不完整,先重启 API 并确认最新运维入口已经注册,再继续做节点接管或发布判断。")
if any(str(item.get("code") or "").strip() == "runtime_build_schema_stale" for item in issues):
operator_hints.append("当前属于典型的“仓库代码已更新,但运行中 API 还是旧 schema”场景先重启 domaincheck-api再重新看 stack-diagnosis 与 node-bootstrap-plan。")
if bool(log_sync.get("enabled", False)) is False and participating_nodes_total > 0:
operator_hints.append("远端日志回传当前关闭,后续如果继续多机联调,海外控制面会继续处于半盲态。")
next_step_action_code = str(next_step.get("action_code") or "").strip()
next_step_focus_ref = dict(next_step.get("focus_ref") or {})
if next_step_action_code in {"bootstrap_run", "run_acceptance"}:
next_step_node_code = str(next_step_focus_ref.get("node_code") or first_handover_gap_node_code or "").strip()
operator_hints.append(
(
f"当前第一动作已经收敛为 {'接入收口' if next_step_action_code == 'bootstrap_run' else '接管验收'}"
+ (f"{next_step_node_code}" if next_step_node_code else "")
+ "页面、CLI、Codex 都应优先围绕这一个节点推进,不再回退到泛化的托管节点修复。"
)
)
resolved_launchpad_target_node_code = str(launchpad_status.get("recommended_target_node_code") or "").strip()
resolved_launchpad_recovery_label = str(launchpad_status.get("recommended_recovery_label") or "").strip()
resolved_launchpad_recovery_summary = str(launchpad_status.get("recommended_recovery_summary") or "").strip()
if (
str(launchpad_status.get("recommended_action_code") or "").strip() not in {"bootstrap_run", "run_acceptance"}
and str(first_handover_gap_recovery.get("action_code") or "").strip() in {"bootstrap_run", "run_acceptance"}
):
resolved_launchpad_target_node_code = (
resolved_launchpad_target_node_code
or str(next_step_focus_ref.get("node_code") or "").strip()
or str(first_handover_gap_recovery.get("node_code") or "").strip()
)
resolved_launchpad_recovery_label = (
resolved_launchpad_recovery_label
or str(first_handover_gap_recovery.get("recovery_label") or "").strip()
)
resolved_launchpad_recovery_summary = (
resolved_launchpad_recovery_summary
or str(first_handover_gap_recovery.get("summary") or "").strip()
)
diagnosis_contract_keys = _normalize_ops_contract_keys(
["ops_stack_diagnosis_contract", "ops_driver_contract"]
+ [contract_key for issue in issues for contract_key in list(issue.get("contract_keys") or [])]
+ list(next_step.get("contract_keys") or [])
)
return {
"generated_at": datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
"diagnosis": {
"contract_key": "ops_stack_diagnosis_contract",
"contract_version": _OPS_CONTRACT_SCHEMA_VERSION,
"registry_version": _OPS_CONTRACT_REGISTRY_VERSION,
"base_url": stack_base_url,
"contract_navigation": _build_ops_contract_navigation(
diagnosis_contract_keys,
primary_contract_key="ops_stack_diagnosis_contract",
registry=contract_registry,
),
"surface_status": surface_status,
"automation_status": automation_status,
"stack_status": stack_status,
"issue_total": len(issues),
"blocking_issue_total": blocking_issue_total,
"warning_issue_total": warning_issue_total,
"info_issue_total": info_issue_total,
"missing_surfaces": missing_surfaces,
"launchpad_recommended_target_node_code": resolved_launchpad_target_node_code,
"launchpad_recommended_recovery_label": resolved_launchpad_recovery_label,
"launchpad_recommended_recovery_summary": resolved_launchpad_recovery_summary,
"launchpad_onboarding_bootstrap_pending_nodes": int(
launchpad_status.get("onboarding_bootstrap_pending_nodes", 0) or 0
),
"launchpad_onboarding_acceptance_ready_nodes": int(
launchpad_status.get("onboarding_acceptance_ready_nodes", 0) or 0
),
"next_step": next_step,
"recommended_actions": recommended_actions[:5],
"operator_decision": operator_decision,
"next_actions": next_actions,
"recommended_commands": recommended_commands,
"issues": issues[:8],
"operator_hints": operator_hints,
"quick_commands": quick_commands,
},
"api": {
"http_status": "200",
"available": True,
**api_info,
},
"surface_matrix": {
"status_counts": surface_status_counts,
"items": [
{
"name": name,
"available": bool(available),
"elapsed_ms": int(
{
"contracts": contracts_section,
"link_snapshot": link_snapshot_section,
"overview": overview_section,
"managed_nodes": nodes_section,
"release_hub": release_hub_section,
"runtime_build_info": build_info_section,
"playbook_runs": playbook_runs_section,
"activity_stream": activity_stream_section,
}.get(name, {}).get("elapsed_ms", 0) or 0
),
}
for name, available in surface_matrix
],
},
"contracts": {
"http_status": str(contracts_section.get("http_status") or "500"),
"available": bool(contracts_section.get("available", False)),
"elapsed_ms": int(contracts_section.get("elapsed_ms", 0) or 0),
"registry_version": str(contracts.get("registry_version") or ""),
"schema_version": str(contracts.get("schema_version") or ""),
"contracts_total": len(contracts_rows),
"contract_keys": [str(item.get("key") or "").strip() for item in contracts_rows if str(item.get("key") or "").strip()],
"error": str(contracts_section.get("error") or ""),
},
"link_snapshot": {
"http_status": str(link_snapshot_section.get("http_status") or "500"),
"available": bool(link_snapshot_section.get("available", False)),
"elapsed_ms": int(link_snapshot_section.get("elapsed_ms", 0) or 0),
"status": str(link_snapshot.get("status") or ""),
"headline": str(link_snapshot.get("headline") or ""),
"next_action_codes": [
str(item.get("action_code") or "").strip()
for item in list(link_snapshot.get("next_actions") or [])
if str(item.get("action_code") or "").strip()
],
"error": str(link_snapshot_section.get("error") or ""),
},
"overview": {
"http_status": str(overview_section.get("http_status") or "500"),
"available": bool(overview_section.get("available", False)),
"elapsed_ms": int(overview_section.get("elapsed_ms", 0) or 0),
"recommendation_key": str(recommendation.get("key") or ""),
"recommendation_title": str(recommendation.get("title") or ""),
"primary_action_code": str(recommendation.get("primary_action_code") or ""),
"secondary_action_code": str(recommendation.get("secondary_action_code") or ""),
"focus_ref": dict(recommendation.get("focus_ref") or {}),
"execution_scene_summary": str(
execution_scene.get("summary")
or (execution_scene.get("participation_summary") or {}).get("summary")
or execution_scene.get("phase_detail")
or ""
),
"execution_scene_counts": dict(execution_scene.get("counts") or {}),
"log_sync": {
"status": str(log_sync.get("status") or log_sync.get("state") or ""),
"status_label": str(log_sync.get("status_label") or ""),
"mode": str(log_sync.get("mode") or ""),
"mode_label": str(log_sync.get("mode_label") or ""),
"enabled": bool(log_sync.get("enabled", False)),
"covered_participating_nodes": int(log_sync.get("covered_participating_node_count", 0) or 0),
"participating_nodes_total": int(
log_sync.get("participating_nodes_total", log_sync.get("participating_node_count", 0)) or 0
),
"missing_sample_node_codes": list(
log_sync.get("missing_sample_node_codes")
or log_sync.get("missing_participating_nodes")
or []
),
},
"error": str(overview_section.get("error") or ""),
},
"managed_nodes": {
"http_status": str(nodes_section.get("http_status") or "500"),
"available": bool(nodes_section.get("available", False)),
"elapsed_ms": int(nodes_section.get("elapsed_ms", 0) or 0),
"total": int(nodes_summary.get("total", len(nodes)) or 0),
"managed_total": int(nodes_summary.get("managed_total", 0) or 0),
"managed_enabled": int(nodes_summary.get("managed_enabled", 0) or 0),
"online": int(nodes_summary.get("online", 0) or 0),
"agent_ready": int(nodes_summary.get("agent_ready", 0) or 0),
"ssh_ready": int(nodes_summary.get("ssh_ready", 0) or 0),
"remote_access_ready": int(nodes_summary.get("remote_access_ready", 0) or 0),
"participating": int(nodes_summary.get("participating", 0) or 0),
"dispatch_active": int(nodes_summary.get("dispatch_active", 0) or 0),
"standby": int(nodes_summary.get("standby", 0) or 0),
"load_syncing": int(nodes_summary.get("load_syncing", 0) or 0),
"queue_retrying_nodes": int(nodes_summary.get("queue_retrying_nodes", 0) or 0),
"queue_dead_letter_nodes": int(nodes_summary.get("queue_dead_letter_nodes", 0) or 0),
"queue_pending_records": int(nodes_summary.get("queue_pending_records", 0) or 0),
"queue_dead_letter_records": int(nodes_summary.get("queue_dead_letter_records", 0) or 0),
"agent_state_counts": dict(nodes_summary.get("status_counts") or {}),
"remote_access_state_counts": dict(nodes_summary.get("remote_access_state_counts") or {}),
"delivery_queue_state_counts": dict(nodes_summary.get("delivery_queue_state_counts") or {}),
"problem_nodes": problem_nodes[:8],
"first_handover_gap_node_code": first_handover_gap_node_code,
"first_handover_gap": first_handover_gap,
"error": str(nodes_section.get("error") or ""),
},
"release_hub": {
"http_status": str(release_hub_section.get("http_status") or "500"),
"available": bool(release_hub_section.get("available", False)),
"elapsed_ms": int(release_hub_section.get("elapsed_ms", 0) or 0),
"latest_release_version": str(latest_release.get("release_version") or ""),
"latest_release_status": str(latest_release.get("status") or ""),
"default_rollout_gate_status": str(default_rollout_gate.get("status") or ""),
"default_rollout_gate_label": str(default_rollout_gate.get("status_label") or ""),
"default_rollout_gate_summary": str(default_rollout_gate.get("summary_text") or default_rollout_gate.get("summary") or ""),
"default_rollout_gate_focus_ref": dict(default_rollout_gate.get("focus_ref") or {}),
"launchpad_status": str(launchpad_status.get("status") or ""),
"launchpad_status_label": str(launchpad_status.get("status_label") or ""),
"launchpad_summary": str(launchpad_status.get("summary_text") or launchpad_status.get("summary") or ""),
"recommended_action_code": str(launchpad_status.get("recommended_action_code") or ""),
"recommended_execution_mode": str(launchpad_status.get("recommended_execution_mode") or ""),
"focus_ref": dict(launchpad_status.get("focus_ref") or {}),
"error": str(release_hub_section.get("error") or ""),
},
"runtime_build_info": {
"http_status": str(build_info_section.get("http_status") or "500"),
"available": bool(build_info_section.get("available", False)),
"elapsed_ms": int(build_info_section.get("elapsed_ms", 0) or 0),
"source": str(build_info.get("source") or ""),
"package_name": str(build_info.get("package_name") or ""),
"generated_at": str(build_info.get("generated_at") or ""),
"commit_sha": str(build_info.get("commit_sha") or ""),
"commit_ref": str(build_info.get("commit_ref") or ""),
"checksum": str(build_info.get("checksum") or ""),
"manifest_path": str(build_info.get("manifest_path") or ""),
"repository_capabilities": repository_capabilities,
"route_surface": route_surface,
"route_surface_missing_keys": route_surface_missing_keys,
"route_surface_declares_bootstrap_plan": route_surface_declares_bootstrap_plan,
"runtime_schema_stale": runtime_schema_stale,
"error": str(build_info_section.get("error") or ""),
},
"playbook_runs": {
"http_status": str(playbook_runs_section.get("http_status") or "500"),
"available": bool(playbook_runs_section.get("available", False)),
"elapsed_ms": int(playbook_runs_section.get("elapsed_ms", 0) or 0),
"recent_total": len(playbook_runs),
"problem_runs_total": len(problem_runs),
"problem_runs": problem_runs[:5],
"error": str(playbook_runs_section.get("error") or ""),
},
"activity_stream": {
"http_status": str(activity_stream_section.get("http_status") or "500"),
"available": bool(activity_stream_section.get("available", False)),
"elapsed_ms": int(activity_stream_section.get("elapsed_ms", 0) or 0),
"recent_total": len(activities),
"status_counts": activity_counts,
"top_items": [
{
"kind": str(item.get("kind") or "").strip(),
"status": str(item.get("status") or item.get("job_status") or "").strip(),
"summary": str(item.get("summary_text") or item.get("summary") or "").strip(),
"occurred_at": str(item.get("occurred_at") or "").strip(),
"focus_ref": dict(item.get("focus_ref") or {}),
"source_focus_ref": dict(item.get("source_focus_ref") or {}),
"ui_intent_kind": str(((item.get("ui_intent") or {}).get("kind") or "")).strip(),
}
for item in activities[:5]
],
"error": str(activity_stream_section.get("error") or ""),
},
}
def get_ops_capabilities() -> dict:
contract_registry = get_ops_contract_registry()
return {
"control_plane": {
"deployment_mode": "overseas-control-plane",
"codex_driver_ready": True,
"ui_button_ready": True,
"daily_entrypoint": "overseas dashboard / ops api",
},
"contract_registry": {
"endpoint": "/api/v1/ops/contracts",
"detail_endpoint_pattern": "/api/v1/ops/contracts/{contract_key}",
"registry_version": str(contract_registry.get("registry_version") or _OPS_CONTRACT_REGISTRY_VERSION),
"schema_version": str(contract_registry.get("schema_version") or _OPS_CONTRACT_SCHEMA_VERSION),
"contracts_total": int(contract_registry.get("contracts_total", 0) or 0),
"contract_keys": [
str(item.get("key") or "").strip()
for item in list(contract_registry.get("contracts") or [])
if str(item.get("key") or "").strip()
],
"schema_docs": {
str(item.get("key") or "").strip(): str(item.get("schema_doc_path") or "").strip()
for item in list(contract_registry.get("contracts") or [])
if str(item.get("key") or "").strip()
},
},
"capabilities": [
{
"key": "cluster_visibility",
"title": "集群可观测",
"current_state": "ready",
"today": "已具备 runtime/cluster/readiness/sync-summary 聚合能力。",
"target": "继续增强节点参与度、诊断报告和远端日志回传。",
},
{
"key": "remote_log_sync",
"title": "远端检测日志回传",
"current_state": "ready",
"today": "已具备日志回传开关、关键/全量模式、聚合样本面板与按节点现场日志钻取。",
"target": "后续如规模继续扩大,再补游标续传、流式 tail 与更长窗口归档。",
},
{
"key": "button_driven_ops",
"title": "后台按钮化运维",
"current_state": "partial",
"today": "已有 runtime actions / ops jobs / ops playbooks开始收口为统一按钮化编排。",
"target": "继续扩大安装、更新、重启、巡检、诊断包收集与发布回滚 playbook。",
},
{
"key": "node_agent",
"title": "节点 Agent",
"current_state": "in_progress",
"today": "已具备 agent token / register / heartbeat / pull / complete 协议与本地执行器骨架。",
"target": "继续增强 stdout/stderr 游标回传、流式日志、诊断包与更多动作模板。",
},
{
"key": "release_management",
"title": "发布包管理",
"current_state": "partial",
"today": "已具备 release / rollout / deploy.release 能力,支持 checksum、切换 current、健康检查、失败回滚以及 Agent / SSH 双通道单节点发布。",
"target": "继续增强批次策略、灰度放量、发布观测面板与一键回滚编排。",
},
],
"action_catalog": [
{"action": "node.bootstrap", "title": "纳管新节点", "transport": "ops-job -> control-plane -> bootstrap-plan"},
{"action": "deploy.release", "title": "发布新版本", "transport": "ops-job -> node-agent / ssh-executor"},
{
"action": "deploy.release.smart_worker",
"title": "按最新包一键 Worker 灰度",
"transport": "ops api -> package release builder -> rollout engine",
},
{
"action": "deploy.release.smart_control",
"title": "按最新包一键 Control 发布",
"transport": "ops api -> package release builder -> rollout engine",
},
{
"action": "deploy.rollout.smart_worker",
"title": "对已有 Release 一键 Worker 灰度",
"transport": "ops api -> rollout engine",
},
{
"action": "deploy.rollout.smart_control",
"title": "对已有 Release 一键 Control 发布",
"transport": "ops api -> rollout engine",
},
{"action": "service.restart", "title": "重启 API / Worker / Sync-Agent", "transport": "ops-job -> node-agent / ssh-executor"},
{"action": "logs.collect", "title": "收集远端日志", "transport": "ops-job -> node-agent / ssh-executor"},
{"action": "diagnostics.collect", "title": "收集诊断包", "transport": "ops-job -> node-agent / ssh-executor"},
{"action": "ops.playbook.execute", "title": "执行标准 playbook", "transport": "ops api -> playbook expander -> ops-jobs"},
{"action": "health.check", "title": "执行巡检", "transport": "ops-job -> node-agent"},
{"action": "deploy.rollout.advance", "title": "推进下一批发布", "transport": "ops api -> rollout engine"},
{"action": "runtime.restart_api", "title": "本机重启 API", "transport": "ops-job -> local-runtime / ssh-executor"},
{"action": "runtime.start_worker", "title": "本机启动 Worker", "transport": "ops-job -> local-runtime / ssh-executor"},
{"action": "runtime.stop_worker", "title": "本机停止 Worker", "transport": "ops-job -> local-runtime / ssh-executor"},
{"action": "runtime.start_sync_agent", "title": "本机启动 Sync Agent", "transport": "ops-job -> local-runtime / ssh-executor"},
{"action": "runtime.stop_sync_agent", "title": "本机停止 Sync Agent", "transport": "ops-job -> local-runtime / ssh-executor"},
{"action": "runtime.pull_tasks", "title": "本机立即拉取任务", "transport": "ops-job -> local-runtime"},
{"action": "runtime.push_sync", "title": "本机立即推送同步", "transport": "ops-job -> local-runtime"},
{"action": "health.snapshot", "title": "采集本机运行时快照", "transport": "ops-job -> local-runtime / ssh-executor"},
],
}
def get_ops_go_live_summary(*, base_url: str = "") -> dict:
normalized_base_url = str(base_url or "").strip() or "http://127.0.0.1:8100"
stack_payload = get_ops_stack_diagnosis(base_url=normalized_base_url)
contracts_payload = get_ops_contract_registry()
launchpad_payload = get_release_launchpad()
build_info_payload = get_runtime_build_info()
nodes_payload = list_managed_nodes_with_agent_state()
overview_payload = get_ops_overview()
diagnosis = dict(stack_payload.get("diagnosis") or {})
diagnosis_issues = [dict(item or {}) for item in list(diagnosis.get("issues") or []) if item]
route_surface = dict(build_info_payload.get("route_surface") or {})
repository_capabilities = dict(build_info_payload.get("repository_capabilities") or {})
nodes_summary = dict(nodes_payload.get("summary") or {})
execution_scene = dict(overview_payload.get("execution_scene") or {})
log_sync = dict(execution_scene.get("log_sync") or overview_payload.get("log_sync") or {})
recommendation = dict(overview_payload.get("recommendation") or {})
launchpad_status = dict(launchpad_payload.get("launchpad_status") or {})
next_step = dict(diagnosis.get("next_step") or {})
operator_decision = dict(diagnosis.get("operator_decision") or {})
contracts = [dict(item or {}) for item in list(contracts_payload.get("contracts") or []) if item]
stack_status = str(diagnosis.get("stack_status") or "").strip() or "unknown"
launchpad_status_code = (
str(launchpad_status.get("status") or "").strip()
or str((stack_payload.get("release_hub") or {}).get("launchpad_status") or "").strip()
)
managed_enabled = int(nodes_summary.get("managed_enabled", 0) or 0)
remote_access_ready = int(nodes_summary.get("remote_access_ready", 0) or 0)
queue_dead_letter_nodes = int(nodes_summary.get("queue_dead_letter_nodes", 0) or 0)
participating_nodes_total = int(log_sync.get("participating_node_count", 0) or 0)
log_sync_covered_nodes = int(log_sync.get("covered_participating_node_count", 0) or 0)
launchpad_onboarding_bootstrap_pending_nodes = int(
launchpad_status.get("onboarding_bootstrap_pending_nodes", 0) or 0
)
launchpad_onboarding_acceptance_ready_nodes = int(
launchpad_status.get("onboarding_acceptance_ready_nodes", 0) or 0
)
start_delivery_issue_total = sum(
1
for item in diagnosis_issues
if str(item.get("code") or "").strip() == "ops_job_start_delivery_failed_local"
)
blocking_reasons: list[str] = []
warnings: list[str] = []
if stack_status == "blocked":
blocking_reasons.append("stack_diagnosis=blocked")
elif stack_status == "attention":
warnings.append("stack_diagnosis=attention")
route_surface_complete = bool(route_surface.get("surface_complete", False))
route_surface_missing_keys = [
str(item or "").strip()
for item in list(route_surface.get("missing_keys") or [])
if str(item or "").strip()
]
route_surface_expected_paths = {
str(key).strip(): str(value).strip()
for key, value in dict(route_surface.get("expected_paths") or {}).items()
if str(key).strip()
}
route_surface_declares_bootstrap_plan = "ops_node_handover_bootstrap_plan" in route_surface_expected_paths
runtime_schema_stale = bool(
repository_capabilities.get("supports_install_command_block", False) and not route_surface_declares_bootstrap_plan
)
if not route_surface_complete:
blocking_reasons.append(
"route_surface_incomplete"
+ (f":{','.join(route_surface_missing_keys)}" if route_surface_missing_keys else "")
)
elif runtime_schema_stale:
warnings.append("runtime_build_schema_stale")
if not contracts:
blocking_reasons.append("contracts_unavailable")
if launchpad_status_code == "blocked":
blocking_reasons.append("release_launchpad=blocked")
elif launchpad_status_code == "attention":
warnings.append("release_launchpad=attention")
if launchpad_onboarding_bootstrap_pending_nodes > 0:
warnings.append(f"launchpad_onboarding_bootstrap_pending={launchpad_onboarding_bootstrap_pending_nodes}")
if launchpad_onboarding_acceptance_ready_nodes > 0:
warnings.append(f"launchpad_onboarding_acceptance_ready={launchpad_onboarding_acceptance_ready_nodes}")
if managed_enabled > 0 and remote_access_ready == 0:
blocking_reasons.append("remote_agent_ready=0")
elif managed_enabled > 0 and remote_access_ready < managed_enabled:
warnings.append(f"remote_agent_ready={remote_access_ready}/{managed_enabled}")
if queue_dead_letter_nodes > 0:
warnings.append(f"queue_dead_letter_nodes={queue_dead_letter_nodes}")
if start_delivery_issue_total > 0:
warnings.append(f"ops_job_start_delivery_failed_local={start_delivery_issue_total}")
if bool(log_sync.get("enabled")) and participating_nodes_total > 0:
if log_sync_covered_nodes == 0:
warnings.append(f"log_sync_waiting_sample=0/{participating_nodes_total}")
elif log_sync_covered_nodes < participating_nodes_total:
warnings.append(f"log_sync_partial={log_sync_covered_nodes}/{participating_nodes_total}")
publish_blocking_reasons: list[str] = []
publish_warnings: list[str] = []
if stack_status == "blocked":
publish_blocking_reasons.append("stack_diagnosis=blocked")
elif stack_status == "attention":
publish_warnings.append("stack_diagnosis=attention")
if not route_surface_complete:
publish_blocking_reasons.append(
"route_surface_incomplete"
+ (f":{','.join(route_surface_missing_keys)}" if route_surface_missing_keys else "")
)
elif runtime_schema_stale:
publish_warnings.append("runtime_build_schema_stale")
if not contracts:
publish_blocking_reasons.append("contracts_unavailable")
if launchpad_status_code == "blocked":
publish_blocking_reasons.append("release_launchpad=blocked")
elif launchpad_status_code == "attention":
publish_warnings.append("release_launchpad=attention")
if launchpad_onboarding_bootstrap_pending_nodes > 0:
publish_warnings.append(f"launchpad_onboarding_bootstrap_pending={launchpad_onboarding_bootstrap_pending_nodes}")
if launchpad_onboarding_acceptance_ready_nodes > 0:
publish_warnings.append(f"launchpad_onboarding_acceptance_ready={launchpad_onboarding_acceptance_ready_nodes}")
if managed_enabled > 0 and remote_access_ready == 0:
publish_blocking_reasons.append("remote_agent_ready=0")
elif managed_enabled > 0 and remote_access_ready < managed_enabled:
publish_warnings.append(f"remote_agent_ready={remote_access_ready}/{managed_enabled}")
if queue_dead_letter_nodes > 0:
publish_blocking_reasons.append(f"queue_dead_letter_nodes={queue_dead_letter_nodes}")
if start_delivery_issue_total > 0:
publish_warnings.append(f"ops_job_start_delivery_failed_local={start_delivery_issue_total}")
if bool(log_sync.get("enabled")) and participating_nodes_total > 0:
if log_sync_covered_nodes == 0:
publish_warnings.append(f"log_sync_waiting_sample=0/{participating_nodes_total}")
elif log_sync_covered_nodes < participating_nodes_total:
publish_warnings.append(f"log_sync_partial={log_sync_covered_nodes}/{participating_nodes_total}")
publish_status = "ready"
if publish_blocking_reasons:
publish_status = "blocked"
elif publish_warnings:
publish_status = "attention"
publish_ready = publish_status == "ready"
if publish_status == "ready":
publish_status_label = "可发布"
publish_summary = "当前执行面、路由面、合同面与发布门禁已收口,可以进入正式发版。"
elif publish_status == "attention":
publish_status_label = "可发布但建议先复核"
publish_summary = "当前没有硬阻断,但仍有上线前关注项,建议先完成复核再正式发版。"
else:
publish_status_label = "暂不可发布"
publish_summary = "当前仍有硬阻断项,不能直接进入正式发版。"
go_live_status = "ready"
if blocking_reasons:
go_live_status = "blocked"
elif warnings:
go_live_status = "attention"
next_step_action_code = (
str(next_step.get("action_code") or "").strip()
or str(recommendation.get("primary_action_code") or "").strip()
or str(launchpad_status.get("recommended_action_code") or "").strip()
)
next_step_reason = (
str(next_step.get("reason") or "").strip()
or str(recommendation.get("reason") or "").strip()
or "来自 overview / launchpad fallback"
)
stack_launchpad_target_node_code = str(
diagnosis.get("launchpad_recommended_target_node_code")
or (dict(next_step.get("focus_ref") or {}).get("node_code") if next_step_action_code in {"bootstrap_run", "run_acceptance"} else "")
or ""
).strip()
stack_launchpad_recovery_label = str(
diagnosis.get("launchpad_recommended_recovery_label")
or ("签发接入工单" if next_step_action_code == "bootstrap_run" else "执行接管验收" if next_step_action_code == "run_acceptance" else "")
or ""
).strip()
stack_launchpad_recovery_summary = str(
diagnosis.get("launchpad_recommended_recovery_summary")
or next_step_reason
or ""
).strip()
launchpad_recommended_target_node_code = str(
launchpad_status.get("recommended_target_node_code") or stack_launchpad_target_node_code or ""
).strip()
launchpad_recommended_recovery_label = str(
launchpad_status.get("recommended_recovery_label") or stack_launchpad_recovery_label or ""
).strip()
launchpad_recommended_recovery_summary = str(
launchpad_status.get("recommended_recovery_summary") or stack_launchpad_recovery_summary or ""
).strip()
operator_lane = str(operator_decision.get("lane") or "").strip()
if not operator_lane:
operator_lane = "recovery" if blocking_reasons else ("verification" if warnings else "go-live")
operator_title = (
str(operator_decision.get("title") or "").strip()
or str(recommendation.get("title") or "").strip()
or str(launchpad_status.get("status_label") or "").strip()
or ("先处理阻塞项" if blocking_reasons else "可以进入上线复核")
)
operator_primary_command_key = str(operator_decision.get("primary_command_key") or "").strip() or next_step_action_code
recommended_commands = {
"stack_summary": build_bash_command("check_ops_center_stack.sh", normalized_base_url, "summary"),
"contracts": build_bash_command("check_ops_contracts.sh", normalized_base_url),
"ops_plane": build_bash_command("check_ops_plane.sh", normalized_base_url),
"release_hub": build_bash_command("check_release_hub.sh", normalized_base_url),
"inspection": build_bash_command("check_ops_inspection.sh", normalized_base_url),
"overview": build_bash_command("drive_ops_center.sh", "overview", normalized_base_url),
"go_live_recover": build_bash_command("drive_ops_center.sh", "go-live-recover", normalized_base_url),
"doctor_export": build_bash_command("drive_ops_center.sh", "doctor-export", "/tmp/domaincheck-go-live", normalized_base_url),
}
if next_step_action_code:
recommended_commands["next_step"] = build_bash_command("drive_ops_center.sh", "driver-resolve", normalized_base_url, next_step_action_code)
if runtime_schema_stale:
recommended_commands["runtime_refresh_recover"] = build_bash_command(
"drive_ops_center.sh",
"runtime-refresh-recover",
normalized_base_url,
)
recommended_commands["api_restart"] = recommended_commands["runtime_refresh_recover"]
recommended_commands["bootstrap_plan_recheck"] = build_bash_command(
"drive_ops_center.sh",
"node-bootstrap-plan",
normalized_base_url,
"mainland-worker-01",
)
if bool(log_sync.get("enabled")):
recommended_commands["log_sync_logs"] = build_bash_command("drive_ops_center.sh", "driver-resolve", normalized_base_url, "open_worker_logs_participating")
recommended_commands["log_sync_inspection"] = build_bash_command("drive_ops_center.sh", "driver-resolve", normalized_base_url, "run_inspection_participating")
return {
"base_url": normalized_base_url,
"generated_at": datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
"go_live_status": go_live_status,
"publish_ready": publish_ready,
"publish_status": publish_status,
"publish_status_label": publish_status_label,
"publish_summary": publish_summary,
"stack_status": stack_status,
"contracts_ready": bool(contracts),
"contracts_total": len(contracts),
"launchpad_status": launchpad_status_code,
"launchpad_status_label": str(launchpad_status.get("status_label") or "").strip(),
"launchpad_recommended_action_code": str(launchpad_status.get("recommended_action_code") or "").strip(),
"launchpad_recommended_target_node_code": launchpad_recommended_target_node_code,
"launchpad_recommended_recovery_label": launchpad_recommended_recovery_label,
"launchpad_recommended_recovery_summary": launchpad_recommended_recovery_summary,
"launchpad_onboarding_bootstrap_pending_nodes": launchpad_onboarding_bootstrap_pending_nodes,
"launchpad_onboarding_acceptance_ready_nodes": launchpad_onboarding_acceptance_ready_nodes,
"route_surface_complete": route_surface_complete,
"route_surface_missing_keys": route_surface_missing_keys,
"route_surface_declares_bootstrap_plan": route_surface_declares_bootstrap_plan,
"runtime_schema_stale": runtime_schema_stale,
"repository_capabilities": repository_capabilities,
"managed_enabled": managed_enabled,
"remote_access_ready": remote_access_ready,
"queue_dead_letter_nodes": queue_dead_letter_nodes,
"activity_start_delivery_issue_total": start_delivery_issue_total,
"participating_nodes_total": participating_nodes_total,
"log_sync_enabled": bool(log_sync.get("enabled")),
"log_sync_state": str(log_sync.get("state") or "").strip(),
"log_sync_mode": str(log_sync.get("mode") or "").strip(),
"log_sync_covered_nodes": log_sync_covered_nodes,
"log_sync_missing_node_codes": [
str(item or "").strip()
for item in list(log_sync.get("missing_participating_nodes") or [])
if str(item or "").strip()
],
"next_step_action_code": next_step_action_code,
"next_step_reason": next_step_reason,
"operator_lane": operator_lane,
"operator_title": operator_title,
"operator_primary_command_key": operator_primary_command_key,
"publish_blocking_reasons": publish_blocking_reasons,
"publish_warnings": publish_warnings,
"blocking_reasons": blocking_reasons,
"warnings": warnings,
"recommended_commands": recommended_commands,
"source_refs": {
"stack_diagnosis_contract_key": str(diagnosis.get("contract_key") or "").strip(),
"contracts_registry_version": str(contracts_payload.get("registry_version") or "").strip(),
"runtime_build_commit_sha": str(build_info_payload.get("commit_sha") or "").strip(),
"release_focus_ref": dict(launchpad_status.get("focus_ref") or {}),
},
}
def _build_doctor_preferred_surface(operator_decision: dict) -> str:
next_focus = dict(operator_decision.get("next_focus") or {})
focus_kind = str(next_focus.get("kind") or "").strip()
lane = str(operator_decision.get("lane") or "").strip()
if focus_kind == "node_scene_log":
return "execution-scene"
if lane == "node_handover":
return "managed-nodes"
if lane == "release":
return "release-hub"
if lane == "observability":
return "execution-scene"
if lane in {"ops_jobs", "steady"}:
return "driver-feed"
return "stack-summary"
def _dedupe_command_list(command_candidates: list[object]) -> list[str]:
normalized_commands: list[str] = []
for candidate in command_candidates:
if isinstance(candidate, (list, tuple)):
for nested in candidate:
nested_command = str(nested or "").strip()
if nested_command and nested_command not in normalized_commands:
normalized_commands.append(nested_command)
continue
command = str(candidate or "").strip()
if command and command not in normalized_commands:
normalized_commands.append(command)
return normalized_commands
def _doctor_recommended_command_map(
*,
base_url: str,
manifest_target: str,
raw_recommended_commands: list[str],
) -> dict[str, str]:
target = str(manifest_target or "").strip() or str(base_url or "").strip() or "http://127.0.0.1:8100"
mapped = {
"doctor_decision": build_bash_command("drive_ops_center.sh", "doctor-decision", target),
}
if str(manifest_target or "").strip():
mapped["go_live_review"] = build_bash_command("drive_ops_center.sh", "go-live-review", manifest_target)
mapped["go_live_signoff"] = build_bash_command("drive_ops_center.sh", "go-live-signoff", manifest_target)
else:
mapped["stack_check"] = build_bash_command("drive_ops_center.sh", "stack-diagnosis", base_url, "summary")
mapped["stack_diagnosis"] = mapped["stack_check"]
mapped["doctor"] = build_bash_command("drive_ops_center.sh", "doctor", base_url)
for index, command in enumerate(_dedupe_command_list(list(raw_recommended_commands or []))[:4]):
key = "primary_recovery" if index == 0 else f"follow_up_{index}"
mapped[key] = command
return mapped
def _doctor_scene_log_problem_node_codes(scene_log_reports: list[dict]) -> list[str]:
problem_nodes: list[str] = []
for item in scene_log_reports:
if not isinstance(item, dict):
continue
node_code = str(item.get("node_code") or "").strip()
if not node_code:
continue
if item.get("ok") is False:
if node_code not in problem_nodes:
problem_nodes.append(node_code)
continue
status = str(item.get("status") or "").strip().lower()
if status and status not in {"ok", "ready", "success"} and node_code not in problem_nodes:
problem_nodes.append(node_code)
return problem_nodes
def _build_live_doctor_payload(base_url: str) -> tuple[dict, dict]:
go_live_summary = get_ops_go_live_summary(base_url=base_url)
stack_payload = get_ops_stack_diagnosis(base_url=base_url)
diagnosis = dict(stack_payload.get("diagnosis") or {})
operator_decision = dict(diagnosis.get("operator_decision") or {})
next_step = dict(diagnosis.get("next_step") or {})
next_actions = dict(diagnosis.get("next_actions") or {})
recommended_commands = dict(diagnosis.get("recommended_commands") or {})
stack_status = str(diagnosis.get("stack_status") or "").strip()
go_live_status = str(go_live_summary.get("go_live_status") or "").strip()
publish_status = str(go_live_summary.get("publish_status") or "").strip()
status = "ready"
if "blocked" in {stack_status, go_live_status, publish_status}:
status = "blocked"
elif "attention" in {stack_status, go_live_status, publish_status}:
status = "attention"
preferred_surface = _build_doctor_preferred_surface(operator_decision)
headline = (
str(operator_decision.get("title") or "").strip()
or str(diagnosis.get("headline") or "").strip()
or str(go_live_summary.get("headline") or "").strip()
or "当前总检主决策还没有形成明确 headline。"
)
detail = (
str(operator_decision.get("summary") or "").strip()
or str(next_step.get("reason") or "").strip()
or str(go_live_summary.get("next_step_reason") or "").strip()
or str(diagnosis.get("blocked_issue_detail") or "").strip()
)
next_action_code = (
str(operator_decision.get("primary_command_key") or "").strip()
or str(next_step.get("action_code") or "").strip()
or str(go_live_summary.get("next_step_action_code") or "").strip()
)
raw_commands = _dedupe_command_list(
[
next_actions.get("primary_command"),
next_actions.get("secondary_command"),
recommended_commands.get(next_action_code),
list(recommended_commands.values())[:3],
]
)
decision = {
"status": status,
"reason_code": str(
operator_decision.get("reason_code") or next_step.get("source") or stack_status or "live_fallback"
).strip(),
"headline": headline,
"detail": detail,
"preferred_surface": preferred_surface,
"next_action_code": next_action_code,
"recommended_commands": raw_commands,
"evidence": {
"blocking_issue_total": int(diagnosis.get("blocking_issue_total", 0) or 0),
"warning_issue_total": int(diagnosis.get("warning_issue_total", 0) or 0),
"missing_surfaces": list(diagnosis.get("missing_surfaces") or []),
},
}
summary = {
"ok": status == "ready",
"required_failures": [],
"optional_unavailable": [],
"scene_log_reports_total": 0,
"scene_log_reports_ok": 0,
"scene_log_status_counts": {},
"scene_log_reports": [],
"contract_surface_gaps": [],
"launchpad_recommended_target_node_code": str(
go_live_summary.get("launchpad_recommended_target_node_code") or ""
).strip(),
"launchpad_recommended_recovery_label": str(
go_live_summary.get("launchpad_recommended_recovery_label") or ""
).strip(),
"launchpad_recommended_recovery_summary": str(
go_live_summary.get("launchpad_recommended_recovery_summary") or ""
).strip(),
"launchpad_onboarding_bootstrap_pending_nodes": int(
go_live_summary.get("launchpad_onboarding_bootstrap_pending_nodes", 0) or 0
),
"launchpad_onboarding_acceptance_ready_nodes": int(
go_live_summary.get("launchpad_onboarding_acceptance_ready_nodes", 0) or 0
),
}
return decision, summary
def get_ops_doctor_decision(*, base_url: str = "", report_dir: str = "") -> dict:
normalized_base_url = str(base_url or "").strip() or "http://127.0.0.1:8100"
normalized_report_dir = str(report_dir or "").strip()
contract_registry = get_ops_contract_registry()
manifest_path: Path | None = None
resolved_report_dir: Path | None = None
if normalized_report_dir:
candidate_path = Path(normalized_report_dir)
if candidate_path.is_dir():
resolved_report_dir = candidate_path
nested_manifest = candidate_path / "manifest.json"
if nested_manifest.is_file():
manifest_path = nested_manifest
elif candidate_path.is_file():
manifest_path = candidate_path
resolved_report_dir = candidate_path.parent
else:
manifest_path, resolved_report_dir = _find_latest_go_live_bundle_manifest()
raw_decision: dict = {}
raw_summary: dict = {}
source = "live_fallback"
if manifest_path is not None and manifest_path.is_file():
manifest = _load_json_path(manifest_path)
artifacts = list(manifest.get("artifacts") or [])
artifact_by_key = {
str(item.get("key") or "").strip(): dict(item)
for item in artifacts
if isinstance(item, dict) and str(item.get("key") or "").strip()
}
doctor_artifact = artifact_by_key.get("doctor_decision") or {}
doctor_artifact_path = Path(str(doctor_artifact.get("path") or "").strip()) if doctor_artifact else None
doctor_payload = _load_json_path(doctor_artifact_path) if doctor_artifact_path else {}
raw_decision = dict(doctor_payload.get("decision") or manifest.get("decision") or {})
raw_summary = dict(doctor_payload.get("summary") or manifest.get("summary") or {})
if raw_decision:
source = "bundle_manifest" if doctor_payload else "doctor_manifest"
if not raw_decision:
raw_decision, raw_summary = _build_live_doctor_payload(normalized_base_url)
source = "live_fallback"
status = str(raw_decision.get("status") or "").strip() or ("ready" if bool(raw_summary.get("ok", False)) else "attention")
status_label = {
"ready": "主决策已就绪",
"attention": "主决策待复核",
"blocked": "主决策阻断",
"missing": "待生成",
}.get(status, status or "主决策未知")
summary_scene_log_reports = [
dict(item)
for item in list(raw_summary.get("scene_log_reports") or [])
if isinstance(item, dict)
]
scene_log_problem_node_codes = _doctor_scene_log_problem_node_codes(summary_scene_log_reports)
manifest_target = str(manifest_path or normalized_report_dir or "").strip()
recommended_commands = _doctor_recommended_command_map(
base_url=normalized_base_url,
manifest_target=manifest_target,
raw_recommended_commands=[
str(item).strip()
for item in list(raw_decision.get("recommended_commands") or [])
if str(item).strip()
],
)
contract_keys = _normalize_ops_contract_keys(
[
"ops_doctor_decision_contract",
"ops_stack_diagnosis_contract",
"ops_driver_contract",
"ops_go_live_bundle_contract",
"ops_go_live_signoff_contract",
]
)
return {
"generated_at": datetime.now().isoformat(timespec="seconds"),
"base_url": normalized_base_url,
"contract_key": "ops_doctor_decision_contract",
"contract_version": str(
(contract_registry.get("contracts_by_key") or {}).get("ops_doctor_decision_contract", {}).get("version")
or _OPS_CONTRACT_SCHEMA_VERSION
).strip(),
"status": status,
"status_label": status_label,
"headline": str(raw_decision.get("headline") or "").strip() or "当前还没有生成总检主决策。",
"detail": str(raw_decision.get("detail") or "").strip(),
"source": source,
"report_dir": str((resolved_report_dir or manifest_path.parent) if manifest_path else normalized_report_dir),
"manifest_path": str(manifest_path or ""),
"decision_available": bool(raw_decision),
"decision": {
"status": status,
"reason_code": str(raw_decision.get("reason_code") or "").strip(),
"headline": str(raw_decision.get("headline") or "").strip(),
"detail": str(raw_decision.get("detail") or "").strip(),
"preferred_surface": str(raw_decision.get("preferred_surface") or "").strip(),
"next_action_code": str(raw_decision.get("next_action_code") or "").strip(),
"recommended_commands": [
str(item).strip()
for item in list(raw_decision.get("recommended_commands") or [])
if str(item).strip()
],
"evidence": dict(raw_decision.get("evidence") or {}),
},
"summary": {
"ok": bool(raw_summary.get("ok", False)),
"required_failures": [
str(item).strip()
for item in list(raw_summary.get("required_failures") or [])
if str(item).strip()
],
"optional_unavailable": [
str(item).strip()
for item in list(raw_summary.get("optional_unavailable") or [])
if str(item).strip()
],
"scene_log_reports_total": int(raw_summary.get("scene_log_reports_total", len(summary_scene_log_reports)) or 0),
"scene_log_reports_ok": int(raw_summary.get("scene_log_reports_ok", 0) or 0),
"scene_log_status_counts": dict(raw_summary.get("scene_log_status_counts") or {}),
"scene_log_reports": summary_scene_log_reports,
"scene_log_problem_node_codes": scene_log_problem_node_codes,
"contract_surface_gaps": [
str(item).strip()
for item in list(raw_summary.get("contract_surface_gaps") or [])
if str(item).strip()
],
"launchpad_recommended_target_node_code": str(
raw_summary.get("launchpad_recommended_target_node_code") or ""
).strip(),
"launchpad_recommended_recovery_label": str(
raw_summary.get("launchpad_recommended_recovery_label") or ""
).strip(),
"launchpad_recommended_recovery_summary": str(
raw_summary.get("launchpad_recommended_recovery_summary") or ""
).strip(),
"launchpad_onboarding_bootstrap_pending_nodes": int(
raw_summary.get("launchpad_onboarding_bootstrap_pending_nodes", 0) or 0
),
"launchpad_onboarding_acceptance_ready_nodes": int(
raw_summary.get("launchpad_onboarding_acceptance_ready_nodes", 0) or 0
),
},
"recommended_commands": recommended_commands,
"contract_navigation": _build_ops_contract_navigation(
contract_keys,
primary_contract_key="ops_doctor_decision_contract",
registry=contract_registry,
),
}
def get_ops_go_live_signoff(*, base_url: str = "", report_dir: str = "") -> dict:
normalized_base_url = str(base_url or "").strip() or "http://127.0.0.1:8100"
normalized_report_dir = str(report_dir or "").strip()
go_live_summary = get_ops_go_live_summary(base_url=normalized_base_url)
stack_payload = get_ops_stack_diagnosis(base_url=normalized_base_url)
driver_feed = get_ops_driver_feed()
codex_brief = get_ops_codex_brief()
release_launchpad = get_release_launchpad()
go_live_review = get_ops_go_live_review(base_url=normalized_base_url, report_dir=normalized_report_dir)
doctor_decision_payload = get_ops_doctor_decision(base_url=normalized_base_url, report_dir=normalized_report_dir)
contract_registry = get_ops_contract_registry()
stack_diagnosis = dict(stack_payload.get("diagnosis") or {})
driver_summary = dict(driver_feed.get("summary") or {})
codex_summary = dict(codex_brief.get("summary") or {})
launchpad_status = dict(release_launchpad.get("launchpad_status") or {})
operator_decision = dict(stack_diagnosis.get("operator_decision") or {})
next_step = dict(stack_diagnosis.get("next_step") or {})
doctor_decision = dict(doctor_decision_payload.get("decision") or {})
launchpad_target_node_codes = {
"go_live_summary": str(go_live_summary.get("launchpad_recommended_target_node_code") or "").strip(),
"driver_feed": str(driver_summary.get("launchpad_recommended_target_node_code") or "").strip(),
"codex_brief": str(codex_summary.get("launchpad_recommended_target_node_code") or "").strip(),
"release_launchpad": str(launchpad_status.get("recommended_target_node_code") or "").strip(),
}
launchpad_recovery_labels = {
"go_live_summary": str(go_live_summary.get("launchpad_recommended_recovery_label") or "").strip(),
"driver_feed": str(driver_summary.get("launchpad_recommended_recovery_label") or "").strip(),
"codex_brief": str(codex_summary.get("launchpad_recommended_recovery_label") or "").strip(),
"release_launchpad": str(launchpad_status.get("recommended_recovery_label") or "").strip(),
}
launchpad_recovery_summaries = {
"go_live_summary": str(go_live_summary.get("launchpad_recommended_recovery_summary") or "").strip(),
"driver_feed": str(driver_summary.get("launchpad_recommended_recovery_summary") or "").strip(),
"codex_brief": str(codex_summary.get("launchpad_recommended_recovery_summary") or "").strip(),
"release_launchpad": str(launchpad_status.get("recommended_recovery_summary") or "").strip(),
}
launchpad_bootstrap_pending_nodes = {
"go_live_summary": int(go_live_summary.get("launchpad_onboarding_bootstrap_pending_nodes", 0) or 0),
"driver_feed": int(driver_summary.get("launchpad_onboarding_bootstrap_pending_nodes", 0) or 0),
"codex_brief": int(codex_summary.get("launchpad_onboarding_bootstrap_pending_nodes", 0) or 0),
"release_launchpad": int(launchpad_status.get("onboarding_bootstrap_pending_nodes", 0) or 0),
}
launchpad_acceptance_ready_nodes = {
"go_live_summary": int(go_live_summary.get("launchpad_onboarding_acceptance_ready_nodes", 0) or 0),
"driver_feed": int(driver_summary.get("launchpad_onboarding_acceptance_ready_nodes", 0) or 0),
"codex_brief": int(codex_summary.get("launchpad_onboarding_acceptance_ready_nodes", 0) or 0),
"release_launchpad": int(launchpad_status.get("onboarding_acceptance_ready_nodes", 0) or 0),
}
def _nonempty_distinct(values: dict[str, str]) -> list[str]:
return sorted({value for value in values.values() if value})
def _distinct_ints(values: dict[str, int]) -> list[int]:
return sorted({int(value) for value in values.values()})
launchpad_alignment = {
"consistent": True,
"target_node_code_consistent": len(_nonempty_distinct(launchpad_target_node_codes)) <= 1,
"recovery_label_consistent": len(_nonempty_distinct(launchpad_recovery_labels)) <= 1,
"recovery_summary_consistent": len(_nonempty_distinct(launchpad_recovery_summaries)) <= 1,
"bootstrap_pending_consistent": len(_distinct_ints(launchpad_bootstrap_pending_nodes)) <= 1,
"acceptance_ready_consistent": len(_distinct_ints(launchpad_acceptance_ready_nodes)) <= 1,
"available_sources": sorted(launchpad_target_node_codes.keys()),
"target_node_codes": launchpad_target_node_codes,
"recovery_labels": launchpad_recovery_labels,
"recovery_summaries": launchpad_recovery_summaries,
"bootstrap_pending_nodes": launchpad_bootstrap_pending_nodes,
"acceptance_ready_nodes": launchpad_acceptance_ready_nodes,
}
launchpad_alignment["consistent"] = all(
bool(launchpad_alignment.get(key))
for key in (
"target_node_code_consistent",
"recovery_label_consistent",
"recovery_summary_consistent",
"bootstrap_pending_consistent",
"acceptance_ready_consistent",
)
)
blocked_reasons: list[str] = []
attention_reasons: list[str] = []
go_live_status = str(go_live_summary.get("go_live_status") or "").strip()
publish_status = str(go_live_summary.get("publish_status") or "").strip()
stack_status = str(stack_diagnosis.get("stack_status") or "").strip()
driver_launch_status = str(driver_summary.get("launch_status") or "").strip()
codex_launch_status = str(codex_summary.get("launch_status") or "").strip()
release_launchpad_status = str(launchpad_status.get("status") or "").strip()
review_status = str(go_live_review.get("status") or "").strip()
doctor_status = str(doctor_decision_payload.get("status") or "").strip()
if go_live_status == "blocked":
blocked_reasons.append("go_live_summary=blocked")
elif go_live_status == "attention":
attention_reasons.append("go_live_summary=attention")
if publish_status == "blocked":
blocked_reasons.append("publish_status=blocked")
elif publish_status == "attention":
attention_reasons.append("publish_status=attention")
if stack_status == "blocked":
blocked_reasons.append("stack_diagnosis=blocked")
elif stack_status == "attention":
attention_reasons.append("stack_diagnosis=attention")
if driver_launch_status == "blocked":
blocked_reasons.append("driver_feed_launch=blocked")
elif driver_launch_status == "attention":
attention_reasons.append("driver_feed_launch=attention")
if codex_launch_status == "blocked":
blocked_reasons.append("codex_brief_launch=blocked")
elif codex_launch_status == "attention":
attention_reasons.append("codex_brief_launch=attention")
if release_launchpad_status == "blocked":
blocked_reasons.append("release_launchpad=blocked")
elif release_launchpad_status == "attention":
attention_reasons.append("release_launchpad=attention")
if review_status == "blocked":
blocked_reasons.append("go_live_review=blocked")
elif review_status in {"attention", "missing"}:
attention_reasons.append(f"go_live_review={review_status}")
if doctor_status == "blocked":
blocked_reasons.append("doctor_decision=blocked")
elif doctor_status in {"attention", "missing"}:
attention_reasons.append(f"doctor_decision={doctor_status}")
if not bool(launchpad_alignment.get("consistent", False)):
attention_reasons.append("launchpad_alignment=inconsistent")
signoff_status = "ready"
if blocked_reasons:
signoff_status = "blocked"
elif attention_reasons:
signoff_status = "attention"
status_label = {
"ready": "可签字上线",
"attention": "待签字复核",
"blocked": "暂不可签字",
}.get(signoff_status, signoff_status or "未知")
if signoff_status == "blocked":
primary_blocked_surface = (
"正式复核"
if review_status == "blocked"
else "总检主决策"
if doctor_status == "blocked"
else "发布门禁"
)
headline = (
f"当前仍存在{primary_blocked_surface}阻断,建议优先处理 "
f"{str(go_live_summary.get('operator_title') or '').strip() or str(operator_decision.get('title') or '').strip() or str(doctor_decision.get('next_action_code') or '').strip() or '阻断项'}"
)
elif signoff_status == "attention":
if review_status in {"attention", "missing"}:
headline = "当前已进入最终收口阶段,但正式复核仍待补口,建议先确认 bundle manifest 结论再签字上线。"
elif doctor_status in {"attention", "missing"}:
headline = "当前已进入最终收口阶段,但总检主决策仍待补口,建议先按主建议完成复核再签字上线。"
else:
headline = (
f"当前已进入最终收口阶段,但仍建议先完成 {str(go_live_summary.get('operator_title') or '').strip() or str(operator_decision.get('title') or '').strip() or 'attention 项'} 再签字上线。"
)
else:
headline = "当前收口、正式复核、主决策、发布门禁与 launchpad 摘要一致,可以进入最终人工签字或正式发布。"
contract_keys = _normalize_ops_contract_keys(
[
"ops_go_live_signoff_contract",
"ops_go_live_review_contract",
"ops_doctor_decision_contract",
"ops_stack_diagnosis_contract",
"ops_driver_contract",
"release_hub_contract",
"ops_observability_contract",
]
)
recommended_commands = {
**dict(go_live_summary.get("recommended_commands") or {}),
**dict(go_live_review.get("recommended_commands") or {}),
**dict(doctor_decision_payload.get("recommended_commands") or {}),
"release_launchpad": build_bash_command("drive_ops_center.sh", "release-launchpad", normalized_base_url),
"driver_feed": build_bash_command("drive_ops_center.sh", "driver-feed", normalized_base_url),
"codex_brief": build_bash_command("drive_ops_center.sh", "codex-brief", normalized_base_url),
"go_live_signoff": build_bash_command(
"drive_ops_center.sh",
"go-live-signoff",
normalized_report_dir or normalized_base_url,
),
}
return {
"generated_at": datetime.now().isoformat(timespec="seconds"),
"base_url": normalized_base_url,
"contract_key": "ops_go_live_signoff_contract",
"contract_version": str(
(contract_registry.get("contracts_by_key") or {}).get("ops_go_live_signoff_contract", {}).get("version")
or _OPS_CONTRACT_SCHEMA_VERSION
).strip(),
"signoff_status": signoff_status,
"status_label": status_label,
"headline": headline,
"release_gate": {
"go_live_status": go_live_status,
"publish_status": publish_status,
"stack_status": stack_status,
"driver_launch_status": driver_launch_status,
"codex_launch_status": codex_launch_status,
"release_launchpad_status": release_launchpad_status,
"go_live_review_status": review_status,
"doctor_decision_status": doctor_status,
},
"blocked_reasons": blocked_reasons,
"attention_reasons": attention_reasons,
"report_dir": str(go_live_review.get("report_dir") or doctor_decision_payload.get("report_dir") or "").strip(),
"manifest_path": str(go_live_review.get("manifest_path") or doctor_decision_payload.get("manifest_path") or "").strip(),
"decision": {
"operator_title": str(go_live_summary.get("operator_title") or "").strip(),
"next_step_action_code": str(go_live_summary.get("next_step_action_code") or "").strip(),
"next_step_reason": str(go_live_summary.get("next_step_reason") or "").strip(),
"preferred_surface": (
"go-live-review"
if review_status in {"blocked", "attention", "missing"}
else str(doctor_decision.get("preferred_surface") or "").strip()
if doctor_status in {"blocked", "attention", "missing"} and str(doctor_decision.get("preferred_surface") or "").strip()
else "release-launchpad"
if release_launchpad_status in {"blocked", "attention"}
else "codex-brief"
if codex_launch_status in {"blocked", "attention"}
else "go-live-summary"
),
"doctor_reason_code": str(doctor_decision.get("reason_code") or "").strip(),
"doctor_preferred_surface": str(doctor_decision.get("preferred_surface") or "").strip(),
"doctor_next_action_code": str(doctor_decision.get("next_action_code") or "").strip(),
"review_status": review_status,
"review_headline": str(go_live_review.get("headline") or "").strip(),
"review_report_dir": str(go_live_review.get("report_dir") or "").strip(),
"review_manifest_path": str(go_live_review.get("manifest_path") or "").strip(),
"stack_operator_title": str(operator_decision.get("title") or "").strip(),
"stack_next_action_code": str(next_step.get("action_code") or "").strip(),
"launchpad_recommended_target_node_code": str(
go_live_summary.get("launchpad_recommended_target_node_code") or ""
).strip(),
"launchpad_recommended_recovery_label": str(
go_live_summary.get("launchpad_recommended_recovery_label") or ""
).strip(),
"launchpad_recommended_recovery_summary": str(
go_live_summary.get("launchpad_recommended_recovery_summary") or ""
).strip(),
},
"launchpad_alignment": launchpad_alignment,
"recommended_commands": recommended_commands,
"contract_navigation": _build_ops_contract_navigation(
contract_keys,
primary_contract_key="ops_go_live_signoff_contract",
registry=contract_registry,
),
"go_live_summary": go_live_summary,
"go_live_review": {
"status": review_status,
"status_label": str(go_live_review.get("status_label") or "").strip(),
"headline": str(go_live_review.get("headline") or "").strip(),
"report_dir": str(go_live_review.get("report_dir") or "").strip(),
"manifest_path": str(go_live_review.get("manifest_path") or "").strip(),
"critical_failures": list(go_live_review.get("critical_failures") or []),
"noncritical_failures": list(go_live_review.get("noncritical_failures") or []),
},
"doctor_decision": {
"status": doctor_status,
"headline": str(doctor_decision_payload.get("headline") or "").strip(),
"detail": str(doctor_decision_payload.get("detail") or "").strip(),
"summary": dict(doctor_decision_payload.get("summary") or {}),
"decision": doctor_decision,
"report_dir": str(doctor_decision_payload.get("report_dir") or "").strip(),
"manifest_path": str(doctor_decision_payload.get("manifest_path") or "").strip(),
},
"stack_diagnosis": {
"stack_status": stack_status,
"operator_decision": operator_decision,
"next_step": next_step,
},
"driver_feed": {
"headline": str(driver_feed.get("headline") or "").strip(),
"summary": driver_summary,
},
"codex_brief": {
"headline": str(codex_brief.get("headline") or "").strip(),
"summary": codex_summary,
"focus": dict(codex_brief.get("focus") or {}),
},
"release_launchpad": {
"status": release_launchpad_status,
"status_label": str(launchpad_status.get("status_label") or "").strip(),
"recommended_action_code": str(launchpad_status.get("recommended_action_code") or "").strip(),
"recommended_execution_mode": str(launchpad_status.get("recommended_execution_mode") or "").strip(),
},
}
def _ops_go_live_bundle_root() -> Path:
return runtime_root() / "ops-center-reports" / "go-live-bundles"
def _load_json_path(path: Path) -> dict:
if not path.is_file():
return {}
try:
payload = json.loads(path.read_text(encoding="utf-8"))
except Exception:
return {}
return payload if isinstance(payload, dict) else {}
def _find_latest_go_live_bundle_manifest() -> tuple[Path | None, Path | None]:
bundle_root = _ops_go_live_bundle_root()
if not bundle_root.exists():
return None, bundle_root
manifest_paths = sorted(bundle_root.glob("*/manifest.json"), key=lambda item: item.stat().st_mtime, reverse=True)
if not manifest_paths:
return None, bundle_root
manifest_path = manifest_paths[0]
return manifest_path, manifest_path.parent
def _resolve_go_live_bundle_manifest(report_dir: str) -> tuple[Path | None, Path | None]:
normalized_report_dir = str(report_dir or "").strip()
if normalized_report_dir:
candidate_path = Path(normalized_report_dir)
if candidate_path.is_dir():
nested_manifest = candidate_path / "manifest.json"
return (nested_manifest if nested_manifest.is_file() else None), candidate_path
if candidate_path.is_file():
return candidate_path, candidate_path.parent
return None, candidate_path
return _find_latest_go_live_bundle_manifest()
def _load_bundle_json_artifact(artifact_by_key: dict[str, dict], key: str) -> dict:
artifact = artifact_by_key.get(str(key or "").strip()) or {}
artifact_path = Path(str(artifact.get("path") or "").strip())
return _load_json_path(artifact_path) if artifact_path else {}
def _load_bundle_env_audit_summary(artifact_by_key: dict[str, dict]) -> dict:
env_audit_artifact = artifact_by_key.get("env_audit") or {}
env_audit_path = Path(str(env_audit_artifact.get("path") or "").strip())
if not env_audit_path.is_file():
return {}
raw_text = env_audit_path.read_text(encoding="utf-8", errors="replace")
marker = "[8/8] condensed env audit summary"
candidate_text = raw_text.split(marker, 1)[1].strip() if marker in raw_text else raw_text.strip()
if not candidate_text:
return {}
try:
loaded_env_audit = json.loads(candidate_text)
except Exception:
loaded_env_audit = {}
return loaded_env_audit if isinstance(loaded_env_audit, dict) else {}
def get_ops_go_live_bundle(*, base_url: str = "", report_dir: str = "") -> dict:
normalized_base_url = str(base_url or "").strip() or "http://127.0.0.1:8100"
contract_registry = get_ops_contract_registry()
manifest_path, resolved_report_dir = _resolve_go_live_bundle_manifest(report_dir)
recommended_commands = {
"go_live_export": build_bash_command("drive_ops_center.sh", "go-live-export"),
"go_live_review": build_bash_command("drive_ops_center.sh", "go-live-review", str(report_dir or "").strip() or "/path/to/go-live-bundle"),
"go_live_signoff": build_bash_command("drive_ops_center.sh", "go-live-signoff", str(report_dir or "").strip() or "/path/to/go-live-bundle"),
"doctor_export": build_bash_command("drive_ops_center.sh", "doctor-export", "/tmp/domaincheck-go-live", normalized_base_url),
}
contract_keys = _normalize_ops_contract_keys(
[
"ops_go_live_bundle_contract",
"ops_go_live_signoff_contract",
"ops_stack_diagnosis_contract",
"ops_driver_contract",
]
)
if manifest_path is None or not manifest_path.is_file():
bundle_root = resolved_report_dir or _ops_go_live_bundle_root()
headline = (
"当前还没有可复核的 go-live bundle建议先导出证据包再进入 manifest 与最终签收复核。"
)
return {
"generated_at": datetime.now().isoformat(timespec="seconds"),
"base_url": normalized_base_url,
"contract_key": "ops_go_live_bundle_contract",
"contract_version": str(
(contract_registry.get("contracts_by_key") or {}).get("ops_go_live_bundle_contract", {}).get("version")
or _OPS_CONTRACT_SCHEMA_VERSION
).strip(),
"status": "missing",
"status_label": "待导出",
"headline": headline,
"bundle_available": False,
"report_dir": str(bundle_root),
"manifest_path": "",
"artifact_total": 0,
"failure_total": 0,
"critical_failures": [],
"noncritical_failures": [],
"env_audit": {},
"summary": {
"ok": False,
"review_status_hint": "missing",
"review_headline": headline,
"launchpad_alignment": {"consistent": False},
},
"recommended_commands": recommended_commands,
"contract_navigation": _build_ops_contract_navigation(
contract_keys,
primary_contract_key="ops_go_live_bundle_contract",
registry=contract_registry,
),
}
manifest = _load_json_path(manifest_path)
artifacts = list(manifest.get("artifacts") or [])
failures = [str(item or "").strip() for item in list(manifest.get("failures") or []) if str(item or "").strip()]
artifact_by_key = {
str(item.get("key") or "").strip(): dict(item)
for item in artifacts
if isinstance(item, dict) and str(item.get("key") or "").strip()
}
summary = dict(manifest.get("summary") or {})
env_audit = _load_bundle_env_audit_summary(artifact_by_key)
review_status_hint = str(summary.get("review_status_hint") or "").strip()
status = review_status_hint or ("ready" if bool(summary.get("ok", False)) else "attention")
status_label = {
"ready": "证据可交付",
"attention": "证据待复核",
"blocked": "证据阻断",
"missing": "待导出",
}.get(status, status or "未知")
headline = (
str(summary.get("review_headline") or "").strip()
or "当前已读取最新 go-live bundle可继续做 manifest 与签收复核。"
)
critical_keys = {"go_live_summary", "stack_diagnosis", "driver_feed", "codex_brief", "release_launchpad"}
critical_failures = [item for item in failures if item in critical_keys]
noncritical_failures = [item for item in failures if item not in critical_keys]
launchpad_alignment = dict(summary.get("launchpad_alignment") or {})
return {
"generated_at": datetime.now().isoformat(timespec="seconds"),
"base_url": normalized_base_url,
"contract_key": "ops_go_live_bundle_contract",
"contract_version": str(
(contract_registry.get("contracts_by_key") or {}).get("ops_go_live_bundle_contract", {}).get("version")
or _OPS_CONTRACT_SCHEMA_VERSION
).strip(),
"status": status,
"status_label": status_label,
"headline": headline,
"bundle_available": True,
"report_dir": str(manifest.get("report_dir") or resolved_report_dir or manifest_path.parent),
"manifest_path": str(manifest_path),
"artifact_total": int(summary.get("artifact_total", len(artifacts)) or len(artifacts)),
"failure_total": int(summary.get("failure_total", len(failures)) or len(failures)),
"critical_failures": critical_failures,
"noncritical_failures": noncritical_failures,
"env_audit": {
"status": str(env_audit.get("status") or "").strip(),
"headline": str(env_audit.get("headline") or "").strip(),
"missing_items": list(env_audit.get("missing_items") or []),
"runtime_may_need_restart": bool(((env_audit.get("runtime") or {}).get("runtime_may_need_restart", False))),
"recommended_actions": [
str(item).strip()
for item in list(env_audit.get("recommended_actions") or [])
if str(item).strip()
],
},
"summary": {
"ok": bool(summary.get("ok", False)),
"review_status_hint": review_status_hint or status,
"review_headline": headline,
"launchpad_recommended_target_node_code": str(
summary.get("launchpad_recommended_target_node_code") or ""
).strip(),
"launchpad_recommended_recovery_label": str(
summary.get("launchpad_recommended_recovery_label") or ""
).strip(),
"launchpad_recommended_recovery_summary": str(
summary.get("launchpad_recommended_recovery_summary") or ""
).strip(),
"launchpad_onboarding_bootstrap_pending_nodes": int(
summary.get("launchpad_onboarding_bootstrap_pending_nodes", 0) or 0
),
"launchpad_onboarding_acceptance_ready_nodes": int(
summary.get("launchpad_onboarding_acceptance_ready_nodes", 0) or 0
),
"launchpad_alignment": launchpad_alignment,
"recommended_reading_order": list(summary.get("recommended_reading_order") or []),
"contract_surface_gaps": list(summary.get("contract_surface_gaps") or []),
"discovered_contract_keys": list(summary.get("discovered_contract_keys") or []),
},
"artifacts": artifacts,
"failures": failures,
"recommended_commands": {
**recommended_commands,
"go_live_review": build_bash_command("drive_ops_center.sh", "go-live-review", str(manifest_path)),
"go_live_signoff": build_bash_command("drive_ops_center.sh", "go-live-signoff", str(manifest_path)),
},
"contract_navigation": _build_ops_contract_navigation(
contract_keys,
primary_contract_key="ops_go_live_bundle_contract",
registry=contract_registry,
),
}
def get_ops_go_live_review(*, base_url: str = "", report_dir: str = "") -> dict:
normalized_base_url = str(base_url or "").strip() or "http://127.0.0.1:8100"
normalized_report_dir = str(report_dir or "").strip()
contract_registry = get_ops_contract_registry()
manifest_path, resolved_report_dir = _resolve_go_live_bundle_manifest(report_dir)
contract_keys = _normalize_ops_contract_keys(
[
"ops_go_live_review_contract",
"ops_go_live_bundle_contract",
"ops_doctor_decision_contract",
"ops_go_live_signoff_contract",
"ops_stack_diagnosis_contract",
"ops_driver_contract",
"release_hub_contract",
]
)
recommended_commands = {
"go_live_export": build_bash_command("drive_ops_center.sh", "go-live-export"),
"go_live_review": build_bash_command("drive_ops_center.sh", "go-live-review", normalized_report_dir or "/path/to/go-live-bundle"),
"go_live_signoff": build_bash_command("drive_ops_center.sh", "go-live-signoff", normalized_report_dir or "/path/to/go-live-bundle"),
"doctor_decision": build_bash_command("drive_ops_center.sh", "doctor-decision", normalized_report_dir or normalized_base_url),
}
if manifest_path is None or not manifest_path.is_file():
headline = "当前还没有可复核的 go-live bundle建议先导出证据包再进入正式复核。"
return {
"generated_at": datetime.now().isoformat(timespec="seconds"),
"base_url": normalized_base_url,
"contract_key": "ops_go_live_review_contract",
"contract_version": str(
(contract_registry.get("contracts_by_key") or {}).get("ops_go_live_review_contract", {}).get("version")
or _OPS_CONTRACT_SCHEMA_VERSION
).strip(),
"status": "missing",
"status_label": "待导出",
"headline": headline,
"report_dir": str(resolved_report_dir or _ops_go_live_bundle_root()),
"manifest_path": "",
"artifact_total": 0,
"failure_total": 0,
"critical_failures": [],
"noncritical_failures": [],
"env_audit": {},
"go_live_summary": {},
"stack_diagnosis": {},
"driver_feed": {},
"codex_brief": {},
"launchpad_alignment": {"consistent": False, "available_sources": []},
"recommended_reading_order": [],
"recommended_next_steps": [
"先执行 go-live-export导出完整上线证据包。",
"导出完成后再执行 go-live-review 或直接打开页面里的证据复核详情。",
],
"recommended_commands": recommended_commands,
"contract_navigation": _build_ops_contract_navigation(
contract_keys,
primary_contract_key="ops_go_live_review_contract",
registry=contract_registry,
),
}
manifest = _load_json_path(manifest_path)
artifacts = list(manifest.get("artifacts") or [])
failures = [str(item or "").strip() for item in list(manifest.get("failures") or []) if str(item or "").strip()]
summary = dict(manifest.get("summary") or {})
artifact_by_key = {
str(item.get("key") or "").strip(): dict(item)
for item in artifacts
if isinstance(item, dict) and str(item.get("key") or "").strip()
}
env_audit = _load_bundle_env_audit_summary(artifact_by_key)
go_live_summary_payload = _load_bundle_json_artifact(artifact_by_key, "go_live_summary")
stack_diagnosis_payload = _load_bundle_json_artifact(artifact_by_key, "stack_diagnosis")
driver_feed_payload = _load_bundle_json_artifact(artifact_by_key, "driver_feed")
codex_brief_payload = _load_bundle_json_artifact(artifact_by_key, "codex_brief")
critical_keys = {"go_live_summary", "stack_diagnosis", "driver_feed", "codex_brief", "release_launchpad"}
critical_failures = sorted(
{
key
for key in critical_keys
if key in failures or not bool((artifact_by_key.get(key) or {}).get("ok", False))
}
)
noncritical_failures = [key for key in failures if key not in critical_keys]
go_live_launchpad_target_node_code = str(go_live_summary_payload.get("launchpad_recommended_target_node_code") or "").strip()
go_live_launchpad_recovery_label = str(go_live_summary_payload.get("launchpad_recommended_recovery_label") or "").strip()
go_live_launchpad_recovery_summary = str(go_live_summary_payload.get("launchpad_recommended_recovery_summary") or "").strip()
go_live_launchpad_bootstrap_pending_nodes = int(go_live_summary_payload.get("launchpad_onboarding_bootstrap_pending_nodes", 0) or 0)
go_live_launchpad_acceptance_ready_nodes = int(go_live_summary_payload.get("launchpad_onboarding_acceptance_ready_nodes", 0) or 0)
stack_diagnosis = dict(stack_diagnosis_payload.get("diagnosis") or {})
driver_feed_summary = dict(driver_feed_payload.get("summary") or {})
codex_brief_summary = dict(codex_brief_payload.get("summary") or {})
launchpad_target_node_codes = {
"go_live_summary": go_live_launchpad_target_node_code,
"stack_diagnosis": str(stack_diagnosis.get("launchpad_recommended_target_node_code") or "").strip(),
"driver_feed": str(driver_feed_summary.get("launchpad_recommended_target_node_code") or "").strip(),
"codex_brief": str(codex_brief_summary.get("launchpad_recommended_target_node_code") or "").strip(),
}
launchpad_recovery_labels = {
"go_live_summary": go_live_launchpad_recovery_label,
"stack_diagnosis": str(stack_diagnosis.get("launchpad_recommended_recovery_label") or "").strip(),
"driver_feed": str(driver_feed_summary.get("launchpad_recommended_recovery_label") or "").strip(),
"codex_brief": str(codex_brief_summary.get("launchpad_recommended_recovery_label") or "").strip(),
}
launchpad_recovery_summaries = {
"go_live_summary": go_live_launchpad_recovery_summary,
"stack_diagnosis": str(stack_diagnosis.get("launchpad_recommended_recovery_summary") or "").strip(),
"driver_feed": str(driver_feed_summary.get("launchpad_recommended_recovery_summary") or "").strip(),
"codex_brief": str(codex_brief_summary.get("launchpad_recommended_recovery_summary") or "").strip(),
}
launchpad_bootstrap_pending_nodes = {
"go_live_summary": go_live_launchpad_bootstrap_pending_nodes,
"stack_diagnosis": int(stack_diagnosis.get("launchpad_onboarding_bootstrap_pending_nodes", 0) or 0),
"driver_feed": int(driver_feed_summary.get("launchpad_onboarding_bootstrap_pending_nodes", 0) or 0),
"codex_brief": int(codex_brief_summary.get("launchpad_onboarding_bootstrap_pending_nodes", 0) or 0),
}
launchpad_acceptance_ready_nodes = {
"go_live_summary": go_live_launchpad_acceptance_ready_nodes,
"stack_diagnosis": int(stack_diagnosis.get("launchpad_onboarding_acceptance_ready_nodes", 0) or 0),
"driver_feed": int(driver_feed_summary.get("launchpad_onboarding_acceptance_ready_nodes", 0) or 0),
"codex_brief": int(codex_brief_summary.get("launchpad_onboarding_acceptance_ready_nodes", 0) or 0),
}
available_alignment_sources = {
source
for source, payload_value in {
"go_live_summary": go_live_summary_payload,
"stack_diagnosis": stack_diagnosis_payload,
"driver_feed": driver_feed_payload,
"codex_brief": codex_brief_payload,
}.items()
if isinstance(payload_value, dict) and payload_value
}
def _nonempty_distinct(values: dict[str, str]) -> list[str]:
return sorted({str(value or "").strip() for key, value in values.items() if key in available_alignment_sources and str(value or "").strip()})
def _distinct_ints(values: dict[str, int]) -> list[int]:
return sorted({int(value) for key, value in values.items() if key in available_alignment_sources})
launchpad_alignment = {
"consistent": True,
"target_node_code_consistent": len(_nonempty_distinct(launchpad_target_node_codes)) <= 1,
"recovery_label_consistent": len(_nonempty_distinct(launchpad_recovery_labels)) <= 1,
"recovery_summary_consistent": len(_nonempty_distinct(launchpad_recovery_summaries)) <= 1,
"bootstrap_pending_consistent": len(_distinct_ints(launchpad_bootstrap_pending_nodes)) <= 1,
"acceptance_ready_consistent": len(_distinct_ints(launchpad_acceptance_ready_nodes)) <= 1,
"available_sources": sorted(available_alignment_sources),
"target_node_codes": launchpad_target_node_codes,
"recovery_labels": launchpad_recovery_labels,
"recovery_summaries": launchpad_recovery_summaries,
"bootstrap_pending_nodes": launchpad_bootstrap_pending_nodes,
"acceptance_ready_nodes": launchpad_acceptance_ready_nodes,
}
launchpad_alignment["consistent"] = all(
bool(launchpad_alignment.get(key))
for key in (
"target_node_code_consistent",
"recovery_label_consistent",
"recovery_summary_consistent",
"bootstrap_pending_consistent",
"acceptance_ready_consistent",
)
)
env_audit_status = str(env_audit.get("status") or "").strip()
env_audit_blocking = env_audit_status == "blocked"
env_audit_attention = env_audit_status == "attention"
env_audit_missing_items = [
str(item).strip()
for item in list(env_audit.get("missing_items") or [])
if str(item).strip()
]
env_audit_recommended_actions = [
str(item).strip()
for item in list(env_audit.get("recommended_actions") or [])
if str(item).strip()
]
env_audit_runtime_may_need_restart = bool(((env_audit.get("runtime") or {}).get("runtime_may_need_restart", False)))
node_agent_env_missing = any(item == "missing_env:/etc/default/domaincheck-node-agent" for item in env_audit_missing_items)
if critical_failures or env_audit_blocking:
status = "blocked"
if critical_failures and env_audit_blocking:
headline = "关键报告存在缺失,且环境审计已给出阻断结论,当前不适合上线。"
elif critical_failures:
headline = "上线证据包仍缺关键报告,暂不建议直接宣称收口完成。"
else:
headline = "环境审计已给出阻断结论,需先修复现场环境后再进入发布门禁。"
elif noncritical_failures or env_audit_attention or not bool(launchpad_alignment.get("consistent")):
status = "attention"
if not bool(launchpad_alignment.get("consistent")):
headline = "核心报告已齐,但 launchpad 摘要在不同 surface 之间存在漂移,建议先复核再发版。"
elif noncritical_failures and env_audit_attention:
headline = "核心报告已齐,但补充报告和环境审计都提示仍有缺口,建议先复核再发版。"
elif noncritical_failures:
headline = "核心报告已齐,但仍有补充报告超时或失败,建议先复核再发版。"
else:
headline = "核心报告已齐,但环境审计仍提示基础缺口,建议补齐后再发版。"
else:
status = "ready"
headline = "核心上线证据已经齐备,可进入最终人工复核或正式发布门禁。"
recommended_next_steps = [
"先看 00_env_audit.txt确认当前机器环境是否存在阻断或注意项",
"先看 02_go_live_summary.json 和 03_stack_diagnosis.json",
"再看 04_driver_feed.json 和 05_codex_brief.json",
"若涉及发布门禁,再看 06_release_launchpad.json",
"若 manifest 里仍有失败项,再看对应 artifact 原文件",
]
if node_agent_env_missing:
recommended_next_steps.extend(
[
"当前缺口指向 Node Agent 未接管:先执行 agent-gap-export拿到首个缺口节点的接管包",
"随后执行 node-bootstrap-plan为缺口节点生成可直接落地的接管脚本",
"如果仓库代码已更新但 bootstrap-plan 输出仍偏旧,先重启控制面 API 再重新导出接管计划",
]
)
if env_audit_runtime_may_need_restart:
recommended_next_steps.extend(
[
"env-audit 已识别运行中 API 可能仍是旧代码:先执行 drive_ops_center.sh runtime-refresh-recover按固定链路完成 API 重启与复检",
"若仍需人工逐条确认,再执行 recommended_actions 里的 API 重启与复检命令",
]
)
if go_live_launchpad_target_node_code:
recommended_next_steps.extend(
[
f"当前 launchpad 缺口已经收敛到节点 {go_live_launchpad_target_node_code},建议优先执行:{go_live_launchpad_recovery_label or '节点恢复'}",
"先对照 02_go_live_summary.json、04_driver_feed.json、05_codex_brief.json 中的 launchpad 字段,确认三处摘要是否一致",
go_live_launchpad_recovery_summary or "如果 recovery_summary 仍为空,再回到 06_release_launchpad.json 查看 gap rows 详情",
]
)
if not bool(launchpad_alignment.get("consistent")):
recommended_next_steps.extend(
[
"当前 bundle 内的 launchpad 摘要在 go_live_summary / stack_diagnosis / driver_feed / codex_brief 之间存在不一致,先不要直接交付。",
"优先复核 02_go_live_summary.json 与 03_stack_diagnosis.json再检查 04_driver_feed.json 与 05_codex_brief.json 是否使用了同一版控制面数据。",
"若只有局部摘要偏旧,先执行 drive_ops_center.sh runtime-refresh-recover并重新导出 go-live bundle。",
]
)
return {
"generated_at": datetime.now().isoformat(timespec="seconds"),
"base_url": normalized_base_url,
"contract_key": "ops_go_live_review_contract",
"contract_version": str(
(contract_registry.get("contracts_by_key") or {}).get("ops_go_live_review_contract", {}).get("version")
or _OPS_CONTRACT_SCHEMA_VERSION
).strip(),
"status": status,
"status_label": {
"ready": "可复核发布",
"attention": "待复核补口",
"blocked": "暂不可发布",
"missing": "待导出",
}.get(status, status or "未知"),
"headline": headline,
"report_dir": str(manifest.get("report_dir") or resolved_report_dir or manifest_path.parent),
"manifest_path": str(manifest_path),
"artifact_total": int(summary.get("artifact_total", len(artifacts)) or len(artifacts)),
"failure_total": int(summary.get("failure_total", len(failures)) or len(failures)),
"critical_failures": critical_failures,
"noncritical_failures": noncritical_failures,
"env_audit": {
"ok": bool((artifact_by_key.get("env_audit") or {}).get("ok", False)),
"status": env_audit_status,
"headline": str(env_audit.get("headline") or "").strip(),
"missing_items": env_audit_missing_items,
"runtime_may_need_restart": env_audit_runtime_may_need_restart,
"recommended_actions": env_audit_recommended_actions,
},
"go_live_summary": {
"go_live_status": str(go_live_summary_payload.get("go_live_status") or "").strip(),
"publish_status": str(go_live_summary_payload.get("publish_status") or "").strip(),
"operator_title": str(go_live_summary_payload.get("operator_title") or "").strip(),
"next_step_action_code": str(go_live_summary_payload.get("next_step_action_code") or "").strip(),
"launchpad_recommended_target_node_code": go_live_launchpad_target_node_code,
"launchpad_recommended_recovery_label": go_live_launchpad_recovery_label,
"launchpad_recommended_recovery_summary": go_live_launchpad_recovery_summary,
"launchpad_onboarding_bootstrap_pending_nodes": go_live_launchpad_bootstrap_pending_nodes,
"launchpad_onboarding_acceptance_ready_nodes": go_live_launchpad_acceptance_ready_nodes,
},
"stack_diagnosis": {
"stack_status": str(stack_diagnosis.get("stack_status") or "").strip(),
"operator_title": str((dict(stack_diagnosis.get("operator_decision") or {})).get("title") or "").strip(),
"next_step_action_code": str((dict(stack_diagnosis.get("next_step") or {})).get("action_code") or "").strip(),
"launchpad_recommended_target_node_code": str(stack_diagnosis.get("launchpad_recommended_target_node_code") or "").strip(),
"launchpad_recommended_recovery_label": str(stack_diagnosis.get("launchpad_recommended_recovery_label") or "").strip(),
"launchpad_recommended_recovery_summary": str(stack_diagnosis.get("launchpad_recommended_recovery_summary") or "").strip(),
"launchpad_onboarding_bootstrap_pending_nodes": int(stack_diagnosis.get("launchpad_onboarding_bootstrap_pending_nodes", 0) or 0),
"launchpad_onboarding_acceptance_ready_nodes": int(stack_diagnosis.get("launchpad_onboarding_acceptance_ready_nodes", 0) or 0),
},
"driver_feed": {
"headline": str(driver_feed_payload.get("headline") or "").strip(),
"launch_status": str((dict(driver_feed_payload.get("automation_coverage") or {})).get("launch_status") or "").strip(),
"launch_ready": bool((dict(driver_feed_payload.get("automation_coverage") or {})).get("launch_ready", False)),
"launchpad_recommended_target_node_code": str(driver_feed_summary.get("launchpad_recommended_target_node_code") or "").strip(),
"launchpad_recommended_recovery_label": str(driver_feed_summary.get("launchpad_recommended_recovery_label") or "").strip(),
},
"codex_brief": {
"headline": str(codex_brief_payload.get("headline") or "").strip(),
"focus_entry_key": str((dict(codex_brief_payload.get("focus") or {})).get("entry_key") or "").strip(),
"launch_status": str((dict(codex_brief_payload.get("automation_coverage") or {})).get("launch_status") or "").strip(),
"launch_ready": bool((dict(codex_brief_payload.get("automation_coverage") or {})).get("launch_ready", False)),
"launchpad_recommended_target_node_code": str(codex_brief_summary.get("launchpad_recommended_target_node_code") or "").strip(),
"launchpad_recommended_recovery_label": str(codex_brief_summary.get("launchpad_recommended_recovery_label") or "").strip(),
},
"launchpad_alignment": launchpad_alignment,
"recommended_reading_order": list(summary.get("recommended_reading_order") or []),
"recommended_next_steps": _dedupe_command_list(recommended_next_steps),
"recommended_commands": {
**recommended_commands,
"go_live_review": build_bash_command("drive_ops_center.sh", "go-live-review", str(manifest_path)),
"go_live_signoff": build_bash_command("drive_ops_center.sh", "go-live-signoff", str(manifest_path)),
"doctor_decision": build_bash_command("drive_ops_center.sh", "doctor-decision", str(manifest_path)),
},
"contract_navigation": _build_ops_contract_navigation(
contract_keys,
primary_contract_key="ops_go_live_review_contract",
registry=contract_registry,
),
}
def get_ops_blueprint() -> dict:
return {
"contract_registry": get_ops_contract_registry(),
"architecture": {
"control_plane": {
"region": "overseas",
"responsibility": [
"web backend",
"ops api",
"release registry",
"task scheduler",
"log aggregation entry",
],
},
"managed_nodes": {
"region": "mainland",
"responsibility": [
"worker/controller runtime",
"node agent",
"systemd execution",
"diagnostics collection",
],
},
},
"execution_model": {
"default_transport": "https-polling",
"future_transport": "websocket",
"fallback_transport": "ssh",
"job_result_contract": {
"required_fields": [
"job_id",
"node_code",
"step",
"started_at",
"finished_at",
"exit_code",
"stdout",
"stderr",
"structured_result",
]
},
},
"mvp_scope": {
"phase_1": [
"ops overview api",
"action catalog api",
"worker participation / standby visibility",
"remote log sync switch",
],
"phase_2": [
"ops jobs table",
"node registration token",
"node heartbeat and pull-task api",
"service restart / health check actions",
],
"phase_3": [
"release package distribution",
"one-click bootstrap",
"rollback",
"streaming logs",
],
},
"operator_model": {
"with_codex": "海外 Codex 作为智能驾驶员,自动分析现网状态并选择合适动作。",
"without_codex": "后台按钮直接创建 ops job由控制面和 agent 自动完成执行与回执。",
},
}
def get_ops_runbook() -> dict:
runtime = get_runtime_status()
readiness = runtime.get("readiness") or {}
worker_runtime = runtime.get("worker") or {}
sync_agent_runtime = runtime.get("sync_agent") or {}
managed_nodes_payload = list_managed_nodes_with_agent_state(participation_payload=runtime.get("detect") or {})
release_launchpad = get_release_launchpad()
control_sequences = _attach_ops_runbook_sequence_resolutions(
_build_ops_runbook_control_sequences(
managed_nodes_payload=managed_nodes_payload,
release_launchpad=release_launchpad,
),
requested_by="api/runbook",
)
return {
"entrypoint": {
"preferred_host": str(settings.sync_target_api_base_url or "").strip() or f"http://127.0.0.1:{settings.api_port}",
"api_prefix": settings.api_prefix,
},
"services": {
"api": {
"service_name": settings.api_service_name,
"status": "running",
},
"worker": {
"service_name": settings.worker_service_name,
"status": _bool_label(bool(worker_runtime.get("running", False))),
},
"sync_agent": {
"service_name": settings.sync_agent_service_name,
"status": _bool_label(bool(sync_agent_runtime.get("running", False))),
},
},
"fast_checks": [
"GET /health",
"GET /api/v1/runtime/status",
"GET /api/v1/runtime/cluster",
"GET /api/v1/runtime/readiness",
"GET /api/v1/ops/stack-diagnosis",
"GET /api/v1/ops/overview",
"GET /api/v1/ops/contracts",
"GET /api/v1/ops/contracts/ops_stack_diagnosis_contract",
"GET /api/v1/ops/releases/launchpad",
"GET /api/v1/ops/activity-stream",
"GET /api/v1/ops/capabilities",
],
"current_readiness": {
"status": str(readiness.get("status") or ""),
"summary": str(readiness.get("summary") or ""),
},
"release_launchpad": release_launchpad,
"control_sequences": control_sequences,
}