11640 lines
550 KiB
Python
11640 lines
550 KiB
Python
from __future__ import annotations
|
||
|
||
import json
|
||
from datetime import datetime, timedelta
|
||
from pathlib import Path
|
||
import re
|
||
import time
|
||
|
||
from app.core.config import settings
|
||
from app.core.files import runtime_root
|
||
from app.services.ops_command_service import build_bash_command
|
||
from app.services.ops_agent_service import (
|
||
execute_managed_node_onboarding_recovery,
|
||
get_managed_node_handover,
|
||
list_ops_job_events_for_jobs,
|
||
list_managed_nodes_with_agent_state,
|
||
)
|
||
from app.services.ops_execution_mode_service import execution_mode_label
|
||
from app.services.ops_job_service import create_ops_job_batch, get_ops_job_summary, list_ops_jobs, sync_managed_nodes_from_cluster
|
||
from app.services.ops_playbook_service import (
|
||
execute_ops_playbook,
|
||
get_ops_playbook,
|
||
get_ops_playbook_run,
|
||
get_recent_ops_playbook_runs,
|
||
list_ops_playbook_run_events,
|
||
preview_ops_playbook,
|
||
)
|
||
from app.services.ops_release_service import (
|
||
build_release_execution_mode_recommendation,
|
||
create_release_and_smart_rollout_from_latest_package,
|
||
get_latest_release_package_metadata,
|
||
create_smart_release_rollout,
|
||
get_latest_release,
|
||
get_release_launchpad,
|
||
get_release_summary,
|
||
list_release_rollouts,
|
||
prepare_latest_release_package,
|
||
)
|
||
from app.services.ops_template_service import get_ops_action_template
|
||
from app.services.build_info_service import get_runtime_build_info
|
||
from app.services.runtime_status_service import get_runtime_status
|
||
from app.services.runtime_settings_service import get_runtime_settings, update_runtime_settings
|
||
from app.services.sync_record_service import get_sync_summary
|
||
|
||
_OPS_INSPECTION_FETCH_LIMIT = 80
|
||
_OPS_ACTIVITY_FETCH_LIMIT = 18
|
||
_OPS_INSPECTION_ACTION_KEYS = ("health.snapshot", "logs.collect", "diagnostics.collect")
|
||
_OPS_CONTRACT_REGISTRY_VERSION = "2026-04-18"
|
||
_OPS_CONTRACT_SCHEMA_VERSION = "v1"
|
||
_REMOTE_LOG_PREVIEW_LINE_RE = re.compile(r"^\[(?P<created_at>[^\]]+)\]\s+\[(?P<node_code>[^\]]+)\]\s+(?P<message>.*)$")
|
||
_BACKEND_DRIVER_ACTION_CODES = {
|
||
"disable_log_sync",
|
||
"enable_log_sync_key",
|
||
"enable_log_sync_full",
|
||
"replay_delivery_queue",
|
||
"flush_delivery_queue",
|
||
"run_scene_logs_key",
|
||
"run_scene_logs_full",
|
||
"open_playbook_dialog",
|
||
"open_action_template_dialog",
|
||
"focus_playbook_run",
|
||
"focus_activity_item",
|
||
"focus_latest_job_events",
|
||
"open_playbook_run_latest_events",
|
||
"handover_first_gap",
|
||
"view_first_gap",
|
||
"open_rollout_dialog",
|
||
"open_release_dialog",
|
||
"open_release_deploy_control",
|
||
"open_release_deploy_worker",
|
||
"open_release_deploy_custom",
|
||
"create_smart_release_rollout_worker",
|
||
"create_smart_release_rollout_control",
|
||
"create_release_rollout_worker",
|
||
"create_release_rollout_control",
|
||
"focus_release_hub",
|
||
"open_worker_logs_participating",
|
||
"open_worker_logs_standby",
|
||
"open_worker_logs",
|
||
"run_inspection_participating",
|
||
"run_inspection_standby",
|
||
"run_standard_inspection",
|
||
"open_diagnostics_participating",
|
||
"open_diagnostics",
|
||
"publish_latest_worker",
|
||
"review_smart_rollout_preview",
|
||
"review_control_rollout",
|
||
"fix_rollout_blockers",
|
||
"fix_managed_nodes",
|
||
"bootstrap_run",
|
||
"run_acceptance",
|
||
"release_package",
|
||
"release_prepare",
|
||
"api-restart",
|
||
}
|
||
_SAFE_AUTO_DRIVER_ACTION_CODES = {
|
||
"disable_log_sync",
|
||
"enable_log_sync_key",
|
||
"enable_log_sync_full",
|
||
"run_scene_logs_key",
|
||
"run_scene_logs_full",
|
||
"open_playbook_dialog",
|
||
"open_action_template_dialog",
|
||
"focus_playbook_run",
|
||
"focus_activity_item",
|
||
"focus_latest_job_events",
|
||
"open_playbook_run_latest_events",
|
||
"handover_first_gap",
|
||
"view_first_gap",
|
||
"open_rollout_dialog",
|
||
"open_release_dialog",
|
||
"open_release_deploy_control",
|
||
"open_release_deploy_worker",
|
||
"open_release_deploy_custom",
|
||
"focus_release_hub",
|
||
"release_package",
|
||
"release_prepare",
|
||
"api-restart",
|
||
"review_smart_rollout_preview",
|
||
"review_control_rollout",
|
||
"fix_rollout_blockers",
|
||
"open_worker_logs_participating",
|
||
"open_worker_logs_standby",
|
||
"open_worker_logs",
|
||
"open_diagnostics_participating",
|
||
"open_diagnostics",
|
||
"run_inspection_participating",
|
||
"run_inspection_standby",
|
||
"run_standard_inspection",
|
||
}
|
||
_GUARDED_AUTO_DRIVER_ACTION_CODES = {
|
||
"replay_delivery_queue",
|
||
"flush_delivery_queue",
|
||
"publish_latest_worker",
|
||
"create_smart_release_rollout_worker",
|
||
"create_smart_release_rollout_control",
|
||
"create_release_rollout_worker",
|
||
"create_release_rollout_control",
|
||
}
|
||
_MIXED_DRIVER_ACTION_CODES = {
|
||
"fix_managed_nodes",
|
||
"bootstrap_run",
|
||
"run_acceptance",
|
||
}
|
||
_UI_ONLY_DRIVER_ACTION_CODES = set()
|
||
|
||
_OPS_CONTRACT_DEFINITIONS = (
|
||
{
|
||
"key": "ops_job_contract",
|
||
"title": "Ops Job Contract",
|
||
"version": _OPS_CONTRACT_SCHEMA_VERSION,
|
||
"status": "active",
|
||
"summary": "冻结 ops job / step / event / policy / approve / cancel / dispatch 的正式执行对象 contract。",
|
||
"schema_doc_path": "docs/schemas/ops_job_contract.md",
|
||
"primary_endpoint": "/api/v1/ops/jobs",
|
||
"discovery_endpoints": [
|
||
"/api/v1/ops/jobs",
|
||
"/api/v1/ops/jobs/{job_id}",
|
||
"/api/v1/ops/jobs/{job_id}/events",
|
||
"/api/v1/ops/jobs",
|
||
"/api/v1/ops/jobs/batch",
|
||
"/api/v1/ops/policy/preview",
|
||
"/api/v1/ops/jobs/{job_id}/approve",
|
||
"/api/v1/ops/jobs/{job_id}/cancel",
|
||
"/api/v1/ops/jobs/{job_id}/dispatch",
|
||
],
|
||
"service_keys": [
|
||
"app.services.ops_job_service",
|
||
"app.services.ops_agent_service",
|
||
],
|
||
"consumers": [
|
||
"ops-center",
|
||
"codex-driver",
|
||
"cli",
|
||
"node-agent",
|
||
"release-hub",
|
||
"playbook-runner",
|
||
],
|
||
"related_contract_keys": [
|
||
"ops_agent_protocol",
|
||
"release_hub_contract",
|
||
"ops_playbook_contract",
|
||
"ops_observability_contract",
|
||
"ops_stack_diagnosis_contract",
|
||
],
|
||
"notes": [
|
||
"ops job 是正式执行颗粒度对象,playbook run 与 rollout 最终都应收敛到它。",
|
||
"approval / cancel / dispatch 不只是按钮动作,而是正式状态迁移。",
|
||
"页面列表默认消费 compact job,详情页再读取 full job + steps + events。",
|
||
],
|
||
},
|
||
{
|
||
"key": "ops_agent_protocol",
|
||
"title": "Node Agent Protocol",
|
||
"version": _OPS_CONTRACT_SCHEMA_VERSION,
|
||
"status": "active",
|
||
"summary": "冻结海外控制面与大陆 Node Agent 之间的认证、注册、心跳、拉任务、完成回执与事件回放 contract。",
|
||
"schema_doc_path": "docs/schemas/ops_agent_protocol.md",
|
||
"primary_endpoint": "/api/v1/ops/agent/bootstrap-plan",
|
||
"discovery_endpoints": [
|
||
"/api/v1/ops/agent/tokens",
|
||
"/api/v1/ops/agent/bootstrap-plan",
|
||
"/api/v1/ops/agent/register",
|
||
"/api/v1/ops/agent/heartbeat",
|
||
"/api/v1/ops/agent/pull",
|
||
"/api/v1/ops/agent/jobs/{job_id}/start",
|
||
"/api/v1/ops/agent/jobs/{job_id}/complete",
|
||
"/api/v1/ops/agent/jobs/{job_id}/events",
|
||
"/api/v1/ops/nodes/{node_code}/delivery-queue",
|
||
"/api/v1/ops/nodes/{node_code}/delivery-queue/records",
|
||
"/api/v1/ops/nodes/{node_code}/delivery-queue/flush",
|
||
"/api/v1/ops/nodes/{node_code}/delivery-queue/replay",
|
||
"/api/v1/ops/nodes/{node_code}/delivery-queue/records/{record_id}/replay",
|
||
"/api/v1/ops/nodes/{node_code}/delivery-queue/records/{record_id}/discard",
|
||
],
|
||
"service_keys": [
|
||
"app.services.ops_agent_service",
|
||
"app.node_agent",
|
||
],
|
||
"consumers": [
|
||
"ops-center",
|
||
"codex-driver",
|
||
"cli",
|
||
"node-agent",
|
||
],
|
||
"related_contract_keys": [
|
||
"ops_job_contract",
|
||
"ops_observability_contract",
|
||
"ops_stack_diagnosis_contract",
|
||
],
|
||
"notes": [
|
||
"Agent 只执行结构化动作,不长期接受任意 shell。",
|
||
"complete / events 回执已经具备 client_request_id / client_event_id 幂等语义。",
|
||
"heartbeat 已纳入 delivery_queue 快照,可暴露 retrying / dead_letter 现场。",
|
||
],
|
||
},
|
||
{
|
||
"key": "release_hub_contract",
|
||
"title": "Release Hub Contract",
|
||
"version": _OPS_CONTRACT_SCHEMA_VERSION,
|
||
"status": "active",
|
||
"summary": "冻结 Release / Rollout / Launchpad / Default Gate 的对象模型与门禁口径。",
|
||
"schema_doc_path": "docs/schemas/release_hub_contract.md",
|
||
"primary_endpoint": "/api/v1/ops/releases/launchpad",
|
||
"discovery_endpoints": [
|
||
"/api/v1/ops/overview",
|
||
"/api/v1/ops/releases",
|
||
"/api/v1/ops/releases/latest",
|
||
"/api/v1/ops/releases/launchpad",
|
||
"/api/v1/ops/releases/{release_id}",
|
||
"/api/v1/ops/releases/{release_id}/rollouts",
|
||
"/api/v1/ops/rollouts/{rollout_id}",
|
||
"/api/v1/ops/rollouts/{rollout_id}/jobs",
|
||
"/api/v1/ops/rollouts/{rollout_id}/advance",
|
||
],
|
||
"service_keys": [
|
||
"app.services.ops_release_service",
|
||
"app.services.ops_service",
|
||
],
|
||
"consumers": [
|
||
"ops-center",
|
||
"codex-driver",
|
||
"cli",
|
||
"release-hub",
|
||
],
|
||
"related_contract_keys": [
|
||
"ops_job_contract",
|
||
"ops_driver_contract",
|
||
"ops_playbook_contract",
|
||
"ops_stack_diagnosis_contract",
|
||
],
|
||
"notes": [
|
||
"发布必须先有 Release,再有 Rollout,不回退到节点 git pull。",
|
||
"default_rollout_gate 是首页、版本区和自动驾驶共用的统一门禁。",
|
||
"Rollout 状态机应继续固定为 planned/running/awaiting_approval/ready_for_next_batch/halted/completed/completed_with_issues。",
|
||
],
|
||
},
|
||
{
|
||
"key": "ops_driver_contract",
|
||
"title": "Ops Driver Contract",
|
||
"version": _OPS_CONTRACT_SCHEMA_VERSION,
|
||
"status": "active",
|
||
"summary": "冻结 overview / driver-feed / codex-brief / driver-actions / runbook sequence 的统一驾驶 contract。",
|
||
"schema_doc_path": "docs/schemas/ops_driver_contract.md",
|
||
"primary_endpoint": "/api/v1/ops/driver-feed",
|
||
"discovery_endpoints": [
|
||
"/api/v1/ops/overview",
|
||
"/api/v1/ops/driver-feed",
|
||
"/api/v1/ops/codex-brief",
|
||
"/api/v1/ops/driver-actions/resolve",
|
||
"/api/v1/ops/driver-actions/execute-resolved",
|
||
"/api/v1/ops/codex-actions/resolve",
|
||
"/api/v1/ops/codex-actions/execute",
|
||
"/api/v1/ops/activity-stream",
|
||
"/api/v1/ops/runbook",
|
||
"/api/v1/ops/runbook/sequences/{sequence_key}/resolve",
|
||
"/api/v1/ops/runbook/sequences/{sequence_key}/execute",
|
||
"/api/v1/ops/driver-actions/preview",
|
||
"/api/v1/ops/driver-actions/execute",
|
||
],
|
||
"service_keys": [
|
||
"app.services.ops_service",
|
||
"app.services.ops_playbook_service",
|
||
],
|
||
"consumers": [
|
||
"ops-center",
|
||
"codex-driver",
|
||
"cli",
|
||
],
|
||
"related_contract_keys": [
|
||
"release_hub_contract",
|
||
"ops_playbook_contract",
|
||
"ops_observability_contract",
|
||
"ops_stack_diagnosis_contract",
|
||
],
|
||
"notes": [
|
||
"后端负责推荐优先级和 ui_intent,页面不再自算。",
|
||
"driver recommendation / runbook sequence / activity stream 是三条不同 contract。",
|
||
"Codex 通过 codex-brief 判断 safe_auto / guarded_auto / ui_only / blocked。",
|
||
"driver-actions/preview 负责统一生成执行链、请求体预览和主次动作 contract 视图。",
|
||
],
|
||
},
|
||
{
|
||
"key": "ops_playbook_contract",
|
||
"title": "Ops Playbook Contract",
|
||
"version": _OPS_CONTRACT_SCHEMA_VERSION,
|
||
"status": "active",
|
||
"summary": "冻结 playbook catalog / preview / run / events 的正式编排 contract。",
|
||
"schema_doc_path": "docs/schemas/ops_playbook_contract.md",
|
||
"primary_endpoint": "/api/v1/ops/playbooks",
|
||
"discovery_endpoints": [
|
||
"/api/v1/ops/playbooks",
|
||
"/api/v1/ops/playbooks/preview",
|
||
"/api/v1/ops/playbooks/execute",
|
||
"/api/v1/ops/playbook-runs",
|
||
"/api/v1/ops/playbook-runs/{run_code}",
|
||
"/api/v1/ops/playbook-runs/{run_code}/events",
|
||
"/api/v1/ops/playbook-runs/{run_code}/rerun",
|
||
"/api/v1/ops/playbook-runs/{run_code}/cancel",
|
||
],
|
||
"service_keys": [
|
||
"app.services.ops_playbook_service",
|
||
"app.services.ops_service",
|
||
],
|
||
"consumers": [
|
||
"ops-center",
|
||
"codex-driver",
|
||
"cli",
|
||
],
|
||
"related_contract_keys": [
|
||
"ops_job_contract",
|
||
"release_hub_contract",
|
||
"ops_driver_contract",
|
||
"ops_observability_contract",
|
||
"ops_stack_diagnosis_contract",
|
||
],
|
||
"notes": [
|
||
"接管验收、标准巡检、现场观察应优先收口为 playbook run。",
|
||
"playbook run 是 ops job 的编排聚合对象,不是单条任务。",
|
||
"页面、CLI、Codex 应围绕同一份 preview / run / events contract 观察与执行。",
|
||
],
|
||
},
|
||
{
|
||
"key": "ops_observability_contract",
|
||
"title": "Ops Observability Contract",
|
||
"version": _OPS_CONTRACT_SCHEMA_VERSION,
|
||
"status": "active",
|
||
"summary": "冻结 execution scene / inspection overview / activity stream / delivery queue 的正式观察面 contract。",
|
||
"schema_doc_path": "docs/schemas/ops_observability_contract.md",
|
||
"primary_endpoint": "/api/v1/ops/overview",
|
||
"discovery_endpoints": [
|
||
"/api/v1/ops/overview",
|
||
"/api/v1/ops/inspection-overview",
|
||
"/api/v1/ops/activity-stream",
|
||
"/api/v1/ops/nodes/{node_code}/scene-log",
|
||
"/api/v1/ops/nodes/{node_code}/delivery-queue",
|
||
"/api/v1/ops/nodes/{node_code}/delivery-queue/records",
|
||
"/api/v1/ops/nodes/{node_code}/delivery-queue/flush",
|
||
"/api/v1/ops/nodes/{node_code}/delivery-queue/replay",
|
||
"/api/v1/ops/nodes/{node_code}/delivery-queue/records/{record_id}/replay",
|
||
"/api/v1/ops/nodes/{node_code}/delivery-queue/records/{record_id}/discard",
|
||
],
|
||
"service_keys": [
|
||
"app.services.ops_service",
|
||
"app.services.ops_agent_service",
|
||
],
|
||
"consumers": [
|
||
"ops-center",
|
||
"codex-driver",
|
||
"cli",
|
||
],
|
||
"related_contract_keys": [
|
||
"ops_job_contract",
|
||
"ops_agent_protocol",
|
||
"ops_driver_contract",
|
||
"ops_playbook_contract",
|
||
"ops_stack_diagnosis_contract",
|
||
],
|
||
"notes": [
|
||
"execution scene 必须明确区分 dispatch_active / recent_only / standby / load_syncing。",
|
||
"inspection overview 必须按节点收口最近 health / worker logs / diagnostics。",
|
||
"delivery queue 当前允许 head_only,但不允许绕过 ops job 直接改远端记录。",
|
||
],
|
||
},
|
||
{
|
||
"key": "ops_stack_diagnosis_contract",
|
||
"title": "Ops Stack Diagnosis Contract",
|
||
"version": _OPS_CONTRACT_SCHEMA_VERSION,
|
||
"status": "active",
|
||
"summary": "冻结海外单脑总检入口的统一诊断 contract,供页面、CLI、Codex、按钮共享同一份第一现场判断。",
|
||
"schema_doc_path": "docs/schemas/ops_stack_diagnosis_contract.md",
|
||
"primary_endpoint": "/api/v1/ops/stack-diagnosis",
|
||
"discovery_endpoints": [
|
||
"/api/v1/ops/go-live-summary",
|
||
"/api/v1/ops/stack-diagnosis",
|
||
"/api/v1/ops/contracts",
|
||
"/api/v1/ops/link-snapshot",
|
||
"/api/v1/ops/overview",
|
||
"/api/v1/ops/nodes",
|
||
"/api/v1/ops/releases/launchpad",
|
||
"/api/v1/ops/playbook-runs",
|
||
"/api/v1/ops/activity-stream",
|
||
],
|
||
"service_keys": [
|
||
"app.services.ops_service",
|
||
],
|
||
"consumers": [
|
||
"ops-center",
|
||
"codex-driver",
|
||
"cli",
|
||
],
|
||
"related_contract_keys": [
|
||
"ops_job_contract",
|
||
"ops_agent_protocol",
|
||
"release_hub_contract",
|
||
"ops_driver_contract",
|
||
"ops_playbook_contract",
|
||
"ops_observability_contract",
|
||
],
|
||
"notes": [
|
||
"这是海外单脑控制面的固定起手式,不允许页面、CLI、Codex 再各自拼一份第一现场诊断。",
|
||
"总检必须同时给出 stack_status、issues、next_step、quick_commands,不能只回一段说明文字。",
|
||
"go-live-summary 是给页面 / CLI / Codex 直接消费的收口摘要层,stack-diagnosis 则保留更细的可解释结构。",
|
||
"当总检 contract 可用后,check_ops_center_stack.sh 应优先直接消费它,而不是继续各算各的。",
|
||
],
|
||
},
|
||
{
|
||
"key": "ops_doctor_decision_contract",
|
||
"title": "Ops Doctor Decision Contract",
|
||
"version": _OPS_CONTRACT_SCHEMA_VERSION,
|
||
"status": "active",
|
||
"summary": "冻结 doctor-decision 主决策 contract,把总检主判断、建议处理面、下一步动作和证据摘要统一暴露给页面、CLI 和 Codex。",
|
||
"schema_doc_path": "docs/schemas/ops_doctor_decision_contract.md",
|
||
"primary_endpoint": "/api/v1/ops/doctor-decision",
|
||
"discovery_endpoints": [
|
||
"/api/v1/ops/doctor-decision",
|
||
"/api/v1/ops/stack-diagnosis",
|
||
"/api/v1/ops/go-live-summary",
|
||
"/api/v1/ops/go-live-bundle",
|
||
"/api/v1/ops/contracts",
|
||
],
|
||
"service_keys": [
|
||
"app.services.ops_service",
|
||
],
|
||
"consumers": [
|
||
"ops-center",
|
||
"codex-driver",
|
||
"cli",
|
||
],
|
||
"related_contract_keys": [
|
||
"ops_stack_diagnosis_contract",
|
||
"ops_driver_contract",
|
||
"ops_go_live_bundle_contract",
|
||
"ops_go_live_signoff_contract",
|
||
"release_hub_contract",
|
||
],
|
||
"notes": [
|
||
"doctor-decision 是页面、CLI、Codex 的统一主决策入口,不应再让不同消费方各算各的“第一处理面”。",
|
||
"优先读取最新导出的 doctor manifest / go-live bundle 中的 doctor_decision,再在缺 bundle 时回落到 live 总检兜底。",
|
||
"返回值必须同时给出 status、preferred_surface、next_action_code 与推荐命令,不能只给一句提示文字。",
|
||
],
|
||
},
|
||
{
|
||
"key": "ops_go_live_signoff_contract",
|
||
"title": "Ops Go-Live Signoff Contract",
|
||
"version": _OPS_CONTRACT_SCHEMA_VERSION,
|
||
"status": "active",
|
||
"summary": "冻结发布前最终签收结论 contract,把收口、发布、自动化、launchpad 对齐状态压成同一份签字判断。",
|
||
"schema_doc_path": "docs/schemas/ops_go_live_signoff_contract.md",
|
||
"primary_endpoint": "/api/v1/ops/go-live-signoff",
|
||
"discovery_endpoints": [
|
||
"/api/v1/ops/go-live-signoff",
|
||
"/api/v1/ops/go-live-summary",
|
||
"/api/v1/ops/stack-diagnosis",
|
||
"/api/v1/ops/driver-feed",
|
||
"/api/v1/ops/codex-brief",
|
||
"/api/v1/ops/releases/launchpad",
|
||
"/api/v1/ops/contracts",
|
||
],
|
||
"service_keys": [
|
||
"app.services.ops_service",
|
||
],
|
||
"consumers": [
|
||
"ops-center",
|
||
"codex-driver",
|
||
"cli",
|
||
],
|
||
"related_contract_keys": [
|
||
"ops_doctor_decision_contract",
|
||
"ops_stack_diagnosis_contract",
|
||
"ops_driver_contract",
|
||
"release_hub_contract",
|
||
"ops_observability_contract",
|
||
],
|
||
"notes": [
|
||
"signoff 不是替代 go-live-summary,而是在其基础上给出最终可否签字上线的统一结论。",
|
||
"当 signoff_status=blocked 时,不允许页面再显示‘基本可上线’这类模糊表述。",
|
||
"launchpad 摘要在 go-live-summary / driver-feed / codex-brief 之间若不一致,必须至少提升为 attention。",
|
||
],
|
||
},
|
||
{
|
||
"key": "ops_go_live_bundle_contract",
|
||
"title": "Ops Go-Live Bundle Contract",
|
||
"version": _OPS_CONTRACT_SCHEMA_VERSION,
|
||
"status": "active",
|
||
"summary": "冻结发布前证据包与 manifest 复核 contract,把 bundle 是否存在、manifest 状态、失败项和交付建议统一暴露给页面、CLI 和 Codex。",
|
||
"schema_doc_path": "docs/schemas/ops_go_live_bundle_contract.md",
|
||
"primary_endpoint": "/api/v1/ops/go-live-bundle",
|
||
"discovery_endpoints": [
|
||
"/api/v1/ops/go-live-bundle",
|
||
"/api/v1/ops/go-live-signoff",
|
||
"/api/v1/ops/go-live-summary",
|
||
"/api/v1/ops/contracts",
|
||
],
|
||
"service_keys": [
|
||
"app.services.ops_service",
|
||
],
|
||
"consumers": [
|
||
"ops-center",
|
||
"codex-driver",
|
||
"cli",
|
||
],
|
||
"related_contract_keys": [
|
||
"ops_doctor_decision_contract",
|
||
"ops_go_live_signoff_contract",
|
||
"ops_stack_diagnosis_contract",
|
||
"ops_driver_contract",
|
||
"release_hub_contract",
|
||
],
|
||
"notes": [
|
||
"页面不应再假设证据包已经导出,而是必须明确显示当前有没有 bundle、manifest 是否可复核。",
|
||
"当 bundle 不存在时,应明确回落到 go-live-export / go-live-review / go-live-signoff 等固定命令,而不是提示人工自行找目录。",
|
||
"这层只负责读取最新 bundle / manifest 与复核摘要,不直接在后端发起 shell 导出。",
|
||
],
|
||
},
|
||
{
|
||
"key": "ops_go_live_review_contract",
|
||
"title": "Ops Go-Live Review Contract",
|
||
"version": _OPS_CONTRACT_SCHEMA_VERSION,
|
||
"status": "active",
|
||
"summary": "冻结 go-live-review 复核结论 contract,把 bundle manifest 的正式复核结果、launchpad 对齐与下一步建议统一暴露给页面、CLI 和 Codex。",
|
||
"schema_doc_path": "docs/schemas/ops_go_live_review_contract.md",
|
||
"primary_endpoint": "/api/v1/ops/go-live-review",
|
||
"discovery_endpoints": [
|
||
"/api/v1/ops/go-live-review",
|
||
"/api/v1/ops/go-live-bundle",
|
||
"/api/v1/ops/doctor-decision",
|
||
"/api/v1/ops/go-live-signoff",
|
||
"/api/v1/ops/contracts",
|
||
],
|
||
"service_keys": [
|
||
"app.services.ops_service",
|
||
],
|
||
"consumers": [
|
||
"ops-center",
|
||
"codex-driver",
|
||
"cli",
|
||
],
|
||
"related_contract_keys": [
|
||
"ops_go_live_bundle_contract",
|
||
"ops_doctor_decision_contract",
|
||
"ops_go_live_signoff_contract",
|
||
"ops_stack_diagnosis_contract",
|
||
"ops_driver_contract",
|
||
"release_hub_contract",
|
||
],
|
||
"notes": [
|
||
"go-live-review 不是简单重复 bundle 状态,而是 bundle manifest 的正式复核结论层。",
|
||
"当 launchpad 摘要在 go_live_summary / stack_diagnosis / driver_feed / codex_brief 之间漂移时,即使关键文件都存在,也必须至少回到 attention。",
|
||
"页面和 Codex 应优先消费这层的 headline / recommended_next_steps,而不是自行拼 review 说明。",
|
||
],
|
||
},
|
||
)
|
||
|
||
|
||
def _bool_label(value: bool) -> str:
|
||
return "enabled" if value else "disabled"
|
||
|
||
|
||
def get_ops_contract_registry() -> dict:
|
||
contracts: list[dict] = []
|
||
contracts_by_key: dict[str, dict] = {}
|
||
for raw_item in _OPS_CONTRACT_DEFINITIONS:
|
||
contract = {
|
||
"key": str(raw_item.get("key") or "").strip(),
|
||
"title": str(raw_item.get("title") or "").strip(),
|
||
"version": str(raw_item.get("version") or _OPS_CONTRACT_SCHEMA_VERSION).strip(),
|
||
"status": str(raw_item.get("status") or "active").strip() or "active",
|
||
"summary": str(raw_item.get("summary") or "").strip(),
|
||
"schema_doc_path": str(raw_item.get("schema_doc_path") or "").strip(),
|
||
"primary_endpoint": str(raw_item.get("primary_endpoint") or "").strip(),
|
||
"discovery_endpoints": [str(item or "").strip() for item in list(raw_item.get("discovery_endpoints") or []) if str(item or "").strip()],
|
||
"service_keys": [str(item or "").strip() for item in list(raw_item.get("service_keys") or []) if str(item or "").strip()],
|
||
"consumers": [str(item or "").strip() for item in list(raw_item.get("consumers") or []) if str(item or "").strip()],
|
||
"related_contract_keys": [
|
||
str(item or "").strip()
|
||
for item in list(raw_item.get("related_contract_keys") or [])
|
||
if str(item or "").strip()
|
||
],
|
||
"notes": [str(item or "").strip() for item in list(raw_item.get("notes") or []) if str(item or "").strip()],
|
||
}
|
||
if not contract["key"]:
|
||
continue
|
||
contracts.append(contract)
|
||
contracts_by_key[contract["key"]] = contract
|
||
|
||
return {
|
||
"registry_version": _OPS_CONTRACT_REGISTRY_VERSION,
|
||
"schema_version": _OPS_CONTRACT_SCHEMA_VERSION,
|
||
"contracts_total": len(contracts),
|
||
"contracts": contracts,
|
||
"contracts_by_key": contracts_by_key,
|
||
"docs_root": "docs/schemas",
|
||
"discovery_entrypoints": [
|
||
"/api/v1/ops/contracts",
|
||
"/api/v1/ops/contracts/{contract_key}",
|
||
"/api/v1/ops/stack-diagnosis",
|
||
"/api/v1/ops/capabilities",
|
||
"/api/v1/ops/overview",
|
||
"/api/v1/ops/driver-feed",
|
||
"/api/v1/ops/codex-brief",
|
||
"/api/v1/ops/releases/launchpad",
|
||
],
|
||
}
|
||
|
||
|
||
def get_ops_contract_detail(contract_key: str) -> dict:
|
||
normalized_contract_key = str(contract_key or "").strip()
|
||
if not normalized_contract_key:
|
||
return {}
|
||
|
||
registry = get_ops_contract_registry()
|
||
contracts_by_key = dict(registry.get("contracts_by_key") or {})
|
||
selected_contract = dict(contracts_by_key.get(normalized_contract_key) or {})
|
||
if not selected_contract:
|
||
return {}
|
||
|
||
related_contracts = [
|
||
dict(contracts_by_key.get(related_key) or {})
|
||
for related_key in list(selected_contract.get("related_contract_keys") or [])
|
||
if str(related_key or "").strip() and dict(contracts_by_key.get(related_key) or {})
|
||
]
|
||
related_contracts = [
|
||
{
|
||
"key": str(item.get("key") or "").strip(),
|
||
"title": str(item.get("title") or "").strip(),
|
||
"summary": str(item.get("summary") or "").strip(),
|
||
"schema_doc_path": str(item.get("schema_doc_path") or "").strip(),
|
||
"primary_endpoint": str(item.get("primary_endpoint") or "").strip(),
|
||
}
|
||
for item in related_contracts
|
||
]
|
||
|
||
return {
|
||
"registry_version": str(registry.get("registry_version") or _OPS_CONTRACT_REGISTRY_VERSION),
|
||
"schema_version": str(registry.get("schema_version") or _OPS_CONTRACT_SCHEMA_VERSION),
|
||
"docs_root": str(registry.get("docs_root") or "docs/schemas"),
|
||
"detail_endpoint": f"/api/v1/ops/contracts/{normalized_contract_key}",
|
||
"contract": selected_contract,
|
||
"related_contracts": related_contracts,
|
||
"related_contract_keys": [
|
||
str(item.get("key") or "").strip()
|
||
for item in related_contracts
|
||
if str(item.get("key") or "").strip()
|
||
],
|
||
}
|
||
|
||
|
||
def _normalize_ops_contract_keys(raw_values: list | tuple | set | None) -> list[str]:
|
||
normalized_values: list[str] = []
|
||
seen_values: set[str] = set()
|
||
for raw_value in list(raw_values or []):
|
||
normalized_value = str(raw_value or "").strip()
|
||
if not normalized_value or normalized_value in seen_values:
|
||
continue
|
||
seen_values.add(normalized_value)
|
||
normalized_values.append(normalized_value)
|
||
return normalized_values
|
||
|
||
|
||
def _build_ops_contract_navigation(
|
||
contract_keys: list | tuple | set | None,
|
||
*,
|
||
primary_contract_key: str = "",
|
||
registry: dict | None = None,
|
||
) -> dict:
|
||
registry_payload = dict(registry or get_ops_contract_registry())
|
||
contracts_by_key = dict(registry_payload.get("contracts_by_key") or {})
|
||
normalized_primary_contract_key = str(primary_contract_key or "").strip()
|
||
ordered_contract_keys = _normalize_ops_contract_keys(contract_keys)
|
||
if normalized_primary_contract_key:
|
||
ordered_contract_keys = [normalized_primary_contract_key] + [
|
||
item for item in ordered_contract_keys if item != normalized_primary_contract_key
|
||
]
|
||
|
||
contracts: list[dict] = []
|
||
for contract_key in ordered_contract_keys:
|
||
raw_contract = dict(contracts_by_key.get(contract_key) or {})
|
||
if not raw_contract:
|
||
continue
|
||
related_contract_keys = _normalize_ops_contract_keys(raw_contract.get("related_contract_keys") or [])
|
||
contracts.append(
|
||
{
|
||
"key": contract_key,
|
||
"title": str(raw_contract.get("title") or "").strip(),
|
||
"status": str(raw_contract.get("status") or "").strip(),
|
||
"version": str(raw_contract.get("version") or _OPS_CONTRACT_SCHEMA_VERSION).strip(),
|
||
"summary": str(raw_contract.get("summary") or "").strip(),
|
||
"primary_endpoint": str(raw_contract.get("primary_endpoint") or "").strip(),
|
||
"schema_doc_path": str(raw_contract.get("schema_doc_path") or "").strip(),
|
||
"detail_endpoint": f"/api/v1/ops/contracts/{contract_key}",
|
||
"discovery_endpoints": [
|
||
str(item).strip()
|
||
for item in list(raw_contract.get("discovery_endpoints") or [])
|
||
if str(item).strip()
|
||
],
|
||
"related_contract_keys": related_contract_keys,
|
||
}
|
||
)
|
||
|
||
return {
|
||
"detail_endpoint_pattern": "/api/v1/ops/contracts/{contract_key}",
|
||
"primary_contract_key": (
|
||
normalized_primary_contract_key
|
||
if normalized_primary_contract_key
|
||
else str((contracts[0] or {}).get("key") or "").strip()
|
||
),
|
||
"contract_keys": (
|
||
[str(item.get("key") or "").strip() for item in contracts if str(item.get("key") or "").strip()]
|
||
if contracts
|
||
else ordered_contract_keys
|
||
),
|
||
"contracts": contracts,
|
||
}
|
||
|
||
|
||
def _ops_contract_keys_from_focus_ref(focus_ref: dict | None) -> list[str]:
|
||
normalized_focus_ref = dict(focus_ref or {})
|
||
kind = str(normalized_focus_ref.get("kind") or "").strip()
|
||
section = str(normalized_focus_ref.get("section") or "").strip()
|
||
|
||
if kind in {"release_hub", "release", "rollout"} or section in {
|
||
"release_launchpad",
|
||
"release_detail",
|
||
"default_rollout_gate",
|
||
}:
|
||
return ["release_hub_contract"]
|
||
if kind in {"execution_scene", "ops_job", "ops_job_event", "activity_stream", "node_scene_log"}:
|
||
return ["ops_observability_contract", "ops_stack_diagnosis_contract"]
|
||
if kind in {"playbook_run", "playbook", "playbook_step"}:
|
||
return ["ops_playbook_contract"]
|
||
if kind in {"managed_node", "node_agent", "delivery_queue"}:
|
||
return ["ops_agent_protocol"]
|
||
return []
|
||
|
||
|
||
def _ops_contract_keys_from_action_code(action_code: str) -> list[str]:
|
||
normalized_action_code = str(action_code or "").strip()
|
||
if not normalized_action_code:
|
||
return []
|
||
if normalized_action_code.startswith("enable_log_sync") or normalized_action_code in {
|
||
"run_inspection_participating",
|
||
"run_inspection_all",
|
||
"focus_execution_scene",
|
||
"focus_activity_item",
|
||
}:
|
||
return ["ops_observability_contract", "ops_stack_diagnosis_contract"]
|
||
if normalized_action_code in {
|
||
"fix_managed_nodes",
|
||
"issue_agent_token",
|
||
"create_bootstrap_plan",
|
||
"open_agent_onboarding",
|
||
"bootstrap_run",
|
||
"run_acceptance",
|
||
}:
|
||
return ["ops_agent_protocol"]
|
||
if normalized_action_code in {
|
||
"open_release_dialog",
|
||
"focus_release_hub",
|
||
"create_release_rollout_worker",
|
||
"create_release_rollout_control",
|
||
"publish_latest_worker",
|
||
"publish_latest_control",
|
||
}:
|
||
return ["release_hub_contract"]
|
||
return []
|
||
|
||
|
||
def _ops_contract_keys_from_job_action(job_action: str) -> list[str]:
|
||
normalized_job_action = str(job_action or "").strip()
|
||
if not normalized_job_action:
|
||
return []
|
||
if normalized_job_action in {"health.snapshot", "logs.collect", "diagnostics.collect", "service.status"}:
|
||
return ["ops_observability_contract", "ops_stack_diagnosis_contract"]
|
||
if normalized_job_action == "node.bootstrap" or normalized_job_action.startswith(("runtime.", "service.", "delivery.queue.")):
|
||
return ["ops_agent_protocol"]
|
||
if normalized_job_action == "deploy.release":
|
||
return ["release_hub_contract"]
|
||
return []
|
||
|
||
|
||
def _suggest_ops_contract_keys_for_codex_entry(entry: dict) -> list[str]:
|
||
normalized_entry = dict(entry or {})
|
||
executor_kind = str(normalized_entry.get("executor_kind") or "driver_action").strip() or "driver_action"
|
||
contract_keys: list[str] = ["ops_driver_contract"]
|
||
if executor_kind == "runbook_sequence":
|
||
contract_keys.append("ops_playbook_contract")
|
||
|
||
for focus_ref in (
|
||
normalized_entry.get("focus_ref"),
|
||
normalized_entry.get("primary_focus_ref"),
|
||
normalized_entry.get("secondary_focus_ref"),
|
||
):
|
||
contract_keys.extend(_ops_contract_keys_from_focus_ref(focus_ref))
|
||
|
||
for action_code in (
|
||
normalized_entry.get("primary_action_code"),
|
||
normalized_entry.get("secondary_action_code"),
|
||
normalized_entry.get("focus_action_code"),
|
||
):
|
||
contract_keys.extend(_ops_contract_keys_from_action_code(str(action_code or "").strip()))
|
||
|
||
return _normalize_ops_contract_keys(contract_keys)
|
||
|
||
|
||
def _suggest_ops_contract_keys_for_activity_item(item: dict) -> list[str]:
|
||
normalized_item = dict(item or {})
|
||
kind = str(normalized_item.get("kind") or "").strip()
|
||
ui_intent = dict(normalized_item.get("ui_intent") or {})
|
||
contract_keys: list[str] = []
|
||
|
||
if kind in {"execution_scene", "log_sync"}:
|
||
contract_keys.extend(["ops_observability_contract", "ops_stack_diagnosis_contract"])
|
||
elif kind in {"playbook_run", "runbook_sequence"}:
|
||
contract_keys.append("ops_playbook_contract")
|
||
if kind == "runbook_sequence":
|
||
contract_keys.append("ops_driver_contract")
|
||
elif kind == "rollout":
|
||
contract_keys.append("release_hub_contract")
|
||
elif kind == "ops_job":
|
||
contract_keys.extend(
|
||
_ops_contract_keys_from_job_action(
|
||
str(normalized_item.get("action") or normalized_item.get("title") or "").strip()
|
||
)
|
||
)
|
||
else:
|
||
contract_keys.append("ops_observability_contract")
|
||
|
||
for focus_ref in (
|
||
normalized_item.get("focus_ref"),
|
||
normalized_item.get("source_focus_ref"),
|
||
):
|
||
contract_keys.extend(_ops_contract_keys_from_focus_ref(focus_ref))
|
||
|
||
for action_code in (
|
||
ui_intent.get("driver_action_code"),
|
||
normalized_item.get("primary_action_code"),
|
||
normalized_item.get("secondary_action_code"),
|
||
normalized_item.get("focus_action_code"),
|
||
):
|
||
contract_keys.extend(_ops_contract_keys_from_action_code(str(action_code or "").strip()))
|
||
|
||
if str(ui_intent.get("kind") or "").strip() == "focus_execution_scene":
|
||
contract_keys.extend(["ops_observability_contract", "ops_stack_diagnosis_contract"])
|
||
|
||
return _normalize_ops_contract_keys(contract_keys)
|
||
|
||
|
||
def _primary_ops_contract_key_for_activity_item(item: dict, contract_keys: list[str]) -> str:
|
||
normalized_item = dict(item or {})
|
||
normalized_contract_keys = _normalize_ops_contract_keys(contract_keys)
|
||
kind = str(normalized_item.get("kind") or "").strip()
|
||
job_action = str(normalized_item.get("action") or normalized_item.get("title") or "").strip()
|
||
|
||
if kind in {"playbook_run", "runbook_sequence"} and "ops_playbook_contract" in normalized_contract_keys:
|
||
return "ops_playbook_contract"
|
||
if kind == "rollout" and "release_hub_contract" in normalized_contract_keys:
|
||
return "release_hub_contract"
|
||
if kind == "ops_job":
|
||
if job_action == "node.bootstrap" and "ops_agent_protocol" in normalized_contract_keys:
|
||
return "ops_agent_protocol"
|
||
if job_action == "deploy.release" and "release_hub_contract" in normalized_contract_keys:
|
||
return "release_hub_contract"
|
||
if "ops_observability_contract" in normalized_contract_keys:
|
||
return "ops_observability_contract"
|
||
if "ops_driver_contract" in normalized_contract_keys:
|
||
return "ops_driver_contract"
|
||
return normalized_contract_keys[0] if normalized_contract_keys else ""
|
||
|
||
|
||
def _attach_driver_feed_contract_navigation(entry: dict) -> dict:
|
||
normalized_entry = dict(entry or {})
|
||
contract_keys = _normalize_ops_contract_keys(
|
||
list(normalized_entry.get("contract_keys") or [])
|
||
+ _suggest_ops_contract_keys_for_codex_entry(normalized_entry)
|
||
)
|
||
return {
|
||
**normalized_entry,
|
||
"contract_keys": contract_keys,
|
||
"contract_navigation": _build_ops_contract_navigation(
|
||
contract_keys,
|
||
primary_contract_key="ops_driver_contract",
|
||
registry=get_ops_contract_registry(),
|
||
),
|
||
}
|
||
|
||
|
||
def _format_node_code_list(raw_values: object, *, limit: int = 5) -> str:
|
||
values: list[str] = []
|
||
seen: set[str] = set()
|
||
for item in list(raw_values or []):
|
||
value = str(item or "").strip()
|
||
if not value or value in seen:
|
||
continue
|
||
seen.add(value)
|
||
values.append(value)
|
||
if not values:
|
||
return "-"
|
||
visible = values[: max(1, int(limit or 5))]
|
||
suffix = f" 等 {len(values)} 台" if len(values) > len(visible) else ""
|
||
return "、".join(visible) + suffix
|
||
|
||
|
||
def _normalize_driver_text_lines(raw_values: object, *, limit: int = 5) -> list[str]:
|
||
normalized: list[str] = []
|
||
seen: set[str] = set()
|
||
for item in list(raw_values or []):
|
||
value = str(item or "").strip()
|
||
if not value or value in seen:
|
||
continue
|
||
seen.add(value)
|
||
normalized.append(value)
|
||
return normalized[: max(1, int(limit or 5))]
|
||
|
||
|
||
def _is_cluster_online_node(node: dict) -> bool:
|
||
return str(node.get("cluster_status") or "").strip() in {"online", "busy"}
|
||
|
||
|
||
def _is_inspection_visible_node(node: dict) -> bool:
|
||
return bool(node.get("cluster_is_effective_worker", False)) and _is_cluster_online_node(node)
|
||
|
||
|
||
def _is_inspection_eligible_node(node: dict) -> bool:
|
||
return (
|
||
bool(node.get("is_managed", False))
|
||
and bool(node.get("is_enabled", False))
|
||
and bool(node.get("is_agent_online", False))
|
||
and _is_inspection_visible_node(node)
|
||
)
|
||
|
||
|
||
def _inspection_job_bucket(job: dict) -> str:
|
||
action = str(job.get("action") or "").strip()
|
||
if action in {"health.snapshot", "diagnostics.collect"}:
|
||
return action
|
||
if action != "logs.collect":
|
||
return ""
|
||
service_name = str((job.get("payload") or {}).get("service_name") or "").strip().lower()
|
||
if "worker" not in service_name:
|
||
return ""
|
||
return "logs.collect"
|
||
|
||
|
||
def _slim_inspection_job(job: dict) -> dict:
|
||
payload = dict(job.get("payload") or {})
|
||
return {
|
||
"id": int(job.get("id") or 0),
|
||
"job_code": str(job.get("job_code") or ""),
|
||
"action": str(job.get("action") or ""),
|
||
"status": str(job.get("status") or ""),
|
||
"target_node_code": str(job.get("target_node_code") or ""),
|
||
"created_at": str(job.get("created_at") or ""),
|
||
"started_at": str(job.get("started_at") or ""),
|
||
"finished_at": str(job.get("finished_at") or ""),
|
||
"updated_at": str(job.get("updated_at") or ""),
|
||
"payload": {
|
||
"service_name": str(payload.get("service_name") or ""),
|
||
"lines": int(payload.get("lines") or 0) if payload.get("lines") not in (None, "") else 0,
|
||
},
|
||
"error_message": str(job.get("error_message") or ""),
|
||
}
|
||
|
||
|
||
def _inspection_status_label(status: str) -> str:
|
||
normalized_status = str(status or "").strip()
|
||
if not normalized_status:
|
||
return "未收口"
|
||
mapping = {
|
||
"success": "成功",
|
||
"failed": "失败",
|
||
"blocked": "阻断",
|
||
"cancelled": "已取消",
|
||
"awaiting_approval": "待审批",
|
||
"dispatching": "待派发",
|
||
"running": "执行中",
|
||
"queued": "排队中",
|
||
}
|
||
return mapping.get(normalized_status, normalized_status)
|
||
|
||
|
||
def _build_inspection_contract_result(job: dict | None, *, action_label: str) -> dict:
|
||
normalized_job = dict(job or {})
|
||
if not normalized_job:
|
||
return {
|
||
"status": "",
|
||
"status_label": "未收口",
|
||
"occurred_at": "",
|
||
"job_code": "",
|
||
"summary": f"最近没有 {action_label} 收口记录。",
|
||
"error_message": "",
|
||
}
|
||
|
||
status = str(normalized_job.get("status") or "").strip()
|
||
occurred_at = (
|
||
str(normalized_job.get("finished_at") or "").strip()
|
||
or str(normalized_job.get("updated_at") or "").strip()
|
||
or str(normalized_job.get("started_at") or "").strip()
|
||
or str(normalized_job.get("created_at") or "").strip()
|
||
)
|
||
job_code = str(normalized_job.get("job_code") or "").strip()
|
||
error_message = str(normalized_job.get("error_message") or "").strip()
|
||
if status == "success":
|
||
summary = f"最近一次{action_label}收口成功。"
|
||
elif status:
|
||
summary = f"最近一次{action_label}状态为 {_inspection_status_label(status)}。"
|
||
else:
|
||
summary = f"最近没有 {action_label} 收口记录。"
|
||
if error_message:
|
||
summary = f"{summary} 错误:{error_message}"
|
||
return {
|
||
"status": status,
|
||
"status_label": _inspection_status_label(status) if status else "未收口",
|
||
"occurred_at": occurred_at,
|
||
"job_code": job_code,
|
||
"summary": summary,
|
||
"error_message": error_message,
|
||
}
|
||
|
||
|
||
def _build_inspection_issue_summary(
|
||
job_map: dict[str, dict],
|
||
*,
|
||
node_code: str = "",
|
||
latest_job: dict | None = None,
|
||
node: dict | None = None,
|
||
) -> dict:
|
||
action_labels = {
|
||
"health.snapshot": "健康快照",
|
||
"logs.collect": "Worker 日志",
|
||
"diagnostics.collect": "诊断包",
|
||
}
|
||
node_context = dict(node or {})
|
||
node_is_visible = _is_inspection_visible_node(node_context)
|
||
node_is_eligible = bool(node_context.get("is_inspection_eligible", False)) or _is_inspection_eligible_node(node_context)
|
||
node_is_participating = bool(node_context.get("cluster_detect_participating", False))
|
||
node_is_managed = bool(node_context.get("is_managed", False))
|
||
node_is_enabled = bool(node_context.get("is_enabled", False))
|
||
node_agent_online = bool(node_context.get("is_agent_online", False))
|
||
node_has_ssh_access = bool(node_context.get("has_ssh_access", False))
|
||
agent_state_label = str(node_context.get("agent_state_label") or "").strip()
|
||
agent_state_reason = str(node_context.get("agent_state_reason") or "").strip()
|
||
preferred_job = (
|
||
job_map.get("diagnostics.collect")
|
||
or job_map.get("logs.collect")
|
||
or job_map.get("health.snapshot")
|
||
or dict(latest_job or {})
|
||
)
|
||
preferred_job_id = int(preferred_job.get("id") or 0)
|
||
preferred_job_code = str(preferred_job.get("job_code") or "").strip()
|
||
preferred_action = str(preferred_job.get("action") or "").strip()
|
||
preferred_target_node_code = str(preferred_job.get("target_node_code") or node_code or "").strip()
|
||
preferred_ui_intent = (
|
||
_build_driver_ui_intent(
|
||
"job_events",
|
||
job_id=preferred_job_id,
|
||
job_code=preferred_job_code,
|
||
action=preferred_action,
|
||
target_node_code=preferred_target_node_code,
|
||
)
|
||
if preferred_job_id > 0
|
||
else {}
|
||
)
|
||
present_statuses = [
|
||
str((job_map.get(key) or {}).get("status") or "").strip()
|
||
for key in _OPS_INSPECTION_ACTION_KEYS
|
||
if str((job_map.get(key) or {}).get("status") or "").strip()
|
||
]
|
||
|
||
if node_is_visible and not node_is_eligible:
|
||
if not node_is_managed:
|
||
return {
|
||
"problem_kind": "handover_gap",
|
||
"problem_label": "未接管",
|
||
"problem_title": "执行节点未接管",
|
||
"problem_level": "danger",
|
||
"problem_keys": [],
|
||
"summary": "节点在线且属于有效执行面,但还没纳入 Node Agent 接管,当前不能稳定下发标准巡检、发布或诊断动作。",
|
||
"recommended_action": "纳管节点",
|
||
"recommended_action_code": "handover_first_gap",
|
||
"ui_intent": {},
|
||
}
|
||
if not node_is_enabled:
|
||
return {
|
||
"problem_kind": "handover_gap",
|
||
"problem_label": "已停用",
|
||
"problem_title": "节点已停用",
|
||
"problem_level": "warning",
|
||
"problem_keys": [],
|
||
"summary": "节点虽然仍在有效执行面中,但当前被标记为停用,先确认是否需要重新启用后再下发巡检或发布动作。",
|
||
"recommended_action": "查看节点",
|
||
"recommended_action_code": "view_first_gap",
|
||
"ui_intent": {},
|
||
}
|
||
if not node_agent_online:
|
||
return {
|
||
"problem_kind": "handover_gap",
|
||
"problem_label": "SSH 已备好" if node_has_ssh_access else (agent_state_label or "待接入"),
|
||
"problem_title": "Agent 未就绪",
|
||
"problem_level": "danger",
|
||
"problem_keys": [],
|
||
"summary": (
|
||
f"节点已纳管,SSH 入口已备好,虽然仍未进入 Agent 在线状态,但已可通过 SSH 执行日志、诊断与部分服务控制。{agent_state_reason}"
|
||
if node_has_ssh_access and agent_state_reason
|
||
else (
|
||
"节点已纳管,SSH 入口已备好,当前已可通过 SSH 执行日志、诊断与部分服务控制;标准巡检编排、正式发布与 Rollout 仍优先依赖 Node Agent 在线。"
|
||
if node_has_ssh_access
|
||
else (
|
||
f"节点已纳管,但仍未进入 Agent 在线状态。{agent_state_reason}"
|
||
if agent_state_reason
|
||
else "节点已纳管,但仍未进入 Agent 在线状态,当前不能稳定执行标准巡检。"
|
||
)
|
||
)
|
||
),
|
||
"recommended_action": "补接入",
|
||
"recommended_action_code": "handover_first_gap",
|
||
"ui_intent": {},
|
||
}
|
||
|
||
if node_is_visible and node_is_participating and not present_statuses:
|
||
return {
|
||
"problem_kind": "participating_no_inspection",
|
||
"problem_label": "现场未巡检",
|
||
"problem_title": "参与检测但未形成巡检",
|
||
"problem_level": "warning",
|
||
"problem_keys": [],
|
||
"summary": "节点当前正在真实参与检测,但还没有形成标准巡检记录,建议先看现场日志或补一轮标准巡检。",
|
||
"recommended_action": "看现场日志",
|
||
"recommended_action_code": "open_worker_logs_participating",
|
||
"ui_intent": {},
|
||
}
|
||
|
||
if node_is_visible and node_is_eligible and not node_is_participating and not present_statuses:
|
||
return {
|
||
"problem_kind": "standby_no_inspection",
|
||
"problem_label": "待命未巡检",
|
||
"problem_title": "在线待命但未巡检",
|
||
"problem_level": "info",
|
||
"problem_keys": [],
|
||
"summary": "节点当前在线可执行,但尚未参与检测,也还没有形成标准巡检记录,适合先做一轮巡检确认是否正常待命。",
|
||
"recommended_action": "补齐标准巡检",
|
||
"recommended_action_code": "run_standard_inspection",
|
||
"ui_intent": {},
|
||
}
|
||
|
||
terminal_problem_actions = [
|
||
key
|
||
for key in _OPS_INSPECTION_ACTION_KEYS
|
||
if str((job_map.get(key) or {}).get("status") or "").strip() in {"failed", "blocked", "cancelled"}
|
||
]
|
||
if terminal_problem_actions:
|
||
key = terminal_problem_actions[0]
|
||
job = job_map.get(key) or {}
|
||
return {
|
||
"problem_kind": "terminal_failure",
|
||
"problem_label": "失败/阻断",
|
||
"problem_title": f"{action_labels[key]}失败",
|
||
"problem_level": "error",
|
||
"problem_keys": terminal_problem_actions,
|
||
"summary": f"{action_labels[key]}当前状态为 {_inspection_status_label(str(job.get('status') or ''))},优先查看事件并重跑标准巡检。",
|
||
"recommended_action": "先看事件",
|
||
"recommended_action_code": "focus_latest_job_events",
|
||
"ui_intent": preferred_ui_intent,
|
||
}
|
||
|
||
running_actions = [
|
||
key
|
||
for key in _OPS_INSPECTION_ACTION_KEYS
|
||
if str((job_map.get(key) or {}).get("status") or "").strip() in {"queued", "dispatching", "running", "awaiting_approval"}
|
||
]
|
||
if running_actions:
|
||
key = running_actions[0]
|
||
job = job_map.get(key) or {}
|
||
return {
|
||
"problem_kind": "inflight_execution",
|
||
"problem_label": "执行中",
|
||
"problem_title": f"{action_labels[key]}执行中",
|
||
"problem_level": "warning",
|
||
"problem_keys": running_actions,
|
||
"summary": f"{action_labels[key]}仍在{_inspection_status_label(str(job.get('status') or ''))},先等待回执或检查对应节点事件流。",
|
||
"recommended_action": "查看事件",
|
||
"recommended_action_code": "focus_latest_job_events",
|
||
"ui_intent": preferred_ui_intent,
|
||
}
|
||
|
||
missing_actions = [key for key in _OPS_INSPECTION_ACTION_KEYS if not str((job_map.get(key) or {}).get("status") or "").strip()]
|
||
if missing_actions:
|
||
missing_problem_map = {
|
||
"health.snapshot": ("missing_health_snapshot", "缺健康快照", "健康快照缺口"),
|
||
"logs.collect": ("missing_worker_logs", "缺 Worker 日志", "Worker 日志缺口"),
|
||
"diagnostics.collect": ("missing_diagnostics", "缺诊断包", "诊断包缺口"),
|
||
}
|
||
if len(missing_actions) == 1:
|
||
problem_kind, problem_label, problem_title = missing_problem_map.get(
|
||
missing_actions[0],
|
||
("missing_inspection_steps", "巡检缺口", "巡检缺口"),
|
||
)
|
||
else:
|
||
problem_kind, problem_label, problem_title = (
|
||
"missing_inspection_steps",
|
||
"巡检缺口",
|
||
"巡检缺口",
|
||
)
|
||
return {
|
||
"problem_kind": problem_kind,
|
||
"problem_label": problem_label,
|
||
"problem_title": problem_title,
|
||
"problem_level": "info",
|
||
"problem_keys": missing_actions,
|
||
"summary": f"当前仍缺少 {'、'.join(action_labels[key] for key in missing_actions)} 的巡检记录。",
|
||
"recommended_action": "补齐标准巡检",
|
||
"recommended_action_code": "run_standard_inspection",
|
||
"ui_intent": {},
|
||
}
|
||
|
||
return {
|
||
"problem_kind": "healthy",
|
||
"problem_label": "已收口",
|
||
"problem_title": "巡检健康",
|
||
"problem_level": "success",
|
||
"problem_keys": [],
|
||
"summary": "当前三类巡检均已成功回执。",
|
||
"recommended_action": "当前无需处理",
|
||
"recommended_action_code": "no_action",
|
||
"ui_intent": {},
|
||
}
|
||
|
||
|
||
def _slim_execution_scene_node(node: dict) -> dict:
|
||
raw_participation_state = str(node.get("participation_state") or "").strip()
|
||
is_dispatch_active = bool(node.get("is_dispatch_active", False))
|
||
if is_dispatch_active or raw_participation_state in {"running", "claimed", "dispatch_active"}:
|
||
participation_bucket = "dispatch_active"
|
||
participation_bucket_label = "执行/已领"
|
||
elif raw_participation_state in {"recent_throughput", "recent_only"}:
|
||
participation_bucket = "recent_only"
|
||
participation_bucket_label = "近窗有吞吐"
|
||
elif raw_participation_state == "load_syncing":
|
||
participation_bucket = "load_syncing"
|
||
participation_bucket_label = "负载待确认"
|
||
elif raw_participation_state in {"standby", "idle"}:
|
||
participation_bucket = "standby"
|
||
participation_bucket_label = "在线待命"
|
||
else:
|
||
participation_bucket = raw_participation_state
|
||
participation_bucket_label = str(node.get("participation_label") or "").strip()
|
||
|
||
return {
|
||
"node_code": str(node.get("node_code") or ""),
|
||
"region": str(node.get("region") or ""),
|
||
"role": str(node.get("role") or ""),
|
||
"status": str(node.get("status") or ""),
|
||
"current_load": int(node.get("current_load", 0) or 0),
|
||
"last_heartbeat_at": str(node.get("last_heartbeat_at") or ""),
|
||
"participation_state": raw_participation_state,
|
||
"participation_state_raw": raw_participation_state,
|
||
"participation_label": str(node.get("participation_label") or ""),
|
||
"participation_reason": str(node.get("participation_reason") or node.get("standby_reason") or ""),
|
||
"participation_bucket": participation_bucket,
|
||
"participation_bucket_label": participation_bucket_label,
|
||
"is_dispatch_active": is_dispatch_active,
|
||
"items_total": int(node.get("items_total", 0) or 0),
|
||
"items_claimed": int(node.get("items_claimed", 0) or 0),
|
||
"items_running": int(node.get("items_running", 0) or 0),
|
||
"items_completed": int(node.get("items_completed", 0) or 0),
|
||
"items_failed": int(node.get("items_failed", 0) or 0),
|
||
"processed_recent": int(node.get("processed_recent", 0) or 0),
|
||
"processed_per_minute": float(node.get("processed_per_minute", 0) or 0),
|
||
"detail": str(node.get("detail") or ""),
|
||
"phase": str(node.get("phase") or ""),
|
||
}
|
||
|
||
|
||
def _preferred_release_label(release_summary: dict) -> str:
|
||
active_by_channel = dict(release_summary.get("active_by_channel") or {})
|
||
for channel in ("stable", "beta", "canary"):
|
||
version = str(active_by_channel.get(channel) or "").strip()
|
||
if version:
|
||
return f"{version} / {channel}"
|
||
return ""
|
||
|
||
|
||
def _preferred_release_for_ops() -> dict:
|
||
for channel in ("stable", "beta", "canary"):
|
||
release = get_latest_release(channel=channel)
|
||
if int(release.get("id") or 0) > 0:
|
||
return release
|
||
return {}
|
||
|
||
|
||
def _normalize_driver_node_codes(raw_node_codes: object) -> list[str]:
|
||
normalized: list[str] = []
|
||
seen: set[str] = set()
|
||
for item in list(raw_node_codes or []):
|
||
node_code = str(item or "").strip()
|
||
if not node_code or node_code in seen:
|
||
continue
|
||
seen.add(node_code)
|
||
normalized.append(node_code)
|
||
return normalized
|
||
|
||
|
||
def _recommend_driver_execution_mode(node_codes: list[str]) -> dict:
|
||
return _recommend_driver_execution_mode_for_supported_modes(
|
||
node_codes,
|
||
supported_modes=["remote-agent", "ssh"],
|
||
default_mode="remote-agent",
|
||
)
|
||
|
||
|
||
def _normalize_driver_supported_execution_modes(
|
||
supported_modes: list[str] | tuple[str, ...] | None = None,
|
||
*,
|
||
default_mode: str = "remote-agent",
|
||
) -> list[str]:
|
||
normalized_default_mode = str(default_mode or "").strip() or "remote-agent"
|
||
normalized_modes: list[str] = []
|
||
seen: set[str] = set()
|
||
for item in list(supported_modes or []):
|
||
mode = str(item or "").strip()
|
||
if not mode or mode in seen:
|
||
continue
|
||
seen.add(mode)
|
||
normalized_modes.append(mode)
|
||
if normalized_modes:
|
||
return normalized_modes
|
||
return [normalized_default_mode]
|
||
|
||
|
||
def _recommend_driver_execution_mode_for_supported_modes(
|
||
node_codes: list[str],
|
||
*,
|
||
supported_modes: list[str] | tuple[str, ...] | None = None,
|
||
default_mode: str = "remote-agent",
|
||
) -> dict:
|
||
normalized_node_codes = _normalize_driver_node_codes(node_codes)
|
||
normalized_supported_modes = _normalize_driver_supported_execution_modes(
|
||
supported_modes,
|
||
default_mode=default_mode,
|
||
)
|
||
normalized_default_mode = (
|
||
str(default_mode or "").strip()
|
||
if str(default_mode or "").strip() in normalized_supported_modes
|
||
else normalized_supported_modes[0]
|
||
)
|
||
if len(normalized_supported_modes) == 1:
|
||
only_mode = normalized_supported_modes[0]
|
||
return {
|
||
"execution_mode": only_mode,
|
||
"reason": f"当前动作仅支持 {only_mode},保持该执行路径。",
|
||
"resolved_nodes": [],
|
||
"missing_node_codes": [],
|
||
"supported_modes": normalized_supported_modes,
|
||
}
|
||
if not normalized_node_codes:
|
||
return {
|
||
"execution_mode": normalized_default_mode,
|
||
"reason": f"当前未指定目标节点,默认继续使用 {normalized_default_mode}。",
|
||
"resolved_nodes": [],
|
||
"missing_node_codes": [],
|
||
"supported_modes": normalized_supported_modes,
|
||
}
|
||
if (
|
||
"local-runtime" in normalized_supported_modes
|
||
and normalized_node_codes
|
||
and all(node_code == settings.node_code for node_code in normalized_node_codes)
|
||
):
|
||
return {
|
||
"execution_mode": "local-runtime",
|
||
"reason": "当前目标节点就是本机控制面节点,推荐直接走 local-runtime。",
|
||
"resolved_nodes": [],
|
||
"missing_node_codes": [],
|
||
"supported_modes": normalized_supported_modes,
|
||
}
|
||
|
||
managed_nodes_payload = list_managed_nodes_with_agent_state()
|
||
managed_nodes = list(managed_nodes_payload.get("nodes") or [])
|
||
managed_map = {
|
||
str(item.get("node_code") or "").strip(): dict(item or {})
|
||
for item in managed_nodes
|
||
if str(item.get("node_code") or "").strip()
|
||
}
|
||
resolved_nodes = [managed_map[node_code] for node_code in normalized_node_codes if node_code in managed_map]
|
||
missing_node_codes = [node_code for node_code in normalized_node_codes if node_code not in managed_map]
|
||
|
||
def _agent_ready(node: dict) -> bool:
|
||
return (
|
||
bool(node.get("is_managed", False))
|
||
and bool(node.get("is_enabled", False))
|
||
and bool(node.get("is_agent_online", False))
|
||
)
|
||
|
||
def _ssh_ready(node: dict) -> bool:
|
||
return (
|
||
bool(node.get("is_managed", False))
|
||
and bool(node.get("is_enabled", False))
|
||
and bool(node.get("has_ssh_access", False))
|
||
)
|
||
|
||
if (
|
||
"remote-agent" in normalized_supported_modes
|
||
and resolved_nodes
|
||
and len(resolved_nodes) == len(normalized_node_codes)
|
||
and all(_agent_ready(node) for node in resolved_nodes)
|
||
):
|
||
return {
|
||
"execution_mode": "remote-agent",
|
||
"reason": "当前目标节点都已进入 Agent 在线状态,默认沿用 remote-agent。",
|
||
"resolved_nodes": resolved_nodes,
|
||
"missing_node_codes": missing_node_codes,
|
||
"supported_modes": normalized_supported_modes,
|
||
}
|
||
if (
|
||
"ssh" in normalized_supported_modes
|
||
and resolved_nodes
|
||
and len(resolved_nodes) == len(normalized_node_codes)
|
||
and all(_ssh_ready(node) for node in resolved_nodes)
|
||
):
|
||
return {
|
||
"execution_mode": "ssh",
|
||
"reason": "当前目标节点 SSH 已全部备好,自动切到 SSH 执行路径。",
|
||
"resolved_nodes": resolved_nodes,
|
||
"missing_node_codes": missing_node_codes,
|
||
"supported_modes": normalized_supported_modes,
|
||
}
|
||
return {
|
||
"execution_mode": normalized_default_mode,
|
||
"reason": f"当前目标节点执行能力不一致,默认保守回退到 {normalized_default_mode}。",
|
||
"resolved_nodes": resolved_nodes,
|
||
"missing_node_codes": missing_node_codes,
|
||
"supported_modes": normalized_supported_modes,
|
||
}
|
||
|
||
|
||
def _recommend_driver_execution_mode_for_playbook(playbook_key: str, node_codes: list[str]) -> dict:
|
||
playbook = dict(get_ops_playbook(playbook_key) or {})
|
||
supported_modes = list(playbook.get("execution_modes") or [])
|
||
default_mode = str(playbook.get("default_execution_mode") or "remote-agent").strip() or "remote-agent"
|
||
return _recommend_driver_execution_mode_for_supported_modes(
|
||
node_codes,
|
||
supported_modes=supported_modes,
|
||
default_mode=default_mode,
|
||
)
|
||
|
||
|
||
def _recommend_driver_execution_mode_for_action_template(template_key: str, node_codes: list[str]) -> dict:
|
||
template = dict(get_ops_action_template(template_key) or {})
|
||
supported_modes = list(template.get("execution_modes") or [])
|
||
default_mode = str(template.get("default_execution_mode") or "remote-agent").strip() or "remote-agent"
|
||
return _recommend_driver_execution_mode_for_supported_modes(
|
||
node_codes,
|
||
supported_modes=supported_modes,
|
||
default_mode=default_mode,
|
||
)
|
||
|
||
|
||
def _build_driver_ui_intent(kind: str, **payload: object) -> dict:
|
||
return {
|
||
"kind": str(kind or "").strip(),
|
||
**payload,
|
||
}
|
||
|
||
|
||
def _normalize_focus_ref(focus_ref: object) -> dict:
|
||
if not isinstance(focus_ref, dict):
|
||
return {}
|
||
normalized: dict[str, object] = {}
|
||
for raw_key, raw_value in focus_ref.items():
|
||
key = str(raw_key or "").strip()
|
||
if not key:
|
||
continue
|
||
if isinstance(raw_value, bool):
|
||
normalized[key] = raw_value
|
||
continue
|
||
if isinstance(raw_value, int):
|
||
if raw_value:
|
||
normalized[key] = raw_value
|
||
continue
|
||
if isinstance(raw_value, float):
|
||
if raw_value:
|
||
normalized[key] = raw_value
|
||
continue
|
||
if isinstance(raw_value, str):
|
||
value = raw_value.strip()
|
||
if value:
|
||
normalized[key] = value
|
||
continue
|
||
if raw_value is not None:
|
||
normalized[key] = raw_value
|
||
return normalized
|
||
|
||
|
||
def _find_activity_stream_item(activity_key: str, *, kind: str = "", scan_limit: int = 200) -> dict:
|
||
normalized_activity_key = str(activity_key or "").strip()
|
||
normalized_kind = str(kind or "").strip()
|
||
if not normalized_activity_key:
|
||
return {}
|
||
activity_stream = get_ops_activity_stream(limit=min(max(scan_limit, 20), 100), scan_limit=max(scan_limit, 80))
|
||
for item in list(activity_stream.get("items") or []):
|
||
if str(item.get("activity_key") or "").strip() != normalized_activity_key:
|
||
continue
|
||
if normalized_kind and str(item.get("kind") or "").strip() != normalized_kind:
|
||
continue
|
||
return dict(item or {})
|
||
return {}
|
||
|
||
|
||
def _merge_focus_ref(focus_ref: object, **extra: object) -> dict:
|
||
merged = _normalize_focus_ref(focus_ref)
|
||
for raw_key, raw_value in extra.items():
|
||
key = str(raw_key or "").strip()
|
||
if not key:
|
||
continue
|
||
if isinstance(raw_value, bool):
|
||
merged[key] = raw_value
|
||
continue
|
||
if isinstance(raw_value, int):
|
||
if raw_value:
|
||
merged[key] = raw_value
|
||
continue
|
||
if isinstance(raw_value, float):
|
||
if raw_value:
|
||
merged[key] = raw_value
|
||
continue
|
||
if isinstance(raw_value, str):
|
||
value = raw_value.strip()
|
||
if value:
|
||
merged[key] = value
|
||
continue
|
||
if raw_value is not None:
|
||
merged[key] = raw_value
|
||
return merged
|
||
|
||
|
||
def _action_payload_with_focus_ref(action_payload: dict | None = None, focus_ref: object = None) -> dict:
|
||
payload = dict(action_payload or {})
|
||
merged_focus_ref = _merge_focus_ref(focus_ref, **_normalize_focus_ref(payload.get("focus_ref")))
|
||
if merged_focus_ref:
|
||
payload["focus_ref"] = merged_focus_ref
|
||
return payload
|
||
|
||
|
||
def _driver_launchpad_context(action_payload: dict) -> dict:
|
||
payload = dict(action_payload or {})
|
||
return {
|
||
"channel": str(payload.get("channel") or "stable").strip() or "stable",
|
||
"control_plane_base_url": str(payload.get("control_plane_base_url") or "").strip(),
|
||
}
|
||
|
||
|
||
def _release_launchpad_preview(release_launchpad: dict, mode: str) -> dict:
|
||
if str(mode or "").strip() == "control":
|
||
return dict(release_launchpad.get("control_rollout_preview") or {})
|
||
return dict(release_launchpad.get("worker_rollout_preview") or {})
|
||
|
||
|
||
def _release_launchpad_operational_rows(release_launchpad: dict, *, modes: tuple[str, ...]) -> list[dict]:
|
||
rows: list[dict] = []
|
||
seen: set[tuple[str, str]] = set()
|
||
for mode in modes:
|
||
preview = _release_launchpad_preview(release_launchpad, mode)
|
||
release_gate = dict((dict(preview.get("policy_preview") or {}).get("release_gate") or {}))
|
||
operational_readiness = dict(release_gate.get("operational_readiness") or {})
|
||
for raw_row in list(operational_readiness.get("rows") or []):
|
||
row = dict(raw_row or {})
|
||
node_code = str(row.get("node_code") or "").strip()
|
||
if not node_code:
|
||
continue
|
||
key = (str(mode or "").strip() or "worker", node_code)
|
||
if key in seen:
|
||
continue
|
||
seen.add(key)
|
||
rows.append(
|
||
{
|
||
**row,
|
||
"mode": str(mode or "").strip() or "worker",
|
||
}
|
||
)
|
||
return rows
|
||
|
||
|
||
def _release_launchpad_gap_rows(release_launchpad: dict) -> list[dict]:
|
||
rows = _release_launchpad_operational_rows(release_launchpad, modes=("worker", "control"))
|
||
|
||
def _row_execution_ready(row: dict) -> bool:
|
||
if "execution_ready" in row:
|
||
return bool(row.get("execution_ready", False))
|
||
return bool(row.get("remote_agent_ready", False))
|
||
|
||
gap_rows = [
|
||
row
|
||
for row in rows
|
||
if not _row_execution_ready(row)
|
||
or str(row.get("inspection_status") or "").strip() in {"running", "attention", "missing"}
|
||
]
|
||
gap_rows.sort(
|
||
key=lambda row: (
|
||
0 if not _row_execution_ready(row) else 1,
|
||
0 if str(row.get("inspection_status") or "").strip() in {"attention", "missing"} else 1,
|
||
0 if str(row.get("inspection_status") or "").strip() == "running" else 1,
|
||
str(row.get("node_code") or ""),
|
||
)
|
||
)
|
||
return gap_rows
|
||
|
||
|
||
def _first_release_launchpad_gap_row(release_launchpad: dict) -> dict:
|
||
gap_rows = _release_launchpad_gap_rows(release_launchpad)
|
||
return dict(gap_rows[0] or {}) if gap_rows else {}
|
||
|
||
|
||
def _release_launchpad_gap_action_context(release_launchpad: dict) -> dict:
|
||
first_gap_row = _first_release_launchpad_gap_row(release_launchpad)
|
||
action_code = str(first_gap_row.get("recovery_action") or "").strip()
|
||
node_code = str(first_gap_row.get("node_code") or "").strip()
|
||
summary = str(
|
||
first_gap_row.get("recovery_summary")
|
||
or first_gap_row.get("onboarding_summary")
|
||
or first_gap_row.get("agent_reason")
|
||
or first_gap_row.get("inspection_reason")
|
||
or ""
|
||
).strip()
|
||
if action_code not in {"bootstrap_run", "run_acceptance"}:
|
||
action_code = ""
|
||
return {
|
||
"action_code": action_code,
|
||
"node_code": node_code,
|
||
"summary": summary,
|
||
"row": first_gap_row,
|
||
}
|
||
|
||
|
||
def _release_launchpad_target_node_codes(release_launchpad: dict, *, mode: str) -> list[str]:
|
||
preview = _release_launchpad_preview(release_launchpad, mode)
|
||
smart_rollout = dict(preview.get("smart_rollout") or {})
|
||
target_node_codes = _normalize_driver_node_codes(smart_rollout.get("target_node_codes") or [])
|
||
if target_node_codes:
|
||
return target_node_codes
|
||
return _normalize_driver_node_codes(
|
||
[str(row.get("node_code") or "").strip() for row in _release_launchpad_operational_rows(release_launchpad, modes=(mode,))]
|
||
)
|
||
|
||
|
||
def _build_release_launchpad_review_payload(*, release_launchpad: dict, mode: str) -> dict:
|
||
normalized_mode = "control" if str(mode or "").strip() == "control" else "worker"
|
||
launchpad = dict(release_launchpad or {})
|
||
latest_release = dict(launchpad.get("latest_release") or {})
|
||
preview = _release_launchpad_preview(launchpad, normalized_mode)
|
||
smart_rollout = dict(preview.get("smart_rollout") or {})
|
||
launchpad_status = dict(launchpad.get("launchpad_status") or {})
|
||
focus_ref = _merge_focus_ref(
|
||
launchpad_status.get("focus_ref"),
|
||
release_id=int(latest_release.get("id") or 0),
|
||
release_version=str(latest_release.get("release_version") or "").strip(),
|
||
channel=str(latest_release.get("channel") or launchpad.get("channel") or "").strip(),
|
||
section="release_launchpad",
|
||
mode=normalized_mode,
|
||
)
|
||
return {
|
||
"mode": normalized_mode,
|
||
"focus_ref": focus_ref,
|
||
"latest_release": latest_release,
|
||
"launchpad_status": launchpad_status,
|
||
"preview": preview,
|
||
"smart_rollout": smart_rollout,
|
||
"target_node_codes": _release_launchpad_target_node_codes(launchpad, mode=normalized_mode),
|
||
"gap_rows": _release_launchpad_gap_rows(launchpad),
|
||
}
|
||
|
||
|
||
def _build_release_launchpad_driver_card(*, release_launchpad: dict) -> dict:
|
||
launchpad = dict(release_launchpad or {})
|
||
launchpad_status = dict(launchpad.get("launchpad_status") or {})
|
||
action_code = str(launchpad_status.get("recommended_action_code") or "").strip()
|
||
if not action_code:
|
||
return {}
|
||
|
||
latest_release = dict(launchpad.get("latest_release") or {})
|
||
latest_package = dict(launchpad.get("latest_package") or {})
|
||
worker_preview = dict(launchpad.get("worker_rollout_preview") or {})
|
||
control_preview = dict(launchpad.get("control_rollout_preview") or {})
|
||
worker_plan = dict(worker_preview.get("smart_rollout") or {})
|
||
control_plan = dict(control_preview.get("smart_rollout") or {})
|
||
first_gap_row = _first_release_launchpad_gap_row(launchpad)
|
||
gap_node_codes = _normalize_driver_node_codes(
|
||
[str(row.get("node_code") or "").strip() for row in _release_launchpad_gap_rows(launchpad)]
|
||
)
|
||
worker_node_codes = _release_launchpad_target_node_codes(launchpad, mode="worker")
|
||
control_node_codes = _release_launchpad_target_node_codes(launchpad, mode="control")
|
||
latest_release_id = int(latest_release.get("id") or 0)
|
||
launchpad_focus_ref = _merge_focus_ref(
|
||
launchpad_status.get("focus_ref"),
|
||
release_id=latest_release_id,
|
||
release_version=str(latest_release.get("release_version") or "").strip(),
|
||
channel=str(latest_release.get("channel") or launchpad.get("channel") or "").strip(),
|
||
)
|
||
release_detail_focus_ref = _merge_focus_ref(
|
||
latest_release.get("focus_ref"),
|
||
release_id=latest_release_id,
|
||
release_version=str(latest_release.get("release_version") or "").strip(),
|
||
channel=str(latest_release.get("channel") or launchpad.get("channel") or "").strip(),
|
||
section="release_detail",
|
||
)
|
||
worker_launchpad_focus_ref = _merge_focus_ref(launchpad_focus_ref, section="release_launchpad", mode="worker")
|
||
control_launchpad_focus_ref = _merge_focus_ref(launchpad_focus_ref, section="release_launchpad", mode="control")
|
||
rollout_gate_focus_ref = _merge_focus_ref(launchpad_focus_ref, section="default_rollout_gate")
|
||
release_label = (
|
||
str(latest_release.get("release_version") or "").strip()
|
||
or str(latest_package.get("package_name") or "").strip()
|
||
or str(launchpad.get("channel") or "stable").strip()
|
||
or "latest"
|
||
)
|
||
shared_action_payload = {
|
||
"channel": str(launchpad.get("channel") or "stable").strip() or "stable",
|
||
"control_plane_base_url": str(launchpad.get("control_plane_base_url") or "").strip(),
|
||
}
|
||
if latest_release_id > 0:
|
||
shared_action_payload["release_id"] = latest_release_id
|
||
|
||
base_card = {
|
||
"level_label": "推荐",
|
||
"tag_type": "info",
|
||
"summary": str(launchpad_status.get("summary") or "").strip(),
|
||
"reason": "",
|
||
"node_codes": [],
|
||
"meta_text": release_label,
|
||
"focus_ref": launchpad_focus_ref,
|
||
"primary_focus_ref": {},
|
||
"secondary_focus_ref": {},
|
||
"primary_type": "primary",
|
||
"disabled": False,
|
||
"primary_action_code": action_code,
|
||
"primary_action_payload": dict(shared_action_payload),
|
||
"secondary_label": "查看版本区" if latest_release_id > 0 else "",
|
||
"secondary_action_code": "focus_release_hub" if latest_release_id > 0 else "",
|
||
"secondary_action_payload": (
|
||
{
|
||
"release_id": latest_release_id,
|
||
}
|
||
if latest_release_id > 0
|
||
else {}
|
||
),
|
||
}
|
||
|
||
if action_code == "publish_latest_worker":
|
||
return {
|
||
**base_card,
|
||
"key": "release-launchpad-ready",
|
||
"title": "当前可以发 Worker 灰度",
|
||
"level_label": "准备就绪",
|
||
"tag_type": "success",
|
||
"reason": str(worker_plan.get("summary") or worker_preview.get("message") or "最新发布包与 Worker 智能 Rollout 预检均已收口。").strip(),
|
||
"node_codes": worker_node_codes,
|
||
"primary_label": "发 Worker 灰度",
|
||
"primary_focus_ref": worker_launchpad_focus_ref,
|
||
"primary_action_payload": _action_payload_with_focus_ref(shared_action_payload, worker_launchpad_focus_ref),
|
||
"secondary_label": "看 Control 预案" if bool(control_plan.get("available", False)) else base_card["secondary_label"],
|
||
"secondary_action_code": "review_control_rollout" if bool(control_plan.get("available", False)) else base_card["secondary_action_code"],
|
||
"secondary_focus_ref": (
|
||
control_launchpad_focus_ref if bool(control_plan.get("available", False)) else release_detail_focus_ref
|
||
),
|
||
"secondary_action_payload": (
|
||
_action_payload_with_focus_ref(shared_action_payload, control_launchpad_focus_ref)
|
||
if bool(control_plan.get("available", False))
|
||
else _action_payload_with_focus_ref(base_card["secondary_action_payload"], release_detail_focus_ref)
|
||
),
|
||
}
|
||
|
||
if action_code == "review_smart_rollout_preview":
|
||
return {
|
||
**base_card,
|
||
"key": "release-launchpad-review-worker",
|
||
"title": "先确认 Worker 灰度预案",
|
||
"level_label": "待确认",
|
||
"tag_type": "warning",
|
||
"reason": str(worker_preview.get("message") or worker_plan.get("summary") or "Worker 智能 Rollout 仍需人工确认。").strip(),
|
||
"node_codes": worker_node_codes,
|
||
"primary_label": "查看 Worker 预案",
|
||
"primary_focus_ref": worker_launchpad_focus_ref,
|
||
"primary_action_payload": _action_payload_with_focus_ref(shared_action_payload, worker_launchpad_focus_ref),
|
||
"secondary_focus_ref": release_detail_focus_ref,
|
||
"secondary_action_payload": _action_payload_with_focus_ref(base_card["secondary_action_payload"], release_detail_focus_ref),
|
||
}
|
||
|
||
if action_code == "review_control_rollout":
|
||
return {
|
||
**base_card,
|
||
"key": "release-launchpad-review-control",
|
||
"title": "先确认 Control 发布预案",
|
||
"level_label": "推荐",
|
||
"tag_type": "warning",
|
||
"reason": str(control_preview.get("message") or control_plan.get("summary") or "当前只有 Control 侧满足预发条件。").strip(),
|
||
"node_codes": control_node_codes,
|
||
"primary_label": "查看 Control 预案",
|
||
"primary_focus_ref": control_launchpad_focus_ref,
|
||
"primary_action_payload": _action_payload_with_focus_ref(shared_action_payload, control_launchpad_focus_ref),
|
||
"secondary_focus_ref": release_detail_focus_ref,
|
||
"secondary_action_payload": _action_payload_with_focus_ref(base_card["secondary_action_payload"], release_detail_focus_ref),
|
||
}
|
||
|
||
if action_code == "fix_rollout_blockers":
|
||
primary_label = "查看 Worker 预案"
|
||
if str(first_gap_row.get("node_code") or "").strip():
|
||
primary_label = "纳管首台节点" if not bool(first_gap_row.get("is_managed", False)) else "补接入首台"
|
||
if str(first_gap_row.get("recovery_action") or "").strip() == "bootstrap_run":
|
||
primary_label = "跑接入收口"
|
||
elif str(first_gap_row.get("recovery_action") or "").strip() == "run_acceptance":
|
||
primary_label = "跑接管验收"
|
||
return {
|
||
**base_card,
|
||
"key": "release-launchpad-fix-blockers",
|
||
"title": "先收口发布门禁",
|
||
"level_label": "最高优先",
|
||
"tag_type": "danger",
|
||
"reason": str(
|
||
first_gap_row.get("agent_reason")
|
||
or first_gap_row.get("inspection_reason")
|
||
or worker_preview.get("message")
|
||
or control_preview.get("message")
|
||
or "Rollout 预检仍存在阻断条件,需先补接管、在线状态或巡检缺口。"
|
||
).strip(),
|
||
"node_codes": gap_node_codes or worker_node_codes or control_node_codes,
|
||
"primary_label": primary_label,
|
||
"primary_focus_ref": rollout_gate_focus_ref,
|
||
"primary_action_payload": _action_payload_with_focus_ref(shared_action_payload, rollout_gate_focus_ref),
|
||
"secondary_focus_ref": release_detail_focus_ref,
|
||
"secondary_action_payload": _action_payload_with_focus_ref(base_card["secondary_action_payload"], release_detail_focus_ref),
|
||
}
|
||
|
||
if action_code in {"bootstrap_run", "run_acceptance"}:
|
||
primary_label = "跑接入收口" if action_code == "bootstrap_run" else "跑接管验收"
|
||
return {
|
||
**base_card,
|
||
"key": f"release-launchpad-{action_code}",
|
||
"title": "先收口执行面接管",
|
||
"level_label": "最高优先" if action_code == "bootstrap_run" else "推荐",
|
||
"tag_type": "warning" if action_code == "bootstrap_run" else "success",
|
||
"reason": str(
|
||
first_gap_row.get("recovery_summary")
|
||
or first_gap_row.get("onboarding_summary")
|
||
or first_gap_row.get("agent_reason")
|
||
or first_gap_row.get("inspection_reason")
|
||
or launchpad_status.get("summary")
|
||
or "当前需要先收口节点接管,再继续发布。"
|
||
).strip(),
|
||
"node_codes": gap_node_codes or ([str(first_gap_row.get("node_code") or "").strip()] if str(first_gap_row.get("node_code") or "").strip() else []),
|
||
"primary_label": primary_label,
|
||
"primary_focus_ref": rollout_gate_focus_ref,
|
||
"primary_action_payload": _action_payload_with_focus_ref(
|
||
{
|
||
**shared_action_payload,
|
||
"node_code": str(first_gap_row.get("node_code") or "").strip(),
|
||
},
|
||
rollout_gate_focus_ref,
|
||
),
|
||
"secondary_focus_ref": release_detail_focus_ref,
|
||
"secondary_action_payload": _action_payload_with_focus_ref(base_card["secondary_action_payload"], release_detail_focus_ref),
|
||
}
|
||
|
||
if action_code == "fix_managed_nodes":
|
||
primary_label = "同步节点"
|
||
if str(first_gap_row.get("node_code") or "").strip():
|
||
primary_label = "纳管首台节点" if not bool(first_gap_row.get("is_managed", False)) else "补接入首台"
|
||
return {
|
||
**base_card,
|
||
"key": "release-launchpad-fix-managed",
|
||
"title": "先补执行面接管",
|
||
"level_label": "推荐",
|
||
"tag_type": "warning",
|
||
"reason": str(
|
||
first_gap_row.get("agent_reason")
|
||
or first_gap_row.get("inspection_reason")
|
||
or "当前没有可用于智能 Rollout 的在线启用执行节点,需要先补齐托管或接入状态。"
|
||
).strip(),
|
||
"node_codes": gap_node_codes,
|
||
"primary_label": primary_label,
|
||
"primary_focus_ref": rollout_gate_focus_ref,
|
||
"primary_action_payload": _action_payload_with_focus_ref(shared_action_payload, rollout_gate_focus_ref),
|
||
"secondary_focus_ref": release_detail_focus_ref,
|
||
"secondary_action_payload": _action_payload_with_focus_ref(base_card["secondary_action_payload"], release_detail_focus_ref),
|
||
}
|
||
|
||
if action_code == "api-restart":
|
||
return {
|
||
**base_card,
|
||
"key": "release-launchpad-runtime-refresh",
|
||
"title": "先刷新控制面运行时",
|
||
"level_label": "最高优先",
|
||
"tag_type": "warning",
|
||
"reason": str(
|
||
launchpad_status.get("summary")
|
||
or "运行中的控制面 API 还没有刷新到当前仓库能力,建议先执行 runtime-refresh-recover。"
|
||
).strip(),
|
||
"node_codes": [],
|
||
"primary_label": "查看刷新收口",
|
||
"primary_focus_ref": worker_launchpad_focus_ref or launchpad_focus_ref,
|
||
"primary_action_payload": _action_payload_with_focus_ref(
|
||
shared_action_payload,
|
||
worker_launchpad_focus_ref or launchpad_focus_ref,
|
||
),
|
||
"secondary_label": "查看版本区" if latest_release_id > 0 else "",
|
||
"secondary_action_code": "focus_release_hub" if latest_release_id > 0 else "",
|
||
"secondary_action_payload": {"release_id": latest_release_id} if latest_release_id > 0 else {},
|
||
"secondary_focus_ref": release_detail_focus_ref if latest_release_id > 0 else {},
|
||
}
|
||
|
||
if action_code == "release_package":
|
||
return {
|
||
**base_card,
|
||
"key": "release-launchpad-package",
|
||
"title": "先准备发布包",
|
||
"level_label": "待补齐",
|
||
"tag_type": "warning",
|
||
"reason": str(latest_package.get("reason") or "当前还没有可用的最新发布包,请先在控制面本机完成打包。").strip(),
|
||
"node_codes": [],
|
||
"primary_label": "打开打包面板",
|
||
"secondary_label": "",
|
||
"secondary_action_code": "",
|
||
"secondary_action_payload": {},
|
||
"primary_focus_ref": worker_launchpad_focus_ref or launchpad_focus_ref,
|
||
"primary_action_payload": _action_payload_with_focus_ref(
|
||
shared_action_payload,
|
||
worker_launchpad_focus_ref or launchpad_focus_ref,
|
||
),
|
||
"secondary_focus_ref": {},
|
||
}
|
||
|
||
if action_code == "release_prepare":
|
||
return {
|
||
**base_card,
|
||
"key": "release-launchpad-final-signoff",
|
||
"title": "先完成最终签收",
|
||
"level_label": "最高优先",
|
||
"tag_type": "danger",
|
||
"reason": str(
|
||
latest_package.get("final_release_report_stale_reason")
|
||
or launchpad_status.get("summary")
|
||
or "当前最新发布包还没有完成最终签收,不能直接进入正式发布动作。"
|
||
).strip(),
|
||
"node_codes": [],
|
||
"primary_label": "执行最终签收",
|
||
"secondary_label": "查看版本区" if latest_release_id > 0 else "",
|
||
"secondary_action_code": "focus_release_hub" if latest_release_id > 0 else "",
|
||
"secondary_action_payload": {"release_id": latest_release_id} if latest_release_id > 0 else {},
|
||
"primary_focus_ref": worker_launchpad_focus_ref or launchpad_focus_ref,
|
||
"primary_action_payload": _action_payload_with_focus_ref(
|
||
shared_action_payload,
|
||
worker_launchpad_focus_ref or launchpad_focus_ref,
|
||
),
|
||
"secondary_focus_ref": release_detail_focus_ref if latest_release_id > 0 else {},
|
||
}
|
||
|
||
return {}
|
||
|
||
|
||
def _runbook_sequence(
|
||
*,
|
||
key: str,
|
||
title: str,
|
||
summary: str,
|
||
reason: str,
|
||
status: str = "ready",
|
||
status_label: str = "",
|
||
tag_type: str = "",
|
||
target_node_codes: list[str] | None = None,
|
||
target_scope_label: str = "",
|
||
step_titles: list[str] | None = None,
|
||
primary_label: str,
|
||
primary_action_code: str,
|
||
primary_action_payload: dict | None = None,
|
||
focus_ref: dict | None = None,
|
||
primary_focus_ref: dict | None = None,
|
||
secondary_label: str = "",
|
||
secondary_action_code: str = "",
|
||
secondary_action_payload: dict | None = None,
|
||
secondary_focus_ref: dict | None = None,
|
||
) -> dict:
|
||
return {
|
||
"key": str(key or "").strip(),
|
||
"title": str(title or "").strip(),
|
||
"summary": str(summary or "").strip(),
|
||
"reason": str(reason or "").strip(),
|
||
"status": str(status or "").strip() or "ready",
|
||
"status_label": str(status_label or "").strip() or str(status or "").strip() or "ready",
|
||
"tag_type": str(tag_type or "").strip() or "info",
|
||
"target_node_codes": _normalize_driver_node_codes(target_node_codes or []),
|
||
"target_scope_label": str(target_scope_label or "").strip(),
|
||
"step_titles": [str(item or "").strip() for item in list(step_titles or []) if str(item or "").strip()],
|
||
"focus_ref": _normalize_focus_ref(focus_ref),
|
||
"primary_label": str(primary_label or "").strip() or "执行",
|
||
"primary_action_code": str(primary_action_code or "").strip(),
|
||
"primary_action_payload": _action_payload_with_focus_ref(primary_action_payload or {}, primary_focus_ref),
|
||
"primary_focus_ref": _normalize_focus_ref(primary_focus_ref),
|
||
"secondary_label": str(secondary_label or "").strip(),
|
||
"secondary_action_code": str(secondary_action_code or "").strip(),
|
||
"secondary_action_payload": _action_payload_with_focus_ref(secondary_action_payload or {}, secondary_focus_ref),
|
||
"secondary_focus_ref": _normalize_focus_ref(secondary_focus_ref),
|
||
}
|
||
|
||
|
||
def _build_release_launchpad_runbook_sequence(*, release_launchpad: dict) -> dict:
|
||
launchpad = dict(release_launchpad or {})
|
||
launchpad_status = dict(launchpad.get("launchpad_status") or {})
|
||
action_code = str(launchpad_status.get("recommended_action_code") or "").strip()
|
||
if not action_code:
|
||
return {}
|
||
|
||
latest_release = dict(launchpad.get("latest_release") or {})
|
||
latest_package = dict(launchpad.get("latest_package") or {})
|
||
worker_preview = dict(launchpad.get("worker_rollout_preview") or {})
|
||
control_preview = dict(launchpad.get("control_rollout_preview") or {})
|
||
worker_plan = dict(worker_preview.get("smart_rollout") or {})
|
||
control_plan = dict(control_preview.get("smart_rollout") or {})
|
||
first_gap_row = _first_release_launchpad_gap_row(launchpad)
|
||
latest_release_id = int(latest_release.get("id") or 0)
|
||
worker_node_codes = _release_launchpad_target_node_codes(launchpad, mode="worker")
|
||
control_node_codes = _release_launchpad_target_node_codes(launchpad, mode="control")
|
||
gap_node_codes = _normalize_driver_node_codes(
|
||
[str(row.get("node_code") or "").strip() for row in _release_launchpad_gap_rows(launchpad)]
|
||
)
|
||
launchpad_focus_ref = _merge_focus_ref(
|
||
launchpad_status.get("focus_ref"),
|
||
release_id=latest_release_id,
|
||
release_version=str(latest_release.get("release_version") or "").strip(),
|
||
channel=str(latest_release.get("channel") or launchpad.get("channel") or "").strip(),
|
||
)
|
||
release_detail_focus_ref = _merge_focus_ref(
|
||
latest_release.get("focus_ref"),
|
||
release_id=latest_release_id,
|
||
release_version=str(latest_release.get("release_version") or "").strip(),
|
||
channel=str(latest_release.get("channel") or launchpad.get("channel") or "").strip(),
|
||
section="release_detail",
|
||
)
|
||
worker_launchpad_focus_ref = _merge_focus_ref(launchpad_focus_ref, section="release_launchpad", mode="worker")
|
||
control_launchpad_focus_ref = _merge_focus_ref(launchpad_focus_ref, section="release_launchpad", mode="control")
|
||
rollout_gate_focus_ref = _merge_focus_ref(launchpad_focus_ref, section="default_rollout_gate")
|
||
release_scope_label = (
|
||
str(latest_release.get("release_version") or "").strip()
|
||
or str(latest_package.get("package_name") or "").strip()
|
||
or str(launchpad.get("channel") or "stable").strip()
|
||
or "latest"
|
||
)
|
||
shared_action_payload = {
|
||
"channel": str(launchpad.get("channel") or "stable").strip() or "stable",
|
||
"control_plane_base_url": str(launchpad.get("control_plane_base_url") or "").strip(),
|
||
}
|
||
if latest_release_id > 0:
|
||
shared_action_payload["release_id"] = latest_release_id
|
||
|
||
base_kwargs = {
|
||
"key": "release_progression",
|
||
"title": "版本发布与放量",
|
||
"summary": str(launchpad_status.get("summary") or "").strip(),
|
||
"status": str(launchpad_status.get("status") or "").strip() or "attention",
|
||
"status_label": str(launchpad_status.get("status_label") or "").strip() or "待处理",
|
||
"target_scope_label": f"Release / Launchpad / {release_scope_label}",
|
||
"focus_ref": launchpad_focus_ref,
|
||
}
|
||
|
||
if action_code == "publish_latest_worker":
|
||
return _runbook_sequence(
|
||
**base_kwargs,
|
||
reason=str(worker_plan.get("summary") or worker_preview.get("message") or "最新发布包与 Worker 智能 Rollout 预检均已收口。").strip(),
|
||
tag_type="success",
|
||
target_node_codes=worker_node_codes,
|
||
step_titles=["确认最新发布包", "执行 Worker 灰度", "观察回执与巡检", "必要时推进 Control"],
|
||
primary_label="直接发 Worker 灰度",
|
||
primary_action_code="publish_latest_worker",
|
||
primary_action_payload=dict(shared_action_payload),
|
||
primary_focus_ref=worker_launchpad_focus_ref,
|
||
secondary_label="查看 Control 预案" if bool(control_plan.get("available", False)) else ("查看版本区" if latest_release_id > 0 else ""),
|
||
secondary_action_code="review_control_rollout" if bool(control_plan.get("available", False)) else ("focus_release_hub" if latest_release_id > 0 else ""),
|
||
secondary_action_payload=dict(shared_action_payload) if bool(control_plan.get("available", False)) else ({"release_id": latest_release_id} if latest_release_id > 0 else {}),
|
||
secondary_focus_ref=control_launchpad_focus_ref if bool(control_plan.get("available", False)) else release_detail_focus_ref,
|
||
)
|
||
|
||
if action_code == "review_smart_rollout_preview":
|
||
return _runbook_sequence(
|
||
**base_kwargs,
|
||
reason=str(worker_preview.get("message") or worker_plan.get("summary") or "Worker 智能 Rollout 仍需人工确认。").strip(),
|
||
tag_type="warning",
|
||
target_node_codes=worker_node_codes,
|
||
step_titles=["查看 Worker 智能预案", "确认门禁与审批", "必要时继续创建", "回看 Release Hub"],
|
||
primary_label="查看 Worker 预案",
|
||
primary_action_code="review_smart_rollout_preview",
|
||
primary_action_payload=dict(shared_action_payload),
|
||
primary_focus_ref=worker_launchpad_focus_ref,
|
||
secondary_label="查看版本区" if latest_release_id > 0 else "",
|
||
secondary_action_code="focus_release_hub" if latest_release_id > 0 else "",
|
||
secondary_action_payload={"release_id": latest_release_id} if latest_release_id > 0 else {},
|
||
secondary_focus_ref=release_detail_focus_ref,
|
||
)
|
||
|
||
if action_code == "review_control_rollout":
|
||
return _runbook_sequence(
|
||
**base_kwargs,
|
||
reason=str(control_preview.get("message") or control_plan.get("summary") or "当前只有 Control 侧满足预发条件。").strip(),
|
||
tag_type="warning",
|
||
target_node_codes=control_node_codes,
|
||
step_titles=["查看 Control 预案", "确认 Control 目标", "确认门禁与审批", "必要时推进发布"],
|
||
primary_label="查看 Control 预案",
|
||
primary_action_code="review_control_rollout",
|
||
primary_action_payload=dict(shared_action_payload),
|
||
primary_focus_ref=control_launchpad_focus_ref,
|
||
secondary_label="查看版本区" if latest_release_id > 0 else "",
|
||
secondary_action_code="focus_release_hub" if latest_release_id > 0 else "",
|
||
secondary_action_payload={"release_id": latest_release_id} if latest_release_id > 0 else {},
|
||
secondary_focus_ref=release_detail_focus_ref,
|
||
)
|
||
|
||
if action_code == "fix_rollout_blockers":
|
||
return _runbook_sequence(
|
||
**base_kwargs,
|
||
reason=str(
|
||
first_gap_row.get("agent_reason")
|
||
or first_gap_row.get("inspection_reason")
|
||
or worker_preview.get("message")
|
||
or control_preview.get("message")
|
||
or "Rollout 预检仍存在阻断条件。"
|
||
).strip(),
|
||
tag_type="danger",
|
||
target_node_codes=gap_node_codes or worker_node_codes or control_node_codes,
|
||
step_titles=["查看 Worker 智能预案", "定位阻断节点", "补接管或巡检缺口", "重新回看预案"],
|
||
primary_label="查看阻断预案",
|
||
primary_action_code="fix_rollout_blockers",
|
||
primary_action_payload=dict(shared_action_payload),
|
||
primary_focus_ref=rollout_gate_focus_ref,
|
||
secondary_label="补执行面接管",
|
||
secondary_action_code="fix_managed_nodes",
|
||
secondary_action_payload=dict(shared_action_payload),
|
||
secondary_focus_ref=rollout_gate_focus_ref,
|
||
)
|
||
|
||
if action_code == "fix_managed_nodes":
|
||
step_titles = ["同步集群节点", "纳管首台缺口", "补齐 Agent 接入", "回到 Launchpad 复检"]
|
||
if str(first_gap_row.get("recovery_action") or "").strip() == "bootstrap_run":
|
||
step_titles = ["定位首台缺口节点", "执行 onboarding.bootstrap", "等待 register / heartbeat", "回到 Launchpad 复检"]
|
||
elif str(first_gap_row.get("recovery_action") or "").strip() == "run_acceptance":
|
||
step_titles = ["定位待验收节点", "执行 onboarding.acceptance", "确认 health / worker / agent", "回到 Launchpad 复检"]
|
||
return _runbook_sequence(
|
||
**base_kwargs,
|
||
reason=str(
|
||
first_gap_row.get("agent_reason")
|
||
or first_gap_row.get("inspection_reason")
|
||
or "当前没有可用于智能 Rollout 的在线启用执行节点,需要先补齐托管或接入状态。"
|
||
).strip(),
|
||
tag_type="warning",
|
||
target_node_codes=gap_node_codes,
|
||
step_titles=step_titles,
|
||
primary_label="补执行面接管",
|
||
primary_action_code="fix_managed_nodes",
|
||
primary_action_payload=dict(shared_action_payload),
|
||
primary_focus_ref=rollout_gate_focus_ref,
|
||
secondary_label="查看版本区" if latest_release_id > 0 else "",
|
||
secondary_action_code="focus_release_hub" if latest_release_id > 0 else "",
|
||
secondary_action_payload={"release_id": latest_release_id} if latest_release_id > 0 else {},
|
||
secondary_focus_ref=release_detail_focus_ref,
|
||
)
|
||
|
||
if action_code in {"bootstrap_run", "run_acceptance"}:
|
||
return _runbook_sequence(
|
||
**base_kwargs,
|
||
reason=str(
|
||
first_gap_row.get("recovery_summary")
|
||
or first_gap_row.get("onboarding_summary")
|
||
or first_gap_row.get("agent_reason")
|
||
or "当前需要先收口节点接管,再继续发布。"
|
||
).strip(),
|
||
tag_type="warning" if action_code == "bootstrap_run" else "success",
|
||
target_node_codes=gap_node_codes or ([str(first_gap_row.get("node_code") or "").strip()] if str(first_gap_row.get("node_code") or "").strip() else []),
|
||
step_titles=(
|
||
["定位首台缺口节点", "执行 onboarding.bootstrap", "等待 register / heartbeat", "回到 Launchpad 复检"]
|
||
if action_code == "bootstrap_run"
|
||
else ["定位待验收节点", "执行 onboarding.acceptance", "确认 health / worker / agent", "回到 Launchpad 复检"]
|
||
),
|
||
primary_label="跑接入收口" if action_code == "bootstrap_run" else "跑接管验收",
|
||
primary_action_code=action_code,
|
||
primary_action_payload={
|
||
**dict(shared_action_payload),
|
||
"node_code": str(first_gap_row.get("node_code") or "").strip(),
|
||
},
|
||
primary_focus_ref=rollout_gate_focus_ref,
|
||
secondary_label="查看版本区" if latest_release_id > 0 else "",
|
||
secondary_action_code="focus_release_hub" if latest_release_id > 0 else "",
|
||
secondary_action_payload={"release_id": latest_release_id} if latest_release_id > 0 else {},
|
||
secondary_focus_ref=release_detail_focus_ref,
|
||
)
|
||
|
||
if action_code == "api-restart":
|
||
return _runbook_sequence(
|
||
**base_kwargs,
|
||
reason=str(
|
||
launchpad_status.get("summary")
|
||
or "运行中的控制面 API 还没有刷新到当前仓库能力,建议先执行 runtime-refresh-recover。"
|
||
).strip(),
|
||
tag_type="warning",
|
||
target_node_codes=[],
|
||
step_titles=["执行 runtime-refresh-recover", "确认 build-info / route surface", "重新拉取 Launchpad", "再决定 Release / Rollout"],
|
||
primary_label="查看刷新收口",
|
||
primary_action_code="api-restart",
|
||
primary_action_payload=dict(shared_action_payload),
|
||
primary_focus_ref=worker_launchpad_focus_ref or launchpad_focus_ref,
|
||
secondary_label="查看版本区" if latest_release_id > 0 else "",
|
||
secondary_action_code="focus_release_hub" if latest_release_id > 0 else "",
|
||
secondary_action_payload={"release_id": latest_release_id} if latest_release_id > 0 else {},
|
||
secondary_focus_ref=release_detail_focus_ref if latest_release_id > 0 else {},
|
||
)
|
||
|
||
if action_code == "release_package":
|
||
return _runbook_sequence(
|
||
**base_kwargs,
|
||
reason=str(latest_package.get("reason") or "当前没有可用的最新发布包,请先在控制面本机完成打包。").strip(),
|
||
tag_type="warning",
|
||
target_node_codes=[],
|
||
step_titles=["打开打包面板", "导入最新发布包", "确认 Release 记录", "重新回看 Launchpad"],
|
||
primary_label="打开打包面板",
|
||
primary_action_code="release_package",
|
||
primary_action_payload=dict(shared_action_payload),
|
||
primary_focus_ref=worker_launchpad_focus_ref or launchpad_focus_ref,
|
||
secondary_label="查看版本区" if latest_release_id > 0 else "",
|
||
secondary_action_code="focus_release_hub" if latest_release_id > 0 else "",
|
||
secondary_action_payload={"release_id": latest_release_id} if latest_release_id > 0 else {},
|
||
secondary_focus_ref=release_detail_focus_ref,
|
||
)
|
||
|
||
if action_code == "release_prepare":
|
||
return _runbook_sequence(
|
||
**base_kwargs,
|
||
reason=str(
|
||
latest_package.get("final_release_report_stale_reason")
|
||
or launchpad_status.get("summary")
|
||
or "当前最新发布包还没有完成最终签收,不能直接进入正式发布动作。"
|
||
).strip(),
|
||
tag_type="danger",
|
||
target_node_codes=[],
|
||
step_titles=["执行最终签收", "确认 verify / smoke / final report", "刷新最新包状态", "回到 Launchpad 复检"],
|
||
primary_label="执行最终签收",
|
||
primary_action_code="release_prepare",
|
||
primary_action_payload=dict(shared_action_payload),
|
||
primary_focus_ref=worker_launchpad_focus_ref or launchpad_focus_ref,
|
||
secondary_label="查看版本区" if latest_release_id > 0 else "",
|
||
secondary_action_code="focus_release_hub" if latest_release_id > 0 else "",
|
||
secondary_action_payload={"release_id": latest_release_id} if latest_release_id > 0 else {},
|
||
secondary_focus_ref=release_detail_focus_ref,
|
||
)
|
||
|
||
return {}
|
||
|
||
|
||
def _build_ops_runbook_control_sequences(
|
||
*,
|
||
managed_nodes_payload: dict | None = None,
|
||
release_launchpad: dict | None = None,
|
||
) -> list[dict]:
|
||
nodes = list((managed_nodes_payload or {}).get("nodes") or [])
|
||
preferred_release = _preferred_release_for_ops()
|
||
preferred_release_id = int(preferred_release.get("id") or 0)
|
||
preferred_release_label = (
|
||
str(preferred_release.get("release_version") or "").strip()
|
||
or str(preferred_release.get("channel") or "").strip()
|
||
or "默认 Release"
|
||
)
|
||
|
||
bootstrap_gap_nodes = [
|
||
item
|
||
for item in nodes
|
||
if bool(item.get("cluster_is_effective_worker", False))
|
||
and (
|
||
not bool(item.get("is_managed", False))
|
||
or not bool(item.get("is_enabled", False))
|
||
or not bool(item.get("is_agent_online", False))
|
||
)
|
||
]
|
||
bootstrap_gap_node_codes = _normalize_driver_node_codes(item.get("node_code") for item in bootstrap_gap_nodes)
|
||
|
||
participating_nodes = [
|
||
item
|
||
for item in nodes
|
||
if bool(item.get("cluster_is_effective_worker", False)) and bool(item.get("is_current_participant", False))
|
||
]
|
||
participating_node_codes = _normalize_driver_node_codes(item.get("node_code") for item in participating_nodes)
|
||
|
||
inspection_ready_nodes = [
|
||
item
|
||
for item in nodes
|
||
if _is_inspection_eligible_node(item)
|
||
]
|
||
inspection_ready_node_codes = _normalize_driver_node_codes(item.get("node_code") for item in inspection_ready_nodes)
|
||
worker_release_node_codes = _normalize_driver_node_codes(
|
||
item.get("node_code")
|
||
for item in inspection_ready_nodes
|
||
if str(item.get("role") or "").strip() == "worker"
|
||
)
|
||
control_release_node_codes = _normalize_driver_node_codes(
|
||
item.get("node_code")
|
||
for item in inspection_ready_nodes
|
||
if str(item.get("role") or "").strip() == "control"
|
||
)
|
||
|
||
scene_target_node_codes = participating_node_codes or inspection_ready_node_codes
|
||
|
||
onboarding_status = "warning" if bootstrap_gap_node_codes else "ready"
|
||
onboarding_status_label = "有待接管节点" if bootstrap_gap_node_codes else "接管链路就绪"
|
||
onboarding_tag_type = "warning" if bootstrap_gap_node_codes else "success"
|
||
onboarding_summary = (
|
||
f"当前有 {len(bootstrap_gap_node_codes)} 台有效执行节点还没进入 Agent 在线状态,建议先从海外控制面生成接入工单。"
|
||
if bootstrap_gap_node_codes
|
||
else "当前没有明显的接管缺口;后续新增大陆机器时,直接从这里生成接入工单。"
|
||
)
|
||
|
||
scene_status = "warning" if scene_target_node_codes else "info"
|
||
scene_status_label = "现场可观察" if scene_target_node_codes else "暂无现场目标"
|
||
scene_tag_type = "warning" if scene_target_node_codes else "info"
|
||
scene_summary = (
|
||
f"当前有 {len(scene_target_node_codes)} 台节点适合作为执行现场观察目标,优先从关键日志开始,再决定是否升级到全量取证。"
|
||
if scene_target_node_codes
|
||
else "当前没有明确的参与节点;后续一旦有节点开始领任务,这里就是海外控制面第一入口。"
|
||
)
|
||
|
||
inspection_status = "ready" if inspection_ready_node_codes else "attention"
|
||
inspection_status_label = "可直接巡检" if inspection_ready_node_codes else "待补接管"
|
||
inspection_tag_type = "success" if inspection_ready_node_codes else "warning"
|
||
inspection_summary = (
|
||
f"当前有 {len(inspection_ready_node_codes)} 台节点已经满足 已纳管 + 已启用 + Agent 在线,可直接走标准巡检。"
|
||
if inspection_ready_node_codes
|
||
else "当前还没有满足标准巡检条件的节点,建议先补齐接管和 Agent 在线状态。"
|
||
)
|
||
|
||
release_status = "ready" if preferred_release_id > 0 else "attention"
|
||
release_status_label = "可进入发布路径" if preferred_release_id > 0 else "待创建 Release"
|
||
release_tag_type = "success" if preferred_release_id > 0 else "warning"
|
||
release_summary = (
|
||
f"当前默认版本 {preferred_release_label} 已可用于点状灰度或 Rollout;先发 Worker,再逐步推进控制面。"
|
||
if preferred_release_id > 0
|
||
else "当前还没有可用 Release;建议先在海外控制面建立版本入口,再进入发布闭环。"
|
||
)
|
||
release_progression_sequence = _build_release_launchpad_runbook_sequence(
|
||
release_launchpad=dict(release_launchpad or {})
|
||
)
|
||
if not release_progression_sequence:
|
||
release_progression_sequence = _runbook_sequence(
|
||
key="release_progression",
|
||
title="版本发布与放量",
|
||
summary=release_summary,
|
||
reason="正式运维时,版本更新应该统一落回 Release / Rollout,而不是回退到各机器单独 pull / restart。",
|
||
status=release_status,
|
||
status_label=release_status_label,
|
||
tag_type=release_tag_type,
|
||
target_node_codes=worker_release_node_codes or control_release_node_codes,
|
||
target_scope_label="Release / Rollout",
|
||
step_titles=["点状灰度 Worker", "健康检查", "必要时推进 Rollout", "最后处理控制面"],
|
||
primary_label="直接创建 Worker Rollout",
|
||
primary_action_code="create_release_rollout_worker",
|
||
primary_action_payload={
|
||
"release_id": preferred_release_id,
|
||
"target_node_codes": worker_release_node_codes,
|
||
},
|
||
secondary_label="查看版本区",
|
||
secondary_action_code="focus_release_hub",
|
||
secondary_action_payload={
|
||
"release_id": preferred_release_id,
|
||
},
|
||
)
|
||
|
||
sequences = [
|
||
_runbook_sequence(
|
||
key="node_onboarding",
|
||
title="纳管新节点",
|
||
summary=onboarding_summary,
|
||
reason="所有新增大陆 controller / worker,第一步都应该先生成 Node Agent 接入工单,而不是手动 SSH 拼命令。",
|
||
status=onboarding_status,
|
||
status_label=onboarding_status_label,
|
||
tag_type=onboarding_tag_type,
|
||
target_node_codes=bootstrap_gap_node_codes,
|
||
target_scope_label="单节点 / 新节点",
|
||
step_titles=["生成接入工单", "复制 Env/脚本", "节点落地执行", "回到控制面看心跳"],
|
||
primary_label="跑接入收口",
|
||
primary_action_code="bootstrap_run",
|
||
primary_action_payload={
|
||
"node_code": bootstrap_gap_node_codes[0] if bootstrap_gap_node_codes else "",
|
||
},
|
||
secondary_label="跑接管验收",
|
||
secondary_action_code="run_acceptance",
|
||
secondary_action_payload={
|
||
"node_code": inspection_ready_node_codes[0] if inspection_ready_node_codes else "",
|
||
},
|
||
),
|
||
_runbook_sequence(
|
||
key="scene_observe",
|
||
title="执行现场观察",
|
||
summary=scene_summary,
|
||
reason="海外控制面遇到“谁在跑、为什么卡住、日志有没有回来”这类问题时,应优先走统一现场观察路径。",
|
||
status=scene_status,
|
||
status_label=scene_status_label,
|
||
tag_type=scene_tag_type,
|
||
target_node_codes=scene_target_node_codes,
|
||
target_scope_label="参与节点 / 现场节点",
|
||
step_titles=["关键日志", "判断是否升级全量", "补诊断包", "回看联调快照"],
|
||
primary_label="执行关键现场日志",
|
||
primary_action_code="run_scene_logs_key",
|
||
primary_action_payload={
|
||
"execution_mode": "remote-agent",
|
||
"auto_approve": True,
|
||
},
|
||
secondary_label="执行全量现场取证",
|
||
secondary_action_code="run_scene_logs_full",
|
||
secondary_action_payload={
|
||
"execution_mode": "remote-agent",
|
||
"auto_approve": True,
|
||
},
|
||
),
|
||
_runbook_sequence(
|
||
key="standard_inspection",
|
||
title="标准巡检",
|
||
summary=inspection_summary,
|
||
reason="现场收口、节点验收和发布前确认,尽量都走同一条标准巡检链,而不是手工拼快照、日志和诊断包。",
|
||
status=inspection_status,
|
||
status_label=inspection_status_label,
|
||
tag_type=inspection_tag_type,
|
||
target_node_codes=inspection_ready_node_codes,
|
||
target_scope_label="已接管执行节点",
|
||
step_titles=["健康快照", "Worker 日志", "诊断包", "统一回看巡检结果"],
|
||
primary_label="直接执行标准巡检",
|
||
primary_action_code="run_standard_inspection",
|
||
primary_action_payload={},
|
||
secondary_label="补充诊断包",
|
||
secondary_action_code="open_diagnostics",
|
||
secondary_action_payload={
|
||
"execution_mode": "remote-agent",
|
||
"auto_approve": True,
|
||
},
|
||
),
|
||
release_progression_sequence,
|
||
]
|
||
return sequences
|
||
|
||
|
||
def _activity_time(*candidates: object) -> str:
|
||
for candidate in candidates:
|
||
value = str(candidate or "").strip()
|
||
if value:
|
||
return value
|
||
return ""
|
||
|
||
|
||
def _truncate_activity_text(value: object, *, limit: int = 240) -> str:
|
||
text = str(value or "").strip()
|
||
if len(text) <= limit:
|
||
return text
|
||
return f"{text[:limit]}..."
|
||
|
||
|
||
def _activity_status_label(status: object, *, kind: str = "") -> str:
|
||
normalized_status = str(status or "").strip()
|
||
mapping = {
|
||
"empty": "无活动",
|
||
"planned": "待推进",
|
||
"ready": "就绪",
|
||
"healthy": "健康",
|
||
"running": "执行中",
|
||
"attention": "待处理",
|
||
"queued": "排队中",
|
||
"dispatching": "派发中",
|
||
"awaiting_approval": "待审批",
|
||
"success": "成功",
|
||
"failed": "失败",
|
||
"blocked": "阻断",
|
||
"cancelled": "已取消",
|
||
"halted": "已暂停",
|
||
"completed_with_issues": "带问题完成",
|
||
"partially_succeeded": "部分成功",
|
||
}
|
||
if normalized_status in mapping:
|
||
return mapping[normalized_status]
|
||
if normalized_status:
|
||
return normalized_status
|
||
if str(kind or "").strip() in {"execution_scene", "log_sync"}:
|
||
return "待观察"
|
||
return "未知"
|
||
|
||
|
||
def _activity_focus_ref(item: dict) -> dict:
|
||
normalized_item = dict(item or {})
|
||
kind = str(normalized_item.get("kind") or "").strip()
|
||
ui_intent = dict(normalized_item.get("ui_intent") or {})
|
||
if kind == "playbook_run":
|
||
return {
|
||
"kind": "playbook_run",
|
||
"run_code": str(normalized_item.get("run_code") or "").strip(),
|
||
"focus_step_key": str((ui_intent.get("focus_step_key") or normalized_item.get("focus_step_key") or "")).strip(),
|
||
"focus_step_title": str((ui_intent.get("focus_step_title") or normalized_item.get("focus_step_title") or "")).strip(),
|
||
}
|
||
if kind == "ops_job":
|
||
return {
|
||
"kind": "ops_job",
|
||
"job_id": int(normalized_item.get("job_id") or 0),
|
||
"job_code": str(normalized_item.get("job_code") or "").strip(),
|
||
"action": str((ui_intent.get("action") or normalized_item.get("title") or "")).strip(),
|
||
"target_node_code": str(
|
||
(
|
||
ui_intent.get("target_node_code")
|
||
or next(iter(normalized_item.get("target_node_codes") or []), "")
|
||
or ""
|
||
)
|
||
).strip(),
|
||
}
|
||
if kind == "rollout":
|
||
return {
|
||
"kind": "rollout",
|
||
"rollout_id": int(normalized_item.get("rollout_id") or 0),
|
||
"rollout_code": str(normalized_item.get("title") or "").strip(),
|
||
"release_id": int((ui_intent.get("release_id") or 0)),
|
||
}
|
||
if kind == "runbook_sequence":
|
||
return {
|
||
"kind": "runbook_sequence",
|
||
"sequence_key": str(normalized_item.get("sequence_key") or "").strip(),
|
||
"driver_action_code": str(ui_intent.get("driver_action_code") or "").strip(),
|
||
}
|
||
if kind in {"execution_scene", "log_sync"}:
|
||
return {
|
||
"kind": "execution_scene",
|
||
"scene_key": str(normalized_item.get("activity_key") or kind).strip(),
|
||
}
|
||
return {
|
||
"kind": kind or "activity",
|
||
"activity_key": str(normalized_item.get("activity_key") or "").strip(),
|
||
}
|
||
|
||
|
||
def _finalize_activity_item(item: dict) -> dict:
|
||
normalized_item = dict(item or {})
|
||
kind = str(normalized_item.get("kind") or "").strip()
|
||
ui_intent = dict(normalized_item.get("ui_intent") or {})
|
||
summary_text = str(normalized_item.get("summary") or "").strip()
|
||
status_label = str(normalized_item.get("status_label") or "").strip() or _activity_status_label(
|
||
normalized_item.get("status"),
|
||
kind=kind,
|
||
)
|
||
target_node_codes = _normalize_driver_node_codes(normalized_item.get("target_node_codes") or [])
|
||
finalized = {
|
||
**normalized_item,
|
||
"summary": summary_text,
|
||
"summary_text": summary_text,
|
||
"status_label": status_label,
|
||
"ui_intent": ui_intent,
|
||
"ui_intent_kind": str(ui_intent.get("kind") or "").strip(),
|
||
"target_node_codes": target_node_codes,
|
||
}
|
||
existing_focus_ref = dict(normalized_item.get("focus_ref") or {})
|
||
existing_source_focus_ref = dict(normalized_item.get("source_focus_ref") or {})
|
||
finalized["focus_ref"] = existing_focus_ref if existing_focus_ref else _activity_focus_ref(finalized)
|
||
finalized["source_focus_ref"] = existing_source_focus_ref
|
||
contract_keys = _normalize_ops_contract_keys(
|
||
list(normalized_item.get("contract_keys") or [])
|
||
+ _suggest_ops_contract_keys_for_activity_item(finalized)
|
||
)
|
||
finalized["contract_keys"] = contract_keys
|
||
finalized["contract_navigation"] = _build_ops_contract_navigation(
|
||
contract_keys,
|
||
primary_contract_key=_primary_ops_contract_key_for_activity_item(finalized, contract_keys),
|
||
registry=get_ops_contract_registry(),
|
||
)
|
||
return finalized
|
||
|
||
|
||
def _activity_stream_summary_state(items: list[dict]) -> tuple[str, str, str]:
|
||
normalized_items = [dict(item or {}) for item in list(items or [])]
|
||
if not normalized_items:
|
||
return "empty", "无活动", "当前没有匹配的活动。"
|
||
|
||
hard_attention_statuses = {
|
||
"attention",
|
||
"failed",
|
||
"blocked",
|
||
"cancelled",
|
||
"halted",
|
||
"completed_with_issues",
|
||
"partially_succeeded",
|
||
"waiting_sample",
|
||
"partial_coverage",
|
||
}
|
||
soft_attention_statuses = {"awaiting_approval"}
|
||
running_statuses = {"running", "dispatching"}
|
||
planned_statuses = {"queued", "planned", "ready"}
|
||
|
||
item_statuses = {str(item.get("status") or "").strip() for item in normalized_items if str(item.get("status") or "").strip()}
|
||
|
||
if item_statuses & hard_attention_statuses:
|
||
return "attention", "待处理", "最近活动里存在需优先处理的现场、任务或发布推进。"
|
||
if item_statuses & soft_attention_statuses:
|
||
return "attention", "待审批", "最近活动里存在待审批或待人工确认的动作。"
|
||
if item_statuses & running_statuses:
|
||
return "running", "执行中", "最近活动里存在正在推进的现场、任务、编排或回传。"
|
||
if item_statuses & planned_statuses:
|
||
return "planned", "待推进", "最近活动里存在已排队或即将继续推进的动作。"
|
||
return "ready", "平稳", "最近活动整体平稳,可按需抽样查看。"
|
||
|
||
|
||
def _is_playbook_child_job(job: dict) -> bool:
|
||
return bool(str(((job.get("metadata") or {}).get("playbook") or {}).get("run_code") or "").strip())
|
||
|
||
|
||
def _build_playbook_run_activity(run: dict) -> dict:
|
||
normalized_run = dict(run or {})
|
||
run_code = str(normalized_run.get("run_code") or "").strip()
|
||
title = (
|
||
str(normalized_run.get("playbook_title") or "").strip()
|
||
or str(normalized_run.get("playbook_key") or "").strip()
|
||
or "标准编排"
|
||
)
|
||
steps = list(normalized_run.get("steps") or [])
|
||
step_count = int(normalized_run.get("step_count") or len(steps) or 0)
|
||
completed_steps = sum(1 for step in steps if str(step.get("status") or "").strip() in {"success", "attention"})
|
||
target_node_codes = _normalize_driver_node_codes(normalized_run.get("target_node_codes") or [])
|
||
focus_summary = str(normalized_run.get("focus_summary") or "").strip()
|
||
if not focus_summary:
|
||
focus_summary = f"当前状态 {str(normalized_run.get('status') or '').strip() or 'queued'},建议进入编排详情继续观察。"
|
||
meta_parts = [run_code]
|
||
group_title = str(normalized_run.get("group_title") or normalized_run.get("group_key") or "").strip()
|
||
if group_title:
|
||
meta_parts.append(group_title)
|
||
execution_mode_label_text = (
|
||
str(normalized_run.get("execution_mode_label") or "").strip()
|
||
or execution_mode_label(str(normalized_run.get("execution_mode") or "").strip())
|
||
)
|
||
if execution_mode_label_text:
|
||
meta_parts.append(f"执行 {execution_mode_label_text}")
|
||
if step_count > 0:
|
||
meta_parts.append(f"步骤 {completed_steps}/{step_count}")
|
||
jobs_total = int(normalized_run.get("jobs_total") or 0)
|
||
if jobs_total > 0:
|
||
meta_parts.append(f"任务 {jobs_total}")
|
||
if target_node_codes:
|
||
meta_parts.append(f"节点 {len(target_node_codes)}")
|
||
return {
|
||
"kind": "playbook_run",
|
||
"activity_key": f"playbook-run:{run_code or 'latest'}",
|
||
"title": title,
|
||
"subtitle": run_code,
|
||
"summary": _truncate_activity_text(focus_summary),
|
||
"meta_text": " / ".join(part for part in meta_parts if part),
|
||
"status": str(normalized_run.get("status") or "").strip(),
|
||
"execution_mode": str(normalized_run.get("execution_mode") or "").strip(),
|
||
"execution_mode_label": execution_mode_label_text,
|
||
"occurred_at": _activity_time(normalized_run.get("updated_at"), normalized_run.get("created_at")),
|
||
"run_code": run_code,
|
||
"focus_step_key": str(normalized_run.get("focus_step_key") or "").strip(),
|
||
"focus_step_title": str(normalized_run.get("focus_step_title") or "").strip(),
|
||
"target_node_codes": target_node_codes,
|
||
"ui_intent": _build_driver_ui_intent(
|
||
"playbook_run_detail",
|
||
run_code=run_code,
|
||
focus_step_key=str(normalized_run.get("focus_step_key") or "").strip(),
|
||
focus_step_title=str(normalized_run.get("focus_step_title") or "").strip(),
|
||
),
|
||
}
|
||
|
||
|
||
def _build_bootstrap_job_activity(job: dict, managed_node: dict | None = None) -> dict:
|
||
normalized_job = dict(job or {})
|
||
normalized_node = dict(managed_node or {})
|
||
job_id = int(normalized_job.get("id") or 0)
|
||
job_code = str(normalized_job.get("job_code") or "").strip()
|
||
status = str(normalized_job.get("status") or "").strip()
|
||
target_node_code = str(normalized_job.get("target_node_code") or "").strip()
|
||
error_message = str(normalized_job.get("error_message") or "").strip()
|
||
result = dict(normalized_job.get("result") or {})
|
||
bootstrap_plan = dict(result.get("bootstrap_plan") or {})
|
||
script_name = str(bootstrap_plan.get("bootstrap_script_name") or "").strip()
|
||
agent_state = str(normalized_node.get("agent_state") or "").strip()
|
||
agent_state_label = str(normalized_node.get("agent_state_label") or "").strip()
|
||
|
||
if error_message and status in {"failed", "blocked", "cancelled"}:
|
||
summary = error_message
|
||
elif status == "awaiting_approval":
|
||
summary = "接入工单等待审批,审批通过后才会生成 bootstrap env、脚本和一键落地命令。"
|
||
elif status in {"queued", "dispatching", "running"}:
|
||
summary = "控制面正在生成节点接入工单,完成后可直接复制 bootstrap env、脚本和一键落地命令。"
|
||
elif status == "success":
|
||
if agent_state in {"online", "online_busy"}:
|
||
summary = "接入工单已生成,目标节点已完成 Agent 接入,可继续执行接管后验收。"
|
||
elif agent_state == "pending_bootstrap":
|
||
summary = "接入工单已生成,节点尚未回连 Agent,可直接复制脚本或一键落地命令。"
|
||
elif agent_state == "runtime_only":
|
||
summary = "接入工单已生成,但当前只有 runtime 心跳,Node Agent 还未真正接管。"
|
||
elif agent_state == "stale":
|
||
summary = "接入工单已生成,但 Agent 心跳已过期,建议先看任务详情和 Agent 日志。"
|
||
else:
|
||
summary = "接入工单已生成,可直接复制 bootstrap env、脚本和一键落地命令。"
|
||
else:
|
||
summary = "节点接入工单最近有新的回执。"
|
||
|
||
execution_mode_label_text = (
|
||
str(normalized_job.get("execution_mode_label") or "").strip()
|
||
or execution_mode_label(str(normalized_job.get("execution_mode") or "control-plane").strip() or "control-plane")
|
||
)
|
||
meta_parts = [
|
||
part
|
||
for part in [
|
||
job_code,
|
||
f"节点 {target_node_code}" if target_node_code else "",
|
||
f"方式 {execution_mode_label_text}" if execution_mode_label_text else "",
|
||
script_name,
|
||
]
|
||
if part
|
||
]
|
||
if agent_state_label:
|
||
meta_parts.append(f"状态 {agent_state_label}")
|
||
|
||
return {
|
||
"kind": "ops_job",
|
||
"activity_key": f"ops-job:{job_id or job_code or 'node.bootstrap'}",
|
||
"action": "node.bootstrap",
|
||
"title": "节点接入工单",
|
||
"subtitle": job_code,
|
||
"summary": _truncate_activity_text(summary),
|
||
"meta_text": " / ".join(meta_parts),
|
||
"status": status,
|
||
"execution_mode": str(normalized_job.get("execution_mode") or "control-plane").strip() or "control-plane",
|
||
"execution_mode_label": execution_mode_label_text,
|
||
"occurred_at": _activity_time(
|
||
normalized_job.get("updated_at"),
|
||
normalized_job.get("finished_at"),
|
||
normalized_job.get("started_at"),
|
||
normalized_job.get("created_at"),
|
||
),
|
||
"job_id": job_id,
|
||
"job_code": job_code,
|
||
"target_node_codes": _normalize_driver_node_codes([target_node_code]),
|
||
"ui_intent": _build_driver_ui_intent(
|
||
"job_detail",
|
||
job_id=job_id,
|
||
job_code=job_code,
|
||
action="node.bootstrap",
|
||
target_node_code=target_node_code,
|
||
),
|
||
}
|
||
|
||
|
||
def _build_ops_job_activity(job: dict, *, managed_node: dict | None = None) -> dict:
|
||
normalized_job = dict(job or {})
|
||
job_id = int(normalized_job.get("id") or 0)
|
||
job_code = str(normalized_job.get("job_code") or "").strip()
|
||
action = str(normalized_job.get("action") or "").strip() or "ops.job"
|
||
if action == "node.bootstrap":
|
||
return _build_bootstrap_job_activity(normalized_job, managed_node)
|
||
target_node_code = str(normalized_job.get("target_node_code") or "").strip()
|
||
target_type = str(normalized_job.get("target_type") or "").strip()
|
||
execution_mode = str(normalized_job.get("execution_mode") or "").strip()
|
||
execution_mode_label_text = (
|
||
str(normalized_job.get("execution_mode_label") or "").strip()
|
||
or execution_mode_label(execution_mode)
|
||
)
|
||
requested_by = str(normalized_job.get("requested_by") or "").strip()
|
||
status = str(normalized_job.get("status") or "").strip()
|
||
error_message = str(normalized_job.get("error_message") or "").strip()
|
||
result = dict(normalized_job.get("result") or {})
|
||
start_delivery_state = str(
|
||
result.get("start_delivery_state")
|
||
or normalized_job.get("start_delivery_state")
|
||
or ""
|
||
).strip()
|
||
start_delivery_error = str(
|
||
result.get("start_delivery_error")
|
||
or normalized_job.get("start_delivery_error")
|
||
or ""
|
||
).strip()
|
||
preferred_summary_text = str(
|
||
normalized_job.get("result_summary_text")
|
||
or result.get("summary_text")
|
||
or result.get("summary")
|
||
or normalized_job.get("summary_text")
|
||
or normalized_job.get("summary")
|
||
or ""
|
||
).strip()
|
||
summary_parts: list[str] = []
|
||
if not preferred_summary_text:
|
||
if target_node_code:
|
||
summary_parts.append(f"目标节点 {target_node_code}")
|
||
elif target_type:
|
||
summary_parts.append(f"目标类型 {target_type}")
|
||
if requested_by:
|
||
summary_parts.append(f"发起 {requested_by}")
|
||
if execution_mode_label_text:
|
||
summary_parts.append(f"方式 {execution_mode_label_text}")
|
||
summary_candidates: list[str] = []
|
||
if preferred_summary_text:
|
||
summary_candidates.append(preferred_summary_text)
|
||
if error_message and status in {"failed", "blocked", "cancelled"}:
|
||
summary_candidates.append(error_message)
|
||
if start_delivery_state == "failed_local":
|
||
summary_candidates.append("节点已接单执行,但开始回执未成功送达控制面。")
|
||
if start_delivery_error:
|
||
summary_candidates.append(f"开始回执失败:{start_delivery_error}")
|
||
meta_parts = [job_code]
|
||
risk_level = str(normalized_job.get("risk_level") or "").strip()
|
||
if risk_level:
|
||
meta_parts.append(f"风险 {risk_level}")
|
||
if execution_mode_label_text:
|
||
meta_parts.append(f"执行 {execution_mode_label_text}")
|
||
effective_status = status
|
||
if start_delivery_state == "failed_local" and status not in {"failed", "blocked", "cancelled"}:
|
||
effective_status = "attention"
|
||
if status:
|
||
meta_parts.append(f"任务 {_activity_status_label(status)}")
|
||
meta_parts.append("开始回执异常")
|
||
merged_focus_ref = _merge_focus_ref(normalized_job.get("focus_ref"), **_normalize_focus_ref(result.get("focus_ref")))
|
||
source_focus_ref = _normalize_focus_ref(result.get("source_focus_ref"))
|
||
summary_text = " / ".join(
|
||
dict.fromkeys(part for part in [*summary_candidates, *summary_parts] if str(part or "").strip())
|
||
) or "标准运维任务最近有新回执。"
|
||
activity = {
|
||
"kind": "ops_job",
|
||
"activity_key": f"ops-job:{job_id or job_code or action}",
|
||
"action": action,
|
||
"title": action,
|
||
"subtitle": job_code,
|
||
"summary": _truncate_activity_text(summary_text),
|
||
"meta_text": " / ".join(part for part in meta_parts if part),
|
||
"status": effective_status,
|
||
"job_status": status,
|
||
"execution_mode": execution_mode,
|
||
"execution_mode_label": execution_mode_label_text,
|
||
"occurred_at": _activity_time(
|
||
normalized_job.get("updated_at"),
|
||
normalized_job.get("finished_at"),
|
||
normalized_job.get("started_at"),
|
||
normalized_job.get("created_at"),
|
||
),
|
||
"job_id": job_id,
|
||
"job_code": job_code,
|
||
"target_node_codes": _normalize_driver_node_codes([target_node_code]),
|
||
"focus_ref": merged_focus_ref,
|
||
"source_focus_ref": source_focus_ref,
|
||
"ui_intent": _build_driver_ui_intent(
|
||
"job_events",
|
||
job_id=job_id,
|
||
job_code=job_code,
|
||
action=action,
|
||
target_node_code=target_node_code,
|
||
),
|
||
}
|
||
if start_delivery_state:
|
||
activity["start_delivery_state"] = start_delivery_state
|
||
if start_delivery_error:
|
||
activity["start_delivery_error"] = start_delivery_error
|
||
if start_delivery_state == "failed_local":
|
||
activity["problem_code"] = "ops_job_start_delivery_failed_local"
|
||
return activity
|
||
|
||
|
||
def _build_rollout_activity(rollout: dict) -> dict:
|
||
normalized_rollout = dict(rollout or {})
|
||
rollout_id = int(normalized_rollout.get("id") or 0)
|
||
rollout_code = str(normalized_rollout.get("rollout_code") or "").strip()
|
||
rollout_policy = dict(normalized_rollout.get("policy") or {})
|
||
rollout_execution_mode = str(rollout_policy.get("execution_mode") or "remote-agent").strip() or "remote-agent"
|
||
rollout_execution_mode_label = (
|
||
str(normalized_rollout.get("execution_mode_label") or "").strip()
|
||
or execution_mode_label(rollout_execution_mode)
|
||
)
|
||
rollout_target_node_codes = _normalize_driver_node_codes(
|
||
[str(item.get("node_code") or "").strip() for item in list(normalized_rollout.get("target_nodes") or [])]
|
||
)
|
||
target_nodes_total = len(rollout_target_node_codes)
|
||
jobs_total = int(normalized_rollout.get("jobs_total") or 0)
|
||
jobs_created = int(normalized_rollout.get("jobs_created") or 0)
|
||
batch_cursor = int(normalized_rollout.get("batch_cursor") or 0)
|
||
batches_total = int(normalized_rollout.get("batches_total") or 0)
|
||
summary_parts: list[str] = []
|
||
if target_nodes_total > 0:
|
||
summary_parts.append(f"目标节点 {target_nodes_total}")
|
||
if jobs_total > 0:
|
||
summary_parts.append(f"任务 {jobs_created}/{jobs_total}")
|
||
if batches_total > 0:
|
||
summary_parts.append(f"批次 {batch_cursor}/{batches_total}")
|
||
result_summary = dict(normalized_rollout.get("result_summary") or {})
|
||
summary_text = str(normalized_rollout.get("summary_text") or normalized_rollout.get("summary") or "").strip()
|
||
if int(result_summary.get("failed", 0) or 0) > 0:
|
||
summary_parts.append(f"失败 {int(result_summary.get('failed', 0) or 0)}")
|
||
elif int(result_summary.get("success", 0) or 0) > 0:
|
||
summary_parts.append(f"成功 {int(result_summary.get('success', 0) or 0)}")
|
||
meta_parts = [f"创建人 {str(normalized_rollout.get('created_by') or '').strip() or '-'}"]
|
||
if rollout_execution_mode_label:
|
||
meta_parts.append(f"执行 {rollout_execution_mode_label}")
|
||
return {
|
||
"kind": "rollout",
|
||
"activity_key": f"rollout:{rollout_id or rollout_code}",
|
||
"title": rollout_code or "Release Rollout",
|
||
"subtitle": f"Release #{int(normalized_rollout.get('release_id') or 0)}" if int(normalized_rollout.get("release_id") or 0) > 0 else "",
|
||
"summary": _truncate_activity_text(summary_text or " / ".join(part for part in summary_parts if part) or "该 rollout 最近有新的推进或回执。"),
|
||
"meta_text": " / ".join(part for part in meta_parts if part),
|
||
"status": str(normalized_rollout.get("status") or "").strip(),
|
||
"status_label": str(normalized_rollout.get("status_label") or "").strip(),
|
||
"execution_mode": rollout_execution_mode,
|
||
"execution_mode_label": rollout_execution_mode_label,
|
||
"occurred_at": _activity_time(normalized_rollout.get("updated_at"), normalized_rollout.get("created_at")),
|
||
"rollout_id": rollout_id,
|
||
"target_node_codes": rollout_target_node_codes,
|
||
"ui_intent": _build_driver_ui_intent(
|
||
"rollout_jobs",
|
||
rollout_id=rollout_id,
|
||
rollout_code=rollout_code,
|
||
release_id=int(normalized_rollout.get("release_id") or 0),
|
||
),
|
||
"focus_ref": dict(normalized_rollout.get("focus_ref") or {}),
|
||
}
|
||
|
||
|
||
def _runbook_sequence_activity_status(sequence: dict) -> str:
|
||
normalized_sequence = dict(sequence or {})
|
||
primary_resolution = dict(normalized_sequence.get("primary_resolution") or {})
|
||
if primary_resolution and primary_resolution.get("ok") is False:
|
||
return "attention"
|
||
normalized_status = str(normalized_sequence.get("status") or "").strip()
|
||
if normalized_status in {"warning", "attention", "blocking"}:
|
||
return "attention"
|
||
if normalized_status in {"ready", "success"}:
|
||
return "ready"
|
||
if normalized_status in {"info", "idle"}:
|
||
return "planned"
|
||
return normalized_status or "planned"
|
||
|
||
|
||
def _build_runbook_sequence_activity(sequence: dict) -> dict:
|
||
normalized_sequence = dict(sequence or {})
|
||
sequence_key = str(normalized_sequence.get("key") or "").strip()
|
||
if not sequence_key:
|
||
return {}
|
||
|
||
title = str(normalized_sequence.get("title") or "").strip() or "标准作业路径"
|
||
status_label = str(normalized_sequence.get("status_label") or normalized_sequence.get("status") or "").strip()
|
||
reason = str(normalized_sequence.get("reason") or "").strip()
|
||
target_scope_label = str(normalized_sequence.get("target_scope_label") or "").strip()
|
||
target_node_codes = _normalize_driver_node_codes(normalized_sequence.get("target_node_codes") or [])
|
||
primary_resolution = dict(normalized_sequence.get("primary_resolution") or {})
|
||
resolved_action_label = (
|
||
str(primary_resolution.get("driver_action_label") or "").strip()
|
||
or str(primary_resolution.get("driver_action_code") or "").strip()
|
||
)
|
||
resolved_action_code = str(primary_resolution.get("driver_action_code") or "").strip()
|
||
resolved_node_codes = _normalize_driver_node_codes(primary_resolution.get("driver_node_codes") or [])
|
||
resolution_message = str(primary_resolution.get("message") or "").strip()
|
||
occurred_at = _activity_time(primary_resolution.get("resolved_at"))
|
||
|
||
summary_parts: list[str] = []
|
||
if primary_resolution and primary_resolution.get("ok") is False:
|
||
summary_parts.append(f"当前解析失败:{resolution_message or '标准作业路径解析失败'}")
|
||
elif resolved_action_label or resolved_action_code:
|
||
summary_parts.append(
|
||
f"当前建议动作:{resolved_action_label or resolved_action_code}"
|
||
+ (
|
||
f" ({resolved_action_code})"
|
||
if resolved_action_code and resolved_action_label and resolved_action_code != resolved_action_label
|
||
else ""
|
||
)
|
||
)
|
||
if resolved_node_codes:
|
||
summary_parts.append(f"目标节点:{_format_node_code_list(resolved_node_codes)}")
|
||
elif target_node_codes:
|
||
summary_parts.append(f"目标节点:{_format_node_code_list(target_node_codes)}")
|
||
if resolution_message:
|
||
summary_parts.append(resolution_message)
|
||
else:
|
||
summary_parts.append(str(normalized_sequence.get("summary") or "").strip() or "当前标准作业路径已就绪。")
|
||
|
||
meta_parts = [status_label, sequence_key]
|
||
if target_scope_label:
|
||
meta_parts.append(target_scope_label)
|
||
if reason:
|
||
meta_parts.append(reason)
|
||
|
||
return {
|
||
"kind": "runbook_sequence",
|
||
"activity_key": f"runbook-sequence:{sequence_key}",
|
||
"title": title,
|
||
"subtitle": sequence_key,
|
||
"summary": _truncate_activity_text(" / ".join(part for part in summary_parts if part)),
|
||
"meta_text": _truncate_activity_text(" / ".join(part for part in meta_parts if part), limit=280),
|
||
"status": _runbook_sequence_activity_status(normalized_sequence),
|
||
"execution_mode": "",
|
||
"execution_mode_label": "",
|
||
"occurred_at": occurred_at,
|
||
"sequence_key": sequence_key,
|
||
"target_node_codes": target_node_codes,
|
||
"focus_ref": _normalize_focus_ref(normalized_sequence.get("focus_ref")),
|
||
"ui_intent": _build_driver_ui_intent(
|
||
"focus_runbook_sequence",
|
||
sequence_key=sequence_key,
|
||
driver_action_code=resolved_action_code or str(normalized_sequence.get("primary_action_code") or "").strip(),
|
||
),
|
||
}
|
||
|
||
|
||
def _matches_activity_query(item: dict, query: str) -> bool:
|
||
normalized_query = str(query or "").strip().lower()
|
||
if not normalized_query:
|
||
return True
|
||
haystack = " ".join(
|
||
[
|
||
str(item.get("activity_key") or ""),
|
||
str(item.get("kind") or ""),
|
||
str(item.get("title") or ""),
|
||
str(item.get("subtitle") or ""),
|
||
str(item.get("summary") or ""),
|
||
str(item.get("meta_text") or ""),
|
||
str(item.get("status") or ""),
|
||
str(item.get("execution_mode") or ""),
|
||
str(item.get("execution_mode_label") or ""),
|
||
str(item.get("run_code") or ""),
|
||
str(item.get("job_code") or ""),
|
||
str(item.get("rollout_id") or ""),
|
||
str(item.get("sequence_key") or ""),
|
||
]
|
||
).lower()
|
||
return normalized_query in haystack
|
||
|
||
|
||
def _driver_activity_status_label(status: str) -> str:
|
||
normalized_status = str(status or "").strip()
|
||
mapping = {
|
||
"failed": "失败",
|
||
"blocked": "阻断",
|
||
"cancelled": "已取消",
|
||
"halted": "已暂停",
|
||
"completed_with_issues": "带问题完成",
|
||
"partially_succeeded": "部分成功",
|
||
"awaiting_approval": "待审批",
|
||
}
|
||
return mapping.get(normalized_status, normalized_status or "异常")
|
||
|
||
|
||
def _pick_driver_activity_focus(items: list[dict]) -> tuple[str, dict]:
|
||
hard_problem_statuses = {"failed", "blocked", "cancelled", "halted", "completed_with_issues", "partially_succeeded"}
|
||
soft_attention_statuses = {"awaiting_approval"}
|
||
candidate_items = [
|
||
dict(item or {})
|
||
for item in list(items or [])
|
||
if str(item.get("kind") or "").strip() in {"ops_job", "rollout"}
|
||
]
|
||
|
||
for item in candidate_items:
|
||
if str(item.get("status") or "").strip() in hard_problem_statuses:
|
||
return "hard", item
|
||
for item in candidate_items:
|
||
if str(item.get("status") or "").strip() in soft_attention_statuses:
|
||
return "soft", item
|
||
return "", {}
|
||
|
||
|
||
def _build_driver_activity_card(level: str, activity_item: dict) -> dict:
|
||
normalized_item = dict(activity_item or {})
|
||
item_kind = str(normalized_item.get("kind") or "").strip()
|
||
item_status = str(normalized_item.get("status") or "").strip()
|
||
ui_intent_kind = str(((normalized_item.get("ui_intent") or {}).get("kind") or "")).strip()
|
||
source_focus_ref = _normalize_focus_ref(normalized_item.get("source_focus_ref"))
|
||
title = str(normalized_item.get("title") or "").strip() or ("Release Rollout" if item_kind == "rollout" else "标准运维任务")
|
||
subtitle = str(normalized_item.get("subtitle") or "").strip()
|
||
summary = str(normalized_item.get("summary") or "").strip()
|
||
meta_text = str(normalized_item.get("meta_text") or "").strip()
|
||
status_label = _driver_activity_status_label(item_status)
|
||
noun_label = "异常 Rollout" if item_kind == "rollout" else "异常任务"
|
||
is_hard_problem = level == "hard"
|
||
|
||
if item_kind == "rollout":
|
||
title_text = "先处理异常 Rollout" if is_hard_problem else "先处理待审批 Rollout"
|
||
primary_label = "查看 Rollout 作业"
|
||
else:
|
||
title_text = "先处理异常任务" if is_hard_problem else "先处理待审批任务"
|
||
primary_label = "查看任务详情" if ui_intent_kind == "job_detail" else "查看任务事件"
|
||
|
||
reason = (
|
||
summary
|
||
or (
|
||
"该活动已经进入失败、阻断或带问题完成状态,继续堆动作只会放大噪音。"
|
||
if is_hard_problem
|
||
else "该活动正在等待人工审批或决策,先落到对应详情最容易判断下一步。"
|
||
)
|
||
)
|
||
detail_summary = f"{title}{' / ' + subtitle if subtitle else ''} 当前状态为 {status_label},建议先进入对应详情。"
|
||
primary_action_code = "focus_activity_item"
|
||
action_payload = {
|
||
"activity_key": str(normalized_item.get("activity_key") or "").strip(),
|
||
"kind": item_kind,
|
||
"status": item_status,
|
||
"ui_intent": dict(normalized_item.get("ui_intent") or {}),
|
||
"focus_ref": _normalize_focus_ref(normalized_item.get("focus_ref")),
|
||
}
|
||
if ui_intent_kind == "job_events" and str(source_focus_ref.get("kind") or "").strip() == "ops_job_event":
|
||
primary_action_code = "focus_latest_job_events"
|
||
action_payload["source_focus_ref"] = source_focus_ref
|
||
|
||
return {
|
||
"key": f"activity-focus-{str(normalized_item.get('activity_key') or item_kind or 'activity').strip()}",
|
||
"title": title_text,
|
||
"level_label": "最高优先" if is_hard_problem else "推荐",
|
||
"tag_type": "danger" if is_hard_problem else "warning",
|
||
"summary": detail_summary,
|
||
"reason": reason,
|
||
"node_codes": [],
|
||
"meta_text": meta_text or subtitle or str(normalized_item.get("activity_key") or "").strip(),
|
||
"primary_label": primary_label,
|
||
"secondary_label": "",
|
||
"primary_type": "danger" if is_hard_problem else "warning",
|
||
"disabled": not str(((normalized_item.get("ui_intent") or {}).get("kind") or "")).strip(),
|
||
"primary_action_code": primary_action_code,
|
||
"secondary_action_code": "",
|
||
"action_payload": action_payload,
|
||
"focus_ref": _normalize_focus_ref(normalized_item.get("focus_ref")),
|
||
"related_activity": {
|
||
"kind": item_kind,
|
||
"title": title,
|
||
"subtitle": subtitle,
|
||
"status": item_status,
|
||
"occurred_at": str(normalized_item.get("occurred_at") or "").strip(),
|
||
},
|
||
}
|
||
|
||
|
||
def _build_driver_activity_focus_entry(activity_item: dict) -> dict:
|
||
normalized_item = dict(activity_item or {})
|
||
activity_key = str(normalized_item.get("activity_key") or "").strip()
|
||
activity_kind = str(normalized_item.get("kind") or "").strip()
|
||
ui_intent = dict(normalized_item.get("ui_intent") or {})
|
||
ui_intent_kind = str(ui_intent.get("kind") or "").strip()
|
||
if not activity_key or not activity_kind:
|
||
return {}
|
||
|
||
focus_ref = _normalize_focus_ref(normalized_item.get("focus_ref"))
|
||
source_focus_ref = _normalize_focus_ref(normalized_item.get("source_focus_ref"))
|
||
focus_action_code = "focus_activity_item" if ui_intent_kind else ""
|
||
if ui_intent_kind == "job_events" and str(source_focus_ref.get("kind") or "").strip() == "ops_job_event":
|
||
focus_action_code = "focus_latest_job_events"
|
||
focus_action_payload = {
|
||
"activity_key": activity_key,
|
||
"kind": activity_kind,
|
||
"status": str(normalized_item.get("status") or "").strip(),
|
||
"ui_intent": ui_intent,
|
||
"focus_ref": focus_ref,
|
||
}
|
||
if source_focus_ref:
|
||
focus_action_payload["source_focus_ref"] = source_focus_ref
|
||
contract_keys = _normalize_ops_contract_keys(
|
||
list(normalized_item.get("contract_keys") or [])
|
||
+ _suggest_ops_contract_keys_for_activity_item(normalized_item)
|
||
)
|
||
|
||
return {
|
||
"key": f"activity:{activity_key}",
|
||
"activity_key": activity_key,
|
||
"activity_kind": activity_kind,
|
||
"title": str(normalized_item.get("title") or "").strip() or activity_kind,
|
||
"subtitle": str(normalized_item.get("subtitle") or "").strip(),
|
||
"summary": str(normalized_item.get("summary_text") or normalized_item.get("summary") or "").strip(),
|
||
"status": str(normalized_item.get("status") or "").strip(),
|
||
"status_label": str(normalized_item.get("status_label") or "").strip(),
|
||
"meta_text": str(normalized_item.get("meta_text") or "").strip(),
|
||
"occurred_at": str(normalized_item.get("occurred_at") or "").strip(),
|
||
"target_node_codes": _normalize_driver_node_codes(normalized_item.get("target_node_codes") or []),
|
||
"focus_ref": focus_ref,
|
||
"source_focus_ref": source_focus_ref,
|
||
"ui_intent": ui_intent,
|
||
"ui_intent_kind": ui_intent_kind,
|
||
"focus_action_code": focus_action_code,
|
||
"focus_action_payload": focus_action_payload if ui_intent_kind else {},
|
||
"observation_only": True,
|
||
"contract_keys": contract_keys,
|
||
"contract_navigation": _build_ops_contract_navigation(
|
||
contract_keys,
|
||
primary_contract_key=_primary_ops_contract_key_for_activity_item(normalized_item, contract_keys),
|
||
registry=get_ops_contract_registry(),
|
||
),
|
||
}
|
||
|
||
|
||
def _build_driver_activity_focus(items: list[dict], *, limit: int = 3) -> list[dict]:
|
||
allowed_scene_kinds = {"execution_scene", "log_sync"}
|
||
allowed_run_kinds = {"playbook_run", "ops_job", "rollout"}
|
||
rows: list[dict] = []
|
||
seen_keys: set[str] = set()
|
||
|
||
for raw_item in list(items or []):
|
||
item = dict(raw_item or {})
|
||
item_kind = str(item.get("kind") or "").strip()
|
||
item_status = str(item.get("status") or "").strip()
|
||
if item_kind in allowed_scene_kinds:
|
||
pass
|
||
elif item_kind == "playbook_run":
|
||
if item_status in {"success", "completed", "healthy"}:
|
||
continue
|
||
elif item_kind in allowed_run_kinds:
|
||
if item_status in {"success", "completed", "ready", "planned"}:
|
||
continue
|
||
else:
|
||
continue
|
||
entry = _build_driver_activity_focus_entry(item)
|
||
entry_key = str(entry.get("key") or "").strip()
|
||
if not entry or not entry_key or entry_key in seen_keys:
|
||
continue
|
||
seen_keys.add(entry_key)
|
||
rows.append(entry)
|
||
if len(rows) >= limit:
|
||
break
|
||
return rows
|
||
|
||
|
||
def _build_execution_scene_log_sync(log_sync: dict, participating_nodes: list[dict]) -> dict:
|
||
normalized_log_sync = dict(log_sync or {})
|
||
enabled = bool(normalized_log_sync.get("enabled", False))
|
||
mode = "full" if str(normalized_log_sync.get("mode") or "").strip().lower() == "full" else "key"
|
||
line_count = int(normalized_log_sync.get("line_count", 0) or 0)
|
||
source_nodes = [
|
||
str(item or "").strip()
|
||
for item in list(normalized_log_sync.get("source_nodes") or [])
|
||
if str(item or "").strip()
|
||
]
|
||
source_nodes = list(dict.fromkeys(source_nodes))
|
||
preview_lines = [
|
||
str(item or "").strip()
|
||
for item in list(normalized_log_sync.get("preview_lines") or [])
|
||
if str(item or "").strip()
|
||
]
|
||
preview_lines = preview_lines[-20:]
|
||
source_node_summaries_map: dict[str, dict] = {}
|
||
for item in list(normalized_log_sync.get("source_node_summaries") or []):
|
||
if not isinstance(item, dict):
|
||
continue
|
||
node_code = str(item.get("node_code") or "").strip()
|
||
if not node_code:
|
||
continue
|
||
source_node_summaries_map[node_code] = {
|
||
"node_code": node_code,
|
||
"line_count": int(item.get("line_count", 0) or 0),
|
||
"key_line_count": int(item.get("key_line_count", 0) or 0),
|
||
"full_line_count": int(item.get("full_line_count", 0) or 0),
|
||
"last_at": str(item.get("last_at") or "").strip(),
|
||
"last_line": str(item.get("last_line") or "").strip(),
|
||
}
|
||
if not source_node_summaries_map and preview_lines:
|
||
for line in preview_lines:
|
||
match = _REMOTE_LOG_PREVIEW_LINE_RE.match(line)
|
||
if not match:
|
||
continue
|
||
node_code = str(match.group("node_code") or "").strip()
|
||
created_at = str(match.group("created_at") or "").strip()
|
||
if not node_code:
|
||
continue
|
||
bucket = source_node_summaries_map.setdefault(
|
||
node_code,
|
||
{
|
||
"node_code": node_code,
|
||
"line_count": 0,
|
||
"key_line_count": 0,
|
||
"full_line_count": 0,
|
||
"last_at": "",
|
||
"last_line": "",
|
||
},
|
||
)
|
||
bucket["line_count"] += 1
|
||
bucket["key_line_count"] += 1
|
||
bucket["last_at"] = created_at
|
||
bucket["last_line"] = line
|
||
for node_code in source_nodes:
|
||
source_node_summaries_map.setdefault(
|
||
node_code,
|
||
{
|
||
"node_code": node_code,
|
||
"line_count": 0,
|
||
"key_line_count": 0,
|
||
"full_line_count": 0,
|
||
"last_at": "",
|
||
"last_line": "",
|
||
},
|
||
)
|
||
source_nodes = list(dict.fromkeys([*source_nodes, *list(source_node_summaries_map.keys())]))
|
||
source_node_summaries = sorted(
|
||
source_node_summaries_map.values(),
|
||
key=lambda item: (
|
||
str(item.get("last_at") or ""),
|
||
str(item.get("node_code") or ""),
|
||
),
|
||
reverse=True,
|
||
)
|
||
source_node_set = set(source_nodes)
|
||
source_node_set.update(source_node_summaries_map.keys())
|
||
participating_node_codes = [
|
||
str(item.get("node_code") or "").strip()
|
||
for item in list(participating_nodes or [])
|
||
if str(item.get("node_code") or "").strip()
|
||
]
|
||
participating_node_codes = list(dict.fromkeys(participating_node_codes))
|
||
participating_node_map = {
|
||
str(item.get("node_code") or "").strip(): dict(item)
|
||
for item in list(participating_nodes or [])
|
||
if str(item.get("node_code") or "").strip()
|
||
}
|
||
covered_participating_nodes = [node_code for node_code in participating_node_codes if node_code in source_node_set]
|
||
missing_participating_nodes = [node_code for node_code in participating_node_codes if node_code not in source_node_set]
|
||
covered_participating_node_summaries = []
|
||
for node_code in covered_participating_nodes:
|
||
source_summary = dict(source_node_summaries_map.get(node_code) or {})
|
||
node_meta = participating_node_map.get(node_code) or {}
|
||
covered_participating_node_summaries.append(
|
||
{
|
||
**source_summary,
|
||
"region": str(node_meta.get("region") or "").strip(),
|
||
"role": str(node_meta.get("role") or "").strip(),
|
||
"status": str(node_meta.get("status") or "").strip(),
|
||
"participation_state": str(node_meta.get("participation_state") or "").strip(),
|
||
"participation_bucket": str(node_meta.get("participation_bucket") or "").strip(),
|
||
"participation_label": str(node_meta.get("participation_label") or "").strip(),
|
||
"participation_reason": str(node_meta.get("participation_reason") or "").strip(),
|
||
}
|
||
)
|
||
missing_participating_node_summaries = []
|
||
for node_code in missing_participating_nodes:
|
||
node_meta = participating_node_map.get(node_code) or {}
|
||
missing_reason_code = "no_sample"
|
||
missing_reason = "当前还没有收到该参与节点的远端日志样本。"
|
||
if not enabled:
|
||
missing_reason_code = "disabled"
|
||
missing_reason = "远端日志回传当前关闭。"
|
||
elif line_count <= 0:
|
||
missing_reason_code = "waiting_sample"
|
||
missing_reason = "日志回传已开启,但当前现场样本尚未形成。"
|
||
missing_participating_node_summaries.append(
|
||
{
|
||
"node_code": node_code,
|
||
"region": str(node_meta.get("region") or "").strip(),
|
||
"role": str(node_meta.get("role") or "").strip(),
|
||
"status": str(node_meta.get("status") or "").strip(),
|
||
"participation_state": str(node_meta.get("participation_state") or "").strip(),
|
||
"participation_bucket": str(node_meta.get("participation_bucket") or "").strip(),
|
||
"participation_label": str(node_meta.get("participation_label") or "").strip(),
|
||
"participation_reason": str(node_meta.get("participation_reason") or "").strip(),
|
||
"missing_reason_code": missing_reason_code,
|
||
"missing_reason": missing_reason,
|
||
}
|
||
)
|
||
|
||
if not enabled:
|
||
state = "disabled"
|
||
status_label = "已关闭"
|
||
status_type = "info"
|
||
description = "远端日志回传当前关闭,海外控制面不会持续收到大陆检测过程日志。"
|
||
recommended_action_label = "建议先开启关键回传"
|
||
elif not participating_node_codes:
|
||
state = "idle"
|
||
status_label = "空闲观察"
|
||
status_type = "success"
|
||
description = "当前没有参与检测的节点,日志回传已就绪,待现场出现时会自动开始镜像样本。"
|
||
recommended_action_label = "当前无需调整"
|
||
elif line_count <= 0:
|
||
state = "waiting_sample"
|
||
status_label = "等待样本"
|
||
status_type = "warning"
|
||
description = "日志回传已开启,但当前还没有收到参与节点的现场日志样本。"
|
||
recommended_action_label = "建议先抓 Worker 日志确认现场输出"
|
||
elif missing_participating_nodes:
|
||
state = "partial_coverage"
|
||
status_label = "部分覆盖"
|
||
status_type = "warning"
|
||
description = f"当前已有 {len(covered_participating_nodes)}/{len(participating_node_codes)} 台参与节点回传日志,仍有节点未被覆盖。"
|
||
recommended_action_label = "建议优先排查未回传节点"
|
||
elif mode == "full":
|
||
state = "full_capture"
|
||
status_label = "全量观察"
|
||
status_type = "success"
|
||
description = "参与节点已经全部被覆盖,当前处于全量日志观察模式,适合短时深度排障。"
|
||
recommended_action_label = "排障结束后可切回关键回传"
|
||
else:
|
||
state = "healthy"
|
||
status_label = "关键覆盖"
|
||
status_type = "success"
|
||
description = "参与节点已经全部被覆盖,当前处于关键日志观察模式,适合常态联调。"
|
||
recommended_action_label = "当前无需调整"
|
||
|
||
return {
|
||
"enabled": enabled,
|
||
"mode": mode,
|
||
"mode_label": "全量回传" if enabled and mode == "full" else ("关键回传" if enabled else "已关闭"),
|
||
"state": state,
|
||
"status_label": status_label,
|
||
"status_type": status_type,
|
||
"description": description,
|
||
"recommended_action_label": recommended_action_label,
|
||
"line_count": line_count,
|
||
"source_node_count": int(normalized_log_sync.get("source_node_count", 0) or len(source_nodes)),
|
||
"source_nodes": source_nodes,
|
||
"source_node_summaries": source_node_summaries,
|
||
"last_at": str(normalized_log_sync.get("last_at") or ""),
|
||
"last_line": str(normalized_log_sync.get("last_line") or ""),
|
||
"preview_lines": preview_lines,
|
||
"preview_line_count": len(preview_lines),
|
||
"participating_node_count": len(participating_node_codes),
|
||
"covered_participating_node_count": len(covered_participating_nodes),
|
||
"missing_participating_node_count": len(missing_participating_nodes),
|
||
"covered_participating_nodes": covered_participating_nodes,
|
||
"missing_participating_nodes": missing_participating_nodes,
|
||
"covered_participating_node_summaries": covered_participating_node_summaries,
|
||
"missing_participating_node_summaries": missing_participating_node_summaries,
|
||
}
|
||
|
||
|
||
def _normalize_scene_log_mode(raw_mode: object, *, fallback: str = "key") -> str:
|
||
normalized_mode = str(raw_mode or "").strip().lower()
|
||
if normalized_mode == "full":
|
||
return "full"
|
||
if normalized_mode == "key":
|
||
return "key"
|
||
return "full" if str(fallback or "").strip().lower() == "full" else "key"
|
||
|
||
|
||
def _derive_scene_log_source_summary_from_records(node_code: str, records: list[dict]) -> dict:
|
||
normalized_node_code = str(node_code or "").strip()
|
||
summary = {
|
||
"node_code": normalized_node_code,
|
||
"line_count": 0,
|
||
"key_line_count": 0,
|
||
"full_line_count": 0,
|
||
"last_at": "",
|
||
"last_line": "",
|
||
}
|
||
for record in list(records or []):
|
||
record_node_code = str(record.get("node_code") or "").strip()
|
||
if normalized_node_code and record_node_code != normalized_node_code:
|
||
continue
|
||
summary["line_count"] += 1
|
||
if str(record.get("mode") or "").strip() == "full":
|
||
summary["full_line_count"] += 1
|
||
else:
|
||
summary["key_line_count"] += 1
|
||
summary["last_at"] = str(record.get("created_at") or "").strip()
|
||
summary["last_line"] = str(record.get("line") or "").strip()
|
||
return summary
|
||
|
||
|
||
def _collect_scene_log_records(
|
||
active_job: dict | None,
|
||
*,
|
||
requested_mode: str,
|
||
) -> list[dict]:
|
||
normalized_active_job = dict(active_job or {})
|
||
events = list(normalized_active_job.get("current_cycle_events") or normalized_active_job.get("recent_events") or [])
|
||
if not events:
|
||
return []
|
||
|
||
current_cycle_token = str(normalized_active_job.get("current_cycle_token") or "").strip()
|
||
normalized_requested_mode = _normalize_scene_log_mode(requested_mode)
|
||
records: list[dict] = []
|
||
for event in reversed(events):
|
||
event_type = str(event.get("event_type") or "").strip()
|
||
if event_type != "worker_log":
|
||
continue
|
||
payload = event.get("payload") if isinstance(event.get("payload"), dict) else {}
|
||
event_cycle_token = str(payload.get("cycle_token") or "").strip()
|
||
if current_cycle_token and event_cycle_token and event_cycle_token != current_cycle_token:
|
||
continue
|
||
event_mode = _normalize_scene_log_mode(payload.get("log_mode") or "key")
|
||
if normalized_requested_mode != "full" and event_mode == "full":
|
||
continue
|
||
created_at = str(event.get("created_at") or "").strip()
|
||
node_code = str(event.get("node_code") or "").strip() or "unknown"
|
||
message = str(event.get("message") or "").strip()
|
||
if not message:
|
||
continue
|
||
records.append(
|
||
{
|
||
"created_at": created_at,
|
||
"node_code": node_code,
|
||
"message": message,
|
||
"mode": event_mode,
|
||
"cycle_token": event_cycle_token,
|
||
"line": f"[{created_at}] [{node_code}] {message}",
|
||
}
|
||
)
|
||
return records
|
||
|
||
|
||
def _parse_ops_timestamp(value: object) -> datetime | None:
|
||
text = str(value or "").strip()
|
||
if not text:
|
||
return None
|
||
try:
|
||
return datetime.fromisoformat(text)
|
||
except ValueError:
|
||
return None
|
||
|
||
|
||
def _collect_scene_log_records_from_ops_jobs(
|
||
*,
|
||
requested_mode: str,
|
||
limit: int = 30,
|
||
max_age_hours: int = 6,
|
||
) -> list[dict]:
|
||
normalized_requested_mode = _normalize_scene_log_mode(requested_mode)
|
||
cutoff = datetime.now() - timedelta(hours=max(1, int(max_age_hours or 6)))
|
||
records: list[dict] = []
|
||
worker_service_name = str(settings.worker_service_name or "domaincheck-worker").strip() or "domaincheck-worker"
|
||
jobs = list_ops_jobs(limit=max(1, min(int(limit or 30), 80)), compact=False)
|
||
for job in reversed(list(jobs or [])):
|
||
if str(job.get("action") or "").strip() != "logs.collect":
|
||
continue
|
||
if str(job.get("status") or "").strip() != "success":
|
||
continue
|
||
result = dict(job.get("result") or {})
|
||
payload = dict(job.get("payload") or {})
|
||
service_name = str(result.get("service_name") or payload.get("service_name") or "").strip()
|
||
if service_name != worker_service_name:
|
||
continue
|
||
node_code = str(job.get("target_node_code") or "").strip()
|
||
if not node_code:
|
||
continue
|
||
created_at = (
|
||
str(job.get("finished_at") or "").strip()
|
||
or str(job.get("updated_at") or "").strip()
|
||
or str(job.get("created_at") or "").strip()
|
||
)
|
||
created_dt = _parse_ops_timestamp(created_at)
|
||
if created_dt and created_dt < cutoff:
|
||
continue
|
||
output_text = str(result.get("stdout") or result.get("log_output") or "").strip()
|
||
if not output_text:
|
||
continue
|
||
output_lines = [str(item or "").strip() for item in output_text.splitlines() if str(item or "").strip()]
|
||
if not output_lines:
|
||
continue
|
||
sample_limit = 12 if normalized_requested_mode == "full" else 4
|
||
for line in output_lines[-sample_limit:]:
|
||
records.append(
|
||
{
|
||
"created_at": created_at,
|
||
"node_code": node_code,
|
||
"message": line,
|
||
"mode": normalized_requested_mode,
|
||
"cycle_token": "",
|
||
"line": f"[{created_at}] [{node_code}] {line}",
|
||
}
|
||
)
|
||
return records
|
||
|
||
|
||
def _merge_execution_scene_log_sync_sources(log_sync: dict | None, *, requested_mode: str) -> dict:
|
||
normalized_log_sync = dict(log_sync or {})
|
||
ops_job_records = _collect_scene_log_records_from_ops_jobs(
|
||
requested_mode=requested_mode,
|
||
limit=30,
|
||
max_age_hours=6,
|
||
)
|
||
if not ops_job_records:
|
||
return normalized_log_sync
|
||
|
||
preview_lines = [
|
||
str(item or "").strip()
|
||
for item in list(normalized_log_sync.get("preview_lines") or [])
|
||
if str(item or "").strip()
|
||
]
|
||
merged_preview_lines = preview_lines + [str(item.get("line") or "").strip() for item in ops_job_records if str(item.get("line") or "").strip()]
|
||
merged_preview_lines = merged_preview_lines[-20:]
|
||
|
||
source_nodes = [
|
||
str(item or "").strip()
|
||
for item in list(normalized_log_sync.get("source_nodes") or [])
|
||
if str(item or "").strip()
|
||
]
|
||
source_nodes.extend(
|
||
str(item.get("node_code") or "").strip()
|
||
for item in ops_job_records
|
||
if str(item.get("node_code") or "").strip()
|
||
)
|
||
source_nodes = list(dict.fromkeys(source_nodes))
|
||
|
||
existing_summaries = {}
|
||
for item in list(normalized_log_sync.get("source_node_summaries") or []):
|
||
if not isinstance(item, dict):
|
||
continue
|
||
node_code = str(item.get("node_code") or "").strip()
|
||
if not node_code:
|
||
continue
|
||
existing_summaries[node_code] = dict(item)
|
||
for node_code in source_nodes:
|
||
derived_summary = _derive_scene_log_source_summary_from_records(
|
||
node_code,
|
||
[record for record in ops_job_records if str(record.get("node_code") or "").strip() == node_code],
|
||
)
|
||
if existing_summaries.get(node_code):
|
||
current_summary = dict(existing_summaries[node_code])
|
||
current_summary["line_count"] = int(current_summary.get("line_count", 0) or 0) + int(derived_summary.get("line_count", 0) or 0)
|
||
current_summary["key_line_count"] = int(current_summary.get("key_line_count", 0) or 0) + int(derived_summary.get("key_line_count", 0) or 0)
|
||
current_summary["full_line_count"] = int(current_summary.get("full_line_count", 0) or 0) + int(derived_summary.get("full_line_count", 0) or 0)
|
||
if str(derived_summary.get("last_at") or "").strip() >= str(current_summary.get("last_at") or "").strip():
|
||
current_summary["last_at"] = str(derived_summary.get("last_at") or current_summary.get("last_at") or "").strip()
|
||
current_summary["last_line"] = str(derived_summary.get("last_line") or current_summary.get("last_line") or "").strip()
|
||
existing_summaries[node_code] = current_summary
|
||
elif int(derived_summary.get("line_count", 0) or 0) > 0:
|
||
existing_summaries[node_code] = derived_summary
|
||
|
||
latest_record = ops_job_records[-1]
|
||
latest_at = str(latest_record.get("created_at") or "").strip()
|
||
latest_line = str(latest_record.get("line") or "").strip()
|
||
current_last_at = str(normalized_log_sync.get("last_at") or "").strip()
|
||
if current_last_at and current_last_at > latest_at:
|
||
latest_at = current_last_at
|
||
latest_line = str(normalized_log_sync.get("last_line") or "").strip()
|
||
|
||
merged_line_count = max(
|
||
int(normalized_log_sync.get("line_count", 0) or 0),
|
||
len(merged_preview_lines),
|
||
sum(int(item.get("line_count", 0) or 0) for item in existing_summaries.values()),
|
||
)
|
||
|
||
return {
|
||
**normalized_log_sync,
|
||
"preview_lines": merged_preview_lines,
|
||
"line_count": merged_line_count,
|
||
"source_nodes": source_nodes,
|
||
"source_node_count": max(int(normalized_log_sync.get("source_node_count", 0) or 0), len(source_nodes)),
|
||
"source_node_summaries": list(existing_summaries.values()),
|
||
"last_at": latest_at,
|
||
"last_line": latest_line,
|
||
}
|
||
|
||
|
||
def _collect_scene_log_records_from_preview_lines(
|
||
preview_lines: list[str] | None,
|
||
*,
|
||
requested_mode: str,
|
||
) -> list[dict]:
|
||
normalized_requested_mode = _normalize_scene_log_mode(requested_mode)
|
||
records: list[dict] = []
|
||
for raw_line in list(preview_lines or []):
|
||
line = str(raw_line or "").strip()
|
||
if not line:
|
||
continue
|
||
match = _REMOTE_LOG_PREVIEW_LINE_RE.match(line)
|
||
if not match:
|
||
continue
|
||
created_at = str(match.group("created_at") or "").strip()
|
||
node_code = str(match.group("node_code") or "").strip() or "unknown"
|
||
message = str(match.group("message") or "").strip()
|
||
if not message:
|
||
continue
|
||
record_mode = "full" if normalized_requested_mode == "full" else "key"
|
||
records.append(
|
||
{
|
||
"created_at": created_at,
|
||
"node_code": node_code,
|
||
"message": message,
|
||
"mode": record_mode,
|
||
"cycle_token": "",
|
||
"line": line,
|
||
}
|
||
)
|
||
return records
|
||
|
||
|
||
def _scene_log_status_meta(status: str, *, log_sync_enabled: bool) -> tuple[str, str]:
|
||
normalized_status = str(status or "").strip()
|
||
mapping = {
|
||
"healthy": ("关键覆盖", "success"),
|
||
"full_capture": ("全量观察", "success"),
|
||
"historical_sample": ("历史样本", "success"),
|
||
"waiting_sample": ("等待样本", "warning"),
|
||
"missing_sample": ("缺少样本", "warning"),
|
||
"disabled": ("已关闭", "info"),
|
||
"standby": ("在线待命", "info"),
|
||
"unknown": ("未发现", "info"),
|
||
}
|
||
if normalized_status in mapping:
|
||
return mapping[normalized_status]
|
||
if not log_sync_enabled:
|
||
return ("已关闭", "info")
|
||
return ("待确认", "info")
|
||
|
||
|
||
def get_ops_node_scene_log(node_code: str, *, limit: int = 80, mode: str = "") -> dict:
|
||
normalized_node_code = str(node_code or "").strip()
|
||
if not normalized_node_code:
|
||
return {}
|
||
|
||
runtime = get_runtime_status()
|
||
detect = dict(runtime.get("detect") or {})
|
||
cluster_nodes = [
|
||
dict(item)
|
||
for item in list((runtime.get("cluster") or {}).get("nodes") or [])
|
||
if str(item.get("node_code") or "").strip()
|
||
]
|
||
execution_scene = _build_ops_execution_scene(detect)
|
||
log_sync = dict(execution_scene.get("log_sync") or {})
|
||
default_mode = _normalize_scene_log_mode(log_sync.get("mode") or "key")
|
||
selected_mode = _normalize_scene_log_mode(mode, fallback=default_mode)
|
||
active_job = dict(detect.get("active_job") or {})
|
||
records = _collect_scene_log_records(active_job, requested_mode=selected_mode)
|
||
if not records:
|
||
records = _collect_scene_log_records_from_preview_lines(
|
||
list(log_sync.get("preview_lines") or []),
|
||
requested_mode=selected_mode,
|
||
)
|
||
node_records = [
|
||
dict(record)
|
||
for record in records
|
||
if str(record.get("node_code") or "").strip() == normalized_node_code
|
||
]
|
||
normalized_limit = max(1, min(int(limit or 80), 400))
|
||
visible_records = node_records[-normalized_limit:]
|
||
|
||
participating_map = {
|
||
str(item.get("node_code") or "").strip(): dict(item)
|
||
for item in list(execution_scene.get("participating_nodes") or [])
|
||
if str(item.get("node_code") or "").strip()
|
||
}
|
||
standby_map = {
|
||
str(item.get("node_code") or "").strip(): dict(item)
|
||
for item in list(execution_scene.get("standby_nodes") or [])
|
||
if str(item.get("node_code") or "").strip()
|
||
}
|
||
cluster_map = {
|
||
str(item.get("node_code") or "").strip(): dict(item)
|
||
for item in cluster_nodes
|
||
if str(item.get("node_code") or "").strip()
|
||
}
|
||
source_summary_map = {
|
||
str(item.get("node_code") or "").strip(): dict(item)
|
||
for item in list(log_sync.get("source_node_summaries") or [])
|
||
if str(item.get("node_code") or "").strip()
|
||
}
|
||
missing_summary_map = {
|
||
str(item.get("node_code") or "").strip(): dict(item)
|
||
for item in list(log_sync.get("missing_participating_node_summaries") or [])
|
||
if str(item.get("node_code") or "").strip()
|
||
}
|
||
covered_summary_map = {
|
||
str(item.get("node_code") or "").strip(): dict(item)
|
||
for item in list(log_sync.get("covered_participating_node_summaries") or [])
|
||
if str(item.get("node_code") or "").strip()
|
||
}
|
||
|
||
node_meta = dict(participating_map.get(normalized_node_code) or standby_map.get(normalized_node_code) or {})
|
||
cluster_meta = dict(cluster_map.get(normalized_node_code) or {})
|
||
source_summary = dict(source_summary_map.get(normalized_node_code) or covered_summary_map.get(normalized_node_code) or {})
|
||
derived_source_summary = _derive_scene_log_source_summary_from_records(normalized_node_code, node_records)
|
||
source_summary = {
|
||
"node_code": normalized_node_code,
|
||
"line_count": max(
|
||
int(source_summary.get("line_count", 0) or 0),
|
||
int(derived_source_summary.get("line_count", 0) or 0),
|
||
),
|
||
"key_line_count": max(
|
||
int(source_summary.get("key_line_count", 0) or 0),
|
||
int(derived_source_summary.get("key_line_count", 0) or 0),
|
||
),
|
||
"full_line_count": max(
|
||
int(source_summary.get("full_line_count", 0) or 0),
|
||
int(derived_source_summary.get("full_line_count", 0) or 0),
|
||
),
|
||
"last_at": str(
|
||
derived_source_summary.get("last_at")
|
||
or source_summary.get("last_at")
|
||
or ""
|
||
).strip(),
|
||
"last_line": str(
|
||
derived_source_summary.get("last_line")
|
||
or source_summary.get("last_line")
|
||
or ""
|
||
).strip(),
|
||
}
|
||
missing_summary = dict(missing_summary_map.get(normalized_node_code) or {})
|
||
|
||
record_available = bool(source_summary.get("line_count", 0) or node_records)
|
||
detect_participating = normalized_node_code in participating_map
|
||
standby_visible = normalized_node_code in standby_map
|
||
log_sync_enabled = bool(log_sync.get("enabled", False))
|
||
|
||
if record_available:
|
||
if detect_participating:
|
||
status = "full_capture" if selected_mode == "full" else "healthy"
|
||
elif standby_visible:
|
||
status = "historical_sample"
|
||
else:
|
||
status = "historical_sample"
|
||
elif missing_summary:
|
||
missing_reason_code = str(missing_summary.get("missing_reason_code") or "").strip()
|
||
if missing_reason_code == "disabled":
|
||
status = "disabled"
|
||
elif missing_reason_code == "waiting_sample":
|
||
status = "waiting_sample"
|
||
else:
|
||
status = "missing_sample"
|
||
elif detect_participating:
|
||
status = "waiting_sample" if log_sync_enabled else "disabled"
|
||
elif standby_visible:
|
||
status = "standby"
|
||
else:
|
||
status = "unknown"
|
||
|
||
status_label, status_type = _scene_log_status_meta(status, log_sync_enabled=log_sync_enabled)
|
||
if record_available and detect_participating:
|
||
summary = f"节点当前正在参与检测,已保留 {int(source_summary.get('line_count', 0) or 0)} 条现场日志样本。"
|
||
elif record_available and standby_visible:
|
||
summary = f"节点当前在线待命,仍可查看最近保留的 {int(source_summary.get('line_count', 0) or 0)} 条现场日志样本。"
|
||
elif record_available:
|
||
summary = f"节点已保留 {int(source_summary.get('line_count', 0) or 0)} 条现场日志样本。"
|
||
elif missing_summary:
|
||
summary = str(missing_summary.get("missing_reason") or "").strip() or "当前还没有收到该节点的现场日志样本。"
|
||
elif detect_participating and log_sync_enabled:
|
||
summary = "节点正在参与检测,但当前现场日志样本仍未形成。"
|
||
elif detect_participating:
|
||
summary = "节点正在参与检测,但远端日志回传当前关闭。"
|
||
elif standby_visible:
|
||
summary = "节点当前在线但未参与本轮检测,暂时没有新的现场日志样本。"
|
||
else:
|
||
summary = "当前没有找到该节点的现场日志记录。"
|
||
|
||
latest_record = dict(visible_records[-1] or {}) if visible_records else {}
|
||
participation_payload = {
|
||
"detect_participating": detect_participating,
|
||
"participation_state": str(node_meta.get("participation_state") or "").strip(),
|
||
"participation_label": str(node_meta.get("participation_label") or "").strip(),
|
||
"participation_reason": str(node_meta.get("participation_reason") or "").strip(),
|
||
"participation_bucket": str(node_meta.get("participation_bucket") or "").strip(),
|
||
"participation_bucket_label": str(node_meta.get("participation_bucket_label") or "").strip(),
|
||
"is_dispatch_active": bool(node_meta.get("is_dispatch_active", False)),
|
||
}
|
||
|
||
return {
|
||
"node_code": normalized_node_code,
|
||
"available": bool(node_meta or cluster_meta or source_summary.get("line_count", 0) or missing_summary or node_records),
|
||
"status": status,
|
||
"status_label": status_label,
|
||
"status_type": status_type,
|
||
"summary": summary,
|
||
"log_sync_enabled": log_sync_enabled,
|
||
"mode": selected_mode,
|
||
"mode_label": "全量回传" if selected_mode == "full" else "关键回传",
|
||
"records_total": len(node_records),
|
||
"records_visible": len(visible_records),
|
||
"records_truncated": len(node_records) > len(visible_records),
|
||
"records": visible_records,
|
||
"latest_record": latest_record,
|
||
"source_summary": source_summary,
|
||
"missing_reason_code": str(missing_summary.get("missing_reason_code") or "").strip(),
|
||
"missing_reason": str(missing_summary.get("missing_reason") or "").strip(),
|
||
"node": {
|
||
"node_code": normalized_node_code,
|
||
"region": str(node_meta.get("region") or cluster_meta.get("region") or missing_summary.get("region") or "").strip(),
|
||
"role": str(node_meta.get("role") or cluster_meta.get("role") or missing_summary.get("role") or "").strip(),
|
||
"status": str(node_meta.get("status") or cluster_meta.get("status") or missing_summary.get("status") or "").strip(),
|
||
"current_load": int(node_meta.get("current_load", cluster_meta.get("current_load", 0)) or 0),
|
||
"last_heartbeat_at": str(
|
||
node_meta.get("last_heartbeat_at") or cluster_meta.get("last_heartbeat_at") or ""
|
||
).strip(),
|
||
},
|
||
"participation": participation_payload,
|
||
"contract_navigation": _build_ops_contract_navigation(
|
||
["ops_observability_contract", "ops_stack_diagnosis_contract"],
|
||
primary_contract_key="ops_observability_contract",
|
||
registry=get_ops_contract_registry(),
|
||
),
|
||
}
|
||
|
||
|
||
def _build_ops_execution_scene(detect: dict) -> dict:
|
||
normalized_detect = dict(detect or {})
|
||
active_job = dict(normalized_detect.get("active_job") or {})
|
||
log_sync = _merge_execution_scene_log_sync_sources(
|
||
normalized_detect.get("log_sync") or {},
|
||
requested_mode=str((normalized_detect.get("log_sync") or {}).get("mode") or "key"),
|
||
)
|
||
participation_summary = normalized_detect.get("participation_summary") or {}
|
||
participating_nodes = [
|
||
_slim_execution_scene_node(item)
|
||
for item in list(normalized_detect.get("participating_nodes") or [])
|
||
if str(item.get("node_code") or "").strip()
|
||
]
|
||
standby_nodes = [
|
||
_slim_execution_scene_node(item)
|
||
for item in list(normalized_detect.get("non_participating_nodes") or [])
|
||
if str(item.get("node_code") or "").strip()
|
||
]
|
||
dispatch_active_nodes = [
|
||
dict(item)
|
||
for item in participating_nodes
|
||
if str(item.get("participation_bucket") or "").strip() == "dispatch_active"
|
||
]
|
||
recent_only_nodes = [
|
||
dict(item)
|
||
for item in participating_nodes
|
||
if str(item.get("participation_bucket") or "").strip() == "recent_only"
|
||
]
|
||
load_syncing_nodes = [
|
||
dict(item)
|
||
for item in [*participating_nodes, *standby_nodes]
|
||
if str(item.get("participation_bucket") or "").strip() == "load_syncing"
|
||
]
|
||
counts = {
|
||
"dispatch_active": int(participation_summary.get("dispatch_active_nodes", len(dispatch_active_nodes)) or len(dispatch_active_nodes)),
|
||
"recent_only": int(participation_summary.get("recent_only_nodes", len(recent_only_nodes)) or len(recent_only_nodes)),
|
||
"standby": int(participation_summary.get("standby_nodes", 0) or 0),
|
||
"load_syncing": int(participation_summary.get("load_syncing_nodes", len(load_syncing_nodes)) or len(load_syncing_nodes)),
|
||
}
|
||
latest_event = dict(active_job.get("latest_cycle_event") or active_job.get("latest_event") or {})
|
||
latest_at_candidates = [
|
||
str(latest_event.get("created_at") or "").strip(),
|
||
str(active_job.get("started_at") or "").strip(),
|
||
str(active_job.get("created_at") or "").strip(),
|
||
str((log_sync or {}).get("last_at") or "").strip(),
|
||
]
|
||
latest_at_candidates.extend(
|
||
str(item.get("last_heartbeat_at") or "").strip()
|
||
for item in [*participating_nodes, *standby_nodes]
|
||
if str(item.get("last_heartbeat_at") or "").strip()
|
||
)
|
||
non_empty_latest_at_candidates = [candidate for candidate in latest_at_candidates if candidate]
|
||
latest_at = sorted(non_empty_latest_at_candidates)[-1] if non_empty_latest_at_candidates else ""
|
||
return {
|
||
"active_job_code": str((normalized_detect.get("active_job") or {}).get("job_code") or ""),
|
||
"phase_label": str(normalized_detect.get("phase_label") or ""),
|
||
"phase_detail": str(normalized_detect.get("phase_detail") or ""),
|
||
"recent_event": str(normalized_detect.get("recent_event") or ""),
|
||
"recent_warning": str(normalized_detect.get("recent_warning") or ""),
|
||
"latest_at": latest_at,
|
||
"summary": str(participation_summary.get("summary") or ""),
|
||
"counts": counts,
|
||
"log_sync": _build_execution_scene_log_sync(log_sync, participating_nodes),
|
||
"participation_summary": {
|
||
"effective_online_nodes": int(participation_summary.get("effective_online_nodes", 0) or 0),
|
||
"participating_nodes": int(participation_summary.get("participating_nodes", 0) or 0),
|
||
"dispatch_active_nodes": int(participation_summary.get("dispatch_active_nodes", 0) or 0),
|
||
"recent_only_nodes": int(participation_summary.get("recent_only_nodes", 0) or 0),
|
||
"non_participating_nodes": int(participation_summary.get("non_participating_nodes", 0) or 0),
|
||
"standby_nodes": int(participation_summary.get("standby_nodes", 0) or 0),
|
||
"load_syncing_nodes": int(participation_summary.get("load_syncing_nodes", 0) or 0),
|
||
"dedicated_worker_nodes": int(participation_summary.get("dedicated_worker_nodes", 0) or 0),
|
||
"controller_worker_nodes": int(participation_summary.get("controller_worker_nodes", 0) or 0),
|
||
"dispatch_active_node_codes": list(participation_summary.get("dispatch_active_node_codes") or []),
|
||
"recent_only_node_codes": list(participation_summary.get("recent_only_node_codes") or []),
|
||
"non_participating_node_codes": list(participation_summary.get("non_participating_node_codes") or []),
|
||
"standby_node_codes": list(participation_summary.get("standby_node_codes") or []),
|
||
"load_syncing_node_codes": list(participation_summary.get("load_syncing_node_codes") or []),
|
||
"summary": str(participation_summary.get("summary") or ""),
|
||
},
|
||
"participating_nodes": participating_nodes,
|
||
"dispatch_active_nodes": dispatch_active_nodes,
|
||
"recent_only_nodes": recent_only_nodes,
|
||
"standby_nodes": standby_nodes,
|
||
"load_syncing_nodes": load_syncing_nodes,
|
||
}
|
||
|
||
|
||
def _should_emit_execution_scene_activity(execution_scene: dict) -> bool:
|
||
normalized_scene = dict(execution_scene or {})
|
||
summary = dict(normalized_scene.get("participation_summary") or {})
|
||
log_sync = dict(normalized_scene.get("log_sync") or {})
|
||
return any(
|
||
[
|
||
str(normalized_scene.get("active_job_code") or "").strip(),
|
||
str(normalized_scene.get("phase_label") or "").strip(),
|
||
str(normalized_scene.get("phase_detail") or "").strip(),
|
||
str(normalized_scene.get("recent_event") or "").strip(),
|
||
str(normalized_scene.get("recent_warning") or "").strip(),
|
||
str(normalized_scene.get("latest_at") or "").strip(),
|
||
int(summary.get("effective_online_nodes", 0) or 0) > 0,
|
||
bool(normalized_scene.get("participating_nodes")),
|
||
bool(normalized_scene.get("standby_nodes")),
|
||
bool(log_sync.get("enabled", False)),
|
||
int(log_sync.get("line_count", 0) or 0) > 0,
|
||
int(log_sync.get("source_node_count", 0) or 0) > 0,
|
||
]
|
||
)
|
||
|
||
|
||
def _should_emit_log_sync_activity(execution_scene: dict) -> bool:
|
||
log_sync = dict((execution_scene or {}).get("log_sync") or {})
|
||
return any(
|
||
[
|
||
bool(log_sync.get("enabled", False)),
|
||
int(log_sync.get("participating_node_count", 0) or 0) > 0,
|
||
int(log_sync.get("line_count", 0) or 0) > 0,
|
||
int(log_sync.get("source_node_count", 0) or 0) > 0,
|
||
int(log_sync.get("missing_participating_node_count", 0) or 0) > 0,
|
||
bool(log_sync.get("preview_lines")),
|
||
str(log_sync.get("last_at") or "").strip(),
|
||
]
|
||
)
|
||
|
||
|
||
def _build_execution_scene_activity(execution_scene: dict) -> dict:
|
||
normalized_scene = dict(execution_scene or {})
|
||
summary = dict(normalized_scene.get("participation_summary") or {})
|
||
participating_nodes = list(normalized_scene.get("participating_nodes") or [])
|
||
standby_nodes = list(normalized_scene.get("standby_nodes") or [])
|
||
dispatch_active_nodes = [
|
||
item for item in participating_nodes
|
||
if bool(item.get("is_dispatch_active", False))
|
||
]
|
||
recent_only_nodes = [
|
||
item for item in participating_nodes
|
||
if str(item.get("participation_state") or "").strip() == "recent_throughput"
|
||
]
|
||
load_syncing_nodes = [
|
||
item for item in standby_nodes
|
||
if str(item.get("participation_state") or "").strip() == "load_syncing"
|
||
]
|
||
pure_standby_nodes = [
|
||
item for item in standby_nodes
|
||
if str(item.get("participation_state") or "").strip() == "standby"
|
||
]
|
||
phase_label = str(normalized_scene.get("phase_label") or "").strip()
|
||
summary_text = (
|
||
str(summary.get("summary") or "").strip()
|
||
or "当前还没有形成统一的执行现场摘要。"
|
||
)
|
||
status = "ready"
|
||
if dispatch_active_nodes or recent_only_nodes:
|
||
status = "running"
|
||
elif load_syncing_nodes:
|
||
status = "attention"
|
||
meta_parts = [
|
||
f"执行/已领 {len(dispatch_active_nodes)}",
|
||
f"近窗 {len(recent_only_nodes)}",
|
||
f"待命 {len(pure_standby_nodes)}",
|
||
f"待确认 {len(load_syncing_nodes)}",
|
||
]
|
||
if phase_label:
|
||
meta_parts.append(f"阶段 {phase_label}")
|
||
scene_node_codes = _normalize_driver_node_codes(
|
||
[str(item.get("node_code") or "").strip() for item in [*participating_nodes, *standby_nodes]]
|
||
)
|
||
return {
|
||
"kind": "execution_scene",
|
||
"activity_key": "execution-scene:live",
|
||
"title": "当前执行现场",
|
||
"subtitle": phase_label,
|
||
"summary": _truncate_activity_text(summary_text),
|
||
"meta_text": " / ".join(part for part in meta_parts if part),
|
||
"status": status,
|
||
"execution_mode": "",
|
||
"execution_mode_label": "",
|
||
"occurred_at": _activity_time(
|
||
normalized_scene.get("latest_at"),
|
||
*[item.get("last_heartbeat_at") for item in participating_nodes],
|
||
*[item.get("last_heartbeat_at") for item in standby_nodes],
|
||
),
|
||
"target_node_codes": scene_node_codes,
|
||
"ui_intent": _build_driver_ui_intent(
|
||
"focus_execution_scene",
|
||
open_log_drawer=False,
|
||
),
|
||
}
|
||
|
||
|
||
def _build_log_sync_activity(execution_scene: dict) -> dict:
|
||
normalized_scene = dict(execution_scene or {})
|
||
log_sync = dict(normalized_scene.get("log_sync") or {})
|
||
enabled = bool(log_sync.get("enabled", False))
|
||
participating_node_count = int(log_sync.get("participating_node_count", 0) or 0)
|
||
covered_node_count = int(log_sync.get("covered_participating_node_count", 0) or 0)
|
||
missing_node_count = int(log_sync.get("missing_participating_node_count", 0) or 0)
|
||
line_count = int(log_sync.get("line_count", 0) or 0)
|
||
source_node_count = int(log_sync.get("source_node_count", 0) or 0)
|
||
missing_nodes = [
|
||
str(item or "").strip()
|
||
for item in list(log_sync.get("missing_participating_nodes") or [])
|
||
if str(item or "").strip()
|
||
]
|
||
status = "ready"
|
||
if participating_node_count > 0 and not enabled:
|
||
status = "attention"
|
||
elif participating_node_count > 0 and (line_count <= 0 or missing_node_count > 0):
|
||
status = "attention"
|
||
elif participating_node_count > 0:
|
||
status = "running"
|
||
description = str(log_sync.get("description") or "").strip() or "远端日志回传当前暂无补充说明。"
|
||
if missing_nodes:
|
||
description = f"{description} 当前仍有未覆盖节点:{'、'.join(missing_nodes)}"
|
||
observed_node_codes = _normalize_driver_node_codes(
|
||
[*list(log_sync.get("source_nodes") or []), *missing_nodes]
|
||
)
|
||
return {
|
||
"kind": "log_sync",
|
||
"activity_key": "execution-scene:log-sync",
|
||
"title": "远端日志回传",
|
||
"subtitle": str(log_sync.get("mode_label") or "").strip(),
|
||
"summary": _truncate_activity_text(description),
|
||
"meta_text": " / ".join(
|
||
part
|
||
for part in [
|
||
f"参与覆盖 {covered_node_count}/{participating_node_count}",
|
||
f"样本 {line_count}",
|
||
f"来源 {source_node_count}",
|
||
f"最近 {str(log_sync.get('last_at') or '').strip()}" if str(log_sync.get("last_at") or "").strip() else "",
|
||
]
|
||
if part
|
||
),
|
||
"status": status,
|
||
"execution_mode": "control-plane",
|
||
"execution_mode_label": execution_mode_label("control-plane"),
|
||
"occurred_at": _activity_time(
|
||
log_sync.get("last_at"),
|
||
normalized_scene.get("latest_at"),
|
||
),
|
||
"target_node_codes": observed_node_codes,
|
||
"ui_intent": _build_driver_ui_intent(
|
||
"focus_execution_scene",
|
||
open_log_drawer=bool(list(log_sync.get("preview_lines") or [])),
|
||
),
|
||
}
|
||
|
||
|
||
def _default_rollout_target_nodes(cluster_nodes: list[dict]) -> list[dict]:
|
||
rows: list[dict] = []
|
||
for node in list(cluster_nodes or []):
|
||
node_code = str(node.get("node_code") or "").strip()
|
||
cluster_status = str(node.get("status") or "").strip()
|
||
if not node_code:
|
||
continue
|
||
if not bool(node.get("is_effective_worker", False)):
|
||
continue
|
||
if cluster_status not in {"online", "busy"}:
|
||
continue
|
||
rows.append(
|
||
{
|
||
"node_code": node_code,
|
||
"region": str(node.get("region") or ""),
|
||
"role": str(node.get("role") or ""),
|
||
"status": cluster_status,
|
||
"current_load": int(node.get("current_load", 0) or 0),
|
||
"is_effective_worker": True,
|
||
}
|
||
)
|
||
return rows
|
||
|
||
|
||
def get_ops_activity_stream(
|
||
*,
|
||
limit: int = _OPS_ACTIVITY_FETCH_LIMIT,
|
||
scan_limit: int | None = None,
|
||
kind: str = "",
|
||
status: str = "",
|
||
execution_mode: str = "",
|
||
query: str = "",
|
||
runtime_status: dict | None = None,
|
||
managed_nodes_payload: dict | None = None,
|
||
) -> dict:
|
||
safe_limit = min(max(int(limit or _OPS_ACTIVITY_FETCH_LIMIT), 1), 100)
|
||
safe_scan_limit = min(max(int(scan_limit or (safe_limit * 4)), safe_limit), 400)
|
||
normalized_kind = str(kind or "").strip()
|
||
normalized_status = str(status or "").strip()
|
||
normalized_execution_mode = str(execution_mode or "").strip()
|
||
normalized_query = str(query or "").strip()
|
||
resolved_runtime_status = dict(runtime_status or {})
|
||
if not resolved_runtime_status:
|
||
resolved_runtime_status = get_runtime_status()
|
||
execution_scene = _build_ops_execution_scene(resolved_runtime_status.get("detect") or {})
|
||
resolved_managed_nodes_payload = dict(managed_nodes_payload or {})
|
||
if not resolved_managed_nodes_payload:
|
||
resolved_managed_nodes_payload = list_managed_nodes_with_agent_state(
|
||
participation_payload=resolved_runtime_status.get("detect") or {}
|
||
)
|
||
managed_node_map = {
|
||
str(item.get("node_code") or "").strip(): dict(item or {})
|
||
for item in list(resolved_managed_nodes_payload.get("nodes") or [])
|
||
if str(item.get("node_code") or "").strip()
|
||
}
|
||
|
||
playbook_runs_payload = get_recent_ops_playbook_runs(limit=safe_scan_limit, scan_limit=max(240, safe_scan_limit * 12))
|
||
playbook_runs = list(playbook_runs_payload.get("runs") or [])
|
||
playbook_items = [_build_playbook_run_activity(run) for run in playbook_runs if str(run.get("run_code") or "").strip()]
|
||
|
||
jobs = list_ops_jobs(limit=safe_scan_limit, compact=False)
|
||
standalone_jobs = [
|
||
job
|
||
for job in jobs
|
||
if not _is_playbook_child_job(job)
|
||
and int(job.get("rollout_id") or 0) <= 0
|
||
]
|
||
job_items = [
|
||
_build_ops_job_activity(
|
||
job,
|
||
managed_node=managed_node_map.get(str(job.get("target_node_code") or "").strip(), {}),
|
||
)
|
||
for job in standalone_jobs
|
||
if int(job.get("id") or 0) > 0
|
||
]
|
||
|
||
rollouts = list_release_rollouts(limit=safe_scan_limit)
|
||
rollout_items = [_build_rollout_activity(rollout) for rollout in rollouts if int(rollout.get("id") or 0) > 0]
|
||
|
||
runbook = get_ops_runbook(
|
||
runtime_status=resolved_runtime_status,
|
||
managed_nodes_payload=resolved_managed_nodes_payload,
|
||
)
|
||
runbook_items = [
|
||
_build_runbook_sequence_activity(sequence)
|
||
for sequence in list(runbook.get("control_sequences") or [])
|
||
if str((sequence or {}).get("key") or "").strip()
|
||
]
|
||
runbook_items = [item for item in runbook_items if item]
|
||
|
||
scene_items: list[dict] = []
|
||
if _should_emit_execution_scene_activity(execution_scene):
|
||
scene_items.append(_build_execution_scene_activity(execution_scene))
|
||
if _should_emit_log_sync_activity(execution_scene):
|
||
scene_items.append(_build_log_sync_activity(execution_scene))
|
||
|
||
all_items = [*scene_items, *playbook_items, *job_items, *rollout_items, *runbook_items]
|
||
all_items = [_finalize_activity_item(item) for item in all_items if item]
|
||
all_items.sort(key=lambda item: (str(item.get("occurred_at") or ""), str(item.get("activity_key") or "")), reverse=True)
|
||
all_items.sort(
|
||
key=lambda item: 1 if str(item.get("kind") or "").strip() == "runbook_sequence" else 0
|
||
)
|
||
|
||
available_kind_counts: dict[str, int] = {}
|
||
available_status_counts: dict[str, int] = {}
|
||
available_execution_mode_counts: dict[str, int] = {}
|
||
for item in all_items:
|
||
item_kind = str(item.get("kind") or "").strip() or "unknown"
|
||
item_status = str(item.get("status") or "").strip() or "unknown"
|
||
item_execution_mode = str(item.get("execution_mode") or "").strip()
|
||
available_kind_counts[item_kind] = int(available_kind_counts.get(item_kind, 0) or 0) + 1
|
||
available_status_counts[item_status] = int(available_status_counts.get(item_status, 0) or 0) + 1
|
||
if item_execution_mode:
|
||
available_execution_mode_counts[item_execution_mode] = (
|
||
int(available_execution_mode_counts.get(item_execution_mode, 0) or 0) + 1
|
||
)
|
||
|
||
filtered_items = [
|
||
item
|
||
for item in all_items
|
||
if (not normalized_kind or str(item.get("kind") or "").strip() == normalized_kind)
|
||
and (not normalized_status or str(item.get("status") or "").strip() == normalized_status)
|
||
and (not normalized_execution_mode or str(item.get("execution_mode") or "").strip() == normalized_execution_mode)
|
||
and _matches_activity_query(item, normalized_query)
|
||
]
|
||
|
||
items = filtered_items[:safe_limit]
|
||
summary_status, summary_status_label, summary_text = _activity_stream_summary_state(filtered_items)
|
||
|
||
kind_counts: dict[str, int] = {}
|
||
status_counts: dict[str, int] = {}
|
||
for item in items:
|
||
kind = str(item.get("kind") or "").strip() or "unknown"
|
||
status = str(item.get("status") or "").strip() or "unknown"
|
||
kind_counts[kind] = int(kind_counts.get(kind, 0) or 0) + 1
|
||
status_counts[status] = int(status_counts.get(status, 0) or 0) + 1
|
||
|
||
summary_contract_keys = _normalize_ops_contract_keys(
|
||
["ops_observability_contract", "ops_stack_diagnosis_contract"]
|
||
+ [
|
||
contract_key
|
||
for item in filtered_items
|
||
for contract_key in list((item or {}).get("contract_keys") or [])
|
||
]
|
||
)
|
||
|
||
return {
|
||
"items": items,
|
||
"contract_navigation": _build_ops_contract_navigation(
|
||
summary_contract_keys,
|
||
primary_contract_key="ops_observability_contract",
|
||
registry=get_ops_contract_registry(),
|
||
),
|
||
"summary": {
|
||
"status": summary_status,
|
||
"status_label": summary_status_label,
|
||
"summary_text": summary_text,
|
||
"total": len(items),
|
||
"filtered_total": len(filtered_items),
|
||
"unfiltered_total": len(all_items),
|
||
"kind_counts": kind_counts,
|
||
"status_counts": status_counts,
|
||
"available_kind_counts": available_kind_counts,
|
||
"available_status_counts": available_status_counts,
|
||
"available_execution_mode_counts": available_execution_mode_counts,
|
||
"latest_at": str(items[0].get("occurred_at") or "") if items else "",
|
||
"limit": safe_limit,
|
||
"scan_limit": safe_scan_limit,
|
||
"filters": {
|
||
"kind": normalized_kind,
|
||
"status": normalized_status,
|
||
"execution_mode": normalized_execution_mode,
|
||
"query": normalized_query,
|
||
},
|
||
},
|
||
}
|
||
|
||
|
||
def _build_driver_recommendations(
|
||
*,
|
||
managed_nodes: list[dict],
|
||
execution_scene: dict,
|
||
release_summary: dict,
|
||
release_gate: dict,
|
||
release_launchpad: dict,
|
||
) -> list[dict]:
|
||
effective_worker_nodes = [
|
||
item
|
||
for item in list(managed_nodes or [])
|
||
if bool(item.get("cluster_is_effective_worker", False))
|
||
and str(item.get("cluster_status") or "").strip() in {"online", "busy"}
|
||
]
|
||
handover_gap_nodes = [
|
||
item
|
||
for item in effective_worker_nodes
|
||
if not (bool(item.get("is_managed", False)) and bool(item.get("is_enabled", False)) and bool(item.get("is_agent_online", False)))
|
||
]
|
||
delivery_dead_letter_nodes = [
|
||
item
|
||
for item in list(managed_nodes or [])
|
||
if str(item.get("delivery_queue_state") or "").strip() == "dead_letter"
|
||
and bool(item.get("is_managed", False))
|
||
and bool(item.get("is_enabled", False))
|
||
]
|
||
delivery_retry_nodes = [
|
||
item
|
||
for item in list(managed_nodes or [])
|
||
if str(item.get("delivery_queue_state") or "").strip() == "retrying"
|
||
and bool(item.get("is_managed", False))
|
||
and bool(item.get("is_enabled", False))
|
||
]
|
||
participating_nodes = [item for item in list(execution_scene.get("participating_nodes") or []) if str(item.get("node_code") or "").strip()]
|
||
standby_nodes = [item for item in list(execution_scene.get("standby_nodes") or []) if str(item.get("node_code") or "").strip()]
|
||
participating_node_codes = [str(item.get("node_code") or "").strip() for item in participating_nodes if str(item.get("node_code") or "").strip()]
|
||
standby_node_codes = [str(item.get("node_code") or "").strip() for item in standby_nodes if str(item.get("node_code") or "").strip()]
|
||
release_launchpad_card = _build_release_launchpad_driver_card(release_launchpad=release_launchpad)
|
||
launchpad_gap_action = _release_launchpad_gap_action_context(release_launchpad)
|
||
log_sync = dict(execution_scene.get("log_sync") or {})
|
||
log_sync_enabled = bool(log_sync.get("enabled", False))
|
||
remote_log_sample_count = int(log_sync.get("line_count", 0) or 0)
|
||
remote_log_source_count = int(log_sync.get("source_node_count", 0) or 0)
|
||
release_label = _preferred_release_label(release_summary)
|
||
preferred_release = dict(release_gate.get("release") or {})
|
||
release_gate_status = str(release_gate.get("status") or "").strip()
|
||
release_gate_execution_mode = str(release_gate.get("execution_mode") or "remote-agent").strip() or "remote-agent"
|
||
release_gate_execution_mode_label = (
|
||
str(release_gate.get("execution_mode_label") or "").strip() or execution_mode_label(release_gate_execution_mode)
|
||
)
|
||
release_gate_summary = str(release_gate.get("summary") or "").strip()
|
||
release_gate_blocking_reasons = [str(item).strip() for item in list(release_gate.get("blocking_reasons") or []) if str(item).strip()]
|
||
release_gate_warning_reasons = [str(item).strip() for item in list(release_gate.get("warning_reasons") or []) if str(item).strip()]
|
||
release_gate_rows = list(((release_gate.get("operational_readiness") or {}).get("rows") or []))
|
||
not_execution_ready_rows = [row for row in release_gate_rows if not bool(row.get("execution_ready", False))]
|
||
inspection_problem_rows = [
|
||
row
|
||
for row in release_gate_rows
|
||
if str(row.get("inspection_status") or "").strip() in {"running", "attention", "missing"}
|
||
]
|
||
playbook_runs_payload = get_recent_ops_playbook_runs(limit=6, scan_limit=240)
|
||
recent_playbook_runs = list(playbook_runs_payload.get("runs") or [])
|
||
attention_playbook_runs = [item for item in recent_playbook_runs if str(item.get("status") or "").strip() == "attention"]
|
||
active_playbook_runs = [item for item in recent_playbook_runs if str(item.get("status") or "").strip() == "running"]
|
||
activity_stream_payload = get_ops_activity_stream(limit=10, scan_limit=80)
|
||
recent_activity_items = list(activity_stream_payload.get("items") or [])
|
||
start_delivery_failed_items = [
|
||
dict(item or {})
|
||
for item in recent_activity_items
|
||
if str(item.get("start_delivery_state") or "").strip() == "failed_local"
|
||
]
|
||
activity_focus_level, activity_focus_item = _pick_driver_activity_focus(recent_activity_items)
|
||
|
||
cards: list[dict] = []
|
||
|
||
if attention_playbook_runs:
|
||
first_attention_run = dict(attention_playbook_runs[0] or {})
|
||
run_code = str(first_attention_run.get("run_code") or "").strip()
|
||
focus_step_title = str(first_attention_run.get("focus_step_title") or "").strip()
|
||
cards.append(
|
||
{
|
||
"key": f"playbook-run-attention-{run_code or 'latest'}",
|
||
"title": "先处理异常编排",
|
||
"level_label": "最高优先",
|
||
"tag_type": "danger",
|
||
"summary": (
|
||
f"{str(first_attention_run.get('playbook_title') or first_attention_run.get('playbook_key') or '标准编排').strip()}"
|
||
f"{' / ' + focus_step_title if focus_step_title else ''} 当前出现异常,建议先看整轮事件流。"
|
||
),
|
||
"reason": str(first_attention_run.get("focus_summary") or "当前编排存在失败、阻断或取消,先处理这轮编排比继续堆动作更重要。").strip(),
|
||
"node_codes": _normalize_driver_node_codes(first_attention_run.get("target_node_codes") or []),
|
||
"meta_text": run_code,
|
||
"primary_label": "查看编排详情",
|
||
"secondary_label": "最近事件",
|
||
"primary_type": "danger",
|
||
"disabled": not run_code,
|
||
"primary_action_code": "focus_playbook_run",
|
||
"secondary_action_code": "open_playbook_run_latest_events",
|
||
"action_payload": {
|
||
"run_code": run_code,
|
||
"focus_step_key": str(first_attention_run.get("focus_step_key") or "").strip(),
|
||
"focus_step_title": focus_step_title,
|
||
},
|
||
}
|
||
)
|
||
elif active_playbook_runs:
|
||
first_active_run = dict(active_playbook_runs[0] or {})
|
||
run_code = str(first_active_run.get("run_code") or "").strip()
|
||
focus_step_title = str(first_active_run.get("focus_step_title") or "").strip()
|
||
cards.append(
|
||
{
|
||
"key": f"playbook-run-running-{run_code or 'latest'}",
|
||
"title": "盯住收口中的编排",
|
||
"level_label": "推荐",
|
||
"tag_type": "warning",
|
||
"summary": (
|
||
f"{str(first_active_run.get('playbook_title') or first_active_run.get('playbook_key') or '标准编排').strip()}"
|
||
f"{' / ' + focus_step_title if focus_step_title else ''} 仍在收口中,建议先看整轮事件流再决定是否补动作。"
|
||
),
|
||
"reason": str(first_active_run.get("focus_summary") or "当前编排还在执行或排队,先看整轮回执和事件流最容易判断是否真卡住。").strip(),
|
||
"node_codes": _normalize_driver_node_codes(first_active_run.get("target_node_codes") or []),
|
||
"meta_text": run_code,
|
||
"primary_label": "查看编排详情",
|
||
"secondary_label": "最近事件",
|
||
"primary_type": "warning",
|
||
"disabled": not run_code,
|
||
"primary_action_code": "focus_playbook_run",
|
||
"secondary_action_code": "open_playbook_run_latest_events",
|
||
"action_payload": {
|
||
"run_code": run_code,
|
||
"focus_step_key": str(first_active_run.get("focus_step_key") or "").strip(),
|
||
"focus_step_title": focus_step_title,
|
||
},
|
||
}
|
||
)
|
||
|
||
if activity_focus_level and activity_focus_item:
|
||
cards.append(_build_driver_activity_card(activity_focus_level, activity_focus_item))
|
||
|
||
if start_delivery_failed_items:
|
||
first_delivery_gap_item = dict(start_delivery_failed_items[0] or {})
|
||
source_focus_ref = _normalize_focus_ref(
|
||
first_delivery_gap_item.get("source_focus_ref") or first_delivery_gap_item.get("focus_ref")
|
||
)
|
||
cards.append(
|
||
{
|
||
"key": "activity-start-delivery-gap",
|
||
"title": "先检查开始回执异常",
|
||
"level_label": "推荐",
|
||
"tag_type": "warning",
|
||
"summary": (
|
||
f"最近有 {len(start_delivery_failed_items)} 条任务已经在节点侧开始执行,但开始回执没能成功送达控制面。"
|
||
),
|
||
"reason": str(first_delivery_gap_item.get("summary") or "").strip()
|
||
or "这通常不代表节点没执行,而是控制面和节点现场之间已经出现开始态失联,建议先看任务事件流。",
|
||
"node_codes": _normalize_driver_node_codes(
|
||
[
|
||
str(source_focus_ref.get("target_node_code") or "").strip(),
|
||
*list(first_delivery_gap_item.get("target_node_codes") or []),
|
||
]
|
||
),
|
||
"meta_text": str(first_delivery_gap_item.get("job_code") or "").strip(),
|
||
"primary_label": "查看任务事件",
|
||
"secondary_label": "",
|
||
"primary_type": "warning",
|
||
"disabled": False,
|
||
"primary_action_code": "focus_latest_job_events",
|
||
"secondary_action_code": "",
|
||
"primary_action_payload": {
|
||
"focus_ref": source_focus_ref,
|
||
},
|
||
"focus_ref": source_focus_ref,
|
||
}
|
||
)
|
||
|
||
if handover_gap_nodes:
|
||
first_gap_node = handover_gap_nodes[0]
|
||
first_gap_node_code = str(first_gap_node.get("node_code") or "").strip()
|
||
launchpad_gap_action_code = str(launchpad_gap_action.get("action_code") or "").strip()
|
||
launchpad_gap_node_code = str(launchpad_gap_action.get("node_code") or "").strip()
|
||
primary_action_code = "handover_first_gap"
|
||
primary_label = "为首台补签 Token" if bool(first_gap_node.get("is_managed", False)) else "纳管首台节点"
|
||
summary = f"当前有 {len(handover_gap_nodes)} 台有效执行节点还没进入标准执行器就绪状态,继续扩机器前要先把接管链路补齐。"
|
||
reason = "节点虽然在线且属于有效执行面,SSH 已可承担日志、诊断和部分服务控制,但标准巡检编排、正式发布与 Rollout 仍以 Agent 在线为准。"
|
||
level_label = "最高优先"
|
||
tag_type = "danger"
|
||
key = "handover-gap"
|
||
if launchpad_gap_action_code in {"bootstrap_run", "run_acceptance"} and (
|
||
not launchpad_gap_node_code or launchpad_gap_node_code == first_gap_node_code
|
||
):
|
||
primary_action_code = launchpad_gap_action_code
|
||
primary_label = "跑接入收口" if launchpad_gap_action_code == "bootstrap_run" else "跑接管验收"
|
||
summary = (
|
||
f"当前有 {len(handover_gap_nodes)} 台有效执行节点还没进入标准执行器就绪状态,"
|
||
"并且 Launchpad 已经给出首台节点的标准收口动作。"
|
||
)
|
||
reason = str(launchpad_gap_action.get("summary") or "").strip() or reason
|
||
level_label = "最高优先" if launchpad_gap_action_code == "bootstrap_run" else "推荐"
|
||
tag_type = "warning" if launchpad_gap_action_code == "bootstrap_run" else "success"
|
||
key = f"handover-gap-{launchpad_gap_action_code}"
|
||
cards.append(
|
||
{
|
||
"key": key,
|
||
"title": "先补接管缺口",
|
||
"level_label": level_label,
|
||
"tag_type": tag_type,
|
||
"summary": summary,
|
||
"reason": reason,
|
||
"node_codes": [str(item.get("node_code") or "").strip() for item in handover_gap_nodes if str(item.get("node_code") or "").strip()],
|
||
"primary_label": primary_label,
|
||
"secondary_label": "查看目标节点",
|
||
"primary_type": "danger",
|
||
"disabled": not first_gap_node_code,
|
||
"primary_action_code": primary_action_code,
|
||
"secondary_action_code": "view_first_gap",
|
||
"primary_action_payload": (
|
||
{"node_code": first_gap_node_code}
|
||
if primary_action_code in {"bootstrap_run", "run_acceptance"}
|
||
else {}
|
||
),
|
||
}
|
||
)
|
||
|
||
if delivery_dead_letter_nodes:
|
||
first_dead_letter_node = dict(delivery_dead_letter_nodes[0] or {})
|
||
affected_node_codes = [
|
||
str(item.get("node_code") or "").strip()
|
||
for item in delivery_dead_letter_nodes
|
||
if str(item.get("node_code") or "").strip()
|
||
]
|
||
cards.append(
|
||
{
|
||
"key": "node-agent-dead-letter",
|
||
"title": "先处理 Agent 死信",
|
||
"level_label": "最高优先",
|
||
"tag_type": "danger",
|
||
"summary": (
|
||
f"当前有 {len(delivery_dead_letter_nodes)} 台托管节点存在 Node Agent 死信记录,"
|
||
"说明部分任务回执或事件已经进入人工介入区。"
|
||
),
|
||
"reason": str(first_dead_letter_node.get("delivery_queue_reason") or "Node Agent 回执队列已经出现死信,继续堆动作会放大控制面与节点现场的不一致。").strip(),
|
||
"node_codes": affected_node_codes,
|
||
"meta_text": _format_node_code_list(affected_node_codes),
|
||
"primary_label": "重放死信队列",
|
||
"secondary_label": "查看 Worker 日志",
|
||
"primary_type": "danger",
|
||
"disabled": not affected_node_codes,
|
||
"primary_action_code": "replay_delivery_queue",
|
||
"secondary_action_code": "open_worker_logs",
|
||
"primary_action_payload": {
|
||
"template_key": "delivery.queue.replay",
|
||
"target_node_codes": affected_node_codes,
|
||
"execution_mode": "remote-agent",
|
||
"auto_approve": False,
|
||
"payload": {
|
||
"limit": min(max(len(affected_node_codes), 1), 20),
|
||
"flush_after_replay": True,
|
||
"reason": "",
|
||
},
|
||
},
|
||
}
|
||
)
|
||
elif delivery_retry_nodes:
|
||
affected_node_codes = [
|
||
str(item.get("node_code") or "").strip()
|
||
for item in delivery_retry_nodes
|
||
if str(item.get("node_code") or "").strip()
|
||
]
|
||
first_retry_node = dict(delivery_retry_nodes[0] or {})
|
||
cards.append(
|
||
{
|
||
"key": "node-agent-retrying",
|
||
"title": "关注 Agent 回执积压",
|
||
"level_label": "推荐",
|
||
"tag_type": "warning",
|
||
"summary": (
|
||
f"当前有 {len(delivery_retry_nodes)} 台托管节点存在待重试回执,"
|
||
"控制面状态可能会短暂落后于节点现场。"
|
||
),
|
||
"reason": str(first_retry_node.get("delivery_queue_reason") or "Node Agent 正在自动回放积压的完成回执或事件,建议先观察日志再继续堆新的运维动作。").strip(),
|
||
"node_codes": affected_node_codes,
|
||
"meta_text": _format_node_code_list(affected_node_codes),
|
||
"primary_label": "冲刷回执队列",
|
||
"secondary_label": "查看 Worker 日志",
|
||
"primary_type": "warning",
|
||
"disabled": not affected_node_codes,
|
||
"primary_action_code": "flush_delivery_queue",
|
||
"secondary_action_code": "open_worker_logs",
|
||
"primary_action_payload": {
|
||
"template_key": "delivery.queue.flush",
|
||
"target_node_codes": affected_node_codes,
|
||
"execution_mode": "remote-agent",
|
||
"auto_approve": True,
|
||
"payload": {
|
||
"limit": min(max(len(affected_node_codes) * 5, 1), 20),
|
||
},
|
||
},
|
||
}
|
||
)
|
||
|
||
preferred_release_id = int(preferred_release.get("id") or 0)
|
||
preferred_release_version = str(preferred_release.get("release_version") or "").strip() or "-"
|
||
preferred_release_channel = str(preferred_release.get("channel") or "").strip() or "stable"
|
||
preferred_release_label = f"{preferred_release_version} / {preferred_release_channel}"
|
||
|
||
if release_launchpad_card:
|
||
cards.append(release_launchpad_card)
|
||
elif preferred_release_id <= 0:
|
||
cards.append(
|
||
{
|
||
"key": "release-missing",
|
||
"title": "先建立 Release",
|
||
"level_label": "待补齐",
|
||
"tag_type": "info",
|
||
"summary": "当前还没有默认 Release,正式 Rollout 入口尚未建立。",
|
||
"reason": "没有 Release 时,节点更新仍然容易回到临时 git pull、手工覆盖或口头版本约定。",
|
||
"node_codes": [],
|
||
"meta_text": "",
|
||
"primary_label": "创建 Release",
|
||
"secondary_label": "",
|
||
"primary_type": "primary",
|
||
"disabled": False,
|
||
"primary_action_code": "open_release_dialog",
|
||
"secondary_action_code": "",
|
||
}
|
||
)
|
||
elif release_gate_status in {"release_not_ready", "artifact_missing"}:
|
||
cards.append(
|
||
{
|
||
"key": "release-finish-metadata",
|
||
"title": "先收口默认 Release",
|
||
"level_label": "最高优先" if release_gate_status == "artifact_missing" else "推荐",
|
||
"tag_type": "danger" if release_gate_status == "artifact_missing" else "warning",
|
||
"summary": release_gate_summary or f"默认 Release {preferred_release_label} 还没有进入可稳定 Rollout 的状态。",
|
||
"reason": "先把 Release 状态、制品地址和版本信息补完整,Rollout 才不会重新退回手工发布路径。",
|
||
"node_codes": [],
|
||
"meta_text": preferred_release_label,
|
||
"primary_label": "查看版本区",
|
||
"secondary_label": "创建 Release",
|
||
"primary_type": "warning",
|
||
"disabled": False,
|
||
"primary_action_code": "focus_release_hub",
|
||
"secondary_action_code": "open_release_dialog",
|
||
}
|
||
)
|
||
elif release_gate_status == "blocked":
|
||
blocker_node_codes = [
|
||
str(item.get("node_code") or "").strip()
|
||
for item in (not_execution_ready_rows or inspection_problem_rows)
|
||
if str(item.get("node_code") or "").strip()
|
||
]
|
||
first_gap_row = dict(not_execution_ready_rows[0] or {})
|
||
first_gap_is_managed = bool(first_gap_row.get("is_managed", False))
|
||
first_gap_code = str(first_gap_row.get("node_code") or "").strip()
|
||
cards.append(
|
||
{
|
||
"key": "release-rollout-blocked",
|
||
"title": "先补 Rollout 前置条件",
|
||
"level_label": "最高优先",
|
||
"tag_type": "danger",
|
||
"summary": release_gate_summary or f"默认 Release {preferred_release_label} 当前仍存在 Rollout 阻断。",
|
||
"reason": (
|
||
release_gate_blocking_reasons[0]
|
||
if release_gate_blocking_reasons
|
||
else f"默认目标节点里仍有节点未通过 {release_gate_execution_mode_label}、巡检或接管门禁。"
|
||
),
|
||
"node_codes": blocker_node_codes,
|
||
"meta_text": preferred_release_label,
|
||
"primary_label": (
|
||
"纳管首台节点"
|
||
if first_gap_code and not first_gap_is_managed
|
||
else ("查看首台缺口" if first_gap_code else "执行标准巡检")
|
||
),
|
||
"secondary_label": "查看版本区",
|
||
"primary_type": "danger",
|
||
"disabled": not blocker_node_codes,
|
||
"primary_action_code": (
|
||
"handover_first_gap"
|
||
if first_gap_code and not first_gap_is_managed
|
||
else ("view_first_gap" if first_gap_code else "run_standard_inspection")
|
||
),
|
||
"secondary_action_code": "focus_release_hub",
|
||
}
|
||
)
|
||
elif release_gate_status == "attention":
|
||
warning_node_codes = [
|
||
str(item.get("node_code") or "").strip()
|
||
for item in inspection_problem_rows
|
||
if str(item.get("node_code") or "").strip()
|
||
]
|
||
cards.append(
|
||
{
|
||
"key": "release-rollout-attention",
|
||
"title": "先补默认 Rollout 巡检",
|
||
"level_label": "推荐",
|
||
"tag_type": "warning",
|
||
"summary": release_gate_summary or f"默认 Release {preferred_release_label} 已接近可发状态,但仍建议先补齐巡检。",
|
||
"reason": (
|
||
release_gate_warning_reasons[0]
|
||
if release_gate_warning_reasons
|
||
else "先补健康快照、日志和诊断包,可以显著降低真正发 Rollout 时的误判风险。"
|
||
),
|
||
"node_codes": warning_node_codes,
|
||
"meta_text": preferred_release_label,
|
||
"primary_label": "执行标准巡检",
|
||
"secondary_label": "查看版本区",
|
||
"primary_type": "warning",
|
||
"disabled": not warning_node_codes,
|
||
"primary_action_code": "run_standard_inspection",
|
||
"secondary_action_code": "focus_release_hub",
|
||
}
|
||
)
|
||
elif release_gate_status == "no_targets":
|
||
cards.append(
|
||
{
|
||
"key": "release-rollout-no-targets",
|
||
"title": "先确认默认 Rollout 目标",
|
||
"level_label": "推荐",
|
||
"tag_type": "warning",
|
||
"summary": release_gate_summary or f"默认 Release {preferred_release_label} 当前没有可直接纳入 Rollout 的默认目标节点。",
|
||
"reason": "通常是当前有效执行面没有在线节点,或节点尚未进入默认 target 集。",
|
||
"node_codes": [],
|
||
"meta_text": preferred_release_label,
|
||
"primary_label": "查看版本区",
|
||
"secondary_label": "",
|
||
"primary_type": "warning",
|
||
"disabled": False,
|
||
"primary_action_code": "focus_release_hub",
|
||
"secondary_action_code": "",
|
||
}
|
||
)
|
||
elif release_gate_status == "ready":
|
||
cards.append(
|
||
{
|
||
"key": "release-rollout-ready",
|
||
"title": "当前可以发起 Rollout",
|
||
"level_label": "准备就绪",
|
||
"tag_type": "success",
|
||
"summary": release_gate_summary or f"默认 Release {preferred_release_label} 已具备进入 Rollout 的门禁条件。",
|
||
"reason": "接管、巡检和默认版本都已经进入可推进状态,现在可以把更新真正纳入 Release / Rollout 闭环。",
|
||
"node_codes": [str(item.get("node_code") or "").strip() for item in list(release_gate.get("default_target_nodes") or []) if str(item.get("node_code") or "").strip()],
|
||
"meta_text": preferred_release_label,
|
||
"primary_label": "Worker 灰度",
|
||
"secondary_label": "Control 发布",
|
||
"primary_type": "success",
|
||
"disabled": False,
|
||
"primary_action_code": "create_release_rollout_worker",
|
||
"primary_action_payload": {
|
||
"release_id": preferred_release_id,
|
||
},
|
||
"secondary_action_code": "create_release_rollout_control",
|
||
"secondary_action_payload": {
|
||
"release_id": preferred_release_id,
|
||
},
|
||
}
|
||
)
|
||
elif preferred_release_id > 0:
|
||
cards.append(
|
||
{
|
||
"key": "release-rollout-fallback",
|
||
"title": "查看默认 Release 门禁",
|
||
"level_label": "推荐",
|
||
"tag_type": "info",
|
||
"summary": release_gate_summary or f"默认 Release {preferred_release_label} 的 Rollout 门禁需要进一步确认。",
|
||
"reason": "当前建议先进入 Release Hub,看默认版本、批次和门禁摘要,再决定下一步。",
|
||
"node_codes": [],
|
||
"meta_text": preferred_release_label,
|
||
"primary_label": "查看版本区",
|
||
"secondary_label": "",
|
||
"primary_type": "primary",
|
||
"disabled": False,
|
||
"primary_action_code": "focus_release_hub",
|
||
"secondary_action_code": "",
|
||
}
|
||
)
|
||
|
||
if participating_node_codes and not log_sync_enabled:
|
||
primary_focus_ref = _build_scene_node_log_focus_ref(
|
||
participating_node_codes[0],
|
||
mode="key",
|
||
limit=80,
|
||
source="remote_log_sync_disabled",
|
||
)
|
||
secondary_focus_ref = _build_scene_node_log_focus_ref(
|
||
participating_node_codes[0],
|
||
mode="full",
|
||
limit=120,
|
||
source="remote_log_sync_disabled",
|
||
)
|
||
cards.append(
|
||
{
|
||
"key": "enable-scene-log-sync",
|
||
"title": "先补现场日志回传",
|
||
"level_label": "最高优先",
|
||
"tag_type": "danger",
|
||
"summary": f"当前有 {len(participating_node_codes)} 台节点正在真实参与检测,但海外控制面还看不到过程日志,排障会处于半盲态。",
|
||
"reason": "先把远端日志回传切到关键模式,才能在不放大噪音的前提下看到阶段、异常、代理与执行过程。",
|
||
"node_codes": participating_node_codes,
|
||
"primary_label": "开启关键回传",
|
||
"secondary_label": "开启全量回传",
|
||
"primary_type": "danger",
|
||
"disabled": False,
|
||
"focus_ref": primary_focus_ref,
|
||
"primary_focus_ref": primary_focus_ref,
|
||
"secondary_focus_ref": secondary_focus_ref,
|
||
"primary_action_code": "enable_log_sync_key",
|
||
"primary_action_payload": {"focus_ref": primary_focus_ref},
|
||
"secondary_action_code": "enable_log_sync_full",
|
||
"secondary_action_payload": {"focus_ref": secondary_focus_ref},
|
||
}
|
||
)
|
||
elif participating_node_codes and remote_log_sample_count <= 0:
|
||
primary_focus_ref = _build_scene_node_log_focus_ref(
|
||
participating_node_codes[0],
|
||
mode=str(log_sync.get("mode") or "key"),
|
||
limit=120,
|
||
source="remote_log_sync_waiting_sample",
|
||
)
|
||
secondary_focus_ref = _merge_focus_ref(
|
||
{},
|
||
kind="execution_scene",
|
||
scene_key="scene.logs",
|
||
source="remote_log_sync_waiting_sample",
|
||
)
|
||
cards.append(
|
||
{
|
||
"key": "inspect-scene-log-gap",
|
||
"title": "排查现场日志样本缺口",
|
||
"level_label": "推荐",
|
||
"tag_type": "warning",
|
||
"summary": f"当前日志回传已开启,但 {len(participating_node_codes)} 台参与节点还没有回传样本,先确认 Worker 现场输出为什么没回来。",
|
||
"reason": "这通常意味着现场仍在执行,但日志镜像链路、周期或样本采集还没形成有效观测,适合先抓 Worker 日志再看巡检。",
|
||
"node_codes": participating_node_codes,
|
||
"meta_text": f"参与节点 {len(participating_node_codes)} 台 / 已回传来源 {remote_log_source_count} 台",
|
||
"primary_label": "看 Worker 日志",
|
||
"secondary_label": "执行标准巡检",
|
||
"primary_type": "warning",
|
||
"disabled": not participating_node_codes,
|
||
"focus_ref": primary_focus_ref,
|
||
"primary_focus_ref": primary_focus_ref,
|
||
"secondary_focus_ref": secondary_focus_ref,
|
||
"primary_action_code": "open_worker_logs_participating",
|
||
"primary_action_payload": {"focus_ref": primary_focus_ref},
|
||
"secondary_action_code": "run_inspection_participating",
|
||
"secondary_action_payload": {"focus_ref": secondary_focus_ref},
|
||
}
|
||
)
|
||
|
||
cards.append(
|
||
{
|
||
"key": "inspect-participating",
|
||
"title": "先看执行现场",
|
||
"level_label": "推荐" if participating_node_codes else "待命",
|
||
"tag_type": "warning" if participating_node_codes else "info",
|
||
"summary": (
|
||
f"当前有 {len(participating_node_codes)} 台节点正在真实参与检测,优先回收它们的标准巡检结果。"
|
||
if participating_node_codes
|
||
else "当前没有节点处于真实执行现场,可先关注待命节点或接管缺口。"
|
||
),
|
||
"reason": "真正参与检测的节点最接近现场,最适合先看健康快照、Worker 日志和诊断包。",
|
||
"node_codes": participating_node_codes,
|
||
"primary_label": "执行标准巡检",
|
||
"secondary_label": "打开诊断模板",
|
||
"primary_type": "primary",
|
||
"disabled": not participating_node_codes,
|
||
"primary_action_code": "run_inspection_participating",
|
||
"secondary_action_code": "open_diagnostics_participating",
|
||
}
|
||
)
|
||
|
||
cards.append(
|
||
{
|
||
"key": "inspect-standby",
|
||
"title": "排查在线未参与节点",
|
||
"level_label": "推荐" if standby_node_codes else "待命",
|
||
"tag_type": "warning" if standby_node_codes else "info",
|
||
"summary": (
|
||
f"当前有 {len(standby_node_codes)} 台节点在线但未参与检测,建议做一轮标准巡检确认是正常待命还是接单异常。"
|
||
if standby_node_codes
|
||
else "当前没有在线未参与节点,执行面结构比较干净。"
|
||
),
|
||
"reason": "这组节点最适合定位“在线但不接单”“控制面兼跑未开始执行”“接管完成但还没真正纳入巡检”类问题。",
|
||
"node_codes": standby_node_codes,
|
||
"primary_label": "执行标准巡检",
|
||
"secondary_label": "看 Worker 日志",
|
||
"primary_type": "primary",
|
||
"disabled": not standby_node_codes,
|
||
"primary_action_code": "run_inspection_standby",
|
||
"secondary_action_code": "open_worker_logs_standby",
|
||
}
|
||
)
|
||
|
||
return cards
|
||
|
||
|
||
def _default_ops_priority_recommendation() -> dict:
|
||
return {
|
||
"source": "static",
|
||
"key": "build-node-agent-plane",
|
||
"priority": "build-node-agent-plane",
|
||
"title": "build-node-agent-plane",
|
||
"summary": "下一阶段应把 SSH/脚本式运维升级为海外控制面 + 节点 Agent 的任务编排模式。",
|
||
"reason": "当前已经具备跨地域心跳、同步、就绪度和日志回传基础,但节点规模继续增加后,人工 SSH 与复制日志的边际成本会快速失控。",
|
||
"level_label": "长期重点",
|
||
"tag_type": "info",
|
||
"primary_label": "",
|
||
"secondary_label": "",
|
||
"primary_action_code": "",
|
||
"secondary_action_code": "",
|
||
"node_codes": [],
|
||
"primary_node_codes": [],
|
||
"secondary_node_codes": [],
|
||
"primary_action_payload": {},
|
||
"secondary_action_payload": {},
|
||
"reasons": [
|
||
"当前已经具备跨地域心跳、同步、就绪度和日志回传基础。",
|
||
"节点数继续增加后,人工 SSH 与复制日志的边际成本会快速失控。",
|
||
"后台按钮化动作需要统一的任务模型、执行回执和日志流通道。",
|
||
],
|
||
}
|
||
|
||
|
||
def _build_ops_priority_recommendation(driver_recommendations: list[dict]) -> dict:
|
||
for raw_item in list(driver_recommendations or []):
|
||
item = dict(raw_item or {})
|
||
if not item:
|
||
continue
|
||
title = str(item.get("title") or item.get("key") or "").strip()
|
||
summary = str(item.get("summary") or item.get("reason") or "").strip()
|
||
if not title and not summary:
|
||
continue
|
||
shared_node_codes = _normalize_driver_node_codes(item.get("node_codes") or [])
|
||
primary_node_codes = _normalize_driver_node_codes(item.get("primary_node_codes") or shared_node_codes)
|
||
secondary_node_codes = _normalize_driver_node_codes(item.get("secondary_node_codes") or shared_node_codes)
|
||
shared_payload = dict(item.get("action_payload") or {})
|
||
return {
|
||
"source": "driver_recommendations",
|
||
"key": str(item.get("key") or "").strip(),
|
||
"priority": title or str(item.get("key") or "").strip() or "当前建议",
|
||
"title": title or str(item.get("key") or "").strip() or "当前建议",
|
||
"summary": summary or "当前已有建议动作,建议先按驾驶建议卡推进。",
|
||
"reason": str(item.get("reason") or "").strip(),
|
||
"level_label": str(item.get("level_label") or "").strip(),
|
||
"tag_type": str(item.get("tag_type") or "").strip(),
|
||
"primary_label": str(item.get("primary_label") or "").strip(),
|
||
"secondary_label": str(item.get("secondary_label") or "").strip(),
|
||
"primary_action_code": str(item.get("primary_action_code") or "").strip(),
|
||
"secondary_action_code": str(item.get("secondary_action_code") or "").strip(),
|
||
"node_codes": shared_node_codes,
|
||
"primary_node_codes": primary_node_codes,
|
||
"secondary_node_codes": secondary_node_codes,
|
||
"primary_action_payload": {
|
||
**shared_payload,
|
||
**dict(item.get("primary_action_payload") or {}),
|
||
},
|
||
"secondary_action_payload": {
|
||
**shared_payload,
|
||
**dict(item.get("secondary_action_payload") or {}),
|
||
},
|
||
"reasons": [
|
||
part
|
||
for part in [
|
||
str(item.get("reason") or "").strip(),
|
||
str(item.get("summary") or "").strip(),
|
||
]
|
||
if part
|
||
],
|
||
}
|
||
return _default_ops_priority_recommendation()
|
||
|
||
|
||
def _driver_feed_status_from_tag_type(tag_type: str) -> str:
|
||
normalized_tag_type = str(tag_type or "").strip()
|
||
if normalized_tag_type in {"danger", "warning"}:
|
||
return "attention"
|
||
if normalized_tag_type == "success":
|
||
return "ready"
|
||
return "planned"
|
||
|
||
|
||
def _build_driver_feed_entry_from_priority_recommendation(recommendation: dict) -> dict:
|
||
normalized_recommendation = dict(recommendation or {})
|
||
key = str(normalized_recommendation.get("key") or "").strip() or "priority-recommendation"
|
||
shared_node_codes = _normalize_driver_node_codes(normalized_recommendation.get("node_codes") or [])
|
||
primary_node_codes = _normalize_driver_node_codes(
|
||
normalized_recommendation.get("primary_node_codes") or shared_node_codes
|
||
)
|
||
secondary_node_codes = _normalize_driver_node_codes(
|
||
normalized_recommendation.get("secondary_node_codes") or shared_node_codes
|
||
)
|
||
primary_action_payload = dict(normalized_recommendation.get("primary_action_payload") or {})
|
||
secondary_action_payload = dict(normalized_recommendation.get("secondary_action_payload") or {})
|
||
title = str(normalized_recommendation.get("title") or normalized_recommendation.get("priority") or key).strip()
|
||
summary = str(normalized_recommendation.get("summary") or "").strip()
|
||
reason = str(normalized_recommendation.get("reason") or "").strip()
|
||
detail_lines = _normalize_driver_text_lines(
|
||
[
|
||
*[str(part or "").strip() for part in list(normalized_recommendation.get("reasons") or [])],
|
||
reason,
|
||
],
|
||
limit=4,
|
||
)
|
||
focus_ref = _normalize_focus_ref(normalized_recommendation.get("focus_ref"))
|
||
primary_focus_ref = _normalize_focus_ref(normalized_recommendation.get("primary_focus_ref"))
|
||
secondary_focus_ref = _normalize_focus_ref(normalized_recommendation.get("secondary_focus_ref"))
|
||
return _attach_driver_feed_contract_navigation({
|
||
"key": f"priority:{key}",
|
||
"kind": "priority_recommendation",
|
||
"lane": "do_now",
|
||
"lane_label": "当前优先",
|
||
"title": title or "当前建议",
|
||
"summary": summary or reason or "当前已有建议动作,建议先从这里推进。",
|
||
"reason": reason,
|
||
"status": _driver_feed_status_from_tag_type(normalized_recommendation.get("tag_type")),
|
||
"level_label": str(normalized_recommendation.get("level_label") or "").strip() or "推荐",
|
||
"tag_type": str(normalized_recommendation.get("tag_type") or "").strip() or "info",
|
||
"meta_text": str(normalized_recommendation.get("source") or "overview.recommendation").strip(),
|
||
"node_codes": shared_node_codes,
|
||
"detail_lines": detail_lines,
|
||
"focus_ref": focus_ref,
|
||
"executor_kind": "driver_action",
|
||
"primary_label": str(normalized_recommendation.get("primary_label") or "").strip(),
|
||
"secondary_label": str(normalized_recommendation.get("secondary_label") or "").strip(),
|
||
"primary_action_code": str(normalized_recommendation.get("primary_action_code") or "").strip(),
|
||
"secondary_action_code": str(normalized_recommendation.get("secondary_action_code") or "").strip(),
|
||
"primary_node_codes": primary_node_codes,
|
||
"secondary_node_codes": secondary_node_codes,
|
||
"primary_focus_ref": primary_focus_ref,
|
||
"secondary_focus_ref": secondary_focus_ref,
|
||
"primary_action_payload": _action_payload_with_focus_ref(primary_action_payload, primary_focus_ref),
|
||
"secondary_action_payload": _action_payload_with_focus_ref(secondary_action_payload, secondary_focus_ref),
|
||
})
|
||
|
||
|
||
def _build_driver_feed_entry_from_driver_recommendation(item: dict) -> dict:
|
||
normalized_item = dict(item or {})
|
||
key = str(normalized_item.get("key") or "").strip() or "driver-recommendation"
|
||
shared_node_codes = _normalize_driver_node_codes(normalized_item.get("node_codes") or [])
|
||
primary_node_codes = _normalize_driver_node_codes(normalized_item.get("primary_node_codes") or shared_node_codes)
|
||
secondary_node_codes = _normalize_driver_node_codes(
|
||
normalized_item.get("secondary_node_codes") or shared_node_codes
|
||
)
|
||
shared_action_payload = dict(normalized_item.get("action_payload") or {})
|
||
primary_action_payload = {
|
||
**shared_action_payload,
|
||
**dict(normalized_item.get("primary_action_payload") or {}),
|
||
}
|
||
secondary_action_payload = {
|
||
**shared_action_payload,
|
||
**dict(normalized_item.get("secondary_action_payload") or {}),
|
||
}
|
||
detail_lines = _normalize_driver_text_lines(
|
||
[
|
||
str(normalized_item.get("reason") or "").strip(),
|
||
str(normalized_item.get("meta_text") or "").strip(),
|
||
],
|
||
limit=3,
|
||
)
|
||
focus_ref = _normalize_focus_ref(normalized_item.get("focus_ref"))
|
||
primary_focus_ref = _normalize_focus_ref(normalized_item.get("primary_focus_ref"))
|
||
secondary_focus_ref = _normalize_focus_ref(normalized_item.get("secondary_focus_ref"))
|
||
return _attach_driver_feed_contract_navigation({
|
||
"key": f"recommendation:{key}",
|
||
"kind": "driver_recommendation",
|
||
"lane": "do_now",
|
||
"lane_label": "建议动作",
|
||
"title": str(normalized_item.get("title") or key).strip() or "驾驶建议",
|
||
"summary": str(normalized_item.get("summary") or normalized_item.get("reason") or "").strip() or "建议先处理这条驾驶动作。",
|
||
"reason": str(normalized_item.get("reason") or "").strip(),
|
||
"status": _driver_feed_status_from_tag_type(normalized_item.get("tag_type")),
|
||
"level_label": str(normalized_item.get("level_label") or "").strip() or "推荐",
|
||
"tag_type": str(normalized_item.get("tag_type") or "").strip() or "info",
|
||
"meta_text": str(normalized_item.get("meta_text") or "").strip(),
|
||
"node_codes": shared_node_codes,
|
||
"detail_lines": detail_lines,
|
||
"focus_ref": focus_ref,
|
||
"executor_kind": "driver_action",
|
||
"primary_label": str(normalized_item.get("primary_label") or "").strip(),
|
||
"secondary_label": str(normalized_item.get("secondary_label") or "").strip(),
|
||
"primary_action_code": str(normalized_item.get("primary_action_code") or "").strip(),
|
||
"secondary_action_code": str(normalized_item.get("secondary_action_code") or "").strip(),
|
||
"primary_node_codes": primary_node_codes,
|
||
"secondary_node_codes": secondary_node_codes,
|
||
"primary_focus_ref": primary_focus_ref,
|
||
"secondary_focus_ref": secondary_focus_ref,
|
||
"primary_action_payload": _action_payload_with_focus_ref(primary_action_payload, primary_focus_ref),
|
||
"secondary_action_payload": _action_payload_with_focus_ref(secondary_action_payload, secondary_focus_ref),
|
||
})
|
||
|
||
|
||
def _build_driver_feed_entry_from_runbook_sequence(sequence: dict) -> dict:
|
||
normalized_sequence = dict(sequence or {})
|
||
sequence_key = str(normalized_sequence.get("key") or "").strip()
|
||
if not sequence_key:
|
||
return {}
|
||
|
||
primary_resolution = dict(normalized_sequence.get("primary_resolution") or {})
|
||
secondary_resolution = dict(normalized_sequence.get("secondary_resolution") or {})
|
||
resolved_action_label = (
|
||
str(primary_resolution.get("driver_action_label") or "").strip()
|
||
or str(primary_resolution.get("driver_action_code") or "").strip()
|
||
)
|
||
resolved_action_code = str(primary_resolution.get("driver_action_code") or "").strip()
|
||
resolved_node_codes = _normalize_driver_node_codes(primary_resolution.get("driver_node_codes") or [])
|
||
secondary_resolved_action_label = (
|
||
str(secondary_resolution.get("driver_action_label") or "").strip()
|
||
or str(secondary_resolution.get("driver_action_code") or "").strip()
|
||
)
|
||
target_node_codes = _normalize_driver_node_codes(normalized_sequence.get("target_node_codes") or [])
|
||
step_titles = [
|
||
str(item or "").strip()
|
||
for item in list(normalized_sequence.get("step_titles") or [])
|
||
if str(item or "").strip()
|
||
]
|
||
detail_lines: list[str] = []
|
||
if primary_resolution and primary_resolution.get("ok") is False:
|
||
detail_lines.append(f"当前解析失败:{str(primary_resolution.get('message') or '解析失败').strip()}")
|
||
elif resolved_action_label or resolved_action_code:
|
||
detail_lines.append(
|
||
f"当前解析:{resolved_action_label or resolved_action_code}"
|
||
+ (
|
||
f" ({resolved_action_code})"
|
||
if resolved_action_code and resolved_action_label and resolved_action_code != resolved_action_label
|
||
else ""
|
||
)
|
||
)
|
||
detail_lines.append(
|
||
f"当前目标:{_format_node_code_list(resolved_node_codes or target_node_codes)}"
|
||
)
|
||
if str(normalized_sequence.get("secondary_label") or "").strip() and secondary_resolved_action_label:
|
||
detail_lines.append(f"备用解析:{secondary_resolved_action_label}")
|
||
if step_titles:
|
||
detail_lines.append(f"标准步骤:{' -> '.join(step_titles)}")
|
||
if str(normalized_sequence.get("reason") or "").strip():
|
||
detail_lines.append(str(normalized_sequence.get("reason") or "").strip())
|
||
focus_ref = _normalize_focus_ref(normalized_sequence.get("focus_ref"))
|
||
primary_focus_ref = _normalize_focus_ref(normalized_sequence.get("primary_focus_ref"))
|
||
secondary_focus_ref = _normalize_focus_ref(normalized_sequence.get("secondary_focus_ref"))
|
||
|
||
return _attach_driver_feed_contract_navigation({
|
||
"key": f"runbook:{sequence_key}",
|
||
"kind": "runbook_sequence",
|
||
"lane": "standard_path",
|
||
"lane_label": "标准路径",
|
||
"title": str(normalized_sequence.get("title") or "").strip() or "标准作业路径",
|
||
"summary": str(normalized_sequence.get("summary") or "").strip() or "当前标准作业路径已就绪。",
|
||
"reason": str(normalized_sequence.get("reason") or "").strip(),
|
||
"status": _runbook_sequence_activity_status(normalized_sequence),
|
||
"level_label": str(normalized_sequence.get("status_label") or normalized_sequence.get("status") or "").strip() or "ready",
|
||
"tag_type": str(normalized_sequence.get("tag_type") or "").strip() or "info",
|
||
"meta_text": str(normalized_sequence.get("target_scope_label") or "").strip(),
|
||
"node_codes": target_node_codes,
|
||
"detail_lines": _normalize_driver_text_lines(detail_lines, limit=5),
|
||
"focus_ref": focus_ref,
|
||
"executor_kind": "runbook_sequence",
|
||
"sequence_key": sequence_key,
|
||
"primary_label": str(normalized_sequence.get("primary_label") or "").strip(),
|
||
"secondary_label": str(normalized_sequence.get("secondary_label") or "").strip(),
|
||
"primary_action_code": resolved_action_code or str(normalized_sequence.get("primary_action_code") or "").strip(),
|
||
"secondary_action_code": (
|
||
str(secondary_resolution.get("driver_action_code") or "").strip()
|
||
or str(normalized_sequence.get("secondary_action_code") or "").strip()
|
||
),
|
||
"primary_node_codes": resolved_node_codes or target_node_codes,
|
||
"secondary_node_codes": _normalize_driver_node_codes(
|
||
secondary_resolution.get("driver_node_codes") or target_node_codes
|
||
),
|
||
"primary_focus_ref": primary_focus_ref,
|
||
"secondary_focus_ref": secondary_focus_ref,
|
||
"primary_action_payload": _action_payload_with_focus_ref(
|
||
primary_resolution.get("driver_action_payload") or {},
|
||
primary_focus_ref,
|
||
),
|
||
"secondary_action_payload": _action_payload_with_focus_ref(
|
||
secondary_resolution.get("driver_action_payload") or {},
|
||
secondary_focus_ref,
|
||
),
|
||
"occurred_at": _activity_time(primary_resolution.get("resolved_at")),
|
||
})
|
||
|
||
|
||
def _build_driver_feed_top_recommendation(entry: dict) -> dict:
|
||
normalized_entry = dict(entry or {})
|
||
if not normalized_entry:
|
||
return {}
|
||
|
||
return {
|
||
"key": str(normalized_entry.get("key") or "").strip(),
|
||
"kind": str(normalized_entry.get("kind") or "").strip(),
|
||
"lane": str(normalized_entry.get("lane") or "").strip(),
|
||
"title": str(normalized_entry.get("title") or "").strip(),
|
||
"summary": str(normalized_entry.get("summary") or "").strip(),
|
||
"reason": str(normalized_entry.get("reason") or "").strip(),
|
||
"status": str(normalized_entry.get("status") or "").strip(),
|
||
"level_label": str(normalized_entry.get("level_label") or "").strip(),
|
||
"tag_type": str(normalized_entry.get("tag_type") or "").strip(),
|
||
"meta_text": str(normalized_entry.get("meta_text") or "").strip(),
|
||
"executor_kind": str(normalized_entry.get("executor_kind") or "").strip(),
|
||
"sequence_key": str(normalized_entry.get("sequence_key") or "").strip(),
|
||
"focus_ref": _normalize_focus_ref(normalized_entry.get("focus_ref")),
|
||
"primary_label": str(normalized_entry.get("primary_label") or "").strip(),
|
||
"secondary_label": str(normalized_entry.get("secondary_label") or "").strip(),
|
||
"primary_action_code": str(normalized_entry.get("primary_action_code") or "").strip(),
|
||
"secondary_action_code": str(normalized_entry.get("secondary_action_code") or "").strip(),
|
||
"node_codes": _normalize_driver_node_codes(normalized_entry.get("node_codes") or []),
|
||
"primary_node_codes": _normalize_driver_node_codes(normalized_entry.get("primary_node_codes") or []),
|
||
"secondary_node_codes": _normalize_driver_node_codes(normalized_entry.get("secondary_node_codes") or []),
|
||
"primary_focus_ref": _normalize_focus_ref(normalized_entry.get("primary_focus_ref")),
|
||
"secondary_focus_ref": _normalize_focus_ref(normalized_entry.get("secondary_focus_ref")),
|
||
"primary_action_payload": dict(normalized_entry.get("primary_action_payload") or {}),
|
||
"secondary_action_payload": dict(normalized_entry.get("secondary_action_payload") or {}),
|
||
"detail_lines": _normalize_driver_text_lines(normalized_entry.get("detail_lines") or [], limit=5),
|
||
"contract_keys": list(normalized_entry.get("contract_keys") or []),
|
||
"contract_navigation": dict(normalized_entry.get("contract_navigation") or {}),
|
||
}
|
||
|
||
|
||
def get_ops_driver_feed() -> dict:
|
||
overview = get_ops_overview()
|
||
go_live_summary = get_ops_go_live_summary()
|
||
runbook = get_ops_runbook()
|
||
recommendation = dict(overview.get("recommendation") or {})
|
||
driver_recommendations = [dict(item or {}) for item in list(overview.get("driver_recommendations") or []) if item]
|
||
control_sequences = [dict(item or {}) for item in list(runbook.get("control_sequences") or []) if item]
|
||
activity_stream = dict(overview.get("activity_stream") or {})
|
||
activity_focus = _build_driver_activity_focus(list(activity_stream.get("items") or []))
|
||
scene_log_observation = _build_scene_log_observation_from_overview(overview)
|
||
|
||
entries: list[dict] = []
|
||
seen_keys: set[str] = set()
|
||
|
||
priority_entry = _build_driver_feed_entry_from_priority_recommendation(recommendation)
|
||
if priority_entry:
|
||
entries.append(priority_entry)
|
||
recommendation_key = str(recommendation.get("key") or "").strip()
|
||
if recommendation_key:
|
||
seen_keys.add(recommendation_key)
|
||
|
||
for raw_item in driver_recommendations:
|
||
raw_key = str(raw_item.get("key") or "").strip()
|
||
if raw_key and raw_key in seen_keys:
|
||
continue
|
||
entry = _build_driver_feed_entry_from_driver_recommendation(raw_item)
|
||
if not entry:
|
||
continue
|
||
entries.append(entry)
|
||
if raw_key:
|
||
seen_keys.add(raw_key)
|
||
|
||
for raw_sequence in control_sequences:
|
||
entry = _build_driver_feed_entry_from_runbook_sequence(raw_sequence)
|
||
if entry:
|
||
entries.append(entry)
|
||
|
||
lane_counts: dict[str, int] = {}
|
||
kind_counts: dict[str, int] = {}
|
||
status_counts: dict[str, int] = {}
|
||
for item in entries:
|
||
lane = str(item.get("lane") or "").strip() or "other"
|
||
kind = str(item.get("kind") or "").strip() or "other"
|
||
status = str(item.get("status") or "").strip() or "other"
|
||
lane_counts[lane] = int(lane_counts.get(lane, 0) or 0) + 1
|
||
kind_counts[kind] = int(kind_counts.get(kind, 0) or 0) + 1
|
||
status_counts[status] = int(status_counts.get(status, 0) or 0) + 1
|
||
|
||
headline = ""
|
||
if entries:
|
||
first_entry = dict(entries[0] or {})
|
||
headline = (
|
||
f"{str(first_entry.get('title') or '').strip()}:{str(first_entry.get('summary') or '').strip()}"
|
||
).strip(":")
|
||
elif activity_focus:
|
||
first_focus = dict(activity_focus[0] or {})
|
||
headline = (
|
||
f"{str(first_focus.get('title') or '').strip()}:{str(first_focus.get('summary') or '').strip()}"
|
||
).strip(":")
|
||
publish_status = str(go_live_summary.get("publish_status") or "").strip()
|
||
publish_status_label = str(go_live_summary.get("publish_status_label") or "").strip()
|
||
publish_summary = str(go_live_summary.get("publish_summary") or "").strip()
|
||
launchpad_recommended_target_node_code = str(
|
||
go_live_summary.get("launchpad_recommended_target_node_code") or ""
|
||
).strip()
|
||
launchpad_recommended_recovery_label = str(
|
||
go_live_summary.get("launchpad_recommended_recovery_label") or ""
|
||
).strip()
|
||
launchpad_recommended_recovery_summary = str(
|
||
go_live_summary.get("launchpad_recommended_recovery_summary") or ""
|
||
).strip()
|
||
launchpad_onboarding_bootstrap_pending_nodes = int(
|
||
go_live_summary.get("launchpad_onboarding_bootstrap_pending_nodes", 0) or 0
|
||
)
|
||
launchpad_onboarding_acceptance_ready_nodes = int(
|
||
go_live_summary.get("launchpad_onboarding_acceptance_ready_nodes", 0) or 0
|
||
)
|
||
if publish_status and publish_status != "ready":
|
||
publish_prefix = publish_status_label or publish_status
|
||
headline = (
|
||
f"{headline}|发布闸门:{publish_prefix}"
|
||
if headline
|
||
else f"发布闸门:{publish_prefix}{'|' + publish_summary if publish_summary else ''}"
|
||
)
|
||
if launchpad_recommended_target_node_code and launchpad_recommended_recovery_label:
|
||
launchpad_prefix = f"{launchpad_recommended_recovery_label}:{launchpad_recommended_target_node_code}"
|
||
headline = f"{headline}|接入缺口:{launchpad_prefix}" if headline else launchpad_prefix
|
||
|
||
summary_contract_keys = _normalize_ops_contract_keys(
|
||
["ops_driver_contract"]
|
||
+ [
|
||
contract_key
|
||
for item in entries
|
||
for contract_key in list((item or {}).get("contract_keys") or [])
|
||
]
|
||
+ [
|
||
contract_key
|
||
for item in activity_focus
|
||
for contract_key in list((item or {}).get("contract_keys") or [])
|
||
]
|
||
+ list(scene_log_observation.get("contract_keys") or [])
|
||
)
|
||
top_recommendation = _build_driver_feed_top_recommendation(entries[0] or {})
|
||
grouped_driver_recommendations = [
|
||
dict(item or {})
|
||
for item in entries
|
||
if str((item or {}).get("kind") or "").strip() in {"priority_recommendation", "driver_recommendation"}
|
||
]
|
||
runbook_sequences = [
|
||
dict(item or {})
|
||
for item in entries
|
||
if str((item or {}).get("kind") or "").strip() == "runbook_sequence"
|
||
]
|
||
automation_entries = [
|
||
_build_codex_brief_entry(raw_entry)
|
||
for raw_entry in entries
|
||
if dict(raw_entry or {})
|
||
]
|
||
automation_entries = [entry for entry in automation_entries if entry]
|
||
automation_coverage = _build_ops_automation_coverage_summary(
|
||
automation_entries,
|
||
activity_focus_total=len(activity_focus),
|
||
scene_log_observation=scene_log_observation,
|
||
go_live_summary=go_live_summary,
|
||
)
|
||
|
||
return {
|
||
"generated_at": datetime.now().isoformat(timespec="seconds"),
|
||
"headline": headline or "当前还没有可展示的驾驶主线。",
|
||
"go_live_summary": go_live_summary,
|
||
"top_recommendation": top_recommendation,
|
||
"driver_recommendations": grouped_driver_recommendations,
|
||
"runbook_sequences": runbook_sequences,
|
||
"activity_focus": activity_focus,
|
||
"scene_log_observation": scene_log_observation,
|
||
"entries": entries,
|
||
"automation_coverage": automation_coverage,
|
||
"contract_navigation": _build_ops_contract_navigation(
|
||
summary_contract_keys,
|
||
primary_contract_key="ops_driver_contract",
|
||
registry=get_ops_contract_registry(),
|
||
),
|
||
"summary": {
|
||
"total": len(entries),
|
||
"lane_counts": lane_counts,
|
||
"kind_counts": kind_counts,
|
||
"status_counts": status_counts,
|
||
"go_live_status": str(go_live_summary.get("go_live_status") or "").strip(),
|
||
"operator_title": str(go_live_summary.get("operator_title") or "").strip(),
|
||
"publish_ready": bool(go_live_summary.get("publish_ready")),
|
||
"publish_status": publish_status,
|
||
"publish_status_label": publish_status_label,
|
||
"launchpad_recommended_target_node_code": launchpad_recommended_target_node_code,
|
||
"launchpad_recommended_recovery_label": launchpad_recommended_recovery_label,
|
||
"launchpad_recommended_recovery_summary": launchpad_recommended_recovery_summary,
|
||
"launchpad_onboarding_bootstrap_pending_nodes": launchpad_onboarding_bootstrap_pending_nodes,
|
||
"launchpad_onboarding_acceptance_ready_nodes": launchpad_onboarding_acceptance_ready_nodes,
|
||
"do_now_total": int(lane_counts.get("do_now", 0) or 0),
|
||
"standard_path_total": int(lane_counts.get("standard_path", 0) or 0),
|
||
"activity_focus_total": len(activity_focus),
|
||
"scene_log_status": str(scene_log_observation.get("status") or "").strip() or "standby",
|
||
"launch_status": str(automation_coverage.get("launch_status") or "").strip(),
|
||
"launch_ready": bool(automation_coverage.get("launch_ready", False)),
|
||
"preview_only_total": int(automation_coverage.get("preview_only_total", 0) or 0),
|
||
"backend_handled_total": int(automation_coverage.get("backend_handled_total", 0) or 0),
|
||
},
|
||
}
|
||
|
||
|
||
def _build_scene_log_observation_from_overview(overview: dict) -> dict:
|
||
normalized_overview = dict(overview or {})
|
||
execution_scene = dict(normalized_overview.get("execution_scene") or {})
|
||
log_sync = dict(execution_scene.get("log_sync") or {})
|
||
participation_summary = dict(execution_scene.get("participation_summary") or {})
|
||
|
||
participating_node_codes = _normalize_ops_node_code_list(
|
||
execution_scene.get("participating_nodes") or [],
|
||
participation_summary.get("dispatch_active_node_codes") or [],
|
||
participation_summary.get("recent_only_node_codes") or [],
|
||
)
|
||
missing_node_codes = _normalize_ops_node_code_list(log_sync.get("missing_participating_nodes") or [])
|
||
source_node_codes = _normalize_ops_node_code_list(
|
||
log_sync.get("source_nodes") or [],
|
||
log_sync.get("source_node_summaries") or [],
|
||
)
|
||
preferred_node_codes = _normalize_ops_node_code_list(
|
||
missing_node_codes,
|
||
participating_node_codes,
|
||
source_node_codes,
|
||
)
|
||
target_node_code = str((preferred_node_codes or [""])[0] or "").strip()
|
||
enabled = bool(log_sync.get("enabled", False))
|
||
mode = _normalize_scene_log_mode(log_sync.get("mode") or "key", fallback="key")
|
||
participating_total = int(
|
||
log_sync.get("participating_node_count", participation_summary.get("participating_nodes", len(participating_node_codes)))
|
||
or participation_summary.get("participating_nodes", len(participating_node_codes))
|
||
or len(participating_node_codes)
|
||
or 0
|
||
)
|
||
covered_total = int(log_sync.get("covered_participating_node_count", 0) or 0)
|
||
line_count = int(log_sync.get("line_count", 0) or 0)
|
||
source_node_count = int(log_sync.get("source_node_count", 0) or 0)
|
||
missing_total = int(log_sync.get("missing_participating_node_count", len(missing_node_codes)) or len(missing_node_codes))
|
||
status = "standby"
|
||
status_label = "待命"
|
||
summary = "当前没有参与检测节点,现场日志观察处于待命状态。"
|
||
source = "driver_scene_log_observation"
|
||
if participating_total > 0 and not enabled:
|
||
status = "disabled"
|
||
status_label = "未开启"
|
||
summary = (
|
||
f"当前有 {participating_total} 台参与节点,但远端日志回传仍关闭,"
|
||
"海外控制面还看不到节点级现场日志。"
|
||
)
|
||
source = "remote_log_sync_disabled"
|
||
elif participating_total > 0 and (
|
||
line_count <= 0 or source_node_count <= 0 or missing_total > 0
|
||
):
|
||
status = "waiting_sample"
|
||
status_label = "等待样本"
|
||
summary = (
|
||
f"远端日志回传已经开启,但参与节点只覆盖 {covered_total}/{participating_total},"
|
||
f"样本 {line_count} 条,仍需继续观察或下钻节点现场日志。"
|
||
)
|
||
source = "remote_log_sync_waiting_sample"
|
||
elif participating_total > 0:
|
||
status = "ready"
|
||
status_label = "可下钻"
|
||
summary = (
|
||
f"远端日志回传已经形成样本,当前覆盖 {covered_total}/{participating_total} 台参与节点,"
|
||
"可以直接下钻节点现场日志。"
|
||
)
|
||
source = "remote_log_sync_ready"
|
||
|
||
focus_ref = (
|
||
_build_scene_node_log_focus_ref(
|
||
target_node_code,
|
||
mode=mode,
|
||
limit=120 if status == "waiting_sample" else 80,
|
||
source=source,
|
||
)
|
||
if target_node_code
|
||
else {}
|
||
)
|
||
return {
|
||
"status": status,
|
||
"status_label": status_label,
|
||
"summary": summary,
|
||
"enabled": enabled,
|
||
"mode": mode,
|
||
"participating_node_count": participating_total,
|
||
"covered_participating_node_count": covered_total,
|
||
"missing_participating_node_count": missing_total,
|
||
"line_count": line_count,
|
||
"source_node_count": source_node_count,
|
||
"target_node_code": target_node_code,
|
||
"target_node_codes": preferred_node_codes,
|
||
"missing_node_codes": missing_node_codes,
|
||
"source_node_codes": source_node_codes,
|
||
"last_at": str(log_sync.get("last_at") or "").strip(),
|
||
"focus_ref": focus_ref,
|
||
"contract_keys": _normalize_ops_contract_keys(
|
||
["ops_observability_contract", "ops_stack_diagnosis_contract"]
|
||
),
|
||
"contract_navigation": _build_ops_contract_navigation(
|
||
["ops_observability_contract", "ops_stack_diagnosis_contract"],
|
||
primary_contract_key="ops_observability_contract",
|
||
registry=get_ops_contract_registry(),
|
||
),
|
||
}
|
||
|
||
|
||
def _build_driver_action_request_preview(
|
||
*,
|
||
executor_kind: str,
|
||
action_code: str,
|
||
action_payload: dict | None = None,
|
||
node_codes: list[str] | None = None,
|
||
sequence_key: str = "",
|
||
secondary: bool = False,
|
||
) -> dict:
|
||
normalized_executor_kind = str(executor_kind or "").strip()
|
||
normalized_action_code = str(action_code or "").strip()
|
||
normalized_action_payload = dict(action_payload or {})
|
||
normalized_node_codes = _normalize_driver_node_codes(node_codes or [])
|
||
normalized_sequence_key = str(sequence_key or "").strip()
|
||
|
||
if normalized_executor_kind == "runbook_sequence" and normalized_sequence_key:
|
||
return {
|
||
"sequence_key": normalized_sequence_key,
|
||
"secondary": bool(secondary),
|
||
"action_payload": normalized_action_payload,
|
||
}
|
||
|
||
return {
|
||
"action_code": normalized_action_code,
|
||
"node_codes": normalized_node_codes,
|
||
"action_payload": normalized_action_payload,
|
||
}
|
||
|
||
|
||
def _driver_action_missing_fields(
|
||
action_code: str,
|
||
*,
|
||
action_payload: dict | None = None,
|
||
node_codes: list[str] | None = None,
|
||
) -> list[str]:
|
||
normalized_action_code = str(action_code or "").strip()
|
||
normalized_action_payload = dict(action_payload or {})
|
||
normalized_node_codes = _normalize_driver_node_codes(node_codes or [])
|
||
missing_fields: list[str] = []
|
||
|
||
if normalized_action_code == "open_playbook_dialog" and not str(normalized_action_payload.get("playbook_key") or "").strip():
|
||
missing_fields.append("playbook_key")
|
||
if normalized_action_code == "open_action_template_dialog" and not str(normalized_action_payload.get("template_key") or "").strip():
|
||
missing_fields.append("template_key")
|
||
if normalized_action_code in {"replay_delivery_queue", "flush_delivery_queue"}:
|
||
target_node_codes = _normalize_driver_node_codes(normalized_action_payload.get("target_node_codes") or normalized_node_codes)
|
||
if not target_node_codes:
|
||
missing_fields.append("target_node_codes")
|
||
if normalized_action_code in {"focus_playbook_run", "open_playbook_run_latest_events"} and not str(
|
||
normalized_action_payload.get("run_code") or ""
|
||
).strip():
|
||
missing_fields.append("run_code")
|
||
if normalized_action_code == "focus_activity_item" and not str(
|
||
(normalized_action_payload.get("ui_intent") or {}).get("kind") or ""
|
||
).strip():
|
||
missing_fields.append("ui_intent.kind")
|
||
if normalized_action_code == "focus_latest_job_events":
|
||
has_ui_intent = bool(str((normalized_action_payload.get("ui_intent") or {}).get("kind") or "").strip())
|
||
source_focus_ref = _normalize_focus_ref(normalized_action_payload.get("source_focus_ref"))
|
||
focus_ref = _normalize_focus_ref(normalized_action_payload.get("focus_ref"))
|
||
has_event_focus = str(source_focus_ref.get("kind") or "").strip() == "ops_job_event"
|
||
has_job_focus = str(focus_ref.get("kind") or "").strip() in {"ops_job", "ops_job_event"}
|
||
if not (has_ui_intent or has_event_focus or has_job_focus):
|
||
missing_fields.append("ui_intent.kind|focus_ref")
|
||
if normalized_action_code in {"handover_first_gap", "view_first_gap"} and not str(
|
||
normalized_action_payload.get("node_code") or (normalized_node_codes[0] if normalized_node_codes else "")
|
||
).strip():
|
||
missing_fields.append("node_code")
|
||
if normalized_action_code in {
|
||
"open_worker_logs_participating",
|
||
"open_worker_logs_standby",
|
||
"open_worker_logs",
|
||
"open_diagnostics_participating",
|
||
"open_diagnostics",
|
||
"run_inspection_participating",
|
||
"run_inspection_standby",
|
||
"run_standard_inspection",
|
||
} and not normalized_node_codes:
|
||
missing_fields.append("node_codes")
|
||
if normalized_action_code in {"create_release_rollout_worker", "create_release_rollout_control"} and int(
|
||
normalized_action_payload.get("release_id") or 0
|
||
) <= 0:
|
||
missing_fields.append("release_id")
|
||
|
||
return missing_fields
|
||
|
||
|
||
def _build_driver_action_execution_profile(
|
||
action_code: str,
|
||
*,
|
||
action_payload: dict | None = None,
|
||
node_codes: list[str] | None = None,
|
||
) -> dict:
|
||
normalized_action_code = str(action_code or "").strip()
|
||
normalized_action_payload = dict(action_payload or {})
|
||
normalized_node_codes = _normalize_driver_node_codes(node_codes or [])
|
||
missing_fields = _driver_action_missing_fields(
|
||
normalized_action_code,
|
||
action_payload=normalized_action_payload,
|
||
node_codes=normalized_node_codes,
|
||
)
|
||
base_profile = {
|
||
"action_code": normalized_action_code,
|
||
"backend_handled": normalized_action_code in _BACKEND_DRIVER_ACTION_CODES,
|
||
"automation_level": "blocked",
|
||
"recommendation": "blocked",
|
||
"executor_mode_hint": "manual-or-ui",
|
||
"risk_level": "unknown",
|
||
"confirm_required": False,
|
||
"ui_only": False,
|
||
"blocked": False,
|
||
"requires_fields": [],
|
||
"missing_fields": missing_fields,
|
||
"reason": "",
|
||
}
|
||
|
||
if not normalized_action_code:
|
||
return {
|
||
**base_profile,
|
||
"blocked": True,
|
||
"reason": "当前条目还没有解析出 driver action,暂时不能自动执行。",
|
||
}
|
||
|
||
if missing_fields:
|
||
return {
|
||
**base_profile,
|
||
"blocked": True,
|
||
"requires_fields": list(dict.fromkeys(missing_fields)),
|
||
"reason": f"当前动作缺少必填上下文:{', '.join(list(dict.fromkeys(missing_fields)))}。",
|
||
}
|
||
|
||
if normalized_action_code not in _BACKEND_DRIVER_ACTION_CODES:
|
||
return {
|
||
**base_profile,
|
||
"automation_level": "ui_only",
|
||
"recommendation": "open_ui",
|
||
"executor_mode_hint": "manual-or-ui",
|
||
"risk_level": "medium",
|
||
"ui_only": True,
|
||
"reason": "当前动作尚未后端化,通常需要页面交互或人工确认后再推进。",
|
||
}
|
||
|
||
if normalized_action_code in _SAFE_AUTO_DRIVER_ACTION_CODES:
|
||
executor_mode_hint = "settings" if normalized_action_code.startswith(("enable_", "disable_")) else "ops-playbook"
|
||
reason = (
|
||
"当前动作只会调整运行时日志回传开关,属于低风险设置变更。"
|
||
if executor_mode_hint == "settings"
|
||
else "当前动作会走标准 playbook / 巡检链路,属于可自动执行的观测与诊断动作。"
|
||
)
|
||
return {
|
||
**base_profile,
|
||
"automation_level": "safe_auto",
|
||
"recommendation": "auto_execute",
|
||
"executor_mode_hint": executor_mode_hint,
|
||
"risk_level": "low",
|
||
"reason": reason,
|
||
}
|
||
|
||
if normalized_action_code in _GUARDED_AUTO_DRIVER_ACTION_CODES:
|
||
return {
|
||
**base_profile,
|
||
"automation_level": "guarded_auto",
|
||
"recommendation": "confirm_then_execute",
|
||
"executor_mode_hint": "smart-release-rollout",
|
||
"risk_level": "high",
|
||
"confirm_required": True,
|
||
"reason": "当前动作会直接创建 Release / Rollout 或推进发布批次,适合在 Codex 或人工确认后执行。",
|
||
}
|
||
|
||
if normalized_action_code in _MIXED_DRIVER_ACTION_CODES:
|
||
return {
|
||
**base_profile,
|
||
"automation_level": "mixed",
|
||
"recommendation": "resolve_first",
|
||
"executor_mode_hint": "cluster-sync-or-ui",
|
||
"risk_level": "medium",
|
||
"reason": "当前动作会先判断接管缺口,可能转为补接入页面,也可能直接同步托管节点,执行前最好先复核解析结果。",
|
||
}
|
||
|
||
if normalized_action_code in _UI_ONLY_DRIVER_ACTION_CODES:
|
||
return {
|
||
**base_profile,
|
||
"automation_level": "ui_only",
|
||
"recommendation": "open_ui",
|
||
"executor_mode_hint": "ui-intent",
|
||
"risk_level": "low",
|
||
"ui_only": True,
|
||
"reason": "当前动作主要用于打开页面、聚焦详情或进入模板,不适合作为无界面自动执行动作。",
|
||
}
|
||
|
||
return {
|
||
**base_profile,
|
||
"automation_level": "ui_only",
|
||
"recommendation": "open_ui",
|
||
"executor_mode_hint": "manual-or-ui",
|
||
"risk_level": "medium",
|
||
"ui_only": True,
|
||
"reason": "当前动作暂时只能作为驾驶建议展示,建议先进入对应工作区再决定下一步。",
|
||
}
|
||
|
||
|
||
def _driver_action_executor_kind_label(executor_kind: str) -> str:
|
||
normalized_executor_kind = str(executor_kind or "").strip()
|
||
if normalized_executor_kind == "runbook_sequence":
|
||
return "runbook sequence"
|
||
return normalized_executor_kind or "driver action"
|
||
|
||
|
||
def _driver_action_execution_chain(executor_kind: str) -> str:
|
||
normalized_executor_kind = str(executor_kind or "").strip()
|
||
if normalized_executor_kind == "runbook_sequence":
|
||
return "ops-driver -> runbook-sequence -> resolved driver action / ui-intent"
|
||
return "ops-driver -> driver-action execute -> playbook / ui-intent / settings"
|
||
|
||
|
||
def _driver_action_target_api(*, executor_kind: str, sequence_key: str = "") -> str:
|
||
normalized_executor_kind = str(executor_kind or "").strip()
|
||
normalized_sequence_key = str(sequence_key or "").strip()
|
||
if normalized_executor_kind == "runbook_sequence" and normalized_sequence_key:
|
||
return f"/api/v1/ops/runbook/sequences/{normalized_sequence_key}/execute"
|
||
return "/api/v1/ops/driver-actions/execute"
|
||
|
||
|
||
def _driver_action_target_method(*, executor_kind: str, sequence_key: str = "") -> str:
|
||
normalized_executor_kind = str(executor_kind or "").strip()
|
||
normalized_sequence_key = str(sequence_key or "").strip()
|
||
if normalized_executor_kind == "runbook_sequence" and normalized_sequence_key:
|
||
return "POST"
|
||
return "POST"
|
||
|
||
|
||
def _driver_action_recommendation_label(recommendation: str) -> str:
|
||
normalized_recommendation = str(recommendation or "").strip()
|
||
return {
|
||
"auto_execute": "可自动执行",
|
||
"confirm_then_execute": "建议确认后执行",
|
||
"resolve_first": "先复核再执行",
|
||
"open_ui": "需要进入工作区",
|
||
"blocked": "当前阻断",
|
||
}.get(normalized_recommendation, normalized_recommendation or "待判断")
|
||
|
||
|
||
def _build_driver_action_contract_request_payload(
|
||
*,
|
||
executor_kind: str,
|
||
action_code: str,
|
||
action_payload: dict | None = None,
|
||
node_codes: list[str] | None = None,
|
||
sequence_key: str = "",
|
||
focus_ref: dict | None = None,
|
||
secondary: bool = False,
|
||
requested_by: str = "api/preview",
|
||
) -> dict:
|
||
normalized_executor_kind = str(executor_kind or "").strip()
|
||
normalized_action_code = str(action_code or "").strip()
|
||
normalized_action_payload = _action_payload_with_focus_ref(action_payload or {}, focus_ref)
|
||
normalized_node_codes = _normalize_driver_node_codes(node_codes or [])
|
||
normalized_sequence_key = str(sequence_key or "").strip()
|
||
normalized_requested_by = str(requested_by or "api/preview").strip() or "api/preview"
|
||
|
||
if normalized_executor_kind == "runbook_sequence" and normalized_sequence_key:
|
||
payload = {
|
||
"secondary": bool(secondary),
|
||
"requested_by": normalized_requested_by,
|
||
}
|
||
if normalized_node_codes:
|
||
payload["node_codes"] = normalized_node_codes
|
||
if normalized_action_payload:
|
||
payload["action_payload"] = normalized_action_payload
|
||
return payload
|
||
|
||
return {
|
||
"action_code": normalized_action_code,
|
||
"node_codes": normalized_node_codes,
|
||
"action_payload": normalized_action_payload,
|
||
"requested_by": normalized_requested_by,
|
||
}
|
||
|
||
|
||
def _build_driver_action_contract_preview_section(
|
||
*,
|
||
executor_kind: str,
|
||
action_code: str,
|
||
label: str = "",
|
||
action_payload: dict | None = None,
|
||
node_codes: list[str] | None = None,
|
||
sequence_key: str = "",
|
||
focus_ref: dict | None = None,
|
||
secondary: bool = False,
|
||
requested_by: str = "api/preview",
|
||
reason: str = "",
|
||
) -> dict:
|
||
normalized_executor_kind = str(executor_kind or "").strip() or "driver_action"
|
||
normalized_action_code = str(action_code or "").strip()
|
||
normalized_label = str(label or "").strip()
|
||
normalized_action_payload = dict(action_payload or {})
|
||
normalized_node_codes = _normalize_driver_node_codes(node_codes or [])
|
||
normalized_sequence_key = str(sequence_key or "").strip()
|
||
execution_profile = _build_driver_action_execution_profile(
|
||
normalized_action_code,
|
||
action_payload=normalized_action_payload,
|
||
node_codes=normalized_node_codes,
|
||
)
|
||
|
||
return {
|
||
"label": normalized_label or ("次动作" if secondary else "主动作"),
|
||
"action_code": normalized_action_code,
|
||
"node_codes": normalized_node_codes,
|
||
"focus_ref": _normalize_focus_ref(focus_ref),
|
||
"target_api": _driver_action_target_api(
|
||
executor_kind=normalized_executor_kind,
|
||
sequence_key=normalized_sequence_key,
|
||
),
|
||
"target_method": _driver_action_target_method(
|
||
executor_kind=normalized_executor_kind,
|
||
sequence_key=normalized_sequence_key,
|
||
),
|
||
"reason": str(reason or execution_profile.get("reason") or "").strip(),
|
||
"recommendation_label": _driver_action_recommendation_label(execution_profile.get("recommendation") or ""),
|
||
"automation_level_label": str(execution_profile.get("automation_level") or "").strip(),
|
||
"risk_level_label": str(execution_profile.get("risk_level") or "").strip(),
|
||
"request_payload": _build_driver_action_contract_request_payload(
|
||
executor_kind=normalized_executor_kind,
|
||
action_code=normalized_action_code,
|
||
action_payload=normalized_action_payload,
|
||
node_codes=normalized_node_codes,
|
||
sequence_key=normalized_sequence_key,
|
||
focus_ref=_normalize_focus_ref(focus_ref),
|
||
secondary=secondary,
|
||
requested_by=requested_by,
|
||
),
|
||
"execution_profile": execution_profile,
|
||
}
|
||
|
||
|
||
def preview_driver_action(payload: dict) -> tuple[bool, str, dict]:
|
||
normalized_payload = dict(payload or {})
|
||
source_label = str(normalized_payload.get("source_label") or "ops-center").strip() or "ops-center"
|
||
title = str(normalized_payload.get("title") or "").strip() or "驾驶动作"
|
||
summary = str(normalized_payload.get("summary") or "").strip()
|
||
reason = str(normalized_payload.get("reason") or "").strip()
|
||
meta_text = str(normalized_payload.get("meta_text") or "").strip()
|
||
executor_kind = str(normalized_payload.get("executor_kind") or "driver_action").strip() or "driver_action"
|
||
sequence_key = str(normalized_payload.get("sequence_key") or "").strip()
|
||
requested_by = str(normalized_payload.get("requested_by") or "api/preview").strip() or "api/preview"
|
||
|
||
contract_registry = get_ops_contract_registry()
|
||
contract = dict((contract_registry.get("contracts_by_key") or {}).get("ops_driver_contract") or {})
|
||
|
||
if executor_kind == "runbook_sequence":
|
||
if not sequence_key:
|
||
return False, "sequence_key 不能为空", {}
|
||
|
||
include_secondary = bool(normalized_payload.get("include_secondary", True))
|
||
primary_ok, primary_message, primary_resolved = resolve_ops_runbook_sequence(
|
||
sequence_key,
|
||
{
|
||
"requested_by": requested_by,
|
||
"secondary": False,
|
||
"node_codes": _normalize_driver_node_codes(normalized_payload.get("primary_node_codes") or []),
|
||
"action_payload": dict(normalized_payload.get("primary_action_payload") or {}),
|
||
},
|
||
)
|
||
if not primary_ok:
|
||
return False, primary_message, dict(primary_resolved or {})
|
||
|
||
primary_label = str(normalized_payload.get("primary_label") or primary_resolved.get("driver_action_label") or "").strip()
|
||
secondary_label = str(normalized_payload.get("secondary_label") or "").strip()
|
||
secondary_action_code = str(normalized_payload.get("secondary_action_code") or "").strip()
|
||
preview_focus_ref = _normalize_focus_ref(
|
||
normalized_payload.get("focus_ref") or (primary_resolved.get("sequence") or {}).get("focus_ref")
|
||
)
|
||
secondary_preview: dict = {}
|
||
if include_secondary or secondary_label or secondary_action_code:
|
||
secondary_ok, secondary_message, secondary_resolved = resolve_ops_runbook_sequence(
|
||
sequence_key,
|
||
{
|
||
"requested_by": requested_by,
|
||
"secondary": True,
|
||
"node_codes": _normalize_driver_node_codes(normalized_payload.get("secondary_node_codes") or []),
|
||
"action_payload": dict(normalized_payload.get("secondary_action_payload") or {}),
|
||
},
|
||
)
|
||
if secondary_ok:
|
||
secondary_preview = _build_driver_action_contract_preview_section(
|
||
executor_kind=executor_kind,
|
||
action_code=str(secondary_resolved.get("driver_action_code") or "").strip(),
|
||
label=str(secondary_label or secondary_resolved.get("driver_action_label") or "").strip(),
|
||
action_payload=dict(secondary_resolved.get("driver_action_payload") or {}),
|
||
node_codes=_normalize_driver_node_codes(secondary_resolved.get("driver_node_codes") or []),
|
||
sequence_key=sequence_key,
|
||
focus_ref=_normalize_focus_ref(
|
||
normalized_payload.get("secondary_focus_ref")
|
||
or (secondary_resolved.get("sequence") or {}).get("action_focus_ref")
|
||
),
|
||
secondary=True,
|
||
requested_by=requested_by,
|
||
reason=reason,
|
||
)
|
||
elif secondary_label or secondary_action_code:
|
||
return False, secondary_message, dict(secondary_resolved or {})
|
||
|
||
return True, "驾驶动作预览已生成", {
|
||
"contract_key": "ops_driver_contract",
|
||
"contract_version": str(contract.get("version") or _OPS_CONTRACT_SCHEMA_VERSION).strip(),
|
||
"contract_schema_doc_path": str(contract.get("schema_doc_path") or "").strip(),
|
||
"contract_primary_endpoint": str(contract.get("primary_endpoint") or "").strip(),
|
||
"preview_endpoint": "/api/v1/ops/driver-actions/preview",
|
||
"source_label": source_label,
|
||
"title": title,
|
||
"summary": summary,
|
||
"reason": reason,
|
||
"meta_text": meta_text,
|
||
"executor_kind": executor_kind,
|
||
"executor_kind_label": _driver_action_executor_kind_label(executor_kind),
|
||
"sequence_key": sequence_key,
|
||
"focus_ref": preview_focus_ref,
|
||
"primary_focus_ref": _normalize_focus_ref(
|
||
normalized_payload.get("primary_focus_ref")
|
||
or (primary_resolved.get("sequence") or {}).get("action_focus_ref")
|
||
),
|
||
"secondary_focus_ref": _normalize_focus_ref(
|
||
normalized_payload.get("secondary_focus_ref")
|
||
or (secondary_preview.get("focus_ref") or {})
|
||
),
|
||
"execution_chain": _driver_action_execution_chain(executor_kind),
|
||
"primary": _build_driver_action_contract_preview_section(
|
||
executor_kind=executor_kind,
|
||
action_code=str(primary_resolved.get("driver_action_code") or "").strip(),
|
||
label=primary_label,
|
||
action_payload=dict(primary_resolved.get("driver_action_payload") or {}),
|
||
node_codes=_normalize_driver_node_codes(primary_resolved.get("driver_node_codes") or []),
|
||
sequence_key=sequence_key,
|
||
focus_ref=_normalize_focus_ref(
|
||
normalized_payload.get("primary_focus_ref")
|
||
or (primary_resolved.get("sequence") or {}).get("action_focus_ref")
|
||
),
|
||
secondary=False,
|
||
requested_by=requested_by,
|
||
reason=reason,
|
||
),
|
||
"secondary": secondary_preview,
|
||
"resolved_primary": dict(primary_resolved or {}),
|
||
}
|
||
|
||
primary_action_code = str(normalized_payload.get("primary_action_code") or "").strip()
|
||
secondary_action_code = str(normalized_payload.get("secondary_action_code") or "").strip()
|
||
if not primary_action_code and not secondary_action_code:
|
||
return False, "当前没有可预览的驾驶动作", {}
|
||
|
||
preview_focus_ref = _normalize_focus_ref(normalized_payload.get("focus_ref"))
|
||
primary_focus_ref = _normalize_focus_ref(normalized_payload.get("primary_focus_ref"))
|
||
secondary_focus_ref = _normalize_focus_ref(normalized_payload.get("secondary_focus_ref"))
|
||
|
||
return True, "驾驶动作预览已生成", {
|
||
"contract_key": "ops_driver_contract",
|
||
"contract_version": str(contract.get("version") or _OPS_CONTRACT_SCHEMA_VERSION).strip(),
|
||
"contract_schema_doc_path": str(contract.get("schema_doc_path") or "").strip(),
|
||
"contract_primary_endpoint": str(contract.get("primary_endpoint") or "").strip(),
|
||
"preview_endpoint": "/api/v1/ops/driver-actions/preview",
|
||
"source_label": source_label,
|
||
"title": title,
|
||
"summary": summary,
|
||
"reason": reason,
|
||
"meta_text": meta_text,
|
||
"executor_kind": executor_kind,
|
||
"executor_kind_label": _driver_action_executor_kind_label(executor_kind),
|
||
"sequence_key": sequence_key,
|
||
"focus_ref": preview_focus_ref,
|
||
"primary_focus_ref": primary_focus_ref,
|
||
"secondary_focus_ref": secondary_focus_ref,
|
||
"execution_chain": _driver_action_execution_chain(executor_kind),
|
||
"primary": (
|
||
_build_driver_action_contract_preview_section(
|
||
executor_kind=executor_kind,
|
||
action_code=primary_action_code,
|
||
label=str(normalized_payload.get("primary_label") or "").strip(),
|
||
action_payload=dict(normalized_payload.get("primary_action_payload") or {}),
|
||
node_codes=_normalize_driver_node_codes(normalized_payload.get("primary_node_codes") or normalized_payload.get("node_codes") or []),
|
||
sequence_key=sequence_key,
|
||
focus_ref=primary_focus_ref,
|
||
secondary=False,
|
||
requested_by=requested_by,
|
||
reason=reason,
|
||
)
|
||
if primary_action_code
|
||
else {}
|
||
),
|
||
"secondary": (
|
||
_build_driver_action_contract_preview_section(
|
||
executor_kind=executor_kind,
|
||
action_code=secondary_action_code,
|
||
label=str(normalized_payload.get("secondary_label") or "").strip(),
|
||
action_payload=dict(normalized_payload.get("secondary_action_payload") or {}),
|
||
node_codes=_normalize_driver_node_codes(normalized_payload.get("secondary_node_codes") or normalized_payload.get("node_codes") or []),
|
||
sequence_key=sequence_key,
|
||
focus_ref=secondary_focus_ref,
|
||
secondary=True,
|
||
requested_by=requested_by,
|
||
reason=reason,
|
||
)
|
||
if secondary_action_code
|
||
else {}
|
||
),
|
||
}
|
||
|
||
|
||
def _build_codex_brief_entry(entry: dict) -> dict:
|
||
normalized_entry = dict(entry or {})
|
||
if not normalized_entry:
|
||
return {}
|
||
|
||
executor_kind = str(normalized_entry.get("executor_kind") or "driver_action").strip() or "driver_action"
|
||
sequence_key = str(normalized_entry.get("sequence_key") or "").strip()
|
||
primary_action_code = str(normalized_entry.get("primary_action_code") or "").strip()
|
||
secondary_action_code = str(normalized_entry.get("secondary_action_code") or "").strip()
|
||
primary_node_codes = _normalize_driver_node_codes(normalized_entry.get("primary_node_codes") or normalized_entry.get("node_codes") or [])
|
||
secondary_node_codes = _normalize_driver_node_codes(normalized_entry.get("secondary_node_codes") or normalized_entry.get("node_codes") or [])
|
||
primary_action_payload = dict(normalized_entry.get("primary_action_payload") or {})
|
||
secondary_action_payload = dict(normalized_entry.get("secondary_action_payload") or {})
|
||
primary_execution = _build_driver_action_execution_profile(
|
||
primary_action_code,
|
||
action_payload=primary_action_payload,
|
||
node_codes=primary_node_codes,
|
||
)
|
||
secondary_execution = (
|
||
_build_driver_action_execution_profile(
|
||
secondary_action_code,
|
||
action_payload=secondary_action_payload,
|
||
node_codes=secondary_node_codes,
|
||
)
|
||
if secondary_action_code
|
||
else {}
|
||
)
|
||
if executor_kind == "runbook_sequence" and sequence_key:
|
||
target_api = f"/api/v1/ops/runbook/sequences/{sequence_key}/execute"
|
||
target_method = "POST"
|
||
else:
|
||
target_api = "/api/v1/ops/driver-actions/execute"
|
||
target_method = "POST"
|
||
|
||
primary_request = _build_driver_action_request_preview(
|
||
executor_kind=executor_kind,
|
||
action_code=primary_action_code,
|
||
action_payload=primary_action_payload,
|
||
node_codes=primary_node_codes,
|
||
sequence_key=sequence_key,
|
||
secondary=False,
|
||
)
|
||
secondary_request = (
|
||
_build_driver_action_request_preview(
|
||
executor_kind=executor_kind,
|
||
action_code=secondary_action_code,
|
||
action_payload=secondary_action_payload,
|
||
node_codes=secondary_node_codes,
|
||
sequence_key=sequence_key,
|
||
secondary=True,
|
||
)
|
||
if secondary_action_code
|
||
else {}
|
||
)
|
||
|
||
recommendation = str(primary_execution.get("recommendation") or "blocked").strip() or "blocked"
|
||
automation_level = str(primary_execution.get("automation_level") or "blocked").strip() or "blocked"
|
||
executor_mode_hint = str(primary_execution.get("executor_mode_hint") or "").strip()
|
||
status_text = {
|
||
"auto_execute": "可自动执行",
|
||
"confirm_then_execute": "建议确认后执行",
|
||
"resolve_first": "先复核再执行",
|
||
"open_ui": "需要进入工作区",
|
||
"blocked": "当前阻断",
|
||
}.get(recommendation, "待判断")
|
||
contract_keys = _suggest_ops_contract_keys_for_codex_entry(normalized_entry)
|
||
|
||
return {
|
||
"key": str(normalized_entry.get("key") or "").strip(),
|
||
"kind": str(normalized_entry.get("kind") or "").strip(),
|
||
"lane": str(normalized_entry.get("lane") or "").strip(),
|
||
"lane_label": str(normalized_entry.get("lane_label") or "").strip(),
|
||
"title": str(normalized_entry.get("title") or "").strip(),
|
||
"summary": str(normalized_entry.get("summary") or "").strip(),
|
||
"reason": str(normalized_entry.get("reason") or "").strip(),
|
||
"status": str(normalized_entry.get("status") or "").strip(),
|
||
"level_label": str(normalized_entry.get("level_label") or "").strip(),
|
||
"tag_type": str(normalized_entry.get("tag_type") or "").strip(),
|
||
"meta_text": str(normalized_entry.get("meta_text") or "").strip(),
|
||
"detail_lines": list(normalized_entry.get("detail_lines") or []),
|
||
"node_codes": _normalize_driver_node_codes(normalized_entry.get("node_codes") or []),
|
||
"executor_kind": executor_kind,
|
||
"sequence_key": sequence_key,
|
||
"focus_ref": _normalize_focus_ref(normalized_entry.get("focus_ref")),
|
||
"primary_focus_ref": _normalize_focus_ref(normalized_entry.get("primary_focus_ref")),
|
||
"secondary_focus_ref": _normalize_focus_ref(normalized_entry.get("secondary_focus_ref")),
|
||
"occurred_at": str(normalized_entry.get("occurred_at") or "").strip(),
|
||
"primary_label": str(normalized_entry.get("primary_label") or "").strip(),
|
||
"secondary_label": str(normalized_entry.get("secondary_label") or "").strip(),
|
||
"primary_action_code": primary_action_code,
|
||
"secondary_action_code": secondary_action_code,
|
||
"contract_keys": contract_keys,
|
||
"contract_navigation": _build_ops_contract_navigation(
|
||
contract_keys,
|
||
primary_contract_key="ops_driver_contract",
|
||
),
|
||
"autopilot": {
|
||
"status_text": status_text,
|
||
"automation_level": automation_level,
|
||
"recommendation": recommendation,
|
||
"executor_mode_hint": executor_mode_hint,
|
||
"confirm_required": bool(primary_execution.get("confirm_required", False)),
|
||
"ui_only": bool(primary_execution.get("ui_only", False)),
|
||
"blocked": bool(primary_execution.get("blocked", False)),
|
||
"backend_handled": bool(primary_execution.get("backend_handled", False)),
|
||
"risk_level": str(primary_execution.get("risk_level") or "unknown").strip() or "unknown",
|
||
"reason": str(primary_execution.get("reason") or "").strip(),
|
||
"target_method": target_method,
|
||
"target_api": target_api,
|
||
},
|
||
"primary_execution": {
|
||
**primary_execution,
|
||
"label": str(normalized_entry.get("primary_label") or "").strip() or primary_action_code,
|
||
"node_codes": primary_node_codes,
|
||
"action_payload": primary_action_payload,
|
||
"request_payload_preview": primary_request,
|
||
},
|
||
"secondary_execution": (
|
||
{
|
||
**secondary_execution,
|
||
"label": str(normalized_entry.get("secondary_label") or "").strip() or secondary_action_code,
|
||
"node_codes": secondary_node_codes,
|
||
"action_payload": secondary_action_payload,
|
||
"request_payload_preview": secondary_request,
|
||
}
|
||
if secondary_action_code
|
||
else {}
|
||
),
|
||
}
|
||
|
||
|
||
def _build_codex_activity_focus_entry(entry: dict) -> dict:
|
||
normalized_entry = dict(entry or {})
|
||
if not normalized_entry:
|
||
return {}
|
||
contract_keys = _suggest_ops_contract_keys_for_codex_entry(normalized_entry)
|
||
return {
|
||
"key": str(normalized_entry.get("key") or "").strip(),
|
||
"activity_key": str(normalized_entry.get("activity_key") or "").strip(),
|
||
"activity_kind": str(normalized_entry.get("activity_kind") or "").strip(),
|
||
"title": str(normalized_entry.get("title") or "").strip(),
|
||
"subtitle": str(normalized_entry.get("subtitle") or "").strip(),
|
||
"summary": str(normalized_entry.get("summary") or "").strip(),
|
||
"status": str(normalized_entry.get("status") or "").strip(),
|
||
"status_label": str(normalized_entry.get("status_label") or "").strip(),
|
||
"meta_text": str(normalized_entry.get("meta_text") or "").strip(),
|
||
"occurred_at": str(normalized_entry.get("occurred_at") or "").strip(),
|
||
"target_node_codes": _normalize_driver_node_codes(normalized_entry.get("target_node_codes") or []),
|
||
"focus_ref": _normalize_focus_ref(normalized_entry.get("focus_ref")),
|
||
"source_focus_ref": _normalize_focus_ref(normalized_entry.get("source_focus_ref")),
|
||
"ui_intent": dict(normalized_entry.get("ui_intent") or {}),
|
||
"ui_intent_kind": str(normalized_entry.get("ui_intent_kind") or "").strip(),
|
||
"focus_action_code": str(normalized_entry.get("focus_action_code") or "").strip(),
|
||
"focus_action_payload": dict(normalized_entry.get("focus_action_payload") or {}),
|
||
"contract_keys": contract_keys,
|
||
"contract_navigation": _build_ops_contract_navigation(
|
||
contract_keys,
|
||
primary_contract_key="ops_driver_contract",
|
||
),
|
||
"observation_only": True,
|
||
}
|
||
|
||
|
||
def _build_codex_action_entry_from_activity_focus(entry: dict) -> dict:
|
||
normalized_entry = dict(entry or {})
|
||
if not normalized_entry:
|
||
return {}
|
||
|
||
key = str(normalized_entry.get("key") or "").strip()
|
||
focus_action_code = str(normalized_entry.get("focus_action_code") or "").strip()
|
||
focus_ref = _normalize_focus_ref(normalized_entry.get("focus_ref"))
|
||
target_node_codes = _normalize_driver_node_codes(normalized_entry.get("target_node_codes") or [])
|
||
focus_action_payload = dict(normalized_entry.get("focus_action_payload") or {})
|
||
status_text = str(normalized_entry.get("status_label") or normalized_entry.get("status") or "").strip()
|
||
summary = str(normalized_entry.get("summary") or "").strip()
|
||
|
||
if not key or not focus_action_code:
|
||
return {}
|
||
|
||
reason = summary or "该焦点属于运行态观察项,建议先进入对应工作区查看上下文。"
|
||
return {
|
||
"key": key,
|
||
"title": str(normalized_entry.get("title") or "").strip() or key,
|
||
"summary": summary,
|
||
"reason": reason,
|
||
"meta_text": str(normalized_entry.get("meta_text") or "").strip(),
|
||
"executor_kind": "driver_action",
|
||
"sequence_key": "",
|
||
"focus_ref": focus_ref,
|
||
"primary_focus_ref": focus_ref,
|
||
"secondary_focus_ref": {},
|
||
"primary_label": "进入工作区",
|
||
"secondary_label": "",
|
||
"primary_action_code": focus_action_code,
|
||
"secondary_action_code": "",
|
||
"autopilot": {
|
||
"status_text": status_text,
|
||
"automation_level": "ui_only",
|
||
"recommendation": "open_ui",
|
||
"executor_mode_hint": "ui-intent",
|
||
"confirm_required": False,
|
||
"ui_only": True,
|
||
"blocked": False,
|
||
"backend_handled": True,
|
||
"risk_level": "low",
|
||
"reason": reason,
|
||
"target_method": "POST",
|
||
"target_api": "/api/v1/ops/driver-actions/execute",
|
||
},
|
||
"primary_execution": {
|
||
"label": "进入工作区",
|
||
"node_codes": target_node_codes,
|
||
"action_payload": focus_action_payload,
|
||
"request_payload_preview": {
|
||
"action_code": focus_action_code,
|
||
"node_codes": target_node_codes,
|
||
"action_payload": focus_action_payload,
|
||
},
|
||
},
|
||
"secondary_execution": {},
|
||
"activity_key": str(normalized_entry.get("activity_key") or "").strip(),
|
||
"activity_kind": str(normalized_entry.get("activity_kind") or "").strip(),
|
||
"status": str(normalized_entry.get("status") or "").strip(),
|
||
"status_label": status_text,
|
||
"occurred_at": str(normalized_entry.get("occurred_at") or "").strip(),
|
||
"target_node_codes": target_node_codes,
|
||
"source_focus_ref": _normalize_focus_ref(normalized_entry.get("source_focus_ref")),
|
||
"ui_intent": dict(normalized_entry.get("ui_intent") or {}),
|
||
"ui_intent_kind": str(normalized_entry.get("ui_intent_kind") or "").strip(),
|
||
"focus_action_code": focus_action_code,
|
||
"focus_action_payload": focus_action_payload,
|
||
"observation_only": True,
|
||
}
|
||
|
||
|
||
def _is_preview_only_driver_action(action_code: str) -> bool:
|
||
normalized_action_code = str(action_code or "").strip()
|
||
if not normalized_action_code:
|
||
return False
|
||
if normalized_action_code.startswith(("open_", "focus_", "view_")):
|
||
return True
|
||
return normalized_action_code in {
|
||
"release_package",
|
||
"handover_first_gap",
|
||
"review_smart_rollout_preview",
|
||
"review_control_rollout",
|
||
"fix_rollout_blockers",
|
||
}
|
||
|
||
|
||
def _build_ops_automation_coverage_summary(
|
||
entries: list[dict] | None = None,
|
||
*,
|
||
activity_focus_total: int = 0,
|
||
scene_log_observation: dict | None = None,
|
||
go_live_summary: dict | None = None,
|
||
) -> dict:
|
||
normalized_entries = [dict(item or {}) for item in list(entries or []) if dict(item or {})]
|
||
normalized_scene_log_observation = dict(scene_log_observation or {})
|
||
normalized_go_live_summary = dict(go_live_summary or {})
|
||
|
||
automation_level_counts: dict[str, int] = {}
|
||
recommendation_counts: dict[str, int] = {}
|
||
executor_kind_counts: dict[str, int] = {}
|
||
backend_handled_total = 0
|
||
preview_only_total = 0
|
||
|
||
for item in normalized_entries:
|
||
autopilot = dict(item.get("autopilot") or {})
|
||
automation_level = str(autopilot.get("automation_level") or "unknown").strip() or "unknown"
|
||
recommendation = str(autopilot.get("recommendation") or "unknown").strip() or "unknown"
|
||
executor_kind = str(item.get("executor_kind") or "unknown").strip() or "unknown"
|
||
action_code = str(item.get("primary_action_code") or "").strip()
|
||
if bool(autopilot.get("backend_handled", False)):
|
||
backend_handled_total += 1
|
||
if _is_preview_only_driver_action(action_code):
|
||
preview_only_total += 1
|
||
automation_level_counts[automation_level] = int(automation_level_counts.get(automation_level, 0) or 0) + 1
|
||
recommendation_counts[recommendation] = int(recommendation_counts.get(recommendation, 0) or 0) + 1
|
||
executor_kind_counts[executor_kind] = int(executor_kind_counts.get(executor_kind, 0) or 0) + 1
|
||
|
||
total = len(normalized_entries)
|
||
safe_auto_total = int(automation_level_counts.get("safe_auto", 0) or 0)
|
||
guarded_auto_total = int(automation_level_counts.get("guarded_auto", 0) or 0)
|
||
mixed_total = int(automation_level_counts.get("mixed", 0) or 0)
|
||
ui_only_total = int(automation_level_counts.get("ui_only", 0) or 0)
|
||
blocked_total = int(automation_level_counts.get("blocked", 0) or 0)
|
||
unknown_total = max(total - safe_auto_total - guarded_auto_total - mixed_total - ui_only_total - blocked_total, 0)
|
||
auto_execute_total = int(recommendation_counts.get("auto_execute", 0) or 0)
|
||
confirm_then_execute_total = int(recommendation_counts.get("confirm_then_execute", 0) or 0)
|
||
resolve_first_total = int(recommendation_counts.get("resolve_first", 0) or 0)
|
||
open_ui_total = int(recommendation_counts.get("open_ui", 0) or 0)
|
||
blocked_recommendation_total = int(recommendation_counts.get("blocked", 0) or 0)
|
||
execution_ready_total = auto_execute_total + confirm_then_execute_total
|
||
human_dependency_total = resolve_first_total + open_ui_total + blocked_recommendation_total
|
||
coverage_ratio = round((backend_handled_total / total), 4) if total > 0 else 1.0
|
||
|
||
launch_status = "ready"
|
||
go_live_status = str(normalized_go_live_summary.get("go_live_status") or "").strip()
|
||
publish_ready = bool(normalized_go_live_summary.get("publish_ready", False))
|
||
scene_log_status = str(normalized_scene_log_observation.get("status") or "").strip() or "standby"
|
||
if blocked_total > 0 or blocked_recommendation_total > 0 or go_live_status == "blocked":
|
||
launch_status = "blocked"
|
||
elif (
|
||
ui_only_total > 0
|
||
or resolve_first_total > 0
|
||
or go_live_status not in {"", "ready"}
|
||
or not publish_ready
|
||
):
|
||
launch_status = "attention"
|
||
|
||
return {
|
||
"total": total,
|
||
"automation_level_counts": automation_level_counts,
|
||
"recommendation_counts": recommendation_counts,
|
||
"executor_kind_counts": executor_kind_counts,
|
||
"safe_auto_total": safe_auto_total,
|
||
"guarded_auto_total": guarded_auto_total,
|
||
"mixed_total": mixed_total,
|
||
"ui_only_total": ui_only_total,
|
||
"blocked_total": blocked_total,
|
||
"unknown_total": unknown_total,
|
||
"auto_execute_total": auto_execute_total,
|
||
"confirm_then_execute_total": confirm_then_execute_total,
|
||
"resolve_first_total": resolve_first_total,
|
||
"open_ui_total": open_ui_total,
|
||
"blocked_recommendation_total": blocked_recommendation_total,
|
||
"backend_handled_total": backend_handled_total,
|
||
"preview_only_total": preview_only_total,
|
||
"execution_ready_total": execution_ready_total,
|
||
"human_dependency_total": human_dependency_total,
|
||
"activity_focus_total": int(activity_focus_total or 0),
|
||
"scene_log_status": scene_log_status,
|
||
"launch_status": launch_status,
|
||
"launch_ready": launch_status == "ready",
|
||
"backend_coverage_ratio": coverage_ratio,
|
||
}
|
||
|
||
|
||
def get_ops_codex_brief() -> dict:
|
||
driver_feed = get_ops_driver_feed()
|
||
go_live_summary = dict(driver_feed.get("go_live_summary") or {})
|
||
contract_registry = get_ops_contract_registry()
|
||
scene_log_observation = dict(driver_feed.get("scene_log_observation") or {})
|
||
activity_focus_entries = [
|
||
_build_codex_activity_focus_entry(raw_entry)
|
||
for raw_entry in list(driver_feed.get("activity_focus") or [])
|
||
if dict(raw_entry or {})
|
||
]
|
||
activity_focus_entries = [entry for entry in activity_focus_entries if entry]
|
||
entries = [
|
||
_build_codex_brief_entry(raw_entry)
|
||
for raw_entry in list(driver_feed.get("entries") or [])
|
||
if dict(raw_entry or {})
|
||
]
|
||
entries = [entry for entry in entries if entry]
|
||
automation_coverage = _build_ops_automation_coverage_summary(
|
||
entries,
|
||
activity_focus_total=len(activity_focus_entries),
|
||
scene_log_observation=scene_log_observation,
|
||
go_live_summary=go_live_summary,
|
||
)
|
||
|
||
focus_entry = dict(entries[0] or {}) if entries else {}
|
||
focus_autopilot = dict(focus_entry.get("autopilot") or {})
|
||
focus_payload = {
|
||
"entry_key": str(focus_entry.get("key") or "").strip(),
|
||
"title": str(focus_entry.get("title") or "").strip(),
|
||
"recommendation": str(focus_autopilot.get("recommendation") or "").strip(),
|
||
"status_text": str(focus_autopilot.get("status_text") or "").strip(),
|
||
"target_api": str(focus_autopilot.get("target_api") or "").strip(),
|
||
"target_method": str(focus_autopilot.get("target_method") or "").strip(),
|
||
"focus_ref": _normalize_focus_ref(focus_entry.get("focus_ref")),
|
||
"primary_focus_ref": _normalize_focus_ref(focus_entry.get("primary_focus_ref")),
|
||
"secondary_focus_ref": _normalize_focus_ref(focus_entry.get("secondary_focus_ref")),
|
||
}
|
||
if not focus_payload["entry_key"]:
|
||
first_activity_focus = next(
|
||
(
|
||
dict(item or {})
|
||
for item in activity_focus_entries
|
||
if str((item or {}).get("focus_action_code") or "").strip()
|
||
),
|
||
{},
|
||
)
|
||
else:
|
||
first_activity_focus = {}
|
||
if not focus_payload["entry_key"] and first_activity_focus:
|
||
activity_focus_contract_keys = _suggest_ops_contract_keys_for_codex_entry(first_activity_focus)
|
||
focus_payload = {
|
||
"entry_key": str(first_activity_focus.get("key") or "").strip(),
|
||
"title": str(first_activity_focus.get("title") or "").strip(),
|
||
"recommendation": "open_ui" if str(first_activity_focus.get("ui_intent_kind") or "").strip() else "",
|
||
"status_text": str(first_activity_focus.get("status_label") or first_activity_focus.get("status") or "").strip(),
|
||
"target_api": "",
|
||
"target_method": "",
|
||
"focus_ref": _normalize_focus_ref(first_activity_focus.get("focus_ref")),
|
||
"primary_focus_ref": _normalize_focus_ref(first_activity_focus.get("focus_ref")),
|
||
"secondary_focus_ref": {},
|
||
"contract_keys": activity_focus_contract_keys,
|
||
"contract_navigation": _build_ops_contract_navigation(
|
||
activity_focus_contract_keys,
|
||
primary_contract_key="ops_driver_contract",
|
||
registry=contract_registry,
|
||
),
|
||
}
|
||
elif (
|
||
not focus_payload["entry_key"]
|
||
and _normalize_focus_ref(scene_log_observation.get("focus_ref"))
|
||
):
|
||
scene_log_contract_keys = _normalize_ops_contract_keys(
|
||
list(scene_log_observation.get("contract_keys") or [])
|
||
+ ["ops_driver_contract"]
|
||
)
|
||
focus_payload = {
|
||
"entry_key": "scene-log-observation",
|
||
"title": "现场日志观察",
|
||
"recommendation": "open_ui",
|
||
"status_text": str(scene_log_observation.get("status_label") or scene_log_observation.get("status") or "").strip(),
|
||
"target_api": "",
|
||
"target_method": "",
|
||
"focus_ref": _normalize_focus_ref(scene_log_observation.get("focus_ref")),
|
||
"primary_focus_ref": _normalize_focus_ref(scene_log_observation.get("focus_ref")),
|
||
"secondary_focus_ref": {},
|
||
"contract_keys": scene_log_contract_keys,
|
||
"contract_navigation": _build_ops_contract_navigation(
|
||
scene_log_contract_keys,
|
||
primary_contract_key="ops_driver_contract",
|
||
registry=contract_registry,
|
||
),
|
||
}
|
||
elif focus_payload["entry_key"]:
|
||
focus_contract_keys = _normalize_ops_contract_keys(
|
||
(focus_entry.get("contract_keys") or [])
|
||
+ _suggest_ops_contract_keys_for_codex_entry(focus_entry)
|
||
)
|
||
focus_payload["contract_keys"] = focus_contract_keys
|
||
focus_payload["contract_navigation"] = _build_ops_contract_navigation(
|
||
focus_contract_keys,
|
||
primary_contract_key="ops_driver_contract",
|
||
registry=contract_registry,
|
||
)
|
||
else:
|
||
focus_payload["contract_keys"] = ["ops_driver_contract"]
|
||
focus_payload["contract_navigation"] = _build_ops_contract_navigation(
|
||
["ops_driver_contract"],
|
||
primary_contract_key="ops_driver_contract",
|
||
registry=contract_registry,
|
||
)
|
||
|
||
summary_contract_keys = _normalize_ops_contract_keys(
|
||
["ops_driver_contract", "ops_stack_diagnosis_contract"]
|
||
+ [
|
||
contract_key
|
||
for item in entries
|
||
for contract_key in list(item.get("contract_keys") or [])
|
||
]
|
||
+ [
|
||
contract_key
|
||
for contract_key in list(focus_payload.get("contract_keys") or [])
|
||
]
|
||
)
|
||
|
||
return {
|
||
"generated_at": datetime.now().isoformat(timespec="seconds"),
|
||
"headline": str(driver_feed.get("headline") or "").strip() or "当前还没有可执行的驾驶主线。",
|
||
"driver_feed": driver_feed,
|
||
"go_live_summary": go_live_summary,
|
||
"scene_log_observation": scene_log_observation,
|
||
"focus": focus_payload,
|
||
"entries": entries,
|
||
"activity_focus": activity_focus_entries,
|
||
"automation_coverage": automation_coverage,
|
||
"contract_navigation": _build_ops_contract_navigation(
|
||
summary_contract_keys,
|
||
primary_contract_key="ops_driver_contract",
|
||
registry=contract_registry,
|
||
),
|
||
"summary": {
|
||
"total": len(entries),
|
||
"automation_level_counts": dict(automation_coverage.get("automation_level_counts") or {}),
|
||
"recommendation_counts": dict(automation_coverage.get("recommendation_counts") or {}),
|
||
"executor_kind_counts": dict(automation_coverage.get("executor_kind_counts") or {}),
|
||
"go_live_status": str(go_live_summary.get("go_live_status") or "").strip(),
|
||
"operator_title": str(go_live_summary.get("operator_title") or "").strip(),
|
||
"publish_ready": bool(go_live_summary.get("publish_ready")),
|
||
"publish_status": str(go_live_summary.get("publish_status") or "").strip(),
|
||
"publish_status_label": str(go_live_summary.get("publish_status_label") or "").strip(),
|
||
"launchpad_recommended_target_node_code": str(
|
||
go_live_summary.get("launchpad_recommended_target_node_code") or ""
|
||
).strip(),
|
||
"launchpad_recommended_recovery_label": str(
|
||
go_live_summary.get("launchpad_recommended_recovery_label") or ""
|
||
).strip(),
|
||
"launchpad_recommended_recovery_summary": str(
|
||
go_live_summary.get("launchpad_recommended_recovery_summary") or ""
|
||
).strip(),
|
||
"launchpad_onboarding_bootstrap_pending_nodes": int(
|
||
go_live_summary.get("launchpad_onboarding_bootstrap_pending_nodes", 0) or 0
|
||
),
|
||
"launchpad_onboarding_acceptance_ready_nodes": int(
|
||
go_live_summary.get("launchpad_onboarding_acceptance_ready_nodes", 0) or 0
|
||
),
|
||
"auto_execute_total": int(automation_coverage.get("auto_execute_total", 0) or 0),
|
||
"confirm_then_execute_total": int(automation_coverage.get("confirm_then_execute_total", 0) or 0),
|
||
"resolve_first_total": int(automation_coverage.get("resolve_first_total", 0) or 0),
|
||
"open_ui_total": int(automation_coverage.get("open_ui_total", 0) or 0),
|
||
"blocked_total": int(automation_coverage.get("blocked_recommendation_total", 0) or 0),
|
||
"activity_focus_total": int(automation_coverage.get("activity_focus_total", 0) or 0),
|
||
"preview_only_total": int(automation_coverage.get("preview_only_total", 0) or 0),
|
||
"backend_handled_total": int(automation_coverage.get("backend_handled_total", 0) or 0),
|
||
"launch_status": str(automation_coverage.get("launch_status") or "").strip(),
|
||
"launch_ready": bool(automation_coverage.get("launch_ready", False)),
|
||
},
|
||
}
|
||
|
||
|
||
def _build_resolved_driver_action_gate(
|
||
preview_data: dict,
|
||
*,
|
||
secondary: bool = False,
|
||
confirm: bool = False,
|
||
) -> dict:
|
||
normalized_preview = dict(preview_data or {})
|
||
preview_section = dict((normalized_preview.get("secondary") if secondary else normalized_preview.get("primary")) or {})
|
||
preview_execution = dict(preview_section.get("execution_profile") or {})
|
||
|
||
recommendation = str(preview_execution.get("recommendation") or "blocked").strip() or "blocked"
|
||
automation_level = str(preview_execution.get("automation_level") or "blocked").strip() or "blocked"
|
||
executor_mode_hint = str(preview_execution.get("executor_mode_hint") or "").strip()
|
||
risk_level = str(preview_execution.get("risk_level") or "unknown").strip() or "unknown"
|
||
confirm_required = bool(preview_execution.get("confirm_required", False))
|
||
ui_only = bool(preview_execution.get("ui_only", False))
|
||
blocked = bool(preview_execution.get("blocked", False))
|
||
reason = str(preview_execution.get("reason") or "").strip()
|
||
target_api = str(preview_section.get("target_api") or "").strip()
|
||
target_method = str(preview_section.get("target_method") or "POST").strip() or "POST"
|
||
request_payload = dict(preview_section.get("request_payload") or {})
|
||
confirm_received = bool(confirm)
|
||
|
||
decision = "blocked"
|
||
decision_reason = reason or "当前动作被标记为阻断,不能自动执行。"
|
||
will_execute = False
|
||
if recommendation == "auto_execute":
|
||
decision = "execute_now"
|
||
will_execute = True
|
||
decision_reason = reason or "当前动作属于 safe_auto,可直接执行。"
|
||
elif recommendation == "confirm_then_execute":
|
||
if confirm_received:
|
||
decision = "execute_confirmed"
|
||
will_execute = True
|
||
decision_reason = reason or "当前动作属于 guarded_auto,已收到确认,允许执行。"
|
||
else:
|
||
decision = "confirmation_required"
|
||
decision_reason = "当前动作属于 guarded_auto,需要显式确认后才能执行。"
|
||
if reason:
|
||
decision_reason = f"{decision_reason} {reason}"
|
||
elif recommendation == "resolve_first":
|
||
decision = "resolve_first"
|
||
decision_reason = reason or "当前动作要求先复核解析结果,不应直接执行。"
|
||
elif recommendation == "open_ui":
|
||
decision = "open_ui"
|
||
decision_reason = reason or "当前动作属于 UI-only,建议进入工作区处理。"
|
||
|
||
decision_label = {
|
||
"execute_now": "直接执行",
|
||
"execute_confirmed": "确认后执行",
|
||
"confirmation_required": "等待确认",
|
||
"resolve_first": "先复核",
|
||
"open_ui": "进入工作区",
|
||
"blocked": "当前阻断",
|
||
}.get(decision, decision)
|
||
|
||
return {
|
||
"decision": decision,
|
||
"decision_label": decision_label,
|
||
"decision_reason": decision_reason,
|
||
"will_execute": will_execute,
|
||
"recommendation": recommendation,
|
||
"recommendation_label": _driver_action_recommendation_label(recommendation),
|
||
"automation_level": automation_level,
|
||
"status_text": "",
|
||
"executor_mode_hint": executor_mode_hint,
|
||
"risk_level": risk_level,
|
||
"confirm_required": confirm_required,
|
||
"confirm_received": confirm_received,
|
||
"ui_only": ui_only,
|
||
"blocked": blocked,
|
||
"secondary": bool(secondary),
|
||
"backend_handled": bool(preview_execution.get("backend_handled", False)),
|
||
"target_api": target_api,
|
||
"target_method": target_method,
|
||
"request_payload": request_payload,
|
||
"focus_ref": _normalize_focus_ref(
|
||
preview_section.get("focus_ref")
|
||
or normalized_preview.get("secondary_focus_ref" if secondary else "primary_focus_ref")
|
||
or normalized_preview.get("focus_ref")
|
||
),
|
||
"action_code": str(preview_section.get("action_code") or "").strip(),
|
||
"sequence_key": str(normalized_preview.get("sequence_key") or "").strip(),
|
||
"label": str(preview_section.get("label") or "").strip(),
|
||
"node_codes": _normalize_driver_node_codes(preview_section.get("node_codes") or []),
|
||
"requires_fields": list(preview_execution.get("requires_fields") or []),
|
||
"missing_fields": list(preview_execution.get("missing_fields") or []),
|
||
}
|
||
|
||
|
||
def resolve_driver_action_request(payload: dict | None = None) -> tuple[bool, str, dict]:
|
||
normalized_payload = dict(payload or {})
|
||
secondary = bool(normalized_payload.get("secondary", False))
|
||
confirm = bool(normalized_payload.get("confirm", False))
|
||
|
||
preview_request = {
|
||
"source_label": str(normalized_payload.get("source_label") or "ops/driver-action").strip() or "ops/driver-action",
|
||
"title": str(normalized_payload.get("title") or "").strip(),
|
||
"summary": str(normalized_payload.get("summary") or "").strip(),
|
||
"reason": str(normalized_payload.get("reason") or "").strip(),
|
||
"meta_text": str(normalized_payload.get("meta_text") or "").strip(),
|
||
"executor_kind": str(normalized_payload.get("executor_kind") or "driver_action").strip() or "driver_action",
|
||
"sequence_key": str(normalized_payload.get("sequence_key") or "").strip(),
|
||
"focus_ref": _normalize_focus_ref(normalized_payload.get("focus_ref")),
|
||
"primary_focus_ref": _normalize_focus_ref(normalized_payload.get("primary_focus_ref")),
|
||
"secondary_focus_ref": _normalize_focus_ref(normalized_payload.get("secondary_focus_ref")),
|
||
"primary_label": str(normalized_payload.get("primary_label") or "").strip(),
|
||
"secondary_label": str(normalized_payload.get("secondary_label") or "").strip(),
|
||
"primary_action_code": str(normalized_payload.get("primary_action_code") or "").strip(),
|
||
"secondary_action_code": str(normalized_payload.get("secondary_action_code") or "").strip(),
|
||
"primary_node_codes": _normalize_driver_node_codes(normalized_payload.get("primary_node_codes") or normalized_payload.get("node_codes") or []),
|
||
"secondary_node_codes": _normalize_driver_node_codes(normalized_payload.get("secondary_node_codes") or normalized_payload.get("node_codes") or []),
|
||
"primary_action_payload": dict(normalized_payload.get("primary_action_payload") or normalized_payload.get("action_payload") or {}),
|
||
"secondary_action_payload": dict(normalized_payload.get("secondary_action_payload") or normalized_payload.get("action_payload") or {}),
|
||
"include_secondary": bool(normalized_payload.get("include_secondary", True)),
|
||
"requested_by": str(normalized_payload.get("requested_by") or "api/driver-action").strip() or "api/driver-action",
|
||
}
|
||
preview_ok, preview_message, preview_data = preview_driver_action(preview_request)
|
||
if not preview_ok:
|
||
return False, preview_message, {
|
||
"preview_request": preview_request,
|
||
"preview": dict(preview_data or {}),
|
||
"selected_section": "secondary" if secondary else "primary",
|
||
}
|
||
|
||
gate = _build_resolved_driver_action_gate(preview_data, secondary=secondary, confirm=confirm)
|
||
return True, "驾驶动作解析完成", {
|
||
"preview_request": preview_request,
|
||
"preview": dict(preview_data or {}),
|
||
"gate": gate,
|
||
"selected_section": "secondary" if secondary else "primary",
|
||
"focus_ref": _normalize_focus_ref(
|
||
gate.get("focus_ref")
|
||
or preview_data.get("secondary_focus_ref" if secondary else "primary_focus_ref")
|
||
or preview_data.get("focus_ref")
|
||
),
|
||
}
|
||
|
||
|
||
def execute_resolved_driver_action(payload: dict | None = None) -> tuple[bool, str, dict]:
|
||
resolve_ok, resolve_message, resolved = resolve_driver_action_request(payload or {})
|
||
if not resolve_ok:
|
||
return False, resolve_message, dict(resolved or {})
|
||
|
||
gate = dict((resolved or {}).get("gate") or {})
|
||
if not bool(gate.get("will_execute", False)):
|
||
return False, str(gate.get("decision_reason") or "当前不允许自动执行").strip() or "当前不允许自动执行", {
|
||
**dict(resolved or {}),
|
||
"executed": False,
|
||
}
|
||
|
||
preview = dict((resolved or {}).get("preview") or {})
|
||
executor_kind = str(preview.get("executor_kind") or "driver_action").strip() or "driver_action"
|
||
request_payload = dict(gate.get("request_payload") or {})
|
||
|
||
if executor_kind == "runbook_sequence":
|
||
sequence_key = str(gate.get("sequence_key") or preview.get("sequence_key") or "").strip()
|
||
if not sequence_key:
|
||
return False, "缺少 sequence_key,无法执行 runbook sequence", {
|
||
**dict(resolved or {}),
|
||
"executed": False,
|
||
}
|
||
execute_ok, execute_message, execute_data = execute_ops_runbook_sequence(sequence_key, request_payload)
|
||
else:
|
||
execute_ok, execute_message, execute_data = execute_driver_action(request_payload)
|
||
|
||
return execute_ok, execute_message, {
|
||
**dict(resolved or {}),
|
||
"executed": bool(execute_ok),
|
||
"execution_result": dict(execute_data or {}),
|
||
}
|
||
|
||
|
||
def _select_codex_brief_entry(codex_brief: dict, entry_key: str = "") -> dict:
|
||
normalized_entry_key = str(entry_key or "").strip()
|
||
entries = [dict(item or {}) for item in list((codex_brief or {}).get("entries") or []) if dict(item or {})]
|
||
activity_focus_entries = [
|
||
_build_codex_action_entry_from_activity_focus(item)
|
||
for item in list((codex_brief or {}).get("activity_focus") or [])
|
||
if dict(item or {})
|
||
]
|
||
activity_focus_entries = [item for item in activity_focus_entries if item]
|
||
focus = dict((codex_brief or {}).get("focus") or {})
|
||
target_entry_key = normalized_entry_key or str(focus.get("entry_key") or "").strip()
|
||
|
||
if target_entry_key:
|
||
for item in entries + activity_focus_entries:
|
||
if str(item.get("key") or "").strip() == target_entry_key:
|
||
return item
|
||
searchable_entries = entries + activity_focus_entries
|
||
return dict(searchable_entries[0] or {}) if searchable_entries else {}
|
||
|
||
|
||
def _build_codex_action_preview_request(
|
||
selected_entry: dict,
|
||
*,
|
||
requested_by: str,
|
||
source_label: str,
|
||
include_secondary: bool = True,
|
||
) -> dict:
|
||
normalized_entry = dict(selected_entry or {})
|
||
autopilot = dict(normalized_entry.get("autopilot") or {})
|
||
primary_execution = dict(normalized_entry.get("primary_execution") or {})
|
||
secondary_execution = dict(normalized_entry.get("secondary_execution") or {})
|
||
requested_by_value = str(requested_by or "api/codex-action").strip() or "api/codex-action"
|
||
source_label_value = str(source_label or "ops/codex-action").strip() or "ops/codex-action"
|
||
|
||
return {
|
||
"source_label": source_label_value,
|
||
"title": str(normalized_entry.get("title") or "").strip() or str(normalized_entry.get("key") or "").strip() or "codex-action",
|
||
"summary": str(normalized_entry.get("summary") or "").strip(),
|
||
"reason": str(normalized_entry.get("reason") or autopilot.get("reason") or "").strip(),
|
||
"meta_text": str(normalized_entry.get("meta_text") or "").strip(),
|
||
"executor_kind": str(normalized_entry.get("executor_kind") or "driver_action").strip() or "driver_action",
|
||
"sequence_key": str(normalized_entry.get("sequence_key") or "").strip(),
|
||
"focus_ref": _normalize_focus_ref(normalized_entry.get("focus_ref")),
|
||
"primary_focus_ref": _normalize_focus_ref(normalized_entry.get("primary_focus_ref")),
|
||
"secondary_focus_ref": _normalize_focus_ref(normalized_entry.get("secondary_focus_ref")),
|
||
"primary_label": str(primary_execution.get("label") or normalized_entry.get("primary_label") or "").strip(),
|
||
"secondary_label": str(secondary_execution.get("label") or normalized_entry.get("secondary_label") or "").strip(),
|
||
"primary_action_code": str(normalized_entry.get("primary_action_code") or "").strip(),
|
||
"secondary_action_code": str(normalized_entry.get("secondary_action_code") or "").strip(),
|
||
"primary_node_codes": _normalize_driver_node_codes(
|
||
primary_execution.get("node_codes") or normalized_entry.get("node_codes") or []
|
||
),
|
||
"secondary_node_codes": _normalize_driver_node_codes(
|
||
secondary_execution.get("node_codes") or normalized_entry.get("node_codes") or []
|
||
),
|
||
"primary_action_payload": dict(primary_execution.get("action_payload") or {}),
|
||
"secondary_action_payload": dict(secondary_execution.get("action_payload") or {}),
|
||
"include_secondary": bool(include_secondary),
|
||
"requested_by": requested_by_value,
|
||
}
|
||
|
||
|
||
def _build_codex_action_gate(
|
||
selected_entry: dict,
|
||
preview_data: dict,
|
||
*,
|
||
secondary: bool = False,
|
||
confirm: bool = False,
|
||
) -> dict:
|
||
normalized_entry = dict(selected_entry or {})
|
||
normalized_preview = dict(preview_data or {})
|
||
autopilot = dict(normalized_entry.get("autopilot") or {})
|
||
preview_section = dict((normalized_preview.get("secondary") if secondary else normalized_preview.get("primary")) or {})
|
||
execution_key = "secondary_execution" if secondary else "primary_execution"
|
||
entry_execution = dict(normalized_entry.get(execution_key) or {})
|
||
preview_execution = dict(preview_section.get("execution_profile") or {})
|
||
|
||
recommendation = str(
|
||
preview_execution.get("recommendation")
|
||
or autopilot.get("recommendation")
|
||
or "blocked"
|
||
).strip() or "blocked"
|
||
automation_level = str(
|
||
preview_execution.get("automation_level")
|
||
or autopilot.get("automation_level")
|
||
or "blocked"
|
||
).strip() or "blocked"
|
||
executor_mode_hint = str(
|
||
preview_execution.get("executor_mode_hint")
|
||
or autopilot.get("executor_mode_hint")
|
||
or ""
|
||
).strip()
|
||
risk_level = str(preview_execution.get("risk_level") or autopilot.get("risk_level") or "unknown").strip() or "unknown"
|
||
confirm_required = bool(preview_execution.get("confirm_required", False) or autopilot.get("confirm_required", False))
|
||
ui_only = bool(preview_execution.get("ui_only", False) or autopilot.get("ui_only", False))
|
||
blocked = bool(preview_execution.get("blocked", False) or autopilot.get("blocked", False))
|
||
reason = str(preview_execution.get("reason") or autopilot.get("reason") or "").strip()
|
||
target_api = str(preview_section.get("target_api") or autopilot.get("target_api") or "").strip()
|
||
target_method = str(preview_section.get("target_method") or autopilot.get("target_method") or "POST").strip() or "POST"
|
||
request_payload = dict(preview_section.get("request_payload") or {})
|
||
confirm_received = bool(confirm)
|
||
|
||
decision = "blocked"
|
||
decision_reason = reason or "当前焦点被标记为阻断,不能自动执行。"
|
||
will_execute = False
|
||
if recommendation == "auto_execute":
|
||
decision = "execute_now"
|
||
will_execute = True
|
||
decision_reason = reason or "当前焦点属于 safe_auto,可直接执行。"
|
||
elif recommendation == "confirm_then_execute":
|
||
if confirm_received:
|
||
decision = "execute_confirmed"
|
||
will_execute = True
|
||
decision_reason = reason or "当前焦点属于 guarded_auto,已收到确认,允许执行。"
|
||
else:
|
||
decision = "confirmation_required"
|
||
decision_reason = "当前焦点属于 guarded_auto,需要显式确认后才能执行。"
|
||
if reason:
|
||
decision_reason = f"{decision_reason} {reason}"
|
||
elif recommendation == "resolve_first":
|
||
decision = "resolve_first"
|
||
decision_reason = reason or "当前焦点要求先复核解析结果,不应直接执行。"
|
||
elif recommendation == "open_ui":
|
||
decision = "open_ui"
|
||
decision_reason = reason or "当前焦点属于 UI-only,建议进入工作区处理。"
|
||
|
||
decision_label = {
|
||
"execute_now": "直接执行",
|
||
"execute_confirmed": "确认后执行",
|
||
"confirmation_required": "等待确认",
|
||
"resolve_first": "先复核",
|
||
"open_ui": "进入工作区",
|
||
"blocked": "当前阻断",
|
||
}.get(decision, decision)
|
||
|
||
return {
|
||
"decision": decision,
|
||
"decision_label": decision_label,
|
||
"decision_reason": decision_reason,
|
||
"will_execute": will_execute,
|
||
"recommendation": recommendation,
|
||
"recommendation_label": _driver_action_recommendation_label(recommendation),
|
||
"automation_level": automation_level,
|
||
"status_text": str(autopilot.get("status_text") or "").strip(),
|
||
"executor_mode_hint": executor_mode_hint,
|
||
"risk_level": risk_level,
|
||
"confirm_required": confirm_required,
|
||
"confirm_received": confirm_received,
|
||
"ui_only": ui_only,
|
||
"blocked": blocked,
|
||
"secondary": bool(secondary),
|
||
"backend_handled": bool(preview_execution.get("backend_handled", False) or autopilot.get("backend_handled", False)),
|
||
"target_api": target_api,
|
||
"target_method": target_method,
|
||
"request_payload": request_payload,
|
||
"focus_ref": _normalize_focus_ref(
|
||
preview_section.get("focus_ref")
|
||
or normalized_entry.get("secondary_focus_ref" if secondary else "primary_focus_ref")
|
||
or normalized_entry.get("focus_ref")
|
||
),
|
||
"action_code": str(
|
||
preview_section.get("action_code")
|
||
or normalized_entry.get("secondary_action_code" if secondary else "primary_action_code")
|
||
or ""
|
||
).strip(),
|
||
"sequence_key": str(normalized_preview.get("sequence_key") or normalized_entry.get("sequence_key") or "").strip(),
|
||
"label": str(
|
||
preview_section.get("label")
|
||
or entry_execution.get("label")
|
||
or normalized_entry.get("secondary_label" if secondary else "primary_label")
|
||
or ""
|
||
).strip(),
|
||
"requires_fields": list(preview_execution.get("requires_fields") or []),
|
||
"missing_fields": list(preview_execution.get("missing_fields") or []),
|
||
}
|
||
|
||
|
||
def resolve_codex_action(payload: dict | None = None) -> tuple[bool, str, dict]:
|
||
normalized_payload = dict(payload or {})
|
||
entry_key = str(normalized_payload.get("entry_key") or "").strip()
|
||
requested_by = str(normalized_payload.get("requested_by") or "api/codex-action").strip() or "api/codex-action"
|
||
source_label = str(normalized_payload.get("source_label") or "ops/codex-action").strip() or "ops/codex-action"
|
||
include_secondary = bool(normalized_payload.get("include_secondary", True))
|
||
secondary = bool(normalized_payload.get("secondary", False))
|
||
confirm = bool(normalized_payload.get("confirm", False))
|
||
|
||
codex_brief = get_ops_codex_brief()
|
||
selected_entry = _select_codex_brief_entry(codex_brief, entry_key=entry_key)
|
||
if not selected_entry:
|
||
return False, "codex brief 当前没有可执行条目", {"entry_key": entry_key}
|
||
|
||
preview_request = _build_codex_action_preview_request(
|
||
selected_entry,
|
||
requested_by=requested_by,
|
||
source_label=source_label,
|
||
include_secondary=include_secondary,
|
||
)
|
||
preview_ok, preview_message, preview_data = preview_driver_action(preview_request)
|
||
if not preview_ok:
|
||
return False, preview_message, {
|
||
"entry_key": str(selected_entry.get("key") or "").strip(),
|
||
"selected_entry": selected_entry,
|
||
"preview_request": preview_request,
|
||
"preview": dict(preview_data or {}),
|
||
}
|
||
|
||
gate = _build_codex_action_gate(selected_entry, preview_data, secondary=secondary, confirm=confirm)
|
||
return True, "Codex 驾驶动作解析完成", {
|
||
"entry_key": str(selected_entry.get("key") or "").strip(),
|
||
"selected_entry": selected_entry,
|
||
"preview_request": preview_request,
|
||
"preview": dict(preview_data or {}),
|
||
"gate": gate,
|
||
"selected_section": "secondary" if secondary else "primary",
|
||
"focus": dict(codex_brief.get("focus") or {}),
|
||
"summary": dict(codex_brief.get("summary") or {}),
|
||
"focus_ref": _normalize_focus_ref(
|
||
gate.get("focus_ref")
|
||
or selected_entry.get("secondary_focus_ref" if secondary else "primary_focus_ref")
|
||
or selected_entry.get("focus_ref")
|
||
),
|
||
}
|
||
|
||
|
||
def execute_codex_action(payload: dict | None = None) -> tuple[bool, str, dict]:
|
||
resolve_ok, resolve_message, resolved = resolve_codex_action(payload or {})
|
||
if not resolve_ok:
|
||
return False, resolve_message, dict(resolved or {})
|
||
|
||
gate = dict((resolved or {}).get("gate") or {})
|
||
if not bool(gate.get("will_execute", False)):
|
||
selected_entry = dict((resolved or {}).get("selected_entry") or {})
|
||
blocked_message = str(gate.get("decision_reason") or "").strip()
|
||
if str(gate.get("decision") or "").strip() == "open_ui":
|
||
blocked_message = blocked_message or "当前焦点属于 UI-only,建议进入工作区处理。"
|
||
if bool(selected_entry.get("observation_only", False)) and "工作区" not in blocked_message:
|
||
blocked_message = f"{blocked_message} 请进入对应工作区查看当前观察项。".strip()
|
||
|
||
return False, blocked_message or "当前不允许自动执行", {
|
||
**dict(resolved or {}),
|
||
"executed": False,
|
||
}
|
||
|
||
selected_entry = dict((resolved or {}).get("selected_entry") or {})
|
||
preview = dict((resolved or {}).get("preview") or {})
|
||
executor_kind = str(preview.get("executor_kind") or selected_entry.get("executor_kind") or "driver_action").strip() or "driver_action"
|
||
request_payload = dict(gate.get("request_payload") or {})
|
||
|
||
if executor_kind == "runbook_sequence":
|
||
sequence_key = str(gate.get("sequence_key") or preview.get("sequence_key") or selected_entry.get("sequence_key") or "").strip()
|
||
if not sequence_key:
|
||
return False, "缺少 sequence_key,无法执行 runbook sequence", {
|
||
**dict(resolved or {}),
|
||
"executed": False,
|
||
}
|
||
execute_ok, execute_message, execute_data = execute_ops_runbook_sequence(sequence_key, request_payload)
|
||
else:
|
||
execute_ok, execute_message, execute_data = execute_driver_action(request_payload)
|
||
|
||
return execute_ok, execute_message, {
|
||
**dict(resolved or {}),
|
||
"executed": bool(execute_ok),
|
||
"execution_result": dict(execute_data or {}),
|
||
}
|
||
|
||
|
||
def execute_driver_action(payload: dict) -> tuple[bool, str, dict]:
|
||
action_code = str((payload or {}).get("action_code") or "").strip()
|
||
requested_by = str((payload or {}).get("requested_by") or "api").strip() or "api"
|
||
node_codes = _normalize_driver_node_codes((payload or {}).get("node_codes") or [])
|
||
action_payload = dict((payload or {}).get("action_payload") or {})
|
||
|
||
if not action_code:
|
||
return False, "action_code 不能为空", {}
|
||
|
||
if action_code not in _BACKEND_DRIVER_ACTION_CODES:
|
||
return True, "该驾驶动作暂需前端交互或尚未后端化", {
|
||
"handled": False,
|
||
"manual_required": True,
|
||
"action_code": action_code,
|
||
"node_codes": node_codes,
|
||
}
|
||
|
||
if action_code == "enable_log_sync_key":
|
||
updated = update_runtime_settings(
|
||
{
|
||
**get_runtime_settings(),
|
||
"worker_log_sync_enabled": True,
|
||
"worker_log_sync_mode": "key",
|
||
}
|
||
)
|
||
return True, "远端日志回传已切到关键模式", {
|
||
"handled": True,
|
||
"action_code": action_code,
|
||
"mode": "settings",
|
||
"runtime_settings": updated,
|
||
}
|
||
|
||
if action_code == "disable_log_sync":
|
||
updated = update_runtime_settings(
|
||
{
|
||
**get_runtime_settings(),
|
||
"worker_log_sync_enabled": False,
|
||
"worker_log_sync_mode": "key",
|
||
}
|
||
)
|
||
return True, "远端日志回传已关闭", {
|
||
"handled": True,
|
||
"action_code": action_code,
|
||
"mode": "settings",
|
||
"runtime_settings": updated,
|
||
}
|
||
|
||
if action_code == "enable_log_sync_full":
|
||
updated = update_runtime_settings(
|
||
{
|
||
**get_runtime_settings(),
|
||
"worker_log_sync_enabled": True,
|
||
"worker_log_sync_mode": "full",
|
||
}
|
||
)
|
||
return True, "远端日志回传已切到全量模式", {
|
||
"handled": True,
|
||
"action_code": action_code,
|
||
"mode": "settings",
|
||
"runtime_settings": updated,
|
||
}
|
||
|
||
if action_code == "open_playbook_dialog":
|
||
playbook_key = str(action_payload.get("playbook_key") or "").strip()
|
||
if not playbook_key:
|
||
return False, "该驾驶动作需要 playbook_key", {"handled": False, "action_code": action_code}
|
||
target_node_codes = _normalize_driver_node_codes(action_payload.get("target_node_codes") or node_codes)
|
||
requested_execution_mode = str(action_payload.get("execution_mode") or "").strip()
|
||
if not requested_execution_mode:
|
||
requested_execution_mode = _recommend_driver_execution_mode_for_playbook(
|
||
playbook_key,
|
||
target_node_codes,
|
||
).get("execution_mode", "")
|
||
preview_ok, preview_message, playbook_preview = preview_ops_playbook(
|
||
{
|
||
"playbook_key": playbook_key,
|
||
"node_codes": target_node_codes,
|
||
"execution_mode": requested_execution_mode,
|
||
"auto_approve": bool(action_payload.get("auto_approve", False)),
|
||
"requested_by": requested_by,
|
||
}
|
||
)
|
||
if not preview_ok:
|
||
return False, preview_message, {"handled": False, "action_code": action_code}
|
||
return True, "", {
|
||
"handled": True,
|
||
"action_code": action_code,
|
||
"mode": "playbook-preview",
|
||
"playbook_preview": playbook_preview,
|
||
"ui_intent": _build_driver_ui_intent(
|
||
"open_playbook_dialog",
|
||
playbook_key=playbook_key,
|
||
target_node_codes=target_node_codes,
|
||
execution_mode=requested_execution_mode,
|
||
auto_approve=bool(action_payload.get("auto_approve", False)),
|
||
),
|
||
}
|
||
|
||
if action_code == "open_action_template_dialog":
|
||
template_key = str(action_payload.get("template_key") or "").strip()
|
||
if not template_key:
|
||
return False, "该驾驶动作需要 template_key", {"handled": False, "action_code": action_code}
|
||
template = dict(get_ops_action_template(template_key) or {})
|
||
target_node_codes = _normalize_driver_node_codes(action_payload.get("target_node_codes") or node_codes)
|
||
requested_execution_mode = str(action_payload.get("execution_mode") or "").strip()
|
||
if not requested_execution_mode:
|
||
requested_execution_mode = _recommend_driver_execution_mode_for_action_template(
|
||
template_key,
|
||
target_node_codes,
|
||
).get("execution_mode", "")
|
||
return True, "", {
|
||
"handled": True,
|
||
"action_code": action_code,
|
||
"mode": "action-template-preview",
|
||
"template_preview": {
|
||
"template": template,
|
||
"target_node_codes": target_node_codes,
|
||
"execution_mode": requested_execution_mode,
|
||
"payload": dict(action_payload.get("payload") or {}),
|
||
},
|
||
"ui_intent": _build_driver_ui_intent(
|
||
"open_action_template_dialog",
|
||
template_key=template_key,
|
||
target_node_codes=target_node_codes,
|
||
execution_mode=requested_execution_mode,
|
||
auto_approve=bool(action_payload.get("auto_approve", False)),
|
||
payload=dict(action_payload.get("payload") or {}),
|
||
),
|
||
}
|
||
|
||
if action_code in {"replay_delivery_queue", "flush_delivery_queue"}:
|
||
template_key = "delivery.queue.replay" if action_code == "replay_delivery_queue" else "delivery.queue.flush"
|
||
target_node_codes = _normalize_driver_node_codes(action_payload.get("target_node_codes") or node_codes)
|
||
if not target_node_codes:
|
||
return False, "该驾驶动作需要 target_node_codes", {"handled": False, "action_code": action_code}
|
||
requested_execution_mode = str(action_payload.get("execution_mode") or "").strip()
|
||
if not requested_execution_mode:
|
||
requested_execution_mode = _recommend_driver_execution_mode_for_action_template(
|
||
template_key,
|
||
target_node_codes,
|
||
).get("execution_mode", "")
|
||
batch_payload = {
|
||
"template_key": template_key,
|
||
"target_node_codes": target_node_codes,
|
||
"requested_by": f"{requested_by}/{action_code}",
|
||
"execution_mode": requested_execution_mode or "remote-agent",
|
||
"auto_approve": bool(action_payload.get("auto_approve", False)),
|
||
"payload": dict(action_payload.get("payload") or {}),
|
||
"metadata": {
|
||
"source": "ops-driver",
|
||
"driver_action_code": action_code,
|
||
},
|
||
}
|
||
ok, message, data = create_ops_job_batch(batch_payload)
|
||
return ok, (message or ("队列修复任务已创建" if ok else "队列修复任务创建失败")), {
|
||
"handled": ok,
|
||
"action_code": action_code,
|
||
"mode": "ops-job-batch",
|
||
"template_key": template_key,
|
||
"execution_mode": batch_payload["execution_mode"],
|
||
"target_node_codes": target_node_codes,
|
||
"job_batch": data,
|
||
}
|
||
|
||
if action_code == "focus_playbook_run":
|
||
run_code = str(action_payload.get("run_code") or "").strip()
|
||
if not run_code:
|
||
return False, "该驾驶动作需要 run_code", {"handled": False, "action_code": action_code}
|
||
run_detail = get_ops_playbook_run(run_code)
|
||
run_events = list_ops_playbook_run_events(
|
||
run_code,
|
||
limit=40,
|
||
step_key=str(action_payload.get("focus_step_key") or "").strip(),
|
||
node_code=str(action_payload.get("node_code") or "").strip(),
|
||
)
|
||
return True, "", {
|
||
"handled": True,
|
||
"action_code": action_code,
|
||
"mode": "playbook-run-focus",
|
||
"playbook_run_focus": {
|
||
"run": run_detail,
|
||
"events": run_events,
|
||
},
|
||
"ui_intent": _build_driver_ui_intent(
|
||
"playbook_run_detail",
|
||
run_code=run_code,
|
||
focus_step_key=str(action_payload.get("focus_step_key") or "").strip(),
|
||
focus_step_title=str(action_payload.get("focus_step_title") or "").strip(),
|
||
),
|
||
}
|
||
|
||
if action_code == "focus_activity_item":
|
||
ui_intent = dict(action_payload.get("ui_intent") or {})
|
||
if not str(ui_intent.get("kind") or "").strip():
|
||
return False, "该驾驶动作需要 ui_intent", {"handled": False, "action_code": action_code}
|
||
activity_key = str(action_payload.get("activity_key") or "").strip()
|
||
activity_kind = str(action_payload.get("kind") or "").strip()
|
||
activity_item = _find_activity_stream_item(activity_key, kind=activity_kind) if activity_key else {}
|
||
if not activity_item:
|
||
activity_item = {
|
||
"activity_key": activity_key,
|
||
"kind": activity_kind,
|
||
"status": str(action_payload.get("status") or "").strip(),
|
||
"summary": str(action_payload.get("summary") or "").strip(),
|
||
"focus_ref": _normalize_focus_ref(action_payload.get("focus_ref")),
|
||
"ui_intent": ui_intent,
|
||
}
|
||
return True, "", {
|
||
"handled": True,
|
||
"action_code": action_code,
|
||
"mode": "activity-focus",
|
||
"activity_focus_preview": {
|
||
"activity": activity_item,
|
||
},
|
||
"ui_intent": ui_intent,
|
||
}
|
||
|
||
if action_code == "focus_latest_job_events":
|
||
ui_intent = dict(action_payload.get("ui_intent") or {})
|
||
if not str(ui_intent.get("kind") or "").strip():
|
||
source_focus_ref = _normalize_focus_ref(action_payload.get("source_focus_ref"))
|
||
focus_ref = _normalize_focus_ref(action_payload.get("focus_ref"))
|
||
resolved_focus_ref = source_focus_ref or focus_ref
|
||
if not resolved_focus_ref:
|
||
return False, "该驾驶动作需要 ui_intent 或 focus_ref", {"handled": False, "action_code": action_code}
|
||
ui_intent = _build_driver_ui_intent(
|
||
"job_events",
|
||
job_id=int(resolved_focus_ref.get("job_id") or focus_ref.get("job_id") or 0),
|
||
job_code=str(resolved_focus_ref.get("job_code") or focus_ref.get("job_code") or "").strip(),
|
||
event_key=str(resolved_focus_ref.get("event_key") or "").strip(),
|
||
target_node_code=str(
|
||
resolved_focus_ref.get("target_node_code") or focus_ref.get("target_node_code") or ""
|
||
).strip(),
|
||
)
|
||
job_id = int(ui_intent.get("job_id") or action_payload.get("job_id") or 0)
|
||
job_events = list_ops_job_events_for_jobs([job_id], limit=60) if job_id > 0 else []
|
||
return True, "", {
|
||
"handled": True,
|
||
"action_code": action_code,
|
||
"mode": "job-events-focus",
|
||
"job_events_preview": {
|
||
"job_id": job_id,
|
||
"job_code": str(ui_intent.get("job_code") or action_payload.get("job_code") or "").strip(),
|
||
"events": job_events,
|
||
},
|
||
"ui_intent": ui_intent,
|
||
}
|
||
|
||
if action_code == "open_playbook_run_latest_events":
|
||
run_code = str(action_payload.get("run_code") or "").strip()
|
||
if not run_code:
|
||
return False, "该驾驶动作需要 run_code", {"handled": False, "action_code": action_code}
|
||
run_events = list_ops_playbook_run_events(
|
||
run_code,
|
||
limit=60,
|
||
step_key=str(action_payload.get("focus_step_key") or "").strip(),
|
||
node_code=str(action_payload.get("node_code") or "").strip(),
|
||
)
|
||
return True, "", {
|
||
"handled": True,
|
||
"action_code": action_code,
|
||
"mode": "playbook-run-events",
|
||
"playbook_run_events_preview": run_events,
|
||
"ui_intent": _build_driver_ui_intent(
|
||
"playbook_run_latest_events",
|
||
run_code=run_code,
|
||
focus_step_key=str(action_payload.get("focus_step_key") or "").strip(),
|
||
focus_step_title=str(action_payload.get("focus_step_title") or "").strip(),
|
||
),
|
||
}
|
||
|
||
if action_code == "open_release_dialog":
|
||
release_package_preview = get_latest_release_package_metadata()
|
||
release_summary = get_release_summary()
|
||
return True, "", {
|
||
"handled": True,
|
||
"action_code": action_code,
|
||
"mode": "release-dialog-preview",
|
||
"release_dialog_preview": {
|
||
"summary": release_summary,
|
||
"release_package_preview": release_package_preview,
|
||
},
|
||
"ui_intent": _build_driver_ui_intent("open_release_dialog"),
|
||
}
|
||
|
||
if action_code == "release_package":
|
||
release_package_preview = get_latest_release_package_metadata()
|
||
return True, "", {
|
||
"handled": True,
|
||
"action_code": action_code,
|
||
"mode": "release-package-preview",
|
||
"release_package_preview": release_package_preview,
|
||
"ui_intent": _build_driver_ui_intent("open_release_dialog"),
|
||
}
|
||
|
||
if action_code == "release_prepare":
|
||
ok, message, data = prepare_latest_release_package(
|
||
requested_by=str(action_payload.get("requested_by") or f"{requested_by}/{action_code}").strip()
|
||
or f"{requested_by}/{action_code}"
|
||
)
|
||
return ok, message, {
|
||
"handled": ok,
|
||
"action_code": action_code,
|
||
"mode": "release-prepare",
|
||
"release_prepare_result": data or {},
|
||
}
|
||
|
||
if action_code == "api-restart":
|
||
launchpad_context = _driver_launchpad_context(action_payload)
|
||
runtime_build_info = get_runtime_build_info()
|
||
recommended_command = build_bash_command(
|
||
"drive_ops_center.sh",
|
||
"runtime-refresh-recover",
|
||
launchpad_context["control_plane_base_url"],
|
||
)
|
||
return True, "建议先刷新控制面运行时,再继续 Launchpad 评估。", {
|
||
"handled": True,
|
||
"action_code": action_code,
|
||
"mode": "runtime-refresh-preview",
|
||
"runtime_build_info": runtime_build_info,
|
||
"recommended_command": recommended_command,
|
||
"ui_intent": _build_driver_ui_intent("open_stack_diagnosis"),
|
||
}
|
||
|
||
if action_code == "publish_latest_worker":
|
||
launchpad_context = _driver_launchpad_context(action_payload)
|
||
ok, message, data = create_release_and_smart_rollout_from_latest_package(
|
||
{
|
||
**action_payload,
|
||
"channel": launchpad_context["channel"],
|
||
"mode": "worker",
|
||
"created_by": str(action_payload.get("created_by") or f"{requested_by}/{action_code}").strip()
|
||
or f"{requested_by}/{action_code}",
|
||
"rollout_created_by": str(
|
||
action_payload.get("rollout_created_by") or f"{requested_by}/{action_code}/rollout"
|
||
).strip()
|
||
or f"{requested_by}/{action_code}/rollout",
|
||
},
|
||
control_plane_base_url=launchpad_context["control_plane_base_url"],
|
||
)
|
||
return ok, message, {
|
||
"handled": ok,
|
||
"action_code": action_code,
|
||
"mode": "smart-release-rollout",
|
||
"smart_release_result": data or {},
|
||
}
|
||
|
||
if action_code in {"review_smart_rollout_preview", "review_control_rollout", "fix_rollout_blockers"}:
|
||
launchpad_context = _driver_launchpad_context(action_payload)
|
||
release_launchpad = get_release_launchpad(
|
||
control_plane_base_url=launchpad_context["control_plane_base_url"],
|
||
channel=launchpad_context["channel"],
|
||
)
|
||
review_mode = "control" if action_code == "review_control_rollout" else "worker"
|
||
review_payload = _build_release_launchpad_review_payload(
|
||
release_launchpad=release_launchpad,
|
||
mode=review_mode,
|
||
)
|
||
latest_release = dict(review_payload.get("latest_release") or {})
|
||
return True, "", {
|
||
"handled": True,
|
||
"action_code": action_code,
|
||
"mode": "release-launchpad-preview",
|
||
"review_mode": review_mode,
|
||
"release_launchpad_review": review_payload,
|
||
"ui_intent": _build_driver_ui_intent(
|
||
"release_launchpad_review",
|
||
mode=review_mode,
|
||
release_id=int(latest_release.get("id") or 0),
|
||
channel=launchpad_context["channel"],
|
||
),
|
||
}
|
||
|
||
if action_code == "fix_managed_nodes":
|
||
launchpad_context = _driver_launchpad_context(action_payload)
|
||
release_launchpad = get_release_launchpad(
|
||
control_plane_base_url=launchpad_context["control_plane_base_url"],
|
||
channel=launchpad_context["channel"],
|
||
)
|
||
first_gap_row = _first_release_launchpad_gap_row(release_launchpad)
|
||
first_gap_node_code = str(first_gap_row.get("node_code") or "").strip()
|
||
if first_gap_node_code:
|
||
handover = get_managed_node_handover(
|
||
first_gap_node_code,
|
||
control_plane_base_url=launchpad_context["control_plane_base_url"],
|
||
)
|
||
return True, "", {
|
||
"handled": True,
|
||
"action_code": action_code,
|
||
"mode": "ui-intent",
|
||
"ui_intent": _build_driver_ui_intent(
|
||
"managed_node_handover",
|
||
node_code=first_gap_node_code,
|
||
),
|
||
"handover": handover,
|
||
}
|
||
sync_result = sync_managed_nodes_from_cluster(dry_run=False)
|
||
return True, "已按集群快照同步托管节点,请继续检查接管状态。", {
|
||
"handled": True,
|
||
"action_code": action_code,
|
||
"mode": "cluster-sync",
|
||
"sync_result": sync_result,
|
||
}
|
||
|
||
if action_code in {"bootstrap_run", "run_acceptance"}:
|
||
target_node_code = str(action_payload.get("node_code") or (node_codes[0] if node_codes else "")).strip()
|
||
if not target_node_code:
|
||
launchpad_context = _driver_launchpad_context(action_payload)
|
||
release_launchpad = get_release_launchpad(
|
||
control_plane_base_url=launchpad_context["control_plane_base_url"],
|
||
channel=launchpad_context["channel"],
|
||
)
|
||
first_gap_row = _first_release_launchpad_gap_row(release_launchpad)
|
||
target_node_code = str(first_gap_row.get("node_code") or "").strip()
|
||
if not target_node_code:
|
||
target_node_code = str(
|
||
(dict(release_launchpad.get("launchpad_status") or {}).get("recommended_target_node_code") or "")
|
||
).strip()
|
||
if not target_node_code:
|
||
return False, "该驾驶动作需要 node_code", {"handled": False, "action_code": action_code}
|
||
ok, message, data = execute_managed_node_onboarding_recovery(
|
||
node_code=target_node_code,
|
||
requested_by=f"{requested_by}/{action_code}",
|
||
control_plane_base_url=str(action_payload.get("control_plane_base_url") or "").strip(),
|
||
root_dir=str(action_payload.get("root_dir") or "").strip(),
|
||
)
|
||
return ok, message, {
|
||
"handled": ok,
|
||
"action_code": action_code,
|
||
"mode": "node-onboarding-recovery",
|
||
"node_code": target_node_code,
|
||
"recovery_result": data or {},
|
||
}
|
||
|
||
if action_code in {
|
||
"create_smart_release_rollout_worker",
|
||
"create_smart_release_rollout_control",
|
||
"create_release_rollout_worker",
|
||
"create_release_rollout_control",
|
||
}:
|
||
rollout_mode = "control" if action_code.endswith("_control") else "worker"
|
||
normalized_payload = {
|
||
**action_payload,
|
||
"mode": rollout_mode,
|
||
"created_by": str(action_payload.get("created_by") or f"{requested_by}/{action_code}").strip()
|
||
or f"{requested_by}/{action_code}",
|
||
"rollout_created_by": str(
|
||
action_payload.get("rollout_created_by") or f"{requested_by}/{action_code}/rollout"
|
||
).strip()
|
||
or f"{requested_by}/{action_code}/rollout",
|
||
"confirm_risky": bool(action_payload.get("confirm_risky", False)),
|
||
}
|
||
if action_code.startswith("create_release_rollout_"):
|
||
release_id = int(action_payload.get("release_id") or (_preferred_release_for_ops().get("id") or 0) or 0)
|
||
if release_id <= 0:
|
||
return False, "该驾驶动作需要 release_id", {"handled": False, "action_code": action_code}
|
||
ok, message, data = create_smart_release_rollout(release_id, normalized_payload)
|
||
else:
|
||
ok, message, data = create_release_and_smart_rollout_from_latest_package(normalized_payload)
|
||
return ok, message, {
|
||
"handled": ok,
|
||
"action_code": action_code,
|
||
"mode": "smart-release-rollout",
|
||
"smart_release_result": data or {},
|
||
}
|
||
|
||
if action_code == "open_rollout_dialog":
|
||
release_id = int(action_payload.get("release_id") or (_preferred_release_for_ops().get("id") or 0) or 0)
|
||
release_summary = get_release_summary()
|
||
release_launchpad = get_release_launchpad()
|
||
return True, "", {
|
||
"handled": True,
|
||
"action_code": action_code,
|
||
"mode": "rollout-dialog-preview",
|
||
"rollout_dialog_preview": {
|
||
"summary": release_summary,
|
||
"launchpad": release_launchpad,
|
||
},
|
||
"ui_intent": _build_driver_ui_intent("open_rollout_dialog", release_id=release_id),
|
||
}
|
||
|
||
if action_code == "focus_release_hub":
|
||
release_id = int(action_payload.get("release_id") or (_preferred_release_for_ops().get("id") or 0) or 0)
|
||
launchpad_context = _driver_launchpad_context(action_payload)
|
||
release_summary = get_release_summary()
|
||
release_launchpad = get_release_launchpad(
|
||
control_plane_base_url=launchpad_context["control_plane_base_url"],
|
||
channel=launchpad_context["channel"],
|
||
)
|
||
return True, "", {
|
||
"handled": True,
|
||
"action_code": action_code,
|
||
"mode": "release-hub-preview",
|
||
"release_hub_preview": {
|
||
"summary": release_summary,
|
||
"launchpad": release_launchpad,
|
||
},
|
||
"ui_intent": _build_driver_ui_intent("focus_release_hub", release_id=release_id),
|
||
}
|
||
|
||
if action_code in {"open_release_deploy_control", "open_release_deploy_worker", "open_release_deploy_custom"}:
|
||
template_key_map = {
|
||
"open_release_deploy_control": "deploy.release.control",
|
||
"open_release_deploy_worker": "deploy.release.worker",
|
||
"open_release_deploy_custom": "deploy.release.custom",
|
||
}
|
||
template_key = template_key_map.get(action_code, "deploy.release.custom")
|
||
template = dict(get_ops_action_template(template_key) or {})
|
||
target_node_codes = _normalize_driver_node_codes(action_payload.get("target_node_codes") or node_codes)
|
||
release_id = int(action_payload.get("release_id") or (_preferred_release_for_ops().get("id") or 0) or 0)
|
||
execution_mode = str(action_payload.get("execution_mode") or "").strip()
|
||
if not execution_mode:
|
||
execution_mode = _recommend_driver_execution_mode(target_node_codes).get("execution_mode", "remote-agent")
|
||
release_summary = get_release_summary()
|
||
release_launchpad = get_release_launchpad()
|
||
return True, "", {
|
||
"handled": True,
|
||
"action_code": action_code,
|
||
"mode": "release-deploy-preview",
|
||
"release_deploy_preview": {
|
||
"template": template,
|
||
"summary": release_summary,
|
||
"launchpad": release_launchpad,
|
||
"target_node_codes": target_node_codes,
|
||
"execution_mode": execution_mode,
|
||
"release_id": release_id,
|
||
},
|
||
"ui_intent": _build_driver_ui_intent(
|
||
"open_release_deploy_template",
|
||
template_key=template_key,
|
||
release_id=release_id,
|
||
target_node_codes=target_node_codes,
|
||
execution_mode=execution_mode,
|
||
),
|
||
}
|
||
|
||
if action_code in {"handover_first_gap", "view_first_gap"}:
|
||
target_node_code = str(action_payload.get("node_code") or (node_codes[0] if node_codes else "")).strip()
|
||
if not target_node_code:
|
||
return False, "该驾驶动作需要 node_code", {"handled": False, "action_code": action_code}
|
||
handover = get_managed_node_handover(
|
||
target_node_code,
|
||
control_plane_base_url=str(action_payload.get("control_plane_base_url") or "").strip(),
|
||
)
|
||
return True, "", {
|
||
"handled": True,
|
||
"action_code": action_code,
|
||
"mode": "handover-preview",
|
||
"handover_preview": handover,
|
||
"ui_intent": _build_driver_ui_intent(
|
||
"managed_node_handover" if action_code == "handover_first_gap" else "managed_node_edit",
|
||
node_code=target_node_code,
|
||
),
|
||
"handover": handover,
|
||
}
|
||
|
||
if not node_codes:
|
||
return False, "该驾驶动作需要 node_codes", {
|
||
"handled": False,
|
||
"action_code": action_code,
|
||
}
|
||
|
||
if action_code in {"open_worker_logs_participating", "open_worker_logs_standby", "open_worker_logs"}:
|
||
playbook_key = "scene.logs.key"
|
||
execution_mode = str(action_payload.get("execution_mode") or "").strip()
|
||
if not execution_mode:
|
||
execution_mode = _recommend_driver_execution_mode(node_codes).get("execution_mode", "remote-agent")
|
||
ok, message, data = execute_ops_playbook(
|
||
{
|
||
"playbook_key": playbook_key,
|
||
"node_codes": node_codes,
|
||
"execution_mode": execution_mode,
|
||
"auto_approve": True,
|
||
"requested_by": f"{requested_by}/{action_code}",
|
||
}
|
||
)
|
||
return ok, message, {
|
||
"handled": ok,
|
||
"action_code": action_code,
|
||
"mode": "ops-playbook",
|
||
"playbook_key": playbook_key,
|
||
"node_codes": node_codes,
|
||
"execution_mode": execution_mode,
|
||
**(data or {}),
|
||
}
|
||
|
||
if action_code in {"run_scene_logs_key", "run_scene_logs_full"}:
|
||
playbook_key = "scene.logs.full" if action_code == "run_scene_logs_full" else "scene.logs.key"
|
||
execution_mode = str(action_payload.get("execution_mode") or "").strip()
|
||
if not execution_mode:
|
||
execution_mode = _recommend_driver_execution_mode(node_codes).get("execution_mode", "remote-agent")
|
||
ok, message, data = execute_ops_playbook(
|
||
{
|
||
"playbook_key": playbook_key,
|
||
"node_codes": node_codes,
|
||
"execution_mode": execution_mode,
|
||
"auto_approve": bool(action_payload.get("auto_approve", True)),
|
||
"requested_by": f"{requested_by}/{action_code}",
|
||
}
|
||
)
|
||
return ok, message, {
|
||
"handled": ok,
|
||
"action_code": action_code,
|
||
"mode": "ops-playbook",
|
||
"playbook_key": playbook_key,
|
||
"node_codes": node_codes,
|
||
"execution_mode": execution_mode,
|
||
**(data or {}),
|
||
}
|
||
|
||
if action_code in {"open_diagnostics_participating", "open_diagnostics"}:
|
||
playbook_key = "scene.diagnostics"
|
||
execution_mode = str(action_payload.get("execution_mode") or "").strip()
|
||
if not execution_mode:
|
||
execution_mode = _recommend_driver_execution_mode(node_codes).get("execution_mode", "remote-agent")
|
||
ok, message, data = execute_ops_playbook(
|
||
{
|
||
"playbook_key": playbook_key,
|
||
"node_codes": node_codes,
|
||
"execution_mode": execution_mode,
|
||
"auto_approve": True,
|
||
"requested_by": f"{requested_by}/{action_code}",
|
||
}
|
||
)
|
||
return ok, message, {
|
||
"handled": ok,
|
||
"action_code": action_code,
|
||
"mode": "ops-playbook",
|
||
"playbook_key": playbook_key,
|
||
"node_codes": node_codes,
|
||
"execution_mode": execution_mode,
|
||
**(data or {}),
|
||
}
|
||
|
||
if action_code in {"run_inspection_participating", "run_inspection_standby", "run_standard_inspection"}:
|
||
playbook_key = "inspection.standard"
|
||
execution_mode = str(action_payload.get("execution_mode") or "").strip()
|
||
if not execution_mode:
|
||
execution_mode = _recommend_driver_execution_mode(node_codes).get("execution_mode", "remote-agent")
|
||
ok, message, data = execute_ops_playbook(
|
||
{
|
||
"playbook_key": playbook_key,
|
||
"node_codes": node_codes,
|
||
"execution_mode": execution_mode,
|
||
"auto_approve": True,
|
||
"requested_by": f"{requested_by}/{action_code}",
|
||
}
|
||
)
|
||
return ok, message, {
|
||
"handled": ok,
|
||
"action_code": action_code,
|
||
"mode": "ops-playbook",
|
||
"playbook_key": playbook_key,
|
||
"node_codes": node_codes,
|
||
"execution_mode": execution_mode,
|
||
**(data or {}),
|
||
}
|
||
|
||
return True, "该驾驶动作暂需前端交互或尚未后端化", {
|
||
"handled": False,
|
||
"manual_required": True,
|
||
"action_code": action_code,
|
||
"node_codes": node_codes,
|
||
}
|
||
|
||
|
||
def _resolve_ops_runbook_sequence_entry(
|
||
sequence: dict,
|
||
*,
|
||
secondary: bool = False,
|
||
payload: dict | None = None,
|
||
requested_by: str = "api",
|
||
) -> tuple[bool, str, dict]:
|
||
normalized_sequence = dict(sequence or {})
|
||
normalized_payload = dict(payload or {})
|
||
normalized_sequence_key = str(normalized_sequence.get("key") or "").strip()
|
||
if not normalized_sequence_key:
|
||
return False, "sequence_key 不能为空", {}
|
||
|
||
action_field = "secondary_action_code" if secondary else "primary_action_code"
|
||
payload_field = "secondary_action_payload" if secondary else "primary_action_payload"
|
||
label_field = "secondary_label" if secondary else "primary_label"
|
||
action_code = str(normalized_sequence.get(action_field) or "").strip()
|
||
if not action_code:
|
||
return False, "当前标准作业路径没有可执行动作", {
|
||
"sequence_key": normalized_sequence_key,
|
||
"secondary": secondary,
|
||
"sequence": normalized_sequence,
|
||
}
|
||
|
||
sequence_node_codes = _normalize_driver_node_codes(normalized_sequence.get("target_node_codes") or [])
|
||
override_node_codes = _normalize_driver_node_codes(normalized_payload.get("node_codes") or [])
|
||
node_codes = override_node_codes or sequence_node_codes
|
||
action_payload = {
|
||
**dict(normalized_sequence.get(payload_field) or {}),
|
||
**dict(normalized_payload.get("action_payload") or {}),
|
||
}
|
||
sequence_focus_ref = _normalize_focus_ref(normalized_sequence.get("focus_ref"))
|
||
action_focus_ref = _normalize_focus_ref(
|
||
normalized_sequence.get("secondary_focus_ref" if secondary else "primary_focus_ref")
|
||
)
|
||
if sequence_focus_ref and not action_payload.get("focus_ref"):
|
||
action_payload["focus_ref"] = action_focus_ref or sequence_focus_ref
|
||
normalized_requested_by = str(requested_by or "api").strip().rstrip("/") or "api"
|
||
driver_requested_by = (
|
||
f"{normalized_requested_by}/{normalized_sequence_key}"
|
||
if normalized_requested_by.endswith("/runbook")
|
||
else f"{normalized_requested_by}/runbook/{normalized_sequence_key}"
|
||
)
|
||
sequence_summary = {
|
||
"key": normalized_sequence_key,
|
||
"title": str(normalized_sequence.get("title") or "").strip(),
|
||
"status": str(normalized_sequence.get("status") or "").strip(),
|
||
"status_label": str(normalized_sequence.get("status_label") or "").strip(),
|
||
"summary": str(normalized_sequence.get("summary") or "").strip(),
|
||
"reason": str(normalized_sequence.get("reason") or "").strip(),
|
||
"secondary": secondary,
|
||
"action_label": str(normalized_sequence.get(label_field) or "").strip(),
|
||
"target_scope_label": str(normalized_sequence.get("target_scope_label") or "").strip(),
|
||
"step_titles": [str(item or "").strip() for item in list(normalized_sequence.get("step_titles") or []) if str(item or "").strip()],
|
||
"focus_ref": sequence_focus_ref,
|
||
"action_focus_ref": action_focus_ref,
|
||
}
|
||
return True, "标准作业路径解析成功", {
|
||
"sequence_key": normalized_sequence_key,
|
||
"sequence": sequence_summary,
|
||
"driver_action_code": action_code,
|
||
"driver_action_label": str(normalized_sequence.get(label_field) or "").strip() or action_code,
|
||
"driver_node_codes": node_codes,
|
||
"driver_action_payload": action_payload,
|
||
"driver_requested_by": driver_requested_by,
|
||
"resolved_at": datetime.now().isoformat(timespec="seconds"),
|
||
}
|
||
|
||
|
||
def _attach_ops_runbook_sequence_resolutions(
|
||
sequences: list[dict],
|
||
*,
|
||
requested_by: str = "api/runbook-snapshot",
|
||
) -> list[dict]:
|
||
annotated_sequences: list[dict] = []
|
||
for raw_sequence in list(sequences or []):
|
||
sequence = dict(raw_sequence or {})
|
||
if not str(sequence.get("key") or "").strip():
|
||
continue
|
||
primary_ok, primary_message, primary_data = _resolve_ops_runbook_sequence_entry(
|
||
sequence,
|
||
secondary=False,
|
||
requested_by=requested_by,
|
||
)
|
||
sequence["primary_resolution"] = {
|
||
"ok": primary_ok,
|
||
"message": primary_message,
|
||
**dict(primary_data or {}),
|
||
}
|
||
if str(sequence.get("secondary_action_code") or "").strip():
|
||
secondary_ok, secondary_message, secondary_data = _resolve_ops_runbook_sequence_entry(
|
||
sequence,
|
||
secondary=True,
|
||
requested_by=requested_by,
|
||
)
|
||
sequence["secondary_resolution"] = {
|
||
"ok": secondary_ok,
|
||
"message": secondary_message,
|
||
**dict(secondary_data or {}),
|
||
}
|
||
else:
|
||
sequence["secondary_resolution"] = {}
|
||
annotated_sequences.append(sequence)
|
||
return annotated_sequences
|
||
|
||
|
||
def resolve_ops_runbook_sequence(sequence_key: str, payload: dict | None = None) -> tuple[bool, str, dict]:
|
||
normalized_sequence_key = str(sequence_key or "").strip()
|
||
normalized_payload = dict(payload or {})
|
||
if not normalized_sequence_key:
|
||
return False, "sequence_key 不能为空", {}
|
||
|
||
requested_by = str(normalized_payload.get("requested_by") or "api").strip() or "api"
|
||
secondary = bool(normalized_payload.get("secondary", False))
|
||
runbook = get_ops_runbook()
|
||
sequences = [
|
||
dict(item or {})
|
||
for item in list(runbook.get("control_sequences") or [])
|
||
if str((item or {}).get("key") or "").strip()
|
||
]
|
||
sequence = next(
|
||
(item for item in sequences if str(item.get("key") or "").strip() == normalized_sequence_key),
|
||
{},
|
||
)
|
||
if not sequence:
|
||
return False, "标准作业路径不存在", {"sequence_key": normalized_sequence_key}
|
||
return _resolve_ops_runbook_sequence_entry(
|
||
sequence,
|
||
secondary=secondary,
|
||
payload=normalized_payload,
|
||
requested_by=requested_by,
|
||
)
|
||
|
||
|
||
def execute_ops_runbook_sequence(sequence_key: str, payload: dict | None = None) -> tuple[bool, str, dict]:
|
||
ok, message, resolved = resolve_ops_runbook_sequence(sequence_key, payload or {})
|
||
if not ok:
|
||
return ok, message, resolved
|
||
|
||
action_code = str(resolved.get("driver_action_code") or "").strip()
|
||
node_codes = _normalize_driver_node_codes(resolved.get("driver_node_codes") or [])
|
||
action_payload = dict(resolved.get("driver_action_payload") or {})
|
||
ok, message, data = execute_driver_action(
|
||
{
|
||
"action_code": action_code,
|
||
"node_codes": node_codes,
|
||
"action_payload": action_payload,
|
||
"requested_by": str(resolved.get("driver_requested_by") or "api/runbook").strip() or "api/runbook",
|
||
}
|
||
)
|
||
return ok, message, {
|
||
**dict(resolved or {}),
|
||
**dict(data or {}),
|
||
}
|
||
|
||
|
||
def get_ops_inspection_overview(
|
||
*,
|
||
managed_nodes: list[dict] | None = None,
|
||
fetch_limit: int = _OPS_INSPECTION_FETCH_LIMIT,
|
||
status: str = "",
|
||
problem_kind: str = "",
|
||
query: str = "",
|
||
only_problem: bool = False,
|
||
only_participating: bool = False,
|
||
) -> dict:
|
||
nodes = list(managed_nodes or [])
|
||
if managed_nodes is None:
|
||
managed_nodes_payload = list_managed_nodes_with_agent_state()
|
||
nodes = list(managed_nodes_payload.get("nodes") or [])
|
||
jobs = list_ops_jobs(limit=fetch_limit, compact=False)
|
||
|
||
inspection_jobs = [job for job in jobs if _inspection_job_bucket(job) and str(job.get("target_node_code") or "").strip()]
|
||
node_map = {
|
||
str(item.get("node_code") or "").strip(): item
|
||
for item in nodes
|
||
if str(item.get("node_code") or "").strip()
|
||
}
|
||
row_map: dict[str, dict] = {}
|
||
|
||
def ensure_row(node_code: str) -> dict:
|
||
if node_code not in row_map:
|
||
node = node_map.get(node_code) or {}
|
||
row_map[node_code] = {
|
||
"node_code": node_code,
|
||
"region": str(node.get("region") or ""),
|
||
"role": str(node.get("role") or ""),
|
||
"is_managed": bool(node.get("is_managed", False)),
|
||
"is_enabled": bool(node.get("is_enabled", False)),
|
||
"is_agent_online": bool(node.get("is_agent_online", False)),
|
||
"has_ssh_access": bool(node.get("has_ssh_access", False)),
|
||
"agent_state": str(node.get("agent_state") or ""),
|
||
"agent_state_label": str(node.get("agent_state_label") or ""),
|
||
"agent_state_reason": str(node.get("agent_state_reason") or ""),
|
||
"remote_access_state": str(node.get("remote_access_state") or ""),
|
||
"remote_access_label": str(node.get("remote_access_label") or ""),
|
||
"remote_access_reason": str(node.get("remote_access_reason") or ""),
|
||
"is_remote_access_ready": bool(node.get("is_remote_access_ready", False)),
|
||
"cluster_status": str(node.get("cluster_status") or ""),
|
||
"cluster_ip": str(node.get("cluster_ip") or ""),
|
||
"cluster_hostname": str(node.get("cluster_hostname") or ""),
|
||
"cluster_is_effective_worker": bool(node.get("cluster_is_effective_worker", False)),
|
||
"cluster_detect_participating": bool(node.get("cluster_detect_participating", False)),
|
||
"participation_state": str(node.get("participation_state") or ""),
|
||
"participation_label": str(node.get("participation_label") or ""),
|
||
"participation_reason": str(node.get("participation_reason") or ""),
|
||
"is_inspection_visible": _is_inspection_visible_node(node),
|
||
"is_inspection_eligible": _is_inspection_eligible_node(node),
|
||
"job_map": {},
|
||
"latest_job": {},
|
||
"last_updated_at": "",
|
||
"success_count": 0,
|
||
"overall_status": "missing",
|
||
"issue_summary": "",
|
||
"problem_kind": "",
|
||
"problem_label": "",
|
||
"problem_title": "",
|
||
"problem_level": "",
|
||
"problem_keys": [],
|
||
"recommended_action": "",
|
||
"recommended_action_code": "",
|
||
}
|
||
return row_map[node_code]
|
||
|
||
for job in inspection_jobs:
|
||
node_code = str(job.get("target_node_code") or "").strip()
|
||
bucket = _inspection_job_bucket(job)
|
||
if not node_code or not bucket:
|
||
continue
|
||
row = ensure_row(node_code)
|
||
if bucket not in row["job_map"]:
|
||
row["job_map"][bucket] = _slim_inspection_job(job)
|
||
candidate_time = str(job.get("updated_at") or job.get("finished_at") or job.get("started_at") or job.get("created_at") or "").strip()
|
||
current_latest_time = str(row.get("last_updated_at") or "").strip()
|
||
if not current_latest_time or candidate_time > current_latest_time:
|
||
row["last_updated_at"] = candidate_time
|
||
row["latest_job"] = _slim_inspection_job(job)
|
||
|
||
for node in nodes:
|
||
node_code = str(node.get("node_code") or "").strip()
|
||
if node_code and _is_inspection_visible_node(node):
|
||
ensure_row(node_code)
|
||
|
||
def derive_overall_status(row: dict, job_map: dict[str, dict]) -> str:
|
||
statuses = [str((job_map.get(key) or {}).get("status") or "").strip() for key in _OPS_INSPECTION_ACTION_KEYS]
|
||
present_statuses = [status for status in statuses if status]
|
||
if not present_statuses:
|
||
if bool(row.get("is_inspection_visible", False)):
|
||
if not bool(row.get("is_inspection_eligible", False)):
|
||
return "attention"
|
||
if bool(row.get("cluster_detect_participating", False)):
|
||
return "attention"
|
||
return "missing"
|
||
if any(status in {"failed", "blocked", "cancelled"} for status in present_statuses):
|
||
return "attention"
|
||
if any(status in {"queued", "dispatching", "running", "awaiting_approval"} for status in present_statuses):
|
||
return "running"
|
||
if all(str((job_map.get(key) or {}).get("status") or "").strip() == "success" for key in _OPS_INSPECTION_ACTION_KEYS):
|
||
return "healthy"
|
||
return "attention"
|
||
|
||
rows: list[dict] = []
|
||
for row in row_map.values():
|
||
job_map = dict(row.get("job_map") or {})
|
||
success_count = sum(1 for key in _OPS_INSPECTION_ACTION_KEYS if str((job_map.get(key) or {}).get("status") or "").strip() == "success")
|
||
issue_summary = _build_inspection_issue_summary(
|
||
job_map,
|
||
node_code=str(row.get("node_code") or ""),
|
||
latest_job=dict(row.get("latest_job") or {}),
|
||
node=row,
|
||
)
|
||
rows.append(
|
||
{
|
||
**row,
|
||
"job_map": job_map,
|
||
"success_count": success_count,
|
||
"overall_status": derive_overall_status(row, job_map),
|
||
"issue_summary": str(issue_summary.get("summary") or ""),
|
||
"problem_kind": str(issue_summary.get("problem_kind") or ""),
|
||
"problem_label": str(issue_summary.get("problem_label") or ""),
|
||
"problem_title": str(issue_summary.get("problem_title") or ""),
|
||
"problem_level": str(issue_summary.get("problem_level") or ""),
|
||
"problem_keys": list(issue_summary.get("problem_keys") or []),
|
||
"recommended_action": str(issue_summary.get("recommended_action") or ""),
|
||
"recommended_action_code": str(issue_summary.get("recommended_action_code") or ""),
|
||
"ui_intent": dict(issue_summary.get("ui_intent") or {}),
|
||
"latest_health_snapshot": _build_inspection_contract_result(
|
||
job_map.get("health.snapshot"),
|
||
action_label="健康快照",
|
||
),
|
||
"latest_worker_logs": _build_inspection_contract_result(
|
||
job_map.get("logs.collect"),
|
||
action_label="Worker 日志",
|
||
),
|
||
"latest_diagnostics": _build_inspection_contract_result(
|
||
job_map.get("diagnostics.collect"),
|
||
action_label="诊断包",
|
||
),
|
||
}
|
||
)
|
||
|
||
status_weight = {"attention": 0, "running": 1, "missing": 2, "healthy": 3}
|
||
# Stable-sort so the primary order is status bucket, then newest update, then node code.
|
||
rows.sort(key=lambda item: str(item.get("node_code") or ""))
|
||
rows.sort(key=lambda item: str(item.get("last_updated_at") or ""), reverse=True)
|
||
rows.sort(key=lambda item: int(status_weight.get(str(item.get("overall_status") or ""), 99)))
|
||
|
||
summary = {
|
||
"healthy": 0,
|
||
"running": 0,
|
||
"attention": 0,
|
||
"missing": 0,
|
||
}
|
||
for row in rows:
|
||
row_overall_status = str(row.get("overall_status") or "missing").strip()
|
||
if row_overall_status in summary:
|
||
summary[row_overall_status] += 1
|
||
|
||
priority_weight = {"attention": 0, "running": 1, "missing": 2, "healthy": 9}
|
||
|
||
normalized_status = str(status or "").strip().lower()
|
||
normalized_problem_kind = str(problem_kind or "").strip().lower()
|
||
normalized_query = str(query or "").strip().lower()
|
||
filtered_rows = list(rows)
|
||
if normalized_status:
|
||
filtered_rows = [
|
||
item for item in filtered_rows if str(item.get("overall_status") or "").strip().lower() == normalized_status
|
||
]
|
||
if normalized_problem_kind:
|
||
filtered_rows = [
|
||
item for item in filtered_rows if str(item.get("problem_kind") or "").strip().lower() == normalized_problem_kind
|
||
]
|
||
if only_problem:
|
||
filtered_rows = [item for item in filtered_rows if str(item.get("overall_status") or "").strip() != "healthy"]
|
||
if only_participating:
|
||
filtered_rows = [item for item in filtered_rows if bool(item.get("cluster_detect_participating", False))]
|
||
if normalized_query:
|
||
filtered_rows = [
|
||
item
|
||
for item in filtered_rows
|
||
if normalized_query in " ".join(
|
||
[
|
||
str(item.get("node_code") or ""),
|
||
str(item.get("region") or ""),
|
||
str(item.get("role") or ""),
|
||
str(item.get("overall_status") or ""),
|
||
str(item.get("problem_kind") or ""),
|
||
str(item.get("problem_label") or ""),
|
||
str(item.get("problem_title") or ""),
|
||
str(item.get("issue_summary") or ""),
|
||
str(item.get("recommended_action") or ""),
|
||
]
|
||
).lower()
|
||
]
|
||
|
||
filtered_summary = {
|
||
"healthy": 0,
|
||
"running": 0,
|
||
"attention": 0,
|
||
"missing": 0,
|
||
}
|
||
for row in filtered_rows:
|
||
row_status = str(row.get("overall_status") or "missing").strip()
|
||
if row_status in filtered_summary:
|
||
filtered_summary[row_status] += 1
|
||
|
||
if not filtered_rows:
|
||
overview_status = "empty"
|
||
overview_status_label = "无匹配结果"
|
||
overview_summary_text = "当前筛选条件下没有匹配节点。"
|
||
elif int(filtered_summary.get("attention", 0) or 0) > 0:
|
||
overview_status = "attention"
|
||
overview_status_label = "待处理"
|
||
overview_summary_text = f"存在 {int(filtered_summary.get('attention', 0) or 0)} 台节点需要优先处理。"
|
||
elif int(filtered_summary.get("running", 0) or 0) > 0:
|
||
overview_status = "running"
|
||
overview_status_label = "执行中"
|
||
overview_summary_text = f"存在 {int(filtered_summary.get('running', 0) or 0)} 台节点的标准巡检仍在执行。"
|
||
elif int(filtered_summary.get("missing", 0) or 0) > 0:
|
||
overview_status = "attention"
|
||
overview_status_label = "待补齐"
|
||
overview_summary_text = f"存在 {int(filtered_summary.get('missing', 0) or 0)} 台节点尚未形成完整巡检收口。"
|
||
else:
|
||
overview_status = "healthy"
|
||
overview_status_label = "健康"
|
||
overview_summary_text = "当前可见节点最近巡检收口正常。"
|
||
|
||
priority_queue = [
|
||
{
|
||
**row,
|
||
"priority_label": "P1" if row.get("overall_status") == "attention" else ("P2" if row.get("overall_status") == "running" else "P3"),
|
||
"priority_type": "danger" if row.get("overall_status") == "attention" else ("warning" if row.get("overall_status") == "running" else "info"),
|
||
"priority_weight": int(priority_weight.get(str(row.get("overall_status") or ""), 99)),
|
||
}
|
||
for row in filtered_rows
|
||
if str(row.get("overall_status") or "") != "healthy"
|
||
]
|
||
priority_queue.sort(key=lambda item: str(item.get("node_code") or ""))
|
||
priority_queue.sort(key=lambda item: str(item.get("last_updated_at") or ""), reverse=True)
|
||
priority_queue.sort(key=lambda item: int(item.get("priority_weight", 99)))
|
||
|
||
available_status_counts = {
|
||
"healthy": int(summary.get("healthy", 0) or 0),
|
||
"running": int(summary.get("running", 0) or 0),
|
||
"attention": int(summary.get("attention", 0) or 0),
|
||
"missing": int(summary.get("missing", 0) or 0),
|
||
}
|
||
available_problem_kind_counts: dict[str, int] = {}
|
||
for row in rows:
|
||
row_problem_kind = str(row.get("problem_kind") or "").strip()
|
||
if not row_problem_kind:
|
||
continue
|
||
available_problem_kind_counts[row_problem_kind] = int(available_problem_kind_counts.get(row_problem_kind, 0) or 0) + 1
|
||
|
||
return {
|
||
"status": overview_status,
|
||
"status_label": overview_status_label,
|
||
"summary_text": overview_summary_text,
|
||
"fetch_limit": int(fetch_limit),
|
||
"jobs_total": len(inspection_jobs),
|
||
"visible_nodes_total": sum(1 for item in nodes if _is_inspection_visible_node(item)),
|
||
"eligible_nodes_total": sum(1 for item in nodes if _is_inspection_eligible_node(item)),
|
||
"handover_gap_nodes_total": sum(
|
||
1 for item in nodes if _is_inspection_visible_node(item) and not _is_inspection_eligible_node(item)
|
||
),
|
||
"participating_nodes_total": sum(1 for item in rows if bool(item.get("cluster_detect_participating", False))),
|
||
"standby_nodes_total": sum(
|
||
1
|
||
for item in rows
|
||
if bool(item.get("is_inspection_visible", False)) and not bool(item.get("cluster_detect_participating", False))
|
||
),
|
||
"rows_total": len(rows),
|
||
"filtered_rows_total": len(filtered_rows),
|
||
"filters": {
|
||
"status": normalized_status,
|
||
"problem_kind": normalized_problem_kind,
|
||
"query": str(query or "").strip(),
|
||
"only_problem": bool(only_problem),
|
||
"only_participating": bool(only_participating),
|
||
},
|
||
"available_status_counts": available_status_counts,
|
||
"available_problem_kind_counts": available_problem_kind_counts,
|
||
"rows": filtered_rows,
|
||
"summary": filtered_summary,
|
||
"counts": {
|
||
"healthy_nodes": int(filtered_summary.get("healthy", 0) or 0),
|
||
"running_nodes": int(filtered_summary.get("running", 0) or 0),
|
||
"attention_nodes": int(filtered_summary.get("attention", 0) or 0),
|
||
"missing_nodes": int(filtered_summary.get("missing", 0) or 0),
|
||
},
|
||
"unfiltered_summary": summary,
|
||
"priority_queue": priority_queue,
|
||
}
|
||
|
||
|
||
def get_ops_overview() -> dict:
|
||
runtime = get_runtime_status()
|
||
readiness = runtime.get("readiness") or {}
|
||
detect = runtime.get("detect") or {}
|
||
cluster = runtime.get("cluster") or {}
|
||
sync_summary = get_sync_summary()
|
||
job_summary = get_ops_job_summary()
|
||
managed_nodes_payload = list_managed_nodes_with_agent_state(participation_payload=detect)
|
||
managed_nodes = list(managed_nodes_payload.get("nodes") or [])
|
||
managed_nodes_summary = managed_nodes_payload.get("summary") or {}
|
||
inspection_overview = get_ops_inspection_overview(managed_nodes=managed_nodes)
|
||
activity_stream = get_ops_activity_stream(
|
||
limit=8,
|
||
runtime_status=runtime,
|
||
managed_nodes_payload=managed_nodes_payload,
|
||
)
|
||
release_summary = get_release_summary()
|
||
preferred_release = _preferred_release_for_ops()
|
||
release_launchpad = get_release_launchpad()
|
||
sync_batches = (sync_summary.get("detect_result_batches") or {}).get("state_counts") or {}
|
||
|
||
online_worker_nodes = int((cluster.get("summary") or {}).get("online_worker_nodes", 0) or 0)
|
||
online_control_nodes = int((cluster.get("summary") or {}).get("online_control_nodes", 0) or 0)
|
||
projected_batches = int(sync_batches.get("projected", 0) or 0)
|
||
failed_batches = int(sync_batches.get("failed", 0) or 0)
|
||
execution_scene = _build_ops_execution_scene(detect)
|
||
default_rollout_targets = _default_rollout_target_nodes(list(cluster.get("nodes") or []))
|
||
release_execution_mode = build_release_execution_mode_recommendation(preferred_release, default_rollout_targets)
|
||
release_gate = dict(release_execution_mode.get("recommended_gate") or {})
|
||
release_gate = {
|
||
**release_gate,
|
||
"default_target_nodes": list(release_gate.get("target_nodes") or []),
|
||
"default_target_node_codes": list(release_gate.get("target_node_codes") or []),
|
||
}
|
||
driver_recommendations = _build_driver_recommendations(
|
||
managed_nodes=managed_nodes,
|
||
execution_scene=execution_scene,
|
||
release_summary=release_summary,
|
||
release_gate=release_gate,
|
||
release_launchpad=release_launchpad,
|
||
)
|
||
recommendation = _build_ops_priority_recommendation(driver_recommendations)
|
||
|
||
return {
|
||
"mode": "centralized-ops",
|
||
"vision": "海外控制面统一接管大陆节点安装、更新、巡检、日志回传与故障诊断。",
|
||
"current_topology": {
|
||
"node_code": settings.node_code,
|
||
"node_region": settings.node_region,
|
||
"node_role": settings.node_role,
|
||
"worker_mode": settings.worker_mode,
|
||
"sync_push_enabled": bool(settings.sync_push_enabled),
|
||
"sync_target_api_base_url": str(settings.sync_target_api_base_url or "").strip(),
|
||
},
|
||
"cluster_summary": {
|
||
"online_control_nodes": online_control_nodes,
|
||
"online_worker_nodes": online_worker_nodes,
|
||
"dedicated_online_worker_nodes": int((cluster.get("summary") or {}).get("dedicated_online_worker_nodes", 0) or 0),
|
||
"busy_nodes": list((cluster.get("summary") or {}).get("busy_nodes") or []),
|
||
},
|
||
"cluster_nodes": [
|
||
{
|
||
"node_code": str(item.get("node_code") or ""),
|
||
"region": str(item.get("region") or ""),
|
||
"role": str(item.get("role") or ""),
|
||
"status": str(item.get("status") or ""),
|
||
"current_load": int(item.get("current_load", 0) or 0),
|
||
"hostname": str(item.get("hostname") or ""),
|
||
"last_heartbeat_at": str(item.get("last_heartbeat_at") or ""),
|
||
"is_effective_worker": bool(item.get("is_effective_worker", False)),
|
||
"detect_participating": bool(item.get("detect_participating", False)),
|
||
}
|
||
for item in list(cluster.get("nodes") or [])
|
||
if str(item.get("node_code") or "").strip()
|
||
],
|
||
"runtime_summary": {
|
||
"readiness_status": str(readiness.get("status") or ""),
|
||
"readiness_summary": str(readiness.get("summary") or ""),
|
||
"phase_label": str(detect.get("phase_label") or ""),
|
||
"phase_detail": str(detect.get("phase_detail") or ""),
|
||
"active_job_code": str((detect.get("active_job") or {}).get("job_code") or ""),
|
||
"progress_percent": float(detect.get("progress_percent", 0) or 0),
|
||
},
|
||
"execution_scene": execution_scene,
|
||
"sync_summary": {
|
||
"enabled": bool(sync_summary.get("enabled", False)),
|
||
"source_region": str(sync_summary.get("source_region") or ""),
|
||
"target_region": str(sync_summary.get("target_region") or ""),
|
||
"projected_batches": projected_batches,
|
||
"failed_batches": failed_batches,
|
||
"records_total": int(sync_summary.get("records_total", 0) or 0),
|
||
},
|
||
"ops_jobs": job_summary,
|
||
"managed_nodes": {
|
||
"total": int(managed_nodes_summary.get("total", len(managed_nodes)) or 0),
|
||
"managed_total": int(managed_nodes_summary.get("managed_total", 0) or 0),
|
||
"enabled": int(managed_nodes_summary.get("managed_enabled", 0) or 0),
|
||
"online": int(managed_nodes_summary.get("online", 0) or 0),
|
||
"stale": int(managed_nodes_summary.get("stale", 0) or 0),
|
||
"pending_bootstrap": int(managed_nodes_summary.get("pending_bootstrap", 0) or 0),
|
||
"runtime_only": int(managed_nodes_summary.get("runtime_only", 0) or 0),
|
||
"unmanaged": int(managed_nodes_summary.get("unmanaged", 0) or 0),
|
||
"token_issue": int(managed_nodes_summary.get("token_issue", 0) or 0),
|
||
"queue_retrying_nodes": int(managed_nodes_summary.get("queue_retrying_nodes", 0) or 0),
|
||
"queue_dead_letter_nodes": int(managed_nodes_summary.get("queue_dead_letter_nodes", 0) or 0),
|
||
"queue_pending_records": int(managed_nodes_summary.get("queue_pending_records", 0) or 0),
|
||
"queue_dead_letter_records": int(managed_nodes_summary.get("queue_dead_letter_records", 0) or 0),
|
||
"agent_state_counts": dict(managed_nodes_summary.get("status_counts") or {}),
|
||
"delivery_queue_state_counts": dict(managed_nodes_summary.get("delivery_queue_state_counts") or {}),
|
||
},
|
||
"inspection": inspection_overview,
|
||
"activity_stream": activity_stream,
|
||
"driver_recommendations": driver_recommendations,
|
||
"release_hub": {
|
||
**release_summary,
|
||
"preferred_release": preferred_release,
|
||
"default_rollout_execution": release_execution_mode,
|
||
"default_rollout_gate": release_gate,
|
||
"default_rollout_gate_options": dict(release_execution_mode.get("gates") or {}),
|
||
"launchpad": release_launchpad,
|
||
},
|
||
"recommendation": recommendation,
|
||
}
|
||
|
||
|
||
def _build_ops_link_snapshot_from_overview(overview: dict) -> dict:
|
||
overview = dict(overview or {})
|
||
topology = dict(overview.get("current_topology") or {})
|
||
cluster_summary = dict(overview.get("cluster_summary") or {})
|
||
runtime_summary = dict(overview.get("runtime_summary") or {})
|
||
execution_scene = dict(overview.get("execution_scene") or {})
|
||
participation_summary = dict(execution_scene.get("participation_summary") or {})
|
||
log_sync = dict(execution_scene.get("log_sync") or {})
|
||
sync_summary = dict(overview.get("sync_summary") or {})
|
||
recommendation = dict(overview.get("recommendation") or {})
|
||
|
||
readiness_status = str(runtime_summary.get("readiness_status") or "").strip() or "attention"
|
||
readiness_summary = str(runtime_summary.get("readiness_summary") or "").strip()
|
||
node_code = str(topology.get("node_code") or "").strip()
|
||
node_region = str(topology.get("node_region") or "").strip()
|
||
node_role = str(topology.get("node_role") or "").strip()
|
||
worker_mode = str(topology.get("worker_mode") or "").strip()
|
||
active_job_code = str(runtime_summary.get("active_job_code") or "").strip()
|
||
phase_label = str(runtime_summary.get("phase_label") or "").strip()
|
||
phase_detail = str(runtime_summary.get("phase_detail") or "").strip()
|
||
progress_percent = float(runtime_summary.get("progress_percent", 0) or 0)
|
||
|
||
online_control_nodes = int(cluster_summary.get("online_control_nodes", 0) or 0)
|
||
online_worker_nodes = int(cluster_summary.get("online_worker_nodes", 0) or 0)
|
||
dedicated_online_worker_nodes = int(cluster_summary.get("dedicated_online_worker_nodes", 0) or 0)
|
||
busy_nodes = list(cluster_summary.get("busy_nodes") or [])
|
||
|
||
effective_online_nodes = int(
|
||
participation_summary.get("effective_online_nodes", online_worker_nodes) or online_worker_nodes or 0
|
||
)
|
||
participating_nodes = int(participation_summary.get("participating_nodes", 0) or 0)
|
||
dispatch_active_nodes = int(participation_summary.get("dispatch_active_nodes", 0) or 0)
|
||
recent_only_nodes = int(participation_summary.get("recent_only_nodes", 0) or 0)
|
||
non_participating_nodes = int(participation_summary.get("non_participating_nodes", 0) or 0)
|
||
standby_nodes = int(participation_summary.get("standby_nodes", 0) or 0)
|
||
load_syncing_nodes = int(participation_summary.get("load_syncing_nodes", 0) or 0)
|
||
dispatch_active_node_codes = list(participation_summary.get("dispatch_active_node_codes") or [])
|
||
non_participating_node_codes = list(participation_summary.get("non_participating_node_codes") or [])
|
||
|
||
log_sync_enabled = bool(log_sync.get("enabled", False))
|
||
log_sync_mode = str(log_sync.get("mode") or "").strip() or "key"
|
||
log_sync_samples = int(log_sync.get("line_count", 0) or 0)
|
||
log_sync_sources = int(log_sync.get("source_node_count", 0) or 0)
|
||
missing_log_nodes = list(log_sync.get("missing_participating_nodes") or [])
|
||
covered_participating_nodes = int(log_sync.get("covered_participating_node_count", 0) or 0)
|
||
participating_node_count = int(log_sync.get("participating_node_count", participating_nodes) or participating_nodes or 0)
|
||
log_sync_last_at = str(log_sync.get("last_at") or "").strip()
|
||
log_sync_preview_lines = [str(item or "").strip() for item in list(log_sync.get("preview_lines") or []) if str(item or "").strip()]
|
||
|
||
sync_enabled = bool(sync_summary.get("enabled", False))
|
||
sync_source_region = str(sync_summary.get("source_region") or "").strip()
|
||
sync_target_region = str(sync_summary.get("target_region") or "").strip()
|
||
projected_batches = int(sync_summary.get("projected_batches", 0) or 0)
|
||
failed_batches = int(sync_summary.get("failed_batches", 0) or 0)
|
||
sync_records_total = int(sync_summary.get("records_total", 0) or 0)
|
||
|
||
if effective_online_nodes <= 0:
|
||
status = "blocking"
|
||
headline = "当前没有有效执行节点,海外控制面暂时无法继续推进联调、巡检或检测任务。"
|
||
elif dispatch_active_nodes > 0 and not log_sync_enabled:
|
||
status = "attention"
|
||
headline = "当前已有执行节点,但远端日志回传仍关闭,海外控制面对现场仍处于半盲态。"
|
||
elif dispatch_active_nodes > 0 and missing_log_nodes:
|
||
status = "attention"
|
||
headline = "当前已有执行节点,但远端日志回传仍未覆盖全部参与节点,建议先补齐现场观测。"
|
||
elif failed_batches > 0:
|
||
status = "attention"
|
||
headline = "当前同步链路存在失败批次,建议先收口跨地域同步状态,再继续放量联调。"
|
||
elif readiness_status != "ready":
|
||
status = readiness_status
|
||
headline = readiness_summary or "当前多机现场仍有待处理事项,建议先按运行中心提示逐项收口。"
|
||
elif active_job_code:
|
||
status = "ready"
|
||
headline = "当前多机现场已进入可观测状态,可以直接从海外控制面继续盯检测任务和节点执行过程。"
|
||
else:
|
||
status = "ready"
|
||
headline = "当前多机骨架已就绪,可以继续拉取任务、做巡检验收或推进发布演练。"
|
||
|
||
summary_lines = [
|
||
f"控制面节点 {node_code or '-'} / 区域 {node_region or '-'} / 角色 {node_role or '-'} / 托管方式 {worker_mode or '-'}",
|
||
f"集群在线:控制面 {online_control_nodes} 台 / 有效执行节点 {effective_online_nodes} 台 / 独立 Worker {dedicated_online_worker_nodes} 台 / 忙碌节点 {_format_node_code_list(busy_nodes)}",
|
||
(
|
||
f"活跃任务 {active_job_code} / 阶段 {phase_label or '-'} / 进度 {progress_percent:.1f}%"
|
||
if active_job_code
|
||
else f"当前无活跃任务 / 就绪度 {readiness_status} / 摘要 {readiness_summary or '-'}"
|
||
),
|
||
(
|
||
f"执行现场:参与 {participating_nodes} 台 / 执行中或已领 {dispatch_active_nodes} 台 / 近窗吞吐 {recent_only_nodes} 台 / 在线未参与 {non_participating_nodes} 台"
|
||
),
|
||
(
|
||
f"远端日志:{'开启' if log_sync_enabled else '关闭'} / 模式 {log_sync_mode} / 覆盖 {covered_participating_nodes}/{participating_node_count} / 样本 {log_sync_samples} / 缺口 {_format_node_code_list(missing_log_nodes)}"
|
||
),
|
||
(
|
||
f"跨地域同步:{'启用' if sync_enabled else '未启用'} / {sync_source_region or '-'} -> {sync_target_region or '-'} / projected {projected_batches} / failed {failed_batches} / records {sync_records_total}"
|
||
),
|
||
]
|
||
if phase_detail:
|
||
summary_lines.append(f"现场阶段说明:{phase_detail}")
|
||
if log_sync_last_at:
|
||
summary_lines.append(f"最近日志回传:{log_sync_last_at}")
|
||
|
||
next_actions: list[dict] = []
|
||
if dispatch_active_nodes > 0 and not log_sync_enabled:
|
||
target_node_codes = list(dispatch_active_node_codes or [])
|
||
next_actions = [
|
||
{
|
||
"label": "开启关键回传",
|
||
"action_code": "enable_log_sync_key",
|
||
"node_codes": target_node_codes,
|
||
"payload": {},
|
||
},
|
||
{
|
||
"label": "开启全量回传",
|
||
"action_code": "enable_log_sync_full",
|
||
"node_codes": target_node_codes,
|
||
"payload": {},
|
||
},
|
||
]
|
||
elif dispatch_active_nodes > 0 and missing_log_nodes:
|
||
target_node_codes = list(dispatch_active_node_codes or missing_log_nodes or [])
|
||
next_actions = [
|
||
{
|
||
"label": "看参与节点日志",
|
||
"action_code": "open_worker_logs_participating",
|
||
"node_codes": target_node_codes,
|
||
"payload": {},
|
||
},
|
||
{
|
||
"label": "执行标准巡检",
|
||
"action_code": "run_inspection_participating",
|
||
"node_codes": target_node_codes,
|
||
"payload": {},
|
||
},
|
||
]
|
||
else:
|
||
primary_label = str(recommendation.get("primary_label") or "").strip()
|
||
primary_action_code = str(recommendation.get("primary_action_code") or "").strip()
|
||
if primary_label or primary_action_code:
|
||
next_actions.append(
|
||
{
|
||
"label": primary_label or "执行建议动作",
|
||
"action_code": primary_action_code,
|
||
"node_codes": list(recommendation.get("primary_node_codes") or recommendation.get("node_codes") or []),
|
||
"payload": dict(recommendation.get("primary_action_payload") or {}),
|
||
}
|
||
)
|
||
secondary_label = str(recommendation.get("secondary_label") or "").strip()
|
||
secondary_action_code = str(recommendation.get("secondary_action_code") or "").strip()
|
||
if secondary_label or secondary_action_code:
|
||
next_actions.append(
|
||
{
|
||
"label": secondary_label or "执行次要动作",
|
||
"action_code": secondary_action_code,
|
||
"node_codes": list(recommendation.get("secondary_node_codes") or recommendation.get("node_codes") or []),
|
||
"payload": dict(recommendation.get("secondary_action_payload") or {}),
|
||
}
|
||
)
|
||
|
||
return {
|
||
"generated_at": datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
|
||
"status": status,
|
||
"headline": headline,
|
||
"summary_lines": summary_lines,
|
||
"control_plane": {
|
||
"node_code": node_code,
|
||
"region": node_region,
|
||
"role": node_role,
|
||
"worker_mode": worker_mode,
|
||
"sync_push_enabled": bool(topology.get("sync_push_enabled", False)),
|
||
"sync_target_api_base_url": str(topology.get("sync_target_api_base_url") or "").strip(),
|
||
},
|
||
"cluster": {
|
||
"online_control_nodes": online_control_nodes,
|
||
"online_worker_nodes": online_worker_nodes,
|
||
"dedicated_online_worker_nodes": dedicated_online_worker_nodes,
|
||
"effective_online_nodes": effective_online_nodes,
|
||
"busy_nodes": list(busy_nodes),
|
||
},
|
||
"execution": {
|
||
"active_job_code": active_job_code,
|
||
"phase_label": phase_label,
|
||
"phase_detail": phase_detail,
|
||
"progress_percent": progress_percent,
|
||
"participating_nodes": participating_nodes,
|
||
"dispatch_active_nodes": dispatch_active_nodes,
|
||
"recent_only_nodes": recent_only_nodes,
|
||
"non_participating_nodes": non_participating_nodes,
|
||
"standby_nodes": standby_nodes,
|
||
"load_syncing_nodes": load_syncing_nodes,
|
||
"dispatch_active_node_codes": list(dispatch_active_node_codes),
|
||
"non_participating_node_codes": list(non_participating_node_codes),
|
||
},
|
||
"log_sync": {
|
||
"enabled": log_sync_enabled,
|
||
"mode": log_sync_mode,
|
||
"line_count": log_sync_samples,
|
||
"source_node_count": log_sync_sources,
|
||
"covered_participating_node_count": covered_participating_nodes,
|
||
"participating_node_count": participating_node_count,
|
||
"missing_participating_nodes": list(missing_log_nodes),
|
||
"last_at": log_sync_last_at,
|
||
"preview_lines": log_sync_preview_lines,
|
||
},
|
||
"sync": {
|
||
"enabled": sync_enabled,
|
||
"source_region": sync_source_region,
|
||
"target_region": sync_target_region,
|
||
"projected_batches": projected_batches,
|
||
"failed_batches": failed_batches,
|
||
"records_total": sync_records_total,
|
||
},
|
||
"recommendation": {
|
||
"key": str(recommendation.get("key") or "").strip(),
|
||
"priority": str(recommendation.get("priority") or "").strip(),
|
||
"summary": str(recommendation.get("summary") or "").strip(),
|
||
"reason": str(recommendation.get("reason") or "").strip(),
|
||
},
|
||
"next_actions": next_actions,
|
||
}
|
||
|
||
|
||
def get_ops_link_snapshot(overview: dict | None = None) -> dict:
|
||
normalized_overview = dict(overview or {})
|
||
if not normalized_overview:
|
||
normalized_overview = get_ops_overview()
|
||
return _build_ops_link_snapshot_from_overview(normalized_overview)
|
||
|
||
|
||
def _normalize_ops_node_code_list(*groups: object) -> list[str]:
|
||
normalized: list[str] = []
|
||
seen: set[str] = set()
|
||
for group in groups:
|
||
if isinstance(group, dict):
|
||
group = [group]
|
||
if not isinstance(group, (list, tuple, set)):
|
||
continue
|
||
for item in group:
|
||
if isinstance(item, dict):
|
||
node_code = str(item.get("node_code") or "").strip()
|
||
else:
|
||
node_code = str(item or "").strip()
|
||
if not node_code or node_code in seen:
|
||
continue
|
||
seen.add(node_code)
|
||
normalized.append(node_code)
|
||
return normalized
|
||
|
||
|
||
def _build_scene_node_log_command(base_url: str, node_code: str, *, limit: int = 80, mode: str = "key") -> str:
|
||
normalized_node_code = str(node_code or "").strip()
|
||
if not normalized_node_code:
|
||
return ""
|
||
normalized_base_url = _ops_stack_api_base_url(base_url)
|
||
normalized_mode = _normalize_scene_log_mode(mode, fallback="key")
|
||
normalized_limit = max(20, min(int(limit or 80), 300))
|
||
return build_bash_command(
|
||
"drive_ops_center.sh",
|
||
"scene-node-log",
|
||
normalized_base_url,
|
||
normalized_node_code,
|
||
normalized_limit,
|
||
normalized_mode,
|
||
)
|
||
|
||
|
||
def _build_scene_node_log_focus_ref(
|
||
node_code: str,
|
||
*,
|
||
mode: str = "key",
|
||
limit: int = 80,
|
||
source: str = "stack_diagnosis",
|
||
) -> dict:
|
||
normalized_node_code = str(node_code or "").strip()
|
||
if not normalized_node_code:
|
||
return {}
|
||
return _merge_focus_ref(
|
||
{},
|
||
kind="node_scene_log",
|
||
node_code=normalized_node_code,
|
||
mode=_normalize_scene_log_mode(mode, fallback="key"),
|
||
limit=max(20, min(int(limit or 80), 300)),
|
||
source=source,
|
||
)
|
||
|
||
|
||
def _build_scene_node_log_commands(
|
||
base_url: str,
|
||
node_codes: list[str] | tuple[str, ...] | set[str],
|
||
*,
|
||
limit: int = 80,
|
||
mode: str = "key",
|
||
max_commands: int = 3,
|
||
) -> list[str]:
|
||
commands: list[str] = []
|
||
for node_code in _normalize_ops_node_code_list(list(node_codes or []))[: max(1, int(max_commands or 3))]:
|
||
command = _build_scene_node_log_command(base_url, node_code, limit=limit, mode=mode)
|
||
if command:
|
||
commands.append(command)
|
||
return commands
|
||
|
||
|
||
def _ops_stack_api_base_url(base_url: str = "") -> str:
|
||
normalized_base_url = str(base_url or "").strip()
|
||
if normalized_base_url:
|
||
return normalized_base_url.rstrip("/")
|
||
return f"http://127.0.0.1:{settings.api_port}"
|
||
|
||
|
||
def _ops_stack_capture(section_key: str, callback) -> dict:
|
||
started_at = time.perf_counter()
|
||
try:
|
||
payload = callback() or {}
|
||
except Exception as exc:
|
||
elapsed_ms = int((time.perf_counter() - started_at) * 1000)
|
||
return {
|
||
"section_key": str(section_key or "").strip(),
|
||
"http_status": "500",
|
||
"available": False,
|
||
"data": {},
|
||
"error": f"{exc.__class__.__name__}: {exc}",
|
||
"elapsed_ms": elapsed_ms,
|
||
}
|
||
if isinstance(payload, dict):
|
||
data = dict(payload or {})
|
||
else:
|
||
data = {"value": payload}
|
||
elapsed_ms = int((time.perf_counter() - started_at) * 1000)
|
||
return {
|
||
"section_key": str(section_key or "").strip(),
|
||
"http_status": "200",
|
||
"available": True,
|
||
"data": data,
|
||
"error": "",
|
||
"elapsed_ms": elapsed_ms,
|
||
}
|
||
|
||
|
||
def _ops_stack_issue_rank(severity: str) -> int:
|
||
return {"blocked": 3, "warning": 2, "info": 1}.get(str(severity or "").strip(), 0)
|
||
|
||
|
||
def _suggest_ops_contract_keys_for_stack_issue(issue: dict) -> list[str]:
|
||
normalized_issue = dict(issue or {})
|
||
layer = str(normalized_issue.get("layer") or "").strip()
|
||
contract_keys: list[str] = ["ops_stack_diagnosis_contract"]
|
||
if layer.startswith("managed_nodes"):
|
||
contract_keys.append("ops_agent_protocol")
|
||
elif layer == "release_hub":
|
||
contract_keys.append("release_hub_contract")
|
||
elif layer == "playbook_runs":
|
||
contract_keys.append("ops_playbook_contract")
|
||
elif layer in {"overview.log_sync", "activity_stream", "link_snapshot"}:
|
||
contract_keys.append("ops_observability_contract")
|
||
elif layer == "contracts":
|
||
contract_keys.append("ops_driver_contract")
|
||
|
||
contract_keys.extend(_ops_contract_keys_from_focus_ref(normalized_issue.get("focus_ref")))
|
||
contract_keys.extend(_ops_contract_keys_from_action_code(str(normalized_issue.get("action_code") or "").strip()))
|
||
return _normalize_ops_contract_keys(contract_keys)
|
||
|
||
|
||
def _build_stack_diagnosis_recommended_commands(
|
||
*,
|
||
stack_base_url: str,
|
||
recommended_actions: list[dict],
|
||
first_handover_gap_node_code: str = "",
|
||
launchpad_status: dict | None = None,
|
||
default_rollout_gate: dict | None = None,
|
||
next_step: dict | None = None,
|
||
) -> dict:
|
||
commands: dict[str, str] = {}
|
||
|
||
commands["stack-diagnosis"] = build_bash_command("drive_ops_center.sh", "stack-diagnosis", stack_base_url, "summary")
|
||
commands["stack-check"] = commands["stack-diagnosis"]
|
||
commands["doctor"] = build_bash_command("drive_ops_center.sh", "doctor", stack_base_url)
|
||
commands["contracts"] = build_bash_command("drive_ops_center.sh", "contracts", stack_base_url)
|
||
commands["runtime-refresh-recover"] = build_bash_command(
|
||
"drive_ops_center.sh",
|
||
"runtime-refresh-recover",
|
||
stack_base_url,
|
||
)
|
||
commands["api-restart"] = commands["runtime-refresh-recover"]
|
||
|
||
for action in list(recommended_actions or []):
|
||
action_code = str(action.get("action_code") or "").strip()
|
||
command = str(action.get("command") or "").strip()
|
||
if action_code and command and action_code not in commands:
|
||
commands[action_code] = command
|
||
|
||
normalized_gap_node_code = str(first_handover_gap_node_code or "").strip()
|
||
if normalized_gap_node_code:
|
||
commands["managed-node-handover"] = build_bash_command("drive_ops_center.sh", "node-handover", stack_base_url, normalized_gap_node_code)
|
||
commands["managed-node-recover"] = build_bash_command("drive_ops_center.sh", "node-recover", stack_base_url, normalized_gap_node_code)
|
||
commands["managed-node-bootstrap-plan"] = build_bash_command("drive_ops_center.sh", "node-bootstrap-plan", stack_base_url, normalized_gap_node_code)
|
||
|
||
normalized_next_step = dict(next_step or {})
|
||
next_step_focus_ref = dict(normalized_next_step.get("focus_ref") or {})
|
||
if str(next_step_focus_ref.get("kind") or "").strip() == "node_scene_log":
|
||
node_code = str(next_step_focus_ref.get("node_code") or "").strip()
|
||
if node_code:
|
||
commands["scene-node-log"] = _build_scene_node_log_command(
|
||
stack_base_url,
|
||
node_code,
|
||
limit=int(next_step_focus_ref.get("limit", 120) or 120),
|
||
mode=str(next_step_focus_ref.get("mode") or "key"),
|
||
)
|
||
|
||
normalized_launchpad_status = dict(launchpad_status or {})
|
||
normalized_default_rollout_gate = dict(default_rollout_gate or {})
|
||
launchpad_focus_ref = dict(
|
||
normalized_launchpad_status.get("focus_ref")
|
||
or normalized_default_rollout_gate.get("focus_ref")
|
||
or {}
|
||
)
|
||
if str(launchpad_focus_ref.get("section") or "").strip() == "release_launchpad":
|
||
commands["focus-release-hub"] = build_bash_command("drive_ops_center.sh", "release-launchpad", stack_base_url)
|
||
mode = str(launchpad_focus_ref.get("mode") or "").strip()
|
||
if mode == "control":
|
||
commands["release-launchpad-control"] = build_bash_command(
|
||
"drive_ops_center.sh",
|
||
"release-launchpad",
|
||
stack_base_url,
|
||
"control",
|
||
)
|
||
else:
|
||
commands["release-launchpad-worker"] = build_bash_command(
|
||
"drive_ops_center.sh",
|
||
"release-launchpad",
|
||
stack_base_url,
|
||
"worker",
|
||
)
|
||
launchpad_recommended_action_code = str(normalized_launchpad_status.get("recommended_action_code") or "").strip()
|
||
launchpad_recommended_target_node_code = str(
|
||
normalized_launchpad_status.get("recommended_target_node_code") or ""
|
||
).strip()
|
||
if (
|
||
launchpad_recommended_action_code in {"bootstrap_run", "run_acceptance"}
|
||
and launchpad_recommended_target_node_code
|
||
):
|
||
commands[launchpad_recommended_action_code] = build_bash_command(
|
||
"drive_ops_center.sh",
|
||
"node-bootstrap-run" if launchpad_recommended_action_code == "bootstrap_run" else "node-acceptance-run",
|
||
stack_base_url,
|
||
launchpad_recommended_target_node_code,
|
||
"cli",
|
||
)
|
||
|
||
return {key: value for key, value in commands.items() if str(value or "").strip()}
|
||
|
||
|
||
def _resolve_first_handover_gap_recovery(
|
||
*,
|
||
first_handover_gap: dict | None = None,
|
||
launchpad_gap_action: dict | None = None,
|
||
launchpad_status: dict | None = None,
|
||
fallback_node_code: str = "",
|
||
) -> dict:
|
||
normalized_handover = dict(first_handover_gap or {})
|
||
stage = dict(normalized_handover.get("stage") or {})
|
||
stage_next_step = dict(stage.get("next_step") or {})
|
||
normalized_launchpad_gap_action = dict(launchpad_gap_action or {})
|
||
normalized_launchpad_status = dict(launchpad_status or {})
|
||
|
||
node_code = str(
|
||
normalized_handover.get("node_code")
|
||
or normalized_launchpad_gap_action.get("node_code")
|
||
or normalized_launchpad_status.get("recommended_target_node_code")
|
||
or fallback_node_code
|
||
or ""
|
||
).strip()
|
||
summary = str(
|
||
normalized_handover.get("summary")
|
||
or stage.get("summary")
|
||
or normalized_launchpad_gap_action.get("summary")
|
||
or normalized_launchpad_status.get("recommended_recovery_summary")
|
||
or ""
|
||
).strip()
|
||
stage_code = str(stage.get("code") or "").strip()
|
||
stage_next_step_code = str(stage_next_step.get("code") or "").strip()
|
||
has_gap_signal = bool(node_code or stage_code or stage_next_step_code or summary)
|
||
|
||
action_code = str(normalized_launchpad_gap_action.get("action_code") or "").strip()
|
||
if not has_gap_signal:
|
||
action_code = ""
|
||
elif action_code not in {"bootstrap_run", "run_acceptance"}:
|
||
if stage_next_step_code in {"run_acceptance", "acceptance", "ready"} or stage_code in {"acceptance_ready"}:
|
||
action_code = "run_acceptance"
|
||
elif stage_next_step_code in {"execute_bootstrap_plan", "bootstrap_run"} or stage_code in {
|
||
"pending_bootstrap",
|
||
"runtime_only",
|
||
"profile_incomplete",
|
||
"ssh_ready",
|
||
"token_issue",
|
||
"stale",
|
||
"agent_pending",
|
||
"unknown",
|
||
"unmanaged",
|
||
}:
|
||
action_code = "bootstrap_run"
|
||
else:
|
||
action_code = ""
|
||
|
||
recovery_label = str(normalized_launchpad_status.get("recommended_recovery_label") or "").strip()
|
||
if not recovery_label and action_code == "bootstrap_run":
|
||
recovery_label = "签发接入工单"
|
||
elif not recovery_label and action_code == "run_acceptance":
|
||
recovery_label = "执行接管验收"
|
||
if not recovery_label:
|
||
recovery_label = str(stage_next_step.get("label") or stage.get("label") or "").strip()
|
||
|
||
focus_ref = dict(
|
||
(dict(normalized_launchpad_gap_action.get("row") or {}).get("focus_ref") or {})
|
||
or normalized_launchpad_status.get("focus_ref")
|
||
or {}
|
||
)
|
||
if node_code and not str(focus_ref.get("node_code") or "").strip():
|
||
focus_ref["node_code"] = node_code
|
||
if node_code and not str(focus_ref.get("kind") or "").strip():
|
||
focus_ref["kind"] = "managed_node"
|
||
|
||
return {
|
||
"action_code": action_code if action_code in {"bootstrap_run", "run_acceptance"} else "",
|
||
"node_code": node_code,
|
||
"summary": summary,
|
||
"recovery_label": recovery_label,
|
||
"focus_ref": focus_ref,
|
||
"stage_code": stage_code,
|
||
"stage_label": str(stage.get("label") or "").strip(),
|
||
"stage_next_step_code": stage_next_step_code,
|
||
}
|
||
|
||
|
||
def _build_stack_diagnosis_operator_decision(
|
||
*,
|
||
issues: list[dict],
|
||
next_step: dict,
|
||
recommended_actions: list[dict],
|
||
recommended_commands: dict,
|
||
first_handover_gap_node_code: str = "",
|
||
launchpad_status: dict | None = None,
|
||
default_rollout_gate: dict | None = None,
|
||
) -> tuple[dict, dict]:
|
||
top_issue = (
|
||
sorted(
|
||
list(issues or []),
|
||
key=lambda item: (
|
||
-_ops_stack_issue_rank(str(item.get("severity") or "").strip()),
|
||
str(item.get("code") or "").strip(),
|
||
),
|
||
)[0]
|
||
if issues
|
||
else {}
|
||
)
|
||
normalized_next_step = dict(next_step or {})
|
||
primary_action_code = str(normalized_next_step.get("action_code") or "").strip()
|
||
next_focus_ref = dict(normalized_next_step.get("focus_ref") or {})
|
||
normalized_launchpad_status = dict(launchpad_status or {})
|
||
normalized_default_rollout_gate = dict(default_rollout_gate or {})
|
||
|
||
operator_decision = {
|
||
"lane": "steady",
|
||
"priority": "low",
|
||
"reason_code": "steady_state",
|
||
"title": "当前总检已进入稳定观察态",
|
||
"summary": "控制面总检、执行现场、纳管与发布骨架当前没有阻断项,可以按既定节奏继续联调或推进发布。",
|
||
"next_focus": next_focus_ref,
|
||
"primary_command_key": primary_action_code,
|
||
"secondary_command_key": "",
|
||
}
|
||
|
||
if str(next_focus_ref.get("kind") or "").strip() == "node_scene_log":
|
||
operator_decision = {
|
||
"lane": "observability",
|
||
"priority": "high" if top_issue else "medium",
|
||
"reason_code": str(top_issue.get("code") or "scene_log_observation").strip(),
|
||
"title": "优先补齐执行现场日志观察",
|
||
"summary": str(top_issue.get("summary") or normalized_next_step.get("reason") or "").strip()
|
||
or "当前已经定位到参与节点现场日志焦点,建议先看现场,再决定是否继续接管或推进发布。",
|
||
"next_focus": next_focus_ref,
|
||
"primary_command_key": "scene-node-log",
|
||
"secondary_command_key": primary_action_code or "open_worker_logs_participating",
|
||
}
|
||
elif str(top_issue.get("code") or "").strip() == "runtime_build_schema_stale":
|
||
secondary_key = "stack-diagnosis" if "stack-diagnosis" in recommended_commands else ""
|
||
if not secondary_key and "stack-check" in recommended_commands:
|
||
secondary_key = "stack-check"
|
||
if "managed-node-bootstrap-plan" in recommended_commands:
|
||
secondary_key = "managed-node-bootstrap-plan"
|
||
operator_decision = {
|
||
"lane": "runtime_recovery",
|
||
"priority": "high",
|
||
"reason_code": "runtime_build_schema_stale",
|
||
"title": "优先重启控制面 API 并复检运行时版本",
|
||
"summary": str(top_issue.get("summary") or normalized_next_step.get("reason") or "").strip()
|
||
or "仓库代码已经具备新能力,但运行中的 API 仍停留在旧 schema,当前第一动作应先让服务吃到最新代码。",
|
||
"next_focus": dict(top_issue.get("focus_ref") or normalized_next_step.get("focus_ref") or {}),
|
||
"primary_command_key": "api-restart",
|
||
"secondary_command_key": secondary_key,
|
||
}
|
||
elif str(top_issue.get("code") or "").strip() == "managed_nodes_agent_pending" or (
|
||
str(top_issue.get("layer") or "").strip().startswith("managed_nodes")
|
||
and str(first_handover_gap_node_code or "").strip()
|
||
):
|
||
managed_primary_key = primary_action_code
|
||
if managed_primary_key not in {"bootstrap_run", "run_acceptance"}:
|
||
managed_primary_key = (
|
||
"managed-node-recover" if "managed-node-recover" in recommended_commands else "managed-node-handover"
|
||
)
|
||
managed_secondary_key = ""
|
||
if managed_primary_key in {"bootstrap_run", "run_acceptance"}:
|
||
if "managed-node-bootstrap-plan" in recommended_commands:
|
||
managed_secondary_key = "managed-node-bootstrap-plan"
|
||
elif "managed-node-handover" in recommended_commands:
|
||
managed_secondary_key = "managed-node-handover"
|
||
elif "fix_managed_nodes" in recommended_commands:
|
||
managed_secondary_key = "fix_managed_nodes"
|
||
elif primary_action_code and primary_action_code != managed_primary_key:
|
||
managed_secondary_key = primary_action_code
|
||
operator_decision = {
|
||
"lane": "node_handover",
|
||
"priority": "high",
|
||
"reason_code": str(top_issue.get("code") or "managed_nodes_agent_pending").strip(),
|
||
"title": "优先执行节点接管收口" if managed_primary_key in {"bootstrap_run", "run_acceptance"} else "优先打通节点接管链路",
|
||
"summary": (
|
||
str(normalized_next_step.get("reason") or "").strip()
|
||
if managed_primary_key in {"bootstrap_run", "run_acceptance"}
|
||
else ""
|
||
)
|
||
or str(top_issue.get("summary") or "").strip()
|
||
or "托管节点还没有进入 remote-agent ready,海外控制面暂时不能把执行统一收口到标准动作链。",
|
||
"next_focus": dict(top_issue.get("focus_ref") or normalized_next_step.get("focus_ref") or {}),
|
||
"primary_command_key": managed_primary_key,
|
||
"secondary_command_key": managed_secondary_key,
|
||
}
|
||
elif (
|
||
str(top_issue.get("code") or "").strip() == "release_rollout_gate_blocked"
|
||
or str(normalized_default_rollout_gate.get("status") or "").strip() == "blocked"
|
||
or str(normalized_launchpad_status.get("recommended_action_code") or "").strip() in {
|
||
"review_smart_rollout_preview",
|
||
"review_control_rollout",
|
||
"fix_rollout_blockers",
|
||
}
|
||
):
|
||
launchpad_focus_ref = dict(
|
||
normalized_default_rollout_gate.get("focus_ref")
|
||
or normalized_launchpad_status.get("focus_ref")
|
||
or normalized_next_step.get("focus_ref")
|
||
or {}
|
||
)
|
||
launchpad_mode = str(launchpad_focus_ref.get("mode") or "").strip()
|
||
primary_key = "release-launchpad-control" if launchpad_mode == "control" else "release-launchpad-worker"
|
||
if primary_key not in recommended_commands:
|
||
primary_key = "focus-release-hub"
|
||
operator_decision = {
|
||
"lane": "release",
|
||
"priority": "high" if str(normalized_default_rollout_gate.get("status") or "").strip() == "blocked" else "medium",
|
||
"reason_code": str(top_issue.get("code") or "release_launchpad_attention").strip(),
|
||
"title": "优先处理发布门禁与 Launchpad 决策",
|
||
"summary": str(top_issue.get("summary") or normalized_default_rollout_gate.get("summary_text") or "").strip()
|
||
or "发布包和 Rollout 入口已经就位,但默认门禁还没完全放行,建议先看 Release Hub 再推进下一批。",
|
||
"next_focus": launchpad_focus_ref,
|
||
"primary_command_key": primary_key,
|
||
"secondary_command_key": primary_action_code or str(normalized_launchpad_status.get("recommended_action_code") or "").strip(),
|
||
}
|
||
elif str(normalized_launchpad_status.get("recommended_action_code") or "").strip() in {"bootstrap_run", "run_acceptance"}:
|
||
launchpad_action_code = str(normalized_launchpad_status.get("recommended_action_code") or "").strip()
|
||
launchpad_target_node_code = str(normalized_launchpad_status.get("recommended_target_node_code") or "").strip()
|
||
launchpad_focus_ref = dict(
|
||
normalized_launchpad_status.get("focus_ref")
|
||
or normalized_default_rollout_gate.get("focus_ref")
|
||
or normalized_next_step.get("focus_ref")
|
||
or {}
|
||
)
|
||
operator_decision = {
|
||
"lane": "node_handover",
|
||
"priority": "high" if launchpad_action_code == "bootstrap_run" else "medium",
|
||
"reason_code": f"release_launchpad_{launchpad_action_code}",
|
||
"title": "优先执行节点接入收口" if launchpad_action_code == "bootstrap_run" else "优先执行节点接管验收",
|
||
"summary": str(normalized_launchpad_status.get("recommended_recovery_summary") or "").strip()
|
||
or str(top_issue.get("summary") or normalized_next_step.get("reason") or "").strip()
|
||
or "Release Launchpad 已经明确给出节点接入缺口,建议先完成该节点的接入收口,再回到发布门禁。",
|
||
"next_focus": {
|
||
**launchpad_focus_ref,
|
||
**({"node_code": launchpad_target_node_code} if launchpad_target_node_code else {}),
|
||
},
|
||
"primary_command_key": launchpad_action_code,
|
||
"secondary_command_key": "focus-release-hub" if "focus-release-hub" in recommended_commands else "",
|
||
}
|
||
elif primary_action_code:
|
||
operator_decision = {
|
||
"lane": "ops_jobs" if primary_action_code.startswith("focus_") else "steady",
|
||
"priority": "medium" if top_issue else "low",
|
||
"reason_code": str(top_issue.get("code") or primary_action_code).strip(),
|
||
"title": "按总检默认下一步继续处理",
|
||
"summary": str(normalized_next_step.get("reason") or "").strip()
|
||
or "当前总检已经给出默认下一步,可直接按统一 driver action 继续推进。",
|
||
"next_focus": next_focus_ref,
|
||
"primary_command_key": primary_action_code,
|
||
"secondary_command_key": "",
|
||
}
|
||
|
||
next_actions = {
|
||
"primary_command_key": str(operator_decision.get("primary_command_key") or "").strip(),
|
||
"primary_command": str(
|
||
recommended_commands.get(str(operator_decision.get("primary_command_key") or "").strip()) or ""
|
||
).strip(),
|
||
"secondary_command_key": str(operator_decision.get("secondary_command_key") or "").strip(),
|
||
"secondary_command": str(
|
||
recommended_commands.get(str(operator_decision.get("secondary_command_key") or "").strip()) or ""
|
||
).strip(),
|
||
}
|
||
return operator_decision, next_actions
|
||
|
||
|
||
def get_ops_stack_diagnosis(*, base_url: str = "") -> dict:
|
||
stack_base_url = _ops_stack_api_base_url(base_url)
|
||
api_info = {
|
||
"service": "domain-api",
|
||
"version": "0.1.0",
|
||
"api_prefix": settings.api_prefix,
|
||
"worker_mode": settings.worker_mode,
|
||
}
|
||
|
||
contracts_section = _ops_stack_capture("contracts", get_ops_contract_registry)
|
||
overview_section = _ops_stack_capture("overview", get_ops_overview)
|
||
overview_data = dict(overview_section.get("data") or {})
|
||
if bool(overview_section.get("available", False)) and overview_data:
|
||
link_snapshot_section = _ops_stack_capture(
|
||
"link_snapshot",
|
||
lambda: get_ops_link_snapshot(overview=overview_data),
|
||
)
|
||
else:
|
||
link_snapshot_section = _ops_stack_capture("link_snapshot", get_ops_link_snapshot)
|
||
nodes_section = _ops_stack_capture("managed_nodes", list_managed_nodes_with_agent_state)
|
||
|
||
def _stack_release_hub_payload() -> dict:
|
||
overview_launchpad = dict(((overview_data.get("release_hub") or {}).get("launchpad") or {}))
|
||
if overview_launchpad:
|
||
return overview_launchpad
|
||
return get_release_launchpad()
|
||
|
||
release_hub_section = _ops_stack_capture("release_hub", _stack_release_hub_payload)
|
||
build_info_section = _ops_stack_capture("runtime_build_info", get_runtime_build_info)
|
||
playbook_runs_section = _ops_stack_capture(
|
||
"playbook_runs",
|
||
lambda: get_recent_ops_playbook_runs(limit=6, scan_limit=240),
|
||
)
|
||
|
||
def _stack_activity_stream_payload() -> dict:
|
||
overview_activity_stream = dict(overview_data.get("activity_stream") or {})
|
||
if overview_activity_stream:
|
||
return overview_activity_stream
|
||
return get_ops_activity_stream(limit=8, scan_limit=80)
|
||
|
||
activity_stream_section = _ops_stack_capture("activity_stream", _stack_activity_stream_payload)
|
||
|
||
contracts = dict(contracts_section.get("data") or {})
|
||
contract_registry = contracts if contracts else get_ops_contract_registry()
|
||
link_snapshot = dict(link_snapshot_section.get("data") or {})
|
||
overview = overview_data
|
||
nodes_payload = dict(nodes_section.get("data") or {})
|
||
launchpad = dict(release_hub_section.get("data") or {})
|
||
build_info = dict(build_info_section.get("data") or {})
|
||
playbook_runs_payload = dict(playbook_runs_section.get("data") or {})
|
||
activity_payload = dict(activity_stream_section.get("data") or {})
|
||
|
||
contracts_rows = list(contracts.get("contracts") or [])
|
||
nodes = list(nodes_payload.get("nodes") or [])
|
||
nodes_summary = dict(nodes_payload.get("summary") or {})
|
||
playbook_runs = list(playbook_runs_payload.get("runs") or playbook_runs_payload.get("playbook_runs") or [])
|
||
activities = list(activity_payload.get("items") or activity_payload.get("activities") or [])
|
||
|
||
recommendation = dict(overview.get("recommendation") or {})
|
||
execution_scene = dict(overview.get("execution_scene") or {})
|
||
log_sync = dict(execution_scene.get("log_sync") or overview.get("log_sync") or {})
|
||
release_hub = dict(overview.get("release_hub") or {})
|
||
default_rollout_gate = dict(release_hub.get("default_rollout_gate") or {})
|
||
launchpad_status = dict(launchpad.get("launchpad_status") or {})
|
||
launchpad_gap_action = _release_launchpad_gap_action_context(launchpad)
|
||
latest_release = dict(launchpad.get("latest_release") or release_hub.get("latest_release") or {})
|
||
route_surface = dict(build_info.get("route_surface") or {})
|
||
repository_capabilities = dict(build_info.get("repository_capabilities") or {})
|
||
route_surface_missing_keys = [
|
||
str(item).strip()
|
||
for item in list(route_surface.get("missing_keys") or [])
|
||
if str(item).strip()
|
||
]
|
||
route_surface_expected_paths = {
|
||
str(key).strip(): str(value).strip()
|
||
for key, value in dict(route_surface.get("expected_paths") or {}).items()
|
||
if str(key).strip()
|
||
}
|
||
repo_supports_install_command_block = bool(repository_capabilities.get("supports_install_command_block", False))
|
||
repo_supports_multi_layout_bootstrap = bool(repository_capabilities.get("supports_multi_layout_bootstrap", False))
|
||
route_surface_declares_bootstrap_plan = "ops_node_handover_bootstrap_plan" in route_surface_expected_paths
|
||
runtime_schema_stale = bool(repo_supports_install_command_block and not route_surface_declares_bootstrap_plan)
|
||
|
||
problem_nodes: list[dict] = []
|
||
for node in nodes:
|
||
node_code = str(node.get("node_code") or "").strip()
|
||
if not node_code:
|
||
continue
|
||
queue_state = str(node.get("delivery_queue_state") or "").strip()
|
||
agent_state = str(node.get("agent_state") or "").strip()
|
||
remote_access_state = str(node.get("remote_access_state") or "").strip()
|
||
if (
|
||
queue_state == "dead_letter"
|
||
or agent_state not in {"online", "online_busy"}
|
||
or remote_access_state in {"agent_pending", "disabled", "unmanaged"}
|
||
):
|
||
problem_nodes.append(
|
||
{
|
||
"node_code": node_code,
|
||
"agent_state": agent_state,
|
||
"remote_access_state": remote_access_state,
|
||
"delivery_queue_state": queue_state,
|
||
"participation_state": str(node.get("participation_state") or "").strip(),
|
||
}
|
||
)
|
||
|
||
first_handover_gap_node = next(
|
||
(
|
||
dict(node or {})
|
||
for node in nodes
|
||
if str(node.get("node_code") or "").strip()
|
||
and (
|
||
str(node.get("agent_state") or "").strip() not in {"online", "online_busy"}
|
||
or str(node.get("remote_access_state") or "").strip() in {"agent_pending", "disabled", "unmanaged"}
|
||
)
|
||
),
|
||
{},
|
||
)
|
||
first_handover_gap_node_code = str(first_handover_gap_node.get("node_code") or "").strip()
|
||
first_handover_gap = (
|
||
get_managed_node_handover(
|
||
first_handover_gap_node_code,
|
||
control_plane_base_url=stack_base_url,
|
||
nodes_payload=nodes_payload,
|
||
)
|
||
if first_handover_gap_node_code
|
||
else {}
|
||
)
|
||
first_handover_gap_recovery = _resolve_first_handover_gap_recovery(
|
||
first_handover_gap=first_handover_gap,
|
||
launchpad_gap_action=launchpad_gap_action,
|
||
launchpad_status=launchpad_status,
|
||
fallback_node_code=first_handover_gap_node_code,
|
||
)
|
||
|
||
problem_runs = [
|
||
{
|
||
"run_code": str(item.get("run_code") or "").strip(),
|
||
"status": str(item.get("status") or "").strip(),
|
||
"status_label": str(item.get("status_label") or "").strip(),
|
||
"focus_step_key": str(item.get("focus_step_key") or "").strip(),
|
||
"focus_summary": str(item.get("focus_summary") or item.get("summary_text") or "").strip(),
|
||
"focus_ref": dict(item.get("focus_ref") or {}),
|
||
}
|
||
for item in playbook_runs
|
||
if str(item.get("status") or "").strip() not in {"success", "completed", "healthy"}
|
||
]
|
||
|
||
activity_counts: dict[str, int] = {}
|
||
for item in activities:
|
||
status_key = str(item.get("status") or item.get("job_status") or "unknown").strip() or "unknown"
|
||
activity_counts[status_key] = int(activity_counts.get(status_key, 0) or 0) + 1
|
||
|
||
start_delivery_failed_items = [
|
||
dict(item or {})
|
||
for item in activities
|
||
if str(item.get("start_delivery_state") or "").strip() == "failed_local"
|
||
]
|
||
|
||
issues: list[dict] = []
|
||
|
||
def add_issue(
|
||
*,
|
||
code: str,
|
||
severity: str,
|
||
layer: str,
|
||
summary: str,
|
||
detail: str = "",
|
||
action_code: str = "",
|
||
focus_ref: dict | None = None,
|
||
commands: list[str] | None = None,
|
||
) -> None:
|
||
contract_keys = _suggest_ops_contract_keys_for_stack_issue(
|
||
{
|
||
"layer": layer,
|
||
"focus_ref": focus_ref or {},
|
||
"action_code": action_code,
|
||
}
|
||
)
|
||
issues.append(
|
||
{
|
||
"code": str(code or "").strip(),
|
||
"severity": str(severity or "").strip() or "warning",
|
||
"layer": str(layer or "").strip(),
|
||
"summary": str(summary or "").strip(),
|
||
"detail": str(detail or "").strip(),
|
||
"action_code": str(action_code or "").strip(),
|
||
"focus_ref": dict(focus_ref or {}),
|
||
"commands": [str(item).strip() for item in list(commands or []) if str(item).strip()],
|
||
"contract_keys": contract_keys,
|
||
"contract_navigation": _build_ops_contract_navigation(
|
||
contract_keys,
|
||
primary_contract_key="ops_stack_diagnosis_contract",
|
||
registry=contract_registry,
|
||
),
|
||
}
|
||
)
|
||
|
||
surface_matrix = [
|
||
("api", True),
|
||
("contracts", bool(contracts_section.get("available", False))),
|
||
("link_snapshot", bool(link_snapshot_section.get("available", False))),
|
||
("overview", bool(overview_section.get("available", False))),
|
||
("managed_nodes", bool(nodes_section.get("available", False))),
|
||
("release_hub", bool(release_hub_section.get("available", False))),
|
||
("runtime_build_info", bool(build_info_section.get("available", False))),
|
||
("playbook_runs", bool(playbook_runs_section.get("available", False))),
|
||
("activity_stream", bool(activity_stream_section.get("available", False))),
|
||
]
|
||
surface_status_counts = {
|
||
"total": len(surface_matrix),
|
||
"available": sum(1 for _, available in surface_matrix if available),
|
||
"missing": sum(1 for _, available in surface_matrix if not available),
|
||
}
|
||
missing_surfaces = [name for name, available in surface_matrix if not available]
|
||
|
||
if not bool(contracts_section.get("available", False)):
|
||
add_issue(
|
||
code="ops_contracts_unavailable",
|
||
severity="warning",
|
||
layer="contracts",
|
||
summary="控制面契约注册表当前未能正常生成,总检无法确认当前运行的是哪一版正式 contract。",
|
||
detail=str(contracts_section.get("error") or "ops contract registry unavailable").strip(),
|
||
commands=[
|
||
build_bash_command("drive_ops_center.sh", "contracts", stack_base_url),
|
||
build_bash_command("drive_ops_center.sh", "doctor", stack_base_url),
|
||
],
|
||
)
|
||
|
||
if not bool(build_info_section.get("available", False)):
|
||
add_issue(
|
||
code="runtime_build_info_unavailable",
|
||
severity="warning",
|
||
layer="runtime_build_info",
|
||
summary="当前无法读取运行中 API 的 build-info,总检暂时无法确认服务是否已经吃到最新路由面。",
|
||
detail=str(build_info_section.get("error") or "runtime build-info unavailable").strip(),
|
||
action_code="api-restart",
|
||
commands=[
|
||
build_bash_command("drive_ops_center.sh", "runtime-refresh-recover", stack_base_url),
|
||
build_bash_command("drive_ops_center.sh", "doctor", stack_base_url),
|
||
],
|
||
)
|
||
elif route_surface and not bool(route_surface.get("surface_complete", False)):
|
||
add_issue(
|
||
code="runtime_route_surface_incomplete",
|
||
severity="warning",
|
||
layer="runtime_build_info.route_surface",
|
||
summary="当前运行中的 API 路由面不完整,说明服务可能还没重启到最新代码,或当前发布包缺少关键运维入口。",
|
||
detail=(
|
||
f"missing_keys={', '.join(route_surface_missing_keys) or '-'};"
|
||
f"mode={str(route_surface.get('mode') or '').strip() or 'unknown'};"
|
||
f"registered_paths_total={int(route_surface.get('registered_paths_total', 0) or 0)}"
|
||
),
|
||
action_code="api-restart",
|
||
focus_ref={
|
||
"kind": "runtime_build_info",
|
||
"section": "route_surface",
|
||
"missing_keys": route_surface_missing_keys,
|
||
},
|
||
commands=[
|
||
build_bash_command("drive_ops_center.sh", "runtime-refresh-recover", stack_base_url),
|
||
build_bash_command("drive_ops_center.sh", "stack-diagnosis", stack_base_url, "summary"),
|
||
build_bash_command("drive_ops_center.sh", "doctor", stack_base_url),
|
||
],
|
||
)
|
||
elif runtime_schema_stale:
|
||
add_issue(
|
||
code="runtime_build_schema_stale",
|
||
severity="warning",
|
||
layer="runtime_build_info.schema",
|
||
summary="仓库已经具备新的节点接管能力,但运行中的 build-info 仍未声明对应路由键,当前更像 API 还没重启到最新代码。",
|
||
detail=(
|
||
f"supports_install_command_block={repo_supports_install_command_block};"
|
||
f"supports_multi_layout_bootstrap={repo_supports_multi_layout_bootstrap};"
|
||
f"route_surface_declares_bootstrap_plan={route_surface_declares_bootstrap_plan}"
|
||
),
|
||
action_code="api-restart",
|
||
focus_ref={
|
||
"kind": "runtime_build_info",
|
||
"section": "schema",
|
||
"expected_route_key": "ops_node_handover_bootstrap_plan",
|
||
},
|
||
commands=[
|
||
build_bash_command("drive_ops_center.sh", "runtime-refresh-recover", stack_base_url),
|
||
build_bash_command("drive_ops_center.sh", "stack-diagnosis", stack_base_url, "summary"),
|
||
build_bash_command("drive_ops_center.sh", "node-bootstrap-plan", stack_base_url, first_handover_gap_node_code or "mainland-worker-01"),
|
||
],
|
||
)
|
||
|
||
if missing_surfaces and any(
|
||
name in missing_surfaces for name in ("link_snapshot", "overview", "managed_nodes", "release_hub")
|
||
):
|
||
add_issue(
|
||
code="ops_surface_partial",
|
||
severity="warning",
|
||
layer="stack",
|
||
summary="海外单脑控制面的关键观察层未全部联通,总检目前只能给出部分收口。",
|
||
detail=f"当前缺失层: {', '.join(missing_surfaces)}",
|
||
commands=[
|
||
build_bash_command("drive_ops_center.sh", "stack-diagnosis", stack_base_url, "full"),
|
||
build_bash_command("drive_ops_center.sh", "doctor", stack_base_url),
|
||
],
|
||
)
|
||
|
||
participating_nodes_total = int(
|
||
log_sync.get("participating_nodes_total", log_sync.get("participating_node_count", 0)) or 0
|
||
)
|
||
participating_node_codes = _normalize_ops_node_code_list(
|
||
execution_scene.get("participating_nodes") or [],
|
||
(execution_scene.get("participation_summary") or {}).get("dispatch_active_node_codes") or [],
|
||
(execution_scene.get("participation_summary") or {}).get("recent_only_node_codes") or [],
|
||
)
|
||
missing_log_node_codes = _normalize_ops_node_code_list(log_sync.get("missing_participating_nodes") or [])
|
||
scene_log_source_node_codes = _normalize_ops_node_code_list(
|
||
log_sync.get("source_nodes") or [],
|
||
log_sync.get("source_node_summaries") or [],
|
||
)
|
||
preferred_scene_log_target_codes = _normalize_ops_node_code_list(
|
||
missing_log_node_codes,
|
||
participating_node_codes,
|
||
scene_log_source_node_codes,
|
||
)
|
||
if bool(log_sync.get("enabled", False)) is False and participating_nodes_total > 0:
|
||
log_sync_action = ""
|
||
for item in list(link_snapshot.get("next_actions") or []):
|
||
action_code = str(item.get("action_code") or "").strip()
|
||
if action_code.startswith("enable_log_sync"):
|
||
log_sync_action = action_code
|
||
break
|
||
if not log_sync_action:
|
||
log_sync_action = "enable_log_sync_key"
|
||
add_issue(
|
||
code="remote_log_sync_disabled",
|
||
severity="warning",
|
||
layer="overview.log_sync",
|
||
summary="现场已有参与检测节点,但远端日志回传仍关闭,海外控制面对执行细节仍处于半盲态。",
|
||
detail=(
|
||
f"参与检测节点 {participating_nodes_total} 台,"
|
||
f"已覆盖 {int(log_sync.get('covered_participating_node_count', 0) or 0)} 台。"
|
||
),
|
||
action_code=log_sync_action,
|
||
focus_ref=_build_scene_node_log_focus_ref(
|
||
(preferred_scene_log_target_codes or participating_node_codes or [""])[0],
|
||
mode="key",
|
||
limit=80,
|
||
source="remote_log_sync_disabled",
|
||
),
|
||
commands=[
|
||
build_bash_command("drive_ops_center.sh", "driver-resolve", stack_base_url, log_sync_action),
|
||
*_build_scene_node_log_commands(
|
||
stack_base_url,
|
||
preferred_scene_log_target_codes,
|
||
limit=80,
|
||
mode="key",
|
||
max_commands=2,
|
||
),
|
||
],
|
||
)
|
||
elif bool(log_sync.get("enabled", False)) and participating_nodes_total > 0 and (
|
||
missing_log_node_codes
|
||
or int(log_sync.get("line_count", 0) or 0) <= 0
|
||
or int(log_sync.get("source_node_count", 0) or 0) <= 0
|
||
):
|
||
waiting_node_codes = preferred_scene_log_target_codes or participating_node_codes
|
||
add_issue(
|
||
code="remote_log_sync_waiting_sample",
|
||
severity="warning",
|
||
layer="overview.log_sync",
|
||
summary="远端日志回传虽然已经开启,但参与检测节点的现场样本仍未完全形成。",
|
||
detail=(
|
||
f"参与检测节点 {participating_nodes_total} 台,"
|
||
f"已覆盖 {int(log_sync.get('covered_participating_node_count', 0) or 0)} 台,"
|
||
f"缺口节点 {_format_node_code_list(missing_log_node_codes)}。"
|
||
),
|
||
action_code="open_worker_logs_participating",
|
||
focus_ref=_build_scene_node_log_focus_ref(
|
||
(waiting_node_codes or [""])[0],
|
||
mode=str(log_sync.get("mode") or "key"),
|
||
limit=120,
|
||
source="remote_log_sync_waiting_sample",
|
||
),
|
||
commands=[
|
||
*_build_scene_node_log_commands(
|
||
stack_base_url,
|
||
waiting_node_codes,
|
||
limit=120,
|
||
mode=str(log_sync.get("mode") or "key"),
|
||
max_commands=3,
|
||
),
|
||
build_bash_command("drive_ops_center.sh", "driver-resolve", stack_base_url, "open_worker_logs_participating"),
|
||
build_bash_command("drive_ops_center.sh", "driver-resolve", stack_base_url, "run_inspection_participating"),
|
||
],
|
||
)
|
||
|
||
managed_enabled = int(nodes_summary.get("managed_enabled", 0) or 0)
|
||
remote_access_ready = int(nodes_summary.get("remote_access_ready", 0) or 0)
|
||
if managed_enabled > 0 and remote_access_ready == 0:
|
||
first_handover_gap_stage = dict(first_handover_gap.get("stage") or {})
|
||
first_handover_gap_next_step = dict(first_handover_gap_stage.get("next_step") or {})
|
||
first_handover_gap_summary = str(first_handover_gap.get("summary") or "").strip()
|
||
managed_nodes_action_code = str(
|
||
first_handover_gap_recovery.get("action_code")
|
||
or launchpad_status.get("recommended_action_code")
|
||
or "fix_managed_nodes"
|
||
).strip()
|
||
add_issue(
|
||
code="managed_nodes_agent_pending",
|
||
severity="blocked",
|
||
layer="managed_nodes",
|
||
summary="托管节点虽然已经纳入 Ops Center,但 0 台 remote-agent 就绪,海外主控还不能统一下发标准执行动作。",
|
||
detail=(
|
||
f"managed_enabled={managed_enabled},remote_access_ready={remote_access_ready};"
|
||
"当前节点多数仍处于 agent_pending / runtime_only / pending_bootstrap。"
|
||
+ (
|
||
f" 首个缺口节点 {first_handover_gap_node_code}:"
|
||
f"{str(first_handover_gap_stage.get('label') or '').strip() or '待处理'},"
|
||
f"{first_handover_gap_summary}"
|
||
if first_handover_gap_node_code and first_handover_gap_summary
|
||
else ""
|
||
)
|
||
),
|
||
action_code=managed_nodes_action_code,
|
||
focus_ref=dict(first_handover_gap_recovery.get("focus_ref") or {}),
|
||
commands=[
|
||
build_bash_command("drive_ops_center.sh", "driver-resolve", stack_base_url, managed_nodes_action_code),
|
||
*(
|
||
[build_bash_command("drive_ops_center.sh", "node-handover", stack_base_url, first_handover_gap_node_code)]
|
||
if first_handover_gap_node_code
|
||
else []
|
||
),
|
||
*(
|
||
[
|
||
build_bash_command("drive_ops_center.sh", "node-bootstrap-plan", stack_base_url, first_handover_gap_node_code)
|
||
]
|
||
if first_handover_gap_node_code
|
||
and str(first_handover_gap_next_step.get("code") or "").strip()
|
||
not in {"ready", "run_acceptance"}
|
||
else []
|
||
),
|
||
*(
|
||
[
|
||
build_bash_command(
|
||
"drive_ops_center.sh",
|
||
"node-bootstrap-run" if managed_nodes_action_code == "bootstrap_run" else "node-acceptance-run",
|
||
stack_base_url,
|
||
first_handover_gap_node_code,
|
||
"cli",
|
||
)
|
||
]
|
||
if first_handover_gap_node_code and managed_nodes_action_code in {"bootstrap_run", "run_acceptance"}
|
||
else []
|
||
),
|
||
build_bash_command("drive_ops_center.sh", "doctor", stack_base_url),
|
||
],
|
||
)
|
||
|
||
queue_dead_letter_nodes = int(nodes_summary.get("queue_dead_letter_nodes", 0) or 0)
|
||
queue_dead_letter_records = int(nodes_summary.get("queue_dead_letter_records", 0) or 0)
|
||
if queue_dead_letter_nodes > 0 or queue_dead_letter_records > 0:
|
||
add_issue(
|
||
code="delivery_queue_dead_letter",
|
||
severity="warning",
|
||
layer="managed_nodes.queue",
|
||
summary="存在 Node Agent 回执死信,后续自动化动作可能反复失败或无法收敛。",
|
||
detail=(
|
||
f"queue_dead_letter_nodes={queue_dead_letter_nodes},"
|
||
f"queue_dead_letter_records={queue_dead_letter_records}"
|
||
),
|
||
commands=[
|
||
build_bash_command("drive_ops_center.sh", "doctor", stack_base_url),
|
||
],
|
||
)
|
||
|
||
if str(default_rollout_gate.get("status") or "").strip() == "blocked":
|
||
add_issue(
|
||
code="release_rollout_gate_blocked",
|
||
severity="warning",
|
||
layer="release_hub",
|
||
summary="ReleaseHub 默认门禁当前阻断,说明虽然发布包已准备好,但正式 Rollout 还不具备条件。",
|
||
detail=str(default_rollout_gate.get("summary_text") or default_rollout_gate.get("summary") or "").strip(),
|
||
action_code=str(launchpad_status.get("recommended_action_code") or "").strip(),
|
||
focus_ref=dict(default_rollout_gate.get("focus_ref") or launchpad_status.get("focus_ref") or {}),
|
||
commands=[
|
||
build_bash_command("drive_ops_center.sh", "release-launchpad", stack_base_url),
|
||
build_bash_command(
|
||
"drive_ops_center.sh",
|
||
"driver-resolve",
|
||
stack_base_url,
|
||
str(launchpad_status.get("recommended_action_code") or "fix_managed_nodes").strip(),
|
||
),
|
||
],
|
||
)
|
||
|
||
if problem_runs:
|
||
add_issue(
|
||
code="playbook_runs_need_attention",
|
||
severity="warning",
|
||
layer="playbook_runs",
|
||
summary="最近存在未完全收口的 playbook run,建议先确认执行序列是否停在中间步骤。",
|
||
detail=f"problem_runs_total={len(problem_runs)}",
|
||
commands=[
|
||
build_bash_command("drive_ops_center.sh", "doctor", stack_base_url),
|
||
],
|
||
)
|
||
|
||
if start_delivery_failed_items:
|
||
first_delivery_gap_item = dict(start_delivery_failed_items[0] or {})
|
||
first_delivery_focus_ref = dict(
|
||
first_delivery_gap_item.get("source_focus_ref")
|
||
or first_delivery_gap_item.get("focus_ref")
|
||
or {}
|
||
)
|
||
first_delivery_job_code = str(first_delivery_gap_item.get("job_code") or "").strip()
|
||
first_delivery_node_code = str(first_delivery_focus_ref.get("target_node_code") or "").strip()
|
||
add_issue(
|
||
code="ops_job_start_delivery_failed_local",
|
||
severity="warning",
|
||
layer="activity_stream.ops_job",
|
||
summary=(
|
||
f"最近有 {len(start_delivery_failed_items)} 条标准运维任务在节点侧已开始执行,但开始回执没有成功送达控制面。"
|
||
),
|
||
detail=" / ".join(
|
||
part
|
||
for part in [
|
||
f"首条任务 {first_delivery_job_code}" if first_delivery_job_code else "",
|
||
f"目标节点 {first_delivery_node_code}" if first_delivery_node_code else "",
|
||
str(first_delivery_gap_item.get("start_delivery_error") or "").strip(),
|
||
str(first_delivery_gap_item.get("summary_text") or first_delivery_gap_item.get("summary") or "").strip(),
|
||
]
|
||
if part
|
||
),
|
||
action_code="focus_latest_job_events",
|
||
focus_ref=first_delivery_focus_ref,
|
||
commands=[
|
||
build_bash_command("drive_ops_center.sh", "driver-resolve", stack_base_url, "focus_latest_job_events"),
|
||
build_bash_command("drive_ops_center.sh", "doctor", stack_base_url),
|
||
],
|
||
)
|
||
|
||
ordered_action_candidates = [
|
||
{
|
||
"source": "overview.primary",
|
||
"action_code": str(recommendation.get("primary_action_code") or "").strip(),
|
||
"title": str(recommendation.get("title") or "").strip(),
|
||
"reason": "来自 overview.recommendation.primary_action_code",
|
||
"focus_ref": dict(recommendation.get("focus_ref") or {}),
|
||
},
|
||
{
|
||
"source": "release_hub.launchpad",
|
||
"action_code": str(launchpad_status.get("recommended_action_code") or "").strip(),
|
||
"title": str(launchpad_status.get("status_label") or "").strip(),
|
||
"reason": "来自 release_launchpad.launchpad_status.recommended_action_code",
|
||
"focus_ref": dict(launchpad_status.get("focus_ref") or {}),
|
||
},
|
||
{
|
||
"source": "overview.secondary",
|
||
"action_code": str(recommendation.get("secondary_action_code") or "").strip(),
|
||
"title": str(recommendation.get("title") or "").strip(),
|
||
"reason": "来自 overview.recommendation.secondary_action_code",
|
||
"focus_ref": dict(recommendation.get("focus_ref") or {}),
|
||
},
|
||
]
|
||
if str(first_handover_gap_recovery.get("action_code") or "").strip() in {"bootstrap_run", "run_acceptance"}:
|
||
ordered_action_candidates.insert(
|
||
0,
|
||
{
|
||
"source": "managed_nodes.first_handover_gap",
|
||
"action_code": str(first_handover_gap_recovery.get("action_code") or "").strip(),
|
||
"title": str(first_handover_gap_recovery.get("recovery_label") or "").strip(),
|
||
"reason": str(first_handover_gap_recovery.get("summary") or "").strip()
|
||
or "来自首个 handover gap 的标准恢复动作。",
|
||
"focus_ref": dict(first_handover_gap_recovery.get("focus_ref") or {}),
|
||
},
|
||
)
|
||
for issue in issues:
|
||
if str(issue.get("code") or "").strip() == "runtime_build_schema_stale":
|
||
ordered_action_candidates.insert(
|
||
0,
|
||
{
|
||
"source": f"issue:{issue.get('code')}",
|
||
"action_code": "api-restart",
|
||
"title": "优先重启控制面 API",
|
||
"reason": str(issue.get("detail") or issue.get("summary") or "").strip(),
|
||
"focus_ref": dict(issue.get("focus_ref") or {}),
|
||
},
|
||
)
|
||
break
|
||
for issue in issues:
|
||
issue_action_code = str(issue.get("action_code") or "").strip()
|
||
if issue_action_code:
|
||
ordered_action_candidates.append(
|
||
{
|
||
"source": f"issue:{issue.get('code')}",
|
||
"action_code": issue_action_code,
|
||
"title": str(issue.get("summary") or "").strip(),
|
||
"reason": str(issue.get("detail") or issue.get("summary") or "").strip(),
|
||
"focus_ref": dict(issue.get("focus_ref") or {}),
|
||
}
|
||
)
|
||
|
||
recommended_actions: list[dict] = []
|
||
seen_action_codes: set[str] = set()
|
||
for item in ordered_action_candidates:
|
||
action_code = str(item.get("action_code") or "").strip()
|
||
if not action_code or action_code in seen_action_codes:
|
||
continue
|
||
seen_action_codes.add(action_code)
|
||
recommended_action_contract_keys = _normalize_ops_contract_keys(
|
||
["ops_stack_diagnosis_contract"]
|
||
+ _ops_contract_keys_from_action_code(action_code)
|
||
+ _ops_contract_keys_from_focus_ref(item.get("focus_ref"))
|
||
)
|
||
recommended_actions.append(
|
||
{
|
||
"action_code": action_code,
|
||
"source": str(item.get("source") or "").strip(),
|
||
"title": str(item.get("title") or "").strip(),
|
||
"reason": str(item.get("reason") or "").strip(),
|
||
"focus_ref": dict(item.get("focus_ref") or {}),
|
||
"command": build_bash_command("drive_ops_center.sh", "driver-resolve", stack_base_url, action_code),
|
||
"contract_keys": recommended_action_contract_keys,
|
||
"contract_navigation": _build_ops_contract_navigation(
|
||
recommended_action_contract_keys,
|
||
primary_contract_key="ops_stack_diagnosis_contract",
|
||
registry=contract_registry,
|
||
),
|
||
}
|
||
)
|
||
|
||
blocking_issue_total = sum(1 for item in issues if str(item.get("severity") or "").strip() == "blocked")
|
||
warning_issue_total = sum(1 for item in issues if str(item.get("severity") or "").strip() == "warning")
|
||
info_issue_total = sum(1 for item in issues if str(item.get("severity") or "").strip() == "info")
|
||
|
||
surface_status = "healthy"
|
||
if missing_surfaces:
|
||
surface_status = "partial"
|
||
|
||
automation_status = "ready"
|
||
if blocking_issue_total > 0:
|
||
automation_status = "blocked"
|
||
elif warning_issue_total > 0:
|
||
automation_status = "attention"
|
||
|
||
stack_status = "ready"
|
||
if automation_status == "blocked":
|
||
stack_status = "blocked"
|
||
elif surface_status != "healthy" or automation_status != "ready":
|
||
stack_status = "attention"
|
||
|
||
top_issue = None
|
||
if issues:
|
||
top_issue = sorted(
|
||
issues,
|
||
key=lambda item: (
|
||
-_ops_stack_issue_rank(str(item.get("severity") or "").strip()),
|
||
str(item.get("code") or "").strip(),
|
||
),
|
||
)[0]
|
||
|
||
next_step = {}
|
||
if recommended_actions:
|
||
first_action = dict(recommended_actions[0] or {})
|
||
next_step_contract_keys = _normalize_ops_contract_keys(
|
||
["ops_stack_diagnosis_contract"]
|
||
+ _ops_contract_keys_from_action_code(str(first_action.get("action_code") or "").strip())
|
||
+ _ops_contract_keys_from_focus_ref(first_action.get("focus_ref"))
|
||
)
|
||
next_step = {
|
||
"action_code": str(first_action.get("action_code") or "").strip(),
|
||
"source": str(first_action.get("source") or "").strip(),
|
||
"reason": str(first_action.get("reason") or "").strip(),
|
||
"command": str(first_action.get("command") or "").strip(),
|
||
"focus_ref": dict(first_action.get("focus_ref") or {}),
|
||
"contract_keys": next_step_contract_keys,
|
||
"contract_navigation": _build_ops_contract_navigation(
|
||
next_step_contract_keys,
|
||
primary_contract_key="ops_stack_diagnosis_contract",
|
||
registry=contract_registry,
|
||
),
|
||
}
|
||
elif top_issue:
|
||
next_step_contract_keys = _normalize_ops_contract_keys(top_issue.get("contract_keys") or ["ops_stack_diagnosis_contract"])
|
||
next_step = {
|
||
"action_code": "",
|
||
"source": f"issue:{str(top_issue.get('code') or '').strip()}",
|
||
"reason": str(top_issue.get("summary") or "").strip(),
|
||
"command": "",
|
||
"focus_ref": dict(top_issue.get("focus_ref") or {}),
|
||
"contract_keys": next_step_contract_keys,
|
||
"contract_navigation": _build_ops_contract_navigation(
|
||
next_step_contract_keys,
|
||
primary_contract_key="ops_stack_diagnosis_contract",
|
||
registry=contract_registry,
|
||
),
|
||
}
|
||
|
||
quick_commands = [
|
||
build_bash_command("drive_ops_center.sh", "stack-diagnosis", stack_base_url, "summary"),
|
||
build_bash_command("drive_ops_center.sh", "doctor", stack_base_url),
|
||
]
|
||
for action in recommended_actions[:3]:
|
||
command = str(action.get("command") or "").strip()
|
||
if command and command not in quick_commands:
|
||
quick_commands.append(command)
|
||
for issue in issues[:5]:
|
||
for command in list(issue.get("commands") or []):
|
||
normalized_command = str(command or "").strip()
|
||
if normalized_command and normalized_command not in quick_commands:
|
||
quick_commands.append(normalized_command)
|
||
if len(quick_commands) >= 8:
|
||
break
|
||
if len(quick_commands) >= 8:
|
||
break
|
||
if not bool(contracts_section.get("available", False)):
|
||
contract_check_command = build_bash_command("drive_ops_center.sh", "contracts", stack_base_url)
|
||
if contract_check_command not in quick_commands:
|
||
quick_commands.append(contract_check_command)
|
||
|
||
recommended_commands = _build_stack_diagnosis_recommended_commands(
|
||
stack_base_url=stack_base_url,
|
||
recommended_actions=recommended_actions,
|
||
first_handover_gap_node_code=first_handover_gap_node_code,
|
||
launchpad_status=launchpad_status,
|
||
default_rollout_gate=default_rollout_gate,
|
||
next_step=next_step,
|
||
)
|
||
operator_decision, next_actions = _build_stack_diagnosis_operator_decision(
|
||
issues=issues,
|
||
next_step=next_step,
|
||
recommended_actions=recommended_actions,
|
||
recommended_commands=recommended_commands,
|
||
first_handover_gap_node_code=first_handover_gap_node_code,
|
||
launchpad_status=launchpad_status,
|
||
default_rollout_gate=default_rollout_gate,
|
||
)
|
||
|
||
operator_hints: list[str] = []
|
||
if stack_status == "blocked":
|
||
operator_hints.append("当前总检已经能看到现场,但海外单脑自动化执行链路仍未正式打通,先处理阻断问题再继续 Rollout 或批量运维。")
|
||
elif stack_status == "attention":
|
||
operator_hints.append("当前总检可用,但仍有缺口;建议先按 next_step 收口,再进入更细粒度脚本。")
|
||
else:
|
||
operator_hints.append("当前海外单脑控制面总检已基本就绪,可以把它作为后续所有联调和发布的固定起手式。")
|
||
if not bool(contracts_section.get("available", False)):
|
||
operator_hints.append("contracts 当前未正常可用,优先先恢复 contract registry,再让页面、CLI、Codex 共享同一份正式契约。")
|
||
if route_surface_missing_keys:
|
||
operator_hints.append("build-info 显示当前运行中的 API 路由面还不完整,先重启 API 并确认最新运维入口已经注册,再继续做节点接管或发布判断。")
|
||
if any(str(item.get("code") or "").strip() == "runtime_build_schema_stale" for item in issues):
|
||
operator_hints.append("当前属于典型的“仓库代码已更新,但运行中 API 还是旧 schema”场景,先重启 domaincheck-api,再重新看 stack-diagnosis 与 node-bootstrap-plan。")
|
||
if bool(log_sync.get("enabled", False)) is False and participating_nodes_total > 0:
|
||
operator_hints.append("远端日志回传当前关闭,后续如果继续多机联调,海外控制面会继续处于半盲态。")
|
||
next_step_action_code = str(next_step.get("action_code") or "").strip()
|
||
next_step_focus_ref = dict(next_step.get("focus_ref") or {})
|
||
if next_step_action_code in {"bootstrap_run", "run_acceptance"}:
|
||
next_step_node_code = str(next_step_focus_ref.get("node_code") or first_handover_gap_node_code or "").strip()
|
||
operator_hints.append(
|
||
(
|
||
f"当前第一动作已经收敛为 {'接入收口' if next_step_action_code == 'bootstrap_run' else '接管验收'}"
|
||
+ (f":{next_step_node_code}" if next_step_node_code else "")
|
||
+ ",页面、CLI、Codex 都应优先围绕这一个节点推进,不再回退到泛化的托管节点修复。"
|
||
)
|
||
)
|
||
|
||
resolved_launchpad_target_node_code = str(launchpad_status.get("recommended_target_node_code") or "").strip()
|
||
resolved_launchpad_recovery_label = str(launchpad_status.get("recommended_recovery_label") or "").strip()
|
||
resolved_launchpad_recovery_summary = str(launchpad_status.get("recommended_recovery_summary") or "").strip()
|
||
if (
|
||
str(launchpad_status.get("recommended_action_code") or "").strip() not in {"bootstrap_run", "run_acceptance"}
|
||
and str(first_handover_gap_recovery.get("action_code") or "").strip() in {"bootstrap_run", "run_acceptance"}
|
||
):
|
||
resolved_launchpad_target_node_code = (
|
||
resolved_launchpad_target_node_code
|
||
or str(next_step_focus_ref.get("node_code") or "").strip()
|
||
or str(first_handover_gap_recovery.get("node_code") or "").strip()
|
||
)
|
||
resolved_launchpad_recovery_label = (
|
||
resolved_launchpad_recovery_label
|
||
or str(first_handover_gap_recovery.get("recovery_label") or "").strip()
|
||
)
|
||
resolved_launchpad_recovery_summary = (
|
||
resolved_launchpad_recovery_summary
|
||
or str(first_handover_gap_recovery.get("summary") or "").strip()
|
||
)
|
||
|
||
diagnosis_contract_keys = _normalize_ops_contract_keys(
|
||
["ops_stack_diagnosis_contract", "ops_driver_contract"]
|
||
+ [contract_key for issue in issues for contract_key in list(issue.get("contract_keys") or [])]
|
||
+ list(next_step.get("contract_keys") or [])
|
||
)
|
||
|
||
return {
|
||
"generated_at": datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
|
||
"diagnosis": {
|
||
"contract_key": "ops_stack_diagnosis_contract",
|
||
"contract_version": _OPS_CONTRACT_SCHEMA_VERSION,
|
||
"registry_version": _OPS_CONTRACT_REGISTRY_VERSION,
|
||
"base_url": stack_base_url,
|
||
"contract_navigation": _build_ops_contract_navigation(
|
||
diagnosis_contract_keys,
|
||
primary_contract_key="ops_stack_diagnosis_contract",
|
||
registry=contract_registry,
|
||
),
|
||
"surface_status": surface_status,
|
||
"automation_status": automation_status,
|
||
"stack_status": stack_status,
|
||
"issue_total": len(issues),
|
||
"blocking_issue_total": blocking_issue_total,
|
||
"warning_issue_total": warning_issue_total,
|
||
"info_issue_total": info_issue_total,
|
||
"missing_surfaces": missing_surfaces,
|
||
"launchpad_recommended_target_node_code": resolved_launchpad_target_node_code,
|
||
"launchpad_recommended_recovery_label": resolved_launchpad_recovery_label,
|
||
"launchpad_recommended_recovery_summary": resolved_launchpad_recovery_summary,
|
||
"launchpad_onboarding_bootstrap_pending_nodes": int(
|
||
launchpad_status.get("onboarding_bootstrap_pending_nodes", 0) or 0
|
||
),
|
||
"launchpad_onboarding_acceptance_ready_nodes": int(
|
||
launchpad_status.get("onboarding_acceptance_ready_nodes", 0) or 0
|
||
),
|
||
"next_step": next_step,
|
||
"recommended_actions": recommended_actions[:5],
|
||
"operator_decision": operator_decision,
|
||
"next_actions": next_actions,
|
||
"recommended_commands": recommended_commands,
|
||
"issues": issues[:8],
|
||
"operator_hints": operator_hints,
|
||
"quick_commands": quick_commands,
|
||
},
|
||
"api": {
|
||
"http_status": "200",
|
||
"available": True,
|
||
**api_info,
|
||
},
|
||
"surface_matrix": {
|
||
"status_counts": surface_status_counts,
|
||
"items": [
|
||
{
|
||
"name": name,
|
||
"available": bool(available),
|
||
"elapsed_ms": int(
|
||
{
|
||
"contracts": contracts_section,
|
||
"link_snapshot": link_snapshot_section,
|
||
"overview": overview_section,
|
||
"managed_nodes": nodes_section,
|
||
"release_hub": release_hub_section,
|
||
"runtime_build_info": build_info_section,
|
||
"playbook_runs": playbook_runs_section,
|
||
"activity_stream": activity_stream_section,
|
||
}.get(name, {}).get("elapsed_ms", 0) or 0
|
||
),
|
||
}
|
||
for name, available in surface_matrix
|
||
],
|
||
},
|
||
"contracts": {
|
||
"http_status": str(contracts_section.get("http_status") or "500"),
|
||
"available": bool(contracts_section.get("available", False)),
|
||
"elapsed_ms": int(contracts_section.get("elapsed_ms", 0) or 0),
|
||
"registry_version": str(contracts.get("registry_version") or ""),
|
||
"schema_version": str(contracts.get("schema_version") or ""),
|
||
"contracts_total": len(contracts_rows),
|
||
"contract_keys": [str(item.get("key") or "").strip() for item in contracts_rows if str(item.get("key") or "").strip()],
|
||
"error": str(contracts_section.get("error") or ""),
|
||
},
|
||
"link_snapshot": {
|
||
"http_status": str(link_snapshot_section.get("http_status") or "500"),
|
||
"available": bool(link_snapshot_section.get("available", False)),
|
||
"elapsed_ms": int(link_snapshot_section.get("elapsed_ms", 0) or 0),
|
||
"status": str(link_snapshot.get("status") or ""),
|
||
"headline": str(link_snapshot.get("headline") or ""),
|
||
"next_action_codes": [
|
||
str(item.get("action_code") or "").strip()
|
||
for item in list(link_snapshot.get("next_actions") or [])
|
||
if str(item.get("action_code") or "").strip()
|
||
],
|
||
"error": str(link_snapshot_section.get("error") or ""),
|
||
},
|
||
"overview": {
|
||
"http_status": str(overview_section.get("http_status") or "500"),
|
||
"available": bool(overview_section.get("available", False)),
|
||
"elapsed_ms": int(overview_section.get("elapsed_ms", 0) or 0),
|
||
"recommendation_key": str(recommendation.get("key") or ""),
|
||
"recommendation_title": str(recommendation.get("title") or ""),
|
||
"primary_action_code": str(recommendation.get("primary_action_code") or ""),
|
||
"secondary_action_code": str(recommendation.get("secondary_action_code") or ""),
|
||
"focus_ref": dict(recommendation.get("focus_ref") or {}),
|
||
"execution_scene_summary": str(
|
||
execution_scene.get("summary")
|
||
or (execution_scene.get("participation_summary") or {}).get("summary")
|
||
or execution_scene.get("phase_detail")
|
||
or ""
|
||
),
|
||
"execution_scene_counts": dict(execution_scene.get("counts") or {}),
|
||
"log_sync": {
|
||
"status": str(log_sync.get("status") or log_sync.get("state") or ""),
|
||
"status_label": str(log_sync.get("status_label") or ""),
|
||
"mode": str(log_sync.get("mode") or ""),
|
||
"mode_label": str(log_sync.get("mode_label") or ""),
|
||
"enabled": bool(log_sync.get("enabled", False)),
|
||
"covered_participating_nodes": int(log_sync.get("covered_participating_node_count", 0) or 0),
|
||
"participating_nodes_total": int(
|
||
log_sync.get("participating_nodes_total", log_sync.get("participating_node_count", 0)) or 0
|
||
),
|
||
"missing_sample_node_codes": list(
|
||
log_sync.get("missing_sample_node_codes")
|
||
or log_sync.get("missing_participating_nodes")
|
||
or []
|
||
),
|
||
},
|
||
"error": str(overview_section.get("error") or ""),
|
||
},
|
||
"managed_nodes": {
|
||
"http_status": str(nodes_section.get("http_status") or "500"),
|
||
"available": bool(nodes_section.get("available", False)),
|
||
"elapsed_ms": int(nodes_section.get("elapsed_ms", 0) or 0),
|
||
"total": int(nodes_summary.get("total", len(nodes)) or 0),
|
||
"managed_total": int(nodes_summary.get("managed_total", 0) or 0),
|
||
"managed_enabled": int(nodes_summary.get("managed_enabled", 0) or 0),
|
||
"online": int(nodes_summary.get("online", 0) or 0),
|
||
"agent_ready": int(nodes_summary.get("agent_ready", 0) or 0),
|
||
"ssh_ready": int(nodes_summary.get("ssh_ready", 0) or 0),
|
||
"remote_access_ready": int(nodes_summary.get("remote_access_ready", 0) or 0),
|
||
"participating": int(nodes_summary.get("participating", 0) or 0),
|
||
"dispatch_active": int(nodes_summary.get("dispatch_active", 0) or 0),
|
||
"standby": int(nodes_summary.get("standby", 0) or 0),
|
||
"load_syncing": int(nodes_summary.get("load_syncing", 0) or 0),
|
||
"queue_retrying_nodes": int(nodes_summary.get("queue_retrying_nodes", 0) or 0),
|
||
"queue_dead_letter_nodes": int(nodes_summary.get("queue_dead_letter_nodes", 0) or 0),
|
||
"queue_pending_records": int(nodes_summary.get("queue_pending_records", 0) or 0),
|
||
"queue_dead_letter_records": int(nodes_summary.get("queue_dead_letter_records", 0) or 0),
|
||
"agent_state_counts": dict(nodes_summary.get("status_counts") or {}),
|
||
"remote_access_state_counts": dict(nodes_summary.get("remote_access_state_counts") or {}),
|
||
"delivery_queue_state_counts": dict(nodes_summary.get("delivery_queue_state_counts") or {}),
|
||
"problem_nodes": problem_nodes[:8],
|
||
"first_handover_gap_node_code": first_handover_gap_node_code,
|
||
"first_handover_gap": first_handover_gap,
|
||
"error": str(nodes_section.get("error") or ""),
|
||
},
|
||
"release_hub": {
|
||
"http_status": str(release_hub_section.get("http_status") or "500"),
|
||
"available": bool(release_hub_section.get("available", False)),
|
||
"elapsed_ms": int(release_hub_section.get("elapsed_ms", 0) or 0),
|
||
"latest_release_version": str(latest_release.get("release_version") or ""),
|
||
"latest_release_status": str(latest_release.get("status") or ""),
|
||
"default_rollout_gate_status": str(default_rollout_gate.get("status") or ""),
|
||
"default_rollout_gate_label": str(default_rollout_gate.get("status_label") or ""),
|
||
"default_rollout_gate_summary": str(default_rollout_gate.get("summary_text") or default_rollout_gate.get("summary") or ""),
|
||
"default_rollout_gate_focus_ref": dict(default_rollout_gate.get("focus_ref") or {}),
|
||
"launchpad_status": str(launchpad_status.get("status") or ""),
|
||
"launchpad_status_label": str(launchpad_status.get("status_label") or ""),
|
||
"launchpad_summary": str(launchpad_status.get("summary_text") or launchpad_status.get("summary") or ""),
|
||
"recommended_action_code": str(launchpad_status.get("recommended_action_code") or ""),
|
||
"recommended_execution_mode": str(launchpad_status.get("recommended_execution_mode") or ""),
|
||
"focus_ref": dict(launchpad_status.get("focus_ref") or {}),
|
||
"error": str(release_hub_section.get("error") or ""),
|
||
},
|
||
"runtime_build_info": {
|
||
"http_status": str(build_info_section.get("http_status") or "500"),
|
||
"available": bool(build_info_section.get("available", False)),
|
||
"elapsed_ms": int(build_info_section.get("elapsed_ms", 0) or 0),
|
||
"source": str(build_info.get("source") or ""),
|
||
"package_name": str(build_info.get("package_name") or ""),
|
||
"generated_at": str(build_info.get("generated_at") or ""),
|
||
"commit_sha": str(build_info.get("commit_sha") or ""),
|
||
"commit_ref": str(build_info.get("commit_ref") or ""),
|
||
"checksum": str(build_info.get("checksum") or ""),
|
||
"manifest_path": str(build_info.get("manifest_path") or ""),
|
||
"repository_capabilities": repository_capabilities,
|
||
"route_surface": route_surface,
|
||
"route_surface_missing_keys": route_surface_missing_keys,
|
||
"route_surface_declares_bootstrap_plan": route_surface_declares_bootstrap_plan,
|
||
"runtime_schema_stale": runtime_schema_stale,
|
||
"error": str(build_info_section.get("error") or ""),
|
||
},
|
||
"playbook_runs": {
|
||
"http_status": str(playbook_runs_section.get("http_status") or "500"),
|
||
"available": bool(playbook_runs_section.get("available", False)),
|
||
"elapsed_ms": int(playbook_runs_section.get("elapsed_ms", 0) or 0),
|
||
"recent_total": len(playbook_runs),
|
||
"problem_runs_total": len(problem_runs),
|
||
"problem_runs": problem_runs[:5],
|
||
"error": str(playbook_runs_section.get("error") or ""),
|
||
},
|
||
"activity_stream": {
|
||
"http_status": str(activity_stream_section.get("http_status") or "500"),
|
||
"available": bool(activity_stream_section.get("available", False)),
|
||
"elapsed_ms": int(activity_stream_section.get("elapsed_ms", 0) or 0),
|
||
"recent_total": len(activities),
|
||
"status_counts": activity_counts,
|
||
"top_items": [
|
||
{
|
||
"kind": str(item.get("kind") or "").strip(),
|
||
"status": str(item.get("status") or item.get("job_status") or "").strip(),
|
||
"summary": str(item.get("summary_text") or item.get("summary") or "").strip(),
|
||
"occurred_at": str(item.get("occurred_at") or "").strip(),
|
||
"focus_ref": dict(item.get("focus_ref") or {}),
|
||
"source_focus_ref": dict(item.get("source_focus_ref") or {}),
|
||
"ui_intent_kind": str(((item.get("ui_intent") or {}).get("kind") or "")).strip(),
|
||
}
|
||
for item in activities[:5]
|
||
],
|
||
"error": str(activity_stream_section.get("error") or ""),
|
||
},
|
||
}
|
||
|
||
|
||
def get_ops_capabilities() -> dict:
|
||
contract_registry = get_ops_contract_registry()
|
||
return {
|
||
"control_plane": {
|
||
"deployment_mode": "overseas-control-plane",
|
||
"codex_driver_ready": True,
|
||
"ui_button_ready": True,
|
||
"daily_entrypoint": "overseas dashboard / ops api",
|
||
},
|
||
"contract_registry": {
|
||
"endpoint": "/api/v1/ops/contracts",
|
||
"detail_endpoint_pattern": "/api/v1/ops/contracts/{contract_key}",
|
||
"registry_version": str(contract_registry.get("registry_version") or _OPS_CONTRACT_REGISTRY_VERSION),
|
||
"schema_version": str(contract_registry.get("schema_version") or _OPS_CONTRACT_SCHEMA_VERSION),
|
||
"contracts_total": int(contract_registry.get("contracts_total", 0) or 0),
|
||
"contract_keys": [
|
||
str(item.get("key") or "").strip()
|
||
for item in list(contract_registry.get("contracts") or [])
|
||
if str(item.get("key") or "").strip()
|
||
],
|
||
"schema_docs": {
|
||
str(item.get("key") or "").strip(): str(item.get("schema_doc_path") or "").strip()
|
||
for item in list(contract_registry.get("contracts") or [])
|
||
if str(item.get("key") or "").strip()
|
||
},
|
||
},
|
||
"capabilities": [
|
||
{
|
||
"key": "cluster_visibility",
|
||
"title": "集群可观测",
|
||
"current_state": "ready",
|
||
"today": "已具备 runtime/cluster/readiness/sync-summary 聚合能力。",
|
||
"target": "继续增强节点参与度、诊断报告和远端日志回传。",
|
||
},
|
||
{
|
||
"key": "remote_log_sync",
|
||
"title": "远端检测日志回传",
|
||
"current_state": "ready",
|
||
"today": "已具备日志回传开关、关键/全量模式、聚合样本面板与按节点现场日志钻取。",
|
||
"target": "后续如规模继续扩大,再补游标续传、流式 tail 与更长窗口归档。",
|
||
},
|
||
{
|
||
"key": "button_driven_ops",
|
||
"title": "后台按钮化运维",
|
||
"current_state": "partial",
|
||
"today": "已有 runtime actions / ops jobs / ops playbooks,开始收口为统一按钮化编排。",
|
||
"target": "继续扩大安装、更新、重启、巡检、诊断包收集与发布回滚 playbook。",
|
||
},
|
||
{
|
||
"key": "node_agent",
|
||
"title": "节点 Agent",
|
||
"current_state": "in_progress",
|
||
"today": "已具备 agent token / register / heartbeat / pull / complete 协议与本地执行器骨架。",
|
||
"target": "继续增强 stdout/stderr 游标回传、流式日志、诊断包与更多动作模板。",
|
||
},
|
||
{
|
||
"key": "release_management",
|
||
"title": "发布包管理",
|
||
"current_state": "partial",
|
||
"today": "已具备 release / rollout / deploy.release 能力,支持 checksum、切换 current、健康检查、失败回滚,以及 Agent / SSH 双通道单节点发布。",
|
||
"target": "继续增强批次策略、灰度放量、发布观测面板与一键回滚编排。",
|
||
},
|
||
],
|
||
"action_catalog": [
|
||
{"action": "node.bootstrap", "title": "纳管新节点", "transport": "ops-job -> control-plane -> bootstrap-plan"},
|
||
{"action": "deploy.release", "title": "发布新版本", "transport": "ops-job -> node-agent / ssh-executor"},
|
||
{
|
||
"action": "deploy.release.smart_worker",
|
||
"title": "按最新包一键 Worker 灰度",
|
||
"transport": "ops api -> package release builder -> rollout engine",
|
||
},
|
||
{
|
||
"action": "deploy.release.smart_control",
|
||
"title": "按最新包一键 Control 发布",
|
||
"transport": "ops api -> package release builder -> rollout engine",
|
||
},
|
||
{
|
||
"action": "deploy.rollout.smart_worker",
|
||
"title": "对已有 Release 一键 Worker 灰度",
|
||
"transport": "ops api -> rollout engine",
|
||
},
|
||
{
|
||
"action": "deploy.rollout.smart_control",
|
||
"title": "对已有 Release 一键 Control 发布",
|
||
"transport": "ops api -> rollout engine",
|
||
},
|
||
{"action": "service.restart", "title": "重启 API / Worker / Sync-Agent", "transport": "ops-job -> node-agent / ssh-executor"},
|
||
{"action": "logs.collect", "title": "收集远端日志", "transport": "ops-job -> node-agent / ssh-executor"},
|
||
{"action": "diagnostics.collect", "title": "收集诊断包", "transport": "ops-job -> node-agent / ssh-executor"},
|
||
{"action": "ops.playbook.execute", "title": "执行标准 playbook", "transport": "ops api -> playbook expander -> ops-jobs"},
|
||
{"action": "health.check", "title": "执行巡检", "transport": "ops-job -> node-agent"},
|
||
{"action": "deploy.rollout.advance", "title": "推进下一批发布", "transport": "ops api -> rollout engine"},
|
||
{"action": "runtime.restart_api", "title": "本机重启 API", "transport": "ops-job -> local-runtime / ssh-executor"},
|
||
{"action": "runtime.start_worker", "title": "本机启动 Worker", "transport": "ops-job -> local-runtime / ssh-executor"},
|
||
{"action": "runtime.stop_worker", "title": "本机停止 Worker", "transport": "ops-job -> local-runtime / ssh-executor"},
|
||
{"action": "runtime.start_sync_agent", "title": "本机启动 Sync Agent", "transport": "ops-job -> local-runtime / ssh-executor"},
|
||
{"action": "runtime.stop_sync_agent", "title": "本机停止 Sync Agent", "transport": "ops-job -> local-runtime / ssh-executor"},
|
||
{"action": "runtime.pull_tasks", "title": "本机立即拉取任务", "transport": "ops-job -> local-runtime"},
|
||
{"action": "runtime.push_sync", "title": "本机立即推送同步", "transport": "ops-job -> local-runtime"},
|
||
{"action": "health.snapshot", "title": "采集本机运行时快照", "transport": "ops-job -> local-runtime / ssh-executor"},
|
||
],
|
||
}
|
||
|
||
|
||
def get_ops_go_live_summary(*, base_url: str = "") -> dict:
|
||
normalized_base_url = str(base_url or "").strip() or "http://127.0.0.1:8100"
|
||
stack_payload = get_ops_stack_diagnosis(base_url=normalized_base_url)
|
||
contracts_payload = get_ops_contract_registry()
|
||
launchpad_payload = get_release_launchpad()
|
||
build_info_payload = get_runtime_build_info()
|
||
nodes_payload = list_managed_nodes_with_agent_state()
|
||
overview_payload = get_ops_overview()
|
||
|
||
diagnosis = dict(stack_payload.get("diagnosis") or {})
|
||
diagnosis_issues = [dict(item or {}) for item in list(diagnosis.get("issues") or []) if item]
|
||
route_surface = dict(build_info_payload.get("route_surface") or {})
|
||
repository_capabilities = dict(build_info_payload.get("repository_capabilities") or {})
|
||
nodes_summary = dict(nodes_payload.get("summary") or {})
|
||
execution_scene = dict(overview_payload.get("execution_scene") or {})
|
||
log_sync = dict(execution_scene.get("log_sync") or overview_payload.get("log_sync") or {})
|
||
recommendation = dict(overview_payload.get("recommendation") or {})
|
||
launchpad_status = dict(launchpad_payload.get("launchpad_status") or {})
|
||
next_step = dict(diagnosis.get("next_step") or {})
|
||
operator_decision = dict(diagnosis.get("operator_decision") or {})
|
||
contracts = [dict(item or {}) for item in list(contracts_payload.get("contracts") or []) if item]
|
||
|
||
stack_status = str(diagnosis.get("stack_status") or "").strip() or "unknown"
|
||
launchpad_status_code = (
|
||
str(launchpad_status.get("status") or "").strip()
|
||
or str((stack_payload.get("release_hub") or {}).get("launchpad_status") or "").strip()
|
||
)
|
||
managed_enabled = int(nodes_summary.get("managed_enabled", 0) or 0)
|
||
remote_access_ready = int(nodes_summary.get("remote_access_ready", 0) or 0)
|
||
queue_dead_letter_nodes = int(nodes_summary.get("queue_dead_letter_nodes", 0) or 0)
|
||
participating_nodes_total = int(log_sync.get("participating_node_count", 0) or 0)
|
||
log_sync_covered_nodes = int(log_sync.get("covered_participating_node_count", 0) or 0)
|
||
launchpad_onboarding_bootstrap_pending_nodes = int(
|
||
launchpad_status.get("onboarding_bootstrap_pending_nodes", 0) or 0
|
||
)
|
||
launchpad_onboarding_acceptance_ready_nodes = int(
|
||
launchpad_status.get("onboarding_acceptance_ready_nodes", 0) or 0
|
||
)
|
||
start_delivery_issue_total = sum(
|
||
1
|
||
for item in diagnosis_issues
|
||
if str(item.get("code") or "").strip() == "ops_job_start_delivery_failed_local"
|
||
)
|
||
|
||
blocking_reasons: list[str] = []
|
||
warnings: list[str] = []
|
||
|
||
if stack_status == "blocked":
|
||
blocking_reasons.append("stack_diagnosis=blocked")
|
||
elif stack_status == "attention":
|
||
warnings.append("stack_diagnosis=attention")
|
||
|
||
route_surface_complete = bool(route_surface.get("surface_complete", False))
|
||
route_surface_missing_keys = [
|
||
str(item or "").strip()
|
||
for item in list(route_surface.get("missing_keys") or [])
|
||
if str(item or "").strip()
|
||
]
|
||
route_surface_expected_paths = {
|
||
str(key).strip(): str(value).strip()
|
||
for key, value in dict(route_surface.get("expected_paths") or {}).items()
|
||
if str(key).strip()
|
||
}
|
||
route_surface_declares_bootstrap_plan = "ops_node_handover_bootstrap_plan" in route_surface_expected_paths
|
||
runtime_schema_stale = bool(
|
||
repository_capabilities.get("supports_install_command_block", False) and not route_surface_declares_bootstrap_plan
|
||
)
|
||
if not route_surface_complete:
|
||
blocking_reasons.append(
|
||
"route_surface_incomplete"
|
||
+ (f":{','.join(route_surface_missing_keys)}" if route_surface_missing_keys else "")
|
||
)
|
||
elif runtime_schema_stale:
|
||
warnings.append("runtime_build_schema_stale")
|
||
|
||
if not contracts:
|
||
blocking_reasons.append("contracts_unavailable")
|
||
|
||
if launchpad_status_code == "blocked":
|
||
blocking_reasons.append("release_launchpad=blocked")
|
||
elif launchpad_status_code == "attention":
|
||
warnings.append("release_launchpad=attention")
|
||
if launchpad_onboarding_bootstrap_pending_nodes > 0:
|
||
warnings.append(f"launchpad_onboarding_bootstrap_pending={launchpad_onboarding_bootstrap_pending_nodes}")
|
||
if launchpad_onboarding_acceptance_ready_nodes > 0:
|
||
warnings.append(f"launchpad_onboarding_acceptance_ready={launchpad_onboarding_acceptance_ready_nodes}")
|
||
|
||
if managed_enabled > 0 and remote_access_ready == 0:
|
||
blocking_reasons.append("remote_agent_ready=0")
|
||
elif managed_enabled > 0 and remote_access_ready < managed_enabled:
|
||
warnings.append(f"remote_agent_ready={remote_access_ready}/{managed_enabled}")
|
||
|
||
if queue_dead_letter_nodes > 0:
|
||
warnings.append(f"queue_dead_letter_nodes={queue_dead_letter_nodes}")
|
||
if start_delivery_issue_total > 0:
|
||
warnings.append(f"ops_job_start_delivery_failed_local={start_delivery_issue_total}")
|
||
|
||
if bool(log_sync.get("enabled")) and participating_nodes_total > 0:
|
||
if log_sync_covered_nodes == 0:
|
||
warnings.append(f"log_sync_waiting_sample=0/{participating_nodes_total}")
|
||
elif log_sync_covered_nodes < participating_nodes_total:
|
||
warnings.append(f"log_sync_partial={log_sync_covered_nodes}/{participating_nodes_total}")
|
||
|
||
publish_blocking_reasons: list[str] = []
|
||
publish_warnings: list[str] = []
|
||
if stack_status == "blocked":
|
||
publish_blocking_reasons.append("stack_diagnosis=blocked")
|
||
elif stack_status == "attention":
|
||
publish_warnings.append("stack_diagnosis=attention")
|
||
if not route_surface_complete:
|
||
publish_blocking_reasons.append(
|
||
"route_surface_incomplete"
|
||
+ (f":{','.join(route_surface_missing_keys)}" if route_surface_missing_keys else "")
|
||
)
|
||
elif runtime_schema_stale:
|
||
publish_warnings.append("runtime_build_schema_stale")
|
||
if not contracts:
|
||
publish_blocking_reasons.append("contracts_unavailable")
|
||
if launchpad_status_code == "blocked":
|
||
publish_blocking_reasons.append("release_launchpad=blocked")
|
||
elif launchpad_status_code == "attention":
|
||
publish_warnings.append("release_launchpad=attention")
|
||
if launchpad_onboarding_bootstrap_pending_nodes > 0:
|
||
publish_warnings.append(f"launchpad_onboarding_bootstrap_pending={launchpad_onboarding_bootstrap_pending_nodes}")
|
||
if launchpad_onboarding_acceptance_ready_nodes > 0:
|
||
publish_warnings.append(f"launchpad_onboarding_acceptance_ready={launchpad_onboarding_acceptance_ready_nodes}")
|
||
if managed_enabled > 0 and remote_access_ready == 0:
|
||
publish_blocking_reasons.append("remote_agent_ready=0")
|
||
elif managed_enabled > 0 and remote_access_ready < managed_enabled:
|
||
publish_warnings.append(f"remote_agent_ready={remote_access_ready}/{managed_enabled}")
|
||
if queue_dead_letter_nodes > 0:
|
||
publish_blocking_reasons.append(f"queue_dead_letter_nodes={queue_dead_letter_nodes}")
|
||
if start_delivery_issue_total > 0:
|
||
publish_warnings.append(f"ops_job_start_delivery_failed_local={start_delivery_issue_total}")
|
||
if bool(log_sync.get("enabled")) and participating_nodes_total > 0:
|
||
if log_sync_covered_nodes == 0:
|
||
publish_warnings.append(f"log_sync_waiting_sample=0/{participating_nodes_total}")
|
||
elif log_sync_covered_nodes < participating_nodes_total:
|
||
publish_warnings.append(f"log_sync_partial={log_sync_covered_nodes}/{participating_nodes_total}")
|
||
|
||
publish_status = "ready"
|
||
if publish_blocking_reasons:
|
||
publish_status = "blocked"
|
||
elif publish_warnings:
|
||
publish_status = "attention"
|
||
publish_ready = publish_status == "ready"
|
||
if publish_status == "ready":
|
||
publish_status_label = "可发布"
|
||
publish_summary = "当前执行面、路由面、合同面与发布门禁已收口,可以进入正式发版。"
|
||
elif publish_status == "attention":
|
||
publish_status_label = "可发布但建议先复核"
|
||
publish_summary = "当前没有硬阻断,但仍有上线前关注项,建议先完成复核再正式发版。"
|
||
else:
|
||
publish_status_label = "暂不可发布"
|
||
publish_summary = "当前仍有硬阻断项,不能直接进入正式发版。"
|
||
|
||
go_live_status = "ready"
|
||
if blocking_reasons:
|
||
go_live_status = "blocked"
|
||
elif warnings:
|
||
go_live_status = "attention"
|
||
|
||
next_step_action_code = (
|
||
str(next_step.get("action_code") or "").strip()
|
||
or str(recommendation.get("primary_action_code") or "").strip()
|
||
or str(launchpad_status.get("recommended_action_code") or "").strip()
|
||
)
|
||
next_step_reason = (
|
||
str(next_step.get("reason") or "").strip()
|
||
or str(recommendation.get("reason") or "").strip()
|
||
or "来自 overview / launchpad fallback"
|
||
)
|
||
stack_launchpad_target_node_code = str(
|
||
diagnosis.get("launchpad_recommended_target_node_code")
|
||
or (dict(next_step.get("focus_ref") or {}).get("node_code") if next_step_action_code in {"bootstrap_run", "run_acceptance"} else "")
|
||
or ""
|
||
).strip()
|
||
stack_launchpad_recovery_label = str(
|
||
diagnosis.get("launchpad_recommended_recovery_label")
|
||
or ("签发接入工单" if next_step_action_code == "bootstrap_run" else "执行接管验收" if next_step_action_code == "run_acceptance" else "")
|
||
or ""
|
||
).strip()
|
||
stack_launchpad_recovery_summary = str(
|
||
diagnosis.get("launchpad_recommended_recovery_summary")
|
||
or next_step_reason
|
||
or ""
|
||
).strip()
|
||
launchpad_recommended_target_node_code = str(
|
||
launchpad_status.get("recommended_target_node_code") or stack_launchpad_target_node_code or ""
|
||
).strip()
|
||
launchpad_recommended_recovery_label = str(
|
||
launchpad_status.get("recommended_recovery_label") or stack_launchpad_recovery_label or ""
|
||
).strip()
|
||
launchpad_recommended_recovery_summary = str(
|
||
launchpad_status.get("recommended_recovery_summary") or stack_launchpad_recovery_summary or ""
|
||
).strip()
|
||
operator_lane = str(operator_decision.get("lane") or "").strip()
|
||
if not operator_lane:
|
||
operator_lane = "recovery" if blocking_reasons else ("verification" if warnings else "go-live")
|
||
operator_title = (
|
||
str(operator_decision.get("title") or "").strip()
|
||
or str(recommendation.get("title") or "").strip()
|
||
or str(launchpad_status.get("status_label") or "").strip()
|
||
or ("先处理阻塞项" if blocking_reasons else "可以进入上线复核")
|
||
)
|
||
operator_primary_command_key = str(operator_decision.get("primary_command_key") or "").strip() or next_step_action_code
|
||
|
||
recommended_commands = {
|
||
"stack_summary": build_bash_command("check_ops_center_stack.sh", normalized_base_url, "summary"),
|
||
"contracts": build_bash_command("check_ops_contracts.sh", normalized_base_url),
|
||
"ops_plane": build_bash_command("check_ops_plane.sh", normalized_base_url),
|
||
"release_hub": build_bash_command("check_release_hub.sh", normalized_base_url),
|
||
"inspection": build_bash_command("check_ops_inspection.sh", normalized_base_url),
|
||
"overview": build_bash_command("drive_ops_center.sh", "overview", normalized_base_url),
|
||
"go_live_recover": build_bash_command("drive_ops_center.sh", "go-live-recover", normalized_base_url),
|
||
"doctor_export": build_bash_command("drive_ops_center.sh", "doctor-export", "/tmp/domaincheck-go-live", normalized_base_url),
|
||
}
|
||
if next_step_action_code:
|
||
recommended_commands["next_step"] = build_bash_command("drive_ops_center.sh", "driver-resolve", normalized_base_url, next_step_action_code)
|
||
if runtime_schema_stale:
|
||
recommended_commands["runtime_refresh_recover"] = build_bash_command(
|
||
"drive_ops_center.sh",
|
||
"runtime-refresh-recover",
|
||
normalized_base_url,
|
||
)
|
||
recommended_commands["api_restart"] = recommended_commands["runtime_refresh_recover"]
|
||
recommended_commands["bootstrap_plan_recheck"] = build_bash_command(
|
||
"drive_ops_center.sh",
|
||
"node-bootstrap-plan",
|
||
normalized_base_url,
|
||
"mainland-worker-01",
|
||
)
|
||
if bool(log_sync.get("enabled")):
|
||
recommended_commands["log_sync_logs"] = build_bash_command("drive_ops_center.sh", "driver-resolve", normalized_base_url, "open_worker_logs_participating")
|
||
recommended_commands["log_sync_inspection"] = build_bash_command("drive_ops_center.sh", "driver-resolve", normalized_base_url, "run_inspection_participating")
|
||
|
||
return {
|
||
"base_url": normalized_base_url,
|
||
"generated_at": datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
|
||
"go_live_status": go_live_status,
|
||
"publish_ready": publish_ready,
|
||
"publish_status": publish_status,
|
||
"publish_status_label": publish_status_label,
|
||
"publish_summary": publish_summary,
|
||
"stack_status": stack_status,
|
||
"contracts_ready": bool(contracts),
|
||
"contracts_total": len(contracts),
|
||
"launchpad_status": launchpad_status_code,
|
||
"launchpad_status_label": str(launchpad_status.get("status_label") or "").strip(),
|
||
"launchpad_recommended_action_code": str(launchpad_status.get("recommended_action_code") or "").strip(),
|
||
"launchpad_recommended_target_node_code": launchpad_recommended_target_node_code,
|
||
"launchpad_recommended_recovery_label": launchpad_recommended_recovery_label,
|
||
"launchpad_recommended_recovery_summary": launchpad_recommended_recovery_summary,
|
||
"launchpad_onboarding_bootstrap_pending_nodes": launchpad_onboarding_bootstrap_pending_nodes,
|
||
"launchpad_onboarding_acceptance_ready_nodes": launchpad_onboarding_acceptance_ready_nodes,
|
||
"route_surface_complete": route_surface_complete,
|
||
"route_surface_missing_keys": route_surface_missing_keys,
|
||
"route_surface_declares_bootstrap_plan": route_surface_declares_bootstrap_plan,
|
||
"runtime_schema_stale": runtime_schema_stale,
|
||
"repository_capabilities": repository_capabilities,
|
||
"managed_enabled": managed_enabled,
|
||
"remote_access_ready": remote_access_ready,
|
||
"queue_dead_letter_nodes": queue_dead_letter_nodes,
|
||
"activity_start_delivery_issue_total": start_delivery_issue_total,
|
||
"participating_nodes_total": participating_nodes_total,
|
||
"log_sync_enabled": bool(log_sync.get("enabled")),
|
||
"log_sync_state": str(log_sync.get("state") or "").strip(),
|
||
"log_sync_mode": str(log_sync.get("mode") or "").strip(),
|
||
"log_sync_covered_nodes": log_sync_covered_nodes,
|
||
"log_sync_missing_node_codes": [
|
||
str(item or "").strip()
|
||
for item in list(log_sync.get("missing_participating_nodes") or [])
|
||
if str(item or "").strip()
|
||
],
|
||
"next_step_action_code": next_step_action_code,
|
||
"next_step_reason": next_step_reason,
|
||
"operator_lane": operator_lane,
|
||
"operator_title": operator_title,
|
||
"operator_primary_command_key": operator_primary_command_key,
|
||
"publish_blocking_reasons": publish_blocking_reasons,
|
||
"publish_warnings": publish_warnings,
|
||
"blocking_reasons": blocking_reasons,
|
||
"warnings": warnings,
|
||
"recommended_commands": recommended_commands,
|
||
"source_refs": {
|
||
"stack_diagnosis_contract_key": str(diagnosis.get("contract_key") or "").strip(),
|
||
"contracts_registry_version": str(contracts_payload.get("registry_version") or "").strip(),
|
||
"runtime_build_commit_sha": str(build_info_payload.get("commit_sha") or "").strip(),
|
||
"release_focus_ref": dict(launchpad_status.get("focus_ref") or {}),
|
||
},
|
||
}
|
||
|
||
|
||
def _build_doctor_preferred_surface(operator_decision: dict) -> str:
|
||
next_focus = dict(operator_decision.get("next_focus") or {})
|
||
focus_kind = str(next_focus.get("kind") or "").strip()
|
||
lane = str(operator_decision.get("lane") or "").strip()
|
||
if focus_kind == "node_scene_log":
|
||
return "execution-scene"
|
||
if lane == "node_handover":
|
||
return "managed-nodes"
|
||
if lane == "release":
|
||
return "release-hub"
|
||
if lane == "observability":
|
||
return "execution-scene"
|
||
if lane in {"ops_jobs", "steady"}:
|
||
return "driver-feed"
|
||
return "stack-summary"
|
||
|
||
|
||
def _dedupe_command_list(command_candidates: list[object]) -> list[str]:
|
||
normalized_commands: list[str] = []
|
||
for candidate in command_candidates:
|
||
if isinstance(candidate, (list, tuple)):
|
||
for nested in candidate:
|
||
nested_command = str(nested or "").strip()
|
||
if nested_command and nested_command not in normalized_commands:
|
||
normalized_commands.append(nested_command)
|
||
continue
|
||
command = str(candidate or "").strip()
|
||
if command and command not in normalized_commands:
|
||
normalized_commands.append(command)
|
||
return normalized_commands
|
||
|
||
|
||
def _doctor_recommended_command_map(
|
||
*,
|
||
base_url: str,
|
||
manifest_target: str,
|
||
raw_recommended_commands: list[str],
|
||
) -> dict[str, str]:
|
||
target = str(manifest_target or "").strip() or str(base_url or "").strip() or "http://127.0.0.1:8100"
|
||
mapped = {
|
||
"doctor_decision": build_bash_command("drive_ops_center.sh", "doctor-decision", target),
|
||
}
|
||
if str(manifest_target or "").strip():
|
||
mapped["go_live_review"] = build_bash_command("drive_ops_center.sh", "go-live-review", manifest_target)
|
||
mapped["go_live_signoff"] = build_bash_command("drive_ops_center.sh", "go-live-signoff", manifest_target)
|
||
else:
|
||
mapped["stack_check"] = build_bash_command("drive_ops_center.sh", "stack-diagnosis", base_url, "summary")
|
||
mapped["stack_diagnosis"] = mapped["stack_check"]
|
||
mapped["doctor"] = build_bash_command("drive_ops_center.sh", "doctor", base_url)
|
||
|
||
for index, command in enumerate(_dedupe_command_list(list(raw_recommended_commands or []))[:4]):
|
||
key = "primary_recovery" if index == 0 else f"follow_up_{index}"
|
||
mapped[key] = command
|
||
return mapped
|
||
|
||
|
||
def _doctor_scene_log_problem_node_codes(scene_log_reports: list[dict]) -> list[str]:
|
||
problem_nodes: list[str] = []
|
||
for item in scene_log_reports:
|
||
if not isinstance(item, dict):
|
||
continue
|
||
node_code = str(item.get("node_code") or "").strip()
|
||
if not node_code:
|
||
continue
|
||
if item.get("ok") is False:
|
||
if node_code not in problem_nodes:
|
||
problem_nodes.append(node_code)
|
||
continue
|
||
status = str(item.get("status") or "").strip().lower()
|
||
if status and status not in {"ok", "ready", "success"} and node_code not in problem_nodes:
|
||
problem_nodes.append(node_code)
|
||
return problem_nodes
|
||
|
||
|
||
def _build_live_doctor_payload(base_url: str) -> tuple[dict, dict]:
|
||
go_live_summary = get_ops_go_live_summary(base_url=base_url)
|
||
stack_payload = get_ops_stack_diagnosis(base_url=base_url)
|
||
diagnosis = dict(stack_payload.get("diagnosis") or {})
|
||
operator_decision = dict(diagnosis.get("operator_decision") or {})
|
||
next_step = dict(diagnosis.get("next_step") or {})
|
||
next_actions = dict(diagnosis.get("next_actions") or {})
|
||
recommended_commands = dict(diagnosis.get("recommended_commands") or {})
|
||
stack_status = str(diagnosis.get("stack_status") or "").strip()
|
||
go_live_status = str(go_live_summary.get("go_live_status") or "").strip()
|
||
publish_status = str(go_live_summary.get("publish_status") or "").strip()
|
||
|
||
status = "ready"
|
||
if "blocked" in {stack_status, go_live_status, publish_status}:
|
||
status = "blocked"
|
||
elif "attention" in {stack_status, go_live_status, publish_status}:
|
||
status = "attention"
|
||
|
||
preferred_surface = _build_doctor_preferred_surface(operator_decision)
|
||
headline = (
|
||
str(operator_decision.get("title") or "").strip()
|
||
or str(diagnosis.get("headline") or "").strip()
|
||
or str(go_live_summary.get("headline") or "").strip()
|
||
or "当前总检主决策还没有形成明确 headline。"
|
||
)
|
||
detail = (
|
||
str(operator_decision.get("summary") or "").strip()
|
||
or str(next_step.get("reason") or "").strip()
|
||
or str(go_live_summary.get("next_step_reason") or "").strip()
|
||
or str(diagnosis.get("blocked_issue_detail") or "").strip()
|
||
)
|
||
next_action_code = (
|
||
str(operator_decision.get("primary_command_key") or "").strip()
|
||
or str(next_step.get("action_code") or "").strip()
|
||
or str(go_live_summary.get("next_step_action_code") or "").strip()
|
||
)
|
||
|
||
raw_commands = _dedupe_command_list(
|
||
[
|
||
next_actions.get("primary_command"),
|
||
next_actions.get("secondary_command"),
|
||
recommended_commands.get(next_action_code),
|
||
list(recommended_commands.values())[:3],
|
||
]
|
||
)
|
||
|
||
decision = {
|
||
"status": status,
|
||
"reason_code": str(
|
||
operator_decision.get("reason_code") or next_step.get("source") or stack_status or "live_fallback"
|
||
).strip(),
|
||
"headline": headline,
|
||
"detail": detail,
|
||
"preferred_surface": preferred_surface,
|
||
"next_action_code": next_action_code,
|
||
"recommended_commands": raw_commands,
|
||
"evidence": {
|
||
"blocking_issue_total": int(diagnosis.get("blocking_issue_total", 0) or 0),
|
||
"warning_issue_total": int(diagnosis.get("warning_issue_total", 0) or 0),
|
||
"missing_surfaces": list(diagnosis.get("missing_surfaces") or []),
|
||
},
|
||
}
|
||
summary = {
|
||
"ok": status == "ready",
|
||
"required_failures": [],
|
||
"optional_unavailable": [],
|
||
"scene_log_reports_total": 0,
|
||
"scene_log_reports_ok": 0,
|
||
"scene_log_status_counts": {},
|
||
"scene_log_reports": [],
|
||
"contract_surface_gaps": [],
|
||
"launchpad_recommended_target_node_code": str(
|
||
go_live_summary.get("launchpad_recommended_target_node_code") or ""
|
||
).strip(),
|
||
"launchpad_recommended_recovery_label": str(
|
||
go_live_summary.get("launchpad_recommended_recovery_label") or ""
|
||
).strip(),
|
||
"launchpad_recommended_recovery_summary": str(
|
||
go_live_summary.get("launchpad_recommended_recovery_summary") or ""
|
||
).strip(),
|
||
"launchpad_onboarding_bootstrap_pending_nodes": int(
|
||
go_live_summary.get("launchpad_onboarding_bootstrap_pending_nodes", 0) or 0
|
||
),
|
||
"launchpad_onboarding_acceptance_ready_nodes": int(
|
||
go_live_summary.get("launchpad_onboarding_acceptance_ready_nodes", 0) or 0
|
||
),
|
||
}
|
||
return decision, summary
|
||
|
||
|
||
def get_ops_doctor_decision(*, base_url: str = "", report_dir: str = "") -> dict:
|
||
normalized_base_url = str(base_url or "").strip() or "http://127.0.0.1:8100"
|
||
normalized_report_dir = str(report_dir or "").strip()
|
||
contract_registry = get_ops_contract_registry()
|
||
|
||
manifest_path: Path | None = None
|
||
resolved_report_dir: Path | None = None
|
||
if normalized_report_dir:
|
||
candidate_path = Path(normalized_report_dir)
|
||
if candidate_path.is_dir():
|
||
resolved_report_dir = candidate_path
|
||
nested_manifest = candidate_path / "manifest.json"
|
||
if nested_manifest.is_file():
|
||
manifest_path = nested_manifest
|
||
elif candidate_path.is_file():
|
||
manifest_path = candidate_path
|
||
resolved_report_dir = candidate_path.parent
|
||
else:
|
||
manifest_path, resolved_report_dir = _find_latest_go_live_bundle_manifest()
|
||
|
||
raw_decision: dict = {}
|
||
raw_summary: dict = {}
|
||
source = "live_fallback"
|
||
|
||
if manifest_path is not None and manifest_path.is_file():
|
||
manifest = _load_json_path(manifest_path)
|
||
artifacts = list(manifest.get("artifacts") or [])
|
||
artifact_by_key = {
|
||
str(item.get("key") or "").strip(): dict(item)
|
||
for item in artifacts
|
||
if isinstance(item, dict) and str(item.get("key") or "").strip()
|
||
}
|
||
doctor_artifact = artifact_by_key.get("doctor_decision") or {}
|
||
doctor_artifact_path = Path(str(doctor_artifact.get("path") or "").strip()) if doctor_artifact else None
|
||
doctor_payload = _load_json_path(doctor_artifact_path) if doctor_artifact_path else {}
|
||
raw_decision = dict(doctor_payload.get("decision") or manifest.get("decision") or {})
|
||
raw_summary = dict(doctor_payload.get("summary") or manifest.get("summary") or {})
|
||
if raw_decision:
|
||
source = "bundle_manifest" if doctor_payload else "doctor_manifest"
|
||
|
||
if not raw_decision:
|
||
raw_decision, raw_summary = _build_live_doctor_payload(normalized_base_url)
|
||
source = "live_fallback"
|
||
|
||
status = str(raw_decision.get("status") or "").strip() or ("ready" if bool(raw_summary.get("ok", False)) else "attention")
|
||
status_label = {
|
||
"ready": "主决策已就绪",
|
||
"attention": "主决策待复核",
|
||
"blocked": "主决策阻断",
|
||
"missing": "待生成",
|
||
}.get(status, status or "主决策未知")
|
||
summary_scene_log_reports = [
|
||
dict(item)
|
||
for item in list(raw_summary.get("scene_log_reports") or [])
|
||
if isinstance(item, dict)
|
||
]
|
||
scene_log_problem_node_codes = _doctor_scene_log_problem_node_codes(summary_scene_log_reports)
|
||
manifest_target = str(manifest_path or normalized_report_dir or "").strip()
|
||
recommended_commands = _doctor_recommended_command_map(
|
||
base_url=normalized_base_url,
|
||
manifest_target=manifest_target,
|
||
raw_recommended_commands=[
|
||
str(item).strip()
|
||
for item in list(raw_decision.get("recommended_commands") or [])
|
||
if str(item).strip()
|
||
],
|
||
)
|
||
contract_keys = _normalize_ops_contract_keys(
|
||
[
|
||
"ops_doctor_decision_contract",
|
||
"ops_stack_diagnosis_contract",
|
||
"ops_driver_contract",
|
||
"ops_go_live_bundle_contract",
|
||
"ops_go_live_signoff_contract",
|
||
]
|
||
)
|
||
|
||
return {
|
||
"generated_at": datetime.now().isoformat(timespec="seconds"),
|
||
"base_url": normalized_base_url,
|
||
"contract_key": "ops_doctor_decision_contract",
|
||
"contract_version": str(
|
||
(contract_registry.get("contracts_by_key") or {}).get("ops_doctor_decision_contract", {}).get("version")
|
||
or _OPS_CONTRACT_SCHEMA_VERSION
|
||
).strip(),
|
||
"status": status,
|
||
"status_label": status_label,
|
||
"headline": str(raw_decision.get("headline") or "").strip() or "当前还没有生成总检主决策。",
|
||
"detail": str(raw_decision.get("detail") or "").strip(),
|
||
"source": source,
|
||
"report_dir": str((resolved_report_dir or manifest_path.parent) if manifest_path else normalized_report_dir),
|
||
"manifest_path": str(manifest_path or ""),
|
||
"decision_available": bool(raw_decision),
|
||
"decision": {
|
||
"status": status,
|
||
"reason_code": str(raw_decision.get("reason_code") or "").strip(),
|
||
"headline": str(raw_decision.get("headline") or "").strip(),
|
||
"detail": str(raw_decision.get("detail") or "").strip(),
|
||
"preferred_surface": str(raw_decision.get("preferred_surface") or "").strip(),
|
||
"next_action_code": str(raw_decision.get("next_action_code") or "").strip(),
|
||
"recommended_commands": [
|
||
str(item).strip()
|
||
for item in list(raw_decision.get("recommended_commands") or [])
|
||
if str(item).strip()
|
||
],
|
||
"evidence": dict(raw_decision.get("evidence") or {}),
|
||
},
|
||
"summary": {
|
||
"ok": bool(raw_summary.get("ok", False)),
|
||
"required_failures": [
|
||
str(item).strip()
|
||
for item in list(raw_summary.get("required_failures") or [])
|
||
if str(item).strip()
|
||
],
|
||
"optional_unavailable": [
|
||
str(item).strip()
|
||
for item in list(raw_summary.get("optional_unavailable") or [])
|
||
if str(item).strip()
|
||
],
|
||
"scene_log_reports_total": int(raw_summary.get("scene_log_reports_total", len(summary_scene_log_reports)) or 0),
|
||
"scene_log_reports_ok": int(raw_summary.get("scene_log_reports_ok", 0) or 0),
|
||
"scene_log_status_counts": dict(raw_summary.get("scene_log_status_counts") or {}),
|
||
"scene_log_reports": summary_scene_log_reports,
|
||
"scene_log_problem_node_codes": scene_log_problem_node_codes,
|
||
"contract_surface_gaps": [
|
||
str(item).strip()
|
||
for item in list(raw_summary.get("contract_surface_gaps") or [])
|
||
if str(item).strip()
|
||
],
|
||
"launchpad_recommended_target_node_code": str(
|
||
raw_summary.get("launchpad_recommended_target_node_code") or ""
|
||
).strip(),
|
||
"launchpad_recommended_recovery_label": str(
|
||
raw_summary.get("launchpad_recommended_recovery_label") or ""
|
||
).strip(),
|
||
"launchpad_recommended_recovery_summary": str(
|
||
raw_summary.get("launchpad_recommended_recovery_summary") or ""
|
||
).strip(),
|
||
"launchpad_onboarding_bootstrap_pending_nodes": int(
|
||
raw_summary.get("launchpad_onboarding_bootstrap_pending_nodes", 0) or 0
|
||
),
|
||
"launchpad_onboarding_acceptance_ready_nodes": int(
|
||
raw_summary.get("launchpad_onboarding_acceptance_ready_nodes", 0) or 0
|
||
),
|
||
},
|
||
"recommended_commands": recommended_commands,
|
||
"contract_navigation": _build_ops_contract_navigation(
|
||
contract_keys,
|
||
primary_contract_key="ops_doctor_decision_contract",
|
||
registry=contract_registry,
|
||
),
|
||
}
|
||
|
||
|
||
def get_ops_go_live_signoff(*, base_url: str = "", report_dir: str = "") -> dict:
|
||
normalized_base_url = str(base_url or "").strip() or "http://127.0.0.1:8100"
|
||
normalized_report_dir = str(report_dir or "").strip()
|
||
go_live_summary = get_ops_go_live_summary(base_url=normalized_base_url)
|
||
stack_payload = get_ops_stack_diagnosis(base_url=normalized_base_url)
|
||
driver_feed = get_ops_driver_feed()
|
||
codex_brief = get_ops_codex_brief()
|
||
release_launchpad = get_release_launchpad()
|
||
go_live_review = get_ops_go_live_review(base_url=normalized_base_url, report_dir=normalized_report_dir)
|
||
doctor_decision_payload = get_ops_doctor_decision(base_url=normalized_base_url, report_dir=normalized_report_dir)
|
||
contract_registry = get_ops_contract_registry()
|
||
|
||
stack_diagnosis = dict(stack_payload.get("diagnosis") or {})
|
||
driver_summary = dict(driver_feed.get("summary") or {})
|
||
codex_summary = dict(codex_brief.get("summary") or {})
|
||
launchpad_status = dict(release_launchpad.get("launchpad_status") or {})
|
||
operator_decision = dict(stack_diagnosis.get("operator_decision") or {})
|
||
next_step = dict(stack_diagnosis.get("next_step") or {})
|
||
doctor_decision = dict(doctor_decision_payload.get("decision") or {})
|
||
|
||
launchpad_target_node_codes = {
|
||
"go_live_summary": str(go_live_summary.get("launchpad_recommended_target_node_code") or "").strip(),
|
||
"driver_feed": str(driver_summary.get("launchpad_recommended_target_node_code") or "").strip(),
|
||
"codex_brief": str(codex_summary.get("launchpad_recommended_target_node_code") or "").strip(),
|
||
"release_launchpad": str(launchpad_status.get("recommended_target_node_code") or "").strip(),
|
||
}
|
||
launchpad_recovery_labels = {
|
||
"go_live_summary": str(go_live_summary.get("launchpad_recommended_recovery_label") or "").strip(),
|
||
"driver_feed": str(driver_summary.get("launchpad_recommended_recovery_label") or "").strip(),
|
||
"codex_brief": str(codex_summary.get("launchpad_recommended_recovery_label") or "").strip(),
|
||
"release_launchpad": str(launchpad_status.get("recommended_recovery_label") or "").strip(),
|
||
}
|
||
launchpad_recovery_summaries = {
|
||
"go_live_summary": str(go_live_summary.get("launchpad_recommended_recovery_summary") or "").strip(),
|
||
"driver_feed": str(driver_summary.get("launchpad_recommended_recovery_summary") or "").strip(),
|
||
"codex_brief": str(codex_summary.get("launchpad_recommended_recovery_summary") or "").strip(),
|
||
"release_launchpad": str(launchpad_status.get("recommended_recovery_summary") or "").strip(),
|
||
}
|
||
launchpad_bootstrap_pending_nodes = {
|
||
"go_live_summary": int(go_live_summary.get("launchpad_onboarding_bootstrap_pending_nodes", 0) or 0),
|
||
"driver_feed": int(driver_summary.get("launchpad_onboarding_bootstrap_pending_nodes", 0) or 0),
|
||
"codex_brief": int(codex_summary.get("launchpad_onboarding_bootstrap_pending_nodes", 0) or 0),
|
||
"release_launchpad": int(launchpad_status.get("onboarding_bootstrap_pending_nodes", 0) or 0),
|
||
}
|
||
launchpad_acceptance_ready_nodes = {
|
||
"go_live_summary": int(go_live_summary.get("launchpad_onboarding_acceptance_ready_nodes", 0) or 0),
|
||
"driver_feed": int(driver_summary.get("launchpad_onboarding_acceptance_ready_nodes", 0) or 0),
|
||
"codex_brief": int(codex_summary.get("launchpad_onboarding_acceptance_ready_nodes", 0) or 0),
|
||
"release_launchpad": int(launchpad_status.get("onboarding_acceptance_ready_nodes", 0) or 0),
|
||
}
|
||
|
||
def _nonempty_distinct(values: dict[str, str]) -> list[str]:
|
||
return sorted({value for value in values.values() if value})
|
||
|
||
def _distinct_ints(values: dict[str, int]) -> list[int]:
|
||
return sorted({int(value) for value in values.values()})
|
||
|
||
launchpad_alignment = {
|
||
"consistent": True,
|
||
"target_node_code_consistent": len(_nonempty_distinct(launchpad_target_node_codes)) <= 1,
|
||
"recovery_label_consistent": len(_nonempty_distinct(launchpad_recovery_labels)) <= 1,
|
||
"recovery_summary_consistent": len(_nonempty_distinct(launchpad_recovery_summaries)) <= 1,
|
||
"bootstrap_pending_consistent": len(_distinct_ints(launchpad_bootstrap_pending_nodes)) <= 1,
|
||
"acceptance_ready_consistent": len(_distinct_ints(launchpad_acceptance_ready_nodes)) <= 1,
|
||
"available_sources": sorted(launchpad_target_node_codes.keys()),
|
||
"target_node_codes": launchpad_target_node_codes,
|
||
"recovery_labels": launchpad_recovery_labels,
|
||
"recovery_summaries": launchpad_recovery_summaries,
|
||
"bootstrap_pending_nodes": launchpad_bootstrap_pending_nodes,
|
||
"acceptance_ready_nodes": launchpad_acceptance_ready_nodes,
|
||
}
|
||
launchpad_alignment["consistent"] = all(
|
||
bool(launchpad_alignment.get(key))
|
||
for key in (
|
||
"target_node_code_consistent",
|
||
"recovery_label_consistent",
|
||
"recovery_summary_consistent",
|
||
"bootstrap_pending_consistent",
|
||
"acceptance_ready_consistent",
|
||
)
|
||
)
|
||
|
||
blocked_reasons: list[str] = []
|
||
attention_reasons: list[str] = []
|
||
|
||
go_live_status = str(go_live_summary.get("go_live_status") or "").strip()
|
||
publish_status = str(go_live_summary.get("publish_status") or "").strip()
|
||
stack_status = str(stack_diagnosis.get("stack_status") or "").strip()
|
||
driver_launch_status = str(driver_summary.get("launch_status") or "").strip()
|
||
codex_launch_status = str(codex_summary.get("launch_status") or "").strip()
|
||
release_launchpad_status = str(launchpad_status.get("status") or "").strip()
|
||
review_status = str(go_live_review.get("status") or "").strip()
|
||
doctor_status = str(doctor_decision_payload.get("status") or "").strip()
|
||
|
||
if go_live_status == "blocked":
|
||
blocked_reasons.append("go_live_summary=blocked")
|
||
elif go_live_status == "attention":
|
||
attention_reasons.append("go_live_summary=attention")
|
||
if publish_status == "blocked":
|
||
blocked_reasons.append("publish_status=blocked")
|
||
elif publish_status == "attention":
|
||
attention_reasons.append("publish_status=attention")
|
||
if stack_status == "blocked":
|
||
blocked_reasons.append("stack_diagnosis=blocked")
|
||
elif stack_status == "attention":
|
||
attention_reasons.append("stack_diagnosis=attention")
|
||
if driver_launch_status == "blocked":
|
||
blocked_reasons.append("driver_feed_launch=blocked")
|
||
elif driver_launch_status == "attention":
|
||
attention_reasons.append("driver_feed_launch=attention")
|
||
if codex_launch_status == "blocked":
|
||
blocked_reasons.append("codex_brief_launch=blocked")
|
||
elif codex_launch_status == "attention":
|
||
attention_reasons.append("codex_brief_launch=attention")
|
||
if release_launchpad_status == "blocked":
|
||
blocked_reasons.append("release_launchpad=blocked")
|
||
elif release_launchpad_status == "attention":
|
||
attention_reasons.append("release_launchpad=attention")
|
||
if review_status == "blocked":
|
||
blocked_reasons.append("go_live_review=blocked")
|
||
elif review_status in {"attention", "missing"}:
|
||
attention_reasons.append(f"go_live_review={review_status}")
|
||
if doctor_status == "blocked":
|
||
blocked_reasons.append("doctor_decision=blocked")
|
||
elif doctor_status in {"attention", "missing"}:
|
||
attention_reasons.append(f"doctor_decision={doctor_status}")
|
||
if not bool(launchpad_alignment.get("consistent", False)):
|
||
attention_reasons.append("launchpad_alignment=inconsistent")
|
||
|
||
signoff_status = "ready"
|
||
if blocked_reasons:
|
||
signoff_status = "blocked"
|
||
elif attention_reasons:
|
||
signoff_status = "attention"
|
||
|
||
status_label = {
|
||
"ready": "可签字上线",
|
||
"attention": "待签字复核",
|
||
"blocked": "暂不可签字",
|
||
}.get(signoff_status, signoff_status or "未知")
|
||
if signoff_status == "blocked":
|
||
primary_blocked_surface = (
|
||
"正式复核"
|
||
if review_status == "blocked"
|
||
else "总检主决策"
|
||
if doctor_status == "blocked"
|
||
else "发布门禁"
|
||
)
|
||
headline = (
|
||
f"当前仍存在{primary_blocked_surface}阻断,建议优先处理 "
|
||
f"{str(go_live_summary.get('operator_title') or '').strip() or str(operator_decision.get('title') or '').strip() or str(doctor_decision.get('next_action_code') or '').strip() or '阻断项'}。"
|
||
)
|
||
elif signoff_status == "attention":
|
||
if review_status in {"attention", "missing"}:
|
||
headline = "当前已进入最终收口阶段,但正式复核仍待补口,建议先确认 bundle manifest 结论再签字上线。"
|
||
elif doctor_status in {"attention", "missing"}:
|
||
headline = "当前已进入最终收口阶段,但总检主决策仍待补口,建议先按主建议完成复核再签字上线。"
|
||
else:
|
||
headline = (
|
||
f"当前已进入最终收口阶段,但仍建议先完成 {str(go_live_summary.get('operator_title') or '').strip() or str(operator_decision.get('title') or '').strip() or 'attention 项'} 再签字上线。"
|
||
)
|
||
else:
|
||
headline = "当前收口、正式复核、主决策、发布门禁与 launchpad 摘要一致,可以进入最终人工签字或正式发布。"
|
||
|
||
contract_keys = _normalize_ops_contract_keys(
|
||
[
|
||
"ops_go_live_signoff_contract",
|
||
"ops_go_live_review_contract",
|
||
"ops_doctor_decision_contract",
|
||
"ops_stack_diagnosis_contract",
|
||
"ops_driver_contract",
|
||
"release_hub_contract",
|
||
"ops_observability_contract",
|
||
]
|
||
)
|
||
recommended_commands = {
|
||
**dict(go_live_summary.get("recommended_commands") or {}),
|
||
**dict(go_live_review.get("recommended_commands") or {}),
|
||
**dict(doctor_decision_payload.get("recommended_commands") or {}),
|
||
"release_launchpad": build_bash_command("drive_ops_center.sh", "release-launchpad", normalized_base_url),
|
||
"driver_feed": build_bash_command("drive_ops_center.sh", "driver-feed", normalized_base_url),
|
||
"codex_brief": build_bash_command("drive_ops_center.sh", "codex-brief", normalized_base_url),
|
||
"go_live_signoff": build_bash_command(
|
||
"drive_ops_center.sh",
|
||
"go-live-signoff",
|
||
normalized_report_dir or normalized_base_url,
|
||
),
|
||
}
|
||
|
||
return {
|
||
"generated_at": datetime.now().isoformat(timespec="seconds"),
|
||
"base_url": normalized_base_url,
|
||
"contract_key": "ops_go_live_signoff_contract",
|
||
"contract_version": str(
|
||
(contract_registry.get("contracts_by_key") or {}).get("ops_go_live_signoff_contract", {}).get("version")
|
||
or _OPS_CONTRACT_SCHEMA_VERSION
|
||
).strip(),
|
||
"signoff_status": signoff_status,
|
||
"status_label": status_label,
|
||
"headline": headline,
|
||
"release_gate": {
|
||
"go_live_status": go_live_status,
|
||
"publish_status": publish_status,
|
||
"stack_status": stack_status,
|
||
"driver_launch_status": driver_launch_status,
|
||
"codex_launch_status": codex_launch_status,
|
||
"release_launchpad_status": release_launchpad_status,
|
||
"go_live_review_status": review_status,
|
||
"doctor_decision_status": doctor_status,
|
||
},
|
||
"blocked_reasons": blocked_reasons,
|
||
"attention_reasons": attention_reasons,
|
||
"report_dir": str(go_live_review.get("report_dir") or doctor_decision_payload.get("report_dir") or "").strip(),
|
||
"manifest_path": str(go_live_review.get("manifest_path") or doctor_decision_payload.get("manifest_path") or "").strip(),
|
||
"decision": {
|
||
"operator_title": str(go_live_summary.get("operator_title") or "").strip(),
|
||
"next_step_action_code": str(go_live_summary.get("next_step_action_code") or "").strip(),
|
||
"next_step_reason": str(go_live_summary.get("next_step_reason") or "").strip(),
|
||
"preferred_surface": (
|
||
"go-live-review"
|
||
if review_status in {"blocked", "attention", "missing"}
|
||
else str(doctor_decision.get("preferred_surface") or "").strip()
|
||
if doctor_status in {"blocked", "attention", "missing"} and str(doctor_decision.get("preferred_surface") or "").strip()
|
||
else "release-launchpad"
|
||
if release_launchpad_status in {"blocked", "attention"}
|
||
else "codex-brief"
|
||
if codex_launch_status in {"blocked", "attention"}
|
||
else "go-live-summary"
|
||
),
|
||
"doctor_reason_code": str(doctor_decision.get("reason_code") or "").strip(),
|
||
"doctor_preferred_surface": str(doctor_decision.get("preferred_surface") or "").strip(),
|
||
"doctor_next_action_code": str(doctor_decision.get("next_action_code") or "").strip(),
|
||
"review_status": review_status,
|
||
"review_headline": str(go_live_review.get("headline") or "").strip(),
|
||
"review_report_dir": str(go_live_review.get("report_dir") or "").strip(),
|
||
"review_manifest_path": str(go_live_review.get("manifest_path") or "").strip(),
|
||
"stack_operator_title": str(operator_decision.get("title") or "").strip(),
|
||
"stack_next_action_code": str(next_step.get("action_code") or "").strip(),
|
||
"launchpad_recommended_target_node_code": str(
|
||
go_live_summary.get("launchpad_recommended_target_node_code") or ""
|
||
).strip(),
|
||
"launchpad_recommended_recovery_label": str(
|
||
go_live_summary.get("launchpad_recommended_recovery_label") or ""
|
||
).strip(),
|
||
"launchpad_recommended_recovery_summary": str(
|
||
go_live_summary.get("launchpad_recommended_recovery_summary") or ""
|
||
).strip(),
|
||
},
|
||
"launchpad_alignment": launchpad_alignment,
|
||
"recommended_commands": recommended_commands,
|
||
"contract_navigation": _build_ops_contract_navigation(
|
||
contract_keys,
|
||
primary_contract_key="ops_go_live_signoff_contract",
|
||
registry=contract_registry,
|
||
),
|
||
"go_live_summary": go_live_summary,
|
||
"go_live_review": {
|
||
"status": review_status,
|
||
"status_label": str(go_live_review.get("status_label") or "").strip(),
|
||
"headline": str(go_live_review.get("headline") or "").strip(),
|
||
"report_dir": str(go_live_review.get("report_dir") or "").strip(),
|
||
"manifest_path": str(go_live_review.get("manifest_path") or "").strip(),
|
||
"critical_failures": list(go_live_review.get("critical_failures") or []),
|
||
"noncritical_failures": list(go_live_review.get("noncritical_failures") or []),
|
||
},
|
||
"doctor_decision": {
|
||
"status": doctor_status,
|
||
"headline": str(doctor_decision_payload.get("headline") or "").strip(),
|
||
"detail": str(doctor_decision_payload.get("detail") or "").strip(),
|
||
"summary": dict(doctor_decision_payload.get("summary") or {}),
|
||
"decision": doctor_decision,
|
||
"report_dir": str(doctor_decision_payload.get("report_dir") or "").strip(),
|
||
"manifest_path": str(doctor_decision_payload.get("manifest_path") or "").strip(),
|
||
},
|
||
"stack_diagnosis": {
|
||
"stack_status": stack_status,
|
||
"operator_decision": operator_decision,
|
||
"next_step": next_step,
|
||
},
|
||
"driver_feed": {
|
||
"headline": str(driver_feed.get("headline") or "").strip(),
|
||
"summary": driver_summary,
|
||
},
|
||
"codex_brief": {
|
||
"headline": str(codex_brief.get("headline") or "").strip(),
|
||
"summary": codex_summary,
|
||
"focus": dict(codex_brief.get("focus") or {}),
|
||
},
|
||
"release_launchpad": {
|
||
"status": release_launchpad_status,
|
||
"status_label": str(launchpad_status.get("status_label") or "").strip(),
|
||
"recommended_action_code": str(launchpad_status.get("recommended_action_code") or "").strip(),
|
||
"recommended_execution_mode": str(launchpad_status.get("recommended_execution_mode") or "").strip(),
|
||
},
|
||
}
|
||
|
||
|
||
def _ops_go_live_bundle_root() -> Path:
|
||
return runtime_root() / "ops-center-reports" / "go-live-bundles"
|
||
|
||
|
||
def _load_json_path(path: Path) -> dict:
|
||
if not path.is_file():
|
||
return {}
|
||
try:
|
||
payload = json.loads(path.read_text(encoding="utf-8"))
|
||
except Exception:
|
||
return {}
|
||
return payload if isinstance(payload, dict) else {}
|
||
|
||
|
||
def _find_latest_go_live_bundle_manifest() -> tuple[Path | None, Path | None]:
|
||
bundle_root = _ops_go_live_bundle_root()
|
||
if not bundle_root.exists():
|
||
return None, bundle_root
|
||
manifest_paths = sorted(bundle_root.glob("*/manifest.json"), key=lambda item: item.stat().st_mtime, reverse=True)
|
||
if not manifest_paths:
|
||
return None, bundle_root
|
||
manifest_path = manifest_paths[0]
|
||
return manifest_path, manifest_path.parent
|
||
|
||
|
||
def _resolve_go_live_bundle_manifest(report_dir: str) -> tuple[Path | None, Path | None]:
|
||
normalized_report_dir = str(report_dir or "").strip()
|
||
if normalized_report_dir:
|
||
candidate_path = Path(normalized_report_dir)
|
||
if candidate_path.is_dir():
|
||
nested_manifest = candidate_path / "manifest.json"
|
||
return (nested_manifest if nested_manifest.is_file() else None), candidate_path
|
||
if candidate_path.is_file():
|
||
return candidate_path, candidate_path.parent
|
||
return None, candidate_path
|
||
return _find_latest_go_live_bundle_manifest()
|
||
|
||
|
||
def _load_bundle_json_artifact(artifact_by_key: dict[str, dict], key: str) -> dict:
|
||
artifact = artifact_by_key.get(str(key or "").strip()) or {}
|
||
artifact_path = Path(str(artifact.get("path") or "").strip())
|
||
return _load_json_path(artifact_path) if artifact_path else {}
|
||
|
||
|
||
def _load_bundle_env_audit_summary(artifact_by_key: dict[str, dict]) -> dict:
|
||
env_audit_artifact = artifact_by_key.get("env_audit") or {}
|
||
env_audit_path = Path(str(env_audit_artifact.get("path") or "").strip())
|
||
if not env_audit_path.is_file():
|
||
return {}
|
||
raw_text = env_audit_path.read_text(encoding="utf-8", errors="replace")
|
||
marker = "[8/8] condensed env audit summary"
|
||
candidate_text = raw_text.split(marker, 1)[1].strip() if marker in raw_text else raw_text.strip()
|
||
if not candidate_text:
|
||
return {}
|
||
try:
|
||
loaded_env_audit = json.loads(candidate_text)
|
||
except Exception:
|
||
loaded_env_audit = {}
|
||
return loaded_env_audit if isinstance(loaded_env_audit, dict) else {}
|
||
|
||
|
||
def get_ops_go_live_bundle(*, base_url: str = "", report_dir: str = "") -> dict:
|
||
normalized_base_url = str(base_url or "").strip() or "http://127.0.0.1:8100"
|
||
contract_registry = get_ops_contract_registry()
|
||
manifest_path, resolved_report_dir = _resolve_go_live_bundle_manifest(report_dir)
|
||
|
||
recommended_commands = {
|
||
"go_live_export": build_bash_command("drive_ops_center.sh", "go-live-export"),
|
||
"go_live_review": build_bash_command("drive_ops_center.sh", "go-live-review", str(report_dir or "").strip() or "/path/to/go-live-bundle"),
|
||
"go_live_signoff": build_bash_command("drive_ops_center.sh", "go-live-signoff", str(report_dir or "").strip() or "/path/to/go-live-bundle"),
|
||
"doctor_export": build_bash_command("drive_ops_center.sh", "doctor-export", "/tmp/domaincheck-go-live", normalized_base_url),
|
||
}
|
||
contract_keys = _normalize_ops_contract_keys(
|
||
[
|
||
"ops_go_live_bundle_contract",
|
||
"ops_go_live_signoff_contract",
|
||
"ops_stack_diagnosis_contract",
|
||
"ops_driver_contract",
|
||
]
|
||
)
|
||
|
||
if manifest_path is None or not manifest_path.is_file():
|
||
bundle_root = resolved_report_dir or _ops_go_live_bundle_root()
|
||
headline = (
|
||
"当前还没有可复核的 go-live bundle,建议先导出证据包,再进入 manifest 与最终签收复核。"
|
||
)
|
||
return {
|
||
"generated_at": datetime.now().isoformat(timespec="seconds"),
|
||
"base_url": normalized_base_url,
|
||
"contract_key": "ops_go_live_bundle_contract",
|
||
"contract_version": str(
|
||
(contract_registry.get("contracts_by_key") or {}).get("ops_go_live_bundle_contract", {}).get("version")
|
||
or _OPS_CONTRACT_SCHEMA_VERSION
|
||
).strip(),
|
||
"status": "missing",
|
||
"status_label": "待导出",
|
||
"headline": headline,
|
||
"bundle_available": False,
|
||
"report_dir": str(bundle_root),
|
||
"manifest_path": "",
|
||
"artifact_total": 0,
|
||
"failure_total": 0,
|
||
"critical_failures": [],
|
||
"noncritical_failures": [],
|
||
"env_audit": {},
|
||
"summary": {
|
||
"ok": False,
|
||
"review_status_hint": "missing",
|
||
"review_headline": headline,
|
||
"launchpad_alignment": {"consistent": False},
|
||
},
|
||
"recommended_commands": recommended_commands,
|
||
"contract_navigation": _build_ops_contract_navigation(
|
||
contract_keys,
|
||
primary_contract_key="ops_go_live_bundle_contract",
|
||
registry=contract_registry,
|
||
),
|
||
}
|
||
|
||
manifest = _load_json_path(manifest_path)
|
||
artifacts = list(manifest.get("artifacts") or [])
|
||
failures = [str(item or "").strip() for item in list(manifest.get("failures") or []) if str(item or "").strip()]
|
||
artifact_by_key = {
|
||
str(item.get("key") or "").strip(): dict(item)
|
||
for item in artifacts
|
||
if isinstance(item, dict) and str(item.get("key") or "").strip()
|
||
}
|
||
summary = dict(manifest.get("summary") or {})
|
||
env_audit = _load_bundle_env_audit_summary(artifact_by_key)
|
||
|
||
review_status_hint = str(summary.get("review_status_hint") or "").strip()
|
||
status = review_status_hint or ("ready" if bool(summary.get("ok", False)) else "attention")
|
||
status_label = {
|
||
"ready": "证据可交付",
|
||
"attention": "证据待复核",
|
||
"blocked": "证据阻断",
|
||
"missing": "待导出",
|
||
}.get(status, status or "未知")
|
||
headline = (
|
||
str(summary.get("review_headline") or "").strip()
|
||
or "当前已读取最新 go-live bundle,可继续做 manifest 与签收复核。"
|
||
)
|
||
|
||
critical_keys = {"go_live_summary", "stack_diagnosis", "driver_feed", "codex_brief", "release_launchpad"}
|
||
critical_failures = [item for item in failures if item in critical_keys]
|
||
noncritical_failures = [item for item in failures if item not in critical_keys]
|
||
launchpad_alignment = dict(summary.get("launchpad_alignment") or {})
|
||
|
||
return {
|
||
"generated_at": datetime.now().isoformat(timespec="seconds"),
|
||
"base_url": normalized_base_url,
|
||
"contract_key": "ops_go_live_bundle_contract",
|
||
"contract_version": str(
|
||
(contract_registry.get("contracts_by_key") or {}).get("ops_go_live_bundle_contract", {}).get("version")
|
||
or _OPS_CONTRACT_SCHEMA_VERSION
|
||
).strip(),
|
||
"status": status,
|
||
"status_label": status_label,
|
||
"headline": headline,
|
||
"bundle_available": True,
|
||
"report_dir": str(manifest.get("report_dir") or resolved_report_dir or manifest_path.parent),
|
||
"manifest_path": str(manifest_path),
|
||
"artifact_total": int(summary.get("artifact_total", len(artifacts)) or len(artifacts)),
|
||
"failure_total": int(summary.get("failure_total", len(failures)) or len(failures)),
|
||
"critical_failures": critical_failures,
|
||
"noncritical_failures": noncritical_failures,
|
||
"env_audit": {
|
||
"status": str(env_audit.get("status") or "").strip(),
|
||
"headline": str(env_audit.get("headline") or "").strip(),
|
||
"missing_items": list(env_audit.get("missing_items") or []),
|
||
"runtime_may_need_restart": bool(((env_audit.get("runtime") or {}).get("runtime_may_need_restart", False))),
|
||
"recommended_actions": [
|
||
str(item).strip()
|
||
for item in list(env_audit.get("recommended_actions") or [])
|
||
if str(item).strip()
|
||
],
|
||
},
|
||
"summary": {
|
||
"ok": bool(summary.get("ok", False)),
|
||
"review_status_hint": review_status_hint or status,
|
||
"review_headline": headline,
|
||
"launchpad_recommended_target_node_code": str(
|
||
summary.get("launchpad_recommended_target_node_code") or ""
|
||
).strip(),
|
||
"launchpad_recommended_recovery_label": str(
|
||
summary.get("launchpad_recommended_recovery_label") or ""
|
||
).strip(),
|
||
"launchpad_recommended_recovery_summary": str(
|
||
summary.get("launchpad_recommended_recovery_summary") or ""
|
||
).strip(),
|
||
"launchpad_onboarding_bootstrap_pending_nodes": int(
|
||
summary.get("launchpad_onboarding_bootstrap_pending_nodes", 0) or 0
|
||
),
|
||
"launchpad_onboarding_acceptance_ready_nodes": int(
|
||
summary.get("launchpad_onboarding_acceptance_ready_nodes", 0) or 0
|
||
),
|
||
"launchpad_alignment": launchpad_alignment,
|
||
"recommended_reading_order": list(summary.get("recommended_reading_order") or []),
|
||
"contract_surface_gaps": list(summary.get("contract_surface_gaps") or []),
|
||
"discovered_contract_keys": list(summary.get("discovered_contract_keys") or []),
|
||
},
|
||
"artifacts": artifacts,
|
||
"failures": failures,
|
||
"recommended_commands": {
|
||
**recommended_commands,
|
||
"go_live_review": build_bash_command("drive_ops_center.sh", "go-live-review", str(manifest_path)),
|
||
"go_live_signoff": build_bash_command("drive_ops_center.sh", "go-live-signoff", str(manifest_path)),
|
||
},
|
||
"contract_navigation": _build_ops_contract_navigation(
|
||
contract_keys,
|
||
primary_contract_key="ops_go_live_bundle_contract",
|
||
registry=contract_registry,
|
||
),
|
||
}
|
||
|
||
|
||
def get_ops_go_live_review(*, base_url: str = "", report_dir: str = "") -> dict:
|
||
normalized_base_url = str(base_url or "").strip() or "http://127.0.0.1:8100"
|
||
normalized_report_dir = str(report_dir or "").strip()
|
||
contract_registry = get_ops_contract_registry()
|
||
manifest_path, resolved_report_dir = _resolve_go_live_bundle_manifest(report_dir)
|
||
contract_keys = _normalize_ops_contract_keys(
|
||
[
|
||
"ops_go_live_review_contract",
|
||
"ops_go_live_bundle_contract",
|
||
"ops_doctor_decision_contract",
|
||
"ops_go_live_signoff_contract",
|
||
"ops_stack_diagnosis_contract",
|
||
"ops_driver_contract",
|
||
"release_hub_contract",
|
||
]
|
||
)
|
||
recommended_commands = {
|
||
"go_live_export": build_bash_command("drive_ops_center.sh", "go-live-export"),
|
||
"go_live_review": build_bash_command("drive_ops_center.sh", "go-live-review", normalized_report_dir or "/path/to/go-live-bundle"),
|
||
"go_live_signoff": build_bash_command("drive_ops_center.sh", "go-live-signoff", normalized_report_dir or "/path/to/go-live-bundle"),
|
||
"doctor_decision": build_bash_command("drive_ops_center.sh", "doctor-decision", normalized_report_dir or normalized_base_url),
|
||
}
|
||
|
||
if manifest_path is None or not manifest_path.is_file():
|
||
headline = "当前还没有可复核的 go-live bundle,建议先导出证据包,再进入正式复核。"
|
||
return {
|
||
"generated_at": datetime.now().isoformat(timespec="seconds"),
|
||
"base_url": normalized_base_url,
|
||
"contract_key": "ops_go_live_review_contract",
|
||
"contract_version": str(
|
||
(contract_registry.get("contracts_by_key") or {}).get("ops_go_live_review_contract", {}).get("version")
|
||
or _OPS_CONTRACT_SCHEMA_VERSION
|
||
).strip(),
|
||
"status": "missing",
|
||
"status_label": "待导出",
|
||
"headline": headline,
|
||
"report_dir": str(resolved_report_dir or _ops_go_live_bundle_root()),
|
||
"manifest_path": "",
|
||
"artifact_total": 0,
|
||
"failure_total": 0,
|
||
"critical_failures": [],
|
||
"noncritical_failures": [],
|
||
"env_audit": {},
|
||
"go_live_summary": {},
|
||
"stack_diagnosis": {},
|
||
"driver_feed": {},
|
||
"codex_brief": {},
|
||
"launchpad_alignment": {"consistent": False, "available_sources": []},
|
||
"recommended_reading_order": [],
|
||
"recommended_next_steps": [
|
||
"先执行 go-live-export,导出完整上线证据包。",
|
||
"导出完成后再执行 go-live-review 或直接打开页面里的证据复核详情。",
|
||
],
|
||
"recommended_commands": recommended_commands,
|
||
"contract_navigation": _build_ops_contract_navigation(
|
||
contract_keys,
|
||
primary_contract_key="ops_go_live_review_contract",
|
||
registry=contract_registry,
|
||
),
|
||
}
|
||
|
||
manifest = _load_json_path(manifest_path)
|
||
artifacts = list(manifest.get("artifacts") or [])
|
||
failures = [str(item or "").strip() for item in list(manifest.get("failures") or []) if str(item or "").strip()]
|
||
summary = dict(manifest.get("summary") or {})
|
||
artifact_by_key = {
|
||
str(item.get("key") or "").strip(): dict(item)
|
||
for item in artifacts
|
||
if isinstance(item, dict) and str(item.get("key") or "").strip()
|
||
}
|
||
env_audit = _load_bundle_env_audit_summary(artifact_by_key)
|
||
go_live_summary_payload = _load_bundle_json_artifact(artifact_by_key, "go_live_summary")
|
||
stack_diagnosis_payload = _load_bundle_json_artifact(artifact_by_key, "stack_diagnosis")
|
||
driver_feed_payload = _load_bundle_json_artifact(artifact_by_key, "driver_feed")
|
||
codex_brief_payload = _load_bundle_json_artifact(artifact_by_key, "codex_brief")
|
||
|
||
critical_keys = {"go_live_summary", "stack_diagnosis", "driver_feed", "codex_brief", "release_launchpad"}
|
||
critical_failures = sorted(
|
||
{
|
||
key
|
||
for key in critical_keys
|
||
if key in failures or not bool((artifact_by_key.get(key) or {}).get("ok", False))
|
||
}
|
||
)
|
||
noncritical_failures = [key for key in failures if key not in critical_keys]
|
||
|
||
go_live_launchpad_target_node_code = str(go_live_summary_payload.get("launchpad_recommended_target_node_code") or "").strip()
|
||
go_live_launchpad_recovery_label = str(go_live_summary_payload.get("launchpad_recommended_recovery_label") or "").strip()
|
||
go_live_launchpad_recovery_summary = str(go_live_summary_payload.get("launchpad_recommended_recovery_summary") or "").strip()
|
||
go_live_launchpad_bootstrap_pending_nodes = int(go_live_summary_payload.get("launchpad_onboarding_bootstrap_pending_nodes", 0) or 0)
|
||
go_live_launchpad_acceptance_ready_nodes = int(go_live_summary_payload.get("launchpad_onboarding_acceptance_ready_nodes", 0) or 0)
|
||
stack_diagnosis = dict(stack_diagnosis_payload.get("diagnosis") or {})
|
||
driver_feed_summary = dict(driver_feed_payload.get("summary") or {})
|
||
codex_brief_summary = dict(codex_brief_payload.get("summary") or {})
|
||
|
||
launchpad_target_node_codes = {
|
||
"go_live_summary": go_live_launchpad_target_node_code,
|
||
"stack_diagnosis": str(stack_diagnosis.get("launchpad_recommended_target_node_code") or "").strip(),
|
||
"driver_feed": str(driver_feed_summary.get("launchpad_recommended_target_node_code") or "").strip(),
|
||
"codex_brief": str(codex_brief_summary.get("launchpad_recommended_target_node_code") or "").strip(),
|
||
}
|
||
launchpad_recovery_labels = {
|
||
"go_live_summary": go_live_launchpad_recovery_label,
|
||
"stack_diagnosis": str(stack_diagnosis.get("launchpad_recommended_recovery_label") or "").strip(),
|
||
"driver_feed": str(driver_feed_summary.get("launchpad_recommended_recovery_label") or "").strip(),
|
||
"codex_brief": str(codex_brief_summary.get("launchpad_recommended_recovery_label") or "").strip(),
|
||
}
|
||
launchpad_recovery_summaries = {
|
||
"go_live_summary": go_live_launchpad_recovery_summary,
|
||
"stack_diagnosis": str(stack_diagnosis.get("launchpad_recommended_recovery_summary") or "").strip(),
|
||
"driver_feed": str(driver_feed_summary.get("launchpad_recommended_recovery_summary") or "").strip(),
|
||
"codex_brief": str(codex_brief_summary.get("launchpad_recommended_recovery_summary") or "").strip(),
|
||
}
|
||
launchpad_bootstrap_pending_nodes = {
|
||
"go_live_summary": go_live_launchpad_bootstrap_pending_nodes,
|
||
"stack_diagnosis": int(stack_diagnosis.get("launchpad_onboarding_bootstrap_pending_nodes", 0) or 0),
|
||
"driver_feed": int(driver_feed_summary.get("launchpad_onboarding_bootstrap_pending_nodes", 0) or 0),
|
||
"codex_brief": int(codex_brief_summary.get("launchpad_onboarding_bootstrap_pending_nodes", 0) or 0),
|
||
}
|
||
launchpad_acceptance_ready_nodes = {
|
||
"go_live_summary": go_live_launchpad_acceptance_ready_nodes,
|
||
"stack_diagnosis": int(stack_diagnosis.get("launchpad_onboarding_acceptance_ready_nodes", 0) or 0),
|
||
"driver_feed": int(driver_feed_summary.get("launchpad_onboarding_acceptance_ready_nodes", 0) or 0),
|
||
"codex_brief": int(codex_brief_summary.get("launchpad_onboarding_acceptance_ready_nodes", 0) or 0),
|
||
}
|
||
|
||
available_alignment_sources = {
|
||
source
|
||
for source, payload_value in {
|
||
"go_live_summary": go_live_summary_payload,
|
||
"stack_diagnosis": stack_diagnosis_payload,
|
||
"driver_feed": driver_feed_payload,
|
||
"codex_brief": codex_brief_payload,
|
||
}.items()
|
||
if isinstance(payload_value, dict) and payload_value
|
||
}
|
||
|
||
def _nonempty_distinct(values: dict[str, str]) -> list[str]:
|
||
return sorted({str(value or "").strip() for key, value in values.items() if key in available_alignment_sources and str(value or "").strip()})
|
||
|
||
def _distinct_ints(values: dict[str, int]) -> list[int]:
|
||
return sorted({int(value) for key, value in values.items() if key in available_alignment_sources})
|
||
|
||
launchpad_alignment = {
|
||
"consistent": True,
|
||
"target_node_code_consistent": len(_nonempty_distinct(launchpad_target_node_codes)) <= 1,
|
||
"recovery_label_consistent": len(_nonempty_distinct(launchpad_recovery_labels)) <= 1,
|
||
"recovery_summary_consistent": len(_nonempty_distinct(launchpad_recovery_summaries)) <= 1,
|
||
"bootstrap_pending_consistent": len(_distinct_ints(launchpad_bootstrap_pending_nodes)) <= 1,
|
||
"acceptance_ready_consistent": len(_distinct_ints(launchpad_acceptance_ready_nodes)) <= 1,
|
||
"available_sources": sorted(available_alignment_sources),
|
||
"target_node_codes": launchpad_target_node_codes,
|
||
"recovery_labels": launchpad_recovery_labels,
|
||
"recovery_summaries": launchpad_recovery_summaries,
|
||
"bootstrap_pending_nodes": launchpad_bootstrap_pending_nodes,
|
||
"acceptance_ready_nodes": launchpad_acceptance_ready_nodes,
|
||
}
|
||
launchpad_alignment["consistent"] = all(
|
||
bool(launchpad_alignment.get(key))
|
||
for key in (
|
||
"target_node_code_consistent",
|
||
"recovery_label_consistent",
|
||
"recovery_summary_consistent",
|
||
"bootstrap_pending_consistent",
|
||
"acceptance_ready_consistent",
|
||
)
|
||
)
|
||
|
||
env_audit_status = str(env_audit.get("status") or "").strip()
|
||
env_audit_blocking = env_audit_status == "blocked"
|
||
env_audit_attention = env_audit_status == "attention"
|
||
env_audit_missing_items = [
|
||
str(item).strip()
|
||
for item in list(env_audit.get("missing_items") or [])
|
||
if str(item).strip()
|
||
]
|
||
env_audit_recommended_actions = [
|
||
str(item).strip()
|
||
for item in list(env_audit.get("recommended_actions") or [])
|
||
if str(item).strip()
|
||
]
|
||
env_audit_runtime_may_need_restart = bool(((env_audit.get("runtime") or {}).get("runtime_may_need_restart", False)))
|
||
node_agent_env_missing = any(item == "missing_env:/etc/default/domaincheck-node-agent" for item in env_audit_missing_items)
|
||
|
||
if critical_failures or env_audit_blocking:
|
||
status = "blocked"
|
||
if critical_failures and env_audit_blocking:
|
||
headline = "关键报告存在缺失,且环境审计已给出阻断结论,当前不适合上线。"
|
||
elif critical_failures:
|
||
headline = "上线证据包仍缺关键报告,暂不建议直接宣称收口完成。"
|
||
else:
|
||
headline = "环境审计已给出阻断结论,需先修复现场环境后再进入发布门禁。"
|
||
elif noncritical_failures or env_audit_attention or not bool(launchpad_alignment.get("consistent")):
|
||
status = "attention"
|
||
if not bool(launchpad_alignment.get("consistent")):
|
||
headline = "核心报告已齐,但 launchpad 摘要在不同 surface 之间存在漂移,建议先复核再发版。"
|
||
elif noncritical_failures and env_audit_attention:
|
||
headline = "核心报告已齐,但补充报告和环境审计都提示仍有缺口,建议先复核再发版。"
|
||
elif noncritical_failures:
|
||
headline = "核心报告已齐,但仍有补充报告超时或失败,建议先复核再发版。"
|
||
else:
|
||
headline = "核心报告已齐,但环境审计仍提示基础缺口,建议补齐后再发版。"
|
||
else:
|
||
status = "ready"
|
||
headline = "核心上线证据已经齐备,可进入最终人工复核或正式发布门禁。"
|
||
|
||
recommended_next_steps = [
|
||
"先看 00_env_audit.txt,确认当前机器环境是否存在阻断或注意项",
|
||
"先看 02_go_live_summary.json 和 03_stack_diagnosis.json",
|
||
"再看 04_driver_feed.json 和 05_codex_brief.json",
|
||
"若涉及发布门禁,再看 06_release_launchpad.json",
|
||
"若 manifest 里仍有失败项,再看对应 artifact 原文件",
|
||
]
|
||
if node_agent_env_missing:
|
||
recommended_next_steps.extend(
|
||
[
|
||
"当前缺口指向 Node Agent 未接管:先执行 agent-gap-export,拿到首个缺口节点的接管包",
|
||
"随后执行 node-bootstrap-plan,为缺口节点生成可直接落地的接管脚本",
|
||
"如果仓库代码已更新但 bootstrap-plan 输出仍偏旧,先重启控制面 API 再重新导出接管计划",
|
||
]
|
||
)
|
||
if env_audit_runtime_may_need_restart:
|
||
recommended_next_steps.extend(
|
||
[
|
||
"env-audit 已识别运行中 API 可能仍是旧代码:先执行 drive_ops_center.sh runtime-refresh-recover,按固定链路完成 API 重启与复检",
|
||
"若仍需人工逐条确认,再执行 recommended_actions 里的 API 重启与复检命令",
|
||
]
|
||
)
|
||
if go_live_launchpad_target_node_code:
|
||
recommended_next_steps.extend(
|
||
[
|
||
f"当前 launchpad 缺口已经收敛到节点 {go_live_launchpad_target_node_code},建议优先执行:{go_live_launchpad_recovery_label or '节点恢复'}",
|
||
"先对照 02_go_live_summary.json、04_driver_feed.json、05_codex_brief.json 中的 launchpad 字段,确认三处摘要是否一致",
|
||
go_live_launchpad_recovery_summary or "如果 recovery_summary 仍为空,再回到 06_release_launchpad.json 查看 gap rows 详情",
|
||
]
|
||
)
|
||
if not bool(launchpad_alignment.get("consistent")):
|
||
recommended_next_steps.extend(
|
||
[
|
||
"当前 bundle 内的 launchpad 摘要在 go_live_summary / stack_diagnosis / driver_feed / codex_brief 之间存在不一致,先不要直接交付。",
|
||
"优先复核 02_go_live_summary.json 与 03_stack_diagnosis.json,再检查 04_driver_feed.json 与 05_codex_brief.json 是否使用了同一版控制面数据。",
|
||
"若只有局部摘要偏旧,先执行 drive_ops_center.sh runtime-refresh-recover,并重新导出 go-live bundle。",
|
||
]
|
||
)
|
||
|
||
return {
|
||
"generated_at": datetime.now().isoformat(timespec="seconds"),
|
||
"base_url": normalized_base_url,
|
||
"contract_key": "ops_go_live_review_contract",
|
||
"contract_version": str(
|
||
(contract_registry.get("contracts_by_key") or {}).get("ops_go_live_review_contract", {}).get("version")
|
||
or _OPS_CONTRACT_SCHEMA_VERSION
|
||
).strip(),
|
||
"status": status,
|
||
"status_label": {
|
||
"ready": "可复核发布",
|
||
"attention": "待复核补口",
|
||
"blocked": "暂不可发布",
|
||
"missing": "待导出",
|
||
}.get(status, status or "未知"),
|
||
"headline": headline,
|
||
"report_dir": str(manifest.get("report_dir") or resolved_report_dir or manifest_path.parent),
|
||
"manifest_path": str(manifest_path),
|
||
"artifact_total": int(summary.get("artifact_total", len(artifacts)) or len(artifacts)),
|
||
"failure_total": int(summary.get("failure_total", len(failures)) or len(failures)),
|
||
"critical_failures": critical_failures,
|
||
"noncritical_failures": noncritical_failures,
|
||
"env_audit": {
|
||
"ok": bool((artifact_by_key.get("env_audit") or {}).get("ok", False)),
|
||
"status": env_audit_status,
|
||
"headline": str(env_audit.get("headline") or "").strip(),
|
||
"missing_items": env_audit_missing_items,
|
||
"runtime_may_need_restart": env_audit_runtime_may_need_restart,
|
||
"recommended_actions": env_audit_recommended_actions,
|
||
},
|
||
"go_live_summary": {
|
||
"go_live_status": str(go_live_summary_payload.get("go_live_status") or "").strip(),
|
||
"publish_status": str(go_live_summary_payload.get("publish_status") or "").strip(),
|
||
"operator_title": str(go_live_summary_payload.get("operator_title") or "").strip(),
|
||
"next_step_action_code": str(go_live_summary_payload.get("next_step_action_code") or "").strip(),
|
||
"launchpad_recommended_target_node_code": go_live_launchpad_target_node_code,
|
||
"launchpad_recommended_recovery_label": go_live_launchpad_recovery_label,
|
||
"launchpad_recommended_recovery_summary": go_live_launchpad_recovery_summary,
|
||
"launchpad_onboarding_bootstrap_pending_nodes": go_live_launchpad_bootstrap_pending_nodes,
|
||
"launchpad_onboarding_acceptance_ready_nodes": go_live_launchpad_acceptance_ready_nodes,
|
||
},
|
||
"stack_diagnosis": {
|
||
"stack_status": str(stack_diagnosis.get("stack_status") or "").strip(),
|
||
"operator_title": str((dict(stack_diagnosis.get("operator_decision") or {})).get("title") or "").strip(),
|
||
"next_step_action_code": str((dict(stack_diagnosis.get("next_step") or {})).get("action_code") or "").strip(),
|
||
"launchpad_recommended_target_node_code": str(stack_diagnosis.get("launchpad_recommended_target_node_code") or "").strip(),
|
||
"launchpad_recommended_recovery_label": str(stack_diagnosis.get("launchpad_recommended_recovery_label") or "").strip(),
|
||
"launchpad_recommended_recovery_summary": str(stack_diagnosis.get("launchpad_recommended_recovery_summary") or "").strip(),
|
||
"launchpad_onboarding_bootstrap_pending_nodes": int(stack_diagnosis.get("launchpad_onboarding_bootstrap_pending_nodes", 0) or 0),
|
||
"launchpad_onboarding_acceptance_ready_nodes": int(stack_diagnosis.get("launchpad_onboarding_acceptance_ready_nodes", 0) or 0),
|
||
},
|
||
"driver_feed": {
|
||
"headline": str(driver_feed_payload.get("headline") or "").strip(),
|
||
"launch_status": str((dict(driver_feed_payload.get("automation_coverage") or {})).get("launch_status") or "").strip(),
|
||
"launch_ready": bool((dict(driver_feed_payload.get("automation_coverage") or {})).get("launch_ready", False)),
|
||
"launchpad_recommended_target_node_code": str(driver_feed_summary.get("launchpad_recommended_target_node_code") or "").strip(),
|
||
"launchpad_recommended_recovery_label": str(driver_feed_summary.get("launchpad_recommended_recovery_label") or "").strip(),
|
||
},
|
||
"codex_brief": {
|
||
"headline": str(codex_brief_payload.get("headline") or "").strip(),
|
||
"focus_entry_key": str((dict(codex_brief_payload.get("focus") or {})).get("entry_key") or "").strip(),
|
||
"launch_status": str((dict(codex_brief_payload.get("automation_coverage") or {})).get("launch_status") or "").strip(),
|
||
"launch_ready": bool((dict(codex_brief_payload.get("automation_coverage") or {})).get("launch_ready", False)),
|
||
"launchpad_recommended_target_node_code": str(codex_brief_summary.get("launchpad_recommended_target_node_code") or "").strip(),
|
||
"launchpad_recommended_recovery_label": str(codex_brief_summary.get("launchpad_recommended_recovery_label") or "").strip(),
|
||
},
|
||
"launchpad_alignment": launchpad_alignment,
|
||
"recommended_reading_order": list(summary.get("recommended_reading_order") or []),
|
||
"recommended_next_steps": _dedupe_command_list(recommended_next_steps),
|
||
"recommended_commands": {
|
||
**recommended_commands,
|
||
"go_live_review": build_bash_command("drive_ops_center.sh", "go-live-review", str(manifest_path)),
|
||
"go_live_signoff": build_bash_command("drive_ops_center.sh", "go-live-signoff", str(manifest_path)),
|
||
"doctor_decision": build_bash_command("drive_ops_center.sh", "doctor-decision", str(manifest_path)),
|
||
},
|
||
"contract_navigation": _build_ops_contract_navigation(
|
||
contract_keys,
|
||
primary_contract_key="ops_go_live_review_contract",
|
||
registry=contract_registry,
|
||
),
|
||
}
|
||
|
||
|
||
def get_ops_blueprint() -> dict:
|
||
return {
|
||
"contract_registry": get_ops_contract_registry(),
|
||
"architecture": {
|
||
"control_plane": {
|
||
"region": "overseas",
|
||
"responsibility": [
|
||
"web backend",
|
||
"ops api",
|
||
"release registry",
|
||
"task scheduler",
|
||
"log aggregation entry",
|
||
],
|
||
},
|
||
"managed_nodes": {
|
||
"region": "mainland",
|
||
"responsibility": [
|
||
"worker/controller runtime",
|
||
"node agent",
|
||
"systemd execution",
|
||
"diagnostics collection",
|
||
],
|
||
},
|
||
},
|
||
"execution_model": {
|
||
"default_transport": "https-polling",
|
||
"future_transport": "websocket",
|
||
"fallback_transport": "ssh",
|
||
"job_result_contract": {
|
||
"required_fields": [
|
||
"job_id",
|
||
"node_code",
|
||
"step",
|
||
"started_at",
|
||
"finished_at",
|
||
"exit_code",
|
||
"stdout",
|
||
"stderr",
|
||
"structured_result",
|
||
]
|
||
},
|
||
},
|
||
"mvp_scope": {
|
||
"phase_1": [
|
||
"ops overview api",
|
||
"action catalog api",
|
||
"worker participation / standby visibility",
|
||
"remote log sync switch",
|
||
],
|
||
"phase_2": [
|
||
"ops jobs table",
|
||
"node registration token",
|
||
"node heartbeat and pull-task api",
|
||
"service restart / health check actions",
|
||
],
|
||
"phase_3": [
|
||
"release package distribution",
|
||
"one-click bootstrap",
|
||
"rollback",
|
||
"streaming logs",
|
||
],
|
||
},
|
||
"operator_model": {
|
||
"with_codex": "海外 Codex 作为智能驾驶员,自动分析现网状态并选择合适动作。",
|
||
"without_codex": "后台按钮直接创建 ops job,由控制面和 agent 自动完成执行与回执。",
|
||
},
|
||
}
|
||
|
||
|
||
def get_ops_runbook(
|
||
*,
|
||
runtime_status: dict | None = None,
|
||
managed_nodes_payload: dict | None = None,
|
||
release_launchpad: dict | None = None,
|
||
) -> dict:
|
||
runtime = dict(runtime_status or {})
|
||
if not runtime:
|
||
runtime = get_runtime_status()
|
||
readiness = runtime.get("readiness") or {}
|
||
worker_runtime = runtime.get("worker") or {}
|
||
sync_agent_runtime = runtime.get("sync_agent") or {}
|
||
resolved_managed_nodes_payload = dict(managed_nodes_payload or {})
|
||
if not resolved_managed_nodes_payload:
|
||
resolved_managed_nodes_payload = list_managed_nodes_with_agent_state(
|
||
participation_payload=runtime.get("detect") or {}
|
||
)
|
||
resolved_release_launchpad = dict(release_launchpad or {})
|
||
if not resolved_release_launchpad:
|
||
resolved_release_launchpad = get_release_launchpad()
|
||
control_sequences = _attach_ops_runbook_sequence_resolutions(
|
||
_build_ops_runbook_control_sequences(
|
||
managed_nodes_payload=resolved_managed_nodes_payload,
|
||
release_launchpad=resolved_release_launchpad,
|
||
),
|
||
requested_by="api/runbook",
|
||
)
|
||
|
||
return {
|
||
"entrypoint": {
|
||
"preferred_host": str(settings.sync_target_api_base_url or "").strip() or f"http://127.0.0.1:{settings.api_port}",
|
||
"api_prefix": settings.api_prefix,
|
||
},
|
||
"services": {
|
||
"api": {
|
||
"service_name": settings.api_service_name,
|
||
"status": "running",
|
||
},
|
||
"worker": {
|
||
"service_name": settings.worker_service_name,
|
||
"status": _bool_label(bool(worker_runtime.get("running", False))),
|
||
},
|
||
"sync_agent": {
|
||
"service_name": settings.sync_agent_service_name,
|
||
"status": _bool_label(bool(sync_agent_runtime.get("running", False))),
|
||
},
|
||
},
|
||
"fast_checks": [
|
||
"GET /health",
|
||
"GET /api/v1/runtime/status",
|
||
"GET /api/v1/runtime/cluster",
|
||
"GET /api/v1/runtime/readiness",
|
||
"GET /api/v1/ops/stack-diagnosis",
|
||
"GET /api/v1/ops/overview",
|
||
"GET /api/v1/ops/contracts",
|
||
"GET /api/v1/ops/contracts/ops_stack_diagnosis_contract",
|
||
"GET /api/v1/ops/releases/launchpad",
|
||
"GET /api/v1/ops/activity-stream",
|
||
"GET /api/v1/ops/capabilities",
|
||
],
|
||
"current_readiness": {
|
||
"status": str(readiness.get("status") or ""),
|
||
"summary": str(readiness.get("summary") or ""),
|
||
},
|
||
"release_launchpad": resolved_release_launchpad,
|
||
"control_sequences": control_sequences,
|
||
}
|