From b9c29481b547a8001d67464f8bdacf35c2f1902b Mon Sep 17 00:00:00 2001 From: Your Name Date: Sat, 18 Apr 2026 23:52:51 +0800 Subject: [PATCH] feat: add ops center and node onboarding flow --- HANDOFF_20260418_2321.md | 98 + HANDOFF_20260418_2344.md | 155 + IMPLEMENTATION_STATUS.md | 135 + README.md | 97 +- TASK_BOARD.md | 148 + docs/08_domainCheck_交付说明.md | 7 + docs/09_domainCheck_交付打包说明.md | 50 +- docs/10_domainCheck_最终交付结论.md | 8 +- docs/12_domainCheck_导航索引.md | 65 + docs/14_domainCheck_正式上线前最终检查单.md | 34 + docs/20_domainCheck_临时海外控制面联调清单.md | 33 +- ...omainCheck_海外主机集中运维与自动化方案.md | 708 + ...heck_海外Codex驾驶员与OpsCenter落地路线.md | 1052 + ...inCheck_终局运维架构设计_海外单脑控制面.md | 1163 + ...mainCheck_NodeAgent协议与ReleaseHub设计.md | 4421 ++++ ..._domainCheck_海外单脑控制面上线收口总表.md | 430 + ...26_domainCheck_发布前运行验证与交付模板.md | 635 + docs/schemas/ops_agent_protocol.md | 816 + docs/schemas/ops_doctor_decision_contract.md | 56 + docs/schemas/ops_driver_contract.md | 797 + docs/schemas/ops_go_live_bundle_contract.md | 59 + docs/schemas/ops_go_live_review_contract.md | 37 + docs/schemas/ops_go_live_signoff_contract.md | 65 + docs/schemas/ops_job_contract.md | 740 + docs/schemas/ops_observability_contract.md | 702 + docs/schemas/ops_playbook_contract.md | 563 + docs/schemas/ops_stack_diagnosis_contract.md | 372 + docs/schemas/release_hub_contract.md | 609 + domain-api/app/api/routes/ops.py | 633 + domain-api/app/api/routes/ops_agent.py | 90 + domain-api/app/api/routes/ops_release.py | 175 + domain-api/app/api/routes/runtime.py | 6 + domain-api/app/core/config.py | 7 + domain-api/app/main.py | 15 +- domain-api/app/node_agent.py | 991 + domain-api/app/schemas/common.py | 4 +- domain-api/app/services/build_info_service.py | 241 + domain-api/app/services/detect_service.py | 104 +- .../app/services/ops_action_executor_core.py | 278 + domain-api/app/services/ops_agent_service.py | 2852 +++ .../app/services/ops_command_service.py | 17 + .../ops_execution_capability_service.py | 79 + .../services/ops_execution_mode_service.py | 60 + domain-api/app/services/ops_job_service.py | 1682 ++ .../app/services/ops_playbook_service.py | 1263 ++ domain-api/app/services/ops_policy_service.py | 494 + .../app/services/ops_release_executor_core.py | 475 + .../app/services/ops_release_service.py | 3448 +++ .../services/ops_runtime_executor_service.py | 420 + domain-api/app/services/ops_service.py | 11611 ++++++++++ .../app/services/ops_template_service.py | 920 + .../app/services/runtime_status_service.py | 304 +- .../app/services/sync_record_service.py | 11 + domain-api/deploy/multi-region/README.md | 1692 +- .../deploy/multi-region/bootstrap_mainland.sh | 2 +- .../multi-region/bootstrap_node_agent.sh | 61 + .../build_node_agent_bootstrap_plan.sh | 91 + .../deploy/multi-region/check_env_drift.sh | 345 + .../deploy/multi-region/check_go_live.sh | 423 + .../multi-region/check_go_live_signoff.sh | 182 + .../deploy/multi-region/check_node_agent.sh | 38 + .../multi-region/check_node_scene_log.sh | 65 + .../multi-region/check_ops_center_stack.sh | 1272 ++ .../multi-region/check_ops_contracts.sh | 55 + .../multi-region/check_ops_inspection.sh | 72 + .../multi-region/check_ops_link_snapshot.sh | 35 + .../deploy/multi-region/check_ops_plane.sh | 609 + .../deploy/multi-region/check_release_hub.sh | 373 + .../multi-region/check_temp_topology.sh | 15 +- .../check_worker_participation.sh | 123 +- .../deploy/multi-region/drive_ops_action.sh | 447 + .../deploy/multi-region/drive_ops_center.sh | 7776 +++++++ .../deploy/multi-region/drive_release_hub.sh | 476 + .../multi-region/export_go_live_bundle.sh | 556 + .../multi-region/init_ops_center_config.sh | 135 + .../install_mainland_controller_quick.sh | 4 +- .../install_mainland_worker_quick.sh | 2 +- .../multi-region/install_overseas_quick.sh | 11 + .../deploy/multi-region/lib_ops_center.sh | 106 + .../multi-region/review_go_live_bundle.sh | 317 + .../domaincheck-node-agent.env.example | 15 + .../domaincheck-ops-center.env.example | 49 + .../deploy/systemd/domain-node-agent.service | 17 + domain-api/tests/test_build_info_service.py | 98 + .../tests/test_drive_ops_center_docs.py | 111 + .../tests/test_node_agent_delivery_queue.py | 284 + .../test_node_agent_release_normalization.py | 41 + .../tests/test_ops_action_executor_core.py | 115 + domain-api/tests/test_ops_agent_service.py | 1181 + domain-api/tests/test_ops_api_routes.py | 96 + domain-api/tests/test_ops_command_service.py | 44 + domain-api/tests/test_ops_job_service.py | 391 + domain-api/tests/test_ops_playbook_service.py | 220 + domain-api/tests/test_ops_policy_service.py | 253 + .../tests/test_ops_release_executor_core.py | 89 + .../test_ops_release_service_contracts.py | 93 + ...est_ops_release_service_execution_modes.py | 432 + .../test_ops_release_service_launchpad.py | 212 + ...st_ops_release_service_package_metadata.py | 915 + .../test_ops_runtime_executor_service.py | 118 + domain-api/tests/test_ops_schema_init.py | 75 + domain-api/tests/test_ops_service_activity.py | 607 + .../tests/test_ops_service_codex_action.py | 925 + .../tests/test_ops_service_codex_brief.py | 497 + .../test_ops_service_contract_registry.py | 295 + .../tests/test_ops_service_doctor_decision.py | 149 + .../test_ops_service_driver_action_preview.py | 274 + .../test_ops_service_driver_action_resolve.py | 433 + .../tests/test_ops_service_driver_actions.py | 1265 ++ .../tests/test_ops_service_driver_feed.py | 358 + .../tests/test_ops_service_execution_modes.py | 24 + .../tests/test_ops_service_go_live_bundle.py | 106 + .../tests/test_ops_service_go_live_review.py | 169 + .../tests/test_ops_service_go_live_signoff.py | 317 + .../tests/test_ops_service_go_live_summary.py | 664 + .../tests/test_ops_service_link_snapshot.py | 182 + .../tests/test_ops_service_node_scene_log.py | 256 + domain-api/tests/test_ops_service_overview.py | 1186 + domain-api/tests/test_ops_service_runbook.py | 283 + .../test_ops_service_runbook_sequence.py | 255 + .../tests/test_ops_service_stack_diagnosis.py | 971 + domain-api/tests/test_ops_template_service.py | 177 + .../tests/test_prepare_final_release.py | 117 + .../tests/test_release_metadata_parity.py | 29 + .../tests/test_release_script_parity.py | 19 + domain-api/tests/test_show_latest_release.py | 162 + domain-web/src/api/modules.ts | 101 + domain-web/src/layouts/MainLayout.vue | 1 + domain-web/src/router/index.ts | 1 + domain-web/src/views/detect/DetectView.vue | 35 + domain-web/src/views/ops/OpsCenterView.vue | 18516 ++++++++++++++++ domain-web/src/views/runtime/RuntimeView.vue | 342 +- domain-web/vite.config.ts | 7 + domainCheck/node_thread_counts.json | 5 +- package_domain_release.ps1 | 80 +- package_domain_release.sh | 328 + prepare_final_release.ps1 | 26 +- prepare_final_release.sh | 55 + show_latest_release.ps1 | 60 +- show_latest_release.sh | 91 + verify_domain_release.ps1 | 27 +- verify_domain_release.sh | 151 + 142 files changed, 89727 insertions(+), 186 deletions(-) create mode 100644 HANDOFF_20260418_2321.md create mode 100644 HANDOFF_20260418_2344.md create mode 100644 IMPLEMENTATION_STATUS.md create mode 100644 TASK_BOARD.md create mode 100644 docs/21_domainCheck_海外主机集中运维与自动化方案.md create mode 100644 docs/22_domainCheck_海外Codex驾驶员与OpsCenter落地路线.md create mode 100644 docs/23_domainCheck_终局运维架构设计_海外单脑控制面.md create mode 100644 docs/24_domainCheck_NodeAgent协议与ReleaseHub设计.md create mode 100644 docs/25_domainCheck_海外单脑控制面上线收口总表.md create mode 100644 docs/26_domainCheck_发布前运行验证与交付模板.md create mode 100644 docs/schemas/ops_agent_protocol.md create mode 100644 docs/schemas/ops_doctor_decision_contract.md create mode 100644 docs/schemas/ops_driver_contract.md create mode 100644 docs/schemas/ops_go_live_bundle_contract.md create mode 100644 docs/schemas/ops_go_live_review_contract.md create mode 100644 docs/schemas/ops_go_live_signoff_contract.md create mode 100644 docs/schemas/ops_job_contract.md create mode 100644 docs/schemas/ops_observability_contract.md create mode 100644 docs/schemas/ops_playbook_contract.md create mode 100644 docs/schemas/ops_stack_diagnosis_contract.md create mode 100644 docs/schemas/release_hub_contract.md create mode 100644 domain-api/app/api/routes/ops.py create mode 100644 domain-api/app/api/routes/ops_agent.py create mode 100644 domain-api/app/api/routes/ops_release.py create mode 100644 domain-api/app/node_agent.py create mode 100644 domain-api/app/services/build_info_service.py create mode 100644 domain-api/app/services/ops_action_executor_core.py create mode 100644 domain-api/app/services/ops_agent_service.py create mode 100644 domain-api/app/services/ops_command_service.py create mode 100644 domain-api/app/services/ops_execution_capability_service.py create mode 100644 domain-api/app/services/ops_execution_mode_service.py create mode 100644 domain-api/app/services/ops_job_service.py create mode 100644 domain-api/app/services/ops_playbook_service.py create mode 100644 domain-api/app/services/ops_policy_service.py create mode 100644 domain-api/app/services/ops_release_executor_core.py create mode 100644 domain-api/app/services/ops_release_service.py create mode 100644 domain-api/app/services/ops_runtime_executor_service.py create mode 100644 domain-api/app/services/ops_service.py create mode 100644 domain-api/app/services/ops_template_service.py create mode 100755 domain-api/deploy/multi-region/bootstrap_node_agent.sh create mode 100755 domain-api/deploy/multi-region/build_node_agent_bootstrap_plan.sh create mode 100644 domain-api/deploy/multi-region/check_env_drift.sh create mode 100644 domain-api/deploy/multi-region/check_go_live.sh create mode 100644 domain-api/deploy/multi-region/check_go_live_signoff.sh create mode 100755 domain-api/deploy/multi-region/check_node_agent.sh create mode 100644 domain-api/deploy/multi-region/check_node_scene_log.sh create mode 100644 domain-api/deploy/multi-region/check_ops_center_stack.sh create mode 100755 domain-api/deploy/multi-region/check_ops_contracts.sh create mode 100755 domain-api/deploy/multi-region/check_ops_inspection.sh create mode 100644 domain-api/deploy/multi-region/check_ops_link_snapshot.sh create mode 100644 domain-api/deploy/multi-region/check_ops_plane.sh create mode 100644 domain-api/deploy/multi-region/check_release_hub.sh create mode 100755 domain-api/deploy/multi-region/drive_ops_action.sh create mode 100755 domain-api/deploy/multi-region/drive_ops_center.sh create mode 100755 domain-api/deploy/multi-region/drive_release_hub.sh create mode 100755 domain-api/deploy/multi-region/export_go_live_bundle.sh create mode 100755 domain-api/deploy/multi-region/init_ops_center_config.sh create mode 100644 domain-api/deploy/multi-region/lib_ops_center.sh create mode 100755 domain-api/deploy/multi-region/review_go_live_bundle.sh create mode 100644 domain-api/deploy/multi-region/templates/domaincheck-node-agent.env.example create mode 100644 domain-api/deploy/multi-region/templates/domaincheck-ops-center.env.example create mode 100644 domain-api/deploy/systemd/domain-node-agent.service create mode 100644 domain-api/tests/test_build_info_service.py create mode 100644 domain-api/tests/test_drive_ops_center_docs.py create mode 100644 domain-api/tests/test_node_agent_delivery_queue.py create mode 100644 domain-api/tests/test_node_agent_release_normalization.py create mode 100644 domain-api/tests/test_ops_action_executor_core.py create mode 100644 domain-api/tests/test_ops_agent_service.py create mode 100644 domain-api/tests/test_ops_api_routes.py create mode 100644 domain-api/tests/test_ops_command_service.py create mode 100644 domain-api/tests/test_ops_job_service.py create mode 100644 domain-api/tests/test_ops_playbook_service.py create mode 100644 domain-api/tests/test_ops_policy_service.py create mode 100644 domain-api/tests/test_ops_release_executor_core.py create mode 100644 domain-api/tests/test_ops_release_service_contracts.py create mode 100644 domain-api/tests/test_ops_release_service_execution_modes.py create mode 100644 domain-api/tests/test_ops_release_service_launchpad.py create mode 100644 domain-api/tests/test_ops_release_service_package_metadata.py create mode 100644 domain-api/tests/test_ops_runtime_executor_service.py create mode 100644 domain-api/tests/test_ops_schema_init.py create mode 100644 domain-api/tests/test_ops_service_activity.py create mode 100644 domain-api/tests/test_ops_service_codex_action.py create mode 100644 domain-api/tests/test_ops_service_codex_brief.py create mode 100644 domain-api/tests/test_ops_service_contract_registry.py create mode 100644 domain-api/tests/test_ops_service_doctor_decision.py create mode 100644 domain-api/tests/test_ops_service_driver_action_preview.py create mode 100644 domain-api/tests/test_ops_service_driver_action_resolve.py create mode 100644 domain-api/tests/test_ops_service_driver_actions.py create mode 100644 domain-api/tests/test_ops_service_driver_feed.py create mode 100644 domain-api/tests/test_ops_service_execution_modes.py create mode 100644 domain-api/tests/test_ops_service_go_live_bundle.py create mode 100644 domain-api/tests/test_ops_service_go_live_review.py create mode 100644 domain-api/tests/test_ops_service_go_live_signoff.py create mode 100644 domain-api/tests/test_ops_service_go_live_summary.py create mode 100644 domain-api/tests/test_ops_service_link_snapshot.py create mode 100644 domain-api/tests/test_ops_service_node_scene_log.py create mode 100644 domain-api/tests/test_ops_service_overview.py create mode 100644 domain-api/tests/test_ops_service_runbook.py create mode 100644 domain-api/tests/test_ops_service_runbook_sequence.py create mode 100644 domain-api/tests/test_ops_service_stack_diagnosis.py create mode 100644 domain-api/tests/test_ops_template_service.py create mode 100644 domain-api/tests/test_prepare_final_release.py create mode 100644 domain-api/tests/test_release_metadata_parity.py create mode 100644 domain-api/tests/test_release_script_parity.py create mode 100644 domain-api/tests/test_show_latest_release.py create mode 100644 domain-web/src/views/ops/OpsCenterView.vue create mode 100755 package_domain_release.sh create mode 100755 prepare_final_release.sh create mode 100755 show_latest_release.sh create mode 100755 verify_domain_release.sh diff --git a/HANDOFF_20260418_2321.md b/HANDOFF_20260418_2321.md new file mode 100644 index 0000000..78c1aac --- /dev/null +++ b/HANDOFF_20260418_2321.md @@ -0,0 +1,98 @@ +# HANDOFF 2026-04-18 23:21 + +## 当前最高优先级 + +大陆节点接管闭环最小路径 + +原因： + +- 当前控制面能力已基本足够 +- 当前最大阻塞不是新功能，而是大陆两台还停在： + - `pending_bootstrap` + - `agent_pending` + - `SSH 未配置` +- 只要这条链路不闭合，项目就仍然停留在“主控骨架可用，但统一接管未落地”的状态 + +## 推荐模型与推理等级 + +推荐： + +- `GPT-5.4` +- `high` + +原因： + +- 当前阶段重心是优先级判断、闭环收口、执行顺序压缩 +- 需要稳定判断力，不适合降到偏快但更容易发散的配置 +- 当前阶段也暂时不需要切到 `xhigh` + +## 下一轮如果继续，第一步先做什么 + +下一轮必须先做： + +1. 录入大陆两台节点的 SSH 信息 +2. 复查 `nodes` +3. 逐台跑 `node-handover` +4. 逐台跑 `node-bootstrap-plan` + +建议顺序： + +- `mainland-controller-01` +- `mainland-worker-01` + +推荐第一组动作： + +```bash +bash domain-api/deploy/multi-region/drive_ops_center.sh node-bind-ssh http://127.0.0.1:8100 mainland-controller-01 22 +bash domain-api/deploy/multi-region/drive_ops_center.sh node-bind-ssh http://127.0.0.1:8100 mainland-worker-01 22 +bash domain-api/deploy/multi-region/drive_ops_center.sh nodes http://127.0.0.1:8100 +bash domain-api/deploy/multi-region/drive_ops_center.sh node-handover http://127.0.0.1:8100 mainland-controller-01 +bash domain-api/deploy/multi-region/drive_ops_center.sh node-bootstrap-plan http://127.0.0.1:8100 mainland-controller-01 +``` + +## 当前不要做的事情 + +现在不要做： + +- 扩写新的架构设计文档 +- 新增页面 +- 新增模块 +- 继续增强控制面专题能力 +- 扩大 Release Hub 范围 +- 扩大 runbook / playbook 范围 +- 做与大陆节点接管闭环无直接关系的功能开发 + +## 当前阶段边界 + +本次已完成： + +- 阶段校准 +- 闭环优先级重排 +- 任务板初始化 +- 状态板初始化 +- handoff 生成 + +本次未进入： + +- 任何新实现 +- 任何新页面或新模块 +- 任何新的控制面增强项 + +## 下一轮完成主批次后的预期 + +如果大陆两台顺利完成： + +- SSH 纳管 +- bootstrap +- acceptance +- `remote_access_ready=3/3` + +则项目总体进度预计进入： + +- `60%~65%` 的稳定上线准备区间 + +那时下一阶段才应该切到： + +- 发布前总检 +- 正式收口 +- 上线证据导出 diff --git a/HANDOFF_20260418_2344.md b/HANDOFF_20260418_2344.md new file mode 100644 index 0000000..6e315de --- /dev/null +++ b/HANDOFF_20260418_2344.md @@ -0,0 +1,155 @@ +# HANDOFF 2026-04-18 23:44 + +## 当前最高优先级 + +仍然是：大陆节点接管闭环最小路径 + +但当前阶段已经从“补 SSH 入口”切到： + +- 修正大陆节点环境内容 +- 让 bootstrap 真正可以落地执行 + +## 今晚已完成的步骤 + +已完成： + +1. 录入 `mainland-controller-01` SSH 信息 +2. 录入 `mainland-worker-01` SSH 信息 +3. 复查 `ops/nodes` +4. 逐台执行 `node-handover` +5. 逐台生成并复核 `node-bootstrap-plan` +6. 复核： + - `go-live-summary` + - `ops/nodes` + - `release-launchpad` +7. 手工验证 controller SSH 登录可达 + +## 当前结果 + +当前指标： + +- `ssh_ready = 2` +- `remote_access_ready = 3` +- 是否达到 `remote_access_ready = 3/3`：是 + +但 A1 仍未闭环，因为： + +- `mainland-controller-01` 仍是 `pending_bootstrap` +- `mainland-worker-01` 仍是 `pending_bootstrap` +- 两台都还没有完成： + - `bootstrap` + - `register / heartbeat` + - `acceptance` + +## 两台节点推进到哪一步 + +### mainland-controller-01 + +已推进到： + +- SSH 已录入 +- `node-handover` 已完成 +- `node-bootstrap-plan` 已生成 +- 手工 SSH 登录成功 +- 尝试执行 bootstrap 片段 + +准确卡点： + +- 远端缺少 node-agent 所需文件 +- bootstrap 执行时报错： + - `bootstrap_node_agent.sh not found under known layouts` + - `Unit domaincheck-node-agent.service could not be found` + +阻塞分类： + +- 主阻塞：环境问题 +- 次阻塞：权限 / 外部条件问题 + - 当前控制面自动 SSH 执行器默认走密钥模式 + - 今晚验证可用的是密码 SSH + +### mainland-worker-01 + +已推进到： + +- SSH 已录入 +- `node-handover` 已完成 +- `node-bootstrap-plan` 已生成 + +准确卡点： + +- 与 controller 同类 +- 远端缺少： + - `domain-api/app/node_agent.py` + - `domain-api/app/api/routes/ops_agent.py` + - `domain-api/app/services/ops_agent_service.py` + - `domain-api/deploy/systemd/domain-node-agent.service` + - `domain-api/deploy/multi-region/templates/domaincheck-node-agent.env.example` + +阻塞分类： + +- 主阻塞：环境问题 + +## 收口结论 + +今晚属于“有明显进展，但 A1 未闭环”的收口结果。 + +明确进展： + +- 从 `remote_access_ready = 1/3` 推进到了 `3/3` +- 两台大陆节点都已进入 `ssh_ready` +- handover 与 bootstrap plan 链路都已打通 + +明确未完成： + +- 两台大陆节点仍未完成 bootstrap +- 因此还没有 register / heartbeat +- 也没有进入 acceptance + +## 下一轮如果继续，第一步先做什么 + +下一轮先做的不是新功能，而是修正大陆节点环境。 + +最小动作： + +1. 确认大陆两台代码目录由 `www` 用户更新到包含 node-agent 文件的最新版本 +2. 确认以下文件在两台大陆节点都存在： + - `domain-api/app/node_agent.py` + - `domain-api/app/api/routes/ops_agent.py` + - `domain-api/app/services/ops_agent_service.py` + - `domain-api/deploy/systemd/domain-node-agent.service` + - `domain-api/deploy/multi-region/templates/domaincheck-node-agent.env.example` +3. 然后重新执行： + - `node-bootstrap-plan` + - `bootstrap` + - `acceptance` + +## 推荐模型与推理等级 + +继续推荐： + +- `GPT-5.4` +- `high` + +## 现在不要做的事情 + +不要做： + +- 新页面 +- 新模块 +- 控制面增强项 +- 新架构文档扩写 +- Candidate B2 +- 为了解决今晚问题而扩展新的运维功能 + +## 明天需要你补充或操作的内容 + +你明天需要安排的最小动作是： + +- 让大陆两台节点的仓库内容更新到当前控制面所需版本 +- 并且注意： + - git / 代码更新应继续使用 `www` + - 不要用 `root` 跑 git + +## 当前一句话状态 + +今晚已经把大陆两台从“未纳管 SSH”推进到“SSH 可执行、bootstrap plan 可生成”，但两台都卡在相同的环境缺口上：远端仓库缺 node-agent 运行文件，因此 A1 还差最后的 bootstrap / register / acceptance 收口。 diff --git a/IMPLEMENTATION_STATUS.md b/IMPLEMENTATION_STATUS.md new file mode 100644 index 0000000..6daaa9f --- /dev/null +++ b/IMPLEMENTATION_STATUS.md @@ -0,0 +1,135 @@ +# IMPLEMENTATION_STATUS + +更新时间：2026-04-18 23:44 CST + +## 当前真实状态 + +阶段判断： + +- 方案与控制面能力建设：约 `78%` +- 真正达到“可稳定上线、海外单脑统一接管大陆节点”：约 `58%` + +当前已成立的事实： + +- Ops Center 主控骨架已基本形成 +- Release Hub 已具备主流程判断能力 +- Node Agent 协议、日志回传、runbook / playbook、节点接管视图已具备基础可用性 +- `drive_ops_center.sh` 已补齐： + - `nodes` + - `node-bind-ssh` + +当前最新节点状态： + +- `managed_enabled = 3` +- `agent_ready = 1` +- `ssh_ready = 2` +- `remote_access_ready = 3` +- `pending_bootstrap = 2` + +节点明细： + +- `overseas-control-01` + - `agent_ready` + - `remote_access_ready = true` +- `mainland-controller-01` + - `pending_bootstrap` + - `remote_access_state = ssh_ready` + - `SSH 已录入` + - `node-handover` 已完成 + - `node-bootstrap-plan` 已生成 + - 手工 SSH 登录已验证可达 +- `mainland-worker-01` + - `pending_bootstrap` + - `remote_access_state = ssh_ready` + - `SSH 已录入` + - `node-handover` 已完成 + - `node-bootstrap-plan` 已生成 + +## 代码闭环 + +已完成或基本具备： + +- 统一查看节点接管状态 +- 统一生成与查看 handover / bootstrap 入口 +- 统一查看 go-live summary +- 统一查看日志回传覆盖情况 +- 统一为托管节点录入 SSH 入口的 CLI 能力 +- controller 与 worker 的 bootstrap plan 均可生成 + +当前代码侧已验证的事实： + +- 当前主控链路可以把大陆节点从 `agent_pending` 推到 `ssh_ready` +- `node-handover` 与 `node-bootstrap-plan` 输出一致 +- `go-live-summary` 已看到 `remote_access_ready = 3` + +结论： + +- 当前代码侧已经可以支撑“接管闭环”的执行顺序 +- 当前主阻塞已不再是“还缺哪块控制面功能” +- 但当前 SSH 自动执行链仍默认走密钥模式，和今晚实际提供的密码登录方式不一致 + +## 外部条件闭环 + +当前仍未闭合的条件： + +- 大陆两台虽然已录入 SSH，但远端仓库未具备 node-agent 所需文件 +- 尚未实际触发大陆两台的 bootstrap 执行 +- 尚未建立两台大陆节点的 register / heartbeat 闭环 +- 尚未完成两台大陆节点的 acceptance 闭环 + +结论： + +- 当前最关键的剩余问题不是新代码，而是大陆节点环境与当前控制面能力不匹配 +- 今晚已经不是“SSH 信息缺失”阶段，而是“大陆节点代码/部署内容缺失”阶段 + +## 明确依赖用户提供的信息 + +今晚已经提供并验证： + +- `mainland-controller-01` + - `ssh_host=121.204.244.188` + - `ssh_user=root` + - 手工 SSH 登录成功 +- `mainland-worker-01` + - `ssh_host=121.204.244.248` + - `ssh_user=root` + +当前仍需要你明天确认或操作的，不再是 SSH 信息，而是以下环境修正路径二选一： + +1. 让大陆两台节点仓库更新到包含 node-agent 文件的最新版本 +2. 或把 node-agent 相关文件以正式发布包 / 正式部署方式补齐到大陆两台 + +在上述环境未修正前，以下链路不能真正闭合： + +- `bootstrap execute` +- `register / heartbeat` +- `acceptance` +- `pending_bootstrap -> 已接管` + +## 当前优先级判断 + +最高优先级： + +- 大陆节点接管闭环最小路径 + +当前不应继续优先推进： + +- 控制面横向增强 +- 额外专题设计 +- 与接管闭环无直接关系的发布能力细化 +- 新的页面或模块 + +本轮新增阻塞分类结论： + +- controller 阻塞：环境问题 +- worker 阻塞：环境问题 +- 自动 SSH 执行器限制：权限 / 外部条件问题 + - 现链路仅按 `auth_mode=key` 工作 + - 今晚实际验证可用的是密码 SSH + +## 完成主批次后的预期 + +如果大陆两台环境补齐，并按最小闭环顺序完成： + +- 项目整体可稳定上线进度预计提升到 `60%~65%` +- 一旦 `remote_access_ready=3/3` 成立，后续就从“架构推进期”切换到“发布收口期” diff --git a/README.md b/README.md index 1b182c1..ede142d 100644 --- a/README.md +++ b/README.md @@ -6,13 +6,24 @@ - 新版后端服务 `domain-api/` - 新版 Web 管理后台 `domain-web/` - 全套交付与部署文档 `docs/` -- Windows 本地启动、打包、验包脚本 +- Windows / Linux 启动、打包、验包脚本 如果你是第一次打开这个仓库，建议先看： - [docs/12_domainCheck_导航索引.md](./docs/12_domainCheck_导航索引.md) - [docs/04_domainCheck_WebLinux改造总体方案.md](./docs/04_domainCheck_WebLinux改造总体方案.md) +如果你当前已经进入“海外单脑控制面 / 多机联调 / 准备上线”阶段，建议直接从下面 4 份开始： + +1. [docs/25_domainCheck_海外单脑控制面上线收口总表.md](./docs/25_domainCheck_海外单脑控制面上线收口总表.md) +2. [docs/23_domainCheck_终局运维架构设计_海外单脑控制面.md](./docs/23_domainCheck_终局运维架构设计_海外单脑控制面.md) +3. [docs/24_domainCheck_NodeAgent协议与ReleaseHub设计.md](./docs/24_domainCheck_NodeAgent协议与ReleaseHub设计.md) +4. [docs/schemas/ops_driver_contract.md](./docs/schemas/ops_driver_contract.md) + +如果已经接近正式发布，建议再补一份： + +- [docs/26_domainCheck_发布前运行验证与交付模板.md](./docs/26_domainCheck_发布前运行验证与交付模板.md) + ## 目录结构 ```text @@ -25,10 +36,14 @@ ├─ start_*.ps1 Windows 启动脚本 ├─ stop_*.ps1 Windows 停止脚本 ├─ smoke_test_stack.ps1 本地整栈自测 -├─ package_domain_release.ps1 交付打包 -├─ verify_domain_release.ps1 交付包校验 -├─ prepare_final_release.ps1 最终发布准备 -└─ show_latest_release.ps1 查看最新交付包 +├─ package_domain_release.ps1 Windows 交付打包 +├─ verify_domain_release.ps1 Windows 交付包校验 +├─ prepare_final_release.ps1 Windows 最终发布准备 +├─ show_latest_release.ps1 Windows 查看最新交付包 +├─ package_domain_release.sh Linux / 海外主机交付打包 +├─ verify_domain_release.sh Linux / 海外主机交付包校验 +├─ prepare_final_release.sh Linux / 海外主机最终发布准备 +└─ show_latest_release.sh Linux / 海外主机查看最新交付包 ``` ## 各目录用途 @@ -118,6 +133,15 @@ - [verify_domain_release.ps1](./verify_domain_release.ps1) - [prepare_final_release.ps1](./prepare_final_release.ps1) - [show_latest_release.ps1](./show_latest_release.ps1) +- [package_domain_release.sh](./package_domain_release.sh) +- [verify_domain_release.sh](./verify_domain_release.sh) +- [prepare_final_release.sh](./prepare_final_release.sh) +- [show_latest_release.sh](./show_latest_release.sh) + +补充说明： + +- `show_latest_release.ps1 / .sh` 现在会校验 `final_release_report.json` 是否真的对应当前 `latest_release.*` +- 不是“目录里有 final report 就算已经完成最终签收” ## 当前状态 @@ -142,6 +166,69 @@ Linux 阶段建议先看： - [docs/11_domainCheck_Linux联调输入清单.md](./docs/11_domainCheck_Linux联调输入清单.md) - [docs/12_domainCheck_导航索引.md](./docs/12_domainCheck_导航索引.md) +如果当前已经切到“海外单脑控制面 + 大陆托管节点 + 发布 / Rollout / 驾驶舱首屏”阶段，建议改为： + +1. [docs/25_domainCheck_海外单脑控制面上线收口总表.md](./docs/25_domainCheck_海外单脑控制面上线收口总表.md) +2. [docs/22_domainCheck_海外Codex驾驶员与OpsCenter落地路线.md](./docs/22_domainCheck_海外Codex驾驶员与OpsCenter落地路线.md) +3. [docs/23_domainCheck_终局运维架构设计_海外单脑控制面.md](./docs/23_domainCheck_终局运维架构设计_海外单脑控制面.md) +4. [docs/24_domainCheck_NodeAgent协议与ReleaseHub设计.md](./docs/24_domainCheck_NodeAgent协议与ReleaseHub设计.md) +5. [docs/schemas/ops_stack_diagnosis_contract.md](./docs/schemas/ops_stack_diagnosis_contract.md) +6. [docs/schemas/ops_driver_contract.md](./docs/schemas/ops_driver_contract.md) + +如果当前已经进入“海外主机集中运维 / 多机联调 / Release Hub”阶段，最直接的统一入口是： + +- [domain-api/deploy/multi-region/drive_ops_center.sh](./domain-api/deploy/multi-region/drive_ops_center.sh) +- [domain-api/deploy/multi-region/drive_release_hub.sh](./domain-api/deploy/multi-region/drive_release_hub.sh) +- [domain-api/deploy/multi-region/drive_ops_action.sh](./domain-api/deploy/multi-region/drive_ops_action.sh) +- [domain-api/deploy/multi-region/init_ops_center_config.sh](./domain-api/deploy/multi-region/init_ops_center_config.sh) + +其中： + +- `drive_ops_center.sh` + 用于海外控制面统一调度检查、Release Hub 和 Driver 动作 +- `drive_release_hub.sh` + 用于直接从命令行走 Release / Rollout 标准接口 +- `drive_ops_action.sh` + 用于直接从命令行走 `ops/driver-actions/execute` +- `init_ops_center_config.sh` + 用于初始化 `/etc/default/domaincheck-ops-center` + +现在 `drive_ops_center.sh` 还额外收进了本机发布链： + +- `release-package` +- `release-verify` +- `release-show` +- `release-launchpad` +- `release-prepare` +- `release-preview` +- `release-preview-smart` + +当前如果要用一条最短路径判断“现在能不能继续收口、能不能继续发布”，推荐先跑： + +```bash +cd /www/wwwroot/getDomain +bash domain-api/deploy/multi-region/drive_ops_center.sh go-live-check http://127.0.0.1:8100 http://127.0.0.1:8100 summary +``` + +然后再按： + +```bash +bash domain-api/deploy/multi-region/drive_ops_center.sh stack-diagnosis +bash domain-api/deploy/multi-region/drive_ops_center.sh driver-feed +bash domain-api/deploy/multi-region/drive_ops_center.sh codex-brief +``` + +继续下钻。 + +如果要把这轮检查直接导出成一整包上线证据，也可以直接执行： + +```bash +bash domain-api/deploy/multi-region/drive_ops_center.sh go-live-export +``` + +它会把 `go-live / stack-diagnosis / driver-feed / codex-brief / release-launchpad / doctor-*` +统一导出到 `domain-api/runtime/ops-center-reports/go-live-bundles/` 下，并生成一份 `manifest.json`。 + ## Git 提交建议 建议提交： diff --git a/TASK_BOARD.md b/TASK_BOARD.md new file mode 100644 index 0000000..af0f5e5 --- /dev/null +++ b/TASK_BOARD.md @@ -0,0 +1,148 @@ +# TASK_BOARD + +更新时间：2026-04-18 23:44 CST + +## 当前主批次 + +唯一主批次：大陆节点接管闭环最小路径 + +目标： + +- 把当前项目的最高优先级从“继续扩控制面能力”切到“完成大陆节点接管闭环” +- 只围绕这条最短路径推进： + - 录入大陆节点 SSH 入口 + - 生成并复核 bootstrap plan + - 执行 bootstrap + - 建立 register / heartbeat + - 执行 acceptance + - 将 `remote_access_ready` 从 `1/3` 提升到 `3/3` + +主批次成功标准： + +- `mainland-controller-01` 不再是 `pending_bootstrap` +- `mainland-worker-01` 不再是 `pending_bootstrap` +- `ssh_ready >= 2` +- `remote_access_ready = 3/3` +- 大陆两台都具备可由海外控制面统一发起标准动作的条件 + +当前完成度： + +- 已完成 `SSH` 入口录入 +- 已完成 `node-handover` +- 已完成 `node-bootstrap-plan` +- 已确认 `ssh_ready = 2` +- 已确认 `remote_access_ready = 3/3` +- 尚未完成： + - `bootstrap` + - `register / heartbeat` + - `acceptance` + - `pending_bootstrap -> 已接管` + +## 主批次拆解 + +### Batch A1 + +名称：大陆节点接管闭环最小路径 + +边界： + +- 只处理大陆两台： + - `mainland-controller-01` + - `mainland-worker-01` +- 只处理接管闭环，不扩展控制面功能 + +任务： + +- 已完成： + - 录入 `mainland-controller-01` SSH 信息 + - 录入 `mainland-worker-01` SSH 信息 + - 复查 `ops/nodes` + - 逐台执行 `node-handover` + - 逐台生成并复核 `node-bootstrap-plan` + - 复核： + - `go-live-summary` + - `ops/nodes` + - `release-launchpad` +- 待完成： + - 执行 controller 的 `bootstrap -> register -> heartbeat` + - 执行 worker 的 `bootstrap -> register -> heartbeat` + - 对两台执行 `acceptance` + - 让两台都退出 `pending_bootstrap` + +阻塞： + +- controller 与 worker 两台大陆节点都缺少 node-agent 相关运行文件 +- 远端实际缺失： + - `domain-api/app/node_agent.py` + - `domain-api/app/api/routes/ops_agent.py` + - `domain-api/app/services/ops_agent_service.py` + - `domain-api/deploy/systemd/domain-node-agent.service` + - `domain-api/deploy/multi-region/templates/domaincheck-node-agent.env.example` +- 因此 bootstrap 片段虽然已生成，但远端执行时无法完成安装与启动 + +阻塞分类： + +- 主阻塞：环境问题 + - 大陆节点上的仓库内容落后于当前控制面所需的 node-agent 文件集 +- 次阻塞：权限 / 外部条件问题 + - 当前 SSH 自动执行链默认走密钥模式 + - 今晚实际可用的是密码 SSH + +## 候选批次 + +### Candidate B1 + +名称：接管闭环完成后的上线收口 + +前提： + +- 仅在 `remote_access_ready = 3/3` 后进入 + +范围： + +- 重新跑 `go-live-check / stack-diagnosis / release-launchpad` +- 收口发布前证据 +- 判断是否达到正式上线窗口 + +### Candidate B2 + +名称：接管闭环完成后的值班自动化细化 + +前提： + +- 仅在大陆两台已进入稳定接管状态后进入 + +范围： + +- 优化运维体验 +- 优化日常值班命令 +- 优化交接与证据导出 + +当前不进入： + +- `Candidate B1` +- `Candidate B2` + +原因： + +- 当前虽然 `remote_access_ready = 3/3` +- 但大陆两台仍停在 `pending_bootstrap` +- A1 还没有真正闭环 + +## 暂停项 + +以下事项当前不继续推进： + +- 新的控制面专题设计 +- 新页面、新模块、新专题文档扩写 +- 与大陆节点接管闭环无直接关系的 Release Hub 增强 +- 与大陆节点接管闭环无直接关系的 runbook / playbook 扩展 +- 更细粒度日志分析功能 +- 额外的驾驶视图、美化项、信息聚合项 +- 任何不能直接推动 `bootstrap -> acceptance -> remote_access_ready=3/3` 的功能开发 + +## 当前判断 + +当前项目不缺“继续横向扩能力”的理由，缺的是“把最后两台大陆节点真正纳入闭环”的收口动作。 + +因此下一轮只围绕主批次 A1 推进。 diff --git a/docs/08_domainCheck_交付说明.md b/docs/08_domainCheck_交付说明.md index da18c6f..128107f 100644 --- a/docs/08_domainCheck_交付说明.md +++ b/docs/08_domainCheck_交付说明.md @@ -120,3 +120,10 @@ python deploy/linux/smoke_test.py --base-url http://127.0.0.1:8100 - Linux 实机联调 - 上线前复测 - 真实服务器灰度验证 + +如果当前已经进入“海外单脑控制面 + 多机联调 + 准备发布”阶段，建议改为按下面顺序执行： + +1. `docs/25_domainCheck_海外单脑控制面上线收口总表.md` +2. `bash domain-api/deploy/multi-region/drive_ops_center.sh go-live-export` +3. `bash domain-api/deploy/multi-region/drive_ops_center.sh go-live-review` +4. `docs/26_domainCheck_发布前运行验证与交付模板.md` diff --git a/docs/09_domainCheck_交付打包说明.md b/docs/09_domainCheck_交付打包说明.md index 579f611..f5868be 100644 --- a/docs/09_domainCheck_交付打包说明.md +++ b/docs/09_domainCheck_交付打包说明.md @@ -2,7 +2,7 @@ ## 一、用途 -用于在 Windows 本地把当前可交付内容整理成一份压缩包，便于： +用于在 Windows 或 Linux 上把当前可交付内容整理成一份发布包，便于： - 发给运维或部署同事 - 存档版本快照 @@ -16,31 +16,60 @@ - `verify_domain_release.ps1` - `prepare_final_release.ps1` - `show_latest_release.ps1` +- `package_domain_release.sh` +- `verify_domain_release.sh` +- `prepare_final_release.sh` +- `show_latest_release.sh` -执行方式： +Windows 执行方式： ```powershell powershell -ExecutionPolicy Bypass -File .\package_domain_release.ps1 ``` +Linux / 海外主机执行方式： + +```bash +bash ./package_domain_release.sh +``` + 如需一键完成“打包 + 验包 + 生成最终准备报告”： ```powershell powershell -ExecutionPolicy Bypass -File .\prepare_final_release.ps1 ``` +```bash +bash ./prepare_final_release.sh +``` + 如需快速查看当前最新交付物： ```powershell powershell -ExecutionPolicy Bypass -File .\show_latest_release.ps1 ``` +```bash +bash ./show_latest_release.sh +``` + +注意： + +- `show_latest_release.*` 现在不只是“显示最近一次 final report 是否存在” +- 它会额外校验 `final_release_report.json` 是否真的对应当前 `latest_release.*` +- 只有 `final_release_report_matches_latest=true` 时，`final_release_ok=true` 才有意义 +- `final_release_gate_decision / blocked_reasons / verify_ok / smoke_test_ok` 用于解释“为什么当前能签 / 不能签” +- Ops Center 里的“直接创建 Release / 一键 Worker 灰度 / 一键 Control 发布” + - 现在会硬性依赖 `final_release_ok=true` + - 如果 final report 缺失、过期或门禁未 ready，前端会禁用按钮，后端接口也会拒绝执行 + ## 三、输出位置 打包后会生成： - `release/domaincheck_release_时间戳/` - `release/domaincheck_release_时间戳.zip` +- `release/domaincheck_release_时间戳.tar.gz` - `release/domaincheck_release_时间戳.sha256.txt` - `release/latest_release.txt` - `release/latest_release.json` @@ -50,6 +79,7 @@ powershell -ExecutionPolicy Bypass -File .\show_latest_release.ps1 - `docs/` - `scripts/` +- `scripts/` 中会同时带上 PowerShell 和 Shell 版发布脚本 - `domain-api/` - `app` - `deploy` @@ -89,6 +119,10 @@ powershell -ExecutionPolicy Bypass -File .\show_latest_release.ps1 powershell -ExecutionPolicy Bypass -File .\verify_domain_release.ps1 ``` +```bash +bash ./verify_domain_release.sh +``` + 5. Windows 联调先跑 `scripts/smoke_test_stack.ps1` 6. Linux 部署前阅读： - `docs/05_domainCheck_Linux部署清单.md` @@ -106,6 +140,18 @@ powershell -ExecutionPolicy Bypass -File .\verify_domain_release.ps1 这样更适合交付、存档和迁移。 +Windows 默认产物为 `.zip`，Linux 默认产物为 `.tar.gz`；两者都会生成同结构的 `release_manifest.json / .sha256.txt / latest_release.*`，便于海外控制面统一管理。 + `latest_release.txt / latest_release.json` 用于快速定位“当前最新一份交付包”，避免人工翻目录。 +若打包目录本身是 Git 仓库，脚本还会自动写入 `commit_sha / commit_ref`；海外控制面在创建 Release 时可以直接回填提交号，减少人工抄写出错。 + `final_release_report.json` 用于记录最近一次“打包 + 验包”的最终状态。 + +`show_latest_release.*` 读取这个报告时，还会校验： + +- `package_name` +- `archive_path` / `zip_path` +- `sha256` + +也就是说，即使目录里留着一份旧的 `final_release_report.json`，只要它不是给当前最新包签出的，`final_release_ok` 也不会误报为 `true`。 diff --git a/docs/10_domainCheck_最终交付结论.md b/docs/10_domainCheck_最终交付结论.md index 0449323..52827ac 100644 --- a/docs/10_domainCheck_最终交付结论.md +++ b/docs/10_domainCheck_最终交付结论.md @@ -65,9 +65,11 @@ 1. 把最新交付包发送到目标 Linux 服务器 2. 按 `docs/05`、`docs/13`、`docs/14` 的顺序执行部署、核对与收口 -3. 部署完成后再做一次正式服务态 `smoke test` -4. 导出一份最终诊断包 -5. 进入灰度上线 +3. 如果当前已经进入海外单脑控制面阶段，再按 `docs/25` 执行统一收口 +4. 用 `docs/26` 完成发布前运行验证、证据导出与最终交付结论 +5. 部署完成后再做一次正式服务态 `smoke test` +6. 导出一份最终诊断包 +7. 进入灰度上线 ## 六、最终判断 diff --git a/docs/12_domainCheck_导航索引.md b/docs/12_domainCheck_导航索引.md index 80fc284..2b36cae 100644 --- a/docs/12_domainCheck_导航索引.md +++ b/docs/12_domainCheck_导航索引.md @@ -40,6 +40,51 @@ - `docs/20_domainCheck_临时海外控制面联调清单.md` +### 10. 海外主机集中运维与自动化方案 + +- `docs/21_domainCheck_海外主机集中运维与自动化方案.md` + +### 11. 海外控制面统一入口 + +- `domain-api/deploy/multi-region/drive_ops_center.sh` +- `domain-api/deploy/multi-region/export_go_live_bundle.sh` +- `domain-api/deploy/multi-region/check_env_drift.sh` +- `domain-api/deploy/multi-region/drive_release_hub.sh` +- `domain-api/deploy/multi-region/drive_ops_action.sh` +- `domain-api/deploy/multi-region/init_ops_center_config.sh` +- `domain-api/deploy/multi-region/check_ops_center_stack.sh` +- `domain-api/deploy/multi-region/check_ops_contracts.sh` + +### 12. 海外 Codex 驾驶员与 Ops Center 落地路线 + +- `docs/22_domainCheck_海外Codex驾驶员与OpsCenter落地路线.md` + +### 13. 终局运维架构设计：海外单脑控制面 + +- `docs/23_domainCheck_终局运维架构设计_海外单脑控制面.md` + +### 14. Node Agent 协议与 Release Hub 设计 + +- `docs/24_domainCheck_NodeAgent协议与ReleaseHub设计.md` + +### 15. 运维 Contract Schema + +- `docs/schemas/ops_agent_protocol.md` +- `docs/schemas/release_hub_contract.md` +- `docs/schemas/ops_driver_contract.md` +- `docs/schemas/ops_job_contract.md` +- `docs/schemas/ops_playbook_contract.md` +- `docs/schemas/ops_observability_contract.md` +- `docs/schemas/ops_stack_diagnosis_contract.md` + +### 16. 海外单脑控制面上线收口总表 + +- `docs/25_domainCheck_海外单脑控制面上线收口总表.md` + +### 17. 发布前运行验证与交付模板 + +- `docs/26_domainCheck_发布前运行验证与交付模板.md` + ## 二、文档阅读顺序 ### 1. 先看总体方案 @@ -58,6 +103,19 @@ - `docs/17_domainCheck_全流程部署实操手册.md` - `docs/18_domainCheck_CentOS9一键复制部署与更新文档.md` - `docs/20_domainCheck_临时海外控制面联调清单.md` +- `docs/21_domainCheck_海外主机集中运维与自动化方案.md` +- `docs/22_domainCheck_海外Codex驾驶员与OpsCenter落地路线.md` +- `docs/23_domainCheck_终局运维架构设计_海外单脑控制面.md` +- `docs/24_domainCheck_NodeAgent协议与ReleaseHub设计.md` +- `docs/25_domainCheck_海外单脑控制面上线收口总表.md` +- `docs/26_domainCheck_发布前运行验证与交付模板.md` +- `docs/schemas/ops_agent_protocol.md` +- `docs/schemas/release_hub_contract.md` +- `docs/schemas/ops_driver_contract.md` +- `docs/schemas/ops_job_contract.md` +- `docs/schemas/ops_playbook_contract.md` +- `docs/schemas/ops_observability_contract.md` +- `docs/schemas/ops_stack_diagnosis_contract.md` ### 3. 如果要回溯历史需求与问题 @@ -89,6 +147,11 @@ - `domain-api/deploy/systemd/domain-worker.service` - `domain-api/deploy/linux/README.md` - `domain-api/deploy/multi-region/README.md` +- `domain-api/deploy/multi-region/drive_ops_center.sh` +- `domain-api/deploy/multi-region/drive_release_hub.sh` +- `domain-api/deploy/multi-region/drive_ops_action.sh` +- `domain-api/deploy/multi-region/check_ops_contracts.sh` +- `domain-api/deploy/multi-region/init_ops_center_config.sh` - `domain-api/deploy/multi-region/bootstrap_overseas.sh` - `domain-api/deploy/multi-region/bootstrap_mainland.sh` - `domain-api/deploy/linux/smoke_test.py` @@ -109,6 +172,8 @@ - 配置迁移与备份 - 打包与验包 - 最终发布准备 +- 海外单脑控制面协议收口 +- Ops Center / Codex / CLI 驾驶 contract 收口 剩余工作只在真实 Linux 环境： diff --git a/docs/14_domainCheck_正式上线前最终检查单.md b/docs/14_domainCheck_正式上线前最终检查单.md index 36bea7c..a5e19f2 100644 --- a/docs/14_domainCheck_正式上线前最终检查单.md +++ b/docs/14_domainCheck_正式上线前最终检查单.md @@ -31,6 +31,12 @@ - 海外控制面 `runtime/sync-summary` - 能看到 `detect_result_batches` +如果当前已经切到“海外单脑控制面统一驾驶”模式，建议同时配套阅读： + +- `docs/25_domainCheck_海外单脑控制面上线收口总表.md` +- `docs/23_domainCheck_终局运维架构设计_海外单脑控制面.md` +- `docs/24_domainCheck_NodeAgent协议与ReleaseHub设计.md` + ## 二、上线前必须确认的结论 上线前至少要确认下面这些结论同时成立： @@ -46,6 +52,34 @@ ## 三、正式上线前执行顺序 +### 0. 先跑一遍统一上线总检 + +如果当前已经进入“海外单脑控制面 + 多地域 / Node Agent”这条正式链路，建议先跑： + +```bash +cd /www/wwwroot/getDomain/domain-api +bash domain-api/deploy/multi-region/drive_ops_center.sh go-live-check http://127.0.0.1:8100 http://152.53.37.118:8100 summary +``` + +期望： + +- `go_live_status` 为 `ready` 或至少没有新的 `blocking_reasons` +- `stack_status` 不为 `blocked` +- `route_surface_complete=true` +- `launchpad_status` 不为 `blocked` +- 如果当前已经纳管执行节点，则 `remote_access_ready` 不应长期为 `0` +- `next_step_action_code` / `operator_title` 应能直接说明当前第一优先动作 +- 如果 `log_sync_enabled=true`，则要额外看 `log_sync_state`、`log_sync_covered_nodes` + +如果这里已经出现 `bootstrap_run / run_acceptance`，说明当前上线前仍有节点接管缺口，应先收口首个缺口节点，再继续下面的业务核验。 + +建议这样理解总检输出： + +- `blocking_reasons` 只放真正会阻断上线的项，例如 `remote_agent_ready=0` +- `warnings` 表示还能继续联调，但现在还不适合直接宣称“全部收口” +- `recommended_commands.next_step` 就是当前最值得优先执行的一条命令 +- `recommended_commands.log_sync_logs` / `recommended_commands.log_sync_inspection` 用来补远端参与节点的日志观测 + ### 1. 核对服务状态 ```bash diff --git a/docs/20_domainCheck_临时海外控制面联调清单.md b/docs/20_domainCheck_临时海外控制面联调清单.md index e044403..eab1255 100644 --- a/docs/20_domainCheck_临时海外控制面联调清单.md +++ b/docs/20_domainCheck_临时海外控制面联调清单.md @@ -165,10 +165,20 @@ bash domain-api/deploy/multi-region/check_worker_participation.sh - `detect_worker_nodes` - 看谁在线 - 看谁是 `worker_online=true` - - 看谁 `detect_participating=true` + - `detect_participating=true` 只能说明它曾被判断为“正在参与” - `detect_job_items` - 看任务到底被哪台节点 `claimed_by` - 这才是“当前真正干活的节点” +- `runtime/status` + - 看 `participation_summary.dispatch_active_nodes` + - 这是“当前真正执行/领任务的节点” + - 看 `participation_summary.non_participating_nodes` + - 这是“在线但未参与的节点” + - 看 `non_participating_nodes[].participation_state` + - `standby` 表示在线待命 + - `load_syncing` 表示负载待确认，不要直接算成“正在干活” + - 看 `log_sync` + - 能直接确认远端日志回传是否开启、是关键还是全量、样本来自哪些节点 ## 8. 常见误判 @@ -182,6 +192,7 @@ bash domain-api/deploy/multi-region/check_worker_participation.sh - “有效执行节点”是可承担任务的在线节点数 - “当前参与检测节点”是当前真的在领任务、跑任务的节点 +- “在线但未参与节点”是已经在线、可承接任务，但当前这轮还没分到任务的节点 如果 controller 没兼跑检测，通常只会看到独立 worker 在真正执行。 @@ -189,6 +200,26 @@ bash domain-api/deploy/multi-region/check_worker_participation.sh 这不一定是 bug，可能只是当前调度没有分到它。应以 `detect_job_items.claimed_by` 为准，不要只看服务在线。 +### 8.4 页面显示“负载待确认” + +这不是新的故障状态，而是为了避免误判： + +- 节点已经上报 `busy` 或有 `current_load` +- 但当前还没看到明确的 `items_claimed / items_running / processed_recent` + +通常是心跳和任务快照还没完全对齐。先等下一轮刷新，再结合 `claimed_by` 和 `participation_summary` 判断，不要立刻当作“这台机器已经在跑检测”。 + +### 8.5 日志控制台看不到大陆节点过程 + +先看 `runtime/status` 或页面里的“远端日志回传”： + +- 若 `enabled=false` + - 说明本来就没开回传 +- 若 `enabled=true` 但 `line_count=0` + - 说明开关已开，但当前还没有远端样本 +- 若 `source_nodes` 里没有目标大陆节点 + - 说明该节点这轮还没回传日志，先看它是否真的在参与检测 + ## 9. 联调完成后的回滚 如果你后续要切回正式海外机器，在大陆 controller 上把目标地址改回正式值即可： diff --git a/docs/21_domainCheck_海外主机集中运维与自动化方案.md b/docs/21_domainCheck_海外主机集中运维与自动化方案.md new file mode 100644 index 0000000..15b21e8 --- /dev/null +++ b/docs/21_domainCheck_海外主机集中运维与自动化方案.md @@ -0,0 +1,708 @@ +# 21 domainCheck 海外主机集中运维与自动化方案 + +如果当前你不是在做“为什么要这么设计”的方案评审，而是已经进入真实收口、准备上线阶段，建议先跳转： + +- `docs/25_domainCheck_海外单脑控制面上线收口总表.md` +- `docs/23_domainCheck_终局运维架构设计_海外单脑控制面.md` +- `docs/24_domainCheck_NodeAgent协议与ReleaseHub设计.md` + +## 一、为什么要升级成“集中运维” + +这段时间的多机联调已经证明一个事实： + +- 现在这套“人在海外主机上看后台，再去大陆机器上手工跑命令、复制日志、判断状态”的方式，能联调，但效率很低 +- 一旦节点数从 `2` 台增加到 `3` 台、`5` 台、`10` 台，复杂度会快速失控 +- 国内机器上又不能稳定部署 Codex，因此不能指望每台机器都具备“本机智能调试能力” +- 检测日志很大，靠人工复制日志片段，不适合做持续诊断 + +所以后续不应该继续堆更多“检查脚本”，而应该把整个体系升级成： + +> 海外主机作为唯一运维控制面，统一接管大陆机器的安装、更新、重启、巡检、日志回流和故障诊断。 + +一句话目标： + +> 只在海外主机操作，大陆机器尽量不需要人工登录。 + +--- + +## 二、最终目标形态 + +### 1. 海外主机承担什么 + +海外主机作为唯一控制面，承担： + +- Web 后台 +- API 控制面 +- 运维任务中心 +- 发布包仓库 +- 节点注册与权限中心 +- 日志汇聚与诊断入口 +- 远程命令编排 +- 巡检结果展示 + +也就是说，后续所有动作都从海外主机发起： + +- 新机器纳管 +- 初始化安装 +- 发布更新 +- 配置下发 +- 服务重启 +- 健康检查 +- 采集诊断包 +- 远端日志查看 +- 故障一键排查 + +### 2. 大陆机器承担什么 + +大陆机器不再承担复杂控制逻辑，只承担： + +- `domaincheck worker/controller` 本体服务 +- 一个轻量 Node Agent +- 本机 systemd / 日志 / 版本 / 健康信息暴露 +- 接收控制面任务并执行 +- 将执行结果、日志、诊断包回传到海外主机 + +这意味着大陆机器以后是“被管理对象”，而不是“人工登录操作对象”。 + +--- + +## 三、推荐方案 + +## 方案 A：海外控制面 + 大陆 Node Agent + +这是我建议的主方案，也是长期最优方案。 + +### 核心思想 + +不要把“SSH 到每台机器执行命令”作为主链路，而是让每台大陆机器常驻一个 Agent： + +- Agent 主动连海外控制面 +- Agent 拉取待执行任务 +- 本地执行 systemd / shell / 发布 / 巡检 +- Agent 把 stdout / stderr / 状态 / 日志游标回传 + +这样做的好处是： + +- 不要求海外机能直接入站打通到大陆机 +- 不要求每次人工 SSH +- 不怕 SSH 权限、跳板机、端口变化导致整套流程断掉 +- 天然适合 NAT、弱网络、多机扩容 + +### 交互方式 + +建议 Agent 使用下面其中一种方式主动连海外控制面： + +#### 首选：HTTPS 长轮询 + +- `agent -> overseas-api` +- 周期拉取任务 +- 周期上报心跳、版本、服务状态、日志摘要 + +优点： + +- 实现最简单 +- 最容易兼容现有 Python / FastAPI 架构 +- 容易先落 MVP + +#### 可升级：WebSocket 常连 + +- 建立长连接 +- 海外控制面可实时下发任务 +- Agent 实时回传执行日志 + +优点： + +- 更实时 +- 日志流式体验更好 + +缺点： + +- 第一版复杂度更高 + +### 为什么 Agent 比 SSH 更优 + +SSH 适合作为： + +- 首次 bootstrap +- 临时人工兜底 +- 非常规应急 + +但不适合作为日常主运维链路，因为： + +- 节点一多，权限和连通性管理会变得脆弱 +- 脚本回显、超时、日志采集很难标准化 +- 人工 SSH 本质上还是“远程手工运维” + +所以最佳做法是： + +- SSH 只用于首次纳管 +- 日常统一走 Agent + +## 三点五、当前已经落地的第一版操作闭环 + +这套方案现在已经不只是设计稿，仓库里已经有一批可以直接使用的统一入口： + +- `domain-api/deploy/multi-region/init_ops_center_config.sh` +- `domain-api/deploy/multi-region/drive_ops_center.sh` +- `domain-api/deploy/multi-region/drive_release_hub.sh` +- `domain-api/deploy/multi-region/drive_ops_action.sh` +- `domain-api/deploy/multi-region/build_node_agent_bootstrap_plan.sh` + +推荐从海外控制面按这个顺序使用： + +### 1. 初始化控制面配置 + +```bash +cd /opt/domaincheck/domain-api +bash domain-api/deploy/multi-region/init_ops_center_config.sh \ + /etc/default/domaincheck-ops-center \ + http://121.204.244.188:8100 \ + http://152.53.37.118:8100 \ + https://api.example.com +``` + +### 2. 查看当前统一配置 + +```bash +bash domain-api/deploy/multi-region/drive_ops_center.sh config +``` + +### 3. 在控制面本机生成发布包 + +```bash +bash domain-api/deploy/multi-region/drive_ops_center.sh release-package +bash domain-api/deploy/multi-region/drive_ops_center.sh release-launchpad +bash domain-api/deploy/multi-region/drive_ops_center.sh release-preview +bash domain-api/deploy/multi-region/drive_ops_center.sh release-preview-smart worker +bash domain-api/deploy/multi-region/drive_ops_center.sh release-show +``` + +### 4. 导出一份联调/交接报告 + +```bash +bash domain-api/deploy/multi-region/drive_ops_center.sh doctor-export +``` + +### 5. 为新节点导出接管计划 + +```bash +bash domain-api/deploy/multi-region/drive_ops_center.sh agent-plan-export \ + /opt/domaincheck/domain-api/runtime/ops-center-reports/agent-plans \ + http://127.0.0.1:8100 \ + mainland-worker-02 \ + mainland \ + worker \ + https://api.example.com \ + /opt/domaincheck +``` + +这五步对应的意义分别是： + +- 先统一控制面默认地址和身份 +- 再确认驾驶舱当前在看哪个环境 +- 再把发布物、发布前预检和发版驾驶舱都收回统一入口，不再额外记根目录脚本 +- 再把现场收敛成可以回看、可以交接的报告 +- 最后为新增节点生成标准接管方案 + +这样后面无论是海外 Codex 自动驾驶、后台按钮，还是人工运维，都不再需要先去大陆机器手工拼命令。 + +## 三点六、现在发布驾驶舱已经进入统一数据口径 + +目前发布侧也已经不是零散接口拼出来的状态，而是收敛成了一套统一判断： + +- `GET /api/v1/ops/releases/launchpad` +- `ops overview -> release_hub.launchpad` +- `check_release_hub.sh` +- `drive_release_hub.sh launchpad` +- 运维中枢页面里的“发布驾驶舱” + +这几处现在复用的是同一套结论，都会统一告诉你： + +- 当前是 `ready / attention / blocked` +- 最新发布包是否可用 +- 最新 Release 是否已经建立 +- Worker 智能灰度是否可发 +- Control 发布是否可发 +- 下一步推荐动作是什么 + +同时，`GET /api/v1/ops/runbook` 里的标准作业路径 `release_progression` 也已经切到同一套发布驾驶舱判断。 +也就是说，海外控制面看到的“标准作业路径”和“发布驾驶舱”不会再出现一套说能发、一套说先补接管的分裂口径。 + +现在又往前推进了一步： + +- `POST /api/v1/ops/runbook/sequences/{sequence_key}/execute` + +后台上的标准作业路径按钮，后续都应该优先走这个入口，再由后端统一分发到 driver action / playbook / rollout 预案，而不是前端自己维护动作分支。 + +这样后续海外 Codex、后台按钮和命令行脚本拿到的就不再是三套不同口径，而是同一套“发布驾驶判断”。 + +## 三点七、现在回执队列也已经进入统一治理入口 + +除了发布驾驶舱，Node Agent 回执队列这一层现在也已经不再是“只能看、不能管”的状态。 + +当前已经统一收口成三类入口： + +- 页面： + - OpsCenter 托管节点表可以直接打开“回执队列治理”抽屉 +- 驾驶建议： + - `dead_letter / retrying` 会优先落到标准动作模板 +- 海外单入口 CLI： + - `drive_ops_center.sh queue-status` + - `drive_ops_center.sh queue-records` + - `drive_ops_center.sh queue-flush` + - `drive_ops_center.sh queue-replay` + - `drive_ops_center.sh queue-replay-record` + - `drive_ops_center.sh queue-discard-record` + +这意味着海外主机已经可以统一处理： + +- 哪台节点存在积压 / 死信 +- 头部记录是什么 +- 什么时候应该立即冲刷 +- 什么时候应该重放 +- 什么时候应该说明原因后丢弃 + +当前阶段仍然明确限制为： + +- `record_visibility=head_only` +- 所有治理动作都落成正式 `ops job` +- 不直接 SSH 到节点改队列文件 + +这样以后即使把远端全量死信记录正式投影到控制面，也只是“扩可见范围”，而不是重新推翻当前模型。 + +--- + +## 四、这套方案要解决的具体问题 + +## 1. 安装部署自动化 + +目标： + +- 海外控制面点一次“新增节点” +- 填一个节点模板 +- 大陆机器自动初始化 + +建议流程： + +1. 海外后台新增节点 +2. 生成 `bootstrap token` +3. 大陆机器只执行一次极短安装命令 +4. Node Agent 注册到海外控制面 +5. 控制面向该节点下发： + - 拉取发布包 + - 解压 + - 生成 `/etc/default/...` + - 安装 systemd + - 启动服务 +6. 回传安装结果 + +这样后续加机器时，不需要再重复人工对照文档逐条敲。 + +## 2. 更新发布自动化 + +目标： + +- 海外主机统一推版本 +- 大陆节点自动拉包、灰度更新、失败回滚 + +这里强烈建议： + +> 后续不要让大陆机器自己跑 git 作为正式更新链路。 + +而是改成： + +- 海外控制面构建发布包 +- 版本号固定 +- Agent 下载指定 release 包 +- 校验 checksum +- 切换当前版本软链 +- 重启服务 +- 回传成功/失败 + +这样比远程 `git pull` 更稳，因为： + +- 不依赖每台机器 git 权限 +- 不怕工作区脏文件 +- 不怕 root/www 用户混用 +- 可回滚 + +推荐目录形态： + +```text +/opt/domaincheck/releases// +/opt/domaincheck/current -> /opt/domaincheck/releases// +``` + +更新时： + +- 下载新版本到 `releases` +- 校验 +- 切换 `current` +- `systemctl restart ...` + +失败就回滚软链。 + +## 3. 远程命令执行自动化 + +目标： + +- 海外控制面能发“结构化任务” +- 不再让人手工复制命令 + +建议任务类型： + +- `service.start` +- `service.stop` +- `service.restart` +- `service.status` +- `deploy.release` +- `config.render` +- `diagnostics.collect` +- `logs.tail` +- `script.run` +- `health.check` + +每个任务统一回传： + +- 任务 ID +- 节点编码 +- 开始时间 / 结束时间 +- exit code +- stdout +- stderr +- 结构化结果 JSON + +这样后面后台才能真正做“操作中心”。 + +## 4. 日志集中回流 + +目标： + +- 海外后台就能看到大陆机器日志 +- 不再人工抄 `journalctl` + +### 日志回流建议分三层 + +#### A. 关键事件流 + +只回传关键事件： + +- 服务启动 +- 服务重启 +- 任务开始 +- 任务完成 +- 代理刷新失败 +- Redis/DB 连接异常 +- 第三方站点异常 + +适合默认长期开启。 + +#### B. 诊断模式日志流 + +像你现在提的“关闭 / 关键 / 全量回传”一样： + +- `off` +- `key` +- `full` + +这是正确方向，应该继续保留。 + +#### C. 诊断包 + +当出现疑难问题时，一键打包： + +- 最近 N 分钟 `journalctl` +- `runtime/status` +- `runtime/cluster` +- `detect_worker.log tail` +- `/etc/default/*` +- 当前版本号 +- 本机健康检查输出 + +然后上传海外主机。 + +这比全量实时传所有日志更经济，也更适合定位问题。 + +## 5. 健康巡检自动化 + +目标： + +- 每台大陆机自动自检 +- 海外后台只看结果 + +建议 Agent 每 30 秒到 60 秒上报： + +- 节点在线状态 +- 当前版本 +- API / Worker / Sync Agent 运行态 +- Redis / PostgreSQL / 磁盘 / 内存 / CPU +- 最近告警 +- 当前任务负载 +- 最近日志时间 + +并把现有脚本升级为 Agent 内部检查项： + +- `check_mainland_controller.sh` +- `check_mainland_worker.sh` +- `check_temp_topology.sh` +- `check_worker_participation.sh` + +后续不是人工执行这些脚本，而是 Agent 周期性执行并结构化上传结果。 + +--- + +## 五、建议的系统分层 + +## 1. 海外控制面 + +建议新增一个“运维控制模块”，逻辑上可放进 `domain-api`，后续再拆独立服务也可以。 + +### 需要的核心对象 + +#### `managed_nodes` + +记录纳管节点： + +- `node_code` +- `region` +- `role` +- `hostname` +- `ip` +- `agent_version` +- `current_release` +- `status` +- `ssh_enabled` +- `agent_last_seen_at` +- `tags` + +#### `ops_jobs` + +记录运维任务： + +建议以后直接以 [ops_job_contract.md](/www/wwwroot/getDomain/docs/schemas/ops_job_contract.md:1) 为正式母本。 + +- `job_id` +- `job_type` +- `target_nodes` +- `payload` +- `created_by` +- `status` +- `started_at` +- `finished_at` + +#### `ops_job_steps` + +记录每个节点执行步骤： + +- `node_code` +- `step_name` +- `status` +- `stdout` +- `stderr` +- `result_json` + +#### `node_log_streams` + +记录日志游标和日志回流状态： + +- `node_code` +- `source` +- `mode` +- `cursor` +- `last_received_at` + +## 2. Node Agent + +建议单独做一个轻量 Python 服务，比如： + +```text +domaincheck-node-agent +``` + +职责： + +- 周期心跳 +- 拉取任务 +- 本地执行 +- 采集日志 +- 上报结果 +- 生成诊断包 + +Agent 尽量独立于业务主程序，不要把它耦合进 `detect_worker.py`。 + +原因： + +- 运维系统不能依赖业务进程是否正常 +- 即使 Worker 崩了，Agent 还应该活着，才能帮你排障 + +--- + +## 六、对你当前项目最合适的落地路线 + +## 第一阶段：先做“控制面集中化” + +目标： + +- 海外后台统一看到所有大陆机器 +- 一键拉诊断 +- 一键执行已有检查脚本 +- 一键切日志回传模式 + +这个阶段先不碰太多发布链路，优先把“看”和“查”统一。 + +### 交付物 + +- Node Agent MVP +- 海外后台“节点管理”页 +- 海外后台“运维任务”页 +- 海外后台“远端日志”页 +- 海外后台“诊断包”页 + +## 第二阶段：再做“一键更新” + +目标： + +- 海外控制面选择版本 +- 指定节点灰度发布 +- 自动回滚 + +### 交付物 + +- 发布包生成器 +- `release manifest` +- Agent 下载 / 校验 / 切换版本 +- 回滚机制 + +## 第三阶段：再做“一键安装新节点” + +目标： + +- 新机器只执行一次 bootstrap +- 其余全部由海外控制面接管 + +### 交付物 + +- bootstrap token +- 安装向导 +- 节点注册流程 +- 环境模板生成器 + +--- + +## 七、比“SSH 全接管”更优的地方 + +你提的方向是： + +> 海外主机配置好 SSH，后续所有大陆机器都由代码内部接管 + +这个方向本身是对的，但如果只做“SSH 全接管”，还不够优秀。 + +更优解是： + +> SSH 只作为纳管和兜底方式；日常统一走 Agent 主动连接。 + +这样好处是： + +- 架构更稳 +- 不依赖每次远程入站打通 +- 不怕某些网络环境 SSH 不稳定 +- 更适合未来节点继续增加 +- 更容易做权限分级、审计、结果留痕 + +所以我建议最终定成： + +### 运维主链路 + +- `海外控制面 -> Agent 任务编排 -> 大陆节点执行 -> 回传结果` + +### 运维兜底链路 + +- `海外控制面 -> SSH -> 大陆节点` + +--- + +## 八、和现有代码如何衔接 + +当前项目已经有一些很好的基础，不需要推翻： + +- 多机节点模型 +- `runtime/status` +- `runtime/cluster` +- `sync-summary` +- 一批巡检脚本 +- 日志回传开关雏形 +- 运行中心 + +这些都应该保留，并演进成： + +### 当前脚本的未来定位 + +- `bootstrap_*.sh` + - 保留为 bootstrap / 应急工具 +- `check_*.sh` + - 演进成 Agent 内部的标准诊断动作 +- `runtime/status` + - 继续作为统一运行态接口 +- `runtime/cluster` + - 继续作为统一集群视图 +- `detect_result_projection / runtime_projection` + - 继续作为跨地域观测基础 + +也就是说，现有代码不是废掉，而是从“人工脚本时代”升级成“控制面编排时代”。 + +--- + +## 九、我建议的最优落地决策 + +如果只选一个方向，我建议直接定成： + +### 最优方案 + +- 海外主机为唯一控制面 +- 大陆节点统一部署 `domaincheck-node-agent` +- Agent 主动访问海外控制面 +- 日常安装、更新、巡检、日志回流全部走 Agent +- SSH 仅保留为 bootstrap 和应急手段 +- 正式更新统一改为 release 包分发，不再把 `git pull` 作为正式更新链路 + +这是当前最值得投入的方向，因为它能同时解决： + +- 调试效率低 +- 多机管理复杂 +- 权限混乱 +- 日志分散 +- 更新不稳定 +- 新增节点成本高 + +--- + +## 十、建议的下一步执行顺序 + +不要再继续先补散点 bug，而是先把运维骨架建起来。 + +建议顺序： + +1. 固化这份方案 +2. 新建 `node-agent` 设计草案 +3. 先做 Agent MVP +4. 先接入： + - 心跳 + - 远程任务 + - 诊断包 + - 日志 tail +5. 海外后台新增： + - 节点管理页 + - 运维任务页 + - 远端日志页 +6. 再把现有检查脚本封成 Agent 动作 +7. 最后再做发布链路自动化 + +--- + +## 十一、结论 + +当前最优策略不是“继续加脚本”，而是： + +> 把 domainCheck 从“多机联调项目”升级成“海外控制面统一接管大陆节点的自动化运维系统”。 + +从长期看，这会比继续靠人工 SSH、人工复制日志、人工比对状态高效很多，也更符合你后续继续扩机器的目标。 diff --git a/docs/22_domainCheck_海外Codex驾驶员与OpsCenter落地路线.md b/docs/22_domainCheck_海外Codex驾驶员与OpsCenter落地路线.md new file mode 100644 index 0000000..3634705 --- /dev/null +++ b/docs/22_domainCheck_海外Codex驾驶员与OpsCenter落地路线.md @@ -0,0 +1,1052 @@ +# 22 domainCheck 海外 Codex 驾驶员与 Ops Center 落地路线 + +如果当前目标已经从“设计 Codex 驾驶员”进入“准备正式上线”，建议先补看： + +- `docs/25_domainCheck_海外单脑控制面上线收口总表.md` +- `docs/schemas/ops_driver_contract.md` +- `docs/schemas/ops_stack_diagnosis_contract.md` + +## 一、目标收口 + +这套体系后续不应该再依赖： + +- 人工登录每台大陆机器 +- 手工 `systemctl` +- 手工复制 `journalctl` +- 人工判断哪台机器在跑、哪台机器没跑 + +要收口成两种都能工作的模式： + +### 模式 A：有 Codex + +- 海外主机部署 Codex +- Codex 作为智能驾驶员 +- 自动读取集群状态、同步状态、参与检测节点、日志回传状态 +- 自动决定下一步执行哪类运维动作 + +### 模式 B：没有 Codex + +- 海外后台照样能点按钮 +- 后台把按钮动作转换成标准化 `ops job` +- 控制面自动分发到节点 +- 节点回传执行结果、日志、诊断摘要 + +一句话： + +> Codex 负责“智能判断”，Ops Center 负责“标准动作执行”。 + +--- + +## 二、推荐最终结构 + +### 1. 海外控制面 + +海外控制面承担： + +- Web 前端 +- API 控制面 +- Ops API +- 发布包管理 +- 节点纳管 +- 日志汇聚入口 +- 运维任务中心 + +### 2. 大陆节点 + +大陆节点统一视为“被托管节点”： + +- controller / worker 主服务 +- node agent +- systemd +- 本机诊断采集器 + +### 3. 三条运维链路 + +#### 主链路：Agent 拉任务 + +- 最终常态方案 +- 适合日常安装、更新、巡检、采日志 + +#### 辅链路：后台按钮 + +- 前端触发动作 +- 后端创建 `ops job` +- 节点 agent 执行 + +#### 兜底链路：SSH + +- 只保留给首次 bootstrap 和紧急救援 +- 不再作为日常主流程 + +--- + +## 三、当前代码基座已经具备的条件 + +当前仓库已经具备这些基础： + +- `runtime/status` +- `runtime/cluster` +- `runtime/readiness` +- `runtime/sync-summary` +- debug handoff / diagnosis +- 远端日志回传开关基础 +- 检测参与节点与待命节点视图基础 + +这意味着我们不是从零开始，而是已经有： + +> 运行态观测基座 + +现在需要补的是： + +> 运维动作编排基座 + +--- + +## 四、建议的落地顺序 + +### 第 1 步：先统一成 Ops API + +本次先补： + +- `/api/v1/ops/overview` +- `/api/v1/ops/capabilities` +- `/api/v1/ops/blueprint` +- `/api/v1/ops/runbook` +- `/api/v1/ops/nodes` +- `/api/v1/ops/jobs` + +这一步的意义不是“已经远控完成”，而是先把控制面语言统一起来。 + +也就是以后无论： + +- Codex 判断 +- 后台按钮 +- 节点 agent 执行 + +都围绕统一的 `ops` 模型展开。 + +后续发布链路也应保持这个原则： + +- `release launchpad` +- `ops overview.driver_recommendations` +- `ops runbook.release_progression` + +三者必须共用同一份发布状态机，而不能各自维护一套“推荐下一步”。 + +同理，标准作业路径也不应只是“展示文案”： + +- `ops runbook.control_sequences` +- `POST /api/v1/ops/runbook/sequences/{sequence_key}/resolve` +- `POST /api/v1/ops/runbook/sequences/{sequence_key}/execute` + +这三层组合起来，才是海外 Codex 驾驶员和后台按钮都能复用的统一作业入口： + +- `control_sequences` + 负责提供当前应该走哪条标准路径 +- `resolve` + 负责把这条路径收口成“此刻真正要执行的 driver action / node_codes / payload” +- `execute` + 负责真正触发后端动作 + +这样海外 Codex 可以先做： + +1. 读取 `runbook` +2. 对目标 sequence 调 `resolve` +3. 判断是自动执行、请求确认，还是只生成建议 +4. 最后再调 `execute` + +同样地，发布审阅动作也不应只是“跳到某个页面再看”： + +- `review_smart_rollout_preview` +- `review_control_rollout` +- `fix_rollout_blockers` + +现在这类动作应该直接返回结构化 `release_launchpad_review` 结果，至少包括： + +- `latest_release` +- `launchpad_status` +- `preview` +- `smart_rollout` +- `target_node_codes` +- `gap_rows` + +这样没有前端时，Codex、CLI、自动化任务也能直接基于 launchpad 预案判断下一步，而不是先依赖 UI 才能继续。 + +这里还要把自动化等级口径固定下来，避免后面又把“审阅”和“执行”混成一类： + +- `review_smart_rollout_preview` +- `review_control_rollout` +- `fix_rollout_blockers` + +这三类动作现在应统一视为： + +- `safe_auto` +- 只读审阅 +- 可以直接由后端执行并返回结构化 `release_launchpad_review` + +而真正会创建发布或放量对象的动作，例如： + +- `publish_latest_worker` +- `create_smart_release_rollout_worker` +- `create_smart_release_rollout_control` +- `create_release_rollout_worker` +- `create_release_rollout_control` + +则必须继续归类为： + +- `guarded_auto` +- 先确认，再执行 +- 必须走正式的发布 / rollout 审批与审计链 + +这样海外 Codex、CLI、按钮驾驶舱三边看到的规则才会一致： + +- “看预案”可以自动跑 +- “创建放量”不能静默自动提交 + +而不是把判断逻辑重新写在 Codex prompt 或后台页面里。 + +同样，`GET /api/v1/ops/codex-brief` 和 `GET /api/v1/ops/stack-diagnosis` 也不应该只返回“人看得懂的摘要”，还要内嵌： + +- `contract_navigation.primary_contract_key` +- `contract_navigation.contract_keys` +- `contract_navigation.detail_endpoint_pattern` + +以及每条 entry / issue 对应的 contract hints。 + +现在这条规则应继续扩展到： + +- `GET /api/v1/ops/driver-feed` +- `GET /api/v1/ops/activity-stream` + +也就是说，不只是 Codex 摘要层，连“驾驶主线”和“统一活动流”本身也必须能直接给出 contract navigation，页面、CLI、海外 Codex 才能真正顺着协议跳转，而不是再各自猜一遍。 + +这样海外 Codex 驾驶员在收到“建议动作”或“总检缺口”后，下一步不是靠 prompt 猜协议，而是可以直接按 contract key 去拉： + +- `GET /api/v1/ops/contracts/{contract_key}` + +这才是真正可持续的“驾驶层”，而不是一次性的提示词工程。 + +现在这条链路还要再收口一层： + +- `GET /api/v1/ops/go-live-summary` + - 作为页面、CLI、海外 Codex 共用的稳定“上线收口摘要” +- `GET /api/v1/ops/codex-brief` + - 继续返回 `go_live_summary` +- `GET /api/v1/ops/driver-feed` + - 继续返回 `go_live_summary` + +也就是说，海外 Codex 驾驶员的固定起手式现在应变成： + +1. 先读 `go-live-summary` +2. 再读 `codex-brief` +3. 真要解释阻断原因、看主车道、抄命令时，再下钻 `stack-diagnosis` + +这样 Codex 不再需要从零拼“当前能不能上线”，而是先接住一份稳定结论，再进入解释层和动作层。 + +这里还要继续固定一个口径： + +- `go_live_summary.go_live_status` + - 回答当前是否已经进入统一收口 / 联调 / 复核车道 +- `go_live_summary.publish_status / publish_ready` + - 回答当前是否已经满足正式发布门禁 + +同时还要把“接入缺口是否已经影响发版门禁”的字段也固定下来，避免海外 Codex 继续自己拼 launchpad gap： + +- `go_live_summary.launchpad_onboarding_bootstrap_pending_nodes` +- `go_live_summary.launchpad_onboarding_acceptance_ready_nodes` +- `go_live_summary.launchpad_recommended_target_node_code` +- `go_live_summary.launchpad_recommended_recovery_label` +- `go_live_summary.launchpad_recommended_recovery_summary` + +这样海外 Codex 的默认判断就应该变成： + +1. 如果 `publish_status = blocked` + - 先看硬阻断 +2. 如果 `publish_status = attention` + - 再优先看 launchpad 是否只是节点接管缺口 +3. 如果 launchpad 已经给出 + - `recommended_target_node_code` + - `recommended_recovery_summary` + 那就直接进入节点级恢复动作 + +而不是再去从 `release_launchpad.worker_rollout_preview.policy_preview.release_gate.rows` 里二次翻译一次。 + +也就是说海外 Codex 的第一轮判断必须先分清： + +- “现在可以继续收口” +- 和 “现在可以正式发版” + +这两个结论不能再混成一句模糊建议。 + +这里再补一个已经适合前后端直接消费的收口字段： + +- `driver-feed.automation_coverage` +- `codex-brief.automation_coverage` + +它们现在应该统一返回至少这些统计： + +- `safe_auto_total` +- `guarded_auto_total` +- `mixed_total` +- `ui_only_total` +- `blocked_total` +- `preview_only_total` +- `backend_handled_total` +- `execution_ready_total` +- `human_dependency_total` +- `launch_status` +- `launch_ready` + +这组字段的定位要固定： + +- `automation_level_counts / recommendation_counts` + 回答“当前动作池的自动化构成” +- `preview_only_total` + 回答“现在有多少动作虽然可自动跑，但本质仍是只读预览/聚焦/审阅” +- `backend_handled_total` + 回答“当前驾驶主线里有多少动作已经真正后端化，不再依赖人工进页面” +- `execution_ready_total` + 回答“此刻可以直接执行或确认后执行的动作总数” +- `human_dependency_total` + 回答“此刻仍需要人工复核、进入工作区、或先排阻断的动作总数” +- `launch_status / launch_ready` + 给页面、CLI、Codex 一个统一的自动化侧收口判断，而不是每层再各自猜 + +这样后续前端顶部摘要、Codex 驾驶提示、CLI 巡检输出，都可以共享同一套“自动化覆盖率”口径。 + +这里还要固定一个前端交互约束，避免页面重新退回“按钮点了只弹几条 toast”的碎片状态： + +- Ops Center 首屏的 + - 执行默认下一步 + - 定位下一步 + - 看下一步契约 + - 进入主处理面板 + - 复制收口命令 + - 复制快检命令 +- 这些动作都应统一回到 + - `驾驶回执条` + +也就是说，首屏动作执行后，页面应该统一返回： + +- 成功 +- 关注 +- 阻断 + +三类结构化回执，而不是： + +- 有的只弹 toast +- 有的静默跳转 +- 有的打开抽屉但没有说明 + +这样值班时第一屏不只是“能点动作”，还必须能在第一屏立即知道： + +- 动作是否已经生效 +- 当前是否已经进入下一个处理面板 +- 还是仍然缺少上下文 / 需要人工继续处理 + +同时，`doctor-export / doctor-decision` 现在也不应只返回： + +- `required_failures` +- `optional_unavailable` +- `recommended_commands` + +还要一起返回： + +- `scene_log_reports_total` +- `scene_log_reports_ok` +- `scene_log_status_counts` +- `scene_log_reports` + +这样海外 Codex 读取一份 `manifest.json` 或 `doctor-decision` 摘要时，不只是知道“总检建议去看某台节点”，还知道： + +- 关键节点现场日志附件有没有真正打出来 +- 打出来的是 `healthy / full_capture` + 还是 `waiting_sample / missing_sample` +- 下一步应该继续看 `driver-feed` + 还是直接下钻具体 `scene-node-log` + +同时，在线驾驶层本身现在也应固定暴露： + +- `driver-feed.scene_log_observation` +- `codex-brief.scene_log_observation` + +这两个字段不是替代 `doctor-export` 附件，而是给“当前这一刻”的在线控制面提供统一口径： + +- 现场日志是否关闭 +- 是否已经开启但仍在等样本 +- 当前最适合继续下钻的节点是谁 +- 对应的正式 `focus_ref` 是什么 + +这样离线交接包看 `scene_log_reports`，在线驾驶层看 `scene_log_observation`，职责就完全分开了。 + +同时，页面侧为了减少额外请求，现在也应优先复用 `GET /api/v1/ops/runbook` 自带的： + +- `primary_resolution` +- `secondary_resolution` + +也就是： + +- 页面首屏直接使用 runbook 内嵌的解析快照 +- CLI / Codex / 自动驾驶器在需要单独复核某条路径时，再额外调用 `resolve` + +这样“运行手册快照”和“单条路径即时解析”就有了清晰分工，不会每个入口都反复打一轮解析请求。 + +当前还额外补了一个最小任务闭环： + +- 可登记托管节点 +- 可创建 `ops job` +- 可查看 `ops job` 执行记录 +- 当前节点可先本机即时执行部分 `runtime.*` 动作 + +这让“后台按钮触发 -> 标准任务留痕”已经开始成型，而不是继续堆零散脚本。 + +当前运维中枢还补了一层很关键的“批量巡检捷径”： + +- 真正参与检测节点 +- 在线未参与节点 +- 全部有效执行节点 + +它们不会直接执行 shell，而是把目标节点批量预填到 `diagnostics.collect` 模板里，再统一创建 `ops job`。 + +这个设计的意义是： + +- 页面先把“节点分组口径”统一好 +- Codex 和后台按钮复用同一套动作模型 +- 后续节点变多时，不需要每次人工挑选机器 + +当前这层快捷入口只纳入： + +- 已纳管 +- 已启用 +- Node Agent 在线 +- 当前属于有效执行节点 + +这样能避免“页面看起来能点，但其实任务根本发不到节点”的假动作。 + +并且这层已经不只是“打开模板”，还开始支持标准巡检序列： + +- `health.snapshot` +- `logs.collect` +- `diagnostics.collect` + +也就是一键把“健康快照 -> Worker 日志 -> 诊断包”整套任务批量排进 `ops job` 队列。 + +这样后续无论： + +- 海外 Codex 自动判断 +- 还是后台按钮人工触发 + +都复用同一套巡检动作编排，而不是再写新的临时脚本。 + +现在 OpsCenter 还要继续并入“执行现场”： + +- 直接看真正参与检测节点 +- 直接看在线但未参与的待命节点 +- 直接看远端日志回传当前是关闭 / 关键 / 全量 +- 并且在同一页上切换日志回传模式 + +也就是说，海外驾驶舱不应该要求操作者再切去 Runtime 页面拼状态，而要在运维中枢里直接完成“观察现场 -> 切日志回传 -> 下发标准巡检”这一整段动作。 + +同时运维中枢开始具备第一版“驾驶建议卡”： + +- 首个缺口节点优先执行接入收口 / 接管验收 +- 先补现场日志回传 +- 先看真正参与检测的节点 +- 再排查在线未参与节点 +- 最后进入 Release / Rollout + +其中“先补现场日志回传”很关键： + +- 如果参与节点存在，但远端日志回传还没开 +- 驾驶建议应优先让操作者开启关键回传 +- 如果日志回传已开但还没有现场样本 +- 则优先抓 Worker 日志，确认为什么现场过程没被镜像回来 +- 日志回传本身也应该成为正式 contract，而不是只有一个布尔开关： + - 当前状态是关闭 / 等待样本 / 部分覆盖 / 关键覆盖 / 全量观察 + - 当前参与节点覆盖了几台 + - 哪些参与节点仍未回传样本 + - 下一步建议动作是什么 + +这一步的意义是把“海外 Codex 驾驶员”的判断逻辑，先沉淀成页面里可见、可点、可回执的正式入口。 + +同样，“参与检测节点”也不能只给一个总数，而要继续拆成： + +- 执行中 / 已领待跑 +- 近窗刚有吞吐 +- 在线待命 +- 负载待确认 + +这样海外控制面看到“在线 3 台、有效执行 3 台”时，才不会误以为这 3 台都在真正做同一种事情。 + +同理，Release / Rollout 也不能只停留在“有没有版本”这一层，而要补成正式门禁对象： + +- `release_hub.default_rollout_gate` +- 里面明确给出： + - 默认 Release 是谁 + - 默认 Rollout 目标节点是谁 + - 当前门禁状态是 `missing_release / release_not_ready / artifact_missing / no_targets / blocked / attention / ready` + - 当前阻断原因、告警原因、推荐动作 + - 当前 remote-agent 就绪数、巡检通过数 + +这样首页驾驶建议、Release Hub、Codex 自动驾驶、后续后台按钮，才能基于同一份门禁判断，而不是页面自己再拼一套“能不能发版”的逻辑。 + +现在这层还要继续后移： + +- `GET /api/v1/ops/overview` 不只返回运行摘要 +- 也直接返回 `driver_recommendations` +- 每条建议带标准化 `action_code`、节点列表和展示文案 + +这样页面只是负责把动作映射到现有按钮，而不是自己再次判断一遍优先级。 + +继续往下一步，就是把能标准化的建议动作直接后端化： + +- `POST /api/v1/ops/driver-actions/execute` +- 先覆盖： + - 开/关现场日志回传 + - 批量标准巡检 + - 批量抓 Worker 日志 + - 批量收诊断包 + +这样页面、脚本、Codex 驾驶员都先走同一个后端执行入口；还不能后端化的动作，再暂时回退到前端交互。 + +并且动作命名也要继续收敛到通用语义： + +- `run_standard_inspection` +- `open_worker_logs` +- `open_diagnostics` + +不要长期把动作名绑死在 `participating / standby` 这种页面上下文里，否则后续同一动作很难在不同入口复用。 + +现在又继续往后收了一层： + +- `GET /api/v1/ops/playbooks` +- `GET /api/v1/ops/playbook-runs` +- `GET /api/v1/ops/playbook-runs/{run_code}` +- `GET /api/v1/ops/playbook-runs/{run_code}/events` +- `GET /api/v1/ops/activity-stream` +- `POST /api/v1/ops/playbook-runs/{run_code}/rerun` +- `POST /api/v1/ops/playbook-runs/{run_code}/cancel` +- `POST /api/v1/ops/playbooks/preview` +- `POST /api/v1/ops/playbooks/execute` + +这层不是替代 `ops job`，而是把一组固定的高层运维意图继续往下展开。 + +同时每一次 playbook 执行，不应该只留下零散的子任务，而应该形成一条正式“编排回执”： + +- 每次执行生成独立 `run_code` +- 每个展开后的 `ops job` 都挂上同一份 playbook metadata +- 控制面可以按 `run_code` 重新聚合： + - 本轮编排是什么 + - 目标节点有哪些 + - 当前卡在哪一步 + - 最近一条可查看事件的任务是哪条 + +这样页面、Codex 驾驶员和后续自动化，都不再需要自己从多条 `ops job` 记录里反推“这一轮到底发生了什么”。 + +继续再往前一步，playbook run 不能只“看”，还要能直接作为运维对象被操作： + +- 直接按 `run_code` 查看详情 +- 直接按 `run_code` 重跑同一轮编排 +- 直接按 `run_code` 取消当前仍未收口的子任务 + +这样运维中心面对的就不再只是“原子 job 列表”，而是正式的“编排回执对象”。 + +再往前收一层，OpsCenter 还要把： + +- playbook run +- standalone ops job +- rollout + +统一压成一条“活动流”。 + +这样海外驾驶舱里看到的就不再是三块割裂入口，而是： + +- 最近发生了什么 +- 这是编排、单任务还是 rollout +- 当前状态是什么 +- 一键应该跳到哪个详情抽屉 + +这一层已经开始落成： + +- 后端聚合 `activity-stream` +- 前端活动流面板直接复用： + - playbook run 详情抽屉 + - ops job 事件抽屉 + - rollout 任务抽屉 + +最近又补了一层很关键的驾驶口径： + +- `runbook_sequence` + +也就是说，统一活动流里现在不只收“已经发生的执行回执”，还会收： + +- 当前固定标准作业路径此刻解析出的下一步动作快照 + +这样海外控制面看到活动流时，能同时得到两类信息： + +- 最近现场到底发生了什么 +- 当前更推荐先进入哪条固定路径 + +并且这条 `runbook_sequence` 活动不是新 job，也不是新 rollout，而是： + +- 对 `ops runbook.control_sequences[*].primary_resolution` + 的时间线化投影 + +它点进去后应直接把页面焦点定位到对应的标准作业路径卡片，而不是再跳去一个无关抽屉。 + +再往前一步，还需要一份正式给“驾驶员”消费的统一结果： + +- `GET /api/v1/ops/driver-feed` + +这份 feed 不应该只是页面内部拼接，而要由后端直接给出： + +- 当前最高优先建议 +- 其他驾驶建议 +- 标准作业路径 + +并且每条都明确声明： + +- 这是 `driver_action` + 还是 +- `runbook_sequence` + +这样海外 Codex、后台按钮和 CLI 才能共享一套真正稳定的执行 contract，而不是继续在前端或 prompt 里二次判断。 + +在这层之上，还要再补一层真正适合“海外 Codex 自动驾驶”的判断结果： + +- `GET /api/v1/ops/codex-brief` + +这层不是重复造一个 feed，而是在 `driver-feed` 之上补齐： + +- 这条动作是 `safe_auto` +- 还是 `guarded_auto` +- 还是 `mixed` +- 还是 `ui_only` +- 或者已经 `blocked` + +同时直接给出： + +- 推荐动作口径： + - `auto_execute` + - `confirm_then_execute` + - `resolve_first` + - `open_ui` + - `blocked` +- 最终应命中的 API： + - `/ops/driver-actions/execute` + - 或 `/ops/runbook/sequences/{sequence_key}/execute` +- 请求预览： + - `action_code / node_codes / action_payload` + - 或 `sequence_key / secondary / action_payload` + +这样海外 Codex 就不需要再把“哪些能自动开、哪些必须人工确认、哪些只能进 UI”重新写死在 prompt 里，而是直接消费控制面给出的统一自动驾驶 contract。 + +并且活动流不该只是“堆最近 20 条”，还要具备正式运维视角： + +- 按 `kind` 筛： + - playbook run + - standalone ops job + - rollout + - runbook sequence +- 按 `status` 筛 +- 按关键词搜： + - 编号 + - 对象 + - 摘要 + - 发起来源 + +这样节点数和动作数上来以后，活动流才能继续保持可用，而不会退化成人工翻时间线。 + +这样后续 Codex 驾驶员与人工操作员共用的就是同一条“最近活动 -> 直接钻取”的驾驶链路。 + +最近又再收一层“可读性 contract”： + +- `GET /api/v1/ops/playbook-runs` 已支持按 `status`、`group_key`、`query` 过滤 +- 回执聚合结果会直接给出： + - `focus_level` + - `focus_step_key` + - `focus_step_title` + - `focus_summary` + - `problem_steps` + - `active_steps` +- 前端不再只展示“这轮 run 有几步”，还会直接提示： + - 当前最该看的步骤是哪一步 + - 它是在执行中，还是已经异常 + - 应该优先去看哪条事件入口 + +这样海外控制面看到的就不是一张“机械回执表”，而是已经带有现场聚焦能力的编排驾驶面板。 + +继续再补一层之后，编排详情也不该只停在“看 step 状态”，还要能直接看整轮事件流： + +- 不再要求操作者点进每一条子任务再翻事件 +- 控制面可以直接按 `run_code` 拉整轮编排的聚合事件 +- 还能继续按 `step_key`、`node_code` 做二次过滤 + +这样 playbook run 在海外控制面里的形态才完整： + +- 能看总状态 +- 能看当前焦点 +- 能看每一步 +- 也能直接回放整轮现场事件 + +再往前收一层，顶部“运维驾驶建议”也不该只盯节点和日志，而要开始直接消费 playbook run： + +- 如果最近存在异常编排，驾驶建议优先提示这轮 `run_code` +- 建议里直接带上当前焦点步骤 +- 点一下就能打开该轮编排详情或最近事件 + +这样海外控制面看到的就是： + +- 先告诉你哪轮编排最值得优先处理 +- 再把你直接带到对应的编排现场 + +这才符合“海外 Codex 驾驶员 / 按钮化驾驶舱”的设计目标。 + +再往前一步，驾驶建议还不该只盯 playbook run，本轮已经开始让它继续消费统一活动流： + +- 如果最近存在异常 `ops job` +- 或存在异常 / 待审批 `rollout` +- recommendation 也应直接把这条 activity 顶上来 +- 点一下直接落到对应 `job events` 或 `rollout jobs` + +也就是说，驾驶建议的优先级已经开始变成： + +- 先看异常编排 +- 再看异常活动 +- 再处理接管缺口、日志回传和巡检动作 + +这样海外控制面才不是“会推荐很多动作”，而是真正会把操作者先送到最值得处理的现场。 + +再继续收一层，driver action 也不该只返回“handled / unhandled”，而要开始返回正式的 `ui-intent`： + +- 后端统一决定这次动作是： + - 直接执行 + - 创建 playbook + - 还是打开某个界面入口 +- 前端不再自己硬编码“这个 action_code 到底该弹哪个抽屉” + +例如现在已经开始固化的 intent 方向： + +- `playbook_run_detail` +- `playbook_run_latest_events` +- `job_events` +- `rollout_jobs` +- `managed_node_handover` +- `managed_node_edit` +- `open_release_dialog` +- `open_rollout_dialog` +- `focus_release_hub` + +这样控制面、Codex 驾驶员、后续自动化脚本三方看到的动作 contract 会越来越一致。 + +这里再补一条新的收口约束： + +- `focus_release_hub` + 不应只是“打开版本区”的导航动作 +- 它现在还应同时返回结构化 `release_hub_preview` + 至少带： + - `summary` + - `launchpad` + +这样页面仍然可以跳到 Release Hub，但 CLI、Codex、自动化任务即使不进页面，也能继续基于 release hub 摘要判断下一步。 + +同样，`release_package` 在打包自动化真正落地前，也不应继续只是“打开打包面板”： + +- 页面侧仍可消费 `ui_intent = open_release_dialog` +- 但后端应同时返回 `release_package_preview` + 例如： + - `available` + - `package_name` + - `release_version_suggestion` + - `reason` + +这样海外 Codex 和 CLI 至少能先判断： + +- 现场是否已经存在可用发布包 +- 建议的版本号是什么 +- 当前卡在“缺包”还是“只是还没进页面确认” + +例如： + +- `inspection.standard` + - `health.snapshot` + - `logs.collect(worker)` + - `diagnostics.collect` +- `scene.logs.key` + - `logs.collect(worker, 120)` +- `scene.diagnostics` + - `diagnostics.collect(200)` + +这样前端按钮、海外 Codex 驾驶员、后续自动化脚本都不需要自己再手动拼“三连动作”，而是统一复用后端定义的 playbook。 + +现在 OpsCenter 页面也已经开始直接接这层 playbook： + +- 页面可读取 `playbooks` +- 页面可做 `playbook preview` +- 标准巡检主入口优先直接调用 `inspection.standard` +- 接管后一键验收优先直接调用 `onboarding.acceptance` + +这意味着“标准巡检”第一次不再由前端自己顺序创建三组模板任务，而是正式走后端编排层。 + +同样的收口方向现在也已经开始覆盖“纳管新节点”与“现场观察”： + +- `node_onboarding` + - 主动作：`bootstrap_run` + - 次动作：`run_acceptance` +- `scene_observe` + - 主动作：`run_scene_logs_key` + - 次动作：`run_scene_logs_full` +- `standard_inspection` + - 主动作：`run_standard_inspection` + - 次动作：`open_diagnostics` +- `release_progression` + - Launchpad 就绪时优先：`publish_latest_worker` + - 只有 Release、但还没形成明确 Launchpad 推荐时：`create_release_rollout_worker` + +其中发布链要继续严格区分两种动作层级： + +- `review_smart_rollout_preview / review_control_rollout / fix_rollout_blockers` + - 属于 `safe_auto` + - 目标是读 launchpad 预案、看门禁缺口、返回结构化审阅结果 +- `publish_latest_worker / create_*_release_rollout_*` + - 属于 `guarded_auto` + - 目标是正式创建发布 / rollout 对象 + - 必须继续要求确认和审计留痕 + +也就是说，Runbook 里的主次步骤已经开始直接落到正式驾驶动作，而不是统一先打开弹窗再让操作者自己决定。 + +也就是说： + +- 有 Codex 时，它读这些状态后自动帮你挑动作 +- 没 Codex 时，你在后台也能沿着同样的优先级直接点下去 + +再往下一层，运维中枢还需要具备“巡检结果收口面板”： + +- 不是只会发 `health.snapshot / logs.collect / diagnostics.collect` +- 而是要按节点把这三类任务最近结果重新聚合 +- 直接看出这台节点是： + - 已健康收口 + - 仍在执行 + - 存在异常 + - 还没形成巡检记录 + +这样海外控制面才真正像驾驶舱，而不是“按钮发射器”。 + +现在这层已经开始正式后移到后端语义： + +- `GET /api/v1/ops/overview` 里直接带 `inspection` +- 也单独提供 `GET /api/v1/ops/inspection-overview` +- 前端优先消费后端聚合结果，本地计算只作为 fallback + +这样后续无论是： + +- 海外 Codex 驾驶员 +- 页面按钮 +- 自动化巡检脚本 + +都读同一份“节点巡检收口状态”，不会再出现页面一套、脚本一套、人工 SQL 又一套的口径分裂。 + +这一层再往前，还应该有“异常节点优先队列”： + +- 先把巡检失败 / 阻断 / 取消的节点顶上来 +- 再显示仍在执行中的节点 +- 最后才是未形成巡检记录的节点 + +并且日志聚合口径要严格： + +- 标准巡检只认 Worker 日志 +- 不能把 Node Agent 的临时排障日志混进 Worker 巡检收口 + +### 第 2 步：补 ops jobs + +这一步后续建议直接以 [ops_job_contract.md](/www/wwwroot/getDomain/docs/schemas/ops_job_contract.md:1) 为正式母本，不再让页面、CLI、Codex、Node Agent 各自定义任务结构。 + +后续新增： + +- `ops_jobs` +- `ops_job_steps` +- `managed_nodes` +- `node_log_streams` + +每个动作必须有： + +- 谁发起 +- 发给哪台机器 +- 目标服务 +- 执行结果 +- stdout/stderr +- 结构化摘要 + +### 第 3 步：补 node agent + +Node Agent 第一版建议只做四类动作： + +- `service.restart` +- `service.status` +- `health.check` +- `logs.collect` + +先把最常用的远程运维闭环跑通，再逐步扩成： + +- `deploy.release` +- `config.render` +- `diagnostics.collect` +- `bootstrap.finish` + +### 第 4 步：补发布与回滚 + +正式收口成： + +- 海外控制面构建 release 包 +- 节点下载 release +- 校验 checksum +- 切换软链 +- 重启服务 +- 失败自动回滚 + +--- + +## 五、Codex 在这里真正扮演什么角色 + +Codex 不应该只是“聊天助手”，而应该是： + +### 1. 运维驾驶员 + +Codex 读取： + +- readiness +- cluster +- sync-summary +- debug diagnosis +- ops overview + +然后给出： + +- 当前是网络问题、配置问题、节点未参与、还是同步问题 +- 下一步先拉日志、先重启、先巡检、还是先暂停同步 + +### 2. 动作编排器 + +当 Codex 确认动作后： + +- 创建 ops job +- 标准化执行 +- 等待回执 +- 汇总结果 + +### 3. 联调加速器 + +以后你不需要再复制大量日志给我看，而是让 Codex 直接在海外控制面读： + +- 节点状态 +- 任务结果 +- 聚合日志 +- 诊断包摘要 + +--- + +## 六、你以后理想的操作方式 + +你只需要在海外控制面做三件事： + +### 1. 配置节点 + +- 节点名称 +- SSH / bootstrap 信息 +- 区域 +- 角色 +- 发布通道 + +### 2. 看状态 + +- 哪些节点在线 +- 哪些节点是有效执行节点 +- 哪些节点正在领任务 +- 哪些节点只是在线待命 +- 哪些节点同步失败 + +### 3. 点动作 + +- 安装节点 +- 更新节点 +- 重启节点 +- 拉日志 +- 巡检 +- 收诊断包 +- 治理 Node Agent 回执积压 / 死信 + +如果装了 Codex，就再多一步： + +- 让 Codex 自动判断“该点哪个动作” + +## 六点五、回执队列已经进入统一驾驶入口 + +当前 Node Agent 回执队列不再只是“后端看得见”，而是已经进入统一驾驶面： + +- OpsCenter 托管节点表可以直接打开“回执队列治理”抽屉 +- `driver_recommendations` 遇到 `dead_letter / retrying` 时，会优先下发标准队列治理动作 +- 海外单入口 CLI 也可以直接查看、冲刷、重放和丢弃 + +这意味着： + +- 没有 Codex 时，人工也能在海外控制面闭环处理 +- 有 Codex 时，Codex 不需要自己发明死信处理逻辑 +- 页面、CLI、Codex 都围绕同一组 `ops job` 入口治理回执队列 + +当前统一 CLI 入口已经支持： + +```bash +cd /opt/domaincheck/domain-api + +# 查看某台节点当前回执队列状态 +bash domain-api/deploy/multi-region/drive_ops_center.sh queue-status mainland-worker-01 + +# 只看可见死信头部记录 +bash domain-api/deploy/multi-region/drive_ops_center.sh queue-records mainland-worker-01 dead_letter + +# 立即冲刷积压回执 +bash domain-api/deploy/multi-region/drive_ops_center.sh queue-flush mainland-worker-01 20 + +# 重放当前可见死信 +bash domain-api/deploy/multi-region/drive_ops_center.sh queue-replay mainland-worker-01 20 + +# 对头部死信执行单条动作 +bash domain-api/deploy/multi-region/drive_ops_center.sh queue-replay-record mainland-worker-01 event-ops-123-2 +bash domain-api/deploy/multi-region/drive_ops_center.sh queue-discard-record mainland-worker-01 event-ops-123-2 "确认已人工归档" +``` + +当前阶段仍然要严格坚持： + +- 所有动作都创建正式 `ops job` +- 当前只操作 `record_visibility=head_only` 的可见头部记录 +- 不直接 SSH 到节点手改队列文件 + +这样以后即使从 `head_only` 扩到“远端全量记录正式投影”，也只是扩展可见范围，不需要推翻当前治理模型。 + +## 六点六、现场观察 runbook 也不再依赖弹窗 + +`scene_observe` 这条标准作业路径现在也开始从“打开 playbook 弹窗”收口到“直接执行正式动作”： + +- 主动作：`run_scene_logs_key` +- 次动作：`run_scene_logs_full` +- 两者都直接落到标准 `ops playbook` 执行链，而不是先让操作者进入模板弹窗二次确认 + +这样海外单脑在排查“谁在跑、为什么卡住、日志有没有回来”时，已经可以直接从 runbook 进入执行态，Codex 和后台按钮也能复用同一条现场观察链路。 + +--- + +## 七、当前最优结论 + +当前最优路线不是继续堆更多手工排查命令，而是： + +1. 以海外主机为唯一运维入口 +2. 用 `ops api + ops jobs + node agent` 接管所有大陆节点 +3. SSH 只做 bootstrap，不做日常主链路 +4. Codex 作为智能驾驶员可选增强，但不是唯一依赖 +5. 没有 Codex 时，后台按钮也能完整跑通 + +这条路线的优点是： + +- 节点越多越省心 +- 跨地域调试成本显著下降 +- 可以沉淀成正式运维平台，而不是临时联调脚本集合 diff --git a/docs/23_domainCheck_终局运维架构设计_海外单脑控制面.md b/docs/23_domainCheck_终局运维架构设计_海外单脑控制面.md new file mode 100644 index 0000000..e59c7af --- /dev/null +++ b/docs/23_domainCheck_终局运维架构设计_海外单脑控制面.md @@ -0,0 +1,1163 @@ +# 23 domainCheck 终局运维架构设计：海外单脑控制面 + +## 一、先回答最核心的问题：`ops job` 是什么 + +`ops job` 不是一条 shell 命令，也不是“点一下按钮马上执行”的临时动作。 + +`ops job` 的本质是： + +> 一条可审计、可编排、可回放、可中断、可回滚的运维意图记录。 + +它表达的是： + +- 谁发起了什么目标 +- 目标作用到哪些节点 +- 按什么策略执行 +- 分成哪些步骤 +- 每一步执行结果是什么 +- 有没有失败 +- 失败后是否回滚 +- 最终系统是否达成目标状态 + +也就是说： + +- shell 是“执行手段” +- `ops job` 是“运维对象” + +最终系统里，不应该由前端、Codex、定时器直接执行 shell。 + +它们都只能做一件事： + +> 创建 `ops job` + +然后由专门的执行器去消费这个 `ops job`。 + +--- + +## 二、为什么最终不要“直接执行 shell” + +因为“直接执行 shell”有 7 个天然缺陷： + +### 1. 无法审计 + +你知道执行了什么命令，但不知道： + +- 谁触发的 +- 为什么触发 +- 是否按预期执行完 +- 执行前后的系统状态 + +### 2. 无法标准化 + +今天你执行： + +- `systemctl restart domaincheck-worker` + +明天可能要补： + +- 重启前健康检查 +- 失败后重试 +- 失败后回滚 +- 成功后采样验证 + +如果直接跑 shell，这些逻辑会散落在脚本、前端、人工脑子里。 + +### 3. 无法并发调度 + +当节点从 `2` 台增加到 `10` 台，直接 shell 无法优雅表达： + +- 先更新 controller +- 再灰度更新一台 worker +- 观察 10 分钟 +- 再滚动更新剩余节点 + +### 4. 无法做回滚 + +shell 通常只描述“怎么做”，不描述“失败怎么办”。 + +### 5. 无法限流和保护 + +比如： + +- 同一时间最多更新 1 台 controller +- 同一地域最多重启 2 台 worker +- 正在跑检测的节点禁止强更 + +这些都不是 shell 擅长的层面。 + +### 6. 无法与 Codex 协作 + +Codex 可以判断，但最终必须落到稳定、标准、可追踪的动作模型，否则还是人工时代。 + +### 7. 无法沉淀成平台 + +你要的不是“几个脚本”，而是“海外一台机器接管全部大陆机器”的正式运维平台。 + +平台核心对象必须是 `ops job`，不是 shell。 + +但这里还有一个非常重要的配套原则： + +- 不直接执行 shell +- 不等于只会给出“你再去跑这几条命令”的人工建议 + +真正终局形态里，还需要一层： + +> 组合恢复入口 + +比如“远端日志回传恢复”这类高频动作，就不应该让人工自己拼： + +1. 开关键回传 +2. 看 overview 有没有生效 +3. 再决定去看哪台节点日志 + +而应该被固化成正式单入口，例如： + +```bash +bash domain-api/deploy/multi-region/drive_ops_center.sh log-sync-recover +``` + +它表达的不是一条 shell，而是一条标准运维意图： + +- 执行恢复动作 +- 读取复核结果 +- 输出下一跳命令 + +也就是说，终局平台既不是“裸 shell”，也不是“只有抽象 job 没有现场入口”，而是： + +- 底层执行对象是 `ops job` +- 上层驾驶入口是组合意图命令 + +这样海外单脑控制面、后台按钮、Codex 驾驶员，才能既保持标准化，又保持现场处理效率。 + +同理，“运行中 API 还是旧代码 / 旧 schema” 这类问题，也不应该再退回到人工只记一条： + +```bash +systemctl restart domaincheck-api +``` + +正式平台里它应该被提升为组合恢复入口，例如： + +```bash +bash domain-api/deploy/multi-region/drive_ops_center.sh runtime-refresh-recover http://127.0.0.1:8100 +``` + +如果当前不是只做单点刷新，而是要把： + +- 运行时刷新 +- 总检复核 +- 上线门禁 +- 下一跳动作判断 + +一起串成一条标准联调车道，那么入口应该继续提升为： + +```bash +bash domain-api/deploy/multi-region/drive_ops_center.sh go-live-recover http://127.0.0.1:8100 +``` + +它表达的依然不是“直接 shell”，而是： + +- 先识别 runtime drift +- 再执行统一恢复 +- 再回读 go-live / stack diagnosis contract +- 最后输出主动作与复核动作 + +这样页面、CLI、Codex 驾驶员就不会再次退回到： + +- 人工猜该不该重启 +- 人工猜重启完先看哪个接口 +- 人工自己拼下一跳命令 + +而是继续保持“上层是组合意图入口、底层才是执行手段”的正式平台结构。 + +同样，“节点接管缺口恢复”也不应该拆成散落在脑子里的三四步： + +- 先看哪台节点有缺口 +- 再看 handover 详情 +- 再生成 bootstrap plan +- 再手工整理下一步命令 + +它同样应该被固化成正式组合入口，例如： + +```bash +bash domain-api/deploy/multi-region/drive_ops_center.sh agent-gap-recover +``` + +它表达的依然不是“直接 SSH 干活”，而是一条标准运维意图： + +- 找到首个接管缺口节点 +- 读取该节点 handover 状态 +- 生成该节点接入方案 +- 输出该节点当前 onboarding 阶段 +- 输出后续检查与执行建议 + +为了不让值班同事继续在脑子里手动拼： + +- `node-handover` +- `node-bootstrap-plan` +- `onboarding.acceptance` + +还应该有一个节点级入口： + +```bash +bash domain-api/deploy/multi-region/drive_ops_center.sh node-onboarding http://127.0.0.1:8100 mainland-worker-01 +``` + +它负责把单节点当前到底处于： + +- 待补资料 +- 待生成 bootstrap +- 待执行 bootstrap +- 可进入 `onboarding.acceptance` + +统一压成一个节点级结果。 + +并且当节点已经进入 `acceptance_ready` 时，不应该再让值班同事自己拼 playbook 参数，而是继续直接给出： + +```bash +bash domain-api/deploy/multi-region/drive_ops_center.sh node-acceptance-plan http://127.0.0.1:8100 mainland-worker-01 +bash domain-api/deploy/multi-region/drive_ops_center.sh node-acceptance-run http://127.0.0.1:8100 mainland-worker-01 ops-center +``` + +也就是： + +- 先看这次验收将创建哪些标准化任务 +- 再把验收统一签发进 ops job / playbook run 体系 + +同理，bootstrap 也不应该只停在“生成脚本块”，还应该有： + +```bash +bash domain-api/deploy/multi-region/drive_ops_center.sh node-bootstrap-preview http://127.0.0.1:8100 mainland-worker-01 +bash domain-api/deploy/multi-region/drive_ops_center.sh node-bootstrap-run http://127.0.0.1:8100 mainland-worker-01 ops-center +``` + +这样： + +- 需要人工落地脚本时，用 `node-bootstrap-plan` +- 需要纳入标准工单追踪时，用 `node-bootstrap-run` +- 需要 ready 之后做验收时，用 `node-acceptance-run` + +再往上还应该有一个真正的节点级自动恢复入口： + +```bash +bash domain-api/deploy/multi-region/drive_ops_center.sh node-recover http://127.0.0.1:8100 mainland-worker-01 +bash domain-api/deploy/multi-region/drive_ops_center.sh node-recover http://127.0.0.1:8100 mainland-worker-01 confirm ops-center +``` + +它的职责不是提供新的能力，而是替值班同事做状态分流： + +- 当前还在接入阶段，就转向 `node-bootstrap-run` +- 当前已经进入验收窗口，就转向 `node-acceptance-run` +- 默认先预览决策，`confirm` 后再真正执行 + +这样平台里真正被沉淀下来的就不是“几条应急命令”，而是： + +- `log-sync-recover` +- `agent-gap-recover` +- `node-onboarding` +- `node-recover` +- 后续还会继续有 `release-rollout-recover` +- `delivery-queue-recover` + +也就是说，终局平台里的高频运维动作，最终都应该沉淀为： + +- 组合恢复入口 +- 正式 preview / execute contract +- 一次动作只执行一次 + +特别是 `node-recover` 这类入口，后续必须固定成： + +- preview 走 `onboarding/recovery/preview` +- confirm 走 `onboarding/recovery/execute` + +如果后端 execute 已经成功接单，CLI / 页面 / Codex 不允许再补跑一次底层： + +- `node-bootstrap-run` +- `node-acceptance-run` + +因为这已经不是“展示问题”，而是会直接造成重复创建 `ops job / playbook run` 的正式执行错误。 + +这里再补一个页面落地约束，避免海外单脑控制面首屏继续堆四五套重复摘要： + +- Ops Center 首屏必须先有一块 + - `单脑驾驶舱首屏` +- 它不是新的数据源 + - 而是把现有 + - `go-live-summary` + - `stack-diagnosis` + - `driver-feed` + - `codex-brief.automation_coverage` + 统一压成一屏结论 + +首屏至少应固定展示： + +- 上线收口状态 +- 发布闸门状态 +- 自动化收口状态 +- 主处理车道 +- 默认下一步 +- 后端接管覆盖 +- 观察层完整度 +- 现场日志覆盖度 + +并且页面分层必须明确： + +- 首屏负责 + - 判断 + - 决策 + - 默认下一步 + - 一键进入处理面板 +- 下方详情区负责 + - 解释原因 + - 展开问题清单 + - 展示推荐动作与命令 + - 提供 contract 下钻 + +也就是说，下方区块不应该再和首屏重复堆一遍相同按钮，而应明确退到“详情 / 复核 / 下钻”角色。 + +这样值班时第一眼先回答四个问题： + +1. 现在能不能上线 +2. 现在能不能正式发版 +3. 现在是不是已经进入自动化可接管状态 +4. 现在默认下一步到底是什么 + +然后才继续下钻到： + +- 问题清单 +- 推荐动作 +- job / playbook / rollout 细节 + +也就是说，页面首屏本身就应该体现“海外单脑”的产品形态，而不是只把多个接口原样堆到一起。 + +> 标准组合恢复入口 + +而不是重新退化回人工 shell 时代。 + +这里还要固定一个已经进入实现层的动作口径： + +- 如果后端已经把首个缺口节点收敛成 `bootstrap_run` + - 页面、CLI、Codex 都优先显示“跑接入收口” +- 如果后端已经把首个缺口节点收敛成 `run_acceptance` + - 页面、CLI、Codex 都优先显示“跑接管验收” +- `fix_managed_nodes` + - 仅保留为兜底入口 + - 不能再覆盖已经明确收敛的单节点恢复动作 + +--- + +## 三、最高级方案的总目标 + +最终形态不是“海外后台 + 若干工具脚本”，而是： + +> 一套以海外控制面为唯一运维大脑、以 `ops job` 为核心对象、以节点 agent 为标准执行器、以 Codex 为智能驾驶员的正式运维平台。 + +一句话理解： + +- 海外控制面负责“决策、编排、发布、审计、汇聚” +- 大陆节点负责“执行、回传、承载业务” +- Codex 负责“分析、建议、触发” + +--- + +## 四、终局架构分层 + +建议直接按 8 层设计。 + +### 1. Control Plane 控制面 + +部署在海外主机。 + +职责： + +- Web UI +- API +- 运维中心 +- 节点目录 +- 发布中心 +- 日志中心 +- 审计中心 + +它是唯一入口。 + +### 2. Intent Layer 意图层 + +所有动作先变成意图： + +- 新增节点 +- 安装节点 +- 更新节点 +- 重启服务 +- 拉取日志 +- 采集诊断 +- 启动检测 +- 暂停检测 +- 回滚版本 + +这些意图统一落成 `ops job`。 + +### 3. Workflow Engine 工作流层 + +把一个运维动作拆成步骤和状态机。 + +例如 `deploy.release`： + +1. 预检 +2. 下载发布包 +3. 校验 checksum +4. 解压到 releases +5. 切换 current +6. 重启服务 +7. 健康检查 +8. 标记成功 +9. 失败则回滚 + +这层不关心 shell 细节，只关心工作流编排。 + +### 4. Scheduler 调度层 + +决定： + +- 哪个 job 先执行 +- 哪些节点能执行 +- 是否需要分批 +- 是否需要串行 +- 是否超过并发上限 +- 是否落在维护窗口 + +### 5. Executor 执行器层 + +真正执行动作的不是前端，也不是 Codex，而是执行器。 + +执行器分 3 类： + +- `node-agent executor` +- `release executor` +- `diagnostic executor` + +兜底再保留： + +- `ssh rescue executor` + +### 6. Node Agent 节点代理层 + +每台大陆节点都跑一个常驻 `domaincheck-node-agent`。 + +职责： + +- 主动注册 +- 主动心跳 +- 拉取待执行 job +- 本地执行命令 +- 采集 stdout/stderr +- 推送结构化结果 +- 推送日志和诊断包 + +### 7. Artifact & Release 发布层 + +发布必须是不可变版本制。 + +不能把正式运维建立在： + +- 节点直接 `git pull` +- 节点工作区可能脏 +- root/www 用户混跑 + +正式模型应该是： + +- 海外控制面构建 release 包 +- release 包有版本号和 checksum +- 节点只下载、校验、切换 +- 回滚也只是在 release 之间切换 + +### 8. Observability 观测与审计层 + +所有动作必须产生： + +- 事件 +- 日志 +- 结构化结果 +- 诊断包 +- 审计记录 + +这层既给前端看，也给 Codex 看。 + +--- + +## 五、`ops job` 的正式对象模型 + +### 1. 一个 `ops job` 至少包含这些字段 + +- `job_code` +- `job_type` +- `requested_by` +- `source` +- `priority` +- `target_selector` +- `execution_policy` +- `desired_state` +- `payload` +- `status` +- `plan` +- `result` +- `created_at` +- `started_at` +- `finished_at` + +### 2. `ops job` 的 5 个核心部分 + +#### A. Intent + +用户真正想做什么。 + +例如： + +- `service.restart` +- `deploy.release` +- `node.bootstrap` + +#### B. Selector + +作用到谁。 + +例如： + +- 指定节点 +- 指定 region=mainland +- 指定 role=worker +- 指定 `hybrid=true` + +#### C. Policy + +怎么执行。 + +例如： + +- 串行 +- 每批 1 台 +- 失败即停止 +- 健康检查失败自动回滚 + +#### D. Plan + +实际拆出来的步骤。 + +例如： + +- 预检 +- 执行 +- 验证 +- 回滚 + +#### E. Result + +最终结果和证据。 + +例如： + +- 成功 / 失败 / 部分成功 +- stdout / stderr 摘要 +- 健康检查结果 +- 诊断包链接 + +--- + +## 六、`ops job` 的状态机 + +建议不要只用简单的 `queued/running/success/failed`。 + +正式版建议： + +- `draft` +- `approved` +- `queued` +- `planning` +- `scheduled` +- `dispatching` +- `running` +- `verifying` +- `partially_succeeded` +- `succeeded` +- `rollback_running` +- `rolled_back` +- `failed` +- `cancelled` +- `timed_out` + +这样后续你看后台时，能明确知道卡在哪一层。 + +--- + +## 七、`ops job` 不是单步骤，而是 DAG 工作流 + +最高级方案里，一个 job 不应只是一条线性步骤，而应支持 DAG。 + +例如“更新大陆两台 worker”： + +- 先跑公共预检 +- 再更新 `mainland-worker-01` +- 成功后更新 `mainland-controller-01` 的 worker 部分 +- 两边都成功后再做集群验证 + +有些步骤可以并行，有些必须串行。 + +所以正式设计应支持： + +- `ops_job_steps` +- `depends_on_step_id` +- `retry_policy` +- `timeout_seconds` +- `rollback_step_id` + +--- + +## 八、节点不再是“机器”，而是“受控资源” + +最高级方案里，要把节点建模成受控资源，而不是一堆 IP。 + +每个节点建议有： + +- `node_code` +- `region` +- `role` +- `capabilities` +- `labels` +- `deploy_channel` +- `agent_version` +- `reachable` +- `maintenance_mode` +- `effective_worker` +- `hybrid_worker` +- `current_release` +- `desired_release` + +这样调度器才能按资源能力派发任务。 + +--- + +## 九、Node Agent 应该怎样设计 + +这是整套方案最关键的一环。 + +### 1. 连接模式 + +首选： + +- Agent 主动连海外控制面 +- 长轮询或 WebSocket + +不推荐以 SSH 为主链路。 + +### 2. Agent 的职责边界 + +Agent 不负责“决策”，只负责： + +- 执行 +- 回传 +- 保持本机状态一致 + +### 3. Agent 的本地能力 + +Agent 至少应内置这些执行器： + +- `systemd executor` +- `release executor` +- `shell executor` +- `log collector` +- `diagnostics collector` +- `config renderer` + +### 4. Agent 的安全限制 + +Agent 必须执行白名单动作。 + +不能允许控制面下发任意 shell 就直接执行。 + +正确方式是： + +- 控制面发 `action + payload` +- agent 解析成受控执行 + +例如： + +- `service.restart` -> `systemctl restart xxx` +- `logs.collect` -> 收集指定日志源 + +shell executor 只能是受限兜底能力，且必须带审批和审计。 + +--- + +## 十、发布系统必须独立出来 + +终局方案里，发布不应该再依赖 git 工作区。 + +建议正式设计为： + +### 1. Release Object + +每个发布包都应有： + +- `release_id` +- `version` +- `commit_sha` +- `build_time` +- `artifact_url` +- `checksum` +- `migration_version` +- `rollback_to` + +### 2. 节点上的目录规范 + +```text +/opt/domaincheck/releases// +/opt/domaincheck/current -> /opt/domaincheck/releases// +/opt/domaincheck/shared/ +``` + +### 3. 更新流程 + +1. 下载发布包 +2. 校验 checksum +3. 解压到新目录 +4. 渲染配置 +5. 停服务 +6. 切 `current` +7. 启服务 +8. 做健康检查 +9. 失败回滚 + +这才是正式运维。 + +--- + +## 十一、日志体系必须分层 + +如果日志不分层，后期还是会被日志量拖死。 + +### 1. Event 事件流 + +默认长期保留，体量小。 + +包含： + +- 服务启动/停止 +- 任务开始/结束 +- 检测阶段切换 +- 代理异常 +- DB/Redis 异常 +- 第三方依赖异常 + +### 2. Job Log 任务日志 + +和 `ops job` 绑定。 + +用于看某次动作： + +- 执行了什么 +- 返回了什么 + +### 3. Runtime Log 运行日志 + +业务服务自身日志。 + +默认不全量常驻上传，只做： + +- 游标读取 +- 按需拉取 +- 临时全量回传 + +### 4. Diagnostics Bundle 诊断包 + +故障时一键采集： + +- env 摘要 +- systemd 状态 +- 关键日志 +- cluster snapshot +- sync summary +- release info + +--- + +## 十二、Codex 的最高级接管方式 + +你说的“海外 Codex 作为智能驾驶员”，最高级不是让 Codex 直接 SSH。 + +最高级方式是： + +### 1. Codex 是 Planner + Operator + +Codex 拿到的是： + +- 控制面所有 API +- 节点状态 +- ops jobs +- 结构化日志 +- 发布记录 + +然后它只做两类事： + +- 分析 +- 创建标准 `ops job` + +### 2. Codex 不直接绕过控制面 + +Codex 不应该： + +- 直接登录每台机器 +- 直接手搓 root 命令 +- 直接破坏审计链 + +它应该永远走控制面 API。 + +### 3. Codex 的价值 + +它的价值不是“替代平台”，而是： + +- 自动读状态 +- 自动识别问题模式 +- 自动建议动作 +- 必要时自动创建低风险 job +- 把复杂联调变成结构化流程 + +--- + +## 十三、没有 Codex 时也必须完整可用 + +这点非常关键。 + +最终平台不能依赖 Codex 才能工作。 + +没有 Codex 时，后台也必须能： + +- 纳管节点 +- 发布更新 +- 一键重启 +- 一键巡检 +- 一键采日志 +- 一键采诊断包 +- 查看执行结果 + +Codex 只是锦上添花，不是唯一入口。 + +--- + +## 十四、最高级方案里的安全体系 + +正式版必须加入： + +### 1. RBAC + +区分： + +- 观察员 +- 运维员 +- 发布员 +- 超级管理员 + +### 2. 审批流 + +高风险动作必须审批： + +- 批量更新 +- controller 重启 +- 清理数据 +- 回滚生产版本 + +### 3. Policy Engine + +可以内置规则： + +- 正在跑检测的 worker 禁止强更 +- 大陆 controller 每次只能更新 1 台 +- `busy` 节点默认拒绝重启 + +### 4. Secrets 管理 + +不把所有 SSH 密钥和密码散落到脚本里。 + +正式版建议独立 secrets 管理，例如： + +- Vault +- 或最少做控制面统一加密存储 + +--- + +## 十五、这套方案为什么是“后续最少返工”的 + +因为它不是按“当前临时联调”设计，而是按“节点继续增长、功能继续增加、Codex 持续接管”设计。 + +它天然支持： + +- 节点从 3 台扩到 30 台 +- 从手工联调升级到一键部署 +- 从后台按钮升级到 Codex 自动驾驶 +- 从局部脚本升级到平台化运维 + +也就是说： + +> 这是终局骨架，不是过渡补丁。 + +--- + +## 十六、我给你的最终建议 + +如果按最高级、最优秀、后续最少反复修改的标准来做，路线应该明确成： + +1. 海外主机做唯一控制面 +2. 一切动作先建模成 `ops job` +3. 节点统一由 `node-agent` 执行 +4. 发布统一走 release 包，不走 git pull +5. 日志统一做事件流 + 任务日志 + 诊断包分层 +6. Codex 只通过控制面 API 接管，不绕过控制面 +7. SSH 只保留 bootstrap 与救援，不做日常主链路 + +这套方案里，`ops job` 就是整个系统的“运维订单”和“执行凭证”。 + +它不是附属品，而是整套平台的中心对象。 + +--- + +## 十七、终局不是“一份大文档”，而是 7 份冻结 contract + +真正能让这套体系后续少返工的，不只是理念对，而是： + +> 概念层有总架构，执行层有正式 contract。 + +最终至少要冻结 7 份正式 contract： + +### 1. `ops_job_contract` + +建议以后直接以 [ops_job_contract.md](/www/wwwroot/getDomain/docs/schemas/ops_job_contract.md:1) 为正式母本。 + +约束： + +- `ops job` 如何创建 +- 如何审批 +- 如何取消 +- 如何派发 +- 如何承诺 job / step / event 结构 + +它定义的是： + +> 一切正式运维动作的执行订单模型 + +### 2. `ops_agent_protocol` + +约束： + +- Node Agent 如何注册 +- 如何心跳 +- 如何拉任务 +- 如何回传事件 +- 如何回传交付队列快照 + +它定义的是： + +> 节点怎样成为“可托管执行器” + +### 3. `ops_driver_contract` + +约束： + +- `overview.driver_recommendations` +- `driver-feed` +- `codex-brief` +- `driver-actions.preview/resolve/execute` +- `activity-stream` + +它定义的是： + +> 海外控制面、CLI、Codex 驾驶员如何共享同一套判断与动作入口 + +### 4. `ops_playbook_contract` + +约束： + +- playbook catalog +- preview +- execute +- playbook run +- playbook events + +它定义的是： + +> 多步标准作业怎样从“按钮集合”升级成正式编排对象 + +### 5. `release_hub_contract` + +约束： + +- Release +- Rollout +- Launchpad +- Default Rollout Gate +- Default Rollout Execution + +它定义的是： + +> 发布到底能不能发、该怎么发、先灰哪一台、失败怎么停 + +### 6. `ops overview / inspection / delivery queue` 收口 contract + +这块现在虽然散落在 `overview`、`inspection`、`activity-stream`、`delivery-queue` 里，但终局一定要收成稳定协议。 + +建议以后直接以 [ops_observability_contract.md](/www/wwwroot/getDomain/docs/schemas/ops_observability_contract.md:1) 为正式母本。 + +至少要固定： + +- 执行现场分组： + - `dispatch_active_nodes` + - `recent_only_nodes` + - `standby_nodes` + - `load_syncing_nodes` +- 巡检收口： + - 最近 `health.snapshot` + - 最近 `logs.collect` + - 最近 `diagnostics.collect` +- 回执队列治理： + - 节点级摘要 + - `head_only` 头部记录 + - `flush / replay / discard` + +它定义的是： + +> 海外控制面如何真正看懂现场，而不是只看到几条状态字串 + +### 7. `ops_stack_diagnosis_contract` + +建议以后直接以 [ops_stack_diagnosis_contract.md](/www/wwwroot/getDomain/docs/schemas/ops_stack_diagnosis_contract.md:1) 为正式母本。 + +至少要固定： + +- `stack_status` +- `issues` +- `next_step` +- `recommended_actions` +- `quick_commands` +- `focus_ref` + +它定义的是： + +> 海外单脑控制面的第一现场总检入口，页面、CLI、Codex 都必须先从这里起步 + +所以总架构文档的职责不是把所有字段都写死，而是明确： + +- 哪 7 份 contract 才是后续实现的正式母本 +- 页面、CLI、Codex 都只能复用这些 contract +- 不允许再额外衍生第 8 套逻辑 + +并且这 7 份 contract 以后不该只存在于文档目录里，还要进入统一 registry： + +- `GET /api/v1/ops/contracts` +- `GET /api/v1/ops/contracts/{contract_key}` + +这样海外控制面、CLI、Codex 驾驶员才能按同一份 contract graph 导航，而不是继续靠人脑记“哪份协议在哪个 md 里”。 + +这一步非常关键，因为一旦没有 contract 冻结，你后面就会再次回到： + +- 页面自己猜 +- CLI 自己拼 +- Codex prompt 自己解释 +- 节点 agent 再走一套临时语义 + +那“单脑控制面”就又会退化成“多脑拼接系统”。 + +--- + +## 十八、单脑控制面的日常闭环应该固定成 6 段 + +你后面不应该再按“这次故障怎么处理”临时想流程，而应该把日常路径固定成下面 6 段。 + +### 1. 接管 + +- 签发 token +- 生成 bootstrap plan +- 节点注册 +- 节点心跳进入托管目录 + +这里的目标不是“节点能连上”，而是： + +> 节点成为正式受控资源 + +### 2. 验收 + +- 执行 `onboarding.acceptance` +- 必要时追加 `inspection.standard` +- 确认： + - service topology 正确 + - agent 在线 + - delivery queue 正常 + - 巡检结果可回看 + +这里的目标不是“接进来就算完”，而是： + +> 接入后立刻进入正式运维 + +### 3. 观察 + +- 看 execution scene +- 看 inspection overview +- 看 activity stream +- 看 delivery queue + +如果现场异常，优先动作应固定成： + +- 开关键日志回传 +- 跑标准巡检 +- 处理死信 / retrying + +而不是重新 SSH 上去抓一遍日志。 + +### 4. 发布 + +- 看 Release Launchpad +- 看 Default Rollout Gate +- 先 worker canary +- 再批次推进 +- 异常即停 +- 必要时回滚 + +这一步必须始终坚持： + +- 节点不 `git pull` +- 发布只认 release artifact +- rollout 只认正式 gate + +### 5. 故障 + +故障时优先路径应固定成： + +1. `driver recommendation` +2. `activity-stream` +3. `playbook run detail` +4. `delivery queue` +5. `inspection / diagnostics` + +也就是说，先走： + +> 结构化现场收口 + +再决定是否需要更重动作。 + +### 6. 救援 + +只有下面场景，才进入 SSH Rescue： + +- Node Agent 没注册成功 +- Node Agent 已失联 +- release 切换中断，当前目录损坏 +- systemd / Python 运行环境坏到 agent 无法自救 + +而且即使进入救援，也应该做到： + +- 仍生成正式 `ops job` +- 仍记录执行人、目标节点、动作来源 +- 仍把结果写回控制面审计链 + +这就是“单脑控制面”的关键边界： + +- 日常动作走标准 contract +- 标准动作走 agent / playbook / rollout +- 救援动作才进入 SSH + +这样以后新增机器、换地域、换发布流程，都只是替换某一层执行器，不需要重写整个体系。 diff --git a/docs/24_domainCheck_NodeAgent协议与ReleaseHub设计.md b/docs/24_domainCheck_NodeAgent协议与ReleaseHub设计.md new file mode 100644 index 0000000..1fab0fc --- /dev/null +++ b/docs/24_domainCheck_NodeAgent协议与ReleaseHub设计.md @@ -0,0 +1,4421 @@ +# 24 domainCheck Node Agent 协议与 Release Hub 设计 + +如果当前已经进入“把方案真正落地到上线前收口”的阶段，建议配套阅读： + +- `docs/25_domainCheck_海外单脑控制面上线收口总表.md` +- `docs/schemas/ops_driver_contract.md` +- `docs/schemas/ops_stack_diagnosis_contract.md` + +## 一、Node Agent 为什么是核心 + +终局架构里，大陆节点不应该再是“需要人工登录的机器”，而应该是： + +> 受控执行节点 + +每个节点通过 `domaincheck-node-agent` 接入海外控制面，承担： + +- 注册 +- 心跳 +- 拉取任务 +- 执行任务 +- 回传结果 +- 回传日志 +- 回传诊断包 + +这样后台按钮和 Codex 都不用再直接碰节点 shell。 + +补一个很关键的落地约束： + +- OpsCenter 顶部的“总检入口”现在应正式优先消费 `GET /api/v1/ops/go-live-summary` +- `GET /api/v1/ops/stack-diagnosis` + - 保留为更细的可解释总检层 + - 用于继续下钻 `issues / operator_decision / next_actions / recommended_commands` +- CLI 总检脚本也应优先消费同一个 contract +- 海外 Codex 驾驶员后续默认也先看这一份总检结果 + +也就是说，Node Agent、Release Hub、页面按钮、Codex 驾驶员，不再各自拼一套“当前现场判断”，而是统一先看： + +> go-live summary + +然后再根据 `stack-diagnosis / overview / release launchpad` 继续下钻。 + +这里还要再固定一个新口径，避免页面和运维动作继续混用“可联调”和“可发版”： + +- `go_live_status` + - 回答“当前是否已经进入统一收口、联调、复核车道” + - 允许出现 `attention` +- `publish_status / publish_ready` + - 回答“当前是否已经满足正式发版门禁” + - 必须明确区分 + - `blocked` + - `attention` + - `ready` + +这里再补一条已经收口成代码规则、后续不要再放松的判断： + +- 如果目标节点的 Node Agent `delivery_queue` 仍然存在 `dead_letter` + - 这不是普通 warning + - 对 `publish_status` 必须按 `blocked` 处理 + - 对 `default_rollout_gate` 也必须按阻断处理 +- 如果只是 `retrying` + - 可以继续保留为 `warning / attention` + - 但不应该再被当成“完全干净的发布现场” + +原因很简单： + +- `dead_letter` 说明控制面到节点的执行回执链路已经出现语义失败 +- 这时候页面如果还显示“可发布”，就会制造最危险的假阳性 +- 所以 `go_live_status` 可以是 `attention` +- 但 `publish_status` 必须更严格，直接阻断正式发版 + +这里还要补一条已经落成代码的事件语义，后续不要再退回“任务成功就算一切成功”的旧口径： + +- 如果 Node Agent 在执行 `jobs/{job_id}/start` 时，开始回执投递失败 + - 允许节点继续本地执行 + - 不允许因为这一次投递失败就直接中断真实任务 +- 但控制面必须把这类现场显性化为单独问题，而不是静默吞掉 + - activity stream 中对应 `ops_job` 应升级为 `attention` + - 保留原始 `job_status` + - 并暴露 + - `start_delivery_state` + - `start_delivery_error` + - `source_focus_ref` +- `ops overview / go-live summary` + - 也必须把它当成正式关注项 + - 不能只在活动流里可见、但首页总检仍显示“可直接上线” + - 当前口径应是 + - `go_live_status = attention` + - `publish_status = attention` + - 默认下一步指向 `focus_latest_job_events` +- `stack-diagnosis` + - 也必须把这类现场提升为可操作 issue + - 默认建议动作应该回到 `focus_latest_job_events` + +这里再固定一个动作层约束，避免后面 UI / CLI / Codex 又各自猜入口： + +- 如果活动焦点已经明确落到 + - `source_focus_ref.kind = ops_job_event` + - 或者问题本身就是“开始回执异常” +- 那么默认焦点动作不应继续使用泛化的 `focus_activity_item` +- 而应直接升级为 + - `focus_latest_job_events` + +原因是： + +- `focus_activity_item` 适合“先进入这条活动对应的详情” +- `focus_latest_job_events` 适合“问题已经明确在 job event 级别” +- 对开始回执异常这类问题，继续停在 activity 级别只会多一跳，降低定位效率 + +原因也很直接： + +- 这类问题不代表节点没执行 +- 但代表控制面和节点现场之间已经出现“开始态失联” +- 如果不显性化，页面会以为“任务没问题，只是还没刷新” +- 实际上这是自动化链路一致性已经开始打折的早期信号 + +也就是说： + +- “总检入口”不只告诉你系统是不是在健康收口 +- 还必须同时告诉你“现在能不能正式发布” + +后续页面、CLI、Codex、按钮动作，都应该优先消费这两个层级，而不是再各自推导一套“看起来差不多能上”的临时判断。 + +这里再补一个已经落地的收口原则： + +- `GET /api/v1/ops/stack-diagnosis` 现在不只返回传统的 + - `issues` + - `recommended_actions` + - `next_step` +- 还会正式返回 + - `diagnosis.operator_decision` + - `diagnosis.next_actions` + - `diagnosis.recommended_commands` + +这三层的定位要固定下来： + +- `operator_decision` + 先回答“当前主处理车道是什么” + 例如：`observability / node_handover / release / ops_jobs / steady` +- `next_actions` + 直接给出主动作和复核动作的下一跳命令 +- `recommended_commands` + 把节点现场日志、接管、Release Hub、driver-resolve 等常用 drill-down 统一收口成可复用命令集 + +这样页面顶部“当前主决策”卡、CLI 的 `check_ops_center_stack.sh`、海外 Codex 驾驶员，就都不该再各自重新推导“当前应该先看日志还是先接管还是先看发布门禁”，而是优先共享后端总检 contract。 + +这里再补一条已经收口成正式操作约束、后续不要再退回去的口径： + +- 如果总检已经识别出 + - `runtime_build_schema_stale` + - `runtime_route_surface_incomplete` + - `runtime_build_info_unavailable` +- 页面、CLI、Codex 驾驶员的第一反应不应该再是“给我一条 `systemctl restart domaincheck-api`” +- 而应该统一提升为标准恢复入口 + - `runtime-refresh-recover` + - `go-live-recover` + +这两个入口的语义要固定： + +- `runtime-refresh-recover` + - 回答“当前运行中的 API 是否还是旧 schema / 旧路由面” + - 给出固定恢复链 + - 再输出下一跳复检命令 +- `go-live-recover` + - 把 `runtime-refresh-recover` + - `stack-diagnosis` + - `stack-next` + - `go-live-check` + - `doctor` + 这条联调收口链直接串起来 + +也就是说： + +- 原子 systemd 重启仍然只是底层执行事实 +- 对页面按钮、CLI、Codex 来说，正式入口必须是组合恢复动作 +- 这样后续总检摘要、主决策卡、推荐命令集，才不会再次退化成“看见 schema 漂移就手工重启” + +再往前一层，现在还应该固定： + +- `GET /api/v1/ops/driver-feed` + - 返回 `go_live_summary` +- `GET /api/v1/ops/driver-feed` + - 继续返回 `automation_coverage` +- `GET /api/v1/ops/codex-brief` + - 返回 `go_live_summary` +- `GET /api/v1/ops/codex-brief` + - 继续返回 `automation_coverage` + +也就是说： + +- 页面顶部先看 `go-live-summary` +- Codex 驾驶员先看 `codex-brief.go_live_summary` +- CLI 总检先看 `go-live-check` +- 真正需要解释“为什么阻断 / 下一步为什么是这个动作”时，再回退到 `stack-diagnosis` + +同时还要固定一条“自动化覆盖率”口径，避免后续页面、CLI、Codex 又各自推导一套“现在已经自动化到什么程度”： + +- `driver_feed.automation_coverage` +- `codex_brief.automation_coverage` + +至少需要稳定包含： + +- `automation_level_counts` +- `recommendation_counts` +- `executor_kind_counts` +- `safe_auto_total` +- `guarded_auto_total` +- `mixed_total` +- `ui_only_total` +- `blocked_total` +- `preview_only_total` +- `backend_handled_total` +- `execution_ready_total` +- `human_dependency_total` +- `launch_status` +- `launch_ready` + +其中语义固定为： + +- `preview_only_total` + 统计那些已经后端化、但本质仍属于“只读预览 / 聚焦 / 审阅”的动作 +- `backend_handled_total` + 统计已经脱离人工页面点击、可以直接走后端 contract 的动作 +- `execution_ready_total` + 统计 `auto_execute + confirm_then_execute` +- `human_dependency_total` + 统计 `resolve_first + open_ui + blocked` +- `launch_status` + 作为“自动化层面的收口判断” + 取值固定为 `ready / attention / blocked` +- `launch_ready` + 只回答“从自动化执行角度看，当前是否已经接近无人值守可上线” + +这样才是完整的： + +- 一份稳定收口摘要 +- 一份可解释总检 +- 一组可执行下钻命令 + +--- + +## 二、当前已经落下的第一版协议骨架 + +当前后端已经补出这些接口骨架： + +- `POST /api/v1/ops/agent/tokens` +- `POST /api/v1/ops/agent/bootstrap-plan` +- `POST /api/v1/ops/agent/register` +- `POST /api/v1/ops/agent/heartbeat` +- `POST /api/v1/ops/agent/pull` +- `POST /api/v1/ops/agent/jobs/{job_id}/start` +- `POST /api/v1/ops/agent/jobs/{job_id}/complete` +- `POST /api/v1/ops/agent/jobs/{job_id}/events` + +这几类接口已经足够表达“节点是怎么接控制面的”。 + +当前仓库里也已经补出第一版 agent 运行骨架： + +- `domain-api/app/node_agent.py` +- `domain-api/deploy/systemd/domain-node-agent.service` +- `domain-api/deploy/multi-region/build_node_agent_bootstrap_plan.sh` +- `domain-api/deploy/multi-region/templates/domaincheck-node-agent.env.example` + +也就是说，这已经不是纯文档方案，而是开始具备正式执行器入口了。 + +另外现在控制面已经不只是“签发一枚 token”，而是可以直接生成完整接入方案： + +- 后端通过 `POST /api/v1/ops/agent/bootstrap-plan` +- 一次返回： + - token + - env 内容 + - 安装命令块 + - 启动检查命令 + +这一步很关键，因为它把“接管新节点”从零散知识点，收敛成了控制面里的标准对象。 + +控制面页面现在也已经补上了第一版“可视化接管态”： + +- `GET /api/v1/ops/nodes` + 不再只返回“已纳管节点”，也会把“集群可见但未纳管”的节点一起带回来 +- `GET /api/v1/ops/nodes/{node_code}/handover` + 可以直接拿到单节点接管详情、阻断项、下一步建议和 bootstrap 默认参数 +- `POST /api/v1/ops/nodes/{node_code}/handover/bootstrap-plan` + 可以按节点当前上下文直接生成新的 bootstrap 方案，而不是让页面或脚本自己再拼 `node_region / node_role` +- `GET /api/v1/ops/nodes/{node_code}/onboarding` + 现在除了返回 `onboarding_stage / acceptance / recommended_actions`，还应返回统一的 `recovery_decision` +- `POST /api/v1/ops/nodes/{node_code}/onboarding/recovery/preview` + 应作为 `node-recover` 的正式 preview 入口 +- `POST /api/v1/ops/nodes/{node_code}/onboarding/recovery/execute` + 应作为 `node-recover confirm` 的正式 execute 入口 +- 节点会被明确区分为： + - `已接管` + - `仅运行态在线` + - `待接入` + - `心跳过期` + - `Token 过期 / 已禁用` + +这意味着海外控制面第一次真正具备了“先看见，再接管”的统一入口，而不是只有接入完成后才能看见节点。 + +这里补一个非常关键的收口原则： + +- `recovery_decision` 必须由后端统一生成 +- 前端、CLI、Codex 驾驶员不再各自重新推导“下一步该 bootstrap 还是该 acceptance” +- 它至少要固定返回： + - `action` + - `label` + - `summary` + - `command_hint` + - `window` + - `stage_code` + - `acceptance_status_code` + +当前口径应固定为： + +- 节点仍处于 + - `pending_bootstrap` + - `runtime_only` + - `stale` + - `agent_pending` + - 以及其他接入中状态 + 时，优先收口为 `bootstrap_run` +- 节点真正进入 + - `acceptance_ready` + - 或 Agent 已在线 + 时，才切到 `run_acceptance` + +这样 `node-recover`、OpsCenter“自动收口建议”、海外 Codex 驾驶员三边看到的恢复结论才会完全一致。 + +这里还要补一条很关键的执行约束，避免页面、CLI、Codex 后面又各自补出“双发一次”的隐藏 bug： + +- `node-recover` + - preview 时优先命中 `onboarding/recovery/preview` + - confirm 时优先命中 `onboarding/recovery/execute` +- 如果控制面已经成功命中正式 `recovery/execute` + - 就表示后端已经完成“选择动作 + 发起执行” + - CLI / 页面 / Codex 不允许再额外补跑一次 + - `node-bootstrap-run` + - 或 `node-acceptance-run` +- 只有在 recovery endpoint 不存在、退回兼容视图时 + - CLI 才可以根据 `recovery_decision / stage_code / acceptance_status_code` + 再本地降级决定下一跳 + +也就是说，正式协议下： + +- 动作选择权在后端 recovery endpoint +- CLI 和页面只是展示恢复结论与执行回执 +- 兼容模式下才允许本地推断 + +这样才能避免后续接管链路出现： + +- preview 看的是一套 +- execute 走的是另一套 +- CLI 还再补跑一次底层动作 + +一旦出现这种双发结构，节点接管、验收、发布回执都会失真，这在上线期是不能接受的。 + +接管通过后，控制面也不应该只是停在“状态已通过”这一步，而是要立刻进入正式运维动作： + +- 单节点执行标准巡检 +- 批量对“真正参与检测节点”发起巡检 +- 批量对“在线未参与节点”发起巡检 +- 节点定向进入 Release / Rollout + +其中“标准巡检”已经统一成固定序列： + +- `health.snapshot` +- `logs.collect` +- `diagnostics.collect` + +后续无论是 Codex 驾驶员还是后台按钮，都应该优先复用这条标准序列，而不是再回到临时 shell 排障。 + +在控制面页面上，这些状态不应该只是文字摘要，还应该继续收口成“驾驶建议卡”： + +- 如果存在有效执行节点但尚未接管，优先进入接管动作 +- 如果存在参与检测节点但现场日志还不可见，优先开启关键日志回传 +- 如果存在真正参与检测节点，优先执行标准巡检 +- 如果存在在线未参与节点，优先排查接单/待命问题 +- 当接管和巡检链路稳定后，再推进 Release / Rollout + +这样 Node Agent 协议、Ops Job 模型和页面按钮才会是同一个体系，而不是三套彼此分裂的运维逻辑。 + +这里的驾驶动作命名也要尽量收敛成通用语义，而不是绑定页面分组： + +- `run_standard_inspection` +- `open_worker_logs` +- `open_diagnostics` + +这样同一套动作才能在： + +- 驾驶建议卡 +- 后台按钮 +- Codex 自动驾驶 +- 后续自动化编排 + +之间稳定复用。 + +再进一步，为了避免前端按钮、Codex 驾驶员、临时脚本各自维护一套“多步动作串联逻辑”，控制面里还需要再上一层： + +- `ops playbook` + +它的职责不是直接执行 shell，而是把一个高层运维意图展开为一组标准 `ops jobs`。 + +例如： + +- `inspection.standard` + - `health.snapshot` + - `logs.collect(worker)` + - `diagnostics.collect` +- `scene.logs.key` + - `logs.collect(worker, 120)` +- `scene.logs.full` + - `logs.collect(worker, 300)` + - `diagnostics.collect(300)` + +这样： + +- 后端推荐的是 playbook +- 前端按钮点的是 playbook +- Codex 驾驶员调用的也是 playbook +- 真正落库和回执的仍然是 `ops jobs` + +整个模型会更稳定，也更适合后面持续扩机器。 + +并且这层已经开始真正进入页面主流程： + +- OpsCenter 已能列出 playbook +- OpsCenter 已能预览 playbook 展开结果 +- “标准巡检”已优先直接调用 `inspection.standard` +- “接管后一键验收”已优先直接调用 `onboarding.acceptance` + +也就是说，控制面 UI、Codex 驾驶员、后端编排这三层终于开始共用同一套多步动作模型。 + +同样，OpsCenter 不应该只会发任务，还要能直接承接执行现场： + +- 把 `participating_nodes` 和 `non_participating_nodes` 一起并入运维视图 +- 把 `log_sync.enabled / mode / source_nodes / last_line` 一起并入运维视图 +- 并允许控制面直接切换远端日志回传模式 + +这样 Codex 驾驶员和人工按钮都会在同一页看到： + +- 谁正在跑 +- 谁在线但没跑 +- 日志有没有回来 +- 要不要临时切到全量回传 + +并且这层不能只停留在页面按钮，还要提供固定 drill-down 命令： + +- `bash domain-api/deploy/multi-region/drive_ops_center.sh scene-node-log [limit] [key|full]` +- `bash domain-api/deploy/multi-region/check_node_scene_log.sh [limit] [key|full]` + +这样海外控制面、CLI 和 Codex 驾驶员在看到“参与检测但缺样本”时，不需要再人工猜下一步看哪台机器，而是直接按节点下钻现场日志。 + +这里再补一个已经落地的恢复流原则： + +- 当 `stack-diagnosis` / `ops overview` 判断为 + - `remote_log_sync_disabled` + - 或“已有参与节点但现场日志仍不可见” +- CLI、页面按钮、Codex 驾驶员都不应该再只给“去看日志”这种观察型建议 +- 而应该优先给出： + +> `log-sync-recover` + +也就是： + +- 先执行 `enable_log_sync_key` 或 `enable_log_sync_full` +- 再自动复核 `execution_scene.log_sync` +- 再自动给出下一跳 drill-down + +当前海外单入口已经有这一条组合命令： + +```bash +bash domain-api/deploy/multi-region/drive_ops_center.sh log-sync-recover +bash domain-api/deploy/multi-region/drive_ops_center.sh log-sync-recover http://127.0.0.1:8100 full confirm cli/ops +``` + +它不是“额外的便捷脚本”，而是终局驾驶 contract 的一个关键落地点： + +- 页面按钮可直接复用 +- Codex 驾驶员可直接复用 +- `stack-diagnosis / doctor / stack-next` 也应该优先推荐它 + +这样“恢复可见性”就不再是人工脑补的一串步骤，而是正式的一跳动作。 + +Node Agent 接管链路也应采用同样原则： + +- 当 `stack-diagnosis` 判断主车道已经进入 `node_handover` +- 或发现 `managed_nodes_agent_pending` +- 不应该只给出零散的 + - `node-handover` + - `node-bootstrap-plan` + - `doctor` +- 而应该优先给出一条组合恢复流： + +> `agent-gap-recover` + +它的目标不是“替代真正的节点落地”，而是把海外控制面需要做的这几步先标准化： + +1. 识别首个接管缺口节点 +2. 输出该节点当前 handover 状态 +3. 生成该节点 bootstrap plan +4. 给出后续检查命令 + +当前海外单入口已经支持： + +```bash +bash domain-api/deploy/multi-region/drive_ops_center.sh agent-gap-check +bash domain-api/deploy/multi-region/drive_ops_center.sh agent-gap-recover +bash domain-api/deploy/multi-region/drive_ops_center.sh agent-gap-export +``` + +而且这条链路也已经具备版本兼容能力： + +- 如果当前控制面 API 还没有节点级 `/handover` + - 退回 `stack-diagnosis` +- 如果当前控制面 API 还没有节点级 `/handover/bootstrap-plan` + - 退回通用 `/ops/agent/bootstrap-plan` + +这样“节点接管”就不再只是页面里的一个弹窗动作，而是正式进入海外单入口、CLI 总检、Codex 驾驶员都可复用的统一恢复模型。 + +其中： + +- `agent-gap-recover` + - 负责把首个缺口节点的状态、bootstrap plan、下一跳建议串成一条恢复流 +- `agent-gap-export` + - 负责把首个缺口节点的接管材料直接导出成目录 + - 默认应至少包含： + - `stack-first-gap-handover` + - `agent-gap-check` + - `node-onboarding` + - `node-bootstrap-plan` + - `node-bootstrap-preview` + - `node-acceptance-plan` + - `manifest.json` + +这样海外控制面就不只是“告诉你下一步做什么”，而是可以直接产出一份可执行、可交接、可归档的节点接管包。 + +另外，`node-bootstrap-plan` 的 condensed summary 还应该主动区分两类情况： + +- 仓库代码本身还不支持最新 bootstrap 字段 +- 仓库代码已经支持，但运行中的 API 还没重启到最新版本 + +后一类在联调里非常常见，所以 summary 必须直接给出： + +- `repo_capability.supports_install_command_block` +- `repo_capability.runtime_may_need_restart` +- `recommended_commands.restart_api_if_runtime_stale` + +这样一眼就能判断，这次缺口到底是“代码没写到位”，还是“代码已经在仓库里，但线上进程还是旧的”。 + +这个判断还必须继续上提到总检 contract： + +- 一旦 `stack-diagnosis` 识别出 `runtime_build_schema_stale` +- 默认 `next_step.action_code` + - 必须优先收敛为 `api-restart` +- `diagnosis.operator_decision.lane` + - 必须优先收敛为 `runtime_recovery` + +因为这类问题的本质不是“节点还没接管”，而是“控制面 API 还是旧 schema / 旧路由面”。 + +如果这里不先做 API 重启，而是继续把人带去执行 `bootstrap_run / fix_managed_nodes` + +- 只会让真正问题继续被掩盖 +- 页面、CLI、Codex 也会一起被带偏 + +另外，bootstrap 计划和安装脚本都必须兼容多种部署目录布局： + +- `${ROOT_DIR}/domainCheck/...` +- `${ROOT_DIR}/domain-api/...` +- `${ROOT_DIR}/deploy/...` + +原因很现实： + +- 开发仓库里脚本位于 `domain-api/deploy/...` +- 实际上线包可能已经把 API 内容平铺到 `domainCheck/...` + +如果 bootstrap 计划只写死其中一种路径，就会在“控制面判定缺口正确，但落地脚本直接找不到 install script”这个位置翻车。 + +同样，这个 drill-down 也应该进入 `doctor-export`： + +- 当 `stack-diagnosis` 已经给出 `node_scene_log` 焦点 +- 导出包自动追加 `15_scene_node_log_.txt` +- `manifest.json.summary.scene_log_reports` 也同步记录这些节点级现场日志附件 + +并且这里的焦点来源现在也应该统一成后端总检 contract： + +- `diagnosis.issues[*].focus_ref` +- `diagnosis.next_step.focus_ref` +- `diagnosis.operator_decision.next_focus` + +只要其中任一层已经明确给出 `node_scene_log`，诊断包就应自动把该节点现场样本带上，而不是再由导出脚本自行重新推导“哪台机器值得看”。 + +这样海外 Codex、后台自动驾驶和人工交接看到的不是一句“建议去看某台节点”，而是一份已经带着关键现场样本的诊断包。 + +再往前一步，驾驶建议本身也不应只存在于前端页面，而应成为 ops overview contract 的一部分： + +- 后端返回 `driver_recommendations` +- 每条建议附带 `action_code` +- 前端只做动作映射，不再重新计算优先级 + +继续往下，`driver_recommendations` 也不能只给“同一组 node_codes + 同一份 payload”。 + +因为真实场景里经常会出现： + +- 主动作是 `创建 Rollout` +- 次动作是 `打开点状灰度模板` +- 两个动作对应的目标节点、release_id、预填参数并不相同 + +所以 contract 还要支持： + +- `primary_node_codes` +- `secondary_node_codes` +- `primary_action_payload` +- `secondary_action_payload` + +这样海外控制面、Codex 驾驶员、CLI 推荐执行器才能真正做到“同一张建议卡，主次动作各走各的参数”，而不是被迫回退成前端手写分支。 + +这样未来换成 Codex 自动驾驶或脚本编排时，也能直接复用同一套推荐逻辑。 + +并且这套 contract 不该只停在“给建议”，而要继续补到“统一执行入口”： + +- `POST /api/v1/ops/driver-actions/preview` +- `POST /api/v1/ops/driver-actions/execute` +- 能后端化的动作先统一走这里 +- 仍需人工选择、弹窗补参数的动作，再回退到前端交互 + +其中 `preview` 这层尤其关键，因为它负责把： + +- execution chain +- target api / method +- 主次动作 request payload +- runbook resolve 结果 + +统一冻结成后端 contract。 + +这样页面、CLI 和海外 Codex 驾驶员都不需要自己重新拼 request preview，也不会再出现“页面自己猜 sequence -> action 映射”的口径漂移。 + +现在这条链路继续往下已经开始接入： + +- `GET /api/v1/ops/playbooks` +- `GET /api/v1/ops/playbook-runs` +- `GET /api/v1/ops/playbook-runs/{run_code}` +- `GET /api/v1/ops/playbook-runs/{run_code}/events` +- `GET /api/v1/ops/activity-stream` +- `POST /api/v1/ops/playbook-runs/{run_code}/rerun` +- `POST /api/v1/ops/playbook-runs/{run_code}/cancel` +- `POST /api/v1/ops/playbooks/preview` +- `POST /api/v1/ops/playbooks/execute` + +也就是说，驾驶动作的后端执行入口不再只是“直接发一个动作模板”，而是可以把多步标准动作整体展开。 + +这样整条链路才是： + +- 后端推荐 +- 后端标准动作执行 +- 前端只做展示和少量交互兜底 + +再进一步，控制面还要具备“巡检结果收口视图”： + +- 一个节点最近的 `health.snapshot` +- 一个节点最近的 `logs.collect` +- 一个节点最近的 `diagnostics.collect` + +要能在页面里按节点重新聚合，而不是让操作者自己翻几十条 `ops job` 去拼结果。 + +这一步很关键，因为它决定了： + +- Codex 驾驶员读取的是“节点收口状态” +- 而不是一堆离散任务记录 + +现在这层也不该继续停留在前端临时聚合，而应该进入正式后端协议： + +- `GET /api/v1/ops/overview` 返回 `inspection` +- `GET /api/v1/ops/inspection-overview` 返回独立巡检收口视图 +- 页面只负责展示，收口排序、异常优先级、Worker 日志口径都以后端为准 + +并且这层 contract 不能只停留在“返回一句总结”，还要正式结构化： + +- `problem_kind` +- `problem_label` +- `problem_level` +- `recommended_action_code` +- `ui_intent` + +这样页面筛选、Codex 自动驾驶、后续策略编排才不需要再从自然语言里反推“这是失败、执行中还是巡检缺口”。 + +同理，动作模板 contract 也不能继续停留在“只有 text / number / select”： + +- `boolean` +- `textarea` +- `text_list` + +这三类字段是运维动作真正高频的表达方式： + +- `boolean` 用来表达是否自动审批、是否自动回滚、是否切 current +- `textarea` 用来承载 artifact URL、长文本备注、临时参数块 +- `text_list` 用来承载服务列表、健康检查 URL 列表、节点列表 + +否则发布、回滚、批量诊断这些动作永远只能靠前端写死表单，无法真正回收到统一模板层。 + +尤其 `deploy.release` 这类动作，应该正式进入模板目录，而不是只存在于 rollout 内部： + +- `deploy.release.control` +- `deploy.release.worker` +- `deploy.release.custom` + +这样单节点灰度验证、点状修复、正式 rollout 之前的预演，才能统一落到同一个 `ops job` 体系里。 + +模板动作再往前还要补一层正式能力： + +- `action template preview` +- `batch policy preview` + +也就是在真正创建 `ops jobs` 之前，控制面先基于： + +- 目标节点集合 +- 当前 execution mode +- 动作 payload +- 集群参与态 / 忙碌态 / control / effective worker 身份 + +给出统一预检结论： + +- `blocked` +- `approval_required` +- `warnings` +- `recommendations` + +这样发布、停 Worker、重启 API 这些高风险动作，就不会变成“点了按钮才知道会出事”，而是先看到： + +- 哪些节点被阻断 +- 哪些节点建议审批 +- 哪些只是需要关注 + +这层能力既服务于页面按钮，也服务于后续的海外 Codex 驾驶员，因为两边都应该调用同一份 policy preview contract，而不是各自写一套风险判断。 + +在 playbook 这一层，还要继续补一条非常关键的 contract： + +- 一次 playbook 执行 = 一次 `playbook run` +- 每个 run 都要有自己的 `run_code` +- 每个子任务都写入统一的 playbook metadata +- 控制面能够把这些子任务重新聚合成： + - run 总状态 + - step 状态 + - 目标节点集合 + - 最近可追事件的任务入口 + +这一步的意义是把“多步动作”从前端临时串联，正式升级成可审计、可追溯、可回放的运维对象。 + +继续补齐 run 级动作之后，控制面就可以直接： + +- 对 playbook run 做状态筛选 +- 按分组收窄到 onboarding / diagnostics / scene +- 按 run_code、创建来源、目标节点做关键词检索 +- 直接查看某轮 run 的 focus summary +- 直接看到 problem steps / active steps + +也就是说，控制面针对 playbook run 的 contract 已经不再只是“能聚合”，而是进一步升级成“能定位焦点、能快速追问题”。 + +再往前一步，playbook run 还必须具备“整轮事件视角”： + +- 不是只查某一条 job 的事件 +- 而是能把同一 `run_code` 下面所有 job 的事件重新聚合 +- 并补齐： + - `step_key` + - `step_title` + - `job_code` + - `job_status` + - `target_node_code` + +这样海外控制面的详情抽屉才能真正像“远端驾驶舱”，因为它看到的是整轮执行过程，而不是零散子任务的碎片。 + +现在这层 contract 还要进一步冻结成“前端不再自行翻译”的正式字段： + +- `playbook run` + - `status_label` + - `summary` + - `summary_text` + - `steps_total` + - `steps_success` + - `steps_running` + - `steps_problem` + - `steps_terminal` + - `focus_ref` +- `playbook run.steps[]` + - `status_label` + - `summary` + - `summary_text` + - `target_node_codes` + - `focus_ref` +- `playbook run events` + - `level_label` + - `job_status_label` + - `summary` + - `summary_text` + - `event_key` + - `occurred_at` + - `focus_ref` + - `source_focus_ref` +- `playbook run events.summary` + - `returned_total` + - `event_type_counts` + - `job_status_counts` + - `filters.limit` + +也就是说，从这一步开始： + +- 前端不再自己把 `running` 翻译成“收口中” +- 前端不再自己拼“当前焦点摘要” +- 前端也不再把“播放焦点”和“远端执行焦点”混成一个对象 + +这里还要再补一条非常重要的消费规则： + +- `playbook run events.focus_ref` + - 始终用于定位到本轮 playbook run 的正式落点 +- `playbook run events.source_focus_ref` + - 始终保留远端 Agent / ops job 原始焦点 + - 允许携带： + - `job_id` + - `job_code` + - `event_key` + - `step_key` + - 其他执行侧专有定位字段 + +这样控制面详情抽屉、Codex 驾驶员和后续 CLI 就不会再遇到一个老问题： + +- 页面需要跳到“这一轮 playbook 的哪个步骤” +- 但排障又需要知道“远端 Agent 具体卡在了哪条事件” + +这两种焦点必须并存，而不能互相覆盖。 +- 前端不再自己推断“应该跳到哪一轮 / 哪一步 / 哪条事件” + +而是统一消费控制面已经产出的结构化别名字段。 + +同样，驾驶建议层也开始和 playbook run 正式打通： + +- recommendation 不再只推荐“去看哪个节点” +- 而是可以直接推荐“先处理哪轮异常编排” +- 并携带： + - `run_code` + - `focus_step_key` + - `focus_step_title` + +这样 recommendation -> playbook run detail -> playbook run events 才形成一条真正闭环的驾驶链路。 + +继续往前收一层，recommendation 也不该只看 playbook run，而要继续消费统一 activity stream： + +- 如果最近存在失败 / 阻断 / 已取消的 standalone `ops job` +- 或存在 `awaiting_approval`、`halted`、`completed_with_issues` 的 rollout +- recommendation 应直接把这条 activity 提升成驾驶建议 +- 并通过统一 `ui_intent` 把前端直接送到： + - `job_events` + - `rollout_jobs` + +这样 recommendation 才真正具备“先聚焦异常活动，再决定是否补动作”的现场驾驶能力。 + +再继续往下一层，控制面还需要统一的“最近活动入口”： + +- 不只是 playbook run +- 也包括 standalone ops jobs +- 以及 rollout 推进记录 + +所以活动流 contract 也应该成为正式后端协议： + +- `GET /api/v1/ops/activity-stream` +- 每条 activity 带： + - `kind` + - `activity_key` + - `status` + - `status_label` + - `occurred_at` + - `summary` + - `summary_text` + - `target_node_codes` + - `ui_intent` + - `focus_ref` + +其中 `ui_intent` 不是让前端自己猜跳哪，而是由后端直接声明： + +- `playbook_run_detail` +- `job_events` +- `rollout_jobs` + +这样后端不仅能告诉前端“这里有一条异常活动”，还能直接告诉它“应该落到哪一个处理界面”。 + +这样海外控制面、后台按钮和 Codex 驾驶员才能共享一套“最近活动 -> 正确落点”的跳转 contract。 + +并且这层不要再停在“页面展示友好字段”，而要正式成为统一驾驶对象： + +- `status_label` + - 直接给页面、CLI、Codex 看的人类可读状态 +- `summary_text` + - 直接给驾驶建议和摘要面板消费的短摘要 +- `target_node_codes` + - 不让调用方再从自然语言里猜目标节点 +- `focus_ref` + - 让页面、CLI、Codex 都能用同一份定位对象跳到： + - `playbook run` + - `ops job` + - `rollout` + - `execution scene` + +这样后续不管是海外 Codex 驾驶员、CLI 还是后台按钮，都能把 activity 当成正式 contract，而不是“页面 table 的一行数据”。 + +同样，`activity-stream` 里的 `ops_job` 项也要冻结一条优先级规则： + +- 如果 job 本身已经带 `result.summary_text` + - activity 的 `summary_text` 优先使用它 +- 如果 job result 里已经带 `focus_ref` + - activity 的 `focus_ref` 优先吸收它 +- 只有在没有明确执行结果摘要时 + - 才回退到“目标节点 / 发起人 / 执行方式”这类模板化描述 + +原因很简单： + +- 模板摘要适合“任务刚创建” +- 结果摘要才适合“任务已经执行过，且远端已经给出结论” + +否则页面上会长期出现这种低信噪比口径： + +- `目标节点 xxx / 发起 web-ui / 方式 远端 Agent` + +但真正更有价值的应该是： + +- `远端 Agent 已回传 120 行 Worker 日志，可直接继续分析` + +这一点对海外 Codex 驾驶员尤其关键，因为它读取 activity-feed 时，优先需要的是“执行结论”，而不是“模板元信息”。 + +这条规则继续往下也要落到 Codex brief： + +- `codex-brief.entries` + - 仍然是可执行主线 +- `codex-brief.activity_focus` + - 则是可观察焦点 +- `codex-brief.scene_log_observation` + - 则是节点级现场日志观察摘要 +- 如果当前没有动作主线，但已有观察焦点 + - `codex-brief.focus` 允许回退到首条 `activity_focus` + - 但推荐行为只能是 `open_ui` + - 不允许伪装成 `auto_execute` +- 如果当前没有动作主线、也没有合适的 activity focus，但现场日志已经识别出明确目标节点 + - `codex-brief.focus` 允许继续回退到 `scene_log_observation.focus_ref` + - 这样海外 Codex 会直接落到 `node_scene_log` + +这样海外 Codex 在“现场正在变化，但暂时没有标准动作要打”的时候，仍然能稳定给出： + +- 先盯哪一条 +- 回到哪个正式落点 +- 为什么现在是观察优先而不是执行优先 + +接下来 driver-feed / runbook 这一层也必须正式冻结： + +- `driver_feed.entries[*]` + - `focus_ref` + - `primary_focus_ref` + - `secondary_focus_ref` +- `runbook_sequences[*]` + - `focus_ref` + - `primary_focus_ref` + - `secondary_focus_ref` + +也就是说： + +- driver-feed 不再只返回“动作码 + 按钮文案” +- runbook 也不再只返回“主次动作” +- 而是必须同时告诉页面、CLI、Codex： + - 这条主线整体应该聚焦哪里 + - 主动作对应哪个正式落点 + - 次动作对应哪个正式落点 + +这样后端才能真正消灭“前端根据 action_code 猜页面落点”的分叉逻辑。 + +这一层现在还要继续补一条非常实用的分流规则： + +- `driver-feed.entries` + - 仍然表示“驾驶动作主线” +- `driver-feed.activity_focus` + - 单独表示“观察与跟踪焦点” + - 不再把观察项硬塞成一条可执行建议 +- `driver-feed.scene_log_observation` + - 单独表示“当前远端日志回传是否已经形成可下钻的节点级现场日志” + +也就是说以后海外控制面和 Codex 同时会拿到两类对象： + +- 一类是可以继续 preview / resolve / execute 的动作主线 +- 一类是当前值得优先盯住的现场对象 + - 例如： + - 正在跑的 `ops job` + - 未收口的 `playbook run` + - 当前 `execution scene` + - 远端 `log sync` + +这样不会再出现一个老问题： + +- 页面把“值得先看”误渲染成“可以直接执行” +- Codex 又把“先看现场”误读成“可以自动发动作” + +同样的冻结规则还要继续往下压到驾驶执行链： + +- `driver-actions/preview` + - `focus_ref` + - `primary_focus_ref` + - `secondary_focus_ref` + - `primary.focus_ref` + - `secondary.focus_ref` +- `driver-actions/resolve` + - `preview_request.focus_ref` + - `preview_request.primary_focus_ref` + - `preview_request.secondary_focus_ref` + - `gate.focus_ref` + - 顶层 `focus_ref` +- `codex-actions/resolve` + - `selected_entry.focus_ref` + - `preview_request.focus_ref` + - `gate.focus_ref` + - 顶层 `focus_ref` + +只有这样页面、CLI、海外 Codex 驾驶员在 preview / resolve / execute 三段链路里，才不会重新猜“当前应该回看哪个 release、哪个 rollout、哪个 execution scene”。这也是后续 NodeAgent、ReleaseHub、OpsCenter 可以共享同一份驾驶协议的关键。 + +进一步说，`activity-stream` 不只是最近事件列表，还应该是正式可筛选的观察面： + +- `kind` +- `status` +- `query` + +这些过滤条件必须后端支持，而不是只在前端本地过滤。这样未来： + +- 海外 Codex 驾驶员 +- CLI 检查脚本 +- 后台页面 + +三者都可以复用同一套活动流收口逻辑。 + +继续升级之后，`driver action` 的协议也不该停在“一个 action_code”： + +- 应返回： + - `handled` + - `mode` + - `ui_intent` +- 其中 `ui_intent` 用来表达： + - 应打开哪个详情页 + - 应聚焦哪一轮 playbook run + - 应进入哪个发布入口 + - 应打开哪个节点接管入口 + +这一步的价值很大，因为它意味着： + +- recommendation 的解释权回到后端 +- 前端变成“执行/跳转承载层” +- 后续如果换成海外 Codex 驾驶员，仍然可以直接复用同一份 intent contract + +- 对某一轮编排做统一重跑 +- 对某一轮编排做统一取消 +- 不需要再由前端把同一批子任务逐条循环处理 + +这正是“海外控制面统一驾驶，大陆节点被托管执行”的关键基础。 + +同理，execution scene contract 也不该只是“参与节点数组 + 待命节点数组”，而要继续收成更明确的运行口径： + +- `dispatch_active_nodes` +- `recent_only_nodes` +- `standby_nodes` +- `load_syncing_nodes` + +以及日志回传本身的正式状态： + +- `disabled` +- `waiting_sample` +- `partial_coverage` +- `healthy` +- `full_capture` + +并且要直接返回： + +- 覆盖了多少参与节点 +- 哪些参与节点还没回传样本 +- 当前推荐动作文案 + +这样页面、Codex 驾驶员和后续自动化脚本都能共享同一套“现场观察 -> 判断缺口 -> 执行动作”的协议，而不是各自再做一次前端猜测。 + +这样未来新增： + +- Codex 自动巡检 +- 后台按钮巡检 +- 节点接管后的首轮验收 +- 发布后的批次验收 + +都能直接复用同一个 inspection contract。 + +在收口视图之上，还应该继续有“异常节点优先队列”： + +- 先处理失败 / 阻断 / 取消 +- 再处理仍在执行中的节点 +- 再处理尚未形成完整巡检记录的节点 + +另外标准巡检里的 `logs.collect` 应明确限定为 Worker 日志收口，不应把 Node Agent 的辅助排障日志混成同一类结果。 + +当前第一版 agent 执行器已经覆盖： + +- `service.start` +- `service.stop` +- `service.restart` +- `service.status` +- `health.snapshot` +- `logs.collect` +- `diagnostics.collect` +- `deploy.release` + +后续继续增强时，只需要在这条主链路上加能力，不需要重换架构。 + +--- + +## 三、Agent 标准执行流程 + +### 1. 签发 token + +控制面给节点签发一枚 agent token。 + +### 2. 节点注册 + +节点启动后调用： + +- `register` + +上报： + +- `node_code` +- `region` +- `role` +- `agent_version` +- `capabilities` +- `labels` +- `hostname` +- `ip` + +### 3. 节点心跳 + +节点周期调用： + +- `heartbeat` + +控制面更新： + +- `last_seen_at` +- agent 元数据 +- 节点在线状态依据 + +### 4. 节点拉任务 + +节点调用： + +- `pull` + +控制面返回分配给该节点的 `ops job`。 + +### 5. 节点开工 + +节点拿到 job 后先调用： + +- `jobs/{id}/start` + +这一步不是可有可无，而是为了把“任务已派发”和“任务已真实开始执行”区分开。 + +### 6. 节点完工 + +执行结束后调用： + +- `jobs/{id}/complete` + +上报： + +- `success / failed / partially_succeeded` +- `stdout` +- `stderr` +- `result json` + +### 7. 节点事件流 + +执行过程中可持续调用： + +- `jobs/{id}/events` + +把中间事件推回控制面。 + +--- + +## 四、为什么要有 Release Hub + +因为正式环境不应该继续依赖： + +- 节点自己 `git pull` +- 工作区状态不确定 +- root/www 用户混跑 + +正式版必须把“版本”变成平台中的一级对象。 + +也就是： + +> 先有 Release，再有 Deploy + +--- + +## 五、当前已经落下的 Release Hub 骨架 + +当前后端已经补出： + +- `POST /api/v1/ops/releases` +- `GET /api/v1/ops/releases` +- `GET /api/v1/ops/releases/latest` +- `GET /api/v1/ops/releases/{id}` +- `POST /api/v1/ops/releases/{id}/activate` +- `POST /api/v1/ops/releases/{id}/rollouts` +- `GET /api/v1/ops/rollouts/{id}` +- `GET /api/v1/ops/rollouts/{id}/jobs` +- `POST /api/v1/ops/rollouts/{id}/advance` + +并落了两类核心对象： + +- `ops_releases` +- `ops_release_rollouts` + +这意味着后面发布系统可以正式围绕： + +- 版本号 +- 渠道 +- commit sha +- artifact url +- checksum +- 激活版本 + +来运作。 + +但是 Release Hub 还不能只回答“有没有版本”，还要回答： + +> 现在这版到底能不能发 + +因此控制面里还需要一个正式门禁对象： + +- `release_hub.default_rollout_gate` + +它至少要返回： + +- 默认 Release +- 默认 Rollout 目标节点 +- 门禁状态： + - `missing_release` + - `release_not_ready` + - `artifact_missing` + - `no_targets` + - `blocked` + - `attention` + - `ready` +- 阻断原因 +- 告警原因 +- 推荐动作 +- `remote_agent_ready_nodes / nodes_total` +- `inspection_healthy_nodes / nodes_total` + +这样首页驾驶建议、Release Hub 顶部摘要、Codex 自动驾驶、真正发 Rollout 前的 preview，才能统一读同一份门禁判断。 + +--- + +## 六、现在这版 rollout 已经不只是“建记录” + +当前 rollout 已经补上三类关键能力： + +- 分批推进 +- rollout 与 ops job 关联 +- 执行结果自动反推 rollout 状态 + +也就是说，控制面不再是一次性给所有节点平铺发任务，而是可以表达： + +- 第一批先发 `1` 台 canary +- 健康后再发后续批次 +- 某批失败则暂停 +- 人工确认后继续下一批 + +当前 rollout 状态会根据 job 结果自动进入： + +- `planned` +- `running` +- `awaiting_approval` +- `ready_for_next_batch` +- `halted` +- `completed` +- `completed_with_issues` + +这已经非常接近正式发布系统的骨架，而不是临时脚本。 + +--- + +## 七、终局版发布链路 + +推荐的正式发布链路： + +1. 海外控制面构建 release +2. 写入 `ops_releases` +3. 标记某个 channel 的 active release +4. 后台点击“部署 release” +5. 生成 `ops job` +6. node-agent 拉取任务 +7. 下载 artifact +8. 校验 checksum +9. 切换版本 +10. 重启服务 +11. 健康检查 +12. 成功或回滚 + +当前后端也已经补出： + +- Release 对象 +- Release 激活 +- Rollout 对象 +- `release -> rollout -> deploy jobs` 分批生成 +- rollout 推进接口 +- 节点执行结果回写后自动刷新 rollout 总结 + +这意味着后续真正接发布包仓库时，只需要把 artifact 构建和分发接入，不需要再重新设计控制面对象模型。 + +--- + +## 八、Node Agent 侧的发布保险丝 + +当前 `deploy.release` 执行链路已经补上： + +- 下载 artifact +- checksum 校验 +- 解压到 `releases/` +- 切换 `current` 软链 +- 重启指定服务 +- 执行健康检查 +- 健康失败时回滚到旧版本 + +健康检查目前支持： + +- `systemctl is-active` +- HTTP URL 探测 + +并且 agent 会把关键事件回推控制面，例如： + +- `deploy_download_started` +- `deploy_checksum_verified` +- `deploy_current_switched` +- `deploy_health_failed` +- `deploy_rollback_completed` + +这套事件流很重要，因为后面后台和 Codex 都是靠这条链路看懂“节点到底做到了哪一步”。 + +所以事件 contract 也要继续冻结，而不能只返回一行原始 message： + +- 每条 job event 至少应带： + - `event_key` + - `event_type` + - `level` + - `level_label` + - `summary` + - `occurred_at` + - `ui_intent` +- 任务事件接口本身还要有 `summary`： + - `returned_total` + - `level_counts` + - `event_type_counts` + - `node_counts` + - `latest_at` + +这样海外控制面打开“任务事件流”时，先看到的是结构化执行回放，而不是先读一大串原始日志文本。 + +--- + +## 九、为什么这套设计后续最省心 + +因为它把三件事情分清楚了： + +- `release` 是版本对象 +- `ops job` 是动作对象 +- `node agent` 是执行对象 + +后面不管节点从 `3` 台增加到 `30` 台，还是 Codex 接管更多动作，这三层都不用推翻。 + +--- + +## 十、接管不是终点，要有“验收后直接进入正式运维”的桥 + +实际落地里最容易断掉的一环，不是 token，也不是 heartbeat，而是： + +> 节点明明已经接进来了，但接下来该做什么，操作员还得自己切页面、自己填 node_code、自己判断先巡检还是先发布。 + +这会把接管成功后的效率重新打回人工模式。 + +所以控制面里要把 Node Agent 接管链路拆成三段： + +### 1. 接入观察 + +用于回答： + +- token 是否有效 +- node agent 是否已经 register +- heartbeat 是否稳定 +- 当前是 `未纳管 / 待接入 / 已接管 / 心跳过期` + +### 2. 标准验收 + +接管完成后，不应该直接默认“节点已经可用”，而是先跑一轮固定验收动作： + +1. `health.snapshot` +2. `service.status(domaincheck-node-agent)` +3. `logs.collect(domaincheck-node-agent)` +4. `service.status(domaincheck-worker)` +5. `logs.collect(domaincheck-worker)` + +这轮验收的意义不是“多做几步”，而是把最常见的接管隐患一次性暴露出来： + +- 服务虽然启动了，但 register 没成功 +- heartbeat 有，但 pull / complete 异常 +- worker 在线，但 Redis / DB / proxy 配置不对 +- systemd 看似 active，但日志里已经反复报错 + +### 3. 正式运维入口 + +如果标准验收通过，控制面不能只显示一个“已通过”标签，而要直接给出两类后续入口： + +- `进入巡检模式` + 默认锁定当前节点，直接预填 `diagnostics.collect` +- `进入发布模式` + 默认锁定当前节点，直接预填目标 release 和单节点 rollout + +这样节点接管就形成了完整闭环： + +> 看见节点 -> 纳管 -> 签发接入方案 -> 接入观察 -> 标准验收 -> 巡检 / 发布 + +而不是“接进来之后又回到人工记忆和手填”。 + +当前前端已经按这个思路落了一版： + +- 接管弹窗里增加“接管验收结论” +- 可一键跑标准验收 +- 验收通过后，直接出现： + - `进入巡检模式` + - `进入发布模式` +- 节点会自动锁定为当前接管节点 +- 发布会默认带上当前选中的 release，没有 release 时直接提示先创建 + +并且 OpsCenter 现在不再只是“有按钮”，而是开始把 contract 直接展示到关键弹窗里： + +- `生成 Node Agent 接入方案` + - 明确显示当前使用 `ops_agent_protocol` + - 展示主入口、schema 文档、发现入口、执行链路 +- `创建 Release` + - 明确显示当前使用 `release_hub_contract` + - 展示 release 主入口、launchpad、schema 文档、发布链路 +- `创建 Rollout` + - 明确显示当前使用 `release_hub_contract` + - 展示 rollout 主入口、launchpad、execution_mode、执行链路 +- `驾驶建议 / 驾驶主线 / 自动驾驶判断` + - 可直接打开“驾驶动作契约预览” + - 展示 `ops_driver_contract`、目标 API、执行链路、request preview + - `overview / driver-feed` 已下沉到后端 `driver-actions/resolve` / `driver-actions/execute-resolved` + - `driver-actions/preview` 保留为基础契约预览与兜底入口 + - `codex-focus-preview` / `codex-focus-run` 进一步下沉到后端 `codex-actions/resolve` / `codex-actions/execute` + - 把“这次点击到底会调用什么”从黑盒按钮变成可审阅对象 + +命令行侧也开始走同一条 contract 预览链： + +- `bash domain-api/deploy/multi-region/drive_ops_center.sh contracts` +- `bash domain-api/deploy/multi-region/drive_ops_center.sh driver-feed` +- `bash domain-api/deploy/multi-region/drive_ops_center.sh driver-focus-preview` +- `bash domain-api/deploy/multi-region/drive_ops_center.sh driver-focus-run` +- `bash domain-api/deploy/multi-region/drive_ops_center.sh driver-preview ...` +- `bash domain-api/deploy/multi-region/drive_ops_center.sh driver-resolve ...` +- `bash domain-api/deploy/multi-region/drive_ops_center.sh driver-run ...` +- `bash domain-api/deploy/multi-region/drive_ops_center.sh runbook-preview ...` +- `bash domain-api/deploy/multi-region/drive_ops_center.sh runbook-run ...` +- `bash domain-api/deploy/multi-region/drive_ops_center.sh codex-focus-preview` +- `bash domain-api/deploy/multi-region/drive_ops_center.sh codex-focus-run` + +这样海外主机上即使不打开页面，也能先看到统一 contract，再决定要不要执行。 + +这里还要再收一条非常重要的约束： + +- `codex-focus-preview / codex-focus-run` + - 不能只认 `codex-brief.entries` + - 当当前没有动作主线、但存在 `codex-brief.activity_focus` 时 + - 应允许选中观察焦点 + - 并统一收口为 `focus_activity_item` + - 其 gate 必须落到 `open_ui` + - 不能伪装成 `auto_execute` + +并且这条命令链现在已经补上了真正面向“单脑驾驶”的默认入口： + +- `driver-feed` + - 看当前驾驶主线 +- `driver-focus-preview` + - 默认消费 `driver-feed.top_recommendation` + - 没传 `entry_key` 时，不再要求人工先抄 `action_code` + - 如果显式传入的是 `activity_focus.key` + - 应自动收口为 `focus_activity_item` + - 只做“进入工作区”的观察焦点，不升级成自动执行 +- `driver-focus-run` + - 先按同一条 focus 生成 `resolve / execute-resolved` 请求 + - 自动兼容： + - 普通 `driver_action` + - `runbook_sequence` + - `secondary` + - `confirm` + +这样后续海外 Codex 驾驶员拿到的就不是“很多离散脚本”，而是一条稳定的动作主线： + +1. 看 `driver-feed` +2. 选 `top_recommendation` +3. 先 `preview` +4. 再按门禁 `run` + +并且 `codex-focus-run` 已经开始真正落地“驾驶门禁”： + +- `safe_auto` + - 直接执行 +- `guarded_auto` + - 默认不执行，只输出审阅结果 + - 只有显式 `confirm` 才放行 +- `resolve_first / open_ui / blocked` + - 一律不执行，只给出原因和目标入口 + +现在这层门禁已经不再由 CLI 本地解释，而是由后端统一返回： + +- `selected_entry` +- `preview` +- `gate` +- `execution_result` + +这样后续海外 Codex 驾驶员就不是“会调接口的脚本”，而是严格受同一份后端 contract 和门禁规则约束。 + +这样后续不管是人工点击、CLI 脚本调用，还是海外 Codex 驾驶员接管，都不会再出现“页面这样理解、后端那样理解、节点执行又是第三套”的口径漂移。 + +Release Hub 自己也要走同一套“标准动作对象”思路，而不是继续靠页面里临时拼表单。 + +当前已经开始往这条线上收： + +- `发节点任务` 不再只是一个通用入口，而是正式区分： + - `deploy.release.control` + - `deploy.release.worker` + - `deploy.release.custom` +- 同时命令行侧的 `check_release_hub.sh` 也会直接输出： + - release 模板摘要 + - 控制面批量预检 + - Worker 批量预检 + - 混合节点预检 + +这样页面按钮、海外 Codex 驾驶员、CLI 自检脚本三者看到的就是同一套发布口径，而不是三套不同的发布逻辑。 + +这一步非常关键，因为它把： + +- Node Agent 接管 +- Ops Job +- Release Hub + +第一次真正接成了一条连续工作流。 + +--- + +## 十一、正式协议收口原则 + +为了让后续开发不再反复改口径，这套体系必须先明确 6 条协议原则： + +### 1. 前后端与 Codex 共用同一份 contract + +同一个动作、同一个节点、同一轮 rollout，必须由后端返回统一结构： + +- 页面直接展示 +- Codex 直接判断 +- CLI 直接检查 + +不能让三边各自“再推导一次”。 + +### 2. 所有远控动作都要先变成对象 + +最终要统一成： + +- `release` +- `rollout` +- `ops job` +- `playbook run` +- `activity` + +而不是： + +- 页面点按钮直接跑 shell +- Codex 临时拼命令 +- 节点本地脚本自己决定做什么 + +### 3. Node Agent 只执行已声明动作 + +Agent 不应该接受任意 shell 文本，而应该只接受标准动作： + +- `service.start` +- `service.stop` +- `service.restart` +- `service.status` +- `health.snapshot` +- `logs.collect` +- `diagnostics.collect` +- `deploy.release` + +这样控制面才能做审计、门禁、批量预检和回滚。 + +### 4. 所有对象都必须能审计 + +至少都要带： + +- `created_at` +- `updated_at` +- `requested_by` +- `target_node_code / target_nodes` +- `status` +- `result` + +后面不管是人、后台还是 Codex 都要能回放过程。 + +### 5. 协议必须支持“逐步增强” + +当前已经落下的是第一版骨架，所以协议要区分： + +- `当前已实现字段` +- `后续建议补充字段` + +这样可以先稳定主链路，再逐步增强，而不是一开始就推翻。 + +### 6. API 返回必须统一包裹 + +当前所有接口都已经走： + +```json +{ + "code": 0, + "message": "ok", + "data": {} +} +``` + +所以后面所有新增接口也应继续遵守： + +- `code=0` 表示成功 +- `code=1` 表示业务失败 +- 具体错误语义继续放进 `message` 和后续建议补的 `detail_code` + +--- + +## 十二、Node Agent HTTP 协议 + +### 1. 公共约束 + +#### 请求头 + +Agent 相关接口统一使用： + +- `Content-Type: application/json` +- `X-Domaincheck-Agent-Token: ` + +其中： + +- `tokens` +- `bootstrap-plan` + +是控制面主动签发，不要求 agent token。 + +而以下接口要求带 token： + +- `register` +- `heartbeat` +- `pull` +- `jobs/{id}/start` +- `jobs/{id}/complete` +- `jobs/{id}/events` + +#### 公共节点载荷 + +当前代码里的 `_base_payload()` 已经收敛出最小节点信息： + +- `node_code` +- `region` +- `role` +- `title` +- `hostname` +- `ip` +- `agent_version` +- `capabilities` +- `labels` +- `metadata.service_names` +- `metadata.delivery_queue` + +这意味着后续页面、Codex 和 agent 日志里看到的节点身份，应该都围绕这组字段统一。 + +其中 `metadata.delivery_queue` 当前已经是正式 contract，而不是预留字段。它至少会带： + +- `state` +- `label` +- `reason` +- `pending_count` +- `dead_letter_count` +- `last_flush_at` +- `oldest_pending_at` +- `oldest_dead_letter_at` + +也就是说，heartbeat 不只是在汇报“Agent 还在线”，也在汇报： + +> 当前本地是否有待补发回执、是否已经出现死信、最近一次补发冲刷是否成功 + +--- + +### 2. 签发 Agent Token + +`POST /api/v1/ops/agent/tokens` + +#### 请求体 + +```json +{ + "node_code": "mainland-worker-02", + "issued_by": "web-ui", + "expires_in_hours": 72, + "metadata": { + "source": "ops-center" + } +} +``` + +#### 当前返回字段 + +- `token` +- `token_preview` +- `record_id` +- `node_code` +- `expires_at` +- `created_at` + +#### 设计要求 + +这一步只做“签发”，不做安装、不做纳管、不做启动。 + +也就是说： + +- token 是凭证对象 +- bootstrap plan 才是接入方案对象 + +--- + +### 3. 生成 Bootstrap Plan + +`POST /api/v1/ops/agent/bootstrap-plan` + +#### 请求体 + +```json +{ + "node_code": "mainland-worker-02", + "node_region": "mainland", + "node_role": "worker", + "issued_by": "web-ui", + "expires_in_hours": 72, + "control_plane_base_url": "https://ops.example.com", + "root_dir": "/opt/domaincheck", + "metadata": { + "source": "ops-center" + } +} +``` + +#### 当前返回字段 + +当前后端 `build_node_agent_bootstrap_plan()` 已返回： + +- token 相关字段 +- `control_plane_base_url` +- `bootstrap_plan.node_code` +- `bootstrap_plan.node_region` +- `bootstrap_plan.node_role` +- `bootstrap_plan.root_dir` +- `bootstrap_plan.env_file` +- `bootstrap_plan.service_name` +- `bootstrap_plan.service_file` +- `bootstrap_plan.install_script_path` +- `bootstrap_plan.env_content` +- `bootstrap_plan.command_lines` +- `bootstrap_plan.command_block` +- `bootstrap_plan.health_checks` +- `bootstrap_plan.health_check_block` +- `bootstrap_plan.bootstrap_script_name` +- `bootstrap_plan.bootstrap_script_path` +- `bootstrap_plan.bootstrap_script_content` +- `bootstrap_plan.bootstrap_run_script_block` + +#### 正式定位 + +`bootstrap_plan` 不只是“给一段命令”，它是： + +> 控制面签发给目标节点的一份标准接管方案 + +后续： + +- 页面按钮 +- Codex 驾驶员 +- CLI 辅助脚本 + +都应该复用这一个对象，不再自己拼接 env 或 shell。 + +--- + +### 4. Agent Register + +`POST /api/v1/ops/agent/register` + +#### 请求体 + +```json +{ + "node_code": "mainland-worker-02", + "region": "mainland", + "role": "worker", + "title": "mainland-worker-02", + "hostname": "host-a", + "ip": "10.0.0.12", + "agent_version": "0.1.0", + "capabilities": [ + "service.start", + "service.stop", + "service.restart", + "service.status", + "health.snapshot", + "logs.collect", + "diagnostics.collect", + "deploy.release" + ], + "labels": {}, + "metadata": { + "service_names": { + "api": "domaincheck-api", + "worker": "domaincheck-worker", + "sync_agent": "domaincheck-sync-agent", + "node_agent": "domaincheck-node-agent" + } + } +} +``` + +#### 当前返回字段 + +- `node_code` +- `expires_at` +- `capabilities` + +#### 后端当前行为 + +当前 `agent_register()` 会： + +- 校验 token 与 `node_code` +- 调 `_upsert_agent_runtime()` +- 更新 `ops_managed_nodes` + +所以 register 的实际语义是： + +> 让节点正式进入“已接管候选 / 已纳管”目录 + +--- + +### 5. Agent Heartbeat + +`POST /api/v1/ops/agent/heartbeat` + +#### 请求体 + +当前与 register 共用同一份最小节点载荷。 + +#### 当前返回字段 + +- `node_code` +- `server_time` +- `expires_at` + +#### 协议语义 + +heartbeat 不是简单“在线 ping”，而是： + +- 刷新 last seen +- 刷新节点身份与 service_names +- 维持控制面里 Node Agent 在线态 +- 同步回执队列现场快照 + +#### 当前已落地的队列快照字段 + +当前 heartbeat 已经会通过 `metadata.delivery_queue` 上报： + +- `state` + - `healthy` + - `retrying` + - `dead_letter` +- `pending_count` +- `dead_letter_count` +- `last_flush_at` +- `oldest_pending_at` +- `oldest_pending_request_id` +- `oldest_pending_kind` +- `oldest_dead_letter_at` +- `oldest_dead_letter_request_id` +- `oldest_dead_letter_kind` + +#### 后续建议补充字段 + +后续建议 heartbeat 再扩： + +- `active_jobs` +- `disk_free_mb` +- `load_average` +- `python_version` + +其中 `local_queue_depth` 这一类信息，当前已经由 `delivery_queue` 快照承接，不再是空白能力。 + +--- + +### 6. Agent Pull + +`POST /api/v1/ops/agent/pull?limit=1` + +#### 请求体 + +```json +{ + "node_code": "mainland-worker-02" +} +``` + +#### 当前后端行为 + +当前 `agent_pull_jobs()` 只派发： + +- `target_node_code = 当前节点` +- `status = queued` +- `execution_mode = remote-agent` + +并在派发时自动改成： + +- `ops_jobs.status = dispatching` +- `ops_job_steps.status = dispatching` + +然后写入事件： + +- `agent_dispatched` + +#### 当前返回字段 + +```json +{ + "jobs": [ + { + "id": 123, + "job_code": "ops-xxx", + "action": "health.snapshot", + "target_node_code": "mainland-worker-02", + "status": "dispatching", + "execution_mode": "remote-agent", + "payload": {}, + "metadata": {}, + "policy": {}, + "steps": [] + } + ], + "count": 1 +} +``` + +#### 正式约束 + +后续就算扩到批量拉取，也应继续遵守： + +- agent 只能拿到属于自己的 job +- agent 不负责选择 job +- 调度权始终在控制面 + +--- + +### 7. 标记任务开始 + +`POST /api/v1/ops/agent/jobs/{job_id}/start` + +#### 请求体 + +```json +{ + "node_code": "mainland-worker-02" +} +``` + +#### 当前后端行为 + +当前 `agent_mark_job_started()` 会： + +- 把 `ops_jobs.status` 改成 `running` +- 把同 job 的 steps 改成 `running` +- 写入 `agent_started` + +#### 设计要求 + +这一步必须保留，不能省略。 + +因为： + +- `dispatching` 表示控制面已派发 +- `running` 表示节点已真实开始执行 + +这两者对排障非常重要。 + +--- + +### 8. 标记任务完成 + +`POST /api/v1/ops/agent/jobs/{job_id}/complete` + +#### 请求体 + +```json +{ + "node_code": "mainland-worker-02", + "client_request_id": "complete-ops-123-1", + "status": "success", + "stdout": "", + "stderr": "", + "result": { + "summary": "ok" + }, + "error_message": "" +} +``` + +#### 当前允许状态 + +- `success` +- `failed` +- `partially_succeeded` + +#### 当前后端行为 + +`agent_complete_job()` 会： + +- 校验 job 属于该节点 +- 基于 `client_request_id` 做完成回执幂等 +- 更新 `ops_jobs.status` +- 写入 `result_json` +- 回写 `error_message` +- 更新全部 `ops_job_steps` +- 写入 `agent_completed` +- 如果 job 属于 rollout，自动刷新 rollout 汇总 + +#### 正式约束 + +节点只负责回写： + +- 结构化结果 +- stdout/stderr +- 失败原因 + +不负责自己修改 rollout 策略、release 状态或下个批次。 + +这些都必须仍由控制面决定。 + +#### 当前已落地的幂等键 + +当前 `complete` 已支持： + +- `client_request_id` + +控制面会把它落到： + +- `ops_jobs.last_agent_complete_request_id` + +因此同一个完成回执即使因为网络抖动被重复补发，也不会再次制造副作用。 + +--- + +### 9. 上报中间事件 + +`POST /api/v1/ops/agent/jobs/{job_id}/events` + +#### 请求体 + +```json +{ + "node_code": "mainland-worker-02", + "client_event_id": "event-ops-123-checksum-1", + "step_id": 0, + "event_type": "deploy_checksum_verified", + "level": "info", + "message": "checksum ok", + "payload": { + "checksum": "sha256:..." + } +} +``` + +#### 当前后端行为 + +`agent_append_job_event()` 会： + +- 校验 token 和 node_code +- 基于 `client_event_id` 做事件幂等 +- 追加结构化事件 +- 返回 `event_id` + +#### 正式要求 + +中间事件不能只传纯文本，至少还要带： + +- `event_type` +- `level` +- `message` +- `payload` + +因为后面： + +- 页面活动流 +- rollout 详情 +- playbook run events +- Codex 驾驶员 + +都依赖结构化事件，而不是只靠日志文本猜。 + +#### 当前已落地的事件去重约束 + +当前事件表已经有： + +- `ops_job_events.client_event_id` +- `uq_ops_job_events_job_client_event` + +所以同一个事件即使被 Agent 重放，也应继续表现为： + +- 控制面接收成功 +- 不重复制造第二条事件 + +--- + +## 十三、Node Agent 状态机 + +### 1. Token 状态机 + +- `issued` +- `disabled` +- `expired` + +当前代码里 `ops_node_tokens` 已具备： + +- `is_enabled` +- `expires_at` +- `last_used_at` + +所以 token 生命周期应该正式解释成： + +- `issued + enabled + 未过期`：可用 +- `issued + disabled`：不可用 +- `issued + 已过期`：不可用 + +--- + +### 2. 节点接管状态机 + +页面与控制面应该统一理解为： + +- `未纳管` +- `待接入` +- `已接管` +- `仅运行态在线` +- `心跳过期` +- `token 过期 / 已禁用` + +建议正式映射为： + +- `discovered` +- `pending_bootstrap` +- `agent_online` +- `runtime_only` +- `stale` +- `token_invalid` + +前端显示名可以中文化，但后端内部最好固定英文状态码。 + +--- + +### 3. Job 状态机 + +当前主链路已经形成： + +- `queued` +- `dispatching` +- `running` +- `success` +- `failed` +- `partially_succeeded` + +正式语义应固定为： + +- `queued`：已创建，尚未派发 +- `dispatching`：控制面已派发给节点，但节点尚未确认开工 +- `running`：节点已开始执行 +- `success`：完成且成功 +- `failed`：完成且失败 +- `partially_succeeded`：部分成功，需要人工关注 + +这套状态机后面不能再随意改名，否则活动流、巡检收口、rollout 汇总都会被打散。 + +--- + +## 十四、`ops job` 的 Agent 执行载荷标准 + +这部分后续建议直接以 [ops_job_contract.md](/www/wwwroot/getDomain/docs/schemas/ops_job_contract.md:1) 为任务对象母本，再由 [ops_agent_protocol.md](/www/wwwroot/getDomain/docs/schemas/ops_agent_protocol.md:1) 继续冻结 Agent 视角下的拉取、开工、完工与事件回执协议。 + +### 1. Agent 可见 job 最小字段 + +Agent 从 `pull` 接口拿到 job 时，最少应该依赖这些字段： + +- `id` +- `job_code` +- `action` +- `target_node_code` +- `execution_mode` +- `payload` +- `metadata` +- `policy` + +其他字段如： + +- `requested_by` +- `risk_level` +- `approval_status` +- `rollout_id` + +虽然不是最小执行必需，但建议一起保留，便于未来 agent 诊断与本地审计。 + +### 2. 结构化动作必须优先 + +当前 agent 已按动作名分发： + +- `supports_structured_action(action)` +- `execute_structured_action(...)` +- `execute_release_action(...)` + +这条设计必须坚持： + +- 先结构化动作 +- 再结构化 payload +- 最后才在执行器内部落成本地命令 + +不能反过来回到“控制面下发整段 shell”。 + +### 3. `deploy.release` 的 payload 基线 + +当前 release 执行器已经支持的正式意图至少包括： + +- artifact 下载 +- checksum 校验 +- 解压到 `releases/` +- 切换 `current` +- 重启服务 +- 健康检查 +- 失败回滚 + +因此 `deploy.release` 的 payload 应长期固定围绕： + +- `release_version` +- `artifact_url` +- `checksum` +- `restart_services` +- `health_check_urls` +- `health_check_timeout_seconds` +- `health_check_retries` +- `health_check_interval_seconds` +- `rollback_on_failure` +- `switch_current` + +这套字段已经足够支撑正式 release 部署。 + +### 4. `logs.collect` 的口径 + +标准巡检里的 `logs.collect` 应优先解释为： + +> 收集目标服务日志，而不是执行任意日志命令 + +正式建议字段： + +- `service_name` +- `lines` +- `include_agent_logs` +- `since_seconds` + +并且默认仍要优先收口： + +- `domaincheck-worker` +- `domaincheck-node-agent` + +而不是允许页面任意填系统命令。 + +--- + +## 十五、Release Hub 正式契约 + +### 1. Release 对象 + +当前 `ops_releases` 已具备： + +- `release_version` +- `channel` +- `commit_sha` +- `status` +- `artifact_url` +- `checksum` +- `notes` +- `metadata` +- `created_by` +- `created_at` +- `activated_at` + +现在正式 contract 还需要进一步固定这些别名字段： + +- `status_label` +- `summary` +- `summary_text` +- `focus_ref` + +正式上应把 Release 理解为： + +> 一份不可变版本记录 + +也就是说： + +- release 创建后，不应该允许直接把它重新指向另一份 artifact +- 同一 `release_version` 只对应一份发布物 + +### 2. Rollout 对象 + +当前 `ops_release_rollouts` 已具备： + +- `release_id` +- `rollout_code` +- `target_selector` +- `target_nodes` +- `policy` +- `status` +- `batch_cursor` +- `batches_total` +- `jobs_total` +- `jobs_created` +- `result_summary` + +而 rollout 这一层也不该再只返回“原始推进计数”，还应稳定返回： + +- `status_label` +- `target_node_codes` +- `summary` +- `summary_text` +- `focus_ref` + +所以 rollout 的正式定义应为： + +> 某个 release 在某批目标节点上的一次分批推进计划 + +### 3. `default_rollout_gate` + +这层是 Release Hub 最关键的后端 contract，必须作为正式对象保留。 + +至少应返回： + +- `status` +- `status_label` +- `status_type` +- `summary` +- `summary_text` +- `release` +- `execution_mode` +- `execution_mode_label` +- `target_node_codes` +- `default_target_node_codes` +- `blocking_reasons` +- `warning_reasons` +- `recommendations` +- `operational_readiness.summary` +- `operational_readiness.rows` +- `focus_ref` + +#### 推荐状态码 + +- `missing_release` +- `release_not_ready` +- `artifact_missing` +- `no_targets` +- `blocked` +- `attention` +- `ready` + +这组状态码后面必须同时服务： + +- 首页驾驶建议 +- Release 页面门禁卡 +- Rollout 预检 +- Codex 自动驾驶 + +### 4. `release_launchpad` + +除了默认门禁外，还应保留更偏“驾驶舱”的发布聚合对象： + +- `latest_package` +- `latest_release` +- `worker_rollout_preview` +- `control_rollout_preview` +- `launchpad_status` + +其中 `launchpad_status` 至少应回答： + +- 当前应该先补什么 +- 推荐动作码是什么 +- 推荐执行方式是什么 +- 是先看 Worker 预案还是先看 Control 预案 +- 当前是否还存在接入缺口 +- 首个缺口节点是谁 +- 这个缺口是 + - `bootstrap_pending` + - 还是 `acceptance_ready` + +并且这层也应该直接给出： + +- `summary_text` +- `focus_ref` +- `recommended_target_node_code` +- `recommended_recovery_label` +- `recommended_recovery_summary` +- `onboarding_bootstrap_pending_nodes` +- `onboarding_acceptance_ready_nodes` + +其中 `focus_ref` 不该只是“回到 release hub”这么模糊，而应该允许继续携带： + +- `section` +- `mode` + +这样页面、CLI、Codex 都不需要自己再推断“该跳回 release hub 的哪个区域”。 + +这里再固定一个已经收口成代码字段的判断口径： + +- 如果 `launchpad_status.recommended_action_code` 已经落在 + - `bootstrap_run` + - `run_acceptance` +- 那么页面、CLI、Codex 不应再只显示笼统的“待补执行面” +- 而应直接把以下字段作为主提示： + - `recommended_target_node_code` + - `recommended_recovery_label` + - `recommended_recovery_summary` + - `onboarding_bootstrap_pending_nodes` + - `onboarding_acceptance_ready_nodes` + +原因是： + +- 真正的阻断并不总是“发布包有问题” +- 很多时候是“发布包已经好了，但节点还没接管完” +- 如果 launchpad 只返回一个大而化之的 `attention` + 前端和 Codex 还得再各自遍历 gap rows 去猜 +- 这会重新制造多套判断逻辑 + +所以现在要把 launchpad 的缺口口径固定成： + +- `launchpad_status` + 直接给驾驶层结论 +- `gap_rows` + 继续给详情层做下钻 + +而不是反过来让驾驶层去解析详情层。 + +当前仓库里这套骨架已经存在，所以后续不要再新造第四套发布摘要模型。 + +--- + +## 十六、Rollout 状态机 + +当前 rollout 状态已经接近正式版，建议固定为： + +- `planned` +- `running` +- `awaiting_approval` +- `ready_for_next_batch` +- `halted` +- `completed` +- `completed_with_issues` + +### 正式语义 + +- `planned`：已创建，尚未开始发批次 +- `running`：当前批次已发出，正在等待 job 结果 +- `awaiting_approval`：下一步推进需要人工确认 +- `ready_for_next_batch`：当前批次通过，可进入后续批次 +- `halted`：由于失败、阻断或人工暂停而停止 +- `completed`：全部批次完成且整体健康 +- `completed_with_issues`：全部批次结束，但存在失败或告警 + +后续不建议再引入新的近义状态名，否则页面和 Codex 会反复适配。 + +--- + +## 十七、错误码、幂等与重试规则 + +### 1. `detail_code` 已经进入 Agent / Ops 主链路 + +现在统一响应体已经有： + +- `code` +- `message` +- `data` + +而且 `detail_code` 已经开始进入顶层或 `data` 中，成为控制面和 Agent 都能消费的结构化错误语义。 + +这层现在的意义不再是“未来建议”，而是： + +> 不再靠中文 message 猜失败原因，而是让页面、Codex 和 Node Agent 共用同一份错误码 + +当前主链路里的错误返回，已经会继续补： + +- `detail_code` + +例如： + +- `agent_token_invalid` +- `agent_token_expired` +- `agent_token_node_mismatch` +- `ops_job_not_owned_by_agent` +- `ops_job_invalid_status_transition` +- `release_checksum_mismatch` +- `release_health_check_failed` + +这样页面和 Codex 就不用再从中文 message 里反推错误类型。 + +### 2. 幂等 contract + +当前已经正式落地两条关键幂等线： + +- `complete` + - 使用 `client_request_id` + - 回写到 `ops_jobs.last_agent_complete_request_id` +- `events` + - 使用 `client_event_id` + - 通过 `uq_ops_job_events_job_client_event` 去重 + +这意味着 Agent 在遇到控制面瞬断、网络抖动或重试补发时，已经具备： + +- 回执不重入 +- 事件不重复落库 + +后续仍建议继续保持幂等的动作有： + +以下动作应该尽量幂等： + +- `register` +- `heartbeat` +- `bootstrap-plan` +- `activate release` +- `rollout preview` + +也就是重复请求最多刷新状态，不应制造重复副作用。 + +### 3. Agent 重试与本地补发规则 + +建议正式约束，并且当前主链路已经落下一版： + +- `register` 失败：指数退避重试 +- `heartbeat` 失败：短退避重试，不立即退出 +- `pull` 失败：进入轮询退避 +- `start/complete/events` 失败：本地保留待补发队列 + +其中现在已经正式落地的是： + +- `complete` +- `events` + +这两类回执在失败时会进入 Agent 本地待补发队列，而不是直接丢失。 + +### 4. 当前已经落地的本地待补发 / 死信队列 + +当前 `domain-api/app/node_agent.py` 已经具备： + +- 本地 `pending` 队列 +- 本地 `dead-letter` 队列 +- 定时 `_flush_delivery_queue()` +- 对永久性 `detail_code` 的死信分流 + +也就是说，Agent 当前不是“请求失败就算了”，而是已经形成： + +1. 优先即时发送 +2. 失败则本地排队 +3. 下个 tick 自动重放 +4. 如果命中永久性错误，则进入死信 + +这层能力当前主要覆盖： + +- `jobs/{id}/complete` +- `jobs/{id}/events` + +### 5. 控制面现在如何看见队列问题 + +这条链路现在也已经不只存在于 Agent 本地。 + +当前控制面会把 heartbeat 里的 `metadata.delivery_queue` 收口成： + +- 节点行字段 + - `delivery_queue_state` + - `delivery_queue_label` + - `delivery_queue_reason` + - `delivery_queue_pending_count` + - `delivery_queue_dead_letter_count` +- 节点汇总字段 + - `queue_retrying_nodes` + - `queue_dead_letter_nodes` + - `queue_pending_records` + - `queue_dead_letter_records` + +并且驾驶建议已经开始优先提升这类问题： + +- 如果存在死信节点，优先建议先处理 Agent 死信 +- 如果存在待重试积压，优先建议先观察回执积压 + +所以这已经不是“隐藏在节点本地目录里的运维细节”，而是正式进入海外控制面的驾驶 contract。 + +--- + +## 十八、终局验收标准 + +当这套体系真正算“上线可用”，至少要满足以下 10 条： + +### 1. 新增节点不再需要人工拼命令 + +控制面能够直接生成： + +- token +- env +- bootstrap script +- health check block + +### 2. 海外控制面能先看见节点，再决定是否接管 + +不是只有接入成功后才看见机器。 + +### 3. 接管完成后可直接跑标准验收 + +而不是再去手工 systemctl / journalctl。 + +### 4. 标准巡检统一走 `ops job` / `playbook run` + +而不是页面每个按钮都各做一套逻辑。 + +### 5. 真正参与检测节点和在线未参与节点口径统一 + +页面、后端、Codex 必须读同一份 execution scene。 + +### 6. 远端日志回传状态正式结构化 + +至少统一成： + +- `disabled` +- `waiting_sample` +- `partial_coverage` +- `healthy` +- `full_capture` + +### 7. 发布必须先有 Release，再有 Rollout + +不能回退到节点直接 `git pull`。 + +### 8. Rollout 必须能分批、暂停、继续、回滚 + +而不是一次性把所有节点推平。 + +### 9. 页面按钮和 Codex 都只创建对象，不直接执行 shell + +最终： + +- 页面创建 `ops job` +- Codex 创建 `ops job` / `playbook run` +- agent 消费 `ops job` + +### 10. 海外主机必须成为唯一运维大脑 + +大陆机器只负责： + +- 执行 +- 回传 +- 承载业务 + +不再承担“人工登录后做决定”的角色。 + +--- + +## 十九、下一步最值得继续补的 6 件事 + +基于当前仓库状态，接下来最值得继续补的，不再是“把主链路补出来”，而是把已经成型的体系收成真正可长期运维的平台。 + +### 1. 给死信队列补正式操作面 + +当前已经有： + +- 本地待补发 +- 本地死信 +- 控制面可见的队列快照 + +下一步最该补的是： + +- 死信详情查看 +- 死信重放 +- 死信丢弃 +- 死信按 `detail_code` 聚类 + +否则“看见死信”之后，操作者还是得回到节点本地处理。 + +### 2. 把 action template preview / batch policy preview 正式后端化 + +在真正创建 `ops jobs` 之前，要让控制面先统一回答： + +- 哪些节点被阻断 +- 哪些节点需要审批 +- 哪些节点只是 warning +- 当前批次策略是否合理 + +这样发布、重启、停 Worker 这类高风险动作，才能在创建对象前先收口风险。 + +### 3. 把接管验收、标准巡检、现场观察彻底收进 playbook run + +现在 playbook run 骨架已经有了，而且其中一部分已经开始从“打开弹窗”进入“直接执行正式动作”。下一步要继续把： + +- `onboarding.acceptance` +- `inspection.standard` +- `scene.logs.key` +- `scene.logs.full` + +都变成真正稳定的编排对象，而不是“部分走编排、部分还停留在前端直触发”。 + +当前已经落地的直接驾驶动作包括： + +- `bootstrap_run` +- `run_acceptance` +- `run_standard_inspection` +- `open_diagnostics` +- `run_scene_logs_key` +- `run_scene_logs_full` + +这意味着纳管收口、标准巡检和现场观察都已经开始直接进入正式执行链，而不是先把操作者送去弹窗二次选择。 + +发布审阅链也应该遵守同样原则： + +- `review_smart_rollout_preview` +- `review_control_rollout` +- `fix_rollout_blockers` + +这几类动作不应只返回“打开 launchpad 审阅页”的 UI intent，而要同时直接返回结构化 `release_launchpad_review` 结果，让自动驾驶、CLI 和后端任务能直接消费 launchpad 预案。 + +并且这里的自动化等级也要彻底锁死： + +- 这三类动作属于 `safe_auto` +- 它们的职责是“审阅”和“解释” +- 不是“创建发布对象” + +所以默认返回应是： + +- 结构化 `release_launchpad_review` +- 可选 `ui_intent = release_launchpad_review` +- 但不直接创建 rollout / publish 记录 + +同理，`focus_release_hub` 也不应继续只是一个纯导航意图： + +- 页面侧仍可消费 `ui_intent = focus_release_hub` +- 但后端同时还应返回 `release_hub_preview` + 至少包含： + - `summary` + - `launchpad` + +这样海外 Codex、CLI、driver-feed 调用方即使不打开页面，也能继续拿这份摘要决定后续是： + +- 继续审阅 launchpad +- 进入发布创建 +- 还是先回补门禁缺口 + +`release_package` 也应遵守同样的“先给后端摘要，再保留 UI 入口”原则： + +- 页面仍可通过 `ui_intent = open_release_dialog` 进入打包区 +- 但动作返回里还应包含 `release_package_preview` + 至少包括： + - `available` + - `package_name` + - `release_version_suggestion` + - `reason` + +这样 Node Agent、Release Hub、海外 Codex 驾驶员三边的判断都能先基于同一份“发布包现场状态”，而不是必须打开页面后才能知道有没有包可发。 + +同理： + +- `open_release_dialog` + 也不应只是纯导航 + 应同时返回 `release_dialog_preview` +- `open_rollout_dialog` + 也不应只是纯导航 + 应同时返回 `rollout_dialog_preview` + +这样就算暂时仍保留页面入口，自动驾驶层也已经可以先读： + +- Release 当前摘要 +- Launchpad 当前摘要 +- 发布包是否存在 + +而不是必须依赖前端抽屉作为唯一信息入口。 + +继续往下一层，模板类动作和发布部署模板动作也应该统一走“预览优先”： + +- `open_playbook_dialog` + 应返回 `playbook_preview` +- `open_action_template_dialog` + 应返回 `template_preview` +- `open_release_deploy_worker / open_release_deploy_control / open_release_deploy_custom` + 应返回 `release_deploy_preview` + +其中 `release_deploy_preview` 至少应包含： + +- `template` +- `summary` +- `launchpad` +- `target_node_codes` +- `execution_mode` +- `release_id` + +这样自动驾驶层拿到的就不是“去某个弹窗”，而是一份已经够继续判断的执行预案。 + +再往前，焦点类动作也不应继续只是“跳到某个详情页”： + +- `focus_playbook_run` + 应返回 `playbook_run_focus` +- `open_playbook_run_latest_events` + 应返回 `playbook_run_events_preview` +- `focus_latest_job_events` + 应返回 `job_events_preview` +- `focus_activity_item` + 应返回 `activity_focus_preview` + +这样 Codex、CLI、OpsCenter 三边在进入具体页面之前，就已经能先读到： + +- 当前编排主状态 +- 最近事件流 +- 当前 job 回执 +- 当前 activity 焦点对象 + +后续“跳到页面”就只是补充查看，而不是唯一信息入口。 + +这里还要把动作分级口径正式固定下来： + +- 只要动作已经满足： + - 后端可直接执行 + - 不修改现场状态 + - 主要返回 `preview / focus / summary / events` +- 那么它就不应继续归在 `ui_only` +- 而应统一归到 `safe_auto` + +当前已经应按这个口径归类的动作包括： + +- `handover_first_gap` +- `view_first_gap` +- `open_playbook_dialog` +- `open_action_template_dialog` +- `focus_playbook_run` +- `focus_activity_item` +- `focus_latest_job_events` +- `open_playbook_run_latest_events` +- `open_release_dialog` +- `open_rollout_dialog` +- `open_release_deploy_control` +- `open_release_deploy_worker` +- `open_release_deploy_custom` +- `focus_release_hub` +- `release_package` + +也就是说，页面仍然可以消费这些动作的 `ui_intent`，但 Codex / CLI / 自动驾驶网关已经可以把它们当成： + +- “可直接执行的只读观察动作” + +而不是： + +- “必须先打开某个页面才能继续” + +而真正进入发布创建层的动作，例如： + +- `publish_latest_worker` +- `create_smart_release_rollout_worker` +- `create_smart_release_rollout_control` +- `create_release_rollout_worker` +- `create_release_rollout_control` + +则必须继续保持： + +- `guarded_auto` +- `confirm_then_execute` +- 正式写入发布与审计链 + +### 4. 把 Release Artifact 管道做成正式不可变发布链 + +现在 Release / Rollout 对象已经成型，下一步最关键的是补齐： + +- 构建产物打包 +- checksum / manifest +- 上传与留存 +- channel 激活 +- 回滚版本选择 + +也就是把“版本对象存在”继续推进到“版本供应链完整”。 + +### 5. 把 SSH Rescue Executor 收成首发接管与紧急救援闭环 + +终局不是回到 SSH 主运维，但 SSH 仍应有正式位置： + +- 首次 bootstrap +- Node Agent 尚未接入时的应急接管 +- Agent 故障时的救援执行 + +这一层应继续做成正式 executor，而不是保留为人工口令。 + +### 6. 把 Node Agent / ReleaseHub / OpsCenter contract 固化为 schema / OpenAPI + +现在最怕的已经不是“没有能力”，而是： + +> 页面、后端、Codex、CLI 在继续迭代时发生字段漂移 + +所以后续上线前，建议正式抽出： + +- `schemas/ops_agent_protocol.md` +- `schemas/release_hub_contract.md` +- `schemas/ops_driver_contract.md` +- `schemas/ops_playbook_contract.md` + +并且不只要有 schema 文件，还要让控制面直接暴露统一索引： + +- `GET /api/v1/ops/contracts` +- `GET /api/v1/ops/contracts/{contract_key}` + +前者负责列出整张 contract map，后者负责按 key 钻取单份协议和相关 contract。 + +控制面前端与海外 CLI 也要统一走这两层入口： + +- 页面默认只拉 registry，避免首屏过重 +- 当用户点击某个 contract 时，再按 key 拉 detail +- detail 返回里必须带 `related_contract_keys` / `related_contracts` +- 这样排查一条链路时，可以从 `ops_stack_diagnosis_contract` 直接跳到 `ops_driver_contract`、`release_hub_contract`、`ops_agent_protocol` + +或者直接写入后端 schema / OpenAPI 注释，让 contract 成为真正受版本控制的对象。 + +--- + +## 二十、海外 Codex 驾驶员的正式闭环 + +如果要把“海外主机单脑控制面”真正做成长期稳定体系，海外 Codex 驾驶员不应该被设计成： + +- 远端 shell 执行器 +- 看日志后临时拼命令的脚本 +- 依赖人工不断复制上下文的助手 + +它应该是严格跑在统一 contract 上的“驾驶层”。 + +### 1. 默认入口只看 3 类信号 + +海外 Codex 驾驶员默认只消费： + +1. `driver-feed` +2. `codex-brief` +3. `activity-stream` + +分别回答： + +- 现在最该处理什么 +- 这一步能不能自动执行 +- 最近执行后现场发生了什么 + +也就是说，驾驶员不应该先去翻零散日志，再猜下一步，而是先看控制面已经结构化好的主线。 + +### 2. 默认执行顺序必须固定 + +标准顺序应固定成： + +1. `driver-feed` + - 取 `top_recommendation` +2. `driver-focus-preview` + - 看 contract / gate / request preview +3. `driver-focus-run` + - 只有后端 gate 允许时才执行 +4. `activity-stream` + - 回看这一步是否真正落地 +5. 必要时钻取： + - `playbook run` + - `ops job` + - `release / rollout` + +这样驾驶员并不是“直接开车”，而是： + +- 先看路标 +- 再看交通规则 +- 然后再执行 + +### 3. 驾驶员永远不直接决定 shell + +正式规则必须收死： + +- 驾驶员不直接 SSH 到节点上拼 shell +- 驾驶员不自己判断某台机器该不该 `git pull` +- 驾驶员不自己决定 systemd 命令如何组合 +- 如果控制面已经把首个缺口节点收敛成 `bootstrap_run / run_acceptance` + - 驾驶员也不能退回成泛化的 `fix_managed_nodes` + - 而应优先围绕同一个节点继续做预览、执行和回执追踪 + +驾驶员只做三件事： + +1. 选择动作对象 +2. 请求后端解析门禁 +3. 创建正式执行对象 + +最终执行必须落到： + +- `ops job` +- `playbook run` +- `release / rollout` +- `node agent task` + +这样才具备： + +- 审计 +- 回放 +- 回滚 +- 风险分级 + +### 4. 驾驶员必须把“看见问题”和“处理问题”拆开 + +终局方案里最容易犯的错，就是把： + +- 看见节点离线 +- 打开日志 +- 重启服务 +- 改配置 +- 再观察 + +全部写成一段混杂的脚本。 + +正确方式应该拆成两层： + +- 观察层 + - `driver-feed` + - `activity-stream` + - `inspection` + - `release launchpad` +- 执行层 + - `driver-actions/execute-resolved` + - `ops job` + - `playbook run` + - `rollout` + +这样后面无论是： + +- 人工值班 +- 海外 Codex 自动驾驶 +- 后台一键按钮 + +都能共用同一套观察口径与执行口径。 + +补充一条正式约束： + +- `driver-feed` + - 顶层 payload 需要带 `contract_navigation` + - 每条 `entries` + - 每条 `activity_focus` + 都要带 `contract_navigation` +- `activity-stream` + - 顶层 payload 需要带 `contract_navigation` + - 每条 `items` + 都要带 `contract_navigation` + +这样海外单脑控制面里的卡片、时间线、CLI drilldown、Codex 驾驶员才能共享同一条“协议跳转链”，而不是 UI 有一套、CLI 有一套、排障时再临时翻文档。 + +### 5. 驾驶员需要一套明确的升级路径 + +真正上线时，建议把海外 Codex 驾驶员分成 4 档能力，而不是一步到位全自动： + +#### L1 观察驾驶 + +- 只看 `driver-feed / codex-brief / activity-stream` +- 只输出建议，不执行 + +#### L2 门禁驾驶 + +- 可以跑 `driver-focus-preview` +- 可以对 `safe_auto` 动作执行 `driver-focus-run` +- 对 `guarded_auto` 只提示确认 + +这里的典型分界现在应明确到动作名，避免后续接入新驾驶员时再自己猜： + +- `safe_auto` + - `review_smart_rollout_preview` + - `review_control_rollout` + - `fix_rollout_blockers` + - `run_standard_inspection` + - `run_scene_logs_key` + - `run_scene_logs_full` +- `guarded_auto` + - `publish_latest_worker` + - `create_smart_release_rollout_worker` + - `create_smart_release_rollout_control` + - `create_release_rollout_worker` + - `create_release_rollout_control` + +#### L3 编排驾驶 + +- 可以创建： + - `playbook run` + - `ops job` + - `release / rollout` +- 但仍不直连 shell + +#### L4 救援驾驶 + +- 仅在 Node Agent 失效或接管初期 +- 允许走受控 `SSH Rescue Executor` +- 仍然必须把动作写回控制面审计链 + +这样自动化能力能逐档放开，而不是一开始就把所有权力交给一段脚本。 + +### 6. 海外主机最终应该成为唯一“决策写入口” + +终局不是“海外主机能看到更多日志”，而是： + +> 所有决策类动作，都只能从海外控制面写入。 + +这意味着： + +- 大陆节点不再自己 `git pull` 决定版本 +- 大陆节点不再靠人工 SSH 决定是否重启 +- 大陆节点只负责执行、回传、承载业务 + +而海外主机负责： + +- 产物选择 +- 风险判断 +- 动作签发 +- 回执汇总 +- 运行态总览 + +这才是真正的“单脑控制面”。 + +### 7. 对你这个项目，最推荐的默认日常路径 + +如果按最省心、后续最少返工的标准，建议以后日常操作尽量收成下面这条路径： + +1. 海外主机打开 OpsCenter +2. 看 `driver-feed.top_recommendation` +3. 先 `driver-focus-preview` +4. 再 `driver-focus-run` +5. 如果动作升级，进入： + - `inspection` + - `playbook run` + - `release hub` +6. 如果执行异常，看： + - `activity-stream` + - 死信队列 + - 回执补发状态 +7. 只有在 Agent 不可用时，才进入 SSH Rescue + +这样后续再新增 1 台大陆机器，不会再变成“重新写一套部署说明”，而只是： + +- 新节点 bootstrap +- Node Agent 注册 +- 控制面验收 +- 进入正式运维 + +架构不会再被机器数量拖垮。 + +--- + +## 二十一、死信队列必须从“可见”升级到“可操作” + +当前这套体系已经能看见： + +- 哪台节点存在 `dead_letter` +- 有多少条死信 +- 最早死信时间是什么 + +这说明“现场可见性”已经有了。 + +但如果没有正式操作面，控制面还是会卡在最后一步： + +- 看到死信 +- 给出建议 +- 最后仍然要人工 SSH 去处理 + +这就违背了“海外单脑控制面”的目标。 + +所以这层必须继续升级成正式对象： + +- 单节点队列总览 +- 队列记录列表 +- 死信详情 +- 单条重放 +- 批量重放 +- 丢弃 +- 主动冲刷 + +其中第一阶段已落地： + +- `GET /api/v1/ops/nodes/{node_code}/delivery-queue` +- `GET /api/v1/ops/nodes/{node_code}/delivery-queue/records` +- `POST /api/v1/ops/nodes/{node_code}/delivery-queue/flush` +- `POST /api/v1/ops/nodes/{node_code}/delivery-queue/replay` +- `POST /api/v1/ops/nodes/{node_code}/delivery-queue/records/{record_id}/replay` +- `POST /api/v1/ops/nodes/{node_code}/delivery-queue/records/{record_id}/discard` + +当前这组入口明确采用： + +- `record_visibility=head_only` + +也就是控制面先稳定看到： + +- 节点级队列状态 +- `pending / dead_letter` 头部记录 + +而不是假装已经具备远端全量死信列表与直接操作能力。 + +也就是说现在已经进入“可操作的第一阶段”： + +- 控制面能看节点级队列状态 +- 控制面能看 `pending / dead_letter` 头部记录 +- 控制面能通过正式 `ops job` 触发冲刷 / 重放 / 丢弃 +- 但还没有把远端全量记录目录完整投影成控制面对象 + +并且这第一阶段现在已经不是“后端接口预留”，而是已经形成了三层统一操作面： + +- OpsCenter 托管节点表可直接打开“回执队列治理”抽屉 +- driver recommendation 遇到 `dead_letter / retrying` 时，会优先落到标准动作模板 +- 海外单入口 CLI 已支持直接查看 / 冲刷 / 重放 / 丢弃 + +也就是说当前这层已经具备明确的人机协作分工： + +- 页面负责看现场、做单条处理、做危险动作确认 +- driver / Codex 负责把“当前最该处理什么”压成建议 +- CLI 负责在海外主机上做统一批处理和交接执行 + +具体到当前仓库，已经形成下面这些稳定入口： + +### 1. 页面入口 + +- OpsCenter 托管节点表： + - `队列` + - `立即冲刷` + - `重放死信` + - `单条重放` + - `单条丢弃` +- 丢弃动作强制要求填写原因 +- 当前页面仍明确遵守： + - `record_visibility=head_only` + +也就是页面不会假装自己能枚举远端完整死信目录，而是只围绕“当前可见头部记录”和“节点级摘要”操作。 + +### 2. 驾驶建议入口 + +当前 `driver_recommendations` 已经升级成： + +- `dead_letter` + - 主动作：直接执行 `replay_delivery_queue` + - 后端落点：创建 `delivery.queue.replay` 的标准 `ops job batch` + - 次动作：查看 Worker 日志 +- `retrying` + - 主动作：直接执行 `flush_delivery_queue` + - 后端落点：创建 `delivery.queue.flush` 的标准 `ops job batch` + - 次动作：查看 Worker 日志 + +这一步很关键，因为它意味着控制面不再只是说“这里有死信，你自己想办法”，也不再只是把人送去模板弹窗，而是已经能直接走正式 `ops job` 链路下发队列治理动作。 + +### 3. 海外单入口 CLI + +当前 `drive_ops_center.sh` 也已经补上了这组命令： + +- `bash domain-api/deploy/multi-region/drive_ops_center.sh queue-status ` +- `bash domain-api/deploy/multi-region/drive_ops_center.sh queue-records [state]` +- `bash domain-api/deploy/multi-region/drive_ops_center.sh queue-flush [limit] [requested_by]` +- `bash domain-api/deploy/multi-region/drive_ops_center.sh queue-replay [limit] [requested_by]` +- `bash domain-api/deploy/multi-region/drive_ops_center.sh queue-replay-record [requested_by]` +- `bash domain-api/deploy/multi-region/drive_ops_center.sh queue-discard-record [requested_by]` +- `bash domain-api/deploy/multi-region/drive_ops_center.sh activity-stream [key=value ...]` +- `bash domain-api/deploy/multi-region/drive_ops_center.sh activity-preview [activity_key] [requested_by]` +- `bash domain-api/deploy/multi-region/drive_ops_center.sh driver-feed` +- `bash domain-api/deploy/multi-region/drive_ops_center.sh driver-focus-preview` +- `bash domain-api/deploy/multi-region/drive_ops_center.sh driver-focus-run` +- `bash domain-api/deploy/multi-region/drive_ops_center.sh codex-brief` +- `bash domain-api/deploy/multi-region/drive_ops_center.sh codex-focus-preview` +- `bash domain-api/deploy/multi-region/drive_ops_center.sh codex-focus-run` + +建议把这组命令视为“海外集中治理 Node Agent 回执队列”的正式入口，而不是临时调试命令。 + +并且现在这组命令的输出口径也要冻结下来： + +- `activity-stream / driver-feed / codex-brief` + - condensed summary 必须打印顶层 `contract_navigation` +- `activity-preview / driver-focus-preview / codex-focus-preview / stack-next` + - condensed summary 必须打印 `preview.contract_key / contract_version / contract_primary_endpoint / preview_endpoint / schema_doc_path` +- 所有 `selected_entry` + - 必须尽可能保留 `contract_navigation` + +这样海外 CLI、页面按钮、未来海外 Codex 驾驶员看到的就不是三套不同“解释文本”，而是一套可追踪到协议、接口和 schema 文档的正式输出。 + +同理，海外单脑控制面导出的交接包也不能只停留在： + +- `stack summary` +- `ops plane` +- `participation` + +还应该把 contract-aware 驾驶面一起固化进去： + +- `contracts registry` +- `driver-feed` +- `activity-stream` +- `codex-brief` +- `stack-next preview` +- `driver-focus preview` +- `activity preview` +- `codex-focus preview` + +并且 preview 类报告允许“当前没有焦点但仍导出文件”，文件尾部只记录 `command_exit_code=1`，这样交接人拿到包以后不会误解成“导出中断”，而能明确判断是“现场当时没有可预览入口”。 + +进一步讲，这个交接包还应该再固定一层机器摘要： + +- `manifest.json` + +它不是替代原始报告，而是给海外 Codex / 后台自动驾驶 / 未来 OpsCenter UI 二次消费的摘要面。 + +正式口径应该固定成： + +- `meta.json` + - 负责说明这包是谁、何时、针对哪个 profile / 哪组 API 生成的 +- `manifest.json` + - 负责说明这包里每份报告是否存在、exit code 是多少、哪些 preview 当前可用、现场暴露了哪些 contract key + - 如果某个 required 报告已经解析出 `http_status` 且不是 `2xx`，也必须记入 `required_failures` +- `*.txt` + - 保留完整原始现场输出，供人类继续钻取 + +并且 `doctor-export` 命令自己的 stdout 也应该回传一份缩略版 `summary + decision`，这样海外 Codex 不必先打开文件树，也能先做第一轮分流判断。 + +进一步往终局落地时，CLI 侧还应该固定一个更适合机器先消费的统一入口： + +- `doctor-decision` + +正式定位应该是： + +- `doctor-export` + - 负责生成完整交接包 +- `doctor-decision` + - 负责读取 `manifest.json` + - 如果没有现成 `manifest.json`，就临时生成一份 export + - 然后只回传 `summary + decision` + - 并且允许用 `-` 明确关闭 overseas 拓扑探测，避免因为默认海外地址不可达把第一轮分流卡死 + +这里的 `summary` 不能只做 surface 成败汇总，还要继续固定带出 launchpad 接管口径： + +- `launchpad_recommended_target_node_code` +- `launchpad_recommended_recovery_label` +- `launchpad_recommended_recovery_summary` +- `launchpad_onboarding_bootstrap_pending_nodes` +- `launchpad_onboarding_acceptance_ready_nodes` + +同时，`decision.evidence` 也要镜像这组字段，确保海外 Codex / 自动驾驶器 / 人工值班 +在只消费 `doctor-decision` 的前提下，就能直接判断： + +- 当前最该先接哪台节点 +- 当前是“还缺 bootstrap”还是“已经 ready，等待 acceptance” +- 是否需要继续下钻 `release-launchpad / driver-feed / codex-brief` + +这样海外单脑控制面的“第一跳”就稳定了： + +- 海外 Codex 先跑 `doctor-decision` +- 后台自动驾驶按钮先跑 `doctor-decision` +- 人工值班也先看 `doctor-decision` + +并且从当前实现口径上，`doctor-decision` 不应再只消费旧的 `next_step_action_code`，而要优先消费： + +- `diagnosis.operator_decision` +- `diagnosis.next_actions` +- `diagnosis.recommended_commands` + +这意味着如果总检已经明确判定“先看现场日志 / 先打通节点接管 / 先看 Release Hub”，那么 `doctor-export` 产出的 `manifest.json` 与 `doctor-decision` 最终给出的 `recommended_commands` 也必须同步落到同一条主决策，不允许再次退化成泛化建议。 + +只有当 `decision.status` 显示需要继续钻取时，才去打开整包 `*.txt` 和 `manifest.json` 深挖。 + +而且 `doctor-export / doctor-decision` 这一层本身也应该固定成“步骤级超时继续导出”： + +- 某个 surface 卡住，不允许整包诊断悬死 +- required report 失败，要进入 `required_failures` +- required report 失败明细，要进入 `required_failure_details` +- optional preview 不可用，要进入 `optional_unavailable` +- optional preview 不可用明细，要进入 `optional_unavailable_details` +- 海外单脑永远优先拿到一份可判断的 `decision`，而不是拿到一个还在转圈的进程 + +并且这些明细不能只停留在“文件名失败”： + +- `reports[].failure.reason_code` +- `reports[].failure.label` +- `reports[].failure.detail` + +至少要能稳定区分： + +- `command_timeout` +- `command_failed` +- `http_unreachable` +- `http_error` +- `result_unparsed` + +再往前一步，控制面 API 自身也应该暴露正式“运行指纹”： + +- `/health` +- `/api/v1/runtime/status` +- `/api/v1/runtime/build-info` + +这些接口至少应该稳定给出： + +- `build.source` +- `build.package_name` +- `build.generated_at` +- `build.commit_sha` +- `build.commit_ref` +- `build.manifest_path` +- `build.route_surface.surface_complete` +- `build.route_surface.missing_paths` + +这样以后看到 `ops/contracts` 返回 404 时，就不用再靠猜： + +- 如果 `build` 缺失，说明运行实例还没切到新版本 +- 如果 `build` 存在但 `route_surface` 缺路由，说明当前实例加载的仍然不是完整协议面 +- 只有 `build` 与 `route_surface` 都完整，才值得继续往 preview / execute 深挖 + +### 4. 当前阶段的明确边界 + +当前仍然要刻意坚持下面这几点，避免以后又滑回“假装可控”： + +- 不直接 SSH 进节点手改 `pending / dead-letter` 文件 +- 不允许页面、CLI、Codex 各自定义不同的 replay / discard 逻辑 +- 所有操作都必须落成正式 `ops job` +- 单条 replay / discard 当前只允许针对控制面可见头部记录 + +这样未来从 `head_only` 升级到“远端全量记录正式投影”时，扩的是可见范围，不是重写整个治理模型 + +推荐以后直接以 [ops_agent_protocol.md](/www/wwwroot/getDomain/docs/schemas/ops_agent_protocol.md:133) 为母本，把 `delivery_queue` 从 heartbeat 快照扩到控制面操作面。 + +正式要求应该固定成： + +- Agent 只负责补发与执行 +- 控制面负责查看、聚类、重放、丢弃 +- 页面、CLI、Codex 不允许各自发明不同的死信处理逻辑 + +--- + +## 二十二、Playbook Run 必须成为正式一等对象 + +当前仓库里 `playbook` 已经不只是“预留想法”，而是已经有： + +- catalog +- preview +- execute +- run detail +- run events +- rerun +- cancel + +这说明骨架已经具备，只差把 contract 冻结清楚。 + +后续最关键的不是再补更多“脚本型能力”，而是把这几类正式收口： + +- `onboarding.bootstrap` +- `onboarding.acceptance` +- `inspection.standard` +- `scene.logs.key` +- `scene.logs.full` +- `scene.diagnostics` + +也就是说： + +- 接管验收 +- 标准巡检 +- 现场观察 + +都应该优先变成 `playbook run`，而不是继续散落在： + +- 页面按钮 +- Driver Action 临时逻辑 +- 手工命令 + +推荐以后直接以 [ops_playbook_contract.md](/www/wwwroot/getDomain/docs/schemas/ops_playbook_contract.md:1) 为正式母本。 + +这样后续： + +- 页面点“标准巡检” +- CLI 执行“现场观察” +- 海外 Codex 驾驶员决定“先验收还是先抓日志” + +最终看到的都是同一类对象： + +- 先 `preview` +- 再 `execute` +- 最后围绕 `playbook run / playbook events` 观察 + +这样控制面才算真正从“按钮集合”进化成“正式编排平台”。 + +--- + +## 二十三、单脑控制面最终要固定成 4 张操作面 + +Node Agent、Release Hub、Driver、Playbook 这些听起来很多，但终局页面不应该把操作者淹没在概念里。 + +最终应该稳定成 4 张操作面。 + +其中“现场面 + 队列面”建议以后直接以 [ops_observability_contract.md](/www/wwwroot/getDomain/docs/schemas/ops_observability_contract.md:1) 为正式母本，避免后面又把这层能力重新拆散到页面局部状态里。 + +### 1. 接管面 + +负责回答： + +- 哪些节点已经纳管 +- 哪些节点只是集群可见但未纳管 +- 哪些节点 token 过期 +- 哪些节点 agent 心跳异常 +- 哪些节点已经通过接管验收 + +这一面主要消费： + +- `ops_agent_protocol` +- `managed nodes` +- `bootstrap-plan` +- `onboarding.acceptance` + +### 2. 现场面 + +负责回答： + +- 哪些节点正在执行 +- 哪些节点在线但未参与 +- 哪些节点近窗有吞吐 +- 日志回传当前是否关闭 / 关键 / 全量 +- 当前应该优先巡检哪台节点 + +这一面主要消费： + +- `overview` +- `inspection` +- `activity-stream` +- `driver_recommendations` +- `playbook runs` + +### 3. 队列面 + +负责回答： + +- 哪台节点的回执队列有积压 +- 哪台节点有死信 +- 头部记录是什么 +- 当前应该冲刷、重放，还是丢弃 + +这一面主要消费： + +- `delivery_queue` +- `delivery_queue.records` +- `flush / replay / discard` + +这里要持续坚持： + +- 当前阶段允许 `head_only` +- 但不允许退回人工 SSH 改文件 + +### 4. 发布面 + +负责回答： + +- 当前默认 release 是谁 +- 默认 gate 是 `blocked / attention / ready` +- 哪些节点可以灰度 +- 哪一批 rollout 正在执行 +- 哪一轮 rollout 需要审批 / 暂停 / 回滚 + +这一面主要消费： + +- `release_hub_contract` +- `launchpad` +- `default_rollout_gate` +- `rollouts` + +这样页面心智模型才会稳定： + +- 接管面：先让节点进体系 +- 现场面：看现场和巡检 +- 队列面：处理回执不一致 +- 发布面：推进版本与回滚 + +这 4 张面板不是 4 套系统，而是： + +> 同一套控制面 contract 的 4 个观察窗口 + +--- + +## 二十四、新节点从“拿到机器”到“可以发版”必须走标准路径 + +后续不应该再写“某台机器特殊处理说明”，而要把新节点接入路径固定为下面 8 步。 + +### 1. 签发接入凭证 + +控制面生成： + +- token +- bootstrap plan +- env 内容 +- systemd 文件 +- 启动检查命令 + +这一步之后，页面、CLI、Codex 都只能复用同一份 bootstrap 对象。 + +### 2. 节点安装 Node Agent + +节点完成： + +- 安装 agent +- 写入 env +- 启动 systemd + +此时节点还只是： + +- `agent installed` + +还不算已经正式接管成功。 + +### 3. 节点注册并进入托管目录 + +节点完成： + +- `register` +- `heartbeat` + +控制面能看见： + +- 节点身份 +- capabilities +- service_names +- delivery queue 摘要 + +这时节点进入： + +- `managed candidate` + +### 4. 立即跑接管验收 + +控制面不应等待人工另开命令，而应立刻执行： + +- `onboarding.acceptance` + +必要时自动追加： + +- `inspection.standard` + +这样新节点接入不会停留在“能连上就算完成”，而会直接走到： + +- 基础服务是否正常 +- 日志是否可收口 +- 诊断是否可采 +- worker / api 拓扑是否正确 + +### 5. 如果有问题，先走现场面而不是手工 SSH + +验收失败后，优先动作应该是： + +- 查看 `activity-stream` +- 打开 `playbook run detail` +- 看 `inspection` +- 看 `delivery queue` + +只有 Node Agent 已明显不可用，才进入 SSH Rescue。 + +### 6. 验收通过后进入发布门禁 + +节点通过验收后，不应马上让它进入发布目标集合，而是先进入： + +- release gate readiness + +也就是明确判断： + +- agent 在线否 +- inspection 健康否 +- delivery queue 正常否 +- 是否允许 remote-agent rollout + +### 7. 先进入 Worker / Control 正确角色集合 + +接入后必须明确： + +- 它是 `worker` +- 还是 `control` +- 是否 `effective_worker` +- 是否允许参与 rollout + +不能让“纳管成功”和“自动进入发版目标”之间没有门槛。 + +### 8. 最后才进入 Release / Rollout + +到这一步才允许： + +- worker canary +- batch rollout +- advance / pause / rollback + +这条标准路径的真正意义是： + +> 新节点接入不再是一次性脚本，而是正式运维生命周期的开始 + +--- + +## 二十五、后续实现时绝不再回退的 10 条边界 + +为了避免系统越来越大以后又退回人工联调，下面 10 条边界建议直接视为终局铁律。 + +### 1. 不再把 `git pull` 当正式发布方式 + +正式发布只认： + +- release artifact +- checksum +- rollout + +### 2. 不再让节点自己决定版本 + +节点只能执行： + +- 下载 +- 校验 +- 切换 +- 回滚 + +不能自己挑版本。 + +### 3. 不再让页面、CLI、Codex 各自拼动作语义 + +所有动作必须收口到： + +- action template +- playbook +- rollout +- delivery queue action + +### 4. 不再把 SSH 当日常主链路 + +SSH 只保留： + +- bootstrap +- rescue + +### 5. 不再让回执队列治理脱离 `ops job` + +无论： + +- flush +- replay +- discard + +都必须形成正式留痕。 + +### 5.1 不再让发布兜底路径退回“打开面板” + +`release_progression` 现在也应该遵守同一原则： + +- 有明确 Launchpad 推荐时，优先走 `publish_latest_worker` +- 已经存在默认 Release、但 Launchpad 还没形成明确推荐时，兜底也要直接走 `create_release_rollout_worker` + +也就是说，不能因为“还没形成完整 launchpad 推荐”就退回到“打开 Worker 点状灰度面板”这种 UI 导航式动作。 + +### 6. 不再让“在线”直接等于“参与检测” + +必须明确区分： + +- 在线 +- 参与执行 +- 在线待命 +- 近窗有吞吐 +- 负载同步中 + +### 7. 不再让页面自己猜跳转落点 + +后端必须给： + +- `ui_intent` +- `run_code` +- `activity_key` +- `focus_step_key` + +### 8. 不再让日志体系只靠全量大文本 + +必须分层： + +- event +- playbook / job log +- runtime log +- diagnostics bundle + +### 9. 不再把 root/www 混合工作区当正常运维模型 + +正式版只允许： + +- release build side +- release deploy side + +角色分清，而不是在目标节点长期维护一份脏 git 工作区。 + +### 10. 不再让“设计口径”和“实现口径”分裂 + +所以现在开始，后续新增能力都建议优先先补到： + +- `ops_agent_protocol.md` +- `ops_driver_contract.md` +- `ops_playbook_contract.md` +- `release_hub_contract.md` + +再去写页面和执行器。 + +这样你这套系统才会越来越像： + +> 正式控制平台 + +而不是： + +> 一直迭代的一组强力调试脚本 + +## 二十六、海外单脑控制面的总检入口也要正式化 + +上面整套设计如果只停留在： + +- 有很多 endpoint +- 有很多检查脚本 +- 有很多 contract 文档 + +那后面现场一复杂，还是会退回“先想想这次该跑哪个脚本”。 + +所以海外单脑控制面还需要一个固定的总检入口： + +- `check_ops_center_stack.sh` + +它的职责不是替代： + +- `check_ops_contracts.sh` +- `check_ops_plane.sh` +- `check_node_agent.sh` +- `check_release_hub.sh` + +而是把这四层先按固定顺序收口成一次总览： + +1. 先看 API 与 contract registry +2. 再看 `link_snapshot / overview` 当前驾驶主线 +3. 再看托管节点、Node Agent、回执队列 +4. 最后看 ReleaseHub 与最近 playbook / activity + +这样后续海外主机第一次接手现场时，默认起手式就固定为： + +```bash +cd /opt/domaincheck/domain-api +bash domain-api/deploy/multi-region/check_ops_center_stack.sh http://127.0.0.1:8100 + +# 如果只想看最终收口，不展开全量原始 JSON +bash domain-api/deploy/multi-region/check_ops_center_stack.sh http://127.0.0.1:8100 summary +``` + +它本质上回答的是 5 个问题： + +- 当前控制面跑的是哪一版 contract +- 当前最该处理的 driver recommendation 是什么 +- 当前有哪些节点已经被正式接管、哪些还没进入稳定执行面 +- 当前回执队列有没有死信或待补发积压 +- 当前 ReleaseHub 是 ready / attention / blocked 哪一种 + +而在正式实现上，这份总检结果不应该只停留在“把几段 JSON 拼起来”，而是要直接形成一层可被人和程序共用的诊断输出： + +- `diagnosis.surface_status` + 说明总检表面层是不是完整联通 +- `diagnosis.automation_status` + 说明海外单脑自动化执行面是不是已经真正可接管 +- `diagnosis.stack_status` + 作为总检最终结论 +- `diagnosis.issues` + 作为问题清单 +- `diagnosis.next_step` + 作为默认下一步动作 +- `diagnosis.quick_commands` + 作为现场立刻可复制执行的命令集合 + +这样一来，总检结果就不只是“读给人看”的说明书，而是后续： + +- 海外 Codex 驾驶 +- 后台按钮自动诊断 +- 交接包导出 +- 标准作业入口编排 + +都可以直接消费的正式 contract。 + +对应的正式入口也应该固定为： + +- `GET /api/v1/ops/stack-diagnosis` + +CLI 总检脚本只是它的终端包装层，而不应该继续长期承担“总检规则真正定义者”的角色。 + +进一步说，CLI 默认动作入口也应该固定下来： + +- `bash domain-api/deploy/multi-region/drive_ops_center.sh stack-next` + +它不再要求海外操作人先从总检结果里人工抄： + +- `action_code` +- `focus_ref` +- `recommended command` + +而是直接消费： + +- `diagnosis.next_step` + +然后统一进入： + +- `driver-actions/resolve` +- `driver-actions/execute-resolved` + +这一步很重要，因为它把“总检”和“下一步执行”正式接成一条链： + +- 页面顶部总检入口 +- 海外 CLI 单入口 +- 后续海外 Codex 驾驶员 + +都不再自己猜“下一步该做什么”，而是统一信任一份正式 contract。 + +如果现场执行 `stack-next` 时拿到的是： + +- `404 Not Found` + +那也不要把它理解成“设计不成立”，而应理解为： + +- 运行中的控制面 API 版本还没升级到包含 `/api/v1/ops/stack-diagnosis` +- 或代码已同步，但 `domaincheck-api` 尚未重启 + +也就是说，后续现场排障里，`404` 属于： + +- 部署版本未对齐问题 + +而不是： + +- 总检 contract 本身设计错误 + +这一步很关键，因为它把“海外主机的第一反应”也变成正式 contract 的一部分，而不是继续依赖人工记忆。 + +也就是说，未来不管是： + +- 人工运维 +- 海外 Codex 驾驶员 +- 后台按钮触发自动诊断 + +都应该优先先看这份总检结果，再决定下一步钻取哪一层。 + +只有这样，这套体系才真正具备： + +> 单入口观察，分层下钻，协议不漂移 + +## 二十七、终局实现顺序不能再乱 + +为了避免后面继续一边联调、一边改口径，这套体系后续实现顺序必须固定。 + +建议严格按下面 6 段推进： + +### 1. 先冻结 contract，再扩页面和执行器 + +优先冻结的母本应固定为： + +- [ops_stack_diagnosis_contract.md](/www/wwwroot/getDomain/docs/schemas/ops_stack_diagnosis_contract.md:1) +- [ops_driver_contract.md](/www/wwwroot/getDomain/docs/schemas/ops_driver_contract.md:1) +- [ops_job_contract.md](/www/wwwroot/getDomain/docs/schemas/ops_job_contract.md:1) +- [ops_agent_protocol.md](/www/wwwroot/getDomain/docs/schemas/ops_agent_protocol.md:1) +- [release_hub_contract.md](/www/wwwroot/getDomain/docs/schemas/release_hub_contract.md:1) +- [ops_observability_contract.md](/www/wwwroot/getDomain/docs/schemas/ops_observability_contract.md:1) + +只有这些 contract 先稳住，页面按钮、CLI、Codex、Node Agent 才能围绕同一套结构化对象工作。 + +### 2. 再固定控制面的统一执行链 + +控制面日常动作主链必须固定为： + +1. `stack-diagnosis` +2. `driver-actions/resolve` +3. `driver-actions/execute-resolved` +4. `ops job` +5. `Node Agent / rollout executor` +6. `event / complete / activity` + +也就是说： + +- 页面不直接执行壳命令 +- CLI 不直接拼最终执行接口 +- Codex 不直接猜动作 target api + +所有高层动作都先下沉为统一 resolve / execute-resolved contract。 + +### 3. 再把 Node Agent 执行面补齐 + +Node Agent 侧必须先补齐： + +- `register` +- `heartbeat` +- `pull` +- `jobs/{job_id}/start` +- `jobs/{job_id}/complete` +- `jobs/{job_id}/events` +- 本地待补发 / 死信队列 + +直到这条链稳定之前，不应把“节点能 SSH 上去”误判成“执行面已经正式接管”。 + +### 4. 再把 Release Hub 收成正式发布面 + +Release Hub 后续必须坚持： + +- 包只认 artifact +- 发布只认 rollout gate +- 批次推进只认正式 rollout / ops job +- 节点不允许直接 `git pull` + +这样后续： + +- worker canary +- controller rollout +- 失败回滚 +- 灰度暂停 + +才会继续沿用同一个正式模型，而不是变回“发布时再单独写一套命令”。 + +### 5. 最后才是 SSH Rescue + +SSH 不应再作为日常主链路，只保留给下面几类场景： + +- Node Agent 尚未接管 +- Node Agent 失联 +- 运行环境损坏到 agent 无法自救 +- release 切换半途损坏，需要紧急修复目录或 systemd + +而且即使走 SSH，也要继续满足： + +- 创建正式 `ops job` +- 留下执行人、节点、动作来源 +- 回写控制面审计链 + +### 6. 页面和 Codex 只能做“消费者”，不能做第二大脑 + +终局架构里： + +- 页面负责展示和触发 +- CLI 负责终端入口 +- Codex 负责智能判断和编排 + +但三者都不能再自己定义： + +- 新状态机 +- 新动作对象 +- 新门禁口径 +- 新下一步推断逻辑 + +否则“海外单脑控制面”就又会退化成三套脑子。 + +## 二十八、上线前必须通过的收口验收 + +如果以后要说“这套系统已经进入正式长期可维护状态”，至少应通过下面 10 条验收。 + +### 1. 总检入口必须真实可用 + +下面命令必须能在海外控制面直接打通： + +```bash +cd /opt/domaincheck/domain-api +bash domain-api/deploy/multi-region/check_ops_center_stack.sh http://127.0.0.1:8100 summary +``` + +而且结果里必须能给出： + +- `diagnosis.stack_status` +- `diagnosis.issues` +- `diagnosis.next_step` + +### 2. 默认下一步必须可以直接消费 + +下面命令必须成立： + +```bash +bash domain-api/deploy/multi-region/drive_ops_center.sh stack-next http://127.0.0.1:8100 +``` + +也就是说，现场不应再需要人工从总检结果里抄： + +- `action_code` +- `focus_ref` +- `command` + +### 3. 所有标准动作都必须落为正式对象 + +后续任何标准动作都必须能落成： + +- `driver action` +- `ops job` +- 或 `playbook run` + +不允许再出现“后台按钮一按，后端直接去某台机器执行一个临时命令，但控制面里没有正式记录”。 + +### 4. Node Agent 接管和 runtime 心跳必须明确区分 + +控制面必须能够直接区分： + +- `已接管` +- `仅运行态在线` +- `待接入` +- `心跳过期` +- `失联` + +绝不能再用“API 能看到节点在线”代替“Node Agent 已正式接管”。 + +### 5. 回执队列必须能治理，不只是能看见 + +必须能在页面或海外单入口完成： + +- `queue-status` +- `queue-records` +- `queue-flush` +- `queue-replay` +- `queue-discard-record` + +否则后续只要出现一次跨地域回执积压，就又会退回到手工删文件。 + +### 6. 远端日志回传必须可开关 + +多机联调时，海外控制面必须能够： + +- 默认关闭，避免长期噪声 +- 调试时打开，拿到远端关键或全量日志 +- 调试结束后关闭 + +否则后续每新增一台大陆机器，都会重新回到“复制日志给 Codex 看”的低效模式。 + +### 7. Release Hub 必须可执行 canary 到 rollout + +至少要能稳定完成： + +- 选默认版本 +- 看 rollout gate +- 发 worker canary +- 看结果 +- 继续批次 +- 异常暂停 / 回滚 + +### 8. 节点发布必须只认 artifact，不认 git pull + +一旦进入正式执行链，节点侧必须坚持： + +- 不 `git pull` +- 不直接以运行目录为发布源 +- 不允许 controller / worker 各自拉不同本地临时代码 + +### 9. 救援必须是正式例外，不是常态 + +如果某类动作每次都需要： + +- SSH 上机 +- 手工重启 +- 手工改 env +- 手工粘日志 + +那说明它还没有进入正式执行面，不能算收口完成。 + +### 10. 页面、CLI、Codex 三端口径必须一致 + +同一条动作在三端看到的： + +- 状态 +- 下一步 +- 风险说明 +- focus_ref +- target_api + +必须一致。 + +只有这样，后面你才不会再遇到： + +- 页面说能做 +- CLI 说不能做 +- Codex 又判断成第三种情况 + +## 二十九、后续绝不能回退的 8 种反模式 + +后续无论现场多急，都不建议再回到下面这些模式。 + +### 1. 新增页面按钮，直接在后端执行 shell + +这是最容易把平台重新打回“脚本集合”的退路。 + +正确做法应始终是： + +- 先 resolve +- 再 execute-resolved +- 最终落成 `ops job` + +### 2. 页面、CLI、Codex 各自拼 request payload + +一旦三端自己拼： + +- `action_code` +- `target_api` +- `focus_ref` +- `execution chain` + +后面就一定会再次口径漂移。 + +### 3. 用 runtime 心跳冒充正式接管 + +“节点在线”不等于： + +- Node Agent 已接管 +- 可以标准发布 +- 可以标准巡检 +- 可以标准回执 + +这两者必须永远分开。 + +### 4. 把发布流程重新做成节点本地更新 + +正式发布链绝不应回退到： + +- `git pull` +- 覆盖运行目录 +- 临时复制文件 + +正确做法只能是： + +- 构建 artifact +- 生成 rollout +- 由执行面标准接收和切换 + +### 5. 把 SSH 当成默认执行器 + +SSH 只能是： + +- bootstrap +- rescue +- 临时过渡 + +而不能成为： + +- 日常发布 +- 日常巡检 +- 日常日志拉取 +- 日常服务控制 + +的默认方式。 + +### 6. 故障时重新回到“复制日志给 Codex” + +终局目标是： + +- 海外控制面统一拿日志 +- Codex 直接消费控制面 contract + +而不是每次故障又退回： + +- 现场机器跑命令 +- 人工复制粘贴 +- 再让 Codex 二次解释 + +### 7. 新增机器时再写一份特殊部署说明 + +以后新增大陆 controller / worker，不应再写： + +- “这台机器例外” +- “这台机器单独执行某个手工命令” +- “这台机器按另一套说明” + +而应继续只走： + +- bootstrap plan +- Node Agent 接管 +- onboarding acceptance +- 标准巡检 / rollout + +### 8. 因为赶时间，就跳过正式审计链 + +只要动作会改变远端节点现场，就必须有： + +- 发起人 +- 动作来源 +- 目标节点 +- 执行结果 +- 回执事件 + +否则后续节点一多，你就再也无法回答： + +- 是谁改的 +- 为什么改 +- 哪次改坏了 +- 哪台节点没跟上 + +这 8 条如果长期守住，后面这套系统才不会越做越散，而会越来越像真正的： + +> 海外单脑控制平台 + +## 三十、这一轮继续冻结的 4 组关键字段 + +这轮设计继续往前收口时，有 4 组字段必须正式冻结，否则后面最容易再次出现“页面一套、后端一套、Codex 一套”的漂移。 + +### 1. Execution Scene 的参与态字段必须固定 + +现在“节点在线”和“节点真正参与检测”已经被证明不能混为一谈。 + +所以后续 `execution_scene` 里每个节点都应该优先稳定返回： + +- `participation_state` +- `participation_label` +- `participation_reason` + +推荐值直接固定成： + +- `dispatch_active` +- `recent_only` +- `standby` +- `load_syncing` + +这样页面“参与检测节点”、CLI 摘要、海外 Codex 驾驶员才能直接区分： + +- 在线但未参与 +- 正在领任务 / 正在执行 +- 近窗刚参与但当前已收口 +- 控制面和现场仍在同步 + +这层后续建议直接以 [ops_observability_contract.md](/www/wwwroot/getDomain/docs/schemas/ops_observability_contract.md:1) 为观察面母本。 + +### 2. 远端日志回传必须从“有无样本”升级成“覆盖到哪些节点” + +远端检测日志回传后续不能再只回答： + +- 开了没 +- 有样本没 + +这还不够支撑多机现场。 + +正式 contract 还要继续稳定返回： + +- `source_node_summaries` +- `covered_participating_node_summaries` +- `missing_participating_node_summaries` + +也就是说，控制面要能直接回答： + +- 哪些参与节点已经有样本 +- 哪些参与节点还没有样本 +- 每台已覆盖节点当前拿到了多少关键行 / 全量行 +- 最近一条样本来自谁 + +这样后面你不需要再自己翻日志去判断“到底是哪台机器没回传”，页面和 Codex 都能直接聚焦缺口节点。 + +### 3. Node Agent 拉到的任务包裹必须固定 + +Node Agent 后续不能只拿到： + +- `action` +- `payload` + +然后自己去猜这是不是 rollout、是不是巡检、是不是日志采样。 + +正式 Agent Job Envelope 应继续带上： + +- `job_type` +- `step_key` +- `step_title` +- `policy` +- `release_context` +- `focus_ref` + +这样同一个 Agent 才能稳定承接： + +- 标准巡检 +- Worker 日志 +- 诊断收集 +- Release 部署 +- Rollout 回滚 + +这层后续建议直接以 [ops_agent_protocol.md](/www/wwwroot/getDomain/docs/schemas/ops_agent_protocol.md:1) 为执行面母本。 + +### 4. Release Hub 不能只显示版本，还必须冻结 manifest / readiness / preview + +Release Hub 如果只停在： + +- latest release +- rollout list + +那本质上还是“看版本页面”，不是正式发布控制面。 + +接下来必须固定的 3 个发布对象是： + +- `artifact manifest` +- `operational_readiness.rows` +- `rollout preview` + +它们分别回答： + +- 这个包到底会改哪些服务、怎么验、怎么回滚 +- 哪台节点为什么能发 / 不能发 +- 真正发 rollout 之前预计会分几批、风险在哪 + +这层后续建议直接以 [release_hub_contract.md](/www/wwwroot/getDomain/docs/schemas/release_hub_contract.md:1) 为发布面母本。 + +这 4 组字段一旦收死，后面再往下做页面、后端、CLI、Codex 自动驾驶时，就不会再反复经历： + +- 先做一个能跑的版本 +- 现场一复杂就发现字段不够 +- 再回头改 contract +- 然后前后端一起返工 + +后续真正高效的节奏应该变成： + +1. 先定 contract +2. 再做后端聚合 +3. 再做页面展示 +4. 最后接 Codex 自动驾驶 + +只有这样，Node Agent、Release Hub、OpsCenter 才会越来越像一个统一系统，而不是三块并排长大的模块。 diff --git a/docs/25_domainCheck_海外单脑控制面上线收口总表.md b/docs/25_domainCheck_海外单脑控制面上线收口总表.md new file mode 100644 index 0000000..b0f9be4 --- /dev/null +++ b/docs/25_domainCheck_海外单脑控制面上线收口总表.md @@ -0,0 +1,430 @@ +# 25 domainCheck 海外单脑控制面上线收口总表 + +## 一、这份文档解决什么问题 + +这份文档不是再讲设计，而是把当前项目进入正式上线前，真正需要收口的事项压成一张总表。 + +适用场景： + +- 海外主机已经作为单脑控制面 +- 大陆节点已经开始纳管 +- Ops Center / Codex Driver / CLI 都已经接入同一套 `ops` contract +- 目标从“能联调”切换到“能稳定上线、能持续运维” + +这份文档优先回答 4 个问题： + +1. 现在能不能继续收口 +2. 现在能不能正式发布 +3. 还有哪些阻断项没清 +4. 下一步到底先跑哪条命令 + +--- + +## 二、上线前统一原则 + +上线前必须统一成下面这条工作方式： + +- 海外主机作为唯一主驾驶席 +- 页面、CLI、Codex 共用同一套后端判断 +- 默认先看 `go-live-summary` +- 真要解释原因时再下钻 `stack-diagnosis` +- 真要执行动作时走 `driver-resolve / execute-resolved` +- 真要发布或放量时走 `Release Hub / rollout` + +不再推荐： + +- 人工分别登录多台大陆机器拼状态 +- 每次上线都从 `journalctl + systemctl + curl` 临时组合判断 +- 前端、CLI、Codex 各自维护不同的默认下一步 + +--- + +## 三、正式上线门禁 + +### 1. 收口门禁 + +至少同时满足： + +- `go_live_summary.go_live_status != blocked` +- `stack_diagnosis.diagnosis.stack_status != blocked` +- `route_surface_complete=true` +- `driver-feed.automation_coverage.launch_status != blocked` +- 首屏默认下一步已经明确，不再是模糊人工判断 + +### 2. 发布门禁 + +至少同时满足： + +- `publish_ready=true` +- `launchpad_status != blocked` +- 不存在未处理的关键 `blocking_reasons` +- 当前发布主车道已经清晰落在： + - `review_smart_rollout_preview` + - `review_control_rollout` + - `publish_latest_worker` + - `create_release_rollout_*` + +### 3. 运维门禁 + +至少同时满足： + +- 海外控制面 API 稳定在线 +- 大陆 controller / worker 心跳稳定 +- 节点接管缺口已经收口，或至少首个缺口节点已有明确恢复动作 +- 远端日志回传可按需开启、复核、关闭 +- 运行中心首屏能直接区分： + - 在线但未参与 + - 正在执行 / 正在领任务 + +--- + +## 四、真正的起手顺序 + +每次准备上线、复核、值班接手时，都按这个顺序来： + +### 第 1 步：看上线收口摘要 + +```bash +cd /www/wwwroot/getDomain +bash domain-api/deploy/multi-region/drive_ops_center.sh go-live-check http://127.0.0.1:8100 http://127.0.0.1:8100 summary +``` + +如果当前是临时海外控制面联调，也可以把第二个地址替换成海外目标 API。 + +第一眼重点只看： + +- `go_live_status` +- `publish_status` +- `blocking_reasons` +- `warnings` +- `next_step_action_code` +- `operator_title` + +如果你怀疑当前是环境本身有漂移，而不是业务链路没收口，先执行： + +```bash +bash domain-api/deploy/multi-region/drive_ops_center.sh env-audit +bash domain-api/deploy/multi-region/drive_ops_center.sh runtime-refresh-recover +bash domain-api/deploy/multi-region/drive_ops_center.sh go-live-recover +``` + +重点只看： + +- `status` +- `headline` +- `missing_items` +- `tooling_items` +- `runtime.preflight_ok` +- `runtime.readiness_status` +- `runtime.route_surface_complete` +- `runtime.repo_capability_drift` +- `runtime.runtime_may_need_restart` +- `recommended_actions` + +如果这里出现： + +- `runtime.repo_capability_drift=true` +- `runtime.runtime_may_need_restart=true` + +则优先执行 `runtime-refresh-recover`，不要先把问题误判成“代码还没写完”。这通常表示： + +- 仓库代码已经更新 +- 但运行中的 `domaincheck-api` 进程还没重启到这版代码 + +`runtime-refresh-recover` 会固定给出： + +- 当前是否真的属于运行时版本漂移 +- 推荐先跑的 `runtime-refresh-recover` 统一恢复入口 +- 重启后应该按什么顺序继续： + - `stack-diagnosis` + - `node-bootstrap-plan` + - `stack-next` + - `doctor-decision` + +如果你已经不想再手工一条条执行，而是想把“刷新运行时 + 再做总检复核”压成一次操作，直接执行： + +```bash +bash domain-api/deploy/multi-region/drive_ops_center.sh go-live-recover +``` + +它会固定串起： + +1. `runtime-refresh-recover` +2. `stack-diagnosis summary` +3. `stack-next` +4. `go-live-check summary` +5. `doctor-decision` + +如果要把这次上线前检查直接导出成一整包证据，而不是手工复制多段终端输出，直接执行： + +```bash +bash domain-api/deploy/multi-region/drive_ops_center.sh go-live-export +``` + +它会统一导出： + +- `env-audit` +- `go-live-check` 摘要 +- `go-live-summary` +- `stack-diagnosis` +- `driver-feed` +- `codex-brief` +- `release-launchpad` +- `doctor-decision` +- `doctor-export` +- `manifest.json` + +其中 `manifest.json` 会直接标记： + +- 环境审计当前是 `ready / attention / blocked` +- 哪些 artifact 成功 +- 哪些 artifact 超时 +- 哪些 endpoint 缺失或返回异常 +- 当前推荐的阅读顺序 + +现在 Ops Center 首屏也会同步读取最新 bundle / manifest： + +- 顶部 `总检决策` +- 详情区 `总检主决策详情` +- API `GET /api/v1/ops/doctor-decision` +- 顶部 `交付证据` +- 详情区 `交付证据详情` +- API `GET /api/v1/ops/go-live-bundle` +- 顶部 `正式复核` +- 详情区 `正式复核详情` +- API `GET /api/v1/ops/go-live-review` + +如果你想先拿到一句“bundle manifest 正式复核是否通过”的统一结论，而不是直接跳到最终签收，也可以先执行： + +```bash +bash domain-api/deploy/multi-region/drive_ops_center.sh go-live-review +``` + +如果你想在 bundle 基础上直接得到一句“现在能不能签字上线”的统一结论，而不是人工再看多份 JSON，直接执行： + +```bash +bash domain-api/deploy/multi-region/drive_ops_center.sh go-live-signoff +``` + +或基于已有 bundle： + +```bash +bash domain-api/deploy/multi-region/drive_ops_center.sh go-live-signoff /path/to/go-live-bundle +``` + +这里的 `go-live-signoff` 已经不是另一套独立摘要，而是固定压缩： + +- `go-live-review` +- `doctor-decision` +- `go-live-summary / 发布门禁 / launchpad 摘要` + +所以页面首屏、CLI 与海外 Codex 看到的是同一份最终签字口径。 + +第一眼重点只看： + +- `signoff_status` +- `headline` +- `release_gate` +- `blocked_reasons` +- `attention_reasons` +- `decision` + +### 第 2 步：看总检详情 + +```bash +bash domain-api/deploy/multi-region/drive_ops_center.sh stack-diagnosis +``` + +重点只看： + +- `stack_status` +- `issues` +- `next_step` +- `quick_commands` + +### 第 3 步：看驾驶主线 + +```bash +bash domain-api/deploy/multi-region/drive_ops_center.sh driver-feed +bash domain-api/deploy/multi-region/drive_ops_center.sh codex-brief +``` + +重点只看： + +- `top_recommendation` +- `automation_coverage` +- `recommended_behavior` +- `focus` + +### 第 4 步：确认默认下一步 + +如果只是预览： + +```bash +bash domain-api/deploy/multi-region/drive_ops_center.sh stack-next +``` + +如果已经确认要执行： + +```bash +bash domain-api/deploy/multi-region/drive_ops_center.sh stack-next run confirm +``` + +--- + +## 五、当前项目的主处理车道 + +上线前遇到问题时，不要发散排查，先归到下面 5 条主车道： + +### 1. 节点接管车道 + +典型信号： + +- `remote_access_ready=0` +- `bootstrap_run` +- `run_acceptance` +- `managed_nodes_agent_pending` + +优先命令： + +```bash +bash domain-api/deploy/multi-region/drive_ops_center.sh agent-gap-check +bash domain-api/deploy/multi-region/drive_ops_center.sh agent-gap-recover +bash domain-api/deploy/multi-region/drive_ops_center.sh node-onboarding http://127.0.0.1:8100 mainland-worker-01 +``` + +### 2. 远端日志车道 + +典型信号： + +- 首屏显示“现场日志覆盖不足” +- `log_sync_enabled=false` +- `line_count=0` +- `source_nodes` 缺节点 + +优先命令： + +```bash +bash domain-api/deploy/multi-region/drive_ops_center.sh log-sync-check +bash domain-api/deploy/multi-region/drive_ops_center.sh log-sync-recover +bash domain-api/deploy/multi-region/drive_ops_center.sh scene-node-log http://127.0.0.1:8100 mainland-worker-01 120 key +``` + +### 3. 总检修复车道 + +典型信号： + +- `stack_status=blocked` +- `surface_status=broken` +- `contracts` / `launchpad` / `activity-stream` 缺口 + +优先命令： + +```bash +bash domain-api/deploy/multi-region/check_ops_center_stack.sh http://127.0.0.1:8100 +bash domain-api/deploy/multi-region/check_ops_contracts.sh http://127.0.0.1:8100 +bash domain-api/deploy/multi-region/check_release_hub.sh http://127.0.0.1:8100 +``` + +### 4. 检测参与车道 + +典型信号： + +- 页面显示在线节点多，但真正跑检测的少 +- `claimed_by` 分布异常 +- 在线但未参与节点过多 + +优先命令： + +```bash +bash domain-api/deploy/multi-region/check_worker_participation.sh +bash domain-api/deploy/multi-region/drive_ops_center.sh activity-stream +``` + +### 5. 发布 / 放量车道 + +典型信号： + +- `publish_ready=true` +- `launchpad_status` 已可进入 Worker / Control rollout +- 默认下一步已落在 Release Hub + +优先命令： + +```bash +bash domain-api/deploy/multi-region/drive_ops_center.sh release-launchpad +bash domain-api/deploy/multi-region/drive_ops_center.sh codex-focus-preview +``` + +--- + +## 六、首屏通过标准 + +Ops Center 首屏如果要算“可上线收工”，至少应满足： + +- 首屏能同时显示： + - 上线收口状态 + - 发布闸门状态 + - 自动化收口状态 + - 主处理车道 + - 默认下一步 + - 后端接管覆盖 + - 观察层完整度 + - 现场日志覆盖度 +- 首屏动作条可直接完成： + - 默认下一步 + - 定位下一步 + - 看契约 + - 复制命令 + - 进入主处理面板 +- 首屏回执条能统一反馈： + - 打开契约成功 + - 定位成功 + - 复制成功 + - 后端动作执行成功 / 警告 / 失败 + +如果这些已经稳定，就说明值班同事和海外 Codex 已经在同一个驾驶台上工作，而不是各自再拼逻辑。 + +--- + +## 七、通过标准后的最小上线动作 + +当上面门禁都通过后，建议最小上线动作顺序为： + +1. 跑 `go-live-check` +2. 看 `stack-diagnosis` +3. 复核 `driver-feed.automation_coverage` +4. 复核 `codex-brief.recommended_behavior` +5. 若进入发布车道，先看 `release-launchpad` +6. 对 `guarded_auto` 动作显式确认后再执行 + +上线前最后一轮建议保留证据： + +- `go-live-summary` 输出 +- `stack-diagnosis` 输出 +- `driver-feed` 输出 +- `codex-brief` 输出 +- `release-launchpad` 输出 +- 前端构建成功记录 + +--- + +## 八、现在这套体系是否已经进入可上线状态 + +按当前仓库现状，可以认为已经具备： + +- 单脑控制面的协议骨架 +- 首屏驾驶舱骨架 +- 总检与驾驶 contract +- Release Hub / Driver / Node Agent 的联动骨架 +- 海外 CLI 单入口 + +但正式宣称“可以上线”，仍建议每次以这份总表复核，而不是只凭某一个页面截图或某一次联调成功就直接跳过。 + +这份文档的定位就是： + +> 后续每次上线、值班接手、发布前复核，都先回到这里。 + +如果你已经准备进入“这次要不要真的发”的最终执行阶段，下一份应直接看： + +- `docs/26_domainCheck_发布前运行验证与交付模板.md` diff --git a/docs/26_domainCheck_发布前运行验证与交付模板.md b/docs/26_domainCheck_发布前运行验证与交付模板.md new file mode 100644 index 0000000..b952a20 --- /dev/null +++ b/docs/26_domainCheck_发布前运行验证与交付模板.md @@ -0,0 +1,635 @@ +# 26 domainCheck 发布前运行验证与交付模板 + +## 一、这份模板怎么用 + +这份文档不是设计说明，而是正式发布前最后一轮执行模板。 + +适用场景： + +- 海外单脑控制面已经搭好 +- 大陆节点已经接入或正在收口 +- 代码已经更新到待发布版本 +- 现在要做最后一轮运行验证、证据留存、交付确认 + +这份模板分成 3 段： + +1. 发布前运行验证 +2. 上线证据导出 +3. 最终交付结论模板 + +--- + +## 二、发布前运行验证 + +### 1. 先跑统一收口摘要 + +```bash +cd /www/wwwroot/getDomain +bash domain-api/deploy/multi-region/drive_ops_center.sh go-live-check http://127.0.0.1:8100 http://127.0.0.1:8100 summary +``` + +记录结果： + +- `go_live_status` +- `publish_status` +- `blocking_reasons` +- `warnings` +- `next_step_action_code` +- `operator_title` +- `launchpad_recommended_target_node_code` +- `launchpad_recommended_recovery_label` +- `launchpad_recommended_recovery_summary` +- `launchpad_onboarding_bootstrap_pending_nodes` +- `launchpad_onboarding_acceptance_ready_nodes` + +通过标准： + +- `go_live_status != blocked` +- `publish_status != blocked` + +### 1.5 先看环境差异是否已收口 + +```bash +bash domain-api/deploy/multi-region/drive_ops_center.sh env-audit +bash domain-api/deploy/multi-region/drive_ops_center.sh runtime-refresh-recover +bash domain-api/deploy/multi-region/drive_ops_center.sh go-live-recover +``` + +重点只看： + +- `status` +- `headline` +- `python.pytest_installed / fastapi_installed / uvicorn_installed` +- `python.runtimes` +- `services` +- `runtime.preflight_ok` +- `runtime.readiness_status` +- `runtime.route_surface_complete` +- `missing_items` +- `tooling_items` + +通过标准建议： + +- `status != blocked` +- `runtime.preflight_ok=true` +- `route_surface_complete=true` +- 如果 `missing_items` 里仍有关键基础项，先补环境，再继续业务复核 +- 如果只有 `tooling_items`，可以继续上线复核，但建议后补本机工具链 +- 如果 `runtime.runtime_may_need_restart=true` + - 不要直接继续接管或发布链 + - 先按 `runtime-refresh-recover` 给出的固定顺序完成 API 重启与复检 +- 如果你希望把这一轮“重启 + 总检 + 默认下一步 + 上线摘要”压成一次复核 + - 直接执行 `go-live-recover` + +### 1.6 先看托管节点接入面是否清楚 + +```bash +bash domain-api/deploy/multi-region/drive_ops_center.sh nodes http://127.0.0.1:8100 +``` + +重点只看： + +- `summary.agent_ready` +- `summary.ssh_ready` +- `summary.remote_access_ready` +- `summary.remote_access_state_counts` +- 每台节点的 `agent_state` +- 每台节点的 `remote_access_state` +- 每台节点的 `ssh_access_label` +- 每台节点的 `ssh_entry` + +通过标准建议： + +- 如果某台节点还是 `agent_pending` + - 先确认是否已经保存 SSH 入口 +- 如果节点尚未保存 SSH 入口，但你已经决定由海外控制面统一接管 + - 先补录： + +```bash +bash domain-api/deploy/multi-region/drive_ops_center.sh node-bind-ssh http://127.0.0.1:8100 mainland-worker-01 121.204.244.248 root 22 +``` + +- 补录后立即复查： + +```bash +bash domain-api/deploy/multi-region/drive_ops_center.sh nodes http://127.0.0.1:8100 +bash domain-api/deploy/multi-region/drive_ops_center.sh node-handover http://127.0.0.1:8100 mainland-worker-01 +``` + +- 如果已经 `agent_ready` + - 说明该节点已进入标准远端执行器接管面 +- 如果还不是 `agent_ready`，但 `ssh_ready` 已经成立 + - 说明至少已经具备海外主控经 SSH 进入节点完成 bootstrap / 验收的前置条件 + +### 2. 再看总检详情 + +```bash +bash domain-api/deploy/multi-region/drive_ops_center.sh stack-diagnosis +``` + +记录结果： + +- `stack_status` +- `issues` +- `next_step` +- `quick_commands` + +通过标准： + +- `stack_status != blocked` +- `issues` 里没有未处理的关键阻断项 + +### 3. 再看驾驶主线 + +```bash +bash domain-api/deploy/multi-region/drive_ops_center.sh driver-feed +bash domain-api/deploy/multi-region/drive_ops_center.sh codex-brief +``` + +记录结果： + +- `top_recommendation` +- `automation_coverage` +- `recommended_behavior` +- `focus` +- `summary.launchpad_recommended_target_node_code` +- `summary.launchpad_recommended_recovery_label` +- `summary.launchpad_onboarding_bootstrap_pending_nodes` +- `summary.launchpad_onboarding_acceptance_ready_nodes` + +通过标准： + +- `automation_coverage.launch_status != blocked` +- 默认下一步已经明确 +- 如果 `summary.launchpad_recommended_target_node_code` 非空 + - 值班同事应能直接知道当前卡在哪台节点 + - 不需要再回 launchpad 明细手工翻 gap rows + +### 4. 若涉及发布，再看 Release Hub + +```bash +bash domain-api/deploy/multi-region/drive_ops_center.sh release-launchpad +``` + +记录结果： + +- `launchpad_status` +- `recommended_action_code` +- `default_rollout_gate_status` +- `recommended_target_node_code` +- `recommended_recovery_label` +- `recommended_recovery_summary` +- `onboarding_bootstrap_pending_nodes` +- `onboarding_acceptance_ready_nodes` + +通过标准： + +- `launchpad_status != blocked` +- `default_rollout_gate_status != blocked` + +特殊判读： + +- 如果 `recommended_action_code=api-restart` + - 不要误判成“节点没接好” + - 这更像是运行中的控制面 API 还没刷新到当前仓库的最新运维能力 + - 应先执行： + +```bash +bash domain-api/deploy/multi-region/drive_ops_center.sh runtime-refresh-recover +``` + + - 然后重新跑： + +```bash +bash domain-api/deploy/multi-region/drive_ops_center.sh stack-diagnosis +bash domain-api/deploy/multi-region/drive_ops_center.sh release-launchpad +``` + +补充说明： + +- 从 `latest_release.json` 直接创建 Release，或执行“基于最新包的一键智能 Rollout”之前 +- 现在不仅要求 `smoke_test_ok=true` +- 还要求当前 `final_release_report.json` 与最新包完全匹配，且 `final_release_ok=true` +- 如果页面按钮被禁用，或 API 返回“最新发布包尚未完成最终签收” + - 优先检查 `final_release_report_stale_reason` + - 再检查 `final_release_gate_decision / final_release_gate_blocked_reasons` + +### 5. 若涉及节点接管，补跑节点验收 + +如果这次发布包含： + +- 新增大陆节点 +- 重新接管旧节点 +- 切换到 Node Agent 主接管模式 + +则不能只看总检摘要，还要对目标节点逐台完成验收。 + +先看验收计划： + +```bash +bash domain-api/deploy/multi-region/drive_ops_center.sh node-acceptance-plan http://127.0.0.1:8100 mainland-worker-01 +``` + +确认无误后执行： + +```bash +bash domain-api/deploy/multi-region/drive_ops_center.sh node-acceptance-run http://127.0.0.1:8100 mainland-worker-01 cli/acceptance +``` + +记录结果： + +- `node_code` +- `status` +- `checks` +- `blocking_issues` +- `warnings` +- `recommended_next_step` + +通过标准： + +- `status != blocked` +- 没有未处理的 `blocking_issues` +- 目标节点已经不再停留在“bootstrap pending / acceptance pending” + +### 6. 若涉及远端执行，补跑日志回传验证 + +如果当前版本准备进入： + +- 多机值班 +- 海外单脑统一接管 +- 远端动作回执与现场日志复核 + +则需要确认日志回传链已经可用，而不是只看节点在线。 + +先检查： + +```bash +bash domain-api/deploy/multi-region/drive_ops_center.sh log-sync-check +``` + +如果存在缺口，再执行恢复： + +```bash +bash domain-api/deploy/multi-region/drive_ops_center.sh log-sync-recover http://127.0.0.1:8100 key confirm cli/log-sync +``` + +必要时抓一份节点现场日志： + +```bash +bash domain-api/deploy/multi-region/drive_ops_center.sh scene-node-log http://127.0.0.1:8100 mainland-worker-01 120 full +``` + +记录结果： + +- `log_sync_enabled` +- `line_count` +- `source_nodes` +- `missing_nodes` +- `scene_log.status` + +通过标准： + +- 关键节点已出现在 `source_nodes` +- `missing_nodes` 不包含当前发布主车道节点 +- 如需人工复核现场，`scene-node-log` 能取到有效日志 + +### 7. 若涉及 Node Agent 交付，补看队列健康 + +如果这轮发布已经进入“控制面发动作、节点拉任务、节点回执结果”的模式，就必须确认 delivery queue 没有假健康。 + +先看队列摘要： + +```bash +bash domain-api/deploy/multi-region/drive_ops_center.sh queue-status http://127.0.0.1:8100 mainland-worker-01 +``` + +如果怀疑有堆积或死信，再看明细： + +```bash +bash domain-api/deploy/multi-region/drive_ops_center.sh queue-records http://127.0.0.1:8100 mainland-worker-01 dead_letter +``` + +如需重放： + +```bash +bash domain-api/deploy/multi-region/drive_ops_center.sh queue-replay http://127.0.0.1:8100 mainland-worker-01 20 cli/queue-replay +``` + +记录结果： + +- `delivery_queue.summary` +- `dead_letter` +- `retrying` +- `pending` +- `replayed_count` + +通过标准： + +- `dead_letter=0` +- 没有持续增长的 `retrying` +- 队列不处于异常堆积状态 + +注意： + +- `dead_letter > 0` 时，不应宣称“可正式发布” +- 这种情况至少按 `publish_status = blocked` 处理 + +### 8. 最后跑一次 doctor 结论 + +前面的检查是分面复核，最后还要再收成一句话，让交班、值班、Codex 驾驶员看到同一结论。 + +```bash +bash domain-api/deploy/multi-region/drive_ops_center.sh doctor-decision +``` + +记录结果： + +- `status` +- `decision` +- `recommended_commands` +- `recommended_reading_order` +- `launchpad_alignment` + +通过标准： + +- `status != blocked` +- `decision` 与前面的 `go-live-check / stack-diagnosis / release-launchpad` 不冲突 +- `recommended_commands` 已经指向明确的下一跳，而不是泛化排查 + +### 9. 若希望直接生成“可否签字上线”的最终摘要 + +如果你不想手工再拼： + +- bundle review +- doctor 结论 +- 发布门禁 + +可以直接执行： + +```bash +bash domain-api/deploy/multi-region/drive_ops_center.sh go-live-signoff +``` + +如果已经有现成证据包，也可以直接基于 manifest 输出： + +```bash +bash domain-api/deploy/multi-region/drive_ops_center.sh go-live-signoff /path/to/go-live-bundle +``` + +这一步现在不是“另一套单独判断”，而是固定复用： + +- `go-live-review` +- `doctor-decision` +- `go-live-summary / 发布门禁 / launchpad 摘要` + +也就是说，页面首屏、CLI 和海外 Codex 看到的最终签收结论，已经开始共享同一份签字口径。 + +重点只看： + +- `signoff_status` +- `headline` +- `release_gate` +- `blocked_reasons` +- `attention_reasons` +- `decision` +- `recommended_commands` + +通过标准： + +- `signoff_status=ready` + - 可进入最终人工签字或正式发布 +- `signoff_status=attention` + - 说明现场已经接近完成，但仍应先清 attention 项 +- `signoff_status=blocked` + - 本轮不应宣称收口完成 + +--- + +## 三、上线证据导出 + +### 1. 导出 bundle + +```bash +bash domain-api/deploy/multi-region/drive_ops_center.sh go-live-export +``` + +这会生成一整包上线前证据。 + +建议至少保留： + +- `00_env_audit.txt` +- `01_go_live_check_summary.txt` +- `02_go_live_summary.json` +- `03_stack_diagnosis.json` +- `04_driver_feed.json` +- `05_codex_brief.json` +- `06_release_launchpad.json` +- `manifest.json` + +如果当前轮次涉及节点接管，建议额外保留： + +- `agent_gap_export.json` +- `node_bootstrap_plan.txt` +- `node_acceptance_plan.json` + +如果当前轮次涉及远端执行与日志回传，建议额外保留： + +- `scene_log_*.txt` +- `doctor_decision.json` + +### 2. 直接复核 bundle + +```bash +bash domain-api/deploy/multi-region/drive_ops_center.sh go-live-review +``` + +如果你已经切到 Ops Center 页面，也可以直接看： + +- 首屏 `总检决策` +- `总检主决策详情` +- API `GET /api/v1/ops/doctor-decision` +- 首屏 `交付证据` +- `交付证据详情` +- API `GET /api/v1/ops/go-live-bundle` +- 首屏 `正式复核` +- `正式复核详情` +- API `GET /api/v1/ops/go-live-review` + +记录结果： + +- `status` +- `headline` +- `critical_failures` +- `noncritical_failures` +- `launchpad_alignment.consistent` +- `env_audit.status` +- `env_audit.missing_items` +- `recommended_next_steps` + +通过标准建议： + +- `status=ready` + - 可进入最终人工确认或正式发布 +- `status=attention` + - 核心报告已齐，但仍需复核补充失败项或环境缺口 + - 也包括 `go_live_summary / stack_diagnosis / driver_feed / codex_brief` 的 launchpad 摘要出现不一致 +- `status=blocked` + - 关键报告缺失，或环境审计已经判定阻断，不应直接宣称收口完成 + +如果 `env_audit.missing_items` 里出现： + +- `missing_env:/etc/default/domaincheck-node-agent` + +则这次 bundle review 已经不是泛泛地提示“环境缺口”，而是明确指向 Node Agent 接管还未收口。此时按下面顺序推进： + +1. `agent-gap-export` +2. `node-bootstrap-plan` +3. 如果 summary 已提示 `runtime_may_need_restart=true` +4. 完成恢复后重新导出 bundle，再做一次 `go-live-review` + +### 3. 生成最终交付包 + +运行验证、bundle 复核都通过后，再生成最终交付包。 + +如果当前机器已经具备 smoke 运行条件，直接执行： + +```bash +bash ./prepare_final_release.sh +``` + +如果当前机器只适合先做离线打包，可先跳过 smoke： + +```bash +DOMAINCHECK_SKIP_SMOKE_TEST=1 bash ./package_domain_release.sh +bash ./verify_domain_release.sh +``` + +后续再回到具备运行条件的机器上执行： + +```bash +bash ./prepare_final_release.sh +``` + +最后统一查看最新交付结果： + +```bash +bash ./show_latest_release.sh +``` + +重点只看： + +- `latest_release.package_name` +- `latest_release.archive_path` +- `latest_release.sha256` +- `latest_release.smoke_test_ok` +- `final_release_ok` +- `final_release_report` +- `final_release_report_ok` +- `final_release_report_matches_latest` +- `final_release_report_stale_reason` +- `final_release_gate_decision` +- `final_release_gate_blocked_reasons` +- `final_release_gate_verify_ok` +- `final_release_gate_smoke_test_ok` + +通过标准： + +- `verify_domain_release.sh` 返回 `ok=true` +- `latest_release.smoke_test_ok=true` +- `final_release_ok=true` +- `final_release_report_matches_latest=true` +- `final_release_gate_decision=ready` +- `final_release_gate_verify_ok=true` +- `final_release_gate_smoke_test_ok=true` +- `final_release_report` 已指向最新一轮生成的正式报告 +4. 先执行 `runtime-refresh-recover` +5. 再重新导出 bootstrap plan + +--- + +## 四、最终交付结论模板 + +下面这段可以直接作为正式发布前的交付结论模板使用。 + +### 模板正文 + +```text +本轮发布前运行验证已完成。 + +一、统一收口结论 +- go_live_status: <填写> +- publish_status: <填写> +- stack_status: <填写> +- automation_coverage.launch_status: <填写> +- launchpad_status: <填写> + +二、默认下一步 +- next_step_action_code: <填写> +- operator_title: <填写> +- launchpad_recommended_target_node_code: <填写> +- launchpad_recommended_recovery_label: <填写> + +三、上线证据包 +- bundle 路径: <填写> +- manifest 复核状态: <填写 ready/attention/blocked> +- go-live-review.headline: <填写> +- manifest.review_status_hint: <填写> +- critical_failures: <填写> +- noncritical_failures: <填写> +- manifest.launchpad_recommended_target_node_code: <填写> +- manifest.launchpad_recommended_recovery_label: <填写> +- manifest.launchpad_onboarding_bootstrap_pending_nodes: <填写> +- manifest.launchpad_onboarding_acceptance_ready_nodes: <填写> +- manifest.launchpad_alignment.consistent: <填写 true/false> + +四、结论 +- <填写：可进入正式发布 / 仍需继续收口 / 暂不可发布> +``` + +### 模板补充说明 + +填写“可进入正式发布”前，建议再做一次人工交叉确认： + +- 发布门禁是否为 `ready` 或至少非 `blocked` +- 当前主车道是否已经明确 +- 是否还存在 `dead_letter` +- 是否还有节点停留在 `bootstrap pending / acceptance pending` +- 是否已经保留可复盘的证据包与 manifest + +--- + +## 五、交付给值班同事时最少要带什么 + +最少带下面 4 样： + +1. `go-live-check` 摘要 +2. `stack-diagnosis` 结果 +3. `driver-feed / codex-brief` 结果 +4. `go-live-export` 生成的 `manifest.json` + +如果值班同事需要直接接手节点问题，再多带 3 样： + +1. `node-acceptance-run` 最近结果 +2. `log-sync-check` 最近结果 +3. `queue-status` 或 `queue-records dead_letter` 最近结果 + +这样交班的人不需要重新从零拼现场。 + +--- + +## 六、真正的收工标准 + +如果要算“这次版本已经可以收工”，建议至少同时满足： + +- 前端构建成功 +- 海外单脑驾驶舱首屏状态正常 +- `go-live-review.status != blocked` +- 发布门禁不为 `blocked` +- Node Agent 目标节点已完成接管验收 +- 远端日志回传链可用或已确认本轮不依赖 +- delivery queue 不存在未处理 `dead_letter` +- 当前默认下一步已经不是“继续补基础设施”，而是进入正式发布 / 观察 / 验收车道 + +到这一步，才适合说： + +> 当前项目已经进入可上线、可交付、可持续运维状态。 diff --git a/docs/schemas/ops_agent_protocol.md b/docs/schemas/ops_agent_protocol.md new file mode 100644 index 0000000..baff6a1 --- /dev/null +++ b/docs/schemas/ops_agent_protocol.md @@ -0,0 +1,816 @@ +# domainCheck Ops Agent Protocol + +## 1. 目标 + +这份文档用于冻结 `Node Agent <-> Overseas Control Plane` 的正式 contract。 + +适用范围： + +- `POST /api/v1/ops/agent/tokens` +- `POST /api/v1/ops/agent/bootstrap-plan` +- `GET /api/v1/ops/nodes/{node_code}/handover` +- `POST /api/v1/ops/nodes/{node_code}/handover/bootstrap-plan` +- `POST /api/v1/ops/agent/register` +- `POST /api/v1/ops/agent/heartbeat` +- `POST /api/v1/ops/agent/pull` +- `POST /api/v1/ops/agent/jobs/{job_id}/start` +- `POST /api/v1/ops/agent/jobs/{job_id}/complete` +- `POST /api/v1/ops/agent/jobs/{job_id}/events` + +设计原则： + +- Agent 只执行声明过的结构化动作 +- 控制面负责调度、门禁、编排和审计 +- 所有回执都必须可重放、可去重、可死信化 + +--- + +## 2. 公共响应包裹 + +所有接口统一返回： + +```json +{ + "code": 0, + "message": "ok", + "detail_code": "", + "data": {} +} +``` + +约束： + +- `code=0` 表示成功 +- `code=1` 表示业务失败 +- `detail_code` 用于结构化错误语义 +- `message` 只承担可读说明，不承担机器判断主逻辑 + +常见 `detail_code`： + +- `agent_token_invalid` +- `agent_token_expired` +- `agent_token_node_mismatch` +- `ops_job_not_owned_by_agent` +- `ops_job_invalid_status_transition` +- `release_checksum_mismatch` +- `release_health_check_failed` + +--- + +## 3. 认证与请求头 + +Agent 相关接口统一使用： + +- `Content-Type: application/json` +- `X-Domaincheck-Agent-Token: ` + +其中： + +- `tokens` +- `bootstrap-plan` + +不要求 Agent Token。 + +其余接口都要求 Token。 + +--- + +## 4. 公共节点载荷 + +`register` 与 `heartbeat` 当前共享 `_base_payload()`。 + +最小字段： + +```json +{ + "node_code": "mainland-worker-02", + "region": "mainland", + "role": "worker", + "title": "mainland-worker-02", + "hostname": "host-a", + "ip": "10.0.0.12", + "agent_version": "0.1.0", + "capabilities": [ + "service.start", + "service.stop", + "service.restart", + "service.status", + "health.snapshot", + "logs.collect", + "diagnostics.collect", + "deploy.release" + ], + "labels": {}, + "metadata": { + "service_names": { + "api": "domaincheck-api", + "worker": "domaincheck-worker", + "sync_agent": "domaincheck-sync-agent", + "node_agent": "domaincheck-node-agent" + }, + "delivery_queue": { + "state": "healthy", + "label": "正常", + "reason": "当前没有待重试回执，也没有死信记录。", + "pending_count": 0, + "dead_letter_count": 0, + "last_flush_at": "", + "oldest_pending_at": "", + "oldest_pending_request_id": "", + "oldest_pending_kind": "", + "oldest_dead_letter_at": "", + "oldest_dead_letter_request_id": "", + "oldest_dead_letter_kind": "" + } + } +} +``` + +说明： + +- `service_names` 是控制面识别节点服务拓扑的最小入口 +- `delivery_queue` 是控制面识别 Agent 回执现场的最小入口 + +--- + +## 5. Bootstrap Contract + +### 5.1 Issue Token + +`POST /api/v1/ops/agent/tokens` + +请求体最小字段： + +```json +{ + "node_code": "mainland-worker-02", + "issued_by": "web-ui", + "expires_in_hours": 72, + "metadata": { + "source": "ops-center" + } +} +``` + +返回体关键字段： + +- `token` +- `token_preview` +- `record_id` +- `node_code` +- `expires_at` +- `created_at` + +### 5.2 Bootstrap Plan + +`POST /api/v1/ops/agent/bootstrap-plan` + +请求体最小字段： + +```json +{ + "node_code": "mainland-worker-02", + "node_region": "mainland", + "node_role": "worker", + "issued_by": "web-ui", + "expires_in_hours": 72, + "control_plane_base_url": "https://ops.example.com", + "root_dir": "/opt/domaincheck", + "metadata": { + "source": "ops-center" + } +} +``` + +返回体关键字段： + +- Token 相关字段 +- `control_plane_base_url` +- `bootstrap_plan.node_code` +- `bootstrap_plan.node_region` +- `bootstrap_plan.node_role` +- `bootstrap_plan.root_dir` +- `bootstrap_plan.env_file` +- `bootstrap_plan.service_name` +- `bootstrap_plan.service_file` +- `bootstrap_plan.install_script_path` +- `bootstrap_plan.env_content` +- `bootstrap_plan.command_lines` +- `bootstrap_plan.command_block` +- `bootstrap_plan.health_checks` +- `bootstrap_plan.health_check_block` +- `bootstrap_plan.bootstrap_script_name` +- `bootstrap_plan.bootstrap_script_path` +- `bootstrap_plan.bootstrap_script_content` +- `bootstrap_plan.bootstrap_run_script_block` + +约束： + +- `bootstrap_plan` 是标准接入方案对象 +- 页面、Codex、CLI 必须复用这一个对象 +- 不允许各端再次手写 env 或 shell 逻辑 + +--- + +## 6. Runtime Contract + +### 6.1 Register + +`POST /api/v1/ops/agent/register` + +语义： + +- 校验 Token 与 `node_code` +- Upsert Agent 运行态 +- 将节点推入托管目录候选态 + +最小成功返回字段： + +- `node_code` +- `expires_at` +- `capabilities` + +### 6.2 Heartbeat + +`POST /api/v1/ops/agent/heartbeat` + +语义： + +- 刷新 `last_seen_at` +- 刷新节点身份、服务名和队列快照 +- 维持控制面中的 Agent 在线态 + +最小成功返回字段： + +- `node_code` +- `server_time` +- `expires_at` + +### 6.3 Pull + +`POST /api/v1/ops/agent/pull?limit=1` + +请求体最小字段： + +```json +{ + "node_code": "mainland-worker-02" +} +``` + +返回体关键字段： + +```json +{ + "jobs": [ + { + "id": 123, + "job_code": "ops-xxx", + "action": "health.snapshot", + "target_node_code": "mainland-worker-02", + "status": "dispatching", + "execution_mode": "remote-agent", + "payload": {}, + "metadata": {}, + "policy": {}, + "steps": [] + } + ], + "count": 1 +} +``` + +约束： + +- Agent 只能拿到属于自己的任务 +- Agent 不负责选择任务 +- 调度权始终在控制面 + +### 6.3.1 Agent Job Envelope + +为了避免 Agent 再从自然语言动作名里“猜执行上下文”，`pull.data.jobs[]` 当前已经按统一任务包裹返回。 + +当前 `pull.data` 顶层也会额外带： + +- `protocol_version` +- `envelope_type` +- `node_code` +- `limit` +- `count` + +推荐最小结构： + +```json +{ + "protocol_version": "ops-agent/v1", + "envelope_type": "agent_job", + "id": 123, + "job_id": 123, + "job_code": "ops-20260418-001", + "job_type": "ops_action", + "action": "logs.collect", + "target_node_code": "mainland-worker-02", + "status": "dispatching", + "execution_mode": "remote-agent", + "step_key": "worker_logs", + "step_title": "收集 Worker 日志", + "job_ref": { + "job_id": 123, + "job_code": "ops-20260418-001", + "action": "logs.collect", + "target_node_code": "mainland-worker-02" + }, + "step_ref": { + "step_id": 456, + "step_key": "worker_logs", + "step_title": "收集 Worker 日志" + }, + "payload": { + "service_name": "domaincheck-worker", + "lines": 120, + "include_agent_logs": false + }, + "policy": { + "timeout_seconds": 120, + "stop_on_failure": true, + "auto_approve": true + }, + "metadata": { + "requested_by": "playbook:inspection.standard", + "source": "ops-center" + }, + "release_context": { + "release_id": 0, + "release_version": "", + "rollout_id": 0, + "rollout_code": "" + }, + "focus_ref": { + "kind": "ops_job", + "job_id": 123, + "job_code": "ops-20260418-001", + "action": "logs.collect", + "target_node_code": "mainland-worker-02" + } +} +``` + +正式要求： + +- Agent 必须优先消费： + - `action` + - `payload` + - `policy` +- Agent 不应通过： + - `step_title` + - `summary` + - `notes` + 去反推真实执行参数 +- 与发布相关的任务应通过： + - `release_context` + 传递 release / rollout 关联，而不是让 Agent 自己查询“当前版本” + +这层的意义是： + +- `ops job` + - 仍是正式执行颗粒度 +- `Agent Job Envelope` + - 是 Node Agent 拿到的稳定执行视图 + +这样同一套 Agent 才能同时承接： + +- 标准巡检 +- 日志收集 +- 诊断采样 +- Release 部署 +- Rollout 回滚 + +### 6.4 Start + +`POST /api/v1/ops/agent/jobs/{job_id}/start` + +请求体最小字段： + +```json +{ + "node_code": "mainland-worker-02" +} +``` + +语义： + +- `dispatching` 表示已派发 +- `running` 表示节点已真实开工 + +### 6.5 Complete + +`POST /api/v1/ops/agent/jobs/{job_id}/complete` + +请求体最小字段： + +```json +{ + "node_code": "mainland-worker-02", + "client_request_id": "complete-ops-123-1", + "status": "success", + "stdout": "", + "stderr": "", + "result": { + "summary": "ok" + }, + "error_message": "" +} +``` + +允许状态： + +- `success` +- `failed` +- `partially_succeeded` + +幂等约束： + +- 使用 `client_request_id` +- 控制面写入 `ops_jobs.last_agent_complete_request_id` +- 同一完成回执可安全重放，不应再次制造副作用 + +推荐补充字段： + +- `duration_ms` +- `result.summary_text` +- `result.focus_ref` + +当前成功返回里也会额外带： + +- `completion_summary.job_id` +- `completion_summary.job_code` +- `completion_summary.status` +- `completion_summary.status_label` +- `completion_summary.result_summary_text` +- `completion_summary.focus_ref` +- `completion_summary.deduplicated` + +这样控制面在不展开 stdout/stderr 的情况下，也能直接把： + +- 任务收口摘要 +- 后续跳转落点 + +并入 activity / inspection / rollout 观察面。 + +### 6.6 Events + +`POST /api/v1/ops/agent/jobs/{job_id}/events` + +请求体最小字段： + +```json +{ + "node_code": "mainland-worker-02", + "client_event_id": "event-ops-123-checksum-1", + "step_id": 0, + "event_type": "deploy_checksum_verified", + "level": "info", + "message": "checksum ok", + "payload": { + "checksum": "sha256:..." + } +} +``` + +去重约束： + +- 使用 `client_event_id` +- 控制面通过唯一索引去重 +- 同一事件可安全重放，不应重复落库 + +推荐补充字段： + +- `occurred_at` +- `summary_text` +- `focus_ref` + +当前成功返回里也会额外带： + +- `event` + - 即标准化后的事件对象 + - 内含 `event_key / summary_text / occurred_at / focus_ref` + +这样事件流就不再只是“原始日志点”，而能直接成为： + +- playbook run events +- rollout events +- activity-stream + +共同消费的现场片段。 + +--- + +## 7. Agent 本地补发队列 + +当前主链路已经正式具备： + +- `pending` 队列 +- `dead-letter` 队列 +- 定时 `_flush_delivery_queue()` +- 永久性错误转死信 + +当前重点覆盖： + +- `jobs/{id}/complete` +- `jobs/{id}/events` + +队列状态： + +- `healthy` +- `retrying` +- `dead_letter` + +语义： + +- `healthy`：无积压，无死信 +- `retrying`：存在待补发回执，Agent 会继续自动重放 +- `dead_letter`：存在永久失败记录，需人工介入 + +控制面消费方式： + +- 节点维度： + - `delivery_queue_state` + - `delivery_queue_label` + - `delivery_queue_reason` + - `delivery_queue_pending_count` + - `delivery_queue_dead_letter_count` +- 汇总维度： + - `queue_retrying_nodes` + - `queue_dead_letter_nodes` +- `queue_pending_records` +- `queue_dead_letter_records` + +### 7.1 当前控制面已可见的最小现场 + +当前至少已经可以通过： + +- `GET /api/v1/ops/nodes` + +看到每台托管节点的队列现场： + +```json +{ + "node_code": "mainland-worker-02", + "delivery_queue_state": "dead_letter", + "delivery_queue_label": "死信 2", + "delivery_queue_reason": "当前存在 2 条死信记录，建议优先查看节点日志或诊断编排。", + "delivery_queue_pending_count": 0, + "delivery_queue_dead_letter_count": 2, + "delivery_queue_last_flush_at": "2026-04-18 06:10:00", + "delivery_queue_oldest_pending_at": "", + "delivery_queue_oldest_pending_request_id": "", + "delivery_queue_oldest_pending_kind": "", + "delivery_queue_oldest_dead_letter_at": "2026-04-18 05:59:00", + "delivery_queue_oldest_dead_letter_request_id": "complete-ops-123-1", + "delivery_queue_oldest_dead_letter_kind": "job_complete" +} +``` + +这已经足够让： + +- 页面显示“节点存在死信” +- `overview / driver-feed / codex-brief` 产出驾驶建议 +- CLI 快速判断现场是不是要先走日志或诊断 + +但这还不够支撑正式运维，因为它只能“看见死信”，还不能“处理死信”。 + +### 7.2 Delivery Queue Record 正式对象 + +后续控制面不应再把死信理解成一个纯计数器，而要把每条待补发 / 死信记录升级成正式对象。 + +推荐最小结构： + +```json +{ + "record_id": "dq-mainland-worker-02-20260418-001", + "node_code": "mainland-worker-02", + "state": "dead_letter", + "request_kind": "job_complete", + "client_request_id": "complete-ops-123-1", + "job_id": 123, + "job_code": "ops-20260418-001", + "target_api": "/api/v1/ops/agent/jobs/123/complete", + "detail_code": "ops_job_invalid_status_transition", + "error_message": "当前任务状态不允许 complete", + "attempt_count": 6, + "first_queued_at": "2026-04-18 05:58:00", + "last_attempt_at": "2026-04-18 05:59:00", + "next_retry_at": "", + "payload_preview": { + "status": "success" + }, + "response_preview": { + "code": 1, + "detail_code": "ops_job_invalid_status_transition" + } +} +``` + +正式约束： + +- `record_id` + - 是控制面侧唯一主键 +- `client_request_id` + - 是 Agent 幂等键 +- `detail_code` + - 是死信聚类主键，不能只靠 `message` +- `payload_preview / response_preview` + - 用于页面和 Codex 快速判断，不必默认展开完整原始报文 + +推荐状态： + +- `pending` +- `retrying` +- `dead_letter` +- `replayed` +- `discarded` + +### 7.3 死信操作面正式入口 + +为了让“看见死信”之后不用回节点本地处理，推荐把操作面固定成下面这组接口。 + +当前已落地的第一阶段能力： + +- `GET /api/v1/ops/nodes/{node_code}/delivery-queue` +- `GET /api/v1/ops/nodes/{node_code}/delivery-queue/records` + +当前控制面返回的记录可见性为： + +- `record_visibility=head_only` + +也就是： + +- 当前已经能稳定查看节点级队列总览 +- 也能看到 `pending / dead_letter` 的头部记录摘要 +- 但还没有把节点本地全量 `pending/*.json / dead-letter/*.json` 正式同步到控制面 + +所以这两条 GET 入口现在属于“正式只读观察面”，而不是完整操作面。 + +#### a. 单节点队列总览 + +- `GET /api/v1/ops/nodes/{node_code}/delivery-queue` + +最小返回体： + +```json +{ + "node_code": "mainland-worker-02", + "summary": { + "state": "dead_letter", + "pending_count": 0, + "dead_letter_count": 2, + "last_flush_at": "2026-04-18 06:10:00" + }, + "oldest_pending_record": {}, + "oldest_dead_letter_record": {} +} +``` + +#### b. 队列记录列表 + +- `GET /api/v1/ops/nodes/{node_code}/delivery-queue/records` + +推荐筛选参数： + +- `state=pending|retrying|dead_letter|replayed|discarded` +- `request_kind=job_complete|job_event` +- `detail_code=...` +- `group_by=detail_code|request_kind|target_api` +- `limit=50` + +当带 `group_by` 时，返回值应优先给出聚类摘要，而不是只返回平铺明细。 + +#### c. 单条死信重放 + +- `POST /api/v1/ops/nodes/{node_code}/delivery-queue/records/{record_id}/replay` + +当前已落地，但要明确当前阶段约束： + +- 控制面仍然是 `record_visibility=head_only` +- 所以单条动作目前只允许针对“当前可见头部记录” +- 真正的执行不是控制面直接改远端文件，而是创建正式 `ops job` +- 由目标节点 `Node Agent` 执行 `delivery.queue.replay` + +最小请求体： + +```json +{ + "requested_by": "web-ui", + "reason": "确认任务状态已修复，重放这条 complete 回执" +} +``` + +#### d. 批量重放 + +- `POST /api/v1/ops/nodes/{node_code}/delivery-queue/replay` + +当前已落地为正式操作面。 + +推荐请求体： + +```json +{ + "requested_by": "codex", + "selector": { + "state": "dead_letter", + "detail_code": "ops_job_invalid_status_transition" + }, + "limit": 20 +} +``` + +#### e. 丢弃死信 + +- `POST /api/v1/ops/nodes/{node_code}/delivery-queue/records/{record_id}/discard` + +当前已落地，但与单条重放一样，仍然只允许针对当前可见头部死信记录发起单条动作。 + +最小请求体： + +```json +{ + "discarded_by": "web-ui", + "reason": "确认这条回执不再需要补发" +} +``` + +#### f. 主动冲刷队列 + +- `POST /api/v1/ops/nodes/{node_code}/delivery-queue/flush` + +当前已落地为正式操作面，执行方式同样是： + +- 控制面创建 `ops job` +- 目标节点 `Node Agent` 执行 `delivery.queue.flush` + +语义： + +- 不是“强行成功” +- 而是立即触发一次 Agent 补发冲刷 +- 结果仍然回到 `pending / retrying / dead_letter / replayed` + +### 7.4 为什么必须做成正式操作面 + +这层不是为了让页面多一个按钮，而是为了保证： + +- 页面可以处理死信 +- CLI 可以处理死信 +- 海外 Codex 驾驶员可以处理死信 + +三者都不需要绕回： + +- 手工 SSH 登录节点 +- 手工删本地文件 +- 手工重放某条 HTTP 请求 + +正式规则应该固定成： + +- “死信查看” 走控制面对象 +- “死信重放 / 丢弃 / 冲刷” 走控制面对象 +- Agent 只负责执行，不负责决定如何处理死信 + +--- + +## 8. Job Status Enum + +当前 Agent 主链路约束： + +- `queued` +- `dispatching` +- `running` +- `success` +- `failed` +- `partially_succeeded` + +语义： + +- `queued`：任务已创建，尚未派发 +- `dispatching`：控制面已派发，但节点尚未确认开工 +- `running`：节点已真实执行 +- `success`：成功完成 +- `failed`：失败完成 +- `partially_succeeded`：部分成功，需关注 + +--- + +## 9. 实现边界 + +Agent 只执行已声明动作： + +- `service.start` +- `service.stop` +- `service.restart` +- `service.status` +- `health.snapshot` +- `logs.collect` +- `diagnostics.collect` +- `deploy.release` + +不允许控制面长期依赖任意 shell 下发。 + +`shell executor` 只能作为受限兜底能力存在，并必须挂审批与审计。 diff --git a/docs/schemas/ops_doctor_decision_contract.md b/docs/schemas/ops_doctor_decision_contract.md new file mode 100644 index 0000000..531b376 --- /dev/null +++ b/docs/schemas/ops_doctor_decision_contract.md @@ -0,0 +1,56 @@ +# ops_doctor_decision_contract + +## Purpose + +冻结 `doctor-decision` 主决策 contract，让页面、CLI、Codex 都消费同一份“当前先看哪一面、先执行什么动作、为什么”的正式结论。 + +## Producer + +- backend service `app.services.ops_service.get_ops_doctor_decision` +- API `GET /api/v1/ops/doctor-decision` + +## Primary Consumers + +- `domain-web` Ops Center 顶部总检入口与主决策详情面板 +- `domain-api/deploy/multi-region/drive_ops_center.sh doctor-decision` +- 海外 Codex 驾驶员 / 自动驾驶策略层 + +## Required Payload + +- `status` +- `status_label` +- `headline` +- `detail` +- `decision` +- `summary` +- `recommended_commands` +- `contract_navigation` + +## Decision Shape + +- `decision.status` +- `decision.reason_code` +- `decision.headline` +- `decision.detail` +- `decision.preferred_surface` +- `decision.next_action_code` +- `decision.recommended_commands` + +## Summary Shape + +- `summary.required_failures` +- `summary.optional_unavailable` +- `summary.scene_log_reports_total` +- `summary.scene_log_reports_ok` +- `summary.scene_log_status_counts` +- `summary.scene_log_problem_node_codes` +- `summary.contract_surface_gaps` +- `summary.launchpad_recommended_target_node_code` +- `summary.launchpad_recommended_recovery_label` +- `summary.launchpad_recommended_recovery_summary` + +## Notes + +- 优先读取最新 `go-live bundle` 中的 `doctor_decision` 产物。 +- 如果 bundle 尚未导出，后端必须回落到 live `stack-diagnosis + go-live-summary` 兜底，而不是直接返回空。 +- `preferred_surface` 必须稳定可消费，不能把面板跳转逻辑继续留给前端猜测。 diff --git a/docs/schemas/ops_driver_contract.md b/docs/schemas/ops_driver_contract.md new file mode 100644 index 0000000..265df3e --- /dev/null +++ b/docs/schemas/ops_driver_contract.md @@ -0,0 +1,797 @@ +# domainCheck Ops Driver Contract + +## 1. 目标 + +这份文档用于冻结 Overseas Ops Center / Codex Driver / CLI 共用的驾驶 contract。 + +核心原则： + +- 后端负责推荐和解释 +- 前端负责展示和触发 +- Codex 负责选择入口和上下文 +- 是否允许自动执行由后端统一门禁 +- 页面与 Codex 都不能各自再推导一套优先级 + +--- + +## 2. 入口接口 + +当前驾驶 contract 主要来自： + +- `GET /api/v1/ops/overview` +- `GET /api/v1/ops/driver-feed` +- `GET /api/v1/ops/codex-brief` +- `POST /api/v1/ops/driver-actions/preview` +- `POST /api/v1/ops/driver-actions/resolve` +- `POST /api/v1/ops/driver-actions/execute` +- `POST /api/v1/ops/driver-actions/execute-resolved` +- `POST /api/v1/ops/codex-actions/resolve` +- `POST /api/v1/ops/codex-actions/execute` + +补充配套： + +- `GET /api/v1/ops/playbooks` +- `GET /api/v1/ops/playbook-runs` +- `GET /api/v1/ops/playbook-runs/{run_code}` +- `GET /api/v1/ops/playbook-runs/{run_code}/events` +- `GET /api/v1/ops/activity-stream` + +--- + +## 3. Driver Recommendation + +来源： + +- `overview.driver_recommendations` + +最小字段： + +```json +{ + "key": "node-agent-dead-letter", + "priority": "先处理 Agent 死信", + "title": "先处理 Agent 死信", + "summary": "当前已有节点出现死信记录，继续堆动作会放大控制面与节点现场的不一致。", + "reason": "Node Agent 回执队列已经出现死信。", + "level_label": "最高优先", + "tag_type": "danger", + "primary_label": "查看 Worker 日志", + "secondary_label": "打开标准巡检", + "primary_action_code": "open_worker_logs", + "secondary_action_code": "open_diagnostics", + "node_codes": [ + "mainland-worker-01" + ], + "primary_node_codes": [ + "mainland-worker-01" + ], + "secondary_node_codes": [ + "mainland-worker-01" + ], + "primary_action_payload": {}, + "secondary_action_payload": {}, + "focus_ref": { + "kind": "ops_job", + "job_id": 101, + "job_code": "ops-20260418064500-a1b2c3" + } +} +``` + +要求： + +- 推荐不只给文案 +- 必须给动作码、节点和预填 payload +- 主次动作允许节点和 payload 不同 +- 推荐对象允许直接携带 `focus_ref / primary_focus_ref / secondary_focus_ref` +- 新客户端优先用 `focus_ref` 做统一定位，而不是再从动作码反推页面落点 + +--- + +## 4. Priority Recommendation + +来源： + +- `overview.recommendation` + +作用： + +- 给首页顶部、CLI 摘要、Codex 首屏判断提供“当前第一优先动作” + +最小字段： + +```json +{ + "source": "driver_recommendations", + "key": "node-agent-dead-letter", + "priority": "先处理 Agent 死信", + "title": "先处理 Agent 死信", + "summary": "当前已有节点出现死信记录。", + "reason": "Node Agent 回执队列已经出现死信。", + "primary_action_code": "open_worker_logs", + "secondary_action_code": "open_diagnostics" +} +``` + +--- + +## 5. Driver Feed + +来源： + +- `GET /api/v1/ops/driver-feed` + +作用： + +- 提供比 `overview` 更直接给驾驶员消费的聚合结果 + +建议最小结构： + +```json +{ + "top_recommendation": {}, + "driver_recommendations": [], + "runbook_sequences": [], + "activity_focus": [], + "automation_coverage": {} +} +``` + +推荐补充约束： + +- `top_recommendation` + - 应该总是来自同一份 `entries` 列表，而不是额外拼一条“影子建议” + - 这样页面、CLI、海外 Codex 在按 `key` 选中时才不会出现口径漂移 +- `driver_recommendations` + - 普通驾驶建议，优先 `executor_kind=driver_action` +- `runbook_sequences` + - 标准作业路径，优先 `executor_kind=runbook_sequence` +- `activity_focus` + - 更偏观察与跟踪，不应伪装成“可立即执行”的驾驶动作 +- `activity_focus[*]` + - 至少应保留： + - `activity_key` + - `activity_kind` + - `summary` + - `status` + - `status_label` + - `occurred_at` + - `focus_ref` + - `source_focus_ref` + - `ui_intent` + - 如果存在可落点观察入口 + - 允许补 `focus_action_code=focus_activity_item` + - 但它的定位仍然是观察入口，而不是标准执行动作 +- `entries[*].focus_ref` + - 是 driver-feed 的统一定位对象 + - 页面 / CLI / Codex 均应优先消费 +- `entries[*].primary_focus_ref` / `entries[*].secondary_focus_ref` + - 是主次动作各自的稳定落点 + - 不允许调用方再根据 `primary_action_code` 自己猜“该跳 Release Hub、Launchpad 还是 Rollout Jobs” +- `automation_coverage` + - 是 Driver Feed 的自动化收口摘要 + - 用于统一回答： + - 当前是否已经达到上线闸门 + - 当前还有多少动作停留在 preview-only + - 当前后端是否已经足够接管首屏默认动作 + - 页面、CLI、Codex 不允许自行再统计一份“自动化覆盖率” + +推荐最小结构： + +```json +{ + "launch_ready": false, + "launch_status": "attention", + "summary": "仍有 3 个驾驶动作停留在 preview-only，暂未达到上线收口标准。", + "total_actions": 12, + "backend_handled_total": 9, + "preview_only_total": 3, + "ui_only_total": 0, + "blocked_total": 0 +} +``` + +其中 `runbook_sequences` 与 `activity_focus` 应继续区分： + +- 这是建议动作 +- 还是固定标准作业路径 +- 还是最近值得优先处理的活动 + +推荐 CLI 消费顺序： + +1. 显式 `entry_key` +2. `top_recommendation` +3. `entries[0]` + +这样海外单入口就能稳定实现： + +- `driver-focus-preview` +- `driver-focus-run` + +--- + +## 6. Codex Brief + +来源： + +- `GET /api/v1/ops/codex-brief` + +作用： + +- 不让 Codex 自己从页面文案里猜“能不能自动执行” + +最小字段： + +```json +{ + "mode": "guarded_auto", + "recommended_behavior": "confirm_then_execute", + "summary": "当前建议先处理 Agent 死信，再进入标准巡检。", + "automation_coverage": { + "launch_ready": false, + "launch_status": "attention", + "summary": "仍有 3 个驾驶动作停留在 preview-only。" + }, + "target": { + "kind": "driver_action", + "action_code": "open_worker_logs", + "node_codes": [ + "mainland-worker-01" + ], + "action_payload": {} + } +} +``` + +推荐 `mode`： + +- `safe_auto` +- `guarded_auto` +- `mixed` +- `ui_only` +- `blocked` + +推荐 `recommended_behavior`： + +- `auto_execute` +- `confirm_then_execute` +- `resolve_first` +- `open_ui` +- `blocked` + +补充约束： + +- `codex-brief.automation_coverage` + - 应与 `driver-feed.automation_coverage` 保持同源 + - 供海外 Codex 驾驶员直接判断： + - 是否允许默认走自动执行 + - 是否必须先预览或人工确认 + - 是否已经达到“可以上线”的自动化收口标准 +- `codex-brief.activity_focus` + - 应直接透传 `driver-feed.activity_focus` + - 供 Codex 在“当前没有可执行主线，但有观察焦点”时继续判断 +- `codex-brief.focus` + - 优先使用 `entries[0]` + - 当 `entries` 为空且 `activity_focus` 不为空时 + - 允许回退到首条带 `focus_action_code` 的 `activity_focus` + - 这时推荐行为应偏向 `open_ui` + - 不应伪装成自动执行动作 + +--- + +## 7. Driver Action Execute + +来源： + +- `POST /api/v1/ops/driver-actions/execute` + +作用： + +- 把能标准化的驾驶动作收进统一后端执行入口 + +最小请求体： + +```json +{ + "action_code": "run_standard_inspection", + "node_codes": [ + "mainland-worker-01" + ], + "action_payload": {} +} +``` + +最小返回体： + +```json +{ + "handled": true, + "mode": "playbook", + "ui_intent": { + "kind": "playbook_run_detail", + "run_code": "run-20260418-001" + } +} +``` + +要求： + +- 不只返回 handled / unhandled +- 还要返回 `mode` +- 还要返回 `ui_intent` + +--- + +## 8. Driver Action Preview + +来源： + +- `POST /api/v1/ops/driver-actions/preview` + +作用： + +- 给 OpsCenter / CLI / Codex Driver 统一生成驾驶动作契约预览 +- 冻结 `execution_chain / target_api / request_payload / 主次动作 section` +- 不再让前端自己拼请求体和执行建议 + +最小请求体： + +```json +{ + "source_label": "driver-feed", + "title": "版本发布与放量", + "summary": "当前可以进入正式发布路径。", + "reason": "统一回到 Release / Rollout 闭环。", + "executor_kind": "runbook_sequence", + "sequence_key": "release_progression", + "primary_label": "Worker 灰度", + "secondary_label": "查看版本区", + "secondary_action_code": "focus_release_hub", + "requested_by": "web-ui/ops-center" +} +``` + +最小返回体： + +```json +{ + "contract_key": "ops_driver_contract", + "contract_version": "v1", + "contract_schema_doc_path": "docs/schemas/ops_driver_contract.md", + "execution_chain": "ops-driver -> runbook-sequence -> resolved driver action / ui-intent", + "focus_ref": { + "kind": "release_hub", + "release_id": 7, + "section": "release_launchpad" + }, + "primary_focus_ref": { + "kind": "release_hub", + "release_id": 7, + "section": "release_launchpad", + "mode": "worker" + }, + "secondary_focus_ref": { + "kind": "release_hub", + "release_id": 7, + "section": "release_detail" + }, + "primary": { + "label": "Worker 灰度", + "action_code": "create_release_rollout_worker", + "target_api": "/api/v1/ops/runbook/sequences/release_progression/execute", + "target_method": "POST", + "recommendation_label": "建议确认后执行", + "automation_level_label": "guarded_auto", + "risk_level_label": "high", + "focus_ref": { + "kind": "release_hub", + "release_id": 7, + "section": "release_launchpad", + "mode": "worker" + }, + "request_payload": { + "secondary": false, + "requested_by": "web-ui/ops-center", + "node_codes": [ + "mainland-controller-01" + ], + "action_payload": { + "release_id": 7, + "focus_ref": { + "kind": "release_hub", + "release_id": 7, + "section": "release_launchpad", + "mode": "worker" + } + } + } + } +} +``` + +要求： + +- preview 必须和 execute 走同一套 action 解析与 runbook resolve 逻辑 +- `primary` / `secondary` section 必须可直接给页面抽屉和 CLI 文本渲染复用 +- `request_payload` 必须是真正可发送到执行接口的 body，而不是页面内部临时对象 +- runbook sequence preview 必须经过后端 resolve，不能由页面自己猜 action_code +- preview 返回体必须保留： + - `focus_ref` + - `primary_focus_ref` + - `secondary_focus_ref` + - `primary.focus_ref` + - `secondary.focus_ref` + +--- + +## 8.3. Driver Action Resolve / Execute Resolved + +来源： + +- `POST /api/v1/ops/driver-actions/resolve` +- `POST /api/v1/ops/driver-actions/execute-resolved` + +作用： + +- 把 `overview.driver_recommendations` / `driver-feed` / 页面按钮 / CLI 对普通驾驶动作的门禁判断统一下沉到后端 +- 页面不再根据 `preview` 自己推导 `直接执行 / 等确认 / 进入 UI / 阻断` +- 后端统一输出： + - `preview_request` + - `preview` + - `gate` + - `selected_section` + - `execution_result` + +`resolve` 最小请求体： + +```json +{ + "source_label": "driver-feed", + "title": "开启关键日志回传", + "executor_kind": "driver_action", + "primary_action_code": "enable_log_sync_key", + "primary_action_payload": {}, + "requested_by": "web-ui/driver-feed", + "secondary": false, + "confirm": false +} +``` + +`resolve` 最小返回体： + +```json +{ + "preview_request": {}, + "preview": {}, + "selected_section": "primary", + "focus_ref": { + "kind": "execution_scene", + "scene_key": "scene.logs" + }, + "gate": { + "decision": "execute_now", + "decision_label": "直接执行", + "decision_reason": "当前动作属于 safe_auto，可直接执行。", + "will_execute": true, + "recommendation": "auto_execute", + "focus_ref": { + "kind": "execution_scene", + "scene_key": "scene.logs" + }, + "action_code": "enable_log_sync_key", + "target_api": "/api/v1/ops/driver-actions/execute" + } +} +``` + +`execute-resolved` 约束： + +- 只有 `gate.will_execute=true` 才允许真正执行 +- `confirmation_required` 只有显式传入 `confirm=true` 才会放行 +- `open_ui / resolve_first / blocked` 必须返回非执行态，由页面或 CLI 继续展示契约 +- `driver-focus-preview / driver-focus-run` + - 允许先从 `driver-feed.top_recommendation` 选出目标项 + - 也允许显式传入 `activity_focus.key` + - 当目标来自 `activity_focus` 时 + - 应自动收口成 `focus_activity_item` + - 只允许进入对应工作区，不应被当成标准执行动作 + - 再把它收口成同一份 `resolve / execute-resolved` 请求体 + - 不允许 CLI 再自己二次推导 gate +- resolve 返回体必须保留： + - `preview_request.focus_ref` + - `preview_request.primary_focus_ref` + - `preview_request.secondary_focus_ref` + - `gate.focus_ref` + - 顶层 `focus_ref` + +--- + +## 8.5. Codex Action Resolve / Execute + +来源： + +- `POST /api/v1/ops/codex-actions/resolve` +- `POST /api/v1/ops/codex-actions/execute` + +作用： + +- 不再让 CLI / Codex 自己从 `codex-brief + preview` 拼一套门禁 +- 由后端统一输出： + - 当前选中的 `selected_entry` + - 对应的 `preview` + - 统一门禁结论 `gate` + - 最终是否允许执行 + +`resolve` 最小请求体： + +```json +{ + "entry_key": "runbook:release_progression", + "requested_by": "cli/ops-center/codex-focus-preview", + "source_label": "cli/codex-focus-preview", + "include_secondary": true, + "confirm": false +} +``` + +`resolve` 最小返回体： + +```json +{ + "entry_key": "runbook:release_progression", + "selected_entry": {}, + "preview_request": {}, + "preview": {}, + "focus_ref": { + "kind": "release_hub", + "release_id": 7, + "section": "release_launchpad", + "mode": "worker" + }, + "gate": { + "decision": "confirmation_required", + "decision_label": "等待确认", + "recommendation": "confirm_then_execute", + "recommendation_label": "建议确认后执行", + "will_execute": false, + "confirm_required": true, + "focus_ref": { + "kind": "release_hub", + "release_id": 7, + "section": "release_launchpad", + "mode": "worker" + }, + "target_api": "/api/v1/ops/runbook/sequences/release_progression/execute", + "target_method": "POST", + "request_payload": { + "secondary": false, + "requested_by": "cli/ops-center/codex-focus-preview", + "action_payload": { + "release_id": 7 + } + } + } +} +``` + +`execute` 要求： + +- 必须复用 `resolve` 的同一套选中逻辑和门禁逻辑 +- `safe_auto` + - 允许直接执行 +- `guarded_auto` + - 只有 `confirm=true` 时允许执行 +- `resolve_first / open_ui / blocked` + - 一律不得执行 +- 返回体必须保留： + - `selected_entry` + - `preview_request` + - `preview` + - `gate` + - `focus_ref` + - `execution_result` + - `executed` + +--- + +## 9. UI Intent + +`ui_intent` 的职责不是让前端猜去哪，而是由后端直接声明： + +- 应打开哪个详情 +- 应聚焦哪轮 run +- 应落到哪个发布入口 + +当前建议固定支持： + +- `playbook_run_detail` +- `playbook_run_latest_events` +- `job_events` +- `rollout_jobs` +- `managed_node_handover` +- `managed_node_edit` +- `open_release_dialog` +- `open_rollout_dialog` +- `focus_release_hub` +- `focus_ref` + +页面只做路由和抽屉承载，不做推荐解释。 + +--- + +## 10. Runbook Sequence + +作用： + +- 表达固定标准作业路径在“此刻”的推荐下一步 + +最小字段： + +```json +{ + "key": "standard_inspection", + "title": "标准巡检", + "status": "warning", + "status_label": "待补接管", + "summary": "当前还没有满足标准巡检条件的节点。", + "target_node_codes": [], + "focus_ref": { + "kind": "release_hub", + "release_id": 7, + "section": "release_launchpad" + }, + "primary_label": "直接执行标准巡检", + "primary_action_code": "run_standard_inspection", + "primary_action_payload": {}, + "primary_focus_ref": {}, + "secondary_label": "打开巡检编排", + "secondary_action_code": "open_playbook_dialog", + "secondary_action_payload": { + "playbook_key": "inspection.standard" + }, + "secondary_focus_ref": {} +} +``` + +要求： + +- Runbook Sequence 是标准路径对象 +- 不是普通活动记录 +- 不是普通推荐卡 +- `focus_ref` + - 表示这条标准路径整体对应的统一观察落点 +- `primary_focus_ref` / `secondary_focus_ref` + - 表示主次动作各自应回看的稳定区域 + +--- + +## 11. Activity Stream + +来源： + +- `GET /api/v1/ops/activity-stream` + +作用： + +- 把最近异常、编排、任务、Rollout 统一压成可钻取活动流 + +每条 activity 最小字段： + +```json +{ + "kind": "playbook_run", + "activity_key": "run-20260418-001", + "status": "failed", + "occurred_at": "2026-04-18 06:00:00", + "summary": "标准巡检在 diagnostics.collect 步骤失败。", + "ui_intent": { + "kind": "playbook_run_detail", + "run_code": "run-20260418-001" + } +} +``` + +推荐 `kind`： + +- `playbook_run` +- `ops_job` +- `rollout` +- `runbook_sequence` + +--- + +## 12. 单脑驾驶舱首屏约束 + +Ops Center 首屏不是自由拼装页面，而是 Driver Contract 的正式消费者。 + +首屏必须优先消费同一组 contract： + +- `overview.recommendation` +- `driver-feed.top_recommendation` +- `driver-feed.automation_coverage` +- `codex-brief` + +首屏建议固定为三层： + +1. 驾驶结论 +2. 驾驶动作条 +3. 驾驶回执条 + +正式要求： + +- 驾驶结论 + - 直接展示当前第一优先动作、自动化收口状态、是否达到上线闸门 +- 驾驶动作条 + - 只放“默认下一步 / 定位下一步 / 看契约 / 复制命令 / 进入主处理面板”这类高频动作 + - 这些按钮不能自己发明新规则，必须消费已有 `action_code / focus_ref / preview` +- 驾驶回执条 + - 是首屏动作后的统一反馈区 + - 用于展示： + - 已打开哪个契约 + - 已定位哪个焦点 + - 已复制哪个命令 + - 后端动作执行成功 / 警告 / 失败 + - 页面不应再把动作结果散落在多个局部 toast 里 + +也就是说： + +- 首屏负责决策与触发 +- 下方详情区负责展开与下钻 +- 不允许详情区反向重写首屏推荐 + +--- + +## 13. 建议优先级顺序 +当前 contract 应优先遵守： + +1. 先看异常编排 +2. 再看异常活动 +3. 再处理 Agent 死信 / 回执积压 +4. 再优先执行首个缺口节点的接入收口或接管验收 +5. 再处理现场日志回传和标准巡检 +6. 最后进入 Release / Rollout + +这条顺序必须由后端维护，不应让前端或 Codex 再自行拼装。 + +补充约束： + +- 当后端已经把首个缺口节点收敛成 `bootstrap_run / run_acceptance` 时： + - `driver_feed.top_recommendation` + - `codex_brief.focus` + - `stack_diagnosis.next_step` + 都必须优先指向这一个节点和这一个动作 +- `fix_managed_nodes` 只能保留为兜底入口，不能再覆盖已经明确收敛的单节点恢复动作 + +--- + +## 14. 页面 / Codex / CLI 共用约束 + +同一条驾驶建议必须由后端一次性给出： + +- 推荐标题 +- 推荐理由 +- 主动作 +- 次动作 +- 节点范围 +- 预填参数 +- UI 落点 +- 是否允许自动执行 + +这样三类入口才会真正共享一套驾驶 contract： + +- OpsCenter 页面 +- 海外 Codex 驾驶员 +- CLI / 运维检查脚本 + +并且三类入口对 `recommended_behavior` 的执行语义也必须一致： + +- `auto_execute` + - 可直接执行 +- `confirm_then_execute` + - 必须显式确认后才能执行 +- `resolve_first` + - 只能先审阅和补上下文，不能直接执行 +- `open_ui` + - 只能进入对应工作区，不能直接执行 +- `blocked` + - 明确阻断，不能执行 diff --git a/docs/schemas/ops_go_live_bundle_contract.md b/docs/schemas/ops_go_live_bundle_contract.md new file mode 100644 index 0000000..af36f4d --- /dev/null +++ b/docs/schemas/ops_go_live_bundle_contract.md @@ -0,0 +1,59 @@ +# ops_go_live_bundle_contract + +## 目标 + +把发布前证据包与 `manifest.json` 复核结论压成一份统一 contract，供： + +- Ops Center 交付面板 +- CLI `go-live-export / go-live-review` +- 海外 Codex 驾驶员 + +共同消费。 + +它解决的问题不是“现在能不能上线”，而是： + +> 现在有没有一份足够完整、足够可信、可交付可复盘的上线证据包 + +## 主接口 + +- `GET /api/v1/ops/go-live-bundle` + +## 核心字段 + +- `status` + - `missing` + - `ready` + - `attention` + - `blocked` +- `status_label` +- `headline` +- `bundle_available` +- `report_dir` +- `manifest_path` +- `artifact_total` +- `failure_total` +- `critical_failures` +- `noncritical_failures` +- `env_audit` +- `summary` +- `recommended_commands` + +## 判定原则 + +- 如果当前还没有 bundle / manifest + - `status = missing` +- 如果 manifest 明确标记 `review_status_hint=blocked` + - `status = blocked` +- 如果 manifest 已存在，但： + - 仍有非关键失败 + - 环境审计提示 attention + - launchpad 摘要存在漂移 + - 则 `status = attention` +- 只有当 manifest 可读、review 为 ready、关键摘要一致 + - 才允许 `status = ready` + +## 设计约束 + +- 页面不直接触发 shell 导出 bundle +- 后端默认只读最新 bundle / manifest，不负责替代 CLI 执行导出 +- 当 bundle 缺失时，必须明确给出固定命令，而不是让操作者自己猜路径 diff --git a/docs/schemas/ops_go_live_review_contract.md b/docs/schemas/ops_go_live_review_contract.md new file mode 100644 index 0000000..c70c302 --- /dev/null +++ b/docs/schemas/ops_go_live_review_contract.md @@ -0,0 +1,37 @@ +# ops_go_live_review_contract + +## Purpose + +冻结 `go-live-review` 复核结论 contract，让页面、CLI、Codex 都能消费同一份 bundle manifest 复核结果，而不是各自再解读一遍 artifact。 + +## Producer + +- backend service `app.services.ops_service.get_ops_go_live_review` +- API `GET /api/v1/ops/go-live-review` +- CLI `bash domain-api/deploy/multi-region/drive_ops_center.sh go-live-review` + +## Primary Consumers + +- `domain-web` Ops Center 证据复核面板 +- 海外 Codex 驾驶员 +- 发布前人工值班复核 + +## Required Payload + +- `status` +- `status_label` +- `headline` +- `artifact_total` +- `failure_total` +- `critical_failures` +- `noncritical_failures` +- `env_audit` +- `launchpad_alignment` +- `recommended_next_steps` +- `recommended_commands` + +## Notes + +- 这层是 manifest 的正式“复核解释层”，不是单纯重复 bundle 基础信息。 +- 如果 `launchpad_alignment.consistent=false`，即使关键文件都存在，也必须至少回到 `attention`。 +- 页面和 Codex 应优先消费这层 `headline / recommended_next_steps`，而不是自行拼 review 口径。 diff --git a/docs/schemas/ops_go_live_signoff_contract.md b/docs/schemas/ops_go_live_signoff_contract.md new file mode 100644 index 0000000..85652d4 --- /dev/null +++ b/docs/schemas/ops_go_live_signoff_contract.md @@ -0,0 +1,65 @@ +# ops_go_live_signoff_contract + +## 目标 + +把发布前最终签收判断压成一份统一 contract，供： + +- Ops Center 首屏 +- CLI `go-live-signoff` +- 海外 Codex 驾驶员 + +共同消费。 + +它不是替代： + +- `go-live-summary` +- `go-live-review` +- `doctor-decision` +- `stack-diagnosis` +- `driver-feed` +- `codex-brief` + +而是在这些 contract 之上给出最终一句： + +> 现在能不能签字上线 + +## 主接口 + +- `GET /api/v1/ops/go-live-signoff` + +## 核心字段 + +- `signoff_status` + - `ready` + - `attention` + - `blocked` +- `status_label` +- `headline` +- `release_gate` +- `blocked_reasons` +- `attention_reasons` +- `decision` +- `launchpad_alignment` +- `recommended_commands` +- `report_dir` +- `manifest_path` + +## 判定原则 + +- 只要 `go_live_summary / go_live_review / doctor_decision / publish_status / stack_status / automation / release_launchpad` 中任一明确阻断 + - `signoff_status = blocked` +- 如果没有阻断，但存在： + - `go-live-review=attention|missing` + - `doctor-decision=attention|missing` + - `attention` + - launchpad 摘要漂移 + - 自动化仍未完全进入 ready + - 则 `signoff_status = attention` +- 只有当收口、正式复核、主决策、发布、自动化、launchpad 摘要一致 + - 才允许 `signoff_status = ready` + +## 设计约束 + +- 页面不允许再自己拼一套“看起来能上” +- CLI 不允许绕过这个 contract 单独下结论 +- Codex 驾驶员优先看这一份，再决定是否继续下钻 diff --git a/docs/schemas/ops_job_contract.md b/docs/schemas/ops_job_contract.md new file mode 100644 index 0000000..3553b0f --- /dev/null +++ b/docs/schemas/ops_job_contract.md @@ -0,0 +1,740 @@ +# domainCheck Ops Job Contract + +## 1. 目标 + +这份文档用于冻结 `ops job` 的正式 contract。 + +它是整套海外单脑控制面里最核心的执行对象。 + +正式定义： + +> `ops job` 是一条可审计、可编排、可派发、可回执、可取消的标准运维任务记录。 + +它不是： + +- 一条 shell 命令 +- 一个页面按钮点击结果 +- 一个仅供 agent 消费的内部对象 + +它应该同时服务于： + +- 页面按钮 +- CLI +- 海外 Codex 驾驶员 +- Node Agent +- Release / Rollout +- Playbook Run + +也就是说： + +- `playbook run` + - 是编排聚合对象 +- `rollout` + - 是发布推进对象 +- `ops job` + - 是真正的执行颗粒度对象 + +--- + +## 2. 入口接口 + +当前已落地的主入口： + +- `GET /api/v1/ops/jobs` +- `GET /api/v1/ops/jobs/{job_id}` +- `GET /api/v1/ops/jobs/{job_id}/events` +- `POST /api/v1/ops/jobs` +- `POST /api/v1/ops/jobs/batch` +- `POST /api/v1/ops/policy/preview` +- `POST /api/v1/ops/jobs/{job_id}/approve` +- `POST /api/v1/ops/jobs/{job_id}/cancel` +- `POST /api/v1/ops/jobs/{job_id}/dispatch` + +配套对象来源： + +- `ops_jobs` +- `ops_job_steps` +- `ops_job_events` + +正式约束： + +- 页面、CLI、Codex 不能各自定义第 2 套任务模型 +- `playbook run` 只能聚合 `ops job` +- `rollout` 只能批量生成 `ops job` +- `Node Agent` 只能拉取和回执 `ops job` + +--- + +## 3. 当前实现里的 Job Object + +来源： + +- `GET /api/v1/ops/jobs` +- `GET /api/v1/ops/jobs/{job_id}` +- 创建 / 审批 / 取消 / 派发后的返回体中的 `job` + +最小字段： + +```json +{ + "id": 101, + "job_code": "ops-20260418064500-a1b2c3", + "action": "logs.collect", + "target_type": "node", + "target_node_code": "mainland-worker-01", + "target_node_codes": [ + "mainland-worker-01" + ], + "status": "queued", + "status_label": "排队中", + "execution_mode": "remote-agent", + "execution_mode_label": "Node Agent", + "requested_by": "web-ui/ops-center", + "payload": {}, + "metadata": {}, + "result": {}, + "summary": "目标节点 mainland-worker-01 / 发起 web-ui/ops-center / 方式 Node Agent", + "summary_text": "目标节点 mainland-worker-01 / 发起 web-ui/ops-center / 方式 Node Agent", + "error_message": "", + "created_at": "2026-04-18 06:45:00", + "started_at": "", + "finished_at": "", + "updated_at": "2026-04-18 06:45:00", + "risk_level": "medium", + "approval_required": false, + "approval_status": "approved", + "approval_status_label": "已审批", + "approved_by": "", + "approved_at": "", + "blocked_reason": "", + "cancellation_reason": "", + "dispatched_at": "", + "target_selector": {}, + "policy": {}, + "rollout_id": 0, + "steps_total": 0, + "steps_running": 0, + "steps_success": 0, + "steps_failed": 0, + "steps_terminal": 0, + "steps_loaded": false, + "focus_ref": { + "kind": "ops_job", + "job_id": 101, + "job_code": "ops-20260418064500-a1b2c3", + "action": "logs.collect", + "target_node_code": "mainland-worker-01" + }, + "steps": [] +} +``` + +正式要求： + +- `job_code` + - 是跨页面、CLI、事件流的稳定任务标识 +- `action` + - 是执行语义主键，不应用 `title` 或自然语言代替 +- `target_type` + - 定义任务作用范围，不应只靠是否传了 `target_node_code` 来猜 +- `execution_mode` + - 是正式执行方式，不是附属备注 +- `policy` + - 是创建时冻结的策略快照，不应由页面二次推导 +- `status_label / approval_status_label` + - 是给页面、CLI、Codex 直接消费的人类可读标签 +- `summary_text` + - 是任务摘要 contract，不要求调用方再用状态和节点自己拼一句中文 +- `steps_total / steps_running / steps_success / steps_failed / steps_terminal` + - 是任务步骤聚合统计，避免前端和 CLI 再自行遍历 steps 统计 +- `focus_ref` + - 是统一定位对象，后续页面、CLI、Codex 应优先消费它，而不是自己重组跳转参数 + +--- + +## 4. Job 状态机 + +### 4.1 当前实现中已使用的状态 + +当前后端已实际使用： + +- `queued` +- `awaiting_approval` +- `blocked` +- `running` +- `success` +- `failed` +- `cancelled` + +这些状态已经是正式 contract 的最小闭环。 + +### 4.2 推荐终局扩展状态 + +终局可继续扩展为： + +- `draft` +- `approved` +- `queued` +- `dispatching` +- `running` +- `verifying` +- `partially_succeeded` +- `rollback_running` +- `rolled_back` +- `success` +- `failed` +- `cancelled` +- `timed_out` + +但要明确区分： + +- 当前实际 API 已稳定承诺的，是最小状态集 +- 终局扩展状态不能破坏当前状态语义 + +--- + +## 5. Execution Mode Contract + +当前实现里正式出现的执行方式： + +- `remote-agent` +- `local-runtime` +- `control-plane` +- `ssh` + +语义必须固定： + +### 5.1 `remote-agent` + +- 默认正式执行方式 +- 由目标节点 Node Agent 拉取并执行 + +### 5.2 `local-runtime` + +- 仅用于当前节点本机即时执行 +- 主要用于本机 runtime 动作 + +### 5.3 `control-plane` + +- 由海外控制面直接执行 +- 适用于控制面内生动作 + +### 5.4 `ssh` + +- 只作为救援和过渡执行器 +- 不能成为大规模正式发布的默认方式 + +正式约束： + +- 发布默认优先 `remote-agent` +- `ssh` 只应是兜底链路 +- 页面、CLI、Codex 不应再自行创造第 5 种 execution mode + +--- + +## 6. Target Type Contract + +当前 contract 至少应允许： + +- `node` +- `batch` +- `nodes` +- `rollout` + +语义建议固定为： + +### 6.1 `node` + +- 单节点任务 + +### 6.2 `batch` / `nodes` + +- 多节点批量任务 +- 当前通常通过 `/ops/jobs/batch` 创建 + +### 6.3 `rollout` + +- 该任务属于某轮 release rollout 推进的一部分 + +正式要求： + +- 如果任务来自 rollout，必须能通过 `rollout_id` 回溯 +- 如果任务来自 playbook，应通过 `metadata` 标出 run 归属 + +--- + +## 7. Job Step Contract + +来源： + +- `GET /api/v1/ops/jobs/{job_id}` + +最小字段： + +```json +{ + "id": 201, + "step_key": "dispatch", + "title": "调度动作", + "node_code": "mainland-worker-01", + "status": "queued", + "stdout": "", + "stderr": "", + "result": {}, + "created_at": "2026-04-18 06:45:00", + "started_at": "", + "finished_at": "", + "updated_at": "2026-04-18 06:45:00" +} +``` + +当前实现说明： + +- 现阶段每条 `ops job` 至少会先生成一个 `dispatch` step +- 以后如果引入真正 DAG / 多步骤执行，也应继续沿用 `ops_job_steps` + +正式约束： + +- `step_key` + - 是步骤主键，不应用标题代替 +- `status` + - 必须与 job 状态联动，但不要求完全相同 +- `stdout / stderr` + - 是步骤级输出，不应与整条 job 的 `result` 混成一层 + +--- + +## 8. Job Event Contract + +来源： + +- `GET /api/v1/ops/jobs/{job_id}/events` + +最小字段建议： + +```json +{ + "id": 301, + "event_key": "job-event:301", + "job_id": 101, + "step_id": 201, + "node_code": "mainland-worker-01", + "client_event_id": "evt-001", + "event_type": "job_dispatch_requested", + "level": "info", + "level_label": "信息", + "message": "任务等待 node-agent 拉取: 101", + "summary": "任务等待 node-agent 拉取: 101", + "payload": {}, + "has_payload": false, + "occurred_at": "2026-04-18 06:45:03", + "created_at": "2026-04-18 06:45:03" +} +``` + +当前代码里事件来源包括： + +- job created +- job blocked +- job awaiting approval +- job approved +- job cancelled +- job dispatch requested +- job executed locally / over ssh / on control plane +- agent start / complete / custom events + +正式要求： + +- `events` 是任务时间线，不是调试附属品 +- 页面、CLI、Codex 都应从这里读“任务做到哪一步了” +- 不应重新从 stdout/stderr 推断关键状态 + +补充约束： + +- `event_key` + - 是事件行的稳定前端 / CLI 标识 +- `summary` + - 是给页面、CLI、Codex 直接消费的短摘要，不要求调用方再拼接 `message` +- `level_label` + - 是面向人看的等级标签 +- `occurred_at` + - 是统一观察面时间字段，允许与 `created_at` 同值 + +### 8.1 Event List Summary Contract + +当前 `GET /api/v1/ops/jobs/{job_id}/events` 建议同时返回： + +```json +{ + "summary": { + "job_id": 101, + "returned_total": 12, + "level_counts": { + "info": 8, + "warning": 3, + "error": 1 + }, + "event_type_counts": { + "job_created": 1, + "job_dispatch_requested": 1, + "executor_received": 1 + }, + "node_counts": { + "mainland-worker-01": 12 + }, + "latest_at": "2026-04-18 06:45:08", + "filters": { + "limit": 50 + } + } +} +``` + +正式要求： + +- 事件接口不只给一串数组 +- 至少要给： + - 当前返回了多少条 + - 等级分布 + - 事件类型分布 + - 最近时间 +- 这样页面、CLI、Codex 才能先判断“有没有异常事件”再决定是否展开明细 + +--- + +## 9. List Contract 与 Compact 模式 + +来源： + +- `GET /api/v1/ops/jobs` + +当前实现特点： + +- 默认 `list_ops_jobs(..., compact=True)` +- 返回 compact job +- `steps` 被置空 +- `payload / metadata / result` 会被压缩 + +因此建议把列表 contract 明确成： + +```json +{ + "jobs": [ + { + "id": 101, + "job_code": "ops-20260418064500-a1b2c3", + "action": "logs.collect", + "status": "queued", + "execution_mode": "remote-agent", + "requested_by": "web-ui/ops-center", + "payload": { + "tail_lines": 120 + }, + "metadata": {}, + "result": {}, + "result_summary": {}, + "is_compact": true, + "steps": [] + } + ] +} +``` + +正式要求： + +- 列表页默认只承诺 compact 视图 +- 详情页再承诺 full job + steps +- 页面不应假设列表结果天然带完整 steps + +--- + +## 10. Create Contract + +来源： + +- `POST /api/v1/ops/jobs` + +最小请求体： + +```json +{ + "action": "logs.collect", + "target_type": "node", + "target_node_code": "mainland-worker-01", + "execution_mode": "remote-agent", + "requested_by": "web-ui/ops-center", + "payload": { + "tail_lines": 120 + }, + "metadata": { + "source": "ops-center" + }, + "auto_approve": false, + "run_now": true +} +``` + +当前实现还支持： + +- `target_selector` +- `rollout_id` + +最小返回体： + +```json +{ + "job": {}, + "executed_immediately": false +} +``` + +关键语义： + +- `executed_immediately=true` + - 说明任务已被控制面即时执行，而不是停留在队列 +- `executed_immediately=false` + - 说明任务进入了： + - `queued` + - `awaiting_approval` + - `blocked` + +正式要求： + +- 创建接口不能只回答“建成功了” +- 必须明确回答： + - 当前状态是什么 + - 是否立即执行 + - 如果没执行，是在等审批、等 agent、还是被阻断 + +--- + +## 11. Batch Create Contract + +来源： + +- `POST /api/v1/ops/jobs/batch` + +最小请求体建议： + +```json +{ + "template_key": "diagnostics.collect", + "target_node_codes": [ + "mainland-worker-01", + "mainland-worker-02" + ], + "execution_mode": "remote-agent", + "requested_by": "web-ui/ops-center", + "payload": {}, + "metadata": {}, + "auto_approve": true +} +``` + +最小返回体建议： + +```json +{ + "jobs": [], + "results": [] +} +``` + +正式要求： + +- batch 只是创建方式，不是新的执行对象 +- batch 最终仍要展开成多条 `ops job` +- 单条失败不能让结果完全丢失，必须保留逐项结果 + +--- + +## 12. Policy Preview Contract + +来源： + +- `POST /api/v1/ops/policy/preview` +- `ops_jobs.policy` + +这层是创建前的正式门禁对象。 + +当前创建任务时，后端已先做： + +- 风险等级判断 +- 阻断判断 +- 审批要求判断 + +建议最小返回字段： + +```json +{ + "blocked": false, + "blocking_reasons": [], + "approval_required": true, + "approval_reasons": [ + "当前目标包含 control 节点" + ], + "warnings": [], + "recommendations": [], + "risk_level": "high", + "target_summary": {}, + "batch_plan": {}, + "cluster_guardrails": {}, + "operational_readiness": {} +} +``` + +正式要求： + +- 页面、CLI、Codex 都应复用同一份 policy preview +- 不能各自再写一套“能不能执行”的判断逻辑 + +--- + +## 13. Approval / Cancel / Dispatch Contract + +### 13.1 Approve + +来源： + +- `POST /api/v1/ops/jobs/{job_id}/approve` + +最小请求体： + +```json +{ + "approved_by": "admin" +} +``` + +语义： + +- 仅 `awaiting_approval` 可审批 +- 审批后回到 `queued` + +### 13.2 Cancel + +来源： + +- `POST /api/v1/ops/jobs/{job_id}/cancel` + +最小请求体： + +```json +{ + "cancelled_by": "admin", + "reason": "本轮灰度暂停" +} +``` + +语义： + +- `success / failed / cancelled` 不可取消 +- 取消后 job 进入 `cancelled` +- steps 同步进入 `cancelled` 或保持终态 + +### 13.3 Dispatch + +来源： + +- `POST /api/v1/ops/jobs/{job_id}/dispatch` + +语义： + +- `queued` 才可派发 +- `control-plane` + - 立即由控制面执行 +- `local-runtime` + - 当前节点本机即时执行 +- `ssh` + - 由 SSH 执行器执行 +- `remote-agent` + - 保持 `queued`，等待对应 agent 拉取 + +这点必须明确： + +> dispatch 不等于一定会立刻变成 running。 + +对于 `remote-agent`，它可能只是： + +- 写入事件 +- 保持排队 +- 等节点稍后 pull + +--- + +## 14. 与 Playbook / Rollout / Agent 的边界 + +### 14.1 与 Playbook + +- `playbook run` + - 负责展开、聚合、重跑、取消整轮编排 +- `ops job` + - 负责单条执行记录 + +### 14.2 与 Rollout + +- `rollout` + - 负责发布批次推进 +- `ops job` + - 负责每个节点每步的实际任务 + +### 14.3 与 Node Agent + +- 控制面创建 `ops job` +- agent 拉取 `ops job` +- agent 回写 `ops_job_events` +- agent 推进 job / step 状态 + +### 14.4 与 Observability + +- `ops_observability_contract` + - 看现场事实 +- `ops_job_contract` + - 看执行动作本身 + +这两层不能混： + +- 观察面告诉你“谁在跑、谁异常” +- 任务面告诉你“这一条任务如何被创建、审批、派发、执行、取消” + +--- + +## 15. 终局约束 + +后续不应再回退到下面几种模式： + +### 1. 页面点按钮直接跑 shell + +错误。 + +正确方式： + +- 页面创建 `ops job` + +### 2. Codex 直接 SSH 改现场 + +错误。 + +正确方式： + +- Codex 创建 `ops job` / `playbook run` + +### 3. 发布系统不经过任务层 + +错误。 + +正确方式： + +- rollout 批量生成 `ops job` + +### 4. 死信治理不留痕 + +错误。 + +正确方式： + +- `flush / replay / discard` 都生成正式 `ops job` + +所以可以把这份文档看成整套系统的“执行订单母本”。 + +后续只要这份 contract 稳住，页面、CLI、Codex、Agent、Release、Playbook 就都能围绕同一个中心对象协作。 diff --git a/docs/schemas/ops_observability_contract.md b/docs/schemas/ops_observability_contract.md new file mode 100644 index 0000000..e081e56 --- /dev/null +++ b/docs/schemas/ops_observability_contract.md @@ -0,0 +1,702 @@ +# domainCheck Ops Observability Contract + +## 1. 目标 + +这份文档用于冻结海外单脑控制面里“看现场”这一层的正式 contract。 + +它不负责： + +- 发布决策 +- Driver 推荐 +- Playbook 编排 + +它负责回答 3 类问题： + +1. 哪些节点现在真正处于执行现场 +2. 这些节点最近巡检收口结果是什么 +3. Node Agent 回执队列现在是否健康 + +也就是说，这份 contract 的职责是： + +> 让页面、CLI、Codex 看到同一份现场事实，而不是各自从日志和状态字串里猜 + +--- + +## 2. 入口接口 + +建议把下面这些接口视为同一观察面的一组正式入口： + +- `GET /api/v1/ops/overview` +- `GET /api/v1/ops/inspection-overview` +- `GET /api/v1/ops/activity-stream` +- `GET /api/v1/ops/nodes/{node_code}/handover` +- `GET /api/v1/ops/nodes/{node_code}/delivery-queue` +- `GET /api/v1/ops/nodes/{node_code}/delivery-queue/records` +- `POST /api/v1/ops/nodes/{node_code}/delivery-queue/flush` +- `POST /api/v1/ops/nodes/{node_code}/delivery-queue/replay` +- `POST /api/v1/ops/nodes/{node_code}/delivery-queue/records/{record_id}/replay` +- `POST /api/v1/ops/nodes/{node_code}/delivery-queue/records/{record_id}/discard` + +配套关系： + +- `overview` + - 首页收口 +- `inspection-overview` + - 节点巡检收口 +- `activity-stream` + - 最近异常活动 +- `handover` + - 单节点接管阻断与下一步建议 +- `delivery-queue` + - Node Agent 回执健康 + +正式要求： + +- 页面不能自己再定义第 5 套“观察口径” +- CLI 不应重新从数据库拼现场 +- Codex 不应通过自然语言日志推断核心状态 + +--- + +## 3. Execution Scene Contract + +来源： + +- `GET /api/v1/ops/overview` +- `overview.execution_scene` + +最小字段： + +```json +{ + "summary": "当前存在 2 台有效执行节点，其中 1 台正在领任务，1 台在线待命。", + "dispatch_active_nodes": [ + { + "node_code": "mainland-worker-01", + "region": "mainland", + "role": "worker", + "status": "busy", + "participation_state": "dispatch_active", + "participation_label": "正在执行", + "participation_reason": "当前存在已领任务和执行中工作负载", + "current_load": 9, + "items_claimed": 9, + "items_running": 5, + "items_completed_recent": 14 + } + ], + "recent_only_nodes": [], + "standby_nodes": [ + { + "node_code": "mainland-controller-01", + "region": "mainland", + "role": "control", + "status": "online", + "participation_state": "standby", + "participation_label": "在线待命", + "participation_reason": "节点在线、有效，但当前没有领任务、执行中任务或近窗吞吐", + "current_load": 0 + } + ], + "load_syncing_nodes": [], + "counts": { + "dispatch_active": 1, + "recent_only": 0, + "standby": 1, + "load_syncing": 0 + } +} +``` + +分组语义必须固定： + +- `dispatch_active_nodes` + - 当前正在领任务、执行任务、或仍有实时工作负载 +- `recent_only_nodes` + - 当前不再运行，但近窗刚有吞吐 +- `standby_nodes` + - 在线、有效，但当前不参与 +- `load_syncing_nodes` + - 节点与控制面状态尚未完全同步 + +正式约束： + +- “在线”不等于“参与检测” +- “有效执行节点”不等于“正在跑” +- 页面展示、CLI 摘要、Codex 判断必须沿用这 4 类分组 + +节点参与态附加字段也应固定： + +- `participation_state` + - 机器判断主键，推荐值： + - `dispatch_active` + - `recent_only` + - `standby` + - `load_syncing` +- `participation_label` + - 直接给页面、CLI、Codex 展示的人类可读标签 +- `participation_reason` + - 用一句稳定摘要解释为什么节点落在当前分组 + +正式要求： + +- 页面必须直接区分： + - 在线但未参与 + - 正在执行或领任务 + - 近窗刚参与但当前已收口 +- 新消费方不允许再从： + - `status` + - `current_load` + - `items_running` + 手工推断“这个节点到底算不算正在参与” + +--- + +## 4. Log Sync Runtime Contract + +来源： + +- `overview.execution_scene.log_sync` +- 或 `overview.log_sync` + +最小字段： + +```json +{ + "status": "partial_coverage", + "status_label": "部分覆盖", + "mode": "key", + "mode_label": "关键回传", + "enabled": true, + "source_nodes": [ + "mainland-worker-01" + ], + "source_node_summaries": [ + { + "node_code": "mainland-worker-01", + "line_count": 42, + "key_line_count": 42, + "full_line_count": 0, + "last_at": "2026-04-18 06:40:00", + "last_line": "域名检测完成: demo.com" + } + ], + "covered_participating_node_summaries": [ + { + "node_code": "mainland-worker-01", + "region": "mainland", + "role": "worker", + "participation_state": "dispatch_active", + "participation_label": "正在执行", + "participation_reason": "当前存在已领任务和执行中工作负载", + "line_count": 42, + "key_line_count": 42, + "full_line_count": 0, + "last_at": "2026-04-18 06:40:00", + "last_line": "域名检测完成: demo.com" + } + ], + "missing_participating_node_summaries": [ + { + "node_code": "mainland-worker-02", + "region": "mainland", + "role": "worker", + "participation_state": "dispatch_active", + "participation_label": "正在执行", + "participation_reason": "当前存在已领任务和执行中工作负载", + "missing_reason": "no_sample" + } + ], + "covered_participating_nodes": 1, + "participating_nodes_total": 2, + "missing_sample_node_codes": [ + "mainland-worker-02" + ], + "last_line_at": "2026-04-18 06:40:00", + "summary": "当前已覆盖 1/2 台参与节点，建议继续观察未回传样本节点。" +} +``` + +推荐状态： + +- `disabled` +- `waiting_sample` +- `partial_coverage` +- `healthy` +- `full_capture` + +这层是“现场可见性”最关键的辅助状态。 + +正式要求： + +- 如果存在参与节点但日志回传还没开，优先推荐开关键回传 +- 如果日志回传已开但没有样本，优先推荐抓 Worker 日志 +- 页面和 Codex 都不能再自己定义另一套日志覆盖结论 + +节点级日志覆盖字段也应固定： + +- `source_nodes` + - 为兼容旧页面保留的节点编码列表 +- `source_node_summaries` + - 当前已有日志样本的节点级统计对象 +- `covered_participating_node_summaries` + - 已参与且已拿到样本的节点对象 +- `missing_participating_node_summaries` + - 已参与但尚未拿到样本的节点对象 + +`source_node_summaries[]` 至少应返回： + +- `node_code` +- `line_count` +- `key_line_count` +- `full_line_count` +- `last_at` +- `last_line` + +`missing_participating_node_summaries[]` 至少应返回： + +- `node_code` +- `region` +- `role` +- `participation_state` +- `participation_label` +- `participation_reason` +- `missing_reason` + +推荐 `missing_reason`： + +- `no_sample` +- `delayed_flush` +- `filtered_out` + +兼容要求： + +- 旧页面仍可继续消费： + - `source_nodes` + - `missing_sample_node_codes` +- 新页面、CLI、Codex 应优先消费： + - `source_node_summaries` + - `covered_participating_node_summaries` + - `missing_participating_node_summaries` + +补充节点级现场入口： + +- `GET /api/v1/ops/nodes/{node_code}/scene-log` + +最小字段： + +```json +{ + "node_code": "mainland-worker-01", + "status": "healthy", + "status_label": "关键覆盖", + "mode": "key", + "mode_label": "关键回传", + "log_sync_enabled": true, + "summary": "节点当前正在参与检测，已保留 42 条现场日志样本。", + "node": { + "region": "mainland", + "role": "worker", + "status": "busy", + "current_load": 6, + "last_heartbeat_at": "2026-04-18 06:40:00" + }, + "participation": { + "detect_participating": true, + "participation_state": "claimed", + "participation_label": "已领待跑", + "participation_bucket": "dispatch_active", + "participation_reason": "已领取任务，等待线程继续执行。" + }, + "source_summary": { + "line_count": 42, + "key_line_count": 42, + "full_line_count": 0, + "last_at": "2026-04-18 06:40:00", + "last_line": "[2026-04-18 06:40:00] [mainland-worker-01] 域名检测完成: demo.com" + }, + "records_total": 42, + "records_visible": 80, + "records": [ + { + "created_at": "2026-04-18 06:38:00", + "node_code": "mainland-worker-01", + "message": "开始检测域名: demo.com", + "mode": "key", + "line": "[2026-04-18 06:38:00] [mainland-worker-01] 开始检测域名: demo.com" + } + ] +} +``` + +节点级入口的正式语义： + +- 这是“看某一台节点当前现场”的正式入口，不允许页面再自己从聚合样本中二次猜测 +- `status` 至少覆盖： + - `healthy` + - `full_capture` + - `historical_sample` + - `waiting_sample` + - `missing_sample` + - `disabled` + - `standby` + - `unknown` +- 当节点当前在线但未参与时，必须明确返回 `standby` +- 当节点正在参与但还没有样本时，必须明确返回 `waiting_sample` +- 当节点已有历史样本但当前不在参与面中时，必须明确返回 `historical_sample` + +--- + +## 5. Inspection Overview Contract + +来源： + +- `GET /api/v1/ops/inspection-overview` +- `overview.inspection` + +最小字段： + +```json +{ + "status": "attention", + "status_label": "待处理", + "summary": "存在 1 台节点最近缺少 Worker 日志收口。", + "counts": { + "healthy_nodes": 1, + "attention_nodes": 1, + "problem_nodes": 0 + }, + "rows": [ + { + "node_code": "mainland-worker-01", + "problem_kind": "missing_worker_logs", + "problem_label": "缺少 Worker 日志收口", + "problem_level": "warning", + "recommended_action_code": "open_worker_logs", + "ui_intent": { + "kind": "node_inspection_detail", + "node_code": "mainland-worker-01" + }, + "latest_health_snapshot": {}, + "latest_worker_logs": {}, + "latest_diagnostics": {} + } + ] +} +``` + +设计要求： + +- 巡检不是几条离散 job 记录 +- 必须按节点重新聚合 +- 必须直接给出： + - `problem_kind` + - `problem_label` + - `problem_level` + - `recommended_action_code` + - `ui_intent` + +这样页面、CLI、Codex 才能真正消费“节点巡检结论”，而不是再从几十条 job 记录里回推问题。 + +--- + +## 6. Inspection Row Contract + +`rows[]` 中每个节点建议最少包含： + +- `node_code` +- `region` +- `role` +- `effective_worker` +- `problem_kind` +- `problem_label` +- `problem_level` +- `recommended_action_code` +- `ui_intent` +- `latest_health_snapshot` +- `latest_worker_logs` +- `latest_diagnostics` + +其中 3 类最近结果建议统一结构： + +```json +{ + "status": "success", + "status_label": "成功", + "occurred_at": "2026-04-18 06:35:00", + "job_code": "ops-20260418-001", + "summary": "最近一次 Worker 日志收集成功，共采样 120 行。" +} +``` + +正式边界： + +- `logs.collect` 默认指 Worker 日志收口 +- Node Agent 辅助排障日志不应混成同一种巡检结果 + +--- + +## 7. Delivery Queue Summary Contract + +来源： + +- `GET /api/v1/ops/nodes/{node_code}/delivery-queue` + +最小字段： + +```json +{ + "node": { + "node_code": "mainland-worker-01", + "title": "mainland-worker-01" + }, + "summary": { + "state": "dead_letter", + "label": "死信", + "reason": "当前有 2 条记录进入死信。", + "pending_count": 3, + "dead_letter_count": 2, + "oldest_pending_at": "2026-04-18 06:20:00", + "oldest_dead_letter_at": "2026-04-18 06:18:00" + }, + "record_visibility": "head_only", + "capabilities": { + "can_flush": true, + "can_replay": true, + "can_discard": true + } +} +``` + +推荐状态： + +- `healthy` +- `retrying` +- `dead_letter` +- `stalled` + +正式约束： + +- 控制面要明确告诉操作者当前是 `head_only` 还是 `full_projection` +- 不允许页面假装能列出远端完整目录 +- 不允许因为能看见死信就绕过 `ops job` 直接改文件 + +--- + +## 8. Delivery Queue Record Contract + +来源： + +- `GET /api/v1/ops/nodes/{node_code}/delivery-queue/records` + +当前阶段建议最小字段： + +```json +{ + "records": [ + { + "record_id": 101, + "state": "dead_letter", + "state_label": "死信", + "request_id": "req-20260418-001", + "kind": "job_complete", + "created_at": "2026-04-18 06:18:00", + "updated_at": "2026-04-18 06:19:00", + "summary": "某次任务完成回执多次补发失败。", + "visibility": "head_only" + } + ], + "summary": { + "returned_records": 1, + "record_visibility": "head_only" + } +} +``` + +当前阶段要求： + +- 单条 `replay / discard` 只针对控制面可见记录 +- 当前 `head_only` 不等于未来不能扩展 +- 以后若进入 `full_projection`，扩的是可见范围，不是重写操作模型 + +--- + +## 9. Delivery Queue Action Contract + +来源： + +- `flush` +- `replay` +- `record replay` +- `record discard` + +正式要求： + +- 所有动作都必须落成正式 `ops job` +- 返回体要能告诉调用方： + - 是否已受理 + - 生成了什么 job + - 建议跳转到哪里观察 + +最小返回体建议： + +```json +{ + "accepted": true, + "job_code": "ops-20260418-queue-001", + "ui_intent": { + "kind": "job_events", + "job_code": "ops-20260418-queue-001" + } +} +``` + +这里的边界必须比“临时脚本”更严格： + +- `discard` 必须带原因 +- `replay` 应支持限流 / limit +- `flush` 应支持显式 `requested_by` + +--- + +## 10. Activity Stream Contract + +来源： + +- `GET /api/v1/ops/activity-stream` + +最小字段建议： + +```json +{ + "items": [ + { + "kind": "ops_job", + "activity_key": "ops-job:101", + "title": "logs.collect", + "subtitle": "ops-20260418064500-a1b2c3", + "summary": "目标节点 mainland-worker-01 / 发起 web-ui / 方式 Node Agent", + "summary_text": "目标节点 mainland-worker-01 / 发起 web-ui / 方式 Node Agent", + "status": "running", + "status_label": "执行中", + "execution_mode": "remote-agent", + "execution_mode_label": "Node Agent", + "target_node_codes": [ + "mainland-worker-01" + ], + "occurred_at": "2026-04-18 06:45:08", + "ui_intent_kind": "job_events", + "focus_ref": { + "kind": "ops_job", + "job_id": 101, + "job_code": "ops-20260418064500-a1b2c3", + "action": "logs.collect", + "target_node_code": "mainland-worker-01" + } + } + ], + "summary": { + "status": "running", + "status_label": "执行中", + "summary_text": "最近活动里存在正在推进的现场、任务、编排或回传。", + "filtered_total": 6, + "unfiltered_total": 18, + "latest_at": "2026-04-18 06:45:08" + } +} +``` + +正式要求： + +- `activity-stream` 不是页面内部 table 数据 +- 它必须是： + - 页面 + - CLI + - Codex 驾驶员 + - 自动驾驶编排器 + 共同消费的“最近活动摘要” + +字段语义需要固定： + +- `summary_text` + - 是直接消费的活动短摘要 +- `status_label` + - 是稳定的人类可读状态标签 +- `target_node_codes` + - 是活动关联的节点口径，不允许调用方再从自然语言里猜 +- `ui_intent_kind` + - 是当前活动默认落点类型 +- `focus_ref` + - 是跨页面 / CLI / Codex 的统一定位对象 + +正式边界： + +- 活动流应保留老字段以兼容现有页面 +- 新消费方优先用： + - `summary_text` + - `status_label` + - `focus_ref` +- 不允许新客户端再从 `title + subtitle + summary` 手工解析定位参数 + +## 11. 与 Driver / Playbook / Release 的边界关系 + +这份 contract 只负责观察面和现场治理，不负责替代其他 contract。 + +边界应明确为： + +- `ops_driver_contract` + - 决定“现在优先做什么” +- `ops_playbook_contract` + - 决定“标准多步动作怎么编排” +- `release_hub_contract` + - 决定“版本怎么发、怎么停、怎么回滚” +- `ops_observability_contract` + - 决定“现场事实是什么、当前缺口在哪里” + +这样后续系统再扩展时，不会又回到“所有东西都写在 overview 一坨大 json 里”的状态。 + +--- + +## 12. 终局验收标准 + +如果这份 contract 真正落地，海外单脑控制面应该做到： + +### 1. 不登录大陆机器也能回答现场问题 + +例如： + +- 谁在跑 +- 谁在线但没跑 +- 谁近窗刚跑过 +- 谁日志没回来 +- 谁队列死信了 + +### 2. 不复制大日志也能先做判断 + +先看： + +- execution scene +- inspection overview +- activity stream +- delivery queue + +再决定是否需要全量日志或 diagnostics bundle。 + +### 3. 页面、CLI、Codex 看到的是同一现场 + +不能出现： + +- 页面说节点正在执行 +- CLI 说节点待命 +- Codex 说节点离线 + +### 4. 所有高风险动作都有正式观察落点 + +也就是： + +- replay 后能看 job +- discard 后能看审计 +- inspection 后能看节点收口 +- rollout 后能看 launchpad / gate / rollout jobs + +这样这套系统才能真正从“调试台”升级成“驾驶舱”。 diff --git a/docs/schemas/ops_playbook_contract.md b/docs/schemas/ops_playbook_contract.md new file mode 100644 index 0000000..e9ada98 --- /dev/null +++ b/docs/schemas/ops_playbook_contract.md @@ -0,0 +1,563 @@ +# domainCheck Ops Playbook Contract + +## 1. 目标 + +这份文档用于冻结 `playbook catalog / preview / run / events` 的正式 contract。 + +这层的定位不是“页面里的一个弹窗”，而是： + +- 控制面的正式编排对象 +- 海外 Codex 驾驶员可消费的标准作业对象 +- Driver / Runbook / Release / Inspection 之间的统一中间层 + +核心原则： + +- 一次标准编排 = 一次 `playbook run` +- 页面、CLI、Codex 必须共用同一份 `playbook` / `playbook run` 结构 +- `playbook run` 只负责创建与聚合正式 `ops job` +- 不允许前端再把“标准巡检 / 接管验收 / 现场日志”退回成零散按钮逻辑 + +建议与 [ops_job_contract.md](/www/wwwroot/getDomain/docs/schemas/ops_job_contract.md:1) 配套阅读。 + +--- + +## 2. 当前接口面 + +当前已落地的主入口： + +- `GET /api/v1/ops/playbooks` +- `POST /api/v1/ops/playbooks/preview` +- `POST /api/v1/ops/playbooks/execute` +- `GET /api/v1/ops/playbook-runs` +- `GET /api/v1/ops/playbook-runs/{run_code}` +- `GET /api/v1/ops/playbook-runs/{run_code}/events` +- `POST /api/v1/ops/playbook-runs/{run_code}/rerun` +- `POST /api/v1/ops/playbook-runs/{run_code}/cancel` + +并且这层已经被以下入口复用： + +- `driver action -> open_playbook_dialog` +- `runbook sequence -> secondary_resolution` +- `inspection` +- `activity-stream` + +所以后续不应再新造第四套“标准作业对象”。 + +--- + +## 3. Playbook Catalog Object + +来源： + +- `GET /api/v1/ops/playbooks` +- `preview / execute` 返回体中的 `playbook` + +最小字段： + +```json +{ + "key": "inspection.standard", + "group_key": "diagnostics", + "group_title": "标准巡检", + "title": "标准巡检", + "description": "按 健康快照 -> Worker 日志 -> 诊断包 的顺序，对目标节点做一次标准联调巡检。", + "target_scope": "node-set", + "default_execution_mode": "remote-agent", + "default_execution_mode_label": "Node Agent", + "execution_modes": [ + "remote-agent", + "ssh" + ], + "execution_mode_options": [ + { + "value": "remote-agent", + "label": "Node Agent", + "is_default": true + }, + { + "value": "ssh", + "label": "SSH", + "is_default": false + } + ], + "default_auto_approve": true, + "stop_on_failure": true, + "steps": [ + { + "step_key": "health", + "title": "采集健康快照", + "template_key": "health.snapshot", + "payload": {} + } + ] +} +``` + +正式约束： + +- `key` + - 是 Playbook 的稳定主键 +- `group_key` + - 用于页面、CLI、Codex 做标准分组，不应用 `title` 猜 +- `execution_modes` + - 是能力白名单，不是建议文案 +- `stop_on_failure` + - 是编排级行为，不是某一步骤的临时说明 + +--- + +## 4. Playbook Preview Contract + +来源： + +- `POST /api/v1/ops/playbooks/preview` + +最小请求体： + +```json +{ + "playbook_key": "inspection.standard", + "node_codes": [ + "mainland-worker-01" + ], + "execution_mode": "remote-agent", + "auto_approve": true, + "requested_by": "web-ui" +} +``` + +最小返回体： + +```json +{ + "playbook": {}, + "requested_by": "web-ui", + "execution_mode": "remote-agent", + "execution_mode_label": "Node Agent", + "auto_approve": true, + "target_nodes_total": 1, + "target_node_codes": [ + "mainland-worker-01" + ], + "step_count": 3, + "expected_jobs_total": 3, + "template_keys": [ + "health.snapshot", + "logs.collect", + "diagnostics.collect" + ], + "steps": [ + { + "order": 1, + "step_key": "health", + "title": "采集健康快照", + "template_key": "health.snapshot", + "action": "health.snapshot", + "payload": {}, + "execution_mode": "remote-agent", + "execution_mode_label": "Node Agent", + "auto_approve": true, + "target_nodes_total": 1, + "expected_jobs": 1 + } + ] +} +``` + +预览阶段必须回答： + +- 最终执行方式是什么 +- 总共会展开多少步 +- 每步会展开成什么模板 +- 总共会生成多少个 `ops job` + +这样页面、CLI、Codex 才能在执行前看见“这次会做什么”。 + +--- + +## 5. Playbook Run Object + +来源： + +- `GET /api/v1/ops/playbook-runs` +- `GET /api/v1/ops/playbook-runs/{run_code}` +- `POST /api/v1/ops/playbooks/execute` + +最小字段： + +```json +{ + "run_code": "playbook-20260418-001", + "playbook_key": "inspection.standard", + "playbook_title": "标准巡检", + "group_key": "diagnostics", + "group_title": "标准巡检", + "requested_by": "web-ui", + "execution_mode": "remote-agent", + "execution_mode_label": "Node Agent", + "stop_on_failure": true, + "step_count": 3, + "expected_jobs_total": 3, + "target_nodes_total": 1, + "target_node_codes": [ + "mainland-worker-01" + ], + "created_at": "2026-04-18 06:30:00", + "updated_at": "2026-04-18 06:31:10", + "status": "running", + "status_label": "收口中", + "status_counts": { + "queued": 1, + "running": 1, + "success": 1 + }, + "jobs_total": 3, + "success_jobs_total": 1, + "terminal_jobs_total": 1, + "completion_percent": 33.3, + "steps_total": 3, + "steps_success": 1, + "steps_running": 1, + "steps_problem": 0, + "steps_terminal": 1, + "focus_level": "warning", + "focus_step_key": "worker_logs", + "focus_step_title": "收集 Worker 日志", + "focus_summary": "收集 Worker 日志 当前仍在执行，先等待回执或检查对应节点事件流。", + "summary": "收集 Worker 日志 当前仍在执行，先等待回执或检查对应节点事件流。", + "summary_text": "收集 Worker 日志 当前仍在执行，先等待回执或检查对应节点事件流。", + "focus_ref": { + "kind": "playbook_run", + "run_code": "playbook-20260418-001", + "playbook_key": "inspection.standard", + "group_key": "diagnostics", + "focus_step_key": "worker_logs", + "focus_step_title": "收集 Worker 日志", + "event_key": "", + "node_code": "" + }, + "problem_steps": [], + "active_steps": [], + "latest_job": {}, + "steps": [] +} +``` + +正式定义： + +> `playbook run` 是一轮标准编排在控制面的聚合对象，不是某一条单独任务。 + +也就是说： + +- `ops job` 是执行颗粒度 +- `playbook run` 是观察、重跑、取消、钻取的编排颗粒度 + +--- + +## 6. Playbook Step Object + +`playbook run.steps[]` 的最小结构： + +```json +{ + "step_key": "worker_logs", + "title": "收集 Worker 日志", + "template_key": "logs.collect", + "order": 2, + "jobs_total": 1, + "nodes_total": 1, + "target_node_codes": [ + "mainland-worker-01" + ], + "status_counts": { + "running": 1 + }, + "terminal_jobs_total": 0, + "success_jobs_total": 0, + "status": "running", + "status_label": "收口中", + "completion_percent": 0.0, + "summary": "收集 Worker 日志 当前处于执行中，可继续观察该步骤事件。", + "summary_text": "收集 Worker 日志 当前处于执行中，可继续观察该步骤事件。", + "focus_ref": { + "kind": "playbook_run", + "run_code": "playbook-20260418-001", + "playbook_key": "inspection.standard", + "group_key": "diagnostics", + "focus_step_key": "worker_logs", + "focus_step_title": "收集 Worker 日志", + "event_key": "", + "node_code": "" + }, + "latest_job": {} +} +``` + +推荐 step 状态： + +- `queued` +- `running` +- `success` +- `attention` + +规则： + +- 只要该步骤出现失败、阻断、取消等终态问题，应收口为 `attention` +- 页面与 Codex 都应优先盯 `problem_steps / active_steps`，而不是自己遍历所有子任务推断 + +--- + +## 7. Playbook Event Stream + +来源： + +- `GET /api/v1/ops/playbook-runs/{run_code}/events` + +最小返回体： + +```json +{ + "run_code": "playbook-20260418-001", + "playbook_run": {}, + "events": [ + { + "job_id": 123, + "job_code": "ops-20260418-123", + "run_code": "playbook-20260418-001", + "job_status": "running", + "job_status_label": "收口中", + "action": "logs.collect", + "target_node_code": "mainland-worker-01", + "step_key": "worker_logs", + "step_title": "收集 Worker 日志", + "event_type": "job_running", + "level": "info", + "level_label": "信息", + "message": "开始收集 Worker 日志", + "summary": "开始收集 Worker 日志", + "summary_text": "开始收集 Worker 日志", + "payload": {}, + "event_key": "job-event:1001", + "occurred_at": "2026-04-18 06:31:00", + "focus_ref": { + "kind": "playbook_run", + "run_code": "playbook-20260418-001", + "playbook_key": "inspection.standard", + "group_key": "diagnostics", + "focus_step_key": "worker_logs", + "focus_step_title": "收集 Worker 日志", + "event_key": "job-event:1001", + "node_code": "mainland-worker-01" + }, + "created_at": "2026-04-18 06:31:00" + } + ], + "summary": { + "total": 1, + "returned_total": 1, + "available_step_counts": { + "worker_logs": 1 + }, + "available_node_counts": { + "mainland-worker-01": 1 + }, + "level_counts": { + "info": 1 + }, + "event_type_counts": { + "job_running": 1 + }, + "job_status_counts": { + "running": 1 + }, + "latest_at": "2026-04-18 06:31:00", + "filters": { + "step_key": "", + "node_code": "", + "limit": 80 + } + } +} +``` + +这层必须支持： + +- 按 `step_key` 过滤 +- 按 `node_code` 过滤 +- 快速看到最近活动时间 + +否则 `playbook run` 只能看总览，不能真正钻取。 + +--- + +## 8. 正式 Playbook Key 语义 + +当前最值得冻结成正式对象的 key： + +### 8.1 `onboarding.bootstrap` + +定位： + +- 生成节点接入工单 + +执行方式： + +- 仅 `control-plane` + +正式语义： + +- 不直接接管节点 +- 只创建标准接入方案对象 + +### 8.2 `onboarding.acceptance` + +定位： + +- 新节点接管后标准验收 + +当前标准步骤： + +1. `health.snapshot` +2. `service.status(node-agent)` +3. `logs.collect(node-agent)` +4. `service.status(worker)` +5. `logs.collect(worker)` + +### 8.3 `inspection.standard` + +定位： + +- 标准巡检 + +当前标准步骤： + +1. `health.snapshot` +2. `logs.collect(worker)` +3. `diagnostics.collect` + +正式要求： + +- 这是 release / rollout 之前最核心的健康观察面 + +### 8.4 `scene.logs.key` + +定位： + +- 关键现场日志样本 + +正式要求： + +- 默认比 `scene.logs.full` 更轻 +- 适合先看现场，再决定是否升级取证 + +### 8.5 `scene.logs.full` + +定位： + +- 全量现场取证 + +正式要求： + +- 应明确比 `scene.logs.key` 更重 +- 默认应同时补诊断包 + +### 8.6 `scene.diagnostics` + +定位： + +- 轻量诊断包 + +正式要求： + +- 适合作为参与节点的常规取证动作 + +--- + +## 9. 执行模式 Contract + +推荐模式： + +- `remote-agent` +- `ssh` +- `control-plane` + +正式约束： + +- `remote-agent` + - 正式日常运维默认模式 +- `ssh` + - 仅用于首发接管与应急救援 +- `control-plane` + - 只适合控制面本地对象生成或极少量控制面动作 + +页面、CLI、Codex 不能自己发明第四种临时模式。 + +--- + +## 10. Playbook Run 状态机 + +推荐状态： + +- `queued` +- `running` +- `success` +- `attention` + +含义： + +- `queued` + - 子任务刚创建，尚未出现有效执行推进 +- `running` + - 已有子任务进入 `dispatching / running / awaiting_approval` +- `success` + - 全部子任务收口成功 +- `attention` + - 任一步骤出现失败、阻断、取消或部分成功，需要人工关注 + +正式规则： + +- 页面上的“标准巡检状态” +- `activity-stream` 中的 playbook run 摘要 +- Driver / Codex 的推荐动作 + +都必须共用这一套状态语义。 + +--- + +## 11. 与 Driver / Runbook / Release 的关系 + +正式收口规则： + +- Driver 不直接“模拟巡检” + - 应优先跳转或创建 `playbook run` +- Runbook Sequence 不直接展开 shell + - 应优先收口到 `driver action` 或 `playbook` +- Release / Rollout 不自己维护另一套巡检摘要 + - 应优先消费 `inspection.standard` 的最近结果 + +也就是说： + +- `scene` 是现场观察对象 +- `inspection` 是标准健康对象 +- `release / rollout` 是发布对象 + +三者不能再各自维护不同的“日志 / 诊断 / 巡检”定义。 + +--- + +## 12. 正式要求 + +后续继续演进时，必须保持： + +1. `playbook catalog` + - 是稳定 contract，而不是页面枚举 +2. `playbook preview` + - 是执行前的唯一结构化预览面 +3. `playbook run` + - 是页面、CLI、Codex 共同观察的一轮编排对象 +4. `playbook events` + - 是整轮编排的钻取视角 +5. 接管验收、标准巡检、现场观察 + - 优先都收进 `playbook` + +这样这套编排才能成为真正的正式平台能力，而不是一组“看起来像流程”的按钮。 diff --git a/docs/schemas/ops_stack_diagnosis_contract.md b/docs/schemas/ops_stack_diagnosis_contract.md new file mode 100644 index 0000000..d9a6757 --- /dev/null +++ b/docs/schemas/ops_stack_diagnosis_contract.md @@ -0,0 +1,372 @@ +# domainCheck Ops Stack Diagnosis Contract + +## 1. 目标 + +这份文档用于冻结海外单脑控制面的“总检入口” contract。 + +它的职责不是替代： + +- `overview` +- `link-snapshot` +- `inspection-overview` +- `release launchpad` +- `activity-stream` + +而是把这些正式 contract 再收口成一份： + +> 第一现场诊断结果 + +让下面这些消费者都先看同一份判断： + +- 海外 Ops Center 页面 +- 海外 Codex 驾驶员 +- CLI 总检脚本 +- 后台按钮自动诊断 + +正式原则： + +- 总检不能只回一段说明文字 +- 总检必须给出结构化问题清单 +- 总检必须给出默认下一步动作 +- 总检必须给出可以直接执行的推荐命令 + +--- + +## 2. 入口接口 + +主入口： + +- `GET /api/v1/ops/go-live-summary` +- `GET /api/v1/ops/stack-diagnosis` + +配套来源： + +- `GET /api/v1/ops/contracts` +- `GET /api/v1/ops/link-snapshot` +- `GET /api/v1/ops/overview` +- `GET /api/v1/ops/nodes` +- `GET /api/v1/ops/releases/launchpad` +- `GET /api/v1/ops/playbook-runs` +- `GET /api/v1/ops/activity-stream` + +推荐 query： + +- `base_url` + +用途： + +- `go-live-summary` + - 给 Ops Center 顶部收口卡、CLI 总检、Codex 驾驶员直接消费 + - 返回更偏“上线判断”的稳定摘要 +- 用于生成 `next_step.command` +- 用于生成 `quick_commands` +- 让 CLI / Codex / 页面复制出来的命令口径一致 + +--- + +## 3. 顶层结构 + +建议最小结构： + +```json +{ + "generated_at": "2026-04-18 09:00:00", + "diagnosis": {}, + "api": {}, + "surface_matrix": {}, + "contracts": {}, + "link_snapshot": {}, + "overview": {}, + "managed_nodes": {}, + "release_hub": {}, + "playbook_runs": {}, + "activity_stream": {} +} +``` + +正式要求： + +- 顶层每一段都必须是稳定 key +- 页面、CLI、Codex 不允许自己再拼第 2 套总检结构 +- `go-live-summary` + - 是 `stack-diagnosis` 的稳定摘要层 + - 必须至少给出： + - `go_live_status` + - `blocking_reasons` + - `warnings` + - `launchpad_recommended_target_node_code` + - `launchpad_recommended_recovery_label` + - `launchpad_recommended_recovery_summary` + - `launchpad_onboarding_bootstrap_pending_nodes` + - `launchpad_onboarding_acceptance_ready_nodes` + - `next_step_action_code` + - `operator_title` + - `recommended_commands` +- `playbook_runs.problem_runs[*]` + - 需要保留 `focus_ref` + - 这样 CLI / Codex / 页面才能直接定位到问题编排的正式焦点 +- `activity_stream.top_items[*]` + - 需要保留： + - `occurred_at` + - `focus_ref` + - `source_focus_ref` + - `ui_intent_kind` + - 这样总检不会把“观察摘要”再次压扁成一段不可定位的纯文本 +- `go-live-summary` + `stack-diagnosis` + `driver-feed.automation_coverage` + - 必须能共同驱动 Ops Center 首屏 + - 不允许页面自己从局部接口重新推导一份“收口状态” + +--- + +## 4. Diagnosis Contract + +来源： + +- `stack_diagnosis.diagnosis` + +最小字段： + +```json +{ + "contract_key": "ops_stack_diagnosis_contract", + "contract_version": "v1", + "registry_version": "2026-04-18", + "base_url": "http://127.0.0.1:8100", + "surface_status": "partial", + "automation_status": "blocked", + "stack_status": "blocked", + "issue_total": 4, + "blocking_issue_total": 1, + "warning_issue_total": 3, + "missing_surfaces": [ + "contracts" + ], + "launchpad_recommended_target_node_code": "mainland-worker-01", + "launchpad_recommended_recovery_label": "待执行接入", + "launchpad_recommended_recovery_summary": "mainland-worker-01 还缺接入收口，先执行 bootstrap。", + "launchpad_onboarding_bootstrap_pending_nodes": 1, + "launchpad_onboarding_acceptance_ready_nodes": 0, + "next_step": { + "action_code": "bootstrap_run", + "source": "issue:managed_nodes_agent_pending", + "reason": "mainland-worker-01 还缺接入收口，先执行 bootstrap。", + "command": "bash domain-api/deploy/multi-region/drive_ops_center.sh driver-resolve http://127.0.0.1:8100 bootstrap_run", + "focus_ref": { + "kind": "managed_node", + "node_code": "mainland-worker-01" + } + }, + "recommended_actions": [], + "issues": [], + "operator_hints": [], + "quick_commands": [] +} +``` + +状态语义必须固定： + +- `surface_status` + - `healthy` + - `partial` + - `broken` +- `automation_status` + - `ready` + - `attention` + - `blocked` +- `stack_status` + - 总结论，面向人和程序 + - 推荐值为 `ready / attention / blocked` + +正式要求： + +- `next_step` 是默认下一步，而不是“可能动作之一” +- 当首个缺口节点已经明确收敛为 `bootstrap_run / run_acceptance` 时，`next_step.action_code` 不应再退回成泛化的 `fix_managed_nodes` +- 当存在 `runtime_build_schema_stale` 时，`next_step.action_code` 必须优先收敛为 `api-restart` +- `issues` 必须按可处理性组织，而不是只堆原始异常文本 +- `quick_commands` 必须是可以直接复制执行的命令 +- `next_step.focus_ref` + - 是首屏“定位下一步”的正式落点 + - 页面、CLI、Codex 不允许再根据 `action_code` 反推要跳去哪个区域 +- `diagnosis.launchpad_recommended_target_node_code` + - 代表 stack-diagnosis 已经把 Release Hub 收敛出的首个接管缺口直接下沉到总检摘要 + - 页面、CLI、doctor-export、Codex 不应再分别回源二次推导 +- `diagnosis.launchpad_onboarding_bootstrap_pending_nodes / launchpad_onboarding_acceptance_ready_nodes` + - 代表当前接管缺口的阶段性计数 + - 用来帮助总检直接区分“待 bootstrap”与“待 acceptance” +- `go-live-summary.launchpad_recommended_target_node_code` + - 是 launchpad 已经收敛出的首个接管缺口节点 + - 页面、CLI、Codex 应优先直接显示它，而不是重新扫描 gap rows +- `go-live-summary.launchpad_onboarding_bootstrap_pending_nodes / launchpad_onboarding_acceptance_ready_nodes` + - 用于区分当前是“待接入”还是“待验收” + - 这两个计数应作为上线收口阶段的正式 warning 来源，而不是页面本地再统计 + +--- + +## 5. Issue Contract + +`diagnosis.issues[]` 最小字段： + +```json +{ + "code": "managed_nodes_agent_pending", + "severity": "blocked", + "layer": "managed_nodes", + "summary": "托管节点虽然已经纳入 Ops Center，但 0 台 remote-agent 就绪，当前第一动作已收敛为首台节点接入收口。", + "detail": "managed_enabled=3，remote_access_ready=0；首个缺口节点 mainland-worker-01：待执行接入。", + "action_code": "bootstrap_run", + "focus_ref": { + "kind": "managed_node", + "node_code": "mainland-worker-01" + }, + "commands": [ + "bash domain-api/deploy/multi-region/drive_ops_center.sh driver-resolve http://127.0.0.1:8100 bootstrap_run", + "bash domain-api/deploy/multi-region/drive_ops_center.sh node-bootstrap-plan http://127.0.0.1:8100 mainland-worker-01" + ] +} +``` + +严重级别建议固定为： + +- `blocked` +- `warning` +- `info` + +正式要求： + +- `summary` 用于列表展示 +- `detail` 用于展开解释 +- `commands` 用于终端直接执行 +- 不允许只有中文文案，没有结构化定位信息 + +对于运行时版本漂移，还应允许出现这类 issue： + +```json +{ + "code": "runtime_build_schema_stale", + "severity": "warning", + "layer": "runtime_build_info.schema", + "summary": "仓库已经具备新的节点接管能力，但运行中的 build-info 仍未声明对应路由键，当前更像 API 还没重启到最新代码。", + "detail": "supports_install_command_block=true；route_surface_declares_bootstrap_plan=false", + "action_code": "api-restart", + "focus_ref": { + "kind": "runtime_build_info", + "section": "schema", + "expected_route_key": "ops_node_handover_bootstrap_plan" + }, + "commands": [ + "bash domain-api/deploy/multi-region/drive_ops_center.sh runtime-refresh-recover http://127.0.0.1:8100", + "bash domain-api/deploy/multi-region/drive_ops_center.sh stack-diagnosis http://127.0.0.1:8100 summary", + "bash domain-api/deploy/multi-region/drive_ops_center.sh node-bootstrap-plan http://127.0.0.1:8100 mainland-worker-01" + ] +} +``` + +--- + +## 6. Surface Matrix Contract + +来源： + +- `stack_diagnosis.surface_matrix` + +最小字段： + +```json +{ + "status_counts": { + "total": 8, + "available": 7, + "missing": 1 + }, + "items": [ + { + "name": "contracts", + "available": false + } + ] +} +``` + +--- + +## 7. Runtime Build Info Extension + +`stack_diagnosis.runtime_build_info` 以及 `/api/v1/runtime/build-info` 建议至少包含： + +```json +{ + "repository_capabilities": { + "supports_install_command_block": true, + "supports_multi_layout_bootstrap": true + }, + "route_surface_declares_bootstrap_plan": false, + "runtime_schema_stale": true +} +``` + +语义要求： + +- `repository_capabilities` + - 表示当前仓库代码本身具备的能力 +- `route_surface_declares_bootstrap_plan` + - 表示运行中的 API build-info 是否已经声明节点级 bootstrap-plan 路由键 +- `runtime_schema_stale` + - 表示“仓库能力已到位，但运行中的 API 结构还停留在旧版” + - 这是典型的“服务没重启到最新代码”信号，不应误判成“功能没开发完” + +作用： + +- 让页面和 CLI 直观看出“缺的是哪一层” +- 避免只看到最终 blocked，却不知道缺的是 contract、launchpad 还是 activity-stream + +--- + +## 7. 与其他 Contract 的关系 + +`ops_stack_diagnosis_contract` 的定位是： + +- 不替代 `ops_driver_contract` +- 不替代 `ops_observability_contract` +- 不替代 `release_hub_contract` + +它负责： + +- 把它们统一收口成第一现场诊断 + +对于 Ops Center 页面还必须补充一条： + +- 首屏负责“结论 + 默认下一步 + 复制命令 + 动作回执” +- 下方详情区负责“issue / surface / launchpad / activity / runbook”的展开与下钻 +- 不允许详情区覆盖首屏的默认下一步结论 + +所以正确消费顺序应该是： + +1. 先看 `stack-diagnosis` +2. 再根据 `issues / next_step / focus_ref` 下钻到： + - `overview` + - `link-snapshot` + - `inspection-overview` + - `release launchpad` + - `activity-stream` + +--- + +## 8. CLI / Codex / 页面统一约束 + +CLI： + +- `check_ops_center_stack.sh` 应优先调用 `GET /api/v1/ops/stack-diagnosis` + +Codex： + +- 默认先读取 `stack-diagnosis` +- 再决定是继续 `driver-resolve`、`doctor` 还是细分检查 + +页面： + +- 顶部总检卡片必须优先消费 `diagnosis` +- 不能重新从多个 endpoint 拼一份影子状态 + +只有这样，海外单脑控制面才能真正进入： + +> 单入口观察，分层下钻，统一执行 diff --git a/docs/schemas/release_hub_contract.md b/docs/schemas/release_hub_contract.md new file mode 100644 index 0000000..46fc80c --- /dev/null +++ b/docs/schemas/release_hub_contract.md @@ -0,0 +1,609 @@ +# domainCheck Release Hub Contract + +## 1. 目标 + +这份文档用于冻结 Release / Rollout / Launchpad / Gate 的正式 contract。 + +核心原则： + +- 先有 Release，再有 Rollout +- Release 是不可变版本对象 +- Rollout 是分批推进对象 +- Gate 是是否可发的统一门禁对象 +- Launchpad 是给页面与 Codex 消费的发布驾驶舱对象 + +建议与 [ops_job_contract.md](/www/wwwroot/getDomain/docs/schemas/ops_job_contract.md:1) 配套阅读，因为 Rollout 的真正执行颗粒度最终仍然是 `ops job`。 + +--- + +## 2. Release Object + +来源： + +- `POST /api/v1/ops/releases` +- `GET /api/v1/ops/releases` +- `GET /api/v1/ops/releases/latest` +- `GET /api/v1/ops/releases/{id}` +- `POST /api/v1/ops/releases/{id}/activate` + +最小字段： + +```json +{ + "id": 1, + "release_version": "2026.04.18+bd6bcb2", + "channel": "stable", + "commit_sha": "bd6bcb2", + "status": "ready", + "status_label": "已就绪", + "artifact_url": "https://release.example.com/domaincheck-2026.04.18.tgz", + "checksum": "sha256:...", + "notes": "", + "metadata": {}, + "created_by": "www", + "created_at": "2026-04-18 05:30:00", + "activated_at": "2026-04-18 05:35:00", + "summary": "2026.04.18+bd6bcb2 已就绪，制品与校验信息齐备，可进入 Rollout。", + "summary_text": "2026.04.18+bd6bcb2 已就绪，制品与校验信息齐备，可进入 Rollout。", + "focus_ref": { + "kind": "release_hub", + "release_id": 1, + "release_version": "2026.04.18+bd6bcb2", + "channel": "stable", + "rollout_id": 0, + "rollout_code": "", + "section": "release_detail" + } +} +``` + +约束： + +- 同一 `release_version` 只对应一份发布物 +- Release 创建后不应重新指向另一份 artifact +- `artifact_url + checksum` 是发布可验证性的最小基线 + +推荐状态： + +- `draft` +- `ready` +- `active` +- `archived` + +--- + +## 3. Rollout Object + +来源： + +- `POST /api/v1/ops/releases/{id}/rollouts` +- `GET /api/v1/ops/rollouts/{id}` +- `GET /api/v1/ops/rollouts/{id}/jobs` +- `POST /api/v1/ops/rollouts/{id}/advance` + +最小字段： + +```json +{ + "id": 11, + "release_id": 1, + "rollout_code": "rollout-20260418-001", + "target_selector": { + "region": "mainland", + "role": "worker" + }, + "target_nodes": [ + "mainland-worker-01", + "mainland-controller-01" + ], + "policy": { + "batch_size": 1, + "require_approval_between_batches": true, + "rollback_on_failure": true + }, + "status": "running", + "status_label": "执行中", + "batch_cursor": 1, + "batches_total": 2, + "jobs_total": 2, + "jobs_created": 1, + "result_summary": {}, + "target_node_codes": [ + "mainland-worker-01", + "mainland-controller-01" + ], + "summary": "当前已覆盖 1/2 台节点，仍有 1 条任务执行中。", + "summary_text": "当前已覆盖 1/2 台节点，仍有 1 条任务执行中。", + "focus_ref": { + "kind": "release_rollout", + "rollout_id": 11, + "rollout_code": "rollout-20260418-001", + "release_id": 1, + "section": "rollout_jobs" + } +} +``` + +正式定义： + +> 某个 Release 在某批目标节点上的一次分批推进计划 + +推荐状态： + +- `planned` +- `running` +- `awaiting_approval` +- `ready_for_next_batch` +- `halted` +- `completed` +- `completed_with_issues` + +--- + +## 4. Default Rollout Gate + +来源： + +- `GET /api/v1/ops/overview` +- `overview.release_hub.default_rollout_gate` + +这层是 Release Hub 最关键的后端 gate contract。 + +最小字段： + +```json +{ + "status": "attention", + "status_label": "待处理", + "status_type": "warning", + "summary": "当前已有 Release，但默认目标节点中仍有未接管或未通过巡检的节点。", + "summary_text": "当前已有 Release，但默认目标节点中仍有未接管或未通过巡检的节点。", + "release": { + "id": 1, + "release_version": "2026.04.18+bd6bcb2", + "status": "ready", + "status_label": "已就绪" + }, + "execution_mode": "remote-agent", + "execution_mode_label": "Node Agent", + "target_node_codes": [ + "mainland-worker-01" + ], + "default_target_node_codes": [ + "mainland-worker-01" + ], + "blocking_reasons": [], + "warning_reasons": [ + "存在未通过标准巡检的节点" + ], + "recommendations": [ + "先执行标准巡检，再进入 Rollout" + ], + "operational_readiness": { + "summary": "remote-agent 就绪 1/2，标准巡检通过 1/2", + "rows": [] + }, + "focus_ref": { + "kind": "release_hub", + "release_id": 1, + "release_version": "2026.04.18+bd6bcb2", + "channel": "stable", + "rollout_id": 0, + "rollout_code": "", + "section": "default_rollout_gate" + } +} +``` + +推荐状态： + +- `missing_release` +- `release_not_ready` +- `artifact_missing` +- `no_targets` +- `blocked` +- `attention` +- `ready` + +要求： + +- 首页驾驶建议 +- Release 页面门禁 +- Rollout 预检 +- Codex 自动驾驶 + +必须共用这同一份 Gate。 + +--- + +## 5. Default Rollout Execution + +来源： + +- `overview.release_hub.default_rollout_execution` +- `overview.release_hub.default_rollout_gate_options` + +作用： + +- 告诉页面 / Codex 这轮默认应使用什么执行方式 +- 给出其他可选 gate / mode + +最小字段： + +```json +{ + "recommended_mode": "remote-agent", + "recommended_mode_label": "Node Agent", + "recommended_gate": {}, + "gates": { + "remote-agent": {}, + "ssh": {} + } +} +``` + +推荐执行模式： + +- `remote-agent` +- `ssh` +- `control-plane` + +说明： + +- 正式发布优先 `remote-agent` +- `ssh` 只作为首发接管与紧急救援兜底 +- `control-plane` 不应用来承载大规模 Worker 发布 + +--- + +## 6. Release Launchpad + +来源： + +- `overview.release_hub.launchpad` + +作用： + +- 给页面与 Codex 提供更偏“驾驶舱”的发布聚合视角 + +最小字段： + +```json +{ + "latest_package": {}, + "latest_release": {}, + "worker_rollout_preview": {}, + "control_rollout_preview": {}, + "launchpad_status": { + "status": "attention", + "status_label": "待处理", + "summary": "建议先完成 Worker 点状灰度，再决定是否推进控制面。", + "summary_text": "建议先完成 Worker 点状灰度，再决定是否推进控制面。", + "recommended_action_code": "open_release_deploy_worker", + "recommended_target_node_code": "mainland-worker-01", + "recommended_recovery_label": "签发接入工单", + "recommended_recovery_summary": "当前节点仍处于接入阶段，建议先签发 onboarding.bootstrap。", + "onboarding_bootstrap_pending_nodes": 1, + "onboarding_acceptance_ready_nodes": 0, + "recommended_mode": "remote-agent", + "recommended_mode_label": "Node Agent", + "focus_ref": { + "kind": "release_hub", + "release_id": 1, + "release_version": "2026.04.18+bd6bcb2", + "channel": "stable", + "rollout_id": 0, + "rollout_code": "", + "section": "release_launchpad", + "mode": "worker" + } + } +} +``` + +`launchpad_status` 至少应回答： + +- 当前先补什么 +- 推荐动作码是什么 +- 当前首个缺口节点是谁 +- 当前恢复动作的人类标签是什么 +- 当前缺口更偏“待接入”还是“待验收” +- 推荐执行方式是什么 +- 先看 Worker 预案还是先看 Control 预案 + +正式约束： + +- `recommended_target_node_code` + - 一旦 launchpad 已经收敛出首个缺口节点，就应直接返回 + - 页面、CLI、Codex 不应再分别从 gap rows 中自行挑第一台 +- `recommended_recovery_label` + - 是给人看的动作标题，例如： + - `签发接入工单` + - `补执行面接管` + - `执行接管验收` +- `recommended_recovery_summary` + - 是解释“为什么当前默认不是发版，而是先补接管”的正式摘要 +- `onboarding_bootstrap_pending_nodes / onboarding_acceptance_ready_nodes` + - 是 launchpad 汇总层面的正式计数 + - 允许被 `go-live-summary`、CLI 摘要、Codex brief 直接复用 + +--- + +## 7. 发布动作 contract + +当前发布相关动作不应再退回“通用发任务”语义，而应正式区分： + +- `deploy.release.control` +- `deploy.release.worker` +- `deploy.release.custom` + +这三个动作必须共用： + +- 同一份 Gate 判断 +- 同一份 Release 版本对象 +- 同一份 Rollout 编排结果 + +--- + +## 8. Artifact 不可变要求 + +上线前建议把 Release 供应链固定为： + +1. 构建产物 +2. 生成 `checksum` +3. 生成 `manifest` +4. 上传 artifact +5. 创建 Release +6. 激活 channel +7. 创建 Rollout + +Node Agent 侧部署链路要求： + +1. 下载 artifact +2. 校验 checksum +3. 解压到 `releases/` +4. 切换 `current` +5. 重启服务 +6. 健康检查 +7. 失败回滚 + +--- + +## 9. Rollout 与 Ops Job 的关系 + +约束： + +- Rollout 是分批推进对象 +- Ops Job 是单次动作对象 +- Rollout 通过一批或多批 `deploy.release.*` Job 落地 + +控制面责任： + +- 决定下一批是否生成 Job +- 汇总 Job 结果 +- 反推 Rollout 状态 +- 决定是否暂停、继续或回滚 + +Node Agent 不负责： + +- 推进下一批 +- 修改 Rollout 策略 +- 修改 Release 状态 + +--- + +## 10. 页面 / Codex 共同消费约束 + +以下对象必须作为稳定后端 contract 存在： + +- `default_rollout_gate` +- `default_rollout_execution` +- `launchpad_status` +- `worker_rollout_preview` +- `control_rollout_preview` + +页面只负责展示和少量交互兜底。 + +Codex 只负责消费后端给出的： + +- 当前是否可发 +- 建议先发哪一侧 +- 应该使用什么 mode +- 下一步推荐动作是什么 + +--- + +## 11. Artifact Manifest Contract + +Release Hub 不能只停在： + +- `artifact_url` +- `checksum` + +这只能证明“有个包”，还不能证明“这个包会把哪些服务改成什么样”。 + +建议把 `release.metadata.manifest` 固定为正式对象。 + +最小结构： + +```json +{ + "artifact_kind": "tarball", + "package_name": "domaincheck-2026.04.18+bd6bcb2.tgz", + "package_size_bytes": 186542331, + "checksum": "sha256:...", + "build_id": "build-20260418-001", + "commit_sha": "bd6bcb2", + "built_at": "2026-04-18 05:20:00", + "included_services": [ + "domaincheck-api", + "domaincheck-worker", + "domaincheck-sync-agent", + "domaincheck-node-agent" + ], + "required_env_keys": [ + "NODE_CODE", + "NODE_REGION", + "NODE_ROLE" + ], + "health_checks": [ + { + "name": "api_health", + "type": "http", + "url": "http://127.0.0.1:8100/health" + } + ], + "rollback_hint": "切回上一个 current 并重启对应服务" +} +``` + +正式要求： + +- Release 创建后，`manifest` 不应再被覆盖成另一份语义不同的发布物 +- Node Agent、页面、Codex 必须共用这同一份 manifest +- 是否可发、怎么回滚、要验哪些服务，不应再散落在： + - shell 脚本 + - 页面表单 + - 人工脑补 + +--- + +## 12. Operational Readiness Row Contract + +`default_rollout_gate.operational_readiness.rows[]` 不应只是“几行提醒文案”，而应成为逐节点门禁对象。 + +最小结构： + +```json +{ + "node_code": "mainland-worker-01", + "region": "mainland", + "role": "worker", + "agent_managed": true, + "execution_mode_ready": true, + "execution_mode": "remote-agent", + "execution_mode_label": "Node Agent", + "inspection_status": "healthy", + "inspection_status_label": "已通过", + "current_release_version": "2026.04.11+1921318", + "desired_release_version": "2026.04.18+bd6bcb2", + "risk_level": "low", + "blocking_reasons": [], + "warning_reasons": [], + "recommended_action_code": "", + "focus_ref": { + "kind": "release_hub", + "release_id": 1, + "release_version": "2026.04.18+bd6bcb2", + "channel": "stable", + "rollout_id": 0, + "rollout_code": "", + "section": "default_rollout_gate" + } +} +``` + +正式要求： + +- `rows[]` 应能直接回答： + - 这台节点能不能进下一轮 rollout + - 不能的话卡在哪 + - 只是 warning 还是 hard block +- 页面、CLI、Codex 不应再从： + - inspection + - runtime + - nodes + 三处数据手工拼门禁结论 + +推荐风险分级： + +- `low` +- `medium` +- `high` +- `blocked` + +--- + +## 13. Rollout Preview Contract + +真正创建 Rollout 之前，还应保留一层正式 preview，而不是由页面自己预估批次和风险。 + +建议入口： + +- `POST /api/v1/ops/releases/{id}/rollouts/preview` + +最小请求体： + +```json +{ + "release_id": 1, + "target_node_codes": [ + "mainland-worker-01", + "mainland-controller-01" + ], + "policy": { + "batch_size": 1, + "require_approval_between_batches": true, + "rollback_on_failure": true + }, + "execution_mode": "remote-agent" +} +``` + +最小返回体： + +```json +{ + "release": {}, + "execution_mode": "remote-agent", + "execution_mode_label": "Node Agent", + "target_nodes_total": 2, + "target_node_codes": [ + "mainland-worker-01", + "mainland-controller-01" + ], + "expected_batches_total": 2, + "expected_jobs_total": 2, + "gate": {}, + "batch_plan": [ + { + "batch_no": 1, + "node_codes": [ + "mainland-worker-01" + ], + "risk_level": "low" + }, + { + "batch_no": 2, + "node_codes": [ + "mainland-controller-01" + ], + "risk_level": "medium" + } + ], + "summary": "建议先灰度 1 台 Worker，再推进 Controller。", + "summary_text": "建议先灰度 1 台 Worker，再推进 Controller。", + "focus_ref": { + "kind": "release_hub", + "release_id": 1, + "release_version": "2026.04.18+bd6bcb2", + "channel": "stable", + "rollout_id": 0, + "rollout_code": "", + "section": "rollout_preview" + } +} +``` + +正式要求： + +- preview 是： + - 页面发 rollout 前的统一预检 + - Codex 自动驾驶发 rollout 前的统一预检 + - CLI 批量发布前的统一预检 +- 不允许前端重新自己算： + - 该分几批 + - 哪台应该先发 + - 哪台属于高风险 + +只有这样，Release Hub 才是真正的发布控制面，而不是“能看版本的页面”。 diff --git a/domain-api/app/api/routes/ops.py b/domain-api/app/api/routes/ops.py new file mode 100644 index 0000000..fe9bfc3 --- /dev/null +++ b/domain-api/app/api/routes/ops.py @@ -0,0 +1,633 @@ +from __future__ import annotations + +from fastapi import APIRouter + +from app.schemas.common import ApiResponse +from app.services.ops_agent_service import ( + build_managed_node_handover_bootstrap_plan, + execute_managed_node_onboarding_bootstrap, + execute_managed_node_onboarding_acceptance, + execute_managed_node_onboarding_recovery, + get_managed_node_delivery_queue, + get_managed_node_handover, + get_managed_node_onboarding, + list_managed_node_delivery_queue_records, + list_managed_nodes_with_agent_state, + list_ops_job_events, + preview_managed_node_onboarding_bootstrap, + preview_managed_node_onboarding_acceptance, + preview_managed_node_onboarding_recovery, + request_managed_node_delivery_queue_action, + summarize_ops_job_events, +) +from app.services.ops_job_service import ( + approve_ops_job, + cancel_ops_job, + create_ops_job, + create_ops_job_batch, + dispatch_ops_job, + get_ops_job, + list_ops_jobs, + sync_managed_nodes_from_cluster, + upsert_managed_node, +) +from app.services.ops_playbook_service import ( + cancel_ops_playbook_run, + execute_ops_playbook, + get_ops_playbook_run, + get_ops_playbooks, + get_recent_ops_playbook_runs, + list_ops_playbook_run_events, + preview_ops_playbook, + rerun_ops_playbook_run, +) +from app.services.ops_policy_service import preview_ops_job_policy +from app.services.ops_service import ( + execute_codex_action, + execute_resolved_driver_action, + get_ops_activity_stream, + get_ops_codex_brief, + get_ops_contract_detail, + get_ops_contract_registry, + get_ops_doctor_decision, + get_ops_driver_feed, + get_ops_go_live_summary, + get_ops_go_live_signoff, + get_ops_go_live_bundle, + get_ops_go_live_review, + get_ops_link_snapshot, + get_ops_node_scene_log, + get_ops_stack_diagnosis, + execute_driver_action, + preview_driver_action, + resolve_codex_action, + resolve_driver_action_request, + resolve_ops_runbook_sequence, + execute_ops_runbook_sequence, + get_ops_blueprint, + get_ops_capabilities, + get_ops_inspection_overview, + get_ops_overview, + get_ops_runbook, +) +from app.services.ops_template_service import get_ops_action_templates + +router = APIRouter(tags=["ops"]) + + +@router.get("/ops/overview", response_model=ApiResponse) +def ops_overview() -> ApiResponse: + return ApiResponse(data=get_ops_overview()) + + +@router.get("/ops/link-snapshot", response_model=ApiResponse) +def ops_link_snapshot() -> ApiResponse: + return ApiResponse(data=get_ops_link_snapshot()) + + +@router.get("/ops/stack-diagnosis", response_model=ApiResponse) +def ops_stack_diagnosis(base_url: str = "") -> ApiResponse: + return ApiResponse(data=get_ops_stack_diagnosis(base_url=base_url)) + + +@router.get("/ops/go-live-summary", response_model=ApiResponse) +def ops_go_live_summary(base_url: str = "") -> ApiResponse: + return ApiResponse(data=get_ops_go_live_summary(base_url=base_url)) + + +@router.get("/ops/doctor-decision", response_model=ApiResponse) +def ops_doctor_decision(base_url: str = "", report_dir: str = "") -> ApiResponse: + return ApiResponse(data=get_ops_doctor_decision(base_url=base_url, report_dir=report_dir)) + + +@router.get("/ops/go-live-signoff", response_model=ApiResponse) +def ops_go_live_signoff(base_url: str = "", report_dir: str = "") -> ApiResponse: + return ApiResponse(data=get_ops_go_live_signoff(base_url=base_url, report_dir=report_dir)) + + +@router.get("/ops/go-live-bundle", response_model=ApiResponse) +def ops_go_live_bundle(base_url: str = "", report_dir: str = "") -> ApiResponse: + return ApiResponse(data=get_ops_go_live_bundle(base_url=base_url, report_dir=report_dir)) + + +@router.get("/ops/go-live-review", response_model=ApiResponse) +def ops_go_live_review(base_url: str = "", report_dir: str = "") -> ApiResponse: + return ApiResponse(data=get_ops_go_live_review(base_url=base_url, report_dir=report_dir)) + + +@router.get("/ops/inspection-overview", response_model=ApiResponse) +def ops_inspection_overview( + limit: int = 80, + status: str = "", + problem_kind: str = "", + query: str = "", + only_problem: bool = False, + only_participating: bool = False, +) -> ApiResponse: + return ApiResponse( + data=get_ops_inspection_overview( + fetch_limit=limit, + status=status, + problem_kind=problem_kind, + query=query, + only_problem=only_problem, + only_participating=only_participating, + ) + ) + + +@router.get("/ops/activity-stream", response_model=ApiResponse) +def ops_activity_stream( + limit: int = 18, + kind: str = "", + status: str = "", + execution_mode: str = "", + query: str = "", +) -> ApiResponse: + return ApiResponse( + data=get_ops_activity_stream( + limit=limit, + kind=kind, + status=status, + execution_mode=execution_mode, + query=query, + ) + ) + + +@router.get("/ops/driver-feed", response_model=ApiResponse) +def ops_driver_feed() -> ApiResponse: + return ApiResponse(data=get_ops_driver_feed()) + + +@router.get("/ops/codex-brief", response_model=ApiResponse) +def ops_codex_brief() -> ApiResponse: + return ApiResponse(data=get_ops_codex_brief()) + + +@router.get("/ops/capabilities", response_model=ApiResponse) +def ops_capabilities() -> ApiResponse: + return ApiResponse(data=get_ops_capabilities()) + + +@router.get("/ops/contracts", response_model=ApiResponse) +def ops_contracts() -> ApiResponse: + return ApiResponse(data=get_ops_contract_registry()) + + +@router.get("/ops/contracts/{contract_key}", response_model=ApiResponse) +def ops_contract_detail(contract_key: str) -> ApiResponse: + data = get_ops_contract_detail(contract_key) + if not data: + return ApiResponse(code=1, message="contract 不存在", data={}) + return ApiResponse(data=data) + + +@router.get("/ops/action-templates", response_model=ApiResponse) +def ops_action_templates() -> ApiResponse: + return ApiResponse(data=get_ops_action_templates()) + + +@router.get("/ops/playbooks", response_model=ApiResponse) +def ops_playbooks() -> ApiResponse: + return ApiResponse(data=get_ops_playbooks()) + + +@router.get("/ops/playbook-runs", response_model=ApiResponse) +def ops_playbook_runs( + limit: int = 12, + status: str = "", + group_key: str = "", + query: str = "", +) -> ApiResponse: + return ApiResponse( + data=get_recent_ops_playbook_runs( + limit=limit, + status=status, + group_key=group_key, + query=query, + ) + ) + + +@router.get("/ops/playbook-runs/{run_code}", response_model=ApiResponse) +def ops_playbook_run_detail(run_code: str) -> ApiResponse: + data = get_ops_playbook_run(run_code) + if not data: + return ApiResponse(code=1, message="playbook run 不存在", data={}) + return ApiResponse(data=data) + + +@router.get("/ops/playbook-runs/{run_code}/events", response_model=ApiResponse) +def ops_playbook_run_events( + run_code: str, + limit: int = 80, + step_key: str = "", + node_code: str = "", +) -> ApiResponse: + data = list_ops_playbook_run_events( + run_code, + limit=limit, + step_key=step_key, + node_code=node_code, + ) + return ApiResponse(data=data) + + +@router.post("/ops/playbook-runs/{run_code}/rerun", response_model=ApiResponse) +def ops_playbook_run_rerun(run_code: str, payload: dict | None = None) -> ApiResponse: + payload = payload or {} + ok, message, data = rerun_ops_playbook_run( + run_code, + requested_by=str(payload.get("requested_by") or "api").strip() or "api", + ) + return ApiResponse(code=0 if ok else 1, message=message, data=data) + + +@router.post("/ops/playbook-runs/{run_code}/cancel", response_model=ApiResponse) +def ops_playbook_run_cancel(run_code: str, payload: dict | None = None) -> ApiResponse: + payload = payload or {} + ok, message, data = cancel_ops_playbook_run( + run_code, + cancelled_by=str(payload.get("cancelled_by") or "api").strip() or "api", + reason=str(payload.get("reason") or "").strip(), + ) + return ApiResponse(code=0 if ok else 1, message=message, data=data) + + +@router.post("/ops/playbooks/preview", response_model=ApiResponse) +def ops_playbook_preview(payload: dict) -> ApiResponse: + ok, message, data = preview_ops_playbook(payload or {}) + return ApiResponse(code=0 if ok else 1, message=message, data=data) + + +@router.post("/ops/playbooks/execute", response_model=ApiResponse) +def ops_playbook_execute(payload: dict) -> ApiResponse: + ok, message, data = execute_ops_playbook(payload or {}) + return ApiResponse(code=0 if ok else 1, message=message, data=data) + + +@router.get("/ops/blueprint", response_model=ApiResponse) +def ops_blueprint() -> ApiResponse: + return ApiResponse(data=get_ops_blueprint()) + + +@router.get("/ops/runbook", response_model=ApiResponse) +def ops_runbook() -> ApiResponse: + return ApiResponse(data=get_ops_runbook()) + + +@router.post("/ops/runbook/sequences/{sequence_key}/resolve", response_model=ApiResponse) +def ops_runbook_sequence_resolve(sequence_key: str, payload: dict | None = None) -> ApiResponse: + ok, message, data = resolve_ops_runbook_sequence(sequence_key, payload or {}) + return ApiResponse(code=0 if ok else 1, message=message, data=data) + + +@router.post("/ops/runbook/sequences/{sequence_key}/execute", response_model=ApiResponse) +def ops_runbook_sequence_execute(sequence_key: str, payload: dict | None = None) -> ApiResponse: + ok, message, data = execute_ops_runbook_sequence(sequence_key, payload or {}) + return ApiResponse(code=0 if ok else 1, message=message, data=data) + + +@router.get("/ops/nodes", response_model=ApiResponse) +def ops_nodes() -> ApiResponse: + return ApiResponse(data=list_managed_nodes_with_agent_state()) + + +@router.get("/ops/nodes/{node_code}/handover", response_model=ApiResponse) +def ops_node_handover( + node_code: str, + control_plane_base_url: str = "", + root_dir: str = "", +) -> ApiResponse: + data = get_managed_node_handover( + node_code, + control_plane_base_url=control_plane_base_url, + root_dir=root_dir, + ) + if not data: + return ApiResponse(code=1, message="节点不存在", data={}) + return ApiResponse(data=data) + + +@router.get("/ops/nodes/{node_code}/onboarding", response_model=ApiResponse) +def ops_node_onboarding( + node_code: str, + control_plane_base_url: str = "", + root_dir: str = "", +) -> ApiResponse: + data = get_managed_node_onboarding( + node_code, + control_plane_base_url=control_plane_base_url, + root_dir=root_dir, + ) + if not data: + return ApiResponse(code=1, message="节点不存在", data={}) + return ApiResponse(data=data) + + +@router.post("/ops/nodes/{node_code}/onboarding/acceptance/preview", response_model=ApiResponse) +def ops_node_onboarding_acceptance_preview(node_code: str, payload: dict | None = None) -> ApiResponse: + normalized_payload = payload or {} + ok, message, data = preview_managed_node_onboarding_acceptance( + node_code=node_code, + requested_by=str(normalized_payload.get("requested_by") or "api").strip() or "api", + control_plane_base_url=str(normalized_payload.get("control_plane_base_url") or "").strip(), + root_dir=str(normalized_payload.get("root_dir") or "").strip(), + ) + return ApiResponse(code=0 if ok else 1, message=message, data=data) + + +@router.post("/ops/nodes/{node_code}/onboarding/bootstrap/preview", response_model=ApiResponse) +def ops_node_onboarding_bootstrap_preview(node_code: str, payload: dict | None = None) -> ApiResponse: + normalized_payload = payload or {} + ok, message, data = preview_managed_node_onboarding_bootstrap( + node_code=node_code, + requested_by=str(normalized_payload.get("requested_by") or "api").strip() or "api", + control_plane_base_url=str(normalized_payload.get("control_plane_base_url") or "").strip(), + root_dir=str(normalized_payload.get("root_dir") or "").strip(), + ) + return ApiResponse(code=0 if ok else 1, message=message, data=data) + + +@router.post("/ops/nodes/{node_code}/onboarding/bootstrap/execute", response_model=ApiResponse) +def ops_node_onboarding_bootstrap_execute(node_code: str, payload: dict | None = None) -> ApiResponse: + normalized_payload = payload or {} + ok, message, data = execute_managed_node_onboarding_bootstrap( + node_code=node_code, + requested_by=str(normalized_payload.get("requested_by") or "api").strip() or "api", + control_plane_base_url=str(normalized_payload.get("control_plane_base_url") or "").strip(), + root_dir=str(normalized_payload.get("root_dir") or "").strip(), + ) + return ApiResponse(code=0 if ok else 1, message=message, data=data) + + +@router.post("/ops/nodes/{node_code}/onboarding/acceptance/execute", response_model=ApiResponse) +def ops_node_onboarding_acceptance_execute(node_code: str, payload: dict | None = None) -> ApiResponse: + normalized_payload = payload or {} + ok, message, data = execute_managed_node_onboarding_acceptance( + node_code=node_code, + requested_by=str(normalized_payload.get("requested_by") or "api").strip() or "api", + control_plane_base_url=str(normalized_payload.get("control_plane_base_url") or "").strip(), + root_dir=str(normalized_payload.get("root_dir") or "").strip(), + ) + return ApiResponse(code=0 if ok else 1, message=message, data=data) + + +@router.post("/ops/nodes/{node_code}/onboarding/recovery/preview", response_model=ApiResponse) +def ops_node_onboarding_recovery_preview(node_code: str, payload: dict | None = None) -> ApiResponse: + normalized_payload = payload or {} + ok, message, data = preview_managed_node_onboarding_recovery( + node_code=node_code, + requested_by=str(normalized_payload.get("requested_by") or "api").strip() or "api", + control_plane_base_url=str(normalized_payload.get("control_plane_base_url") or "").strip(), + root_dir=str(normalized_payload.get("root_dir") or "").strip(), + ) + return ApiResponse(code=0 if ok else 1, message=message, data=data) + + +@router.post("/ops/nodes/{node_code}/onboarding/recovery/execute", response_model=ApiResponse) +def ops_node_onboarding_recovery_execute(node_code: str, payload: dict | None = None) -> ApiResponse: + normalized_payload = payload or {} + ok, message, data = execute_managed_node_onboarding_recovery( + node_code=node_code, + requested_by=str(normalized_payload.get("requested_by") or "api").strip() or "api", + control_plane_base_url=str(normalized_payload.get("control_plane_base_url") or "").strip(), + root_dir=str(normalized_payload.get("root_dir") or "").strip(), + ) + return ApiResponse(code=0 if ok else 1, message=message, data=data) + + +@router.get("/ops/nodes/{node_code}/scene-log", response_model=ApiResponse) +def ops_node_scene_log( + node_code: str, + limit: int = 80, + mode: str = "", +) -> ApiResponse: + data = get_ops_node_scene_log(node_code, limit=limit, mode=mode) + if not data: + return ApiResponse(code=1, message="节点不存在", data={}) + return ApiResponse(data=data) + + +@router.post("/ops/nodes/{node_code}/handover/bootstrap-plan", response_model=ApiResponse) +def ops_node_handover_bootstrap_plan(node_code: str, payload: dict | None = None) -> ApiResponse: + normalized_payload = payload or {} + ok, message, data = build_managed_node_handover_bootstrap_plan( + node_code=node_code, + issued_by=str(normalized_payload.get("issued_by") or "api").strip() or "api", + expires_in_hours=int(normalized_payload.get("expires_in_hours") or 72), + control_plane_base_url=str(normalized_payload.get("control_plane_base_url") or "").strip(), + root_dir=str(normalized_payload.get("root_dir") or "").strip(), + metadata=normalized_payload.get("metadata") or {}, + ) + return ApiResponse(code=0 if ok else 1, message=message, data=data) + + +@router.get("/ops/nodes/{node_code}/delivery-queue", response_model=ApiResponse) +def ops_node_delivery_queue(node_code: str) -> ApiResponse: + data = get_managed_node_delivery_queue(node_code) + if not data: + return ApiResponse(code=1, message="节点不存在", data={}) + return ApiResponse(data=data) + + +@router.get("/ops/nodes/{node_code}/delivery-queue/records", response_model=ApiResponse) +def ops_node_delivery_queue_records( + node_code: str, + state: str = "", + limit: int = 50, +) -> ApiResponse: + data = list_managed_node_delivery_queue_records( + node_code, + state=state, + limit=limit, + ) + if not data: + return ApiResponse(code=1, message="节点不存在", data={}) + return ApiResponse(data=data) + + +@router.post("/ops/nodes/{node_code}/delivery-queue/flush", response_model=ApiResponse) +def ops_node_delivery_queue_flush(node_code: str, payload: dict | None = None) -> ApiResponse: + ok, message, data = request_managed_node_delivery_queue_action( + node_code, + "delivery.queue.flush", + payload=payload or {}, + ) + return ApiResponse(code=0 if ok else 1, message=message, data=data) + + +@router.post("/ops/nodes/{node_code}/delivery-queue/replay", response_model=ApiResponse) +def ops_node_delivery_queue_replay(node_code: str, payload: dict | None = None) -> ApiResponse: + ok, message, data = request_managed_node_delivery_queue_action( + node_code, + "delivery.queue.replay", + payload=payload or {}, + ) + return ApiResponse(code=0 if ok else 1, message=message, data=data) + + +@router.post("/ops/nodes/{node_code}/delivery-queue/records/{record_id}/replay", response_model=ApiResponse) +def ops_node_delivery_queue_record_replay( + node_code: str, + record_id: str, + payload: dict | None = None, +) -> ApiResponse: + ok, message, data = request_managed_node_delivery_queue_action( + node_code, + "delivery.queue.replay", + payload=payload or {}, + record_id=record_id, + ) + return ApiResponse(code=0 if ok else 1, message=message, data=data) + + +@router.post("/ops/nodes/{node_code}/delivery-queue/records/{record_id}/discard", response_model=ApiResponse) +def ops_node_delivery_queue_record_discard( + node_code: str, + record_id: str, + payload: dict | None = None, +) -> ApiResponse: + ok, message, data = request_managed_node_delivery_queue_action( + node_code, + "delivery.queue.discard", + payload=payload or {}, + record_id=record_id, + ) + return ApiResponse(code=0 if ok else 1, message=message, data=data) + + +@router.post("/ops/nodes", response_model=ApiResponse) +def ops_nodes_upsert(payload: dict) -> ApiResponse: + ok, message, data = upsert_managed_node(payload) + return ApiResponse(code=0 if ok else 1, message=message, data=data) + + +@router.post("/ops/nodes/sync-from-cluster", response_model=ApiResponse) +def ops_nodes_sync_from_cluster(dry_run: bool = False) -> ApiResponse: + return ApiResponse(data=sync_managed_nodes_from_cluster(dry_run=dry_run)) + + +@router.get("/ops/jobs", response_model=ApiResponse) +def ops_jobs(limit: int = 20, compact: bool = True, rollout_id: int = 0) -> ApiResponse: + normalized_rollout_id = int(rollout_id or 0) + jobs = list_ops_jobs( + limit=limit, + compact=compact, + rollout_id=normalized_rollout_id if normalized_rollout_id > 0 else None, + ) + page_status_counts: dict[str, int] = {} + for item in jobs: + item_status = str(item.get("status") or "").strip() or "unknown" + page_status_counts[item_status] = int(page_status_counts.get(item_status, 0) or 0) + 1 + return ApiResponse( + data={ + "jobs": jobs, + "filters": { + "limit": int(limit or 20), + "compact": bool(compact), + "rollout_id": normalized_rollout_id, + }, + "page_summary": { + "jobs_total": len(jobs), + "status_counts": page_status_counts, + }, + } + ) + + +@router.get("/ops/jobs/{job_id}", response_model=ApiResponse) +def ops_job_detail(job_id: int) -> ApiResponse: + data = get_ops_job(job_id) + if not data: + return ApiResponse(code=1, message="任务不存在", data={}) + return ApiResponse(data=data) + + +@router.get("/ops/jobs/{job_id}/events", response_model=ApiResponse) +def ops_job_events(job_id: int, limit: int = 50) -> ApiResponse: + events = list_ops_job_events(job_id, limit=limit) + return ApiResponse( + data={ + "events": events, + "summary": summarize_ops_job_events(events, job_id=job_id, limit=limit), + } + ) + + +@router.post("/ops/driver-actions/execute", response_model=ApiResponse) +def ops_driver_action_execute(payload: dict) -> ApiResponse: + ok, message, data = execute_driver_action(payload or {}) + return ApiResponse(code=0 if ok else 1, message=message, data=data) + + +@router.post("/ops/driver-actions/preview", response_model=ApiResponse) +def ops_driver_action_preview(payload: dict) -> ApiResponse: + ok, message, data = preview_driver_action(payload or {}) + return ApiResponse(code=0 if ok else 1, message=message, data=data) + + +@router.post("/ops/driver-actions/resolve", response_model=ApiResponse) +def ops_driver_action_resolve(payload: dict | None = None) -> ApiResponse: + ok, message, data = resolve_driver_action_request(payload or {}) + return ApiResponse(code=0 if ok else 1, message=message, data=data) + + +@router.post("/ops/driver-actions/execute-resolved", response_model=ApiResponse) +def ops_driver_action_execute_resolved(payload: dict | None = None) -> ApiResponse: + ok, message, data = execute_resolved_driver_action(payload or {}) + return ApiResponse(code=0 if ok else 1, message=message, data=data) + + +@router.post("/ops/codex-actions/resolve", response_model=ApiResponse) +def ops_codex_action_resolve(payload: dict | None = None) -> ApiResponse: + ok, message, data = resolve_codex_action(payload or {}) + return ApiResponse(code=0 if ok else 1, message=message, data=data) + + +@router.post("/ops/codex-actions/execute", response_model=ApiResponse) +def ops_codex_action_execute(payload: dict | None = None) -> ApiResponse: + ok, message, data = execute_codex_action(payload or {}) + return ApiResponse(code=0 if ok else 1, message=message, data=data) + + +@router.post("/ops/jobs", response_model=ApiResponse) +def ops_jobs_create(payload: dict) -> ApiResponse: + ok, message, data = create_ops_job(payload) + return ApiResponse(code=0 if ok else 1, message=message, data=data) + + +@router.post("/ops/jobs/batch", response_model=ApiResponse) +def ops_jobs_create_batch(payload: dict) -> ApiResponse: + ok, message, data = create_ops_job_batch(payload) + return ApiResponse(code=0 if ok else 1, message=message, data=data) + + +@router.post("/ops/policy/preview", response_model=ApiResponse) +def ops_policy_preview(payload: dict) -> ApiResponse: + return ApiResponse(data=preview_ops_job_policy(payload)) + + +@router.post("/ops/jobs/{job_id}/approve", response_model=ApiResponse) +def ops_job_approve(job_id: int, payload: dict | None = None) -> ApiResponse: + payload = payload or {} + ok, message, data = approve_ops_job(job_id, approved_by=str(payload.get("approved_by") or "api").strip() or "api") + return ApiResponse(code=0 if ok else 1, message=message, data=data) + + +@router.post("/ops/jobs/{job_id}/cancel", response_model=ApiResponse) +def ops_job_cancel(job_id: int, payload: dict | None = None) -> ApiResponse: + payload = payload or {} + ok, message, data = cancel_ops_job( + job_id, + cancelled_by=str(payload.get("cancelled_by") or "api").strip() or "api", + reason=str(payload.get("reason") or "").strip(), + ) + return ApiResponse(code=0 if ok else 1, message=message, data=data) + + +@router.post("/ops/jobs/{job_id}/dispatch", response_model=ApiResponse) +def ops_job_dispatch(job_id: int) -> ApiResponse: + ok, message, data = dispatch_ops_job(job_id) + return ApiResponse(code=0 if ok else 1, message=message, data=data) diff --git a/domain-api/app/api/routes/ops_agent.py b/domain-api/app/api/routes/ops_agent.py new file mode 100644 index 0000000..7d7607e --- /dev/null +++ b/domain-api/app/api/routes/ops_agent.py @@ -0,0 +1,90 @@ +from typing import Optional + +from fastapi import APIRouter, Header + +from app.schemas.common import ApiResponse +from app.services.ops_agent_service import ( + agent_append_job_event, + agent_complete_job, + agent_heartbeat, + agent_mark_job_started, + agent_pull_jobs, + agent_register, + build_node_agent_bootstrap_plan, + issue_node_agent_token, +) + +router = APIRouter(tags=["ops-agent"]) + + +def _resolve_agent_token(x_domaincheck_agent_token: Optional[str]) -> str: + return str(x_domaincheck_agent_token or "").strip() + + +def _build_agent_response(ok: bool, message: str, data: object) -> ApiResponse: + detail_code = None + if isinstance(data, dict): + detail_code = str(data.get("detail_code") or "").strip() or None + return ApiResponse(code=0 if ok else 1, message=message, data=data, detail_code=detail_code) + + +@router.post("/ops/agent/tokens", response_model=ApiResponse) +def ops_agent_issue_token(payload: dict) -> ApiResponse: + ok, message, data = issue_node_agent_token( + node_code=str(payload.get("node_code") or "").strip(), + issued_by=str(payload.get("issued_by") or "api").strip() or "api", + expires_in_hours=int(payload.get("expires_in_hours") or 72), + metadata=payload.get("metadata") or {}, + ) + return _build_agent_response(ok, message, data) + + +@router.post("/ops/agent/bootstrap-plan", response_model=ApiResponse) +def ops_agent_bootstrap_plan(payload: dict) -> ApiResponse: + ok, message, data = build_node_agent_bootstrap_plan( + node_code=str(payload.get("node_code") or "").strip(), + node_region=str(payload.get("node_region") or "mainland").strip() or "mainland", + node_role=str(payload.get("node_role") or "worker").strip() or "worker", + issued_by=str(payload.get("issued_by") or "api").strip() or "api", + expires_in_hours=int(payload.get("expires_in_hours") or 72), + control_plane_base_url=str(payload.get("control_plane_base_url") or "").strip(), + root_dir=str(payload.get("root_dir") or "/opt/domaincheck").strip() or "/opt/domaincheck", + metadata=payload.get("metadata") or {}, + ) + return _build_agent_response(ok, message, data) + + +@router.post("/ops/agent/register", response_model=ApiResponse) +def ops_agent_register(payload: dict, x_domaincheck_agent_token: Optional[str] = Header(default=None)) -> ApiResponse: + ok, message, data = agent_register(payload, token=_resolve_agent_token(x_domaincheck_agent_token)) + return _build_agent_response(ok, message, data) + + +@router.post("/ops/agent/heartbeat", response_model=ApiResponse) +def ops_agent_heartbeat(payload: dict, x_domaincheck_agent_token: Optional[str] = Header(default=None)) -> ApiResponse: + ok, message, data = agent_heartbeat(payload, token=_resolve_agent_token(x_domaincheck_agent_token)) + return _build_agent_response(ok, message, data) + + +@router.post("/ops/agent/pull", response_model=ApiResponse) +def ops_agent_pull(payload: dict, limit: int = 1, x_domaincheck_agent_token: Optional[str] = Header(default=None)) -> ApiResponse: + ok, message, data = agent_pull_jobs(payload, token=_resolve_agent_token(x_domaincheck_agent_token), limit=limit) + return _build_agent_response(ok, message, data) + + +@router.post("/ops/agent/jobs/{job_id}/start", response_model=ApiResponse) +def ops_agent_job_start(job_id: int, payload: dict, x_domaincheck_agent_token: Optional[str] = Header(default=None)) -> ApiResponse: + ok, message, data = agent_mark_job_started(job_id, payload, token=_resolve_agent_token(x_domaincheck_agent_token)) + return _build_agent_response(ok, message, data) + + +@router.post("/ops/agent/jobs/{job_id}/complete", response_model=ApiResponse) +def ops_agent_job_complete(job_id: int, payload: dict, x_domaincheck_agent_token: Optional[str] = Header(default=None)) -> ApiResponse: + ok, message, data = agent_complete_job(job_id, payload, token=_resolve_agent_token(x_domaincheck_agent_token)) + return _build_agent_response(ok, message, data) + + +@router.post("/ops/agent/jobs/{job_id}/events", response_model=ApiResponse) +def ops_agent_job_event(job_id: int, payload: dict, x_domaincheck_agent_token: Optional[str] = Header(default=None)) -> ApiResponse: + ok, message, data = agent_append_job_event(job_id, payload, token=_resolve_agent_token(x_domaincheck_agent_token)) + return _build_agent_response(ok, message, data) diff --git a/domain-api/app/api/routes/ops_release.py b/domain-api/app/api/routes/ops_release.py new file mode 100644 index 0000000..e3146b9 --- /dev/null +++ b/domain-api/app/api/routes/ops_release.py @@ -0,0 +1,175 @@ +from __future__ import annotations + +from fastapi import APIRouter, Request +from fastapi.responses import FileResponse + +from app.schemas.common import ApiResponse +from app.services.ops_release_service import ( + advance_release_rollout, + activate_release, + create_release, + create_release_and_smart_rollout_from_latest_package, + create_release_from_latest_package, + create_release_rollout, + create_smart_release_rollout, + get_latest_release_package_metadata, + get_release_launchpad, + get_latest_release, + get_release_package_archive_path, + get_release_package_sha256_path, + get_release, + get_release_rollout, + list_release_rollout_jobs, + list_release_rollouts, + list_releases, + preview_release_from_latest_package, + preview_smart_release_rollout, + preview_smart_release_rollout_from_latest_package, +) + +router = APIRouter(tags=["ops-release"]) + + +@router.get("/ops/releases", response_model=ApiResponse) +def ops_releases(limit: int = 20, channel: str | None = None) -> ApiResponse: + return ApiResponse(data={"releases": list_releases(limit=limit, channel=channel)}) + + +@router.get("/ops/releases/latest", response_model=ApiResponse) +def ops_releases_latest(channel: str = "stable") -> ApiResponse: + return ApiResponse(data=get_latest_release(channel=channel)) + + +@router.get("/ops/releases/package-metadata/latest", response_model=ApiResponse) +def ops_releases_package_metadata_latest() -> ApiResponse: + return ApiResponse(data=get_latest_release_package_metadata()) + + +@router.get("/ops/releases/launchpad", response_model=ApiResponse) +def ops_releases_launchpad(request: Request, channel: str = "stable") -> ApiResponse: + derived_base_url = str(request.base_url).rstrip("/") + return ApiResponse(data=get_release_launchpad(control_plane_base_url=derived_base_url, channel=channel)) + + +@router.get("/ops/releases/packages/{package_name}/download") +def ops_release_package_download(package_name: str): + try: + archive_path = get_release_package_archive_path(package_name) + except ValueError: + return ApiResponse(code=1, message="发布包名称不合法", data={}) + if not archive_path: + return ApiResponse(code=1, message="发布包不存在", data={}) + media_type = "application/octet-stream" + archive_name = archive_path.name.lower() + if archive_name.endswith(".tar.gz"): + media_type = "application/gzip" + elif archive_name.endswith(".zip"): + media_type = "application/zip" + return FileResponse(path=archive_path, filename=archive_path.name, media_type=media_type) + + +@router.get("/ops/releases/packages/{package_name}/sha256") +def ops_release_package_sha256(package_name: str): + try: + sha256_path = get_release_package_sha256_path(package_name) + except ValueError: + return ApiResponse(code=1, message="发布包名称不合法", data={}) + if not sha256_path: + return ApiResponse(code=1, message="发布包校验文件不存在", data={}) + return FileResponse(path=sha256_path, filename=sha256_path.name, media_type="text/plain; charset=utf-8") + + +@router.get("/ops/releases/{release_id}", response_model=ApiResponse) +def ops_releases_detail(release_id: int) -> ApiResponse: + data = get_release(release_id) + if not data: + return ApiResponse(code=1, message="Release 不存在", data={}) + return ApiResponse(data=data) + + +@router.post("/ops/releases", response_model=ApiResponse) +def ops_releases_create(payload: dict) -> ApiResponse: + ok, message, data = create_release(payload) + return ApiResponse(code=0 if ok else 1, message=message, data=data) + + +@router.post("/ops/releases/from-package/latest", response_model=ApiResponse) +def ops_releases_create_from_latest_package(request: Request, payload: dict | None = None) -> ApiResponse: + derived_base_url = str(request.base_url).rstrip("/") + ok, message, data = create_release_from_latest_package(payload or {}, control_plane_base_url=derived_base_url) + return ApiResponse(code=0 if ok else 1, message=message, data=data) + + +@router.post("/ops/releases/from-package/latest/preview", response_model=ApiResponse) +def ops_releases_preview_from_latest_package(request: Request, payload: dict | None = None) -> ApiResponse: + derived_base_url = str(request.base_url).rstrip("/") + ok, message, data = preview_release_from_latest_package(payload or {}, control_plane_base_url=derived_base_url) + return ApiResponse(code=0 if ok else 1, message=message, data=data) + + +@router.post("/ops/releases/from-package/latest/smart-rollout", response_model=ApiResponse) +def ops_releases_create_from_latest_package_and_smart_rollout(request: Request, payload: dict | None = None) -> ApiResponse: + derived_base_url = str(request.base_url).rstrip("/") + ok, message, data = create_release_and_smart_rollout_from_latest_package( + payload or {}, + control_plane_base_url=derived_base_url, + ) + return ApiResponse(code=0 if ok else 1, message=message, data=data) + + +@router.post("/ops/releases/from-package/latest/smart-rollout-preview", response_model=ApiResponse) +def ops_releases_preview_from_latest_package_and_smart_rollout(request: Request, payload: dict | None = None) -> ApiResponse: + derived_base_url = str(request.base_url).rstrip("/") + ok, message, data = preview_smart_release_rollout_from_latest_package( + payload or {}, + control_plane_base_url=derived_base_url, + ) + return ApiResponse(code=0 if ok else 1, message=message, data=data) + + +@router.post("/ops/releases/{release_id}/activate", response_model=ApiResponse) +def ops_releases_activate(release_id: int) -> ApiResponse: + ok, message, data = activate_release(release_id) + return ApiResponse(code=0 if ok else 1, message=message, data=data) + + +@router.post("/ops/releases/{release_id}/smart-rollout-preview", response_model=ApiResponse) +def ops_releases_smart_rollout_preview(release_id: int, payload: dict | None = None) -> ApiResponse: + ok, message, data = preview_smart_release_rollout(release_id, payload or {}) + return ApiResponse(code=0 if ok else 1, message=message, data=data) + + +@router.post("/ops/releases/{release_id}/smart-rollout", response_model=ApiResponse) +def ops_releases_smart_rollout(release_id: int, payload: dict | None = None) -> ApiResponse: + ok, message, data = create_smart_release_rollout(release_id, payload or {}) + return ApiResponse(code=0 if ok else 1, message=message, data=data) + + +@router.get("/ops/releases/{release_id}/rollouts", response_model=ApiResponse) +def ops_release_rollouts(release_id: int, limit: int = 20) -> ApiResponse: + return ApiResponse(data={"rollouts": list_release_rollouts(release_id=release_id, limit=limit)}) + + +@router.get("/ops/rollouts/{rollout_id}", response_model=ApiResponse) +def ops_rollout_detail(rollout_id: int) -> ApiResponse: + data = get_release_rollout(rollout_id) + if not data: + return ApiResponse(code=1, message="Rollout 不存在", data={}) + return ApiResponse(data=data) + + +@router.get("/ops/rollouts/{rollout_id}/jobs", response_model=ApiResponse) +def ops_rollout_jobs(rollout_id: int, limit: int = 200) -> ApiResponse: + return ApiResponse(data={"jobs": list_release_rollout_jobs(rollout_id, limit=limit)}) + + +@router.post("/ops/rollouts/{rollout_id}/advance", response_model=ApiResponse) +def ops_rollout_advance(rollout_id: int, payload: dict | None = None) -> ApiResponse: + ok, message, data = advance_release_rollout(rollout_id, payload or {}) + return ApiResponse(code=0 if ok else 1, message=message, data=data) + + +@router.post("/ops/releases/{release_id}/rollouts", response_model=ApiResponse) +def ops_release_rollouts_create(release_id: int, payload: dict) -> ApiResponse: + ok, message, data = create_release_rollout(release_id, payload) + return ApiResponse(code=0 if ok else 1, message=message, data=data) diff --git a/domain-api/app/api/routes/runtime.py b/domain-api/app/api/routes/runtime.py index 821e22e..f8bccac 100644 --- a/domain-api/app/api/routes/runtime.py +++ b/domain-api/app/api/routes/runtime.py @@ -3,6 +3,7 @@ from typing import Optional from fastapi import APIRouter, Header from app.schemas.common import ApiResponse +from app.services.build_info_service import get_runtime_build_info from app.services.cluster_runtime_service import get_cluster_snapshot from app.services.debug_event_service import get_debug_diagnosis, get_debug_event_overview, get_debug_handoff_report, ingest_debug_event, list_debug_events from app.services.runtime_control_service import runtime_action @@ -37,6 +38,11 @@ def runtime_cluster() -> ApiResponse: return ApiResponse(data=get_cluster_snapshot()) +@router.get("/runtime/build-info", response_model=ApiResponse) +def runtime_build_info() -> ApiResponse: + return ApiResponse(data=get_runtime_build_info()) + + @router.get("/runtime/sync-summary", response_model=ApiResponse) def runtime_sync_summary() -> ApiResponse: return ApiResponse(data=get_sync_summary()) diff --git a/domain-api/app/core/config.py b/domain-api/app/core/config.py index 2bc5bd7..a7f5083 100644 --- a/domain-api/app/core/config.py +++ b/domain-api/app/core/config.py @@ -42,6 +42,13 @@ class Settings(BaseSettings): sync_shared_token: str = "" sync_batch_size: int = 200 sync_poll_interval_seconds: int = 30 + build_manifest_path: str = "" + build_commit_sha: str = "" + build_commit_ref: str = "" + build_generated_at: str = "" + build_package_name: str = "" + build_checksum: str = "" + build_source_label: str = "" @property def cors_origins_list(self) -> list[str]: diff --git a/domain-api/app/main.py b/domain-api/app/main.py index 4ab0b6a..e8728ec 100644 --- a/domain-api/app/main.py +++ b/domain-api/app/main.py @@ -3,9 +3,13 @@ import threading from fastapi import FastAPI from fastapi.middleware.cors import CORSMiddleware -from app.api.routes import auth, dashboard, settings as settings_routes, imports, detect, domains, exports, logs, runtime, juming, sensitive_words +from app.api.routes import auth, dashboard, settings as settings_routes, imports, detect, domains, exports, logs, runtime, juming, ops, ops_agent, ops_release, sensitive_words from app.core.config import settings as app_settings +from app.services.build_info_service import get_runtime_build_info, remember_registered_route_paths from app.services.cluster_runtime_service import ensure_runtime_schema, register_local_control_heartbeat +from app.services.ops_agent_service import ensure_ops_agent_schema +from app.services.ops_job_service import ensure_ops_schema +from app.services.ops_release_service import ensure_ops_release_schema _heartbeat_stop_event = threading.Event() @@ -36,6 +40,10 @@ app.add_middleware( @app.on_event("startup") def on_startup() -> None: ensure_runtime_schema() + ensure_ops_schema() + ensure_ops_agent_schema() + ensure_ops_release_schema() + remember_registered_route_paths(route.path for route in app.routes) register_local_control_heartbeat() _heartbeat_stop_event.clear() threading.Thread(target=_control_heartbeat_loop, name="control-heartbeat", daemon=True).start() @@ -56,6 +64,7 @@ def health() -> dict: "worker_mode": app_settings.worker_mode, "api_host": app_settings.api_host, "api_port": app_settings.api_port, + "build": get_runtime_build_info(route_paths=(route.path for route in app.routes)), } @@ -69,4 +78,8 @@ app.include_router(exports.router, prefix=app_settings.api_prefix) app.include_router(logs.router, prefix=app_settings.api_prefix) app.include_router(runtime.router, prefix=app_settings.api_prefix) app.include_router(juming.router, prefix=app_settings.api_prefix) +app.include_router(ops.router, prefix=app_settings.api_prefix) +app.include_router(ops_agent.router, prefix=app_settings.api_prefix) +app.include_router(ops_release.router, prefix=app_settings.api_prefix) app.include_router(sensitive_words.router, prefix=app_settings.api_prefix) +remember_registered_route_paths(route.path for route in app.routes) diff --git a/domain-api/app/node_agent.py b/domain-api/app/node_agent.py new file mode 100644 index 0000000..c707d45 --- /dev/null +++ b/domain-api/app/node_agent.py @@ -0,0 +1,991 @@ +from __future__ import annotations + +import json +import os +import socket +import subprocess +import time +import urllib.error +import urllib.request +from datetime import datetime +from uuid import uuid4 + +from app.services.ops_action_executor_core import ( + build_service_name_map, + execute_structured_action, + supports_structured_action, +) +from app.services.ops_release_executor_core import ( + execute_release_action, + normalize_release_health_check_services as _release_normalize_health_check_services, + normalize_text_list as _release_normalize_text_list, +) + + +CONTROL_PLANE_BASE_URL = str(os.getenv("OPS_CONTROL_PLANE_BASE_URL", "")).strip().rstrip("/") +AGENT_TOKEN = str(os.getenv("OPS_AGENT_TOKEN", "")).strip() +NODE_CODE = str(os.getenv("NODE_CODE", "")).strip() +NODE_REGION = str(os.getenv("NODE_REGION", "mainland")).strip() or "mainland" +NODE_ROLE = str(os.getenv("NODE_ROLE", "worker")).strip() or "worker" +AGENT_POLL_INTERVAL_SECONDS = max(2, int(os.getenv("OPS_AGENT_POLL_INTERVAL_SECONDS", "5") or 5)) +WORKER_SERVICE_NAME = str(os.getenv("WORKER_SERVICE_NAME", os.getenv("WORKER_SERVICE", "domaincheck-worker"))).strip() or "domaincheck-worker" +API_SERVICE_NAME = str(os.getenv("API_SERVICE_NAME", "domaincheck-api")).strip() or "domaincheck-api" +SYNC_AGENT_SERVICE_NAME = str(os.getenv("SYNC_AGENT_SERVICE_NAME", "domaincheck-sync-agent")).strip() or "domaincheck-sync-agent" +NODE_AGENT_SERVICE_NAME = str(os.getenv("NODE_AGENT_SERVICE_NAME", "domaincheck-node-agent")).strip() or "domaincheck-node-agent" +AGENT_VERSION = "0.1.0" +_APP_DIR = os.path.dirname(os.path.abspath(__file__)) +_PROJECT_DIR = os.path.dirname(_APP_DIR) +_DEFAULT_QUEUE_DIR = os.path.join(_PROJECT_DIR, "runtime", "node-agent-queue", NODE_CODE or "unbound") +AGENT_QUEUE_DIR = str(os.getenv("OPS_AGENT_QUEUE_DIR", _DEFAULT_QUEUE_DIR)).strip() or _DEFAULT_QUEUE_DIR +AGENT_QUEUE_FLUSH_LIMIT = max(1, int(os.getenv("OPS_AGENT_QUEUE_FLUSH_LIMIT", "20") or 20)) +_PERMANENT_DELIVERY_DETAIL_CODES = { + "agent_node_code_required", + "ops_job_not_owned_by_agent", + "ops_job_invalid_status", +} +_LAST_QUEUE_FLUSH_SUMMARY = { + "scanned": 0, + "delivered": 0, + "deferred": 0, + "dead_letter": 0, + "last_flush_at": "", +} + + +def _normalize_text_list(raw_value: object) -> list[str]: + """Keep node-agent payload normalization aligned with release executor helpers.""" + return _release_normalize_text_list(raw_value) + + +def _normalize_release_health_check_services( + payload: dict | None, + restart_services: list[str] | None, +) -> list[str]: + """Backward-compatible wrapper used by node-agent tests and payload shaping.""" + return _release_normalize_health_check_services( + dict(payload or {}), + list(restart_services or []), + default_api_service_name=API_SERVICE_NAME, + ) + + +def _log(message: str) -> None: + print(f"{datetime.now().isoformat(sep=' ', timespec='seconds')} [node-agent] {message}", flush=True) + + +def _json_env(name: str, fallback: object) -> object: + raw = str(os.getenv(name, "")).strip() + if not raw: + return fallback + try: + return json.loads(raw) + except Exception: + return fallback + + +AGENT_CAPABILITIES = _json_env( + "OPS_AGENT_CAPABILITIES", + [ + "service.start", + "service.stop", + "service.restart", + "service.status", + "runtime.start_worker", + "runtime.stop_worker", + "runtime.restart_api", + "runtime.start_sync_agent", + "runtime.stop_sync_agent", + "health.snapshot", + "logs.collect", + "diagnostics.collect", + "delivery.queue.flush", + "delivery.queue.replay", + "delivery.queue.discard", + "deploy.release", + ], +) +AGENT_LABELS = _json_env("OPS_AGENT_LABELS", {}) + + +class AgentResponseError(RuntimeError): + def __init__(self, message: str, *, detail_code: str = "", payload: dict | None = None): + super().__init__(message) + self.detail_code = str(detail_code or "").strip() + self.payload = dict(payload or {}) + + +def _now_text() -> str: + return datetime.now().isoformat(sep=" ", timespec="seconds") + + +def _queue_pending_dir() -> str: + return os.path.join(AGENT_QUEUE_DIR, "pending") + + +def _queue_dead_letter_dir() -> str: + return os.path.join(AGENT_QUEUE_DIR, "dead-letter") + + +def _queue_discarded_dir() -> str: + return os.path.join(AGENT_QUEUE_DIR, "discarded") + + +def _ensure_queue_dirs() -> None: + for path in (AGENT_QUEUE_DIR, _queue_pending_dir(), _queue_dead_letter_dir(), _queue_discarded_dir()): + os.makedirs(path, exist_ok=True) + + +def _write_json_file(path: str, payload: dict) -> None: + temp_path = f"{path}.tmp" + with open(temp_path, "w", encoding="utf-8") as handle: + json.dump(payload, handle, ensure_ascii=False, indent=2) + os.replace(temp_path, path) + + +def _read_json_file(path: str) -> dict: + with open(path, "r", encoding="utf-8") as handle: + return json.load(handle) + + +def _new_request_id(prefix: str, job_id: int) -> str: + normalized_prefix = str(prefix or "request").strip() or "request" + return f"{normalized_prefix}-{int(job_id or 0)}-{int(time.time() * 1000)}-{uuid4().hex[:10]}" + + +def _queue_file_names(directory: str) -> list[str]: + _ensure_queue_dirs() + try: + return sorted(file_name for file_name in os.listdir(directory) if file_name.endswith(".json")) + except FileNotFoundError: + return [] + + +def _queue_head_record(directory: str) -> dict: + file_names = _queue_file_names(directory) + if not file_names: + return {} + file_path = os.path.join(directory, file_names[0]) + try: + return _read_json_file(file_path) + except Exception: + return {} + + +def _queue_record_id(record: dict, *, file_name: str = "") -> str: + request_id = str((record or {}).get("request_id") or "").strip() + if request_id: + return request_id + normalized_file_name = str(file_name or "").strip() + if normalized_file_name.endswith(".json"): + return normalized_file_name[:-5] + return normalized_file_name + + +def _queue_entries(state: str) -> list[dict]: + normalized_state = str(state or "").strip() + if normalized_state == "pending": + directory = _queue_pending_dir() + elif normalized_state == "dead_letter": + directory = _queue_dead_letter_dir() + else: + return [] + + entries: list[dict] = [] + for file_name in _queue_file_names(directory): + file_path = os.path.join(directory, file_name) + try: + record = _read_json_file(file_path) + except Exception: + continue + entries.append( + { + "state": normalized_state, + "file_name": file_name, + "file_path": file_path, + "record_id": _queue_record_id(record, file_name=file_name), + "record": dict(record or {}), + } + ) + return entries + + +def _matches_delivery_selector(entry: dict, selector: dict, *, allowed_states: set[str] | None = None) -> bool: + normalized_selector = dict(selector or {}) + record = dict(entry.get("record") or {}) + state = str(entry.get("state") or "").strip() + if allowed_states and state not in allowed_states: + return False + + selected_state = str(normalized_selector.get("state") or "").strip() + if selected_state and state != selected_state: + return False + + selected_record_id = str( + normalized_selector.get("record_id") + or normalized_selector.get("request_id") + or "" + ).strip() + if selected_record_id and str(entry.get("record_id") or "").strip() != selected_record_id: + return False + + selected_request_kind = str(normalized_selector.get("request_kind") or "").strip() + if selected_request_kind and str(record.get("kind") or "").strip() != selected_request_kind: + return False + + selected_detail_code = str(normalized_selector.get("detail_code") or "").strip() + if selected_detail_code and str(record.get("last_detail_code") or "").strip() != selected_detail_code: + return False + + return True + + +def _trim_queue_preview(records: list[dict], *, limit: int = 5) -> list[dict]: + safe_limit = max(1, int(limit or 5)) + preview: list[dict] = [] + for entry in list(records or [])[:safe_limit]: + record = dict(entry.get("record") or {}) + preview.append( + { + "record_id": str(entry.get("record_id") or "").strip(), + "state": str(entry.get("state") or "").strip(), + "request_kind": str(record.get("kind") or "").strip(), + "request_id": str(record.get("request_id") or "").strip(), + "detail_code": str(record.get("last_detail_code") or "").strip(), + "created_at": str(record.get("created_at") or "").strip(), + "updated_at": str(record.get("updated_at") or "").strip(), + } + ) + return preview + + +def _normalize_queue_limit(raw_value: object, *, default: int = 20, minimum: int = 1, maximum: int = 200) -> int: + try: + value = int(raw_value or default) + except Exception: + value = int(default) + return max(minimum, min(maximum, value)) + + +def _replay_dead_letter_records(payload: dict | None = None) -> tuple[bool, str, dict]: + normalized_payload = dict(payload or {}) + selector = dict(normalized_payload.get("selector") or {}) + selector["state"] = str(selector.get("state") or "dead_letter").strip() or "dead_letter" + limit = _normalize_queue_limit(normalized_payload.get("limit"), default=20) + flush_after_replay = bool(normalized_payload.get("flush_after_replay", True)) + reason = str(normalized_payload.get("reason") or "").strip() + + matched_entries = [ + entry + for entry in _queue_entries("dead_letter") + if _matches_delivery_selector(entry, selector, allowed_states={"dead_letter"}) + ][:limit] + if not matched_entries: + return False, "未找到符合条件的死信记录", { + "selector": selector, + "limit": limit, + "queue": _delivery_queue_snapshot(), + } + + replayed_total = 0 + for entry in matched_entries: + record = dict(entry.get("record") or {}) + record["updated_at"] = _now_text() + record["replay_count"] = int(record.get("replay_count") or 0) + 1 + record["last_replay_at"] = _now_text() + if reason: + record["last_replay_reason"] = reason + record.pop("dead_letter_at", None) + record.pop("dead_letter_reason", None) + _store_pending_delivery(record) + try: + os.remove(str(entry.get("file_path") or "")) + except FileNotFoundError: + pass + replayed_total += 1 + + flush_summary = {} + if flush_after_replay and replayed_total > 0: + flush_summary = _flush_delivery_queue(limit=replayed_total) + + result = { + "selector": selector, + "limit": limit, + "replayed_total": replayed_total, + "flush_after_replay": flush_after_replay, + "flush_summary": flush_summary, + "matched_records": _trim_queue_preview(matched_entries), + "queue": _delivery_queue_snapshot(), + } + return True, f"已重放 {replayed_total} 条死信记录", result + + +def _store_discarded_delivery(record: dict, *, discarded_by: str = "", reason: str = "") -> str: + _ensure_queue_dirs() + discarded_record = dict(record or {}) + discarded_record["discarded_at"] = _now_text() + discarded_record["discarded_by"] = str(discarded_by or "").strip() + discarded_record["discard_reason"] = str(reason or "").strip() + file_name = f"{int(time.time() * 1000)}-{uuid4().hex}.json" + file_path = os.path.join(_queue_discarded_dir(), file_name) + _write_json_file(file_path, discarded_record) + return file_path + + +def _discard_dead_letter_records(payload: dict | None = None) -> tuple[bool, str, dict]: + normalized_payload = dict(payload or {}) + selector = dict(normalized_payload.get("selector") or {}) + selector["state"] = str(selector.get("state") or "dead_letter").strip() or "dead_letter" + limit = _normalize_queue_limit(normalized_payload.get("limit"), default=20) + discarded_by = str(normalized_payload.get("discarded_by") or "").strip() + reason = str(normalized_payload.get("reason") or "").strip() + + matched_entries = [ + entry + for entry in _queue_entries("dead_letter") + if _matches_delivery_selector(entry, selector, allowed_states={"dead_letter"}) + ][:limit] + if not matched_entries: + return False, "未找到符合条件的死信记录", { + "selector": selector, + "limit": limit, + "queue": _delivery_queue_snapshot(), + } + + discarded_total = 0 + for entry in matched_entries: + _store_discarded_delivery( + dict(entry.get("record") or {}), + discarded_by=discarded_by, + reason=reason, + ) + try: + os.remove(str(entry.get("file_path") or "")) + except FileNotFoundError: + pass + discarded_total += 1 + + result = { + "selector": selector, + "limit": limit, + "discarded_total": discarded_total, + "discarded_by": discarded_by, + "reason": reason, + "matched_records": _trim_queue_preview(matched_entries), + "queue": _delivery_queue_snapshot(), + } + return True, f"已丢弃 {discarded_total} 条死信记录", result + + +def _delivery_queue_snapshot() -> dict: + pending_dir = _queue_pending_dir() + dead_letter_dir = _queue_dead_letter_dir() + pending_files = _queue_file_names(pending_dir) + dead_letter_files = _queue_file_names(dead_letter_dir) + pending_head = _queue_head_record(pending_dir) + dead_letter_head = _queue_head_record(dead_letter_dir) + pending_count = len(pending_files) + dead_letter_count = len(dead_letter_files) + + state = "healthy" + label = "正常" + reason = "当前没有待重试回执，也没有死信记录。" + if dead_letter_count > 0: + state = "dead_letter" + label = f"死信 {dead_letter_count}" + reason = "存在语义失败的回执/事件，自动重试已停止，建议人工查看。" + elif pending_count > 0: + state = "retrying" + label = f"待重试 {pending_count}" + reason = "存在待重试的回执/事件，Node Agent 会在后续 heartbeat/poll 周期继续回放。" + elif not pending_head and not dead_letter_head: + label = "正常" + + return { + "state": state, + "label": label, + "reason": reason, + "pending_count": pending_count, + "dead_letter_count": dead_letter_count, + "oldest_pending_at": str(pending_head.get("created_at") or "").strip(), + "oldest_pending_request_id": str(pending_head.get("request_id") or "").strip(), + "oldest_pending_kind": str(pending_head.get("kind") or "").strip(), + "oldest_dead_letter_at": str( + dead_letter_head.get("dead_letter_at") or dead_letter_head.get("created_at") or "" + ).strip(), + "oldest_dead_letter_request_id": str(dead_letter_head.get("request_id") or "").strip(), + "oldest_dead_letter_kind": str(dead_letter_head.get("kind") or "").strip(), + "last_flush_at": str(_LAST_QUEUE_FLUSH_SUMMARY.get("last_flush_at") or "").strip(), + "last_flush_delivered": int(_LAST_QUEUE_FLUSH_SUMMARY.get("delivered") or 0), + "last_flush_deferred": int(_LAST_QUEUE_FLUSH_SUMMARY.get("deferred") or 0), + "last_flush_dead_letter": int(_LAST_QUEUE_FLUSH_SUMMARY.get("dead_letter") or 0), + } + + +def _response_detail_code(response: dict) -> str: + if not isinstance(response, dict): + return "" + top_level = str(response.get("detail_code") or "").strip() + if top_level: + return top_level + data = response.get("data") + if isinstance(data, dict): + return str(data.get("detail_code") or "").strip() + return "" + + +def _ensure_ok_response(response: dict, fallback_message: str) -> dict: + raw_code = response.get("code", 1) + try: + normalized_code = int(raw_code if raw_code not in (None, "") else 1) + except Exception: + normalized_code = 1 + if normalized_code == 0: + return response + raise AgentResponseError( + str(response.get("message") or fallback_message), + detail_code=_response_detail_code(response), + payload=(response.get("data") if isinstance(response.get("data"), dict) else {}), + ) + + +def _build_delivery_record(kind: str, path: str, payload: dict, request_id: str) -> dict: + return { + "kind": str(kind or "").strip() or "delivery", + "path": str(path or "").strip(), + "payload": dict(payload or {}), + "request_id": str(request_id or "").strip(), + "attempt_count": 0, + "created_at": _now_text(), + "updated_at": _now_text(), + "last_error": "", + "last_detail_code": "", + } + + +def _store_pending_delivery(record: dict) -> str: + _ensure_queue_dirs() + file_name = f"{int(time.time() * 1000)}-{uuid4().hex}.json" + file_path = os.path.join(_queue_pending_dir(), file_name) + _write_json_file(file_path, record) + return file_path + + +def _store_dead_letter_delivery(record: dict, *, reason: str, detail_code: str = "") -> str: + _ensure_queue_dirs() + dead_record = dict(record or {}) + dead_record["dead_letter_reason"] = str(reason or "").strip() + dead_record["dead_letter_at"] = _now_text() + dead_record["last_error"] = str(reason or "").strip() + dead_record["last_detail_code"] = str(detail_code or "").strip() + file_name = f"{int(time.time() * 1000)}-{uuid4().hex}.json" + file_path = os.path.join(_queue_dead_letter_dir(), file_name) + _write_json_file(file_path, dead_record) + return file_path + + +def _dispatch_delivery_record(record: dict) -> dict: + kind = str(record.get("kind") or "delivery").strip() or "delivery" + path = str(record.get("path") or "").strip() + payload = record.get("payload") or {} + timeout = 15 if kind == "job_event" else 30 + response = _post(path, payload, timeout=timeout) + return _ensure_ok_response(response, f"{kind} failed") + + +def _flush_delivery_queue(limit: int | None = None) -> dict: + global _LAST_QUEUE_FLUSH_SUMMARY + _ensure_queue_dirs() + safe_limit = max(1, int(limit or AGENT_QUEUE_FLUSH_LIMIT or 1)) + summary = { + "scanned": 0, + "delivered": 0, + "deferred": 0, + "dead_letter": 0, + } + file_names = sorted( + file_name + for file_name in os.listdir(_queue_pending_dir()) + if file_name.endswith(".json") + )[:safe_limit] + for file_name in file_names: + summary["scanned"] += 1 + file_path = os.path.join(_queue_pending_dir(), file_name) + try: + record = _read_json_file(file_path) + except Exception as exc: + _store_dead_letter_delivery({"file_name": file_name}, reason=f"invalid queue record: {exc}") + try: + os.remove(file_path) + except FileNotFoundError: + pass + summary["dead_letter"] += 1 + continue + + try: + _dispatch_delivery_record(record) + try: + os.remove(file_path) + except FileNotFoundError: + pass + summary["delivered"] += 1 + except AgentResponseError as exc: + record["attempt_count"] = int(record.get("attempt_count") or 0) + 1 + record["updated_at"] = _now_text() + record["last_error"] = str(exc) + record["last_detail_code"] = exc.detail_code + if exc.detail_code in _PERMANENT_DELIVERY_DETAIL_CODES: + _store_dead_letter_delivery(record, reason=str(exc), detail_code=exc.detail_code) + try: + os.remove(file_path) + except FileNotFoundError: + pass + summary["dead_letter"] += 1 + _log( + f"delivery moved to dead-letter: kind={record.get('kind')} request_id={record.get('request_id')} " + f"detail_code={exc.detail_code or '-'} message={exc}" + ) + else: + _write_json_file(file_path, record) + summary["deferred"] += 1 + except Exception as exc: + record["attempt_count"] = int(record.get("attempt_count") or 0) + 1 + record["updated_at"] = _now_text() + record["last_error"] = str(exc) + _write_json_file(file_path, record) + summary["deferred"] += 1 + summary["last_flush_at"] = _now_text() + _LAST_QUEUE_FLUSH_SUMMARY = dict(summary) + return summary + + +def _deliver_or_queue( + *, + kind: str, + path: str, + payload: dict, + request_id: str, + timeout: int = 30, +) -> dict: + record = _build_delivery_record(kind, path, payload, request_id) + try: + response = _post(path, payload, timeout=timeout) + _ensure_ok_response(response, f"{kind} failed") + return {"state": "delivered", "request_id": request_id, "detail_code": "", "response": response} + except AgentResponseError as exc: + if exc.detail_code in _PERMANENT_DELIVERY_DETAIL_CODES: + file_path = _store_dead_letter_delivery(record, reason=str(exc), detail_code=exc.detail_code) + _log( + f"{kind} dead-lettered: request_id={request_id} detail_code={exc.detail_code or '-'} " + f"message={exc} file={file_path}" + ) + return {"state": "dead_letter", "request_id": request_id, "detail_code": exc.detail_code, "file_path": file_path} + file_path = _store_pending_delivery({**record, "last_error": str(exc), "last_detail_code": exc.detail_code}) + _log( + f"{kind} queued for retry: request_id={request_id} detail_code={exc.detail_code or '-'} " + f"message={exc} file={file_path}" + ) + return {"state": "queued", "request_id": request_id, "detail_code": exc.detail_code, "file_path": file_path} + except Exception as exc: + file_path = _store_pending_delivery({**record, "last_error": str(exc)}) + _log(f"{kind} queued for retry: request_id={request_id} message={exc} file={file_path}") + return {"state": "queued", "request_id": request_id, "detail_code": "", "file_path": file_path} + + +def _headers() -> dict[str, str]: + return { + "Content-Type": "application/json", + "X-Domaincheck-Agent-Token": AGENT_TOKEN, + } + + +def _request(method: str, path: str, payload: dict | None = None, timeout: int = 30) -> dict: + if not CONTROL_PLANE_BASE_URL: + raise RuntimeError("OPS_CONTROL_PLANE_BASE_URL 未配置") + if not AGENT_TOKEN: + raise RuntimeError("OPS_AGENT_TOKEN 未配置") + url = f"{CONTROL_PLANE_BASE_URL}{path}" + data = json.dumps(payload or {}, ensure_ascii=False).encode("utf-8") + request = urllib.request.Request(url=url, data=data, headers=_headers(), method=method.upper()) + with urllib.request.urlopen(request, timeout=timeout) as response: + body = response.read().decode("utf-8", errors="ignore") + return json.loads(body or "{}") + + +def _post(path: str, payload: dict, timeout: int = 30) -> dict: + return _request("POST", path, payload, timeout=timeout) + + +def _hostname() -> str: + try: + return socket.gethostname() + except Exception: + return "" + + +def _ip() -> str: + try: + return socket.gethostbyname(socket.gethostname()) + except Exception: + return "" + + +def _base_payload() -> dict: + return { + "node_code": NODE_CODE, + "region": NODE_REGION, + "role": NODE_ROLE, + "title": NODE_CODE, + "hostname": _hostname(), + "ip": _ip(), + "agent_version": AGENT_VERSION, + "capabilities": AGENT_CAPABILITIES, + "labels": AGENT_LABELS, + "metadata": { + "service_names": { + "api": API_SERVICE_NAME, + "worker": WORKER_SERVICE_NAME, + "sync_agent": SYNC_AGENT_SERVICE_NAME, + "node_agent": NODE_AGENT_SERVICE_NAME, + }, + "delivery_queue": _delivery_queue_snapshot(), + }, + } + + +def _run(command: list[str], timeout: int = 60) -> tuple[int, str, str]: + completed = subprocess.run(command, capture_output=True, text=True, timeout=timeout) + return completed.returncode, completed.stdout.strip(), completed.stderr.strip() + + +def _execute_action( + action: str, + payload: dict, + *, + job_id: int | None = None, + job_context: dict | None = None, +) -> tuple[bool, str, dict]: + normalized_action = str(action or "").strip() + if normalized_action == "delivery.queue.flush": + limit = _normalize_queue_limit((payload or {}).get("limit"), default=20) + summary = _flush_delivery_queue(limit=limit) + return True, "Delivery Queue 已执行冲刷", { + "action": normalized_action, + "limit": limit, + "flush_summary": summary, + "queue": _delivery_queue_snapshot(), + } + if normalized_action == "delivery.queue.replay": + return _replay_dead_letter_records(payload) + if normalized_action == "delivery.queue.discard": + return _discard_dead_letter_records(payload) + if supports_structured_action(normalized_action): + return execute_structured_action( + normalized_action, + payload, + service_names=build_service_name_map( + api_service_name=API_SERVICE_NAME, + worker_service_name=WORKER_SERVICE_NAME, + sync_agent_service_name=SYNC_AGENT_SERVICE_NAME, + node_agent_service_name=NODE_AGENT_SERVICE_NAME, + ), + runner=_run, + host_context={ + "hostname": _hostname(), + "ip": _ip(), + }, + ) + if normalized_action == "deploy.release": + return execute_release_action( + dict(payload or {}), + run_command=_run, + default_api_service_name=API_SERVICE_NAME, + event_callback=( + (lambda event_type, message, level="info", payload=None: _job_event( + int(job_id), + event_type=event_type, + message=message, + level=level, + payload={ + **dict(payload or {}), + "release_context": dict((job_context or {}).get("release_context") or {}), + "step_key": str((job_context or {}).get("step_key") or "").strip(), + }, + summary_text=message, + focus_ref=dict((job_context or {}).get("focus_ref") or {}), + occurred_at=_now_text(), + )) + if job_id + else None + ), + urlopen_func=urllib.request.urlopen, + user_agent=f"domaincheck-node-agent/{AGENT_VERSION}", + ) + return False, f"unsupported action: {normalized_action}", {"action": normalized_action} + + +def _register() -> None: + response = _post("/api/v1/ops/agent/register", _base_payload()) + _ensure_ok_response(response, "agent register failed") + _log(f"registered: {response.get('message')}") + + +def _heartbeat() -> None: + response = _post("/api/v1/ops/agent/heartbeat", _base_payload()) + _ensure_ok_response(response, "agent heartbeat failed") + + +def _pull_jobs() -> list[dict]: + response = _post(f"/api/v1/ops/agent/pull?limit=1", {"node_code": NODE_CODE}) + _ensure_ok_response(response, "agent pull failed") + data = response.get("data") or {} + return list(data.get("jobs") or []) + + +def _normalize_agent_job(job: dict) -> dict: + normalized_job = dict(job or {}) + focus_ref = normalized_job.get("focus_ref") if isinstance(normalized_job.get("focus_ref"), dict) else {} + release_context = ( + normalized_job.get("release_context") if isinstance(normalized_job.get("release_context"), dict) else {} + ) + step_ref = normalized_job.get("step_ref") if isinstance(normalized_job.get("step_ref"), dict) else {} + job_id = int(normalized_job.get("job_id") or normalized_job.get("id") or 0) + step_key = str( + normalized_job.get("step_key") + or step_ref.get("step_key") + or "dispatch" + ).strip() or "dispatch" + step_title = str( + normalized_job.get("step_title") + or step_ref.get("step_title") + or normalized_job.get("summary") + or normalized_job.get("action") + or step_key + ).strip() or step_key + if not focus_ref: + focus_ref = { + "kind": "ops_job", + "job_id": job_id, + "job_code": str(normalized_job.get("job_code") or "").strip(), + "action": str(normalized_job.get("action") or "").strip(), + "target_node_code": str(normalized_job.get("target_node_code") or "").strip(), + } + return { + **normalized_job, + "job_id": job_id, + "job_code": str(normalized_job.get("job_code") or "").strip(), + "job_type": str(normalized_job.get("job_type") or "ops_action").strip() or "ops_action", + "action": str(normalized_job.get("action") or "").strip(), + "payload": dict(normalized_job.get("payload") or {}), + "policy": dict(normalized_job.get("policy") or {}), + "focus_ref": focus_ref, + "release_context": dict(release_context or {}), + "step_key": step_key, + "step_title": step_title, + "step_ref": { + "step_id": int(step_ref.get("step_id") or 0), + "step_key": step_key, + "step_title": step_title, + }, + } + + +def _job_start(job_id: int, *, job: dict | None = None) -> None: + normalized_job = _normalize_agent_job(job or {"job_id": job_id}) + response = _post( + f"/api/v1/ops/agent/jobs/{job_id}/start", + { + "node_code": NODE_CODE, + "job_code": normalized_job.get("job_code") or "", + "action": normalized_job.get("action") or "", + "step_key": normalized_job.get("step_key") or "", + "focus_ref": normalized_job.get("focus_ref") or {}, + }, + ) + _ensure_ok_response(response, "job start failed") + + +def _job_complete( + job_id: int, + *, + status: str, + stdout: str, + stderr: str, + result: dict, + error_message: str = "", + client_request_id: str | None = None, + duration_ms: int | None = None, + summary_text: str = "", + focus_ref: dict | None = None, + step_ref: dict | None = None, + release_context: dict | None = None, +) -> dict: + request_id = str(client_request_id or "").strip() or _new_request_id("complete", job_id) + payload = { + "node_code": NODE_CODE, + "status": status, + "stdout": stdout, + "stderr": stderr, + "result": result, + "error_message": error_message, + "client_request_id": request_id, + } + if duration_ms is not None: + payload["duration_ms"] = max(0, int(duration_ms or 0)) + if str(summary_text or "").strip(): + payload["summary_text"] = str(summary_text).strip() + if isinstance(focus_ref, dict) and focus_ref: + payload["focus_ref"] = dict(focus_ref) + if isinstance(step_ref, dict) and step_ref: + payload["step_ref"] = dict(step_ref) + if isinstance(release_context, dict) and release_context: + payload["release_context"] = dict(release_context) + return _deliver_or_queue( + kind="job_complete", + path=f"/api/v1/ops/agent/jobs/{job_id}/complete", + payload=payload, + request_id=request_id, + timeout=30, + ) + + +def _job_event( + job_id: int, + *, + event_type: str, + message: str, + level: str = "info", + payload: dict | None = None, + client_event_id: str | None = None, + summary_text: str = "", + focus_ref: dict | None = None, + occurred_at: str = "", +) -> dict: + request_id = str(client_event_id or "").strip() or _new_request_id("event", job_id) + delivery_payload = { + "node_code": NODE_CODE, + "event_type": event_type, + "message": message, + "level": level, + "payload": payload or {}, + "client_event_id": request_id, + } + if str(summary_text or "").strip(): + delivery_payload["summary_text"] = str(summary_text).strip() + if isinstance(focus_ref, dict) and focus_ref: + delivery_payload["focus_ref"] = dict(focus_ref) + if str(occurred_at or "").strip(): + delivery_payload["occurred_at"] = str(occurred_at).strip() + return _deliver_or_queue( + kind="job_event", + path=f"/api/v1/ops/agent/jobs/{job_id}/events", + payload=delivery_payload, + request_id=request_id, + timeout=15, + ) + + +def _process_job(job: dict) -> None: + normalized_job = _normalize_agent_job(job) + job_id = int(normalized_job.get("job_id") or 0) + action = str(normalized_job.get("action") or "").strip() + payload = dict(normalized_job.get("payload") or {}) + if job_id <= 0 or not action: + return + + started_at = time.monotonic() + _log( + "job start: " + f"id={job_id} code={normalized_job.get('job_code') or '-'} " + f"type={normalized_job.get('job_type') or '-'} " + f"step={normalized_job.get('step_key') or '-'} action={action}" + ) + start_delivery_state = "delivered" + start_delivery_error = "" + try: + _job_start(job_id, job=normalized_job) + except Exception as exc: + start_delivery_state = "failed_local" + start_delivery_error = str(exc) + _log( + "job start delivery failed, continue locally: " + f"id={job_id} code={normalized_job.get('job_code') or '-'} action={action} error={exc}" + ) + _job_event( + job_id, + event_type="executor_received", + message=f"node agent accepted action {action}", + summary_text=f"已接单 {action}", + focus_ref=dict(normalized_job.get("focus_ref") or {}), + occurred_at=_now_text(), + payload={ + "action": action, + "job_code": normalized_job.get("job_code") or "", + "job_type": normalized_job.get("job_type") or "", + "step_key": normalized_job.get("step_key") or "", + "step_title": normalized_job.get("step_title") or "", + "release_context": dict(normalized_job.get("release_context") or {}), + "start_delivery_state": start_delivery_state, + "start_delivery_error": start_delivery_error, + }, + ) + ok, message, result = _execute_action(action, payload, job_id=job_id, job_context=normalized_job) + stdout = str(result.get("stdout") or "") + stderr = str(result.get("stderr") or "") + duration_ms = max(0, int((time.monotonic() - started_at) * 1000)) + summary_text = str(result.get("summary_text") or result.get("summary") or message).strip() + delivery = _job_complete( + job_id, + status="success" if ok else "failed", + stdout=stdout, + stderr=stderr, + result=result, + error_message="" if ok else message, + duration_ms=duration_ms, + summary_text=summary_text, + focus_ref=dict(normalized_job.get("focus_ref") or {}), + step_ref=dict(normalized_job.get("step_ref") or {}), + release_context=dict(normalized_job.get("release_context") or {}), + ) + _log( + "job complete: " + f"id={job_id} code={normalized_job.get('job_code') or '-'} " + f"action={action} ok={ok} duration_ms={duration_ms} delivery={delivery.get('state')}" + ) + + +def main() -> None: + if not NODE_CODE: + raise RuntimeError("NODE_CODE 未配置") + _log(f"starting node agent: node={NODE_CODE} role={NODE_ROLE} region={NODE_REGION}") + _ensure_queue_dirs() + _register() + last_heartbeat_at = 0.0 + + while True: + now = time.time() + try: + delivery_summary = _flush_delivery_queue(limit=AGENT_QUEUE_FLUSH_LIMIT) + if delivery_summary["delivered"] or delivery_summary["dead_letter"]: + _log(f"delivery queue flush: {delivery_summary}") + if now - last_heartbeat_at >= 15: + _heartbeat() + last_heartbeat_at = now + jobs = _pull_jobs() + if jobs: + for job in jobs: + _process_job(job) + else: + time.sleep(AGENT_POLL_INTERVAL_SECONDS) + except urllib.error.HTTPError as exc: + _log(f"http error: {exc.code}") + time.sleep(AGENT_POLL_INTERVAL_SECONDS) + except urllib.error.URLError as exc: + _log(f"url error: {exc}") + time.sleep(AGENT_POLL_INTERVAL_SECONDS) + except Exception as exc: + _log(f"loop error: {exc}") + time.sleep(AGENT_POLL_INTERVAL_SECONDS) + + +if __name__ == "__main__": + main() diff --git a/domain-api/app/schemas/common.py b/domain-api/app/schemas/common.py index ab3f490..3dc0fa2 100644 --- a/domain-api/app/schemas/common.py +++ b/domain-api/app/schemas/common.py @@ -1,8 +1,10 @@ +from typing import Any, Optional + from pydantic import BaseModel -from typing import Any class ApiResponse(BaseModel): code: int = 0 message: str = "ok" data: Any = None + detail_code: Optional[str] = None diff --git a/domain-api/app/services/build_info_service.py b/domain-api/app/services/build_info_service.py new file mode 100644 index 0000000..09e0d74 --- /dev/null +++ b/domain-api/app/services/build_info_service.py @@ -0,0 +1,241 @@ +from __future__ import annotations + +import json +import subprocess +from pathlib import Path +from typing import Iterable + +from app.core.config import settings + + +DOMAIN_API_ROOT = Path(__file__).resolve().parents[2] +WORKSPACE_ROOT = DOMAIN_API_ROOT.parent +_REGISTERED_ROUTE_PATHS: set[str] = set() + + +def remember_registered_route_paths(route_paths: Iterable[str] | None) -> None: + global _REGISTERED_ROUTE_PATHS + normalized = { + str(item).strip() + for item in list(route_paths or []) + if str(item).strip() + } + _REGISTERED_ROUTE_PATHS = normalized + + +def _read_json(path: Path) -> dict: + try: + return json.loads(path.read_text(encoding="utf-8")) + except Exception: + return {} + + +def _discover_manifest_candidate() -> tuple[Path | None, str]: + configured = str(settings.build_manifest_path or "").strip() + if configured: + path = Path(configured) + if path.exists(): + return path, "configured_manifest" + + root_manifest = WORKSPACE_ROOT / "release_manifest.json" + if root_manifest.exists(): + return root_manifest, "release_manifest" + + latest_release = WORKSPACE_ROOT / "release" / "latest_release.json" + if latest_release.exists(): + return latest_release, "latest_release" + + return None, "" + + +def _build_info_from_env() -> dict | None: + if not any( + [ + str(settings.build_commit_sha or "").strip(), + str(settings.build_commit_ref or "").strip(), + str(settings.build_generated_at or "").strip(), + str(settings.build_package_name or "").strip(), + str(settings.build_checksum or "").strip(), + ] + ): + return None + + return { + "source": str(settings.build_source_label or "").strip() or "env", + "package_name": str(settings.build_package_name or "").strip(), + "generated_at": str(settings.build_generated_at or "").strip(), + "commit_sha": str(settings.build_commit_sha or "").strip(), + "commit_ref": str(settings.build_commit_ref or "").strip(), + "checksum": str(settings.build_checksum or "").strip(), + "manifest_path": str(settings.build_manifest_path or "").strip(), + } + + +def _build_info_from_manifest() -> dict | None: + manifest_path, source = _discover_manifest_candidate() + if manifest_path is None: + return None + + payload = _read_json(manifest_path) + if not payload: + return { + "source": source, + "package_name": "", + "generated_at": "", + "commit_sha": "", + "commit_ref": "", + "checksum": "", + "manifest_path": str(manifest_path), + } + + checksum = str(payload.get("checksum") or payload.get("sha256") or "").strip() + return { + "source": source, + "package_name": str(payload.get("package_name") or "").strip(), + "generated_at": str(payload.get("generated_at") or "").strip(), + "commit_sha": str(payload.get("commit_sha") or "").strip(), + "commit_ref": str(payload.get("commit_ref") or "").strip(), + "checksum": checksum, + "manifest_path": str(manifest_path), + } + + +def _build_info_from_git() -> dict | None: + try: + inside_worktree = subprocess.run( + ["git", "-C", str(WORKSPACE_ROOT), "rev-parse", "--is-inside-work-tree"], + check=False, + capture_output=True, + text=True, + timeout=2, + ) + except Exception: + return None + + if inside_worktree.returncode != 0 or str(inside_worktree.stdout or "").strip() != "true": + return None + + def _read_git(*args: str) -> str: + try: + result = subprocess.run( + ["git", "-C", str(WORKSPACE_ROOT), *args], + check=False, + capture_output=True, + text=True, + timeout=2, + ) + except Exception: + return "" + if result.returncode != 0: + return "" + return str(result.stdout or "").strip() + + return { + "source": "git", + "package_name": "", + "generated_at": "", + "commit_sha": _read_git("rev-parse", "--short=12", "HEAD"), + "commit_ref": _read_git("rev-parse", "--abbrev-ref", "HEAD"), + "checksum": "", + "manifest_path": "", + } + + +def _resolve_build_identity() -> dict: + return ( + _build_info_from_env() + or _build_info_from_manifest() + or _build_info_from_git() + or { + "source": "unknown", + "package_name": "", + "generated_at": "", + "commit_sha": "", + "commit_ref": "", + "checksum": "", + "manifest_path": "", + } + ) + + +def _expected_route_paths() -> dict[str, str]: + prefix = str(settings.api_prefix or "/api/v1").rstrip("/") + return { + "ops_contracts": f"{prefix}/ops/contracts", + "ops_contract_detail": f"{prefix}/ops/contracts/{{contract_key}}", + "ops_stack_diagnosis": f"{prefix}/ops/stack-diagnosis", + "ops_node_handover": f"{prefix}/ops/nodes/{{node_code}}/handover", + "ops_node_onboarding": f"{prefix}/ops/nodes/{{node_code}}/onboarding", + "ops_node_onboarding_bootstrap_preview": f"{prefix}/ops/nodes/{{node_code}}/onboarding/bootstrap/preview", + "ops_node_onboarding_bootstrap_execute": f"{prefix}/ops/nodes/{{node_code}}/onboarding/bootstrap/execute", + "ops_node_onboarding_acceptance_preview": f"{prefix}/ops/nodes/{{node_code}}/onboarding/acceptance/preview", + "ops_node_onboarding_acceptance_execute": f"{prefix}/ops/nodes/{{node_code}}/onboarding/acceptance/execute", + "ops_node_onboarding_recovery_preview": f"{prefix}/ops/nodes/{{node_code}}/onboarding/recovery/preview", + "ops_node_onboarding_recovery_execute": f"{prefix}/ops/nodes/{{node_code}}/onboarding/recovery/execute", + "ops_node_scene_log": f"{prefix}/ops/nodes/{{node_code}}/scene-log", + "ops_node_handover_bootstrap_plan": f"{prefix}/ops/nodes/{{node_code}}/handover/bootstrap-plan", + "ops_driver_feed": f"{prefix}/ops/driver-feed", + "ops_codex_brief": f"{prefix}/ops/codex-brief", + "ops_activity_stream": f"{prefix}/ops/activity-stream", + "ops_runbook_resolve": f"{prefix}/ops/runbook/sequences/{{sequence_key}}/resolve", + "ops_runbook_execute": f"{prefix}/ops/runbook/sequences/{{sequence_key}}/execute", + "runtime_build_info": f"{prefix}/runtime/build-info", + } + + +def _read_text(path: Path) -> str: + try: + return path.read_text(encoding="utf-8", errors="replace") + except Exception: + return "" + + +def _repository_capabilities() -> dict: + ops_agent_service_text = _read_text(DOMAIN_API_ROOT / "app" / "services" / "ops_agent_service.py") + bootstrap_node_agent_text = _read_text(DOMAIN_API_ROOT / "deploy" / "multi-region" / "bootstrap_node_agent.sh") + return { + "supports_install_command_block": ( + "install_command_block" in ops_agent_service_text + and "_candidate_node_agent_install_paths" in ops_agent_service_text + ), + "supports_multi_layout_bootstrap": "resolve_project_root" in bootstrap_node_agent_text, + } + + +def _build_route_surface(route_paths: Iterable[str] | None = None) -> dict: + expected = _expected_route_paths() + normalized_paths = { + str(item).strip() + for item in (list(route_paths) if route_paths is not None else list(_REGISTERED_ROUTE_PATHS)) + if str(item).strip() + } + mode = "registered" if normalized_paths else "declared_contract" + flags = {key: (path in normalized_paths if normalized_paths else True) for key, path in expected.items()} + missing_keys = [key for key, available in flags.items() if not available] + return { + "mode": mode, + "registered_paths_total": len(normalized_paths), + "surface_flags": flags, + "expected_paths": expected, + "missing_keys": missing_keys, + "missing_paths": [expected[key] for key in missing_keys], + "surface_complete": len(missing_keys) == 0, + } + + +def get_runtime_build_info(route_paths: Iterable[str] | None = None) -> dict: + build = _resolve_build_identity() + route_surface = _build_route_surface(route_paths=route_paths) + return { + "source": str(build.get("source") or "").strip() or "unknown", + "package_name": str(build.get("package_name") or "").strip(), + "generated_at": str(build.get("generated_at") or "").strip(), + "commit_sha": str(build.get("commit_sha") or "").strip(), + "commit_ref": str(build.get("commit_ref") or "").strip(), + "checksum": str(build.get("checksum") or "").strip(), + "manifest_path": str(build.get("manifest_path") or "").strip(), + "workspace_root": str(WORKSPACE_ROOT), + "domain_api_root": str(DOMAIN_API_ROOT), + "repository_capabilities": _repository_capabilities(), + "route_surface": route_surface, + } diff --git a/domain-api/app/services/detect_service.py b/domain-api/app/services/detect_service.py index 32cccb7..fb58f5e 100644 --- a/domain-api/app/services/detect_service.py +++ b/domain-api/app/services/detect_service.py @@ -147,15 +147,50 @@ def _build_remote_log_lines( mode: str, limit: int = 240, ) -> list[str]: + return _build_remote_log_snapshot(active_job, enabled=enabled, mode=mode, limit=limit)["lines"] + + +def _build_remote_log_snapshot( + active_job: dict | None, + *, + enabled: bool, + mode: str, + limit: int = 240, +) -> dict: if not enabled: - return [] + return { + "lines": [], + "line_count": 0, + "last_at": "", + "last_line": "", + "source_nodes": [], + "source_node_count": 0, + } if not active_job: - return [] + return { + "lines": [], + "line_count": 0, + "last_at": "", + "last_line": "", + "source_nodes": [], + "source_node_count": 0, + } events = list(active_job.get("current_cycle_events") or active_job.get("recent_events") or []) if not events: - return [] + return { + "lines": [], + "line_count": 0, + "last_at": "", + "last_line": "", + "source_nodes": [], + "source_node_count": 0, + } lines: list[str] = [] + source_nodes: set[str] = set() + source_node_summaries: dict[str, dict] = {} + last_at = "" + last_line = "" normalized_mode = str(mode or "key").strip().lower() if normalized_mode not in {"key", "full"}: normalized_mode = "key" @@ -180,8 +215,47 @@ def _build_remote_log_lines( continue if len(message) > _REMOTE_LOG_MAX_CHARS: message = f"{message[:_REMOTE_LOG_MAX_CHARS]}..." - lines.append(f"[{created_at}] [{node_code}] {message}") - return lines[-max(1, int(limit or 240)) :] + formatted_line = f"[{created_at}] [{node_code}] {message}" + lines.append(formatted_line) + source_nodes.add(node_code) + node_summary = source_node_summaries.setdefault( + node_code, + { + "node_code": node_code, + "line_count": 0, + "key_line_count": 0, + "full_line_count": 0, + "last_at": "", + "last_line": "", + }, + ) + node_summary["line_count"] += 1 + if event_mode == "full": + node_summary["full_line_count"] += 1 + else: + node_summary["key_line_count"] += 1 + node_summary["last_at"] = created_at + node_summary["last_line"] = formatted_line + last_at = created_at + last_line = formatted_line + sliced_lines = lines[-max(1, int(limit or 240)) :] + sorted_source_node_summaries = sorted( + source_node_summaries.values(), + key=lambda item: ( + str(item.get("last_at") or ""), + str(item.get("node_code") or ""), + ), + reverse=True, + ) + return { + "lines": sliced_lines, + "line_count": len(sliced_lines), + "last_at": last_at, + "last_line": last_line, + "source_nodes": sorted(source_nodes), + "source_node_count": len(source_nodes), + "source_node_summaries": sorted_source_node_summaries, + } def _resolve_remote_log_lines( @@ -195,6 +269,17 @@ def _resolve_remote_log_lines( return _build_remote_log_lines(active_job, enabled=enabled, mode=mode, limit=limit) +def _resolve_remote_log_snapshot( + active_job: dict | None, + runs: list[dict], + *, + enabled: bool, + mode: str, + limit: int = 240, +) -> dict: + return _build_remote_log_snapshot(active_job, enabled=enabled, mode=mode, limit=limit) + + def _load_runtime_state() -> dict: try: redis_client = get_redis() @@ -404,13 +489,14 @@ def get_detect_status() -> dict: runs = sync_detect_runs(runtime_snapshot, progress, settings_summary, active_job=active_job) worker_log_sync_enabled = bool(runtime_settings.get("worker_log_sync_enabled", False)) worker_log_sync_mode = str(runtime_settings.get("worker_log_sync_mode", "key") or "key") - remote_log_lines = _resolve_remote_log_lines( + remote_log_snapshot = _resolve_remote_log_snapshot( active_job, runs, enabled=worker_log_sync_enabled, mode=worker_log_sync_mode, limit=240, ) + remote_log_lines = list(remote_log_snapshot.get("lines") or []) dependency_alerts = _extract_dependency_alerts(recent_lines) append_detect_result_projection_if_changed( detect={ @@ -465,6 +551,12 @@ def get_detect_status() -> dict: "recent_warning": recent_proxy_warning, "log_lines": recent_lines, "remote_log_lines": remote_log_lines, + "remote_log_line_count": int(remote_log_snapshot.get("line_count", 0) or 0), + "remote_log_last_at": str(remote_log_snapshot.get("last_at") or ""), + "remote_log_last_line": str(remote_log_snapshot.get("last_line") or ""), + "remote_log_nodes": list(remote_log_snapshot.get("source_nodes") or []), + "remote_log_node_count": int(remote_log_snapshot.get("source_node_count", 0) or 0), + "remote_log_node_summaries": list(remote_log_snapshot.get("source_node_summaries") or []), "runs": runs, "active_job": active_job, "worker_log_sync_enabled": worker_log_sync_enabled, diff --git a/domain-api/app/services/ops_action_executor_core.py b/domain-api/app/services/ops_action_executor_core.py new file mode 100644 index 0000000..781e903 --- /dev/null +++ b/domain-api/app/services/ops_action_executor_core.py @@ -0,0 +1,278 @@ +from __future__ import annotations + +from pathlib import Path +from typing import Protocol + +from app.core.config import settings + + +STRUCTURED_ACTIONS = { + "health.snapshot", + "service.status", + "service.restart", + "service.start", + "service.stop", + "logs.collect", + "diagnostics.collect", + "runtime.start_worker", + "runtime.stop_worker", + "runtime.restart_api", + "runtime.start_sync_agent", + "runtime.stop_sync_agent", +} + + +class CommandRunner(Protocol): + def __call__(self, command: list[str], *, timeout: int = 60) -> tuple[int, str, str]: + ... + + +def supports_structured_action(action: str) -> bool: + return str(action or "").strip() in STRUCTURED_ACTIONS + + +def trim_output(text: str, limit: int) -> str: + normalized = str(text or "").strip() + if len(normalized) <= int(limit): + return normalized + return normalized[-int(limit):] + + +def _read_tail_lines(path: Path, *, max_lines: int) -> list[str]: + if not path.exists() or not path.is_file(): + return [] + with path.open("r", encoding="utf-8", errors="replace") as handle: + return handle.read().splitlines()[-max_lines:] + + +def _runtime_logs_dir() -> Path: + return Path(__file__).resolve().parents[2] / "runtime" / "logs" + + +def _service_log_file_candidates(service_name: str, *, service_names: dict[str, str]) -> list[Path]: + normalized_service_name = str(service_name or "").strip() + if not normalized_service_name: + return [] + + domain_root = Path(settings.domain_root) + runtime_logs_dir = _runtime_logs_dir() + candidates: list[Path] = [] + + def append_candidate(path: Path) -> None: + if path not in candidates: + candidates.append(path) + + if normalized_service_name == (service_names.get("worker") or "domaincheck-worker"): + append_candidate(domain_root / "detect_worker.log") + append_candidate(domain_root / "logs" / "detect_worker.log") + elif normalized_service_name == (service_names.get("api") or "domaincheck-api"): + append_candidate(runtime_logs_dir / "domain-api.stderr.log") + append_candidate(runtime_logs_dir / "domain-api.stdout.log") + append_candidate(domain_root / "logs" / "app.log") + + return candidates + + +def _collect_log_file_fallback( + service_name: str, + *, + lines: int, + service_names: dict[str, str], +) -> tuple[str, list[str]]: + aggregated_lines: list[str] = [] + used_paths: list[str] = [] + for path in _service_log_file_candidates(service_name, service_names=service_names): + current_lines = _read_tail_lines(path, max_lines=lines) + if not current_lines: + continue + aggregated_lines.extend(current_lines) + used_paths.append(str(path)) + if not aggregated_lines: + return "", [] + return "\n".join(aggregated_lines[-lines:]), used_paths + + +def build_service_name_map( + *, + api_service_name: str, + worker_service_name: str, + sync_agent_service_name: str, + node_agent_service_name: str = "domaincheck-node-agent", +) -> dict[str, str]: + return { + "api": str(api_service_name or "").strip() or "domaincheck-api", + "worker": str(worker_service_name or "").strip() or "domaincheck-worker", + "sync_agent": str(sync_agent_service_name or "").strip() or "domaincheck-sync-agent", + "node_agent": str(node_agent_service_name or "").strip() or "domaincheck-node-agent", + } + + +def service_name_from_payload(payload: dict, *, default_worker_service_name: str) -> str: + service_name = str((payload or {}).get("service_name") or "").strip() + return service_name or str(default_worker_service_name or "").strip() or "domaincheck-worker" + + +def service_name_for_action(action: str, payload: dict, service_names: dict[str, str]) -> str: + normalized_action = str(action or "").strip() + if normalized_action in {"service.status", "service.restart", "service.start", "service.stop"}: + return service_name_from_payload(payload, default_worker_service_name=service_names.get("worker") or "") + runtime_action_map = { + "runtime.start_worker": service_names.get("worker") or "domaincheck-worker", + "runtime.stop_worker": service_names.get("worker") or "domaincheck-worker", + "runtime.restart_api": service_names.get("api") or "domaincheck-api", + "runtime.start_sync_agent": service_names.get("sync_agent") or "domaincheck-sync-agent", + "runtime.stop_sync_agent": service_names.get("sync_agent") or "domaincheck-sync-agent", + } + return str(runtime_action_map.get(normalized_action) or "").strip() + + +def systemctl_action_name(action: str) -> str: + normalized_action = str(action or "").strip() + if normalized_action in {"service.restart", "runtime.restart_api"}: + return "restart" + if normalized_action in {"service.start", "runtime.start_worker", "runtime.start_sync_agent"}: + return "start" + if normalized_action in {"service.stop", "runtime.stop_worker", "runtime.stop_sync_agent"}: + return "stop" + return "" + + +def execute_structured_action( + action: str, + payload: dict | None, + *, + service_names: dict[str, str], + runner: CommandRunner, + host_context: dict | None = None, +) -> tuple[bool, str, dict]: + normalized_action = str(action or "").strip() + normalized_payload = dict(payload or {}) + normalized_service_names = { + str(key or "").strip(): str(value or "").strip() + for key, value in dict(service_names or {}).items() + if str(key or "").strip() and str(value or "").strip() + } + host_details = { + key: str((host_context or {}).get(key) or "").strip() + for key in ("hostname", "ip") + if str((host_context or {}).get(key) or "").strip() + } + + if normalized_action == "health.snapshot": + checks: dict[str, dict] = {} + for key, service_name in normalized_service_names.items(): + code, stdout, stderr = runner(["systemctl", "is-active", service_name], timeout=15) + checks[key] = { + "service_name": service_name, + "returncode": int(code or 0), + "state": stdout or stderr, + "ok": int(code or 0) == 0 and (stdout or stderr) == "active", + } + result = {"checks": checks} + result.update(host_details) + return True, "health snapshot collected", result + + if normalized_action == "service.status": + service_name = service_name_from_payload( + normalized_payload, + default_worker_service_name=normalized_service_names.get("worker") or "", + ) + code, stdout, stderr = runner(["systemctl", "status", service_name, "--no-pager", "-l"], timeout=45) + result = { + "service_name": service_name, + "stdout": trim_output(stdout, 12000), + "stderr": trim_output(stderr, 4000), + } + result.update(host_details) + return int(code or 0) == 0, stdout or stderr or f"{service_name} status collected", result + + systemctl_action = systemctl_action_name(normalized_action) + if systemctl_action: + service_name = service_name_for_action(normalized_action, normalized_payload, normalized_service_names) + if not service_name: + return False, f"当前动作缺少 service_name: {normalized_action}", {"action": normalized_action} + code, stdout, stderr = runner(["systemctl", systemctl_action, service_name], timeout=45) + result = { + "service_name": service_name, + "systemctl_action": systemctl_action, + "stdout": trim_output(stdout, 12000), + "stderr": trim_output(stderr, 4000), + } + result.update(host_details) + return int(code or 0) == 0, stdout or stderr or f"{service_name} {systemctl_action} completed", result + + if normalized_action == "logs.collect": + service_name = service_name_from_payload( + normalized_payload, + default_worker_service_name=normalized_service_names.get("worker") or "", + ) + lines = max(20, min(int(normalized_payload.get("lines") or 120), 500)) + code, stdout, stderr = runner(["journalctl", "-u", service_name, "-n", str(lines), "--no-pager"], timeout=45) + journal_output = stdout or stderr + fallback_output, fallback_paths = _collect_log_file_fallback( + service_name, + lines=lines, + service_names=normalized_service_names, + ) + collection_source = "journal" + effective_output = journal_output + ok = int(code or 0) == 0 + if (not ok or not stdout.strip()) and fallback_output: + collection_source = "file_fallback" + effective_output = fallback_output + ok = True + result = { + "service_name": service_name, + "lines": lines, + "collection_source": collection_source, + "fallback_used": bool(collection_source == "file_fallback"), + "fallback_reason": trim_output(journal_output, 4000) if collection_source == "file_fallback" else "", + "log_paths": fallback_paths, + "journal_returncode": int(code or 0), + "stdout": trim_output(effective_output, 20000), + "stderr": trim_output(stderr, 4000), + } + result.update(host_details) + return ok, effective_output or f"{service_name} logs collected", result + + if normalized_action == "diagnostics.collect": + lines = max(20, min(int(normalized_payload.get("lines") or 200), 800)) + diagnostics: dict[str, object] = { + "services": {}, + "lines": lines, + } + diagnostics.update(host_details) + for key, service_name in normalized_service_names.items(): + active_code, active_stdout, active_stderr = runner(["systemctl", "is-active", service_name], timeout=15) + status_code, status_stdout, status_stderr = runner( + ["systemctl", "status", service_name, "--no-pager", "-l"], + timeout=45, + ) + log_code, log_stdout, log_stderr = runner( + ["journalctl", "-u", service_name, "-n", str(lines), "--no-pager"], + timeout=45, + ) + log_output = log_stdout or log_stderr + fallback_output, fallback_paths = _collect_log_file_fallback( + service_name, + lines=lines, + service_names=normalized_service_names, + ) + log_source = "journal" + if (int(log_code or 0) != 0 or not log_stdout.strip()) and fallback_output: + log_output = fallback_output + log_source = "file_fallback" + diagnostics["services"][key] = { + "service_name": service_name, + "active_returncode": int(active_code or 0), + "active_state": active_stdout or active_stderr, + "status_returncode": int(status_code or 0), + "status_output": trim_output(status_stdout or status_stderr, 12000), + "log_returncode": int(log_code or 0), + "log_source": log_source, + "log_paths": fallback_paths, + "log_output": trim_output(log_output, 20000), + } + return True, "diagnostics collected", diagnostics + + return False, f"unsupported action: {normalized_action}", {"action": normalized_action} diff --git a/domain-api/app/services/ops_agent_service.py b/domain-api/app/services/ops_agent_service.py new file mode 100644 index 0000000..79f75b1 --- /dev/null +++ b/domain-api/app/services/ops_agent_service.py @@ -0,0 +1,2852 @@ +from __future__ import annotations + +import hashlib +import json +import secrets +import shlex +import threading +from datetime import datetime, timedelta + +from app.core.config import settings +from app.core.db import get_db +from app.services.ops_command_service import build_bash_command +from app.services.ops_job_service import create_ops_job, ensure_ops_schema, get_ops_job +from app.services.ops_template_service import build_ops_template_payload, get_ops_action_template + + +_AGENT_SCHEMA_SQL = """ +CREATE TABLE IF NOT EXISTS ops_node_tokens ( + id BIGSERIAL PRIMARY KEY, + token_hash VARCHAR(128) NOT NULL UNIQUE, + node_code VARCHAR(64) NOT NULL DEFAULT '', + purpose VARCHAR(32) NOT NULL DEFAULT 'agent', + issued_by VARCHAR(64) NOT NULL DEFAULT 'system', + is_enabled BOOLEAN NOT NULL DEFAULT TRUE, + expires_at TIMESTAMP, + last_used_at TIMESTAMP, + metadata_json JSONB, + created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP +); + +CREATE INDEX IF NOT EXISTS idx_ops_node_tokens_node_code +ON ops_node_tokens(node_code, is_enabled, purpose); + +ALTER TABLE ops_node_tokens ADD COLUMN IF NOT EXISTS node_code VARCHAR(64) NOT NULL DEFAULT ''; +ALTER TABLE ops_node_tokens ADD COLUMN IF NOT EXISTS purpose VARCHAR(32) NOT NULL DEFAULT 'agent'; +ALTER TABLE ops_node_tokens ADD COLUMN IF NOT EXISTS issued_by VARCHAR(64) NOT NULL DEFAULT 'system'; +ALTER TABLE ops_node_tokens ADD COLUMN IF NOT EXISTS is_enabled BOOLEAN NOT NULL DEFAULT TRUE; +ALTER TABLE ops_node_tokens ADD COLUMN IF NOT EXISTS expires_at TIMESTAMP; +ALTER TABLE ops_node_tokens ADD COLUMN IF NOT EXISTS last_used_at TIMESTAMP; +ALTER TABLE ops_node_tokens ADD COLUMN IF NOT EXISTS metadata_json JSONB; +ALTER TABLE ops_node_tokens ADD COLUMN IF NOT EXISTS created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP; +ALTER TABLE ops_node_tokens ADD COLUMN IF NOT EXISTS updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP; + +CREATE TABLE IF NOT EXISTS ops_job_events ( + id BIGSERIAL PRIMARY KEY, + job_id BIGINT REFERENCES ops_jobs(id) ON DELETE CASCADE, + step_id BIGINT REFERENCES ops_job_steps(id) ON DELETE SET NULL, + node_code VARCHAR(64) NOT NULL DEFAULT '', + client_event_id VARCHAR(128) NOT NULL DEFAULT '', + event_type VARCHAR(64) NOT NULL DEFAULT '', + level VARCHAR(16) NOT NULL DEFAULT 'info', + message TEXT NOT NULL DEFAULT '', + payload_json JSONB, + created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP +); + +ALTER TABLE ops_job_events ADD COLUMN IF NOT EXISTS job_id BIGINT; +ALTER TABLE ops_job_events ADD COLUMN IF NOT EXISTS step_id BIGINT; +ALTER TABLE ops_job_events ADD COLUMN IF NOT EXISTS node_code VARCHAR(64) NOT NULL DEFAULT ''; +ALTER TABLE ops_job_events ADD COLUMN IF NOT EXISTS client_event_id VARCHAR(128) NOT NULL DEFAULT ''; +ALTER TABLE ops_job_events ADD COLUMN IF NOT EXISTS event_type VARCHAR(64) NOT NULL DEFAULT ''; +ALTER TABLE ops_job_events ADD COLUMN IF NOT EXISTS level VARCHAR(16) NOT NULL DEFAULT 'info'; +ALTER TABLE ops_job_events ADD COLUMN IF NOT EXISTS message TEXT NOT NULL DEFAULT ''; +ALTER TABLE ops_job_events ADD COLUMN IF NOT EXISTS payload_json JSONB; +ALTER TABLE ops_job_events ADD COLUMN IF NOT EXISTS created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP; + +CREATE INDEX IF NOT EXISTS idx_ops_job_events_job_created +ON ops_job_events(job_id, created_at DESC); + +CREATE UNIQUE INDEX IF NOT EXISTS uq_ops_job_events_job_client_event +ON ops_job_events(job_id, client_event_id) +WHERE client_event_id <> ''; + +ALTER TABLE ops_jobs ADD COLUMN IF NOT EXISTS last_agent_complete_request_id VARCHAR(128) NOT NULL DEFAULT ''; +""" +_OPS_AGENT_SCHEMA_LOCK = threading.Lock() +_OPS_AGENT_SCHEMA_READY = False +_OPS_AGENT_SCHEMA_ADVISORY_LOCK_KEY = 90421802 + + +def ensure_ops_agent_schema() -> None: + global _OPS_AGENT_SCHEMA_READY + ensure_ops_schema() + if _OPS_AGENT_SCHEMA_READY: + return + with _OPS_AGENT_SCHEMA_LOCK: + if _OPS_AGENT_SCHEMA_READY: + return + with get_db() as conn: + conn.autocommit = False + with conn.cursor() as cur: + cur.execute("SELECT pg_advisory_xact_lock(%s)", (_OPS_AGENT_SCHEMA_ADVISORY_LOCK_KEY,)) + cur.execute(_AGENT_SCHEMA_SQL) + conn.commit() + _OPS_AGENT_SCHEMA_READY = True + + +def _hash_token(token: str) -> str: + return hashlib.sha256(str(token or "").encode("utf-8")).hexdigest() + + +def _decode_json(value: object) -> dict: + if isinstance(value, dict): + return value + if value in (None, ""): + return {} + try: + return json.loads(value) + except Exception: + return {} + + +def _format_time(value: object) -> str: + if isinstance(value, datetime): + return value.isoformat(sep=" ", timespec="seconds") + return "" + + +def _ops_job_event_level_label(level: object) -> str: + normalized_level = str(level or "").strip().lower() + mapping = { + "debug": "调试", + "info": "信息", + "success": "成功", + "warning": "警告", + "error": "错误", + "critical": "严重", + } + return mapping.get(normalized_level, normalized_level or "信息") + + +def _build_ops_job_event_row(row: tuple) -> dict: + created_at = _format_time(row[9]) + payload = _decode_json(row[8]) + job_id = int(row[1]) if row[1] else 0 + event_id = int(row[0]) + event_type = str(row[5] or "") + level = str(row[6] or "") + message = str(row[7] or "") + occurred_at = str(payload.get("occurred_at") or "").strip() or created_at + summary_text = str(payload.get("summary_text") or "").strip() or message + focus_ref = payload.get("focus_ref") if isinstance(payload.get("focus_ref"), dict) else {} + if not focus_ref: + focus_ref = { + "kind": "ops_job_event", + "job_id": job_id, + "focus_event_id": event_id, + "event_type": event_type, + } + return { + "id": event_id, + "event_key": f"job-event:{event_id}", + "job_id": job_id, + "step_id": int(row[2]) if row[2] else 0, + "node_code": str(row[3] or ""), + "client_event_id": str(row[4] or ""), + "event_type": event_type, + "level": level, + "level_label": _ops_job_event_level_label(level), + "message": message, + "summary": message, + "summary_text": summary_text, + "payload": payload, + "has_payload": bool(payload), + "created_at": created_at, + "occurred_at": occurred_at, + "focus_ref": focus_ref, + "ui_intent": { + "kind": "job_detail", + "job_id": job_id, + "focus_event_id": event_id, + "event_type": event_type, + }, + } + + +def summarize_ops_job_events(events: list[dict], *, job_id: int = 0, limit: int = 50) -> dict: + level_counts: dict[str, int] = {} + event_type_counts: dict[str, int] = {} + node_counts: dict[str, int] = {} + latest_at = "" + for event in list(events or []): + event_level = str(event.get("level") or "").strip() or "info" + event_type = str(event.get("event_type") or "").strip() or "event" + node_code = str(event.get("node_code") or "").strip() + occurred_at = str(event.get("occurred_at") or event.get("created_at") or "").strip() + level_counts[event_level] = int(level_counts.get(event_level, 0) or 0) + 1 + event_type_counts[event_type] = int(event_type_counts.get(event_type, 0) or 0) + 1 + if node_code: + node_counts[node_code] = int(node_counts.get(node_code, 0) or 0) + 1 + if occurred_at and occurred_at > latest_at: + latest_at = occurred_at + return { + "job_id": int(job_id or 0), + "returned_total": len(events or []), + "level_counts": level_counts, + "event_type_counts": event_type_counts, + "node_counts": node_counts, + "latest_at": latest_at, + "filters": { + "limit": int(limit or 50), + }, + } + + +def _parse_time(value: object) -> datetime | None: + if isinstance(value, datetime): + return value + raw_value = str(value or "").strip() + if not raw_value: + return None + normalized = raw_value.replace("Z", "+00:00") + try: + return datetime.fromisoformat(normalized) + except Exception: + pass + for fmt in ("%Y-%m-%d %H:%M:%S", "%Y-%m-%d %H:%M:%S.%f"): + try: + return datetime.strptime(raw_value, fmt) + except ValueError: + continue + return None + + +def _agent_error(message: str, detail_code: str, data: dict | None = None) -> tuple[bool, str, dict]: + payload = dict(data or {}) + payload["detail_code"] = str(detail_code or "").strip() + return False, message, payload + + +def _agent_state_from_context( + *, + last_seen_at: str, + latest_token: dict, + cluster_status: str, + current_load: int, +) -> tuple[str, str, str]: + now = datetime.now() + last_seen_dt = _parse_time(last_seen_at) + token_status = str(latest_token.get("token_status") or "absent").strip() or "absent" + + if last_seen_dt: + current_time = datetime.now(last_seen_dt.tzinfo) if last_seen_dt.tzinfo else now + age_seconds = max(0, int((current_time - last_seen_dt).total_seconds())) + if age_seconds <= 180: + if cluster_status == "busy" or int(current_load or 0) > 0: + return "online_busy", "已接管", f"Agent 最近心跳 {age_seconds} 秒前，节点当前负载 {int(current_load or 0)}。" + return "online", "已接管", f"Agent 最近心跳 {age_seconds} 秒前。" + return "stale", "心跳过期", f"Agent 最近心跳 {age_seconds} 秒前，已超过在线阈值。" + + if token_status == "active": + return "pending_bootstrap", "待接入", "已签发有效 Agent Token，但节点尚未完成 register/heartbeat。" + if token_status == "expired": + return "token_expired", "Token 过期", "已签发 Agent Token，但该 Token 已过期。" + if token_status == "disabled": + return "token_disabled", "Token 已禁用", "最近一枚 Agent Token 已被禁用，需重新签发。" + + if cluster_status in {"online", "busy"}: + return "runtime_only", "仅运行态在线", "控制面能看到 runtime 心跳，但 Node Agent 尚未接管。" + return "unmanaged", "未接管", "当前节点尚未生成有效 Agent 接入状态。" + + +def _ssh_access_context(node: dict) -> tuple[bool, str, str]: + ssh_host = str(node.get("ssh_host") or "").strip() + ssh_user = str(node.get("ssh_user") or "").strip() + if ssh_host and ssh_user: + return True, "SSH 已备好", f"已保存 SSH 入口 {ssh_user}@{ssh_host}，后续可由海外控制面 SSH 执行器接管。" + if ssh_host and not ssh_user: + return False, "SSH 待完善", "已填写 SSH 主机，但还缺少 ssh_user。" + if ssh_user and not ssh_host: + return False, "SSH 待完善", "已填写 ssh_user，但还缺少 ssh_host。" + return False, "SSH 未配置", "当前还没有保存 SSH 入口。" + + +def _remote_access_context(node: dict) -> tuple[str, str, str, bool]: + is_managed = bool(node.get("is_managed", False)) + is_enabled = bool(node.get("is_enabled", False)) + is_agent_online = bool(node.get("is_agent_online", False)) + ssh_ready, ssh_label, ssh_reason = _ssh_access_context(node) + + if not is_managed: + return "unmanaged", "未纳管", "当前节点还没有进入海外控制面的托管清单。", False + if not is_enabled: + return "disabled", "已停用", "当前节点已纳管，但被标记为停用。", False + if is_agent_online and ssh_ready: + return "hybrid_ready", "Agent + SSH", "Node Agent 已在线，SSH 入口也已备好。", True + if is_agent_online: + return "agent_ready", "Agent 在线", "当前节点已经进入标准自动执行器就绪状态。", True + if ssh_ready: + return ( + "ssh_ready", + "SSH 可执行", + f"{ssh_reason} 当前已可执行日志、诊断和部分服务控制；正式发布与标准 Rollout 仍优先依赖 Node Agent。", + True, + ) + return "agent_pending", "待接入", "当前还没有可用 Agent，也没有保存 SSH 入口。", False + + +def _delivery_queue_context(metadata: dict) -> tuple[str, str, str, dict]: + snapshot = dict((metadata or {}).get("delivery_queue") or {}) + pending_count = int(snapshot.get("pending_count", 0) or 0) + dead_letter_count = int(snapshot.get("dead_letter_count", 0) or 0) + last_flush_at = str(snapshot.get("last_flush_at") or "").strip() + oldest_pending_at = str(snapshot.get("oldest_pending_at") or "").strip() + oldest_dead_letter_at = str(snapshot.get("oldest_dead_letter_at") or "").strip() + + if not snapshot: + return "unknown", "未上报", "Node Agent 尚未上报回执队列状态。", {} + if dead_letter_count > 0: + reason = f"当前存在 {dead_letter_count} 条死信记录，建议优先查看节点日志或诊断编排。" + if oldest_dead_letter_at: + reason += f" 最早死信时间：{oldest_dead_letter_at}。" + return "dead_letter", f"死信 {dead_letter_count}", reason, snapshot + if pending_count > 0: + reason = f"当前存在 {pending_count} 条待重试回执，Node Agent 会继续自动回放。" + if oldest_pending_at: + reason += f" 最早积压时间：{oldest_pending_at}。" + return "retrying", f"待重试 {pending_count}", reason, snapshot + reason = "当前没有待重试回执，也没有死信记录。" + if last_flush_at: + reason += f" 最近一次队列冲刷：{last_flush_at}。" + return "healthy", "正常", reason, snapshot + + +def _summarize_managed_node_states(nodes: list[dict]) -> dict: + status_counts: dict[str, int] = {} + remote_access_counts: dict[str, int] = {} + delivery_queue_state_counts: dict[str, int] = {} + for item in nodes: + state = str(item.get("agent_state") or "").strip() or "unknown" + status_counts[state] = int(status_counts.get(state, 0) or 0) + 1 + remote_state = str(item.get("remote_access_state") or "").strip() or "unknown" + remote_access_counts[remote_state] = int(remote_access_counts.get(remote_state, 0) or 0) + 1 + queue_state = str(item.get("delivery_queue_state") or "").strip() or "unknown" + delivery_queue_state_counts[queue_state] = int(delivery_queue_state_counts.get(queue_state, 0) or 0) + 1 + managed_nodes = [item for item in nodes if bool(item.get("is_managed", False))] + participating_nodes = [item for item in nodes if bool(item.get("is_current_participant", False))] + dispatch_active_nodes = [item for item in nodes if bool(item.get("is_dispatch_active", False))] + recent_only_nodes = [ + item for item in nodes if str(item.get("participation_state") or "").strip() == "recent_throughput" + ] + standby_nodes = [item for item in nodes if str(item.get("participation_state") or "").strip() == "standby"] + load_syncing_nodes = [item for item in nodes if str(item.get("participation_state") or "").strip() == "load_syncing"] + return { + "status_counts": status_counts, + "online": int(status_counts.get("online", 0) or 0) + int(status_counts.get("online_busy", 0) or 0), + "stale": int(status_counts.get("stale", 0) or 0), + "pending_bootstrap": int(status_counts.get("pending_bootstrap", 0) or 0), + "runtime_only": int(status_counts.get("runtime_only", 0) or 0), + "unmanaged": int(status_counts.get("unmanaged", 0) or 0), + "token_issue": int(status_counts.get("token_expired", 0) or 0) + int(status_counts.get("token_disabled", 0) or 0), + "ssh_ready": int(remote_access_counts.get("ssh_ready", 0) or 0) + int(remote_access_counts.get("hybrid_ready", 0) or 0), + "agent_ready": int(remote_access_counts.get("agent_ready", 0) or 0) + int(remote_access_counts.get("hybrid_ready", 0) or 0), + "remote_access_ready": sum(1 for item in nodes if bool(item.get("is_remote_access_ready", False))), + "managed_total": len(managed_nodes), + "managed_enabled": sum(1 for item in managed_nodes if bool(item.get("is_enabled", False))), + "queue_retrying_nodes": sum( + 1 for item in managed_nodes if str(item.get("delivery_queue_state") or "").strip() == "retrying" + ), + "queue_dead_letter_nodes": sum( + 1 for item in managed_nodes if str(item.get("delivery_queue_state") or "").strip() == "dead_letter" + ), + "queue_pending_records": sum(int(item.get("delivery_queue_pending_count", 0) or 0) for item in managed_nodes), + "queue_dead_letter_records": sum( + int(item.get("delivery_queue_dead_letter_count", 0) or 0) for item in managed_nodes + ), + "participating": len(participating_nodes), + "dispatch_active": len(dispatch_active_nodes), + "recent_only": len(recent_only_nodes), + "standby": len(standby_nodes), + "load_syncing": len(load_syncing_nodes), + "non_participating": len(standby_nodes) + len(load_syncing_nodes), + "total": len(nodes), + "remote_access_state_counts": remote_access_counts, + "delivery_queue_state_counts": delivery_queue_state_counts, + } + + +def _normalize_detect_participation_rows(payload: dict | None = None) -> dict[str, dict]: + normalized_payload = dict(payload or {}) + if "detect" in normalized_payload and isinstance(normalized_payload.get("detect"), dict): + normalized_payload = dict(normalized_payload.get("detect") or {}) + + rows: dict[str, dict] = {} + for collection_key in ("participating_nodes", "non_participating_nodes", "standby_nodes"): + for item in list(normalized_payload.get(collection_key) or []): + node_code = str((item or {}).get("node_code") or "").strip() + if not node_code: + continue + rows[node_code] = dict(item or {}) + return rows + + +def append_ops_job_event( + *, + job_id: int | None, + step_id: int | None = None, + node_code: str = "", + client_event_id: str = "", + event_type: str, + message: str, + level: str = "info", + payload: dict | None = None, +) -> int: + ensure_ops_agent_schema() + with get_db() as conn: + with conn.cursor() as cur: + cur.execute( + """ + INSERT INTO ops_job_events ( + job_id, step_id, node_code, client_event_id, event_type, level, message, payload_json, created_at + ) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, CURRENT_TIMESTAMP) + ON CONFLICT (job_id, client_event_id) WHERE client_event_id <> '' + DO UPDATE SET + step_id = COALESCE(EXCLUDED.step_id, ops_job_events.step_id), + node_code = CASE WHEN EXCLUDED.node_code <> '' THEN EXCLUDED.node_code ELSE ops_job_events.node_code END, + event_type = EXCLUDED.event_type, + level = EXCLUDED.level, + message = EXCLUDED.message, + payload_json = EXCLUDED.payload_json + RETURNING id + """, + ( + int(job_id) if job_id else None, + int(step_id) if step_id else None, + str(node_code or "").strip(), + str(client_event_id or "").strip()[:128], + str(event_type or "").strip() or "event", + str(level or "info").strip() or "info", + str(message or "").strip()[:2000], + json.dumps(payload or {}, ensure_ascii=False), + ), + ) + event_id = int(cur.fetchone()[0]) + conn.commit() + return event_id + + +def list_ops_job_events(job_id: int, limit: int = 50) -> list[dict]: + ensure_ops_agent_schema() + safe_limit = min(max(int(limit or 50), 1), 200) + with get_db() as conn: + with conn.cursor() as cur: + cur.execute( + """ + SELECT id, job_id, step_id, node_code, client_event_id, event_type, level, message, payload_json, created_at + FROM ops_job_events + WHERE job_id = %s + ORDER BY id DESC + LIMIT %s + """, + (int(job_id), safe_limit), + ) + rows = cur.fetchall() + return [_build_ops_job_event_row(row) for row in rows] + + +def get_ops_job_event(event_id: int) -> dict: + ensure_ops_agent_schema() + with get_db() as conn: + with conn.cursor() as cur: + cur.execute( + """ + SELECT id, job_id, step_id, node_code, client_event_id, event_type, level, message, payload_json, created_at + FROM ops_job_events + WHERE id = %s + LIMIT 1 + """, + (int(event_id),), + ) + row = cur.fetchone() + return _build_ops_job_event_row(row) if row else {} + + +def list_ops_job_events_for_jobs(job_ids: list[int], limit: int = 120) -> list[dict]: + ensure_ops_agent_schema() + normalized_job_ids = sorted({int(item) for item in list(job_ids or []) if int(item or 0) > 0}) + if not normalized_job_ids: + return [] + safe_limit = min(max(int(limit or 120), 1), 800) + placeholders = ", ".join(["%s"] * len(normalized_job_ids)) + with get_db() as conn: + with conn.cursor() as cur: + cur.execute( + f""" + SELECT id, job_id, step_id, node_code, client_event_id, event_type, level, message, payload_json, created_at + FROM ops_job_events + WHERE job_id IN ({placeholders}) + ORDER BY id DESC + LIMIT %s + """, + (*normalized_job_ids, safe_limit), + ) + rows = cur.fetchall() + return [_build_ops_job_event_row(row) for row in rows] + + +def list_managed_nodes_with_agent_state(*, participation_payload: dict | None = None) -> dict: + from app.services.cluster_runtime_service import get_cluster_snapshot + from app.services.ops_job_service import list_managed_nodes + + ensure_ops_agent_schema() + managed_nodes = list_managed_nodes() + cluster_nodes = list((get_cluster_snapshot().get("nodes") or [])) + cluster_map = { + str(item.get("node_code") or "").strip(): item + for item in cluster_nodes + if str(item.get("node_code") or "").strip() + } + detect_participation_map = _normalize_detect_participation_rows(participation_payload) + if not detect_participation_map: + try: + from app.services.runtime_status_service import get_runtime_status + + runtime_payload = get_runtime_status() + detect_participation_map = _normalize_detect_participation_rows(runtime_payload.get("detect") or {}) + except Exception: + detect_participation_map = {} + + latest_tokens: dict[str, dict] = {} + latest_jobs: dict[str, dict] = {} + with get_db() as conn: + with conn.cursor() as cur: + cur.execute( + """ + SELECT DISTINCT ON (node_code) + node_code, id, issued_by, is_enabled, expires_at, last_used_at, metadata_json, created_at, updated_at + FROM ops_node_tokens + WHERE purpose = 'agent' + ORDER BY node_code ASC, created_at DESC, id DESC + """ + ) + for row in cur.fetchall(): + node_code = str(row[0] or "").strip() + if not node_code: + continue + expires_at = _format_time(row[4]) + token_status = "active" + expires_dt = _parse_time(row[4]) + if not bool(row[3]): + token_status = "disabled" + elif expires_dt: + current_time = datetime.now(expires_dt.tzinfo) if expires_dt.tzinfo else datetime.now() + if expires_dt < current_time: + token_status = "expired" + latest_tokens[node_code] = { + "record_id": int(row[1]), + "issued_by": str(row[2] or ""), + "is_enabled": bool(row[3]), + "expires_at": expires_at, + "last_used_at": _format_time(row[5]), + "metadata": _decode_json(row[6]), + "created_at": _format_time(row[7]), + "updated_at": _format_time(row[8]), + "token_status": token_status, + } + + cur.execute( + """ + SELECT DISTINCT ON (target_node_code) + target_node_code, id, job_code, action, status, execution_mode, created_at, updated_at + FROM ops_jobs + WHERE target_node_code <> '' + ORDER BY target_node_code ASC, created_at DESC, id DESC + """ + ) + for row in cur.fetchall(): + node_code = str(row[0] or "").strip() + if not node_code: + continue + latest_jobs[node_code] = { + "id": int(row[1]), + "job_code": str(row[2] or ""), + "action": str(row[3] or ""), + "status": str(row[4] or ""), + "execution_mode": str(row[5] or ""), + "created_at": _format_time(row[6]), + "updated_at": _format_time(row[7]), + } + + enriched_nodes: list[dict] = [] + seen_node_codes: set[str] = set() + for node in managed_nodes: + node_code = str(node.get("node_code") or "").strip() + if not node_code: + continue + seen_node_codes.add(node_code) + metadata = dict(node.get("metadata") or {}) + cluster_node = cluster_map.get(node_code, {}) + latest_token = latest_tokens.get(node_code, {}) + latest_job = latest_jobs.get(node_code, {}) + participation_row = dict(detect_participation_map.get(node_code) or {}) + last_seen_at = str(node.get("last_seen_at") or "").strip() or str(metadata.get("last_seen_at") or "").strip() + cluster_status = str(cluster_node.get("status") or metadata.get("cluster_status") or "").strip() + current_load = int(cluster_node.get("current_load", 0) or 0) + ( + delivery_queue_state, + delivery_queue_label, + delivery_queue_reason, + delivery_queue_snapshot, + ) = _delivery_queue_context(metadata) + agent_state, agent_state_label, agent_state_reason = _agent_state_from_context( + last_seen_at=last_seen_at, + latest_token=latest_token, + cluster_status=cluster_status, + current_load=current_load, + ) + capabilities = list(metadata.get("capabilities") or []) + merged_node = { + **node, + "is_managed": True, + "last_seen_at": last_seen_at, + "agent_state": agent_state, + "agent_state_label": agent_state_label, + "agent_state_reason": agent_state_reason, + "is_agent_online": agent_state in {"online", "online_busy"}, + "capabilities": capabilities, + "capabilities_count": len(capabilities), + "agent_version": str(metadata.get("agent_version") or "").strip(), + "agent_hostname": str(metadata.get("hostname") or "").strip(), + "agent_ip": str(metadata.get("ip") or "").strip(), + "cluster_hostname": str(cluster_node.get("hostname") or "").strip(), + "cluster_ip": str(cluster_node.get("ip") or "").strip(), + "cluster_status": cluster_status, + "cluster_current_load": current_load, + "cluster_last_heartbeat_at": str(cluster_node.get("last_heartbeat_at") or metadata.get("last_heartbeat_at") or "").strip(), + "cluster_is_effective_worker": bool(cluster_node.get("is_effective_worker", metadata.get("is_effective_worker", False))), + "cluster_detect_participating": bool( + participation_row.get("is_current_participant", False) + or cluster_node.get("detect_participating", metadata.get("detect_participating", False)) + ), + "participation_state": str(participation_row.get("participation_state") or "").strip(), + "participation_label": str(participation_row.get("participation_label") or "").strip(), + "participation_reason": str(participation_row.get("participation_reason") or "").strip(), + "is_current_participant": bool(participation_row.get("is_current_participant", False)), + "is_dispatch_active": bool(participation_row.get("is_dispatch_active", False)), + "items_total": int(participation_row.get("items_total", 0) or 0), + "items_pending": int(participation_row.get("items_pending", 0) or 0), + "items_claimed": int(participation_row.get("items_claimed", 0) or 0), + "items_running": int(participation_row.get("items_running", 0) or 0), + "items_completed": int(participation_row.get("items_completed", 0) or 0), + "items_failed": int(participation_row.get("items_failed", 0) or 0), + "processed_recent": int(participation_row.get("processed_recent", 0) or 0), + "processed_per_minute": float(participation_row.get("processed_per_minute", 0) or 0), + "delivery_queue_state": delivery_queue_state, + "delivery_queue_label": delivery_queue_label, + "delivery_queue_reason": delivery_queue_reason, + "delivery_queue_pending_count": int(delivery_queue_snapshot.get("pending_count", 0) or 0), + "delivery_queue_dead_letter_count": int(delivery_queue_snapshot.get("dead_letter_count", 0) or 0), + "delivery_queue_last_flush_at": str(delivery_queue_snapshot.get("last_flush_at") or "").strip(), + "delivery_queue_oldest_pending_at": str(delivery_queue_snapshot.get("oldest_pending_at") or "").strip(), + "delivery_queue_oldest_pending_request_id": str( + delivery_queue_snapshot.get("oldest_pending_request_id") or "" + ).strip(), + "delivery_queue_oldest_pending_kind": str(delivery_queue_snapshot.get("oldest_pending_kind") or "").strip(), + "delivery_queue_oldest_dead_letter_at": str( + delivery_queue_snapshot.get("oldest_dead_letter_at") or "" + ).strip(), + "delivery_queue_oldest_dead_letter_request_id": str( + delivery_queue_snapshot.get("oldest_dead_letter_request_id") or "" + ).strip(), + "delivery_queue_oldest_dead_letter_kind": str( + delivery_queue_snapshot.get("oldest_dead_letter_kind") or "" + ).strip(), + "latest_token": latest_token, + "latest_job": latest_job, + } + ssh_ready, ssh_access_label, ssh_access_reason = _ssh_access_context(merged_node) + remote_access_state, remote_access_label, remote_access_reason, is_remote_access_ready = _remote_access_context( + merged_node + ) + + enriched_nodes.append( + { + **merged_node, + "has_ssh_access": ssh_ready, + "ssh_access_label": ssh_access_label, + "ssh_access_reason": ssh_access_reason, + "remote_access_state": remote_access_state, + "remote_access_label": remote_access_label, + "remote_access_reason": remote_access_reason, + "is_remote_access_ready": is_remote_access_ready, + } + ) + + for node_code, cluster_node in cluster_map.items(): + if not node_code or node_code in seen_node_codes: + continue + metadata = dict(cluster_node.get("metadata") or {}) + latest_token = latest_tokens.get(node_code, {}) + latest_job = latest_jobs.get(node_code, {}) + participation_row = dict(detect_participation_map.get(node_code) or {}) + cluster_status = str(cluster_node.get("status") or "").strip() + current_load = int(cluster_node.get("current_load", 0) or 0) + ( + delivery_queue_state, + delivery_queue_label, + delivery_queue_reason, + delivery_queue_snapshot, + ) = _delivery_queue_context(metadata) + agent_state, agent_state_label, agent_state_reason = _agent_state_from_context( + last_seen_at="", + latest_token=latest_token, + cluster_status=cluster_status, + current_load=current_load, + ) + capabilities = list(metadata.get("capabilities") or []) + fallback_node = { + "node_code": node_code, + "region": str(cluster_node.get("region") or "").strip(), + "role": str(cluster_node.get("role") or "").strip(), + "title": str(cluster_node.get("hostname") or node_code).strip(), + "ssh_host": "", + "ssh_port": 22, + "ssh_user": "", + "auth_mode": "key", + "deploy_channel": "", + "is_enabled": False, + "is_managed": False, + "metadata": { + **metadata, + "imported_from_cluster_fallback": True, + }, + "created_at": "", + "last_seen_at": "", + "updated_at": "", + "agent_state": agent_state, + "agent_state_label": agent_state_label, + "agent_state_reason": agent_state_reason, + "is_agent_online": agent_state in {"online", "online_busy"}, + "capabilities": capabilities, + "capabilities_count": len(capabilities), + "agent_version": str(metadata.get("agent_version") or "").strip(), + "agent_hostname": str(cluster_node.get("hostname") or metadata.get("hostname") or "").strip(), + "agent_ip": str(cluster_node.get("ip") or metadata.get("ip") or "").strip(), + "cluster_hostname": str(cluster_node.get("hostname") or "").strip(), + "cluster_ip": str(cluster_node.get("ip") or "").strip(), + "cluster_status": cluster_status, + "cluster_current_load": current_load, + "cluster_last_heartbeat_at": str(cluster_node.get("last_heartbeat_at") or "").strip(), + "cluster_is_effective_worker": bool(cluster_node.get("is_effective_worker", False)), + "cluster_detect_participating": bool( + participation_row.get("is_current_participant", False) or cluster_node.get("detect_participating", False) + ), + "participation_state": str(participation_row.get("participation_state") or "").strip(), + "participation_label": str(participation_row.get("participation_label") or "").strip(), + "participation_reason": str(participation_row.get("participation_reason") or "").strip(), + "is_current_participant": bool(participation_row.get("is_current_participant", False)), + "is_dispatch_active": bool(participation_row.get("is_dispatch_active", False)), + "items_total": int(participation_row.get("items_total", 0) or 0), + "items_pending": int(participation_row.get("items_pending", 0) or 0), + "items_claimed": int(participation_row.get("items_claimed", 0) or 0), + "items_running": int(participation_row.get("items_running", 0) or 0), + "items_completed": int(participation_row.get("items_completed", 0) or 0), + "items_failed": int(participation_row.get("items_failed", 0) or 0), + "processed_recent": int(participation_row.get("processed_recent", 0) or 0), + "processed_per_minute": float(participation_row.get("processed_per_minute", 0) or 0), + "delivery_queue_state": delivery_queue_state, + "delivery_queue_label": delivery_queue_label, + "delivery_queue_reason": delivery_queue_reason, + "delivery_queue_pending_count": int(delivery_queue_snapshot.get("pending_count", 0) or 0), + "delivery_queue_dead_letter_count": int(delivery_queue_snapshot.get("dead_letter_count", 0) or 0), + "delivery_queue_last_flush_at": str(delivery_queue_snapshot.get("last_flush_at") or "").strip(), + "delivery_queue_oldest_pending_at": str(delivery_queue_snapshot.get("oldest_pending_at") or "").strip(), + "delivery_queue_oldest_pending_request_id": str( + delivery_queue_snapshot.get("oldest_pending_request_id") or "" + ).strip(), + "delivery_queue_oldest_pending_kind": str(delivery_queue_snapshot.get("oldest_pending_kind") or "").strip(), + "delivery_queue_oldest_dead_letter_at": str( + delivery_queue_snapshot.get("oldest_dead_letter_at") or "" + ).strip(), + "delivery_queue_oldest_dead_letter_request_id": str( + delivery_queue_snapshot.get("oldest_dead_letter_request_id") or "" + ).strip(), + "delivery_queue_oldest_dead_letter_kind": str( + delivery_queue_snapshot.get("oldest_dead_letter_kind") or "" + ).strip(), + "latest_token": latest_token, + "latest_job": latest_job, + } + ssh_ready, ssh_access_label, ssh_access_reason = _ssh_access_context(fallback_node) + remote_access_state, remote_access_label, remote_access_reason, is_remote_access_ready = _remote_access_context( + fallback_node + ) + enriched_nodes.append( + { + **fallback_node, + "has_ssh_access": ssh_ready, + "ssh_access_label": ssh_access_label, + "ssh_access_reason": ssh_access_reason, + "remote_access_state": remote_access_state, + "remote_access_label": remote_access_label, + "remote_access_reason": remote_access_reason, + "is_remote_access_ready": is_remote_access_ready, + } + ) + + enriched_nodes.sort( + key=lambda item: ( + 0 if bool(item.get("is_managed", False)) else 1, + str(item.get("region") or ""), + str(item.get("role") or ""), + str(item.get("node_code") or ""), + ) + ) + + return { + "nodes": enriched_nodes, + "summary": _summarize_managed_node_states(enriched_nodes), + } + + +def _build_managed_node_handover_stage(node: dict) -> dict: + normalized_node = dict(node or {}) + node_code = str(normalized_node.get("node_code") or "").strip() + agent_state = str(normalized_node.get("agent_state") or "").strip() + agent_state_label = str(normalized_node.get("agent_state_label") or "").strip() or "未知" + agent_state_reason = str(normalized_node.get("agent_state_reason") or "").strip() + remote_access_state = str(normalized_node.get("remote_access_state") or "").strip() + remote_access_label = str(normalized_node.get("remote_access_label") or "").strip() or "未就绪" + remote_access_reason = str(normalized_node.get("remote_access_reason") or "").strip() + token_status = str(((normalized_node.get("latest_token") or {}).get("token_status") or "")).strip() + is_managed = bool(normalized_node.get("is_managed", False)) + is_enabled = bool(normalized_node.get("is_enabled", False)) + has_ssh_access = bool(normalized_node.get("has_ssh_access", False)) + + if not is_managed: + return { + "code": "unmanaged", + "label": "未纳管", + "summary": "节点已出现在集群视图，但还没保存到海外控制面的托管清单。", + "next_step": { + "code": "save_managed_node", + "label": "先保存节点资料", + "summary": "先补齐 SSH 入口、身份和部署通道，再生成 Node Agent 接入方案。", + "ui_intent": { + "kind": "managed_node_edit", + "node_code": node_code, + }, + }, + } + if not is_enabled: + return { + "code": "disabled", + "label": "已停用", + "summary": "节点虽然已经纳管，但当前被标记为停用，标准运维动作不会继续下发。", + "next_step": { + "code": "enable_managed_node", + "label": "检查并重新启用", + "summary": "确认该节点仍应参与运维后，先在托管节点面板重新启用。", + "ui_intent": { + "kind": "managed_node_edit", + "node_code": node_code, + }, + }, + } + if agent_state in {"online", "online_busy"}: + return { + "code": "ready", + "label": "接管完成", + "summary": "Node Agent 已经在线，节点已经进入标准自动执行链路。", + "next_step": { + "code": "run_acceptance", + "label": "执行接管验收", + "summary": "建议立刻跑一轮 onboarding.acceptance 或标准巡检，确认日志、健康快照和发布链路都打通。", + "ui_intent": { + "kind": "open_playbook_dialog", + "playbook_key": "onboarding.acceptance", + "target_node_codes": [node_code] if node_code else [], + "execution_mode": "remote-agent", + "auto_approve": True, + }, + }, + } + if agent_state == "stale": + return { + "code": "stale", + "label": "心跳过期", + "summary": "节点曾经接入过，但 Node Agent 心跳已经过期，当前不能视为稳定可控。", + "next_step": { + "code": "recover_agent", + "label": "优先恢复 Agent", + "summary": "先检查 domaincheck-node-agent 服务、网络连通性和控制面地址，再决定是否重新签发接入方案。", + "ui_intent": { + "kind": "managed_node_handover", + "node_code": node_code, + }, + }, + } + if token_status in {"expired", "disabled"} or agent_state in {"token_expired", "token_disabled"}: + return { + "code": "token_issue", + "label": "Token 失效", + "summary": "最近一枚 Node Agent Token 已不可继续使用，需要重新生成接入方案。", + "next_step": { + "code": "reissue_bootstrap_plan", + "label": "重新签发接入方案", + "summary": "重新生成 bootstrap env 和一键落地片段，再到目标节点执行。", + "ui_intent": { + "kind": "managed_node_handover", + "node_code": node_code, + }, + }, + } + if agent_state == "pending_bootstrap": + return { + "code": "pending_bootstrap", + "label": "待执行接入", + "summary": "控制面已经签发了有效 Token，当前卡在目标节点尚未执行 bootstrap 片段。", + "next_step": { + "code": "execute_bootstrap_plan", + "label": "去目标机执行接入片段", + "summary": "把 env、脚本或一键落地片段拷到目标机执行，然后观察 register / heartbeat 是否建立。", + "ui_intent": { + "kind": "managed_node_handover", + "node_code": node_code, + }, + }, + } + if remote_access_state == "ssh_ready" or has_ssh_access: + return { + "code": "ssh_ready", + "label": remote_access_label or "SSH 已备好", + "summary": "SSH 入口已经可用，可以从海外控制面统一生成并下发 Node Agent 接入方案。", + "next_step": { + "code": "issue_bootstrap_plan", + "label": "生成接入方案", + "summary": "下一步就是签发 bootstrap env、脚本和一键落地命令，然后在目标节点启动 domaincheck-node-agent。", + "ui_intent": { + "kind": "managed_node_handover", + "node_code": node_code, + }, + }, + } + if agent_state == "runtime_only": + return { + "code": "runtime_only", + "label": agent_state_label or "仅运行态在线", + "summary": "控制面能看到 runtime 心跳，但 Node Agent 还没进入标准接管链路。", + "next_step": { + "code": "complete_ssh_profile", + "label": "补齐 SSH 入口并生成接入方案", + "summary": "先把 SSH 主机和账号补齐，再从海外控制面统一签发 Node Agent 接入方案。", + "ui_intent": { + "kind": "managed_node_edit", + "node_code": node_code, + }, + }, + } + return { + "code": "profile_incomplete", + "label": remote_access_label or agent_state_label or "待完善", + "summary": remote_access_reason or agent_state_reason or "当前接管资料还不完整，尚不能进入标准自动执行链路。", + "next_step": { + "code": "complete_ssh_profile", + "label": "补齐节点接入资料", + "summary": "至少补齐 SSH 入口，再生成标准 Node Agent 接入方案。", + "ui_intent": { + "kind": "managed_node_edit", + "node_code": node_code, + }, + }, + } + + +def _build_managed_node_handover_payload( + *, + node: dict, + control_plane_base_url: str = "", + root_dir: str = "", +) -> dict: + normalized_node = dict(node or {}) + normalized_node_code = str(normalized_node.get("node_code") or "").strip() + if not normalized_node_code: + return {} + + node = normalized_node + metadata = dict(node.get("metadata") or {}) + latest_token = dict(node.get("latest_token") or {}) + latest_job = dict(node.get("latest_job") or {}) + control_plane_value = _normalize_control_plane_base_url(control_plane_base_url) + if not control_plane_value: + control_plane_value = ( + _normalize_control_plane_base_url(str(metadata.get("ops_control_plane_base_url") or "")) + or f"http://127.0.0.1:{settings.api_port}" + ) + resolved_root_dir = ( + str(root_dir or metadata.get("root_dir") or metadata.get("deploy_root_dir") or "/opt/domaincheck").strip() + or "/opt/domaincheck" + ) + + stage = _build_managed_node_handover_stage(node) + + blocking_items: list[str] = [] + attention_items: list[str] = [] + + if not bool(node.get("is_managed", False)): + blocking_items.append("当前节点还未纳入海外控制面的托管清单。") + if bool(node.get("is_managed", False)) and not bool(node.get("is_enabled", False)): + blocking_items.append("当前节点已纳管，但被标记为停用。") + if not bool(node.get("has_ssh_access", False)): + attention_items.append("当前还没有完整 SSH 入口，无法从海外控制面直接接管该节点。") + + agent_state = str(node.get("agent_state") or "").strip() + if agent_state == "pending_bootstrap": + blocking_items.append("已签发有效 Token，但目标节点尚未执行 bootstrap 片段。") + elif agent_state == "stale": + blocking_items.append("Node Agent 心跳已过期，需先恢复服务或重新接入。") + elif agent_state in {"token_expired", "token_disabled"}: + blocking_items.append("最近一枚 Node Agent Token 已失效，需要重新签发。") + + if str(node.get("delivery_queue_state") or "").strip() == "dead_letter": + attention_items.append( + f"当前节点存在 {int(node.get('delivery_queue_dead_letter_count', 0) or 0)} 条死信回执，建议先查看 delivery queue。" + ) + elif str(node.get("delivery_queue_state") or "").strip() == "retrying": + attention_items.append( + f"当前节点还有 {int(node.get('delivery_queue_pending_count', 0) or 0)} 条待重试回执，Node Agent 会继续自动回放。" + ) + + return { + "node_code": normalized_node_code, + "summary": str(stage.get("summary") or "").strip(), + "stage": stage, + "node": node, + "control_plane_base_url": control_plane_value, + "root_dir": resolved_root_dir, + "readiness": { + "is_managed": bool(node.get("is_managed", False)), + "is_enabled": bool(node.get("is_enabled", False)), + "has_ssh_access": bool(node.get("has_ssh_access", False)), + "is_agent_online": bool(node.get("is_agent_online", False)), + "is_remote_access_ready": bool(node.get("is_remote_access_ready", False)), + }, + "ssh_profile": { + "host": str(node.get("ssh_host") or "").strip(), + "port": int(node.get("ssh_port", 22) or 22), + "user": str(node.get("ssh_user") or "").strip(), + "auth_mode": str(node.get("auth_mode") or "").strip(), + "label": str(node.get("ssh_access_label") or "").strip(), + "reason": str(node.get("ssh_access_reason") or "").strip(), + "ssh_command_hint": ( + f"ssh -p {int(node.get('ssh_port', 22) or 22)} " + f"{str(node.get('ssh_user') or '').strip()}@{str(node.get('ssh_host') or '').strip()}" + if str(node.get("ssh_host") or "").strip() and str(node.get("ssh_user") or "").strip() + else "" + ), + }, + "cluster": { + "status": str(node.get("cluster_status") or "").strip(), + "current_load": int(node.get("cluster_current_load", 0) or 0), + "last_heartbeat_at": str(node.get("cluster_last_heartbeat_at") or "").strip(), + "hostname": str(node.get("cluster_hostname") or "").strip(), + "ip": str(node.get("cluster_ip") or "").strip(), + "is_effective_worker": bool(node.get("cluster_is_effective_worker", False)), + }, + "participation": { + "state": str(node.get("participation_state") or "").strip(), + "label": str(node.get("participation_label") or "").strip(), + "reason": str(node.get("participation_reason") or "").strip(), + "is_current_participant": bool(node.get("is_current_participant", False)), + "is_dispatch_active": bool(node.get("is_dispatch_active", False)), + "items_claimed": int(node.get("items_claimed", 0) or 0), + "items_running": int(node.get("items_running", 0) or 0), + "items_completed": int(node.get("items_completed", 0) or 0), + "processed_recent": int(node.get("processed_recent", 0) or 0), + "processed_per_minute": float(node.get("processed_per_minute", 0) or 0), + }, + "agent_token": latest_token, + "latest_job": latest_job, + "delivery_queue": { + "state": str(node.get("delivery_queue_state") or "").strip(), + "label": str(node.get("delivery_queue_label") or "").strip(), + "reason": str(node.get("delivery_queue_reason") or "").strip(), + "pending_count": int(node.get("delivery_queue_pending_count", 0) or 0), + "dead_letter_count": int(node.get("delivery_queue_dead_letter_count", 0) or 0), + "last_flush_at": str(node.get("delivery_queue_last_flush_at") or "").strip(), + }, + "blocking_items": blocking_items, + "attention_items": attention_items, + "bootstrap_defaults": { + "node_code": normalized_node_code, + "node_region": str(node.get("region") or "mainland").strip() or "mainland", + "node_role": str(node.get("role") or "worker").strip() or "worker", + "control_plane_base_url": control_plane_value, + "root_dir": resolved_root_dir, + "expires_in_hours": 72, + "issued_by": "api", + "metadata": { + "issued_from": "ops-managed-node-handover", + "node_region": str(node.get("region") or "mainland").strip() or "mainland", + "node_role": str(node.get("role") or "worker").strip() or "worker", + "remote_access_state": str(node.get("remote_access_state") or "").strip(), + "agent_state": agent_state, + }, + }, + "endpoints": { + "detail": f"/api/v1/ops/nodes/{normalized_node_code}/handover", + "bootstrap_plan": f"/api/v1/ops/nodes/{normalized_node_code}/handover/bootstrap-plan", + }, + } + + +def get_managed_node_handover( + node_code: str, + *, + control_plane_base_url: str = "", + root_dir: str = "", + participation_payload: dict | None = None, + nodes_payload: dict | None = None, +) -> dict: + normalized_node_code = str(node_code or "").strip() + if not normalized_node_code: + return {} + + resolved_nodes_payload = dict(nodes_payload or {}) + if not resolved_nodes_payload: + resolved_nodes_payload = list_managed_nodes_with_agent_state(participation_payload=participation_payload) + nodes = list(resolved_nodes_payload.get("nodes") or []) + node = next( + (dict(item or {}) for item in nodes if str(item.get("node_code") or "").strip() == normalized_node_code), + {}, + ) + if not node: + return {} + return _build_managed_node_handover_payload( + node=node, + control_plane_base_url=control_plane_base_url, + root_dir=root_dir, + ) + + +def get_managed_node_onboarding( + node_code: str, + *, + control_plane_base_url: str = "", + root_dir: str = "", + participation_payload: dict | None = None, + nodes_payload: dict | None = None, +) -> dict: + handover = get_managed_node_handover( + node_code, + control_plane_base_url=control_plane_base_url, + root_dir=root_dir, + participation_payload=participation_payload, + nodes_payload=nodes_payload, + ) + if not handover: + return {} + + node = dict(handover.get("node") or {}) + stage = dict(handover.get("stage") or {}) + readiness = dict(handover.get("readiness") or {}) + normalized_node_code = str(handover.get("node_code") or node_code or "").strip() + stage_code = str(stage.get("code") or "").strip() + has_ssh_access = bool(readiness.get("has_ssh_access", False)) + is_agent_online = bool(readiness.get("is_agent_online", False)) + is_remote_access_ready = bool(readiness.get("is_remote_access_ready", False)) + + acceptance_execution_mode = "" + acceptance_status_code = "blocked" + acceptance_status_label = "暂不可验收" + acceptance_summary = "当前还没进入标准接管就绪状态，暂不建议直接跑 onboarding.acceptance。" + if is_agent_online: + acceptance_execution_mode = "remote-agent" + acceptance_status_code = "ready" + acceptance_status_label = "可远端验收" + acceptance_summary = "Node Agent 已在线，可以直接跑 onboarding.acceptance 做标准接管验收。" + elif has_ssh_access: + acceptance_execution_mode = "ssh" + acceptance_status_code = "ssh_only" + acceptance_status_label = "可走 SSH 验收" + acceptance_summary = "当前 Agent 还没在线，但 SSH 已备好，可先走 SSH 模式做接管验收。" + + onboarding_stage_code = stage_code or "unknown" + onboarding_stage_label = str(stage.get("label") or "").strip() or "待判断" + onboarding_summary = str(handover.get("summary") or stage.get("summary") or "").strip() + if acceptance_status_code in {"ready", "ssh_only"} and stage_code == "ready": + onboarding_stage_code = "acceptance_ready" + onboarding_stage_label = "待接管验收" + onboarding_summary = acceptance_summary + + acceptance_payload = { + "playbook_key": "onboarding.acceptance", + "target_node_codes": [normalized_node_code] if normalized_node_code else [], + "execution_mode": acceptance_execution_mode or "remote-agent", + "auto_approve": True, + } + bootstrap_payload = { + "node_code": normalized_node_code, + "node_region": str(node.get("region") or "mainland").strip() or "mainland", + "node_role": str(node.get("role") or "worker").strip() or "worker", + "control_plane_base_url": str(handover.get("control_plane_base_url") or "").strip(), + "root_dir": str(handover.get("root_dir") or "/opt/domaincheck").strip() or "/opt/domaincheck", + } + next_actions = [] + if stage_code in {"pending_bootstrap", "runtime_only", "profile_incomplete", "ssh_ready", "token_issue", "stale"}: + next_actions.append( + { + "key": "bootstrap_plan", + "label": "生成接入方案", + "summary": "先生成并执行 Node Agent bootstrap 方案，再观察 register / heartbeat 是否建立。", + "command_hint": ( + build_bash_command("drive_ops_center.sh", "node-bootstrap-plan", "http://127.0.0.1:8100", normalized_node_code) + if normalized_node_code + else "" + ), + } + ) + next_actions.append( + { + "key": "bootstrap_run", + "label": "签发接入工单", + "summary": "把本节点 bootstrap 收编进 ops playbook / ops job，形成标准可追踪的纳管回执。", + "command_hint": ( + build_bash_command("drive_ops_center.sh", "node-bootstrap-run", "http://127.0.0.1:8100", normalized_node_code) + if normalized_node_code + else "" + ), + } + ) + if acceptance_status_code in {"ready", "ssh_only"}: + next_actions.append( + { + "key": "run_acceptance", + "label": "执行接管验收", + "summary": acceptance_summary, + "command_hint": ( + "通过 driver / runbook 执行 onboarding.acceptance，确认 health / node-agent / worker 三段验收全部通过。" + ), + } + ) + + recovery_action = "noop" + recovery_label = "当前无需额外恢复动作" + recovery_summary = onboarding_summary or "当前节点暂无需要执行的接管恢复动作。" + recovery_command_hint = "" + recovery_needs_confirmation = True + recovery_window = "none" + if onboarding_stage_code == "acceptance_ready" or is_agent_online: + recovery_action = "run_acceptance" + recovery_label = "执行接管验收" + recovery_summary = acceptance_summary or "当前节点已经进入验收窗口，建议直接执行 onboarding.acceptance。" + recovery_command_hint = ( + build_bash_command("drive_ops_center.sh", "node-acceptance-run", "http://127.0.0.1:8100", normalized_node_code, "cli") + if normalized_node_code + else "" + ) + recovery_window = "acceptance" + elif onboarding_stage_code in { + "pending_bootstrap", + "runtime_only", + "profile_incomplete", + "ssh_ready", + "token_issue", + "stale", + "agent_pending", + "unknown", + "", + }: + recovery_action = "bootstrap_run" + recovery_label = "签发接入工单" + recovery_summary = "当前节点仍处于接入阶段，建议先签发 onboarding.bootstrap 并等待 register / heartbeat 建立。" + recovery_command_hint = ( + build_bash_command("drive_ops_center.sh", "node-bootstrap-run", "http://127.0.0.1:8100", normalized_node_code, "cli") + if normalized_node_code + else "" + ) + recovery_window = "bootstrap" + + return { + "node_code": normalized_node_code, + "summary": onboarding_summary, + "onboarding_stage": { + "code": onboarding_stage_code, + "label": onboarding_stage_label, + "summary": onboarding_summary, + }, + "handover": handover, + "acceptance": { + "status_code": acceptance_status_code, + "status_label": acceptance_status_label, + "summary": acceptance_summary, + "execution_mode": acceptance_execution_mode, + "is_agent_online": is_agent_online, + "has_ssh_access": has_ssh_access, + "is_remote_access_ready": is_remote_access_ready, + "playbook": acceptance_payload, + }, + "recommended_actions": next_actions, + "recovery_decision": { + "action": recovery_action, + "label": recovery_label, + "summary": recovery_summary, + "command_hint": recovery_command_hint, + "needs_confirmation": recovery_needs_confirmation, + "window": recovery_window, + "stage_code": onboarding_stage_code, + "acceptance_status_code": acceptance_status_code, + }, + "bootstrap_plan_request": bootstrap_payload, + } + + +def preview_managed_node_onboarding_bootstrap( + *, + node_code: str, + requested_by: str = "api", + control_plane_base_url: str = "", + root_dir: str = "", + participation_payload: dict | None = None, + nodes_payload: dict | None = None, +) -> tuple[bool, str, dict]: + onboarding = get_managed_node_onboarding( + node_code, + control_plane_base_url=control_plane_base_url, + root_dir=root_dir, + participation_payload=participation_payload, + nodes_payload=nodes_payload, + ) + if not onboarding: + return False, "节点不存在", {} + + normalized_node_code = str(onboarding.get("node_code") or node_code or "").strip() + bootstrap_plan_request = dict(onboarding.get("bootstrap_plan_request") or {}) + normalized_requested_by = str(requested_by or "api").strip() or "api" + + from app.services.ops_playbook_service import preview_ops_playbook + + ok, message, data = preview_ops_playbook( + { + "playbook_key": "onboarding.bootstrap", + "node_codes": [normalized_node_code] if normalized_node_code else [], + "execution_mode": "control-plane", + "auto_approve": True, + "requested_by": normalized_requested_by, + "payload": bootstrap_plan_request, + } + ) + follow_up = { + "next_stage_code": "acceptance_ready", + "summary": ( + str((((data or {}).get("playbook_run") or {}).get("focus_summary") or "")).strip() + or "接入工单预览已生成；正式执行后应继续观察 Node Agent register / heartbeat，并确认 onboarding_stage 是否进入 acceptance_ready。" + ), + "recommended_commands": [ + build_bash_command("drive_ops_center.sh", "node-recover", "http://127.0.0.1:8100", normalized_node_code), + build_bash_command("drive_ops_center.sh", "node-onboarding", "http://127.0.0.1:8100", normalized_node_code), + build_bash_command("drive_ops_center.sh", "node-handover", "http://127.0.0.1:8100", normalized_node_code), + ] if normalized_node_code else [], + "focus_ref": dict((((data or {}).get("playbook_preview") or {}).get("focus_ref") or {})), + } + return ok, message, { + "node_code": normalized_node_code, + "requested_by": normalized_requested_by, + "execution_mode": "control-plane", + "bootstrap_plan_request": bootstrap_plan_request, + "onboarding": onboarding, + "playbook_preview": data, + "follow_up": follow_up, + } + + +def execute_managed_node_onboarding_bootstrap( + *, + node_code: str, + requested_by: str = "api", + control_plane_base_url: str = "", + root_dir: str = "", + participation_payload: dict | None = None, + nodes_payload: dict | None = None, +) -> tuple[bool, str, dict]: + onboarding = get_managed_node_onboarding( + node_code, + control_plane_base_url=control_plane_base_url, + root_dir=root_dir, + participation_payload=participation_payload, + nodes_payload=nodes_payload, + ) + if not onboarding: + return False, "节点不存在", {} + + normalized_node_code = str(onboarding.get("node_code") or node_code or "").strip() + bootstrap_plan_request = dict(onboarding.get("bootstrap_plan_request") or {}) + normalized_requested_by = str(requested_by or "api").strip() or "api" + + from app.services.ops_playbook_service import execute_ops_playbook + + ok, message, data = execute_ops_playbook( + { + "playbook_key": "onboarding.bootstrap", + "node_codes": [normalized_node_code] if normalized_node_code else [], + "execution_mode": "control-plane", + "auto_approve": True, + "requested_by": normalized_requested_by, + "payload": bootstrap_plan_request, + } + ) + follow_up = { + "next_stage_code": "acceptance_ready", + "summary": ( + str((((data or {}).get("playbook_run") or {}).get("focus_summary") or "")).strip() + or "接入工单已签发；接下来应继续观察 Node Agent register / heartbeat，并确认 onboarding_stage 是否进入 acceptance_ready。" + ), + "recommended_commands": [ + build_bash_command("drive_ops_center.sh", "node-recover", "http://127.0.0.1:8100", normalized_node_code), + build_bash_command("drive_ops_center.sh", "node-onboarding", "http://127.0.0.1:8100", normalized_node_code), + build_bash_command("drive_ops_center.sh", "node-handover", "http://127.0.0.1:8100", normalized_node_code), + ] if normalized_node_code else [], + "focus_ref": dict((((data or {}).get("playbook_run") or {}).get("focus_ref") or {})), + } + return ok, message, { + "node_code": normalized_node_code, + "requested_by": normalized_requested_by, + "execution_mode": "control-plane", + "bootstrap_plan_request": bootstrap_plan_request, + "onboarding": onboarding, + "playbook_run": data, + "follow_up": follow_up, + } + + +def preview_managed_node_onboarding_acceptance( + *, + node_code: str, + requested_by: str = "api", + control_plane_base_url: str = "", + root_dir: str = "", + participation_payload: dict | None = None, + nodes_payload: dict | None = None, +) -> tuple[bool, str, dict]: + onboarding = get_managed_node_onboarding( + node_code, + control_plane_base_url=control_plane_base_url, + root_dir=root_dir, + participation_payload=participation_payload, + nodes_payload=nodes_payload, + ) + if not onboarding: + return False, "节点不存在", {} + + acceptance = dict(onboarding.get("acceptance") or {}) + acceptance_status_code = str(acceptance.get("status_code") or "").strip() + if acceptance_status_code not in {"ready", "ssh_only"}: + return False, ( + str(acceptance.get("summary") or "当前节点暂不可执行接管验收").strip() or "当前节点暂不可执行接管验收" + ), {"onboarding": onboarding} + + from app.services.ops_playbook_service import preview_ops_playbook + + normalized_node_code = str(onboarding.get("node_code") or node_code or "").strip() + normalized_requested_by = str(requested_by or "api").strip() or "api" + execution_mode = str(acceptance.get("execution_mode") or "remote-agent").strip() or "remote-agent" + ok, message, data = preview_ops_playbook( + { + "playbook_key": "onboarding.acceptance", + "node_codes": [normalized_node_code] if normalized_node_code else [], + "execution_mode": execution_mode, + "auto_approve": True, + "requested_by": normalized_requested_by, + } + ) + follow_up = { + "next_stage_code": "ready", + "summary": ( + str((((data or {}).get("playbook_run") or {}).get("focus_summary") or "")).strip() + or "接管验收预览已生成；正式执行后应继续确认 health / node-agent / worker 三段验收全部通过。" + ), + "recommended_commands": [ + build_bash_command("drive_ops_center.sh", "node-onboarding", "http://127.0.0.1:8100", normalized_node_code), + build_bash_command("drive_ops_center.sh", "doctor", "http://127.0.0.1:8100"), + ] if normalized_node_code else [], + "focus_ref": dict((((data or {}).get("playbook_preview") or {}).get("focus_ref") or {})), + } + return ok, message, { + "node_code": normalized_node_code, + "requested_by": normalized_requested_by, + "execution_mode": execution_mode, + "onboarding": onboarding, + "acceptance": acceptance, + "playbook_preview": data, + "follow_up": follow_up, + } + + +def execute_managed_node_onboarding_acceptance( + *, + node_code: str, + requested_by: str = "api", + control_plane_base_url: str = "", + root_dir: str = "", + participation_payload: dict | None = None, + nodes_payload: dict | None = None, +) -> tuple[bool, str, dict]: + onboarding = get_managed_node_onboarding( + node_code, + control_plane_base_url=control_plane_base_url, + root_dir=root_dir, + participation_payload=participation_payload, + nodes_payload=nodes_payload, + ) + if not onboarding: + return False, "节点不存在", {} + + acceptance = dict(onboarding.get("acceptance") or {}) + acceptance_status_code = str(acceptance.get("status_code") or "").strip() + if acceptance_status_code not in {"ready", "ssh_only"}: + return False, ( + str(acceptance.get("summary") or "当前节点暂不可执行接管验收").strip() or "当前节点暂不可执行接管验收" + ), {"onboarding": onboarding} + + from app.services.ops_playbook_service import execute_ops_playbook + + normalized_node_code = str(onboarding.get("node_code") or node_code or "").strip() + normalized_requested_by = str(requested_by or "api").strip() or "api" + execution_mode = str(acceptance.get("execution_mode") or "remote-agent").strip() or "remote-agent" + ok, message, data = execute_ops_playbook( + { + "playbook_key": "onboarding.acceptance", + "node_codes": [normalized_node_code] if normalized_node_code else [], + "execution_mode": execution_mode, + "auto_approve": True, + "requested_by": normalized_requested_by, + } + ) + follow_up = { + "next_stage_code": "ready", + "summary": ( + str((((data or {}).get("playbook_run") or {}).get("focus_summary") or "")).strip() + or "接管验收已发起；接下来应继续确认 health / node-agent / worker 三段验收全部通过。" + ), + "recommended_commands": [ + build_bash_command("drive_ops_center.sh", "node-onboarding", "http://127.0.0.1:8100", normalized_node_code), + build_bash_command("drive_ops_center.sh", "doctor", "http://127.0.0.1:8100"), + ] if normalized_node_code else [], + "focus_ref": dict((((data or {}).get("playbook_run") or {}).get("focus_ref") or {})), + } + return ok, message, { + "node_code": normalized_node_code, + "requested_by": normalized_requested_by, + "execution_mode": execution_mode, + "onboarding": onboarding, + "acceptance": acceptance, + "playbook_run": data, + "follow_up": follow_up, + } + + +def preview_managed_node_onboarding_recovery( + *, + node_code: str, + requested_by: str = "api", + control_plane_base_url: str = "", + root_dir: str = "", + participation_payload: dict | None = None, + nodes_payload: dict | None = None, +) -> tuple[bool, str, dict]: + onboarding = get_managed_node_onboarding( + node_code, + control_plane_base_url=control_plane_base_url, + root_dir=root_dir, + participation_payload=participation_payload, + nodes_payload=nodes_payload, + ) + if not onboarding: + return False, "节点不存在", {} + + recovery = dict(onboarding.get("recovery_decision") or {}) + recovery_action = str(recovery.get("action") or "").strip() + if recovery_action == "bootstrap_run": + ok, message, data = preview_managed_node_onboarding_bootstrap( + node_code=str(onboarding.get("node_code") or node_code or "").strip(), + requested_by=requested_by, + control_plane_base_url=control_plane_base_url, + root_dir=root_dir, + participation_payload=participation_payload, + nodes_payload=nodes_payload, + ) + return ok, message, { + "recovery_action": recovery_action, + "recovery_decision": recovery, + "onboarding": onboarding, + "selected_preview": data, + } + if recovery_action == "run_acceptance": + ok, message, data = preview_managed_node_onboarding_acceptance( + node_code=str(onboarding.get("node_code") or node_code or "").strip(), + requested_by=requested_by, + control_plane_base_url=control_plane_base_url, + root_dir=root_dir, + participation_payload=participation_payload, + nodes_payload=nodes_payload, + ) + return ok, message, { + "recovery_action": recovery_action, + "recovery_decision": recovery, + "onboarding": onboarding, + "selected_preview": data, + } + return False, ( + str(recovery.get("summary") or "当前节点暂无可执行恢复动作").strip() or "当前节点暂无可执行恢复动作" + ), { + "recovery_action": recovery_action, + "recovery_decision": recovery, + "onboarding": onboarding, + } + + +def execute_managed_node_onboarding_recovery( + *, + node_code: str, + requested_by: str = "api", + control_plane_base_url: str = "", + root_dir: str = "", + participation_payload: dict | None = None, + nodes_payload: dict | None = None, +) -> tuple[bool, str, dict]: + onboarding = get_managed_node_onboarding( + node_code, + control_plane_base_url=control_plane_base_url, + root_dir=root_dir, + participation_payload=participation_payload, + nodes_payload=nodes_payload, + ) + if not onboarding: + return False, "节点不存在", {} + + recovery = dict(onboarding.get("recovery_decision") or {}) + recovery_action = str(recovery.get("action") or "").strip() + if recovery_action == "bootstrap_run": + ok, message, data = execute_managed_node_onboarding_bootstrap( + node_code=str(onboarding.get("node_code") or node_code or "").strip(), + requested_by=requested_by, + control_plane_base_url=control_plane_base_url, + root_dir=root_dir, + participation_payload=participation_payload, + nodes_payload=nodes_payload, + ) + return ok, message, { + "recovery_action": recovery_action, + "recovery_decision": recovery, + "onboarding": onboarding, + "selected_run": data, + } + if recovery_action == "run_acceptance": + ok, message, data = execute_managed_node_onboarding_acceptance( + node_code=str(onboarding.get("node_code") or node_code or "").strip(), + requested_by=requested_by, + control_plane_base_url=control_plane_base_url, + root_dir=root_dir, + participation_payload=participation_payload, + nodes_payload=nodes_payload, + ) + return ok, message, { + "recovery_action": recovery_action, + "recovery_decision": recovery, + "onboarding": onboarding, + "selected_run": data, + } + return False, ( + str(recovery.get("summary") or "当前节点暂无可执行恢复动作").strip() or "当前节点暂无可执行恢复动作" + ), { + "recovery_action": recovery_action, + "recovery_decision": recovery, + "onboarding": onboarding, + } + + +def build_managed_node_handover_bootstrap_plan( + *, + node_code: str, + issued_by: str = "api", + expires_in_hours: int = 72, + control_plane_base_url: str = "", + root_dir: str = "", + metadata: dict | None = None, +) -> tuple[bool, str, dict]: + handover = get_managed_node_handover( + node_code, + control_plane_base_url=control_plane_base_url, + root_dir=root_dir, + ) + if not handover: + return False, "节点不存在", {} + + node = dict(handover.get("node") or {}) + bootstrap_defaults = dict(handover.get("bootstrap_defaults") or {}) + normalized_node_code = str(node.get("node_code") or node_code or "").strip() + ok, message, data = build_node_agent_bootstrap_plan( + node_code=normalized_node_code, + node_region=str( + bootstrap_defaults.get("node_region") or node.get("region") or "mainland" + ).strip() + or "mainland", + node_role=str(bootstrap_defaults.get("node_role") or node.get("role") or "worker").strip() or "worker", + issued_by=str(issued_by or "api").strip() or "api", + expires_in_hours=int(expires_in_hours or bootstrap_defaults.get("expires_in_hours") or 72), + control_plane_base_url=str( + control_plane_base_url or bootstrap_defaults.get("control_plane_base_url") or handover.get("control_plane_base_url") or "" + ).strip(), + root_dir=str(root_dir or bootstrap_defaults.get("root_dir") or handover.get("root_dir") or "/opt/domaincheck").strip() + or "/opt/domaincheck", + metadata={ + **dict(bootstrap_defaults.get("metadata") or {}), + **dict(metadata or {}), + "issued_from": str((metadata or {}).get("issued_from") or "ops-managed-node-handover").strip() + or "ops-managed-node-handover", + }, + ) + if not ok: + return False, message, data + + refreshed_handover = get_managed_node_handover( + normalized_node_code, + control_plane_base_url=str(handover.get("control_plane_base_url") or "").strip(), + root_dir=str(handover.get("root_dir") or "").strip(), + ) + return True, message, { + **data, + "handover": refreshed_handover, + "requested_via": "managed-node-handover", + } + + +def _agent_job_type(job: dict) -> str: + normalized_job = dict(job or {}) + metadata = dict(normalized_job.get("metadata") or {}) + payload = dict(normalized_job.get("payload") or {}) + explicit_job_type = str(metadata.get("job_type") or payload.get("job_type") or "").strip() + if explicit_job_type: + return explicit_job_type + action = str(normalized_job.get("action") or "").strip() + rollout_id = int(normalized_job.get("rollout_id") or 0) + if rollout_id > 0: + return "release_rollout" + if action == "deploy.release": + return "release_deploy" + if action in {"health.snapshot", "logs.collect", "diagnostics.collect"}: + return "inspection" + return "ops_action" + + +def _agent_release_context(job: dict) -> dict: + normalized_job = dict(job or {}) + metadata = dict(normalized_job.get("metadata") or {}) + payload = dict(normalized_job.get("payload") or {}) + return { + "release_id": int(payload.get("release_id") or metadata.get("release_id") or 0), + "release_version": str(payload.get("release_version") or metadata.get("release_version") or "").strip(), + "rollout_id": int( + normalized_job.get("rollout_id") + or payload.get("rollout_id") + or metadata.get("rollout_id") + or 0 + ), + "rollout_code": str(payload.get("rollout_code") or metadata.get("rollout_code") or "").strip(), + } + + +def _agent_job_envelope(job: dict) -> dict: + normalized_job = dict(job or {}) + steps = list(normalized_job.get("steps") or []) + first_step = dict(steps[0] or {}) if steps else {} + metadata = dict(normalized_job.get("metadata") or {}) + focus_ref = dict(normalized_job.get("focus_ref") or {}) + if not focus_ref: + focus_ref = { + "kind": "ops_job", + "job_id": int(normalized_job.get("id") or 0), + "job_code": str(normalized_job.get("job_code") or "").strip(), + "action": str(normalized_job.get("action") or "").strip(), + "target_node_code": str(normalized_job.get("target_node_code") or "").strip(), + } + step_key = str(first_step.get("step_key") or metadata.get("step_key") or "dispatch").strip() or "dispatch" + step_title = ( + str( + first_step.get("title") + or metadata.get("step_title") + or normalized_job.get("summary") + or normalized_job.get("action") + or "" + ).strip() + or step_key + ) + envelope = { + **normalized_job, + "protocol_version": "ops-agent/v1", + "envelope_type": "agent_job", + "job_id": int(normalized_job.get("id") or 0), + "job_type": _agent_job_type(normalized_job), + "step_key": step_key, + "step_title": step_title, + "policy": dict(normalized_job.get("policy") or {}), + "release_context": _agent_release_context(normalized_job), + "focus_ref": focus_ref, + "job_ref": { + "job_id": int(normalized_job.get("id") or 0), + "job_code": str(normalized_job.get("job_code") or "").strip(), + "action": str(normalized_job.get("action") or "").strip(), + "target_node_code": str(normalized_job.get("target_node_code") or "").strip(), + }, + "step_ref": { + "step_id": int(first_step.get("id") or 0), + "step_key": step_key, + "step_title": step_title, + }, + } + return envelope + + +def _completion_summary(job: dict, *, client_request_id: str = "", deduplicated: bool = False) -> dict: + normalized_job = dict(job or {}) + result = dict(normalized_job.get("result") or {}) + return { + "job_id": int(normalized_job.get("id") or 0), + "job_code": str(normalized_job.get("job_code") or "").strip(), + "status": str(normalized_job.get("status") or "").strip(), + "status_label": str(normalized_job.get("status_label") or "").strip(), + "deduplicated": bool(deduplicated), + "client_request_id": str(client_request_id or "").strip(), + "result_summary_text": str(result.get("summary_text") or result.get("summary") or "").strip(), + "focus_ref": dict(normalized_job.get("focus_ref") or {}), + } + + +def _slim_delivery_queue_head_record(node: dict, *, state: str) -> dict: + normalized_state = str(state or "").strip() + if normalized_state not in {"pending", "dead_letter"}: + return {} + + if normalized_state == "pending": + created_at = str(node.get("delivery_queue_oldest_pending_at") or "").strip() + request_id = str(node.get("delivery_queue_oldest_pending_request_id") or "").strip() + request_kind = str(node.get("delivery_queue_oldest_pending_kind") or "").strip() + else: + created_at = str(node.get("delivery_queue_oldest_dead_letter_at") or "").strip() + request_id = str(node.get("delivery_queue_oldest_dead_letter_request_id") or "").strip() + request_kind = str(node.get("delivery_queue_oldest_dead_letter_kind") or "").strip() + + if not any((created_at, request_id, request_kind)): + return {} + + node_code = str(node.get("node_code") or "").strip() + return { + "record_key": f"{node_code}:{normalized_state}:{request_id or request_kind or created_at or 'head'}", + "record_id": request_id, + "node_code": node_code, + "state": normalized_state, + "label": "待重试头部记录" if normalized_state == "pending" else "死信头部记录", + "request_kind": request_kind, + "client_request_id": request_id, + "created_at": created_at if normalized_state == "pending" else "", + "dead_letter_at": created_at if normalized_state == "dead_letter" else "", + "detail_code": "", + "error_message": "", + "record_source": "agent-heartbeat-snapshot", + "record_visibility": "head_only", + "payload_visibility": "unavailable", + "response_visibility": "unavailable", + } + + +def get_managed_node_delivery_queue(node_code: str) -> dict: + normalized_node_code = str(node_code or "").strip() + if not normalized_node_code: + return {} + + payload = list_managed_nodes_with_agent_state() + node = next( + ( + dict(item or {}) + for item in list(payload.get("nodes") or []) + if str(item.get("node_code") or "").strip() == normalized_node_code + ), + {}, + ) + if not node: + return {} + + pending_head = _slim_delivery_queue_head_record(node, state="pending") + dead_letter_head = _slim_delivery_queue_head_record(node, state="dead_letter") + record_visibility = "head_only" + capabilities = { + "can_flush": True, + "can_replay": True, + "can_discard": True, + } + + return { + "node_code": normalized_node_code, + "node": { + "node_code": normalized_node_code, + "title": str(node.get("title") or "").strip(), + "region": str(node.get("region") or "").strip(), + "role": str(node.get("role") or "").strip(), + "is_managed": bool(node.get("is_managed", False)), + "is_enabled": bool(node.get("is_enabled", False)), + "is_agent_online": bool(node.get("is_agent_online", False)), + "agent_state": str(node.get("agent_state") or "").strip(), + "agent_state_label": str(node.get("agent_state_label") or "").strip(), + "remote_access_state": str(node.get("remote_access_state") or "").strip(), + "remote_access_label": str(node.get("remote_access_label") or "").strip(), + }, + "summary": { + "state": str(node.get("delivery_queue_state") or "").strip(), + "label": str(node.get("delivery_queue_label") or "").strip(), + "reason": str(node.get("delivery_queue_reason") or "").strip(), + "pending_count": int(node.get("delivery_queue_pending_count", 0) or 0), + "dead_letter_count": int(node.get("delivery_queue_dead_letter_count", 0) or 0), + "last_flush_at": str(node.get("delivery_queue_last_flush_at") or "").strip(), + "oldest_pending_at": str(node.get("delivery_queue_oldest_pending_at") or "").strip(), + "oldest_dead_letter_at": str(node.get("delivery_queue_oldest_dead_letter_at") or "").strip(), + }, + "record_visibility": record_visibility, + "capabilities": capabilities, + "records_capability": { + "record_source": "agent-heartbeat-snapshot", + "record_visibility": record_visibility, + "full_records_available": False, + "available_actions": [ + { + "key": "delivery.queue.flush", + "label": "立即冲刷", + "endpoint": f"/api/v1/ops/nodes/{normalized_node_code}/delivery-queue/flush", + "record_scope": "pending", + "execution_mode": "remote-agent", + }, + { + "key": "delivery.queue.replay", + "label": "重放死信", + "endpoint": f"/api/v1/ops/nodes/{normalized_node_code}/delivery-queue/replay", + "record_scope": "dead_letter", + "execution_mode": "remote-agent", + }, + { + "key": "delivery.queue.discard", + "label": "丢弃死信", + "endpoint": f"/api/v1/ops/nodes/{normalized_node_code}/delivery-queue/records/{{record_id}}/discard", + "record_scope": "dead_letter", + "execution_mode": "remote-agent", + }, + ], + "next_step": "当前控制面仍以头部记录视角展示队列；已支持通过正式 Ops Job 触发冲刷、重放和丢弃动作。", + }, + "head_records": { + "pending": pending_head, + "dead_letter": dead_letter_head, + }, + } + + +def list_managed_node_delivery_queue_records( + node_code: str, + *, + state: str = "", + limit: int = 50, +) -> dict: + normalized_node_code = str(node_code or "").strip() + normalized_state = str(state or "").strip() + safe_limit = min(max(int(limit or 50), 1), 200) + queue_payload = get_managed_node_delivery_queue(normalized_node_code) + if not queue_payload: + return {} + + candidate_records = [ + record + for record in ( + dict((queue_payload.get("head_records") or {}).get("pending") or {}), + dict((queue_payload.get("head_records") or {}).get("dead_letter") or {}), + ) + if record + ] + if normalized_state: + candidate_records = [ + record for record in candidate_records if str(record.get("state") or "").strip() == normalized_state + ] + records = candidate_records[:safe_limit] + + available_state_counts: dict[str, int] = {} + for record in candidate_records: + record_state = str(record.get("state") or "").strip() or "unknown" + available_state_counts[record_state] = int(available_state_counts.get(record_state, 0) or 0) + 1 + + return { + "node_code": normalized_node_code, + "records": records, + "summary": { + "total": len(records), + "returned_records": len(records), + "available_total": len(candidate_records), + "available_state_counts": available_state_counts, + "record_visibility": "head_only", + "filters": { + "state": normalized_state, + "limit": safe_limit, + }, + }, + "records_capability": dict(queue_payload.get("records_capability") or {}), + "record_visibility": str((queue_payload.get("record_visibility") or "head_only")).strip() or "head_only", + "capabilities": dict(queue_payload.get("capabilities") or {}), + } + + +def _delivery_queue_visible_record_id(record: dict | None = None) -> str: + normalized_record = dict(record or {}) + return ( + str(normalized_record.get("record_id") or "").strip() + or str(normalized_record.get("client_request_id") or "").strip() + or str(normalized_record.get("record_key") or "").strip() + ) + + +def _find_visible_delivery_queue_record(queue_payload: dict, record_id: str) -> dict: + normalized_record_id = str(record_id or "").strip() + if not normalized_record_id: + return {} + for state in ("pending", "dead_letter"): + record = dict(((queue_payload.get("head_records") or {}).get(state) or {})) + if record and _delivery_queue_visible_record_id(record) == normalized_record_id: + return record + return {} + + +def _build_delivery_queue_selector_from_template_payload( + template_key: str, + normalized_payload: dict, + *, + force_state: str = "", +) -> dict: + selector: dict[str, object] = {} + if str(force_state or "").strip(): + selector["state"] = str(force_state or "").strip() + elif template_key in {"delivery.queue.replay", "delivery.queue.discard"}: + selector["state"] = "dead_letter" + + record_id = str(normalized_payload.get("record_id") or "").strip() + request_kind = str(normalized_payload.get("request_kind") or "").strip() + detail_code = str(normalized_payload.get("detail_code") or "").strip() + if record_id: + selector["record_id"] = record_id + if request_kind: + selector["request_kind"] = request_kind + if detail_code: + selector["detail_code"] = detail_code + return selector + + +def request_managed_node_delivery_queue_action( + node_code: str, + action_key: str, + *, + payload: dict | None = None, + record_id: str = "", +) -> tuple[bool, str, dict]: + normalized_node_code = str(node_code or "").strip() + normalized_action_key = str(action_key or "").strip() + raw_payload = dict(payload or {}) + if not normalized_node_code: + return False, "node_code 不能为空", {} + if normalized_action_key not in {"delivery.queue.flush", "delivery.queue.replay", "delivery.queue.discard"}: + return False, "当前不支持该 delivery queue 动作", {"action_key": normalized_action_key} + + queue_payload = get_managed_node_delivery_queue(normalized_node_code) + if not queue_payload: + return False, "节点不存在", {} + + template = get_ops_action_template(normalized_action_key) + if not template: + return False, "delivery queue 动作模板不存在", {"action_key": normalized_action_key} + + effective_payload = dict(raw_payload) + if str(record_id or "").strip(): + effective_payload["record_id"] = str(record_id or "").strip() + normalize_ok, normalize_message, normalized_action_payload = build_ops_template_payload( + normalized_action_key, + effective_payload, + ) + if not normalize_ok: + return False, normalize_message, {"action_key": normalized_action_key} + + visible_record = {} + if str(record_id or "").strip(): + visible_record = _find_visible_delivery_queue_record(queue_payload, str(record_id or "").strip()) + if not visible_record: + return False, "当前控制面仅支持对可见头部记录执行单条动作", { + "action_key": normalized_action_key, + "node_code": normalized_node_code, + "record_id": str(record_id or "").strip(), + "record_visibility": str(((queue_payload.get("records_capability") or {}).get("record_visibility") or "")), + } + if normalized_action_key in {"delivery.queue.replay", "delivery.queue.discard"}: + if str(visible_record.get("state") or "").strip() != "dead_letter": + return False, "单条重放/丢弃当前仅支持死信头部记录", { + "action_key": normalized_action_key, + "node_code": normalized_node_code, + "record": visible_record, + } + normalized_action_payload["record_id"] = _delivery_queue_visible_record_id(visible_record) + + selector = _build_delivery_queue_selector_from_template_payload( + normalized_action_key, + normalized_action_payload, + ) + action_payload: dict[str, object] = {} + if normalized_action_key == "delivery.queue.flush": + action_payload["limit"] = int(normalized_action_payload.get("limit") or 20) + elif normalized_action_key == "delivery.queue.replay": + action_payload["selector"] = selector + action_payload["limit"] = int(normalized_action_payload.get("limit") or 20) + action_payload["flush_after_replay"] = bool(normalized_action_payload.get("flush_after_replay", True)) + if str(normalized_action_payload.get("reason") or "").strip(): + action_payload["reason"] = str(normalized_action_payload.get("reason") or "").strip() + elif normalized_action_key == "delivery.queue.discard": + action_payload["selector"] = selector + action_payload["limit"] = int(normalized_action_payload.get("limit") or 20) + action_payload["discarded_by"] = ( + str(raw_payload.get("discarded_by") or raw_payload.get("requested_by") or "web-ui").strip() or "web-ui" + ) + action_payload["reason"] = str(normalized_action_payload.get("reason") or "").strip() + + requested_by = str(raw_payload.get("requested_by") or "web-ui").strip() or "web-ui" + ok, message, data = create_ops_job( + { + "action": normalized_action_key, + "target_type": "node", + "target_node_code": normalized_node_code, + "requested_by": requested_by, + "execution_mode": "remote-agent", + "auto_approve": bool(template.get("default_auto_approve", False)), + "payload": action_payload, + "metadata": { + "delivery_queue_action": { + "action_key": normalized_action_key, + "template_key": normalized_action_key, + "selector": selector, + "record_visibility": str(((queue_payload.get("records_capability") or {}).get("record_visibility") or "")), + "requested_from": "ops-delivery-queue", + "visible_record": visible_record, + "queue_summary": dict(queue_payload.get("summary") or {}), + } + }, + } + ) + if not ok: + return False, message, data + + action_label = str(template.get("title") or normalized_action_key).strip() + return True, f"{action_label}任务已创建", { + "node_code": normalized_node_code, + "action_key": normalized_action_key, + "action_label": action_label, + "queue": { + "summary": dict(queue_payload.get("summary") or {}), + "records_capability": dict(queue_payload.get("records_capability") or {}), + }, + "selector": selector, + "visible_record": visible_record, + "job": dict(data.get("job") or {}), + "executed_immediately": bool(data.get("executed_immediately", False)), + } + + +def issue_node_agent_token( + *, + node_code: str, + issued_by: str = "api", + expires_in_hours: int = 72, + metadata: dict | None = None, +) -> tuple[bool, str, dict]: + ensure_ops_agent_schema() + normalized_node_code = str(node_code or "").strip() + if not normalized_node_code: + return False, "node_code 不能为空", {} + + token = secrets.token_urlsafe(32) + token_hash = _hash_token(token) + expires_at = datetime.now() + timedelta(hours=max(1, min(int(expires_in_hours or 72), 24 * 30))) + with get_db() as conn: + with conn.cursor() as cur: + cur.execute( + """ + INSERT INTO ops_node_tokens ( + token_hash, node_code, purpose, issued_by, is_enabled, expires_at, metadata_json, created_at, updated_at + ) VALUES (%s, %s, 'agent', %s, TRUE, %s, %s, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP) + RETURNING id, expires_at, created_at + """, + ( + token_hash, + normalized_node_code, + str(issued_by or "api").strip() or "api", + expires_at, + json.dumps(metadata or {}, ensure_ascii=False), + ), + ) + row = cur.fetchone() + conn.commit() + + return True, "节点 Agent 令牌已签发", { + "token": token, + "token_preview": f"{token[:6]}...{token[-4:]}", + "record_id": int(row[0]), + "node_code": normalized_node_code, + "expires_at": _format_time(row[1]), + "created_at": _format_time(row[2]), + } + + +def _normalize_control_plane_base_url(raw_value: str) -> str: + normalized = str(raw_value or "").strip().rstrip("/") + if normalized.endswith("/api/v1"): + normalized = normalized[: -len("/api/v1")] + return normalized + + +def _candidate_node_agent_install_paths(root_dir: str) -> list[str]: + normalized_root_dir = str(root_dir or "/opt/domaincheck").strip().rstrip("/") or "/opt/domaincheck" + return [ + f"{normalized_root_dir}/domainCheck/deploy/multi-region/bootstrap_node_agent.sh", + f"{normalized_root_dir}/domain-api/deploy/multi-region/bootstrap_node_agent.sh", + f"{normalized_root_dir}/deploy/multi-region/bootstrap_node_agent.sh", + ] + + +def _build_node_agent_install_block(root_dir: str) -> tuple[str, str]: + candidates = _candidate_node_agent_install_paths(root_dir) + preferred_path = candidates[0] + lines = [ + f"ROOT_DIR={shlex.quote(root_dir)}", + "INSTALL_SCRIPT=", + "for candidate in \\", + ] + for index, candidate in enumerate(candidates): + suffix = " \\" if index < len(candidates) - 1 else "" + lines.append(f" {shlex.quote(candidate)}{suffix}") + lines.extend( + [ + "do", + " if [[ -f \"${candidate}\" ]]; then", + " INSTALL_SCRIPT=\"${candidate}\"", + " break", + " fi", + "done", + "", + "if [[ -z \"${INSTALL_SCRIPT}\" ]]; then", + " echo \"bootstrap_node_agent.sh not found under known layouts\" >&2", + " exit 1", + "fi", + "", + "bash \"${INSTALL_SCRIPT}\" \"${ROOT_DIR}\"", + ] + ) + return preferred_path, "\n".join(lines) + + +def _build_node_agent_bootstrap_script( + *, + node_code: str, + root_dir: str, + install_script_path: str, + install_command_block: str, + env_content: str, + health_checks: list[str], +) -> tuple[str, str, str]: + script_name = f"bootstrap-node-agent-{node_code}.sh" + script_path = f"/tmp/{script_name}" + script_lines = [ + "#!/usr/bin/env bash", + "set -euo pipefail", + "", + f"ROOT_DIR={shlex.quote(root_dir)}", + f"INSTALL_SCRIPT={shlex.quote(install_script_path)}", + "ENV_FILE=/etc/default/domaincheck-node-agent", + "SERVICE_NAME=domaincheck-node-agent", + "", + "# Resolve bootstrap_node_agent.sh across supported repo layouts.", + *install_command_block.splitlines(), + "", + "cat >\"${ENV_FILE}\" <<'EOF'", + env_content, + "EOF", + "", + "systemctl restart \"${SERVICE_NAME}\"", + "systemctl status \"${SERVICE_NAME}\" --no-pager -l", + "journalctl -u \"${SERVICE_NAME}\" -n 60 --no-pager", + ] + if health_checks: + script_lines.extend( + [ + "", + "echo", + "echo '[health-checks]'", + ] + ) + for item in health_checks: + normalized_item = str(item or "").strip() + if normalized_item: + script_lines.append(normalized_item) + script_content = "\n".join(script_lines).strip() + "\n" + write_script_block = "\n".join( + [ + f"cat >{script_path} <<'EOF'", + script_content.rstrip("\n"), + "EOF", + f"chmod +x {script_path}", + ] + ) + run_script_block = "\n".join( + [ + write_script_block, + f"bash {script_path}", + ] + ) + return script_name, script_content, run_script_block + + +def build_node_agent_bootstrap_plan( + *, + node_code: str, + node_region: str = "mainland", + node_role: str = "worker", + issued_by: str = "api", + expires_in_hours: int = 72, + control_plane_base_url: str = "", + root_dir: str = "/opt/domaincheck", + metadata: dict | None = None, +) -> tuple[bool, str, dict]: + normalized_node_code = str(node_code or "").strip() + if not normalized_node_code: + return False, "node_code 不能为空", {} + + normalized_region = str(node_region or "mainland").strip() or "mainland" + normalized_role = str(node_role or "worker").strip() or "worker" + normalized_root_dir = str(root_dir or "/opt/domaincheck").strip() or "/opt/domaincheck" + normalized_control_plane_base_url = _normalize_control_plane_base_url(control_plane_base_url) + if not normalized_control_plane_base_url: + normalized_control_plane_base_url = f"http://127.0.0.1:{settings.api_port}" + + token_ok, token_message, token_data = issue_node_agent_token( + node_code=normalized_node_code, + issued_by=issued_by, + expires_in_hours=expires_in_hours, + metadata=metadata or {}, + ) + if not token_ok: + return False, token_message, {} + + env_lines = [ + f"OPS_CONTROL_PLANE_BASE_URL={normalized_control_plane_base_url}", + f"OPS_AGENT_TOKEN={token_data.get('token') or ''}", + "OPS_AGENT_POLL_INTERVAL_SECONDS=5", + "", + f"NODE_CODE={normalized_node_code}", + f"NODE_REGION={normalized_region}", + f"NODE_ROLE={normalized_role}", + "", + f"WORKER_SERVICE_NAME={settings.worker_service_name}", + f"API_SERVICE_NAME={settings.api_service_name}", + f"SYNC_AGENT_SERVICE_NAME={settings.sync_agent_service_name}", + "NODE_AGENT_SERVICE_NAME=domaincheck-node-agent", + "", + 'OPS_AGENT_CAPABILITIES=["service.start","service.stop","service.restart","service.status","runtime.start_worker","runtime.stop_worker","runtime.restart_api","runtime.start_sync_agent","runtime.stop_sync_agent","health.snapshot","logs.collect","diagnostics.collect","deploy.release"]', + "OPS_AGENT_LABELS={}", + ] + env_content = "\n".join(env_lines) + install_script_path, install_command_block = _build_node_agent_install_block(normalized_root_dir) + command_lines = [ + install_command_block, + "cat >/etc/default/domaincheck-node-agent <<'EOF'", + env_content, + "EOF", + "systemctl restart domaincheck-node-agent", + "systemctl status domaincheck-node-agent --no-pager -l", + "journalctl -u domaincheck-node-agent -n 60 --no-pager", + ] + command_block = "\n".join(command_lines) + health_checks = [ + "systemctl status domaincheck-node-agent --no-pager -l", + "journalctl -u domaincheck-node-agent -n 60 --no-pager", + f"curl -s {normalized_control_plane_base_url}/api/v1/ops/overview", + ] + bootstrap_script_name, bootstrap_script_content, bootstrap_run_script_block = _build_node_agent_bootstrap_script( + node_code=normalized_node_code, + root_dir=normalized_root_dir, + install_script_path=install_script_path, + install_command_block=install_command_block, + env_content=env_content, + health_checks=health_checks, + ) + + return True, "节点 Agent 接入方案已生成", { + **token_data, + "control_plane_base_url": normalized_control_plane_base_url, + "bootstrap_plan": { + "node_code": normalized_node_code, + "node_region": normalized_region, + "node_role": normalized_role, + "root_dir": normalized_root_dir, + "env_file": "/etc/default/domaincheck-node-agent", + "service_name": "domaincheck-node-agent", + "service_file": "/etc/systemd/system/domaincheck-node-agent.service", + "install_script_path": install_script_path, + "install_script_candidates": _candidate_node_agent_install_paths(normalized_root_dir), + "install_command_block": install_command_block, + "env_content": env_content, + "command_lines": command_lines, + "command_block": command_block, + "health_checks": health_checks, + "health_check_block": "\n".join(health_checks), + "bootstrap_script_name": bootstrap_script_name, + "bootstrap_script_path": f"/tmp/{bootstrap_script_name}", + "bootstrap_script_content": bootstrap_script_content, + "bootstrap_run_script_block": bootstrap_run_script_block, + }, + } + + +def _authenticate_agent_token(token: str, *, expected_node_code: str | None = None) -> tuple[bool, str, dict]: + ensure_ops_agent_schema() + token_hash = _hash_token(token) + with get_db() as conn: + with conn.cursor() as cur: + cur.execute( + """ + SELECT id, node_code, purpose, is_enabled, expires_at, metadata_json + FROM ops_node_tokens + WHERE token_hash = %s + LIMIT 1 + """, + (token_hash,), + ) + row = cur.fetchone() + if not row: + return _agent_error("agent token 无效", "agent_token_invalid") + record_id = int(row[0]) + node_code = str(row[1] or "") + purpose = str(row[2] or "") + is_enabled = bool(row[3]) + expires_at = row[4] + metadata = _decode_json(row[5]) + if purpose != "agent" or not is_enabled: + return _agent_error("agent token 不可用", "agent_token_unavailable") + if isinstance(expires_at, datetime): + current_time = datetime.now(expires_at.tzinfo) if expires_at.tzinfo else datetime.now() + if expires_at < current_time: + return _agent_error("agent token 已过期", "agent_token_expired") + normalized_expected = str(expected_node_code or "").strip() + if normalized_expected and normalized_expected != node_code: + return _agent_error( + "agent token 与节点不匹配", + "agent_token_node_mismatch", + {"expected_node_code": normalized_expected, "token_node_code": node_code}, + ) + cur.execute( + """ + UPDATE ops_node_tokens + SET last_used_at = CURRENT_TIMESTAMP, updated_at = CURRENT_TIMESTAMP + WHERE id = %s + """, + (record_id,), + ) + conn.commit() + return True, "ok", { + "record_id": record_id, + "node_code": node_code, + "metadata": metadata, + "expires_at": _format_time(expires_at), + } + + +def _upsert_agent_runtime(node_code: str, payload: dict) -> None: + normalized_node_code = str(node_code or "").strip() + if not normalized_node_code: + return + + region = str(payload.get("region") or "unknown").strip() or "unknown" + role = str(payload.get("role") or "worker").strip() or "worker" + title = str(payload.get("title") or normalized_node_code).strip() or normalized_node_code + metadata = payload.get("metadata") or {} + ssh_host = str(payload.get("ssh_host") or "").strip() + ssh_user = str(payload.get("ssh_user") or "").strip() + ssh_port = max(1, int(payload.get("ssh_port") or 22)) + auth_mode = str(payload.get("auth_mode") or "key").strip() or "key" + deploy_channel = str(payload.get("deploy_channel") or "stable").strip() or "stable" + + merged_metadata = { + **metadata, + "agent_version": str(payload.get("agent_version") or metadata.get("agent_version") or "").strip(), + "last_seen_at": datetime.now().isoformat(timespec="seconds"), + "capabilities": payload.get("capabilities") or metadata.get("capabilities") or [], + "labels": payload.get("labels") or metadata.get("labels") or {}, + "hostname": str(payload.get("hostname") or metadata.get("hostname") or "").strip(), + "ip": str(payload.get("ip") or metadata.get("ip") or "").strip(), + } + + with get_db() as conn: + with conn.cursor() as cur: + cur.execute( + """ + INSERT INTO ops_managed_nodes ( + node_code, region, role, title, ssh_host, ssh_port, ssh_user, auth_mode, deploy_channel, is_enabled, metadata_json, created_at, last_seen_at, updated_at + ) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, TRUE, %s, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP) + ON CONFLICT (node_code) DO UPDATE SET + region = EXCLUDED.region, + role = EXCLUDED.role, + title = EXCLUDED.title, + ssh_host = CASE WHEN EXCLUDED.ssh_host <> '' THEN EXCLUDED.ssh_host ELSE ops_managed_nodes.ssh_host END, + ssh_port = EXCLUDED.ssh_port, + ssh_user = CASE WHEN EXCLUDED.ssh_user <> '' THEN EXCLUDED.ssh_user ELSE ops_managed_nodes.ssh_user END, + auth_mode = EXCLUDED.auth_mode, + deploy_channel = EXCLUDED.deploy_channel, + metadata_json = EXCLUDED.metadata_json, + last_seen_at = CURRENT_TIMESTAMP, + updated_at = CURRENT_TIMESTAMP + """, + ( + normalized_node_code, + region, + role, + title, + ssh_host, + ssh_port, + ssh_user, + auth_mode, + deploy_channel, + json.dumps(merged_metadata, ensure_ascii=False), + ), + ) + conn.commit() + + +def agent_register(payload: dict, *, token: str) -> tuple[bool, str, dict]: + node_code = str(payload.get("node_code") or "").strip() + if not node_code: + return _agent_error("node_code 不能为空", "agent_node_code_required") + ok, message, auth = _authenticate_agent_token(token, expected_node_code=node_code) + if not ok: + return False, message, auth + _upsert_agent_runtime(node_code, payload) + return True, "Agent 注册成功", { + "node_code": node_code, + "expires_at": auth.get("expires_at", ""), + "capabilities": payload.get("capabilities") or [], + } + + +def agent_heartbeat(payload: dict, *, token: str) -> tuple[bool, str, dict]: + node_code = str(payload.get("node_code") or "").strip() + if not node_code: + return _agent_error("node_code 不能为空", "agent_node_code_required") + ok, message, auth = _authenticate_agent_token(token, expected_node_code=node_code) + if not ok: + return False, message, auth + _upsert_agent_runtime(node_code, payload) + return True, "heartbeat ok", { + "node_code": node_code, + "server_time": _format_time(datetime.now()), + "expires_at": auth.get("expires_at", ""), + } + + +def agent_pull_jobs(payload: dict, *, token: str, limit: int = 1) -> tuple[bool, str, dict]: + node_code = str(payload.get("node_code") or "").strip() + if not node_code: + return _agent_error("node_code 不能为空", "agent_node_code_required") + ok, message, _auth = _authenticate_agent_token(token, expected_node_code=node_code) + if not ok: + return False, message, _auth + + safe_limit = min(max(int(limit or 1), 1), 10) + with get_db() as conn: + conn.autocommit = False + with conn.cursor() as cur: + cur.execute( + """ + SELECT id + FROM ops_jobs + WHERE target_node_code = %s + AND status = 'queued' + AND execution_mode = 'remote-agent' + ORDER BY created_at ASC, id ASC + LIMIT %s + FOR UPDATE SKIP LOCKED + """, + (node_code, safe_limit), + ) + rows = cur.fetchall() + job_ids = [int(row[0]) for row in rows] + jobs: list[dict] = [] + for job_id in job_ids: + cur.execute( + """ + UPDATE ops_jobs + SET status = 'dispatching', started_at = COALESCE(started_at, CURRENT_TIMESTAMP), dispatched_at = CURRENT_TIMESTAMP, updated_at = CURRENT_TIMESTAMP + WHERE id = %s + """, + (job_id,), + ) + cur.execute( + """ + UPDATE ops_job_steps + SET status = 'dispatching', started_at = COALESCE(started_at, CURRENT_TIMESTAMP), updated_at = CURRENT_TIMESTAMP + WHERE job_id = %s + """, + (job_id,), + ) + append_ops_job_event( + job_id=job_id, + node_code=node_code, + event_type="agent_dispatched", + message=f"任务已派发给节点 {node_code}", + payload={"node_code": node_code}, + ) + jobs.append(_agent_job_envelope(get_ops_job(job_id))) + conn.commit() + return True, "ok", { + "jobs": jobs, + "count": len(jobs), + "node_code": node_code, + "limit": safe_limit, + "protocol_version": "ops-agent/v1", + "envelope_type": "agent_job", + } + + +def agent_mark_job_started(job_id: int, payload: dict, *, token: str) -> tuple[bool, str, dict]: + node_code = str(payload.get("node_code") or "").strip() + if not node_code: + return _agent_error("node_code 不能为空", "agent_node_code_required") + ok, message, _auth = _authenticate_agent_token(token, expected_node_code=node_code) + if not ok: + return False, message, _auth + + with get_db() as conn: + conn.autocommit = False + with conn.cursor() as cur: + cur.execute( + """ + UPDATE ops_jobs + SET status = 'running', started_at = COALESCE(started_at, CURRENT_TIMESTAMP), updated_at = CURRENT_TIMESTAMP + WHERE id = %s AND target_node_code = %s + RETURNING id + """, + (int(job_id), node_code), + ) + row = cur.fetchone() + if not row: + conn.rollback() + return _agent_error( + "任务不存在或不属于当前节点", + "ops_job_not_owned_by_agent", + {"job_id": int(job_id), "node_code": node_code}, + ) + cur.execute( + """ + UPDATE ops_job_steps + SET status = 'running', started_at = COALESCE(started_at, CURRENT_TIMESTAMP), updated_at = CURRENT_TIMESTAMP + WHERE job_id = %s + RETURNING id + """, + (int(job_id),), + ) + step_rows = cur.fetchall() + step_ids = [int(item[0]) for item in step_rows] + append_ops_job_event( + job_id=int(job_id), + node_code=node_code, + event_type="agent_started", + message=f"节点 {node_code} 已开始执行任务", + payload={"step_ids": step_ids}, + ) + conn.commit() + job = get_ops_job(int(job_id)) + return True, "任务已标记为运行中", { + "job": job, + "accepted_job": _agent_job_envelope(job), + "focus_ref": dict(job.get("focus_ref") or {}), + } + + +def agent_complete_job(job_id: int, payload: dict, *, token: str) -> tuple[bool, str, dict]: + node_code = str(payload.get("node_code") or "").strip() + if not node_code: + return _agent_error("node_code 不能为空", "agent_node_code_required") + ok, message, _auth = _authenticate_agent_token(token, expected_node_code=node_code) + if not ok: + return False, message, _auth + + job_status = str(payload.get("status") or "").strip().lower() + if job_status not in {"success", "failed", "partially_succeeded"}: + return _agent_error( + "status 必须是 success / failed / partially_succeeded", + "ops_job_invalid_status", + {"allowed_statuses": ["success", "failed", "partially_succeeded"]}, + ) + + stdout_text = str(payload.get("stdout") or "") + stderr_text = str(payload.get("stderr") or "") + result = dict(payload.get("result") or {}) + error_message = str(payload.get("error_message") or "").strip() + client_request_id = str(payload.get("client_request_id") or "").strip()[:128] + duration_ms = max(0, int(payload.get("duration_ms") or 0)) + summary_text = str(payload.get("summary_text") or "").strip() + focus_ref = payload.get("focus_ref") if isinstance(payload.get("focus_ref"), dict) else {} + if duration_ms and "duration_ms" not in result: + result["duration_ms"] = duration_ms + if summary_text and "summary_text" not in result and "summary" not in result: + result["summary_text"] = summary_text + if focus_ref and "focus_ref" not in result: + result["focus_ref"] = focus_ref + event_level = "info" if job_status == "success" else ("warning" if job_status == "partially_succeeded" else "error") + + with get_db() as conn: + conn.autocommit = False + with conn.cursor() as cur: + cur.execute( + """ + SELECT id, COALESCE(last_agent_complete_request_id, '') + FROM ops_jobs + WHERE id = %s AND target_node_code = %s + LIMIT 1 + """, + (int(job_id), node_code), + ) + job_row = cur.fetchone() + if not job_row: + conn.rollback() + return _agent_error( + "任务不存在或不属于当前节点", + "ops_job_not_owned_by_agent", + {"job_id": int(job_id), "node_code": node_code}, + ) + current_request_id = str(job_row[1] or "") + if client_request_id and current_request_id == client_request_id: + conn.rollback() + deduplicated_job = get_ops_job(int(job_id)) + return True, "任务结果已幂等回写", { + "job": deduplicated_job, + "deduplicated": True, + "client_request_id": client_request_id, + "completion_summary": _completion_summary( + deduplicated_job, + client_request_id=client_request_id, + deduplicated=True, + ), + } + + cur.execute( + """ + UPDATE ops_jobs + SET + status = %s, + result_json = %s, + error_message = %s, + last_agent_complete_request_id = CASE WHEN %s <> '' THEN %s ELSE last_agent_complete_request_id END, + finished_at = CURRENT_TIMESTAMP, + updated_at = CURRENT_TIMESTAMP + WHERE id = %s + """, + ( + job_status, + json.dumps(result, ensure_ascii=False), + error_message if job_status != "success" else "", + client_request_id, + client_request_id, + int(job_id), + ), + ) + cur.execute( + """ + UPDATE ops_job_steps + SET + status = %s, + stdout_text = %s, + stderr_text = %s, + result_json = %s, + finished_at = CURRENT_TIMESTAMP, + updated_at = CURRENT_TIMESTAMP + WHERE job_id = %s + RETURNING id + """, + ( + job_status, + stdout_text, + stderr_text, + json.dumps(result, ensure_ascii=False), + int(job_id), + ), + ) + step_rows = cur.fetchall() + step_ids = [int(item[0]) for item in step_rows] + append_ops_job_event( + job_id=int(job_id), + node_code=node_code, + client_event_id=(f"complete:{client_request_id}" if client_request_id else ""), + event_type="agent_completed", + message=f"节点 {node_code} 已完成任务，状态: {job_status}", + level=event_level, + payload={ + "step_ids": step_ids, + "result": result, + "duration_ms": duration_ms, + "summary_text": str(result.get("summary_text") or result.get("summary") or summary_text).strip(), + "focus_ref": focus_ref, + }, + ) + conn.commit() + from app.services.ops_release_service import refresh_release_rollout_for_job + + refresh_release_rollout_for_job(int(job_id)) + job = get_ops_job(int(job_id)) + return True, "任务结果已回写", { + "job": job, + "deduplicated": False, + "client_request_id": client_request_id, + "completion_summary": _completion_summary(job, client_request_id=client_request_id), + } + + +def agent_append_job_event(job_id: int, payload: dict, *, token: str) -> tuple[bool, str, dict]: + node_code = str(payload.get("node_code") or "").strip() + if not node_code: + return _agent_error("node_code 不能为空", "agent_node_code_required") + ok, message, _auth = _authenticate_agent_token(token, expected_node_code=node_code) + if not ok: + return False, message, _auth + client_event_id = str(payload.get("client_event_id") or "").strip()[:128] + event_payload = dict(payload.get("payload") or {}) + occurred_at = str(payload.get("occurred_at") or "").strip() + summary_text = str(payload.get("summary_text") or "").strip() + focus_ref = payload.get("focus_ref") if isinstance(payload.get("focus_ref"), dict) else {} + if occurred_at and "occurred_at" not in event_payload: + event_payload["occurred_at"] = occurred_at + if summary_text and "summary_text" not in event_payload: + event_payload["summary_text"] = summary_text + if focus_ref and "focus_ref" not in event_payload: + event_payload["focus_ref"] = focus_ref + event_id = append_ops_job_event( + job_id=int(job_id), + step_id=int(payload.get("step_id") or 0) or None, + node_code=node_code, + client_event_id=client_event_id, + event_type=str(payload.get("event_type") or "agent_event").strip() or "agent_event", + level=str(payload.get("level") or "info").strip() or "info", + message=str(payload.get("message") or "").strip()[:2000] or "agent event", + payload=event_payload, + ) + return True, "事件已写入", { + "event_id": event_id, + "client_event_id": client_event_id, + "event": get_ops_job_event(event_id), + } diff --git a/domain-api/app/services/ops_command_service.py b/domain-api/app/services/ops_command_service.py new file mode 100644 index 0000000..b8580a0 --- /dev/null +++ b/domain-api/app/services/ops_command_service.py @@ -0,0 +1,17 @@ +from __future__ import annotations + + +OPS_MULTI_REGION_DIR = "domain-api/deploy/multi-region" + + +def ops_script_path(script_name: str) -> str: + normalized_script_name = str(script_name or "").strip().lstrip("/") + if not normalized_script_name: + return OPS_MULTI_REGION_DIR + return f"{OPS_MULTI_REGION_DIR}/{normalized_script_name}" + + +def build_bash_command(script_name: str, *args: object) -> str: + command_parts = ["bash", ops_script_path(script_name)] + command_parts.extend(str(arg or "").strip() for arg in args if str(arg or "").strip()) + return " ".join(command_parts) diff --git a/domain-api/app/services/ops_execution_capability_service.py b/domain-api/app/services/ops_execution_capability_service.py new file mode 100644 index 0000000..ecb7f14 --- /dev/null +++ b/domain-api/app/services/ops_execution_capability_service.py @@ -0,0 +1,79 @@ +from __future__ import annotations + +from app.services.ops_action_executor_core import STRUCTURED_ACTIONS + + +_LOCAL_RUNTIME_ONLY_ACTIONS = { + "runtime.push_sync", + "runtime.pull_tasks", +} + +_REMOTE_AGENT_ONLY_ACTIONS = { + "delivery.queue.flush", + "delivery.queue.replay", + "delivery.queue.discard", +} + +_LOCAL_RUNTIME_ACTIONS = set(STRUCTURED_ACTIONS) | _LOCAL_RUNTIME_ONLY_ACTIONS | {"deploy.release"} +_SSH_ACTIONS = set(STRUCTURED_ACTIONS) | {"deploy.release"} +_REMOTE_AGENT_ACTIONS = set(STRUCTURED_ACTIONS) | _REMOTE_AGENT_ONLY_ACTIONS | {"deploy.release"} +_CONTROL_PLANE_ACTIONS = { + "node.bootstrap", +} + + +def supports_local_runtime_action(action: str) -> bool: + return str(action or "").strip() in _LOCAL_RUNTIME_ACTIONS + + +def supports_ssh_action(action: str) -> bool: + return str(action or "").strip() in _SSH_ACTIONS + + +def supports_remote_agent_action(action: str) -> bool: + return str(action or "").strip() in _REMOTE_AGENT_ACTIONS + + +def supports_control_plane_action(action: str) -> bool: + return str(action or "").strip() in _CONTROL_PLANE_ACTIONS + + +def validate_action_execution_mode( + action: str, + execution_mode: str, + *, + target_node_code: str = "", + current_node_code: str = "", +) -> tuple[bool, str]: + normalized_action = str(action or "").strip() + normalized_mode = str(execution_mode or "remote-agent").strip() or "remote-agent" + normalized_target_node_code = str(target_node_code or "").strip() + normalized_current_node_code = str(current_node_code or "").strip() + + if normalized_mode == "local-runtime": + if not supports_local_runtime_action(normalized_action): + return False, f"{normalized_action} 当前不支持 local-runtime 执行方式。" + if ( + normalized_target_node_code + and normalized_current_node_code + and normalized_target_node_code != normalized_current_node_code + ): + return False, "local-runtime 仅支持当前控制面本机节点。" + return True, "" + + if normalized_mode == "control-plane": + if not supports_control_plane_action(normalized_action): + return False, f"{normalized_action} 当前不支持 control-plane 执行方式。" + return True, "" + + if normalized_mode == "ssh": + if not supports_ssh_action(normalized_action): + return False, f"{normalized_action} 当前不支持 ssh 执行方式。" + return True, "" + + if normalized_mode == "remote-agent": + if not supports_remote_agent_action(normalized_action): + return False, f"{normalized_action} 当前不支持 remote-agent 执行方式。" + return True, "" + + return False, f"未知执行方式: {normalized_mode}" diff --git a/domain-api/app/services/ops_execution_mode_service.py b/domain-api/app/services/ops_execution_mode_service.py new file mode 100644 index 0000000..a3279cc --- /dev/null +++ b/domain-api/app/services/ops_execution_mode_service.py @@ -0,0 +1,60 @@ +from __future__ import annotations + + +def normalize_execution_modes(raw_modes: object, fallback_mode: str = "remote-agent") -> list[str]: + normalized_modes: list[str] = [] + seen: set[str] = set() + fallback = str(fallback_mode or "remote-agent").strip() or "remote-agent" + for item in list(raw_modes or []): + mode = str(item or "").strip() + if not mode or mode in seen: + continue + seen.add(mode) + normalized_modes.append(mode) + if fallback and fallback not in seen: + normalized_modes.insert(0, fallback) + return normalized_modes or [fallback] + + +def execution_mode_label(execution_mode: str) -> str: + normalized_mode = str(execution_mode or "remote-agent").strip() or "remote-agent" + if normalized_mode == "ssh": + return "SSH" + if normalized_mode == "control-plane": + return "控制面" + if normalized_mode == "local-runtime": + return "本机运行时" + if normalized_mode == "remote-agent": + return "远端 Agent" + return normalized_mode + + +def build_execution_mode_options(raw_modes: object, fallback_mode: str = "remote-agent") -> list[dict]: + return [ + { + "value": mode, + "label": execution_mode_label(mode), + } + for mode in normalize_execution_modes(raw_modes, fallback_mode) + ] + + +def decorate_execution_mode_fields( + payload: dict | None = None, + *, + default_key: str = "default_execution_mode", + modes_key: str = "execution_modes", +) -> dict: + normalized_payload = dict(payload or {}) + default_mode = str(normalized_payload.get(default_key) or "remote-agent").strip() or "remote-agent" + execution_mode_options = build_execution_mode_options(normalized_payload.get(modes_key) or [], default_mode) + normalized_payload[default_key] = default_mode + normalized_payload[modes_key] = [str(item.get("value") or "").strip() for item in execution_mode_options if str(item.get("value") or "").strip()] + normalized_payload["default_execution_mode_label"] = execution_mode_label(default_mode) + normalized_payload["execution_mode_options"] = execution_mode_options + normalized_payload["execution_mode_labels"] = [ + str(item.get("label") or "").strip() + for item in execution_mode_options + if str(item.get("label") or "").strip() + ] + return normalized_payload diff --git a/domain-api/app/services/ops_job_service.py b/domain-api/app/services/ops_job_service.py new file mode 100644 index 0000000..104ae47 --- /dev/null +++ b/domain-api/app/services/ops_job_service.py @@ -0,0 +1,1682 @@ +from __future__ import annotations + +import json +import threading +from datetime import datetime +from uuid import uuid4 + +from app.core.config import settings +from app.core.db import get_db +from app.services.ops_execution_capability_service import ( + supports_control_plane_action, + supports_local_runtime_action, +) +from app.services.ops_execution_mode_service import execution_mode_label +from app.services.ops_runtime_executor_service import ( + execute_local_support_action, + execute_ssh_action, + supports_ssh_execution, +) +from app.services.runtime_control_service import runtime_action +from app.services.runtime_status_service import get_runtime_status + + +_OPS_SCHEMA_SQL = """ +CREATE TABLE IF NOT EXISTS ops_managed_nodes ( + node_code VARCHAR(64) PRIMARY KEY, + region VARCHAR(32) NOT NULL DEFAULT 'unknown', + role VARCHAR(32) NOT NULL DEFAULT 'worker', + title VARCHAR(128) NOT NULL DEFAULT '', + ssh_host VARCHAR(255) NOT NULL DEFAULT '', + ssh_port INTEGER NOT NULL DEFAULT 22, + ssh_user VARCHAR(64) NOT NULL DEFAULT '', + auth_mode VARCHAR(32) NOT NULL DEFAULT 'key', + deploy_channel VARCHAR(64) NOT NULL DEFAULT 'stable', + is_enabled BOOLEAN NOT NULL DEFAULT TRUE, + metadata_json JSONB, + created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + last_seen_at TIMESTAMP, + updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP +); + +CREATE TABLE IF NOT EXISTS ops_jobs ( + id BIGSERIAL PRIMARY KEY, + job_code VARCHAR(64) NOT NULL UNIQUE, + action VARCHAR(128) NOT NULL DEFAULT '', + target_type VARCHAR(32) NOT NULL DEFAULT 'node', + target_node_code VARCHAR(64) NOT NULL DEFAULT '', + status VARCHAR(32) NOT NULL DEFAULT 'queued', + execution_mode VARCHAR(32) NOT NULL DEFAULT 'remote-agent', + requested_by VARCHAR(64) NOT NULL DEFAULT 'api', + payload_json JSONB, + metadata_json JSONB, + result_json JSONB, + error_message TEXT NOT NULL DEFAULT '', + created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + started_at TIMESTAMP, + finished_at TIMESTAMP, + updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP +); + +CREATE INDEX IF NOT EXISTS idx_ops_jobs_status_created +ON ops_jobs(status, created_at DESC); + +CREATE TABLE IF NOT EXISTS ops_job_steps ( + id BIGSERIAL PRIMARY KEY, + job_id BIGINT NOT NULL REFERENCES ops_jobs(id) ON DELETE CASCADE, + step_key VARCHAR(64) NOT NULL DEFAULT '', + title VARCHAR(128) NOT NULL DEFAULT '', + node_code VARCHAR(64) NOT NULL DEFAULT '', + status VARCHAR(32) NOT NULL DEFAULT 'queued', + stdout_text TEXT NOT NULL DEFAULT '', + stderr_text TEXT NOT NULL DEFAULT '', + result_json JSONB, + created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + started_at TIMESTAMP, + finished_at TIMESTAMP, + updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP +); + +CREATE INDEX IF NOT EXISTS idx_ops_job_steps_job_created +ON ops_job_steps(job_id, created_at ASC); + +ALTER TABLE ops_jobs ADD COLUMN IF NOT EXISTS risk_level VARCHAR(16) NOT NULL DEFAULT 'medium'; +ALTER TABLE ops_jobs ADD COLUMN IF NOT EXISTS approval_required BOOLEAN NOT NULL DEFAULT FALSE; +ALTER TABLE ops_jobs ADD COLUMN IF NOT EXISTS approval_status VARCHAR(32) NOT NULL DEFAULT 'not_required'; +ALTER TABLE ops_jobs ADD COLUMN IF NOT EXISTS approved_by VARCHAR(64) NOT NULL DEFAULT ''; +ALTER TABLE ops_jobs ADD COLUMN IF NOT EXISTS approved_at TIMESTAMP; +ALTER TABLE ops_jobs ADD COLUMN IF NOT EXISTS blocked_reason TEXT NOT NULL DEFAULT ''; +ALTER TABLE ops_jobs ADD COLUMN IF NOT EXISTS cancellation_reason TEXT NOT NULL DEFAULT ''; +ALTER TABLE ops_jobs ADD COLUMN IF NOT EXISTS dispatched_at TIMESTAMP; +ALTER TABLE ops_jobs ADD COLUMN IF NOT EXISTS target_selector_json JSONB; +ALTER TABLE ops_jobs ADD COLUMN IF NOT EXISTS policy_json JSONB; +ALTER TABLE ops_jobs ADD COLUMN IF NOT EXISTS rollout_id BIGINT; +ALTER TABLE ops_jobs ADD COLUMN IF NOT EXISTS metadata_json JSONB; +""" + +_LOCAL_RUNTIME_ACTIONS = { + "runtime.start_worker": "start_worker", + "runtime.stop_worker": "stop_worker", + "runtime.restart_api": "restart_api", + "runtime.start_sync_agent": "start_sync_agent", + "runtime.stop_sync_agent": "stop_sync_agent", + "runtime.push_sync": "push_sync", + "runtime.pull_tasks": "pull_tasks", +} + +_OPS_SCHEMA_LOCK = threading.Lock() +_OPS_SCHEMA_READY = False +_OPS_SCHEMA_ADVISORY_LOCK_KEY = 90421801 + + +def ensure_ops_schema() -> None: + global _OPS_SCHEMA_READY + if _OPS_SCHEMA_READY: + return + with _OPS_SCHEMA_LOCK: + if _OPS_SCHEMA_READY: + return + with get_db() as conn: + conn.autocommit = False + with conn.cursor() as cur: + cur.execute("SELECT pg_advisory_xact_lock(%s)", (_OPS_SCHEMA_ADVISORY_LOCK_KEY,)) + cur.execute(_OPS_SCHEMA_SQL) + conn.commit() + _OPS_SCHEMA_READY = True + + +def _decode_json(value: object) -> dict: + if isinstance(value, dict): + return value + if value in (None, ""): + return {} + try: + return json.loads(value) + except Exception: + return {} + + +def _ts(value: object) -> str: + if isinstance(value, datetime): + return value.isoformat(sep=" ", timespec="seconds") + return "" + + +def _serialize_step_row(row: tuple) -> dict: + return { + "id": int(row[0]), + "step_key": str(row[1] or ""), + "title": str(row[2] or ""), + "node_code": str(row[3] or ""), + "status": str(row[4] or ""), + "stdout": str(row[5] or ""), + "stderr": str(row[6] or ""), + "result": _decode_json(row[7]), + "created_at": _ts(row[8]), + "started_at": _ts(row[9]), + "finished_at": _ts(row[10]), + "updated_at": _ts(row[11]), + } + + +def _job_status_label(status: object) -> str: + normalized_status = str(status or "").strip() + mapping = { + "queued": "排队中", + "awaiting_approval": "待审批", + "blocked": "阻断", + "dispatching": "派发中", + "running": "执行中", + "success": "成功", + "failed": "失败", + "cancelled": "已取消", + "partially_succeeded": "部分成功", + "completed_with_issues": "带问题完成", + } + return mapping.get(normalized_status, normalized_status or "未知") + + +def _job_approval_status_label(approval_status: object, *, approval_required: bool = False) -> str: + normalized_status = str(approval_status or "").strip() + if not normalized_status or normalized_status == "not_required": + return "待审批" if bool(approval_required) else "无需审批" + mapping = { + "approved": "已审批", + "blocked": "已阻断", + "cancelled": "已取消", + } + return mapping.get(normalized_status, normalized_status) + + +def _build_job_summary_text(job: dict) -> str: + normalized_job = dict(job or {}) + status = str(normalized_job.get("status") or "").strip() + target_node_code = str(normalized_job.get("target_node_code") or "").strip() + execution_mode_label_text = str(normalized_job.get("execution_mode_label") or "").strip() + requested_by = str(normalized_job.get("requested_by") or "").strip() + error_message = str(normalized_job.get("error_message") or "").strip() + blocked_reason = str(normalized_job.get("blocked_reason") or "").strip() + cancellation_reason = str(normalized_job.get("cancellation_reason") or "").strip() + + if cancellation_reason and status == "cancelled": + return cancellation_reason + if blocked_reason and status == "blocked": + return blocked_reason + if error_message and status in {"failed", "blocked", "cancelled", "completed_with_issues"}: + return error_message + if status == "awaiting_approval": + return "任务已创建，等待审批后继续派发。" + if status == "running": + if target_node_code: + return f"目标节点 {target_node_code} 正在执行该任务。" + return "任务正在执行中。" + if status == "success": + if target_node_code: + return f"目标节点 {target_node_code} 已完成该任务。" + return "任务已执行完成。" + + parts: list[str] = [] + if target_node_code: + parts.append(f"目标节点 {target_node_code}") + if requested_by: + parts.append(f"发起 {requested_by}") + if execution_mode_label_text: + parts.append(f"方式 {execution_mode_label_text}") + if parts: + return " / ".join(parts) + return f"当前状态 {_job_status_label(status)}。" + + +def _build_job_step_counters(steps: list[dict] | None = None) -> dict: + normalized_steps = list(steps or []) + total = len(normalized_steps) + running = sum(1 for item in normalized_steps if str(item.get("status") or "").strip() in {"running", "dispatching"}) + success = sum(1 for item in normalized_steps if str(item.get("status") or "").strip() == "success") + failed = sum(1 for item in normalized_steps if str(item.get("status") or "").strip() in {"failed", "blocked", "cancelled"}) + terminal = sum( + 1 + for item in normalized_steps + if str(item.get("status") or "").strip() in {"success", "failed", "blocked", "cancelled", "partially_succeeded"} + ) + return { + "steps_total": total, + "steps_running": running, + "steps_success": success, + "steps_failed": failed, + "steps_terminal": terminal, + } + + +def _serialize_job_row(row: tuple, *, steps: list[dict] | None = None) -> dict: + normalized_execution_mode = str(row[6] or "") + normalized_steps = list(steps or []) + approval_required = bool(row[17]) + serialized = { + "id": int(row[0]), + "job_code": str(row[1] or ""), + "action": str(row[2] or ""), + "target_type": str(row[3] or ""), + "target_node_code": str(row[4] or ""), + "status": str(row[5] or ""), + "status_label": _job_status_label(row[5]), + "execution_mode": normalized_execution_mode, + "execution_mode_label": execution_mode_label(normalized_execution_mode), + "requested_by": str(row[7] or ""), + "payload": _decode_json(row[8]), + "metadata": _decode_json(row[9]), + "result": _decode_json(row[10]), + "error_message": str(row[11] or ""), + "created_at": _ts(row[12]), + "started_at": _ts(row[13]), + "finished_at": _ts(row[14]), + "updated_at": _ts(row[15]), + "risk_level": str(row[16] or ""), + "approval_required": approval_required, + "approval_status": str(row[18] or ""), + "approval_status_label": _job_approval_status_label(row[18], approval_required=approval_required), + "approved_by": str(row[19] or ""), + "approved_at": _ts(row[20]), + "blocked_reason": str(row[21] or ""), + "cancellation_reason": str(row[22] or ""), + "dispatched_at": _ts(row[23]), + "target_selector": _decode_json(row[24]), + "policy": _decode_json(row[25]), + "rollout_id": int(row[26]) if row[26] else 0, + "steps": normalized_steps, + "steps_loaded": steps is not None, + "is_compact": False, + "target_node_codes": [str(row[4] or "")] if str(row[4] or "").strip() else [], + "focus_ref": { + "kind": "ops_job", + "job_id": int(row[0]), + "job_code": str(row[1] or ""), + "action": str(row[2] or ""), + "target_node_code": str(row[4] or ""), + }, + } + serialized.update(_build_job_step_counters(normalized_steps)) + serialized["summary"] = _build_job_summary_text(serialized) + serialized["summary_text"] = str(serialized.get("summary") or "") + return serialized + + +def _serialize_node_row(row: tuple) -> dict: + return { + "node_code": str(row[0] or ""), + "region": str(row[1] or ""), + "role": str(row[2] or ""), + "title": str(row[3] or ""), + "ssh_host": str(row[4] or ""), + "ssh_port": int(row[5] or 22), + "ssh_user": str(row[6] or ""), + "auth_mode": str(row[7] or ""), + "deploy_channel": str(row[8] or ""), + "is_enabled": bool(row[9]), + "metadata": _decode_json(row[10]), + "created_at": _ts(row[11]), + "last_seen_at": _ts(row[12]), + "updated_at": _ts(row[13]), + } + + +def _pick_text_value(payload: dict, key: str, fallback: str = "", *, default: str = "") -> str: + if key in payload: + normalized = str(payload.get(key) or "").strip() + if normalized: + return normalized + normalized_fallback = str(fallback or "").strip() + if normalized_fallback: + return normalized_fallback + return str(default or "").strip() + + +def _pick_int_value(payload: dict, key: str, fallback: int, *, default: int = 0, minimum: int = 0) -> int: + if key in payload and payload.get(key) not in (None, ""): + try: + return max(int(payload.get(key) or default), minimum) + except Exception: + pass + try: + return max(int(fallback or default), minimum) + except Exception: + return max(int(default), minimum) + + +def _compact_value(value: object, *, depth: int = 0) -> object: + if isinstance(value, str): + return value if len(value) <= 240 else f"{value[:240]}..." + if isinstance(value, (int, float, bool)) or value is None: + return value + if depth >= 2: + if isinstance(value, list): + return f"" + if isinstance(value, dict): + return f"" + return str(value) + if isinstance(value, list): + limited = [_compact_value(item, depth=depth + 1) for item in value[:6]] + if len(value) > 6: + limited.append(f"...(+{len(value) - 6} more)") + return limited + if isinstance(value, dict): + compacted: dict[str, object] = {} + for index, (key, item) in enumerate(value.items()): + if index >= 8: + compacted["__truncated__"] = f"+{len(value) - 8} more keys" + break + compacted[str(key)] = _compact_value(item, depth=depth + 1) + return compacted + return str(value) + + +def _compact_job(job: dict) -> dict: + compacted = dict(job or {}) + compacted["payload"] = _compact_value(job.get("payload") or {}) + compacted["metadata"] = _compact_value(job.get("metadata") or {}) + compacted["result"] = _compact_value(job.get("result") or {}) + compacted["steps"] = [] + compacted["is_compact"] = True + compacted["result_summary"] = _compact_value(job.get("result") or {}) + return compacted + + +def _build_job_code() -> str: + stamp = datetime.now().strftime("%Y%m%d%H%M%S") + return f"ops-{stamp}-{uuid4().hex[:6]}" + + +def _normalize_requested_by(value: object) -> str: + normalized = str(value or "api").strip() or "api" + return normalized[:64] + + +def upsert_managed_node(payload: dict) -> tuple[bool, str, dict]: + ensure_ops_schema() + node_code = str(payload.get("node_code") or "").strip() + if not node_code: + return False, "node_code 不能为空", {} + + with get_db() as conn: + with conn.cursor() as cur: + cur.execute( + """ + SELECT + node_code, region, role, title, ssh_host, ssh_port, ssh_user, auth_mode, deploy_channel, is_enabled, metadata_json, created_at, last_seen_at, updated_at + FROM ops_managed_nodes + WHERE node_code = %s + LIMIT 1 + """, + (node_code,), + ) + existing_row = cur.fetchone() + existing_node = _serialize_node_row(existing_row) if existing_row else {} + existing_metadata = dict(existing_node.get("metadata") or {}) + incoming_metadata = dict(payload.get("metadata") or {}) + + region = _pick_text_value(payload, "region", str(existing_node.get("region") or ""), default="unknown") or "unknown" + role = _pick_text_value(payload, "role", str(existing_node.get("role") or ""), default="worker") or "worker" + title = _pick_text_value(payload, "title", str(existing_node.get("title") or ""), default=node_code) or node_code + ssh_host = _pick_text_value(payload, "ssh_host", str(existing_node.get("ssh_host") or "")) + ssh_port = _pick_int_value(payload, "ssh_port", int(existing_node.get("ssh_port") or 22), default=22, minimum=1) + ssh_user = _pick_text_value(payload, "ssh_user", str(existing_node.get("ssh_user") or "")) + auth_mode = _pick_text_value(payload, "auth_mode", str(existing_node.get("auth_mode") or ""), default="key") or "key" + deploy_channel = _pick_text_value( + payload, + "deploy_channel", + str(existing_node.get("deploy_channel") or ""), + default="stable", + ) or "stable" + is_enabled = bool(payload["is_enabled"]) if "is_enabled" in payload else bool(existing_node.get("is_enabled", True)) + metadata = { + **existing_metadata, + **incoming_metadata, + } + cur.execute( + """ + INSERT INTO ops_managed_nodes ( + node_code, region, role, title, ssh_host, ssh_port, ssh_user, auth_mode, deploy_channel, is_enabled, metadata_json, updated_at + ) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, CURRENT_TIMESTAMP) + ON CONFLICT (node_code) DO UPDATE SET + region = EXCLUDED.region, + role = EXCLUDED.role, + title = EXCLUDED.title, + ssh_host = EXCLUDED.ssh_host, + ssh_port = EXCLUDED.ssh_port, + ssh_user = EXCLUDED.ssh_user, + auth_mode = EXCLUDED.auth_mode, + deploy_channel = EXCLUDED.deploy_channel, + is_enabled = EXCLUDED.is_enabled, + metadata_json = EXCLUDED.metadata_json, + updated_at = CURRENT_TIMESTAMP + RETURNING + node_code, region, role, title, ssh_host, ssh_port, ssh_user, auth_mode, deploy_channel, is_enabled, metadata_json, created_at, last_seen_at, updated_at + """, + ( + node_code, + region, + role, + title, + ssh_host, + ssh_port, + ssh_user, + auth_mode, + deploy_channel, + is_enabled, + json.dumps(metadata, ensure_ascii=False), + ), + ) + row = cur.fetchone() + conn.commit() + return True, "托管节点已保存", {"node": _serialize_node_row(row)} + + +def list_managed_nodes() -> list[dict]: + ensure_ops_schema() + with get_db() as conn: + with conn.cursor() as cur: + cur.execute( + """ + SELECT + node_code, region, role, title, ssh_host, ssh_port, ssh_user, auth_mode, deploy_channel, is_enabled, metadata_json, created_at, last_seen_at, updated_at + FROM ops_managed_nodes + ORDER BY region ASC, role ASC, node_code ASC + """ + ) + rows = cur.fetchall() + return [_serialize_node_row(row) for row in rows] + + +def sync_managed_nodes_from_cluster(*, dry_run: bool = False) -> dict: + from app.services.cluster_runtime_service import get_cluster_snapshot + + snapshot = get_cluster_snapshot() + cluster_nodes = list(snapshot.get("nodes") or []) + synced_nodes: list[dict] = [] + + for cluster_node in cluster_nodes: + node_code = str(cluster_node.get("node_code") or "").strip() + if not node_code: + continue + payload = { + "node_code": node_code, + "region": str(cluster_node.get("region") or "unknown").strip() or "unknown", + "role": str(cluster_node.get("role") or "worker").strip() or "worker", + "title": str(cluster_node.get("hostname") or node_code).strip() or node_code, + "ssh_host": ( + str(cluster_node.get("ip") or "").strip() + if str(cluster_node.get("ip") or "").strip() not in {"", "127.0.0.1", "::1", "localhost"} + else "" + ), + "metadata": { + "imported_from_cluster": True, + "cluster_status": str(cluster_node.get("status") or ""), + "is_effective_worker": bool(cluster_node.get("is_effective_worker", False)), + "detect_participating": bool(cluster_node.get("detect_participating", False)), + "last_heartbeat_at": str(cluster_node.get("last_heartbeat_at") or ""), + "source_metadata": cluster_node.get("metadata") or {}, + }, + } + synced_nodes.append(payload) + if not dry_run: + upsert_managed_node(payload) + + return { + "dry_run": dry_run, + "cluster_nodes_total": len(cluster_nodes), + "synced_count": len(synced_nodes), + "nodes": synced_nodes, + } + + +def _create_step(cur, *, job_id: int, step_key: str, title: str, node_code: str, status: str) -> int: + cur.execute( + """ + INSERT INTO ops_job_steps ( + job_id, step_key, title, node_code, status, created_at, updated_at + ) VALUES (%s, %s, %s, %s, %s, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP) + RETURNING id + """, + (job_id, step_key, title, node_code, status), + ) + return int(cur.fetchone()[0]) + + +def _update_step( + cur, + *, + step_id: int, + status: str, + stdout_text: str = "", + stderr_text: str = "", + result: dict | None = None, + started: bool = False, + finished: bool = False, +) -> None: + started_sql = "started_at = COALESCE(started_at, CURRENT_TIMESTAMP)," if started else "" + finished_sql = "finished_at = CURRENT_TIMESTAMP," if finished else "" + cur.execute( + f""" + UPDATE ops_job_steps + SET + status = %s, + stdout_text = %s, + stderr_text = %s, + result_json = %s, + {started_sql} + {finished_sql} + updated_at = CURRENT_TIMESTAMP + WHERE id = %s + """, + ( + status, + stdout_text, + stderr_text, + json.dumps(result or {}, ensure_ascii=False), + step_id, + ), + ) + + +def _update_job( + cur, + *, + job_id: int, + status: str, + result: dict | None = None, + error_message: str = "", + started: bool = False, + finished: bool = False, +) -> None: + started_sql = "started_at = COALESCE(started_at, CURRENT_TIMESTAMP)," if started else "" + finished_sql = "finished_at = CURRENT_TIMESTAMP," if finished else "" + cur.execute( + f""" + UPDATE ops_jobs + SET + status = %s, + result_json = %s, + error_message = %s, + {started_sql} + {finished_sql} + updated_at = CURRENT_TIMESTAMP + WHERE id = %s + """, + ( + status, + json.dumps(result or {}, ensure_ascii=False), + error_message, + job_id, + ), + ) + + +def _get_job_row(cur, job_id: int) -> tuple | None: + cur.execute( + """ + SELECT + id, job_code, action, target_type, target_node_code, status, execution_mode, requested_by, + payload_json, metadata_json, result_json, error_message, created_at, started_at, finished_at, updated_at, + risk_level, approval_required, approval_status, approved_by, approved_at, blocked_reason, + cancellation_reason, dispatched_at, target_selector_json, policy_json, rollout_id + FROM ops_jobs + WHERE id = %s + """, + (job_id,), + ) + return cur.fetchone() + + +def _get_steps(cur, job_id: int) -> list[dict]: + cur.execute( + """ + SELECT + id, step_key, title, node_code, status, stdout_text, stderr_text, result_json, created_at, started_at, finished_at, updated_at + FROM ops_job_steps + WHERE job_id = %s + ORDER BY id ASC + """, + (job_id,), + ) + return [_serialize_step_row(row) for row in cur.fetchall()] + + +def _execute_local_job(action: str, payload: dict | None = None) -> tuple[bool, str, dict]: + if action == "health.snapshot": + payload = get_runtime_status() + return True, "运行时快照采集成功", {"runtime": payload} + + runtime_action_name = _LOCAL_RUNTIME_ACTIONS.get(action) + if runtime_action_name: + ok, message, data = runtime_action(runtime_action_name) + return ok, message, data + + if supports_ssh_execution(action): + return execute_local_support_action(action, payload) + + return False, f"当前未实现本机即时执行动作: {action}", {} + + +def _resolve_bootstrap_target_defaults(target_node_code: str) -> dict: + normalized_node_code = str(target_node_code or "").strip() + if not normalized_node_code: + return {"node_region": "mainland", "node_role": "worker"} + + managed_node = next( + (item for item in list_managed_nodes() if str(item.get("node_code") or "").strip() == normalized_node_code), + {}, + ) + if managed_node: + return { + "node_region": str(managed_node.get("region") or "mainland").strip() or "mainland", + "node_role": str(managed_node.get("role") or "worker").strip() or "worker", + } + + from app.services.cluster_runtime_service import get_cluster_snapshot + + cluster = get_cluster_snapshot() + cluster_node = next( + (item for item in list(cluster.get("nodes") or []) if str(item.get("node_code") or "").strip() == normalized_node_code), + {}, + ) + return { + "node_region": str(cluster_node.get("region") or "mainland").strip() or "mainland", + "node_role": str(cluster_node.get("role") or "worker").strip() or "worker", + } + + +def _execute_control_plane_job( + action: str, + *, + target_node_code: str, + payload: dict | None = None, + requested_by: str = "api", + metadata: dict | None = None, +) -> tuple[bool, str, dict]: + normalized_payload = dict(payload or {}) + normalized_metadata = dict(metadata or {}) + + if action == "node.bootstrap": + target_defaults = _resolve_bootstrap_target_defaults(target_node_code) + node_region = str(normalized_payload.get("node_region") or target_defaults.get("node_region") or "mainland").strip() or "mainland" + node_role = str(normalized_payload.get("node_role") or target_defaults.get("node_role") or "worker").strip() or "worker" + try: + expires_in_hours = int(normalized_payload.get("expires_in_hours") or 72) + except Exception: + expires_in_hours = 72 + from app.services.ops_agent_service import build_node_agent_bootstrap_plan + + ok, message, data = build_node_agent_bootstrap_plan( + node_code=str(target_node_code or "").strip(), + node_region=node_region, + node_role=node_role, + issued_by=str(requested_by or "api").strip() or "api", + expires_in_hours=expires_in_hours, + control_plane_base_url=str(normalized_payload.get("control_plane_base_url") or "").strip(), + root_dir=str(normalized_payload.get("root_dir") or "/opt/domaincheck").strip() or "/opt/domaincheck", + metadata={ + **normalized_metadata, + "issued_from": "ops-job/control-plane", + "target_node_code": str(target_node_code or "").strip(), + "node_region": node_region, + "node_role": node_role, + }, + ) + if ok: + return True, message, { + **data, + "action": action, + "target_node_code": str(target_node_code or "").strip(), + "node_region": node_region, + "node_role": node_role, + "execution_mode": "control-plane", + } + return False, message, data + + return False, f"当前未实现控制面执行动作: {action}", {} + + +def _execute_local_job_record(job_id: int) -> tuple[bool, str, dict]: + ensure_ops_schema() + with get_db() as conn: + conn.autocommit = False + with conn.cursor() as cur: + row = _get_job_row(cur, int(job_id)) + if not row: + conn.rollback() + return False, "任务不存在", {} + job = _serialize_job_row(row) + if str(job.get("execution_mode") or "") != "local-runtime": + conn.rollback() + return False, "当前任务不是本机即时执行任务", {"job": job} + if str(job.get("target_node_code") or "") != settings.node_code: + conn.rollback() + return False, "当前任务目标节点不是本机", {"job": job} + if str(job.get("status") or "") != "queued": + conn.rollback() + return False, f"当前任务状态为 {job.get('status') or ''}，不可执行", {"job": job} + + cur.execute( + """ + UPDATE ops_jobs + SET + status = 'running', + started_at = COALESCE(started_at, CURRENT_TIMESTAMP), + dispatched_at = COALESCE(dispatched_at, CURRENT_TIMESTAMP), + updated_at = CURRENT_TIMESTAMP + WHERE id = %s + """, + (int(job_id),), + ) + cur.execute( + """ + UPDATE ops_job_steps + SET + status = 'running', + started_at = COALESCE(started_at, CURRENT_TIMESTAMP), + updated_at = CURRENT_TIMESTAMP + WHERE job_id = %s + """, + (int(job_id),), + ) + conn.commit() + + ok, message, result = _execute_local_job(str(job.get("action") or ""), dict(job.get("payload") or {})) + + with get_db() as conn: + conn.autocommit = False + with conn.cursor() as cur: + cur.execute( + """ + UPDATE ops_job_steps + SET + status = %s, + stdout_text = %s, + stderr_text = %s, + result_json = %s, + finished_at = CURRENT_TIMESTAMP, + updated_at = CURRENT_TIMESTAMP + WHERE job_id = %s + """, + ( + "success" if ok else "failed", + message if ok else "", + "" if ok else message, + json.dumps(result or {}, ensure_ascii=False), + int(job_id), + ), + ) + cur.execute( + """ + UPDATE ops_jobs + SET + status = %s, + result_json = %s, + error_message = %s, + finished_at = CURRENT_TIMESTAMP, + updated_at = CURRENT_TIMESTAMP + WHERE id = %s + """, + ( + "success" if ok else "failed", + json.dumps(result or {}, ensure_ascii=False), + "" if ok else message, + int(job_id), + ), + ) + conn.commit() + from app.services.ops_release_service import refresh_release_rollout_for_job + + refresh_release_rollout_for_job(int(job_id)) + return ok, message, {"job": get_ops_job(int(job_id))} + + +def _execute_control_plane_job_record(job_id: int) -> tuple[bool, str, dict]: + ensure_ops_schema() + with get_db() as conn: + conn.autocommit = False + with conn.cursor() as cur: + row = _get_job_row(cur, int(job_id)) + if not row: + conn.rollback() + return False, "任务不存在", {} + job = _serialize_job_row(row) + if str(job.get("execution_mode") or "") != "control-plane": + conn.rollback() + return False, "当前任务不是控制面执行任务", {"job": job} + if str(job.get("status") or "") != "queued": + conn.rollback() + return False, f"当前任务状态为 {job.get('status') or ''}，不可执行", {"job": job} + + cur.execute( + """ + UPDATE ops_jobs + SET + status = 'running', + started_at = COALESCE(started_at, CURRENT_TIMESTAMP), + dispatched_at = COALESCE(dispatched_at, CURRENT_TIMESTAMP), + updated_at = CURRENT_TIMESTAMP + WHERE id = %s + """, + (int(job_id),), + ) + cur.execute( + """ + UPDATE ops_job_steps + SET + status = 'running', + started_at = COALESCE(started_at, CURRENT_TIMESTAMP), + updated_at = CURRENT_TIMESTAMP + WHERE job_id = %s + """, + (int(job_id),), + ) + conn.commit() + + ok, message, result = _execute_control_plane_job( + str(job.get("action") or ""), + target_node_code=str(job.get("target_node_code") or ""), + payload=dict(job.get("payload") or {}), + requested_by=str(job.get("requested_by") or "api"), + metadata=dict(job.get("metadata") or {}), + ) + + with get_db() as conn: + conn.autocommit = False + with conn.cursor() as cur: + cur.execute( + """ + UPDATE ops_job_steps + SET + status = %s, + stdout_text = %s, + stderr_text = %s, + result_json = %s, + finished_at = CURRENT_TIMESTAMP, + updated_at = CURRENT_TIMESTAMP + WHERE job_id = %s + """, + ( + "success" if ok else "failed", + message if ok else "", + "" if ok else message, + json.dumps(result or {}, ensure_ascii=False), + int(job_id), + ), + ) + cur.execute( + """ + UPDATE ops_jobs + SET + status = %s, + result_json = %s, + error_message = %s, + finished_at = CURRENT_TIMESTAMP, + updated_at = CURRENT_TIMESTAMP + WHERE id = %s + """, + ( + "success" if ok else "failed", + json.dumps(result or {}, ensure_ascii=False), + "" if ok else message, + int(job_id), + ), + ) + conn.commit() + from app.services.ops_agent_service import append_ops_job_event + + append_ops_job_event( + job_id=int(job_id), + node_code=str(job.get("target_node_code") or ""), + event_type="job_executed_on_control_plane", + level="info" if ok else "error", + message=f"任务已在控制面执行: {str(job.get('action') or '')}", + payload=result, + ) + from app.services.ops_release_service import refresh_release_rollout_for_job + + refresh_release_rollout_for_job(int(job_id)) + return ok, message, {"job": get_ops_job(int(job_id))} + + +def _execute_ssh_job_record(job_id: int) -> tuple[bool, str, dict]: + ensure_ops_schema() + with get_db() as conn: + conn.autocommit = False + with conn.cursor() as cur: + row = _get_job_row(cur, int(job_id)) + if not row: + conn.rollback() + return False, "任务不存在", {} + job = _serialize_job_row(row) + if str(job.get("execution_mode") or "") != "ssh": + conn.rollback() + return False, "当前任务不是 SSH 执行任务", {"job": job} + if str(job.get("status") or "") != "queued": + conn.rollback() + return False, f"当前任务状态为 {job.get('status') or ''}，不可执行", {"job": job} + + target_node_code = str(job.get("target_node_code") or "").strip() + managed_node = next( + (item for item in list_managed_nodes() if str(item.get("node_code") or "").strip() == target_node_code), + {}, + ) + if not managed_node: + conn.rollback() + return False, "目标节点未纳入托管清单，无法执行 SSH 任务", {"job": job} + + cur.execute( + """ + UPDATE ops_jobs + SET + status = 'running', + started_at = COALESCE(started_at, CURRENT_TIMESTAMP), + dispatched_at = COALESCE(dispatched_at, CURRENT_TIMESTAMP), + updated_at = CURRENT_TIMESTAMP + WHERE id = %s + """, + (int(job_id),), + ) + cur.execute( + """ + UPDATE ops_job_steps + SET + status = 'running', + started_at = COALESCE(started_at, CURRENT_TIMESTAMP), + updated_at = CURRENT_TIMESTAMP + WHERE job_id = %s + """, + (int(job_id),), + ) + conn.commit() + + ok, message, result = execute_ssh_action( + managed_node, + str(job.get("action") or ""), + dict(job.get("payload") or {}), + ) + + with get_db() as conn: + conn.autocommit = False + with conn.cursor() as cur: + cur.execute( + """ + UPDATE ops_job_steps + SET + status = %s, + stdout_text = %s, + stderr_text = %s, + result_json = %s, + finished_at = CURRENT_TIMESTAMP, + updated_at = CURRENT_TIMESTAMP + WHERE job_id = %s + """, + ( + "success" if ok else "failed", + message if ok else "", + "" if ok else message, + json.dumps(result or {}, ensure_ascii=False), + int(job_id), + ), + ) + cur.execute( + """ + UPDATE ops_jobs + SET + status = %s, + result_json = %s, + error_message = %s, + finished_at = CURRENT_TIMESTAMP, + updated_at = CURRENT_TIMESTAMP + WHERE id = %s + """, + ( + "success" if ok else "failed", + json.dumps(result or {}, ensure_ascii=False), + "" if ok else message, + int(job_id), + ), + ) + conn.commit() + from app.services.ops_agent_service import append_ops_job_event + + append_ops_job_event( + job_id=int(job_id), + node_code=str(job.get("target_node_code") or ""), + event_type="job_executed_over_ssh", + level="info" if ok else "error", + message=f"任务已通过 SSH 执行: {str(job.get('action') or '')}", + payload=result, + ) + from app.services.ops_release_service import refresh_release_rollout_for_job + + refresh_release_rollout_for_job(int(job_id)) + return ok, message, {"job": get_ops_job(int(job_id))} + + +def create_ops_job(payload: dict) -> tuple[bool, str, dict]: + ensure_ops_schema() + action = str(payload.get("action") or "").strip() + if not action: + return False, "action 不能为空", {} + + target_type = str(payload.get("target_type") or "node").strip() or "node" + target_node_code = str(payload.get("target_node_code") or settings.node_code).strip() or settings.node_code + target_selector = payload.get("target_selector") or {} + requested_by = _normalize_requested_by(payload.get("requested_by")) + input_payload = payload.get("payload") or {} + metadata = payload.get("metadata") or {} + auto_approve = bool(payload.get("auto_approve", False)) + rollout_id = int(payload.get("rollout_id") or 0) or None + local_action = target_node_code == settings.node_code and supports_local_runtime_action(action) + control_plane_action = supports_control_plane_action(action) + requested_execution_mode = ( + "local-runtime" + if local_action + else ("control-plane" if control_plane_action else str(payload.get("execution_mode") or "remote-agent").strip() or "remote-agent") + ) + + from app.services.ops_policy_service import preview_ops_job_policy + + policy_preview = preview_ops_job_policy( + { + "action": action, + "target_type": target_type, + "target_node_code": target_node_code, + "target_selector": target_selector, + "execution_mode": requested_execution_mode, + "payload": input_payload, + } + ) + risk_level = str(policy_preview.get("risk_level") or "medium") + approval_required = bool(policy_preview.get("approval_required", False)) + blocked = bool(policy_preview.get("blocked", False)) + blocked_reason = "；".join(list(policy_preview.get("blocking_reasons") or [])) + + execution_mode = requested_execution_mode + run_now = ( + bool(payload.get("run_now", True)) + if (local_action or control_plane_action or execution_mode == "ssh") + and not blocked + and not (approval_required and not auto_approve) + else False + ) + job_code = _build_job_code() + initial_status = "blocked" if blocked else ("awaiting_approval" if approval_required and not auto_approve else "queued") + approval_status = "blocked" if blocked else ("approved" if (not approval_required or auto_approve) else "pending") + + with get_db() as conn: + conn.autocommit = False + with conn.cursor() as cur: + cur.execute( + """ + INSERT INTO ops_jobs ( + job_code, action, target_type, target_node_code, status, execution_mode, requested_by, payload_json, metadata_json, + created_at, updated_at, risk_level, approval_required, approval_status, approved_by, approved_at, + blocked_reason, target_selector_json, policy_json, rollout_id + ) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP, %s, %s, %s, %s, %s, %s, %s, %s, %s) + RETURNING id + """, + ( + job_code, + action, + target_type, + target_node_code, + initial_status, + execution_mode, + requested_by, + json.dumps(input_payload, ensure_ascii=False), + json.dumps(metadata, ensure_ascii=False), + risk_level, + approval_required, + approval_status, + requested_by if approval_required and auto_approve else "", + datetime.now() if approval_required and auto_approve else None, + blocked_reason, + json.dumps(target_selector, ensure_ascii=False), + json.dumps(policy_preview, ensure_ascii=False), + rollout_id, + ), + ) + job_id = int(cur.fetchone()[0]) + step_id = _create_step( + cur, + job_id=job_id, + step_key="dispatch", + title="调度动作", + node_code=target_node_code, + status=initial_status, + ) + + if blocked: + policy_result = { + "dispatch_state": "blocked", + "policy": policy_preview, + "hint": "请先解决阻断条件，再重新创建或重新排队任务。", + } + _update_job(cur, job_id=job_id, status="blocked", result=policy_result, error_message=blocked_reason) + _update_step( + cur, + step_id=step_id, + status="blocked", + stderr_text=blocked_reason, + result=policy_result, + finished=True, + ) + conn.commit() + from app.services.ops_agent_service import append_ops_job_event + + append_ops_job_event( + job_id=job_id, + node_code=target_node_code, + event_type="job_blocked", + level="warning", + message=f"任务已被策略阻断: {action}", + payload={"policy": policy_preview}, + ) + return False, blocked_reason or "任务被策略阻断", {"job": get_ops_job(job_id), "executed_immediately": False} + + if approval_required and not auto_approve: + approval_result = { + "dispatch_state": "awaiting_approval", + "policy": policy_preview, + "hint": "该任务需要审批后才能进入执行队列。", + } + _update_job(cur, job_id=job_id, status="awaiting_approval", result=approval_result) + _update_step( + cur, + step_id=step_id, + status="awaiting_approval", + result=approval_result, + ) + conn.commit() + from app.services.ops_agent_service import append_ops_job_event + + append_ops_job_event( + job_id=job_id, + node_code=target_node_code, + event_type="job_awaiting_approval", + level="info", + message=f"任务等待审批: {action}", + payload={"policy": policy_preview}, + ) + return True, "运维任务已创建，等待审批", {"job": get_ops_job(job_id), "executed_immediately": False} + + if run_now: + _update_job(cur, job_id=job_id, status="running", started=True) + _update_step(cur, step_id=step_id, status="running", started=True) + if execution_mode == "control-plane": + ok, message, result = _execute_control_plane_job( + action, + target_node_code=target_node_code, + payload=dict(input_payload or {}), + requested_by=requested_by, + metadata=dict(metadata or {}), + ) + elif execution_mode == "ssh": + managed_node = next( + (item for item in list_managed_nodes() if str(item.get("node_code") or "").strip() == target_node_code), + {}, + ) + if not managed_node: + ok, message, result = ( + False, + "目标节点未纳入托管清单，无法执行 SSH 任务", + {"target_node_code": target_node_code, "executor": "ssh"}, + ) + else: + ok, message, result = execute_ssh_action( + managed_node, + action, + dict(input_payload or {}), + ) + else: + ok, message, result = _execute_local_job(action, dict(input_payload or {})) + _update_step( + cur, + step_id=step_id, + status="success" if ok else "failed", + stdout_text=message if ok else "", + stderr_text="" if ok else message, + result=result, + finished=True, + ) + _update_job( + cur, + job_id=job_id, + status="success" if ok else "failed", + result=result, + error_message="" if ok else message, + started=True, + finished=True, + ) + conn.commit() + from app.services.ops_agent_service import append_ops_job_event + + append_ops_job_event( + job_id=job_id, + node_code=target_node_code, + event_type=( + "job_executed_on_control_plane" + if execution_mode == "control-plane" + else ("job_executed_over_ssh" if execution_mode == "ssh" else "job_executed_locally") + ), + level="info" if ok else "error", + message=( + f"任务已在控制面即时执行: {action}" + if execution_mode == "control-plane" + else (f"任务已通过 SSH 即时执行: {action}" if execution_mode == "ssh" else f"任务已在本机即时执行: {action}") + ), + payload=result, + ) + return ok, message, {"job": get_ops_job(job_id), "executed_immediately": True} + + queued_result = { + "dispatch_state": "queued", + "waiting_for": ( + "node-agent" + if execution_mode == "remote-agent" + else ("ssh-executor" if execution_mode == "ssh" else execution_mode) + ), + "hint": ( + "后续可由海外控制面的控制面执行器生成接入工单。" + if execution_mode == "control-plane" + else ( + "该任务将由海外控制面的 SSH 执行器直接连到目标节点执行。" + if execution_mode == "ssh" + else "后续可由海外控制面的 node-agent / SSH 执行器消费该任务。" + ) + ), + "policy": policy_preview, + } + _update_job(cur, job_id=job_id, status="queued", result=queued_result) + conn.commit() + from app.services.ops_agent_service import append_ops_job_event + + append_ops_job_event( + job_id=job_id, + node_code=target_node_code, + event_type="job_created", + level="info", + message=f"任务已创建并进入队列: {action}", + payload={"execution_mode": execution_mode, "policy": policy_preview}, + ) + return True, "运维任务已创建，等待执行器接管", {"job": get_ops_job(job_id), "executed_immediately": False} + + +def create_ops_job_batch(payload: dict) -> tuple[bool, str, dict]: + ensure_ops_schema() + template_key = str(payload.get("template_key") or "").strip() + target_node_codes = [] + seen_node_codes: set[str] = set() + for item in list(payload.get("target_node_codes") or []): + node_code = str(item or "").strip() + if not node_code or node_code in seen_node_codes: + continue + seen_node_codes.add(node_code) + target_node_codes.append(node_code) + if not target_node_codes: + return False, "target_node_codes 不能为空", {} + + requested_by = _normalize_requested_by(payload.get("requested_by")) + input_payload = payload.get("payload") or {} + metadata = payload.get("metadata") or {} + execution_mode = str(payload.get("execution_mode") or "").strip() + auto_approve = payload.get("auto_approve") + + from app.services.ops_template_service import build_ops_template_payload, get_ops_action_template + + template = get_ops_action_template(template_key) + if not template: + return False, "动作模板不存在", {} + + normalize_ok, normalize_message, normalized_action_payload = build_ops_template_payload(template_key, input_payload) + if not normalize_ok: + return False, normalize_message, {} + + action = str(template.get("action") or "").strip() + execution_mode = execution_mode or str(template.get("default_execution_mode") or "remote-agent").strip() or "remote-agent" + if auto_approve is None: + auto_approve = bool(template.get("default_auto_approve", False)) + else: + auto_approve = bool(auto_approve) + + created_jobs: list[dict] = [] + item_results: list[dict] = [] + status_counts: dict[str, int] = {} + created_count = 0 + blocked_count = 0 + awaiting_approval_count = 0 + failed_count = 0 + + for node_code in target_node_codes: + ok, message, data = create_ops_job( + { + "action": action, + "target_type": "node", + "target_node_code": node_code, + "requested_by": requested_by, + "execution_mode": execution_mode, + "auto_approve": auto_approve, + "payload": normalized_action_payload, + "metadata": metadata, + } + ) + job = (data.get("job") or {}) if isinstance(data, dict) else {} + compact_job = _compact_job(job) if job else {} + job_status = str(job.get("status") or ("failed" if not ok else "unknown")).strip() or "unknown" + status_counts[job_status] = int(status_counts.get(job_status, 0) or 0) + 1 + if compact_job: + created_jobs.append(compact_job) + created_count += 1 + if job_status == "blocked": + blocked_count += 1 + elif job_status == "awaiting_approval": + awaiting_approval_count += 1 + elif not ok and not job: + failed_count += 1 + item_results.append( + { + "node_code": node_code, + "ok": ok, + "message": message, + "status": job_status, + "job": compact_job, + } + ) + + if created_count <= 0: + return False, "未能创建任何运维任务", { + "template": template, + "results": item_results, + "status_counts": status_counts, + } + + summary = { + "template_key": template_key, + "action": action, + "requested_by": requested_by, + "execution_mode": execution_mode, + "auto_approve": auto_approve, + "target_nodes_total": len(target_node_codes), + "created_count": created_count, + "blocked_count": blocked_count, + "awaiting_approval_count": awaiting_approval_count, + "failed_count": failed_count, + "status_counts": status_counts, + } + message = ( + f"已创建 {created_count} 个运维任务" + f"；待审批 {awaiting_approval_count}" + f"；阻断 {blocked_count}" + f"；失败 {failed_count}" + ) + return True, message, { + "template": template, + "summary": summary, + "metadata": _compact_value(metadata), + "jobs": created_jobs, + "results": item_results, + } + + +def list_ops_jobs(limit: int = 20, *, rollout_id: int | None = None, compact: bool = True) -> list[dict]: + ensure_ops_schema() + limited = min(max(int(limit or 20), 1), 10000) + conditions: list[str] = [] + params: list[object] = [] + if rollout_id is not None and int(rollout_id or 0) > 0: + conditions.append("rollout_id = %s") + params.append(int(rollout_id)) + where_clause = f"WHERE {' AND '.join(conditions)}" if conditions else "" + with get_db() as conn: + with conn.cursor() as cur: + cur.execute( + f""" + SELECT + id, job_code, action, target_type, target_node_code, status, execution_mode, requested_by, + payload_json, metadata_json, result_json, error_message, created_at, started_at, finished_at, updated_at, + risk_level, approval_required, approval_status, approved_by, approved_at, blocked_reason, + cancellation_reason, dispatched_at, target_selector_json, policy_json, rollout_id + FROM ops_jobs + {where_clause} + ORDER BY id DESC + LIMIT %s + """, + (*params, limited), + ) + rows = cur.fetchall() + jobs = [_serialize_job_row(row) for row in rows] + if compact: + return [_compact_job(job) for job in jobs] + return jobs + + +def get_ops_job(job_id: int) -> dict: + ensure_ops_schema() + with get_db() as conn: + with conn.cursor() as cur: + row = _get_job_row(cur, int(job_id)) + if not row: + return {} + steps = _get_steps(cur, int(job_id)) + return _serialize_job_row(row, steps=steps) + + +def get_ops_job_summary() -> dict: + ensure_ops_schema() + with get_db() as conn: + with conn.cursor() as cur: + cur.execute( + """ + SELECT status, count(*) + FROM ops_jobs + GROUP BY status + """ + ) + rows = cur.fetchall() + status_counts = {str(row[0] or ""): int(row[1] or 0) for row in rows} + return { + "status_counts": status_counts, + "queued": int(status_counts.get("queued", 0) or 0), + "running": int(status_counts.get("running", 0) or 0), + "awaiting_approval": int(status_counts.get("awaiting_approval", 0) or 0), + "blocked": int(status_counts.get("blocked", 0) or 0), + "cancelled": int(status_counts.get("cancelled", 0) or 0), + "success": int(status_counts.get("success", 0) or 0), + "failed": int(status_counts.get("failed", 0) or 0), + "total": sum(status_counts.values()), + } + + +def approve_ops_job(job_id: int, *, approved_by: str = "api") -> tuple[bool, str, dict]: + ensure_ops_schema() + normalized_approved_by = str(approved_by or "api").strip() or "api" + with get_db() as conn: + conn.autocommit = False + with conn.cursor() as cur: + row = _get_job_row(cur, int(job_id)) + if not row: + conn.rollback() + return False, "任务不存在", {} + job = _serialize_job_row(row) + if str(job.get("status") or "") != "awaiting_approval": + conn.rollback() + return False, "当前任务不处于待审批状态", {"job": job} + cur.execute( + """ + UPDATE ops_jobs + SET + status = 'queued', + approval_status = 'approved', + approved_by = %s, + approved_at = CURRENT_TIMESTAMP, + updated_at = CURRENT_TIMESTAMP + WHERE id = %s + """, + (normalized_approved_by, int(job_id)), + ) + cur.execute( + """ + UPDATE ops_job_steps + SET + status = 'queued', + updated_at = CURRENT_TIMESTAMP + WHERE job_id = %s + """, + (int(job_id),), + ) + conn.commit() + from app.services.ops_agent_service import append_ops_job_event + + append_ops_job_event( + job_id=int(job_id), + node_code=str((get_ops_job(int(job_id)) or {}).get("target_node_code") or ""), + event_type="job_approved", + level="info", + message=f"任务审批通过: {job_id}", + payload={"approved_by": normalized_approved_by}, + ) + from app.services.ops_release_service import refresh_release_rollout_for_job + + refresh_release_rollout_for_job(int(job_id)) + return True, "任务已审批通过并进入队列", {"job": get_ops_job(int(job_id))} + + +def cancel_ops_job(job_id: int, *, cancelled_by: str = "api", reason: str = "") -> tuple[bool, str, dict]: + ensure_ops_schema() + normalized_cancelled_by = str(cancelled_by or "api").strip() or "api" + normalized_reason = str(reason or "").strip() + with get_db() as conn: + conn.autocommit = False + with conn.cursor() as cur: + row = _get_job_row(cur, int(job_id)) + if not row: + conn.rollback() + return False, "任务不存在", {} + job = _serialize_job_row(row) + current_status = str(job.get("status") or "") + if current_status in {"success", "failed", "cancelled"}: + conn.rollback() + return False, f"当前任务状态为 {current_status}，不可取消", {"job": job} + result = dict(job.get("result") or {}) + result["cancelled_by"] = normalized_cancelled_by + if normalized_reason: + result["cancel_reason"] = normalized_reason + cur.execute( + """ + UPDATE ops_jobs + SET + status = 'cancelled', + cancellation_reason = %s, + result_json = %s, + updated_at = CURRENT_TIMESTAMP, + finished_at = CASE WHEN finished_at IS NULL THEN CURRENT_TIMESTAMP ELSE finished_at END + WHERE id = %s + """, + ( + normalized_reason, + json.dumps(result, ensure_ascii=False), + int(job_id), + ), + ) + cur.execute( + """ + UPDATE ops_job_steps + SET + status = CASE WHEN status IN ('success', 'failed', 'cancelled') THEN status ELSE 'cancelled' END, + stderr_text = CASE WHEN %s <> '' THEN %s ELSE stderr_text END, + updated_at = CURRENT_TIMESTAMP, + finished_at = CASE WHEN finished_at IS NULL THEN CURRENT_TIMESTAMP ELSE finished_at END + WHERE job_id = %s + """, + ( + normalized_reason, + normalized_reason, + int(job_id), + ), + ) + conn.commit() + from app.services.ops_agent_service import append_ops_job_event + + append_ops_job_event( + job_id=int(job_id), + node_code=str((get_ops_job(int(job_id)) or {}).get("target_node_code") or ""), + event_type="job_cancelled", + level="warning", + message=f"任务已取消: {job_id}", + payload={"cancelled_by": normalized_cancelled_by, "reason": normalized_reason}, + ) + from app.services.ops_release_service import refresh_release_rollout_for_job + + refresh_release_rollout_for_job(int(job_id)) + return True, "任务已取消", {"job": get_ops_job(int(job_id))} + + +def dispatch_ops_job(job_id: int) -> tuple[bool, str, dict]: + ensure_ops_schema() + with get_db() as conn: + with conn.cursor() as cur: + row = _get_job_row(cur, int(job_id)) + if not row: + return False, "任务不存在", {} + job = _serialize_job_row(row) + + status = str(job.get("status") or "") + if status != "queued": + return False, f"当前任务状态为 {status}，不可派发", {"job": job} + + target_node_code = str(job.get("target_node_code") or "") + execution_mode = str(job.get("execution_mode") or "") + if execution_mode == "control-plane": + from app.services.ops_agent_service import append_ops_job_event + + append_ops_job_event( + job_id=int(job_id), + node_code=target_node_code, + event_type="job_dispatch_requested", + level="info", + message=f"请求控制面执行任务: {job_id}", + payload={"action": str(job.get('action') or ''), "execution_mode": execution_mode}, + ) + ok, message, data = _execute_control_plane_job_record(int(job_id)) + return ok, f"已按控制面执行派发：{message}", data + + if execution_mode == "local-runtime" and target_node_code == settings.node_code: + from app.services.ops_agent_service import append_ops_job_event + + append_ops_job_event( + job_id=int(job_id), + node_code=target_node_code, + event_type="job_dispatch_requested", + level="info", + message=f"请求本机即时执行任务: {job_id}", + payload={"action": str(job.get('action') or '')}, + ) + ok, message, data = _execute_local_job_record(int(job_id)) + return ok, f"已按本机即时执行派发：{message}", data + + if execution_mode == "ssh": + from app.services.ops_agent_service import append_ops_job_event + + append_ops_job_event( + job_id=int(job_id), + node_code=target_node_code, + event_type="job_dispatch_requested", + level="info", + message=f"请求 SSH 执行器执行任务: {job_id}", + payload={"action": str(job.get('action') or ''), "execution_mode": execution_mode}, + ) + ok, message, data = _execute_ssh_job_record(int(job_id)) + return ok, f"已按 SSH 执行派发：{message}", data + + from app.services.ops_agent_service import append_ops_job_event + + append_ops_job_event( + job_id=int(job_id), + node_code=target_node_code, + event_type="job_dispatch_requested", + level="info", + message=f"任务等待 node-agent 拉取: {job_id}", + payload={"execution_mode": execution_mode}, + ) + return True, "任务保持 queued，等待对应 node-agent 拉取", {"job": job} diff --git a/domain-api/app/services/ops_playbook_service.py b/domain-api/app/services/ops_playbook_service.py new file mode 100644 index 0000000..f5493ff --- /dev/null +++ b/domain-api/app/services/ops_playbook_service.py @@ -0,0 +1,1263 @@ +from __future__ import annotations + +from copy import deepcopy +from uuid import uuid4 + +from app.core.config import settings +from app.services.ops_execution_mode_service import ( + build_execution_mode_options, + decorate_execution_mode_fields, + execution_mode_label, +) +from app.services.ops_agent_service import list_managed_nodes_with_agent_state, list_ops_job_events_for_jobs +from app.services.ops_job_service import cancel_ops_job, create_ops_job_batch, list_ops_jobs +from app.services.ops_template_service import build_ops_template_payload, get_ops_action_template + +_NODE_AGENT_SERVICE_NAME = "domaincheck-node-agent" +_PLAYBOOK_RUN_ACTIVE_STATUSES = {"queued", "dispatching", "running", "awaiting_approval"} +_PLAYBOOK_RUN_TERMINAL_PROBLEM_STATUSES = {"failed", "blocked", "cancelled"} + + +def _normalize_focus_ref(focus_ref: object) -> dict: + if not isinstance(focus_ref, dict): + return {} + normalized: dict[str, object] = {} + for raw_key, raw_value in focus_ref.items(): + key = str(raw_key or "").strip() + if not key: + continue + if isinstance(raw_value, str): + value = raw_value.strip() + if value: + normalized[key] = value + continue + if isinstance(raw_value, bool): + normalized[key] = raw_value + continue + if isinstance(raw_value, (int, float)): + if raw_value: + normalized[key] = raw_value + continue + if raw_value is not None: + normalized[key] = raw_value + return normalized + + +def _normalize_node_codes(raw_node_codes: object) -> list[str]: + normalized: list[str] = [] + seen: set[str] = set() + for item in list(raw_node_codes or []): + node_code = str(item or "").strip() + if not node_code or node_code in seen: + continue + seen.add(node_code) + normalized.append(node_code) + return normalized + + +def _build_playbook_run_code() -> str: + return f"pbr-{uuid4().hex[:10]}" + + +def _slim_playbook_job(job: dict | None = None) -> dict: + normalized_job = dict(job or {}) + status = str(normalized_job.get("status") or "") + return { + "id": int(normalized_job.get("id") or 0), + "job_code": str(normalized_job.get("job_code") or ""), + "action": str(normalized_job.get("action") or ""), + "status": status, + "status_label": _playbook_run_status_label(status), + "target_node_code": str(normalized_job.get("target_node_code") or ""), + "created_at": str(normalized_job.get("created_at") or ""), + "started_at": str(normalized_job.get("started_at") or ""), + "finished_at": str(normalized_job.get("finished_at") or ""), + "updated_at": str(normalized_job.get("updated_at") or ""), + "error_message": str(normalized_job.get("error_message") or ""), + } + + +def _increment_status(status_counts: dict[str, int], status: str) -> None: + normalized_status = str(status or "").strip() or "unknown" + status_counts[normalized_status] = int(status_counts.get(normalized_status, 0) or 0) + 1 + + +def _latest_job_time(job: dict) -> str: + return ( + str(job.get("updated_at") or "").strip() + or str(job.get("finished_at") or "").strip() + or str(job.get("started_at") or "").strip() + or str(job.get("created_at") or "").strip() + ) + + +def _derive_run_status(*, status_counts: dict[str, int], jobs_total: int) -> str: + if jobs_total <= 0: + return "queued" + if sum(int(status_counts.get(status, 0) or 0) for status in _PLAYBOOK_RUN_TERMINAL_PROBLEM_STATUSES) > 0: + return "attention" + if sum(int(status_counts.get(status, 0) or 0) for status in _PLAYBOOK_RUN_ACTIVE_STATUSES) > 0: + return "running" + if int(status_counts.get("success", 0) or 0) >= jobs_total: + return "success" + return "queued" + + +def _terminal_problem_label(status_counts: dict[str, int]) -> str: + if int(status_counts.get("failed", 0) or 0) > 0: + return "失败" + if int(status_counts.get("blocked", 0) or 0) > 0: + return "阻断" + if int(status_counts.get("cancelled", 0) or 0) > 0: + return "已取消" + return "" + + +def _active_status_label(status_counts: dict[str, int]) -> str: + if int(status_counts.get("running", 0) or 0) > 0: + return "执行中" + if int(status_counts.get("dispatching", 0) or 0) > 0: + return "待派发" + if int(status_counts.get("awaiting_approval", 0) or 0) > 0: + return "待审批" + if int(status_counts.get("queued", 0) or 0) > 0: + return "排队中" + return "" + + +def _playbook_run_status_label(status: object) -> str: + normalized_status = str(status or "").strip() + mapping = { + "queued": "已创建", + "running": "收口中", + "success": "成功", + "attention": "待关注", + } + return mapping.get(normalized_status, normalized_status or "未知") + + +def _build_playbook_run_focus_ref( + run: dict | None = None, + *, + step_key: str = "", + step_title: str = "", + event_key: str = "", + node_code: str = "", +) -> dict: + normalized_run = dict(run or {}) + return { + "kind": "playbook_run", + "run_code": str(normalized_run.get("run_code") or "").strip(), + "playbook_key": str(normalized_run.get("playbook_key") or "").strip(), + "group_key": str(normalized_run.get("group_key") or "").strip(), + "focus_step_key": str(step_key or normalized_run.get("focus_step_key") or "").strip(), + "focus_step_title": str(step_title or normalized_run.get("focus_step_title") or "").strip(), + "event_key": str(event_key or "").strip(), + "node_code": str(node_code or "").strip(), + } + + +def _build_playbook_step_summary_text(step: dict | None = None) -> str: + normalized_step = dict(step or {}) + title = str(normalized_step.get("title") or normalized_step.get("step_key") or "步骤").strip() + status = str(normalized_step.get("status") or "").strip() + status_counts = dict(normalized_step.get("status_counts") or {}) + if status == "attention": + problem_label = _terminal_problem_label(status_counts) or "异常" + return f"{title} 当前出现{problem_label}，建议先查看该步骤事件流。" + if status == "running": + active_label = _active_status_label(status_counts) or "处理中" + return f"{title} 当前处于{active_label}，可继续观察该步骤事件。" + if status == "success": + return f"{title} 已完成收口。" + if status == "queued": + return f"{title} 已创建，等待后续任务推进。" + return f"{title} 当前状态为 {_playbook_run_status_label(status)}。" + + +def _build_playbook_run_summary_text(run: dict | None = None) -> str: + normalized_run = dict(run or {}) + focus_summary = str(normalized_run.get("focus_summary") or "").strip() + if focus_summary: + return focus_summary + title = str(normalized_run.get("playbook_title") or normalized_run.get("playbook_key") or "编排").strip() + status = str(normalized_run.get("status") or "").strip() + if status == "success": + return f"{title} 已全部收口。" + if status == "attention": + return f"{title} 当前存在需要优先处理的异常步骤。" + if status == "running": + return f"{title} 仍在执行中，可继续查看步骤与事件流。" + if status == "queued": + return f"{title} 已创建，等待执行推进。" + return f"{title} 当前状态为 {_playbook_run_status_label(status)}。" + + +def _build_bootstrap_target_state_counts(target_node_codes: list[str], managed_node_map: dict[str, dict]) -> dict[str, int]: + counts = { + "total": 0, + "online": 0, + "pending_bootstrap": 0, + "runtime_only": 0, + "stale": 0, + "unmanaged": 0, + "unknown": 0, + } + normalized_codes = _normalize_node_codes(target_node_codes) + counts["total"] = len(normalized_codes) + for node_code in normalized_codes: + node = dict(managed_node_map.get(node_code) or {}) + if not node: + counts["unknown"] += 1 + continue + state = str(node.get("agent_state") or "").strip() + if state in {"online", "online_busy"}: + counts["online"] += 1 + continue + if state in {"pending_bootstrap", "runtime_only", "stale"}: + counts[state] += 1 + continue + if bool(node.get("is_managed", False)): + counts["unknown"] += 1 + else: + counts["unmanaged"] += 1 + return counts + + +def _format_bootstrap_state_parts(state_counts: dict[str, int]) -> list[str]: + parts: list[str] = [] + if int(state_counts.get("online", 0) or 0) > 0: + parts.append(f"已在线 {int(state_counts.get('online', 0) or 0)}") + if int(state_counts.get("pending_bootstrap", 0) or 0) > 0: + parts.append(f"待回连 {int(state_counts.get('pending_bootstrap', 0) or 0)}") + if int(state_counts.get("runtime_only", 0) or 0) > 0: + parts.append(f"仅 runtime 在线 {int(state_counts.get('runtime_only', 0) or 0)}") + if int(state_counts.get("stale", 0) or 0) > 0: + parts.append(f"心跳过期 {int(state_counts.get('stale', 0) or 0)}") + if int(state_counts.get("unmanaged", 0) or 0) > 0: + parts.append(f"未纳管 {int(state_counts.get('unmanaged', 0) or 0)}") + if int(state_counts.get("unknown", 0) or 0) > 0: + parts.append(f"待确认 {int(state_counts.get('unknown', 0) or 0)}") + return parts + + +def _enrich_onboarding_bootstrap_run(run: dict, managed_node_map: dict[str, dict]) -> None: + normalized_run = dict(run or {}) + target_node_codes = _normalize_node_codes(normalized_run.get("target_node_codes") or []) + if not target_node_codes: + return + + state_counts = _build_bootstrap_target_state_counts(target_node_codes, managed_node_map) + state_parts = _format_bootstrap_state_parts(state_counts) + total = int(state_counts.get("total", 0) or 0) + run["bootstrap_state_counts"] = state_counts + + run_status = str(run.get("status") or "").strip() + if run_status == "attention": + summary = str(run.get("focus_summary") or "").strip() or "接入工单编排当前出现异常。" + run["focus_summary"] = f"{summary} 可直接进入任务详情查看 bootstrap 结果或错误回执。" + return + + if run_status == "running": + run["focus_level"] = "warning" + run["focus_summary"] = ( + f"正在为 {total} 个节点生成接入工单，完成后可直接复制 bootstrap 脚本、env 和一键落地命令。" + ) + return + + if run_status == "queued": + run["focus_level"] = "info" + run["focus_summary"] = "接入工单编排已创建，等待控制面生成 bootstrap 方案。" + return + + if run_status != "success": + return + + if total == 1: + node_code = target_node_codes[0] + node = dict(managed_node_map.get(node_code) or {}) + node_state = str(node.get("agent_state") or "").strip() + if node_state in {"online", "online_busy"}: + run["focus_level"] = "success" + run["focus_summary"] = "接入工单已生成，目标节点已完成 Agent 接入，可继续执行接管后验收。" + return + if node_state == "pending_bootstrap": + run["focus_level"] = "warning" + run["focus_summary"] = "接入工单已生成，节点待执行 bootstrap 脚本并回连 Agent。" + return + if node_state == "runtime_only": + run["focus_level"] = "warning" + run["focus_summary"] = "接入工单已生成，但当前只有 runtime 心跳，Node Agent 还未真正接管。" + return + if node_state == "stale": + run["focus_level"] = "warning" + run["focus_summary"] = "接入工单已生成，但 Agent 心跳已过期，建议先看任务详情和 Agent 日志。" + return + + run["focus_level"] = "success" if int(state_counts.get("online", 0) or 0) >= total else "warning" + if int(state_counts.get("online", 0) or 0) >= total: + run["focus_summary"] = f"已为 {total} 个节点生成接入工单，目标节点已全部完成 Agent 接入，可继续执行接管后验收。" + return + + state_text = " / ".join(state_parts) if state_parts else "状态待确认" + run["focus_summary"] = ( + f"已为 {total} 个节点生成接入工单，当前状态：{state_text}。" + " 下一步可进入任务详情复制 bootstrap 脚本，并在节点执行后再做接管验收。" + ) + + +def _build_playbook_job_metadata( + *, + playbook: dict, + preview: dict, + step: dict, + run_code: str, + requested_by: str, + execution_mode: str, +) -> dict: + return { + "playbook": { + "run_code": run_code, + "key": str(playbook.get("key") or "").strip(), + "title": str(playbook.get("title") or "").strip(), + "group_key": str(playbook.get("group_key") or "").strip(), + "group_title": str(playbook.get("group_title") or "").strip(), + "requested_by": requested_by, + "execution_mode": execution_mode, + "execution_mode_label": execution_mode_label(execution_mode), + "step_key": str(step.get("step_key") or "").strip(), + "step_title": str(step.get("title") or "").strip(), + "step_order": int(step.get("order") or 0), + "step_count": int(preview.get("step_count") or 0), + "expected_jobs_total": int(preview.get("expected_jobs_total") or 0), + "target_nodes_total": int(preview.get("target_nodes_total") or 0), + "stop_on_failure": bool(playbook.get("stop_on_failure", True)), + "template_key": str(step.get("template_key") or "").strip(), + } + } + + +def _list_playbook_run_jobs(run_code: str, *, scan_limit: int = 1000) -> list[dict]: + normalized_run_code = str(run_code or "").strip() + if not normalized_run_code: + return [] + safe_scan_limit = min(max(int(scan_limit or 1000), 1), 2000) + jobs = list_ops_jobs(limit=safe_scan_limit, compact=False) + return [ + job + for job in jobs + if str(((job.get("metadata") or {}).get("playbook") or {}).get("run_code") or "").strip() == normalized_run_code + ] + + +def _group_playbook_runs_from_jobs(jobs: list[dict], *, limit: int = 12, run_code_filter: str = "") -> dict: + safe_limit = min(max(int(limit or 12), 1), 200) + normalized_filter = str(run_code_filter or "").strip() + grouped_runs: dict[str, dict] = {} + + for job in jobs: + playbook_meta = dict((job.get("metadata") or {}).get("playbook") or {}) + run_code = str(playbook_meta.get("run_code") or "").strip() + if not run_code: + continue + if normalized_filter and run_code != normalized_filter: + continue + step_key = str(playbook_meta.get("step_key") or "").strip() + step_order = int(playbook_meta.get("step_order") or 0) + step_title = str(playbook_meta.get("step_title") or step_key or "").strip() + template_key = str(playbook_meta.get("template_key") or "").strip() + job_status = str(job.get("status") or "").strip() + target_node_code = str(job.get("target_node_code") or "").strip() + latest_time = _latest_job_time(job) + + run = grouped_runs.get(run_code) + if not run: + run = { + "run_code": run_code, + "playbook_key": str(playbook_meta.get("key") or "").strip(), + "playbook_title": str(playbook_meta.get("title") or "").strip(), + "group_key": str(playbook_meta.get("group_key") or "").strip(), + "group_title": str(playbook_meta.get("group_title") or "").strip(), + "requested_by": str(playbook_meta.get("requested_by") or job.get("requested_by") or "").strip(), + "execution_mode": str(playbook_meta.get("execution_mode") or job.get("execution_mode") or "").strip(), + "execution_mode_label": ( + str(playbook_meta.get("execution_mode_label") or "").strip() + or execution_mode_label(str(playbook_meta.get("execution_mode") or job.get("execution_mode") or "").strip()) + ), + "stop_on_failure": bool(playbook_meta.get("stop_on_failure", True)), + "step_count": int(playbook_meta.get("step_count") or 0), + "expected_jobs_total": int(playbook_meta.get("expected_jobs_total") or 0), + "target_nodes_total": int(playbook_meta.get("target_nodes_total") or 0), + "target_node_codes": [], + "created_at": str(job.get("created_at") or ""), + "updated_at": latest_time, + "status_counts": {}, + "jobs_total": 0, + "success_jobs_total": 0, + "terminal_jobs_total": 0, + "steps": [], + "latest_job": {}, + "_target_node_set": set(), + "_step_map": {}, + } + grouped_runs[run_code] = run + + run["jobs_total"] = int(run.get("jobs_total", 0) or 0) + 1 + _increment_status(run["status_counts"], job_status) + if job_status == "success": + run["success_jobs_total"] = int(run.get("success_jobs_total", 0) or 0) + 1 + if job_status == "success" or job_status in _PLAYBOOK_RUN_TERMINAL_PROBLEM_STATUSES: + run["terminal_jobs_total"] = int(run.get("terminal_jobs_total", 0) or 0) + 1 + if target_node_code and target_node_code not in run["_target_node_set"]: + run["_target_node_set"].add(target_node_code) + run["target_node_codes"].append(target_node_code) + created_at = str(job.get("created_at") or "").strip() + if created_at and (not str(run.get("created_at") or "").strip() or created_at < str(run.get("created_at") or "").strip()): + run["created_at"] = created_at + if latest_time and latest_time >= str(run.get("updated_at") or "").strip(): + run["updated_at"] = latest_time + run["latest_job"] = _slim_playbook_job(job) + + step_map = run["_step_map"] + step_row = step_map.get(step_key) + if not step_row: + step_row = { + "step_key": step_key, + "title": step_title, + "template_key": template_key, + "order": step_order, + "jobs_total": 0, + "nodes_total": 0, + "status_counts": {}, + "terminal_jobs_total": 0, + "success_jobs_total": 0, + "latest_job": {}, + "_node_set": set(), + } + step_map[step_key] = step_row + run["steps"].append(step_row) + step_row["jobs_total"] = int(step_row.get("jobs_total", 0) or 0) + 1 + _increment_status(step_row["status_counts"], job_status) + if job_status == "success": + step_row["success_jobs_total"] = int(step_row.get("success_jobs_total", 0) or 0) + 1 + if job_status == "success" or job_status in _PLAYBOOK_RUN_TERMINAL_PROBLEM_STATUSES: + step_row["terminal_jobs_total"] = int(step_row.get("terminal_jobs_total", 0) or 0) + 1 + if target_node_code and target_node_code not in step_row["_node_set"]: + step_row["_node_set"].add(target_node_code) + step_row["nodes_total"] = len(step_row["_node_set"]) + current_latest = _latest_job_time(step_row.get("latest_job") or {}) + if latest_time and latest_time >= current_latest: + step_row["latest_job"] = _slim_playbook_job(job) + + runs = list(grouped_runs.values()) + managed_node_map: dict[str, dict] = {} + try: + managed_nodes_payload = list_managed_nodes_with_agent_state() + managed_node_map = { + str(item.get("node_code") or "").strip(): dict(item or {}) + for item in list(managed_nodes_payload.get("nodes") or []) + if str(item.get("node_code") or "").strip() + } + except Exception: + managed_node_map = {} + + for run in runs: + run["target_nodes_total"] = int(run.get("target_nodes_total") or len(run.get("target_node_codes") or [])) + run["status"] = _derive_run_status( + status_counts=dict(run.get("status_counts") or {}), + jobs_total=int(run.get("jobs_total", 0) or 0), + ) + run["completion_percent"] = round( + (int(run.get("terminal_jobs_total", 0) or 0) / max(int(run.get("jobs_total", 0) or 0), 1)) * 100, + 1, + ) + for step in list(run.get("steps") or []): + step["status"] = _derive_run_status( + status_counts=dict(step.get("status_counts") or {}), + jobs_total=int(step.get("jobs_total", 0) or 0), + ) + step["status_label"] = _playbook_run_status_label(step.get("status")) + step["completion_percent"] = round( + (int(step.get("terminal_jobs_total", 0) or 0) / max(int(step.get("jobs_total", 0) or 0), 1)) * 100, + 1, + ) + step["target_node_codes"] = sorted(str(code or "").strip() for code in step.get("_node_set") or [] if str(code or "").strip()) + step["summary"] = _build_playbook_step_summary_text(step) + step["summary_text"] = str(step.get("summary") or "") + step["focus_ref"] = _build_playbook_run_focus_ref( + run, + step_key=str(step.get("step_key") or ""), + step_title=str(step.get("title") or step.get("step_key") or ""), + ) + step.pop("_node_set", None) + run["steps"] = sorted( + list(run.get("steps") or []), + key=lambda item: (int(item.get("order") or 0), str(item.get("step_key") or "")), + ) + problem_steps: list[dict] = [] + active_steps: list[dict] = [] + for step in list(run.get("steps") or []): + step_status_counts = dict(step.get("status_counts") or {}) + if str(step.get("status") or "") == "attention": + problem_steps.append( + { + "step_key": str(step.get("step_key") or ""), + "title": str(step.get("title") or step.get("step_key") or ""), + "status": str(step.get("status") or ""), + "reason_label": _terminal_problem_label(step_status_counts), + "latest_job": dict(step.get("latest_job") or {}), + } + ) + elif str(step.get("status") or "") in {"running", "queued"}: + active_steps.append( + { + "step_key": str(step.get("step_key") or ""), + "title": str(step.get("title") or step.get("step_key") or ""), + "status": str(step.get("status") or ""), + "reason_label": _active_status_label(step_status_counts), + "latest_job": dict(step.get("latest_job") or {}), + } + ) + run["problem_steps"] = problem_steps + run["active_steps"] = active_steps + if problem_steps: + first_problem = problem_steps[0] + run["focus_level"] = "danger" + run["focus_step_key"] = str(first_problem.get("step_key") or "") + run["focus_step_title"] = str(first_problem.get("title") or "") + run["focus_summary"] = ( + f"{str(first_problem.get('title') or first_problem.get('step_key') or '步骤').strip()} 当前出现" + f"{str(first_problem.get('reason_label') or '异常').strip()}，建议先看事件，再决定是否整轮重跑。" + ) + elif active_steps: + first_active = active_steps[0] + run["focus_level"] = "warning" + run["focus_step_key"] = str(first_active.get("step_key") or "") + run["focus_step_title"] = str(first_active.get("title") or "") + run["focus_summary"] = ( + f"{str(first_active.get('title') or first_active.get('step_key') or '步骤').strip()} 当前处于" + f"{str(first_active.get('reason_label') or '处理中').strip()}，建议先看事件和节点现场输出。" + ) + elif str(run.get("status") or "") == "success": + run["focus_level"] = "success" + run["focus_step_key"] = "" + run["focus_step_title"] = "" + run["focus_summary"] = "当前整轮编排已全部收口。" + else: + run["focus_level"] = "info" + run["focus_step_key"] = "" + run["focus_step_title"] = "" + run["focus_summary"] = "当前编排已创建，等待节点继续执行。" + if str(run.get("playbook_key") or "").strip() == "onboarding.bootstrap": + _enrich_onboarding_bootstrap_run(run, managed_node_map) + run["status_label"] = _playbook_run_status_label(run.get("status")) + run["steps_total"] = len(list(run.get("steps") or [])) + run["steps_success"] = sum( + 1 for step in list(run.get("steps") or []) if str(step.get("status") or "").strip() == "success" + ) + run["steps_running"] = sum( + 1 for step in list(run.get("steps") or []) if str(step.get("status") or "").strip() in {"running", "queued"} + ) + run["steps_problem"] = sum( + 1 for step in list(run.get("steps") or []) if str(step.get("status") or "").strip() == "attention" + ) + run["steps_terminal"] = sum( + 1 for step in list(run.get("steps") or []) if str(step.get("status") or "").strip() in {"success", "attention"} + ) + run["summary"] = _build_playbook_run_summary_text(run) + run["summary_text"] = str(run.get("summary") or "") + run["focus_ref"] = _build_playbook_run_focus_ref(run) + run.pop("_target_node_set", None) + run.pop("_step_map", None) + + runs.sort( + key=lambda item: ( + str(item.get("updated_at") or ""), + str(item.get("created_at") or ""), + str(item.get("run_code") or ""), + ), + reverse=True, + ) + return {"runs": runs[:safe_limit]} + + +def _playbook_catalog() -> list[dict]: + return [ + { + "key": "onboarding.bootstrap", + "group_key": "onboarding", + "group_title": "接管验收", + "title": "生成节点接入工单", + "description": "在控制面为目标节点生成一轮可追踪的 Node Agent 接入工单，产出 env、脚本和一键落地命令。", + "target_scope": "node-set", + "default_execution_mode": "control-plane", + "execution_modes": ["control-plane"], + "default_auto_approve": True, + "stop_on_failure": True, + "steps": [ + { + "step_key": "bootstrap", + "title": "生成 Node Agent 接入工单", + "template_key": "node.bootstrap", + "payload": {}, + } + ], + }, + { + "key": "onboarding.acceptance", + "group_key": "onboarding", + "group_title": "接管验收", + "title": "接管后验收", + "description": "按 健康快照 -> Node Agent 状态/日志 -> Worker 状态/日志 的顺序，对新接管节点做一轮标准接管验收。", + "target_scope": "node-set", + "default_execution_mode": "remote-agent", + "execution_modes": ["remote-agent", "ssh"], + "default_auto_approve": True, + "stop_on_failure": True, + "steps": [ + { + "step_key": "health", + "title": "采集健康快照", + "template_key": "health.snapshot", + "payload": {}, + }, + { + "step_key": "node_agent_status", + "title": "采集 Node Agent 状态", + "template_key": "service.status", + "payload": { + "service_name": _NODE_AGENT_SERVICE_NAME, + }, + }, + { + "step_key": "node_agent_logs", + "title": "收集 Node Agent 日志", + "template_key": "logs.collect", + "payload": { + "service_name": _NODE_AGENT_SERVICE_NAME, + "lines": 120, + }, + }, + { + "step_key": "worker_status", + "title": "采集 Worker 状态", + "template_key": "service.status", + "payload": { + "service_name": settings.worker_service_name, + }, + }, + { + "step_key": "worker_logs", + "title": "收集 Worker 日志", + "template_key": "logs.collect", + "payload": { + "service_name": settings.worker_service_name, + "lines": 120, + }, + }, + ], + }, + { + "key": "inspection.standard", + "group_key": "diagnostics", + "group_title": "标准巡检", + "title": "标准巡检", + "description": "按 健康快照 -> Worker 日志 -> 诊断包 的顺序，对目标节点做一次标准联调巡检。", + "target_scope": "node-set", + "default_execution_mode": "remote-agent", + "execution_modes": ["remote-agent", "ssh"], + "default_auto_approve": True, + "stop_on_failure": True, + "steps": [ + { + "step_key": "health", + "title": "采集健康快照", + "template_key": "health.snapshot", + "payload": {}, + }, + { + "step_key": "worker_logs", + "title": "收集 Worker 日志", + "template_key": "logs.collect", + "payload": { + "service_name": settings.worker_service_name, + "lines": 120, + }, + }, + { + "step_key": "diagnostics", + "title": "收集诊断包", + "template_key": "diagnostics.collect", + "payload": { + "lines": 200, + }, + }, + ], + }, + { + "key": "scene.logs.key", + "group_key": "scene", + "group_title": "执行现场", + "title": "关键现场日志", + "description": "只回收 Worker 的关键日志样本，适合先看现场而不放大量级。", + "target_scope": "node-set", + "default_execution_mode": "remote-agent", + "execution_modes": ["remote-agent", "ssh"], + "default_auto_approve": True, + "stop_on_failure": True, + "steps": [ + { + "step_key": "worker_logs", + "title": "收集 Worker 关键日志", + "template_key": "logs.collect", + "payload": { + "service_name": settings.worker_service_name, + "lines": 120, + }, + } + ], + }, + { + "key": "scene.logs.full", + "group_key": "scene", + "group_title": "执行现场", + "title": "全量现场取证", + "description": "回收较大日志样本并补一份诊断包，适合深入排查远端执行面。", + "target_scope": "node-set", + "default_execution_mode": "remote-agent", + "execution_modes": ["remote-agent", "ssh"], + "default_auto_approve": True, + "stop_on_failure": True, + "steps": [ + { + "step_key": "worker_logs", + "title": "收集 Worker 全量日志", + "template_key": "logs.collect", + "payload": { + "service_name": settings.worker_service_name, + "lines": 300, + }, + }, + { + "step_key": "diagnostics", + "title": "收集诊断包", + "template_key": "diagnostics.collect", + "payload": { + "lines": 300, + }, + }, + ], + }, + { + "key": "scene.diagnostics", + "group_key": "scene", + "group_title": "执行现场", + "title": "现场诊断包", + "description": "直接回收诊断包，适合对参与节点做一次较轻量的远端取证。", + "target_scope": "node-set", + "default_execution_mode": "remote-agent", + "execution_modes": ["remote-agent", "ssh"], + "default_auto_approve": True, + "stop_on_failure": True, + "steps": [ + { + "step_key": "diagnostics", + "title": "收集诊断包", + "template_key": "diagnostics.collect", + "payload": { + "lines": 200, + }, + } + ], + }, + ] + + +def get_ops_playbooks() -> dict: + playbooks = [decorate_execution_mode_fields(item) for item in deepcopy(_playbook_catalog())] + return { + "items": playbooks, + "summary": { + "total": len(playbooks), + "groups": sorted({str(item.get("group_key") or "").strip() for item in playbooks if str(item.get("group_key") or "").strip()}), + }, + } + + +def get_ops_playbook(playbook_key: str) -> dict: + normalized_key = str(playbook_key or "").strip() + if not normalized_key: + return {} + for item in _playbook_catalog(): + if str(item.get("key") or "").strip() == normalized_key: + return decorate_execution_mode_fields(deepcopy(item)) + return {} + + +def preview_ops_playbook(payload: dict | None = None) -> tuple[bool, str, dict]: + source_payload = dict(payload or {}) + playbook_key = str(source_payload.get("playbook_key") or "").strip() + if not playbook_key: + return False, "playbook_key 不能为空", {} + + playbook = get_ops_playbook(playbook_key) + if not playbook: + return False, "playbook 不存在", {} + + node_codes = _normalize_node_codes(source_payload.get("node_codes") or []) + if not node_codes: + return False, "该 playbook 需要 node_codes", {} + + requested_by = str(source_payload.get("requested_by") or "api").strip() or "api" + supported_execution_modes = [ + str(item or "").strip() + for item in list(playbook.get("execution_modes") or []) + if str(item or "").strip() + ] + if not supported_execution_modes: + supported_execution_modes = [str(playbook.get("default_execution_mode") or "remote-agent").strip() or "remote-agent"] + execution_mode = str(source_payload.get("execution_mode") or playbook.get("default_execution_mode") or "remote-agent").strip() or "remote-agent" + if execution_mode not in supported_execution_modes: + return False, f"该 playbook 仅支持 {' / '.join(supported_execution_modes)}", { + "playbook": { + "key": playbook_key, + "title": str(playbook.get("title") or "").strip(), + "execution_modes": supported_execution_modes, + "default_execution_mode": str(playbook.get("default_execution_mode") or "").strip(), + "default_execution_mode_label": str(playbook.get("default_execution_mode_label") or "").strip(), + "execution_mode_options": build_execution_mode_options( + supported_execution_modes, + str(playbook.get("default_execution_mode") or "").strip() or "remote-agent", + ), + } + } + auto_approve = bool(source_payload.get("auto_approve", playbook.get("default_auto_approve", True))) + + steps_preview: list[dict] = [] + template_keys: list[str] = [] + for index, step in enumerate(list(playbook.get("steps") or []), start=1): + template_key = str(step.get("template_key") or "").strip() + template = get_ops_action_template(template_key) + if not template: + return False, f"playbook step 缺少动作模板: {template_key}", {} + normalize_ok, normalize_message, normalized_payload = build_ops_template_payload(template_key, step.get("payload") or {}) + if not normalize_ok: + return False, normalize_message, {} + template_keys.append(template_key) + steps_preview.append( + { + "order": index, + "step_key": str(step.get("step_key") or f"step-{index}").strip() or f"step-{index}", + "title": str(step.get("title") or template.get("title") or template_key).strip() or template_key, + "template_key": template_key, + "action": str(template.get("action") or "").strip(), + "payload": normalized_payload, + "execution_mode": execution_mode, + "execution_mode_label": execution_mode_label(execution_mode), + "auto_approve": auto_approve, + "target_nodes_total": len(node_codes), + "expected_jobs": len(node_codes), + } + ) + + preview = { + "playbook": { + "key": playbook_key, + "title": str(playbook.get("title") or "").strip(), + "group_key": str(playbook.get("group_key") or "").strip(), + "group_title": str(playbook.get("group_title") or "").strip(), + "description": str(playbook.get("description") or "").strip(), + "stop_on_failure": bool(playbook.get("stop_on_failure", True)), + "default_execution_mode": str(playbook.get("default_execution_mode") or "").strip(), + "default_execution_mode_label": str(playbook.get("default_execution_mode_label") or "").strip(), + "execution_modes": supported_execution_modes, + "execution_mode_options": build_execution_mode_options( + supported_execution_modes, + str(playbook.get("default_execution_mode") or "").strip() or "remote-agent", + ), + }, + "requested_by": requested_by, + "execution_mode": execution_mode, + "execution_mode_label": execution_mode_label(execution_mode), + "auto_approve": auto_approve, + "target_nodes_total": len(node_codes), + "target_node_codes": node_codes, + "step_count": len(steps_preview), + "expected_jobs_total": len(node_codes) * len(steps_preview), + "template_keys": template_keys, + "steps": steps_preview, + } + return True, "playbook 预览生成成功", preview + + +def get_recent_ops_playbook_runs( + *, + limit: int = 12, + scan_limit: int = 360, + status: str = "", + group_key: str = "", + query: str = "", +) -> dict: + safe_limit = min(max(int(limit or 12), 1), 50) + safe_scan_limit = min(max(int(scan_limit or max(safe_limit * 30, 120)), safe_limit), 1000) + normalized_status = str(status or "").strip() + normalized_group_key = str(group_key or "").strip() + normalized_query = str(query or "").strip().lower() + jobs = list_ops_jobs(limit=safe_scan_limit, compact=False) + grouped = _group_playbook_runs_from_jobs(jobs, limit=safe_scan_limit) + all_runs = list(grouped.get("runs") or []) + available_status_counts: dict[str, int] = {} + available_group_counts: dict[str, int] = {} + for run in all_runs: + run_status = str(run.get("status") or "").strip() or "queued" + available_status_counts[run_status] = int(available_status_counts.get(run_status, 0) or 0) + 1 + run_group_key = str(run.get("group_key") or "").strip() + if run_group_key: + available_group_counts[run_group_key] = int(available_group_counts.get(run_group_key, 0) or 0) + 1 + + filtered_runs: list[dict] = [] + for run in all_runs: + run_status = str(run.get("status") or "").strip() + run_group_key = str(run.get("group_key") or "").strip() + haystack = " ".join( + [ + str(run.get("run_code") or ""), + str(run.get("playbook_key") or ""), + str(run.get("playbook_title") or ""), + str(run.get("requested_by") or ""), + " ".join(list(run.get("target_node_codes") or [])), + ] + ).lower() + if normalized_status and run_status != normalized_status: + continue + if normalized_group_key and run_group_key != normalized_group_key: + continue + if normalized_query and normalized_query not in haystack: + continue + filtered_runs.append(run) + + runs = filtered_runs[:safe_limit] + summary = { + "total": len(runs), + "filtered_total": len(filtered_runs), + "unfiltered_total": len(all_runs), + "success": sum(1 for item in runs if str(item.get("status") or "") == "success"), + "running": sum(1 for item in runs if str(item.get("status") or "") == "running"), + "attention": sum(1 for item in runs if str(item.get("status") or "") == "attention"), + "queued": sum(1 for item in runs if str(item.get("status") or "") == "queued"), + "scanned_jobs": len(jobs), + "available_status_counts": available_status_counts, + "available_group_counts": available_group_counts, + "filters": { + "status": normalized_status, + "group_key": normalized_group_key, + "query": normalized_query, + }, + } + return { + "runs": runs, + "summary": summary, + } + + +def get_ops_playbook_run(run_code: str, *, scan_limit: int = 1000) -> dict: + normalized_run_code = str(run_code or "").strip() + if not normalized_run_code: + return {} + matched_jobs = _list_playbook_run_jobs(normalized_run_code, scan_limit=scan_limit) + grouped = _group_playbook_runs_from_jobs(matched_jobs, limit=1, run_code_filter=normalized_run_code) + return next(iter(grouped.get("runs") or []), {}) + + +def list_ops_playbook_run_events( + run_code: str, + *, + limit: int = 80, + step_key: str = "", + node_code: str = "", + scan_limit: int = 1000, +) -> dict: + normalized_run_code = str(run_code or "").strip() + normalized_step_key = str(step_key or "").strip() + normalized_node_code = str(node_code or "").strip() + safe_limit = min(max(int(limit or 80), 1), 300) + matched_jobs = _list_playbook_run_jobs(normalized_run_code, scan_limit=scan_limit) + if not matched_jobs: + return { + "run_code": normalized_run_code, + "events": [], + "summary": { + "total": 0, + "returned_total": 0, + "available_step_counts": {}, + "available_node_counts": {}, + "level_counts": {}, + "event_type_counts": {}, + "job_status_counts": {}, + "latest_at": "", + "filters": { + "step_key": normalized_step_key, + "node_code": normalized_node_code, + "limit": safe_limit, + }, + }, + } + + playbook_run = _group_playbook_runs_from_jobs(matched_jobs, limit=1, run_code_filter=normalized_run_code) + run_payload = next(iter(playbook_run.get("runs") or []), {}) + job_map = {int(job.get("id") or 0): dict(job or {}) for job in matched_jobs if int(job.get("id") or 0) > 0} + raw_events = list_ops_job_events_for_jobs(list(job_map.keys()), limit=min(max(safe_limit * 4, 120), 800)) + + available_step_counts: dict[str, int] = {} + available_node_counts: dict[str, int] = {} + level_counts: dict[str, int] = {} + event_type_counts: dict[str, int] = {} + job_status_counts: dict[str, int] = {} + events: list[dict] = [] + latest_at = "" + + for event in raw_events: + job = job_map.get(int(event.get("job_id") or 0), {}) + playbook_meta = dict((job.get("metadata") or {}).get("playbook") or {}) + event_step_key = str(playbook_meta.get("step_key") or "").strip() + event_step_title = str(playbook_meta.get("step_title") or event_step_key or "").strip() + event_node_code = str(event.get("node_code") or job.get("target_node_code") or "").strip() + if event_step_key: + available_step_counts[event_step_key] = int(available_step_counts.get(event_step_key, 0) or 0) + 1 + if event_node_code: + available_node_counts[event_node_code] = int(available_node_counts.get(event_node_code, 0) or 0) + 1 + event_level = str(event.get("level") or "").strip() or "info" + level_counts[event_level] = int(level_counts.get(event_level, 0) or 0) + 1 + event_type = str(event.get("event_type") or "").strip() or "event" + event_type_counts[event_type] = int(event_type_counts.get(event_type, 0) or 0) + 1 + job_status = str(job.get("status") or "").strip() or "queued" + job_status_counts[job_status] = int(job_status_counts.get(job_status, 0) or 0) + 1 + event_created_at = str(event.get("created_at") or "").strip() + if event_created_at and event_created_at > latest_at: + latest_at = event_created_at + if normalized_step_key and event_step_key != normalized_step_key: + continue + if normalized_node_code and event_node_code != normalized_node_code: + continue + event_summary_text = str(event.get("summary_text") or event.get("summary") or event.get("message") or "").strip() + event_source_focus_ref = _normalize_focus_ref(event.get("focus_ref")) + events.append( + { + **event, + "run_code": normalized_run_code, + "job_code": str(job.get("job_code") or ""), + "job_status": job_status, + "job_status_label": _playbook_run_status_label(job_status), + "action": str(job.get("action") or ""), + "target_node_code": str(job.get("target_node_code") or ""), + "step_key": event_step_key, + "step_title": event_step_title, + "summary_text": event_summary_text, + "source_focus_ref": event_source_focus_ref, + "focus_ref": _build_playbook_run_focus_ref( + run_payload, + step_key=event_step_key, + step_title=event_step_title, + event_key=str(event.get("event_key") or "").strip(), + node_code=event_node_code, + ), + } + ) + + return { + "run_code": normalized_run_code, + "playbook_run": run_payload, + "events": events[:safe_limit], + "summary": { + "total": len(events), + "returned_total": len(events), + "available_step_counts": available_step_counts, + "available_node_counts": available_node_counts, + "level_counts": level_counts, + "event_type_counts": event_type_counts, + "job_status_counts": job_status_counts, + "latest_at": latest_at, + "filters": { + "step_key": normalized_step_key, + "node_code": normalized_node_code, + "limit": safe_limit, + }, + }, + } + + +def rerun_ops_playbook_run(run_code: str, *, requested_by: str = "api") -> tuple[bool, str, dict]: + playbook_run = get_ops_playbook_run(run_code) + if not playbook_run: + return False, "playbook run 不存在", {} + playbook_key = str(playbook_run.get("playbook_key") or "").strip() + node_codes = _normalize_node_codes(playbook_run.get("target_node_codes") or []) + if not playbook_key or not node_codes: + return False, "当前 playbook run 缺少重跑所需上下文", {"playbook_run": playbook_run} + return execute_ops_playbook( + { + "playbook_key": playbook_key, + "node_codes": node_codes, + "execution_mode": str(playbook_run.get("execution_mode") or "remote-agent").strip() or "remote-agent", + "auto_approve": True, + "requested_by": f"{str(requested_by or 'api').strip() or 'api'}/rerun/{playbook_key}", + } + ) + + +def cancel_ops_playbook_run(run_code: str, *, cancelled_by: str = "api", reason: str = "") -> tuple[bool, str, dict]: + playbook_run = get_ops_playbook_run(run_code) + if not playbook_run: + return False, "playbook run 不存在", {} + + normalized_run_code = str(run_code or "").strip() + matched_jobs = _list_playbook_run_jobs(normalized_run_code, scan_limit=2000) + if not matched_jobs: + return False, "当前 playbook run 没有关联任务", {"playbook_run": playbook_run} + + cancellable_jobs = [ + job + for job in matched_jobs + if str(job.get("status") or "").strip() not in {"success", "failed", "cancelled"} + ] + if not cancellable_jobs: + return False, "当前 playbook run 已全部收口，无需取消", { + "playbook_run": playbook_run, + "jobs_total": len(matched_jobs), + "cancelled_count": 0, + } + + cancelled_count = 0 + failed_count = 0 + results: list[dict] = [] + for job in cancellable_jobs: + job_id = int(job.get("id") or 0) + ok, message, data = cancel_ops_job( + job_id, + cancelled_by=str(cancelled_by or "api").strip() or "api", + reason=reason, + ) + if ok: + cancelled_count += 1 + else: + failed_count += 1 + results.append( + { + "job_id": job_id, + "job_code": str(job.get("job_code") or ""), + "ok": ok, + "message": message, + "status": str(((data or {}).get("job") or {}).get("status") or job.get("status") or ""), + } + ) + + latest_run = get_ops_playbook_run(run_code) + return failed_count <= 0, ( + f"已取消 {cancelled_count} 条编排子任务" + if failed_count <= 0 + else f"编排取消部分完成，成功 {cancelled_count}，失败 {failed_count}" + ), { + "playbook_run": latest_run or playbook_run, + "jobs_total": len(matched_jobs), + "cancellable_jobs_total": len(cancellable_jobs), + "cancelled_count": cancelled_count, + "failed_count": failed_count, + "results": results, + } + + +def execute_ops_playbook(payload: dict | None = None) -> tuple[bool, str, dict]: + preview_ok, preview_message, preview = preview_ops_playbook(payload) + if not preview_ok: + return False, preview_message, {} + + playbook = dict(preview.get("playbook") or {}) + requested_by = str(preview.get("requested_by") or "api").strip() or "api" + node_codes = _normalize_node_codes(preview.get("target_node_codes") or []) + execution_mode = str(preview.get("execution_mode") or "remote-agent").strip() or "remote-agent" + auto_approve = bool(preview.get("auto_approve", True)) + stop_on_failure = bool(playbook.get("stop_on_failure", True)) + run_code = _build_playbook_run_code() + + results: list[dict] = [] + successful_steps = 0 + failed_steps = 0 + + for step in list(preview.get("steps") or []): + template_key = str(step.get("template_key") or "").strip() + step_key = str(step.get("step_key") or template_key).strip() or template_key + ok, message, data = create_ops_job_batch( + { + "template_key": template_key, + "target_node_codes": node_codes, + "execution_mode": execution_mode, + "auto_approve": auto_approve, + "requested_by": f"{requested_by}/playbook/{playbook.get('key') or ''}/{step_key}", + "payload": dict(step.get("payload") or {}), + "metadata": _build_playbook_job_metadata( + playbook=playbook, + preview=preview, + step=step, + run_code=run_code, + requested_by=requested_by, + execution_mode=execution_mode, + ), + } + ) + results.append( + { + "run_code": run_code, + "step_key": step_key, + "template_key": template_key, + "title": str(step.get("title") or "").strip(), + "ok": ok, + "message": message, + "data": data or {}, + } + ) + if ok: + successful_steps += 1 + continue + failed_steps += 1 + if stop_on_failure: + playbook_runs = get_recent_ops_playbook_runs(limit=20, scan_limit=max(len(node_codes) * max(len(results), 1) * 4, 120)) + playbook_run = next( + (item for item in list(playbook_runs.get("runs") or []) if str(item.get("run_code") or "").strip() == run_code), + {}, + ) + return False, message, { + "playbook": playbook, + "mode": "ops-playbook", + "run_code": run_code, + "requested_by": requested_by, + "execution_mode": execution_mode, + "target_node_codes": node_codes, + "successful_steps": successful_steps, + "failed_steps": failed_steps, + "results": results, + "preview": preview, + "playbook_run": playbook_run, + } + + playbook_runs = get_recent_ops_playbook_runs(limit=20, scan_limit=max(len(node_codes) * max(len(results), 1) * 4, 120)) + playbook_run = next( + (item for item in list(playbook_runs.get("runs") or []) if str(item.get("run_code") or "").strip() == run_code), + {}, + ) + return failed_steps <= 0, ( + f"playbook {playbook.get('title') or playbook.get('key') or ''} 已创建 {successful_steps} 个步骤，回执 {run_code}" + if failed_steps <= 0 + else f"playbook 已部分创建，成功 {successful_steps}，失败 {failed_steps}，回执 {run_code}" + ), { + "playbook": playbook, + "mode": "ops-playbook", + "run_code": run_code, + "requested_by": requested_by, + "execution_mode": execution_mode, + "target_node_codes": node_codes, + "successful_steps": successful_steps, + "failed_steps": failed_steps, + "results": results, + "preview": preview, + "playbook_run": playbook_run, + } diff --git a/domain-api/app/services/ops_policy_service.py b/domain-api/app/services/ops_policy_service.py new file mode 100644 index 0000000..eddc79f --- /dev/null +++ b/domain-api/app/services/ops_policy_service.py @@ -0,0 +1,494 @@ +from __future__ import annotations + +from app.core.config import settings +from app.services.cluster_runtime_service import get_cluster_snapshot +from app.services.ops_execution_capability_service import validate_action_execution_mode +from app.services.ops_release_service import ( + _calculate_total_batches, + _normalize_rollout_policy, + _resolve_rollout_targets, + build_release_rollout_gate, + get_release, +) + + +_LOW_RISK_ACTIONS = { + "health.snapshot", + "runtime.push_sync", + "runtime.pull_tasks", + "logs.collect", + "diagnostics.collect", + "delivery.queue.flush", +} + +_MEDIUM_RISK_ACTIONS = { + "runtime.start_worker", + "runtime.start_sync_agent", + "service.status", + "health.check", + "delivery.queue.replay", +} + +_HIGH_RISK_ACTIONS = { + "runtime.stop_worker", + "runtime.stop_sync_agent", + "runtime.restart_api", + "service.restart", + "deploy.release", + "delivery.queue.discard", +} + +_CRITICAL_RISK_ACTIONS = { + "deploy.rollback", + "node.bootstrap", + "cluster.reconfigure", +} + + +def _normalize_node_code_list(raw_value: object) -> list[str]: + if isinstance(raw_value, list): + return [str(item).strip() for item in raw_value if str(item).strip()] + text = str(raw_value or "").replace("\r", "\n").strip() + if not text: + return [] + normalized = text.replace(",", "\n") + return [item.strip() for item in normalized.split("\n") if item.strip()] + + +def _dedupe_text_items(items: list[str]) -> list[str]: + return list(dict.fromkeys(str(item or "").strip() for item in items if str(item or "").strip())) + + +def _risk_level_for_action(action: str) -> str: + if action in _LOW_RISK_ACTIONS: + return "low" + if action in _HIGH_RISK_ACTIONS: + return "high" + if action in _CRITICAL_RISK_ACTIONS: + return "critical" + return "medium" + + +def _compact_target_node(item: dict) -> dict: + return { + "node_code": str(item.get("node_code") or ""), + "region": str(item.get("region") or ""), + "role": str(item.get("role") or ""), + "status": str(item.get("status") or ""), + "current_load": int(item.get("current_load", 0) or 0), + "is_effective_worker": bool(item.get("is_effective_worker", False)), + "detect_participating": bool(item.get("detect_participating", False)), + "is_enabled": bool(item.get("is_enabled", True)), + "deploy_channel": str(item.get("deploy_channel") or ""), + } + + +def _preview_action_payload_guardrails( + action: str, + payload: dict, + *, + execution_mode: str, + target_nodes_total: int, +) -> dict: + warnings: list[str] = [] + blocking_reasons: list[str] = [] + approval_reasons: list[str] = [] + recommendations: list[str] = [] + + if action == "deploy.release": + release_version = str(payload.get("release_version") or "").strip() + artifact_url = str(payload.get("artifact_url") or "").strip() + if not release_version: + blocking_reasons.append("deploy.release 缺少 release_version，当前不能创建正式发布任务。") + if not artifact_url: + blocking_reasons.append("deploy.release 缺少 artifact_url，当前不能创建正式发布任务。") + if execution_mode not in {"remote-agent", "ssh"}: + blocking_reasons.append("deploy.release 仅支持 remote-agent 或 ssh 执行方式。") + if target_nodes_total > 1: + recommendations.append("建议先单节点 canary 验证发布任务，再逐步扩到更多节点。") + + if action == "node.bootstrap": + if execution_mode != "control-plane": + blocking_reasons.append("node.bootstrap 仅支持 control-plane 执行方式。") + if target_nodes_total > 1: + recommendations.append("接管动作建议按单节点节奏推进，先确认首台节点接入成功后再继续放量。") + + if action in _CRITICAL_RISK_ACTIONS: + approval_reasons.append("该动作属于 critical 风险动作，正式环境必须审批。") + elif action in _HIGH_RISK_ACTIONS: + approval_reasons.append("该动作属于 high 风险动作，建议审批后执行。") + + return { + "warnings": _dedupe_text_items(warnings), + "blocking_reasons": _dedupe_text_items(blocking_reasons), + "approval_reasons": _dedupe_text_items(approval_reasons), + "recommendations": _dedupe_text_items(recommendations), + } + + +def _preview_single_node_policy( + action: str, + *, + target_node_code: str, + cluster: dict, + payload: dict | None = None, + execution_mode: str = "remote-agent", + target_type: str = "node", + include_payload_guardrails: bool = True, +) -> dict: + payload = dict(payload or {}) + nodes = list(cluster.get("nodes") or []) + summary = cluster.get("summary") or {} + target_node = next((item for item in nodes if str(item.get("node_code") or "").strip() == target_node_code), {}) + + risk_level = _risk_level_for_action(action) + warnings: list[str] = [] + blocking_reasons: list[str] = [] + approval_reasons: list[str] = [] + recommendations: list[str] = [] + + execution_supported, execution_reason = validate_action_execution_mode( + action, + execution_mode, + target_node_code=target_node_code, + current_node_code=settings.node_code, + ) + if not execution_supported and execution_reason: + blocking_reasons.append(execution_reason) + + if target_type == "node" and target_node_code and not target_node: + warnings.append("目标节点当前未出现在集群快照中，可能尚未注册或已离线。") + + node_status = str(target_node.get("status") or "") + node_role = str(target_node.get("role") or "") + node_current_load = int(target_node.get("current_load", 0) or 0) + node_effective_worker = bool(target_node.get("is_effective_worker", False)) + node_detect_participating = bool(target_node.get("detect_participating", False)) + + if target_node: + if node_status in {"busy"} and action in {"runtime.stop_worker", "runtime.restart_api", "deploy.release", "deploy.rollback", "service.restart"}: + blocking_reasons.append("目标节点当前处于 busy 状态，不适合直接执行中断类动作。") + + if node_detect_participating and action in {"runtime.stop_worker", "runtime.restart_api", "deploy.release", "deploy.rollback", "service.restart"}: + blocking_reasons.append("目标节点正在参与检测，需先迁移负载或人工确认后再执行。") + + if node_role == "control" and action in {"runtime.restart_api", "deploy.release", "deploy.rollback", "service.restart"}: + approval_reasons.append("目标节点是 control 节点，建议强制走审批或维护窗口。") + + if node_effective_worker and int(summary.get("online_worker_nodes", 0) or 0) <= 1 and action in {"runtime.stop_worker", "deploy.release", "deploy.rollback", "service.restart"}: + blocking_reasons.append("当前有效执行节点数不足 2，执行该动作可能导致检测能力中断。") + + if node_current_load > 0: + warnings.append(f"目标节点当前负载为 {node_current_load}，建议优先观察或转移任务。") + + if include_payload_guardrails: + payload_guardrails = _preview_action_payload_guardrails( + action, + payload, + execution_mode=execution_mode, + target_nodes_total=1, + ) + warnings.extend(payload_guardrails.get("warnings") or []) + blocking_reasons.extend(payload_guardrails.get("blocking_reasons") or []) + approval_reasons.extend(payload_guardrails.get("approval_reasons") or []) + recommendations.extend(payload_guardrails.get("recommendations") or []) + + warnings = _dedupe_text_items(warnings) + blocking_reasons = _dedupe_text_items(blocking_reasons) + approval_reasons = _dedupe_text_items(approval_reasons) + recommendations = _dedupe_text_items(recommendations) + + approval_required = bool(approval_reasons) + blocked = bool(blocking_reasons) + + return { + "action": action, + "target_type": target_type, + "target_node_code": target_node_code, + "execution_mode": execution_mode, + "risk_level": risk_level, + "approval_required": approval_required, + "blocked": blocked, + "blocking_reasons": blocking_reasons, + "approval_reasons": approval_reasons, + "warnings": warnings, + "recommendations": recommendations, + "target_node": { + "node_code": str(target_node.get("node_code") or ""), + "region": str(target_node.get("region") or ""), + "role": node_role, + "status": node_status, + "current_load": node_current_load, + "is_effective_worker": node_effective_worker, + "detect_participating": node_detect_participating, + } if target_node else {}, + "cluster_guardrails": { + "online_worker_nodes": int(summary.get("online_worker_nodes", 0) or 0), + "dedicated_online_worker_nodes": int(summary.get("dedicated_online_worker_nodes", 0) or 0), + "online_control_nodes": int(summary.get("online_control_nodes", 0) or 0), + "busy_nodes": list(summary.get("busy_nodes") or []), + }, + } + + +def _preview_node_batch_policy(action: str, payload: dict, cluster: dict) -> dict: + target_node_codes = _normalize_node_code_list(payload.get("target_node_codes") or payload.get("node_codes")) + execution_mode = str(payload.get("execution_mode") or "remote-agent").strip() or "remote-agent" + action_payload = dict(payload.get("payload") or {}) + summary = cluster.get("summary") or {} + + warnings: list[str] = [] + blocking_reasons: list[str] = [] + approval_reasons: list[str] = [] + recommendations: list[str] = [] + + if not target_node_codes: + blocking_reasons.append("当前没有可预检的目标节点。") + + shared_guardrails = _preview_action_payload_guardrails( + action, + action_payload, + execution_mode=execution_mode, + target_nodes_total=len(target_node_codes), + ) + warnings.extend(shared_guardrails.get("warnings") or []) + blocking_reasons.extend(shared_guardrails.get("blocking_reasons") or []) + approval_reasons.extend(shared_guardrails.get("approval_reasons") or []) + recommendations.extend(shared_guardrails.get("recommendations") or []) + + node_previews = [ + _preview_single_node_policy( + action, + target_node_code=node_code, + cluster=cluster, + payload=action_payload, + execution_mode=execution_mode, + target_type="batch", + include_payload_guardrails=False, + ) + for node_code in target_node_codes + ] + + blocked_nodes = [item for item in node_previews if bool(item.get("blocked", False))] + approval_nodes = [item for item in node_previews if bool(item.get("approval_required", False))] + warning_nodes = [item for item in node_previews if list(item.get("warnings") or [])] + missing_nodes = [item for item in node_previews if not (item.get("target_node") or {}).get("node_code")] + target_control_nodes = [item for item in node_previews if str((item.get("target_node") or {}).get("role") or "") == "control"] + + if blocked_nodes: + blocking_reasons.append(f"共有 {len(blocked_nodes)} 个目标节点存在节点级阻断，需先处理后再批量执行。") + if approval_nodes: + approval_reasons.append(f"共有 {len(approval_nodes)} 个目标节点建议先审批。") + if warning_nodes: + warnings.append(f"共有 {len(warning_nodes)} 个目标节点存在需关注事项。") + if missing_nodes: + warnings.append(f"共有 {len(missing_nodes)} 个目标节点未出现在当前集群快照中。") + if target_control_nodes and len(target_node_codes) > 1: + recommendations.append("建议把 control 节点放到批次尾部，优先在 worker 或空闲节点验证。") + + warnings = _dedupe_text_items(warnings) + blocking_reasons = _dedupe_text_items(blocking_reasons) + approval_reasons = _dedupe_text_items(approval_reasons) + recommendations = _dedupe_text_items(recommendations) + + return { + "action": action, + "target_type": "batch", + "execution_mode": execution_mode, + "risk_level": _risk_level_for_action(action), + "approval_required": bool(approval_reasons), + "blocked": bool(blocking_reasons), + "blocking_reasons": blocking_reasons, + "approval_reasons": approval_reasons, + "warnings": warnings, + "recommendations": recommendations, + "target_summary": { + "nodes_total": len(target_node_codes), + "blocked_nodes": len(blocked_nodes), + "approval_nodes": len(approval_nodes), + "warning_nodes": len(warning_nodes), + "missing_nodes": len(missing_nodes), + }, + "target_nodes": [ + { + **(item.get("target_node") or {}), + "node_code": str((item.get("target_node") or {}).get("node_code") or item.get("target_node_code") or ""), + "risk_level": str(item.get("risk_level") or ""), + "approval_required": bool(item.get("approval_required", False)), + "blocked": bool(item.get("blocked", False)), + "blocking_reasons": list(item.get("blocking_reasons") or []), + "approval_reasons": list(item.get("approval_reasons") or []), + "warnings": list(item.get("warnings") or []), + } + for item in node_previews + ], + "cluster_guardrails": { + "online_worker_nodes": int(summary.get("online_worker_nodes", 0) or 0), + "dedicated_online_worker_nodes": int(summary.get("dedicated_online_worker_nodes", 0) or 0), + "online_control_nodes": int(summary.get("online_control_nodes", 0) or 0), + "busy_nodes": list(summary.get("busy_nodes") or []), + }, + } + + +def _preview_rollout_policy(action: str, payload: dict, cluster: dict) -> dict: + selector = payload.get("target_selector") or {} + input_policy = payload.get("policy") or {} + release_id = int(payload.get("release_id") or 0) + input_release = dict(payload.get("release") or {}) + release = input_release or (get_release(release_id) if release_id > 0 else {}) + target_nodes = _resolve_rollout_targets(selector) + normalized_policy = _normalize_rollout_policy(input_policy, total_targets=len(target_nodes)) + execution_mode = str(normalized_policy.get("execution_mode") or "remote-agent").strip() or "remote-agent" + summary = cluster.get("summary") or {} + release_gate = build_release_rollout_gate( + release, + target_nodes, + execution_mode=execution_mode, + ) + operational_readiness = dict(release_gate.get("operational_readiness") or {}) + + risk_level = _risk_level_for_action(action) + warnings: list[str] = [] + blocking_reasons: list[str] = [] + approval_reasons: list[str] = [] + recommendations: list[str] = [] + + target_count = len(target_nodes) + first_batch_size = int(normalized_policy.get("first_batch_size") or 0) + batch_size = int(normalized_policy.get("batch_size") or 0) + batches_total = _calculate_total_batches(target_count, normalized_policy) + first_batch_targets = target_nodes[:first_batch_size] if first_batch_size > 0 else [] + + online_worker_nodes = int(summary.get("online_worker_nodes", 0) or 0) + dedicated_online_worker_nodes = int(summary.get("dedicated_online_worker_nodes", 0) or 0) + online_control_nodes = int(summary.get("online_control_nodes", 0) or 0) + + target_control_nodes = [item for item in target_nodes if str(item.get("role") or "") == "control"] + target_worker_nodes = [item for item in target_nodes if str(item.get("role") or "") == "worker"] + target_effective_workers = [item for item in target_nodes if bool(item.get("is_effective_worker", False))] + first_batch_effective_workers = [item for item in first_batch_targets if bool(item.get("is_effective_worker", False))] + first_batch_control_nodes = [item for item in first_batch_targets if str(item.get("role") or "") == "control"] + + busy_targets = [item for item in target_nodes if int(item.get("current_load", 0) or 0) > 0] + participating_targets = [item for item in target_nodes if bool(item.get("detect_participating", False))] + offline_targets = [item for item in target_nodes if str(item.get("status") or "") not in {"online", "busy"}] + + if target_count <= 0: + blocking_reasons.append("当前筛选条件未命中任何可 rollout 的目标节点。") + if release_id > 0 and not input_release and not release: + blocking_reasons.append("指定的 Release 不存在，当前无法预检该 rollout。") + gate_status = str(release_gate.get("status") or "").strip() + if gate_status in {"release_not_ready", "artifact_missing"}: + warnings.append(str(release_gate.get("summary") or "").strip()) + + if offline_targets: + warnings.append(f"命中了 {len(offline_targets)} 个非在线节点，若保留 only_online=false，发布时可能出现排队或失败。") + + if busy_targets: + warnings.append(f"命中了 {len(busy_targets)} 个当前有负载的节点，建议优先确认任务是否可中断。") + + if participating_targets: + warnings.append(f"命中了 {len(participating_targets)} 个正在参与检测的节点，建议优先滚动到空闲节点。") + + if target_control_nodes: + approval_reasons.append("本次 rollout 覆盖 control 节点，建议强制审批，并把 control 放在最后一批。") + + if first_batch_effective_workers and online_worker_nodes > 0 and len(first_batch_effective_workers) >= online_worker_nodes: + blocking_reasons.append("第一批会覆盖全部有效执行节点，检测能力可能瞬时归零。请缩小 first_batch_size。") + + if first_batch_control_nodes and online_control_nodes > 0 and len(first_batch_control_nodes) >= online_control_nodes: + blocking_reasons.append("第一批会覆盖全部在线控制面，运维中枢本身可能不可用。请调整节点顺序或缩小 first_batch_size。") + + if first_batch_size > 1 and target_control_nodes: + warnings.append("当前第一批大于 1，且目标中包含 control 节点；默认建议先 worker、后 control。") + + if action in _CRITICAL_RISK_ACTIONS: + approval_reasons.append("该动作属于 critical 风险动作，正式环境必须审批。") + elif action in _HIGH_RISK_ACTIONS: + approval_reasons.append("该动作属于 high 风险动作，建议审批后执行。") + + blocking_reasons.extend(list(release_gate.get("blocking_reasons") or [])) + warnings.extend(list(release_gate.get("warning_reasons") or [])) + recommendations.extend(list(release_gate.get("recommendations") or [])) + + readiness_summary = dict(operational_readiness.get("summary") or {}) + if target_control_nodes and int(readiness_summary.get("inspection_healthy_nodes", 0) or 0) < target_count: + approval_reasons.append("本次目标中存在巡检未完全通过的节点，建议先完成标准巡检并经人工确认后再发布。") + + if target_count > 1 and first_batch_size > 1: + recommendations.append("正式环境建议 first_batch_size=1，从单节点 canary 开始。") + if target_control_nodes: + recommendations.append("建议把 control 节点放在 rollout 末尾，并确保 worker 已先通过健康检查。") + if busy_targets or participating_targets: + recommendations.append("建议先等参与检测的节点降为空闲，再推进该批次。") + + warnings = list(dict.fromkeys(item for item in warnings if str(item or "").strip())) + blocking_reasons = list(dict.fromkeys(item for item in blocking_reasons if str(item or "").strip())) + approval_reasons = list(dict.fromkeys(item for item in approval_reasons if str(item or "").strip())) + recommendations = list(dict.fromkeys(item for item in recommendations if str(item or "").strip())) + + approval_required = bool(approval_reasons) + blocked = bool(blocking_reasons) + + return { + "action": action, + "target_type": "rollout", + "risk_level": risk_level, + "approval_required": approval_required, + "blocked": blocked, + "blocking_reasons": blocking_reasons, + "approval_reasons": approval_reasons, + "warnings": warnings, + "recommendations": recommendations, + "target_summary": { + "nodes_total": target_count, + "worker_nodes": len(target_worker_nodes), + "control_nodes": len(target_control_nodes), + "effective_worker_nodes": len(target_effective_workers), + "busy_nodes": len(busy_targets), + "participating_nodes": len(participating_targets), + "offline_nodes": len(offline_targets), + }, + "batch_plan": { + "first_batch_size": first_batch_size, + "batch_size": batch_size, + "batches_total": batches_total, + "first_batch_nodes": [_compact_target_node(item) for item in first_batch_targets], + "remaining_after_first_batch": max(target_count - len(first_batch_targets), 0), + }, + "cluster_guardrails": { + "online_worker_nodes": online_worker_nodes, + "dedicated_online_worker_nodes": dedicated_online_worker_nodes, + "online_control_nodes": online_control_nodes, + "busy_nodes": list(summary.get("busy_nodes") or []), + }, + "release": release, + "release_gate": release_gate, + "target_nodes": [_compact_target_node(item) for item in target_nodes], + "operational_readiness": operational_readiness, + } + + +def preview_ops_job_policy(payload: dict) -> dict: + action = str(payload.get("action") or "").strip() + target_type = str(payload.get("target_type") or "node").strip() or "node" + target_node_code = str(payload.get("target_node_code") or "").strip() + execution_mode = str(payload.get("execution_mode") or "remote-agent").strip() or "remote-agent" + action_payload = dict(payload.get("payload") or {}) + + cluster = get_cluster_snapshot() + if target_type == "rollout": + return _preview_rollout_policy(action, payload, cluster) + if target_type in {"batch", "nodes"} or list(payload.get("target_node_codes") or []): + return _preview_node_batch_policy(action, payload, cluster) + + return _preview_single_node_policy( + action, + target_node_code=target_node_code, + cluster=cluster, + payload=action_payload, + execution_mode=execution_mode, + target_type=target_type, + include_payload_guardrails=True, + ) diff --git a/domain-api/app/services/ops_release_executor_core.py b/domain-api/app/services/ops_release_executor_core.py new file mode 100644 index 0000000..77fbeac --- /dev/null +++ b/domain-api/app/services/ops_release_executor_core.py @@ -0,0 +1,475 @@ +from __future__ import annotations + +import hashlib +import inspect +import json +import shutil +import tarfile +import textwrap +import time +import urllib.error +import urllib.request +from datetime import datetime +from pathlib import Path + + +def normalize_text_list(raw_value: object) -> list[str]: + if isinstance(raw_value, list): + return [str(item).strip() for item in raw_value if str(item).strip()] + text = str(raw_value or "").replace("\r", "\n").strip() + if not text: + return [] + normalized = text.replace(",", "\n") + return [item.strip() for item in normalized.split("\n") if item.strip()] + + +def coerce_bool(raw_value: object, default: bool = False) -> bool: + if raw_value is None: + return bool(default) + if isinstance(raw_value, bool): + return raw_value + if isinstance(raw_value, (int, float)): + return bool(raw_value) + normalized = str(raw_value or "").strip().lower() + if normalized in {"1", "true", "yes", "y", "on", "enable", "enabled"}: + return True + if normalized in {"0", "false", "no", "n", "off", "disable", "disabled"}: + return False + return bool(default) if normalized == "" else bool(normalized) + + +def normalize_release_health_check_services( + payload: dict, + restart_services: list[str], + *, + default_api_service_name: str, +) -> list[str]: + raw_health_check_services = payload.get("health_check_services") + if raw_health_check_services is None: + health_check_service_source = restart_services or [default_api_service_name] + else: + health_check_service_source = raw_health_check_services + return normalize_text_list(health_check_service_source) + + +def collect_service_state(run_command, service_name: str) -> dict: + code, stdout, stderr = run_command(["systemctl", "is-active", service_name], timeout=15) + state = stdout or stderr + return { + "service_name": service_name, + "returncode": int(code or 0), + "state": state, + "ok": int(code or 0) == 0 and state == "active", + } + + +def check_health_url(url: str, timeout: int, *, user_agent: str, urlopen_func) -> dict: + request = urllib.request.Request( + url=str(url).strip(), + headers={"User-Agent": str(user_agent or "domaincheck-ops/0.1").strip()}, + method="GET", + ) + try: + with urlopen_func(request, timeout=timeout) as response: + body = response.read(4000).decode("utf-8", errors="ignore") + status_code = int(getattr(response, "status", 0) or response.getcode() or 0) + return { + "url": str(url).strip(), + "ok": 200 <= status_code < 400, + "http_status": status_code, + "body_preview": body[-1000:], + } + except urllib.error.HTTPError as exc: + body = exc.read().decode("utf-8", errors="ignore") if exc.fp else "" + return { + "url": str(url).strip(), + "ok": False, + "http_status": int(exc.code or 0), + "body_preview": body[-1000:], + "error": str(exc), + } + except Exception as exc: + return { + "url": str(url).strip(), + "ok": False, + "http_status": 0, + "body_preview": "", + "error": str(exc), + } + + +def run_release_health_checks( + *, + urls: list[str], + services: list[str], + timeout: int, + retries: int, + interval_seconds: int, + run_command, + user_agent: str, + urlopen_func, +) -> tuple[bool, dict]: + normalized_urls = [str(item).strip() for item in urls if str(item).strip()] + normalized_services = [str(item).strip() for item in services if str(item).strip()] + attempts: list[dict] = [] + total_attempts = max(1, retries + 1) + + for attempt_index in range(1, total_attempts + 1): + service_results = [collect_service_state(run_command, service_name) for service_name in normalized_services] + url_results = [ + check_health_url(url, timeout=timeout, user_agent=user_agent, urlopen_func=urlopen_func) + for url in normalized_urls + ] + services_ok = all(bool(item.get("ok", False)) for item in service_results) if service_results else True + urls_ok = all(bool(item.get("ok", False)) for item in url_results) if url_results else True + attempt_payload = { + "attempt": attempt_index, + "services": service_results, + "urls": url_results, + "ok": services_ok and urls_ok, + } + attempts.append(attempt_payload) + if attempt_payload["ok"]: + return True, { + "ok": True, + "attempts": attempts, + "services_checked": normalized_services, + "urls_checked": normalized_urls, + } + if attempt_index < total_attempts: + time.sleep(max(0, interval_seconds)) + + return False, { + "ok": False, + "attempts": attempts, + "services_checked": normalized_services, + "urls_checked": normalized_urls, + } + + +def safe_extract_tar(archive: tarfile.TarFile, target_dir: Path) -> None: + target_dir_resolved = target_dir.resolve() + members = archive.getmembers() + for member in members: + member_path = (target_dir / member.name).resolve() + if not str(member_path).startswith(str(target_dir_resolved)): + raise RuntimeError(f"unsafe archive member: {member.name}") + try: + archive.extractall(target_dir, members=members, filter="data") + except TypeError: + archive.extractall(target_dir, members=members) + + +def execute_release_action( + payload: dict, + *, + run_command, + default_api_service_name: str, + event_callback=None, + urlopen_func=None, + user_agent: str = "domaincheck-ops/0.1", +) -> tuple[bool, str, dict]: + normalized_payload = dict(payload or {}) + event_callback = event_callback or (lambda event_type, message, level="info", payload=None: None) + urlopen_func = urlopen_func or urllib.request.urlopen + + release_version = str(normalized_payload.get("release_version") or "").strip() + artifact_url = str(normalized_payload.get("artifact_url") or "").strip() + checksum = str(normalized_payload.get("checksum") or "").strip().lower() + install_root = Path(str(normalized_payload.get("install_root") or "/opt/domaincheck")).resolve() + switch_current = coerce_bool(normalized_payload.get("switch_current", True), default=True) + restart_services = normalize_text_list(normalized_payload.get("restart_services")) + health_check_urls = normalize_text_list(normalized_payload.get("health_check_urls")) + health_check_services = normalize_release_health_check_services( + normalized_payload, + restart_services, + default_api_service_name=default_api_service_name, + ) + health_check_timeout_seconds = max(2, int(normalized_payload.get("health_check_timeout_seconds") or 10)) + health_check_retries = max(0, int(normalized_payload.get("health_check_retries") or 2)) + health_check_interval_seconds = max(0, int(normalized_payload.get("health_check_interval_seconds") or 2)) + rollback_on_failure = coerce_bool(normalized_payload.get("rollback_on_failure", True), default=True) + + if not release_version: + return False, "release_version missing", {} + if not artifact_url: + return False, "artifact_url missing", {} + + downloads_dir = install_root / "downloads" + releases_dir = install_root / "releases" + target_dir = releases_dir / release_version + temp_dir = releases_dir / f"{release_version}.tmp" + artifact_path = downloads_dir / f"{release_version}.tar.gz" + current_link = install_root / "current" + previous_current_target = "" + downloads_dir.mkdir(parents=True, exist_ok=True) + releases_dir.mkdir(parents=True, exist_ok=True) + + if current_link.exists(): + try: + previous_current_target = str(current_link.resolve(strict=True)) + except Exception: + try: + previous_current_target = str(current_link.resolve()) + except Exception: + previous_current_target = "" + + event_callback( + "deploy_download_started", + f"开始下载发布包: {release_version}", + payload={"artifact_url": artifact_url}, + ) + request = urllib.request.Request( + url=artifact_url, + headers={"User-Agent": str(user_agent or "domaincheck-ops/0.1").strip()}, + method="GET", + ) + with urlopen_func(request, timeout=120) as response: + artifact_bytes = response.read() + artifact_path.write_bytes(artifact_bytes) + event_callback( + "deploy_download_completed", + f"发布包下载完成: {release_version}", + payload={"artifact_path": str(artifact_path), "size_bytes": len(artifact_bytes)}, + ) + + calculated_checksum = hashlib.sha256(artifact_bytes).hexdigest().lower() + if checksum and checksum != calculated_checksum: + return False, "release checksum mismatch", { + "expected_checksum": checksum, + "calculated_checksum": calculated_checksum, + } + event_callback( + "deploy_checksum_verified", + f"发布包校验通过: {release_version}", + payload={"checksum": calculated_checksum}, + ) + + if temp_dir.exists(): + shutil.rmtree(temp_dir, ignore_errors=True) + + temp_dir.mkdir(parents=True, exist_ok=True) + with tarfile.open(artifact_path, "r:gz") as archive: + safe_extract_tar(archive, temp_dir) + + if target_dir.exists(): + extracted_target = target_dir + shutil.rmtree(temp_dir, ignore_errors=True) + else: + temp_dir.rename(target_dir) + extracted_target = target_dir + + meta_path = extracted_target / ".release-meta.json" + meta_path.write_text( + json.dumps( + { + "release_version": release_version, + "artifact_url": artifact_url, + "checksum": calculated_checksum, + "deployed_at": datetime.now().isoformat(sep=" ", timespec="seconds"), + }, + ensure_ascii=False, + indent=2, + ), + encoding="utf-8", + ) + + if switch_current: + if current_link.is_symlink() or current_link.is_file(): + current_link.unlink(missing_ok=True) + elif current_link.exists(): + raise RuntimeError(f"current link path exists and is not a symlink/file: {current_link}") + current_link.symlink_to(extracted_target) + event_callback( + "deploy_current_switched", + f"current 已切换到 {release_version}", + payload={"current_link": str(current_link), "target": str(extracted_target)}, + ) + + restarted: list[dict] = [] + for service_name in restart_services: + normalized_service_name = str(service_name or "").strip() + if not normalized_service_name: + continue + event_callback( + "deploy_service_restart", + f"重启服务: {normalized_service_name}", + payload={"service_name": normalized_service_name}, + ) + code, stdout, stderr = run_command(["systemctl", "restart", normalized_service_name], timeout=30) + restarted.append( + { + "service_name": normalized_service_name, + "returncode": int(code or 0), + "stdout": stdout, + "stderr": stderr, + } + ) + if int(code or 0) != 0: + return False, f"restart failed: {normalized_service_name}", { + "release_version": release_version, + "release_dir": str(extracted_target), + "artifact_path": str(artifact_path), + "checksum": calculated_checksum, + "previous_current_target": previous_current_target, + "restart_results": restarted, + } + + health_ok, health_result = run_release_health_checks( + urls=health_check_urls, + services=health_check_services, + timeout=health_check_timeout_seconds, + retries=health_check_retries, + interval_seconds=health_check_interval_seconds, + run_command=run_command, + user_agent=user_agent, + urlopen_func=urlopen_func, + ) + if not health_ok: + rollback_result = { + "attempted": rollback_on_failure, + "restored_to": previous_current_target, + "restart_results": [], + "health_result": health_result, + } + event_callback( + "deploy_health_failed", + f"发布健康检查失败: {release_version}", + level="error", + payload=health_result, + ) + if rollback_on_failure and previous_current_target: + if current_link.is_symlink() or current_link.is_file(): + current_link.unlink(missing_ok=True) + elif current_link.exists(): + raise RuntimeError(f"current link path exists and is not a symlink/file: {current_link}") + current_link.symlink_to(Path(previous_current_target)) + for service_name in restart_services: + normalized_service_name = str(service_name or "").strip() + if not normalized_service_name: + continue + code, stdout, stderr = run_command(["systemctl", "restart", normalized_service_name], timeout=30) + rollback_result["restart_results"].append( + { + "service_name": normalized_service_name, + "returncode": int(code or 0), + "stdout": stdout, + "stderr": stderr, + } + ) + rollback_result["post_rollback_health"] = run_release_health_checks( + urls=health_check_urls, + services=health_check_services, + timeout=health_check_timeout_seconds, + retries=0, + interval_seconds=0, + run_command=run_command, + user_agent=user_agent, + urlopen_func=urlopen_func, + )[1] + event_callback( + "deploy_rollback_completed", + f"已回滚到旧版本: {previous_current_target}", + level="warning", + payload=rollback_result, + ) + return False, "release health check failed", { + "release_version": release_version, + "release_dir": str(extracted_target), + "artifact_path": str(artifact_path), + "checksum": calculated_checksum, + "current_link": str(current_link), + "previous_current_target": previous_current_target, + "restart_results": restarted, + "health_check": health_result, + "rollback": rollback_result, + } + + event_callback( + "deploy_health_passed", + f"发布健康检查通过: {release_version}", + payload=health_result, + ) + return True, "release deployed", { + "release_version": release_version, + "release_dir": str(extracted_target), + "artifact_path": str(artifact_path), + "checksum": calculated_checksum, + "current_link": str(current_link), + "previous_current_target": previous_current_target, + "restart_results": restarted, + "health_check": health_result, + } + + +def build_remote_release_action_script( + payload: dict, + *, + default_api_service_name: str = "domaincheck-api", + user_agent: str = "domaincheck-ssh/0.1", +) -> str: + helper_functions = [ + normalize_text_list, + coerce_bool, + normalize_release_health_check_services, + collect_service_state, + check_health_url, + run_release_health_checks, + safe_extract_tar, + execute_release_action, + ] + helper_source = "\n\n".join( + textwrap.dedent(inspect.getsource(func)).strip("\n") + for func in helper_functions + ) + return f"""from __future__ import annotations + +import hashlib +import json +import shutil +import tarfile +import time +import urllib.error +import urllib.request +from datetime import datetime +from pathlib import Path + + +{helper_source} + + +PAYLOAD = {json.dumps(dict(payload or {{}}), ensure_ascii=False)} +DEFAULT_API_SERVICE_NAME = {json.dumps(str(default_api_service_name or 'domaincheck-api'), ensure_ascii=False)} +USER_AGENT = {json.dumps(str(user_agent or 'domaincheck-ssh/0.1'), ensure_ascii=False)} + + +def _run(command, timeout=60): + import subprocess + + completed = subprocess.run(command, capture_output=True, text=True, timeout=timeout) + return int(completed.returncode or 0), str(completed.stdout or "").strip(), str(completed.stderr or "").strip() + + +def _event_callback(event_type, message, level="info", payload=None): + return None + + +def main(): + try: + ok, message, result = execute_release_action( + PAYLOAD, + run_command=_run, + default_api_service_name=DEFAULT_API_SERVICE_NAME, + event_callback=_event_callback, + urlopen_func=urllib.request.urlopen, + user_agent=USER_AGENT, + ) + except Exception as exc: + ok, message, result = False, str(exc), {{"exception": str(exc), "action": "deploy.release"}} + print(json.dumps({{"ok": ok, "message": message, "result": result}}, ensure_ascii=False)) + raise SystemExit(0 if ok else 1) + + +if __name__ == "__main__": + main() +""" diff --git a/domain-api/app/services/ops_release_service.py b/domain-api/app/services/ops_release_service.py new file mode 100644 index 0000000..10871c5 --- /dev/null +++ b/domain-api/app/services/ops_release_service.py @@ -0,0 +1,3448 @@ +from __future__ import annotations + +import json +import os +import re +import subprocess +from datetime import datetime +from math import ceil +from pathlib import Path +from threading import Lock +from uuid import uuid4 + +from app.core.db import get_db +from app.services.build_info_service import get_runtime_build_info +from app.services.ops_command_service import build_bash_command +from app.services.ops_execution_mode_service import execution_mode_label + + +_RELEASE_SCHEMA_SQL = """ +CREATE TABLE IF NOT EXISTS ops_releases ( + id BIGSERIAL PRIMARY KEY, + release_version VARCHAR(128) NOT NULL UNIQUE, + channel VARCHAR(64) NOT NULL DEFAULT 'stable', + commit_sha VARCHAR(64) NOT NULL DEFAULT '', + status VARCHAR(32) NOT NULL DEFAULT 'draft', + artifact_url TEXT NOT NULL DEFAULT '', + checksum VARCHAR(128) NOT NULL DEFAULT '', + notes TEXT NOT NULL DEFAULT '', + metadata_json JSONB, + created_by VARCHAR(64) NOT NULL DEFAULT 'api', + created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + activated_at TIMESTAMP, + updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP +); + +CREATE INDEX IF NOT EXISTS idx_ops_releases_channel_status +ON ops_releases(channel, status, created_at DESC); + +CREATE TABLE IF NOT EXISTS ops_release_rollouts ( + id BIGSERIAL PRIMARY KEY, + release_id BIGINT NOT NULL REFERENCES ops_releases(id) ON DELETE CASCADE, + rollout_code VARCHAR(64) NOT NULL DEFAULT '', + target_selector_json JSONB, + target_nodes_json JSONB, + policy_json JSONB, + status VARCHAR(32) NOT NULL DEFAULT 'planned', + batch_cursor INTEGER NOT NULL DEFAULT 0, + batches_total INTEGER NOT NULL DEFAULT 0, + jobs_total INTEGER NOT NULL DEFAULT 0, + jobs_created INTEGER NOT NULL DEFAULT 0, + result_summary_json JSONB, + created_by VARCHAR(64) NOT NULL DEFAULT 'api', + created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, + updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP +); + +CREATE INDEX IF NOT EXISTS idx_ops_release_rollouts_release +ON ops_release_rollouts(release_id, created_at DESC); + +ALTER TABLE ops_release_rollouts ADD COLUMN IF NOT EXISTS rollout_code VARCHAR(64) NOT NULL DEFAULT ''; +ALTER TABLE ops_release_rollouts ADD COLUMN IF NOT EXISTS target_nodes_json JSONB; +ALTER TABLE ops_release_rollouts ADD COLUMN IF NOT EXISTS batch_cursor INTEGER NOT NULL DEFAULT 0; +ALTER TABLE ops_release_rollouts ADD COLUMN IF NOT EXISTS batches_total INTEGER NOT NULL DEFAULT 0; +ALTER TABLE ops_release_rollouts ADD COLUMN IF NOT EXISTS jobs_total INTEGER NOT NULL DEFAULT 0; +ALTER TABLE ops_release_rollouts ADD COLUMN IF NOT EXISTS jobs_created INTEGER NOT NULL DEFAULT 0; +ALTER TABLE ops_release_rollouts ADD COLUMN IF NOT EXISTS result_summary_json JSONB; +""" + +_ACTIVE_JOB_STATUSES = {"queued", "dispatching", "running", "awaiting_approval"} +_ISSUE_JOB_STATUSES = {"failed", "cancelled", "blocked", "partially_succeeded"} +_ROLLOUT_INSPECTION_ACTION_KEYS = ("health.snapshot", "logs.collect", "diagnostics.collect") +_SAFE_RELEASE_PACKAGE_NAME_RE = re.compile(r"^[A-Za-z0-9._-]+$") +_RELEASE_SCHEMA_LOCK = Lock() +_RELEASE_SCHEMA_READY = False + + +def ensure_ops_release_schema() -> None: + global _RELEASE_SCHEMA_READY + + if _RELEASE_SCHEMA_READY: + return + + with _RELEASE_SCHEMA_LOCK: + if _RELEASE_SCHEMA_READY: + return + with get_db() as conn: + conn.autocommit = False + with conn.cursor() as cur: + cur.execute(_RELEASE_SCHEMA_SQL) + conn.commit() + _RELEASE_SCHEMA_READY = True + + +def _decode_json(value: object) -> dict: + if isinstance(value, dict): + return value + if value in (None, ""): + return {} + try: + return json.loads(value) + except Exception: + return {} + + +def _decode_items(value: object) -> list[dict]: + decoded = _decode_json(value) + items = decoded.get("items") if isinstance(decoded, dict) else [] + return [item for item in list(items or []) if isinstance(item, dict)] + + +def _ts(value: object) -> str: + if isinstance(value, datetime): + return value.isoformat(sep=" ", timespec="seconds") + return "" + + +def _release_status_label(status: object) -> str: + normalized_status = str(status or "").strip() + mapping = { + "draft": "草稿", + "ready": "已就绪", + "active": "已激活", + "archived": "已归档", + } + return mapping.get(normalized_status, normalized_status or "未知") + + +def _rollout_status_label(status: object) -> str: + normalized_status = str(status or "").strip() + mapping = { + "planned": "待推进", + "running": "执行中", + "awaiting_approval": "待审批", + "ready_for_next_batch": "可推进下一批", + "halted": "已暂停", + "completed": "已完成", + "completed_with_issues": "带问题完成", + "failed": "失败", + } + return mapping.get(normalized_status, normalized_status or "未知") + + +def _build_release_focus_ref( + release: dict | None = None, + *, + rollout_id: int = 0, + rollout_code: str = "", + section: str = "", +) -> dict: + normalized_release = dict(release or {}) + return { + "kind": "release_hub", + "release_id": int(normalized_release.get("id") or 0), + "release_version": str(normalized_release.get("release_version") or "").strip(), + "channel": str(normalized_release.get("channel") or "").strip(), + "rollout_id": int(rollout_id or 0), + "rollout_code": str(rollout_code or "").strip(), + "section": str(section or "").strip(), + } + + +def _build_rollout_focus_ref(rollout: dict | None = None, *, section: str = "rollout_jobs") -> dict: + normalized_rollout = dict(rollout or {}) + return { + "kind": "release_rollout", + "rollout_id": int(normalized_rollout.get("id") or 0), + "rollout_code": str(normalized_rollout.get("rollout_code") or "").strip(), + "release_id": int(normalized_rollout.get("release_id") or 0), + "section": str(section or "").strip(), + } + + +def _build_release_summary_text(release: dict | None = None) -> str: + normalized_release = dict(release or {}) + status = str(normalized_release.get("status") or "").strip() + release_version = str(normalized_release.get("release_version") or "").strip() or "当前版本" + artifact_url = str(normalized_release.get("artifact_url") or "").strip() + checksum = str(normalized_release.get("checksum") or "").strip() + + if status == "active": + return f"{release_version} 当前为激活版本，可继续进入 Rollout 评估。" + if status == "ready": + if artifact_url and checksum: + return f"{release_version} 已就绪，制品与校验信息齐备，可进入 Rollout。" + if artifact_url: + return f"{release_version} 已就绪，但建议补齐 checksum 后再推进正式发布。" + return f"{release_version} 已就绪，但仍缺少可直接下发的制品地址。" + if status == "draft": + return f"{release_version} 仍为草稿，建议先补齐制品、校验信息与说明。" + if status == "archived": + return f"{release_version} 已归档，通常不再作为默认发布入口。" + return f"{release_version} 当前状态为 {_release_status_label(status)}。" + + +def _build_rollout_summary_text(rollout: dict | None = None) -> str: + normalized_rollout = dict(rollout or {}) + result_summary = dict(normalized_rollout.get("result_summary") or {}) + status = str(normalized_rollout.get("status") or result_summary.get("status") or "").strip() + target_nodes = list(normalized_rollout.get("target_nodes") or []) + total_targets = len(target_nodes) or int(normalized_rollout.get("jobs_total") or 0) + launched_targets = int(result_summary.get("launched_targets", normalized_rollout.get("batch_cursor", 0)) or 0) + remaining_targets = int(result_summary.get("remaining_targets", max(total_targets - launched_targets, 0)) or 0) + active_jobs = int(result_summary.get("active_jobs", 0) or 0) + issue_jobs = int(result_summary.get("issue_jobs", 0) or 0) + batches_total = int(normalized_rollout.get("batches_total") or result_summary.get("batches_total") or 0) + batch_index = int(result_summary.get("current_batch_index", normalized_rollout.get("batch_cursor", 0)) or 0) + + if status == "running": + return f"当前已覆盖 {launched_targets}/{total_targets} 台节点，仍有 {active_jobs} 条任务执行中。" + if status == "awaiting_approval": + return f"当前批次等待审批，已覆盖 {launched_targets}/{total_targets} 台节点。" + if status == "ready_for_next_batch": + return f"当前批次已收口，可继续推进下一批；剩余节点 {remaining_targets} 台。" + if status == "halted": + return f"当前 rollout 已暂停，存在 {issue_jobs} 条待处理问题任务。" + if status == "completed": + return f"全部 {total_targets} 台目标节点已完成发布收口。" + if status == "completed_with_issues": + return f"发布已完成，但仍残留 {issue_jobs} 条问题任务，建议先复盘再继续。" + if status == "failed": + return f"当前 rollout 执行失败，建议进入任务列表查看失败节点与阻断原因。" + if status == "planned": + batch_text = f"{batch_index}/{batches_total}" if batches_total > 0 else "0/0" + return f"当前 rollout 已创建，目标节点 {total_targets} 台，批次 {batch_text}，等待首批推进。" + return f"当前 rollout 状态为 {_rollout_status_label(status)}。" + + +def _extract_release_manifest(payload: dict | None = None) -> dict: + normalized_payload = dict(payload or {}) + metadata = dict(normalized_payload.get("metadata") or {}) + manifest = dict(metadata.get("manifest") or normalized_payload.get("manifest") or {}) + artifact_kind = ( + str(manifest.get("artifact_kind") or "").strip() + or str(normalized_payload.get("package_type") or manifest.get("package_type") or "").strip() + ) + package_name = ( + str(manifest.get("package_name") or "").strip() + or str(normalized_payload.get("package_name") or "").strip() + or str(normalized_payload.get("release_version") or "").strip() + ) + checksum = ( + str(normalized_payload.get("checksum") or "").strip() + or str(normalized_payload.get("sha256") or "").strip() + or str(manifest.get("checksum") or "").strip() + ) + commit_sha = ( + str(normalized_payload.get("commit_sha") or "").strip() + or str(manifest.get("commit_sha") or "").strip() + ) + built_at = ( + str(manifest.get("built_at") or "").strip() + or str(normalized_payload.get("generated_at") or "").strip() + or str(normalized_payload.get("created_at") or "").strip() + ) + normalized_manifest = { + "artifact_kind": artifact_kind, + "package_name": package_name, + "package_size_bytes": int(manifest.get("package_size_bytes", 0) or 0), + "checksum": checksum, + "build_id": str(manifest.get("build_id") or "").strip(), + "commit_sha": commit_sha, + "built_at": built_at, + "included_services": list(manifest.get("included_services") or []), + "required_env_keys": list(manifest.get("required_env_keys") or []), + "health_checks": list(manifest.get("health_checks") or []), + "rollback_hint": str(manifest.get("rollback_hint") or "").strip(), + } + if any( + [ + normalized_manifest["artifact_kind"], + normalized_manifest["package_name"], + normalized_manifest["checksum"], + normalized_manifest["build_id"], + normalized_manifest["commit_sha"], + normalized_manifest["built_at"], + normalized_manifest["included_services"], + normalized_manifest["required_env_keys"], + normalized_manifest["health_checks"], + normalized_manifest["rollback_hint"], + normalized_manifest["package_size_bytes"] > 0, + ] + ): + return normalized_manifest + return {} + + +def _extract_node_release_version(*candidates: dict) -> str: + keys = ( + "current_release_version", + "release_version", + "active_release_version", + "current_version", + "deployed_release_version", + ) + for candidate in candidates: + normalized_candidate = dict(candidate or {}) + for key in keys: + value = str(normalized_candidate.get(key) or "").strip() + if value: + return value + metadata = dict(normalized_candidate.get("metadata") or {}) + for key in keys: + value = str(metadata.get(key) or "").strip() + if value: + return value + return "" + + +def _build_rollout_readiness_focus_ref(release: dict | None = None, *, node_code: str = "") -> dict: + focus_ref = _build_release_focus_ref(release, section="default_rollout_gate") + if str(node_code or "").strip(): + focus_ref["node_code"] = str(node_code).strip() + return focus_ref + + +def _rollout_readiness_risk_level(*, blocking_reasons: list[str], inspection_status: str) -> str: + if list(blocking_reasons or []): + return "blocked" + normalized_status = str(inspection_status or "").strip() + if normalized_status == "attention": + return "high" + if normalized_status in {"missing", "running"}: + return "medium" + return "low" + + +def _parse_time(value: object) -> datetime | None: + if isinstance(value, datetime): + return value + raw_value = str(value or "").strip() + if not raw_value: + return None + normalized = raw_value.replace("Z", "+00:00") + try: + return datetime.fromisoformat(normalized) + except Exception: + pass + for fmt in ("%Y-%m-%d %H:%M:%S", "%Y-%m-%d %H:%M:%S.%f"): + try: + return datetime.strptime(raw_value, fmt) + except ValueError: + continue + return None + + +def _serialize_release_row(row: tuple) -> dict: + serialized = { + "id": int(row[0]), + "release_version": str(row[1] or ""), + "channel": str(row[2] or ""), + "commit_sha": str(row[3] or ""), + "status": str(row[4] or ""), + "artifact_url": str(row[5] or ""), + "checksum": str(row[6] or ""), + "notes": str(row[7] or ""), + "metadata": _decode_json(row[8]), + "created_by": str(row[9] or ""), + "created_at": _ts(row[10]), + "activated_at": _ts(row[11]), + "updated_at": _ts(row[12]), + } + serialized["manifest"] = _extract_release_manifest(serialized) + serialized["status_label"] = _release_status_label(serialized.get("status")) + serialized["summary"] = _build_release_summary_text(serialized) + serialized["summary_text"] = str(serialized.get("summary") or "") + serialized["focus_ref"] = _build_release_focus_ref(serialized, section="release_detail") + return serialized + + +def _serialize_rollout_row(row: tuple) -> dict: + target_nodes = _decode_items(row[4]) + policy = _decode_json(row[5]) + serialized = { + "id": int(row[0]), + "release_id": int(row[1]), + "rollout_code": str(row[2] or ""), + "target_selector": _decode_json(row[3]), + "target_nodes": target_nodes, + "target_node_codes": [ + str(item.get("node_code") or "").strip() + for item in target_nodes + if str(item.get("node_code") or "").strip() + ], + "policy": policy, + "status": str(row[6] or ""), + "batch_cursor": int(row[7] or 0), + "batches_total": int(row[8] or 0), + "jobs_total": int(row[9] or 0), + "jobs_created": int(row[10] or 0), + "result_summary": _decode_json(row[11]), + "created_by": str(row[12] or ""), + "created_at": _ts(row[13]), + "updated_at": _ts(row[14]), + } + serialized["status_label"] = _rollout_status_label(serialized.get("status")) + serialized["summary"] = _build_rollout_summary_text(serialized) + serialized["summary_text"] = str(serialized.get("summary") or "") + serialized["focus_ref"] = _build_rollout_focus_ref(serialized) + return serialized + + +def _build_rollout_code() -> str: + return f"rollout-{datetime.now().strftime('%Y%m%d%H%M%S')}-{uuid4().hex[:4]}" + + +def _normalize_rollout_policy(policy: dict, *, total_targets: int) -> dict: + input_policy = dict(policy or {}) + safe_total = max(int(total_targets or 0), 0) + requested_batch_size = int(input_policy.get("batch_size") or safe_total or 1) + batch_size = max(1, min(requested_batch_size, safe_total or requested_batch_size)) + requested_first_batch_size = int( + input_policy.get("first_batch_size") + or input_policy.get("canary_size") + or batch_size + ) + first_batch_size = max(1, min(requested_first_batch_size, safe_total or requested_first_batch_size)) + if safe_total == 0: + first_batch_size = 0 + batch_size = 0 + elif batch_size < first_batch_size: + batch_size = first_batch_size + + deploy_payload = dict(input_policy.get("deploy_payload") or {}) + deploy_payload.setdefault("restart_services", []) + deploy_payload.setdefault("health_check_urls", []) + deploy_payload.setdefault("health_check_timeout_seconds", 10) + deploy_payload.setdefault("health_check_retries", 2) + deploy_payload.setdefault("health_check_interval_seconds", 2) + deploy_payload.setdefault("rollback_on_failure", True) + deploy_payload.setdefault("switch_current", True) + + normalized = { + "auto_start": bool(input_policy.get("auto_start", True)), + "auto_dispatch": bool(input_policy.get("auto_dispatch", False)), + "auto_approve": bool(input_policy.get("auto_approve", False)), + "auto_advance": bool(input_policy.get("auto_advance", True)), + "continue_on_failure": bool(input_policy.get("continue_on_failure", False)), + "continue_on_partial": bool(input_policy.get("continue_on_partial", False)), + "execution_mode": str(input_policy.get("execution_mode") or "remote-agent").strip() or "remote-agent", + "batch_size": batch_size, + "first_batch_size": first_batch_size, + "pause_seconds": max(0, int(input_policy.get("pause_seconds") or 0)), + "deploy_payload": deploy_payload, + } + return normalized + + +def _calculate_total_batches(total_targets: int, policy: dict) -> int: + safe_total = max(int(total_targets or 0), 0) + if safe_total <= 0: + return 0 + first_batch_size = max(1, int(policy.get("first_batch_size") or 1)) + batch_size = max(1, int(policy.get("batch_size") or first_batch_size)) + if safe_total <= first_batch_size: + return 1 + remaining = safe_total - first_batch_size + return 1 + int(ceil(remaining / batch_size)) + + +def _decode_payload_json(value: object) -> dict: + return _decode_json(value) + + +def _release_package_root() -> Path: + configured = str(os.environ.get("DOMAINCHECK_RELEASE_ROOT") or "").strip() + if configured: + return Path(configured).expanduser().resolve() + return Path(__file__).resolve().parents[3] / "release" + + +def _release_repo_root() -> Path: + return Path(__file__).resolve().parents[3] + + +def _trim_release_command_output(value: object, limit: int = 4000) -> str: + text = str(value or "").strip() + if len(text) <= int(limit): + return text + return text[-int(limit):] + + +def _safe_release_package_name(package_name: str) -> str: + normalized = str(package_name or "").strip() + if not normalized or not _SAFE_RELEASE_PACKAGE_NAME_RE.fullmatch(normalized): + raise ValueError("invalid package_name") + return normalized + + +def _normalize_public_base_url(raw_value: str) -> str: + normalized = str(raw_value or "").strip().rstrip("/") + if normalized.endswith("/api/v1"): + normalized = normalized[: -len("/api/v1")] + return normalized + + +def _build_absolute_release_package_url(base_url: str, raw_path: str) -> str: + normalized_base = _normalize_public_base_url(base_url) + normalized_path = str(raw_path or "").strip() + if not normalized_path: + return "" + if normalized_path.startswith("http://") or normalized_path.startswith("https://"): + return normalized_path + if not normalized_base: + return "" + if normalized_path.startswith("/"): + return f"{normalized_base}{normalized_path}" + return f"{normalized_base}/{normalized_path}" + + +def _read_latest_release_metadata_file() -> dict: + release_root = _release_package_root() + latest_json_path = release_root / "latest_release.json" + if not latest_json_path.exists(): + return {} + try: + return json.loads(latest_json_path.read_text(encoding="utf-8")) + except Exception: + return {} + + +def _read_final_release_report_file() -> dict: + release_root = _release_package_root() + report_path = release_root / "final_release_report.json" + if not report_path.exists(): + return {} + try: + return json.loads(report_path.read_text(encoding="utf-8")) + except Exception: + return {} + + +def _normalize_release_package_metadata(payload: dict) -> dict: + raw_payload = dict(payload or {}) + package_name = str(raw_payload.get("package_name") or "").strip() + if not package_name: + return {} + + archive_path_value = ( + str(raw_payload.get("archive_path") or "").strip() + or str(raw_payload.get("zip_path") or "").strip() + ) + sha256_path_value = str(raw_payload.get("sha256_path") or "").strip() + staging_path_value = str(raw_payload.get("staging_path") or "").strip() + package_type = str(raw_payload.get("package_type") or "").strip() + archive_filename = "" + if archive_path_value: + archive_filename = Path(archive_path_value).name + if not package_type: + if archive_filename.endswith(".tar.gz"): + package_type = "tar.gz" + elif archive_filename.endswith(".zip"): + package_type = "zip" + + smoke_test_ok = bool(raw_payload.get("smoke_test_ok", False)) + manifest_smoke = {} + commit_sha = str(raw_payload.get("commit_sha") or "").strip() + commit_ref = str(raw_payload.get("commit_ref") or "").strip() + manifest = {} + final_release_report = _read_final_release_report_file() + manifest_path = Path(staging_path_value) / "release_manifest.json" if staging_path_value else None + if manifest_path and manifest_path.exists(): + try: + manifest = json.loads(manifest_path.read_text(encoding="utf-8")) + except Exception: + manifest = {} + manifest_smoke = manifest.get("smoke_test") or {} + if not commit_sha: + commit_sha = str(manifest.get("commit_sha") or "").strip() + if not commit_ref: + commit_ref = str(manifest.get("commit_ref") or "").strip() + smoke_test_report = str( + raw_payload.get("smoke_test_report") + or manifest_smoke.get("report") + or "" + ).strip() + smoke_test_message = str( + raw_payload.get("smoke_test_message") + or manifest_smoke.get("message") + or "" + ).strip() + if "\n" in smoke_test_message or len(smoke_test_message) > 240: + smoke_test_message = f"详见 {smoke_test_report}" if smoke_test_report else smoke_test_message.splitlines()[0][:240] + + final_release_report_available = bool(final_release_report) + final_release_report_ok = bool(final_release_report.get("ok", False)) if final_release_report_available else False + final_release_latest = dict(final_release_report.get("latest_release") or {}) if final_release_report_available else {} + final_release_report_package_name = str(final_release_latest.get("package_name") or "").strip() + final_release_report_archive_path = str( + final_release_latest.get("archive_path") or final_release_latest.get("zip_path") or "" + ).strip() + final_release_report_sha256 = str(final_release_latest.get("sha256") or "").strip().lower() + current_archive_path = archive_path_value.strip() + current_sha256 = str(raw_payload.get("sha256") or "").strip().lower() + final_release_report_matches_latest = False + final_release_report_stale_reason = "" + if final_release_report_available: + mismatches: list[str] = [] + if final_release_report_package_name != package_name: + mismatches.append("package_name") + if final_release_report_archive_path != current_archive_path: + mismatches.append("archive_path") + if final_release_report_sha256 != current_sha256: + mismatches.append("sha256") + final_release_report_matches_latest = not mismatches + if mismatches: + final_release_report_stale_reason = "final_release_report does not match latest_release: " + ", ".join(mismatches) + + final_release_gate = dict(final_release_report.get("release_gate") or {}) if final_release_report_available else {} + final_release_gate_decision = str(final_release_gate.get("decision") or "").strip() + final_release_gate_blocked_reasons = [ + str(item).strip() + for item in list(final_release_gate.get("blocked_reasons") or []) + if str(item).strip() + ] + final_release_gate_verify_ok = ( + bool(final_release_gate.get("verify_ok")) if "verify_ok" in final_release_gate else None + ) + final_release_gate_smoke_test_ok = ( + bool(final_release_gate.get("smoke_test_ok")) if "smoke_test_ok" in final_release_gate else None + ) + final_release_ok = bool( + final_release_report_available + and final_release_report_ok + and final_release_report_matches_latest + ) + + summary_parts = [f"导入自本机发布包 {archive_filename or package_name}"] + generated_at = str(raw_payload.get("generated_at") or "").strip() + if generated_at: + summary_parts.append(f"生成时间 {generated_at}") + if commit_sha: + summary_parts.append(f"提交 {commit_sha}") + summary_parts.append(f"自检{'通过' if smoke_test_ok else '未执行或未通过'}") + if final_release_ok: + summary_parts.append("最终签收通过") + elif final_release_report_available: + summary_parts.append("最终签收未通过") + + return { + "available": True, + "package_name": package_name, + "package_type": package_type or "unknown", + "generated_at": generated_at, + "archive_path": archive_path_value, + "archive_filename": archive_filename, + "archive_present": bool(archive_path_value and Path(archive_path_value).exists()), + "sha256_path": sha256_path_value, + "sha256_present": bool(sha256_path_value and Path(sha256_path_value).exists()), + "sha256": str(raw_payload.get("sha256") or "").strip(), + "commit_sha": commit_sha, + "commit_ref": commit_ref, + "staging_path": staging_path_value, + "staging_present": bool(staging_path_value and Path(staging_path_value).exists()), + "smoke_test_ok": smoke_test_ok, + "smoke_test_message": smoke_test_message, + "smoke_test_report": smoke_test_report, + "final_release_ok": final_release_ok, + "final_release_report_available": final_release_report_available, + "final_release_report_ok": final_release_report_ok if final_release_report_available else None, + "final_release_report_matches_latest": final_release_report_matches_latest if final_release_report_available else None, + "final_release_report_package_name": final_release_report_package_name, + "final_release_report_stale_reason": final_release_report_stale_reason, + "final_release_gate_decision": final_release_gate_decision, + "final_release_gate_blocked_reasons": final_release_gate_blocked_reasons, + "final_release_gate_verify_ok": final_release_gate_verify_ok, + "final_release_gate_smoke_test_ok": final_release_gate_smoke_test_ok, + "download_path": f"/api/v1/ops/releases/packages/{package_name}/download", + "sha256_download_path": f"/api/v1/ops/releases/packages/{package_name}/sha256", + "release_version_suggestion": package_name, + "notes_suggestion": "；".join(summary_parts), + "metadata": { + "package_name": package_name, + "package_type": package_type or "unknown", + "generated_at": generated_at, + "archive_filename": archive_filename, + "archive_path": archive_path_value, + "sha256_path": sha256_path_value, + "commit_sha": commit_sha, + "commit_ref": commit_ref, + "staging_path": staging_path_value, + "smoke_test_ok": smoke_test_ok, + "smoke_test_message": smoke_test_message, + "smoke_test_report": smoke_test_report, + "final_release_report": final_release_report, + "manifest": manifest, + }, + } + + +def get_latest_release_package_metadata() -> dict: + payload = _read_latest_release_metadata_file() + normalized = _normalize_release_package_metadata(payload) + if normalized: + return normalized + return { + "available": False, + "reason": "latest_release.json 不存在或不可解析", + } + + +def _validate_latest_release_package_final_gate(package_metadata: dict | None = None) -> tuple[bool, str, dict]: + normalized_package = dict(package_metadata or {}) + if not bool(normalized_package.get("available")): + message = str(normalized_package.get("reason") or "当前没有可用的本机发布包") + return False, message, { + "package": normalized_package, + "final_release_gate": { + "ready": False, + "status": "package_unavailable", + "status_label": "发布包不可用", + "summary": message, + "blocked_reasons": ["package_unavailable"], + }, + } + + if bool(normalized_package.get("final_release_ok")): + return True, "", { + "package": normalized_package, + "final_release_gate": { + "ready": True, + "status": "ready", + "status_label": "最终签收通过", + "summary": "最新发布包已完成最终签收，可进入 Release 创建或智能 Rollout。", + "blocked_reasons": [], + "decision": str(normalized_package.get("final_release_gate_decision") or "").strip(), + }, + } + + blocked_reasons: list[str] = [] + if not bool(normalized_package.get("smoke_test_ok")): + blocked_reasons.append("smoke_test_failed_or_skipped") + if not bool(normalized_package.get("final_release_report_available")): + blocked_reasons.append("final_release_report_missing") + if bool(normalized_package.get("final_release_report_available")) and not bool( + normalized_package.get("final_release_report_ok") + ): + blocked_reasons.append("final_release_report_not_ok") + if normalized_package.get("final_release_report_matches_latest") is False: + blocked_reasons.append("final_release_report_not_matching_latest") + if normalized_package.get("final_release_gate_verify_ok") is False: + blocked_reasons.append("verify_failed") + if normalized_package.get("final_release_gate_smoke_test_ok") is False: + blocked_reasons.append("smoke_test_failed") + + decision = str(normalized_package.get("final_release_gate_decision") or "").strip() + if decision and decision != "ready": + blocked_reasons.append(f"final_release_gate_{decision}") + + for item in list(normalized_package.get("final_release_gate_blocked_reasons") or []): + normalized_item = str(item or "").strip() + if normalized_item: + blocked_reasons.append(normalized_item) + + deduped_blocked_reasons: list[str] = [] + for item in blocked_reasons: + if item and item not in deduped_blocked_reasons: + deduped_blocked_reasons.append(item) + + hints: list[str] = [] + stale_reason = str(normalized_package.get("final_release_report_stale_reason") or "").strip() + if stale_reason: + hints.append(stale_reason) + if not bool(normalized_package.get("final_release_report_available")): + hints.append("缺少 final_release_report.json") + elif not bool(normalized_package.get("final_release_report_ok")): + hints.append("final_release_report.json 尚未给出 ok=true") + if not bool(normalized_package.get("smoke_test_ok")): + hints.append("smoke test 未通过") + if normalized_package.get("final_release_gate_verify_ok") is False: + hints.append("verify 未通过") + if normalized_package.get("final_release_gate_smoke_test_ok") is False: + hints.append("最终签收门禁中的 smoke_test_ok=false") + + summary = "最新发布包尚未完成最终签收，暂不允许创建 Release 或发起基于最新包的智能 Rollout。" + if hints: + summary = f"{summary} {hints[0]}" + + return False, summary, { + "package": normalized_package, + "final_release_gate": { + "ready": False, + "status": "blocked", + "status_label": "最终签收未完成", + "summary": summary, + "blocked_reasons": deduped_blocked_reasons, + "decision": decision, + "report_available": bool(normalized_package.get("final_release_report_available")), + "report_ok": bool(normalized_package.get("final_release_report_ok")), + "report_matches_latest": normalized_package.get("final_release_report_matches_latest"), + "stale_reason": stale_reason, + "verify_ok": normalized_package.get("final_release_gate_verify_ok"), + "smoke_test_ok": normalized_package.get("final_release_gate_smoke_test_ok"), + }, + } + + +def prepare_latest_release_package(*, requested_by: str = "api/release-prepare") -> tuple[bool, str, dict]: + repo_root = _release_repo_root() + script_path = repo_root / "prepare_final_release.sh" + if not script_path.exists(): + return False, "prepare_final_release.sh 不存在", { + "requested_by": str(requested_by or "").strip() or "api/release-prepare", + "script_path": str(script_path), + } + + completed = subprocess.run( + ["bash", str(script_path)], + cwd=str(repo_root), + capture_output=True, + text=True, + timeout=1200, + ) + package = get_latest_release_package_metadata() + gate_ok, gate_message, gate_data = _validate_latest_release_package_final_gate(package) + result = { + "requested_by": str(requested_by or "").strip() or "api/release-prepare", + "script_path": str(script_path), + "command": ["bash", str(script_path)], + "returncode": int(completed.returncode or 0), + "stdout_preview": _trim_release_command_output(completed.stdout, 12000), + "stderr_preview": _trim_release_command_output(completed.stderr, 4000), + "package": package, + "final_release_gate": dict(gate_data.get("final_release_gate") or {}), + } + if completed.returncode != 0: + message = _trim_release_command_output(completed.stderr or completed.stdout, 400) or "最终签收准备执行失败" + return False, message, result + if not gate_ok: + return False, gate_message, result + return True, "最新发布包最终签收已完成", result + + +def get_release_package_archive_path(package_name: str) -> Path | None: + normalized_name = _safe_release_package_name(package_name) + release_root = _release_package_root() + for suffix in (".tar.gz", ".zip"): + candidate = release_root / f"{normalized_name}{suffix}" + if candidate.exists() and candidate.is_file(): + return candidate + metadata = _normalize_release_package_metadata(_read_latest_release_metadata_file()) + if metadata and str(metadata.get("package_name") or "").strip() == normalized_name: + archive_path_value = str(metadata.get("archive_path") or "").strip() + if archive_path_value: + candidate = Path(archive_path_value) + if candidate.exists() and candidate.is_file(): + return candidate + return None + + +def get_release_package_sha256_path(package_name: str) -> Path | None: + normalized_name = _safe_release_package_name(package_name) + release_root = _release_package_root() + candidate = release_root / f"{normalized_name}.sha256.txt" + if candidate.exists() and candidate.is_file(): + return candidate + metadata = _normalize_release_package_metadata(_read_latest_release_metadata_file()) + if metadata and str(metadata.get("package_name") or "").strip() == normalized_name: + sha256_path_value = str(metadata.get("sha256_path") or "").strip() + if sha256_path_value: + resolved = Path(sha256_path_value) + if resolved.exists() and resolved.is_file(): + return resolved + return None + + +def _get_release_by_version(release_version: str) -> dict: + ensure_ops_release_schema() + normalized_release_version = str(release_version or "").strip() + if not normalized_release_version: + return {} + with get_db() as conn: + with conn.cursor() as cur: + cur.execute( + """ + SELECT + id, release_version, channel, commit_sha, status, artifact_url, checksum, notes, metadata_json, created_by, created_at, activated_at, updated_at + FROM ops_releases + WHERE release_version = %s + LIMIT 1 + """, + (normalized_release_version,), + ) + row = cur.fetchone() + return _serialize_release_row(row) if row else {} + + +def _inspection_job_bucket_for_rollout(action: str, payload: dict) -> str: + normalized_action = str(action or "").strip() + if normalized_action in {"health.snapshot", "diagnostics.collect"}: + return normalized_action + if normalized_action != "logs.collect": + return "" + service_name = str((payload or {}).get("service_name") or "").strip().lower() + if "worker" not in service_name: + return "" + return "logs.collect" + + +def _rollout_inspection_status_label(status: str) -> str: + normalized_status = str(status or "").strip() + mapping = { + "healthy": "巡检通过", + "running": "巡检进行中", + "attention": "巡检待关注", + "missing": "缺少巡检", + } + return mapping.get(normalized_status, normalized_status or "未知") + + +def _rollout_agent_state_from_context( + *, + last_seen_at: str, + latest_token: dict, + cluster_status: str, + current_load: int, +) -> tuple[str, str, str]: + now = datetime.now() + last_seen_dt = _parse_time(last_seen_at) + token_status = str(latest_token.get("token_status") or "absent").strip() or "absent" + + if last_seen_dt: + current_time = datetime.now(last_seen_dt.tzinfo) if last_seen_dt.tzinfo else now + age_seconds = max(0, int((current_time - last_seen_dt).total_seconds())) + if age_seconds <= 180: + if cluster_status == "busy" or int(current_load or 0) > 0: + return "online_busy", "已接管", f"Agent 最近心跳 {age_seconds} 秒前，节点当前负载 {int(current_load or 0)}。" + return "online", "已接管", f"Agent 最近心跳 {age_seconds} 秒前。" + return "stale", "心跳过期", f"Agent 最近心跳 {age_seconds} 秒前，已超过在线阈值。" + + if token_status == "active": + return "pending_bootstrap", "待接入", "已签发有效 Agent Token，但节点尚未完成 register/heartbeat。" + if token_status == "expired": + return "token_expired", "Token 过期", "已签发 Agent Token，但该 Token 已过期。" + if token_status == "disabled": + return "token_disabled", "Token 已禁用", "最近一枚 Agent Token 已被禁用，需重新签发。" + if cluster_status in {"online", "busy"}: + return "runtime_only", "仅运行态在线", "控制面能看到 runtime 心跳，但 Node Agent 尚未接管。" + return "unmanaged", "未接管", "当前节点尚未生成有效 Agent 接入状态。" + + +def _rollout_execution_mode_label(execution_mode: str) -> str: + normalized_execution_mode = str(execution_mode or "remote-agent").strip() or "remote-agent" + return execution_mode_label(normalized_execution_mode) + + +def _rollout_delivery_queue_context(metadata: dict | None = None) -> tuple[str, str, str, dict]: + snapshot = dict((metadata or {}).get("delivery_queue") or {}) + pending_count = int(snapshot.get("pending_count", 0) or 0) + dead_letter_count = int(snapshot.get("dead_letter_count", 0) or 0) + last_flush_at = str(snapshot.get("last_flush_at") or "").strip() + oldest_pending_at = str(snapshot.get("oldest_pending_at") or "").strip() + oldest_dead_letter_at = str(snapshot.get("oldest_dead_letter_at") or "").strip() + + if not snapshot: + return "unknown", "未上报", "Node Agent 尚未上报回执队列状态。", {} + if dead_letter_count > 0: + reason = f"当前存在 {dead_letter_count} 条死信记录，说明该节点的 Agent 回执链路尚未完全收口。" + if oldest_dead_letter_at: + reason += f" 最早死信时间：{oldest_dead_letter_at}。" + return "dead_letter", f"死信 {dead_letter_count}", reason, snapshot + if pending_count > 0: + reason = f"当前存在 {pending_count} 条待重试回执，Node Agent 会继续自动回放。" + if oldest_pending_at: + reason += f" 最早积压时间：{oldest_pending_at}。" + return "retrying", f"待重试 {pending_count}", reason, snapshot + + reason = "当前没有待重试回执，也没有死信记录。" + if last_flush_at: + reason += f" 最近一次队列冲刷：{last_flush_at}。" + return "healthy", "正常", reason, snapshot + + +def build_rollout_target_operational_readiness( + target_nodes: list[dict], + *, + execution_mode: str = "remote-agent", + desired_release: dict | None = None, +) -> dict: + from app.services.cluster_runtime_service import get_cluster_snapshot + from app.services.ops_agent_service import ensure_ops_agent_schema, get_managed_node_onboarding + from app.services.ops_job_service import list_managed_nodes + + ensure_ops_agent_schema() + normalized_execution_mode = str(execution_mode or "remote-agent").strip() or "remote-agent" + normalized_desired_release = dict(desired_release or {}) + desired_release_version = str(normalized_desired_release.get("release_version") or "").strip() + normalized_target_nodes = [ + dict(item or {}) + for item in list(target_nodes or []) + if str((item or {}).get("node_code") or "").strip() + ] + target_node_codes = [str(item.get("node_code") or "").strip() for item in normalized_target_nodes] + if not target_node_codes: + return { + "execution_mode": normalized_execution_mode, + "summary": { + "nodes_total": 0, + "managed_nodes": 0, + "managed_enabled_nodes": 0, + "agent_online_nodes": 0, + "ssh_ready_nodes": 0, + "remote_agent_ready_nodes": 0, + "execution_ready_nodes": 0, + "inspection_healthy_nodes": 0, + "inspection_running_nodes": 0, + "inspection_attention_nodes": 0, + "inspection_missing_nodes": 0, + }, + "rows": [], + "blocking_reasons": [], + "warning_reasons": [], + "recommendations": [], + } + + cluster_snapshot = get_cluster_snapshot() + cluster_map = { + str(item.get("node_code") or "").strip(): item + for item in list(cluster_snapshot.get("nodes") or []) + if str(item.get("node_code") or "").strip() + } + managed_nodes = list_managed_nodes() + managed_map = { + str(item.get("node_code") or "").strip(): item + for item in managed_nodes + if str(item.get("node_code") or "").strip() + } + + latest_tokens: dict[str, dict] = {} + latest_inspection_jobs: dict[tuple[str, str], dict] = {} + placeholders = ", ".join(["%s"] * len(target_node_codes)) + with get_db() as conn: + with conn.cursor() as cur: + cur.execute( + f""" + SELECT DISTINCT ON (node_code) + node_code, id, is_enabled, expires_at, last_used_at, metadata_json, created_at, updated_at + FROM ops_node_tokens + WHERE purpose = 'agent' AND node_code IN ({placeholders}) + ORDER BY node_code ASC, created_at DESC, id DESC + """, + tuple(target_node_codes), + ) + for row in cur.fetchall(): + node_code = str(row[0] or "").strip() + if not node_code: + continue + expires_at = _ts(row[3]) + token_status = "active" + expires_dt = _parse_time(row[3]) + if not bool(row[2]): + token_status = "disabled" + elif expires_dt: + current_time = datetime.now(expires_dt.tzinfo) if expires_dt.tzinfo else datetime.now() + if expires_dt < current_time: + token_status = "expired" + latest_tokens[node_code] = { + "record_id": int(row[1]), + "is_enabled": bool(row[2]), + "expires_at": expires_at, + "last_used_at": _ts(row[4]), + "metadata": _decode_json(row[5]), + "created_at": _ts(row[6]), + "updated_at": _ts(row[7]), + "token_status": token_status, + } + + cur.execute( + f""" + SELECT + id, target_node_code, action, status, payload_json, created_at, updated_at + FROM ops_jobs + WHERE target_node_code IN ({placeholders}) + AND action IN ('health.snapshot', 'logs.collect', 'diagnostics.collect') + ORDER BY created_at DESC, id DESC + """, + tuple(target_node_codes), + ) + for row in cur.fetchall(): + node_code = str(row[1] or "").strip() + if not node_code: + continue + payload = _decode_payload_json(row[4]) + bucket = _inspection_job_bucket_for_rollout(str(row[2] or ""), payload) + if not bucket: + continue + key = (node_code, bucket) + if key in latest_inspection_jobs: + continue + latest_inspection_jobs[key] = { + "id": int(row[0]), + "status": str(row[3] or ""), + "created_at": _ts(row[5]), + "updated_at": _ts(row[6]), + } + + rows: list[dict] = [] + blocking_reasons: list[str] = [] + warning_reasons: list[str] = [] + recommendations: list[str] = [] + + unmanaged_nodes: list[str] = [] + disabled_nodes: list[str] = [] + agent_offline_nodes: list[str] = [] + ssh_unready_nodes: list[str] = [] + inspection_attention_nodes: list[str] = [] + inspection_missing_nodes: list[str] = [] + onboarding_bootstrap_pending_nodes: list[str] = [] + onboarding_acceptance_ready_nodes: list[str] = [] + onboarding_noop_nodes: list[str] = [] + queue_dead_letter_nodes: list[str] = [] + queue_retrying_nodes: list[str] = [] + + for target in normalized_target_nodes: + node_code = str(target.get("node_code") or "").strip() + cluster_node = cluster_map.get(node_code, {}) + managed = managed_map.get(node_code, {}) + latest_token = latest_tokens.get(node_code, {}) + metadata = dict(managed.get("metadata") or {}) + ( + delivery_queue_state, + delivery_queue_label, + delivery_queue_reason, + delivery_queue_snapshot, + ) = _rollout_delivery_queue_context(metadata) + last_seen_at = str(managed.get("last_seen_at") or "").strip() or str(metadata.get("last_seen_at") or "").strip() + cluster_status = str(cluster_node.get("status") or target.get("status") or "").strip() + current_load = int(cluster_node.get("current_load", target.get("current_load", 0)) or 0) + onboarding = get_managed_node_onboarding(node_code) if node_code else {} + onboarding_stage = dict(onboarding.get("onboarding_stage") or {}) + recovery_decision = dict(onboarding.get("recovery_decision") or {}) + onboarding_stage_code = str(onboarding_stage.get("code") or "").strip() + onboarding_stage_label = str(onboarding_stage.get("label") or "").strip() + onboarding_summary = str(onboarding.get("summary") or onboarding_stage.get("summary") or "").strip() + recovery_action = str(recovery_decision.get("action") or "").strip() + recovery_label = str(recovery_decision.get("label") or "").strip() + recovery_summary = str(recovery_decision.get("summary") or "").strip() + recovery_command_hint = str(recovery_decision.get("command_hint") or "").strip() + recovery_window = str(recovery_decision.get("window") or "").strip() + agent_state, agent_state_label, agent_state_reason = _rollout_agent_state_from_context( + last_seen_at=last_seen_at, + latest_token=latest_token, + cluster_status=cluster_status, + current_load=current_load, + ) + is_managed = bool(managed) + is_enabled = bool(managed.get("is_enabled", target.get("is_enabled", False))) + is_agent_online = agent_state in {"online", "online_busy"} + remote_agent_ready = is_managed and is_enabled and is_agent_online + has_ssh_access = bool(str(managed.get("ssh_host") or "").strip() and str(managed.get("ssh_user") or "").strip()) + ssh_ready = is_managed and is_enabled and has_ssh_access + execution_ready = remote_agent_ready if normalized_execution_mode == "remote-agent" else ssh_ready + + inspection_status_map = { + bucket: str((latest_inspection_jobs.get((node_code, bucket)) or {}).get("status") or "").strip() + for bucket in _ROLLOUT_INSPECTION_ACTION_KEYS + } + present_statuses = [status for status in inspection_status_map.values() if status] + if not present_statuses: + inspection_status = "missing" + inspection_reason = "当前缺少健康快照、Worker 日志和诊断包的巡检记录。" + elif any(status in {"failed", "blocked", "cancelled"} for status in present_statuses): + inspection_status = "attention" + inspection_reason = "最近巡检存在失败、阻断或取消记录。" + elif any(status in {"queued", "dispatching", "running", "awaiting_approval"} for status in present_statuses): + inspection_status = "running" + inspection_reason = "最近巡检仍在执行、排队或待审批中。" + elif all(inspection_status_map.get(bucket) == "success" for bucket in _ROLLOUT_INSPECTION_ACTION_KEYS): + inspection_status = "healthy" + inspection_reason = "最近一轮健康快照、Worker 日志和诊断包均已成功。" + else: + inspection_status = "attention" + missing_actions = [ + bucket for bucket in _ROLLOUT_INSPECTION_ACTION_KEYS if not inspection_status_map.get(bucket) + ] + inspection_reason = ( + f"最近巡检仍缺少 {'、'.join(missing_actions)} 的成功记录。" + if missing_actions + else "最近巡检尚未形成完整健康结论。" + ) + + inspection_success_count = sum(1 for bucket in _ROLLOUT_INSPECTION_ACTION_KEYS if inspection_status_map.get(bucket) == "success") + inspection_ready = inspection_status == "healthy" + row_blocking_reasons: list[str] = [] + row_warning_reasons: list[str] = [] + + if not is_managed: + row_blocking_reasons.append("当前节点尚未纳管，不能直接进入正式发布执行面。") + elif not is_enabled: + row_blocking_reasons.append("当前节点已纳管但未启用，需先恢复启用状态。") + elif normalized_execution_mode == "remote-agent" and not is_agent_online: + row_blocking_reasons.append("当前节点 Node Agent 未在线，暂不适合 remote-agent 发布。") + elif normalized_execution_mode == "ssh" and not has_ssh_access: + row_blocking_reasons.append("当前节点尚未配置完整 SSH 入口，暂不适合 SSH 发布。") + + if delivery_queue_state == "dead_letter": + row_blocking_reasons.append("当前节点 Node Agent 回执队列存在死信，说明执行回执链路未收口，暂不适合正式发布。") + elif delivery_queue_state == "retrying": + row_warning_reasons.append("当前节点 Node Agent 回执队列仍在自动重试，建议等回执链路收口后再发布。") + + if inspection_status == "attention": + row_warning_reasons.append("最近巡检存在失败、阻断或取消记录，建议先看巡检收口。") + elif inspection_status == "missing": + row_warning_reasons.append("当前缺少健康快照、Worker 日志和诊断包的完整巡检记录。") + elif inspection_status == "running": + row_warning_reasons.append("最近巡检仍在执行中，建议等待收口后再推进发布。") + + if recovery_action == "bootstrap_run": + onboarding_bootstrap_pending_nodes.append(node_code) + if not execution_ready: + row_warning_reasons.append("当前节点还在接入阶段，建议先执行 onboarding.bootstrap。") + elif recovery_action == "run_acceptance": + onboarding_acceptance_ready_nodes.append(node_code) + if not execution_ready: + row_warning_reasons.append("当前节点已进入接管验收窗口，建议先执行 onboarding.acceptance。") + else: + onboarding_noop_nodes.append(node_code) + + current_release_version = _extract_node_release_version(target, cluster_node, managed, metadata) + risk_level = _rollout_readiness_risk_level( + blocking_reasons=row_blocking_reasons, + inspection_status=inspection_status, + ) + if row_blocking_reasons: + if recovery_action in {"bootstrap_run", "run_acceptance"}: + recommended_action_code = recovery_action + else: + recommended_action_code = "handover_first_gap" if not is_managed else "fix_managed_nodes" + elif inspection_status in {"attention", "missing", "running"}: + recommended_action_code = "run_standard_inspection" + else: + recommended_action_code = "" + + if not is_managed: + unmanaged_nodes.append(node_code) + elif not is_enabled: + disabled_nodes.append(node_code) + elif not is_agent_online: + agent_offline_nodes.append(node_code) + if is_managed and is_enabled and not has_ssh_access: + ssh_unready_nodes.append(node_code) + + if inspection_status == "attention": + inspection_attention_nodes.append(node_code) + elif inspection_status == "missing": + inspection_missing_nodes.append(node_code) + if delivery_queue_state == "dead_letter": + queue_dead_letter_nodes.append(node_code) + elif delivery_queue_state == "retrying": + queue_retrying_nodes.append(node_code) + + rows.append( + { + "node_code": node_code, + "region": str(target.get("region") or cluster_node.get("region") or ""), + "role": str(target.get("role") or cluster_node.get("role") or ""), + "status": cluster_status, + "current_load": current_load, + "agent_managed": is_managed, + "is_managed": is_managed, + "is_enabled": is_enabled, + "agent_state": agent_state, + "agent_state_label": agent_state_label, + "agent_reason": agent_state_reason, + "is_agent_online": is_agent_online, + "has_ssh_access": has_ssh_access, + "ssh_ready": ssh_ready, + "remote_agent_ready": remote_agent_ready, + "execution_ready": execution_ready, + "execution_mode": normalized_execution_mode, + "execution_mode_label": _rollout_execution_mode_label(normalized_execution_mode), + "inspection_status": inspection_status, + "inspection_status_label": _rollout_inspection_status_label(inspection_status), + "inspection_reason": inspection_reason, + "inspection_success_count": inspection_success_count, + "inspection_ready": inspection_ready, + "inspection_status_map": inspection_status_map, + "delivery_queue_state": delivery_queue_state, + "delivery_queue_label": delivery_queue_label, + "delivery_queue_reason": delivery_queue_reason, + "delivery_queue_pending_count": int(delivery_queue_snapshot.get("pending_count", 0) or 0), + "delivery_queue_dead_letter_count": int(delivery_queue_snapshot.get("dead_letter_count", 0) or 0), + "current_release_version": current_release_version, + "desired_release_version": desired_release_version, + "onboarding_stage_code": onboarding_stage_code, + "onboarding_stage_label": onboarding_stage_label, + "onboarding_summary": onboarding_summary, + "recovery_action": recovery_action, + "recovery_label": recovery_label, + "recovery_summary": recovery_summary, + "recovery_command_hint": recovery_command_hint, + "recovery_window": recovery_window, + "risk_level": risk_level, + "blocking_reasons": row_blocking_reasons, + "warning_reasons": row_warning_reasons, + "recommended_action_code": recommended_action_code, + "focus_ref": _build_rollout_readiness_focus_ref(normalized_desired_release, node_code=node_code), + } + ) + + if normalized_execution_mode == "remote-agent": + if unmanaged_nodes: + blocking_reasons.append(f"有 {len(unmanaged_nodes)} 台目标节点尚未纳管，remote-agent rollout 无法下发。") + if disabled_nodes: + blocking_reasons.append(f"有 {len(disabled_nodes)} 台目标节点已纳管但未启用，需先恢复启用。") + if agent_offline_nodes: + blocking_reasons.append(f"有 {len(agent_offline_nodes)} 台目标节点 Agent 未在线，当前不适合直接 remote-agent 发布。") + elif normalized_execution_mode == "ssh": + if unmanaged_nodes: + blocking_reasons.append(f"有 {len(unmanaged_nodes)} 台目标节点尚未纳管，SSH 发布无法直接下发。") + if disabled_nodes: + blocking_reasons.append(f"有 {len(disabled_nodes)} 台目标节点已纳管但未启用，需先恢复启用。") + if ssh_unready_nodes: + blocking_reasons.append(f"有 {len(ssh_unready_nodes)} 台目标节点尚未配置完整 SSH 入口，当前不适合直接 SSH 发布。") + + if inspection_attention_nodes: + warning_reasons.append(f"有 {len(inspection_attention_nodes)} 台目标节点最近巡检待关注，建议先看巡检结果再发布。") + if inspection_missing_nodes: + warning_reasons.append(f"有 {len(inspection_missing_nodes)} 台目标节点缺少完整巡检记录，建议先补齐标准巡检。") + if queue_dead_letter_nodes: + blocking_reasons.append(f"有 {len(queue_dead_letter_nodes)} 台目标节点存在 Node Agent 死信回执，当前不能进入正式发布。") + if queue_retrying_nodes: + warning_reasons.append(f"有 {len(queue_retrying_nodes)} 台目标节点仍在自动回放回执，建议等待回执链路收口。") + + if normalized_execution_mode == "remote-agent" and (unmanaged_nodes or disabled_nodes or agent_offline_nodes): + recommendations.append("先把目标节点全部收敛到 已纳管 + 已启用 + Agent 在线，再推进 remote-agent rollout。") + if normalized_execution_mode == "ssh" and (unmanaged_nodes or disabled_nodes or ssh_unready_nodes): + recommendations.append("先把目标节点全部收敛到 已纳管 + 已启用 + SSH 已备好，再推进 SSH 发布。") + if inspection_attention_nodes or inspection_missing_nodes: + recommendations.append("先对待发布节点执行 inspection.standard，至少拿到 健康快照 + Worker 日志 + 诊断包 的最新回执。") + if queue_dead_letter_nodes: + recommendations.append("先处理 Node Agent 死信队列并确认回执链路恢复正常，再推进正式发布。") + elif queue_retrying_nodes: + recommendations.append("先等待 Node Agent 自动回放收口，确认待重试回执清空后再推进发布。") + if onboarding_bootstrap_pending_nodes: + recommendations.append( + f"有 {len(onboarding_bootstrap_pending_nodes)} 台目标节点仍停留在接入阶段，建议先执行 onboarding.bootstrap。" + ) + if onboarding_acceptance_ready_nodes: + recommendations.append( + f"有 {len(onboarding_acceptance_ready_nodes)} 台目标节点已进入验收窗口，建议先执行 onboarding.acceptance。" + ) + + summary = { + "nodes_total": len(rows), + "managed_nodes": sum(1 for row in rows if bool(row.get("is_managed", False))), + "managed_enabled_nodes": sum(1 for row in rows if bool(row.get("is_managed", False)) and bool(row.get("is_enabled", False))), + "agent_online_nodes": sum(1 for row in rows if bool(row.get("is_agent_online", False))), + "ssh_ready_nodes": sum(1 for row in rows if bool(row.get("ssh_ready", False))), + "remote_agent_ready_nodes": sum(1 for row in rows if bool(row.get("remote_agent_ready", False))), + "execution_ready_nodes": sum(1 for row in rows if bool(row.get("execution_ready", False))), + "inspection_healthy_nodes": sum(1 for row in rows if str(row.get("inspection_status") or "") == "healthy"), + "inspection_running_nodes": sum(1 for row in rows if str(row.get("inspection_status") or "") == "running"), + "inspection_attention_nodes": sum(1 for row in rows if str(row.get("inspection_status") or "") == "attention"), + "inspection_missing_nodes": sum(1 for row in rows if str(row.get("inspection_status") or "") == "missing"), + "queue_retrying_nodes": len(queue_retrying_nodes), + "queue_dead_letter_nodes": len(queue_dead_letter_nodes), + "onboarding_bootstrap_pending_nodes": len(onboarding_bootstrap_pending_nodes), + "onboarding_acceptance_ready_nodes": len(onboarding_acceptance_ready_nodes), + "onboarding_noop_nodes": len(onboarding_noop_nodes), + } + + rows.sort( + key=lambda item: ( + 0 if bool(item.get("execution_ready", False)) else 1, + 0 if str(item.get("inspection_status") or "") == "healthy" else 1, + str(item.get("role") or ""), + str(item.get("node_code") or ""), + ) + ) + + return { + "execution_mode": normalized_execution_mode, + "summary": summary, + "rows": rows, + "blocking_reasons": blocking_reasons, + "warning_reasons": warning_reasons, + "recommendations": recommendations, + } + + +def build_release_rollout_gate(release: dict, target_nodes: list[dict], *, execution_mode: str = "remote-agent") -> dict: + normalized_release = dict(release or {}) + execution_mode_label = _rollout_execution_mode_label(execution_mode) + target_rows = [ + dict(item or {}) + for item in list(target_nodes or []) + if str((item or {}).get("node_code") or "").strip() + ] + readiness = build_rollout_target_operational_readiness( + target_rows, + execution_mode=execution_mode, + desired_release=normalized_release, + ) + readiness_summary = dict(readiness.get("summary") or {}) + blocking_reasons = [str(item).strip() for item in list(readiness.get("blocking_reasons") or []) if str(item).strip()] + warning_reasons = [str(item).strip() for item in list(readiness.get("warning_reasons") or []) if str(item).strip()] + recommendations = [str(item).strip() for item in list(readiness.get("recommendations") or []) if str(item).strip()] + + release_id = int(normalized_release.get("id") or 0) + release_status = str(normalized_release.get("status") or "").strip() + release_version = str(normalized_release.get("release_version") or "-").strip() or "-" + release_artifact_url = str(normalized_release.get("artifact_url") or "").strip() + is_preview_release = release_id <= 0 and bool(release_version not in {"", "-"} or release_status or release_artifact_url) + + if release_id <= 0 and not is_preview_release: + gate_status = "missing_release" + gate_label = "缺少 Release" + gate_type = "info" + gate_summary = "当前还没有默认 Release，正式 Rollout 入口尚未建立。" + elif release_status not in {"ready", "active"}: + gate_status = "release_preview_not_ready" if is_preview_release else "release_not_ready" + gate_label = "Release 预览未就绪" if is_preview_release else "Release 未就绪" + gate_type = "warning" + gate_summary = ( + f"预览 Release {release_version} 当前状态为 {release_status or 'draft'}，建议先收口到 ready 或 active。" + if is_preview_release + else f"默认 Release {release_version} 当前状态为 {release_status or 'draft'}，建议先收口到 ready 或 active。" + ) + elif not release_artifact_url: + gate_status = "artifact_missing" + gate_label = "预览缺少制品" if is_preview_release else "缺少制品" + gate_type = "warning" + gate_summary = ( + "当前预览 Release 尚未填写 artifact_url，Rollout 下发链路仍不完整。" + if is_preview_release + else "默认 Release 尚未填写 artifact_url，Rollout 下发链路仍不完整。" + ) + elif not target_rows: + gate_status = "no_targets" + gate_label = "无默认目标" + gate_type = "warning" + gate_summary = ( + "当前没有在线且可视为有效执行面的预览 Rollout 目标节点。" + if is_preview_release + else "当前没有在线且可视为有效执行面的默认 Rollout 目标节点。" + ) + elif blocking_reasons: + gate_status = "blocked" + gate_label = "存在阻断" + gate_type = "danger" + gate_summary = ( + f"{'预览 Release 已推导完成' if is_preview_release else '默认 Release 已选定'}，" + f"但目标节点里仅 {int(readiness_summary.get('execution_ready_nodes', 0) or 0)}/" + f"{int(readiness_summary.get('nodes_total', 0) or 0)} 台 {execution_mode_label} 就绪，暂不适合直接发 Rollout。" + ) + elif warning_reasons: + gate_status = "attention" + gate_label = "待补巡检" + gate_type = "warning" + gate_summary = ( + f"{'预览 Release 已可用于 Rollout' if is_preview_release else '默认 Release 已可用于 Rollout'}，" + f"但当前仅 {int(readiness_summary.get('inspection_healthy_nodes', 0) or 0)}/" + f"{int(readiness_summary.get('nodes_total', 0) or 0)} 台目标节点完成健康巡检。" + ) + else: + gate_status = "preview_ready" if is_preview_release else "ready" + gate_label = "预览可发 Rollout" if is_preview_release else "可发 Rollout" + gate_type = "success" + gate_summary = ( + "当前基于最新发布包推导出的 Release 预览与目标节点均已通过门禁，已具备进入定向或灰度 Rollout 的条件。" + if is_preview_release + else "默认 Release 与默认 Rollout 目标均已通过门禁，当前可直接进入定向或灰度 Rollout。" + ) + + gate = { + "status": gate_status, + "status_label": gate_label, + "status_type": gate_type, + "execution_mode": str(execution_mode or "remote-agent").strip() or "remote-agent", + "execution_mode_label": execution_mode_label, + "summary": gate_summary, + "release": { + "id": release_id, + "release_version": str(normalized_release.get("release_version") or ""), + "channel": str(normalized_release.get("channel") or ""), + "status": release_status, + "status_label": _release_status_label(release_status), + "artifact_url": release_artifact_url, + "is_preview": is_preview_release, + }, + "is_preview_release": is_preview_release, + "target_nodes": target_rows, + "target_node_codes": [str(item.get("node_code") or "").strip() for item in target_rows if str(item.get("node_code") or "").strip()], + "blocking_reasons": blocking_reasons, + "warning_reasons": warning_reasons, + "recommendations": recommendations, + "operational_readiness": { + "summary": readiness_summary, + "rows": list(readiness.get("rows") or []), + }, + } + gate["summary_text"] = str(gate.get("summary") or "").strip() + gate["focus_ref"] = _build_release_focus_ref( + normalized_release, + section="default_rollout_gate", + ) + return gate + + +def _release_gate_status_rank(status: str) -> int: + normalized_status = str(status or "").strip() + if normalized_status in {"ready", "preview_ready"}: + return 0 + if normalized_status == "attention": + return 1 + if normalized_status == "blocked": + return 2 + if normalized_status == "no_targets": + return 3 + if normalized_status == "artifact_missing": + return 4 + if normalized_status in {"release_not_ready", "release_preview_not_ready"}: + return 5 + if normalized_status == "missing_release": + return 6 + return 7 + + +def _build_release_execution_mode_reason( + *, + recommended_mode: str, + recommended_gate: dict, + alternate_gate: dict, +) -> str: + recommended_label = _rollout_execution_mode_label(recommended_mode) + alternate_mode = str(alternate_gate.get("execution_mode") or "").strip() + alternate_label = _rollout_execution_mode_label(alternate_mode) if alternate_mode else "" + recommended_status = str(recommended_gate.get("status") or "").strip() + alternate_status = str(alternate_gate.get("status") or "").strip() + recommended_summary = dict((recommended_gate.get("operational_readiness") or {}).get("summary") or {}) + alternate_summary = dict((alternate_gate.get("operational_readiness") or {}).get("summary") or {}) + recommended_ready_nodes = int(recommended_summary.get("execution_ready_nodes", 0) or 0) + alternate_ready_nodes = int(alternate_summary.get("execution_ready_nodes", 0) or 0) + + if recommended_status in {"missing_release", "release_not_ready", "release_preview_not_ready", "artifact_missing"}: + return "当前 Release 门禁本身尚未收口，执行模式推荐仅用于预览后续下发路径。" + if recommended_status == "no_targets": + return "当前默认目标节点为空，暂时还无法形成稳定的发布执行路径。" + + if recommended_mode == "ssh": + if _release_gate_status_rank(recommended_status) < _release_gate_status_rank(alternate_status): + return "当前目标节点通过 SSH 路径的门禁更完整，建议先走 SSH 发布。" + if recommended_ready_nodes > alternate_ready_nodes: + return "当前目标节点的 SSH 就绪度高于 remote-agent，就此版本建议先走 SSH 发布。" + return "当前标准 Agent 路径尚未完全收口，SSH 更适合作为当前发布入口。" + + if alternate_mode == "ssh" and recommended_ready_nodes >= alternate_ready_nodes: + return "当前目标节点已满足标准 remote-agent 发布链路，默认优先沿用 remote-agent。" + return f"当前 {recommended_label} 路径不劣于 {alternate_label or '备用路径'}，默认继续使用 {recommended_label}。" + + +def build_release_execution_mode_recommendation(release: dict, target_nodes: list[dict]) -> dict: + mode_candidates = ("remote-agent", "ssh") + gates = { + mode: build_release_rollout_gate(release, target_nodes, execution_mode=mode) + for mode in mode_candidates + } + ranked_modes = sorted( + mode_candidates, + key=lambda mode: ( + _release_gate_status_rank(gates[mode].get("status")), + -int(((gates[mode].get("operational_readiness") or {}).get("summary") or {}).get("execution_ready_nodes", 0) or 0), + len(list(gates[mode].get("blocking_reasons") or [])), + len(list(gates[mode].get("warning_reasons") or [])), + 0 if mode == "remote-agent" else 1, + ), + ) + recommended_mode = ranked_modes[0] if ranked_modes else "remote-agent" + fallback_mode = ranked_modes[1] if len(ranked_modes) > 1 else "" + recommended_gate = dict(gates.get(recommended_mode) or {}) + fallback_gate = dict(gates.get(fallback_mode) or {}) + + options = [] + for mode in mode_candidates: + gate = dict(gates.get(mode) or {}) + readiness_summary = dict((gate.get("operational_readiness") or {}).get("summary") or {}) + options.append( + { + "mode": mode, + "mode_label": _rollout_execution_mode_label(mode), + "status": str(gate.get("status") or ""), + "status_label": str(gate.get("status_label") or ""), + "summary": str(gate.get("summary") or ""), + "execution_ready_nodes": int(readiness_summary.get("execution_ready_nodes", 0) or 0), + "nodes_total": int(readiness_summary.get("nodes_total", 0) or 0), + } + ) + + return { + "recommended_mode": recommended_mode, + "recommended_mode_label": _rollout_execution_mode_label(recommended_mode), + "fallback_mode": fallback_mode, + "fallback_mode_label": _rollout_execution_mode_label(fallback_mode) if fallback_mode else "", + "reason": _build_release_execution_mode_reason( + recommended_mode=recommended_mode, + recommended_gate=recommended_gate, + alternate_gate=fallback_gate, + ), + "options": options, + "recommended_gate": recommended_gate, + "gates": gates, + } + + +def _resolve_rollout_targets(selector: dict) -> list[dict]: + from app.services.cluster_runtime_service import get_cluster_snapshot + from app.services.ops_job_service import list_managed_nodes + + selector = selector or {} + requested_node_codes_list = [ + str(item).strip() + for item in list(selector.get("node_codes") or []) + if str(item).strip() + ] + requested_node_codes = set(requested_node_codes_list) + requested_node_code_order = { + node_code: index for index, node_code in enumerate(requested_node_codes_list) + } + requested_region = str(selector.get("region") or "").strip() + requested_role = str(selector.get("role") or "").strip() + only_enabled = bool(selector.get("only_enabled", True)) + only_effective_workers = bool(selector.get("only_effective_workers", False)) + only_online = bool(selector.get("only_online", False)) + + managed_nodes = list_managed_nodes() + managed_map = { + str(item.get("node_code") or ""): item + for item in managed_nodes + if str(item.get("node_code") or "") + } + cluster_nodes = list((get_cluster_snapshot().get("nodes") or [])) + + merged: dict[str, dict] = {} + for cluster_node in cluster_nodes: + node_code = str(cluster_node.get("node_code") or "").strip() + if not node_code: + continue + managed = managed_map.get(node_code, {}) + merged[node_code] = { + "node_code": node_code, + "region": str(cluster_node.get("region") or managed.get("region") or ""), + "role": str(cluster_node.get("role") or managed.get("role") or ""), + "status": str(cluster_node.get("status") or ""), + "current_load": int(cluster_node.get("current_load", 0) or 0), + "is_effective_worker": bool(cluster_node.get("is_effective_worker", False)), + "detect_participating": bool(cluster_node.get("detect_participating", False)), + "is_enabled": bool(managed.get("is_enabled", True)), + "ssh_host": str(managed.get("ssh_host") or ""), + "ssh_user": str(managed.get("ssh_user") or ""), + "ssh_port": int(managed.get("ssh_port") or 22), + "auth_mode": str(managed.get("auth_mode") or "key"), + "deploy_channel": str(managed.get("deploy_channel") or "stable"), + "title": str(managed.get("title") or node_code), + } + for managed in managed_nodes: + node_code = str(managed.get("node_code") or "").strip() + if node_code and node_code not in merged: + merged[node_code] = { + "node_code": node_code, + "region": str(managed.get("region") or ""), + "role": str(managed.get("role") or ""), + "status": "", + "current_load": 0, + "is_effective_worker": False, + "detect_participating": False, + "is_enabled": bool(managed.get("is_enabled", True)), + "ssh_host": str(managed.get("ssh_host") or ""), + "ssh_user": str(managed.get("ssh_user") or ""), + "ssh_port": int(managed.get("ssh_port") or 22), + "auth_mode": str(managed.get("auth_mode") or "key"), + "deploy_channel": str(managed.get("deploy_channel") or "stable"), + "title": str(managed.get("title") or node_code), + } + + targets: list[dict] = [] + for item in merged.values(): + if requested_node_codes and str(item.get("node_code") or "") not in requested_node_codes: + continue + if requested_region and str(item.get("region") or "") != requested_region: + continue + if requested_role and str(item.get("role") or "") != requested_role: + continue + if only_enabled and not bool(item.get("is_enabled", False)): + continue + if only_effective_workers and not bool(item.get("is_effective_worker", False)): + continue + if only_online and str(item.get("status") or "") != "online": + continue + targets.append(item) + + role_order = { + "worker": 0, + "control": 1, + } + + return sorted( + targets, + key=lambda item: ( + requested_node_code_order.get(str(item.get("node_code") or ""), 10_000), + str(item.get("region") or ""), + role_order.get(str(item.get("role") or ""), 9), + str(item.get("node_code") or ""), + ), + ) + + +def create_release(payload: dict) -> tuple[bool, str, dict]: + ensure_ops_release_schema() + release_version = str(payload.get("release_version") or "").strip() + if not release_version: + return False, "release_version 不能为空", {} + + channel = str(payload.get("channel") or "stable").strip() or "stable" + commit_sha = str(payload.get("commit_sha") or "").strip() + status = str(payload.get("status") or "draft").strip() or "draft" + artifact_url = str(payload.get("artifact_url") or "").strip() + checksum = str(payload.get("checksum") or "").strip() + notes = str(payload.get("notes") or "") + metadata = payload.get("metadata") or {} + created_by = str(payload.get("created_by") or "api").strip() or "api" + + with get_db() as conn: + with conn.cursor() as cur: + cur.execute( + """ + INSERT INTO ops_releases ( + release_version, channel, commit_sha, status, artifact_url, checksum, notes, metadata_json, created_by, created_at, updated_at + ) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP) + RETURNING + id, release_version, channel, commit_sha, status, artifact_url, checksum, notes, metadata_json, created_by, created_at, activated_at, updated_at + """, + ( + release_version, + channel, + commit_sha, + status, + artifact_url, + checksum, + notes, + json.dumps(metadata, ensure_ascii=False), + created_by, + ), + ) + row = cur.fetchone() + conn.commit() + return True, "Release 已创建", {"release": _serialize_release_row(row)} + + +def _materialize_release_preview_payload(create_payload: dict) -> dict: + payload = dict(create_payload or {}) + return { + "id": 0, + "release_version": str(payload.get("release_version") or "").strip(), + "channel": str(payload.get("channel") or "stable").strip() or "stable", + "commit_sha": str(payload.get("commit_sha") or "").strip(), + "status": str(payload.get("status") or "ready").strip() or "ready", + "artifact_url": str(payload.get("artifact_url") or "").strip(), + "checksum": str(payload.get("checksum") or "").strip(), + "notes": str(payload.get("notes") or ""), + "metadata": dict(payload.get("metadata") or {}), + "created_by": str(payload.get("created_by") or "api/package-import").strip() or "api/package-import", + "created_at": "", + "activated_at": "", + "updated_at": "", + "is_preview": True, + } + + +def _resolve_release_from_latest_package( + payload: dict | None = None, + *, + control_plane_base_url: str = "", +) -> tuple[bool, str, dict]: + normalized_payload = dict(payload or {}) + package_metadata = get_latest_release_package_metadata() + if not bool(package_metadata.get("available")): + return False, str(package_metadata.get("reason") or "当前没有可用的本机发布包"), {"package": package_metadata} + + release_version = ( + str(normalized_payload.get("release_version") or "").strip() + or str(package_metadata.get("release_version_suggestion") or "").strip() + or str(package_metadata.get("package_name") or "").strip() + ) + if not release_version: + return False, "最新发布包缺少 release_version 建议值", {"package": package_metadata} + + existing_release = _get_release_by_version(release_version) + + normalized_base_url = _normalize_public_base_url( + str(normalized_payload.get("control_plane_base_url") or "").strip() or control_plane_base_url + ) + artifact_url = _build_absolute_release_package_url(normalized_base_url, str(package_metadata.get("download_path") or "")) + sha256_download_url = _build_absolute_release_package_url( + normalized_base_url, str(package_metadata.get("sha256_download_path") or "") + ) + if existing_release and not artifact_url: + release_preview = dict(existing_release or {}) + return True, "已解析最新发布包对应的 Release 信息", { + "release": dict(existing_release or {}), + "release_preview": release_preview, + "preview_source": "existing_release", + "deduplicated": True, + "package": package_metadata, + "resolved_artifact_url": "", + "resolved_sha256_download_url": sha256_download_url, + "release_payload": {}, + } + if not artifact_url: + return False, "当前无法为最新发布包生成可下载的 artifact_url", {"package": package_metadata} + + notes = str(normalized_payload.get("notes") or "").strip() or str(package_metadata.get("notes_suggestion") or "").strip() + notes_suffix = str(normalized_payload.get("notes_suffix") or "").strip() + if notes_suffix: + notes = f"{notes}\n{notes_suffix}".strip() if notes else notes_suffix + + package_source_metadata = { + **dict(package_metadata.get("metadata") or {}), + "download_path": str(package_metadata.get("download_path") or "").strip(), + "sha256_download_path": str(package_metadata.get("sha256_download_path") or "").strip(), + "download_url": artifact_url, + "sha256_download_url": sha256_download_url, + } + release_metadata = { + **dict(normalized_payload.get("metadata") or {}), + "created_from": "latest_release_package", + "package_source": package_source_metadata, + } + + create_payload = { + "release_version": release_version, + "channel": str(normalized_payload.get("channel") or "stable").strip() or "stable", + "status": str(normalized_payload.get("status") or "ready").strip() or "ready", + "commit_sha": ( + str(normalized_payload.get("commit_sha") or "").strip() + or str(package_metadata.get("commit_sha") or "").strip() + or str(package_source_metadata.get("commit_sha") or "").strip() + ), + "artifact_url": artifact_url, + "checksum": str(normalized_payload.get("checksum") or "").strip() or str(package_metadata.get("sha256") or "").strip(), + "notes": notes, + "metadata": release_metadata, + "created_by": str(normalized_payload.get("created_by") or "api/package-import").strip() or "api/package-import", + } + release_preview = existing_release or _materialize_release_preview_payload(create_payload) + return True, "已解析最新发布包对应的 Release 信息", { + "release": dict(existing_release or {}), + "release_preview": release_preview, + "preview_source": "existing_release" if existing_release else "latest_package_draft", + "deduplicated": bool(existing_release), + "package": package_metadata, + "resolved_artifact_url": artifact_url, + "resolved_sha256_download_url": sha256_download_url, + "release_payload": create_payload, + } + + +def create_release_from_latest_package( + payload: dict | None = None, + *, + control_plane_base_url: str = "", +) -> tuple[bool, str, dict]: + ok, message, resolved = _resolve_release_from_latest_package( + payload, + control_plane_base_url=control_plane_base_url, + ) + if not ok: + return False, message, resolved + + gate_ok, gate_message, gate_data = _validate_latest_release_package_final_gate(resolved.get("package") or {}) + if not gate_ok: + return False, gate_message, { + **dict(resolved or {}), + **dict(gate_data or {}), + } + + existing_release = dict(resolved.get("release") or {}) + if existing_release: + return True, "Release 已存在，按幂等返回", { + "release": existing_release, + "deduplicated": True, + "package": dict(resolved.get("package") or {}), + "final_release_gate": dict(gate_data.get("final_release_gate") or {}), + "resolved_artifact_url": str(resolved.get("resolved_artifact_url") or ""), + "resolved_sha256_download_url": str(resolved.get("resolved_sha256_download_url") or ""), + "preview_source": "existing_release", + } + + create_payload = dict(resolved.get("release_payload") or {}) + ok, message, data = create_release(create_payload) + if ok: + data["deduplicated"] = False + data["package"] = dict(resolved.get("package") or {}) + data["final_release_gate"] = dict(gate_data.get("final_release_gate") or {}) + data["resolved_artifact_url"] = str(resolved.get("resolved_artifact_url") or "") + data["resolved_sha256_download_url"] = str(resolved.get("resolved_sha256_download_url") or "") + data["preview_source"] = "latest_package_draft" + return ok, message, data + + +def preview_release_from_latest_package( + payload: dict | None = None, + *, + control_plane_base_url: str = "", +) -> tuple[bool, str, dict]: + ok, message, resolved = _resolve_release_from_latest_package( + payload, + control_plane_base_url=control_plane_base_url, + ) + if not ok: + return False, message, resolved + + preview_release = dict(resolved.get("release_preview") or resolved.get("release") or {}) + return True, "最新发布包预览已生成", { + **dict(resolved or {}), + "release": preview_release, + "release_created": False, + "release_deduplicated": bool(resolved.get("deduplicated", False)), + } + + +def _normalize_smart_rollout_mode(raw_value: object) -> str: + normalized = str(raw_value or "").strip().lower() + if normalized in {"control", "controller"}: + return "control" + if normalized in {"auto", ""}: + return "auto" + return "worker" + + +def _smart_rollout_mode_label(mode: str) -> str: + return "Control 发布" if str(mode or "").strip() == "control" else "Worker 灰度" + + +def _smart_rollout_node_role(node: dict) -> str: + return str(node.get("role") or node.get("cluster_role") or "").strip().lower() + + +def _smart_rollout_node_region(node: dict) -> str: + return str(node.get("region") or node.get("cluster_region") or "").strip().lower() + + +def _smart_rollout_node_status(node: dict) -> str: + return str(node.get("cluster_status") or node.get("status") or "").strip().lower() + + +def _is_smart_rollout_online_node(node: dict) -> bool: + return _smart_rollout_node_status(node) in {"online", "busy"} + + +def _smart_rollout_delivery_queue_state(node: dict) -> str: + return str( + node.get("delivery_queue_state") + or ((node.get("metadata") or {}).get("delivery_queue") or {}).get("state") + or "" + ).strip().lower() + + +def _is_smart_rollout_remote_ready(node: dict) -> bool: + queue_state = _smart_rollout_delivery_queue_state(node) + return bool( + node.get("is_managed", False) + and node.get("is_enabled", False) + and node.get("is_agent_online", False) + and queue_state != "dead_letter" + ) + + +def _is_smart_rollout_ssh_ready(node: dict) -> bool: + queue_state = _smart_rollout_delivery_queue_state(node) + return bool( + node.get("is_managed", False) + and node.get("is_enabled", False) + and node.get("has_ssh_access", False) + and queue_state != "dead_letter" + ) + + +def _is_smart_rollout_execution_ready(node: dict, execution_mode: str) -> bool: + if str(execution_mode or "").strip() == "ssh": + return _is_smart_rollout_ssh_ready(node) + return _is_smart_rollout_remote_ready(node) + + +def _pick_smart_rollout_execution_mode(candidates: list[dict]) -> tuple[str, list[dict]]: + remote_ready_candidates = [item for item in list(candidates or []) if _is_smart_rollout_remote_ready(item)] + ssh_ready_candidates = [item for item in list(candidates or []) if _is_smart_rollout_ssh_ready(item)] + if ssh_ready_candidates and len(ssh_ready_candidates) > len(remote_ready_candidates): + return "ssh", ssh_ready_candidates + if remote_ready_candidates: + return "remote-agent", remote_ready_candidates + if ssh_ready_candidates: + return "ssh", ssh_ready_candidates + return "", [] + + +def _is_smart_rollout_effective_worker(node: dict) -> bool: + if "cluster_is_effective_worker" in node: + return bool(node.get("cluster_is_effective_worker", False)) + if "is_effective_worker" in node: + return bool(node.get("is_effective_worker", False)) + return _smart_rollout_node_role(node) == "worker" + + +def _pick_smart_rollout_dominant_region(nodes: list[dict]) -> str: + counts: dict[str, int] = {} + for item in nodes: + region = _smart_rollout_node_region(item) + if not region: + continue + counts[region] = int(counts.get(region, 0) or 0) + 1 + if not counts: + return "" + return sorted(counts.items(), key=lambda item: (-int(item[1]), str(item[0])))[0][0] + + +def _sort_smart_rollout_candidates(nodes: list[dict]) -> list[dict]: + def _candidate_key(node: dict) -> tuple: + status = _smart_rollout_node_status(node) + status_rank = 0 if status == "online" else (1 if status == "busy" else 9) + current_load = int(node.get("cluster_current_load", node.get("current_load", 0)) or 0) + participating = bool(node.get("cluster_detect_participating", node.get("detect_participating", False))) + access_rank = 0 if _is_smart_rollout_remote_ready(node) else (1 if _is_smart_rollout_ssh_ready(node) else 2) + return ( + access_rank, + 0 if not participating else 1, + status_rank, + current_load, + str(node.get("node_code") or ""), + ) + + return sorted(list(nodes or []), key=_candidate_key) + + +def _build_smart_rollout_role_policy(mode: str, *, execution_mode: str = "remote-agent") -> dict: + if str(mode or "").strip() == "control": + deploy_payload = { + "restart_services": ["domaincheck-api", "domaincheck-worker", "domaincheck-sync-agent"], + "health_check_urls": ["http://127.0.0.1:8100/health"], + "health_check_services": ["domaincheck-api", "domaincheck-worker", "domaincheck-sync-agent"], + "health_check_timeout_seconds": 10, + "health_check_retries": 2, + "health_check_interval_seconds": 2, + "rollback_on_failure": True, + "switch_current": True, + } + else: + deploy_payload = { + "restart_services": ["domaincheck-worker"], + "health_check_urls": [], + "health_check_services": ["domaincheck-worker"], + "health_check_timeout_seconds": 10, + "health_check_retries": 2, + "health_check_interval_seconds": 2, + "rollback_on_failure": True, + "switch_current": True, + } + return { + "auto_start": True, + "auto_dispatch": False, + "auto_advance": True, + "auto_approve": False, + "continue_on_failure": False, + "continue_on_partial": False, + "execution_mode": str(execution_mode or "remote-agent").strip() or "remote-agent", + "first_batch_size": 1, + "batch_size": 1, + "deploy_payload": deploy_payload, + } + + +def build_smart_release_rollout_plan(mode: str = "auto") -> dict: + from app.services.ops_agent_service import list_managed_nodes_with_agent_state + + normalized_mode = _normalize_smart_rollout_mode(mode) + managed_nodes_payload = list_managed_nodes_with_agent_state() + source_nodes = list(managed_nodes_payload.get("nodes") or []) + online_enabled_nodes = [ + dict(item or {}) + for item in source_nodes + if bool((item or {}).get("is_enabled", False)) and _is_smart_rollout_online_node(dict(item or {})) + ] + if not online_enabled_nodes: + return { + "available": False, + "mode": normalized_mode, + "mode_label": _smart_rollout_mode_label(normalized_mode), + "reason": "当前没有在线且启用的节点可用于智能 Rollout。", + "target_selector": {}, + "policy": {}, + "target_nodes": [], + } + + effective_worker_nodes = [item for item in online_enabled_nodes if _is_smart_rollout_effective_worker(item)] + remote_ready_worker_nodes = [item for item in effective_worker_nodes if _is_smart_rollout_remote_ready(item)] + ssh_ready_worker_nodes = [item for item in effective_worker_nodes if _is_smart_rollout_ssh_ready(item)] + preferred_ready_worker_nodes = ( + ssh_ready_worker_nodes if len(ssh_ready_worker_nodes) > len(remote_ready_worker_nodes) else remote_ready_worker_nodes + ) + dominant_region = _pick_smart_rollout_dominant_region(preferred_ready_worker_nodes or effective_worker_nodes or online_enabled_nodes) + resolved_mode = normalized_mode + if resolved_mode == "auto": + resolved_mode = "worker" if effective_worker_nodes else "control" + + def _matches_mode(node: dict) -> bool: + role = _smart_rollout_node_role(node) + if resolved_mode == "control": + return role == "control" + return role == "worker" and _is_smart_rollout_effective_worker(node) + + regional_candidates = [ + item + for item in online_enabled_nodes + if _matches_mode(item) and (not dominant_region or _smart_rollout_node_region(item) == dominant_region) + ] + fallback_candidates = [item for item in online_enabled_nodes if _matches_mode(item)] + ranked_candidates = _sort_smart_rollout_candidates(regional_candidates or fallback_candidates) + execution_mode, ready_candidates = _pick_smart_rollout_execution_mode(ranked_candidates) + candidates = ready_candidates or ranked_candidates + if not candidates: + return { + "available": False, + "mode": resolved_mode, + "mode_label": _smart_rollout_mode_label(resolved_mode), + "reason": "当前没有满足角色条件的在线节点可用于智能 Rollout。", + "target_selector": {}, + "policy": {}, + "target_nodes": [], + } + if not ready_candidates: + dead_letter_nodes = [ + str(item.get("node_code") or "").strip() + for item in ranked_candidates + if _smart_rollout_delivery_queue_state(item) == "dead_letter" and str(item.get("node_code") or "").strip() + ] + reason = "当前命中的在线节点尚未进入可执行发布状态，remote-agent 与 SSH 两条路径都还不够稳定，暂不建议一键创建 Rollout。" + if dead_letter_nodes: + reason = ( + f"当前命中的候选节点里有 {len(dead_letter_nodes)} 台存在 Node Agent 死信回执，" + "需先收口 delivery queue，再继续智能 Rollout。" + ) + return { + "available": False, + "mode": resolved_mode, + "mode_label": _smart_rollout_mode_label(resolved_mode), + "reason": reason, + "target_selector": {}, + "policy": {}, + "target_nodes": [ + { + "node_code": str(item.get("node_code") or ""), + "region": str(item.get("region") or ""), + "role": str(item.get("role") or ""), + "status": _smart_rollout_node_status(item), + "current_load": int(item.get("cluster_current_load", item.get("current_load", 0)) or 0), + "remote_agent_ready": _is_smart_rollout_remote_ready(item), + "ssh_ready": _is_smart_rollout_ssh_ready(item), + "execution_ready": False, + "delivery_queue_state": _smart_rollout_delivery_queue_state(item), + } + for item in ranked_candidates + ], + } + + node_codes = [str(item.get("node_code") or "").strip() for item in candidates if str(item.get("node_code") or "").strip()] + batch_size = 1 if len(node_codes) <= 2 else (2 if len(node_codes) <= 5 else 3) + policy = _build_smart_rollout_role_policy(resolved_mode, execution_mode=execution_mode) + policy["batch_size"] = batch_size + selector = { + "node_codes": node_codes, + "region": "", + "role": "", + "only_enabled": True, + "only_online": True, + "only_effective_workers": False, + "smart_generated": True, + "smart_rollout_mode": resolved_mode, + } + return { + "available": True, + "mode": resolved_mode, + "mode_label": _smart_rollout_mode_label(resolved_mode), + "reason": "", + "summary": ( + f"已按 {_smart_rollout_mode_label(resolved_mode)} 预选 {len(node_codes)} 台节点，" + f"执行路径采用 {_rollout_execution_mode_label(execution_mode)}；" + f"优先锁定 {dominant_region or _smart_rollout_node_region(candidates[0]) or '当前区域'}；" + f"首批 1 台，后续每批 {batch_size} 台。" + ), + "dominant_region": dominant_region or _smart_rollout_node_region(candidates[0]), + "execution_mode": execution_mode, + "execution_mode_label": _rollout_execution_mode_label(execution_mode), + "target_selector": selector, + "policy": policy, + "target_nodes": [ + { + "node_code": str(item.get("node_code") or ""), + "region": str(item.get("region") or ""), + "role": str(item.get("role") or ""), + "status": _smart_rollout_node_status(item), + "current_load": int(item.get("cluster_current_load", item.get("current_load", 0)) or 0), + "remote_agent_ready": _is_smart_rollout_remote_ready(item), + "ssh_ready": _is_smart_rollout_ssh_ready(item), + "execution_ready": _is_smart_rollout_execution_ready(item, execution_mode), + "detect_participating": bool(item.get("cluster_detect_participating", False)), + "delivery_queue_state": _smart_rollout_delivery_queue_state(item), + } + for item in candidates + ], + } + + +def _preview_smart_release_rollout_for_release( + release: dict, + normalized_payload: dict, + rollout_mode: str, +) -> tuple[bool, str, dict]: + from app.services.ops_policy_service import preview_ops_job_policy + + safe_release = dict(release or {}) + smart_rollout = build_smart_release_rollout_plan(rollout_mode) + data = { + "release": safe_release, + "release_created": False, + "rollout_created": False, + "requires_confirmation": False, + "blocked": False, + "smart_rollout": smart_rollout, + "preview": {}, + "rollout_payload": {}, + } + if not bool(smart_rollout.get("available", False)): + return True, "当前无法自动生成智能 Rollout", data + + rollout_payload = { + "created_by": str(normalized_payload.get("rollout_created_by") or normalized_payload.get("created_by") or "api/smart-rollout").strip() + or "api/smart-rollout", + "target_selector": dict(smart_rollout.get("target_selector") or {}), + "policy": dict(smart_rollout.get("policy") or {}), + } + preview = preview_ops_job_policy( + { + "action": "deploy.release", + "target_type": "rollout", + "release_id": int(safe_release.get("id") or 0), + "release": safe_release, + "target_selector": rollout_payload["target_selector"], + "policy": rollout_payload["policy"], + } + ) + data["preview"] = preview + data["rollout_payload"] = rollout_payload + if bool(preview.get("blocked", False)): + data["blocked"] = True + return True, "智能 Rollout 预检未通过", data + + confirm_risky = bool(normalized_payload.get("confirm_risky", False)) + if (list(preview.get("warnings") or []) or bool(preview.get("approval_required", False))) and not confirm_risky: + data["requires_confirmation"] = True + return True, "智能 Rollout 需要确认", data + + return True, "智能 Rollout 预检通过", data + + +def _create_smart_release_rollout_for_release( + release: dict, + normalized_payload: dict, + rollout_mode: str, +) -> tuple[bool, str, dict]: + release_id = int((release or {}).get("id") or 0) + if release_id <= 0: + return False, "智能 Rollout 需要有效的 release_id", {} + + preview_ok, preview_message, data = _preview_smart_release_rollout_for_release(release, normalized_payload, rollout_mode) + if not preview_ok: + return False, preview_message, data + if preview_message in {"当前无法自动生成智能 Rollout", "智能 Rollout 预检未通过", "智能 Rollout 需要确认"}: + return True, preview_message, data + + rollout_payload = dict(data.get("rollout_payload") or {}) + rollout_ok, rollout_message, rollout_data = create_release_rollout(release_id, rollout_payload) + data.update(dict(rollout_data or {})) + data["rollout_created"] = bool(rollout_ok and dict(rollout_data or {}).get("rollout")) + data["requires_confirmation"] = False + data["blocked"] = False + if rollout_ok: + return True, rollout_message, data + return False, rollout_message, data + + +def preview_smart_release_rollout( + release_id: int, + payload: dict | None = None, +) -> tuple[bool, str, dict]: + normalized_payload = dict(payload or {}) + safe_release_id = int(release_id or normalized_payload.get("release_id") or 0) + if safe_release_id <= 0: + return False, "智能 Rollout 需要有效的 release_id", {} + + release = get_release(safe_release_id) + if not release: + return False, "Release 不存在", {} + + rollout_mode = _normalize_smart_rollout_mode(normalized_payload.get("mode")) + ok, message, data = _preview_smart_release_rollout_for_release(release, normalized_payload, rollout_mode) + merged_data = { + **dict(data or {}), + "release": release, + "release_created": False, + "release_deduplicated": False, + } + return ok, message, merged_data + + +def create_smart_release_rollout( + release_id: int, + payload: dict | None = None, +) -> tuple[bool, str, dict]: + normalized_payload = dict(payload or {}) + safe_release_id = int(release_id or normalized_payload.get("release_id") or 0) + if safe_release_id <= 0: + return False, "智能 Rollout 需要有效的 release_id", {} + + release = get_release(safe_release_id) + if not release: + return False, "Release 不存在", {} + + rollout_mode = _normalize_smart_rollout_mode(normalized_payload.get("mode")) + ok, message, data = _create_smart_release_rollout_for_release(release, normalized_payload, rollout_mode) + merged_data = { + **dict(data or {}), + "release": release, + "release_created": False, + } + return ok, message, merged_data + + +def preview_smart_release_rollout_from_latest_package( + payload: dict | None = None, + *, + control_plane_base_url: str = "", +) -> tuple[bool, str, dict]: + normalized_payload = dict(payload or {}) + preview_ok, preview_message, preview_data = preview_release_from_latest_package( + normalized_payload, + control_plane_base_url=control_plane_base_url, + ) + if not preview_ok: + return False, preview_message, preview_data + + release = dict(preview_data.get("release") or {}) + rollout_mode = _normalize_smart_rollout_mode(normalized_payload.get("mode")) + rollout_ok, rollout_message, rollout_data = _preview_smart_release_rollout_for_release( + release, + normalized_payload, + rollout_mode, + ) + data = { + **dict(preview_data or {}), + **dict(rollout_data or {}), + "release": release, + "release_created": False, + "release_deduplicated": bool(dict(preview_data or {}).get("release_deduplicated", False)), + "created_release_message": preview_message, + } + return rollout_ok, rollout_message, data + + +def create_release_and_smart_rollout_from_latest_package( + payload: dict | None = None, + *, + control_plane_base_url: str = "", +) -> tuple[bool, str, dict]: + normalized_payload = dict(payload or {}) + rollout_mode = _normalize_smart_rollout_mode(normalized_payload.get("mode")) + + release_ok, release_message, release_data = create_release_from_latest_package( + normalized_payload, + control_plane_base_url=control_plane_base_url, + ) + if not release_ok: + return False, release_message, release_data + + release = dict(release_data.get("release") or {}) + if int(release.get("id") or 0) <= 0: + return False, "Release 创建结果缺少有效 release_id", release_data + + rollout_ok, rollout_message, rollout_data = _create_smart_release_rollout_for_release(release, normalized_payload, rollout_mode) + data = { + **dict(release_data or {}), + **dict(rollout_data or {}), + "release": release, + "release_created": not bool(dict(release_data or {}).get("deduplicated", False)), + "release_deduplicated": bool(dict(release_data or {}).get("deduplicated", False)), + "created_release_message": release_message, + } + if not bool(data.get("rollout_created", False)) and rollout_message == "当前无法自动生成智能 Rollout": + return True, "Release 已创建，但当前无法自动生成智能 Rollout", data + if not bool(data.get("rollout_created", False)) and rollout_message == "智能 Rollout 预检未通过": + return True, "Release 已创建，但智能 Rollout 预检未通过", data + if bool(data.get("requires_confirmation", False)) and rollout_message == "智能 Rollout 需要确认": + return True, "Release 已创建，智能 Rollout 需要确认", data + return rollout_ok, rollout_message, data + + +def list_releases(limit: int = 20, *, channel: str | None = None) -> list[dict]: + ensure_ops_release_schema() + safe_limit = min(max(int(limit or 20), 1), 100) + conditions: list[str] = [] + params: list[object] = [] + if str(channel or "").strip(): + conditions.append("channel = %s") + params.append(str(channel).strip()) + where_clause = f"WHERE {' AND '.join(conditions)}" if conditions else "" + + with get_db() as conn: + with conn.cursor() as cur: + cur.execute( + f""" + SELECT + id, release_version, channel, commit_sha, status, artifact_url, checksum, notes, metadata_json, created_by, created_at, activated_at, updated_at + FROM ops_releases + {where_clause} + ORDER BY created_at DESC, id DESC + LIMIT %s + """, + (*params, safe_limit), + ) + rows = cur.fetchall() + return [_serialize_release_row(row) for row in rows] + + +def get_release(release_id: int) -> dict: + ensure_ops_release_schema() + with get_db() as conn: + with conn.cursor() as cur: + cur.execute( + """ + SELECT + id, release_version, channel, commit_sha, status, artifact_url, checksum, notes, metadata_json, created_by, created_at, activated_at, updated_at + FROM ops_releases + WHERE id = %s + """, + (int(release_id),), + ) + row = cur.fetchone() + return _serialize_release_row(row) if row else {} + + +def get_latest_release(*, channel: str = "stable") -> dict: + ensure_ops_release_schema() + normalized_channel = str(channel or "stable").strip() or "stable" + with get_db() as conn: + with conn.cursor() as cur: + cur.execute( + """ + SELECT + id, release_version, channel, commit_sha, status, artifact_url, checksum, notes, metadata_json, created_by, created_at, activated_at, updated_at + FROM ops_releases + WHERE channel = %s + ORDER BY + CASE WHEN status = 'active' THEN 0 WHEN status = 'ready' THEN 1 ELSE 2 END, + created_at DESC, + id DESC + LIMIT 1 + """, + (normalized_channel,), + ) + row = cur.fetchone() + return _serialize_release_row(row) if row else {} + + +def _compact_package_metadata(payload: dict) -> dict: + package = dict(payload or {}) + return { + "available": bool(package.get("available", False)), + "package_name": str(package.get("package_name") or ""), + "package_type": str(package.get("package_type") or ""), + "generated_at": str(package.get("generated_at") or ""), + "commit_sha": str(package.get("commit_sha") or ""), + "commit_ref": str(package.get("commit_ref") or ""), + "smoke_test_ok": bool(package.get("smoke_test_ok", False)), + "smoke_test_message": str(package.get("smoke_test_message") or ""), + "final_release_ok": bool(package.get("final_release_ok", False)), + "final_release_report_available": bool(package.get("final_release_report_available", False)), + "final_release_report_ok": package.get("final_release_report_ok"), + "final_release_report_matches_latest": package.get("final_release_report_matches_latest"), + "final_release_report_package_name": str(package.get("final_release_report_package_name") or ""), + "final_release_report_stale_reason": str(package.get("final_release_report_stale_reason") or ""), + "final_release_gate_decision": str(package.get("final_release_gate_decision") or ""), + "final_release_gate_blocked_reasons": list(package.get("final_release_gate_blocked_reasons") or []), + "final_release_gate_verify_ok": package.get("final_release_gate_verify_ok"), + "final_release_gate_smoke_test_ok": package.get("final_release_gate_smoke_test_ok"), + "download_path": str(package.get("download_path") or ""), + "sha256_download_path": str(package.get("sha256_download_path") or ""), + "reason": str(package.get("reason") or ""), + "manifest": _extract_release_manifest(package), + } + + +def _compact_launchpad_runtime_build_info(payload: dict) -> dict: + build_info = dict(payload or {}) + route_surface = dict(build_info.get("route_surface") or {}) + repository_capabilities = dict(build_info.get("repository_capabilities") or {}) + route_surface_missing_keys = [ + str(item).strip() + for item in list(route_surface.get("missing_keys") or []) + if str(item).strip() + ] + route_surface_expected_paths = { + str(key).strip(): str(value).strip() + for key, value in dict(route_surface.get("expected_paths") or {}).items() + if str(key).strip() + } + repo_supports_install_command_block = bool(repository_capabilities.get("supports_install_command_block", False)) + route_surface_declares_bootstrap_plan = "ops_node_handover_bootstrap_plan" in route_surface_expected_paths + runtime_schema_stale = bool( + repo_supports_install_command_block + and ( + "ops_node_handover_bootstrap_plan" in route_surface_missing_keys + or not route_surface_declares_bootstrap_plan + ) + ) + return { + "source": str(build_info.get("source") or "").strip(), + "package_name": str(build_info.get("package_name") or "").strip(), + "commit_sha": str(build_info.get("commit_sha") or "").strip(), + "commit_ref": str(build_info.get("commit_ref") or "").strip(), + "generated_at": str(build_info.get("generated_at") or "").strip(), + "route_surface_complete": bool(route_surface.get("surface_complete", False)), + "route_surface_missing_keys": route_surface_missing_keys, + "route_surface_declares_bootstrap_plan": route_surface_declares_bootstrap_plan, + "repository_supports_install_command_block": repo_supports_install_command_block, + "runtime_schema_stale": runtime_schema_stale, + } + + +def _compact_release_payload(payload: dict) -> dict: + release = dict(payload or {}) + compact = { + "id": int(release.get("id") or 0), + "release_version": str(release.get("release_version") or ""), + "channel": str(release.get("channel") or ""), + "commit_sha": str(release.get("commit_sha") or ""), + "status": str(release.get("status") or ""), + "artifact_url": str(release.get("artifact_url") or ""), + "artifact_url_present": bool(str(release.get("artifact_url") or "").strip()), + "checksum": str(release.get("checksum") or ""), + "created_by": str(release.get("created_by") or ""), + "created_at": str(release.get("created_at") or ""), + "updated_at": str(release.get("updated_at") or ""), + "is_preview": bool(release.get("is_preview", False)), + "manifest": _extract_release_manifest(release), + } + compact["status_label"] = str(release.get("status_label") or "").strip() or _release_status_label(compact.get("status")) + compact["summary"] = str(release.get("summary") or "").strip() or _build_release_summary_text(compact) + compact["summary_text"] = str(release.get("summary_text") or compact.get("summary") or "").strip() + compact["focus_ref"] = dict(release.get("focus_ref") or {}) or _build_release_focus_ref(compact, section="release_detail") + return compact + + +def _compact_policy_preview(preview: dict) -> dict: + data = dict(preview or {}) + return { + "risk_level": str(data.get("risk_level") or ""), + "approval_required": bool(data.get("approval_required", False)), + "blocked": bool(data.get("blocked", False)), + "blocking_reasons": list(data.get("blocking_reasons") or []), + "approval_reasons": list(data.get("approval_reasons") or []), + "warnings": list(data.get("warnings") or []), + "recommendations": list(data.get("recommendations") or []), + "target_summary": dict(data.get("target_summary") or {}), + "batch_plan": dict(data.get("batch_plan") or {}), + "cluster_guardrails": dict(data.get("cluster_guardrails") or {}), + "release_gate": dict(data.get("release_gate") or {}), + } + + +def _compact_smart_rollout_plan(payload: dict) -> dict: + smart_rollout = dict(payload or {}) + selector = dict(smart_rollout.get("target_selector") or {}) + return { + "available": bool(smart_rollout.get("available", False)), + "mode": str(smart_rollout.get("mode") or ""), + "mode_label": str(smart_rollout.get("mode_label") or ""), + "execution_mode": str(smart_rollout.get("execution_mode") or ""), + "execution_mode_label": str(smart_rollout.get("execution_mode_label") or ""), + "reason": str(smart_rollout.get("reason") or ""), + "summary": str(smart_rollout.get("summary") or smart_rollout.get("reason") or ""), + "dominant_region": str(smart_rollout.get("dominant_region") or ""), + "target_node_codes": list(selector.get("node_codes") or []), + "target_nodes": [ + { + "node_code": str(item.get("node_code") or ""), + "region": str(item.get("region") or ""), + "role": str(item.get("role") or ""), + "status": str(item.get("status") or ""), + "current_load": int(item.get("current_load", 0) or 0), + "remote_agent_ready": bool(item.get("remote_agent_ready", False)), + "ssh_ready": bool(item.get("ssh_ready", False)), + "execution_ready": bool(item.get("execution_ready", False)), + "detect_participating": bool(item.get("detect_participating", False)), + "delivery_queue_state": str(item.get("delivery_queue_state") or ""), + } + for item in list(smart_rollout.get("target_nodes") or []) + if str(item.get("node_code") or "").strip() + ], + "policy": dict(smart_rollout.get("policy") or {}), + } + + +def _compact_latest_package_preview(message: str, data: dict) -> dict: + payload = dict(data or {}) + return { + "message": str(message or ""), + "preview_source": str(payload.get("preview_source") or ""), + "deduplicated": bool(payload.get("release_deduplicated", payload.get("deduplicated", False))), + "release": _compact_release_payload(payload.get("release") or {}), + "package": _compact_package_metadata(payload.get("package") or {}), + "resolved_artifact_url": str(payload.get("resolved_artifact_url") or ""), + "resolved_sha256_download_url": str(payload.get("resolved_sha256_download_url") or ""), + } + + +def _compact_latest_smart_rollout_preview(message: str, data: dict) -> dict: + payload = dict(data or {}) + preview = _compact_policy_preview(payload.get("preview") or {}) + smart_rollout = _compact_smart_rollout_plan(payload.get("smart_rollout") or {}) + release = _compact_release_payload(payload.get("release") or {}) + requires_confirmation = bool(payload.get("requires_confirmation", False)) + blocked = bool(payload.get("blocked", False) or preview.get("blocked", False)) + preview_batch_plan = dict(preview.get("batch_plan") or {}) + preview_target_summary = dict(preview.get("target_summary") or {}) + rollout_preview = { + "release": release, + "execution_mode": str(smart_rollout.get("execution_mode") or ""), + "execution_mode_label": str(smart_rollout.get("execution_mode_label") or ""), + "target_nodes_total": int(preview_target_summary.get("nodes_total", len(list(smart_rollout.get("target_nodes") or []))) or 0), + "target_node_codes": list(smart_rollout.get("target_node_codes") or []), + "expected_batches_total": int(preview_batch_plan.get("batches_total", 0) or 0), + "expected_jobs_total": int(preview_target_summary.get("nodes_total", len(list(smart_rollout.get("target_nodes") or []))) or 0), + "gate": dict(preview.get("release_gate") or {}), + "batch_plan": dict(preview_batch_plan), + "summary": str( + (dict(preview.get("release_gate") or {}).get("summary") or "") + or smart_rollout.get("summary") + or message + or "" + ).strip(), + "focus_ref": _build_release_focus_ref( + release, + section="rollout_preview", + ), + } + rollout_preview["summary_text"] = str(rollout_preview.get("summary") or "") + return { + "message": str(message or ""), + "preview_source": str(payload.get("preview_source") or ""), + "release": release, + "package": _compact_package_metadata(payload.get("package") or {}), + "smart_rollout": smart_rollout, + "policy_preview": preview, + "rollout_preview": rollout_preview, + "requires_confirmation": requires_confirmation, + "blocked": blocked, + "rollout_created": bool(payload.get("rollout_created", False)), + "release_deduplicated": bool(payload.get("release_deduplicated", payload.get("deduplicated", False))), + } + + +def _launchpad_preview_execution_context(preview: dict) -> tuple[str, str]: + smart_rollout = dict((preview or {}).get("smart_rollout") or {}) + execution_mode = str(smart_rollout.get("execution_mode") or "").strip() + execution_mode_label = str(smart_rollout.get("execution_mode_label") or "").strip() + if execution_mode or execution_mode_label: + return execution_mode, execution_mode_label or _rollout_execution_mode_label(execution_mode) + release_gate = dict((dict(preview or {}).get("policy_preview") or {}).get("release_gate") or {}) + execution_mode = str(release_gate.get("execution_mode") or "").strip() + execution_mode_label = str(release_gate.get("execution_mode_label") or "").strip() + if execution_mode or execution_mode_label: + return execution_mode, execution_mode_label or _rollout_execution_mode_label(execution_mode) + return "", "" + + +def _decorate_release_launchpad_status( + status: dict, + *, + gap_row: dict, + onboarding_bootstrap_pending_nodes: int, + onboarding_acceptance_ready_nodes: int, +) -> dict: + normalized_status = dict(status or {}) + normalized_gap_row = dict(gap_row or {}) + normalized_status["onboarding_bootstrap_pending_nodes"] = int(onboarding_bootstrap_pending_nodes or 0) + normalized_status["onboarding_acceptance_ready_nodes"] = int(onboarding_acceptance_ready_nodes or 0) + normalized_status["onboarding_gap_nodes_total"] = int( + int(onboarding_bootstrap_pending_nodes or 0) + int(onboarding_acceptance_ready_nodes or 0) + ) + normalized_status["recommended_target_node_code"] = str( + normalized_status.get("recommended_target_node_code") or normalized_gap_row.get("node_code") or "" + ).strip() + normalized_status["recommended_recovery_label"] = str( + normalized_status.get("recommended_recovery_label") or normalized_gap_row.get("recovery_label") or "" + ).strip() + normalized_status["recommended_recovery_summary"] = str( + normalized_status.get("recommended_recovery_summary") + or normalized_gap_row.get("recovery_summary") + or normalized_gap_row.get("onboarding_summary") + or "" + ).strip() + normalized_status["recommended_onboarding_stage_code"] = str( + normalized_gap_row.get("onboarding_stage_code") or "" + ).strip() + normalized_status["recommended_onboarding_stage_label"] = str( + normalized_gap_row.get("onboarding_stage_label") or "" + ).strip() + return normalized_status + + +def _build_release_launchpad_status( + *, + package: dict, + latest_package_preview: dict, + worker_preview: dict, + control_preview: dict, + runtime_build_info: dict | None = None, +) -> dict: + if not bool(package.get("available", False)): + status = { + "status": "blocked", + "status_label": "缺少发布包", + "summary": "当前没有可用的最新发布包，请先在控制面本机完成打包。", + "recommended_action_code": "release_package", + "recommended_command": build_bash_command("drive_ops_center.sh", "release-package"), + } + status["summary_text"] = str(status.get("summary") or "") + status["focus_ref"] = _build_release_focus_ref({}, section="release_launchpad") + return status + + package_final_gate_ok = package.get("final_release_ok") + if package_final_gate_ok is False: + stale_reason = str(package.get("final_release_report_stale_reason") or "").strip() + blocked_reasons = [ + str(item).strip() + for item in list(package.get("final_release_gate_blocked_reasons") or []) + if str(item).strip() + ] + gate_decision = str(package.get("final_release_gate_decision") or "").strip() + summary = "最新发布包尚未完成最终签收，当前应先完成最终验包与签收，再进入 Release / Rollout。" + if stale_reason: + summary = f"{summary} {stale_reason}" + elif blocked_reasons: + summary = f"{summary} 当前阻断：{' / '.join(blocked_reasons)}" + elif gate_decision: + summary = f"{summary} 当前门禁状态：{gate_decision}" + status = { + "status": "blocked", + "status_label": "待完成最终签收", + "summary": summary, + "recommended_action_code": "release_prepare", + "recommended_command": build_bash_command("drive_ops_center.sh", "release-prepare"), + "recommended_execution_mode": "", + "recommended_execution_mode_label": "", + "final_release_gate_ready": False, + "final_release_gate_decision": gate_decision, + "final_release_gate_blocked_reasons": blocked_reasons, + } + status["summary_text"] = str(status.get("summary") or "") + status["focus_ref"] = _build_release_focus_ref({}, section="release_launchpad") + return status + + runtime_refresh_state = dict(runtime_build_info or {}) + runtime_schema_stale = bool(runtime_refresh_state.get("runtime_schema_stale", False)) + route_surface_complete = bool(runtime_refresh_state.get("route_surface_complete", True)) + route_surface_missing_keys = [ + str(item).strip() + for item in list(runtime_refresh_state.get("route_surface_missing_keys") or []) + if str(item).strip() + ] + if runtime_schema_stale or not route_surface_complete: + summary = "运行中的控制面 API 还没有刷新到当前仓库的最新运维能力，当前 Release Launchpad 判断可能偏旧，建议先完成运行时刷新与复检。" + if runtime_schema_stale: + summary = "仓库已经具备新的节点接管与发布能力，但运行中的控制面 API 路由面仍旧，建议先执行运行时刷新，再回到 Launchpad 继续判断。" + elif route_surface_missing_keys: + summary = f"{summary} 当前缺少路由键：{', '.join(route_surface_missing_keys)}。" + status = { + "status": "attention", + "status_label": "待刷新运行时", + "summary": summary, + "recommended_action_code": "api-restart", + "recommended_command": build_bash_command("drive_ops_center.sh", "runtime-refresh-recover"), + "recommended_execution_mode": "", + "recommended_execution_mode_label": "", + "runtime_refresh_required": True, + "runtime_schema_stale": runtime_schema_stale, + "runtime_route_surface_complete": route_surface_complete, + "runtime_route_surface_missing_keys": route_surface_missing_keys, + } + status["summary_text"] = str(status.get("summary") or "") + status["focus_ref"] = _build_release_focus_ref({}, section="release_launchpad") + return status + + worker_plan = dict(worker_preview.get("smart_rollout") or {}) + worker_policy = dict(worker_preview.get("policy_preview") or {}) + control_plan = dict(control_preview.get("smart_rollout") or {}) + control_policy = dict(control_preview.get("policy_preview") or {}) + worker_execution_mode, worker_execution_mode_label = _launchpad_preview_execution_context(worker_preview) + control_execution_mode, control_execution_mode_label = _launchpad_preview_execution_context(control_preview) + worker_release_gate = dict(worker_policy.get("release_gate") or {}) + control_release_gate = dict(control_policy.get("release_gate") or {}) + candidate_rows = [ + dict(item or {}) + for item in ( + list(((worker_release_gate.get("operational_readiness") or {}).get("rows") or [])) + + list(((control_release_gate.get("operational_readiness") or {}).get("rows") or [])) + ) + if dict(item or {}) + ] + gap_row = next( + ( + row + for row in candidate_rows + if str(row.get("recovery_action") or "").strip() in {"bootstrap_run", "run_acceptance"} + ), + {}, + ) + gap_recovery_action = str(gap_row.get("recovery_action") or "").strip() + gap_recovery_label = str(gap_row.get("recovery_label") or "").strip() + gap_recovery_summary = str( + gap_row.get("recovery_summary") + or gap_row.get("onboarding_summary") + or gap_row.get("agent_reason") + or "" + ).strip() + gap_node_code = str(gap_row.get("node_code") or "").strip() + onboarding_bootstrap_pending_nodes = sum( + 1 for row in candidate_rows if str(row.get("recovery_action") or "").strip() == "bootstrap_run" + ) + onboarding_acceptance_ready_nodes = sum( + 1 for row in candidate_rows if str(row.get("recovery_action") or "").strip() == "run_acceptance" + ) + + if bool(worker_policy.get("blocked", False)) or bool(control_policy.get("blocked", False)): + recommended_action_code = "fix_rollout_blockers" + recommended_command = build_bash_command("drive_ops_center.sh", "release-preview-smart", "worker") + summary = "最新发布包已就绪，但当前 Rollout 预检存在阻断条件，需先修复节点接管、在线状态或巡检缺口。" + if gap_recovery_action in {"bootstrap_run", "run_acceptance"} and gap_node_code: + recommended_action_code = gap_recovery_action + if gap_recovery_action == "bootstrap_run": + recommended_command = build_bash_command("drive_ops_center.sh", "node-bootstrap-run", "http://127.0.0.1:8100", gap_node_code, "cli") + else: + recommended_command = build_bash_command("drive_ops_center.sh", "node-acceptance-run", "http://127.0.0.1:8100", gap_node_code, "cli") + summary = gap_recovery_summary or summary + status = { + "status": "blocked", + "status_label": "发布门禁阻断", + "summary": summary, + "recommended_action_code": recommended_action_code, + "recommended_command": recommended_command, + "recommended_execution_mode": worker_execution_mode or control_execution_mode, + "recommended_execution_mode_label": worker_execution_mode_label or control_execution_mode_label, + "recommended_target_node_code": gap_node_code, + "recommended_recovery_label": gap_recovery_label, + } + status["summary_text"] = str(status.get("summary") or "") + status["focus_ref"] = _build_release_focus_ref({}, section="release_launchpad") + return _decorate_release_launchpad_status( + status, + gap_row=gap_row, + onboarding_bootstrap_pending_nodes=onboarding_bootstrap_pending_nodes, + onboarding_acceptance_ready_nodes=onboarding_acceptance_ready_nodes, + ) + + if bool(worker_preview.get("requires_confirmation", False)) or bool(control_preview.get("requires_confirmation", False)): + status = { + "status": "attention", + "status_label": "待人工确认", + "summary": "最新发布包已经满足预发条件，但智能 Rollout 仍有告警或审批要求，建议先人工确认后再推进。", + "recommended_action_code": "review_smart_rollout_preview", + "recommended_command": build_bash_command("drive_ops_center.sh", "release-preview-smart", "worker"), + "recommended_execution_mode": worker_execution_mode or control_execution_mode, + "recommended_execution_mode_label": worker_execution_mode_label or control_execution_mode_label, + } + status["summary_text"] = str(status.get("summary") or "") + status["focus_ref"] = _build_release_focus_ref({}, section="release_launchpad") + return _decorate_release_launchpad_status( + status, + gap_row=gap_row, + onboarding_bootstrap_pending_nodes=onboarding_bootstrap_pending_nodes, + onboarding_acceptance_ready_nodes=onboarding_acceptance_ready_nodes, + ) + + if bool(worker_plan.get("available", False)): + status = { + "status": "ready", + "status_label": "可发 Worker 灰度", + "summary": "最新发布包、Release 记录与 Worker 智能 Rollout 预检均已收口，当前可以直接进入 Worker 灰度发布。", + "recommended_action_code": "publish_latest_worker", + "recommended_command": build_bash_command("drive_ops_center.sh", "publish-latest", "worker"), + "recommended_execution_mode": worker_execution_mode, + "recommended_execution_mode_label": worker_execution_mode_label, + } + status["summary_text"] = str(status.get("summary") or "") + status["focus_ref"] = _build_release_focus_ref({}, section="release_launchpad") + return _decorate_release_launchpad_status( + status, + gap_row=gap_row, + onboarding_bootstrap_pending_nodes=onboarding_bootstrap_pending_nodes, + onboarding_acceptance_ready_nodes=onboarding_acceptance_ready_nodes, + ) + + if bool(control_plan.get("available", False)): + status = { + "status": "attention", + "status_label": "仅 Control 可发", + "summary": "当前 Control 侧满足预发条件，但 Worker 侧还未形成可用灰度面，建议先收敛 Worker 节点状态。", + "recommended_action_code": "review_control_rollout", + "recommended_command": build_bash_command("drive_ops_center.sh", "release-preview-smart", "control"), + "recommended_execution_mode": control_execution_mode, + "recommended_execution_mode_label": control_execution_mode_label, + } + status["summary_text"] = str(status.get("summary") or "") + status["focus_ref"] = _build_release_focus_ref({}, section="release_launchpad") + return _decorate_release_launchpad_status( + status, + gap_row=gap_row, + onboarding_bootstrap_pending_nodes=onboarding_bootstrap_pending_nodes, + onboarding_acceptance_ready_nodes=onboarding_acceptance_ready_nodes, + ) + + status = { + "status": "attention", + "status_label": "待补执行面", + "summary": "最新发布包已经可用，但当前没有可用于智能 Rollout 的在线启用执行节点，需要先补齐托管或接入状态。", + "recommended_action_code": "fix_managed_nodes", + "recommended_command": build_bash_command("drive_ops_center.sh", "doctor"), + "recommended_execution_mode": worker_execution_mode or control_execution_mode, + "recommended_execution_mode_label": worker_execution_mode_label or control_execution_mode_label, + } + if gap_recovery_action in {"bootstrap_run", "run_acceptance"} and gap_node_code: + status["recommended_action_code"] = gap_recovery_action + status["recommended_command"] = ( + build_bash_command("drive_ops_center.sh", "node-bootstrap-run", "http://127.0.0.1:8100", gap_node_code, "cli") + if gap_recovery_action == "bootstrap_run" + else build_bash_command("drive_ops_center.sh", "node-acceptance-run", "http://127.0.0.1:8100", gap_node_code, "cli") + ) + status["summary"] = gap_recovery_summary or status["summary"] + status["recommended_target_node_code"] = gap_node_code + status["recommended_recovery_label"] = gap_recovery_label + status["summary_text"] = str(status.get("summary") or "") + status["focus_ref"] = _build_release_focus_ref({}, section="release_launchpad") + return _decorate_release_launchpad_status( + status, + gap_row=gap_row, + onboarding_bootstrap_pending_nodes=onboarding_bootstrap_pending_nodes, + onboarding_acceptance_ready_nodes=onboarding_acceptance_ready_nodes, + ) + + +def get_release_launchpad(*, control_plane_base_url: str = "", channel: str = "stable") -> dict: + package = get_latest_release_package_metadata() + release_summary = get_release_summary() + latest_release = get_latest_release(channel=channel) + runtime_build_info = _compact_launchpad_runtime_build_info(get_runtime_build_info()) + + preview_ok, preview_message, preview_data = preview_release_from_latest_package( + {"channel": channel, "created_by": "api/release-launchpad"}, + control_plane_base_url=control_plane_base_url, + ) + worker_ok, worker_message, worker_data = preview_smart_release_rollout_from_latest_package( + { + "channel": channel, + "mode": "worker", + "created_by": "api/release-launchpad", + "rollout_created_by": "api/release-launchpad/worker", + }, + control_plane_base_url=control_plane_base_url, + ) + control_ok, control_message, control_data = preview_smart_release_rollout_from_latest_package( + { + "channel": channel, + "mode": "control", + "created_by": "api/release-launchpad", + "rollout_created_by": "api/release-launchpad/control", + }, + control_plane_base_url=control_plane_base_url, + ) + + latest_package_preview = _compact_latest_package_preview(preview_message, preview_data if preview_ok else {"package": package}) + worker_preview = _compact_latest_smart_rollout_preview(worker_message, worker_data if worker_ok else {"package": package}) + control_preview = _compact_latest_smart_rollout_preview(control_message, control_data if control_ok else {"package": package}) + compact_latest_release = _compact_release_payload(latest_release) + launchpad_status = _build_release_launchpad_status( + package=_compact_package_metadata(package), + latest_package_preview=latest_package_preview, + worker_preview=worker_preview, + control_preview=control_preview, + runtime_build_info=runtime_build_info, + ) + launchpad_status["focus_ref"] = _build_release_focus_ref( + compact_latest_release, + section="release_launchpad", + ) + + return { + "channel": channel, + "control_plane_base_url": _normalize_public_base_url(control_plane_base_url), + "latest_package": _compact_package_metadata(package), + "latest_release": compact_latest_release, + "release_summary": release_summary, + "runtime_build_info": runtime_build_info, + "latest_package_preview": latest_package_preview, + "worker_rollout_preview": worker_preview, + "control_rollout_preview": control_preview, + "launchpad_status": launchpad_status, + } + + +def activate_release(release_id: int) -> tuple[bool, str, dict]: + ensure_ops_release_schema() + release = get_release(release_id) + if not release: + return False, "Release 不存在", {} + channel = str(release.get("channel") or "stable") + + with get_db() as conn: + conn.autocommit = False + with conn.cursor() as cur: + cur.execute( + """ + UPDATE ops_releases + SET + status = CASE WHEN id = %s THEN 'active' ELSE status END, + activated_at = CASE WHEN id = %s THEN CURRENT_TIMESTAMP ELSE activated_at END, + updated_at = CURRENT_TIMESTAMP + WHERE channel = %s AND id = %s + """, + (int(release_id), int(release_id), channel, int(release_id)), + ) + cur.execute( + """ + UPDATE ops_releases + SET + status = CASE WHEN status = 'active' THEN 'ready' ELSE status END, + updated_at = CURRENT_TIMESTAMP + WHERE channel = %s AND id <> %s + """, + (channel, int(release_id)), + ) + conn.commit() + + return True, "Release 已激活", {"release": get_release(int(release_id))} + + +def get_release_summary() -> dict: + ensure_ops_release_schema() + with get_db() as conn: + with conn.cursor() as cur: + cur.execute( + """ + SELECT status, count(*) + FROM ops_releases + GROUP BY status + """ + ) + release_rows = cur.fetchall() + cur.execute( + """ + SELECT channel, release_version + FROM ops_releases + WHERE status = 'active' + ORDER BY channel ASC, activated_at DESC NULLS LAST, id DESC + """ + ) + active_rows = cur.fetchall() + cur.execute( + """ + SELECT status, count(*) + FROM ops_release_rollouts + GROUP BY status + """ + ) + rollout_rows = cur.fetchall() + status_counts = {str(row[0] or ""): int(row[1] or 0) for row in release_rows} + active_by_channel = {str(row[0] or ""): str(row[1] or "") for row in active_rows} + rollout_status_counts = {str(row[0] or ""): int(row[1] or 0) for row in rollout_rows} + return { + "status_counts": status_counts, + "total": sum(status_counts.values()), + "active_by_channel": active_by_channel, + "rollout_status_counts": rollout_status_counts, + "rollouts_total": sum(rollout_status_counts.values()), + } + + +def _get_rollout_row(rollout_id: int) -> tuple | None: + with get_db() as conn: + with conn.cursor() as cur: + cur.execute( + """ + SELECT + id, release_id, rollout_code, target_selector_json, target_nodes_json, policy_json, status, + batch_cursor, batches_total, jobs_total, jobs_created, result_summary_json, created_by, created_at, updated_at + FROM ops_release_rollouts + WHERE id = %s + """, + (int(rollout_id),), + ) + return cur.fetchone() + + +def _list_rollout_jobs(rollout_id: int, *, total_targets: int) -> list[dict]: + from app.services.ops_job_service import list_ops_jobs + + safe_limit = max(50, min(max(total_targets * 2, total_targets + 10), 10000)) + return list_ops_jobs(limit=safe_limit, rollout_id=int(rollout_id)) + + +def _compute_rollout_summary(rollout: dict, jobs: list[dict]) -> dict: + target_nodes = list(rollout.get("target_nodes") or []) + target_node_codes = [str(item.get("node_code") or "").strip() for item in target_nodes if str(item.get("node_code") or "").strip()] + launched_node_codes = [] + status_counts: dict[str, int] = {} + waiting_approval_targets = 0 + + for job in jobs: + status = str(job.get("status") or "") + target_node_code = str(job.get("target_node_code") or "").strip() + status_counts[status] = int(status_counts.get(status, 0) or 0) + 1 + if target_node_code and target_node_code not in launched_node_codes: + launched_node_codes.append(target_node_code) + if status == "awaiting_approval": + waiting_approval_targets += 1 + + launched_target_codes = set(launched_node_codes) + remaining_targets = [item for item in target_nodes if str(item.get("node_code") or "").strip() not in launched_target_codes] + remaining_target_codes = [str(item.get("node_code") or "").strip() for item in remaining_targets] + + active_jobs = sum(int(status_counts.get(status, 0) or 0) for status in _ACTIVE_JOB_STATUSES) + issue_jobs = sum(int(status_counts.get(status, 0) or 0) for status in _ISSUE_JOB_STATUSES) + success_jobs = int(status_counts.get("success", 0) or 0) + failed_jobs = int(status_counts.get("failed", 0) or 0) + cancelled_jobs = int(status_counts.get("cancelled", 0) or 0) + blocked_jobs = int(status_counts.get("blocked", 0) or 0) + partial_jobs = int(status_counts.get("partially_succeeded", 0) or 0) + + policy = dict(rollout.get("policy") or {}) + continue_on_failure = bool(policy.get("continue_on_failure", False)) + continue_on_partial = bool(policy.get("continue_on_partial", False)) + + halted_by_failures = (failed_jobs > 0 or cancelled_jobs > 0 or blocked_jobs > 0) and not continue_on_failure + halted_by_partial = partial_jobs > 0 and not continue_on_partial + halted = halted_by_failures or halted_by_partial + next_batch_ready = active_jobs == 0 and bool(remaining_target_codes) and not halted + + if active_jobs > 0: + status = "awaiting_approval" if active_jobs == waiting_approval_targets else "running" + elif not jobs and remaining_target_codes: + status = "planned" + elif next_batch_ready: + status = "ready_for_next_batch" + elif remaining_target_codes and halted: + status = "halted" + elif not remaining_target_codes: + if issue_jobs <= 0 and success_jobs > 0: + status = "completed" + elif success_jobs <= 0 and issue_jobs > 0: + status = "failed" + else: + status = "completed_with_issues" + else: + status = str(rollout.get("status") or "planned") + + launched_count = len(launched_target_codes) + total_targets = len(target_node_codes) + batches_total = max(int(rollout.get("batches_total") or 0), _calculate_total_batches(total_targets, policy)) + batch_size = max(1, int(policy.get("batch_size") or 1)) if total_targets > 0 else 0 + current_batch_index = int(ceil(launched_count / batch_size)) if launched_count > 0 and batch_size > 0 else 0 + + return { + "status": status, + "status_counts": status_counts, + "jobs_created": int(len(jobs)), + "active_jobs": active_jobs, + "issue_jobs": issue_jobs, + "success_jobs": success_jobs, + "failed_jobs": failed_jobs, + "cancelled_jobs": cancelled_jobs, + "blocked_jobs": blocked_jobs, + "partial_jobs": partial_jobs, + "launched_targets": launched_count, + "remaining_targets": len(remaining_target_codes), + "remaining_target_codes": remaining_target_codes, + "launched_target_codes": list(launched_target_codes), + "next_batch_ready": next_batch_ready, + "halted": halted, + "current_batch_index": current_batch_index, + "batches_total": batches_total, + } + + +def refresh_release_rollout(rollout_id: int, *, advance_if_possible: bool = True) -> dict: + ensure_ops_release_schema() + row = _get_rollout_row(int(rollout_id)) + if not row: + return {} + rollout = _serialize_rollout_row(row) + jobs = _list_rollout_jobs(int(rollout_id), total_targets=len(list(rollout.get("target_nodes") or []))) + summary = _compute_rollout_summary(rollout, jobs) + + with get_db() as conn: + with conn.cursor() as cur: + cur.execute( + """ + UPDATE ops_release_rollouts + SET + status = %s, + batch_cursor = %s, + batches_total = %s, + jobs_total = %s, + jobs_created = %s, + result_summary_json = %s, + updated_at = CURRENT_TIMESTAMP + WHERE id = %s + """, + ( + str(summary.get("status") or "planned"), + int(summary.get("launched_targets") or 0), + int(summary.get("batches_total") or 0), + len(list(rollout.get("target_nodes") or [])), + int(summary.get("jobs_created") or 0), + json.dumps(summary, ensure_ascii=False), + int(rollout_id), + ), + ) + conn.commit() + + refreshed_row = _get_rollout_row(int(rollout_id)) + refreshed = _serialize_rollout_row(refreshed_row) if refreshed_row else {} + refreshed["jobs"] = jobs + if advance_if_possible and bool((refreshed.get("policy") or {}).get("auto_advance", False)) and bool((refreshed.get("result_summary") or {}).get("next_batch_ready", False)): + ok, _message, data = _enqueue_rollout_batch(int(rollout_id), created_by=str(refreshed.get("created_by") or "api"), reason="auto_advance") + if ok and isinstance(data, dict) and data.get("rollout"): + return data.get("rollout") or refreshed + return refreshed + + +def refresh_release_rollout_for_job(job_id: int) -> dict: + from app.services.ops_job_service import get_ops_job + + job = get_ops_job(int(job_id)) + rollout_id = int(job.get("rollout_id") or 0) + if rollout_id <= 0: + return {} + return refresh_release_rollout(rollout_id) + + +def _build_release_job_payload(release: dict, rollout: dict) -> dict: + policy = dict(rollout.get("policy") or {}) + deploy_payload = dict(policy.get("deploy_payload") or {}) + return { + "release_id": int(release.get("id") or 0), + "rollout_id": int(rollout.get("id") or 0), + "release_version": str(release.get("release_version") or ""), + "artifact_url": str(release.get("artifact_url") or ""), + "checksum": str(release.get("checksum") or ""), + "channel": str(release.get("channel") or ""), + "commit_sha": str(release.get("commit_sha") or ""), + "notes": str(release.get("notes") or ""), + **deploy_payload, + } + + +def _enqueue_rollout_batch(rollout_id: int, *, created_by: str, reason: str = "manual_advance") -> tuple[bool, str, dict]: + ensure_ops_release_schema() + rollout = refresh_release_rollout(int(rollout_id), advance_if_possible=False) + if not rollout: + return False, "Rollout 不存在", {} + + release = get_release(int(rollout.get("release_id") or 0)) + if not release: + return False, "Release 不存在", {"rollout": rollout} + + summary = dict(rollout.get("result_summary") or {}) + if int(summary.get("active_jobs") or 0) > 0: + return False, "当前已有批次正在执行，暂不可推进下一批", {"rollout": rollout} + if bool(summary.get("halted", False)): + return False, "当前 rollout 已暂停，请先处理失败/阻断节点", {"rollout": rollout} + if int(summary.get("remaining_targets") or 0) <= 0: + return False, "已无剩余目标节点", {"rollout": rollout} + + target_nodes = list(rollout.get("target_nodes") or []) + launched_node_codes = set(summary.get("launched_target_codes") or []) + remaining_targets = [ + item + for item in target_nodes + if str(item.get("node_code") or "").strip() and str(item.get("node_code") or "").strip() not in launched_node_codes + ] + if not remaining_targets: + return False, "已无剩余目标节点", {"rollout": rollout} + + policy = dict(rollout.get("policy") or {}) + batch_size = int(policy.get("first_batch_size") or 1) if int(summary.get("jobs_created") or 0) == 0 else int(policy.get("batch_size") or 1) + batch_targets = remaining_targets[: max(1, batch_size)] + + from app.services.ops_job_service import create_ops_job, dispatch_ops_job + + jobs: list[dict] = [] + auto_dispatch = bool(policy.get("auto_dispatch", False)) + auto_approve = bool(policy.get("auto_approve", False)) + execution_mode = str(policy.get("execution_mode") or "remote-agent").strip() or "remote-agent" + job_payload = _build_release_job_payload(release, rollout) + + for target in batch_targets: + target_node_code = str(target.get("node_code") or "").strip() + if not target_node_code: + continue + job_ok, _job_message, job_data = create_ops_job( + { + "action": "deploy.release", + "target_type": "node", + "target_node_code": target_node_code, + "requested_by": created_by, + "execution_mode": execution_mode, + "auto_approve": auto_approve, + "rollout_id": int(rollout.get("id") or 0), + "payload": { + **job_payload, + "target_node_code": target_node_code, + }, + } + ) + job = (job_data.get("job") or {}) if isinstance(job_data, dict) else {} + if job: + jobs.append(job) + if auto_dispatch and str(job.get("status") or "") == "queued": + dispatch_ops_job(int(job.get("id") or 0)) + elif not job_ok: + continue + + refreshed = refresh_release_rollout(int(rollout_id), advance_if_possible=False) + refreshed.setdefault("result_summary", {}) + refreshed["result_summary"]["last_enqueue_reason"] = reason + return True, "已生成下一批发布任务", {"rollout": refreshed, "jobs": jobs} + + +def create_release_rollout(release_id: int, payload: dict) -> tuple[bool, str, dict]: + ensure_ops_release_schema() + release = get_release(release_id) + if not release: + return False, "Release 不存在", {} + + selector = payload.get("target_selector") or {} + created_by = str(payload.get("created_by") or "api").strip() or "api" + target_nodes = _resolve_rollout_targets(selector) + if not target_nodes: + return False, "未命中任何目标节点", {"release": release} + + normalized_policy = _normalize_rollout_policy(payload.get("policy") or {}, total_targets=len(target_nodes)) + rollout_code = _build_rollout_code() + batches_total = _calculate_total_batches(len(target_nodes), normalized_policy) + + with get_db() as conn: + conn.autocommit = False + with conn.cursor() as cur: + cur.execute( + """ + INSERT INTO ops_release_rollouts ( + release_id, rollout_code, target_selector_json, target_nodes_json, policy_json, status, + batch_cursor, batches_total, jobs_total, jobs_created, result_summary_json, created_by, created_at, updated_at + ) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, %s, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP) + RETURNING + id, release_id, rollout_code, target_selector_json, target_nodes_json, policy_json, status, + batch_cursor, batches_total, jobs_total, jobs_created, result_summary_json, created_by, created_at, updated_at + """, + ( + int(release_id), + rollout_code, + json.dumps(selector, ensure_ascii=False), + json.dumps({"items": target_nodes}, ensure_ascii=False), + json.dumps(normalized_policy, ensure_ascii=False), + "planned", + 0, + batches_total, + len(target_nodes), + 0, + json.dumps({}, ensure_ascii=False), + created_by, + ), + ) + row = cur.fetchone() + conn.commit() + + rollout = _serialize_rollout_row(row) + if bool(normalized_policy.get("auto_start", True)): + return _enqueue_rollout_batch(int(rollout.get("id") or 0), created_by=created_by, reason="initial_batch") + + refreshed = refresh_release_rollout(int(rollout.get("id") or 0), advance_if_possible=False) + return True, "Release rollout 已创建", {"rollout": refreshed, "jobs": []} + + +def advance_release_rollout(rollout_id: int, payload: dict | None = None) -> tuple[bool, str, dict]: + payload = payload or {} + created_by = str(payload.get("created_by") or "api").strip() or "api" + return _enqueue_rollout_batch(int(rollout_id), created_by=created_by, reason="manual_advance") + + +def list_release_rollouts(*, release_id: int | None = None, limit: int = 20) -> list[dict]: + ensure_ops_release_schema() + safe_limit = min(max(int(limit or 20), 1), 100) + conditions: list[str] = [] + params: list[object] = [] + if release_id is not None and int(release_id or 0) > 0: + conditions.append("release_id = %s") + params.append(int(release_id)) + where_clause = f"WHERE {' AND '.join(conditions)}" if conditions else "" + with get_db() as conn: + with conn.cursor() as cur: + cur.execute( + f""" + SELECT + id, release_id, rollout_code, target_selector_json, target_nodes_json, policy_json, status, + batch_cursor, batches_total, jobs_total, jobs_created, result_summary_json, created_by, created_at, updated_at + FROM ops_release_rollouts + {where_clause} + ORDER BY created_at DESC, id DESC + LIMIT %s + """, + (*params, safe_limit), + ) + rows = cur.fetchall() + return [_serialize_rollout_row(row) for row in rows] + + +def get_release_rollout(rollout_id: int, *, refresh: bool = True) -> dict: + ensure_ops_release_schema() + if refresh: + refreshed = refresh_release_rollout(int(rollout_id), advance_if_possible=False) + if refreshed: + return refreshed + row = _get_rollout_row(int(rollout_id)) + return _serialize_rollout_row(row) if row else {} + + +def list_release_rollout_jobs(rollout_id: int, *, limit: int = 200) -> list[dict]: + from app.services.ops_job_service import list_ops_jobs + + safe_limit = min(max(int(limit or 200), 1), 2000) + return list_ops_jobs(limit=safe_limit, rollout_id=int(rollout_id)) diff --git a/domain-api/app/services/ops_runtime_executor_service.py b/domain-api/app/services/ops_runtime_executor_service.py new file mode 100644 index 0000000..ac84a4c --- /dev/null +++ b/domain-api/app/services/ops_runtime_executor_service.py @@ -0,0 +1,420 @@ +from __future__ import annotations + +import json +import subprocess + +from app.core.config import settings +from app.services.ops_action_executor_core import ( + STRUCTURED_ACTIONS, + build_service_name_map, + execute_structured_action, + trim_output, +) +from app.services.ops_release_executor_core import build_remote_release_action_script, execute_release_action + + +_SSH_SUPPORTED_ACTIONS = set(STRUCTURED_ACTIONS) | {"deploy.release"} +_SSH_CONNECT_TIMEOUT_SECONDS = 12 +_SSH_REMOTE_TIMEOUT_SECONDS = { + "health.snapshot": 30, + "service.status": 45, + "service.restart": 45, + "service.start": 45, + "service.stop": 45, + "logs.collect": 45, + "diagnostics.collect": 90, + "runtime.start_worker": 45, + "runtime.stop_worker": 45, + "runtime.restart_api": 45, + "runtime.start_sync_agent": 45, + "runtime.stop_sync_agent": 45, + "deploy.release": 180, +} + + +def supports_ssh_execution(action: str) -> bool: + return str(action or "").strip() in _SSH_SUPPORTED_ACTIONS + + +def execute_local_support_action(action: str, payload: dict | None = None) -> tuple[bool, str, dict]: + normalized_action = str(action or "").strip() + if normalized_action not in _SSH_SUPPORTED_ACTIONS: + return False, f"当前未实现本机支持动作: {normalized_action}", {} + if normalized_action == "deploy.release": + return execute_release_action( + dict(payload or {}), + run_command=_run_local_command, + default_api_service_name=settings.api_service_name, + user_agent="domaincheck-local-runtime/0.1", + ) + return execute_structured_action( + normalized_action, + dict(payload or {}), + service_names=_service_name_map(), + runner=_run_local_command, + ) + + +def execute_ssh_action(node: dict, action: str, payload: dict | None = None) -> tuple[bool, str, dict]: + normalized_action = str(action or "").strip() + if normalized_action not in _SSH_SUPPORTED_ACTIONS: + return False, f"当前 SSH 执行器暂不支持动作: {normalized_action}", {} + + ssh_host = str(node.get("ssh_host") or "").strip() + ssh_user = str(node.get("ssh_user") or "").strip() + ssh_port = max(1, int(node.get("ssh_port") or 22)) + node_code = str(node.get("node_code") or "").strip() + if not ssh_host or not ssh_user: + return False, "目标节点缺少 SSH 主机或用户信息", { + "node_code": node_code, + "ssh_host": ssh_host, + "ssh_user": ssh_user, + "ssh_port": ssh_port, + } + + normalized_payload = dict(payload or {}) + service_names = _service_name_map() + if normalized_action == "deploy.release": + remote_script = build_remote_release_action_script( + normalized_payload, + default_api_service_name=service_names.get("api") or "domaincheck-api", + user_agent="domaincheck-ssh/0.1", + ) + else: + remote_script = _build_remote_python_script( + action=normalized_action, + payload=normalized_payload, + service_names=service_names, + ) + remote_command = "\n".join( + [ + "set -euo pipefail", + "if command -v python3 >/dev/null 2>&1; then", + " PYTHON_BIN=python3", + "elif command -v python >/dev/null 2>&1; then", + " PYTHON_BIN=python", + "else", + ' echo "{\\"ok\\": false, \\"message\\": \\"python interpreter missing on remote node\\", \\"result\\": {\\"executor\\": \\"ssh\\"}}"', + " exit 127", + "fi", + '"${PYTHON_BIN}" - <<\'PY\'', + remote_script.rstrip("\n"), + "PY", + ] + ) + ssh_command = [ + "ssh", + "-o", + "BatchMode=yes", + "-o", + "PreferredAuthentications=publickey", + "-o", + "StrictHostKeyChecking=accept-new", + "-o", + f"ConnectTimeout={_SSH_CONNECT_TIMEOUT_SECONDS}", + "-p", + str(ssh_port), + f"{ssh_user}@{ssh_host}", + remote_command, + ] + timeout_seconds = int(_SSH_REMOTE_TIMEOUT_SECONDS.get(normalized_action, 45) or 45) + _SSH_CONNECT_TIMEOUT_SECONDS + completed = subprocess.run( + ssh_command, + capture_output=True, + text=True, + timeout=timeout_seconds, + ) + + transport = { + "executor": "ssh", + "node_code": node_code, + "ssh_host": ssh_host, + "ssh_user": ssh_user, + "ssh_port": ssh_port, + "action": normalized_action, + "returncode": int(completed.returncode or 0), + } + parsed_ok, parsed_payload = _extract_json_from_output(completed.stdout or "") + if parsed_ok: + ok = bool(parsed_payload.get("ok", False)) + message = str(parsed_payload.get("message") or "").strip() or ("SSH 动作执行成功" if ok else "SSH 动作执行失败") + result = dict(parsed_payload.get("result") or {}) + result["transport"] = { + **transport, + "stdout_preview": trim_output(completed.stdout, 4000), + "stderr_preview": trim_output(completed.stderr, 2000), + } + if completed.returncode != 0 and ok: + ok = False + message = f"SSH 命令返回码异常: {completed.returncode}" + return ok, message, result + + stderr_preview = trim_output(completed.stderr, 4000) + stdout_preview = trim_output(completed.stdout, 4000) + message = stderr_preview or stdout_preview or f"SSH 执行失败，返回码 {completed.returncode}" + return False, message, { + "executor": "ssh", + "action": normalized_action, + "transport": { + **transport, + "stdout_preview": stdout_preview, + "stderr_preview": stderr_preview, + }, + } + + +def _service_name_map() -> dict[str, str]: + return build_service_name_map( + api_service_name=settings.api_service_name, + worker_service_name=settings.worker_service_name, + sync_agent_service_name=settings.sync_agent_service_name, + ) + + +def _run_local_command(command: list[str], *, timeout: int = 60) -> tuple[int, str, str]: + completed = subprocess.run(command, capture_output=True, text=True, timeout=timeout) + return int(completed.returncode or 0), str(completed.stdout or "").strip(), str(completed.stderr or "").strip() + + +def _build_remote_python_script(*, action: str, payload: dict, service_names: dict[str, str]) -> str: + return f"""import json +import subprocess +import sys +from pathlib import Path + +SERVICE_NAMES = {json.dumps(service_names, ensure_ascii=False)} +ACTION = {json.dumps(str(action or '').strip(), ensure_ascii=False)} +PAYLOAD = {json.dumps(dict(payload or {}), ensure_ascii=False)} + + +def trim_output(text, limit): + value = str(text or "").strip() + if len(value) <= int(limit): + return value + return value[-int(limit):] + + +def run(cmd, timeout=60): + completed = subprocess.run(cmd, capture_output=True, text=True, timeout=timeout) + return int(completed.returncode or 0), str(completed.stdout or "").strip(), str(completed.stderr or "").strip() + + +def read_tail_lines(path, max_lines): + file_path = Path(path) + if not file_path.exists() or not file_path.is_file(): + return [] + with file_path.open("r", encoding="utf-8", errors="replace") as handle: + return handle.read().splitlines()[-int(max_lines):] + + +def candidate_domain_roots(): + roots = [] + for candidate in ( + Path.cwd() / "domainCheck", + Path("/opt/domaincheck/domainCheck"), + Path("/www/wwwroot/getDomain/domainCheck"), + ): + if candidate not in roots: + roots.append(candidate) + return roots + + +def candidate_runtime_logs_dirs(): + dirs = [] + for candidate in ( + Path.cwd() / "domain-api" / "runtime" / "logs", + Path("/opt/domaincheck/domain-api/runtime/logs"), + Path("/www/wwwroot/getDomain/domain-api/runtime/logs"), + ): + if candidate not in dirs: + dirs.append(candidate) + return dirs + + +def service_log_file_candidates(service_name): + normalized_service_name = str(service_name or "").strip() + if not normalized_service_name: + return [] + candidates = [] + + def append_candidate(path): + if path not in candidates: + candidates.append(path) + + if normalized_service_name == (SERVICE_NAMES.get("worker") or "domaincheck-worker"): + for domain_root in candidate_domain_roots(): + append_candidate(domain_root / "detect_worker.log") + append_candidate(domain_root / "logs" / "detect_worker.log") + elif normalized_service_name == (SERVICE_NAMES.get("api") or "domaincheck-api"): + for runtime_logs_dir in candidate_runtime_logs_dirs(): + append_candidate(runtime_logs_dir / "domain-api.stderr.log") + append_candidate(runtime_logs_dir / "domain-api.stdout.log") + for domain_root in candidate_domain_roots(): + append_candidate(domain_root / "logs" / "app.log") + + return candidates + + +def collect_log_file_fallback(service_name, lines): + aggregated_lines = [] + used_paths = [] + for path in service_log_file_candidates(service_name): + current_lines = read_tail_lines(path, lines) + if not current_lines: + continue + aggregated_lines.extend(current_lines) + used_paths.append(str(path)) + if not aggregated_lines: + return "", [] + return "\\n".join(aggregated_lines[-int(lines):]), used_paths + + +def service_name_from_payload(payload): + value = str((payload or {{}}).get("service_name") or "").strip() + return value or SERVICE_NAMES.get("worker") or "domaincheck-worker" + + +def service_name_for_action(action, payload): + if action in ("service.status", "service.restart", "service.start", "service.stop"): + return service_name_from_payload(payload) + runtime_action_map = {{ + "runtime.start_worker": SERVICE_NAMES.get("worker") or "domaincheck-worker", + "runtime.stop_worker": SERVICE_NAMES.get("worker") or "domaincheck-worker", + "runtime.restart_api": SERVICE_NAMES.get("api") or "domaincheck-api", + "runtime.start_sync_agent": SERVICE_NAMES.get("sync_agent") or "domaincheck-sync-agent", + "runtime.stop_sync_agent": SERVICE_NAMES.get("sync_agent") or "domaincheck-sync-agent", + }} + return str(runtime_action_map.get(action) or "").strip() + + +def systemctl_action_name(action): + if action in ("service.restart", "runtime.restart_api"): + return "restart" + if action in ("service.start", "runtime.start_worker", "runtime.start_sync_agent"): + return "start" + if action in ("service.stop", "runtime.stop_worker", "runtime.stop_sync_agent"): + return "stop" + return "" + + +def execute(action, payload): + if action == "health.snapshot": + checks = {{}} + for key, service_name in SERVICE_NAMES.items(): + code, stdout, stderr = run(["systemctl", "is-active", service_name], timeout=15) + checks[key] = {{ + "service_name": service_name, + "returncode": int(code or 0), + "state": stdout or stderr, + "ok": int(code or 0) == 0 and (stdout or stderr) == "active", + }} + return True, "health snapshot collected", {{"checks": checks}} + + if action == "service.status": + service_name = service_name_from_payload(payload) + code, stdout, stderr = run(["systemctl", "status", service_name, "--no-pager", "-l"], timeout=45) + result = {{ + "service_name": service_name, + "stdout": trim_output(stdout, 12000), + "stderr": trim_output(stderr, 4000), + }} + return int(code or 0) == 0, stdout or stderr or f"{{service_name}} status collected", result + + systemctl_action = systemctl_action_name(action) + if systemctl_action: + service_name = service_name_for_action(action, payload) + if not service_name: + return False, f"missing service name for action: {{action}}", {{"action": action}} + code, stdout, stderr = run(["systemctl", systemctl_action, service_name], timeout=45) + result = {{ + "service_name": service_name, + "systemctl_action": systemctl_action, + "stdout": trim_output(stdout, 12000), + "stderr": trim_output(stderr, 4000), + }} + return int(code or 0) == 0, stdout or stderr or f"{{service_name}} {{systemctl_action}} completed", result + + if action == "logs.collect": + service_name = service_name_from_payload(payload) + lines = max(20, min(int((payload or {{}}).get("lines") or 120), 500)) + code, stdout, stderr = run(["journalctl", "-u", service_name, "-n", str(lines), "--no-pager"], timeout=45) + journal_output = stdout or stderr + fallback_output, fallback_paths = collect_log_file_fallback(service_name, lines) + collection_source = "journal" + effective_output = journal_output + ok = int(code or 0) == 0 + if (not ok or not stdout.strip()) and fallback_output: + collection_source = "file_fallback" + effective_output = fallback_output + ok = True + result = {{ + "service_name": service_name, + "lines": lines, + "collection_source": collection_source, + "fallback_used": bool(collection_source == "file_fallback"), + "fallback_reason": trim_output(journal_output, 4000) if collection_source == "file_fallback" else "", + "log_paths": fallback_paths, + "journal_returncode": int(code or 0), + "stdout": trim_output(effective_output, 20000), + "stderr": trim_output(stderr, 4000), + }} + return ok, effective_output or f"{{service_name}} logs collected", result + + if action == "diagnostics.collect": + lines = max(20, min(int((payload or {{}}).get("lines") or 200), 800)) + diagnostics = {{ + "services": {{}}, + "lines": lines, + }} + for key, service_name in SERVICE_NAMES.items(): + active_code, active_stdout, active_stderr = run(["systemctl", "is-active", service_name], timeout=15) + status_code, status_stdout, status_stderr = run(["systemctl", "status", service_name, "--no-pager", "-l"], timeout=45) + log_code, log_stdout, log_stderr = run(["journalctl", "-u", service_name, "-n", str(lines), "--no-pager"], timeout=45) + log_output = log_stdout or log_stderr + fallback_output, fallback_paths = collect_log_file_fallback(service_name, lines) + log_source = "journal" + if (int(log_code or 0) != 0 or not log_stdout.strip()) and fallback_output: + log_output = fallback_output + log_source = "file_fallback" + diagnostics["services"][key] = {{ + "service_name": service_name, + "active_returncode": int(active_code or 0), + "active_state": active_stdout or active_stderr, + "status_returncode": int(status_code or 0), + "status_output": trim_output(status_stdout or status_stderr, 12000), + "log_returncode": int(log_code or 0), + "log_source": log_source, + "log_paths": fallback_paths, + "log_output": trim_output(log_output, 20000), + }} + return True, "diagnostics collected", diagnostics + + return False, f"unsupported action: {{action}}", {{"action": action}} + + +def main(): + try: + ok, message, result = execute(ACTION, PAYLOAD) + except Exception as exc: + ok, message, result = False, str(exc), {{"exception": str(exc), "action": ACTION}} + print(json.dumps({{"ok": ok, "message": message, "result": result}}, ensure_ascii=False)) + raise SystemExit(0 if ok else 1) + + +if __name__ == "__main__": + main() +""" + + +def _extract_json_from_output(text: str) -> tuple[bool, dict]: + for raw_line in reversed(str(text or "").splitlines()): + line = str(raw_line or "").strip() + if not line: + continue + try: + data = json.loads(line) + except Exception: + continue + if isinstance(data, dict) and "ok" in data: + return True, data + return False, {} diff --git a/domain-api/app/services/ops_service.py b/domain-api/app/services/ops_service.py new file mode 100644 index 0000000..f7a97cc --- /dev/null +++ b/domain-api/app/services/ops_service.py @@ -0,0 +1,11611 @@ +from __future__ import annotations + +import json +from datetime import datetime, timedelta +from pathlib import Path +import re +import time + +from app.core.config import settings +from app.core.files import runtime_root +from app.services.ops_command_service import build_bash_command +from app.services.ops_agent_service import ( + execute_managed_node_onboarding_recovery, + get_managed_node_handover, + list_ops_job_events_for_jobs, + list_managed_nodes_with_agent_state, +) +from app.services.ops_execution_mode_service import execution_mode_label +from app.services.ops_job_service import create_ops_job_batch, get_ops_job_summary, list_ops_jobs, sync_managed_nodes_from_cluster +from app.services.ops_playbook_service import ( + execute_ops_playbook, + get_ops_playbook, + get_ops_playbook_run, + get_recent_ops_playbook_runs, + list_ops_playbook_run_events, + preview_ops_playbook, +) +from app.services.ops_release_service import ( + build_release_execution_mode_recommendation, + create_release_and_smart_rollout_from_latest_package, + get_latest_release_package_metadata, + create_smart_release_rollout, + get_latest_release, + get_release_launchpad, + get_release_summary, + list_release_rollouts, + prepare_latest_release_package, +) +from app.services.ops_template_service import get_ops_action_template +from app.services.build_info_service import get_runtime_build_info +from app.services.runtime_status_service import get_runtime_status +from app.services.runtime_settings_service import get_runtime_settings, update_runtime_settings +from app.services.sync_record_service import get_sync_summary + +_OPS_INSPECTION_FETCH_LIMIT = 80 +_OPS_ACTIVITY_FETCH_LIMIT = 18 +_OPS_INSPECTION_ACTION_KEYS = ("health.snapshot", "logs.collect", "diagnostics.collect") +_OPS_CONTRACT_REGISTRY_VERSION = "2026-04-18" +_OPS_CONTRACT_SCHEMA_VERSION = "v1" +_REMOTE_LOG_PREVIEW_LINE_RE = re.compile(r"^\[(?P[^\]]+)\]\s+\[(?P[^\]]+)\]\s+(?P.*)$") +_BACKEND_DRIVER_ACTION_CODES = { + "disable_log_sync", + "enable_log_sync_key", + "enable_log_sync_full", + "replay_delivery_queue", + "flush_delivery_queue", + "run_scene_logs_key", + "run_scene_logs_full", + "open_playbook_dialog", + "open_action_template_dialog", + "focus_playbook_run", + "focus_activity_item", + "focus_latest_job_events", + "open_playbook_run_latest_events", + "handover_first_gap", + "view_first_gap", + "open_rollout_dialog", + "open_release_dialog", + "open_release_deploy_control", + "open_release_deploy_worker", + "open_release_deploy_custom", + "create_smart_release_rollout_worker", + "create_smart_release_rollout_control", + "create_release_rollout_worker", + "create_release_rollout_control", + "focus_release_hub", + "open_worker_logs_participating", + "open_worker_logs_standby", + "open_worker_logs", + "run_inspection_participating", + "run_inspection_standby", + "run_standard_inspection", + "open_diagnostics_participating", + "open_diagnostics", + "publish_latest_worker", + "review_smart_rollout_preview", + "review_control_rollout", + "fix_rollout_blockers", + "fix_managed_nodes", + "bootstrap_run", + "run_acceptance", + "release_package", + "release_prepare", + "api-restart", +} +_SAFE_AUTO_DRIVER_ACTION_CODES = { + "disable_log_sync", + "enable_log_sync_key", + "enable_log_sync_full", + "run_scene_logs_key", + "run_scene_logs_full", + "open_playbook_dialog", + "open_action_template_dialog", + "focus_playbook_run", + "focus_activity_item", + "focus_latest_job_events", + "open_playbook_run_latest_events", + "handover_first_gap", + "view_first_gap", + "open_rollout_dialog", + "open_release_dialog", + "open_release_deploy_control", + "open_release_deploy_worker", + "open_release_deploy_custom", + "focus_release_hub", + "release_package", + "release_prepare", + "api-restart", + "review_smart_rollout_preview", + "review_control_rollout", + "fix_rollout_blockers", + "open_worker_logs_participating", + "open_worker_logs_standby", + "open_worker_logs", + "open_diagnostics_participating", + "open_diagnostics", + "run_inspection_participating", + "run_inspection_standby", + "run_standard_inspection", +} +_GUARDED_AUTO_DRIVER_ACTION_CODES = { + "replay_delivery_queue", + "flush_delivery_queue", + "publish_latest_worker", + "create_smart_release_rollout_worker", + "create_smart_release_rollout_control", + "create_release_rollout_worker", + "create_release_rollout_control", +} +_MIXED_DRIVER_ACTION_CODES = { + "fix_managed_nodes", + "bootstrap_run", + "run_acceptance", +} +_UI_ONLY_DRIVER_ACTION_CODES = set() + +_OPS_CONTRACT_DEFINITIONS = ( + { + "key": "ops_job_contract", + "title": "Ops Job Contract", + "version": _OPS_CONTRACT_SCHEMA_VERSION, + "status": "active", + "summary": "冻结 ops job / step / event / policy / approve / cancel / dispatch 的正式执行对象 contract。", + "schema_doc_path": "docs/schemas/ops_job_contract.md", + "primary_endpoint": "/api/v1/ops/jobs", + "discovery_endpoints": [ + "/api/v1/ops/jobs", + "/api/v1/ops/jobs/{job_id}", + "/api/v1/ops/jobs/{job_id}/events", + "/api/v1/ops/jobs", + "/api/v1/ops/jobs/batch", + "/api/v1/ops/policy/preview", + "/api/v1/ops/jobs/{job_id}/approve", + "/api/v1/ops/jobs/{job_id}/cancel", + "/api/v1/ops/jobs/{job_id}/dispatch", + ], + "service_keys": [ + "app.services.ops_job_service", + "app.services.ops_agent_service", + ], + "consumers": [ + "ops-center", + "codex-driver", + "cli", + "node-agent", + "release-hub", + "playbook-runner", + ], + "related_contract_keys": [ + "ops_agent_protocol", + "release_hub_contract", + "ops_playbook_contract", + "ops_observability_contract", + "ops_stack_diagnosis_contract", + ], + "notes": [ + "ops job 是正式执行颗粒度对象，playbook run 与 rollout 最终都应收敛到它。", + "approval / cancel / dispatch 不只是按钮动作，而是正式状态迁移。", + "页面列表默认消费 compact job，详情页再读取 full job + steps + events。", + ], + }, + { + "key": "ops_agent_protocol", + "title": "Node Agent Protocol", + "version": _OPS_CONTRACT_SCHEMA_VERSION, + "status": "active", + "summary": "冻结海外控制面与大陆 Node Agent 之间的认证、注册、心跳、拉任务、完成回执与事件回放 contract。", + "schema_doc_path": "docs/schemas/ops_agent_protocol.md", + "primary_endpoint": "/api/v1/ops/agent/bootstrap-plan", + "discovery_endpoints": [ + "/api/v1/ops/agent/tokens", + "/api/v1/ops/agent/bootstrap-plan", + "/api/v1/ops/agent/register", + "/api/v1/ops/agent/heartbeat", + "/api/v1/ops/agent/pull", + "/api/v1/ops/agent/jobs/{job_id}/start", + "/api/v1/ops/agent/jobs/{job_id}/complete", + "/api/v1/ops/agent/jobs/{job_id}/events", + "/api/v1/ops/nodes/{node_code}/delivery-queue", + "/api/v1/ops/nodes/{node_code}/delivery-queue/records", + "/api/v1/ops/nodes/{node_code}/delivery-queue/flush", + "/api/v1/ops/nodes/{node_code}/delivery-queue/replay", + "/api/v1/ops/nodes/{node_code}/delivery-queue/records/{record_id}/replay", + "/api/v1/ops/nodes/{node_code}/delivery-queue/records/{record_id}/discard", + ], + "service_keys": [ + "app.services.ops_agent_service", + "app.node_agent", + ], + "consumers": [ + "ops-center", + "codex-driver", + "cli", + "node-agent", + ], + "related_contract_keys": [ + "ops_job_contract", + "ops_observability_contract", + "ops_stack_diagnosis_contract", + ], + "notes": [ + "Agent 只执行结构化动作，不长期接受任意 shell。", + "complete / events 回执已经具备 client_request_id / client_event_id 幂等语义。", + "heartbeat 已纳入 delivery_queue 快照，可暴露 retrying / dead_letter 现场。", + ], + }, + { + "key": "release_hub_contract", + "title": "Release Hub Contract", + "version": _OPS_CONTRACT_SCHEMA_VERSION, + "status": "active", + "summary": "冻结 Release / Rollout / Launchpad / Default Gate 的对象模型与门禁口径。", + "schema_doc_path": "docs/schemas/release_hub_contract.md", + "primary_endpoint": "/api/v1/ops/releases/launchpad", + "discovery_endpoints": [ + "/api/v1/ops/overview", + "/api/v1/ops/releases", + "/api/v1/ops/releases/latest", + "/api/v1/ops/releases/launchpad", + "/api/v1/ops/releases/{release_id}", + "/api/v1/ops/releases/{release_id}/rollouts", + "/api/v1/ops/rollouts/{rollout_id}", + "/api/v1/ops/rollouts/{rollout_id}/jobs", + "/api/v1/ops/rollouts/{rollout_id}/advance", + ], + "service_keys": [ + "app.services.ops_release_service", + "app.services.ops_service", + ], + "consumers": [ + "ops-center", + "codex-driver", + "cli", + "release-hub", + ], + "related_contract_keys": [ + "ops_job_contract", + "ops_driver_contract", + "ops_playbook_contract", + "ops_stack_diagnosis_contract", + ], + "notes": [ + "发布必须先有 Release，再有 Rollout，不回退到节点 git pull。", + "default_rollout_gate 是首页、版本区和自动驾驶共用的统一门禁。", + "Rollout 状态机应继续固定为 planned/running/awaiting_approval/ready_for_next_batch/halted/completed/completed_with_issues。", + ], + }, + { + "key": "ops_driver_contract", + "title": "Ops Driver Contract", + "version": _OPS_CONTRACT_SCHEMA_VERSION, + "status": "active", + "summary": "冻结 overview / driver-feed / codex-brief / driver-actions / runbook sequence 的统一驾驶 contract。", + "schema_doc_path": "docs/schemas/ops_driver_contract.md", + "primary_endpoint": "/api/v1/ops/driver-feed", + "discovery_endpoints": [ + "/api/v1/ops/overview", + "/api/v1/ops/driver-feed", + "/api/v1/ops/codex-brief", + "/api/v1/ops/driver-actions/resolve", + "/api/v1/ops/driver-actions/execute-resolved", + "/api/v1/ops/codex-actions/resolve", + "/api/v1/ops/codex-actions/execute", + "/api/v1/ops/activity-stream", + "/api/v1/ops/runbook", + "/api/v1/ops/runbook/sequences/{sequence_key}/resolve", + "/api/v1/ops/runbook/sequences/{sequence_key}/execute", + "/api/v1/ops/driver-actions/preview", + "/api/v1/ops/driver-actions/execute", + ], + "service_keys": [ + "app.services.ops_service", + "app.services.ops_playbook_service", + ], + "consumers": [ + "ops-center", + "codex-driver", + "cli", + ], + "related_contract_keys": [ + "release_hub_contract", + "ops_playbook_contract", + "ops_observability_contract", + "ops_stack_diagnosis_contract", + ], + "notes": [ + "后端负责推荐优先级和 ui_intent，页面不再自算。", + "driver recommendation / runbook sequence / activity stream 是三条不同 contract。", + "Codex 通过 codex-brief 判断 safe_auto / guarded_auto / ui_only / blocked。", + "driver-actions/preview 负责统一生成执行链、请求体预览和主次动作 contract 视图。", + ], + }, + { + "key": "ops_playbook_contract", + "title": "Ops Playbook Contract", + "version": _OPS_CONTRACT_SCHEMA_VERSION, + "status": "active", + "summary": "冻结 playbook catalog / preview / run / events 的正式编排 contract。", + "schema_doc_path": "docs/schemas/ops_playbook_contract.md", + "primary_endpoint": "/api/v1/ops/playbooks", + "discovery_endpoints": [ + "/api/v1/ops/playbooks", + "/api/v1/ops/playbooks/preview", + "/api/v1/ops/playbooks/execute", + "/api/v1/ops/playbook-runs", + "/api/v1/ops/playbook-runs/{run_code}", + "/api/v1/ops/playbook-runs/{run_code}/events", + "/api/v1/ops/playbook-runs/{run_code}/rerun", + "/api/v1/ops/playbook-runs/{run_code}/cancel", + ], + "service_keys": [ + "app.services.ops_playbook_service", + "app.services.ops_service", + ], + "consumers": [ + "ops-center", + "codex-driver", + "cli", + ], + "related_contract_keys": [ + "ops_job_contract", + "release_hub_contract", + "ops_driver_contract", + "ops_observability_contract", + "ops_stack_diagnosis_contract", + ], + "notes": [ + "接管验收、标准巡检、现场观察应优先收口为 playbook run。", + "playbook run 是 ops job 的编排聚合对象，不是单条任务。", + "页面、CLI、Codex 应围绕同一份 preview / run / events contract 观察与执行。", + ], + }, + { + "key": "ops_observability_contract", + "title": "Ops Observability Contract", + "version": _OPS_CONTRACT_SCHEMA_VERSION, + "status": "active", + "summary": "冻结 execution scene / inspection overview / activity stream / delivery queue 的正式观察面 contract。", + "schema_doc_path": "docs/schemas/ops_observability_contract.md", + "primary_endpoint": "/api/v1/ops/overview", + "discovery_endpoints": [ + "/api/v1/ops/overview", + "/api/v1/ops/inspection-overview", + "/api/v1/ops/activity-stream", + "/api/v1/ops/nodes/{node_code}/scene-log", + "/api/v1/ops/nodes/{node_code}/delivery-queue", + "/api/v1/ops/nodes/{node_code}/delivery-queue/records", + "/api/v1/ops/nodes/{node_code}/delivery-queue/flush", + "/api/v1/ops/nodes/{node_code}/delivery-queue/replay", + "/api/v1/ops/nodes/{node_code}/delivery-queue/records/{record_id}/replay", + "/api/v1/ops/nodes/{node_code}/delivery-queue/records/{record_id}/discard", + ], + "service_keys": [ + "app.services.ops_service", + "app.services.ops_agent_service", + ], + "consumers": [ + "ops-center", + "codex-driver", + "cli", + ], + "related_contract_keys": [ + "ops_job_contract", + "ops_agent_protocol", + "ops_driver_contract", + "ops_playbook_contract", + "ops_stack_diagnosis_contract", + ], + "notes": [ + "execution scene 必须明确区分 dispatch_active / recent_only / standby / load_syncing。", + "inspection overview 必须按节点收口最近 health / worker logs / diagnostics。", + "delivery queue 当前允许 head_only，但不允许绕过 ops job 直接改远端记录。", + ], + }, + { + "key": "ops_stack_diagnosis_contract", + "title": "Ops Stack Diagnosis Contract", + "version": _OPS_CONTRACT_SCHEMA_VERSION, + "status": "active", + "summary": "冻结海外单脑总检入口的统一诊断 contract，供页面、CLI、Codex、按钮共享同一份第一现场判断。", + "schema_doc_path": "docs/schemas/ops_stack_diagnosis_contract.md", + "primary_endpoint": "/api/v1/ops/stack-diagnosis", + "discovery_endpoints": [ + "/api/v1/ops/go-live-summary", + "/api/v1/ops/stack-diagnosis", + "/api/v1/ops/contracts", + "/api/v1/ops/link-snapshot", + "/api/v1/ops/overview", + "/api/v1/ops/nodes", + "/api/v1/ops/releases/launchpad", + "/api/v1/ops/playbook-runs", + "/api/v1/ops/activity-stream", + ], + "service_keys": [ + "app.services.ops_service", + ], + "consumers": [ + "ops-center", + "codex-driver", + "cli", + ], + "related_contract_keys": [ + "ops_job_contract", + "ops_agent_protocol", + "release_hub_contract", + "ops_driver_contract", + "ops_playbook_contract", + "ops_observability_contract", + ], + "notes": [ + "这是海外单脑控制面的固定起手式，不允许页面、CLI、Codex 再各自拼一份第一现场诊断。", + "总检必须同时给出 stack_status、issues、next_step、quick_commands，不能只回一段说明文字。", + "go-live-summary 是给页面 / CLI / Codex 直接消费的收口摘要层，stack-diagnosis 则保留更细的可解释结构。", + "当总检 contract 可用后，check_ops_center_stack.sh 应优先直接消费它，而不是继续各算各的。", + ], + }, + { + "key": "ops_doctor_decision_contract", + "title": "Ops Doctor Decision Contract", + "version": _OPS_CONTRACT_SCHEMA_VERSION, + "status": "active", + "summary": "冻结 doctor-decision 主决策 contract，把总检主判断、建议处理面、下一步动作和证据摘要统一暴露给页面、CLI 和 Codex。", + "schema_doc_path": "docs/schemas/ops_doctor_decision_contract.md", + "primary_endpoint": "/api/v1/ops/doctor-decision", + "discovery_endpoints": [ + "/api/v1/ops/doctor-decision", + "/api/v1/ops/stack-diagnosis", + "/api/v1/ops/go-live-summary", + "/api/v1/ops/go-live-bundle", + "/api/v1/ops/contracts", + ], + "service_keys": [ + "app.services.ops_service", + ], + "consumers": [ + "ops-center", + "codex-driver", + "cli", + ], + "related_contract_keys": [ + "ops_stack_diagnosis_contract", + "ops_driver_contract", + "ops_go_live_bundle_contract", + "ops_go_live_signoff_contract", + "release_hub_contract", + ], + "notes": [ + "doctor-decision 是页面、CLI、Codex 的统一主决策入口，不应再让不同消费方各算各的“第一处理面”。", + "优先读取最新导出的 doctor manifest / go-live bundle 中的 doctor_decision，再在缺 bundle 时回落到 live 总检兜底。", + "返回值必须同时给出 status、preferred_surface、next_action_code 与推荐命令，不能只给一句提示文字。", + ], + }, + { + "key": "ops_go_live_signoff_contract", + "title": "Ops Go-Live Signoff Contract", + "version": _OPS_CONTRACT_SCHEMA_VERSION, + "status": "active", + "summary": "冻结发布前最终签收结论 contract，把收口、发布、自动化、launchpad 对齐状态压成同一份签字判断。", + "schema_doc_path": "docs/schemas/ops_go_live_signoff_contract.md", + "primary_endpoint": "/api/v1/ops/go-live-signoff", + "discovery_endpoints": [ + "/api/v1/ops/go-live-signoff", + "/api/v1/ops/go-live-summary", + "/api/v1/ops/stack-diagnosis", + "/api/v1/ops/driver-feed", + "/api/v1/ops/codex-brief", + "/api/v1/ops/releases/launchpad", + "/api/v1/ops/contracts", + ], + "service_keys": [ + "app.services.ops_service", + ], + "consumers": [ + "ops-center", + "codex-driver", + "cli", + ], + "related_contract_keys": [ + "ops_doctor_decision_contract", + "ops_stack_diagnosis_contract", + "ops_driver_contract", + "release_hub_contract", + "ops_observability_contract", + ], + "notes": [ + "signoff 不是替代 go-live-summary，而是在其基础上给出最终可否签字上线的统一结论。", + "当 signoff_status=blocked 时，不允许页面再显示‘基本可上线’这类模糊表述。", + "launchpad 摘要在 go-live-summary / driver-feed / codex-brief 之间若不一致，必须至少提升为 attention。", + ], + }, + { + "key": "ops_go_live_bundle_contract", + "title": "Ops Go-Live Bundle Contract", + "version": _OPS_CONTRACT_SCHEMA_VERSION, + "status": "active", + "summary": "冻结发布前证据包与 manifest 复核 contract，把 bundle 是否存在、manifest 状态、失败项和交付建议统一暴露给页面、CLI 和 Codex。", + "schema_doc_path": "docs/schemas/ops_go_live_bundle_contract.md", + "primary_endpoint": "/api/v1/ops/go-live-bundle", + "discovery_endpoints": [ + "/api/v1/ops/go-live-bundle", + "/api/v1/ops/go-live-signoff", + "/api/v1/ops/go-live-summary", + "/api/v1/ops/contracts", + ], + "service_keys": [ + "app.services.ops_service", + ], + "consumers": [ + "ops-center", + "codex-driver", + "cli", + ], + "related_contract_keys": [ + "ops_doctor_decision_contract", + "ops_go_live_signoff_contract", + "ops_stack_diagnosis_contract", + "ops_driver_contract", + "release_hub_contract", + ], + "notes": [ + "页面不应再假设证据包已经导出，而是必须明确显示当前有没有 bundle、manifest 是否可复核。", + "当 bundle 不存在时，应明确回落到 go-live-export / go-live-review / go-live-signoff 等固定命令，而不是提示人工自行找目录。", + "这层只负责读取最新 bundle / manifest 与复核摘要，不直接在后端发起 shell 导出。", + ], + }, + { + "key": "ops_go_live_review_contract", + "title": "Ops Go-Live Review Contract", + "version": _OPS_CONTRACT_SCHEMA_VERSION, + "status": "active", + "summary": "冻结 go-live-review 复核结论 contract，把 bundle manifest 的正式复核结果、launchpad 对齐与下一步建议统一暴露给页面、CLI 和 Codex。", + "schema_doc_path": "docs/schemas/ops_go_live_review_contract.md", + "primary_endpoint": "/api/v1/ops/go-live-review", + "discovery_endpoints": [ + "/api/v1/ops/go-live-review", + "/api/v1/ops/go-live-bundle", + "/api/v1/ops/doctor-decision", + "/api/v1/ops/go-live-signoff", + "/api/v1/ops/contracts", + ], + "service_keys": [ + "app.services.ops_service", + ], + "consumers": [ + "ops-center", + "codex-driver", + "cli", + ], + "related_contract_keys": [ + "ops_go_live_bundle_contract", + "ops_doctor_decision_contract", + "ops_go_live_signoff_contract", + "ops_stack_diagnosis_contract", + "ops_driver_contract", + "release_hub_contract", + ], + "notes": [ + "go-live-review 不是简单重复 bundle 状态，而是 bundle manifest 的正式复核结论层。", + "当 launchpad 摘要在 go_live_summary / stack_diagnosis / driver_feed / codex_brief 之间漂移时，即使关键文件都存在，也必须至少回到 attention。", + "页面和 Codex 应优先消费这层的 headline / recommended_next_steps，而不是自行拼 review 说明。", + ], + }, +) + + +def _bool_label(value: bool) -> str: + return "enabled" if value else "disabled" + + +def get_ops_contract_registry() -> dict: + contracts: list[dict] = [] + contracts_by_key: dict[str, dict] = {} + for raw_item in _OPS_CONTRACT_DEFINITIONS: + contract = { + "key": str(raw_item.get("key") or "").strip(), + "title": str(raw_item.get("title") or "").strip(), + "version": str(raw_item.get("version") or _OPS_CONTRACT_SCHEMA_VERSION).strip(), + "status": str(raw_item.get("status") or "active").strip() or "active", + "summary": str(raw_item.get("summary") or "").strip(), + "schema_doc_path": str(raw_item.get("schema_doc_path") or "").strip(), + "primary_endpoint": str(raw_item.get("primary_endpoint") or "").strip(), + "discovery_endpoints": [str(item or "").strip() for item in list(raw_item.get("discovery_endpoints") or []) if str(item or "").strip()], + "service_keys": [str(item or "").strip() for item in list(raw_item.get("service_keys") or []) if str(item or "").strip()], + "consumers": [str(item or "").strip() for item in list(raw_item.get("consumers") or []) if str(item or "").strip()], + "related_contract_keys": [ + str(item or "").strip() + for item in list(raw_item.get("related_contract_keys") or []) + if str(item or "").strip() + ], + "notes": [str(item or "").strip() for item in list(raw_item.get("notes") or []) if str(item or "").strip()], + } + if not contract["key"]: + continue + contracts.append(contract) + contracts_by_key[contract["key"]] = contract + + return { + "registry_version": _OPS_CONTRACT_REGISTRY_VERSION, + "schema_version": _OPS_CONTRACT_SCHEMA_VERSION, + "contracts_total": len(contracts), + "contracts": contracts, + "contracts_by_key": contracts_by_key, + "docs_root": "docs/schemas", + "discovery_entrypoints": [ + "/api/v1/ops/contracts", + "/api/v1/ops/contracts/{contract_key}", + "/api/v1/ops/stack-diagnosis", + "/api/v1/ops/capabilities", + "/api/v1/ops/overview", + "/api/v1/ops/driver-feed", + "/api/v1/ops/codex-brief", + "/api/v1/ops/releases/launchpad", + ], + } + + +def get_ops_contract_detail(contract_key: str) -> dict: + normalized_contract_key = str(contract_key or "").strip() + if not normalized_contract_key: + return {} + + registry = get_ops_contract_registry() + contracts_by_key = dict(registry.get("contracts_by_key") or {}) + selected_contract = dict(contracts_by_key.get(normalized_contract_key) or {}) + if not selected_contract: + return {} + + related_contracts = [ + dict(contracts_by_key.get(related_key) or {}) + for related_key in list(selected_contract.get("related_contract_keys") or []) + if str(related_key or "").strip() and dict(contracts_by_key.get(related_key) or {}) + ] + related_contracts = [ + { + "key": str(item.get("key") or "").strip(), + "title": str(item.get("title") or "").strip(), + "summary": str(item.get("summary") or "").strip(), + "schema_doc_path": str(item.get("schema_doc_path") or "").strip(), + "primary_endpoint": str(item.get("primary_endpoint") or "").strip(), + } + for item in related_contracts + ] + + return { + "registry_version": str(registry.get("registry_version") or _OPS_CONTRACT_REGISTRY_VERSION), + "schema_version": str(registry.get("schema_version") or _OPS_CONTRACT_SCHEMA_VERSION), + "docs_root": str(registry.get("docs_root") or "docs/schemas"), + "detail_endpoint": f"/api/v1/ops/contracts/{normalized_contract_key}", + "contract": selected_contract, + "related_contracts": related_contracts, + "related_contract_keys": [ + str(item.get("key") or "").strip() + for item in related_contracts + if str(item.get("key") or "").strip() + ], + } + + +def _normalize_ops_contract_keys(raw_values: list | tuple | set | None) -> list[str]: + normalized_values: list[str] = [] + seen_values: set[str] = set() + for raw_value in list(raw_values or []): + normalized_value = str(raw_value or "").strip() + if not normalized_value or normalized_value in seen_values: + continue + seen_values.add(normalized_value) + normalized_values.append(normalized_value) + return normalized_values + + +def _build_ops_contract_navigation( + contract_keys: list | tuple | set | None, + *, + primary_contract_key: str = "", + registry: dict | None = None, +) -> dict: + registry_payload = dict(registry or get_ops_contract_registry()) + contracts_by_key = dict(registry_payload.get("contracts_by_key") or {}) + normalized_primary_contract_key = str(primary_contract_key or "").strip() + ordered_contract_keys = _normalize_ops_contract_keys(contract_keys) + if normalized_primary_contract_key: + ordered_contract_keys = [normalized_primary_contract_key] + [ + item for item in ordered_contract_keys if item != normalized_primary_contract_key + ] + + contracts: list[dict] = [] + for contract_key in ordered_contract_keys: + raw_contract = dict(contracts_by_key.get(contract_key) or {}) + if not raw_contract: + continue + related_contract_keys = _normalize_ops_contract_keys(raw_contract.get("related_contract_keys") or []) + contracts.append( + { + "key": contract_key, + "title": str(raw_contract.get("title") or "").strip(), + "status": str(raw_contract.get("status") or "").strip(), + "version": str(raw_contract.get("version") or _OPS_CONTRACT_SCHEMA_VERSION).strip(), + "summary": str(raw_contract.get("summary") or "").strip(), + "primary_endpoint": str(raw_contract.get("primary_endpoint") or "").strip(), + "schema_doc_path": str(raw_contract.get("schema_doc_path") or "").strip(), + "detail_endpoint": f"/api/v1/ops/contracts/{contract_key}", + "discovery_endpoints": [ + str(item).strip() + for item in list(raw_contract.get("discovery_endpoints") or []) + if str(item).strip() + ], + "related_contract_keys": related_contract_keys, + } + ) + + return { + "detail_endpoint_pattern": "/api/v1/ops/contracts/{contract_key}", + "primary_contract_key": ( + normalized_primary_contract_key + if normalized_primary_contract_key + else str((contracts[0] or {}).get("key") or "").strip() + ), + "contract_keys": ( + [str(item.get("key") or "").strip() for item in contracts if str(item.get("key") or "").strip()] + if contracts + else ordered_contract_keys + ), + "contracts": contracts, + } + + +def _ops_contract_keys_from_focus_ref(focus_ref: dict | None) -> list[str]: + normalized_focus_ref = dict(focus_ref or {}) + kind = str(normalized_focus_ref.get("kind") or "").strip() + section = str(normalized_focus_ref.get("section") or "").strip() + + if kind in {"release_hub", "release", "rollout"} or section in { + "release_launchpad", + "release_detail", + "default_rollout_gate", + }: + return ["release_hub_contract"] + if kind in {"execution_scene", "ops_job", "ops_job_event", "activity_stream", "node_scene_log"}: + return ["ops_observability_contract", "ops_stack_diagnosis_contract"] + if kind in {"playbook_run", "playbook", "playbook_step"}: + return ["ops_playbook_contract"] + if kind in {"managed_node", "node_agent", "delivery_queue"}: + return ["ops_agent_protocol"] + return [] + + +def _ops_contract_keys_from_action_code(action_code: str) -> list[str]: + normalized_action_code = str(action_code or "").strip() + if not normalized_action_code: + return [] + if normalized_action_code.startswith("enable_log_sync") or normalized_action_code in { + "run_inspection_participating", + "run_inspection_all", + "focus_execution_scene", + "focus_activity_item", + }: + return ["ops_observability_contract", "ops_stack_diagnosis_contract"] + if normalized_action_code in { + "fix_managed_nodes", + "issue_agent_token", + "create_bootstrap_plan", + "open_agent_onboarding", + "bootstrap_run", + "run_acceptance", + }: + return ["ops_agent_protocol"] + if normalized_action_code in { + "open_release_dialog", + "focus_release_hub", + "create_release_rollout_worker", + "create_release_rollout_control", + "publish_latest_worker", + "publish_latest_control", + }: + return ["release_hub_contract"] + return [] + + +def _ops_contract_keys_from_job_action(job_action: str) -> list[str]: + normalized_job_action = str(job_action or "").strip() + if not normalized_job_action: + return [] + if normalized_job_action in {"health.snapshot", "logs.collect", "diagnostics.collect", "service.status"}: + return ["ops_observability_contract", "ops_stack_diagnosis_contract"] + if normalized_job_action == "node.bootstrap" or normalized_job_action.startswith(("runtime.", "service.", "delivery.queue.")): + return ["ops_agent_protocol"] + if normalized_job_action == "deploy.release": + return ["release_hub_contract"] + return [] + + +def _suggest_ops_contract_keys_for_codex_entry(entry: dict) -> list[str]: + normalized_entry = dict(entry or {}) + executor_kind = str(normalized_entry.get("executor_kind") or "driver_action").strip() or "driver_action" + contract_keys: list[str] = ["ops_driver_contract"] + if executor_kind == "runbook_sequence": + contract_keys.append("ops_playbook_contract") + + for focus_ref in ( + normalized_entry.get("focus_ref"), + normalized_entry.get("primary_focus_ref"), + normalized_entry.get("secondary_focus_ref"), + ): + contract_keys.extend(_ops_contract_keys_from_focus_ref(focus_ref)) + + for action_code in ( + normalized_entry.get("primary_action_code"), + normalized_entry.get("secondary_action_code"), + normalized_entry.get("focus_action_code"), + ): + contract_keys.extend(_ops_contract_keys_from_action_code(str(action_code or "").strip())) + + return _normalize_ops_contract_keys(contract_keys) + + +def _suggest_ops_contract_keys_for_activity_item(item: dict) -> list[str]: + normalized_item = dict(item or {}) + kind = str(normalized_item.get("kind") or "").strip() + ui_intent = dict(normalized_item.get("ui_intent") or {}) + contract_keys: list[str] = [] + + if kind in {"execution_scene", "log_sync"}: + contract_keys.extend(["ops_observability_contract", "ops_stack_diagnosis_contract"]) + elif kind in {"playbook_run", "runbook_sequence"}: + contract_keys.append("ops_playbook_contract") + if kind == "runbook_sequence": + contract_keys.append("ops_driver_contract") + elif kind == "rollout": + contract_keys.append("release_hub_contract") + elif kind == "ops_job": + contract_keys.extend( + _ops_contract_keys_from_job_action( + str(normalized_item.get("action") or normalized_item.get("title") or "").strip() + ) + ) + else: + contract_keys.append("ops_observability_contract") + + for focus_ref in ( + normalized_item.get("focus_ref"), + normalized_item.get("source_focus_ref"), + ): + contract_keys.extend(_ops_contract_keys_from_focus_ref(focus_ref)) + + for action_code in ( + ui_intent.get("driver_action_code"), + normalized_item.get("primary_action_code"), + normalized_item.get("secondary_action_code"), + normalized_item.get("focus_action_code"), + ): + contract_keys.extend(_ops_contract_keys_from_action_code(str(action_code or "").strip())) + + if str(ui_intent.get("kind") or "").strip() == "focus_execution_scene": + contract_keys.extend(["ops_observability_contract", "ops_stack_diagnosis_contract"]) + + return _normalize_ops_contract_keys(contract_keys) + + +def _primary_ops_contract_key_for_activity_item(item: dict, contract_keys: list[str]) -> str: + normalized_item = dict(item or {}) + normalized_contract_keys = _normalize_ops_contract_keys(contract_keys) + kind = str(normalized_item.get("kind") or "").strip() + job_action = str(normalized_item.get("action") or normalized_item.get("title") or "").strip() + + if kind in {"playbook_run", "runbook_sequence"} and "ops_playbook_contract" in normalized_contract_keys: + return "ops_playbook_contract" + if kind == "rollout" and "release_hub_contract" in normalized_contract_keys: + return "release_hub_contract" + if kind == "ops_job": + if job_action == "node.bootstrap" and "ops_agent_protocol" in normalized_contract_keys: + return "ops_agent_protocol" + if job_action == "deploy.release" and "release_hub_contract" in normalized_contract_keys: + return "release_hub_contract" + if "ops_observability_contract" in normalized_contract_keys: + return "ops_observability_contract" + if "ops_driver_contract" in normalized_contract_keys: + return "ops_driver_contract" + return normalized_contract_keys[0] if normalized_contract_keys else "" + + +def _attach_driver_feed_contract_navigation(entry: dict) -> dict: + normalized_entry = dict(entry or {}) + contract_keys = _normalize_ops_contract_keys( + list(normalized_entry.get("contract_keys") or []) + + _suggest_ops_contract_keys_for_codex_entry(normalized_entry) + ) + return { + **normalized_entry, + "contract_keys": contract_keys, + "contract_navigation": _build_ops_contract_navigation( + contract_keys, + primary_contract_key="ops_driver_contract", + registry=get_ops_contract_registry(), + ), + } + + +def _format_node_code_list(raw_values: object, *, limit: int = 5) -> str: + values: list[str] = [] + seen: set[str] = set() + for item in list(raw_values or []): + value = str(item or "").strip() + if not value or value in seen: + continue + seen.add(value) + values.append(value) + if not values: + return "-" + visible = values[: max(1, int(limit or 5))] + suffix = f" 等 {len(values)} 台" if len(values) > len(visible) else "" + return "、".join(visible) + suffix + + +def _normalize_driver_text_lines(raw_values: object, *, limit: int = 5) -> list[str]: + normalized: list[str] = [] + seen: set[str] = set() + for item in list(raw_values or []): + value = str(item or "").strip() + if not value or value in seen: + continue + seen.add(value) + normalized.append(value) + return normalized[: max(1, int(limit or 5))] + + +def _is_cluster_online_node(node: dict) -> bool: + return str(node.get("cluster_status") or "").strip() in {"online", "busy"} + + +def _is_inspection_visible_node(node: dict) -> bool: + return bool(node.get("cluster_is_effective_worker", False)) and _is_cluster_online_node(node) + + +def _is_inspection_eligible_node(node: dict) -> bool: + return ( + bool(node.get("is_managed", False)) + and bool(node.get("is_enabled", False)) + and bool(node.get("is_agent_online", False)) + and _is_inspection_visible_node(node) + ) + + +def _inspection_job_bucket(job: dict) -> str: + action = str(job.get("action") or "").strip() + if action in {"health.snapshot", "diagnostics.collect"}: + return action + if action != "logs.collect": + return "" + service_name = str((job.get("payload") or {}).get("service_name") or "").strip().lower() + if "worker" not in service_name: + return "" + return "logs.collect" + + +def _slim_inspection_job(job: dict) -> dict: + payload = dict(job.get("payload") or {}) + return { + "id": int(job.get("id") or 0), + "job_code": str(job.get("job_code") or ""), + "action": str(job.get("action") or ""), + "status": str(job.get("status") or ""), + "target_node_code": str(job.get("target_node_code") or ""), + "created_at": str(job.get("created_at") or ""), + "started_at": str(job.get("started_at") or ""), + "finished_at": str(job.get("finished_at") or ""), + "updated_at": str(job.get("updated_at") or ""), + "payload": { + "service_name": str(payload.get("service_name") or ""), + "lines": int(payload.get("lines") or 0) if payload.get("lines") not in (None, "") else 0, + }, + "error_message": str(job.get("error_message") or ""), + } + + +def _inspection_status_label(status: str) -> str: + normalized_status = str(status or "").strip() + if not normalized_status: + return "未收口" + mapping = { + "success": "成功", + "failed": "失败", + "blocked": "阻断", + "cancelled": "已取消", + "awaiting_approval": "待审批", + "dispatching": "待派发", + "running": "执行中", + "queued": "排队中", + } + return mapping.get(normalized_status, normalized_status) + + +def _build_inspection_contract_result(job: dict | None, *, action_label: str) -> dict: + normalized_job = dict(job or {}) + if not normalized_job: + return { + "status": "", + "status_label": "未收口", + "occurred_at": "", + "job_code": "", + "summary": f"最近没有 {action_label} 收口记录。", + "error_message": "", + } + + status = str(normalized_job.get("status") or "").strip() + occurred_at = ( + str(normalized_job.get("finished_at") or "").strip() + or str(normalized_job.get("updated_at") or "").strip() + or str(normalized_job.get("started_at") or "").strip() + or str(normalized_job.get("created_at") or "").strip() + ) + job_code = str(normalized_job.get("job_code") or "").strip() + error_message = str(normalized_job.get("error_message") or "").strip() + if status == "success": + summary = f"最近一次{action_label}收口成功。" + elif status: + summary = f"最近一次{action_label}状态为 {_inspection_status_label(status)}。" + else: + summary = f"最近没有 {action_label} 收口记录。" + if error_message: + summary = f"{summary} 错误：{error_message}" + return { + "status": status, + "status_label": _inspection_status_label(status) if status else "未收口", + "occurred_at": occurred_at, + "job_code": job_code, + "summary": summary, + "error_message": error_message, + } + + +def _build_inspection_issue_summary( + job_map: dict[str, dict], + *, + node_code: str = "", + latest_job: dict | None = None, + node: dict | None = None, +) -> dict: + action_labels = { + "health.snapshot": "健康快照", + "logs.collect": "Worker 日志", + "diagnostics.collect": "诊断包", + } + node_context = dict(node or {}) + node_is_visible = _is_inspection_visible_node(node_context) + node_is_eligible = bool(node_context.get("is_inspection_eligible", False)) or _is_inspection_eligible_node(node_context) + node_is_participating = bool(node_context.get("cluster_detect_participating", False)) + node_is_managed = bool(node_context.get("is_managed", False)) + node_is_enabled = bool(node_context.get("is_enabled", False)) + node_agent_online = bool(node_context.get("is_agent_online", False)) + node_has_ssh_access = bool(node_context.get("has_ssh_access", False)) + agent_state_label = str(node_context.get("agent_state_label") or "").strip() + agent_state_reason = str(node_context.get("agent_state_reason") or "").strip() + preferred_job = ( + job_map.get("diagnostics.collect") + or job_map.get("logs.collect") + or job_map.get("health.snapshot") + or dict(latest_job or {}) + ) + preferred_job_id = int(preferred_job.get("id") or 0) + preferred_job_code = str(preferred_job.get("job_code") or "").strip() + preferred_action = str(preferred_job.get("action") or "").strip() + preferred_target_node_code = str(preferred_job.get("target_node_code") or node_code or "").strip() + preferred_ui_intent = ( + _build_driver_ui_intent( + "job_events", + job_id=preferred_job_id, + job_code=preferred_job_code, + action=preferred_action, + target_node_code=preferred_target_node_code, + ) + if preferred_job_id > 0 + else {} + ) + present_statuses = [ + str((job_map.get(key) or {}).get("status") or "").strip() + for key in _OPS_INSPECTION_ACTION_KEYS + if str((job_map.get(key) or {}).get("status") or "").strip() + ] + + if node_is_visible and not node_is_eligible: + if not node_is_managed: + return { + "problem_kind": "handover_gap", + "problem_label": "未接管", + "problem_title": "执行节点未接管", + "problem_level": "danger", + "problem_keys": [], + "summary": "节点在线且属于有效执行面，但还没纳入 Node Agent 接管，当前不能稳定下发标准巡检、发布或诊断动作。", + "recommended_action": "纳管节点", + "recommended_action_code": "handover_first_gap", + "ui_intent": {}, + } + if not node_is_enabled: + return { + "problem_kind": "handover_gap", + "problem_label": "已停用", + "problem_title": "节点已停用", + "problem_level": "warning", + "problem_keys": [], + "summary": "节点虽然仍在有效执行面中，但当前被标记为停用，先确认是否需要重新启用后再下发巡检或发布动作。", + "recommended_action": "查看节点", + "recommended_action_code": "view_first_gap", + "ui_intent": {}, + } + if not node_agent_online: + return { + "problem_kind": "handover_gap", + "problem_label": "SSH 已备好" if node_has_ssh_access else (agent_state_label or "待接入"), + "problem_title": "Agent 未就绪", + "problem_level": "danger", + "problem_keys": [], + "summary": ( + f"节点已纳管，SSH 入口已备好，虽然仍未进入 Agent 在线状态，但已可通过 SSH 执行日志、诊断与部分服务控制。{agent_state_reason}" + if node_has_ssh_access and agent_state_reason + else ( + "节点已纳管，SSH 入口已备好，当前已可通过 SSH 执行日志、诊断与部分服务控制；标准巡检编排、正式发布与 Rollout 仍优先依赖 Node Agent 在线。" + if node_has_ssh_access + else ( + f"节点已纳管，但仍未进入 Agent 在线状态。{agent_state_reason}" + if agent_state_reason + else "节点已纳管，但仍未进入 Agent 在线状态，当前不能稳定执行标准巡检。" + ) + ) + ), + "recommended_action": "补接入", + "recommended_action_code": "handover_first_gap", + "ui_intent": {}, + } + + if node_is_visible and node_is_participating and not present_statuses: + return { + "problem_kind": "participating_no_inspection", + "problem_label": "现场未巡检", + "problem_title": "参与检测但未形成巡检", + "problem_level": "warning", + "problem_keys": [], + "summary": "节点当前正在真实参与检测，但还没有形成标准巡检记录，建议先看现场日志或补一轮标准巡检。", + "recommended_action": "看现场日志", + "recommended_action_code": "open_worker_logs_participating", + "ui_intent": {}, + } + + if node_is_visible and node_is_eligible and not node_is_participating and not present_statuses: + return { + "problem_kind": "standby_no_inspection", + "problem_label": "待命未巡检", + "problem_title": "在线待命但未巡检", + "problem_level": "info", + "problem_keys": [], + "summary": "节点当前在线可执行，但尚未参与检测，也还没有形成标准巡检记录，适合先做一轮巡检确认是否正常待命。", + "recommended_action": "补齐标准巡检", + "recommended_action_code": "run_standard_inspection", + "ui_intent": {}, + } + + terminal_problem_actions = [ + key + for key in _OPS_INSPECTION_ACTION_KEYS + if str((job_map.get(key) or {}).get("status") or "").strip() in {"failed", "blocked", "cancelled"} + ] + if terminal_problem_actions: + key = terminal_problem_actions[0] + job = job_map.get(key) or {} + return { + "problem_kind": "terminal_failure", + "problem_label": "失败/阻断", + "problem_title": f"{action_labels[key]}失败", + "problem_level": "error", + "problem_keys": terminal_problem_actions, + "summary": f"{action_labels[key]}当前状态为 {_inspection_status_label(str(job.get('status') or ''))}，优先查看事件并重跑标准巡检。", + "recommended_action": "先看事件", + "recommended_action_code": "focus_latest_job_events", + "ui_intent": preferred_ui_intent, + } + + running_actions = [ + key + for key in _OPS_INSPECTION_ACTION_KEYS + if str((job_map.get(key) or {}).get("status") or "").strip() in {"queued", "dispatching", "running", "awaiting_approval"} + ] + if running_actions: + key = running_actions[0] + job = job_map.get(key) or {} + return { + "problem_kind": "inflight_execution", + "problem_label": "执行中", + "problem_title": f"{action_labels[key]}执行中", + "problem_level": "warning", + "problem_keys": running_actions, + "summary": f"{action_labels[key]}仍在{_inspection_status_label(str(job.get('status') or ''))}，先等待回执或检查对应节点事件流。", + "recommended_action": "查看事件", + "recommended_action_code": "focus_latest_job_events", + "ui_intent": preferred_ui_intent, + } + + missing_actions = [key for key in _OPS_INSPECTION_ACTION_KEYS if not str((job_map.get(key) or {}).get("status") or "").strip()] + if missing_actions: + missing_problem_map = { + "health.snapshot": ("missing_health_snapshot", "缺健康快照", "健康快照缺口"), + "logs.collect": ("missing_worker_logs", "缺 Worker 日志", "Worker 日志缺口"), + "diagnostics.collect": ("missing_diagnostics", "缺诊断包", "诊断包缺口"), + } + if len(missing_actions) == 1: + problem_kind, problem_label, problem_title = missing_problem_map.get( + missing_actions[0], + ("missing_inspection_steps", "巡检缺口", "巡检缺口"), + ) + else: + problem_kind, problem_label, problem_title = ( + "missing_inspection_steps", + "巡检缺口", + "巡检缺口", + ) + return { + "problem_kind": problem_kind, + "problem_label": problem_label, + "problem_title": problem_title, + "problem_level": "info", + "problem_keys": missing_actions, + "summary": f"当前仍缺少 {'、'.join(action_labels[key] for key in missing_actions)} 的巡检记录。", + "recommended_action": "补齐标准巡检", + "recommended_action_code": "run_standard_inspection", + "ui_intent": {}, + } + + return { + "problem_kind": "healthy", + "problem_label": "已收口", + "problem_title": "巡检健康", + "problem_level": "success", + "problem_keys": [], + "summary": "当前三类巡检均已成功回执。", + "recommended_action": "当前无需处理", + "recommended_action_code": "no_action", + "ui_intent": {}, + } + + +def _slim_execution_scene_node(node: dict) -> dict: + raw_participation_state = str(node.get("participation_state") or "").strip() + is_dispatch_active = bool(node.get("is_dispatch_active", False)) + if is_dispatch_active or raw_participation_state in {"running", "claimed", "dispatch_active"}: + participation_bucket = "dispatch_active" + participation_bucket_label = "执行/已领" + elif raw_participation_state in {"recent_throughput", "recent_only"}: + participation_bucket = "recent_only" + participation_bucket_label = "近窗有吞吐" + elif raw_participation_state == "load_syncing": + participation_bucket = "load_syncing" + participation_bucket_label = "负载待确认" + elif raw_participation_state in {"standby", "idle"}: + participation_bucket = "standby" + participation_bucket_label = "在线待命" + else: + participation_bucket = raw_participation_state + participation_bucket_label = str(node.get("participation_label") or "").strip() + + return { + "node_code": str(node.get("node_code") or ""), + "region": str(node.get("region") or ""), + "role": str(node.get("role") or ""), + "status": str(node.get("status") or ""), + "current_load": int(node.get("current_load", 0) or 0), + "last_heartbeat_at": str(node.get("last_heartbeat_at") or ""), + "participation_state": raw_participation_state, + "participation_state_raw": raw_participation_state, + "participation_label": str(node.get("participation_label") or ""), + "participation_reason": str(node.get("participation_reason") or node.get("standby_reason") or ""), + "participation_bucket": participation_bucket, + "participation_bucket_label": participation_bucket_label, + "is_dispatch_active": is_dispatch_active, + "items_total": int(node.get("items_total", 0) or 0), + "items_claimed": int(node.get("items_claimed", 0) or 0), + "items_running": int(node.get("items_running", 0) or 0), + "items_completed": int(node.get("items_completed", 0) or 0), + "items_failed": int(node.get("items_failed", 0) or 0), + "processed_recent": int(node.get("processed_recent", 0) or 0), + "processed_per_minute": float(node.get("processed_per_minute", 0) or 0), + "detail": str(node.get("detail") or ""), + "phase": str(node.get("phase") or ""), + } + + +def _preferred_release_label(release_summary: dict) -> str: + active_by_channel = dict(release_summary.get("active_by_channel") or {}) + for channel in ("stable", "beta", "canary"): + version = str(active_by_channel.get(channel) or "").strip() + if version: + return f"{version} / {channel}" + return "" + + +def _preferred_release_for_ops() -> dict: + for channel in ("stable", "beta", "canary"): + release = get_latest_release(channel=channel) + if int(release.get("id") or 0) > 0: + return release + return {} + + +def _normalize_driver_node_codes(raw_node_codes: object) -> list[str]: + normalized: list[str] = [] + seen: set[str] = set() + for item in list(raw_node_codes or []): + node_code = str(item or "").strip() + if not node_code or node_code in seen: + continue + seen.add(node_code) + normalized.append(node_code) + return normalized + + +def _recommend_driver_execution_mode(node_codes: list[str]) -> dict: + return _recommend_driver_execution_mode_for_supported_modes( + node_codes, + supported_modes=["remote-agent", "ssh"], + default_mode="remote-agent", + ) + + +def _normalize_driver_supported_execution_modes( + supported_modes: list[str] | tuple[str, ...] | None = None, + *, + default_mode: str = "remote-agent", +) -> list[str]: + normalized_default_mode = str(default_mode or "").strip() or "remote-agent" + normalized_modes: list[str] = [] + seen: set[str] = set() + for item in list(supported_modes or []): + mode = str(item or "").strip() + if not mode or mode in seen: + continue + seen.add(mode) + normalized_modes.append(mode) + if normalized_modes: + return normalized_modes + return [normalized_default_mode] + + +def _recommend_driver_execution_mode_for_supported_modes( + node_codes: list[str], + *, + supported_modes: list[str] | tuple[str, ...] | None = None, + default_mode: str = "remote-agent", +) -> dict: + normalized_node_codes = _normalize_driver_node_codes(node_codes) + normalized_supported_modes = _normalize_driver_supported_execution_modes( + supported_modes, + default_mode=default_mode, + ) + normalized_default_mode = ( + str(default_mode or "").strip() + if str(default_mode or "").strip() in normalized_supported_modes + else normalized_supported_modes[0] + ) + if len(normalized_supported_modes) == 1: + only_mode = normalized_supported_modes[0] + return { + "execution_mode": only_mode, + "reason": f"当前动作仅支持 {only_mode}，保持该执行路径。", + "resolved_nodes": [], + "missing_node_codes": [], + "supported_modes": normalized_supported_modes, + } + if not normalized_node_codes: + return { + "execution_mode": normalized_default_mode, + "reason": f"当前未指定目标节点，默认继续使用 {normalized_default_mode}。", + "resolved_nodes": [], + "missing_node_codes": [], + "supported_modes": normalized_supported_modes, + } + if ( + "local-runtime" in normalized_supported_modes + and normalized_node_codes + and all(node_code == settings.node_code for node_code in normalized_node_codes) + ): + return { + "execution_mode": "local-runtime", + "reason": "当前目标节点就是本机控制面节点，推荐直接走 local-runtime。", + "resolved_nodes": [], + "missing_node_codes": [], + "supported_modes": normalized_supported_modes, + } + + managed_nodes_payload = list_managed_nodes_with_agent_state() + managed_nodes = list(managed_nodes_payload.get("nodes") or []) + managed_map = { + str(item.get("node_code") or "").strip(): dict(item or {}) + for item in managed_nodes + if str(item.get("node_code") or "").strip() + } + resolved_nodes = [managed_map[node_code] for node_code in normalized_node_codes if node_code in managed_map] + missing_node_codes = [node_code for node_code in normalized_node_codes if node_code not in managed_map] + + def _agent_ready(node: dict) -> bool: + return ( + bool(node.get("is_managed", False)) + and bool(node.get("is_enabled", False)) + and bool(node.get("is_agent_online", False)) + ) + + def _ssh_ready(node: dict) -> bool: + return ( + bool(node.get("is_managed", False)) + and bool(node.get("is_enabled", False)) + and bool(node.get("has_ssh_access", False)) + ) + + if ( + "remote-agent" in normalized_supported_modes + and resolved_nodes + and len(resolved_nodes) == len(normalized_node_codes) + and all(_agent_ready(node) for node in resolved_nodes) + ): + return { + "execution_mode": "remote-agent", + "reason": "当前目标节点都已进入 Agent 在线状态，默认沿用 remote-agent。", + "resolved_nodes": resolved_nodes, + "missing_node_codes": missing_node_codes, + "supported_modes": normalized_supported_modes, + } + if ( + "ssh" in normalized_supported_modes + and resolved_nodes + and len(resolved_nodes) == len(normalized_node_codes) + and all(_ssh_ready(node) for node in resolved_nodes) + ): + return { + "execution_mode": "ssh", + "reason": "当前目标节点 SSH 已全部备好，自动切到 SSH 执行路径。", + "resolved_nodes": resolved_nodes, + "missing_node_codes": missing_node_codes, + "supported_modes": normalized_supported_modes, + } + return { + "execution_mode": normalized_default_mode, + "reason": f"当前目标节点执行能力不一致，默认保守回退到 {normalized_default_mode}。", + "resolved_nodes": resolved_nodes, + "missing_node_codes": missing_node_codes, + "supported_modes": normalized_supported_modes, + } + + +def _recommend_driver_execution_mode_for_playbook(playbook_key: str, node_codes: list[str]) -> dict: + playbook = dict(get_ops_playbook(playbook_key) or {}) + supported_modes = list(playbook.get("execution_modes") or []) + default_mode = str(playbook.get("default_execution_mode") or "remote-agent").strip() or "remote-agent" + return _recommend_driver_execution_mode_for_supported_modes( + node_codes, + supported_modes=supported_modes, + default_mode=default_mode, + ) + + +def _recommend_driver_execution_mode_for_action_template(template_key: str, node_codes: list[str]) -> dict: + template = dict(get_ops_action_template(template_key) or {}) + supported_modes = list(template.get("execution_modes") or []) + default_mode = str(template.get("default_execution_mode") or "remote-agent").strip() or "remote-agent" + return _recommend_driver_execution_mode_for_supported_modes( + node_codes, + supported_modes=supported_modes, + default_mode=default_mode, + ) + + +def _build_driver_ui_intent(kind: str, **payload: object) -> dict: + return { + "kind": str(kind or "").strip(), + **payload, + } + + +def _normalize_focus_ref(focus_ref: object) -> dict: + if not isinstance(focus_ref, dict): + return {} + normalized: dict[str, object] = {} + for raw_key, raw_value in focus_ref.items(): + key = str(raw_key or "").strip() + if not key: + continue + if isinstance(raw_value, bool): + normalized[key] = raw_value + continue + if isinstance(raw_value, int): + if raw_value: + normalized[key] = raw_value + continue + if isinstance(raw_value, float): + if raw_value: + normalized[key] = raw_value + continue + if isinstance(raw_value, str): + value = raw_value.strip() + if value: + normalized[key] = value + continue + if raw_value is not None: + normalized[key] = raw_value + return normalized + + +def _find_activity_stream_item(activity_key: str, *, kind: str = "", scan_limit: int = 200) -> dict: + normalized_activity_key = str(activity_key or "").strip() + normalized_kind = str(kind or "").strip() + if not normalized_activity_key: + return {} + activity_stream = get_ops_activity_stream(limit=min(max(scan_limit, 20), 100), scan_limit=max(scan_limit, 80)) + for item in list(activity_stream.get("items") or []): + if str(item.get("activity_key") or "").strip() != normalized_activity_key: + continue + if normalized_kind and str(item.get("kind") or "").strip() != normalized_kind: + continue + return dict(item or {}) + return {} + + +def _merge_focus_ref(focus_ref: object, **extra: object) -> dict: + merged = _normalize_focus_ref(focus_ref) + for raw_key, raw_value in extra.items(): + key = str(raw_key or "").strip() + if not key: + continue + if isinstance(raw_value, bool): + merged[key] = raw_value + continue + if isinstance(raw_value, int): + if raw_value: + merged[key] = raw_value + continue + if isinstance(raw_value, float): + if raw_value: + merged[key] = raw_value + continue + if isinstance(raw_value, str): + value = raw_value.strip() + if value: + merged[key] = value + continue + if raw_value is not None: + merged[key] = raw_value + return merged + + +def _action_payload_with_focus_ref(action_payload: dict | None = None, focus_ref: object = None) -> dict: + payload = dict(action_payload or {}) + merged_focus_ref = _merge_focus_ref(focus_ref, **_normalize_focus_ref(payload.get("focus_ref"))) + if merged_focus_ref: + payload["focus_ref"] = merged_focus_ref + return payload + + +def _driver_launchpad_context(action_payload: dict) -> dict: + payload = dict(action_payload or {}) + return { + "channel": str(payload.get("channel") or "stable").strip() or "stable", + "control_plane_base_url": str(payload.get("control_plane_base_url") or "").strip(), + } + + +def _release_launchpad_preview(release_launchpad: dict, mode: str) -> dict: + if str(mode or "").strip() == "control": + return dict(release_launchpad.get("control_rollout_preview") or {}) + return dict(release_launchpad.get("worker_rollout_preview") or {}) + + +def _release_launchpad_operational_rows(release_launchpad: dict, *, modes: tuple[str, ...]) -> list[dict]: + rows: list[dict] = [] + seen: set[tuple[str, str]] = set() + for mode in modes: + preview = _release_launchpad_preview(release_launchpad, mode) + release_gate = dict((dict(preview.get("policy_preview") or {}).get("release_gate") or {})) + operational_readiness = dict(release_gate.get("operational_readiness") or {}) + for raw_row in list(operational_readiness.get("rows") or []): + row = dict(raw_row or {}) + node_code = str(row.get("node_code") or "").strip() + if not node_code: + continue + key = (str(mode or "").strip() or "worker", node_code) + if key in seen: + continue + seen.add(key) + rows.append( + { + **row, + "mode": str(mode or "").strip() or "worker", + } + ) + return rows + + +def _release_launchpad_gap_rows(release_launchpad: dict) -> list[dict]: + rows = _release_launchpad_operational_rows(release_launchpad, modes=("worker", "control")) + + def _row_execution_ready(row: dict) -> bool: + if "execution_ready" in row: + return bool(row.get("execution_ready", False)) + return bool(row.get("remote_agent_ready", False)) + + gap_rows = [ + row + for row in rows + if not _row_execution_ready(row) + or str(row.get("inspection_status") or "").strip() in {"running", "attention", "missing"} + ] + gap_rows.sort( + key=lambda row: ( + 0 if not _row_execution_ready(row) else 1, + 0 if str(row.get("inspection_status") or "").strip() in {"attention", "missing"} else 1, + 0 if str(row.get("inspection_status") or "").strip() == "running" else 1, + str(row.get("node_code") or ""), + ) + ) + return gap_rows + + +def _first_release_launchpad_gap_row(release_launchpad: dict) -> dict: + gap_rows = _release_launchpad_gap_rows(release_launchpad) + return dict(gap_rows[0] or {}) if gap_rows else {} + + +def _release_launchpad_gap_action_context(release_launchpad: dict) -> dict: + first_gap_row = _first_release_launchpad_gap_row(release_launchpad) + action_code = str(first_gap_row.get("recovery_action") or "").strip() + node_code = str(first_gap_row.get("node_code") or "").strip() + summary = str( + first_gap_row.get("recovery_summary") + or first_gap_row.get("onboarding_summary") + or first_gap_row.get("agent_reason") + or first_gap_row.get("inspection_reason") + or "" + ).strip() + if action_code not in {"bootstrap_run", "run_acceptance"}: + action_code = "" + return { + "action_code": action_code, + "node_code": node_code, + "summary": summary, + "row": first_gap_row, + } + + +def _release_launchpad_target_node_codes(release_launchpad: dict, *, mode: str) -> list[str]: + preview = _release_launchpad_preview(release_launchpad, mode) + smart_rollout = dict(preview.get("smart_rollout") or {}) + target_node_codes = _normalize_driver_node_codes(smart_rollout.get("target_node_codes") or []) + if target_node_codes: + return target_node_codes + return _normalize_driver_node_codes( + [str(row.get("node_code") or "").strip() for row in _release_launchpad_operational_rows(release_launchpad, modes=(mode,))] + ) + + +def _build_release_launchpad_review_payload(*, release_launchpad: dict, mode: str) -> dict: + normalized_mode = "control" if str(mode or "").strip() == "control" else "worker" + launchpad = dict(release_launchpad or {}) + latest_release = dict(launchpad.get("latest_release") or {}) + preview = _release_launchpad_preview(launchpad, normalized_mode) + smart_rollout = dict(preview.get("smart_rollout") or {}) + launchpad_status = dict(launchpad.get("launchpad_status") or {}) + focus_ref = _merge_focus_ref( + launchpad_status.get("focus_ref"), + release_id=int(latest_release.get("id") or 0), + release_version=str(latest_release.get("release_version") or "").strip(), + channel=str(latest_release.get("channel") or launchpad.get("channel") or "").strip(), + section="release_launchpad", + mode=normalized_mode, + ) + return { + "mode": normalized_mode, + "focus_ref": focus_ref, + "latest_release": latest_release, + "launchpad_status": launchpad_status, + "preview": preview, + "smart_rollout": smart_rollout, + "target_node_codes": _release_launchpad_target_node_codes(launchpad, mode=normalized_mode), + "gap_rows": _release_launchpad_gap_rows(launchpad), + } + + +def _build_release_launchpad_driver_card(*, release_launchpad: dict) -> dict: + launchpad = dict(release_launchpad or {}) + launchpad_status = dict(launchpad.get("launchpad_status") or {}) + action_code = str(launchpad_status.get("recommended_action_code") or "").strip() + if not action_code: + return {} + + latest_release = dict(launchpad.get("latest_release") or {}) + latest_package = dict(launchpad.get("latest_package") or {}) + worker_preview = dict(launchpad.get("worker_rollout_preview") or {}) + control_preview = dict(launchpad.get("control_rollout_preview") or {}) + worker_plan = dict(worker_preview.get("smart_rollout") or {}) + control_plan = dict(control_preview.get("smart_rollout") or {}) + first_gap_row = _first_release_launchpad_gap_row(launchpad) + gap_node_codes = _normalize_driver_node_codes( + [str(row.get("node_code") or "").strip() for row in _release_launchpad_gap_rows(launchpad)] + ) + worker_node_codes = _release_launchpad_target_node_codes(launchpad, mode="worker") + control_node_codes = _release_launchpad_target_node_codes(launchpad, mode="control") + latest_release_id = int(latest_release.get("id") or 0) + launchpad_focus_ref = _merge_focus_ref( + launchpad_status.get("focus_ref"), + release_id=latest_release_id, + release_version=str(latest_release.get("release_version") or "").strip(), + channel=str(latest_release.get("channel") or launchpad.get("channel") or "").strip(), + ) + release_detail_focus_ref = _merge_focus_ref( + latest_release.get("focus_ref"), + release_id=latest_release_id, + release_version=str(latest_release.get("release_version") or "").strip(), + channel=str(latest_release.get("channel") or launchpad.get("channel") or "").strip(), + section="release_detail", + ) + worker_launchpad_focus_ref = _merge_focus_ref(launchpad_focus_ref, section="release_launchpad", mode="worker") + control_launchpad_focus_ref = _merge_focus_ref(launchpad_focus_ref, section="release_launchpad", mode="control") + rollout_gate_focus_ref = _merge_focus_ref(launchpad_focus_ref, section="default_rollout_gate") + release_label = ( + str(latest_release.get("release_version") or "").strip() + or str(latest_package.get("package_name") or "").strip() + or str(launchpad.get("channel") or "stable").strip() + or "latest" + ) + shared_action_payload = { + "channel": str(launchpad.get("channel") or "stable").strip() or "stable", + "control_plane_base_url": str(launchpad.get("control_plane_base_url") or "").strip(), + } + if latest_release_id > 0: + shared_action_payload["release_id"] = latest_release_id + + base_card = { + "level_label": "推荐", + "tag_type": "info", + "summary": str(launchpad_status.get("summary") or "").strip(), + "reason": "", + "node_codes": [], + "meta_text": release_label, + "focus_ref": launchpad_focus_ref, + "primary_focus_ref": {}, + "secondary_focus_ref": {}, + "primary_type": "primary", + "disabled": False, + "primary_action_code": action_code, + "primary_action_payload": dict(shared_action_payload), + "secondary_label": "查看版本区" if latest_release_id > 0 else "", + "secondary_action_code": "focus_release_hub" if latest_release_id > 0 else "", + "secondary_action_payload": ( + { + "release_id": latest_release_id, + } + if latest_release_id > 0 + else {} + ), + } + + if action_code == "publish_latest_worker": + return { + **base_card, + "key": "release-launchpad-ready", + "title": "当前可以发 Worker 灰度", + "level_label": "准备就绪", + "tag_type": "success", + "reason": str(worker_plan.get("summary") or worker_preview.get("message") or "最新发布包与 Worker 智能 Rollout 预检均已收口。").strip(), + "node_codes": worker_node_codes, + "primary_label": "发 Worker 灰度", + "primary_focus_ref": worker_launchpad_focus_ref, + "primary_action_payload": _action_payload_with_focus_ref(shared_action_payload, worker_launchpad_focus_ref), + "secondary_label": "看 Control 预案" if bool(control_plan.get("available", False)) else base_card["secondary_label"], + "secondary_action_code": "review_control_rollout" if bool(control_plan.get("available", False)) else base_card["secondary_action_code"], + "secondary_focus_ref": ( + control_launchpad_focus_ref if bool(control_plan.get("available", False)) else release_detail_focus_ref + ), + "secondary_action_payload": ( + _action_payload_with_focus_ref(shared_action_payload, control_launchpad_focus_ref) + if bool(control_plan.get("available", False)) + else _action_payload_with_focus_ref(base_card["secondary_action_payload"], release_detail_focus_ref) + ), + } + + if action_code == "review_smart_rollout_preview": + return { + **base_card, + "key": "release-launchpad-review-worker", + "title": "先确认 Worker 灰度预案", + "level_label": "待确认", + "tag_type": "warning", + "reason": str(worker_preview.get("message") or worker_plan.get("summary") or "Worker 智能 Rollout 仍需人工确认。").strip(), + "node_codes": worker_node_codes, + "primary_label": "查看 Worker 预案", + "primary_focus_ref": worker_launchpad_focus_ref, + "primary_action_payload": _action_payload_with_focus_ref(shared_action_payload, worker_launchpad_focus_ref), + "secondary_focus_ref": release_detail_focus_ref, + "secondary_action_payload": _action_payload_with_focus_ref(base_card["secondary_action_payload"], release_detail_focus_ref), + } + + if action_code == "review_control_rollout": + return { + **base_card, + "key": "release-launchpad-review-control", + "title": "先确认 Control 发布预案", + "level_label": "推荐", + "tag_type": "warning", + "reason": str(control_preview.get("message") or control_plan.get("summary") or "当前只有 Control 侧满足预发条件。").strip(), + "node_codes": control_node_codes, + "primary_label": "查看 Control 预案", + "primary_focus_ref": control_launchpad_focus_ref, + "primary_action_payload": _action_payload_with_focus_ref(shared_action_payload, control_launchpad_focus_ref), + "secondary_focus_ref": release_detail_focus_ref, + "secondary_action_payload": _action_payload_with_focus_ref(base_card["secondary_action_payload"], release_detail_focus_ref), + } + + if action_code == "fix_rollout_blockers": + primary_label = "查看 Worker 预案" + if str(first_gap_row.get("node_code") or "").strip(): + primary_label = "纳管首台节点" if not bool(first_gap_row.get("is_managed", False)) else "补接入首台" + if str(first_gap_row.get("recovery_action") or "").strip() == "bootstrap_run": + primary_label = "跑接入收口" + elif str(first_gap_row.get("recovery_action") or "").strip() == "run_acceptance": + primary_label = "跑接管验收" + return { + **base_card, + "key": "release-launchpad-fix-blockers", + "title": "先收口发布门禁", + "level_label": "最高优先", + "tag_type": "danger", + "reason": str( + first_gap_row.get("agent_reason") + or first_gap_row.get("inspection_reason") + or worker_preview.get("message") + or control_preview.get("message") + or "Rollout 预检仍存在阻断条件，需先补接管、在线状态或巡检缺口。" + ).strip(), + "node_codes": gap_node_codes or worker_node_codes or control_node_codes, + "primary_label": primary_label, + "primary_focus_ref": rollout_gate_focus_ref, + "primary_action_payload": _action_payload_with_focus_ref(shared_action_payload, rollout_gate_focus_ref), + "secondary_focus_ref": release_detail_focus_ref, + "secondary_action_payload": _action_payload_with_focus_ref(base_card["secondary_action_payload"], release_detail_focus_ref), + } + + if action_code in {"bootstrap_run", "run_acceptance"}: + primary_label = "跑接入收口" if action_code == "bootstrap_run" else "跑接管验收" + return { + **base_card, + "key": f"release-launchpad-{action_code}", + "title": "先收口执行面接管", + "level_label": "最高优先" if action_code == "bootstrap_run" else "推荐", + "tag_type": "warning" if action_code == "bootstrap_run" else "success", + "reason": str( + first_gap_row.get("recovery_summary") + or first_gap_row.get("onboarding_summary") + or first_gap_row.get("agent_reason") + or first_gap_row.get("inspection_reason") + or launchpad_status.get("summary") + or "当前需要先收口节点接管，再继续发布。" + ).strip(), + "node_codes": gap_node_codes or ([str(first_gap_row.get("node_code") or "").strip()] if str(first_gap_row.get("node_code") or "").strip() else []), + "primary_label": primary_label, + "primary_focus_ref": rollout_gate_focus_ref, + "primary_action_payload": _action_payload_with_focus_ref( + { + **shared_action_payload, + "node_code": str(first_gap_row.get("node_code") or "").strip(), + }, + rollout_gate_focus_ref, + ), + "secondary_focus_ref": release_detail_focus_ref, + "secondary_action_payload": _action_payload_with_focus_ref(base_card["secondary_action_payload"], release_detail_focus_ref), + } + + if action_code == "fix_managed_nodes": + primary_label = "同步节点" + if str(first_gap_row.get("node_code") or "").strip(): + primary_label = "纳管首台节点" if not bool(first_gap_row.get("is_managed", False)) else "补接入首台" + return { + **base_card, + "key": "release-launchpad-fix-managed", + "title": "先补执行面接管", + "level_label": "推荐", + "tag_type": "warning", + "reason": str( + first_gap_row.get("agent_reason") + or first_gap_row.get("inspection_reason") + or "当前没有可用于智能 Rollout 的在线启用执行节点，需要先补齐托管或接入状态。" + ).strip(), + "node_codes": gap_node_codes, + "primary_label": primary_label, + "primary_focus_ref": rollout_gate_focus_ref, + "primary_action_payload": _action_payload_with_focus_ref(shared_action_payload, rollout_gate_focus_ref), + "secondary_focus_ref": release_detail_focus_ref, + "secondary_action_payload": _action_payload_with_focus_ref(base_card["secondary_action_payload"], release_detail_focus_ref), + } + + if action_code == "api-restart": + return { + **base_card, + "key": "release-launchpad-runtime-refresh", + "title": "先刷新控制面运行时", + "level_label": "最高优先", + "tag_type": "warning", + "reason": str( + launchpad_status.get("summary") + or "运行中的控制面 API 还没有刷新到当前仓库能力，建议先执行 runtime-refresh-recover。" + ).strip(), + "node_codes": [], + "primary_label": "查看刷新收口", + "primary_focus_ref": worker_launchpad_focus_ref or launchpad_focus_ref, + "primary_action_payload": _action_payload_with_focus_ref( + shared_action_payload, + worker_launchpad_focus_ref or launchpad_focus_ref, + ), + "secondary_label": "查看版本区" if latest_release_id > 0 else "", + "secondary_action_code": "focus_release_hub" if latest_release_id > 0 else "", + "secondary_action_payload": {"release_id": latest_release_id} if latest_release_id > 0 else {}, + "secondary_focus_ref": release_detail_focus_ref if latest_release_id > 0 else {}, + } + + if action_code == "release_package": + return { + **base_card, + "key": "release-launchpad-package", + "title": "先准备发布包", + "level_label": "待补齐", + "tag_type": "warning", + "reason": str(latest_package.get("reason") or "当前还没有可用的最新发布包，请先在控制面本机完成打包。").strip(), + "node_codes": [], + "primary_label": "打开打包面板", + "secondary_label": "", + "secondary_action_code": "", + "secondary_action_payload": {}, + "primary_focus_ref": worker_launchpad_focus_ref or launchpad_focus_ref, + "primary_action_payload": _action_payload_with_focus_ref( + shared_action_payload, + worker_launchpad_focus_ref or launchpad_focus_ref, + ), + "secondary_focus_ref": {}, + } + + if action_code == "release_prepare": + return { + **base_card, + "key": "release-launchpad-final-signoff", + "title": "先完成最终签收", + "level_label": "最高优先", + "tag_type": "danger", + "reason": str( + latest_package.get("final_release_report_stale_reason") + or launchpad_status.get("summary") + or "当前最新发布包还没有完成最终签收，不能直接进入正式发布动作。" + ).strip(), + "node_codes": [], + "primary_label": "执行最终签收", + "secondary_label": "查看版本区" if latest_release_id > 0 else "", + "secondary_action_code": "focus_release_hub" if latest_release_id > 0 else "", + "secondary_action_payload": {"release_id": latest_release_id} if latest_release_id > 0 else {}, + "primary_focus_ref": worker_launchpad_focus_ref or launchpad_focus_ref, + "primary_action_payload": _action_payload_with_focus_ref( + shared_action_payload, + worker_launchpad_focus_ref or launchpad_focus_ref, + ), + "secondary_focus_ref": release_detail_focus_ref if latest_release_id > 0 else {}, + } + + return {} + + +def _runbook_sequence( + *, + key: str, + title: str, + summary: str, + reason: str, + status: str = "ready", + status_label: str = "", + tag_type: str = "", + target_node_codes: list[str] | None = None, + target_scope_label: str = "", + step_titles: list[str] | None = None, + primary_label: str, + primary_action_code: str, + primary_action_payload: dict | None = None, + focus_ref: dict | None = None, + primary_focus_ref: dict | None = None, + secondary_label: str = "", + secondary_action_code: str = "", + secondary_action_payload: dict | None = None, + secondary_focus_ref: dict | None = None, +) -> dict: + return { + "key": str(key or "").strip(), + "title": str(title or "").strip(), + "summary": str(summary or "").strip(), + "reason": str(reason or "").strip(), + "status": str(status or "").strip() or "ready", + "status_label": str(status_label or "").strip() or str(status or "").strip() or "ready", + "tag_type": str(tag_type or "").strip() or "info", + "target_node_codes": _normalize_driver_node_codes(target_node_codes or []), + "target_scope_label": str(target_scope_label or "").strip(), + "step_titles": [str(item or "").strip() for item in list(step_titles or []) if str(item or "").strip()], + "focus_ref": _normalize_focus_ref(focus_ref), + "primary_label": str(primary_label or "").strip() or "执行", + "primary_action_code": str(primary_action_code or "").strip(), + "primary_action_payload": _action_payload_with_focus_ref(primary_action_payload or {}, primary_focus_ref), + "primary_focus_ref": _normalize_focus_ref(primary_focus_ref), + "secondary_label": str(secondary_label or "").strip(), + "secondary_action_code": str(secondary_action_code or "").strip(), + "secondary_action_payload": _action_payload_with_focus_ref(secondary_action_payload or {}, secondary_focus_ref), + "secondary_focus_ref": _normalize_focus_ref(secondary_focus_ref), + } + + +def _build_release_launchpad_runbook_sequence(*, release_launchpad: dict) -> dict: + launchpad = dict(release_launchpad or {}) + launchpad_status = dict(launchpad.get("launchpad_status") or {}) + action_code = str(launchpad_status.get("recommended_action_code") or "").strip() + if not action_code: + return {} + + latest_release = dict(launchpad.get("latest_release") or {}) + latest_package = dict(launchpad.get("latest_package") or {}) + worker_preview = dict(launchpad.get("worker_rollout_preview") or {}) + control_preview = dict(launchpad.get("control_rollout_preview") or {}) + worker_plan = dict(worker_preview.get("smart_rollout") or {}) + control_plan = dict(control_preview.get("smart_rollout") or {}) + first_gap_row = _first_release_launchpad_gap_row(launchpad) + latest_release_id = int(latest_release.get("id") or 0) + worker_node_codes = _release_launchpad_target_node_codes(launchpad, mode="worker") + control_node_codes = _release_launchpad_target_node_codes(launchpad, mode="control") + gap_node_codes = _normalize_driver_node_codes( + [str(row.get("node_code") or "").strip() for row in _release_launchpad_gap_rows(launchpad)] + ) + launchpad_focus_ref = _merge_focus_ref( + launchpad_status.get("focus_ref"), + release_id=latest_release_id, + release_version=str(latest_release.get("release_version") or "").strip(), + channel=str(latest_release.get("channel") or launchpad.get("channel") or "").strip(), + ) + release_detail_focus_ref = _merge_focus_ref( + latest_release.get("focus_ref"), + release_id=latest_release_id, + release_version=str(latest_release.get("release_version") or "").strip(), + channel=str(latest_release.get("channel") or launchpad.get("channel") or "").strip(), + section="release_detail", + ) + worker_launchpad_focus_ref = _merge_focus_ref(launchpad_focus_ref, section="release_launchpad", mode="worker") + control_launchpad_focus_ref = _merge_focus_ref(launchpad_focus_ref, section="release_launchpad", mode="control") + rollout_gate_focus_ref = _merge_focus_ref(launchpad_focus_ref, section="default_rollout_gate") + release_scope_label = ( + str(latest_release.get("release_version") or "").strip() + or str(latest_package.get("package_name") or "").strip() + or str(launchpad.get("channel") or "stable").strip() + or "latest" + ) + shared_action_payload = { + "channel": str(launchpad.get("channel") or "stable").strip() or "stable", + "control_plane_base_url": str(launchpad.get("control_plane_base_url") or "").strip(), + } + if latest_release_id > 0: + shared_action_payload["release_id"] = latest_release_id + + base_kwargs = { + "key": "release_progression", + "title": "版本发布与放量", + "summary": str(launchpad_status.get("summary") or "").strip(), + "status": str(launchpad_status.get("status") or "").strip() or "attention", + "status_label": str(launchpad_status.get("status_label") or "").strip() or "待处理", + "target_scope_label": f"Release / Launchpad / {release_scope_label}", + "focus_ref": launchpad_focus_ref, + } + + if action_code == "publish_latest_worker": + return _runbook_sequence( + **base_kwargs, + reason=str(worker_plan.get("summary") or worker_preview.get("message") or "最新发布包与 Worker 智能 Rollout 预检均已收口。").strip(), + tag_type="success", + target_node_codes=worker_node_codes, + step_titles=["确认最新发布包", "执行 Worker 灰度", "观察回执与巡检", "必要时推进 Control"], + primary_label="直接发 Worker 灰度", + primary_action_code="publish_latest_worker", + primary_action_payload=dict(shared_action_payload), + primary_focus_ref=worker_launchpad_focus_ref, + secondary_label="查看 Control 预案" if bool(control_plan.get("available", False)) else ("查看版本区" if latest_release_id > 0 else ""), + secondary_action_code="review_control_rollout" if bool(control_plan.get("available", False)) else ("focus_release_hub" if latest_release_id > 0 else ""), + secondary_action_payload=dict(shared_action_payload) if bool(control_plan.get("available", False)) else ({"release_id": latest_release_id} if latest_release_id > 0 else {}), + secondary_focus_ref=control_launchpad_focus_ref if bool(control_plan.get("available", False)) else release_detail_focus_ref, + ) + + if action_code == "review_smart_rollout_preview": + return _runbook_sequence( + **base_kwargs, + reason=str(worker_preview.get("message") or worker_plan.get("summary") or "Worker 智能 Rollout 仍需人工确认。").strip(), + tag_type="warning", + target_node_codes=worker_node_codes, + step_titles=["查看 Worker 智能预案", "确认门禁与审批", "必要时继续创建", "回看 Release Hub"], + primary_label="查看 Worker 预案", + primary_action_code="review_smart_rollout_preview", + primary_action_payload=dict(shared_action_payload), + primary_focus_ref=worker_launchpad_focus_ref, + secondary_label="查看版本区" if latest_release_id > 0 else "", + secondary_action_code="focus_release_hub" if latest_release_id > 0 else "", + secondary_action_payload={"release_id": latest_release_id} if latest_release_id > 0 else {}, + secondary_focus_ref=release_detail_focus_ref, + ) + + if action_code == "review_control_rollout": + return _runbook_sequence( + **base_kwargs, + reason=str(control_preview.get("message") or control_plan.get("summary") or "当前只有 Control 侧满足预发条件。").strip(), + tag_type="warning", + target_node_codes=control_node_codes, + step_titles=["查看 Control 预案", "确认 Control 目标", "确认门禁与审批", "必要时推进发布"], + primary_label="查看 Control 预案", + primary_action_code="review_control_rollout", + primary_action_payload=dict(shared_action_payload), + primary_focus_ref=control_launchpad_focus_ref, + secondary_label="查看版本区" if latest_release_id > 0 else "", + secondary_action_code="focus_release_hub" if latest_release_id > 0 else "", + secondary_action_payload={"release_id": latest_release_id} if latest_release_id > 0 else {}, + secondary_focus_ref=release_detail_focus_ref, + ) + + if action_code == "fix_rollout_blockers": + return _runbook_sequence( + **base_kwargs, + reason=str( + first_gap_row.get("agent_reason") + or first_gap_row.get("inspection_reason") + or worker_preview.get("message") + or control_preview.get("message") + or "Rollout 预检仍存在阻断条件。" + ).strip(), + tag_type="danger", + target_node_codes=gap_node_codes or worker_node_codes or control_node_codes, + step_titles=["查看 Worker 智能预案", "定位阻断节点", "补接管或巡检缺口", "重新回看预案"], + primary_label="查看阻断预案", + primary_action_code="fix_rollout_blockers", + primary_action_payload=dict(shared_action_payload), + primary_focus_ref=rollout_gate_focus_ref, + secondary_label="补执行面接管", + secondary_action_code="fix_managed_nodes", + secondary_action_payload=dict(shared_action_payload), + secondary_focus_ref=rollout_gate_focus_ref, + ) + + if action_code == "fix_managed_nodes": + step_titles = ["同步集群节点", "纳管首台缺口", "补齐 Agent 接入", "回到 Launchpad 复检"] + if str(first_gap_row.get("recovery_action") or "").strip() == "bootstrap_run": + step_titles = ["定位首台缺口节点", "执行 onboarding.bootstrap", "等待 register / heartbeat", "回到 Launchpad 复检"] + elif str(first_gap_row.get("recovery_action") or "").strip() == "run_acceptance": + step_titles = ["定位待验收节点", "执行 onboarding.acceptance", "确认 health / worker / agent", "回到 Launchpad 复检"] + return _runbook_sequence( + **base_kwargs, + reason=str( + first_gap_row.get("agent_reason") + or first_gap_row.get("inspection_reason") + or "当前没有可用于智能 Rollout 的在线启用执行节点，需要先补齐托管或接入状态。" + ).strip(), + tag_type="warning", + target_node_codes=gap_node_codes, + step_titles=step_titles, + primary_label="补执行面接管", + primary_action_code="fix_managed_nodes", + primary_action_payload=dict(shared_action_payload), + primary_focus_ref=rollout_gate_focus_ref, + secondary_label="查看版本区" if latest_release_id > 0 else "", + secondary_action_code="focus_release_hub" if latest_release_id > 0 else "", + secondary_action_payload={"release_id": latest_release_id} if latest_release_id > 0 else {}, + secondary_focus_ref=release_detail_focus_ref, + ) + + if action_code in {"bootstrap_run", "run_acceptance"}: + return _runbook_sequence( + **base_kwargs, + reason=str( + first_gap_row.get("recovery_summary") + or first_gap_row.get("onboarding_summary") + or first_gap_row.get("agent_reason") + or "当前需要先收口节点接管，再继续发布。" + ).strip(), + tag_type="warning" if action_code == "bootstrap_run" else "success", + target_node_codes=gap_node_codes or ([str(first_gap_row.get("node_code") or "").strip()] if str(first_gap_row.get("node_code") or "").strip() else []), + step_titles=( + ["定位首台缺口节点", "执行 onboarding.bootstrap", "等待 register / heartbeat", "回到 Launchpad 复检"] + if action_code == "bootstrap_run" + else ["定位待验收节点", "执行 onboarding.acceptance", "确认 health / worker / agent", "回到 Launchpad 复检"] + ), + primary_label="跑接入收口" if action_code == "bootstrap_run" else "跑接管验收", + primary_action_code=action_code, + primary_action_payload={ + **dict(shared_action_payload), + "node_code": str(first_gap_row.get("node_code") or "").strip(), + }, + primary_focus_ref=rollout_gate_focus_ref, + secondary_label="查看版本区" if latest_release_id > 0 else "", + secondary_action_code="focus_release_hub" if latest_release_id > 0 else "", + secondary_action_payload={"release_id": latest_release_id} if latest_release_id > 0 else {}, + secondary_focus_ref=release_detail_focus_ref, + ) + + if action_code == "api-restart": + return _runbook_sequence( + **base_kwargs, + reason=str( + launchpad_status.get("summary") + or "运行中的控制面 API 还没有刷新到当前仓库能力，建议先执行 runtime-refresh-recover。" + ).strip(), + tag_type="warning", + target_node_codes=[], + step_titles=["执行 runtime-refresh-recover", "确认 build-info / route surface", "重新拉取 Launchpad", "再决定 Release / Rollout"], + primary_label="查看刷新收口", + primary_action_code="api-restart", + primary_action_payload=dict(shared_action_payload), + primary_focus_ref=worker_launchpad_focus_ref or launchpad_focus_ref, + secondary_label="查看版本区" if latest_release_id > 0 else "", + secondary_action_code="focus_release_hub" if latest_release_id > 0 else "", + secondary_action_payload={"release_id": latest_release_id} if latest_release_id > 0 else {}, + secondary_focus_ref=release_detail_focus_ref if latest_release_id > 0 else {}, + ) + + if action_code == "release_package": + return _runbook_sequence( + **base_kwargs, + reason=str(latest_package.get("reason") or "当前没有可用的最新发布包，请先在控制面本机完成打包。").strip(), + tag_type="warning", + target_node_codes=[], + step_titles=["打开打包面板", "导入最新发布包", "确认 Release 记录", "重新回看 Launchpad"], + primary_label="打开打包面板", + primary_action_code="release_package", + primary_action_payload=dict(shared_action_payload), + primary_focus_ref=worker_launchpad_focus_ref or launchpad_focus_ref, + secondary_label="查看版本区" if latest_release_id > 0 else "", + secondary_action_code="focus_release_hub" if latest_release_id > 0 else "", + secondary_action_payload={"release_id": latest_release_id} if latest_release_id > 0 else {}, + secondary_focus_ref=release_detail_focus_ref, + ) + + if action_code == "release_prepare": + return _runbook_sequence( + **base_kwargs, + reason=str( + latest_package.get("final_release_report_stale_reason") + or launchpad_status.get("summary") + or "当前最新发布包还没有完成最终签收，不能直接进入正式发布动作。" + ).strip(), + tag_type="danger", + target_node_codes=[], + step_titles=["执行最终签收", "确认 verify / smoke / final report", "刷新最新包状态", "回到 Launchpad 复检"], + primary_label="执行最终签收", + primary_action_code="release_prepare", + primary_action_payload=dict(shared_action_payload), + primary_focus_ref=worker_launchpad_focus_ref or launchpad_focus_ref, + secondary_label="查看版本区" if latest_release_id > 0 else "", + secondary_action_code="focus_release_hub" if latest_release_id > 0 else "", + secondary_action_payload={"release_id": latest_release_id} if latest_release_id > 0 else {}, + secondary_focus_ref=release_detail_focus_ref, + ) + + return {} + + +def _build_ops_runbook_control_sequences( + *, + managed_nodes_payload: dict | None = None, + release_launchpad: dict | None = None, +) -> list[dict]: + nodes = list((managed_nodes_payload or {}).get("nodes") or []) + preferred_release = _preferred_release_for_ops() + preferred_release_id = int(preferred_release.get("id") or 0) + preferred_release_label = ( + str(preferred_release.get("release_version") or "").strip() + or str(preferred_release.get("channel") or "").strip() + or "默认 Release" + ) + + bootstrap_gap_nodes = [ + item + for item in nodes + if bool(item.get("cluster_is_effective_worker", False)) + and ( + not bool(item.get("is_managed", False)) + or not bool(item.get("is_enabled", False)) + or not bool(item.get("is_agent_online", False)) + ) + ] + bootstrap_gap_node_codes = _normalize_driver_node_codes(item.get("node_code") for item in bootstrap_gap_nodes) + + participating_nodes = [ + item + for item in nodes + if bool(item.get("cluster_is_effective_worker", False)) and bool(item.get("is_current_participant", False)) + ] + participating_node_codes = _normalize_driver_node_codes(item.get("node_code") for item in participating_nodes) + + inspection_ready_nodes = [ + item + for item in nodes + if _is_inspection_eligible_node(item) + ] + inspection_ready_node_codes = _normalize_driver_node_codes(item.get("node_code") for item in inspection_ready_nodes) + worker_release_node_codes = _normalize_driver_node_codes( + item.get("node_code") + for item in inspection_ready_nodes + if str(item.get("role") or "").strip() == "worker" + ) + control_release_node_codes = _normalize_driver_node_codes( + item.get("node_code") + for item in inspection_ready_nodes + if str(item.get("role") or "").strip() == "control" + ) + + scene_target_node_codes = participating_node_codes or inspection_ready_node_codes + + onboarding_status = "warning" if bootstrap_gap_node_codes else "ready" + onboarding_status_label = "有待接管节点" if bootstrap_gap_node_codes else "接管链路就绪" + onboarding_tag_type = "warning" if bootstrap_gap_node_codes else "success" + onboarding_summary = ( + f"当前有 {len(bootstrap_gap_node_codes)} 台有效执行节点还没进入 Agent 在线状态，建议先从海外控制面生成接入工单。" + if bootstrap_gap_node_codes + else "当前没有明显的接管缺口；后续新增大陆机器时，直接从这里生成接入工单。" + ) + + scene_status = "warning" if scene_target_node_codes else "info" + scene_status_label = "现场可观察" if scene_target_node_codes else "暂无现场目标" + scene_tag_type = "warning" if scene_target_node_codes else "info" + scene_summary = ( + f"当前有 {len(scene_target_node_codes)} 台节点适合作为执行现场观察目标，优先从关键日志开始，再决定是否升级到全量取证。" + if scene_target_node_codes + else "当前没有明确的参与节点；后续一旦有节点开始领任务，这里就是海外控制面第一入口。" + ) + + inspection_status = "ready" if inspection_ready_node_codes else "attention" + inspection_status_label = "可直接巡检" if inspection_ready_node_codes else "待补接管" + inspection_tag_type = "success" if inspection_ready_node_codes else "warning" + inspection_summary = ( + f"当前有 {len(inspection_ready_node_codes)} 台节点已经满足 已纳管 + 已启用 + Agent 在线，可直接走标准巡检。" + if inspection_ready_node_codes + else "当前还没有满足标准巡检条件的节点，建议先补齐接管和 Agent 在线状态。" + ) + + release_status = "ready" if preferred_release_id > 0 else "attention" + release_status_label = "可进入发布路径" if preferred_release_id > 0 else "待创建 Release" + release_tag_type = "success" if preferred_release_id > 0 else "warning" + release_summary = ( + f"当前默认版本 {preferred_release_label} 已可用于点状灰度或 Rollout；先发 Worker，再逐步推进控制面。" + if preferred_release_id > 0 + else "当前还没有可用 Release；建议先在海外控制面建立版本入口，再进入发布闭环。" + ) + release_progression_sequence = _build_release_launchpad_runbook_sequence( + release_launchpad=dict(release_launchpad or {}) + ) + if not release_progression_sequence: + release_progression_sequence = _runbook_sequence( + key="release_progression", + title="版本发布与放量", + summary=release_summary, + reason="正式运维时，版本更新应该统一落回 Release / Rollout，而不是回退到各机器单独 pull / restart。", + status=release_status, + status_label=release_status_label, + tag_type=release_tag_type, + target_node_codes=worker_release_node_codes or control_release_node_codes, + target_scope_label="Release / Rollout", + step_titles=["点状灰度 Worker", "健康检查", "必要时推进 Rollout", "最后处理控制面"], + primary_label="直接创建 Worker Rollout", + primary_action_code="create_release_rollout_worker", + primary_action_payload={ + "release_id": preferred_release_id, + "target_node_codes": worker_release_node_codes, + }, + secondary_label="查看版本区", + secondary_action_code="focus_release_hub", + secondary_action_payload={ + "release_id": preferred_release_id, + }, + ) + + sequences = [ + _runbook_sequence( + key="node_onboarding", + title="纳管新节点", + summary=onboarding_summary, + reason="所有新增大陆 controller / worker，第一步都应该先生成 Node Agent 接入工单，而不是手动 SSH 拼命令。", + status=onboarding_status, + status_label=onboarding_status_label, + tag_type=onboarding_tag_type, + target_node_codes=bootstrap_gap_node_codes, + target_scope_label="单节点 / 新节点", + step_titles=["生成接入工单", "复制 Env/脚本", "节点落地执行", "回到控制面看心跳"], + primary_label="跑接入收口", + primary_action_code="bootstrap_run", + primary_action_payload={ + "node_code": bootstrap_gap_node_codes[0] if bootstrap_gap_node_codes else "", + }, + secondary_label="跑接管验收", + secondary_action_code="run_acceptance", + secondary_action_payload={ + "node_code": inspection_ready_node_codes[0] if inspection_ready_node_codes else "", + }, + ), + _runbook_sequence( + key="scene_observe", + title="执行现场观察", + summary=scene_summary, + reason="海外控制面遇到“谁在跑、为什么卡住、日志有没有回来”这类问题时，应优先走统一现场观察路径。", + status=scene_status, + status_label=scene_status_label, + tag_type=scene_tag_type, + target_node_codes=scene_target_node_codes, + target_scope_label="参与节点 / 现场节点", + step_titles=["关键日志", "判断是否升级全量", "补诊断包", "回看联调快照"], + primary_label="执行关键现场日志", + primary_action_code="run_scene_logs_key", + primary_action_payload={ + "execution_mode": "remote-agent", + "auto_approve": True, + }, + secondary_label="执行全量现场取证", + secondary_action_code="run_scene_logs_full", + secondary_action_payload={ + "execution_mode": "remote-agent", + "auto_approve": True, + }, + ), + _runbook_sequence( + key="standard_inspection", + title="标准巡检", + summary=inspection_summary, + reason="现场收口、节点验收和发布前确认，尽量都走同一条标准巡检链，而不是手工拼快照、日志和诊断包。", + status=inspection_status, + status_label=inspection_status_label, + tag_type=inspection_tag_type, + target_node_codes=inspection_ready_node_codes, + target_scope_label="已接管执行节点", + step_titles=["健康快照", "Worker 日志", "诊断包", "统一回看巡检结果"], + primary_label="直接执行标准巡检", + primary_action_code="run_standard_inspection", + primary_action_payload={}, + secondary_label="补充诊断包", + secondary_action_code="open_diagnostics", + secondary_action_payload={ + "execution_mode": "remote-agent", + "auto_approve": True, + }, + ), + release_progression_sequence, + ] + return sequences + + +def _activity_time(*candidates: object) -> str: + for candidate in candidates: + value = str(candidate or "").strip() + if value: + return value + return "" + + +def _truncate_activity_text(value: object, *, limit: int = 240) -> str: + text = str(value or "").strip() + if len(text) <= limit: + return text + return f"{text[:limit]}..." + + +def _activity_status_label(status: object, *, kind: str = "") -> str: + normalized_status = str(status or "").strip() + mapping = { + "empty": "无活动", + "planned": "待推进", + "ready": "就绪", + "healthy": "健康", + "running": "执行中", + "attention": "待处理", + "queued": "排队中", + "dispatching": "派发中", + "awaiting_approval": "待审批", + "success": "成功", + "failed": "失败", + "blocked": "阻断", + "cancelled": "已取消", + "halted": "已暂停", + "completed_with_issues": "带问题完成", + "partially_succeeded": "部分成功", + } + if normalized_status in mapping: + return mapping[normalized_status] + if normalized_status: + return normalized_status + if str(kind or "").strip() in {"execution_scene", "log_sync"}: + return "待观察" + return "未知" + + +def _activity_focus_ref(item: dict) -> dict: + normalized_item = dict(item or {}) + kind = str(normalized_item.get("kind") or "").strip() + ui_intent = dict(normalized_item.get("ui_intent") or {}) + if kind == "playbook_run": + return { + "kind": "playbook_run", + "run_code": str(normalized_item.get("run_code") or "").strip(), + "focus_step_key": str((ui_intent.get("focus_step_key") or normalized_item.get("focus_step_key") or "")).strip(), + "focus_step_title": str((ui_intent.get("focus_step_title") or normalized_item.get("focus_step_title") or "")).strip(), + } + if kind == "ops_job": + return { + "kind": "ops_job", + "job_id": int(normalized_item.get("job_id") or 0), + "job_code": str(normalized_item.get("job_code") or "").strip(), + "action": str((ui_intent.get("action") or normalized_item.get("title") or "")).strip(), + "target_node_code": str( + ( + ui_intent.get("target_node_code") + or next(iter(normalized_item.get("target_node_codes") or []), "") + or "" + ) + ).strip(), + } + if kind == "rollout": + return { + "kind": "rollout", + "rollout_id": int(normalized_item.get("rollout_id") or 0), + "rollout_code": str(normalized_item.get("title") or "").strip(), + "release_id": int((ui_intent.get("release_id") or 0)), + } + if kind == "runbook_sequence": + return { + "kind": "runbook_sequence", + "sequence_key": str(normalized_item.get("sequence_key") or "").strip(), + "driver_action_code": str(ui_intent.get("driver_action_code") or "").strip(), + } + if kind in {"execution_scene", "log_sync"}: + return { + "kind": "execution_scene", + "scene_key": str(normalized_item.get("activity_key") or kind).strip(), + } + return { + "kind": kind or "activity", + "activity_key": str(normalized_item.get("activity_key") or "").strip(), + } + + +def _finalize_activity_item(item: dict) -> dict: + normalized_item = dict(item or {}) + kind = str(normalized_item.get("kind") or "").strip() + ui_intent = dict(normalized_item.get("ui_intent") or {}) + summary_text = str(normalized_item.get("summary") or "").strip() + status_label = str(normalized_item.get("status_label") or "").strip() or _activity_status_label( + normalized_item.get("status"), + kind=kind, + ) + target_node_codes = _normalize_driver_node_codes(normalized_item.get("target_node_codes") or []) + finalized = { + **normalized_item, + "summary": summary_text, + "summary_text": summary_text, + "status_label": status_label, + "ui_intent": ui_intent, + "ui_intent_kind": str(ui_intent.get("kind") or "").strip(), + "target_node_codes": target_node_codes, + } + existing_focus_ref = dict(normalized_item.get("focus_ref") or {}) + existing_source_focus_ref = dict(normalized_item.get("source_focus_ref") or {}) + finalized["focus_ref"] = existing_focus_ref if existing_focus_ref else _activity_focus_ref(finalized) + finalized["source_focus_ref"] = existing_source_focus_ref + contract_keys = _normalize_ops_contract_keys( + list(normalized_item.get("contract_keys") or []) + + _suggest_ops_contract_keys_for_activity_item(finalized) + ) + finalized["contract_keys"] = contract_keys + finalized["contract_navigation"] = _build_ops_contract_navigation( + contract_keys, + primary_contract_key=_primary_ops_contract_key_for_activity_item(finalized, contract_keys), + registry=get_ops_contract_registry(), + ) + return finalized + + +def _activity_stream_summary_state(items: list[dict]) -> tuple[str, str, str]: + normalized_items = [dict(item or {}) for item in list(items or [])] + if not normalized_items: + return "empty", "无活动", "当前没有匹配的活动。" + + hard_attention_statuses = { + "attention", + "failed", + "blocked", + "cancelled", + "halted", + "completed_with_issues", + "partially_succeeded", + "waiting_sample", + "partial_coverage", + } + soft_attention_statuses = {"awaiting_approval"} + running_statuses = {"running", "dispatching"} + planned_statuses = {"queued", "planned", "ready"} + + item_statuses = {str(item.get("status") or "").strip() for item in normalized_items if str(item.get("status") or "").strip()} + + if item_statuses & hard_attention_statuses: + return "attention", "待处理", "最近活动里存在需优先处理的现场、任务或发布推进。" + if item_statuses & soft_attention_statuses: + return "attention", "待审批", "最近活动里存在待审批或待人工确认的动作。" + if item_statuses & running_statuses: + return "running", "执行中", "最近活动里存在正在推进的现场、任务、编排或回传。" + if item_statuses & planned_statuses: + return "planned", "待推进", "最近活动里存在已排队或即将继续推进的动作。" + return "ready", "平稳", "最近活动整体平稳，可按需抽样查看。" + + +def _is_playbook_child_job(job: dict) -> bool: + return bool(str(((job.get("metadata") or {}).get("playbook") or {}).get("run_code") or "").strip()) + + +def _build_playbook_run_activity(run: dict) -> dict: + normalized_run = dict(run or {}) + run_code = str(normalized_run.get("run_code") or "").strip() + title = ( + str(normalized_run.get("playbook_title") or "").strip() + or str(normalized_run.get("playbook_key") or "").strip() + or "标准编排" + ) + steps = list(normalized_run.get("steps") or []) + step_count = int(normalized_run.get("step_count") or len(steps) or 0) + completed_steps = sum(1 for step in steps if str(step.get("status") or "").strip() in {"success", "attention"}) + target_node_codes = _normalize_driver_node_codes(normalized_run.get("target_node_codes") or []) + focus_summary = str(normalized_run.get("focus_summary") or "").strip() + if not focus_summary: + focus_summary = f"当前状态 {str(normalized_run.get('status') or '').strip() or 'queued'}，建议进入编排详情继续观察。" + meta_parts = [run_code] + group_title = str(normalized_run.get("group_title") or normalized_run.get("group_key") or "").strip() + if group_title: + meta_parts.append(group_title) + execution_mode_label_text = ( + str(normalized_run.get("execution_mode_label") or "").strip() + or execution_mode_label(str(normalized_run.get("execution_mode") or "").strip()) + ) + if execution_mode_label_text: + meta_parts.append(f"执行 {execution_mode_label_text}") + if step_count > 0: + meta_parts.append(f"步骤 {completed_steps}/{step_count}") + jobs_total = int(normalized_run.get("jobs_total") or 0) + if jobs_total > 0: + meta_parts.append(f"任务 {jobs_total}") + if target_node_codes: + meta_parts.append(f"节点 {len(target_node_codes)}") + return { + "kind": "playbook_run", + "activity_key": f"playbook-run:{run_code or 'latest'}", + "title": title, + "subtitle": run_code, + "summary": _truncate_activity_text(focus_summary), + "meta_text": " / ".join(part for part in meta_parts if part), + "status": str(normalized_run.get("status") or "").strip(), + "execution_mode": str(normalized_run.get("execution_mode") or "").strip(), + "execution_mode_label": execution_mode_label_text, + "occurred_at": _activity_time(normalized_run.get("updated_at"), normalized_run.get("created_at")), + "run_code": run_code, + "focus_step_key": str(normalized_run.get("focus_step_key") or "").strip(), + "focus_step_title": str(normalized_run.get("focus_step_title") or "").strip(), + "target_node_codes": target_node_codes, + "ui_intent": _build_driver_ui_intent( + "playbook_run_detail", + run_code=run_code, + focus_step_key=str(normalized_run.get("focus_step_key") or "").strip(), + focus_step_title=str(normalized_run.get("focus_step_title") or "").strip(), + ), + } + + +def _build_bootstrap_job_activity(job: dict, managed_node: dict | None = None) -> dict: + normalized_job = dict(job or {}) + normalized_node = dict(managed_node or {}) + job_id = int(normalized_job.get("id") or 0) + job_code = str(normalized_job.get("job_code") or "").strip() + status = str(normalized_job.get("status") or "").strip() + target_node_code = str(normalized_job.get("target_node_code") or "").strip() + error_message = str(normalized_job.get("error_message") or "").strip() + result = dict(normalized_job.get("result") or {}) + bootstrap_plan = dict(result.get("bootstrap_plan") or {}) + script_name = str(bootstrap_plan.get("bootstrap_script_name") or "").strip() + agent_state = str(normalized_node.get("agent_state") or "").strip() + agent_state_label = str(normalized_node.get("agent_state_label") or "").strip() + + if error_message and status in {"failed", "blocked", "cancelled"}: + summary = error_message + elif status == "awaiting_approval": + summary = "接入工单等待审批，审批通过后才会生成 bootstrap env、脚本和一键落地命令。" + elif status in {"queued", "dispatching", "running"}: + summary = "控制面正在生成节点接入工单，完成后可直接复制 bootstrap env、脚本和一键落地命令。" + elif status == "success": + if agent_state in {"online", "online_busy"}: + summary = "接入工单已生成，目标节点已完成 Agent 接入，可继续执行接管后验收。" + elif agent_state == "pending_bootstrap": + summary = "接入工单已生成，节点尚未回连 Agent，可直接复制脚本或一键落地命令。" + elif agent_state == "runtime_only": + summary = "接入工单已生成，但当前只有 runtime 心跳，Node Agent 还未真正接管。" + elif agent_state == "stale": + summary = "接入工单已生成，但 Agent 心跳已过期，建议先看任务详情和 Agent 日志。" + else: + summary = "接入工单已生成，可直接复制 bootstrap env、脚本和一键落地命令。" + else: + summary = "节点接入工单最近有新的回执。" + + execution_mode_label_text = ( + str(normalized_job.get("execution_mode_label") or "").strip() + or execution_mode_label(str(normalized_job.get("execution_mode") or "control-plane").strip() or "control-plane") + ) + meta_parts = [ + part + for part in [ + job_code, + f"节点 {target_node_code}" if target_node_code else "", + f"方式 {execution_mode_label_text}" if execution_mode_label_text else "", + script_name, + ] + if part + ] + if agent_state_label: + meta_parts.append(f"状态 {agent_state_label}") + + return { + "kind": "ops_job", + "activity_key": f"ops-job:{job_id or job_code or 'node.bootstrap'}", + "action": "node.bootstrap", + "title": "节点接入工单", + "subtitle": job_code, + "summary": _truncate_activity_text(summary), + "meta_text": " / ".join(meta_parts), + "status": status, + "execution_mode": str(normalized_job.get("execution_mode") or "control-plane").strip() or "control-plane", + "execution_mode_label": execution_mode_label_text, + "occurred_at": _activity_time( + normalized_job.get("updated_at"), + normalized_job.get("finished_at"), + normalized_job.get("started_at"), + normalized_job.get("created_at"), + ), + "job_id": job_id, + "job_code": job_code, + "target_node_codes": _normalize_driver_node_codes([target_node_code]), + "ui_intent": _build_driver_ui_intent( + "job_detail", + job_id=job_id, + job_code=job_code, + action="node.bootstrap", + target_node_code=target_node_code, + ), + } + + +def _build_ops_job_activity(job: dict, *, managed_node: dict | None = None) -> dict: + normalized_job = dict(job or {}) + job_id = int(normalized_job.get("id") or 0) + job_code = str(normalized_job.get("job_code") or "").strip() + action = str(normalized_job.get("action") or "").strip() or "ops.job" + if action == "node.bootstrap": + return _build_bootstrap_job_activity(normalized_job, managed_node) + target_node_code = str(normalized_job.get("target_node_code") or "").strip() + target_type = str(normalized_job.get("target_type") or "").strip() + execution_mode = str(normalized_job.get("execution_mode") or "").strip() + execution_mode_label_text = ( + str(normalized_job.get("execution_mode_label") or "").strip() + or execution_mode_label(execution_mode) + ) + requested_by = str(normalized_job.get("requested_by") or "").strip() + status = str(normalized_job.get("status") or "").strip() + error_message = str(normalized_job.get("error_message") or "").strip() + result = dict(normalized_job.get("result") or {}) + start_delivery_state = str( + result.get("start_delivery_state") + or normalized_job.get("start_delivery_state") + or "" + ).strip() + start_delivery_error = str( + result.get("start_delivery_error") + or normalized_job.get("start_delivery_error") + or "" + ).strip() + preferred_summary_text = str( + normalized_job.get("result_summary_text") + or result.get("summary_text") + or result.get("summary") + or normalized_job.get("summary_text") + or normalized_job.get("summary") + or "" + ).strip() + summary_parts: list[str] = [] + if not preferred_summary_text: + if target_node_code: + summary_parts.append(f"目标节点 {target_node_code}") + elif target_type: + summary_parts.append(f"目标类型 {target_type}") + if requested_by: + summary_parts.append(f"发起 {requested_by}") + if execution_mode_label_text: + summary_parts.append(f"方式 {execution_mode_label_text}") + summary_candidates: list[str] = [] + if preferred_summary_text: + summary_candidates.append(preferred_summary_text) + if error_message and status in {"failed", "blocked", "cancelled"}: + summary_candidates.append(error_message) + if start_delivery_state == "failed_local": + summary_candidates.append("节点已接单执行，但开始回执未成功送达控制面。") + if start_delivery_error: + summary_candidates.append(f"开始回执失败：{start_delivery_error}") + meta_parts = [job_code] + risk_level = str(normalized_job.get("risk_level") or "").strip() + if risk_level: + meta_parts.append(f"风险 {risk_level}") + if execution_mode_label_text: + meta_parts.append(f"执行 {execution_mode_label_text}") + effective_status = status + if start_delivery_state == "failed_local" and status not in {"failed", "blocked", "cancelled"}: + effective_status = "attention" + if status: + meta_parts.append(f"任务 {_activity_status_label(status)}") + meta_parts.append("开始回执异常") + merged_focus_ref = _merge_focus_ref(normalized_job.get("focus_ref"), **_normalize_focus_ref(result.get("focus_ref"))) + source_focus_ref = _normalize_focus_ref(result.get("source_focus_ref")) + summary_text = " / ".join( + dict.fromkeys(part for part in [*summary_candidates, *summary_parts] if str(part or "").strip()) + ) or "标准运维任务最近有新回执。" + activity = { + "kind": "ops_job", + "activity_key": f"ops-job:{job_id or job_code or action}", + "action": action, + "title": action, + "subtitle": job_code, + "summary": _truncate_activity_text(summary_text), + "meta_text": " / ".join(part for part in meta_parts if part), + "status": effective_status, + "job_status": status, + "execution_mode": execution_mode, + "execution_mode_label": execution_mode_label_text, + "occurred_at": _activity_time( + normalized_job.get("updated_at"), + normalized_job.get("finished_at"), + normalized_job.get("started_at"), + normalized_job.get("created_at"), + ), + "job_id": job_id, + "job_code": job_code, + "target_node_codes": _normalize_driver_node_codes([target_node_code]), + "focus_ref": merged_focus_ref, + "source_focus_ref": source_focus_ref, + "ui_intent": _build_driver_ui_intent( + "job_events", + job_id=job_id, + job_code=job_code, + action=action, + target_node_code=target_node_code, + ), + } + if start_delivery_state: + activity["start_delivery_state"] = start_delivery_state + if start_delivery_error: + activity["start_delivery_error"] = start_delivery_error + if start_delivery_state == "failed_local": + activity["problem_code"] = "ops_job_start_delivery_failed_local" + return activity + + +def _build_rollout_activity(rollout: dict) -> dict: + normalized_rollout = dict(rollout or {}) + rollout_id = int(normalized_rollout.get("id") or 0) + rollout_code = str(normalized_rollout.get("rollout_code") or "").strip() + rollout_policy = dict(normalized_rollout.get("policy") or {}) + rollout_execution_mode = str(rollout_policy.get("execution_mode") or "remote-agent").strip() or "remote-agent" + rollout_execution_mode_label = ( + str(normalized_rollout.get("execution_mode_label") or "").strip() + or execution_mode_label(rollout_execution_mode) + ) + rollout_target_node_codes = _normalize_driver_node_codes( + [str(item.get("node_code") or "").strip() for item in list(normalized_rollout.get("target_nodes") or [])] + ) + target_nodes_total = len(rollout_target_node_codes) + jobs_total = int(normalized_rollout.get("jobs_total") or 0) + jobs_created = int(normalized_rollout.get("jobs_created") or 0) + batch_cursor = int(normalized_rollout.get("batch_cursor") or 0) + batches_total = int(normalized_rollout.get("batches_total") or 0) + summary_parts: list[str] = [] + if target_nodes_total > 0: + summary_parts.append(f"目标节点 {target_nodes_total}") + if jobs_total > 0: + summary_parts.append(f"任务 {jobs_created}/{jobs_total}") + if batches_total > 0: + summary_parts.append(f"批次 {batch_cursor}/{batches_total}") + result_summary = dict(normalized_rollout.get("result_summary") or {}) + summary_text = str(normalized_rollout.get("summary_text") or normalized_rollout.get("summary") or "").strip() + if int(result_summary.get("failed", 0) or 0) > 0: + summary_parts.append(f"失败 {int(result_summary.get('failed', 0) or 0)}") + elif int(result_summary.get("success", 0) or 0) > 0: + summary_parts.append(f"成功 {int(result_summary.get('success', 0) or 0)}") + meta_parts = [f"创建人 {str(normalized_rollout.get('created_by') or '').strip() or '-'}"] + if rollout_execution_mode_label: + meta_parts.append(f"执行 {rollout_execution_mode_label}") + return { + "kind": "rollout", + "activity_key": f"rollout:{rollout_id or rollout_code}", + "title": rollout_code or "Release Rollout", + "subtitle": f"Release #{int(normalized_rollout.get('release_id') or 0)}" if int(normalized_rollout.get("release_id") or 0) > 0 else "", + "summary": _truncate_activity_text(summary_text or " / ".join(part for part in summary_parts if part) or "该 rollout 最近有新的推进或回执。"), + "meta_text": " / ".join(part for part in meta_parts if part), + "status": str(normalized_rollout.get("status") or "").strip(), + "status_label": str(normalized_rollout.get("status_label") or "").strip(), + "execution_mode": rollout_execution_mode, + "execution_mode_label": rollout_execution_mode_label, + "occurred_at": _activity_time(normalized_rollout.get("updated_at"), normalized_rollout.get("created_at")), + "rollout_id": rollout_id, + "target_node_codes": rollout_target_node_codes, + "ui_intent": _build_driver_ui_intent( + "rollout_jobs", + rollout_id=rollout_id, + rollout_code=rollout_code, + release_id=int(normalized_rollout.get("release_id") or 0), + ), + "focus_ref": dict(normalized_rollout.get("focus_ref") or {}), + } + + +def _runbook_sequence_activity_status(sequence: dict) -> str: + normalized_sequence = dict(sequence or {}) + primary_resolution = dict(normalized_sequence.get("primary_resolution") or {}) + if primary_resolution and primary_resolution.get("ok") is False: + return "attention" + normalized_status = str(normalized_sequence.get("status") or "").strip() + if normalized_status in {"warning", "attention", "blocking"}: + return "attention" + if normalized_status in {"ready", "success"}: + return "ready" + if normalized_status in {"info", "idle"}: + return "planned" + return normalized_status or "planned" + + +def _build_runbook_sequence_activity(sequence: dict) -> dict: + normalized_sequence = dict(sequence or {}) + sequence_key = str(normalized_sequence.get("key") or "").strip() + if not sequence_key: + return {} + + title = str(normalized_sequence.get("title") or "").strip() or "标准作业路径" + status_label = str(normalized_sequence.get("status_label") or normalized_sequence.get("status") or "").strip() + reason = str(normalized_sequence.get("reason") or "").strip() + target_scope_label = str(normalized_sequence.get("target_scope_label") or "").strip() + target_node_codes = _normalize_driver_node_codes(normalized_sequence.get("target_node_codes") or []) + primary_resolution = dict(normalized_sequence.get("primary_resolution") or {}) + resolved_action_label = ( + str(primary_resolution.get("driver_action_label") or "").strip() + or str(primary_resolution.get("driver_action_code") or "").strip() + ) + resolved_action_code = str(primary_resolution.get("driver_action_code") or "").strip() + resolved_node_codes = _normalize_driver_node_codes(primary_resolution.get("driver_node_codes") or []) + resolution_message = str(primary_resolution.get("message") or "").strip() + occurred_at = _activity_time(primary_resolution.get("resolved_at")) + + summary_parts: list[str] = [] + if primary_resolution and primary_resolution.get("ok") is False: + summary_parts.append(f"当前解析失败：{resolution_message or '标准作业路径解析失败'}") + elif resolved_action_label or resolved_action_code: + summary_parts.append( + f"当前建议动作：{resolved_action_label or resolved_action_code}" + + ( + f" ({resolved_action_code})" + if resolved_action_code and resolved_action_label and resolved_action_code != resolved_action_label + else "" + ) + ) + if resolved_node_codes: + summary_parts.append(f"目标节点：{_format_node_code_list(resolved_node_codes)}") + elif target_node_codes: + summary_parts.append(f"目标节点：{_format_node_code_list(target_node_codes)}") + if resolution_message: + summary_parts.append(resolution_message) + else: + summary_parts.append(str(normalized_sequence.get("summary") or "").strip() or "当前标准作业路径已就绪。") + + meta_parts = [status_label, sequence_key] + if target_scope_label: + meta_parts.append(target_scope_label) + if reason: + meta_parts.append(reason) + + return { + "kind": "runbook_sequence", + "activity_key": f"runbook-sequence:{sequence_key}", + "title": title, + "subtitle": sequence_key, + "summary": _truncate_activity_text(" / ".join(part for part in summary_parts if part)), + "meta_text": _truncate_activity_text(" / ".join(part for part in meta_parts if part), limit=280), + "status": _runbook_sequence_activity_status(normalized_sequence), + "execution_mode": "", + "execution_mode_label": "", + "occurred_at": occurred_at, + "sequence_key": sequence_key, + "target_node_codes": target_node_codes, + "focus_ref": _normalize_focus_ref(normalized_sequence.get("focus_ref")), + "ui_intent": _build_driver_ui_intent( + "focus_runbook_sequence", + sequence_key=sequence_key, + driver_action_code=resolved_action_code or str(normalized_sequence.get("primary_action_code") or "").strip(), + ), + } + + +def _matches_activity_query(item: dict, query: str) -> bool: + normalized_query = str(query or "").strip().lower() + if not normalized_query: + return True + haystack = " ".join( + [ + str(item.get("activity_key") or ""), + str(item.get("kind") or ""), + str(item.get("title") or ""), + str(item.get("subtitle") or ""), + str(item.get("summary") or ""), + str(item.get("meta_text") or ""), + str(item.get("status") or ""), + str(item.get("execution_mode") or ""), + str(item.get("execution_mode_label") or ""), + str(item.get("run_code") or ""), + str(item.get("job_code") or ""), + str(item.get("rollout_id") or ""), + str(item.get("sequence_key") or ""), + ] + ).lower() + return normalized_query in haystack + + +def _driver_activity_status_label(status: str) -> str: + normalized_status = str(status or "").strip() + mapping = { + "failed": "失败", + "blocked": "阻断", + "cancelled": "已取消", + "halted": "已暂停", + "completed_with_issues": "带问题完成", + "partially_succeeded": "部分成功", + "awaiting_approval": "待审批", + } + return mapping.get(normalized_status, normalized_status or "异常") + + +def _pick_driver_activity_focus(items: list[dict]) -> tuple[str, dict]: + hard_problem_statuses = {"failed", "blocked", "cancelled", "halted", "completed_with_issues", "partially_succeeded"} + soft_attention_statuses = {"awaiting_approval"} + candidate_items = [ + dict(item or {}) + for item in list(items or []) + if str(item.get("kind") or "").strip() in {"ops_job", "rollout"} + ] + + for item in candidate_items: + if str(item.get("status") or "").strip() in hard_problem_statuses: + return "hard", item + for item in candidate_items: + if str(item.get("status") or "").strip() in soft_attention_statuses: + return "soft", item + return "", {} + + +def _build_driver_activity_card(level: str, activity_item: dict) -> dict: + normalized_item = dict(activity_item or {}) + item_kind = str(normalized_item.get("kind") or "").strip() + item_status = str(normalized_item.get("status") or "").strip() + ui_intent_kind = str(((normalized_item.get("ui_intent") or {}).get("kind") or "")).strip() + source_focus_ref = _normalize_focus_ref(normalized_item.get("source_focus_ref")) + title = str(normalized_item.get("title") or "").strip() or ("Release Rollout" if item_kind == "rollout" else "标准运维任务") + subtitle = str(normalized_item.get("subtitle") or "").strip() + summary = str(normalized_item.get("summary") or "").strip() + meta_text = str(normalized_item.get("meta_text") or "").strip() + status_label = _driver_activity_status_label(item_status) + noun_label = "异常 Rollout" if item_kind == "rollout" else "异常任务" + is_hard_problem = level == "hard" + + if item_kind == "rollout": + title_text = "先处理异常 Rollout" if is_hard_problem else "先处理待审批 Rollout" + primary_label = "查看 Rollout 作业" + else: + title_text = "先处理异常任务" if is_hard_problem else "先处理待审批任务" + primary_label = "查看任务详情" if ui_intent_kind == "job_detail" else "查看任务事件" + + reason = ( + summary + or ( + "该活动已经进入失败、阻断或带问题完成状态，继续堆动作只会放大噪音。" + if is_hard_problem + else "该活动正在等待人工审批或决策，先落到对应详情最容易判断下一步。" + ) + ) + detail_summary = f"{title}{' / ' + subtitle if subtitle else ''} 当前状态为 {status_label}，建议先进入对应详情。" + primary_action_code = "focus_activity_item" + action_payload = { + "activity_key": str(normalized_item.get("activity_key") or "").strip(), + "kind": item_kind, + "status": item_status, + "ui_intent": dict(normalized_item.get("ui_intent") or {}), + "focus_ref": _normalize_focus_ref(normalized_item.get("focus_ref")), + } + if ui_intent_kind == "job_events" and str(source_focus_ref.get("kind") or "").strip() == "ops_job_event": + primary_action_code = "focus_latest_job_events" + action_payload["source_focus_ref"] = source_focus_ref + + return { + "key": f"activity-focus-{str(normalized_item.get('activity_key') or item_kind or 'activity').strip()}", + "title": title_text, + "level_label": "最高优先" if is_hard_problem else "推荐", + "tag_type": "danger" if is_hard_problem else "warning", + "summary": detail_summary, + "reason": reason, + "node_codes": [], + "meta_text": meta_text or subtitle or str(normalized_item.get("activity_key") or "").strip(), + "primary_label": primary_label, + "secondary_label": "", + "primary_type": "danger" if is_hard_problem else "warning", + "disabled": not str(((normalized_item.get("ui_intent") or {}).get("kind") or "")).strip(), + "primary_action_code": primary_action_code, + "secondary_action_code": "", + "action_payload": action_payload, + "focus_ref": _normalize_focus_ref(normalized_item.get("focus_ref")), + "related_activity": { + "kind": item_kind, + "title": title, + "subtitle": subtitle, + "status": item_status, + "occurred_at": str(normalized_item.get("occurred_at") or "").strip(), + }, + } + + +def _build_driver_activity_focus_entry(activity_item: dict) -> dict: + normalized_item = dict(activity_item or {}) + activity_key = str(normalized_item.get("activity_key") or "").strip() + activity_kind = str(normalized_item.get("kind") or "").strip() + ui_intent = dict(normalized_item.get("ui_intent") or {}) + ui_intent_kind = str(ui_intent.get("kind") or "").strip() + if not activity_key or not activity_kind: + return {} + + focus_ref = _normalize_focus_ref(normalized_item.get("focus_ref")) + source_focus_ref = _normalize_focus_ref(normalized_item.get("source_focus_ref")) + focus_action_code = "focus_activity_item" if ui_intent_kind else "" + if ui_intent_kind == "job_events" and str(source_focus_ref.get("kind") or "").strip() == "ops_job_event": + focus_action_code = "focus_latest_job_events" + focus_action_payload = { + "activity_key": activity_key, + "kind": activity_kind, + "status": str(normalized_item.get("status") or "").strip(), + "ui_intent": ui_intent, + "focus_ref": focus_ref, + } + if source_focus_ref: + focus_action_payload["source_focus_ref"] = source_focus_ref + contract_keys = _normalize_ops_contract_keys( + list(normalized_item.get("contract_keys") or []) + + _suggest_ops_contract_keys_for_activity_item(normalized_item) + ) + + return { + "key": f"activity:{activity_key}", + "activity_key": activity_key, + "activity_kind": activity_kind, + "title": str(normalized_item.get("title") or "").strip() or activity_kind, + "subtitle": str(normalized_item.get("subtitle") or "").strip(), + "summary": str(normalized_item.get("summary_text") or normalized_item.get("summary") or "").strip(), + "status": str(normalized_item.get("status") or "").strip(), + "status_label": str(normalized_item.get("status_label") or "").strip(), + "meta_text": str(normalized_item.get("meta_text") or "").strip(), + "occurred_at": str(normalized_item.get("occurred_at") or "").strip(), + "target_node_codes": _normalize_driver_node_codes(normalized_item.get("target_node_codes") or []), + "focus_ref": focus_ref, + "source_focus_ref": source_focus_ref, + "ui_intent": ui_intent, + "ui_intent_kind": ui_intent_kind, + "focus_action_code": focus_action_code, + "focus_action_payload": focus_action_payload if ui_intent_kind else {}, + "observation_only": True, + "contract_keys": contract_keys, + "contract_navigation": _build_ops_contract_navigation( + contract_keys, + primary_contract_key=_primary_ops_contract_key_for_activity_item(normalized_item, contract_keys), + registry=get_ops_contract_registry(), + ), + } + + +def _build_driver_activity_focus(items: list[dict], *, limit: int = 3) -> list[dict]: + allowed_scene_kinds = {"execution_scene", "log_sync"} + allowed_run_kinds = {"playbook_run", "ops_job", "rollout"} + rows: list[dict] = [] + seen_keys: set[str] = set() + + for raw_item in list(items or []): + item = dict(raw_item or {}) + item_kind = str(item.get("kind") or "").strip() + item_status = str(item.get("status") or "").strip() + if item_kind in allowed_scene_kinds: + pass + elif item_kind == "playbook_run": + if item_status in {"success", "completed", "healthy"}: + continue + elif item_kind in allowed_run_kinds: + if item_status in {"success", "completed", "ready", "planned"}: + continue + else: + continue + entry = _build_driver_activity_focus_entry(item) + entry_key = str(entry.get("key") or "").strip() + if not entry or not entry_key or entry_key in seen_keys: + continue + seen_keys.add(entry_key) + rows.append(entry) + if len(rows) >= limit: + break + return rows + + +def _build_execution_scene_log_sync(log_sync: dict, participating_nodes: list[dict]) -> dict: + normalized_log_sync = dict(log_sync or {}) + enabled = bool(normalized_log_sync.get("enabled", False)) + mode = "full" if str(normalized_log_sync.get("mode") or "").strip().lower() == "full" else "key" + line_count = int(normalized_log_sync.get("line_count", 0) or 0) + source_nodes = [ + str(item or "").strip() + for item in list(normalized_log_sync.get("source_nodes") or []) + if str(item or "").strip() + ] + source_nodes = list(dict.fromkeys(source_nodes)) + preview_lines = [ + str(item or "").strip() + for item in list(normalized_log_sync.get("preview_lines") or []) + if str(item or "").strip() + ] + preview_lines = preview_lines[-20:] + source_node_summaries_map: dict[str, dict] = {} + for item in list(normalized_log_sync.get("source_node_summaries") or []): + if not isinstance(item, dict): + continue + node_code = str(item.get("node_code") or "").strip() + if not node_code: + continue + source_node_summaries_map[node_code] = { + "node_code": node_code, + "line_count": int(item.get("line_count", 0) or 0), + "key_line_count": int(item.get("key_line_count", 0) or 0), + "full_line_count": int(item.get("full_line_count", 0) or 0), + "last_at": str(item.get("last_at") or "").strip(), + "last_line": str(item.get("last_line") or "").strip(), + } + if not source_node_summaries_map and preview_lines: + for line in preview_lines: + match = _REMOTE_LOG_PREVIEW_LINE_RE.match(line) + if not match: + continue + node_code = str(match.group("node_code") or "").strip() + created_at = str(match.group("created_at") or "").strip() + if not node_code: + continue + bucket = source_node_summaries_map.setdefault( + node_code, + { + "node_code": node_code, + "line_count": 0, + "key_line_count": 0, + "full_line_count": 0, + "last_at": "", + "last_line": "", + }, + ) + bucket["line_count"] += 1 + bucket["key_line_count"] += 1 + bucket["last_at"] = created_at + bucket["last_line"] = line + for node_code in source_nodes: + source_node_summaries_map.setdefault( + node_code, + { + "node_code": node_code, + "line_count": 0, + "key_line_count": 0, + "full_line_count": 0, + "last_at": "", + "last_line": "", + }, + ) + source_nodes = list(dict.fromkeys([*source_nodes, *list(source_node_summaries_map.keys())])) + source_node_summaries = sorted( + source_node_summaries_map.values(), + key=lambda item: ( + str(item.get("last_at") or ""), + str(item.get("node_code") or ""), + ), + reverse=True, + ) + source_node_set = set(source_nodes) + source_node_set.update(source_node_summaries_map.keys()) + participating_node_codes = [ + str(item.get("node_code") or "").strip() + for item in list(participating_nodes or []) + if str(item.get("node_code") or "").strip() + ] + participating_node_codes = list(dict.fromkeys(participating_node_codes)) + participating_node_map = { + str(item.get("node_code") or "").strip(): dict(item) + for item in list(participating_nodes or []) + if str(item.get("node_code") or "").strip() + } + covered_participating_nodes = [node_code for node_code in participating_node_codes if node_code in source_node_set] + missing_participating_nodes = [node_code for node_code in participating_node_codes if node_code not in source_node_set] + covered_participating_node_summaries = [] + for node_code in covered_participating_nodes: + source_summary = dict(source_node_summaries_map.get(node_code) or {}) + node_meta = participating_node_map.get(node_code) or {} + covered_participating_node_summaries.append( + { + **source_summary, + "region": str(node_meta.get("region") or "").strip(), + "role": str(node_meta.get("role") or "").strip(), + "status": str(node_meta.get("status") or "").strip(), + "participation_state": str(node_meta.get("participation_state") or "").strip(), + "participation_bucket": str(node_meta.get("participation_bucket") or "").strip(), + "participation_label": str(node_meta.get("participation_label") or "").strip(), + "participation_reason": str(node_meta.get("participation_reason") or "").strip(), + } + ) + missing_participating_node_summaries = [] + for node_code in missing_participating_nodes: + node_meta = participating_node_map.get(node_code) or {} + missing_reason_code = "no_sample" + missing_reason = "当前还没有收到该参与节点的远端日志样本。" + if not enabled: + missing_reason_code = "disabled" + missing_reason = "远端日志回传当前关闭。" + elif line_count <= 0: + missing_reason_code = "waiting_sample" + missing_reason = "日志回传已开启，但当前现场样本尚未形成。" + missing_participating_node_summaries.append( + { + "node_code": node_code, + "region": str(node_meta.get("region") or "").strip(), + "role": str(node_meta.get("role") or "").strip(), + "status": str(node_meta.get("status") or "").strip(), + "participation_state": str(node_meta.get("participation_state") or "").strip(), + "participation_bucket": str(node_meta.get("participation_bucket") or "").strip(), + "participation_label": str(node_meta.get("participation_label") or "").strip(), + "participation_reason": str(node_meta.get("participation_reason") or "").strip(), + "missing_reason_code": missing_reason_code, + "missing_reason": missing_reason, + } + ) + + if not enabled: + state = "disabled" + status_label = "已关闭" + status_type = "info" + description = "远端日志回传当前关闭，海外控制面不会持续收到大陆检测过程日志。" + recommended_action_label = "建议先开启关键回传" + elif not participating_node_codes: + state = "idle" + status_label = "空闲观察" + status_type = "success" + description = "当前没有参与检测的节点，日志回传已就绪，待现场出现时会自动开始镜像样本。" + recommended_action_label = "当前无需调整" + elif line_count <= 0: + state = "waiting_sample" + status_label = "等待样本" + status_type = "warning" + description = "日志回传已开启，但当前还没有收到参与节点的现场日志样本。" + recommended_action_label = "建议先抓 Worker 日志确认现场输出" + elif missing_participating_nodes: + state = "partial_coverage" + status_label = "部分覆盖" + status_type = "warning" + description = f"当前已有 {len(covered_participating_nodes)}/{len(participating_node_codes)} 台参与节点回传日志，仍有节点未被覆盖。" + recommended_action_label = "建议优先排查未回传节点" + elif mode == "full": + state = "full_capture" + status_label = "全量观察" + status_type = "success" + description = "参与节点已经全部被覆盖，当前处于全量日志观察模式，适合短时深度排障。" + recommended_action_label = "排障结束后可切回关键回传" + else: + state = "healthy" + status_label = "关键覆盖" + status_type = "success" + description = "参与节点已经全部被覆盖，当前处于关键日志观察模式，适合常态联调。" + recommended_action_label = "当前无需调整" + + return { + "enabled": enabled, + "mode": mode, + "mode_label": "全量回传" if enabled and mode == "full" else ("关键回传" if enabled else "已关闭"), + "state": state, + "status_label": status_label, + "status_type": status_type, + "description": description, + "recommended_action_label": recommended_action_label, + "line_count": line_count, + "source_node_count": int(normalized_log_sync.get("source_node_count", 0) or len(source_nodes)), + "source_nodes": source_nodes, + "source_node_summaries": source_node_summaries, + "last_at": str(normalized_log_sync.get("last_at") or ""), + "last_line": str(normalized_log_sync.get("last_line") or ""), + "preview_lines": preview_lines, + "preview_line_count": len(preview_lines), + "participating_node_count": len(participating_node_codes), + "covered_participating_node_count": len(covered_participating_nodes), + "missing_participating_node_count": len(missing_participating_nodes), + "covered_participating_nodes": covered_participating_nodes, + "missing_participating_nodes": missing_participating_nodes, + "covered_participating_node_summaries": covered_participating_node_summaries, + "missing_participating_node_summaries": missing_participating_node_summaries, + } + + +def _normalize_scene_log_mode(raw_mode: object, *, fallback: str = "key") -> str: + normalized_mode = str(raw_mode or "").strip().lower() + if normalized_mode == "full": + return "full" + if normalized_mode == "key": + return "key" + return "full" if str(fallback or "").strip().lower() == "full" else "key" + + +def _derive_scene_log_source_summary_from_records(node_code: str, records: list[dict]) -> dict: + normalized_node_code = str(node_code or "").strip() + summary = { + "node_code": normalized_node_code, + "line_count": 0, + "key_line_count": 0, + "full_line_count": 0, + "last_at": "", + "last_line": "", + } + for record in list(records or []): + record_node_code = str(record.get("node_code") or "").strip() + if normalized_node_code and record_node_code != normalized_node_code: + continue + summary["line_count"] += 1 + if str(record.get("mode") or "").strip() == "full": + summary["full_line_count"] += 1 + else: + summary["key_line_count"] += 1 + summary["last_at"] = str(record.get("created_at") or "").strip() + summary["last_line"] = str(record.get("line") or "").strip() + return summary + + +def _collect_scene_log_records( + active_job: dict | None, + *, + requested_mode: str, +) -> list[dict]: + normalized_active_job = dict(active_job or {}) + events = list(normalized_active_job.get("current_cycle_events") or normalized_active_job.get("recent_events") or []) + if not events: + return [] + + current_cycle_token = str(normalized_active_job.get("current_cycle_token") or "").strip() + normalized_requested_mode = _normalize_scene_log_mode(requested_mode) + records: list[dict] = [] + for event in reversed(events): + event_type = str(event.get("event_type") or "").strip() + if event_type != "worker_log": + continue + payload = event.get("payload") if isinstance(event.get("payload"), dict) else {} + event_cycle_token = str(payload.get("cycle_token") or "").strip() + if current_cycle_token and event_cycle_token and event_cycle_token != current_cycle_token: + continue + event_mode = _normalize_scene_log_mode(payload.get("log_mode") or "key") + if normalized_requested_mode != "full" and event_mode == "full": + continue + created_at = str(event.get("created_at") or "").strip() + node_code = str(event.get("node_code") or "").strip() or "unknown" + message = str(event.get("message") or "").strip() + if not message: + continue + records.append( + { + "created_at": created_at, + "node_code": node_code, + "message": message, + "mode": event_mode, + "cycle_token": event_cycle_token, + "line": f"[{created_at}] [{node_code}] {message}", + } + ) + return records + + +def _parse_ops_timestamp(value: object) -> datetime | None: + text = str(value or "").strip() + if not text: + return None + try: + return datetime.fromisoformat(text) + except ValueError: + return None + + +def _collect_scene_log_records_from_ops_jobs( + *, + requested_mode: str, + limit: int = 30, + max_age_hours: int = 6, +) -> list[dict]: + normalized_requested_mode = _normalize_scene_log_mode(requested_mode) + cutoff = datetime.now() - timedelta(hours=max(1, int(max_age_hours or 6))) + records: list[dict] = [] + worker_service_name = str(settings.worker_service_name or "domaincheck-worker").strip() or "domaincheck-worker" + jobs = list_ops_jobs(limit=max(1, min(int(limit or 30), 80)), compact=False) + for job in reversed(list(jobs or [])): + if str(job.get("action") or "").strip() != "logs.collect": + continue + if str(job.get("status") or "").strip() != "success": + continue + result = dict(job.get("result") or {}) + payload = dict(job.get("payload") or {}) + service_name = str(result.get("service_name") or payload.get("service_name") or "").strip() + if service_name != worker_service_name: + continue + node_code = str(job.get("target_node_code") or "").strip() + if not node_code: + continue + created_at = ( + str(job.get("finished_at") or "").strip() + or str(job.get("updated_at") or "").strip() + or str(job.get("created_at") or "").strip() + ) + created_dt = _parse_ops_timestamp(created_at) + if created_dt and created_dt < cutoff: + continue + output_text = str(result.get("stdout") or result.get("log_output") or "").strip() + if not output_text: + continue + output_lines = [str(item or "").strip() for item in output_text.splitlines() if str(item or "").strip()] + if not output_lines: + continue + sample_limit = 12 if normalized_requested_mode == "full" else 4 + for line in output_lines[-sample_limit:]: + records.append( + { + "created_at": created_at, + "node_code": node_code, + "message": line, + "mode": normalized_requested_mode, + "cycle_token": "", + "line": f"[{created_at}] [{node_code}] {line}", + } + ) + return records + + +def _merge_execution_scene_log_sync_sources(log_sync: dict | None, *, requested_mode: str) -> dict: + normalized_log_sync = dict(log_sync or {}) + ops_job_records = _collect_scene_log_records_from_ops_jobs( + requested_mode=requested_mode, + limit=30, + max_age_hours=6, + ) + if not ops_job_records: + return normalized_log_sync + + preview_lines = [ + str(item or "").strip() + for item in list(normalized_log_sync.get("preview_lines") or []) + if str(item or "").strip() + ] + merged_preview_lines = preview_lines + [str(item.get("line") or "").strip() for item in ops_job_records if str(item.get("line") or "").strip()] + merged_preview_lines = merged_preview_lines[-20:] + + source_nodes = [ + str(item or "").strip() + for item in list(normalized_log_sync.get("source_nodes") or []) + if str(item or "").strip() + ] + source_nodes.extend( + str(item.get("node_code") or "").strip() + for item in ops_job_records + if str(item.get("node_code") or "").strip() + ) + source_nodes = list(dict.fromkeys(source_nodes)) + + existing_summaries = {} + for item in list(normalized_log_sync.get("source_node_summaries") or []): + if not isinstance(item, dict): + continue + node_code = str(item.get("node_code") or "").strip() + if not node_code: + continue + existing_summaries[node_code] = dict(item) + for node_code in source_nodes: + derived_summary = _derive_scene_log_source_summary_from_records( + node_code, + [record for record in ops_job_records if str(record.get("node_code") or "").strip() == node_code], + ) + if existing_summaries.get(node_code): + current_summary = dict(existing_summaries[node_code]) + current_summary["line_count"] = int(current_summary.get("line_count", 0) or 0) + int(derived_summary.get("line_count", 0) or 0) + current_summary["key_line_count"] = int(current_summary.get("key_line_count", 0) or 0) + int(derived_summary.get("key_line_count", 0) or 0) + current_summary["full_line_count"] = int(current_summary.get("full_line_count", 0) or 0) + int(derived_summary.get("full_line_count", 0) or 0) + if str(derived_summary.get("last_at") or "").strip() >= str(current_summary.get("last_at") or "").strip(): + current_summary["last_at"] = str(derived_summary.get("last_at") or current_summary.get("last_at") or "").strip() + current_summary["last_line"] = str(derived_summary.get("last_line") or current_summary.get("last_line") or "").strip() + existing_summaries[node_code] = current_summary + elif int(derived_summary.get("line_count", 0) or 0) > 0: + existing_summaries[node_code] = derived_summary + + latest_record = ops_job_records[-1] + latest_at = str(latest_record.get("created_at") or "").strip() + latest_line = str(latest_record.get("line") or "").strip() + current_last_at = str(normalized_log_sync.get("last_at") or "").strip() + if current_last_at and current_last_at > latest_at: + latest_at = current_last_at + latest_line = str(normalized_log_sync.get("last_line") or "").strip() + + merged_line_count = max( + int(normalized_log_sync.get("line_count", 0) or 0), + len(merged_preview_lines), + sum(int(item.get("line_count", 0) or 0) for item in existing_summaries.values()), + ) + + return { + **normalized_log_sync, + "preview_lines": merged_preview_lines, + "line_count": merged_line_count, + "source_nodes": source_nodes, + "source_node_count": max(int(normalized_log_sync.get("source_node_count", 0) or 0), len(source_nodes)), + "source_node_summaries": list(existing_summaries.values()), + "last_at": latest_at, + "last_line": latest_line, + } + + +def _collect_scene_log_records_from_preview_lines( + preview_lines: list[str] | None, + *, + requested_mode: str, +) -> list[dict]: + normalized_requested_mode = _normalize_scene_log_mode(requested_mode) + records: list[dict] = [] + for raw_line in list(preview_lines or []): + line = str(raw_line or "").strip() + if not line: + continue + match = _REMOTE_LOG_PREVIEW_LINE_RE.match(line) + if not match: + continue + created_at = str(match.group("created_at") or "").strip() + node_code = str(match.group("node_code") or "").strip() or "unknown" + message = str(match.group("message") or "").strip() + if not message: + continue + record_mode = "full" if normalized_requested_mode == "full" else "key" + records.append( + { + "created_at": created_at, + "node_code": node_code, + "message": message, + "mode": record_mode, + "cycle_token": "", + "line": line, + } + ) + return records + + +def _scene_log_status_meta(status: str, *, log_sync_enabled: bool) -> tuple[str, str]: + normalized_status = str(status or "").strip() + mapping = { + "healthy": ("关键覆盖", "success"), + "full_capture": ("全量观察", "success"), + "historical_sample": ("历史样本", "success"), + "waiting_sample": ("等待样本", "warning"), + "missing_sample": ("缺少样本", "warning"), + "disabled": ("已关闭", "info"), + "standby": ("在线待命", "info"), + "unknown": ("未发现", "info"), + } + if normalized_status in mapping: + return mapping[normalized_status] + if not log_sync_enabled: + return ("已关闭", "info") + return ("待确认", "info") + + +def get_ops_node_scene_log(node_code: str, *, limit: int = 80, mode: str = "") -> dict: + normalized_node_code = str(node_code or "").strip() + if not normalized_node_code: + return {} + + runtime = get_runtime_status() + detect = dict(runtime.get("detect") or {}) + cluster_nodes = [ + dict(item) + for item in list((runtime.get("cluster") or {}).get("nodes") or []) + if str(item.get("node_code") or "").strip() + ] + execution_scene = _build_ops_execution_scene(detect) + log_sync = dict(execution_scene.get("log_sync") or {}) + default_mode = _normalize_scene_log_mode(log_sync.get("mode") or "key") + selected_mode = _normalize_scene_log_mode(mode, fallback=default_mode) + active_job = dict(detect.get("active_job") or {}) + records = _collect_scene_log_records(active_job, requested_mode=selected_mode) + if not records: + records = _collect_scene_log_records_from_preview_lines( + list(log_sync.get("preview_lines") or []), + requested_mode=selected_mode, + ) + node_records = [ + dict(record) + for record in records + if str(record.get("node_code") or "").strip() == normalized_node_code + ] + normalized_limit = max(1, min(int(limit or 80), 400)) + visible_records = node_records[-normalized_limit:] + + participating_map = { + str(item.get("node_code") or "").strip(): dict(item) + for item in list(execution_scene.get("participating_nodes") or []) + if str(item.get("node_code") or "").strip() + } + standby_map = { + str(item.get("node_code") or "").strip(): dict(item) + for item in list(execution_scene.get("standby_nodes") or []) + if str(item.get("node_code") or "").strip() + } + cluster_map = { + str(item.get("node_code") or "").strip(): dict(item) + for item in cluster_nodes + if str(item.get("node_code") or "").strip() + } + source_summary_map = { + str(item.get("node_code") or "").strip(): dict(item) + for item in list(log_sync.get("source_node_summaries") or []) + if str(item.get("node_code") or "").strip() + } + missing_summary_map = { + str(item.get("node_code") or "").strip(): dict(item) + for item in list(log_sync.get("missing_participating_node_summaries") or []) + if str(item.get("node_code") or "").strip() + } + covered_summary_map = { + str(item.get("node_code") or "").strip(): dict(item) + for item in list(log_sync.get("covered_participating_node_summaries") or []) + if str(item.get("node_code") or "").strip() + } + + node_meta = dict(participating_map.get(normalized_node_code) or standby_map.get(normalized_node_code) or {}) + cluster_meta = dict(cluster_map.get(normalized_node_code) or {}) + source_summary = dict(source_summary_map.get(normalized_node_code) or covered_summary_map.get(normalized_node_code) or {}) + derived_source_summary = _derive_scene_log_source_summary_from_records(normalized_node_code, node_records) + source_summary = { + "node_code": normalized_node_code, + "line_count": max( + int(source_summary.get("line_count", 0) or 0), + int(derived_source_summary.get("line_count", 0) or 0), + ), + "key_line_count": max( + int(source_summary.get("key_line_count", 0) or 0), + int(derived_source_summary.get("key_line_count", 0) or 0), + ), + "full_line_count": max( + int(source_summary.get("full_line_count", 0) or 0), + int(derived_source_summary.get("full_line_count", 0) or 0), + ), + "last_at": str( + derived_source_summary.get("last_at") + or source_summary.get("last_at") + or "" + ).strip(), + "last_line": str( + derived_source_summary.get("last_line") + or source_summary.get("last_line") + or "" + ).strip(), + } + missing_summary = dict(missing_summary_map.get(normalized_node_code) or {}) + + record_available = bool(source_summary.get("line_count", 0) or node_records) + detect_participating = normalized_node_code in participating_map + standby_visible = normalized_node_code in standby_map + log_sync_enabled = bool(log_sync.get("enabled", False)) + + if record_available: + if detect_participating: + status = "full_capture" if selected_mode == "full" else "healthy" + elif standby_visible: + status = "historical_sample" + else: + status = "historical_sample" + elif missing_summary: + missing_reason_code = str(missing_summary.get("missing_reason_code") or "").strip() + if missing_reason_code == "disabled": + status = "disabled" + elif missing_reason_code == "waiting_sample": + status = "waiting_sample" + else: + status = "missing_sample" + elif detect_participating: + status = "waiting_sample" if log_sync_enabled else "disabled" + elif standby_visible: + status = "standby" + else: + status = "unknown" + + status_label, status_type = _scene_log_status_meta(status, log_sync_enabled=log_sync_enabled) + if record_available and detect_participating: + summary = f"节点当前正在参与检测，已保留 {int(source_summary.get('line_count', 0) or 0)} 条现场日志样本。" + elif record_available and standby_visible: + summary = f"节点当前在线待命，仍可查看最近保留的 {int(source_summary.get('line_count', 0) or 0)} 条现场日志样本。" + elif record_available: + summary = f"节点已保留 {int(source_summary.get('line_count', 0) or 0)} 条现场日志样本。" + elif missing_summary: + summary = str(missing_summary.get("missing_reason") or "").strip() or "当前还没有收到该节点的现场日志样本。" + elif detect_participating and log_sync_enabled: + summary = "节点正在参与检测，但当前现场日志样本仍未形成。" + elif detect_participating: + summary = "节点正在参与检测，但远端日志回传当前关闭。" + elif standby_visible: + summary = "节点当前在线但未参与本轮检测，暂时没有新的现场日志样本。" + else: + summary = "当前没有找到该节点的现场日志记录。" + + latest_record = dict(visible_records[-1] or {}) if visible_records else {} + participation_payload = { + "detect_participating": detect_participating, + "participation_state": str(node_meta.get("participation_state") or "").strip(), + "participation_label": str(node_meta.get("participation_label") or "").strip(), + "participation_reason": str(node_meta.get("participation_reason") or "").strip(), + "participation_bucket": str(node_meta.get("participation_bucket") or "").strip(), + "participation_bucket_label": str(node_meta.get("participation_bucket_label") or "").strip(), + "is_dispatch_active": bool(node_meta.get("is_dispatch_active", False)), + } + + return { + "node_code": normalized_node_code, + "available": bool(node_meta or cluster_meta or source_summary.get("line_count", 0) or missing_summary or node_records), + "status": status, + "status_label": status_label, + "status_type": status_type, + "summary": summary, + "log_sync_enabled": log_sync_enabled, + "mode": selected_mode, + "mode_label": "全量回传" if selected_mode == "full" else "关键回传", + "records_total": len(node_records), + "records_visible": len(visible_records), + "records_truncated": len(node_records) > len(visible_records), + "records": visible_records, + "latest_record": latest_record, + "source_summary": source_summary, + "missing_reason_code": str(missing_summary.get("missing_reason_code") or "").strip(), + "missing_reason": str(missing_summary.get("missing_reason") or "").strip(), + "node": { + "node_code": normalized_node_code, + "region": str(node_meta.get("region") or cluster_meta.get("region") or missing_summary.get("region") or "").strip(), + "role": str(node_meta.get("role") or cluster_meta.get("role") or missing_summary.get("role") or "").strip(), + "status": str(node_meta.get("status") or cluster_meta.get("status") or missing_summary.get("status") or "").strip(), + "current_load": int(node_meta.get("current_load", cluster_meta.get("current_load", 0)) or 0), + "last_heartbeat_at": str( + node_meta.get("last_heartbeat_at") or cluster_meta.get("last_heartbeat_at") or "" + ).strip(), + }, + "participation": participation_payload, + "contract_navigation": _build_ops_contract_navigation( + ["ops_observability_contract", "ops_stack_diagnosis_contract"], + primary_contract_key="ops_observability_contract", + registry=get_ops_contract_registry(), + ), + } + + +def _build_ops_execution_scene(detect: dict) -> dict: + normalized_detect = dict(detect or {}) + active_job = dict(normalized_detect.get("active_job") or {}) + log_sync = _merge_execution_scene_log_sync_sources( + normalized_detect.get("log_sync") or {}, + requested_mode=str((normalized_detect.get("log_sync") or {}).get("mode") or "key"), + ) + participation_summary = normalized_detect.get("participation_summary") or {} + participating_nodes = [ + _slim_execution_scene_node(item) + for item in list(normalized_detect.get("participating_nodes") or []) + if str(item.get("node_code") or "").strip() + ] + standby_nodes = [ + _slim_execution_scene_node(item) + for item in list(normalized_detect.get("non_participating_nodes") or []) + if str(item.get("node_code") or "").strip() + ] + dispatch_active_nodes = [ + dict(item) + for item in participating_nodes + if str(item.get("participation_bucket") or "").strip() == "dispatch_active" + ] + recent_only_nodes = [ + dict(item) + for item in participating_nodes + if str(item.get("participation_bucket") or "").strip() == "recent_only" + ] + load_syncing_nodes = [ + dict(item) + for item in [*participating_nodes, *standby_nodes] + if str(item.get("participation_bucket") or "").strip() == "load_syncing" + ] + counts = { + "dispatch_active": int(participation_summary.get("dispatch_active_nodes", len(dispatch_active_nodes)) or len(dispatch_active_nodes)), + "recent_only": int(participation_summary.get("recent_only_nodes", len(recent_only_nodes)) or len(recent_only_nodes)), + "standby": int(participation_summary.get("standby_nodes", 0) or 0), + "load_syncing": int(participation_summary.get("load_syncing_nodes", len(load_syncing_nodes)) or len(load_syncing_nodes)), + } + latest_event = dict(active_job.get("latest_cycle_event") or active_job.get("latest_event") or {}) + latest_at_candidates = [ + str(latest_event.get("created_at") or "").strip(), + str(active_job.get("started_at") or "").strip(), + str(active_job.get("created_at") or "").strip(), + str((log_sync or {}).get("last_at") or "").strip(), + ] + latest_at_candidates.extend( + str(item.get("last_heartbeat_at") or "").strip() + for item in [*participating_nodes, *standby_nodes] + if str(item.get("last_heartbeat_at") or "").strip() + ) + non_empty_latest_at_candidates = [candidate for candidate in latest_at_candidates if candidate] + latest_at = sorted(non_empty_latest_at_candidates)[-1] if non_empty_latest_at_candidates else "" + return { + "active_job_code": str((normalized_detect.get("active_job") or {}).get("job_code") or ""), + "phase_label": str(normalized_detect.get("phase_label") or ""), + "phase_detail": str(normalized_detect.get("phase_detail") or ""), + "recent_event": str(normalized_detect.get("recent_event") or ""), + "recent_warning": str(normalized_detect.get("recent_warning") or ""), + "latest_at": latest_at, + "summary": str(participation_summary.get("summary") or ""), + "counts": counts, + "log_sync": _build_execution_scene_log_sync(log_sync, participating_nodes), + "participation_summary": { + "effective_online_nodes": int(participation_summary.get("effective_online_nodes", 0) or 0), + "participating_nodes": int(participation_summary.get("participating_nodes", 0) or 0), + "dispatch_active_nodes": int(participation_summary.get("dispatch_active_nodes", 0) or 0), + "recent_only_nodes": int(participation_summary.get("recent_only_nodes", 0) or 0), + "non_participating_nodes": int(participation_summary.get("non_participating_nodes", 0) or 0), + "standby_nodes": int(participation_summary.get("standby_nodes", 0) or 0), + "load_syncing_nodes": int(participation_summary.get("load_syncing_nodes", 0) or 0), + "dedicated_worker_nodes": int(participation_summary.get("dedicated_worker_nodes", 0) or 0), + "controller_worker_nodes": int(participation_summary.get("controller_worker_nodes", 0) or 0), + "dispatch_active_node_codes": list(participation_summary.get("dispatch_active_node_codes") or []), + "recent_only_node_codes": list(participation_summary.get("recent_only_node_codes") or []), + "non_participating_node_codes": list(participation_summary.get("non_participating_node_codes") or []), + "standby_node_codes": list(participation_summary.get("standby_node_codes") or []), + "load_syncing_node_codes": list(participation_summary.get("load_syncing_node_codes") or []), + "summary": str(participation_summary.get("summary") or ""), + }, + "participating_nodes": participating_nodes, + "dispatch_active_nodes": dispatch_active_nodes, + "recent_only_nodes": recent_only_nodes, + "standby_nodes": standby_nodes, + "load_syncing_nodes": load_syncing_nodes, + } + + +def _should_emit_execution_scene_activity(execution_scene: dict) -> bool: + normalized_scene = dict(execution_scene or {}) + summary = dict(normalized_scene.get("participation_summary") or {}) + log_sync = dict(normalized_scene.get("log_sync") or {}) + return any( + [ + str(normalized_scene.get("active_job_code") or "").strip(), + str(normalized_scene.get("phase_label") or "").strip(), + str(normalized_scene.get("phase_detail") or "").strip(), + str(normalized_scene.get("recent_event") or "").strip(), + str(normalized_scene.get("recent_warning") or "").strip(), + str(normalized_scene.get("latest_at") or "").strip(), + int(summary.get("effective_online_nodes", 0) or 0) > 0, + bool(normalized_scene.get("participating_nodes")), + bool(normalized_scene.get("standby_nodes")), + bool(log_sync.get("enabled", False)), + int(log_sync.get("line_count", 0) or 0) > 0, + int(log_sync.get("source_node_count", 0) or 0) > 0, + ] + ) + + +def _should_emit_log_sync_activity(execution_scene: dict) -> bool: + log_sync = dict((execution_scene or {}).get("log_sync") or {}) + return any( + [ + bool(log_sync.get("enabled", False)), + int(log_sync.get("participating_node_count", 0) or 0) > 0, + int(log_sync.get("line_count", 0) or 0) > 0, + int(log_sync.get("source_node_count", 0) or 0) > 0, + int(log_sync.get("missing_participating_node_count", 0) or 0) > 0, + bool(log_sync.get("preview_lines")), + str(log_sync.get("last_at") or "").strip(), + ] + ) + + +def _build_execution_scene_activity(execution_scene: dict) -> dict: + normalized_scene = dict(execution_scene or {}) + summary = dict(normalized_scene.get("participation_summary") or {}) + participating_nodes = list(normalized_scene.get("participating_nodes") or []) + standby_nodes = list(normalized_scene.get("standby_nodes") or []) + dispatch_active_nodes = [ + item for item in participating_nodes + if bool(item.get("is_dispatch_active", False)) + ] + recent_only_nodes = [ + item for item in participating_nodes + if str(item.get("participation_state") or "").strip() == "recent_throughput" + ] + load_syncing_nodes = [ + item for item in standby_nodes + if str(item.get("participation_state") or "").strip() == "load_syncing" + ] + pure_standby_nodes = [ + item for item in standby_nodes + if str(item.get("participation_state") or "").strip() == "standby" + ] + phase_label = str(normalized_scene.get("phase_label") or "").strip() + summary_text = ( + str(summary.get("summary") or "").strip() + or "当前还没有形成统一的执行现场摘要。" + ) + status = "ready" + if dispatch_active_nodes or recent_only_nodes: + status = "running" + elif load_syncing_nodes: + status = "attention" + meta_parts = [ + f"执行/已领 {len(dispatch_active_nodes)}", + f"近窗 {len(recent_only_nodes)}", + f"待命 {len(pure_standby_nodes)}", + f"待确认 {len(load_syncing_nodes)}", + ] + if phase_label: + meta_parts.append(f"阶段 {phase_label}") + scene_node_codes = _normalize_driver_node_codes( + [str(item.get("node_code") or "").strip() for item in [*participating_nodes, *standby_nodes]] + ) + return { + "kind": "execution_scene", + "activity_key": "execution-scene:live", + "title": "当前执行现场", + "subtitle": phase_label, + "summary": _truncate_activity_text(summary_text), + "meta_text": " / ".join(part for part in meta_parts if part), + "status": status, + "execution_mode": "", + "execution_mode_label": "", + "occurred_at": _activity_time( + normalized_scene.get("latest_at"), + *[item.get("last_heartbeat_at") for item in participating_nodes], + *[item.get("last_heartbeat_at") for item in standby_nodes], + ), + "target_node_codes": scene_node_codes, + "ui_intent": _build_driver_ui_intent( + "focus_execution_scene", + open_log_drawer=False, + ), + } + + +def _build_log_sync_activity(execution_scene: dict) -> dict: + normalized_scene = dict(execution_scene or {}) + log_sync = dict(normalized_scene.get("log_sync") or {}) + enabled = bool(log_sync.get("enabled", False)) + participating_node_count = int(log_sync.get("participating_node_count", 0) or 0) + covered_node_count = int(log_sync.get("covered_participating_node_count", 0) or 0) + missing_node_count = int(log_sync.get("missing_participating_node_count", 0) or 0) + line_count = int(log_sync.get("line_count", 0) or 0) + source_node_count = int(log_sync.get("source_node_count", 0) or 0) + missing_nodes = [ + str(item or "").strip() + for item in list(log_sync.get("missing_participating_nodes") or []) + if str(item or "").strip() + ] + status = "ready" + if participating_node_count > 0 and not enabled: + status = "attention" + elif participating_node_count > 0 and (line_count <= 0 or missing_node_count > 0): + status = "attention" + elif participating_node_count > 0: + status = "running" + description = str(log_sync.get("description") or "").strip() or "远端日志回传当前暂无补充说明。" + if missing_nodes: + description = f"{description} 当前仍有未覆盖节点：{'、'.join(missing_nodes)}" + observed_node_codes = _normalize_driver_node_codes( + [*list(log_sync.get("source_nodes") or []), *missing_nodes] + ) + return { + "kind": "log_sync", + "activity_key": "execution-scene:log-sync", + "title": "远端日志回传", + "subtitle": str(log_sync.get("mode_label") or "").strip(), + "summary": _truncate_activity_text(description), + "meta_text": " / ".join( + part + for part in [ + f"参与覆盖 {covered_node_count}/{participating_node_count}", + f"样本 {line_count}", + f"来源 {source_node_count}", + f"最近 {str(log_sync.get('last_at') or '').strip()}" if str(log_sync.get("last_at") or "").strip() else "", + ] + if part + ), + "status": status, + "execution_mode": "control-plane", + "execution_mode_label": execution_mode_label("control-plane"), + "occurred_at": _activity_time( + log_sync.get("last_at"), + normalized_scene.get("latest_at"), + ), + "target_node_codes": observed_node_codes, + "ui_intent": _build_driver_ui_intent( + "focus_execution_scene", + open_log_drawer=bool(list(log_sync.get("preview_lines") or [])), + ), + } + + +def _default_rollout_target_nodes(cluster_nodes: list[dict]) -> list[dict]: + rows: list[dict] = [] + for node in list(cluster_nodes or []): + node_code = str(node.get("node_code") or "").strip() + cluster_status = str(node.get("status") or "").strip() + if not node_code: + continue + if not bool(node.get("is_effective_worker", False)): + continue + if cluster_status not in {"online", "busy"}: + continue + rows.append( + { + "node_code": node_code, + "region": str(node.get("region") or ""), + "role": str(node.get("role") or ""), + "status": cluster_status, + "current_load": int(node.get("current_load", 0) or 0), + "is_effective_worker": True, + } + ) + return rows + + +def get_ops_activity_stream( + *, + limit: int = _OPS_ACTIVITY_FETCH_LIMIT, + scan_limit: int | None = None, + kind: str = "", + status: str = "", + execution_mode: str = "", + query: str = "", +) -> dict: + safe_limit = min(max(int(limit or _OPS_ACTIVITY_FETCH_LIMIT), 1), 100) + safe_scan_limit = min(max(int(scan_limit or (safe_limit * 4)), safe_limit), 400) + normalized_kind = str(kind or "").strip() + normalized_status = str(status or "").strip() + normalized_execution_mode = str(execution_mode or "").strip() + normalized_query = str(query or "").strip() + runtime_status = get_runtime_status() + execution_scene = _build_ops_execution_scene(runtime_status.get("detect") or {}) + managed_nodes_payload = list_managed_nodes_with_agent_state() + managed_node_map = { + str(item.get("node_code") or "").strip(): dict(item or {}) + for item in list(managed_nodes_payload.get("nodes") or []) + if str(item.get("node_code") or "").strip() + } + + playbook_runs_payload = get_recent_ops_playbook_runs(limit=safe_scan_limit, scan_limit=max(240, safe_scan_limit * 12)) + playbook_runs = list(playbook_runs_payload.get("runs") or []) + playbook_items = [_build_playbook_run_activity(run) for run in playbook_runs if str(run.get("run_code") or "").strip()] + + jobs = list_ops_jobs(limit=safe_scan_limit, compact=False) + standalone_jobs = [ + job + for job in jobs + if not _is_playbook_child_job(job) + and int(job.get("rollout_id") or 0) <= 0 + ] + job_items = [ + _build_ops_job_activity( + job, + managed_node=managed_node_map.get(str(job.get("target_node_code") or "").strip(), {}), + ) + for job in standalone_jobs + if int(job.get("id") or 0) > 0 + ] + + rollouts = list_release_rollouts(limit=safe_scan_limit) + rollout_items = [_build_rollout_activity(rollout) for rollout in rollouts if int(rollout.get("id") or 0) > 0] + + runbook = get_ops_runbook() + runbook_items = [ + _build_runbook_sequence_activity(sequence) + for sequence in list(runbook.get("control_sequences") or []) + if str((sequence or {}).get("key") or "").strip() + ] + runbook_items = [item for item in runbook_items if item] + + scene_items: list[dict] = [] + if _should_emit_execution_scene_activity(execution_scene): + scene_items.append(_build_execution_scene_activity(execution_scene)) + if _should_emit_log_sync_activity(execution_scene): + scene_items.append(_build_log_sync_activity(execution_scene)) + + all_items = [*scene_items, *playbook_items, *job_items, *rollout_items, *runbook_items] + all_items = [_finalize_activity_item(item) for item in all_items if item] + all_items.sort(key=lambda item: (str(item.get("occurred_at") or ""), str(item.get("activity_key") or "")), reverse=True) + all_items.sort( + key=lambda item: 1 if str(item.get("kind") or "").strip() == "runbook_sequence" else 0 + ) + + available_kind_counts: dict[str, int] = {} + available_status_counts: dict[str, int] = {} + available_execution_mode_counts: dict[str, int] = {} + for item in all_items: + item_kind = str(item.get("kind") or "").strip() or "unknown" + item_status = str(item.get("status") or "").strip() or "unknown" + item_execution_mode = str(item.get("execution_mode") or "").strip() + available_kind_counts[item_kind] = int(available_kind_counts.get(item_kind, 0) or 0) + 1 + available_status_counts[item_status] = int(available_status_counts.get(item_status, 0) or 0) + 1 + if item_execution_mode: + available_execution_mode_counts[item_execution_mode] = ( + int(available_execution_mode_counts.get(item_execution_mode, 0) or 0) + 1 + ) + + filtered_items = [ + item + for item in all_items + if (not normalized_kind or str(item.get("kind") or "").strip() == normalized_kind) + and (not normalized_status or str(item.get("status") or "").strip() == normalized_status) + and (not normalized_execution_mode or str(item.get("execution_mode") or "").strip() == normalized_execution_mode) + and _matches_activity_query(item, normalized_query) + ] + + items = filtered_items[:safe_limit] + summary_status, summary_status_label, summary_text = _activity_stream_summary_state(filtered_items) + + kind_counts: dict[str, int] = {} + status_counts: dict[str, int] = {} + for item in items: + kind = str(item.get("kind") or "").strip() or "unknown" + status = str(item.get("status") or "").strip() or "unknown" + kind_counts[kind] = int(kind_counts.get(kind, 0) or 0) + 1 + status_counts[status] = int(status_counts.get(status, 0) or 0) + 1 + + summary_contract_keys = _normalize_ops_contract_keys( + ["ops_observability_contract", "ops_stack_diagnosis_contract"] + + [ + contract_key + for item in filtered_items + for contract_key in list((item or {}).get("contract_keys") or []) + ] + ) + + return { + "items": items, + "contract_navigation": _build_ops_contract_navigation( + summary_contract_keys, + primary_contract_key="ops_observability_contract", + registry=get_ops_contract_registry(), + ), + "summary": { + "status": summary_status, + "status_label": summary_status_label, + "summary_text": summary_text, + "total": len(items), + "filtered_total": len(filtered_items), + "unfiltered_total": len(all_items), + "kind_counts": kind_counts, + "status_counts": status_counts, + "available_kind_counts": available_kind_counts, + "available_status_counts": available_status_counts, + "available_execution_mode_counts": available_execution_mode_counts, + "latest_at": str(items[0].get("occurred_at") or "") if items else "", + "limit": safe_limit, + "scan_limit": safe_scan_limit, + "filters": { + "kind": normalized_kind, + "status": normalized_status, + "execution_mode": normalized_execution_mode, + "query": normalized_query, + }, + }, + } + + +def _build_driver_recommendations( + *, + managed_nodes: list[dict], + execution_scene: dict, + release_summary: dict, + release_gate: dict, + release_launchpad: dict, +) -> list[dict]: + effective_worker_nodes = [ + item + for item in list(managed_nodes or []) + if bool(item.get("cluster_is_effective_worker", False)) + and str(item.get("cluster_status") or "").strip() in {"online", "busy"} + ] + handover_gap_nodes = [ + item + for item in effective_worker_nodes + if not (bool(item.get("is_managed", False)) and bool(item.get("is_enabled", False)) and bool(item.get("is_agent_online", False))) + ] + delivery_dead_letter_nodes = [ + item + for item in list(managed_nodes or []) + if str(item.get("delivery_queue_state") or "").strip() == "dead_letter" + and bool(item.get("is_managed", False)) + and bool(item.get("is_enabled", False)) + ] + delivery_retry_nodes = [ + item + for item in list(managed_nodes or []) + if str(item.get("delivery_queue_state") or "").strip() == "retrying" + and bool(item.get("is_managed", False)) + and bool(item.get("is_enabled", False)) + ] + participating_nodes = [item for item in list(execution_scene.get("participating_nodes") or []) if str(item.get("node_code") or "").strip()] + standby_nodes = [item for item in list(execution_scene.get("standby_nodes") or []) if str(item.get("node_code") or "").strip()] + participating_node_codes = [str(item.get("node_code") or "").strip() for item in participating_nodes if str(item.get("node_code") or "").strip()] + standby_node_codes = [str(item.get("node_code") or "").strip() for item in standby_nodes if str(item.get("node_code") or "").strip()] + release_launchpad_card = _build_release_launchpad_driver_card(release_launchpad=release_launchpad) + launchpad_gap_action = _release_launchpad_gap_action_context(release_launchpad) + log_sync = dict(execution_scene.get("log_sync") or {}) + log_sync_enabled = bool(log_sync.get("enabled", False)) + remote_log_sample_count = int(log_sync.get("line_count", 0) or 0) + remote_log_source_count = int(log_sync.get("source_node_count", 0) or 0) + release_label = _preferred_release_label(release_summary) + preferred_release = dict(release_gate.get("release") or {}) + release_gate_status = str(release_gate.get("status") or "").strip() + release_gate_execution_mode = str(release_gate.get("execution_mode") or "remote-agent").strip() or "remote-agent" + release_gate_execution_mode_label = ( + str(release_gate.get("execution_mode_label") or "").strip() or execution_mode_label(release_gate_execution_mode) + ) + release_gate_summary = str(release_gate.get("summary") or "").strip() + release_gate_blocking_reasons = [str(item).strip() for item in list(release_gate.get("blocking_reasons") or []) if str(item).strip()] + release_gate_warning_reasons = [str(item).strip() for item in list(release_gate.get("warning_reasons") or []) if str(item).strip()] + release_gate_rows = list(((release_gate.get("operational_readiness") or {}).get("rows") or [])) + not_execution_ready_rows = [row for row in release_gate_rows if not bool(row.get("execution_ready", False))] + inspection_problem_rows = [ + row + for row in release_gate_rows + if str(row.get("inspection_status") or "").strip() in {"running", "attention", "missing"} + ] + playbook_runs_payload = get_recent_ops_playbook_runs(limit=6, scan_limit=240) + recent_playbook_runs = list(playbook_runs_payload.get("runs") or []) + attention_playbook_runs = [item for item in recent_playbook_runs if str(item.get("status") or "").strip() == "attention"] + active_playbook_runs = [item for item in recent_playbook_runs if str(item.get("status") or "").strip() == "running"] + activity_stream_payload = get_ops_activity_stream(limit=10, scan_limit=80) + recent_activity_items = list(activity_stream_payload.get("items") or []) + start_delivery_failed_items = [ + dict(item or {}) + for item in recent_activity_items + if str(item.get("start_delivery_state") or "").strip() == "failed_local" + ] + activity_focus_level, activity_focus_item = _pick_driver_activity_focus(recent_activity_items) + + cards: list[dict] = [] + + if attention_playbook_runs: + first_attention_run = dict(attention_playbook_runs[0] or {}) + run_code = str(first_attention_run.get("run_code") or "").strip() + focus_step_title = str(first_attention_run.get("focus_step_title") or "").strip() + cards.append( + { + "key": f"playbook-run-attention-{run_code or 'latest'}", + "title": "先处理异常编排", + "level_label": "最高优先", + "tag_type": "danger", + "summary": ( + f"{str(first_attention_run.get('playbook_title') or first_attention_run.get('playbook_key') or '标准编排').strip()}" + f"{' / ' + focus_step_title if focus_step_title else ''} 当前出现异常，建议先看整轮事件流。" + ), + "reason": str(first_attention_run.get("focus_summary") or "当前编排存在失败、阻断或取消，先处理这轮编排比继续堆动作更重要。").strip(), + "node_codes": _normalize_driver_node_codes(first_attention_run.get("target_node_codes") or []), + "meta_text": run_code, + "primary_label": "查看编排详情", + "secondary_label": "最近事件", + "primary_type": "danger", + "disabled": not run_code, + "primary_action_code": "focus_playbook_run", + "secondary_action_code": "open_playbook_run_latest_events", + "action_payload": { + "run_code": run_code, + "focus_step_key": str(first_attention_run.get("focus_step_key") or "").strip(), + "focus_step_title": focus_step_title, + }, + } + ) + elif active_playbook_runs: + first_active_run = dict(active_playbook_runs[0] or {}) + run_code = str(first_active_run.get("run_code") or "").strip() + focus_step_title = str(first_active_run.get("focus_step_title") or "").strip() + cards.append( + { + "key": f"playbook-run-running-{run_code or 'latest'}", + "title": "盯住收口中的编排", + "level_label": "推荐", + "tag_type": "warning", + "summary": ( + f"{str(first_active_run.get('playbook_title') or first_active_run.get('playbook_key') or '标准编排').strip()}" + f"{' / ' + focus_step_title if focus_step_title else ''} 仍在收口中，建议先看整轮事件流再决定是否补动作。" + ), + "reason": str(first_active_run.get("focus_summary") or "当前编排还在执行或排队，先看整轮回执和事件流最容易判断是否真卡住。").strip(), + "node_codes": _normalize_driver_node_codes(first_active_run.get("target_node_codes") or []), + "meta_text": run_code, + "primary_label": "查看编排详情", + "secondary_label": "最近事件", + "primary_type": "warning", + "disabled": not run_code, + "primary_action_code": "focus_playbook_run", + "secondary_action_code": "open_playbook_run_latest_events", + "action_payload": { + "run_code": run_code, + "focus_step_key": str(first_active_run.get("focus_step_key") or "").strip(), + "focus_step_title": focus_step_title, + }, + } + ) + + if activity_focus_level and activity_focus_item: + cards.append(_build_driver_activity_card(activity_focus_level, activity_focus_item)) + + if start_delivery_failed_items: + first_delivery_gap_item = dict(start_delivery_failed_items[0] or {}) + source_focus_ref = _normalize_focus_ref( + first_delivery_gap_item.get("source_focus_ref") or first_delivery_gap_item.get("focus_ref") + ) + cards.append( + { + "key": "activity-start-delivery-gap", + "title": "先检查开始回执异常", + "level_label": "推荐", + "tag_type": "warning", + "summary": ( + f"最近有 {len(start_delivery_failed_items)} 条任务已经在节点侧开始执行，但开始回执没能成功送达控制面。" + ), + "reason": str(first_delivery_gap_item.get("summary") or "").strip() + or "这通常不代表节点没执行，而是控制面和节点现场之间已经出现开始态失联，建议先看任务事件流。", + "node_codes": _normalize_driver_node_codes( + [ + str(source_focus_ref.get("target_node_code") or "").strip(), + *list(first_delivery_gap_item.get("target_node_codes") or []), + ] + ), + "meta_text": str(first_delivery_gap_item.get("job_code") or "").strip(), + "primary_label": "查看任务事件", + "secondary_label": "", + "primary_type": "warning", + "disabled": False, + "primary_action_code": "focus_latest_job_events", + "secondary_action_code": "", + "primary_action_payload": { + "focus_ref": source_focus_ref, + }, + "focus_ref": source_focus_ref, + } + ) + + if handover_gap_nodes: + first_gap_node = handover_gap_nodes[0] + first_gap_node_code = str(first_gap_node.get("node_code") or "").strip() + launchpad_gap_action_code = str(launchpad_gap_action.get("action_code") or "").strip() + launchpad_gap_node_code = str(launchpad_gap_action.get("node_code") or "").strip() + primary_action_code = "handover_first_gap" + primary_label = "为首台补签 Token" if bool(first_gap_node.get("is_managed", False)) else "纳管首台节点" + summary = f"当前有 {len(handover_gap_nodes)} 台有效执行节点还没进入标准执行器就绪状态，继续扩机器前要先把接管链路补齐。" + reason = "节点虽然在线且属于有效执行面，SSH 已可承担日志、诊断和部分服务控制，但标准巡检编排、正式发布与 Rollout 仍以 Agent 在线为准。" + level_label = "最高优先" + tag_type = "danger" + key = "handover-gap" + if launchpad_gap_action_code in {"bootstrap_run", "run_acceptance"} and ( + not launchpad_gap_node_code or launchpad_gap_node_code == first_gap_node_code + ): + primary_action_code = launchpad_gap_action_code + primary_label = "跑接入收口" if launchpad_gap_action_code == "bootstrap_run" else "跑接管验收" + summary = ( + f"当前有 {len(handover_gap_nodes)} 台有效执行节点还没进入标准执行器就绪状态，" + "并且 Launchpad 已经给出首台节点的标准收口动作。" + ) + reason = str(launchpad_gap_action.get("summary") or "").strip() or reason + level_label = "最高优先" if launchpad_gap_action_code == "bootstrap_run" else "推荐" + tag_type = "warning" if launchpad_gap_action_code == "bootstrap_run" else "success" + key = f"handover-gap-{launchpad_gap_action_code}" + cards.append( + { + "key": key, + "title": "先补接管缺口", + "level_label": level_label, + "tag_type": tag_type, + "summary": summary, + "reason": reason, + "node_codes": [str(item.get("node_code") or "").strip() for item in handover_gap_nodes if str(item.get("node_code") or "").strip()], + "primary_label": primary_label, + "secondary_label": "查看目标节点", + "primary_type": "danger", + "disabled": not first_gap_node_code, + "primary_action_code": primary_action_code, + "secondary_action_code": "view_first_gap", + "primary_action_payload": ( + {"node_code": first_gap_node_code} + if primary_action_code in {"bootstrap_run", "run_acceptance"} + else {} + ), + } + ) + + if delivery_dead_letter_nodes: + first_dead_letter_node = dict(delivery_dead_letter_nodes[0] or {}) + affected_node_codes = [ + str(item.get("node_code") or "").strip() + for item in delivery_dead_letter_nodes + if str(item.get("node_code") or "").strip() + ] + cards.append( + { + "key": "node-agent-dead-letter", + "title": "先处理 Agent 死信", + "level_label": "最高优先", + "tag_type": "danger", + "summary": ( + f"当前有 {len(delivery_dead_letter_nodes)} 台托管节点存在 Node Agent 死信记录，" + "说明部分任务回执或事件已经进入人工介入区。" + ), + "reason": str(first_dead_letter_node.get("delivery_queue_reason") or "Node Agent 回执队列已经出现死信，继续堆动作会放大控制面与节点现场的不一致。").strip(), + "node_codes": affected_node_codes, + "meta_text": _format_node_code_list(affected_node_codes), + "primary_label": "重放死信队列", + "secondary_label": "查看 Worker 日志", + "primary_type": "danger", + "disabled": not affected_node_codes, + "primary_action_code": "replay_delivery_queue", + "secondary_action_code": "open_worker_logs", + "primary_action_payload": { + "template_key": "delivery.queue.replay", + "target_node_codes": affected_node_codes, + "execution_mode": "remote-agent", + "auto_approve": False, + "payload": { + "limit": min(max(len(affected_node_codes), 1), 20), + "flush_after_replay": True, + "reason": "", + }, + }, + } + ) + elif delivery_retry_nodes: + affected_node_codes = [ + str(item.get("node_code") or "").strip() + for item in delivery_retry_nodes + if str(item.get("node_code") or "").strip() + ] + first_retry_node = dict(delivery_retry_nodes[0] or {}) + cards.append( + { + "key": "node-agent-retrying", + "title": "关注 Agent 回执积压", + "level_label": "推荐", + "tag_type": "warning", + "summary": ( + f"当前有 {len(delivery_retry_nodes)} 台托管节点存在待重试回执，" + "控制面状态可能会短暂落后于节点现场。" + ), + "reason": str(first_retry_node.get("delivery_queue_reason") or "Node Agent 正在自动回放积压的完成回执或事件，建议先观察日志再继续堆新的运维动作。").strip(), + "node_codes": affected_node_codes, + "meta_text": _format_node_code_list(affected_node_codes), + "primary_label": "冲刷回执队列", + "secondary_label": "查看 Worker 日志", + "primary_type": "warning", + "disabled": not affected_node_codes, + "primary_action_code": "flush_delivery_queue", + "secondary_action_code": "open_worker_logs", + "primary_action_payload": { + "template_key": "delivery.queue.flush", + "target_node_codes": affected_node_codes, + "execution_mode": "remote-agent", + "auto_approve": True, + "payload": { + "limit": min(max(len(affected_node_codes) * 5, 1), 20), + }, + }, + } + ) + + preferred_release_id = int(preferred_release.get("id") or 0) + preferred_release_version = str(preferred_release.get("release_version") or "").strip() or "-" + preferred_release_channel = str(preferred_release.get("channel") or "").strip() or "stable" + preferred_release_label = f"{preferred_release_version} / {preferred_release_channel}" + + if release_launchpad_card: + cards.append(release_launchpad_card) + elif preferred_release_id <= 0: + cards.append( + { + "key": "release-missing", + "title": "先建立 Release", + "level_label": "待补齐", + "tag_type": "info", + "summary": "当前还没有默认 Release，正式 Rollout 入口尚未建立。", + "reason": "没有 Release 时，节点更新仍然容易回到临时 git pull、手工覆盖或口头版本约定。", + "node_codes": [], + "meta_text": "", + "primary_label": "创建 Release", + "secondary_label": "", + "primary_type": "primary", + "disabled": False, + "primary_action_code": "open_release_dialog", + "secondary_action_code": "", + } + ) + elif release_gate_status in {"release_not_ready", "artifact_missing"}: + cards.append( + { + "key": "release-finish-metadata", + "title": "先收口默认 Release", + "level_label": "最高优先" if release_gate_status == "artifact_missing" else "推荐", + "tag_type": "danger" if release_gate_status == "artifact_missing" else "warning", + "summary": release_gate_summary or f"默认 Release {preferred_release_label} 还没有进入可稳定 Rollout 的状态。", + "reason": "先把 Release 状态、制品地址和版本信息补完整，Rollout 才不会重新退回手工发布路径。", + "node_codes": [], + "meta_text": preferred_release_label, + "primary_label": "查看版本区", + "secondary_label": "创建 Release", + "primary_type": "warning", + "disabled": False, + "primary_action_code": "focus_release_hub", + "secondary_action_code": "open_release_dialog", + } + ) + elif release_gate_status == "blocked": + blocker_node_codes = [ + str(item.get("node_code") or "").strip() + for item in (not_execution_ready_rows or inspection_problem_rows) + if str(item.get("node_code") or "").strip() + ] + first_gap_row = dict(not_execution_ready_rows[0] or {}) + first_gap_is_managed = bool(first_gap_row.get("is_managed", False)) + first_gap_code = str(first_gap_row.get("node_code") or "").strip() + cards.append( + { + "key": "release-rollout-blocked", + "title": "先补 Rollout 前置条件", + "level_label": "最高优先", + "tag_type": "danger", + "summary": release_gate_summary or f"默认 Release {preferred_release_label} 当前仍存在 Rollout 阻断。", + "reason": ( + release_gate_blocking_reasons[0] + if release_gate_blocking_reasons + else f"默认目标节点里仍有节点未通过 {release_gate_execution_mode_label}、巡检或接管门禁。" + ), + "node_codes": blocker_node_codes, + "meta_text": preferred_release_label, + "primary_label": ( + "纳管首台节点" + if first_gap_code and not first_gap_is_managed + else ("查看首台缺口" if first_gap_code else "执行标准巡检") + ), + "secondary_label": "查看版本区", + "primary_type": "danger", + "disabled": not blocker_node_codes, + "primary_action_code": ( + "handover_first_gap" + if first_gap_code and not first_gap_is_managed + else ("view_first_gap" if first_gap_code else "run_standard_inspection") + ), + "secondary_action_code": "focus_release_hub", + } + ) + elif release_gate_status == "attention": + warning_node_codes = [ + str(item.get("node_code") or "").strip() + for item in inspection_problem_rows + if str(item.get("node_code") or "").strip() + ] + cards.append( + { + "key": "release-rollout-attention", + "title": "先补默认 Rollout 巡检", + "level_label": "推荐", + "tag_type": "warning", + "summary": release_gate_summary or f"默认 Release {preferred_release_label} 已接近可发状态，但仍建议先补齐巡检。", + "reason": ( + release_gate_warning_reasons[0] + if release_gate_warning_reasons + else "先补健康快照、日志和诊断包，可以显著降低真正发 Rollout 时的误判风险。" + ), + "node_codes": warning_node_codes, + "meta_text": preferred_release_label, + "primary_label": "执行标准巡检", + "secondary_label": "查看版本区", + "primary_type": "warning", + "disabled": not warning_node_codes, + "primary_action_code": "run_standard_inspection", + "secondary_action_code": "focus_release_hub", + } + ) + elif release_gate_status == "no_targets": + cards.append( + { + "key": "release-rollout-no-targets", + "title": "先确认默认 Rollout 目标", + "level_label": "推荐", + "tag_type": "warning", + "summary": release_gate_summary or f"默认 Release {preferred_release_label} 当前没有可直接纳入 Rollout 的默认目标节点。", + "reason": "通常是当前有效执行面没有在线节点，或节点尚未进入默认 target 集。", + "node_codes": [], + "meta_text": preferred_release_label, + "primary_label": "查看版本区", + "secondary_label": "", + "primary_type": "warning", + "disabled": False, + "primary_action_code": "focus_release_hub", + "secondary_action_code": "", + } + ) + elif release_gate_status == "ready": + cards.append( + { + "key": "release-rollout-ready", + "title": "当前可以发起 Rollout", + "level_label": "准备就绪", + "tag_type": "success", + "summary": release_gate_summary or f"默认 Release {preferred_release_label} 已具备进入 Rollout 的门禁条件。", + "reason": "接管、巡检和默认版本都已经进入可推进状态，现在可以把更新真正纳入 Release / Rollout 闭环。", + "node_codes": [str(item.get("node_code") or "").strip() for item in list(release_gate.get("default_target_nodes") or []) if str(item.get("node_code") or "").strip()], + "meta_text": preferred_release_label, + "primary_label": "Worker 灰度", + "secondary_label": "Control 发布", + "primary_type": "success", + "disabled": False, + "primary_action_code": "create_release_rollout_worker", + "primary_action_payload": { + "release_id": preferred_release_id, + }, + "secondary_action_code": "create_release_rollout_control", + "secondary_action_payload": { + "release_id": preferred_release_id, + }, + } + ) + elif preferred_release_id > 0: + cards.append( + { + "key": "release-rollout-fallback", + "title": "查看默认 Release 门禁", + "level_label": "推荐", + "tag_type": "info", + "summary": release_gate_summary or f"默认 Release {preferred_release_label} 的 Rollout 门禁需要进一步确认。", + "reason": "当前建议先进入 Release Hub，看默认版本、批次和门禁摘要，再决定下一步。", + "node_codes": [], + "meta_text": preferred_release_label, + "primary_label": "查看版本区", + "secondary_label": "", + "primary_type": "primary", + "disabled": False, + "primary_action_code": "focus_release_hub", + "secondary_action_code": "", + } + ) + + if participating_node_codes and not log_sync_enabled: + primary_focus_ref = _build_scene_node_log_focus_ref( + participating_node_codes[0], + mode="key", + limit=80, + source="remote_log_sync_disabled", + ) + secondary_focus_ref = _build_scene_node_log_focus_ref( + participating_node_codes[0], + mode="full", + limit=120, + source="remote_log_sync_disabled", + ) + cards.append( + { + "key": "enable-scene-log-sync", + "title": "先补现场日志回传", + "level_label": "最高优先", + "tag_type": "danger", + "summary": f"当前有 {len(participating_node_codes)} 台节点正在真实参与检测，但海外控制面还看不到过程日志，排障会处于半盲态。", + "reason": "先把远端日志回传切到关键模式，才能在不放大噪音的前提下看到阶段、异常、代理与执行过程。", + "node_codes": participating_node_codes, + "primary_label": "开启关键回传", + "secondary_label": "开启全量回传", + "primary_type": "danger", + "disabled": False, + "focus_ref": primary_focus_ref, + "primary_focus_ref": primary_focus_ref, + "secondary_focus_ref": secondary_focus_ref, + "primary_action_code": "enable_log_sync_key", + "primary_action_payload": {"focus_ref": primary_focus_ref}, + "secondary_action_code": "enable_log_sync_full", + "secondary_action_payload": {"focus_ref": secondary_focus_ref}, + } + ) + elif participating_node_codes and remote_log_sample_count <= 0: + primary_focus_ref = _build_scene_node_log_focus_ref( + participating_node_codes[0], + mode=str(log_sync.get("mode") or "key"), + limit=120, + source="remote_log_sync_waiting_sample", + ) + secondary_focus_ref = _merge_focus_ref( + {}, + kind="execution_scene", + scene_key="scene.logs", + source="remote_log_sync_waiting_sample", + ) + cards.append( + { + "key": "inspect-scene-log-gap", + "title": "排查现场日志样本缺口", + "level_label": "推荐", + "tag_type": "warning", + "summary": f"当前日志回传已开启，但 {len(participating_node_codes)} 台参与节点还没有回传样本，先确认 Worker 现场输出为什么没回来。", + "reason": "这通常意味着现场仍在执行，但日志镜像链路、周期或样本采集还没形成有效观测，适合先抓 Worker 日志再看巡检。", + "node_codes": participating_node_codes, + "meta_text": f"参与节点 {len(participating_node_codes)} 台 / 已回传来源 {remote_log_source_count} 台", + "primary_label": "看 Worker 日志", + "secondary_label": "执行标准巡检", + "primary_type": "warning", + "disabled": not participating_node_codes, + "focus_ref": primary_focus_ref, + "primary_focus_ref": primary_focus_ref, + "secondary_focus_ref": secondary_focus_ref, + "primary_action_code": "open_worker_logs_participating", + "primary_action_payload": {"focus_ref": primary_focus_ref}, + "secondary_action_code": "run_inspection_participating", + "secondary_action_payload": {"focus_ref": secondary_focus_ref}, + } + ) + + cards.append( + { + "key": "inspect-participating", + "title": "先看执行现场", + "level_label": "推荐" if participating_node_codes else "待命", + "tag_type": "warning" if participating_node_codes else "info", + "summary": ( + f"当前有 {len(participating_node_codes)} 台节点正在真实参与检测，优先回收它们的标准巡检结果。" + if participating_node_codes + else "当前没有节点处于真实执行现场，可先关注待命节点或接管缺口。" + ), + "reason": "真正参与检测的节点最接近现场，最适合先看健康快照、Worker 日志和诊断包。", + "node_codes": participating_node_codes, + "primary_label": "执行标准巡检", + "secondary_label": "打开诊断模板", + "primary_type": "primary", + "disabled": not participating_node_codes, + "primary_action_code": "run_inspection_participating", + "secondary_action_code": "open_diagnostics_participating", + } + ) + + cards.append( + { + "key": "inspect-standby", + "title": "排查在线未参与节点", + "level_label": "推荐" if standby_node_codes else "待命", + "tag_type": "warning" if standby_node_codes else "info", + "summary": ( + f"当前有 {len(standby_node_codes)} 台节点在线但未参与检测，建议做一轮标准巡检确认是正常待命还是接单异常。" + if standby_node_codes + else "当前没有在线未参与节点，执行面结构比较干净。" + ), + "reason": "这组节点最适合定位“在线但不接单”“控制面兼跑未开始执行”“接管完成但还没真正纳入巡检”类问题。", + "node_codes": standby_node_codes, + "primary_label": "执行标准巡检", + "secondary_label": "看 Worker 日志", + "primary_type": "primary", + "disabled": not standby_node_codes, + "primary_action_code": "run_inspection_standby", + "secondary_action_code": "open_worker_logs_standby", + } + ) + + return cards + + +def _default_ops_priority_recommendation() -> dict: + return { + "source": "static", + "key": "build-node-agent-plane", + "priority": "build-node-agent-plane", + "title": "build-node-agent-plane", + "summary": "下一阶段应把 SSH/脚本式运维升级为海外控制面 + 节点 Agent 的任务编排模式。", + "reason": "当前已经具备跨地域心跳、同步、就绪度和日志回传基础，但节点规模继续增加后，人工 SSH 与复制日志的边际成本会快速失控。", + "level_label": "长期重点", + "tag_type": "info", + "primary_label": "", + "secondary_label": "", + "primary_action_code": "", + "secondary_action_code": "", + "node_codes": [], + "primary_node_codes": [], + "secondary_node_codes": [], + "primary_action_payload": {}, + "secondary_action_payload": {}, + "reasons": [ + "当前已经具备跨地域心跳、同步、就绪度和日志回传基础。", + "节点数继续增加后，人工 SSH 与复制日志的边际成本会快速失控。", + "后台按钮化动作需要统一的任务模型、执行回执和日志流通道。", + ], + } + + +def _build_ops_priority_recommendation(driver_recommendations: list[dict]) -> dict: + for raw_item in list(driver_recommendations or []): + item = dict(raw_item or {}) + if not item: + continue + title = str(item.get("title") or item.get("key") or "").strip() + summary = str(item.get("summary") or item.get("reason") or "").strip() + if not title and not summary: + continue + shared_node_codes = _normalize_driver_node_codes(item.get("node_codes") or []) + primary_node_codes = _normalize_driver_node_codes(item.get("primary_node_codes") or shared_node_codes) + secondary_node_codes = _normalize_driver_node_codes(item.get("secondary_node_codes") or shared_node_codes) + shared_payload = dict(item.get("action_payload") or {}) + return { + "source": "driver_recommendations", + "key": str(item.get("key") or "").strip(), + "priority": title or str(item.get("key") or "").strip() or "当前建议", + "title": title or str(item.get("key") or "").strip() or "当前建议", + "summary": summary or "当前已有建议动作，建议先按驾驶建议卡推进。", + "reason": str(item.get("reason") or "").strip(), + "level_label": str(item.get("level_label") or "").strip(), + "tag_type": str(item.get("tag_type") or "").strip(), + "primary_label": str(item.get("primary_label") or "").strip(), + "secondary_label": str(item.get("secondary_label") or "").strip(), + "primary_action_code": str(item.get("primary_action_code") or "").strip(), + "secondary_action_code": str(item.get("secondary_action_code") or "").strip(), + "node_codes": shared_node_codes, + "primary_node_codes": primary_node_codes, + "secondary_node_codes": secondary_node_codes, + "primary_action_payload": { + **shared_payload, + **dict(item.get("primary_action_payload") or {}), + }, + "secondary_action_payload": { + **shared_payload, + **dict(item.get("secondary_action_payload") or {}), + }, + "reasons": [ + part + for part in [ + str(item.get("reason") or "").strip(), + str(item.get("summary") or "").strip(), + ] + if part + ], + } + return _default_ops_priority_recommendation() + + +def _driver_feed_status_from_tag_type(tag_type: str) -> str: + normalized_tag_type = str(tag_type or "").strip() + if normalized_tag_type in {"danger", "warning"}: + return "attention" + if normalized_tag_type == "success": + return "ready" + return "planned" + + +def _build_driver_feed_entry_from_priority_recommendation(recommendation: dict) -> dict: + normalized_recommendation = dict(recommendation or {}) + key = str(normalized_recommendation.get("key") or "").strip() or "priority-recommendation" + shared_node_codes = _normalize_driver_node_codes(normalized_recommendation.get("node_codes") or []) + primary_node_codes = _normalize_driver_node_codes( + normalized_recommendation.get("primary_node_codes") or shared_node_codes + ) + secondary_node_codes = _normalize_driver_node_codes( + normalized_recommendation.get("secondary_node_codes") or shared_node_codes + ) + primary_action_payload = dict(normalized_recommendation.get("primary_action_payload") or {}) + secondary_action_payload = dict(normalized_recommendation.get("secondary_action_payload") or {}) + title = str(normalized_recommendation.get("title") or normalized_recommendation.get("priority") or key).strip() + summary = str(normalized_recommendation.get("summary") or "").strip() + reason = str(normalized_recommendation.get("reason") or "").strip() + detail_lines = _normalize_driver_text_lines( + [ + *[str(part or "").strip() for part in list(normalized_recommendation.get("reasons") or [])], + reason, + ], + limit=4, + ) + focus_ref = _normalize_focus_ref(normalized_recommendation.get("focus_ref")) + primary_focus_ref = _normalize_focus_ref(normalized_recommendation.get("primary_focus_ref")) + secondary_focus_ref = _normalize_focus_ref(normalized_recommendation.get("secondary_focus_ref")) + return _attach_driver_feed_contract_navigation({ + "key": f"priority:{key}", + "kind": "priority_recommendation", + "lane": "do_now", + "lane_label": "当前优先", + "title": title or "当前建议", + "summary": summary or reason or "当前已有建议动作，建议先从这里推进。", + "reason": reason, + "status": _driver_feed_status_from_tag_type(normalized_recommendation.get("tag_type")), + "level_label": str(normalized_recommendation.get("level_label") or "").strip() or "推荐", + "tag_type": str(normalized_recommendation.get("tag_type") or "").strip() or "info", + "meta_text": str(normalized_recommendation.get("source") or "overview.recommendation").strip(), + "node_codes": shared_node_codes, + "detail_lines": detail_lines, + "focus_ref": focus_ref, + "executor_kind": "driver_action", + "primary_label": str(normalized_recommendation.get("primary_label") or "").strip(), + "secondary_label": str(normalized_recommendation.get("secondary_label") or "").strip(), + "primary_action_code": str(normalized_recommendation.get("primary_action_code") or "").strip(), + "secondary_action_code": str(normalized_recommendation.get("secondary_action_code") or "").strip(), + "primary_node_codes": primary_node_codes, + "secondary_node_codes": secondary_node_codes, + "primary_focus_ref": primary_focus_ref, + "secondary_focus_ref": secondary_focus_ref, + "primary_action_payload": _action_payload_with_focus_ref(primary_action_payload, primary_focus_ref), + "secondary_action_payload": _action_payload_with_focus_ref(secondary_action_payload, secondary_focus_ref), + }) + + +def _build_driver_feed_entry_from_driver_recommendation(item: dict) -> dict: + normalized_item = dict(item or {}) + key = str(normalized_item.get("key") or "").strip() or "driver-recommendation" + shared_node_codes = _normalize_driver_node_codes(normalized_item.get("node_codes") or []) + primary_node_codes = _normalize_driver_node_codes(normalized_item.get("primary_node_codes") or shared_node_codes) + secondary_node_codes = _normalize_driver_node_codes( + normalized_item.get("secondary_node_codes") or shared_node_codes + ) + shared_action_payload = dict(normalized_item.get("action_payload") or {}) + primary_action_payload = { + **shared_action_payload, + **dict(normalized_item.get("primary_action_payload") or {}), + } + secondary_action_payload = { + **shared_action_payload, + **dict(normalized_item.get("secondary_action_payload") or {}), + } + detail_lines = _normalize_driver_text_lines( + [ + str(normalized_item.get("reason") or "").strip(), + str(normalized_item.get("meta_text") or "").strip(), + ], + limit=3, + ) + focus_ref = _normalize_focus_ref(normalized_item.get("focus_ref")) + primary_focus_ref = _normalize_focus_ref(normalized_item.get("primary_focus_ref")) + secondary_focus_ref = _normalize_focus_ref(normalized_item.get("secondary_focus_ref")) + return _attach_driver_feed_contract_navigation({ + "key": f"recommendation:{key}", + "kind": "driver_recommendation", + "lane": "do_now", + "lane_label": "建议动作", + "title": str(normalized_item.get("title") or key).strip() or "驾驶建议", + "summary": str(normalized_item.get("summary") or normalized_item.get("reason") or "").strip() or "建议先处理这条驾驶动作。", + "reason": str(normalized_item.get("reason") or "").strip(), + "status": _driver_feed_status_from_tag_type(normalized_item.get("tag_type")), + "level_label": str(normalized_item.get("level_label") or "").strip() or "推荐", + "tag_type": str(normalized_item.get("tag_type") or "").strip() or "info", + "meta_text": str(normalized_item.get("meta_text") or "").strip(), + "node_codes": shared_node_codes, + "detail_lines": detail_lines, + "focus_ref": focus_ref, + "executor_kind": "driver_action", + "primary_label": str(normalized_item.get("primary_label") or "").strip(), + "secondary_label": str(normalized_item.get("secondary_label") or "").strip(), + "primary_action_code": str(normalized_item.get("primary_action_code") or "").strip(), + "secondary_action_code": str(normalized_item.get("secondary_action_code") or "").strip(), + "primary_node_codes": primary_node_codes, + "secondary_node_codes": secondary_node_codes, + "primary_focus_ref": primary_focus_ref, + "secondary_focus_ref": secondary_focus_ref, + "primary_action_payload": _action_payload_with_focus_ref(primary_action_payload, primary_focus_ref), + "secondary_action_payload": _action_payload_with_focus_ref(secondary_action_payload, secondary_focus_ref), + }) + + +def _build_driver_feed_entry_from_runbook_sequence(sequence: dict) -> dict: + normalized_sequence = dict(sequence or {}) + sequence_key = str(normalized_sequence.get("key") or "").strip() + if not sequence_key: + return {} + + primary_resolution = dict(normalized_sequence.get("primary_resolution") or {}) + secondary_resolution = dict(normalized_sequence.get("secondary_resolution") or {}) + resolved_action_label = ( + str(primary_resolution.get("driver_action_label") or "").strip() + or str(primary_resolution.get("driver_action_code") or "").strip() + ) + resolved_action_code = str(primary_resolution.get("driver_action_code") or "").strip() + resolved_node_codes = _normalize_driver_node_codes(primary_resolution.get("driver_node_codes") or []) + secondary_resolved_action_label = ( + str(secondary_resolution.get("driver_action_label") or "").strip() + or str(secondary_resolution.get("driver_action_code") or "").strip() + ) + target_node_codes = _normalize_driver_node_codes(normalized_sequence.get("target_node_codes") or []) + step_titles = [ + str(item or "").strip() + for item in list(normalized_sequence.get("step_titles") or []) + if str(item or "").strip() + ] + detail_lines: list[str] = [] + if primary_resolution and primary_resolution.get("ok") is False: + detail_lines.append(f"当前解析失败：{str(primary_resolution.get('message') or '解析失败').strip()}") + elif resolved_action_label or resolved_action_code: + detail_lines.append( + f"当前解析：{resolved_action_label or resolved_action_code}" + + ( + f" ({resolved_action_code})" + if resolved_action_code and resolved_action_label and resolved_action_code != resolved_action_label + else "" + ) + ) + detail_lines.append( + f"当前目标：{_format_node_code_list(resolved_node_codes or target_node_codes)}" + ) + if str(normalized_sequence.get("secondary_label") or "").strip() and secondary_resolved_action_label: + detail_lines.append(f"备用解析：{secondary_resolved_action_label}") + if step_titles: + detail_lines.append(f"标准步骤：{' -> '.join(step_titles)}") + if str(normalized_sequence.get("reason") or "").strip(): + detail_lines.append(str(normalized_sequence.get("reason") or "").strip()) + focus_ref = _normalize_focus_ref(normalized_sequence.get("focus_ref")) + primary_focus_ref = _normalize_focus_ref(normalized_sequence.get("primary_focus_ref")) + secondary_focus_ref = _normalize_focus_ref(normalized_sequence.get("secondary_focus_ref")) + + return _attach_driver_feed_contract_navigation({ + "key": f"runbook:{sequence_key}", + "kind": "runbook_sequence", + "lane": "standard_path", + "lane_label": "标准路径", + "title": str(normalized_sequence.get("title") or "").strip() or "标准作业路径", + "summary": str(normalized_sequence.get("summary") or "").strip() or "当前标准作业路径已就绪。", + "reason": str(normalized_sequence.get("reason") or "").strip(), + "status": _runbook_sequence_activity_status(normalized_sequence), + "level_label": str(normalized_sequence.get("status_label") or normalized_sequence.get("status") or "").strip() or "ready", + "tag_type": str(normalized_sequence.get("tag_type") or "").strip() or "info", + "meta_text": str(normalized_sequence.get("target_scope_label") or "").strip(), + "node_codes": target_node_codes, + "detail_lines": _normalize_driver_text_lines(detail_lines, limit=5), + "focus_ref": focus_ref, + "executor_kind": "runbook_sequence", + "sequence_key": sequence_key, + "primary_label": str(normalized_sequence.get("primary_label") or "").strip(), + "secondary_label": str(normalized_sequence.get("secondary_label") or "").strip(), + "primary_action_code": resolved_action_code or str(normalized_sequence.get("primary_action_code") or "").strip(), + "secondary_action_code": ( + str(secondary_resolution.get("driver_action_code") or "").strip() + or str(normalized_sequence.get("secondary_action_code") or "").strip() + ), + "primary_node_codes": resolved_node_codes or target_node_codes, + "secondary_node_codes": _normalize_driver_node_codes( + secondary_resolution.get("driver_node_codes") or target_node_codes + ), + "primary_focus_ref": primary_focus_ref, + "secondary_focus_ref": secondary_focus_ref, + "primary_action_payload": _action_payload_with_focus_ref( + primary_resolution.get("driver_action_payload") or {}, + primary_focus_ref, + ), + "secondary_action_payload": _action_payload_with_focus_ref( + secondary_resolution.get("driver_action_payload") or {}, + secondary_focus_ref, + ), + "occurred_at": _activity_time(primary_resolution.get("resolved_at")), + }) + + +def _build_driver_feed_top_recommendation(entry: dict) -> dict: + normalized_entry = dict(entry or {}) + if not normalized_entry: + return {} + + return { + "key": str(normalized_entry.get("key") or "").strip(), + "kind": str(normalized_entry.get("kind") or "").strip(), + "lane": str(normalized_entry.get("lane") or "").strip(), + "title": str(normalized_entry.get("title") or "").strip(), + "summary": str(normalized_entry.get("summary") or "").strip(), + "reason": str(normalized_entry.get("reason") or "").strip(), + "status": str(normalized_entry.get("status") or "").strip(), + "level_label": str(normalized_entry.get("level_label") or "").strip(), + "tag_type": str(normalized_entry.get("tag_type") or "").strip(), + "meta_text": str(normalized_entry.get("meta_text") or "").strip(), + "executor_kind": str(normalized_entry.get("executor_kind") or "").strip(), + "sequence_key": str(normalized_entry.get("sequence_key") or "").strip(), + "focus_ref": _normalize_focus_ref(normalized_entry.get("focus_ref")), + "primary_label": str(normalized_entry.get("primary_label") or "").strip(), + "secondary_label": str(normalized_entry.get("secondary_label") or "").strip(), + "primary_action_code": str(normalized_entry.get("primary_action_code") or "").strip(), + "secondary_action_code": str(normalized_entry.get("secondary_action_code") or "").strip(), + "node_codes": _normalize_driver_node_codes(normalized_entry.get("node_codes") or []), + "primary_node_codes": _normalize_driver_node_codes(normalized_entry.get("primary_node_codes") or []), + "secondary_node_codes": _normalize_driver_node_codes(normalized_entry.get("secondary_node_codes") or []), + "primary_focus_ref": _normalize_focus_ref(normalized_entry.get("primary_focus_ref")), + "secondary_focus_ref": _normalize_focus_ref(normalized_entry.get("secondary_focus_ref")), + "primary_action_payload": dict(normalized_entry.get("primary_action_payload") or {}), + "secondary_action_payload": dict(normalized_entry.get("secondary_action_payload") or {}), + "detail_lines": _normalize_driver_text_lines(normalized_entry.get("detail_lines") or [], limit=5), + "contract_keys": list(normalized_entry.get("contract_keys") or []), + "contract_navigation": dict(normalized_entry.get("contract_navigation") or {}), + } + + +def get_ops_driver_feed() -> dict: + overview = get_ops_overview() + go_live_summary = get_ops_go_live_summary() + runbook = get_ops_runbook() + recommendation = dict(overview.get("recommendation") or {}) + driver_recommendations = [dict(item or {}) for item in list(overview.get("driver_recommendations") or []) if item] + control_sequences = [dict(item or {}) for item in list(runbook.get("control_sequences") or []) if item] + activity_stream = dict(overview.get("activity_stream") or {}) + activity_focus = _build_driver_activity_focus(list(activity_stream.get("items") or [])) + scene_log_observation = _build_scene_log_observation_from_overview(overview) + + entries: list[dict] = [] + seen_keys: set[str] = set() + + priority_entry = _build_driver_feed_entry_from_priority_recommendation(recommendation) + if priority_entry: + entries.append(priority_entry) + recommendation_key = str(recommendation.get("key") or "").strip() + if recommendation_key: + seen_keys.add(recommendation_key) + + for raw_item in driver_recommendations: + raw_key = str(raw_item.get("key") or "").strip() + if raw_key and raw_key in seen_keys: + continue + entry = _build_driver_feed_entry_from_driver_recommendation(raw_item) + if not entry: + continue + entries.append(entry) + if raw_key: + seen_keys.add(raw_key) + + for raw_sequence in control_sequences: + entry = _build_driver_feed_entry_from_runbook_sequence(raw_sequence) + if entry: + entries.append(entry) + + lane_counts: dict[str, int] = {} + kind_counts: dict[str, int] = {} + status_counts: dict[str, int] = {} + for item in entries: + lane = str(item.get("lane") or "").strip() or "other" + kind = str(item.get("kind") or "").strip() or "other" + status = str(item.get("status") or "").strip() or "other" + lane_counts[lane] = int(lane_counts.get(lane, 0) or 0) + 1 + kind_counts[kind] = int(kind_counts.get(kind, 0) or 0) + 1 + status_counts[status] = int(status_counts.get(status, 0) or 0) + 1 + + headline = "" + if entries: + first_entry = dict(entries[0] or {}) + headline = ( + f"{str(first_entry.get('title') or '').strip()}：{str(first_entry.get('summary') or '').strip()}" + ).strip("：") + elif activity_focus: + first_focus = dict(activity_focus[0] or {}) + headline = ( + f"{str(first_focus.get('title') or '').strip()}：{str(first_focus.get('summary') or '').strip()}" + ).strip("：") + publish_status = str(go_live_summary.get("publish_status") or "").strip() + publish_status_label = str(go_live_summary.get("publish_status_label") or "").strip() + publish_summary = str(go_live_summary.get("publish_summary") or "").strip() + launchpad_recommended_target_node_code = str( + go_live_summary.get("launchpad_recommended_target_node_code") or "" + ).strip() + launchpad_recommended_recovery_label = str( + go_live_summary.get("launchpad_recommended_recovery_label") or "" + ).strip() + launchpad_recommended_recovery_summary = str( + go_live_summary.get("launchpad_recommended_recovery_summary") or "" + ).strip() + launchpad_onboarding_bootstrap_pending_nodes = int( + go_live_summary.get("launchpad_onboarding_bootstrap_pending_nodes", 0) or 0 + ) + launchpad_onboarding_acceptance_ready_nodes = int( + go_live_summary.get("launchpad_onboarding_acceptance_ready_nodes", 0) or 0 + ) + if publish_status and publish_status != "ready": + publish_prefix = publish_status_label or publish_status + headline = ( + f"{headline}｜发布闸门：{publish_prefix}" + if headline + else f"发布闸门：{publish_prefix}{'｜' + publish_summary if publish_summary else ''}" + ) + if launchpad_recommended_target_node_code and launchpad_recommended_recovery_label: + launchpad_prefix = f"{launchpad_recommended_recovery_label}：{launchpad_recommended_target_node_code}" + headline = f"{headline}｜接入缺口：{launchpad_prefix}" if headline else launchpad_prefix + + summary_contract_keys = _normalize_ops_contract_keys( + ["ops_driver_contract"] + + [ + contract_key + for item in entries + for contract_key in list((item or {}).get("contract_keys") or []) + ] + + [ + contract_key + for item in activity_focus + for contract_key in list((item or {}).get("contract_keys") or []) + ] + + list(scene_log_observation.get("contract_keys") or []) + ) + top_recommendation = _build_driver_feed_top_recommendation(entries[0] or {}) + grouped_driver_recommendations = [ + dict(item or {}) + for item in entries + if str((item or {}).get("kind") or "").strip() in {"priority_recommendation", "driver_recommendation"} + ] + runbook_sequences = [ + dict(item or {}) + for item in entries + if str((item or {}).get("kind") or "").strip() == "runbook_sequence" + ] + automation_entries = [ + _build_codex_brief_entry(raw_entry) + for raw_entry in entries + if dict(raw_entry or {}) + ] + automation_entries = [entry for entry in automation_entries if entry] + automation_coverage = _build_ops_automation_coverage_summary( + automation_entries, + activity_focus_total=len(activity_focus), + scene_log_observation=scene_log_observation, + go_live_summary=go_live_summary, + ) + + return { + "generated_at": datetime.now().isoformat(timespec="seconds"), + "headline": headline or "当前还没有可展示的驾驶主线。", + "go_live_summary": go_live_summary, + "top_recommendation": top_recommendation, + "driver_recommendations": grouped_driver_recommendations, + "runbook_sequences": runbook_sequences, + "activity_focus": activity_focus, + "scene_log_observation": scene_log_observation, + "entries": entries, + "automation_coverage": automation_coverage, + "contract_navigation": _build_ops_contract_navigation( + summary_contract_keys, + primary_contract_key="ops_driver_contract", + registry=get_ops_contract_registry(), + ), + "summary": { + "total": len(entries), + "lane_counts": lane_counts, + "kind_counts": kind_counts, + "status_counts": status_counts, + "go_live_status": str(go_live_summary.get("go_live_status") or "").strip(), + "operator_title": str(go_live_summary.get("operator_title") or "").strip(), + "publish_ready": bool(go_live_summary.get("publish_ready")), + "publish_status": publish_status, + "publish_status_label": publish_status_label, + "launchpad_recommended_target_node_code": launchpad_recommended_target_node_code, + "launchpad_recommended_recovery_label": launchpad_recommended_recovery_label, + "launchpad_recommended_recovery_summary": launchpad_recommended_recovery_summary, + "launchpad_onboarding_bootstrap_pending_nodes": launchpad_onboarding_bootstrap_pending_nodes, + "launchpad_onboarding_acceptance_ready_nodes": launchpad_onboarding_acceptance_ready_nodes, + "do_now_total": int(lane_counts.get("do_now", 0) or 0), + "standard_path_total": int(lane_counts.get("standard_path", 0) or 0), + "activity_focus_total": len(activity_focus), + "scene_log_status": str(scene_log_observation.get("status") or "").strip() or "standby", + "launch_status": str(automation_coverage.get("launch_status") or "").strip(), + "launch_ready": bool(automation_coverage.get("launch_ready", False)), + "preview_only_total": int(automation_coverage.get("preview_only_total", 0) or 0), + "backend_handled_total": int(automation_coverage.get("backend_handled_total", 0) or 0), + }, + } + + +def _build_scene_log_observation_from_overview(overview: dict) -> dict: + normalized_overview = dict(overview or {}) + execution_scene = dict(normalized_overview.get("execution_scene") or {}) + log_sync = dict(execution_scene.get("log_sync") or {}) + participation_summary = dict(execution_scene.get("participation_summary") or {}) + + participating_node_codes = _normalize_ops_node_code_list( + execution_scene.get("participating_nodes") or [], + participation_summary.get("dispatch_active_node_codes") or [], + participation_summary.get("recent_only_node_codes") or [], + ) + missing_node_codes = _normalize_ops_node_code_list(log_sync.get("missing_participating_nodes") or []) + source_node_codes = _normalize_ops_node_code_list( + log_sync.get("source_nodes") or [], + log_sync.get("source_node_summaries") or [], + ) + preferred_node_codes = _normalize_ops_node_code_list( + missing_node_codes, + participating_node_codes, + source_node_codes, + ) + target_node_code = str((preferred_node_codes or [""])[0] or "").strip() + enabled = bool(log_sync.get("enabled", False)) + mode = _normalize_scene_log_mode(log_sync.get("mode") or "key", fallback="key") + participating_total = int( + log_sync.get("participating_node_count", participation_summary.get("participating_nodes", len(participating_node_codes))) + or participation_summary.get("participating_nodes", len(participating_node_codes)) + or len(participating_node_codes) + or 0 + ) + covered_total = int(log_sync.get("covered_participating_node_count", 0) or 0) + line_count = int(log_sync.get("line_count", 0) or 0) + source_node_count = int(log_sync.get("source_node_count", 0) or 0) + missing_total = int(log_sync.get("missing_participating_node_count", len(missing_node_codes)) or len(missing_node_codes)) + status = "standby" + status_label = "待命" + summary = "当前没有参与检测节点，现场日志观察处于待命状态。" + source = "driver_scene_log_observation" + if participating_total > 0 and not enabled: + status = "disabled" + status_label = "未开启" + summary = ( + f"当前有 {participating_total} 台参与节点，但远端日志回传仍关闭，" + "海外控制面还看不到节点级现场日志。" + ) + source = "remote_log_sync_disabled" + elif participating_total > 0 and ( + line_count <= 0 or source_node_count <= 0 or missing_total > 0 + ): + status = "waiting_sample" + status_label = "等待样本" + summary = ( + f"远端日志回传已经开启，但参与节点只覆盖 {covered_total}/{participating_total}，" + f"样本 {line_count} 条，仍需继续观察或下钻节点现场日志。" + ) + source = "remote_log_sync_waiting_sample" + elif participating_total > 0: + status = "ready" + status_label = "可下钻" + summary = ( + f"远端日志回传已经形成样本，当前覆盖 {covered_total}/{participating_total} 台参与节点，" + "可以直接下钻节点现场日志。" + ) + source = "remote_log_sync_ready" + + focus_ref = ( + _build_scene_node_log_focus_ref( + target_node_code, + mode=mode, + limit=120 if status == "waiting_sample" else 80, + source=source, + ) + if target_node_code + else {} + ) + return { + "status": status, + "status_label": status_label, + "summary": summary, + "enabled": enabled, + "mode": mode, + "participating_node_count": participating_total, + "covered_participating_node_count": covered_total, + "missing_participating_node_count": missing_total, + "line_count": line_count, + "source_node_count": source_node_count, + "target_node_code": target_node_code, + "target_node_codes": preferred_node_codes, + "missing_node_codes": missing_node_codes, + "source_node_codes": source_node_codes, + "last_at": str(log_sync.get("last_at") or "").strip(), + "focus_ref": focus_ref, + "contract_keys": _normalize_ops_contract_keys( + ["ops_observability_contract", "ops_stack_diagnosis_contract"] + ), + "contract_navigation": _build_ops_contract_navigation( + ["ops_observability_contract", "ops_stack_diagnosis_contract"], + primary_contract_key="ops_observability_contract", + registry=get_ops_contract_registry(), + ), + } + + +def _build_driver_action_request_preview( + *, + executor_kind: str, + action_code: str, + action_payload: dict | None = None, + node_codes: list[str] | None = None, + sequence_key: str = "", + secondary: bool = False, +) -> dict: + normalized_executor_kind = str(executor_kind or "").strip() + normalized_action_code = str(action_code or "").strip() + normalized_action_payload = dict(action_payload or {}) + normalized_node_codes = _normalize_driver_node_codes(node_codes or []) + normalized_sequence_key = str(sequence_key or "").strip() + + if normalized_executor_kind == "runbook_sequence" and normalized_sequence_key: + return { + "sequence_key": normalized_sequence_key, + "secondary": bool(secondary), + "action_payload": normalized_action_payload, + } + + return { + "action_code": normalized_action_code, + "node_codes": normalized_node_codes, + "action_payload": normalized_action_payload, + } + + +def _driver_action_missing_fields( + action_code: str, + *, + action_payload: dict | None = None, + node_codes: list[str] | None = None, +) -> list[str]: + normalized_action_code = str(action_code or "").strip() + normalized_action_payload = dict(action_payload or {}) + normalized_node_codes = _normalize_driver_node_codes(node_codes or []) + missing_fields: list[str] = [] + + if normalized_action_code == "open_playbook_dialog" and not str(normalized_action_payload.get("playbook_key") or "").strip(): + missing_fields.append("playbook_key") + if normalized_action_code == "open_action_template_dialog" and not str(normalized_action_payload.get("template_key") or "").strip(): + missing_fields.append("template_key") + if normalized_action_code in {"replay_delivery_queue", "flush_delivery_queue"}: + target_node_codes = _normalize_driver_node_codes(normalized_action_payload.get("target_node_codes") or normalized_node_codes) + if not target_node_codes: + missing_fields.append("target_node_codes") + if normalized_action_code in {"focus_playbook_run", "open_playbook_run_latest_events"} and not str( + normalized_action_payload.get("run_code") or "" + ).strip(): + missing_fields.append("run_code") + if normalized_action_code == "focus_activity_item" and not str( + (normalized_action_payload.get("ui_intent") or {}).get("kind") or "" + ).strip(): + missing_fields.append("ui_intent.kind") + if normalized_action_code == "focus_latest_job_events": + has_ui_intent = bool(str((normalized_action_payload.get("ui_intent") or {}).get("kind") or "").strip()) + source_focus_ref = _normalize_focus_ref(normalized_action_payload.get("source_focus_ref")) + focus_ref = _normalize_focus_ref(normalized_action_payload.get("focus_ref")) + has_event_focus = str(source_focus_ref.get("kind") or "").strip() == "ops_job_event" + has_job_focus = str(focus_ref.get("kind") or "").strip() in {"ops_job", "ops_job_event"} + if not (has_ui_intent or has_event_focus or has_job_focus): + missing_fields.append("ui_intent.kind|focus_ref") + if normalized_action_code in {"handover_first_gap", "view_first_gap"} and not str( + normalized_action_payload.get("node_code") or (normalized_node_codes[0] if normalized_node_codes else "") + ).strip(): + missing_fields.append("node_code") + if normalized_action_code in { + "open_worker_logs_participating", + "open_worker_logs_standby", + "open_worker_logs", + "open_diagnostics_participating", + "open_diagnostics", + "run_inspection_participating", + "run_inspection_standby", + "run_standard_inspection", + } and not normalized_node_codes: + missing_fields.append("node_codes") + if normalized_action_code in {"create_release_rollout_worker", "create_release_rollout_control"} and int( + normalized_action_payload.get("release_id") or 0 + ) <= 0: + missing_fields.append("release_id") + + return missing_fields + + +def _build_driver_action_execution_profile( + action_code: str, + *, + action_payload: dict | None = None, + node_codes: list[str] | None = None, +) -> dict: + normalized_action_code = str(action_code or "").strip() + normalized_action_payload = dict(action_payload or {}) + normalized_node_codes = _normalize_driver_node_codes(node_codes or []) + missing_fields = _driver_action_missing_fields( + normalized_action_code, + action_payload=normalized_action_payload, + node_codes=normalized_node_codes, + ) + base_profile = { + "action_code": normalized_action_code, + "backend_handled": normalized_action_code in _BACKEND_DRIVER_ACTION_CODES, + "automation_level": "blocked", + "recommendation": "blocked", + "executor_mode_hint": "manual-or-ui", + "risk_level": "unknown", + "confirm_required": False, + "ui_only": False, + "blocked": False, + "requires_fields": [], + "missing_fields": missing_fields, + "reason": "", + } + + if not normalized_action_code: + return { + **base_profile, + "blocked": True, + "reason": "当前条目还没有解析出 driver action，暂时不能自动执行。", + } + + if missing_fields: + return { + **base_profile, + "blocked": True, + "requires_fields": list(dict.fromkeys(missing_fields)), + "reason": f"当前动作缺少必填上下文：{', '.join(list(dict.fromkeys(missing_fields)))}。", + } + + if normalized_action_code not in _BACKEND_DRIVER_ACTION_CODES: + return { + **base_profile, + "automation_level": "ui_only", + "recommendation": "open_ui", + "executor_mode_hint": "manual-or-ui", + "risk_level": "medium", + "ui_only": True, + "reason": "当前动作尚未后端化，通常需要页面交互或人工确认后再推进。", + } + + if normalized_action_code in _SAFE_AUTO_DRIVER_ACTION_CODES: + executor_mode_hint = "settings" if normalized_action_code.startswith(("enable_", "disable_")) else "ops-playbook" + reason = ( + "当前动作只会调整运行时日志回传开关，属于低风险设置变更。" + if executor_mode_hint == "settings" + else "当前动作会走标准 playbook / 巡检链路，属于可自动执行的观测与诊断动作。" + ) + return { + **base_profile, + "automation_level": "safe_auto", + "recommendation": "auto_execute", + "executor_mode_hint": executor_mode_hint, + "risk_level": "low", + "reason": reason, + } + + if normalized_action_code in _GUARDED_AUTO_DRIVER_ACTION_CODES: + return { + **base_profile, + "automation_level": "guarded_auto", + "recommendation": "confirm_then_execute", + "executor_mode_hint": "smart-release-rollout", + "risk_level": "high", + "confirm_required": True, + "reason": "当前动作会直接创建 Release / Rollout 或推进发布批次，适合在 Codex 或人工确认后执行。", + } + + if normalized_action_code in _MIXED_DRIVER_ACTION_CODES: + return { + **base_profile, + "automation_level": "mixed", + "recommendation": "resolve_first", + "executor_mode_hint": "cluster-sync-or-ui", + "risk_level": "medium", + "reason": "当前动作会先判断接管缺口，可能转为补接入页面，也可能直接同步托管节点，执行前最好先复核解析结果。", + } + + if normalized_action_code in _UI_ONLY_DRIVER_ACTION_CODES: + return { + **base_profile, + "automation_level": "ui_only", + "recommendation": "open_ui", + "executor_mode_hint": "ui-intent", + "risk_level": "low", + "ui_only": True, + "reason": "当前动作主要用于打开页面、聚焦详情或进入模板，不适合作为无界面自动执行动作。", + } + + return { + **base_profile, + "automation_level": "ui_only", + "recommendation": "open_ui", + "executor_mode_hint": "manual-or-ui", + "risk_level": "medium", + "ui_only": True, + "reason": "当前动作暂时只能作为驾驶建议展示，建议先进入对应工作区再决定下一步。", + } + + +def _driver_action_executor_kind_label(executor_kind: str) -> str: + normalized_executor_kind = str(executor_kind or "").strip() + if normalized_executor_kind == "runbook_sequence": + return "runbook sequence" + return normalized_executor_kind or "driver action" + + +def _driver_action_execution_chain(executor_kind: str) -> str: + normalized_executor_kind = str(executor_kind or "").strip() + if normalized_executor_kind == "runbook_sequence": + return "ops-driver -> runbook-sequence -> resolved driver action / ui-intent" + return "ops-driver -> driver-action execute -> playbook / ui-intent / settings" + + +def _driver_action_target_api(*, executor_kind: str, sequence_key: str = "") -> str: + normalized_executor_kind = str(executor_kind or "").strip() + normalized_sequence_key = str(sequence_key or "").strip() + if normalized_executor_kind == "runbook_sequence" and normalized_sequence_key: + return f"/api/v1/ops/runbook/sequences/{normalized_sequence_key}/execute" + return "/api/v1/ops/driver-actions/execute" + + +def _driver_action_target_method(*, executor_kind: str, sequence_key: str = "") -> str: + normalized_executor_kind = str(executor_kind or "").strip() + normalized_sequence_key = str(sequence_key or "").strip() + if normalized_executor_kind == "runbook_sequence" and normalized_sequence_key: + return "POST" + return "POST" + + +def _driver_action_recommendation_label(recommendation: str) -> str: + normalized_recommendation = str(recommendation or "").strip() + return { + "auto_execute": "可自动执行", + "confirm_then_execute": "建议确认后执行", + "resolve_first": "先复核再执行", + "open_ui": "需要进入工作区", + "blocked": "当前阻断", + }.get(normalized_recommendation, normalized_recommendation or "待判断") + + +def _build_driver_action_contract_request_payload( + *, + executor_kind: str, + action_code: str, + action_payload: dict | None = None, + node_codes: list[str] | None = None, + sequence_key: str = "", + focus_ref: dict | None = None, + secondary: bool = False, + requested_by: str = "api/preview", +) -> dict: + normalized_executor_kind = str(executor_kind or "").strip() + normalized_action_code = str(action_code or "").strip() + normalized_action_payload = _action_payload_with_focus_ref(action_payload or {}, focus_ref) + normalized_node_codes = _normalize_driver_node_codes(node_codes or []) + normalized_sequence_key = str(sequence_key or "").strip() + normalized_requested_by = str(requested_by or "api/preview").strip() or "api/preview" + + if normalized_executor_kind == "runbook_sequence" and normalized_sequence_key: + payload = { + "secondary": bool(secondary), + "requested_by": normalized_requested_by, + } + if normalized_node_codes: + payload["node_codes"] = normalized_node_codes + if normalized_action_payload: + payload["action_payload"] = normalized_action_payload + return payload + + return { + "action_code": normalized_action_code, + "node_codes": normalized_node_codes, + "action_payload": normalized_action_payload, + "requested_by": normalized_requested_by, + } + + +def _build_driver_action_contract_preview_section( + *, + executor_kind: str, + action_code: str, + label: str = "", + action_payload: dict | None = None, + node_codes: list[str] | None = None, + sequence_key: str = "", + focus_ref: dict | None = None, + secondary: bool = False, + requested_by: str = "api/preview", + reason: str = "", +) -> dict: + normalized_executor_kind = str(executor_kind or "").strip() or "driver_action" + normalized_action_code = str(action_code or "").strip() + normalized_label = str(label or "").strip() + normalized_action_payload = dict(action_payload or {}) + normalized_node_codes = _normalize_driver_node_codes(node_codes or []) + normalized_sequence_key = str(sequence_key or "").strip() + execution_profile = _build_driver_action_execution_profile( + normalized_action_code, + action_payload=normalized_action_payload, + node_codes=normalized_node_codes, + ) + + return { + "label": normalized_label or ("次动作" if secondary else "主动作"), + "action_code": normalized_action_code, + "node_codes": normalized_node_codes, + "focus_ref": _normalize_focus_ref(focus_ref), + "target_api": _driver_action_target_api( + executor_kind=normalized_executor_kind, + sequence_key=normalized_sequence_key, + ), + "target_method": _driver_action_target_method( + executor_kind=normalized_executor_kind, + sequence_key=normalized_sequence_key, + ), + "reason": str(reason or execution_profile.get("reason") or "").strip(), + "recommendation_label": _driver_action_recommendation_label(execution_profile.get("recommendation") or ""), + "automation_level_label": str(execution_profile.get("automation_level") or "").strip(), + "risk_level_label": str(execution_profile.get("risk_level") or "").strip(), + "request_payload": _build_driver_action_contract_request_payload( + executor_kind=normalized_executor_kind, + action_code=normalized_action_code, + action_payload=normalized_action_payload, + node_codes=normalized_node_codes, + sequence_key=normalized_sequence_key, + focus_ref=_normalize_focus_ref(focus_ref), + secondary=secondary, + requested_by=requested_by, + ), + "execution_profile": execution_profile, + } + + +def preview_driver_action(payload: dict) -> tuple[bool, str, dict]: + normalized_payload = dict(payload or {}) + source_label = str(normalized_payload.get("source_label") or "ops-center").strip() or "ops-center" + title = str(normalized_payload.get("title") or "").strip() or "驾驶动作" + summary = str(normalized_payload.get("summary") or "").strip() + reason = str(normalized_payload.get("reason") or "").strip() + meta_text = str(normalized_payload.get("meta_text") or "").strip() + executor_kind = str(normalized_payload.get("executor_kind") or "driver_action").strip() or "driver_action" + sequence_key = str(normalized_payload.get("sequence_key") or "").strip() + requested_by = str(normalized_payload.get("requested_by") or "api/preview").strip() or "api/preview" + + contract_registry = get_ops_contract_registry() + contract = dict((contract_registry.get("contracts_by_key") or {}).get("ops_driver_contract") or {}) + + if executor_kind == "runbook_sequence": + if not sequence_key: + return False, "sequence_key 不能为空", {} + + include_secondary = bool(normalized_payload.get("include_secondary", True)) + primary_ok, primary_message, primary_resolved = resolve_ops_runbook_sequence( + sequence_key, + { + "requested_by": requested_by, + "secondary": False, + "node_codes": _normalize_driver_node_codes(normalized_payload.get("primary_node_codes") or []), + "action_payload": dict(normalized_payload.get("primary_action_payload") or {}), + }, + ) + if not primary_ok: + return False, primary_message, dict(primary_resolved or {}) + + primary_label = str(normalized_payload.get("primary_label") or primary_resolved.get("driver_action_label") or "").strip() + secondary_label = str(normalized_payload.get("secondary_label") or "").strip() + secondary_action_code = str(normalized_payload.get("secondary_action_code") or "").strip() + preview_focus_ref = _normalize_focus_ref( + normalized_payload.get("focus_ref") or (primary_resolved.get("sequence") or {}).get("focus_ref") + ) + secondary_preview: dict = {} + if include_secondary or secondary_label or secondary_action_code: + secondary_ok, secondary_message, secondary_resolved = resolve_ops_runbook_sequence( + sequence_key, + { + "requested_by": requested_by, + "secondary": True, + "node_codes": _normalize_driver_node_codes(normalized_payload.get("secondary_node_codes") or []), + "action_payload": dict(normalized_payload.get("secondary_action_payload") or {}), + }, + ) + if secondary_ok: + secondary_preview = _build_driver_action_contract_preview_section( + executor_kind=executor_kind, + action_code=str(secondary_resolved.get("driver_action_code") or "").strip(), + label=str(secondary_label or secondary_resolved.get("driver_action_label") or "").strip(), + action_payload=dict(secondary_resolved.get("driver_action_payload") or {}), + node_codes=_normalize_driver_node_codes(secondary_resolved.get("driver_node_codes") or []), + sequence_key=sequence_key, + focus_ref=_normalize_focus_ref( + normalized_payload.get("secondary_focus_ref") + or (secondary_resolved.get("sequence") or {}).get("action_focus_ref") + ), + secondary=True, + requested_by=requested_by, + reason=reason, + ) + elif secondary_label or secondary_action_code: + return False, secondary_message, dict(secondary_resolved or {}) + + return True, "驾驶动作预览已生成", { + "contract_key": "ops_driver_contract", + "contract_version": str(contract.get("version") or _OPS_CONTRACT_SCHEMA_VERSION).strip(), + "contract_schema_doc_path": str(contract.get("schema_doc_path") or "").strip(), + "contract_primary_endpoint": str(contract.get("primary_endpoint") or "").strip(), + "preview_endpoint": "/api/v1/ops/driver-actions/preview", + "source_label": source_label, + "title": title, + "summary": summary, + "reason": reason, + "meta_text": meta_text, + "executor_kind": executor_kind, + "executor_kind_label": _driver_action_executor_kind_label(executor_kind), + "sequence_key": sequence_key, + "focus_ref": preview_focus_ref, + "primary_focus_ref": _normalize_focus_ref( + normalized_payload.get("primary_focus_ref") + or (primary_resolved.get("sequence") or {}).get("action_focus_ref") + ), + "secondary_focus_ref": _normalize_focus_ref( + normalized_payload.get("secondary_focus_ref") + or (secondary_preview.get("focus_ref") or {}) + ), + "execution_chain": _driver_action_execution_chain(executor_kind), + "primary": _build_driver_action_contract_preview_section( + executor_kind=executor_kind, + action_code=str(primary_resolved.get("driver_action_code") or "").strip(), + label=primary_label, + action_payload=dict(primary_resolved.get("driver_action_payload") or {}), + node_codes=_normalize_driver_node_codes(primary_resolved.get("driver_node_codes") or []), + sequence_key=sequence_key, + focus_ref=_normalize_focus_ref( + normalized_payload.get("primary_focus_ref") + or (primary_resolved.get("sequence") or {}).get("action_focus_ref") + ), + secondary=False, + requested_by=requested_by, + reason=reason, + ), + "secondary": secondary_preview, + "resolved_primary": dict(primary_resolved or {}), + } + + primary_action_code = str(normalized_payload.get("primary_action_code") or "").strip() + secondary_action_code = str(normalized_payload.get("secondary_action_code") or "").strip() + if not primary_action_code and not secondary_action_code: + return False, "当前没有可预览的驾驶动作", {} + + preview_focus_ref = _normalize_focus_ref(normalized_payload.get("focus_ref")) + primary_focus_ref = _normalize_focus_ref(normalized_payload.get("primary_focus_ref")) + secondary_focus_ref = _normalize_focus_ref(normalized_payload.get("secondary_focus_ref")) + + return True, "驾驶动作预览已生成", { + "contract_key": "ops_driver_contract", + "contract_version": str(contract.get("version") or _OPS_CONTRACT_SCHEMA_VERSION).strip(), + "contract_schema_doc_path": str(contract.get("schema_doc_path") or "").strip(), + "contract_primary_endpoint": str(contract.get("primary_endpoint") or "").strip(), + "preview_endpoint": "/api/v1/ops/driver-actions/preview", + "source_label": source_label, + "title": title, + "summary": summary, + "reason": reason, + "meta_text": meta_text, + "executor_kind": executor_kind, + "executor_kind_label": _driver_action_executor_kind_label(executor_kind), + "sequence_key": sequence_key, + "focus_ref": preview_focus_ref, + "primary_focus_ref": primary_focus_ref, + "secondary_focus_ref": secondary_focus_ref, + "execution_chain": _driver_action_execution_chain(executor_kind), + "primary": ( + _build_driver_action_contract_preview_section( + executor_kind=executor_kind, + action_code=primary_action_code, + label=str(normalized_payload.get("primary_label") or "").strip(), + action_payload=dict(normalized_payload.get("primary_action_payload") or {}), + node_codes=_normalize_driver_node_codes(normalized_payload.get("primary_node_codes") or normalized_payload.get("node_codes") or []), + sequence_key=sequence_key, + focus_ref=primary_focus_ref, + secondary=False, + requested_by=requested_by, + reason=reason, + ) + if primary_action_code + else {} + ), + "secondary": ( + _build_driver_action_contract_preview_section( + executor_kind=executor_kind, + action_code=secondary_action_code, + label=str(normalized_payload.get("secondary_label") or "").strip(), + action_payload=dict(normalized_payload.get("secondary_action_payload") or {}), + node_codes=_normalize_driver_node_codes(normalized_payload.get("secondary_node_codes") or normalized_payload.get("node_codes") or []), + sequence_key=sequence_key, + focus_ref=secondary_focus_ref, + secondary=True, + requested_by=requested_by, + reason=reason, + ) + if secondary_action_code + else {} + ), + } + + +def _build_codex_brief_entry(entry: dict) -> dict: + normalized_entry = dict(entry or {}) + if not normalized_entry: + return {} + + executor_kind = str(normalized_entry.get("executor_kind") or "driver_action").strip() or "driver_action" + sequence_key = str(normalized_entry.get("sequence_key") or "").strip() + primary_action_code = str(normalized_entry.get("primary_action_code") or "").strip() + secondary_action_code = str(normalized_entry.get("secondary_action_code") or "").strip() + primary_node_codes = _normalize_driver_node_codes(normalized_entry.get("primary_node_codes") or normalized_entry.get("node_codes") or []) + secondary_node_codes = _normalize_driver_node_codes(normalized_entry.get("secondary_node_codes") or normalized_entry.get("node_codes") or []) + primary_action_payload = dict(normalized_entry.get("primary_action_payload") or {}) + secondary_action_payload = dict(normalized_entry.get("secondary_action_payload") or {}) + primary_execution = _build_driver_action_execution_profile( + primary_action_code, + action_payload=primary_action_payload, + node_codes=primary_node_codes, + ) + secondary_execution = ( + _build_driver_action_execution_profile( + secondary_action_code, + action_payload=secondary_action_payload, + node_codes=secondary_node_codes, + ) + if secondary_action_code + else {} + ) + if executor_kind == "runbook_sequence" and sequence_key: + target_api = f"/api/v1/ops/runbook/sequences/{sequence_key}/execute" + target_method = "POST" + else: + target_api = "/api/v1/ops/driver-actions/execute" + target_method = "POST" + + primary_request = _build_driver_action_request_preview( + executor_kind=executor_kind, + action_code=primary_action_code, + action_payload=primary_action_payload, + node_codes=primary_node_codes, + sequence_key=sequence_key, + secondary=False, + ) + secondary_request = ( + _build_driver_action_request_preview( + executor_kind=executor_kind, + action_code=secondary_action_code, + action_payload=secondary_action_payload, + node_codes=secondary_node_codes, + sequence_key=sequence_key, + secondary=True, + ) + if secondary_action_code + else {} + ) + + recommendation = str(primary_execution.get("recommendation") or "blocked").strip() or "blocked" + automation_level = str(primary_execution.get("automation_level") or "blocked").strip() or "blocked" + executor_mode_hint = str(primary_execution.get("executor_mode_hint") or "").strip() + status_text = { + "auto_execute": "可自动执行", + "confirm_then_execute": "建议确认后执行", + "resolve_first": "先复核再执行", + "open_ui": "需要进入工作区", + "blocked": "当前阻断", + }.get(recommendation, "待判断") + contract_keys = _suggest_ops_contract_keys_for_codex_entry(normalized_entry) + + return { + "key": str(normalized_entry.get("key") or "").strip(), + "kind": str(normalized_entry.get("kind") or "").strip(), + "lane": str(normalized_entry.get("lane") or "").strip(), + "lane_label": str(normalized_entry.get("lane_label") or "").strip(), + "title": str(normalized_entry.get("title") or "").strip(), + "summary": str(normalized_entry.get("summary") or "").strip(), + "reason": str(normalized_entry.get("reason") or "").strip(), + "status": str(normalized_entry.get("status") or "").strip(), + "level_label": str(normalized_entry.get("level_label") or "").strip(), + "tag_type": str(normalized_entry.get("tag_type") or "").strip(), + "meta_text": str(normalized_entry.get("meta_text") or "").strip(), + "detail_lines": list(normalized_entry.get("detail_lines") or []), + "node_codes": _normalize_driver_node_codes(normalized_entry.get("node_codes") or []), + "executor_kind": executor_kind, + "sequence_key": sequence_key, + "focus_ref": _normalize_focus_ref(normalized_entry.get("focus_ref")), + "primary_focus_ref": _normalize_focus_ref(normalized_entry.get("primary_focus_ref")), + "secondary_focus_ref": _normalize_focus_ref(normalized_entry.get("secondary_focus_ref")), + "occurred_at": str(normalized_entry.get("occurred_at") or "").strip(), + "primary_label": str(normalized_entry.get("primary_label") or "").strip(), + "secondary_label": str(normalized_entry.get("secondary_label") or "").strip(), + "primary_action_code": primary_action_code, + "secondary_action_code": secondary_action_code, + "contract_keys": contract_keys, + "contract_navigation": _build_ops_contract_navigation( + contract_keys, + primary_contract_key="ops_driver_contract", + ), + "autopilot": { + "status_text": status_text, + "automation_level": automation_level, + "recommendation": recommendation, + "executor_mode_hint": executor_mode_hint, + "confirm_required": bool(primary_execution.get("confirm_required", False)), + "ui_only": bool(primary_execution.get("ui_only", False)), + "blocked": bool(primary_execution.get("blocked", False)), + "backend_handled": bool(primary_execution.get("backend_handled", False)), + "risk_level": str(primary_execution.get("risk_level") or "unknown").strip() or "unknown", + "reason": str(primary_execution.get("reason") or "").strip(), + "target_method": target_method, + "target_api": target_api, + }, + "primary_execution": { + **primary_execution, + "label": str(normalized_entry.get("primary_label") or "").strip() or primary_action_code, + "node_codes": primary_node_codes, + "action_payload": primary_action_payload, + "request_payload_preview": primary_request, + }, + "secondary_execution": ( + { + **secondary_execution, + "label": str(normalized_entry.get("secondary_label") or "").strip() or secondary_action_code, + "node_codes": secondary_node_codes, + "action_payload": secondary_action_payload, + "request_payload_preview": secondary_request, + } + if secondary_action_code + else {} + ), + } + + +def _build_codex_activity_focus_entry(entry: dict) -> dict: + normalized_entry = dict(entry or {}) + if not normalized_entry: + return {} + contract_keys = _suggest_ops_contract_keys_for_codex_entry(normalized_entry) + return { + "key": str(normalized_entry.get("key") or "").strip(), + "activity_key": str(normalized_entry.get("activity_key") or "").strip(), + "activity_kind": str(normalized_entry.get("activity_kind") or "").strip(), + "title": str(normalized_entry.get("title") or "").strip(), + "subtitle": str(normalized_entry.get("subtitle") or "").strip(), + "summary": str(normalized_entry.get("summary") or "").strip(), + "status": str(normalized_entry.get("status") or "").strip(), + "status_label": str(normalized_entry.get("status_label") or "").strip(), + "meta_text": str(normalized_entry.get("meta_text") or "").strip(), + "occurred_at": str(normalized_entry.get("occurred_at") or "").strip(), + "target_node_codes": _normalize_driver_node_codes(normalized_entry.get("target_node_codes") or []), + "focus_ref": _normalize_focus_ref(normalized_entry.get("focus_ref")), + "source_focus_ref": _normalize_focus_ref(normalized_entry.get("source_focus_ref")), + "ui_intent": dict(normalized_entry.get("ui_intent") or {}), + "ui_intent_kind": str(normalized_entry.get("ui_intent_kind") or "").strip(), + "focus_action_code": str(normalized_entry.get("focus_action_code") or "").strip(), + "focus_action_payload": dict(normalized_entry.get("focus_action_payload") or {}), + "contract_keys": contract_keys, + "contract_navigation": _build_ops_contract_navigation( + contract_keys, + primary_contract_key="ops_driver_contract", + ), + "observation_only": True, + } + + +def _build_codex_action_entry_from_activity_focus(entry: dict) -> dict: + normalized_entry = dict(entry or {}) + if not normalized_entry: + return {} + + key = str(normalized_entry.get("key") or "").strip() + focus_action_code = str(normalized_entry.get("focus_action_code") or "").strip() + focus_ref = _normalize_focus_ref(normalized_entry.get("focus_ref")) + target_node_codes = _normalize_driver_node_codes(normalized_entry.get("target_node_codes") or []) + focus_action_payload = dict(normalized_entry.get("focus_action_payload") or {}) + status_text = str(normalized_entry.get("status_label") or normalized_entry.get("status") or "").strip() + summary = str(normalized_entry.get("summary") or "").strip() + + if not key or not focus_action_code: + return {} + + reason = summary or "该焦点属于运行态观察项，建议先进入对应工作区查看上下文。" + return { + "key": key, + "title": str(normalized_entry.get("title") or "").strip() or key, + "summary": summary, + "reason": reason, + "meta_text": str(normalized_entry.get("meta_text") or "").strip(), + "executor_kind": "driver_action", + "sequence_key": "", + "focus_ref": focus_ref, + "primary_focus_ref": focus_ref, + "secondary_focus_ref": {}, + "primary_label": "进入工作区", + "secondary_label": "", + "primary_action_code": focus_action_code, + "secondary_action_code": "", + "autopilot": { + "status_text": status_text, + "automation_level": "ui_only", + "recommendation": "open_ui", + "executor_mode_hint": "ui-intent", + "confirm_required": False, + "ui_only": True, + "blocked": False, + "backend_handled": True, + "risk_level": "low", + "reason": reason, + "target_method": "POST", + "target_api": "/api/v1/ops/driver-actions/execute", + }, + "primary_execution": { + "label": "进入工作区", + "node_codes": target_node_codes, + "action_payload": focus_action_payload, + "request_payload_preview": { + "action_code": focus_action_code, + "node_codes": target_node_codes, + "action_payload": focus_action_payload, + }, + }, + "secondary_execution": {}, + "activity_key": str(normalized_entry.get("activity_key") or "").strip(), + "activity_kind": str(normalized_entry.get("activity_kind") or "").strip(), + "status": str(normalized_entry.get("status") or "").strip(), + "status_label": status_text, + "occurred_at": str(normalized_entry.get("occurred_at") or "").strip(), + "target_node_codes": target_node_codes, + "source_focus_ref": _normalize_focus_ref(normalized_entry.get("source_focus_ref")), + "ui_intent": dict(normalized_entry.get("ui_intent") or {}), + "ui_intent_kind": str(normalized_entry.get("ui_intent_kind") or "").strip(), + "focus_action_code": focus_action_code, + "focus_action_payload": focus_action_payload, + "observation_only": True, + } + + +def _is_preview_only_driver_action(action_code: str) -> bool: + normalized_action_code = str(action_code or "").strip() + if not normalized_action_code: + return False + if normalized_action_code.startswith(("open_", "focus_", "view_")): + return True + return normalized_action_code in { + "release_package", + "handover_first_gap", + "review_smart_rollout_preview", + "review_control_rollout", + "fix_rollout_blockers", + } + + +def _build_ops_automation_coverage_summary( + entries: list[dict] | None = None, + *, + activity_focus_total: int = 0, + scene_log_observation: dict | None = None, + go_live_summary: dict | None = None, +) -> dict: + normalized_entries = [dict(item or {}) for item in list(entries or []) if dict(item or {})] + normalized_scene_log_observation = dict(scene_log_observation or {}) + normalized_go_live_summary = dict(go_live_summary or {}) + + automation_level_counts: dict[str, int] = {} + recommendation_counts: dict[str, int] = {} + executor_kind_counts: dict[str, int] = {} + backend_handled_total = 0 + preview_only_total = 0 + + for item in normalized_entries: + autopilot = dict(item.get("autopilot") or {}) + automation_level = str(autopilot.get("automation_level") or "unknown").strip() or "unknown" + recommendation = str(autopilot.get("recommendation") or "unknown").strip() or "unknown" + executor_kind = str(item.get("executor_kind") or "unknown").strip() or "unknown" + action_code = str(item.get("primary_action_code") or "").strip() + if bool(autopilot.get("backend_handled", False)): + backend_handled_total += 1 + if _is_preview_only_driver_action(action_code): + preview_only_total += 1 + automation_level_counts[automation_level] = int(automation_level_counts.get(automation_level, 0) or 0) + 1 + recommendation_counts[recommendation] = int(recommendation_counts.get(recommendation, 0) or 0) + 1 + executor_kind_counts[executor_kind] = int(executor_kind_counts.get(executor_kind, 0) or 0) + 1 + + total = len(normalized_entries) + safe_auto_total = int(automation_level_counts.get("safe_auto", 0) or 0) + guarded_auto_total = int(automation_level_counts.get("guarded_auto", 0) or 0) + mixed_total = int(automation_level_counts.get("mixed", 0) or 0) + ui_only_total = int(automation_level_counts.get("ui_only", 0) or 0) + blocked_total = int(automation_level_counts.get("blocked", 0) or 0) + unknown_total = max(total - safe_auto_total - guarded_auto_total - mixed_total - ui_only_total - blocked_total, 0) + auto_execute_total = int(recommendation_counts.get("auto_execute", 0) or 0) + confirm_then_execute_total = int(recommendation_counts.get("confirm_then_execute", 0) or 0) + resolve_first_total = int(recommendation_counts.get("resolve_first", 0) or 0) + open_ui_total = int(recommendation_counts.get("open_ui", 0) or 0) + blocked_recommendation_total = int(recommendation_counts.get("blocked", 0) or 0) + execution_ready_total = auto_execute_total + confirm_then_execute_total + human_dependency_total = resolve_first_total + open_ui_total + blocked_recommendation_total + coverage_ratio = round((backend_handled_total / total), 4) if total > 0 else 1.0 + + launch_status = "ready" + go_live_status = str(normalized_go_live_summary.get("go_live_status") or "").strip() + publish_ready = bool(normalized_go_live_summary.get("publish_ready", False)) + scene_log_status = str(normalized_scene_log_observation.get("status") or "").strip() or "standby" + if blocked_total > 0 or blocked_recommendation_total > 0 or go_live_status == "blocked": + launch_status = "blocked" + elif ( + ui_only_total > 0 + or resolve_first_total > 0 + or go_live_status not in {"", "ready"} + or not publish_ready + ): + launch_status = "attention" + + return { + "total": total, + "automation_level_counts": automation_level_counts, + "recommendation_counts": recommendation_counts, + "executor_kind_counts": executor_kind_counts, + "safe_auto_total": safe_auto_total, + "guarded_auto_total": guarded_auto_total, + "mixed_total": mixed_total, + "ui_only_total": ui_only_total, + "blocked_total": blocked_total, + "unknown_total": unknown_total, + "auto_execute_total": auto_execute_total, + "confirm_then_execute_total": confirm_then_execute_total, + "resolve_first_total": resolve_first_total, + "open_ui_total": open_ui_total, + "blocked_recommendation_total": blocked_recommendation_total, + "backend_handled_total": backend_handled_total, + "preview_only_total": preview_only_total, + "execution_ready_total": execution_ready_total, + "human_dependency_total": human_dependency_total, + "activity_focus_total": int(activity_focus_total or 0), + "scene_log_status": scene_log_status, + "launch_status": launch_status, + "launch_ready": launch_status == "ready", + "backend_coverage_ratio": coverage_ratio, + } + + +def get_ops_codex_brief() -> dict: + driver_feed = get_ops_driver_feed() + go_live_summary = dict(driver_feed.get("go_live_summary") or {}) + contract_registry = get_ops_contract_registry() + scene_log_observation = dict(driver_feed.get("scene_log_observation") or {}) + activity_focus_entries = [ + _build_codex_activity_focus_entry(raw_entry) + for raw_entry in list(driver_feed.get("activity_focus") or []) + if dict(raw_entry or {}) + ] + activity_focus_entries = [entry for entry in activity_focus_entries if entry] + entries = [ + _build_codex_brief_entry(raw_entry) + for raw_entry in list(driver_feed.get("entries") or []) + if dict(raw_entry or {}) + ] + entries = [entry for entry in entries if entry] + automation_coverage = _build_ops_automation_coverage_summary( + entries, + activity_focus_total=len(activity_focus_entries), + scene_log_observation=scene_log_observation, + go_live_summary=go_live_summary, + ) + + focus_entry = dict(entries[0] or {}) if entries else {} + focus_autopilot = dict(focus_entry.get("autopilot") or {}) + focus_payload = { + "entry_key": str(focus_entry.get("key") or "").strip(), + "title": str(focus_entry.get("title") or "").strip(), + "recommendation": str(focus_autopilot.get("recommendation") or "").strip(), + "status_text": str(focus_autopilot.get("status_text") or "").strip(), + "target_api": str(focus_autopilot.get("target_api") or "").strip(), + "target_method": str(focus_autopilot.get("target_method") or "").strip(), + "focus_ref": _normalize_focus_ref(focus_entry.get("focus_ref")), + "primary_focus_ref": _normalize_focus_ref(focus_entry.get("primary_focus_ref")), + "secondary_focus_ref": _normalize_focus_ref(focus_entry.get("secondary_focus_ref")), + } + if not focus_payload["entry_key"]: + first_activity_focus = next( + ( + dict(item or {}) + for item in activity_focus_entries + if str((item or {}).get("focus_action_code") or "").strip() + ), + {}, + ) + else: + first_activity_focus = {} + if not focus_payload["entry_key"] and first_activity_focus: + activity_focus_contract_keys = _suggest_ops_contract_keys_for_codex_entry(first_activity_focus) + focus_payload = { + "entry_key": str(first_activity_focus.get("key") or "").strip(), + "title": str(first_activity_focus.get("title") or "").strip(), + "recommendation": "open_ui" if str(first_activity_focus.get("ui_intent_kind") or "").strip() else "", + "status_text": str(first_activity_focus.get("status_label") or first_activity_focus.get("status") or "").strip(), + "target_api": "", + "target_method": "", + "focus_ref": _normalize_focus_ref(first_activity_focus.get("focus_ref")), + "primary_focus_ref": _normalize_focus_ref(first_activity_focus.get("focus_ref")), + "secondary_focus_ref": {}, + "contract_keys": activity_focus_contract_keys, + "contract_navigation": _build_ops_contract_navigation( + activity_focus_contract_keys, + primary_contract_key="ops_driver_contract", + registry=contract_registry, + ), + } + elif ( + not focus_payload["entry_key"] + and _normalize_focus_ref(scene_log_observation.get("focus_ref")) + ): + scene_log_contract_keys = _normalize_ops_contract_keys( + list(scene_log_observation.get("contract_keys") or []) + + ["ops_driver_contract"] + ) + focus_payload = { + "entry_key": "scene-log-observation", + "title": "现场日志观察", + "recommendation": "open_ui", + "status_text": str(scene_log_observation.get("status_label") or scene_log_observation.get("status") or "").strip(), + "target_api": "", + "target_method": "", + "focus_ref": _normalize_focus_ref(scene_log_observation.get("focus_ref")), + "primary_focus_ref": _normalize_focus_ref(scene_log_observation.get("focus_ref")), + "secondary_focus_ref": {}, + "contract_keys": scene_log_contract_keys, + "contract_navigation": _build_ops_contract_navigation( + scene_log_contract_keys, + primary_contract_key="ops_driver_contract", + registry=contract_registry, + ), + } + elif focus_payload["entry_key"]: + focus_contract_keys = _normalize_ops_contract_keys( + (focus_entry.get("contract_keys") or []) + + _suggest_ops_contract_keys_for_codex_entry(focus_entry) + ) + focus_payload["contract_keys"] = focus_contract_keys + focus_payload["contract_navigation"] = _build_ops_contract_navigation( + focus_contract_keys, + primary_contract_key="ops_driver_contract", + registry=contract_registry, + ) + else: + focus_payload["contract_keys"] = ["ops_driver_contract"] + focus_payload["contract_navigation"] = _build_ops_contract_navigation( + ["ops_driver_contract"], + primary_contract_key="ops_driver_contract", + registry=contract_registry, + ) + + summary_contract_keys = _normalize_ops_contract_keys( + ["ops_driver_contract", "ops_stack_diagnosis_contract"] + + [ + contract_key + for item in entries + for contract_key in list(item.get("contract_keys") or []) + ] + + [ + contract_key + for contract_key in list(focus_payload.get("contract_keys") or []) + ] + ) + + return { + "generated_at": datetime.now().isoformat(timespec="seconds"), + "headline": str(driver_feed.get("headline") or "").strip() or "当前还没有可执行的驾驶主线。", + "driver_feed": driver_feed, + "go_live_summary": go_live_summary, + "scene_log_observation": scene_log_observation, + "focus": focus_payload, + "entries": entries, + "activity_focus": activity_focus_entries, + "automation_coverage": automation_coverage, + "contract_navigation": _build_ops_contract_navigation( + summary_contract_keys, + primary_contract_key="ops_driver_contract", + registry=contract_registry, + ), + "summary": { + "total": len(entries), + "automation_level_counts": dict(automation_coverage.get("automation_level_counts") or {}), + "recommendation_counts": dict(automation_coverage.get("recommendation_counts") or {}), + "executor_kind_counts": dict(automation_coverage.get("executor_kind_counts") or {}), + "go_live_status": str(go_live_summary.get("go_live_status") or "").strip(), + "operator_title": str(go_live_summary.get("operator_title") or "").strip(), + "publish_ready": bool(go_live_summary.get("publish_ready")), + "publish_status": str(go_live_summary.get("publish_status") or "").strip(), + "publish_status_label": str(go_live_summary.get("publish_status_label") or "").strip(), + "launchpad_recommended_target_node_code": str( + go_live_summary.get("launchpad_recommended_target_node_code") or "" + ).strip(), + "launchpad_recommended_recovery_label": str( + go_live_summary.get("launchpad_recommended_recovery_label") or "" + ).strip(), + "launchpad_recommended_recovery_summary": str( + go_live_summary.get("launchpad_recommended_recovery_summary") or "" + ).strip(), + "launchpad_onboarding_bootstrap_pending_nodes": int( + go_live_summary.get("launchpad_onboarding_bootstrap_pending_nodes", 0) or 0 + ), + "launchpad_onboarding_acceptance_ready_nodes": int( + go_live_summary.get("launchpad_onboarding_acceptance_ready_nodes", 0) or 0 + ), + "auto_execute_total": int(automation_coverage.get("auto_execute_total", 0) or 0), + "confirm_then_execute_total": int(automation_coverage.get("confirm_then_execute_total", 0) or 0), + "resolve_first_total": int(automation_coverage.get("resolve_first_total", 0) or 0), + "open_ui_total": int(automation_coverage.get("open_ui_total", 0) or 0), + "blocked_total": int(automation_coverage.get("blocked_recommendation_total", 0) or 0), + "activity_focus_total": int(automation_coverage.get("activity_focus_total", 0) or 0), + "preview_only_total": int(automation_coverage.get("preview_only_total", 0) or 0), + "backend_handled_total": int(automation_coverage.get("backend_handled_total", 0) or 0), + "launch_status": str(automation_coverage.get("launch_status") or "").strip(), + "launch_ready": bool(automation_coverage.get("launch_ready", False)), + }, + } + + +def _build_resolved_driver_action_gate( + preview_data: dict, + *, + secondary: bool = False, + confirm: bool = False, +) -> dict: + normalized_preview = dict(preview_data or {}) + preview_section = dict((normalized_preview.get("secondary") if secondary else normalized_preview.get("primary")) or {}) + preview_execution = dict(preview_section.get("execution_profile") or {}) + + recommendation = str(preview_execution.get("recommendation") or "blocked").strip() or "blocked" + automation_level = str(preview_execution.get("automation_level") or "blocked").strip() or "blocked" + executor_mode_hint = str(preview_execution.get("executor_mode_hint") or "").strip() + risk_level = str(preview_execution.get("risk_level") or "unknown").strip() or "unknown" + confirm_required = bool(preview_execution.get("confirm_required", False)) + ui_only = bool(preview_execution.get("ui_only", False)) + blocked = bool(preview_execution.get("blocked", False)) + reason = str(preview_execution.get("reason") or "").strip() + target_api = str(preview_section.get("target_api") or "").strip() + target_method = str(preview_section.get("target_method") or "POST").strip() or "POST" + request_payload = dict(preview_section.get("request_payload") or {}) + confirm_received = bool(confirm) + + decision = "blocked" + decision_reason = reason or "当前动作被标记为阻断，不能自动执行。" + will_execute = False + if recommendation == "auto_execute": + decision = "execute_now" + will_execute = True + decision_reason = reason or "当前动作属于 safe_auto，可直接执行。" + elif recommendation == "confirm_then_execute": + if confirm_received: + decision = "execute_confirmed" + will_execute = True + decision_reason = reason or "当前动作属于 guarded_auto，已收到确认，允许执行。" + else: + decision = "confirmation_required" + decision_reason = "当前动作属于 guarded_auto，需要显式确认后才能执行。" + if reason: + decision_reason = f"{decision_reason} {reason}" + elif recommendation == "resolve_first": + decision = "resolve_first" + decision_reason = reason or "当前动作要求先复核解析结果，不应直接执行。" + elif recommendation == "open_ui": + decision = "open_ui" + decision_reason = reason or "当前动作属于 UI-only，建议进入工作区处理。" + + decision_label = { + "execute_now": "直接执行", + "execute_confirmed": "确认后执行", + "confirmation_required": "等待确认", + "resolve_first": "先复核", + "open_ui": "进入工作区", + "blocked": "当前阻断", + }.get(decision, decision) + + return { + "decision": decision, + "decision_label": decision_label, + "decision_reason": decision_reason, + "will_execute": will_execute, + "recommendation": recommendation, + "recommendation_label": _driver_action_recommendation_label(recommendation), + "automation_level": automation_level, + "status_text": "", + "executor_mode_hint": executor_mode_hint, + "risk_level": risk_level, + "confirm_required": confirm_required, + "confirm_received": confirm_received, + "ui_only": ui_only, + "blocked": blocked, + "secondary": bool(secondary), + "backend_handled": bool(preview_execution.get("backend_handled", False)), + "target_api": target_api, + "target_method": target_method, + "request_payload": request_payload, + "focus_ref": _normalize_focus_ref( + preview_section.get("focus_ref") + or normalized_preview.get("secondary_focus_ref" if secondary else "primary_focus_ref") + or normalized_preview.get("focus_ref") + ), + "action_code": str(preview_section.get("action_code") or "").strip(), + "sequence_key": str(normalized_preview.get("sequence_key") or "").strip(), + "label": str(preview_section.get("label") or "").strip(), + "node_codes": _normalize_driver_node_codes(preview_section.get("node_codes") or []), + "requires_fields": list(preview_execution.get("requires_fields") or []), + "missing_fields": list(preview_execution.get("missing_fields") or []), + } + + +def resolve_driver_action_request(payload: dict | None = None) -> tuple[bool, str, dict]: + normalized_payload = dict(payload or {}) + secondary = bool(normalized_payload.get("secondary", False)) + confirm = bool(normalized_payload.get("confirm", False)) + + preview_request = { + "source_label": str(normalized_payload.get("source_label") or "ops/driver-action").strip() or "ops/driver-action", + "title": str(normalized_payload.get("title") or "").strip(), + "summary": str(normalized_payload.get("summary") or "").strip(), + "reason": str(normalized_payload.get("reason") or "").strip(), + "meta_text": str(normalized_payload.get("meta_text") or "").strip(), + "executor_kind": str(normalized_payload.get("executor_kind") or "driver_action").strip() or "driver_action", + "sequence_key": str(normalized_payload.get("sequence_key") or "").strip(), + "focus_ref": _normalize_focus_ref(normalized_payload.get("focus_ref")), + "primary_focus_ref": _normalize_focus_ref(normalized_payload.get("primary_focus_ref")), + "secondary_focus_ref": _normalize_focus_ref(normalized_payload.get("secondary_focus_ref")), + "primary_label": str(normalized_payload.get("primary_label") or "").strip(), + "secondary_label": str(normalized_payload.get("secondary_label") or "").strip(), + "primary_action_code": str(normalized_payload.get("primary_action_code") or "").strip(), + "secondary_action_code": str(normalized_payload.get("secondary_action_code") or "").strip(), + "primary_node_codes": _normalize_driver_node_codes(normalized_payload.get("primary_node_codes") or normalized_payload.get("node_codes") or []), + "secondary_node_codes": _normalize_driver_node_codes(normalized_payload.get("secondary_node_codes") or normalized_payload.get("node_codes") or []), + "primary_action_payload": dict(normalized_payload.get("primary_action_payload") or normalized_payload.get("action_payload") or {}), + "secondary_action_payload": dict(normalized_payload.get("secondary_action_payload") or normalized_payload.get("action_payload") or {}), + "include_secondary": bool(normalized_payload.get("include_secondary", True)), + "requested_by": str(normalized_payload.get("requested_by") or "api/driver-action").strip() or "api/driver-action", + } + preview_ok, preview_message, preview_data = preview_driver_action(preview_request) + if not preview_ok: + return False, preview_message, { + "preview_request": preview_request, + "preview": dict(preview_data or {}), + "selected_section": "secondary" if secondary else "primary", + } + + gate = _build_resolved_driver_action_gate(preview_data, secondary=secondary, confirm=confirm) + return True, "驾驶动作解析完成", { + "preview_request": preview_request, + "preview": dict(preview_data or {}), + "gate": gate, + "selected_section": "secondary" if secondary else "primary", + "focus_ref": _normalize_focus_ref( + gate.get("focus_ref") + or preview_data.get("secondary_focus_ref" if secondary else "primary_focus_ref") + or preview_data.get("focus_ref") + ), + } + + +def execute_resolved_driver_action(payload: dict | None = None) -> tuple[bool, str, dict]: + resolve_ok, resolve_message, resolved = resolve_driver_action_request(payload or {}) + if not resolve_ok: + return False, resolve_message, dict(resolved or {}) + + gate = dict((resolved or {}).get("gate") or {}) + if not bool(gate.get("will_execute", False)): + return False, str(gate.get("decision_reason") or "当前不允许自动执行").strip() or "当前不允许自动执行", { + **dict(resolved or {}), + "executed": False, + } + + preview = dict((resolved or {}).get("preview") or {}) + executor_kind = str(preview.get("executor_kind") or "driver_action").strip() or "driver_action" + request_payload = dict(gate.get("request_payload") or {}) + + if executor_kind == "runbook_sequence": + sequence_key = str(gate.get("sequence_key") or preview.get("sequence_key") or "").strip() + if not sequence_key: + return False, "缺少 sequence_key，无法执行 runbook sequence", { + **dict(resolved or {}), + "executed": False, + } + execute_ok, execute_message, execute_data = execute_ops_runbook_sequence(sequence_key, request_payload) + else: + execute_ok, execute_message, execute_data = execute_driver_action(request_payload) + + return execute_ok, execute_message, { + **dict(resolved or {}), + "executed": bool(execute_ok), + "execution_result": dict(execute_data or {}), + } + + +def _select_codex_brief_entry(codex_brief: dict, entry_key: str = "") -> dict: + normalized_entry_key = str(entry_key or "").strip() + entries = [dict(item or {}) for item in list((codex_brief or {}).get("entries") or []) if dict(item or {})] + activity_focus_entries = [ + _build_codex_action_entry_from_activity_focus(item) + for item in list((codex_brief or {}).get("activity_focus") or []) + if dict(item or {}) + ] + activity_focus_entries = [item for item in activity_focus_entries if item] + focus = dict((codex_brief or {}).get("focus") or {}) + target_entry_key = normalized_entry_key or str(focus.get("entry_key") or "").strip() + + if target_entry_key: + for item in entries + activity_focus_entries: + if str(item.get("key") or "").strip() == target_entry_key: + return item + searchable_entries = entries + activity_focus_entries + return dict(searchable_entries[0] or {}) if searchable_entries else {} + + +def _build_codex_action_preview_request( + selected_entry: dict, + *, + requested_by: str, + source_label: str, + include_secondary: bool = True, +) -> dict: + normalized_entry = dict(selected_entry or {}) + autopilot = dict(normalized_entry.get("autopilot") or {}) + primary_execution = dict(normalized_entry.get("primary_execution") or {}) + secondary_execution = dict(normalized_entry.get("secondary_execution") or {}) + requested_by_value = str(requested_by or "api/codex-action").strip() or "api/codex-action" + source_label_value = str(source_label or "ops/codex-action").strip() or "ops/codex-action" + + return { + "source_label": source_label_value, + "title": str(normalized_entry.get("title") or "").strip() or str(normalized_entry.get("key") or "").strip() or "codex-action", + "summary": str(normalized_entry.get("summary") or "").strip(), + "reason": str(normalized_entry.get("reason") or autopilot.get("reason") or "").strip(), + "meta_text": str(normalized_entry.get("meta_text") or "").strip(), + "executor_kind": str(normalized_entry.get("executor_kind") or "driver_action").strip() or "driver_action", + "sequence_key": str(normalized_entry.get("sequence_key") or "").strip(), + "focus_ref": _normalize_focus_ref(normalized_entry.get("focus_ref")), + "primary_focus_ref": _normalize_focus_ref(normalized_entry.get("primary_focus_ref")), + "secondary_focus_ref": _normalize_focus_ref(normalized_entry.get("secondary_focus_ref")), + "primary_label": str(primary_execution.get("label") or normalized_entry.get("primary_label") or "").strip(), + "secondary_label": str(secondary_execution.get("label") or normalized_entry.get("secondary_label") or "").strip(), + "primary_action_code": str(normalized_entry.get("primary_action_code") or "").strip(), + "secondary_action_code": str(normalized_entry.get("secondary_action_code") or "").strip(), + "primary_node_codes": _normalize_driver_node_codes( + primary_execution.get("node_codes") or normalized_entry.get("node_codes") or [] + ), + "secondary_node_codes": _normalize_driver_node_codes( + secondary_execution.get("node_codes") or normalized_entry.get("node_codes") or [] + ), + "primary_action_payload": dict(primary_execution.get("action_payload") or {}), + "secondary_action_payload": dict(secondary_execution.get("action_payload") or {}), + "include_secondary": bool(include_secondary), + "requested_by": requested_by_value, + } + + +def _build_codex_action_gate( + selected_entry: dict, + preview_data: dict, + *, + secondary: bool = False, + confirm: bool = False, +) -> dict: + normalized_entry = dict(selected_entry or {}) + normalized_preview = dict(preview_data or {}) + autopilot = dict(normalized_entry.get("autopilot") or {}) + preview_section = dict((normalized_preview.get("secondary") if secondary else normalized_preview.get("primary")) or {}) + execution_key = "secondary_execution" if secondary else "primary_execution" + entry_execution = dict(normalized_entry.get(execution_key) or {}) + preview_execution = dict(preview_section.get("execution_profile") or {}) + + recommendation = str( + preview_execution.get("recommendation") + or autopilot.get("recommendation") + or "blocked" + ).strip() or "blocked" + automation_level = str( + preview_execution.get("automation_level") + or autopilot.get("automation_level") + or "blocked" + ).strip() or "blocked" + executor_mode_hint = str( + preview_execution.get("executor_mode_hint") + or autopilot.get("executor_mode_hint") + or "" + ).strip() + risk_level = str(preview_execution.get("risk_level") or autopilot.get("risk_level") or "unknown").strip() or "unknown" + confirm_required = bool(preview_execution.get("confirm_required", False) or autopilot.get("confirm_required", False)) + ui_only = bool(preview_execution.get("ui_only", False) or autopilot.get("ui_only", False)) + blocked = bool(preview_execution.get("blocked", False) or autopilot.get("blocked", False)) + reason = str(preview_execution.get("reason") or autopilot.get("reason") or "").strip() + target_api = str(preview_section.get("target_api") or autopilot.get("target_api") or "").strip() + target_method = str(preview_section.get("target_method") or autopilot.get("target_method") or "POST").strip() or "POST" + request_payload = dict(preview_section.get("request_payload") or {}) + confirm_received = bool(confirm) + + decision = "blocked" + decision_reason = reason or "当前焦点被标记为阻断，不能自动执行。" + will_execute = False + if recommendation == "auto_execute": + decision = "execute_now" + will_execute = True + decision_reason = reason or "当前焦点属于 safe_auto，可直接执行。" + elif recommendation == "confirm_then_execute": + if confirm_received: + decision = "execute_confirmed" + will_execute = True + decision_reason = reason or "当前焦点属于 guarded_auto，已收到确认，允许执行。" + else: + decision = "confirmation_required" + decision_reason = "当前焦点属于 guarded_auto，需要显式确认后才能执行。" + if reason: + decision_reason = f"{decision_reason} {reason}" + elif recommendation == "resolve_first": + decision = "resolve_first" + decision_reason = reason or "当前焦点要求先复核解析结果，不应直接执行。" + elif recommendation == "open_ui": + decision = "open_ui" + decision_reason = reason or "当前焦点属于 UI-only，建议进入工作区处理。" + + decision_label = { + "execute_now": "直接执行", + "execute_confirmed": "确认后执行", + "confirmation_required": "等待确认", + "resolve_first": "先复核", + "open_ui": "进入工作区", + "blocked": "当前阻断", + }.get(decision, decision) + + return { + "decision": decision, + "decision_label": decision_label, + "decision_reason": decision_reason, + "will_execute": will_execute, + "recommendation": recommendation, + "recommendation_label": _driver_action_recommendation_label(recommendation), + "automation_level": automation_level, + "status_text": str(autopilot.get("status_text") or "").strip(), + "executor_mode_hint": executor_mode_hint, + "risk_level": risk_level, + "confirm_required": confirm_required, + "confirm_received": confirm_received, + "ui_only": ui_only, + "blocked": blocked, + "secondary": bool(secondary), + "backend_handled": bool(preview_execution.get("backend_handled", False) or autopilot.get("backend_handled", False)), + "target_api": target_api, + "target_method": target_method, + "request_payload": request_payload, + "focus_ref": _normalize_focus_ref( + preview_section.get("focus_ref") + or normalized_entry.get("secondary_focus_ref" if secondary else "primary_focus_ref") + or normalized_entry.get("focus_ref") + ), + "action_code": str( + preview_section.get("action_code") + or normalized_entry.get("secondary_action_code" if secondary else "primary_action_code") + or "" + ).strip(), + "sequence_key": str(normalized_preview.get("sequence_key") or normalized_entry.get("sequence_key") or "").strip(), + "label": str( + preview_section.get("label") + or entry_execution.get("label") + or normalized_entry.get("secondary_label" if secondary else "primary_label") + or "" + ).strip(), + "requires_fields": list(preview_execution.get("requires_fields") or []), + "missing_fields": list(preview_execution.get("missing_fields") or []), + } + + +def resolve_codex_action(payload: dict | None = None) -> tuple[bool, str, dict]: + normalized_payload = dict(payload or {}) + entry_key = str(normalized_payload.get("entry_key") or "").strip() + requested_by = str(normalized_payload.get("requested_by") or "api/codex-action").strip() or "api/codex-action" + source_label = str(normalized_payload.get("source_label") or "ops/codex-action").strip() or "ops/codex-action" + include_secondary = bool(normalized_payload.get("include_secondary", True)) + secondary = bool(normalized_payload.get("secondary", False)) + confirm = bool(normalized_payload.get("confirm", False)) + + codex_brief = get_ops_codex_brief() + selected_entry = _select_codex_brief_entry(codex_brief, entry_key=entry_key) + if not selected_entry: + return False, "codex brief 当前没有可执行条目", {"entry_key": entry_key} + + preview_request = _build_codex_action_preview_request( + selected_entry, + requested_by=requested_by, + source_label=source_label, + include_secondary=include_secondary, + ) + preview_ok, preview_message, preview_data = preview_driver_action(preview_request) + if not preview_ok: + return False, preview_message, { + "entry_key": str(selected_entry.get("key") or "").strip(), + "selected_entry": selected_entry, + "preview_request": preview_request, + "preview": dict(preview_data or {}), + } + + gate = _build_codex_action_gate(selected_entry, preview_data, secondary=secondary, confirm=confirm) + return True, "Codex 驾驶动作解析完成", { + "entry_key": str(selected_entry.get("key") or "").strip(), + "selected_entry": selected_entry, + "preview_request": preview_request, + "preview": dict(preview_data or {}), + "gate": gate, + "selected_section": "secondary" if secondary else "primary", + "focus": dict(codex_brief.get("focus") or {}), + "summary": dict(codex_brief.get("summary") or {}), + "focus_ref": _normalize_focus_ref( + gate.get("focus_ref") + or selected_entry.get("secondary_focus_ref" if secondary else "primary_focus_ref") + or selected_entry.get("focus_ref") + ), + } + + +def execute_codex_action(payload: dict | None = None) -> tuple[bool, str, dict]: + resolve_ok, resolve_message, resolved = resolve_codex_action(payload or {}) + if not resolve_ok: + return False, resolve_message, dict(resolved or {}) + + gate = dict((resolved or {}).get("gate") or {}) + if not bool(gate.get("will_execute", False)): + selected_entry = dict((resolved or {}).get("selected_entry") or {}) + blocked_message = str(gate.get("decision_reason") or "").strip() + if str(gate.get("decision") or "").strip() == "open_ui": + blocked_message = blocked_message or "当前焦点属于 UI-only，建议进入工作区处理。" + if bool(selected_entry.get("observation_only", False)) and "工作区" not in blocked_message: + blocked_message = f"{blocked_message} 请进入对应工作区查看当前观察项。".strip() + + return False, blocked_message or "当前不允许自动执行", { + **dict(resolved or {}), + "executed": False, + } + + selected_entry = dict((resolved or {}).get("selected_entry") or {}) + preview = dict((resolved or {}).get("preview") or {}) + executor_kind = str(preview.get("executor_kind") or selected_entry.get("executor_kind") or "driver_action").strip() or "driver_action" + request_payload = dict(gate.get("request_payload") or {}) + + if executor_kind == "runbook_sequence": + sequence_key = str(gate.get("sequence_key") or preview.get("sequence_key") or selected_entry.get("sequence_key") or "").strip() + if not sequence_key: + return False, "缺少 sequence_key，无法执行 runbook sequence", { + **dict(resolved or {}), + "executed": False, + } + execute_ok, execute_message, execute_data = execute_ops_runbook_sequence(sequence_key, request_payload) + else: + execute_ok, execute_message, execute_data = execute_driver_action(request_payload) + + return execute_ok, execute_message, { + **dict(resolved or {}), + "executed": bool(execute_ok), + "execution_result": dict(execute_data or {}), + } + + +def execute_driver_action(payload: dict) -> tuple[bool, str, dict]: + action_code = str((payload or {}).get("action_code") or "").strip() + requested_by = str((payload or {}).get("requested_by") or "api").strip() or "api" + node_codes = _normalize_driver_node_codes((payload or {}).get("node_codes") or []) + action_payload = dict((payload or {}).get("action_payload") or {}) + + if not action_code: + return False, "action_code 不能为空", {} + + if action_code not in _BACKEND_DRIVER_ACTION_CODES: + return True, "该驾驶动作暂需前端交互或尚未后端化", { + "handled": False, + "manual_required": True, + "action_code": action_code, + "node_codes": node_codes, + } + + if action_code == "enable_log_sync_key": + updated = update_runtime_settings( + { + **get_runtime_settings(), + "worker_log_sync_enabled": True, + "worker_log_sync_mode": "key", + } + ) + return True, "远端日志回传已切到关键模式", { + "handled": True, + "action_code": action_code, + "mode": "settings", + "runtime_settings": updated, + } + + if action_code == "disable_log_sync": + updated = update_runtime_settings( + { + **get_runtime_settings(), + "worker_log_sync_enabled": False, + "worker_log_sync_mode": "key", + } + ) + return True, "远端日志回传已关闭", { + "handled": True, + "action_code": action_code, + "mode": "settings", + "runtime_settings": updated, + } + + if action_code == "enable_log_sync_full": + updated = update_runtime_settings( + { + **get_runtime_settings(), + "worker_log_sync_enabled": True, + "worker_log_sync_mode": "full", + } + ) + return True, "远端日志回传已切到全量模式", { + "handled": True, + "action_code": action_code, + "mode": "settings", + "runtime_settings": updated, + } + + if action_code == "open_playbook_dialog": + playbook_key = str(action_payload.get("playbook_key") or "").strip() + if not playbook_key: + return False, "该驾驶动作需要 playbook_key", {"handled": False, "action_code": action_code} + target_node_codes = _normalize_driver_node_codes(action_payload.get("target_node_codes") or node_codes) + requested_execution_mode = str(action_payload.get("execution_mode") or "").strip() + if not requested_execution_mode: + requested_execution_mode = _recommend_driver_execution_mode_for_playbook( + playbook_key, + target_node_codes, + ).get("execution_mode", "") + preview_ok, preview_message, playbook_preview = preview_ops_playbook( + { + "playbook_key": playbook_key, + "node_codes": target_node_codes, + "execution_mode": requested_execution_mode, + "auto_approve": bool(action_payload.get("auto_approve", False)), + "requested_by": requested_by, + } + ) + if not preview_ok: + return False, preview_message, {"handled": False, "action_code": action_code} + return True, "", { + "handled": True, + "action_code": action_code, + "mode": "playbook-preview", + "playbook_preview": playbook_preview, + "ui_intent": _build_driver_ui_intent( + "open_playbook_dialog", + playbook_key=playbook_key, + target_node_codes=target_node_codes, + execution_mode=requested_execution_mode, + auto_approve=bool(action_payload.get("auto_approve", False)), + ), + } + + if action_code == "open_action_template_dialog": + template_key = str(action_payload.get("template_key") or "").strip() + if not template_key: + return False, "该驾驶动作需要 template_key", {"handled": False, "action_code": action_code} + template = dict(get_ops_action_template(template_key) or {}) + target_node_codes = _normalize_driver_node_codes(action_payload.get("target_node_codes") or node_codes) + requested_execution_mode = str(action_payload.get("execution_mode") or "").strip() + if not requested_execution_mode: + requested_execution_mode = _recommend_driver_execution_mode_for_action_template( + template_key, + target_node_codes, + ).get("execution_mode", "") + return True, "", { + "handled": True, + "action_code": action_code, + "mode": "action-template-preview", + "template_preview": { + "template": template, + "target_node_codes": target_node_codes, + "execution_mode": requested_execution_mode, + "payload": dict(action_payload.get("payload") or {}), + }, + "ui_intent": _build_driver_ui_intent( + "open_action_template_dialog", + template_key=template_key, + target_node_codes=target_node_codes, + execution_mode=requested_execution_mode, + auto_approve=bool(action_payload.get("auto_approve", False)), + payload=dict(action_payload.get("payload") or {}), + ), + } + + if action_code in {"replay_delivery_queue", "flush_delivery_queue"}: + template_key = "delivery.queue.replay" if action_code == "replay_delivery_queue" else "delivery.queue.flush" + target_node_codes = _normalize_driver_node_codes(action_payload.get("target_node_codes") or node_codes) + if not target_node_codes: + return False, "该驾驶动作需要 target_node_codes", {"handled": False, "action_code": action_code} + requested_execution_mode = str(action_payload.get("execution_mode") or "").strip() + if not requested_execution_mode: + requested_execution_mode = _recommend_driver_execution_mode_for_action_template( + template_key, + target_node_codes, + ).get("execution_mode", "") + batch_payload = { + "template_key": template_key, + "target_node_codes": target_node_codes, + "requested_by": f"{requested_by}/{action_code}", + "execution_mode": requested_execution_mode or "remote-agent", + "auto_approve": bool(action_payload.get("auto_approve", False)), + "payload": dict(action_payload.get("payload") or {}), + "metadata": { + "source": "ops-driver", + "driver_action_code": action_code, + }, + } + ok, message, data = create_ops_job_batch(batch_payload) + return ok, (message or ("队列修复任务已创建" if ok else "队列修复任务创建失败")), { + "handled": ok, + "action_code": action_code, + "mode": "ops-job-batch", + "template_key": template_key, + "execution_mode": batch_payload["execution_mode"], + "target_node_codes": target_node_codes, + "job_batch": data, + } + + if action_code == "focus_playbook_run": + run_code = str(action_payload.get("run_code") or "").strip() + if not run_code: + return False, "该驾驶动作需要 run_code", {"handled": False, "action_code": action_code} + run_detail = get_ops_playbook_run(run_code) + run_events = list_ops_playbook_run_events( + run_code, + limit=40, + step_key=str(action_payload.get("focus_step_key") or "").strip(), + node_code=str(action_payload.get("node_code") or "").strip(), + ) + return True, "", { + "handled": True, + "action_code": action_code, + "mode": "playbook-run-focus", + "playbook_run_focus": { + "run": run_detail, + "events": run_events, + }, + "ui_intent": _build_driver_ui_intent( + "playbook_run_detail", + run_code=run_code, + focus_step_key=str(action_payload.get("focus_step_key") or "").strip(), + focus_step_title=str(action_payload.get("focus_step_title") or "").strip(), + ), + } + + if action_code == "focus_activity_item": + ui_intent = dict(action_payload.get("ui_intent") or {}) + if not str(ui_intent.get("kind") or "").strip(): + return False, "该驾驶动作需要 ui_intent", {"handled": False, "action_code": action_code} + activity_key = str(action_payload.get("activity_key") or "").strip() + activity_kind = str(action_payload.get("kind") or "").strip() + activity_item = _find_activity_stream_item(activity_key, kind=activity_kind) if activity_key else {} + if not activity_item: + activity_item = { + "activity_key": activity_key, + "kind": activity_kind, + "status": str(action_payload.get("status") or "").strip(), + "summary": str(action_payload.get("summary") or "").strip(), + "focus_ref": _normalize_focus_ref(action_payload.get("focus_ref")), + "ui_intent": ui_intent, + } + return True, "", { + "handled": True, + "action_code": action_code, + "mode": "activity-focus", + "activity_focus_preview": { + "activity": activity_item, + }, + "ui_intent": ui_intent, + } + + if action_code == "focus_latest_job_events": + ui_intent = dict(action_payload.get("ui_intent") or {}) + if not str(ui_intent.get("kind") or "").strip(): + source_focus_ref = _normalize_focus_ref(action_payload.get("source_focus_ref")) + focus_ref = _normalize_focus_ref(action_payload.get("focus_ref")) + resolved_focus_ref = source_focus_ref or focus_ref + if not resolved_focus_ref: + return False, "该驾驶动作需要 ui_intent 或 focus_ref", {"handled": False, "action_code": action_code} + ui_intent = _build_driver_ui_intent( + "job_events", + job_id=int(resolved_focus_ref.get("job_id") or focus_ref.get("job_id") or 0), + job_code=str(resolved_focus_ref.get("job_code") or focus_ref.get("job_code") or "").strip(), + event_key=str(resolved_focus_ref.get("event_key") or "").strip(), + target_node_code=str( + resolved_focus_ref.get("target_node_code") or focus_ref.get("target_node_code") or "" + ).strip(), + ) + job_id = int(ui_intent.get("job_id") or action_payload.get("job_id") or 0) + job_events = list_ops_job_events_for_jobs([job_id], limit=60) if job_id > 0 else [] + return True, "", { + "handled": True, + "action_code": action_code, + "mode": "job-events-focus", + "job_events_preview": { + "job_id": job_id, + "job_code": str(ui_intent.get("job_code") or action_payload.get("job_code") or "").strip(), + "events": job_events, + }, + "ui_intent": ui_intent, + } + + if action_code == "open_playbook_run_latest_events": + run_code = str(action_payload.get("run_code") or "").strip() + if not run_code: + return False, "该驾驶动作需要 run_code", {"handled": False, "action_code": action_code} + run_events = list_ops_playbook_run_events( + run_code, + limit=60, + step_key=str(action_payload.get("focus_step_key") or "").strip(), + node_code=str(action_payload.get("node_code") or "").strip(), + ) + return True, "", { + "handled": True, + "action_code": action_code, + "mode": "playbook-run-events", + "playbook_run_events_preview": run_events, + "ui_intent": _build_driver_ui_intent( + "playbook_run_latest_events", + run_code=run_code, + focus_step_key=str(action_payload.get("focus_step_key") or "").strip(), + focus_step_title=str(action_payload.get("focus_step_title") or "").strip(), + ), + } + + if action_code == "open_release_dialog": + release_package_preview = get_latest_release_package_metadata() + release_summary = get_release_summary() + return True, "", { + "handled": True, + "action_code": action_code, + "mode": "release-dialog-preview", + "release_dialog_preview": { + "summary": release_summary, + "release_package_preview": release_package_preview, + }, + "ui_intent": _build_driver_ui_intent("open_release_dialog"), + } + + if action_code == "release_package": + release_package_preview = get_latest_release_package_metadata() + return True, "", { + "handled": True, + "action_code": action_code, + "mode": "release-package-preview", + "release_package_preview": release_package_preview, + "ui_intent": _build_driver_ui_intent("open_release_dialog"), + } + + if action_code == "release_prepare": + ok, message, data = prepare_latest_release_package( + requested_by=str(action_payload.get("requested_by") or f"{requested_by}/{action_code}").strip() + or f"{requested_by}/{action_code}" + ) + return ok, message, { + "handled": ok, + "action_code": action_code, + "mode": "release-prepare", + "release_prepare_result": data or {}, + } + + if action_code == "api-restart": + launchpad_context = _driver_launchpad_context(action_payload) + runtime_build_info = get_runtime_build_info() + recommended_command = build_bash_command( + "drive_ops_center.sh", + "runtime-refresh-recover", + launchpad_context["control_plane_base_url"], + ) + return True, "建议先刷新控制面运行时，再继续 Launchpad 评估。", { + "handled": True, + "action_code": action_code, + "mode": "runtime-refresh-preview", + "runtime_build_info": runtime_build_info, + "recommended_command": recommended_command, + "ui_intent": _build_driver_ui_intent("open_stack_diagnosis"), + } + + if action_code == "publish_latest_worker": + launchpad_context = _driver_launchpad_context(action_payload) + ok, message, data = create_release_and_smart_rollout_from_latest_package( + { + **action_payload, + "channel": launchpad_context["channel"], + "mode": "worker", + "created_by": str(action_payload.get("created_by") or f"{requested_by}/{action_code}").strip() + or f"{requested_by}/{action_code}", + "rollout_created_by": str( + action_payload.get("rollout_created_by") or f"{requested_by}/{action_code}/rollout" + ).strip() + or f"{requested_by}/{action_code}/rollout", + }, + control_plane_base_url=launchpad_context["control_plane_base_url"], + ) + return ok, message, { + "handled": ok, + "action_code": action_code, + "mode": "smart-release-rollout", + "smart_release_result": data or {}, + } + + if action_code in {"review_smart_rollout_preview", "review_control_rollout", "fix_rollout_blockers"}: + launchpad_context = _driver_launchpad_context(action_payload) + release_launchpad = get_release_launchpad( + control_plane_base_url=launchpad_context["control_plane_base_url"], + channel=launchpad_context["channel"], + ) + review_mode = "control" if action_code == "review_control_rollout" else "worker" + review_payload = _build_release_launchpad_review_payload( + release_launchpad=release_launchpad, + mode=review_mode, + ) + latest_release = dict(review_payload.get("latest_release") or {}) + return True, "", { + "handled": True, + "action_code": action_code, + "mode": "release-launchpad-preview", + "review_mode": review_mode, + "release_launchpad_review": review_payload, + "ui_intent": _build_driver_ui_intent( + "release_launchpad_review", + mode=review_mode, + release_id=int(latest_release.get("id") or 0), + channel=launchpad_context["channel"], + ), + } + + if action_code == "fix_managed_nodes": + launchpad_context = _driver_launchpad_context(action_payload) + release_launchpad = get_release_launchpad( + control_plane_base_url=launchpad_context["control_plane_base_url"], + channel=launchpad_context["channel"], + ) + first_gap_row = _first_release_launchpad_gap_row(release_launchpad) + first_gap_node_code = str(first_gap_row.get("node_code") or "").strip() + if first_gap_node_code: + handover = get_managed_node_handover( + first_gap_node_code, + control_plane_base_url=launchpad_context["control_plane_base_url"], + ) + return True, "", { + "handled": True, + "action_code": action_code, + "mode": "ui-intent", + "ui_intent": _build_driver_ui_intent( + "managed_node_handover", + node_code=first_gap_node_code, + ), + "handover": handover, + } + sync_result = sync_managed_nodes_from_cluster(dry_run=False) + return True, "已按集群快照同步托管节点，请继续检查接管状态。", { + "handled": True, + "action_code": action_code, + "mode": "cluster-sync", + "sync_result": sync_result, + } + + if action_code in {"bootstrap_run", "run_acceptance"}: + target_node_code = str(action_payload.get("node_code") or (node_codes[0] if node_codes else "")).strip() + if not target_node_code: + launchpad_context = _driver_launchpad_context(action_payload) + release_launchpad = get_release_launchpad( + control_plane_base_url=launchpad_context["control_plane_base_url"], + channel=launchpad_context["channel"], + ) + first_gap_row = _first_release_launchpad_gap_row(release_launchpad) + target_node_code = str(first_gap_row.get("node_code") or "").strip() + if not target_node_code: + target_node_code = str( + (dict(release_launchpad.get("launchpad_status") or {}).get("recommended_target_node_code") or "") + ).strip() + if not target_node_code: + return False, "该驾驶动作需要 node_code", {"handled": False, "action_code": action_code} + ok, message, data = execute_managed_node_onboarding_recovery( + node_code=target_node_code, + requested_by=f"{requested_by}/{action_code}", + control_plane_base_url=str(action_payload.get("control_plane_base_url") or "").strip(), + root_dir=str(action_payload.get("root_dir") or "").strip(), + ) + return ok, message, { + "handled": ok, + "action_code": action_code, + "mode": "node-onboarding-recovery", + "node_code": target_node_code, + "recovery_result": data or {}, + } + + if action_code in { + "create_smart_release_rollout_worker", + "create_smart_release_rollout_control", + "create_release_rollout_worker", + "create_release_rollout_control", + }: + rollout_mode = "control" if action_code.endswith("_control") else "worker" + normalized_payload = { + **action_payload, + "mode": rollout_mode, + "created_by": str(action_payload.get("created_by") or f"{requested_by}/{action_code}").strip() + or f"{requested_by}/{action_code}", + "rollout_created_by": str( + action_payload.get("rollout_created_by") or f"{requested_by}/{action_code}/rollout" + ).strip() + or f"{requested_by}/{action_code}/rollout", + "confirm_risky": bool(action_payload.get("confirm_risky", False)), + } + if action_code.startswith("create_release_rollout_"): + release_id = int(action_payload.get("release_id") or (_preferred_release_for_ops().get("id") or 0) or 0) + if release_id <= 0: + return False, "该驾驶动作需要 release_id", {"handled": False, "action_code": action_code} + ok, message, data = create_smart_release_rollout(release_id, normalized_payload) + else: + ok, message, data = create_release_and_smart_rollout_from_latest_package(normalized_payload) + return ok, message, { + "handled": ok, + "action_code": action_code, + "mode": "smart-release-rollout", + "smart_release_result": data or {}, + } + + if action_code == "open_rollout_dialog": + release_id = int(action_payload.get("release_id") or (_preferred_release_for_ops().get("id") or 0) or 0) + release_summary = get_release_summary() + release_launchpad = get_release_launchpad() + return True, "", { + "handled": True, + "action_code": action_code, + "mode": "rollout-dialog-preview", + "rollout_dialog_preview": { + "summary": release_summary, + "launchpad": release_launchpad, + }, + "ui_intent": _build_driver_ui_intent("open_rollout_dialog", release_id=release_id), + } + + if action_code == "focus_release_hub": + release_id = int(action_payload.get("release_id") or (_preferred_release_for_ops().get("id") or 0) or 0) + launchpad_context = _driver_launchpad_context(action_payload) + release_summary = get_release_summary() + release_launchpad = get_release_launchpad( + control_plane_base_url=launchpad_context["control_plane_base_url"], + channel=launchpad_context["channel"], + ) + return True, "", { + "handled": True, + "action_code": action_code, + "mode": "release-hub-preview", + "release_hub_preview": { + "summary": release_summary, + "launchpad": release_launchpad, + }, + "ui_intent": _build_driver_ui_intent("focus_release_hub", release_id=release_id), + } + + if action_code in {"open_release_deploy_control", "open_release_deploy_worker", "open_release_deploy_custom"}: + template_key_map = { + "open_release_deploy_control": "deploy.release.control", + "open_release_deploy_worker": "deploy.release.worker", + "open_release_deploy_custom": "deploy.release.custom", + } + template_key = template_key_map.get(action_code, "deploy.release.custom") + template = dict(get_ops_action_template(template_key) or {}) + target_node_codes = _normalize_driver_node_codes(action_payload.get("target_node_codes") or node_codes) + release_id = int(action_payload.get("release_id") or (_preferred_release_for_ops().get("id") or 0) or 0) + execution_mode = str(action_payload.get("execution_mode") or "").strip() + if not execution_mode: + execution_mode = _recommend_driver_execution_mode(target_node_codes).get("execution_mode", "remote-agent") + release_summary = get_release_summary() + release_launchpad = get_release_launchpad() + return True, "", { + "handled": True, + "action_code": action_code, + "mode": "release-deploy-preview", + "release_deploy_preview": { + "template": template, + "summary": release_summary, + "launchpad": release_launchpad, + "target_node_codes": target_node_codes, + "execution_mode": execution_mode, + "release_id": release_id, + }, + "ui_intent": _build_driver_ui_intent( + "open_release_deploy_template", + template_key=template_key, + release_id=release_id, + target_node_codes=target_node_codes, + execution_mode=execution_mode, + ), + } + + if action_code in {"handover_first_gap", "view_first_gap"}: + target_node_code = str(action_payload.get("node_code") or (node_codes[0] if node_codes else "")).strip() + if not target_node_code: + return False, "该驾驶动作需要 node_code", {"handled": False, "action_code": action_code} + handover = get_managed_node_handover( + target_node_code, + control_plane_base_url=str(action_payload.get("control_plane_base_url") or "").strip(), + ) + return True, "", { + "handled": True, + "action_code": action_code, + "mode": "handover-preview", + "handover_preview": handover, + "ui_intent": _build_driver_ui_intent( + "managed_node_handover" if action_code == "handover_first_gap" else "managed_node_edit", + node_code=target_node_code, + ), + "handover": handover, + } + + if not node_codes: + return False, "该驾驶动作需要 node_codes", { + "handled": False, + "action_code": action_code, + } + + if action_code in {"open_worker_logs_participating", "open_worker_logs_standby", "open_worker_logs"}: + playbook_key = "scene.logs.key" + execution_mode = str(action_payload.get("execution_mode") or "").strip() + if not execution_mode: + execution_mode = _recommend_driver_execution_mode(node_codes).get("execution_mode", "remote-agent") + ok, message, data = execute_ops_playbook( + { + "playbook_key": playbook_key, + "node_codes": node_codes, + "execution_mode": execution_mode, + "auto_approve": True, + "requested_by": f"{requested_by}/{action_code}", + } + ) + return ok, message, { + "handled": ok, + "action_code": action_code, + "mode": "ops-playbook", + "playbook_key": playbook_key, + "node_codes": node_codes, + "execution_mode": execution_mode, + **(data or {}), + } + + if action_code in {"run_scene_logs_key", "run_scene_logs_full"}: + playbook_key = "scene.logs.full" if action_code == "run_scene_logs_full" else "scene.logs.key" + execution_mode = str(action_payload.get("execution_mode") or "").strip() + if not execution_mode: + execution_mode = _recommend_driver_execution_mode(node_codes).get("execution_mode", "remote-agent") + ok, message, data = execute_ops_playbook( + { + "playbook_key": playbook_key, + "node_codes": node_codes, + "execution_mode": execution_mode, + "auto_approve": bool(action_payload.get("auto_approve", True)), + "requested_by": f"{requested_by}/{action_code}", + } + ) + return ok, message, { + "handled": ok, + "action_code": action_code, + "mode": "ops-playbook", + "playbook_key": playbook_key, + "node_codes": node_codes, + "execution_mode": execution_mode, + **(data or {}), + } + + if action_code in {"open_diagnostics_participating", "open_diagnostics"}: + playbook_key = "scene.diagnostics" + execution_mode = str(action_payload.get("execution_mode") or "").strip() + if not execution_mode: + execution_mode = _recommend_driver_execution_mode(node_codes).get("execution_mode", "remote-agent") + ok, message, data = execute_ops_playbook( + { + "playbook_key": playbook_key, + "node_codes": node_codes, + "execution_mode": execution_mode, + "auto_approve": True, + "requested_by": f"{requested_by}/{action_code}", + } + ) + return ok, message, { + "handled": ok, + "action_code": action_code, + "mode": "ops-playbook", + "playbook_key": playbook_key, + "node_codes": node_codes, + "execution_mode": execution_mode, + **(data or {}), + } + + if action_code in {"run_inspection_participating", "run_inspection_standby", "run_standard_inspection"}: + playbook_key = "inspection.standard" + execution_mode = str(action_payload.get("execution_mode") or "").strip() + if not execution_mode: + execution_mode = _recommend_driver_execution_mode(node_codes).get("execution_mode", "remote-agent") + ok, message, data = execute_ops_playbook( + { + "playbook_key": playbook_key, + "node_codes": node_codes, + "execution_mode": execution_mode, + "auto_approve": True, + "requested_by": f"{requested_by}/{action_code}", + } + ) + return ok, message, { + "handled": ok, + "action_code": action_code, + "mode": "ops-playbook", + "playbook_key": playbook_key, + "node_codes": node_codes, + "execution_mode": execution_mode, + **(data or {}), + } + + return True, "该驾驶动作暂需前端交互或尚未后端化", { + "handled": False, + "manual_required": True, + "action_code": action_code, + "node_codes": node_codes, + } + + +def _resolve_ops_runbook_sequence_entry( + sequence: dict, + *, + secondary: bool = False, + payload: dict | None = None, + requested_by: str = "api", +) -> tuple[bool, str, dict]: + normalized_sequence = dict(sequence or {}) + normalized_payload = dict(payload or {}) + normalized_sequence_key = str(normalized_sequence.get("key") or "").strip() + if not normalized_sequence_key: + return False, "sequence_key 不能为空", {} + + action_field = "secondary_action_code" if secondary else "primary_action_code" + payload_field = "secondary_action_payload" if secondary else "primary_action_payload" + label_field = "secondary_label" if secondary else "primary_label" + action_code = str(normalized_sequence.get(action_field) or "").strip() + if not action_code: + return False, "当前标准作业路径没有可执行动作", { + "sequence_key": normalized_sequence_key, + "secondary": secondary, + "sequence": normalized_sequence, + } + + sequence_node_codes = _normalize_driver_node_codes(normalized_sequence.get("target_node_codes") or []) + override_node_codes = _normalize_driver_node_codes(normalized_payload.get("node_codes") or []) + node_codes = override_node_codes or sequence_node_codes + action_payload = { + **dict(normalized_sequence.get(payload_field) or {}), + **dict(normalized_payload.get("action_payload") or {}), + } + sequence_focus_ref = _normalize_focus_ref(normalized_sequence.get("focus_ref")) + action_focus_ref = _normalize_focus_ref( + normalized_sequence.get("secondary_focus_ref" if secondary else "primary_focus_ref") + ) + if sequence_focus_ref and not action_payload.get("focus_ref"): + action_payload["focus_ref"] = action_focus_ref or sequence_focus_ref + normalized_requested_by = str(requested_by or "api").strip().rstrip("/") or "api" + driver_requested_by = ( + f"{normalized_requested_by}/{normalized_sequence_key}" + if normalized_requested_by.endswith("/runbook") + else f"{normalized_requested_by}/runbook/{normalized_sequence_key}" + ) + sequence_summary = { + "key": normalized_sequence_key, + "title": str(normalized_sequence.get("title") or "").strip(), + "status": str(normalized_sequence.get("status") or "").strip(), + "status_label": str(normalized_sequence.get("status_label") or "").strip(), + "summary": str(normalized_sequence.get("summary") or "").strip(), + "reason": str(normalized_sequence.get("reason") or "").strip(), + "secondary": secondary, + "action_label": str(normalized_sequence.get(label_field) or "").strip(), + "target_scope_label": str(normalized_sequence.get("target_scope_label") or "").strip(), + "step_titles": [str(item or "").strip() for item in list(normalized_sequence.get("step_titles") or []) if str(item or "").strip()], + "focus_ref": sequence_focus_ref, + "action_focus_ref": action_focus_ref, + } + return True, "标准作业路径解析成功", { + "sequence_key": normalized_sequence_key, + "sequence": sequence_summary, + "driver_action_code": action_code, + "driver_action_label": str(normalized_sequence.get(label_field) or "").strip() or action_code, + "driver_node_codes": node_codes, + "driver_action_payload": action_payload, + "driver_requested_by": driver_requested_by, + "resolved_at": datetime.now().isoformat(timespec="seconds"), + } + + +def _attach_ops_runbook_sequence_resolutions( + sequences: list[dict], + *, + requested_by: str = "api/runbook-snapshot", +) -> list[dict]: + annotated_sequences: list[dict] = [] + for raw_sequence in list(sequences or []): + sequence = dict(raw_sequence or {}) + if not str(sequence.get("key") or "").strip(): + continue + primary_ok, primary_message, primary_data = _resolve_ops_runbook_sequence_entry( + sequence, + secondary=False, + requested_by=requested_by, + ) + sequence["primary_resolution"] = { + "ok": primary_ok, + "message": primary_message, + **dict(primary_data or {}), + } + if str(sequence.get("secondary_action_code") or "").strip(): + secondary_ok, secondary_message, secondary_data = _resolve_ops_runbook_sequence_entry( + sequence, + secondary=True, + requested_by=requested_by, + ) + sequence["secondary_resolution"] = { + "ok": secondary_ok, + "message": secondary_message, + **dict(secondary_data or {}), + } + else: + sequence["secondary_resolution"] = {} + annotated_sequences.append(sequence) + return annotated_sequences + + +def resolve_ops_runbook_sequence(sequence_key: str, payload: dict | None = None) -> tuple[bool, str, dict]: + normalized_sequence_key = str(sequence_key or "").strip() + normalized_payload = dict(payload or {}) + if not normalized_sequence_key: + return False, "sequence_key 不能为空", {} + + requested_by = str(normalized_payload.get("requested_by") or "api").strip() or "api" + secondary = bool(normalized_payload.get("secondary", False)) + runbook = get_ops_runbook() + sequences = [ + dict(item or {}) + for item in list(runbook.get("control_sequences") or []) + if str((item or {}).get("key") or "").strip() + ] + sequence = next( + (item for item in sequences if str(item.get("key") or "").strip() == normalized_sequence_key), + {}, + ) + if not sequence: + return False, "标准作业路径不存在", {"sequence_key": normalized_sequence_key} + return _resolve_ops_runbook_sequence_entry( + sequence, + secondary=secondary, + payload=normalized_payload, + requested_by=requested_by, + ) + + +def execute_ops_runbook_sequence(sequence_key: str, payload: dict | None = None) -> tuple[bool, str, dict]: + ok, message, resolved = resolve_ops_runbook_sequence(sequence_key, payload or {}) + if not ok: + return ok, message, resolved + + action_code = str(resolved.get("driver_action_code") or "").strip() + node_codes = _normalize_driver_node_codes(resolved.get("driver_node_codes") or []) + action_payload = dict(resolved.get("driver_action_payload") or {}) + ok, message, data = execute_driver_action( + { + "action_code": action_code, + "node_codes": node_codes, + "action_payload": action_payload, + "requested_by": str(resolved.get("driver_requested_by") or "api/runbook").strip() or "api/runbook", + } + ) + return ok, message, { + **dict(resolved or {}), + **dict(data or {}), + } + + +def get_ops_inspection_overview( + *, + managed_nodes: list[dict] | None = None, + fetch_limit: int = _OPS_INSPECTION_FETCH_LIMIT, + status: str = "", + problem_kind: str = "", + query: str = "", + only_problem: bool = False, + only_participating: bool = False, +) -> dict: + nodes = list(managed_nodes or []) + if managed_nodes is None: + managed_nodes_payload = list_managed_nodes_with_agent_state() + nodes = list(managed_nodes_payload.get("nodes") or []) + jobs = list_ops_jobs(limit=fetch_limit, compact=False) + + inspection_jobs = [job for job in jobs if _inspection_job_bucket(job) and str(job.get("target_node_code") or "").strip()] + node_map = { + str(item.get("node_code") or "").strip(): item + for item in nodes + if str(item.get("node_code") or "").strip() + } + row_map: dict[str, dict] = {} + + def ensure_row(node_code: str) -> dict: + if node_code not in row_map: + node = node_map.get(node_code) or {} + row_map[node_code] = { + "node_code": node_code, + "region": str(node.get("region") or ""), + "role": str(node.get("role") or ""), + "is_managed": bool(node.get("is_managed", False)), + "is_enabled": bool(node.get("is_enabled", False)), + "is_agent_online": bool(node.get("is_agent_online", False)), + "has_ssh_access": bool(node.get("has_ssh_access", False)), + "agent_state": str(node.get("agent_state") or ""), + "agent_state_label": str(node.get("agent_state_label") or ""), + "agent_state_reason": str(node.get("agent_state_reason") or ""), + "remote_access_state": str(node.get("remote_access_state") or ""), + "remote_access_label": str(node.get("remote_access_label") or ""), + "remote_access_reason": str(node.get("remote_access_reason") or ""), + "is_remote_access_ready": bool(node.get("is_remote_access_ready", False)), + "cluster_status": str(node.get("cluster_status") or ""), + "cluster_ip": str(node.get("cluster_ip") or ""), + "cluster_hostname": str(node.get("cluster_hostname") or ""), + "cluster_is_effective_worker": bool(node.get("cluster_is_effective_worker", False)), + "cluster_detect_participating": bool(node.get("cluster_detect_participating", False)), + "participation_state": str(node.get("participation_state") or ""), + "participation_label": str(node.get("participation_label") or ""), + "participation_reason": str(node.get("participation_reason") or ""), + "is_inspection_visible": _is_inspection_visible_node(node), + "is_inspection_eligible": _is_inspection_eligible_node(node), + "job_map": {}, + "latest_job": {}, + "last_updated_at": "", + "success_count": 0, + "overall_status": "missing", + "issue_summary": "", + "problem_kind": "", + "problem_label": "", + "problem_title": "", + "problem_level": "", + "problem_keys": [], + "recommended_action": "", + "recommended_action_code": "", + } + return row_map[node_code] + + for job in inspection_jobs: + node_code = str(job.get("target_node_code") or "").strip() + bucket = _inspection_job_bucket(job) + if not node_code or not bucket: + continue + row = ensure_row(node_code) + if bucket not in row["job_map"]: + row["job_map"][bucket] = _slim_inspection_job(job) + candidate_time = str(job.get("updated_at") or job.get("finished_at") or job.get("started_at") or job.get("created_at") or "").strip() + current_latest_time = str(row.get("last_updated_at") or "").strip() + if not current_latest_time or candidate_time > current_latest_time: + row["last_updated_at"] = candidate_time + row["latest_job"] = _slim_inspection_job(job) + + for node in nodes: + node_code = str(node.get("node_code") or "").strip() + if node_code and _is_inspection_visible_node(node): + ensure_row(node_code) + + def derive_overall_status(row: dict, job_map: dict[str, dict]) -> str: + statuses = [str((job_map.get(key) or {}).get("status") or "").strip() for key in _OPS_INSPECTION_ACTION_KEYS] + present_statuses = [status for status in statuses if status] + if not present_statuses: + if bool(row.get("is_inspection_visible", False)): + if not bool(row.get("is_inspection_eligible", False)): + return "attention" + if bool(row.get("cluster_detect_participating", False)): + return "attention" + return "missing" + if any(status in {"failed", "blocked", "cancelled"} for status in present_statuses): + return "attention" + if any(status in {"queued", "dispatching", "running", "awaiting_approval"} for status in present_statuses): + return "running" + if all(str((job_map.get(key) or {}).get("status") or "").strip() == "success" for key in _OPS_INSPECTION_ACTION_KEYS): + return "healthy" + return "attention" + + rows: list[dict] = [] + for row in row_map.values(): + job_map = dict(row.get("job_map") or {}) + success_count = sum(1 for key in _OPS_INSPECTION_ACTION_KEYS if str((job_map.get(key) or {}).get("status") or "").strip() == "success") + issue_summary = _build_inspection_issue_summary( + job_map, + node_code=str(row.get("node_code") or ""), + latest_job=dict(row.get("latest_job") or {}), + node=row, + ) + rows.append( + { + **row, + "job_map": job_map, + "success_count": success_count, + "overall_status": derive_overall_status(row, job_map), + "issue_summary": str(issue_summary.get("summary") or ""), + "problem_kind": str(issue_summary.get("problem_kind") or ""), + "problem_label": str(issue_summary.get("problem_label") or ""), + "problem_title": str(issue_summary.get("problem_title") or ""), + "problem_level": str(issue_summary.get("problem_level") or ""), + "problem_keys": list(issue_summary.get("problem_keys") or []), + "recommended_action": str(issue_summary.get("recommended_action") or ""), + "recommended_action_code": str(issue_summary.get("recommended_action_code") or ""), + "ui_intent": dict(issue_summary.get("ui_intent") or {}), + "latest_health_snapshot": _build_inspection_contract_result( + job_map.get("health.snapshot"), + action_label="健康快照", + ), + "latest_worker_logs": _build_inspection_contract_result( + job_map.get("logs.collect"), + action_label="Worker 日志", + ), + "latest_diagnostics": _build_inspection_contract_result( + job_map.get("diagnostics.collect"), + action_label="诊断包", + ), + } + ) + + status_weight = {"attention": 0, "running": 1, "missing": 2, "healthy": 3} + # Stable-sort so the primary order is status bucket, then newest update, then node code. + rows.sort(key=lambda item: str(item.get("node_code") or "")) + rows.sort(key=lambda item: str(item.get("last_updated_at") or ""), reverse=True) + rows.sort(key=lambda item: int(status_weight.get(str(item.get("overall_status") or ""), 99))) + + summary = { + "healthy": 0, + "running": 0, + "attention": 0, + "missing": 0, + } + for row in rows: + row_overall_status = str(row.get("overall_status") or "missing").strip() + if row_overall_status in summary: + summary[row_overall_status] += 1 + + priority_weight = {"attention": 0, "running": 1, "missing": 2, "healthy": 9} + + normalized_status = str(status or "").strip().lower() + normalized_problem_kind = str(problem_kind or "").strip().lower() + normalized_query = str(query or "").strip().lower() + filtered_rows = list(rows) + if normalized_status: + filtered_rows = [ + item for item in filtered_rows if str(item.get("overall_status") or "").strip().lower() == normalized_status + ] + if normalized_problem_kind: + filtered_rows = [ + item for item in filtered_rows if str(item.get("problem_kind") or "").strip().lower() == normalized_problem_kind + ] + if only_problem: + filtered_rows = [item for item in filtered_rows if str(item.get("overall_status") or "").strip() != "healthy"] + if only_participating: + filtered_rows = [item for item in filtered_rows if bool(item.get("cluster_detect_participating", False))] + if normalized_query: + filtered_rows = [ + item + for item in filtered_rows + if normalized_query in " ".join( + [ + str(item.get("node_code") or ""), + str(item.get("region") or ""), + str(item.get("role") or ""), + str(item.get("overall_status") or ""), + str(item.get("problem_kind") or ""), + str(item.get("problem_label") or ""), + str(item.get("problem_title") or ""), + str(item.get("issue_summary") or ""), + str(item.get("recommended_action") or ""), + ] + ).lower() + ] + + filtered_summary = { + "healthy": 0, + "running": 0, + "attention": 0, + "missing": 0, + } + for row in filtered_rows: + row_status = str(row.get("overall_status") or "missing").strip() + if row_status in filtered_summary: + filtered_summary[row_status] += 1 + + if not filtered_rows: + overview_status = "empty" + overview_status_label = "无匹配结果" + overview_summary_text = "当前筛选条件下没有匹配节点。" + elif int(filtered_summary.get("attention", 0) or 0) > 0: + overview_status = "attention" + overview_status_label = "待处理" + overview_summary_text = f"存在 {int(filtered_summary.get('attention', 0) or 0)} 台节点需要优先处理。" + elif int(filtered_summary.get("running", 0) or 0) > 0: + overview_status = "running" + overview_status_label = "执行中" + overview_summary_text = f"存在 {int(filtered_summary.get('running', 0) or 0)} 台节点的标准巡检仍在执行。" + elif int(filtered_summary.get("missing", 0) or 0) > 0: + overview_status = "attention" + overview_status_label = "待补齐" + overview_summary_text = f"存在 {int(filtered_summary.get('missing', 0) or 0)} 台节点尚未形成完整巡检收口。" + else: + overview_status = "healthy" + overview_status_label = "健康" + overview_summary_text = "当前可见节点最近巡检收口正常。" + + priority_queue = [ + { + **row, + "priority_label": "P1" if row.get("overall_status") == "attention" else ("P2" if row.get("overall_status") == "running" else "P3"), + "priority_type": "danger" if row.get("overall_status") == "attention" else ("warning" if row.get("overall_status") == "running" else "info"), + "priority_weight": int(priority_weight.get(str(row.get("overall_status") or ""), 99)), + } + for row in filtered_rows + if str(row.get("overall_status") or "") != "healthy" + ] + priority_queue.sort(key=lambda item: str(item.get("node_code") or "")) + priority_queue.sort(key=lambda item: str(item.get("last_updated_at") or ""), reverse=True) + priority_queue.sort(key=lambda item: int(item.get("priority_weight", 99))) + + available_status_counts = { + "healthy": int(summary.get("healthy", 0) or 0), + "running": int(summary.get("running", 0) or 0), + "attention": int(summary.get("attention", 0) or 0), + "missing": int(summary.get("missing", 0) or 0), + } + available_problem_kind_counts: dict[str, int] = {} + for row in rows: + row_problem_kind = str(row.get("problem_kind") or "").strip() + if not row_problem_kind: + continue + available_problem_kind_counts[row_problem_kind] = int(available_problem_kind_counts.get(row_problem_kind, 0) or 0) + 1 + + return { + "status": overview_status, + "status_label": overview_status_label, + "summary_text": overview_summary_text, + "fetch_limit": int(fetch_limit), + "jobs_total": len(inspection_jobs), + "visible_nodes_total": sum(1 for item in nodes if _is_inspection_visible_node(item)), + "eligible_nodes_total": sum(1 for item in nodes if _is_inspection_eligible_node(item)), + "handover_gap_nodes_total": sum( + 1 for item in nodes if _is_inspection_visible_node(item) and not _is_inspection_eligible_node(item) + ), + "participating_nodes_total": sum(1 for item in rows if bool(item.get("cluster_detect_participating", False))), + "standby_nodes_total": sum( + 1 + for item in rows + if bool(item.get("is_inspection_visible", False)) and not bool(item.get("cluster_detect_participating", False)) + ), + "rows_total": len(rows), + "filtered_rows_total": len(filtered_rows), + "filters": { + "status": normalized_status, + "problem_kind": normalized_problem_kind, + "query": str(query or "").strip(), + "only_problem": bool(only_problem), + "only_participating": bool(only_participating), + }, + "available_status_counts": available_status_counts, + "available_problem_kind_counts": available_problem_kind_counts, + "rows": filtered_rows, + "summary": filtered_summary, + "counts": { + "healthy_nodes": int(filtered_summary.get("healthy", 0) or 0), + "running_nodes": int(filtered_summary.get("running", 0) or 0), + "attention_nodes": int(filtered_summary.get("attention", 0) or 0), + "missing_nodes": int(filtered_summary.get("missing", 0) or 0), + }, + "unfiltered_summary": summary, + "priority_queue": priority_queue, + } + + +def get_ops_overview() -> dict: + runtime = get_runtime_status() + readiness = runtime.get("readiness") or {} + detect = runtime.get("detect") or {} + cluster = runtime.get("cluster") or {} + sync_summary = get_sync_summary() + job_summary = get_ops_job_summary() + managed_nodes_payload = list_managed_nodes_with_agent_state(participation_payload=detect) + managed_nodes = list(managed_nodes_payload.get("nodes") or []) + managed_nodes_summary = managed_nodes_payload.get("summary") or {} + inspection_overview = get_ops_inspection_overview(managed_nodes=managed_nodes) + activity_stream = get_ops_activity_stream(limit=8) + release_summary = get_release_summary() + preferred_release = _preferred_release_for_ops() + release_launchpad = get_release_launchpad() + sync_batches = (sync_summary.get("detect_result_batches") or {}).get("state_counts") or {} + + online_worker_nodes = int((cluster.get("summary") or {}).get("online_worker_nodes", 0) or 0) + online_control_nodes = int((cluster.get("summary") or {}).get("online_control_nodes", 0) or 0) + projected_batches = int(sync_batches.get("projected", 0) or 0) + failed_batches = int(sync_batches.get("failed", 0) or 0) + execution_scene = _build_ops_execution_scene(detect) + default_rollout_targets = _default_rollout_target_nodes(list(cluster.get("nodes") or [])) + release_execution_mode = build_release_execution_mode_recommendation(preferred_release, default_rollout_targets) + release_gate = dict(release_execution_mode.get("recommended_gate") or {}) + release_gate = { + **release_gate, + "default_target_nodes": list(release_gate.get("target_nodes") or []), + "default_target_node_codes": list(release_gate.get("target_node_codes") or []), + } + driver_recommendations = _build_driver_recommendations( + managed_nodes=managed_nodes, + execution_scene=execution_scene, + release_summary=release_summary, + release_gate=release_gate, + release_launchpad=release_launchpad, + ) + recommendation = _build_ops_priority_recommendation(driver_recommendations) + + return { + "mode": "centralized-ops", + "vision": "海外控制面统一接管大陆节点安装、更新、巡检、日志回传与故障诊断。", + "current_topology": { + "node_code": settings.node_code, + "node_region": settings.node_region, + "node_role": settings.node_role, + "worker_mode": settings.worker_mode, + "sync_push_enabled": bool(settings.sync_push_enabled), + "sync_target_api_base_url": str(settings.sync_target_api_base_url or "").strip(), + }, + "cluster_summary": { + "online_control_nodes": online_control_nodes, + "online_worker_nodes": online_worker_nodes, + "dedicated_online_worker_nodes": int((cluster.get("summary") or {}).get("dedicated_online_worker_nodes", 0) or 0), + "busy_nodes": list((cluster.get("summary") or {}).get("busy_nodes") or []), + }, + "cluster_nodes": [ + { + "node_code": str(item.get("node_code") or ""), + "region": str(item.get("region") or ""), + "role": str(item.get("role") or ""), + "status": str(item.get("status") or ""), + "current_load": int(item.get("current_load", 0) or 0), + "hostname": str(item.get("hostname") or ""), + "last_heartbeat_at": str(item.get("last_heartbeat_at") or ""), + "is_effective_worker": bool(item.get("is_effective_worker", False)), + "detect_participating": bool(item.get("detect_participating", False)), + } + for item in list(cluster.get("nodes") or []) + if str(item.get("node_code") or "").strip() + ], + "runtime_summary": { + "readiness_status": str(readiness.get("status") or ""), + "readiness_summary": str(readiness.get("summary") or ""), + "phase_label": str(detect.get("phase_label") or ""), + "phase_detail": str(detect.get("phase_detail") or ""), + "active_job_code": str((detect.get("active_job") or {}).get("job_code") or ""), + "progress_percent": float(detect.get("progress_percent", 0) or 0), + }, + "execution_scene": execution_scene, + "sync_summary": { + "enabled": bool(sync_summary.get("enabled", False)), + "source_region": str(sync_summary.get("source_region") or ""), + "target_region": str(sync_summary.get("target_region") or ""), + "projected_batches": projected_batches, + "failed_batches": failed_batches, + "records_total": int(sync_summary.get("records_total", 0) or 0), + }, + "ops_jobs": job_summary, + "managed_nodes": { + "total": int(managed_nodes_summary.get("total", len(managed_nodes)) or 0), + "managed_total": int(managed_nodes_summary.get("managed_total", 0) or 0), + "enabled": int(managed_nodes_summary.get("managed_enabled", 0) or 0), + "online": int(managed_nodes_summary.get("online", 0) or 0), + "stale": int(managed_nodes_summary.get("stale", 0) or 0), + "pending_bootstrap": int(managed_nodes_summary.get("pending_bootstrap", 0) or 0), + "runtime_only": int(managed_nodes_summary.get("runtime_only", 0) or 0), + "unmanaged": int(managed_nodes_summary.get("unmanaged", 0) or 0), + "token_issue": int(managed_nodes_summary.get("token_issue", 0) or 0), + "queue_retrying_nodes": int(managed_nodes_summary.get("queue_retrying_nodes", 0) or 0), + "queue_dead_letter_nodes": int(managed_nodes_summary.get("queue_dead_letter_nodes", 0) or 0), + "queue_pending_records": int(managed_nodes_summary.get("queue_pending_records", 0) or 0), + "queue_dead_letter_records": int(managed_nodes_summary.get("queue_dead_letter_records", 0) or 0), + "agent_state_counts": dict(managed_nodes_summary.get("status_counts") or {}), + "delivery_queue_state_counts": dict(managed_nodes_summary.get("delivery_queue_state_counts") or {}), + }, + "inspection": inspection_overview, + "activity_stream": activity_stream, + "driver_recommendations": driver_recommendations, + "release_hub": { + **release_summary, + "preferred_release": preferred_release, + "default_rollout_execution": release_execution_mode, + "default_rollout_gate": release_gate, + "default_rollout_gate_options": dict(release_execution_mode.get("gates") or {}), + "launchpad": release_launchpad, + }, + "recommendation": recommendation, + } + + +def _build_ops_link_snapshot_from_overview(overview: dict) -> dict: + overview = dict(overview or {}) + topology = dict(overview.get("current_topology") or {}) + cluster_summary = dict(overview.get("cluster_summary") or {}) + runtime_summary = dict(overview.get("runtime_summary") or {}) + execution_scene = dict(overview.get("execution_scene") or {}) + participation_summary = dict(execution_scene.get("participation_summary") or {}) + log_sync = dict(execution_scene.get("log_sync") or {}) + sync_summary = dict(overview.get("sync_summary") or {}) + recommendation = dict(overview.get("recommendation") or {}) + + readiness_status = str(runtime_summary.get("readiness_status") or "").strip() or "attention" + readiness_summary = str(runtime_summary.get("readiness_summary") or "").strip() + node_code = str(topology.get("node_code") or "").strip() + node_region = str(topology.get("node_region") or "").strip() + node_role = str(topology.get("node_role") or "").strip() + worker_mode = str(topology.get("worker_mode") or "").strip() + active_job_code = str(runtime_summary.get("active_job_code") or "").strip() + phase_label = str(runtime_summary.get("phase_label") or "").strip() + phase_detail = str(runtime_summary.get("phase_detail") or "").strip() + progress_percent = float(runtime_summary.get("progress_percent", 0) or 0) + + online_control_nodes = int(cluster_summary.get("online_control_nodes", 0) or 0) + online_worker_nodes = int(cluster_summary.get("online_worker_nodes", 0) or 0) + dedicated_online_worker_nodes = int(cluster_summary.get("dedicated_online_worker_nodes", 0) or 0) + busy_nodes = list(cluster_summary.get("busy_nodes") or []) + + effective_online_nodes = int( + participation_summary.get("effective_online_nodes", online_worker_nodes) or online_worker_nodes or 0 + ) + participating_nodes = int(participation_summary.get("participating_nodes", 0) or 0) + dispatch_active_nodes = int(participation_summary.get("dispatch_active_nodes", 0) or 0) + recent_only_nodes = int(participation_summary.get("recent_only_nodes", 0) or 0) + non_participating_nodes = int(participation_summary.get("non_participating_nodes", 0) or 0) + standby_nodes = int(participation_summary.get("standby_nodes", 0) or 0) + load_syncing_nodes = int(participation_summary.get("load_syncing_nodes", 0) or 0) + dispatch_active_node_codes = list(participation_summary.get("dispatch_active_node_codes") or []) + non_participating_node_codes = list(participation_summary.get("non_participating_node_codes") or []) + + log_sync_enabled = bool(log_sync.get("enabled", False)) + log_sync_mode = str(log_sync.get("mode") or "").strip() or "key" + log_sync_samples = int(log_sync.get("line_count", 0) or 0) + log_sync_sources = int(log_sync.get("source_node_count", 0) or 0) + missing_log_nodes = list(log_sync.get("missing_participating_nodes") or []) + covered_participating_nodes = int(log_sync.get("covered_participating_node_count", 0) or 0) + participating_node_count = int(log_sync.get("participating_node_count", participating_nodes) or participating_nodes or 0) + log_sync_last_at = str(log_sync.get("last_at") or "").strip() + log_sync_preview_lines = [str(item or "").strip() for item in list(log_sync.get("preview_lines") or []) if str(item or "").strip()] + + sync_enabled = bool(sync_summary.get("enabled", False)) + sync_source_region = str(sync_summary.get("source_region") or "").strip() + sync_target_region = str(sync_summary.get("target_region") or "").strip() + projected_batches = int(sync_summary.get("projected_batches", 0) or 0) + failed_batches = int(sync_summary.get("failed_batches", 0) or 0) + sync_records_total = int(sync_summary.get("records_total", 0) or 0) + + if effective_online_nodes <= 0: + status = "blocking" + headline = "当前没有有效执行节点，海外控制面暂时无法继续推进联调、巡检或检测任务。" + elif dispatch_active_nodes > 0 and not log_sync_enabled: + status = "attention" + headline = "当前已有执行节点，但远端日志回传仍关闭，海外控制面对现场仍处于半盲态。" + elif dispatch_active_nodes > 0 and missing_log_nodes: + status = "attention" + headline = "当前已有执行节点，但远端日志回传仍未覆盖全部参与节点，建议先补齐现场观测。" + elif failed_batches > 0: + status = "attention" + headline = "当前同步链路存在失败批次，建议先收口跨地域同步状态，再继续放量联调。" + elif readiness_status != "ready": + status = readiness_status + headline = readiness_summary or "当前多机现场仍有待处理事项，建议先按运行中心提示逐项收口。" + elif active_job_code: + status = "ready" + headline = "当前多机现场已进入可观测状态，可以直接从海外控制面继续盯检测任务和节点执行过程。" + else: + status = "ready" + headline = "当前多机骨架已就绪，可以继续拉取任务、做巡检验收或推进发布演练。" + + summary_lines = [ + f"控制面节点 {node_code or '-'} / 区域 {node_region or '-'} / 角色 {node_role or '-'} / 托管方式 {worker_mode or '-'}", + f"集群在线：控制面 {online_control_nodes} 台 / 有效执行节点 {effective_online_nodes} 台 / 独立 Worker {dedicated_online_worker_nodes} 台 / 忙碌节点 {_format_node_code_list(busy_nodes)}", + ( + f"活跃任务 {active_job_code} / 阶段 {phase_label or '-'} / 进度 {progress_percent:.1f}%" + if active_job_code + else f"当前无活跃任务 / 就绪度 {readiness_status} / 摘要 {readiness_summary or '-'}" + ), + ( + f"执行现场：参与 {participating_nodes} 台 / 执行中或已领 {dispatch_active_nodes} 台 / 近窗吞吐 {recent_only_nodes} 台 / 在线未参与 {non_participating_nodes} 台" + ), + ( + f"远端日志：{'开启' if log_sync_enabled else '关闭'} / 模式 {log_sync_mode} / 覆盖 {covered_participating_nodes}/{participating_node_count} / 样本 {log_sync_samples} / 缺口 {_format_node_code_list(missing_log_nodes)}" + ), + ( + f"跨地域同步：{'启用' if sync_enabled else '未启用'} / {sync_source_region or '-'} -> {sync_target_region or '-'} / projected {projected_batches} / failed {failed_batches} / records {sync_records_total}" + ), + ] + if phase_detail: + summary_lines.append(f"现场阶段说明：{phase_detail}") + if log_sync_last_at: + summary_lines.append(f"最近日志回传：{log_sync_last_at}") + + next_actions: list[dict] = [] + if dispatch_active_nodes > 0 and not log_sync_enabled: + target_node_codes = list(dispatch_active_node_codes or []) + next_actions = [ + { + "label": "开启关键回传", + "action_code": "enable_log_sync_key", + "node_codes": target_node_codes, + "payload": {}, + }, + { + "label": "开启全量回传", + "action_code": "enable_log_sync_full", + "node_codes": target_node_codes, + "payload": {}, + }, + ] + elif dispatch_active_nodes > 0 and missing_log_nodes: + target_node_codes = list(dispatch_active_node_codes or missing_log_nodes or []) + next_actions = [ + { + "label": "看参与节点日志", + "action_code": "open_worker_logs_participating", + "node_codes": target_node_codes, + "payload": {}, + }, + { + "label": "执行标准巡检", + "action_code": "run_inspection_participating", + "node_codes": target_node_codes, + "payload": {}, + }, + ] + else: + primary_label = str(recommendation.get("primary_label") or "").strip() + primary_action_code = str(recommendation.get("primary_action_code") or "").strip() + if primary_label or primary_action_code: + next_actions.append( + { + "label": primary_label or "执行建议动作", + "action_code": primary_action_code, + "node_codes": list(recommendation.get("primary_node_codes") or recommendation.get("node_codes") or []), + "payload": dict(recommendation.get("primary_action_payload") or {}), + } + ) + secondary_label = str(recommendation.get("secondary_label") or "").strip() + secondary_action_code = str(recommendation.get("secondary_action_code") or "").strip() + if secondary_label or secondary_action_code: + next_actions.append( + { + "label": secondary_label or "执行次要动作", + "action_code": secondary_action_code, + "node_codes": list(recommendation.get("secondary_node_codes") or recommendation.get("node_codes") or []), + "payload": dict(recommendation.get("secondary_action_payload") or {}), + } + ) + + return { + "generated_at": datetime.now().strftime("%Y-%m-%d %H:%M:%S"), + "status": status, + "headline": headline, + "summary_lines": summary_lines, + "control_plane": { + "node_code": node_code, + "region": node_region, + "role": node_role, + "worker_mode": worker_mode, + "sync_push_enabled": bool(topology.get("sync_push_enabled", False)), + "sync_target_api_base_url": str(topology.get("sync_target_api_base_url") or "").strip(), + }, + "cluster": { + "online_control_nodes": online_control_nodes, + "online_worker_nodes": online_worker_nodes, + "dedicated_online_worker_nodes": dedicated_online_worker_nodes, + "effective_online_nodes": effective_online_nodes, + "busy_nodes": list(busy_nodes), + }, + "execution": { + "active_job_code": active_job_code, + "phase_label": phase_label, + "phase_detail": phase_detail, + "progress_percent": progress_percent, + "participating_nodes": participating_nodes, + "dispatch_active_nodes": dispatch_active_nodes, + "recent_only_nodes": recent_only_nodes, + "non_participating_nodes": non_participating_nodes, + "standby_nodes": standby_nodes, + "load_syncing_nodes": load_syncing_nodes, + "dispatch_active_node_codes": list(dispatch_active_node_codes), + "non_participating_node_codes": list(non_participating_node_codes), + }, + "log_sync": { + "enabled": log_sync_enabled, + "mode": log_sync_mode, + "line_count": log_sync_samples, + "source_node_count": log_sync_sources, + "covered_participating_node_count": covered_participating_nodes, + "participating_node_count": participating_node_count, + "missing_participating_nodes": list(missing_log_nodes), + "last_at": log_sync_last_at, + "preview_lines": log_sync_preview_lines, + }, + "sync": { + "enabled": sync_enabled, + "source_region": sync_source_region, + "target_region": sync_target_region, + "projected_batches": projected_batches, + "failed_batches": failed_batches, + "records_total": sync_records_total, + }, + "recommendation": { + "key": str(recommendation.get("key") or "").strip(), + "priority": str(recommendation.get("priority") or "").strip(), + "summary": str(recommendation.get("summary") or "").strip(), + "reason": str(recommendation.get("reason") or "").strip(), + }, + "next_actions": next_actions, + } + + +def get_ops_link_snapshot(overview: dict | None = None) -> dict: + normalized_overview = dict(overview or {}) + if not normalized_overview: + normalized_overview = get_ops_overview() + return _build_ops_link_snapshot_from_overview(normalized_overview) + + +def _normalize_ops_node_code_list(*groups: object) -> list[str]: + normalized: list[str] = [] + seen: set[str] = set() + for group in groups: + if isinstance(group, dict): + group = [group] + if not isinstance(group, (list, tuple, set)): + continue + for item in group: + if isinstance(item, dict): + node_code = str(item.get("node_code") or "").strip() + else: + node_code = str(item or "").strip() + if not node_code or node_code in seen: + continue + seen.add(node_code) + normalized.append(node_code) + return normalized + + +def _build_scene_node_log_command(base_url: str, node_code: str, *, limit: int = 80, mode: str = "key") -> str: + normalized_node_code = str(node_code or "").strip() + if not normalized_node_code: + return "" + normalized_base_url = _ops_stack_api_base_url(base_url) + normalized_mode = _normalize_scene_log_mode(mode, fallback="key") + normalized_limit = max(20, min(int(limit or 80), 300)) + return build_bash_command( + "drive_ops_center.sh", + "scene-node-log", + normalized_base_url, + normalized_node_code, + normalized_limit, + normalized_mode, + ) + + +def _build_scene_node_log_focus_ref( + node_code: str, + *, + mode: str = "key", + limit: int = 80, + source: str = "stack_diagnosis", +) -> dict: + normalized_node_code = str(node_code or "").strip() + if not normalized_node_code: + return {} + return _merge_focus_ref( + {}, + kind="node_scene_log", + node_code=normalized_node_code, + mode=_normalize_scene_log_mode(mode, fallback="key"), + limit=max(20, min(int(limit or 80), 300)), + source=source, + ) + + +def _build_scene_node_log_commands( + base_url: str, + node_codes: list[str] | tuple[str, ...] | set[str], + *, + limit: int = 80, + mode: str = "key", + max_commands: int = 3, +) -> list[str]: + commands: list[str] = [] + for node_code in _normalize_ops_node_code_list(list(node_codes or []))[: max(1, int(max_commands or 3))]: + command = _build_scene_node_log_command(base_url, node_code, limit=limit, mode=mode) + if command: + commands.append(command) + return commands + + +def _ops_stack_api_base_url(base_url: str = "") -> str: + normalized_base_url = str(base_url or "").strip() + if normalized_base_url: + return normalized_base_url.rstrip("/") + return f"http://127.0.0.1:{settings.api_port}" + + +def _ops_stack_capture(section_key: str, callback) -> dict: + started_at = time.perf_counter() + try: + payload = callback() or {} + except Exception as exc: + elapsed_ms = int((time.perf_counter() - started_at) * 1000) + return { + "section_key": str(section_key or "").strip(), + "http_status": "500", + "available": False, + "data": {}, + "error": f"{exc.__class__.__name__}: {exc}", + "elapsed_ms": elapsed_ms, + } + if isinstance(payload, dict): + data = dict(payload or {}) + else: + data = {"value": payload} + elapsed_ms = int((time.perf_counter() - started_at) * 1000) + return { + "section_key": str(section_key or "").strip(), + "http_status": "200", + "available": True, + "data": data, + "error": "", + "elapsed_ms": elapsed_ms, + } + + +def _ops_stack_issue_rank(severity: str) -> int: + return {"blocked": 3, "warning": 2, "info": 1}.get(str(severity or "").strip(), 0) + + +def _suggest_ops_contract_keys_for_stack_issue(issue: dict) -> list[str]: + normalized_issue = dict(issue or {}) + layer = str(normalized_issue.get("layer") or "").strip() + contract_keys: list[str] = ["ops_stack_diagnosis_contract"] + if layer.startswith("managed_nodes"): + contract_keys.append("ops_agent_protocol") + elif layer == "release_hub": + contract_keys.append("release_hub_contract") + elif layer == "playbook_runs": + contract_keys.append("ops_playbook_contract") + elif layer in {"overview.log_sync", "activity_stream", "link_snapshot"}: + contract_keys.append("ops_observability_contract") + elif layer == "contracts": + contract_keys.append("ops_driver_contract") + + contract_keys.extend(_ops_contract_keys_from_focus_ref(normalized_issue.get("focus_ref"))) + contract_keys.extend(_ops_contract_keys_from_action_code(str(normalized_issue.get("action_code") or "").strip())) + return _normalize_ops_contract_keys(contract_keys) + + +def _build_stack_diagnosis_recommended_commands( + *, + stack_base_url: str, + recommended_actions: list[dict], + first_handover_gap_node_code: str = "", + launchpad_status: dict | None = None, + default_rollout_gate: dict | None = None, + next_step: dict | None = None, +) -> dict: + commands: dict[str, str] = {} + + commands["stack-diagnosis"] = build_bash_command("drive_ops_center.sh", "stack-diagnosis", stack_base_url, "summary") + commands["stack-check"] = commands["stack-diagnosis"] + commands["doctor"] = build_bash_command("drive_ops_center.sh", "doctor", stack_base_url) + commands["contracts"] = build_bash_command("drive_ops_center.sh", "contracts", stack_base_url) + commands["runtime-refresh-recover"] = build_bash_command( + "drive_ops_center.sh", + "runtime-refresh-recover", + stack_base_url, + ) + commands["api-restart"] = commands["runtime-refresh-recover"] + + for action in list(recommended_actions or []): + action_code = str(action.get("action_code") or "").strip() + command = str(action.get("command") or "").strip() + if action_code and command and action_code not in commands: + commands[action_code] = command + + normalized_gap_node_code = str(first_handover_gap_node_code or "").strip() + if normalized_gap_node_code: + commands["managed-node-handover"] = build_bash_command("drive_ops_center.sh", "node-handover", stack_base_url, normalized_gap_node_code) + commands["managed-node-recover"] = build_bash_command("drive_ops_center.sh", "node-recover", stack_base_url, normalized_gap_node_code) + commands["managed-node-bootstrap-plan"] = build_bash_command("drive_ops_center.sh", "node-bootstrap-plan", stack_base_url, normalized_gap_node_code) + + normalized_next_step = dict(next_step or {}) + next_step_focus_ref = dict(normalized_next_step.get("focus_ref") or {}) + if str(next_step_focus_ref.get("kind") or "").strip() == "node_scene_log": + node_code = str(next_step_focus_ref.get("node_code") or "").strip() + if node_code: + commands["scene-node-log"] = _build_scene_node_log_command( + stack_base_url, + node_code, + limit=int(next_step_focus_ref.get("limit", 120) or 120), + mode=str(next_step_focus_ref.get("mode") or "key"), + ) + + normalized_launchpad_status = dict(launchpad_status or {}) + normalized_default_rollout_gate = dict(default_rollout_gate or {}) + launchpad_focus_ref = dict( + normalized_launchpad_status.get("focus_ref") + or normalized_default_rollout_gate.get("focus_ref") + or {} + ) + if str(launchpad_focus_ref.get("section") or "").strip() == "release_launchpad": + commands["focus-release-hub"] = build_bash_command("drive_ops_center.sh", "release-launchpad", stack_base_url) + mode = str(launchpad_focus_ref.get("mode") or "").strip() + if mode == "control": + commands["release-launchpad-control"] = build_bash_command( + "drive_ops_center.sh", + "release-launchpad", + stack_base_url, + "control", + ) + else: + commands["release-launchpad-worker"] = build_bash_command( + "drive_ops_center.sh", + "release-launchpad", + stack_base_url, + "worker", + ) + launchpad_recommended_action_code = str(normalized_launchpad_status.get("recommended_action_code") or "").strip() + launchpad_recommended_target_node_code = str( + normalized_launchpad_status.get("recommended_target_node_code") or "" + ).strip() + if ( + launchpad_recommended_action_code in {"bootstrap_run", "run_acceptance"} + and launchpad_recommended_target_node_code + ): + commands[launchpad_recommended_action_code] = build_bash_command( + "drive_ops_center.sh", + "node-bootstrap-run" if launchpad_recommended_action_code == "bootstrap_run" else "node-acceptance-run", + stack_base_url, + launchpad_recommended_target_node_code, + "cli", + ) + + return {key: value for key, value in commands.items() if str(value or "").strip()} + + +def _resolve_first_handover_gap_recovery( + *, + first_handover_gap: dict | None = None, + launchpad_gap_action: dict | None = None, + launchpad_status: dict | None = None, + fallback_node_code: str = "", +) -> dict: + normalized_handover = dict(first_handover_gap or {}) + stage = dict(normalized_handover.get("stage") or {}) + stage_next_step = dict(stage.get("next_step") or {}) + normalized_launchpad_gap_action = dict(launchpad_gap_action or {}) + normalized_launchpad_status = dict(launchpad_status or {}) + + node_code = str( + normalized_handover.get("node_code") + or normalized_launchpad_gap_action.get("node_code") + or normalized_launchpad_status.get("recommended_target_node_code") + or fallback_node_code + or "" + ).strip() + summary = str( + normalized_handover.get("summary") + or stage.get("summary") + or normalized_launchpad_gap_action.get("summary") + or normalized_launchpad_status.get("recommended_recovery_summary") + or "" + ).strip() + stage_code = str(stage.get("code") or "").strip() + stage_next_step_code = str(stage_next_step.get("code") or "").strip() + has_gap_signal = bool(node_code or stage_code or stage_next_step_code or summary) + + action_code = str(normalized_launchpad_gap_action.get("action_code") or "").strip() + if not has_gap_signal: + action_code = "" + elif action_code not in {"bootstrap_run", "run_acceptance"}: + if stage_next_step_code in {"run_acceptance", "acceptance", "ready"} or stage_code in {"acceptance_ready"}: + action_code = "run_acceptance" + elif stage_next_step_code in {"execute_bootstrap_plan", "bootstrap_run"} or stage_code in { + "pending_bootstrap", + "runtime_only", + "profile_incomplete", + "ssh_ready", + "token_issue", + "stale", + "agent_pending", + "unknown", + "unmanaged", + }: + action_code = "bootstrap_run" + else: + action_code = "" + + recovery_label = str(normalized_launchpad_status.get("recommended_recovery_label") or "").strip() + if not recovery_label and action_code == "bootstrap_run": + recovery_label = "签发接入工单" + elif not recovery_label and action_code == "run_acceptance": + recovery_label = "执行接管验收" + if not recovery_label: + recovery_label = str(stage_next_step.get("label") or stage.get("label") or "").strip() + + focus_ref = dict( + (dict(normalized_launchpad_gap_action.get("row") or {}).get("focus_ref") or {}) + or normalized_launchpad_status.get("focus_ref") + or {} + ) + if node_code and not str(focus_ref.get("node_code") or "").strip(): + focus_ref["node_code"] = node_code + if node_code and not str(focus_ref.get("kind") or "").strip(): + focus_ref["kind"] = "managed_node" + + return { + "action_code": action_code if action_code in {"bootstrap_run", "run_acceptance"} else "", + "node_code": node_code, + "summary": summary, + "recovery_label": recovery_label, + "focus_ref": focus_ref, + "stage_code": stage_code, + "stage_label": str(stage.get("label") or "").strip(), + "stage_next_step_code": stage_next_step_code, + } + + +def _build_stack_diagnosis_operator_decision( + *, + issues: list[dict], + next_step: dict, + recommended_actions: list[dict], + recommended_commands: dict, + first_handover_gap_node_code: str = "", + launchpad_status: dict | None = None, + default_rollout_gate: dict | None = None, +) -> tuple[dict, dict]: + top_issue = ( + sorted( + list(issues or []), + key=lambda item: ( + -_ops_stack_issue_rank(str(item.get("severity") or "").strip()), + str(item.get("code") or "").strip(), + ), + )[0] + if issues + else {} + ) + normalized_next_step = dict(next_step or {}) + primary_action_code = str(normalized_next_step.get("action_code") or "").strip() + next_focus_ref = dict(normalized_next_step.get("focus_ref") or {}) + normalized_launchpad_status = dict(launchpad_status or {}) + normalized_default_rollout_gate = dict(default_rollout_gate or {}) + + operator_decision = { + "lane": "steady", + "priority": "low", + "reason_code": "steady_state", + "title": "当前总检已进入稳定观察态", + "summary": "控制面总检、执行现场、纳管与发布骨架当前没有阻断项，可以按既定节奏继续联调或推进发布。", + "next_focus": next_focus_ref, + "primary_command_key": primary_action_code, + "secondary_command_key": "", + } + + if str(next_focus_ref.get("kind") or "").strip() == "node_scene_log": + operator_decision = { + "lane": "observability", + "priority": "high" if top_issue else "medium", + "reason_code": str(top_issue.get("code") or "scene_log_observation").strip(), + "title": "优先补齐执行现场日志观察", + "summary": str(top_issue.get("summary") or normalized_next_step.get("reason") or "").strip() + or "当前已经定位到参与节点现场日志焦点，建议先看现场，再决定是否继续接管或推进发布。", + "next_focus": next_focus_ref, + "primary_command_key": "scene-node-log", + "secondary_command_key": primary_action_code or "open_worker_logs_participating", + } + elif str(top_issue.get("code") or "").strip() == "runtime_build_schema_stale": + secondary_key = "stack-diagnosis" if "stack-diagnosis" in recommended_commands else "" + if not secondary_key and "stack-check" in recommended_commands: + secondary_key = "stack-check" + if "managed-node-bootstrap-plan" in recommended_commands: + secondary_key = "managed-node-bootstrap-plan" + operator_decision = { + "lane": "runtime_recovery", + "priority": "high", + "reason_code": "runtime_build_schema_stale", + "title": "优先重启控制面 API 并复检运行时版本", + "summary": str(top_issue.get("summary") or normalized_next_step.get("reason") or "").strip() + or "仓库代码已经具备新能力，但运行中的 API 仍停留在旧 schema，当前第一动作应先让服务吃到最新代码。", + "next_focus": dict(top_issue.get("focus_ref") or normalized_next_step.get("focus_ref") or {}), + "primary_command_key": "api-restart", + "secondary_command_key": secondary_key, + } + elif str(top_issue.get("code") or "").strip() == "managed_nodes_agent_pending" or ( + str(top_issue.get("layer") or "").strip().startswith("managed_nodes") + and str(first_handover_gap_node_code or "").strip() + ): + managed_primary_key = primary_action_code + if managed_primary_key not in {"bootstrap_run", "run_acceptance"}: + managed_primary_key = ( + "managed-node-recover" if "managed-node-recover" in recommended_commands else "managed-node-handover" + ) + managed_secondary_key = "" + if managed_primary_key in {"bootstrap_run", "run_acceptance"}: + if "managed-node-bootstrap-plan" in recommended_commands: + managed_secondary_key = "managed-node-bootstrap-plan" + elif "managed-node-handover" in recommended_commands: + managed_secondary_key = "managed-node-handover" + elif "fix_managed_nodes" in recommended_commands: + managed_secondary_key = "fix_managed_nodes" + elif primary_action_code and primary_action_code != managed_primary_key: + managed_secondary_key = primary_action_code + operator_decision = { + "lane": "node_handover", + "priority": "high", + "reason_code": str(top_issue.get("code") or "managed_nodes_agent_pending").strip(), + "title": "优先执行节点接管收口" if managed_primary_key in {"bootstrap_run", "run_acceptance"} else "优先打通节点接管链路", + "summary": ( + str(normalized_next_step.get("reason") or "").strip() + if managed_primary_key in {"bootstrap_run", "run_acceptance"} + else "" + ) + or str(top_issue.get("summary") or "").strip() + or "托管节点还没有进入 remote-agent ready，海外控制面暂时不能把执行统一收口到标准动作链。", + "next_focus": dict(top_issue.get("focus_ref") or normalized_next_step.get("focus_ref") or {}), + "primary_command_key": managed_primary_key, + "secondary_command_key": managed_secondary_key, + } + elif ( + str(top_issue.get("code") or "").strip() == "release_rollout_gate_blocked" + or str(normalized_default_rollout_gate.get("status") or "").strip() == "blocked" + or str(normalized_launchpad_status.get("recommended_action_code") or "").strip() in { + "review_smart_rollout_preview", + "review_control_rollout", + "fix_rollout_blockers", + } + ): + launchpad_focus_ref = dict( + normalized_default_rollout_gate.get("focus_ref") + or normalized_launchpad_status.get("focus_ref") + or normalized_next_step.get("focus_ref") + or {} + ) + launchpad_mode = str(launchpad_focus_ref.get("mode") or "").strip() + primary_key = "release-launchpad-control" if launchpad_mode == "control" else "release-launchpad-worker" + if primary_key not in recommended_commands: + primary_key = "focus-release-hub" + operator_decision = { + "lane": "release", + "priority": "high" if str(normalized_default_rollout_gate.get("status") or "").strip() == "blocked" else "medium", + "reason_code": str(top_issue.get("code") or "release_launchpad_attention").strip(), + "title": "优先处理发布门禁与 Launchpad 决策", + "summary": str(top_issue.get("summary") or normalized_default_rollout_gate.get("summary_text") or "").strip() + or "发布包和 Rollout 入口已经就位，但默认门禁还没完全放行，建议先看 Release Hub 再推进下一批。", + "next_focus": launchpad_focus_ref, + "primary_command_key": primary_key, + "secondary_command_key": primary_action_code or str(normalized_launchpad_status.get("recommended_action_code") or "").strip(), + } + elif str(normalized_launchpad_status.get("recommended_action_code") or "").strip() in {"bootstrap_run", "run_acceptance"}: + launchpad_action_code = str(normalized_launchpad_status.get("recommended_action_code") or "").strip() + launchpad_target_node_code = str(normalized_launchpad_status.get("recommended_target_node_code") or "").strip() + launchpad_focus_ref = dict( + normalized_launchpad_status.get("focus_ref") + or normalized_default_rollout_gate.get("focus_ref") + or normalized_next_step.get("focus_ref") + or {} + ) + operator_decision = { + "lane": "node_handover", + "priority": "high" if launchpad_action_code == "bootstrap_run" else "medium", + "reason_code": f"release_launchpad_{launchpad_action_code}", + "title": "优先执行节点接入收口" if launchpad_action_code == "bootstrap_run" else "优先执行节点接管验收", + "summary": str(normalized_launchpad_status.get("recommended_recovery_summary") or "").strip() + or str(top_issue.get("summary") or normalized_next_step.get("reason") or "").strip() + or "Release Launchpad 已经明确给出节点接入缺口，建议先完成该节点的接入收口，再回到发布门禁。", + "next_focus": { + **launchpad_focus_ref, + **({"node_code": launchpad_target_node_code} if launchpad_target_node_code else {}), + }, + "primary_command_key": launchpad_action_code, + "secondary_command_key": "focus-release-hub" if "focus-release-hub" in recommended_commands else "", + } + elif primary_action_code: + operator_decision = { + "lane": "ops_jobs" if primary_action_code.startswith("focus_") else "steady", + "priority": "medium" if top_issue else "low", + "reason_code": str(top_issue.get("code") or primary_action_code).strip(), + "title": "按总检默认下一步继续处理", + "summary": str(normalized_next_step.get("reason") or "").strip() + or "当前总检已经给出默认下一步，可直接按统一 driver action 继续推进。", + "next_focus": next_focus_ref, + "primary_command_key": primary_action_code, + "secondary_command_key": "", + } + + next_actions = { + "primary_command_key": str(operator_decision.get("primary_command_key") or "").strip(), + "primary_command": str( + recommended_commands.get(str(operator_decision.get("primary_command_key") or "").strip()) or "" + ).strip(), + "secondary_command_key": str(operator_decision.get("secondary_command_key") or "").strip(), + "secondary_command": str( + recommended_commands.get(str(operator_decision.get("secondary_command_key") or "").strip()) or "" + ).strip(), + } + return operator_decision, next_actions + + +def get_ops_stack_diagnosis(*, base_url: str = "") -> dict: + stack_base_url = _ops_stack_api_base_url(base_url) + api_info = { + "service": "domain-api", + "version": "0.1.0", + "api_prefix": settings.api_prefix, + "worker_mode": settings.worker_mode, + } + + contracts_section = _ops_stack_capture("contracts", get_ops_contract_registry) + overview_section = _ops_stack_capture("overview", get_ops_overview) + overview_data = dict(overview_section.get("data") or {}) + if bool(overview_section.get("available", False)) and overview_data: + link_snapshot_section = _ops_stack_capture( + "link_snapshot", + lambda: get_ops_link_snapshot(overview=overview_data), + ) + else: + link_snapshot_section = _ops_stack_capture("link_snapshot", get_ops_link_snapshot) + nodes_section = _ops_stack_capture("managed_nodes", list_managed_nodes_with_agent_state) + + def _stack_release_hub_payload() -> dict: + overview_launchpad = dict(((overview_data.get("release_hub") or {}).get("launchpad") or {})) + if overview_launchpad: + return overview_launchpad + return get_release_launchpad() + + release_hub_section = _ops_stack_capture("release_hub", _stack_release_hub_payload) + build_info_section = _ops_stack_capture("runtime_build_info", get_runtime_build_info) + playbook_runs_section = _ops_stack_capture( + "playbook_runs", + lambda: get_recent_ops_playbook_runs(limit=6, scan_limit=240), + ) + + def _stack_activity_stream_payload() -> dict: + overview_activity_stream = dict(overview_data.get("activity_stream") or {}) + if overview_activity_stream: + return overview_activity_stream + return get_ops_activity_stream(limit=8, scan_limit=80) + + activity_stream_section = _ops_stack_capture("activity_stream", _stack_activity_stream_payload) + + contracts = dict(contracts_section.get("data") or {}) + contract_registry = contracts if contracts else get_ops_contract_registry() + link_snapshot = dict(link_snapshot_section.get("data") or {}) + overview = overview_data + nodes_payload = dict(nodes_section.get("data") or {}) + launchpad = dict(release_hub_section.get("data") or {}) + build_info = dict(build_info_section.get("data") or {}) + playbook_runs_payload = dict(playbook_runs_section.get("data") or {}) + activity_payload = dict(activity_stream_section.get("data") or {}) + + contracts_rows = list(contracts.get("contracts") or []) + nodes = list(nodes_payload.get("nodes") or []) + nodes_summary = dict(nodes_payload.get("summary") or {}) + playbook_runs = list(playbook_runs_payload.get("runs") or playbook_runs_payload.get("playbook_runs") or []) + activities = list(activity_payload.get("items") or activity_payload.get("activities") or []) + + recommendation = dict(overview.get("recommendation") or {}) + execution_scene = dict(overview.get("execution_scene") or {}) + log_sync = dict(execution_scene.get("log_sync") or overview.get("log_sync") or {}) + release_hub = dict(overview.get("release_hub") or {}) + default_rollout_gate = dict(release_hub.get("default_rollout_gate") or {}) + launchpad_status = dict(launchpad.get("launchpad_status") or {}) + launchpad_gap_action = _release_launchpad_gap_action_context(launchpad) + latest_release = dict(launchpad.get("latest_release") or release_hub.get("latest_release") or {}) + route_surface = dict(build_info.get("route_surface") or {}) + repository_capabilities = dict(build_info.get("repository_capabilities") or {}) + route_surface_missing_keys = [ + str(item).strip() + for item in list(route_surface.get("missing_keys") or []) + if str(item).strip() + ] + route_surface_expected_paths = { + str(key).strip(): str(value).strip() + for key, value in dict(route_surface.get("expected_paths") or {}).items() + if str(key).strip() + } + repo_supports_install_command_block = bool(repository_capabilities.get("supports_install_command_block", False)) + repo_supports_multi_layout_bootstrap = bool(repository_capabilities.get("supports_multi_layout_bootstrap", False)) + route_surface_declares_bootstrap_plan = "ops_node_handover_bootstrap_plan" in route_surface_expected_paths + runtime_schema_stale = bool(repo_supports_install_command_block and not route_surface_declares_bootstrap_plan) + + problem_nodes: list[dict] = [] + for node in nodes: + node_code = str(node.get("node_code") or "").strip() + if not node_code: + continue + queue_state = str(node.get("delivery_queue_state") or "").strip() + agent_state = str(node.get("agent_state") or "").strip() + remote_access_state = str(node.get("remote_access_state") or "").strip() + if ( + queue_state == "dead_letter" + or agent_state not in {"online", "online_busy"} + or remote_access_state in {"agent_pending", "disabled", "unmanaged"} + ): + problem_nodes.append( + { + "node_code": node_code, + "agent_state": agent_state, + "remote_access_state": remote_access_state, + "delivery_queue_state": queue_state, + "participation_state": str(node.get("participation_state") or "").strip(), + } + ) + + first_handover_gap_node = next( + ( + dict(node or {}) + for node in nodes + if str(node.get("node_code") or "").strip() + and ( + str(node.get("agent_state") or "").strip() not in {"online", "online_busy"} + or str(node.get("remote_access_state") or "").strip() in {"agent_pending", "disabled", "unmanaged"} + ) + ), + {}, + ) + first_handover_gap_node_code = str(first_handover_gap_node.get("node_code") or "").strip() + first_handover_gap = ( + get_managed_node_handover( + first_handover_gap_node_code, + control_plane_base_url=stack_base_url, + nodes_payload=nodes_payload, + ) + if first_handover_gap_node_code + else {} + ) + first_handover_gap_recovery = _resolve_first_handover_gap_recovery( + first_handover_gap=first_handover_gap, + launchpad_gap_action=launchpad_gap_action, + launchpad_status=launchpad_status, + fallback_node_code=first_handover_gap_node_code, + ) + + problem_runs = [ + { + "run_code": str(item.get("run_code") or "").strip(), + "status": str(item.get("status") or "").strip(), + "status_label": str(item.get("status_label") or "").strip(), + "focus_step_key": str(item.get("focus_step_key") or "").strip(), + "focus_summary": str(item.get("focus_summary") or item.get("summary_text") or "").strip(), + "focus_ref": dict(item.get("focus_ref") or {}), + } + for item in playbook_runs + if str(item.get("status") or "").strip() not in {"success", "completed", "healthy"} + ] + + activity_counts: dict[str, int] = {} + for item in activities: + status_key = str(item.get("status") or item.get("job_status") or "unknown").strip() or "unknown" + activity_counts[status_key] = int(activity_counts.get(status_key, 0) or 0) + 1 + + start_delivery_failed_items = [ + dict(item or {}) + for item in activities + if str(item.get("start_delivery_state") or "").strip() == "failed_local" + ] + + issues: list[dict] = [] + + def add_issue( + *, + code: str, + severity: str, + layer: str, + summary: str, + detail: str = "", + action_code: str = "", + focus_ref: dict | None = None, + commands: list[str] | None = None, + ) -> None: + contract_keys = _suggest_ops_contract_keys_for_stack_issue( + { + "layer": layer, + "focus_ref": focus_ref or {}, + "action_code": action_code, + } + ) + issues.append( + { + "code": str(code or "").strip(), + "severity": str(severity or "").strip() or "warning", + "layer": str(layer or "").strip(), + "summary": str(summary or "").strip(), + "detail": str(detail or "").strip(), + "action_code": str(action_code or "").strip(), + "focus_ref": dict(focus_ref or {}), + "commands": [str(item).strip() for item in list(commands or []) if str(item).strip()], + "contract_keys": contract_keys, + "contract_navigation": _build_ops_contract_navigation( + contract_keys, + primary_contract_key="ops_stack_diagnosis_contract", + registry=contract_registry, + ), + } + ) + + surface_matrix = [ + ("api", True), + ("contracts", bool(contracts_section.get("available", False))), + ("link_snapshot", bool(link_snapshot_section.get("available", False))), + ("overview", bool(overview_section.get("available", False))), + ("managed_nodes", bool(nodes_section.get("available", False))), + ("release_hub", bool(release_hub_section.get("available", False))), + ("runtime_build_info", bool(build_info_section.get("available", False))), + ("playbook_runs", bool(playbook_runs_section.get("available", False))), + ("activity_stream", bool(activity_stream_section.get("available", False))), + ] + surface_status_counts = { + "total": len(surface_matrix), + "available": sum(1 for _, available in surface_matrix if available), + "missing": sum(1 for _, available in surface_matrix if not available), + } + missing_surfaces = [name for name, available in surface_matrix if not available] + + if not bool(contracts_section.get("available", False)): + add_issue( + code="ops_contracts_unavailable", + severity="warning", + layer="contracts", + summary="控制面契约注册表当前未能正常生成，总检无法确认当前运行的是哪一版正式 contract。", + detail=str(contracts_section.get("error") or "ops contract registry unavailable").strip(), + commands=[ + build_bash_command("drive_ops_center.sh", "contracts", stack_base_url), + build_bash_command("drive_ops_center.sh", "doctor", stack_base_url), + ], + ) + + if not bool(build_info_section.get("available", False)): + add_issue( + code="runtime_build_info_unavailable", + severity="warning", + layer="runtime_build_info", + summary="当前无法读取运行中 API 的 build-info，总检暂时无法确认服务是否已经吃到最新路由面。", + detail=str(build_info_section.get("error") or "runtime build-info unavailable").strip(), + action_code="api-restart", + commands=[ + build_bash_command("drive_ops_center.sh", "runtime-refresh-recover", stack_base_url), + build_bash_command("drive_ops_center.sh", "doctor", stack_base_url), + ], + ) + elif route_surface and not bool(route_surface.get("surface_complete", False)): + add_issue( + code="runtime_route_surface_incomplete", + severity="warning", + layer="runtime_build_info.route_surface", + summary="当前运行中的 API 路由面不完整，说明服务可能还没重启到最新代码，或当前发布包缺少关键运维入口。", + detail=( + f"missing_keys={', '.join(route_surface_missing_keys) or '-'}；" + f"mode={str(route_surface.get('mode') or '').strip() or 'unknown'}；" + f"registered_paths_total={int(route_surface.get('registered_paths_total', 0) or 0)}" + ), + action_code="api-restart", + focus_ref={ + "kind": "runtime_build_info", + "section": "route_surface", + "missing_keys": route_surface_missing_keys, + }, + commands=[ + build_bash_command("drive_ops_center.sh", "runtime-refresh-recover", stack_base_url), + build_bash_command("drive_ops_center.sh", "stack-diagnosis", stack_base_url, "summary"), + build_bash_command("drive_ops_center.sh", "doctor", stack_base_url), + ], + ) + elif runtime_schema_stale: + add_issue( + code="runtime_build_schema_stale", + severity="warning", + layer="runtime_build_info.schema", + summary="仓库已经具备新的节点接管能力，但运行中的 build-info 仍未声明对应路由键，当前更像 API 还没重启到最新代码。", + detail=( + f"supports_install_command_block={repo_supports_install_command_block}；" + f"supports_multi_layout_bootstrap={repo_supports_multi_layout_bootstrap}；" + f"route_surface_declares_bootstrap_plan={route_surface_declares_bootstrap_plan}" + ), + action_code="api-restart", + focus_ref={ + "kind": "runtime_build_info", + "section": "schema", + "expected_route_key": "ops_node_handover_bootstrap_plan", + }, + commands=[ + build_bash_command("drive_ops_center.sh", "runtime-refresh-recover", stack_base_url), + build_bash_command("drive_ops_center.sh", "stack-diagnosis", stack_base_url, "summary"), + build_bash_command("drive_ops_center.sh", "node-bootstrap-plan", stack_base_url, first_handover_gap_node_code or "mainland-worker-01"), + ], + ) + + if missing_surfaces and any( + name in missing_surfaces for name in ("link_snapshot", "overview", "managed_nodes", "release_hub") + ): + add_issue( + code="ops_surface_partial", + severity="warning", + layer="stack", + summary="海外单脑控制面的关键观察层未全部联通，总检目前只能给出部分收口。", + detail=f"当前缺失层: {', '.join(missing_surfaces)}", + commands=[ + build_bash_command("drive_ops_center.sh", "stack-diagnosis", stack_base_url, "full"), + build_bash_command("drive_ops_center.sh", "doctor", stack_base_url), + ], + ) + + participating_nodes_total = int( + log_sync.get("participating_nodes_total", log_sync.get("participating_node_count", 0)) or 0 + ) + participating_node_codes = _normalize_ops_node_code_list( + execution_scene.get("participating_nodes") or [], + (execution_scene.get("participation_summary") or {}).get("dispatch_active_node_codes") or [], + (execution_scene.get("participation_summary") or {}).get("recent_only_node_codes") or [], + ) + missing_log_node_codes = _normalize_ops_node_code_list(log_sync.get("missing_participating_nodes") or []) + scene_log_source_node_codes = _normalize_ops_node_code_list( + log_sync.get("source_nodes") or [], + log_sync.get("source_node_summaries") or [], + ) + preferred_scene_log_target_codes = _normalize_ops_node_code_list( + missing_log_node_codes, + participating_node_codes, + scene_log_source_node_codes, + ) + if bool(log_sync.get("enabled", False)) is False and participating_nodes_total > 0: + log_sync_action = "" + for item in list(link_snapshot.get("next_actions") or []): + action_code = str(item.get("action_code") or "").strip() + if action_code.startswith("enable_log_sync"): + log_sync_action = action_code + break + if not log_sync_action: + log_sync_action = "enable_log_sync_key" + add_issue( + code="remote_log_sync_disabled", + severity="warning", + layer="overview.log_sync", + summary="现场已有参与检测节点，但远端日志回传仍关闭，海外控制面对执行细节仍处于半盲态。", + detail=( + f"参与检测节点 {participating_nodes_total} 台，" + f"已覆盖 {int(log_sync.get('covered_participating_node_count', 0) or 0)} 台。" + ), + action_code=log_sync_action, + focus_ref=_build_scene_node_log_focus_ref( + (preferred_scene_log_target_codes or participating_node_codes or [""])[0], + mode="key", + limit=80, + source="remote_log_sync_disabled", + ), + commands=[ + build_bash_command("drive_ops_center.sh", "driver-resolve", stack_base_url, log_sync_action), + *_build_scene_node_log_commands( + stack_base_url, + preferred_scene_log_target_codes, + limit=80, + mode="key", + max_commands=2, + ), + ], + ) + elif bool(log_sync.get("enabled", False)) and participating_nodes_total > 0 and ( + missing_log_node_codes + or int(log_sync.get("line_count", 0) or 0) <= 0 + or int(log_sync.get("source_node_count", 0) or 0) <= 0 + ): + waiting_node_codes = preferred_scene_log_target_codes or participating_node_codes + add_issue( + code="remote_log_sync_waiting_sample", + severity="warning", + layer="overview.log_sync", + summary="远端日志回传虽然已经开启，但参与检测节点的现场样本仍未完全形成。", + detail=( + f"参与检测节点 {participating_nodes_total} 台，" + f"已覆盖 {int(log_sync.get('covered_participating_node_count', 0) or 0)} 台，" + f"缺口节点 {_format_node_code_list(missing_log_node_codes)}。" + ), + action_code="open_worker_logs_participating", + focus_ref=_build_scene_node_log_focus_ref( + (waiting_node_codes or [""])[0], + mode=str(log_sync.get("mode") or "key"), + limit=120, + source="remote_log_sync_waiting_sample", + ), + commands=[ + *_build_scene_node_log_commands( + stack_base_url, + waiting_node_codes, + limit=120, + mode=str(log_sync.get("mode") or "key"), + max_commands=3, + ), + build_bash_command("drive_ops_center.sh", "driver-resolve", stack_base_url, "open_worker_logs_participating"), + build_bash_command("drive_ops_center.sh", "driver-resolve", stack_base_url, "run_inspection_participating"), + ], + ) + + managed_enabled = int(nodes_summary.get("managed_enabled", 0) or 0) + remote_access_ready = int(nodes_summary.get("remote_access_ready", 0) or 0) + if managed_enabled > 0 and remote_access_ready == 0: + first_handover_gap_stage = dict(first_handover_gap.get("stage") or {}) + first_handover_gap_next_step = dict(first_handover_gap_stage.get("next_step") or {}) + first_handover_gap_summary = str(first_handover_gap.get("summary") or "").strip() + managed_nodes_action_code = str( + first_handover_gap_recovery.get("action_code") + or launchpad_status.get("recommended_action_code") + or "fix_managed_nodes" + ).strip() + add_issue( + code="managed_nodes_agent_pending", + severity="blocked", + layer="managed_nodes", + summary="托管节点虽然已经纳入 Ops Center，但 0 台 remote-agent 就绪，海外主控还不能统一下发标准执行动作。", + detail=( + f"managed_enabled={managed_enabled}，remote_access_ready={remote_access_ready}；" + "当前节点多数仍处于 agent_pending / runtime_only / pending_bootstrap。" + + ( + f" 首个缺口节点 {first_handover_gap_node_code}：" + f"{str(first_handover_gap_stage.get('label') or '').strip() or '待处理'}，" + f"{first_handover_gap_summary}" + if first_handover_gap_node_code and first_handover_gap_summary + else "" + ) + ), + action_code=managed_nodes_action_code, + focus_ref=dict(first_handover_gap_recovery.get("focus_ref") or {}), + commands=[ + build_bash_command("drive_ops_center.sh", "driver-resolve", stack_base_url, managed_nodes_action_code), + *( + [build_bash_command("drive_ops_center.sh", "node-handover", stack_base_url, first_handover_gap_node_code)] + if first_handover_gap_node_code + else [] + ), + *( + [ + build_bash_command("drive_ops_center.sh", "node-bootstrap-plan", stack_base_url, first_handover_gap_node_code) + ] + if first_handover_gap_node_code + and str(first_handover_gap_next_step.get("code") or "").strip() + not in {"ready", "run_acceptance"} + else [] + ), + *( + [ + build_bash_command( + "drive_ops_center.sh", + "node-bootstrap-run" if managed_nodes_action_code == "bootstrap_run" else "node-acceptance-run", + stack_base_url, + first_handover_gap_node_code, + "cli", + ) + ] + if first_handover_gap_node_code and managed_nodes_action_code in {"bootstrap_run", "run_acceptance"} + else [] + ), + build_bash_command("drive_ops_center.sh", "doctor", stack_base_url), + ], + ) + + queue_dead_letter_nodes = int(nodes_summary.get("queue_dead_letter_nodes", 0) or 0) + queue_dead_letter_records = int(nodes_summary.get("queue_dead_letter_records", 0) or 0) + if queue_dead_letter_nodes > 0 or queue_dead_letter_records > 0: + add_issue( + code="delivery_queue_dead_letter", + severity="warning", + layer="managed_nodes.queue", + summary="存在 Node Agent 回执死信，后续自动化动作可能反复失败或无法收敛。", + detail=( + f"queue_dead_letter_nodes={queue_dead_letter_nodes}，" + f"queue_dead_letter_records={queue_dead_letter_records}" + ), + commands=[ + build_bash_command("drive_ops_center.sh", "doctor", stack_base_url), + ], + ) + + if str(default_rollout_gate.get("status") or "").strip() == "blocked": + add_issue( + code="release_rollout_gate_blocked", + severity="warning", + layer="release_hub", + summary="ReleaseHub 默认门禁当前阻断，说明虽然发布包已准备好，但正式 Rollout 还不具备条件。", + detail=str(default_rollout_gate.get("summary_text") or default_rollout_gate.get("summary") or "").strip(), + action_code=str(launchpad_status.get("recommended_action_code") or "").strip(), + focus_ref=dict(default_rollout_gate.get("focus_ref") or launchpad_status.get("focus_ref") or {}), + commands=[ + build_bash_command("drive_ops_center.sh", "release-launchpad", stack_base_url), + build_bash_command( + "drive_ops_center.sh", + "driver-resolve", + stack_base_url, + str(launchpad_status.get("recommended_action_code") or "fix_managed_nodes").strip(), + ), + ], + ) + + if problem_runs: + add_issue( + code="playbook_runs_need_attention", + severity="warning", + layer="playbook_runs", + summary="最近存在未完全收口的 playbook run，建议先确认执行序列是否停在中间步骤。", + detail=f"problem_runs_total={len(problem_runs)}", + commands=[ + build_bash_command("drive_ops_center.sh", "doctor", stack_base_url), + ], + ) + + if start_delivery_failed_items: + first_delivery_gap_item = dict(start_delivery_failed_items[0] or {}) + first_delivery_focus_ref = dict( + first_delivery_gap_item.get("source_focus_ref") + or first_delivery_gap_item.get("focus_ref") + or {} + ) + first_delivery_job_code = str(first_delivery_gap_item.get("job_code") or "").strip() + first_delivery_node_code = str(first_delivery_focus_ref.get("target_node_code") or "").strip() + add_issue( + code="ops_job_start_delivery_failed_local", + severity="warning", + layer="activity_stream.ops_job", + summary=( + f"最近有 {len(start_delivery_failed_items)} 条标准运维任务在节点侧已开始执行，但开始回执没有成功送达控制面。" + ), + detail=" / ".join( + part + for part in [ + f"首条任务 {first_delivery_job_code}" if first_delivery_job_code else "", + f"目标节点 {first_delivery_node_code}" if first_delivery_node_code else "", + str(first_delivery_gap_item.get("start_delivery_error") or "").strip(), + str(first_delivery_gap_item.get("summary_text") or first_delivery_gap_item.get("summary") or "").strip(), + ] + if part + ), + action_code="focus_latest_job_events", + focus_ref=first_delivery_focus_ref, + commands=[ + build_bash_command("drive_ops_center.sh", "driver-resolve", stack_base_url, "focus_latest_job_events"), + build_bash_command("drive_ops_center.sh", "doctor", stack_base_url), + ], + ) + + ordered_action_candidates = [ + { + "source": "overview.primary", + "action_code": str(recommendation.get("primary_action_code") or "").strip(), + "title": str(recommendation.get("title") or "").strip(), + "reason": "来自 overview.recommendation.primary_action_code", + "focus_ref": dict(recommendation.get("focus_ref") or {}), + }, + { + "source": "release_hub.launchpad", + "action_code": str(launchpad_status.get("recommended_action_code") or "").strip(), + "title": str(launchpad_status.get("status_label") or "").strip(), + "reason": "来自 release_launchpad.launchpad_status.recommended_action_code", + "focus_ref": dict(launchpad_status.get("focus_ref") or {}), + }, + { + "source": "overview.secondary", + "action_code": str(recommendation.get("secondary_action_code") or "").strip(), + "title": str(recommendation.get("title") or "").strip(), + "reason": "来自 overview.recommendation.secondary_action_code", + "focus_ref": dict(recommendation.get("focus_ref") or {}), + }, + ] + if str(first_handover_gap_recovery.get("action_code") or "").strip() in {"bootstrap_run", "run_acceptance"}: + ordered_action_candidates.insert( + 0, + { + "source": "managed_nodes.first_handover_gap", + "action_code": str(first_handover_gap_recovery.get("action_code") or "").strip(), + "title": str(first_handover_gap_recovery.get("recovery_label") or "").strip(), + "reason": str(first_handover_gap_recovery.get("summary") or "").strip() + or "来自首个 handover gap 的标准恢复动作。", + "focus_ref": dict(first_handover_gap_recovery.get("focus_ref") or {}), + }, + ) + for issue in issues: + if str(issue.get("code") or "").strip() == "runtime_build_schema_stale": + ordered_action_candidates.insert( + 0, + { + "source": f"issue:{issue.get('code')}", + "action_code": "api-restart", + "title": "优先重启控制面 API", + "reason": str(issue.get("detail") or issue.get("summary") or "").strip(), + "focus_ref": dict(issue.get("focus_ref") or {}), + }, + ) + break + for issue in issues: + issue_action_code = str(issue.get("action_code") or "").strip() + if issue_action_code: + ordered_action_candidates.append( + { + "source": f"issue:{issue.get('code')}", + "action_code": issue_action_code, + "title": str(issue.get("summary") or "").strip(), + "reason": str(issue.get("detail") or issue.get("summary") or "").strip(), + "focus_ref": dict(issue.get("focus_ref") or {}), + } + ) + + recommended_actions: list[dict] = [] + seen_action_codes: set[str] = set() + for item in ordered_action_candidates: + action_code = str(item.get("action_code") or "").strip() + if not action_code or action_code in seen_action_codes: + continue + seen_action_codes.add(action_code) + recommended_action_contract_keys = _normalize_ops_contract_keys( + ["ops_stack_diagnosis_contract"] + + _ops_contract_keys_from_action_code(action_code) + + _ops_contract_keys_from_focus_ref(item.get("focus_ref")) + ) + recommended_actions.append( + { + "action_code": action_code, + "source": str(item.get("source") or "").strip(), + "title": str(item.get("title") or "").strip(), + "reason": str(item.get("reason") or "").strip(), + "focus_ref": dict(item.get("focus_ref") or {}), + "command": build_bash_command("drive_ops_center.sh", "driver-resolve", stack_base_url, action_code), + "contract_keys": recommended_action_contract_keys, + "contract_navigation": _build_ops_contract_navigation( + recommended_action_contract_keys, + primary_contract_key="ops_stack_diagnosis_contract", + registry=contract_registry, + ), + } + ) + + blocking_issue_total = sum(1 for item in issues if str(item.get("severity") or "").strip() == "blocked") + warning_issue_total = sum(1 for item in issues if str(item.get("severity") or "").strip() == "warning") + info_issue_total = sum(1 for item in issues if str(item.get("severity") or "").strip() == "info") + + surface_status = "healthy" + if missing_surfaces: + surface_status = "partial" + + automation_status = "ready" + if blocking_issue_total > 0: + automation_status = "blocked" + elif warning_issue_total > 0: + automation_status = "attention" + + stack_status = "ready" + if automation_status == "blocked": + stack_status = "blocked" + elif surface_status != "healthy" or automation_status != "ready": + stack_status = "attention" + + top_issue = None + if issues: + top_issue = sorted( + issues, + key=lambda item: ( + -_ops_stack_issue_rank(str(item.get("severity") or "").strip()), + str(item.get("code") or "").strip(), + ), + )[0] + + next_step = {} + if recommended_actions: + first_action = dict(recommended_actions[0] or {}) + next_step_contract_keys = _normalize_ops_contract_keys( + ["ops_stack_diagnosis_contract"] + + _ops_contract_keys_from_action_code(str(first_action.get("action_code") or "").strip()) + + _ops_contract_keys_from_focus_ref(first_action.get("focus_ref")) + ) + next_step = { + "action_code": str(first_action.get("action_code") or "").strip(), + "source": str(first_action.get("source") or "").strip(), + "reason": str(first_action.get("reason") or "").strip(), + "command": str(first_action.get("command") or "").strip(), + "focus_ref": dict(first_action.get("focus_ref") or {}), + "contract_keys": next_step_contract_keys, + "contract_navigation": _build_ops_contract_navigation( + next_step_contract_keys, + primary_contract_key="ops_stack_diagnosis_contract", + registry=contract_registry, + ), + } + elif top_issue: + next_step_contract_keys = _normalize_ops_contract_keys(top_issue.get("contract_keys") or ["ops_stack_diagnosis_contract"]) + next_step = { + "action_code": "", + "source": f"issue:{str(top_issue.get('code') or '').strip()}", + "reason": str(top_issue.get("summary") or "").strip(), + "command": "", + "focus_ref": dict(top_issue.get("focus_ref") or {}), + "contract_keys": next_step_contract_keys, + "contract_navigation": _build_ops_contract_navigation( + next_step_contract_keys, + primary_contract_key="ops_stack_diagnosis_contract", + registry=contract_registry, + ), + } + + quick_commands = [ + build_bash_command("drive_ops_center.sh", "stack-diagnosis", stack_base_url, "summary"), + build_bash_command("drive_ops_center.sh", "doctor", stack_base_url), + ] + for action in recommended_actions[:3]: + command = str(action.get("command") or "").strip() + if command and command not in quick_commands: + quick_commands.append(command) + for issue in issues[:5]: + for command in list(issue.get("commands") or []): + normalized_command = str(command or "").strip() + if normalized_command and normalized_command not in quick_commands: + quick_commands.append(normalized_command) + if len(quick_commands) >= 8: + break + if len(quick_commands) >= 8: + break + if not bool(contracts_section.get("available", False)): + contract_check_command = build_bash_command("drive_ops_center.sh", "contracts", stack_base_url) + if contract_check_command not in quick_commands: + quick_commands.append(contract_check_command) + + recommended_commands = _build_stack_diagnosis_recommended_commands( + stack_base_url=stack_base_url, + recommended_actions=recommended_actions, + first_handover_gap_node_code=first_handover_gap_node_code, + launchpad_status=launchpad_status, + default_rollout_gate=default_rollout_gate, + next_step=next_step, + ) + operator_decision, next_actions = _build_stack_diagnosis_operator_decision( + issues=issues, + next_step=next_step, + recommended_actions=recommended_actions, + recommended_commands=recommended_commands, + first_handover_gap_node_code=first_handover_gap_node_code, + launchpad_status=launchpad_status, + default_rollout_gate=default_rollout_gate, + ) + + operator_hints: list[str] = [] + if stack_status == "blocked": + operator_hints.append("当前总检已经能看到现场，但海外单脑自动化执行链路仍未正式打通，先处理阻断问题再继续 Rollout 或批量运维。") + elif stack_status == "attention": + operator_hints.append("当前总检可用，但仍有缺口；建议先按 next_step 收口，再进入更细粒度脚本。") + else: + operator_hints.append("当前海外单脑控制面总检已基本就绪，可以把它作为后续所有联调和发布的固定起手式。") + if not bool(contracts_section.get("available", False)): + operator_hints.append("contracts 当前未正常可用，优先先恢复 contract registry，再让页面、CLI、Codex 共享同一份正式契约。") + if route_surface_missing_keys: + operator_hints.append("build-info 显示当前运行中的 API 路由面还不完整，先重启 API 并确认最新运维入口已经注册，再继续做节点接管或发布判断。") + if any(str(item.get("code") or "").strip() == "runtime_build_schema_stale" for item in issues): + operator_hints.append("当前属于典型的“仓库代码已更新，但运行中 API 还是旧 schema”场景，先重启 domaincheck-api，再重新看 stack-diagnosis 与 node-bootstrap-plan。") + if bool(log_sync.get("enabled", False)) is False and participating_nodes_total > 0: + operator_hints.append("远端日志回传当前关闭，后续如果继续多机联调，海外控制面会继续处于半盲态。") + next_step_action_code = str(next_step.get("action_code") or "").strip() + next_step_focus_ref = dict(next_step.get("focus_ref") or {}) + if next_step_action_code in {"bootstrap_run", "run_acceptance"}: + next_step_node_code = str(next_step_focus_ref.get("node_code") or first_handover_gap_node_code or "").strip() + operator_hints.append( + ( + f"当前第一动作已经收敛为 {'接入收口' if next_step_action_code == 'bootstrap_run' else '接管验收'}" + + (f"：{next_step_node_code}" if next_step_node_code else "") + + "，页面、CLI、Codex 都应优先围绕这一个节点推进，不再回退到泛化的托管节点修复。" + ) + ) + + resolved_launchpad_target_node_code = str(launchpad_status.get("recommended_target_node_code") or "").strip() + resolved_launchpad_recovery_label = str(launchpad_status.get("recommended_recovery_label") or "").strip() + resolved_launchpad_recovery_summary = str(launchpad_status.get("recommended_recovery_summary") or "").strip() + if ( + str(launchpad_status.get("recommended_action_code") or "").strip() not in {"bootstrap_run", "run_acceptance"} + and str(first_handover_gap_recovery.get("action_code") or "").strip() in {"bootstrap_run", "run_acceptance"} + ): + resolved_launchpad_target_node_code = ( + resolved_launchpad_target_node_code + or str(next_step_focus_ref.get("node_code") or "").strip() + or str(first_handover_gap_recovery.get("node_code") or "").strip() + ) + resolved_launchpad_recovery_label = ( + resolved_launchpad_recovery_label + or str(first_handover_gap_recovery.get("recovery_label") or "").strip() + ) + resolved_launchpad_recovery_summary = ( + resolved_launchpad_recovery_summary + or str(first_handover_gap_recovery.get("summary") or "").strip() + ) + + diagnosis_contract_keys = _normalize_ops_contract_keys( + ["ops_stack_diagnosis_contract", "ops_driver_contract"] + + [contract_key for issue in issues for contract_key in list(issue.get("contract_keys") or [])] + + list(next_step.get("contract_keys") or []) + ) + + return { + "generated_at": datetime.now().strftime("%Y-%m-%d %H:%M:%S"), + "diagnosis": { + "contract_key": "ops_stack_diagnosis_contract", + "contract_version": _OPS_CONTRACT_SCHEMA_VERSION, + "registry_version": _OPS_CONTRACT_REGISTRY_VERSION, + "base_url": stack_base_url, + "contract_navigation": _build_ops_contract_navigation( + diagnosis_contract_keys, + primary_contract_key="ops_stack_diagnosis_contract", + registry=contract_registry, + ), + "surface_status": surface_status, + "automation_status": automation_status, + "stack_status": stack_status, + "issue_total": len(issues), + "blocking_issue_total": blocking_issue_total, + "warning_issue_total": warning_issue_total, + "info_issue_total": info_issue_total, + "missing_surfaces": missing_surfaces, + "launchpad_recommended_target_node_code": resolved_launchpad_target_node_code, + "launchpad_recommended_recovery_label": resolved_launchpad_recovery_label, + "launchpad_recommended_recovery_summary": resolved_launchpad_recovery_summary, + "launchpad_onboarding_bootstrap_pending_nodes": int( + launchpad_status.get("onboarding_bootstrap_pending_nodes", 0) or 0 + ), + "launchpad_onboarding_acceptance_ready_nodes": int( + launchpad_status.get("onboarding_acceptance_ready_nodes", 0) or 0 + ), + "next_step": next_step, + "recommended_actions": recommended_actions[:5], + "operator_decision": operator_decision, + "next_actions": next_actions, + "recommended_commands": recommended_commands, + "issues": issues[:8], + "operator_hints": operator_hints, + "quick_commands": quick_commands, + }, + "api": { + "http_status": "200", + "available": True, + **api_info, + }, + "surface_matrix": { + "status_counts": surface_status_counts, + "items": [ + { + "name": name, + "available": bool(available), + "elapsed_ms": int( + { + "contracts": contracts_section, + "link_snapshot": link_snapshot_section, + "overview": overview_section, + "managed_nodes": nodes_section, + "release_hub": release_hub_section, + "runtime_build_info": build_info_section, + "playbook_runs": playbook_runs_section, + "activity_stream": activity_stream_section, + }.get(name, {}).get("elapsed_ms", 0) or 0 + ), + } + for name, available in surface_matrix + ], + }, + "contracts": { + "http_status": str(contracts_section.get("http_status") or "500"), + "available": bool(contracts_section.get("available", False)), + "elapsed_ms": int(contracts_section.get("elapsed_ms", 0) or 0), + "registry_version": str(contracts.get("registry_version") or ""), + "schema_version": str(contracts.get("schema_version") or ""), + "contracts_total": len(contracts_rows), + "contract_keys": [str(item.get("key") or "").strip() for item in contracts_rows if str(item.get("key") or "").strip()], + "error": str(contracts_section.get("error") or ""), + }, + "link_snapshot": { + "http_status": str(link_snapshot_section.get("http_status") or "500"), + "available": bool(link_snapshot_section.get("available", False)), + "elapsed_ms": int(link_snapshot_section.get("elapsed_ms", 0) or 0), + "status": str(link_snapshot.get("status") or ""), + "headline": str(link_snapshot.get("headline") or ""), + "next_action_codes": [ + str(item.get("action_code") or "").strip() + for item in list(link_snapshot.get("next_actions") or []) + if str(item.get("action_code") or "").strip() + ], + "error": str(link_snapshot_section.get("error") or ""), + }, + "overview": { + "http_status": str(overview_section.get("http_status") or "500"), + "available": bool(overview_section.get("available", False)), + "elapsed_ms": int(overview_section.get("elapsed_ms", 0) or 0), + "recommendation_key": str(recommendation.get("key") or ""), + "recommendation_title": str(recommendation.get("title") or ""), + "primary_action_code": str(recommendation.get("primary_action_code") or ""), + "secondary_action_code": str(recommendation.get("secondary_action_code") or ""), + "focus_ref": dict(recommendation.get("focus_ref") or {}), + "execution_scene_summary": str( + execution_scene.get("summary") + or (execution_scene.get("participation_summary") or {}).get("summary") + or execution_scene.get("phase_detail") + or "" + ), + "execution_scene_counts": dict(execution_scene.get("counts") or {}), + "log_sync": { + "status": str(log_sync.get("status") or log_sync.get("state") or ""), + "status_label": str(log_sync.get("status_label") or ""), + "mode": str(log_sync.get("mode") or ""), + "mode_label": str(log_sync.get("mode_label") or ""), + "enabled": bool(log_sync.get("enabled", False)), + "covered_participating_nodes": int(log_sync.get("covered_participating_node_count", 0) or 0), + "participating_nodes_total": int( + log_sync.get("participating_nodes_total", log_sync.get("participating_node_count", 0)) or 0 + ), + "missing_sample_node_codes": list( + log_sync.get("missing_sample_node_codes") + or log_sync.get("missing_participating_nodes") + or [] + ), + }, + "error": str(overview_section.get("error") or ""), + }, + "managed_nodes": { + "http_status": str(nodes_section.get("http_status") or "500"), + "available": bool(nodes_section.get("available", False)), + "elapsed_ms": int(nodes_section.get("elapsed_ms", 0) or 0), + "total": int(nodes_summary.get("total", len(nodes)) or 0), + "managed_total": int(nodes_summary.get("managed_total", 0) or 0), + "managed_enabled": int(nodes_summary.get("managed_enabled", 0) or 0), + "online": int(nodes_summary.get("online", 0) or 0), + "agent_ready": int(nodes_summary.get("agent_ready", 0) or 0), + "ssh_ready": int(nodes_summary.get("ssh_ready", 0) or 0), + "remote_access_ready": int(nodes_summary.get("remote_access_ready", 0) or 0), + "participating": int(nodes_summary.get("participating", 0) or 0), + "dispatch_active": int(nodes_summary.get("dispatch_active", 0) or 0), + "standby": int(nodes_summary.get("standby", 0) or 0), + "load_syncing": int(nodes_summary.get("load_syncing", 0) or 0), + "queue_retrying_nodes": int(nodes_summary.get("queue_retrying_nodes", 0) or 0), + "queue_dead_letter_nodes": int(nodes_summary.get("queue_dead_letter_nodes", 0) or 0), + "queue_pending_records": int(nodes_summary.get("queue_pending_records", 0) or 0), + "queue_dead_letter_records": int(nodes_summary.get("queue_dead_letter_records", 0) or 0), + "agent_state_counts": dict(nodes_summary.get("status_counts") or {}), + "remote_access_state_counts": dict(nodes_summary.get("remote_access_state_counts") or {}), + "delivery_queue_state_counts": dict(nodes_summary.get("delivery_queue_state_counts") or {}), + "problem_nodes": problem_nodes[:8], + "first_handover_gap_node_code": first_handover_gap_node_code, + "first_handover_gap": first_handover_gap, + "error": str(nodes_section.get("error") or ""), + }, + "release_hub": { + "http_status": str(release_hub_section.get("http_status") or "500"), + "available": bool(release_hub_section.get("available", False)), + "elapsed_ms": int(release_hub_section.get("elapsed_ms", 0) or 0), + "latest_release_version": str(latest_release.get("release_version") or ""), + "latest_release_status": str(latest_release.get("status") or ""), + "default_rollout_gate_status": str(default_rollout_gate.get("status") or ""), + "default_rollout_gate_label": str(default_rollout_gate.get("status_label") or ""), + "default_rollout_gate_summary": str(default_rollout_gate.get("summary_text") or default_rollout_gate.get("summary") or ""), + "default_rollout_gate_focus_ref": dict(default_rollout_gate.get("focus_ref") or {}), + "launchpad_status": str(launchpad_status.get("status") or ""), + "launchpad_status_label": str(launchpad_status.get("status_label") or ""), + "launchpad_summary": str(launchpad_status.get("summary_text") or launchpad_status.get("summary") or ""), + "recommended_action_code": str(launchpad_status.get("recommended_action_code") or ""), + "recommended_execution_mode": str(launchpad_status.get("recommended_execution_mode") or ""), + "focus_ref": dict(launchpad_status.get("focus_ref") or {}), + "error": str(release_hub_section.get("error") or ""), + }, + "runtime_build_info": { + "http_status": str(build_info_section.get("http_status") or "500"), + "available": bool(build_info_section.get("available", False)), + "elapsed_ms": int(build_info_section.get("elapsed_ms", 0) or 0), + "source": str(build_info.get("source") or ""), + "package_name": str(build_info.get("package_name") or ""), + "generated_at": str(build_info.get("generated_at") or ""), + "commit_sha": str(build_info.get("commit_sha") or ""), + "commit_ref": str(build_info.get("commit_ref") or ""), + "checksum": str(build_info.get("checksum") or ""), + "manifest_path": str(build_info.get("manifest_path") or ""), + "repository_capabilities": repository_capabilities, + "route_surface": route_surface, + "route_surface_missing_keys": route_surface_missing_keys, + "route_surface_declares_bootstrap_plan": route_surface_declares_bootstrap_plan, + "runtime_schema_stale": runtime_schema_stale, + "error": str(build_info_section.get("error") or ""), + }, + "playbook_runs": { + "http_status": str(playbook_runs_section.get("http_status") or "500"), + "available": bool(playbook_runs_section.get("available", False)), + "elapsed_ms": int(playbook_runs_section.get("elapsed_ms", 0) or 0), + "recent_total": len(playbook_runs), + "problem_runs_total": len(problem_runs), + "problem_runs": problem_runs[:5], + "error": str(playbook_runs_section.get("error") or ""), + }, + "activity_stream": { + "http_status": str(activity_stream_section.get("http_status") or "500"), + "available": bool(activity_stream_section.get("available", False)), + "elapsed_ms": int(activity_stream_section.get("elapsed_ms", 0) or 0), + "recent_total": len(activities), + "status_counts": activity_counts, + "top_items": [ + { + "kind": str(item.get("kind") or "").strip(), + "status": str(item.get("status") or item.get("job_status") or "").strip(), + "summary": str(item.get("summary_text") or item.get("summary") or "").strip(), + "occurred_at": str(item.get("occurred_at") or "").strip(), + "focus_ref": dict(item.get("focus_ref") or {}), + "source_focus_ref": dict(item.get("source_focus_ref") or {}), + "ui_intent_kind": str(((item.get("ui_intent") or {}).get("kind") or "")).strip(), + } + for item in activities[:5] + ], + "error": str(activity_stream_section.get("error") or ""), + }, + } + + +def get_ops_capabilities() -> dict: + contract_registry = get_ops_contract_registry() + return { + "control_plane": { + "deployment_mode": "overseas-control-plane", + "codex_driver_ready": True, + "ui_button_ready": True, + "daily_entrypoint": "overseas dashboard / ops api", + }, + "contract_registry": { + "endpoint": "/api/v1/ops/contracts", + "detail_endpoint_pattern": "/api/v1/ops/contracts/{contract_key}", + "registry_version": str(contract_registry.get("registry_version") or _OPS_CONTRACT_REGISTRY_VERSION), + "schema_version": str(contract_registry.get("schema_version") or _OPS_CONTRACT_SCHEMA_VERSION), + "contracts_total": int(contract_registry.get("contracts_total", 0) or 0), + "contract_keys": [ + str(item.get("key") or "").strip() + for item in list(contract_registry.get("contracts") or []) + if str(item.get("key") or "").strip() + ], + "schema_docs": { + str(item.get("key") or "").strip(): str(item.get("schema_doc_path") or "").strip() + for item in list(contract_registry.get("contracts") or []) + if str(item.get("key") or "").strip() + }, + }, + "capabilities": [ + { + "key": "cluster_visibility", + "title": "集群可观测", + "current_state": "ready", + "today": "已具备 runtime/cluster/readiness/sync-summary 聚合能力。", + "target": "继续增强节点参与度、诊断报告和远端日志回传。", + }, + { + "key": "remote_log_sync", + "title": "远端检测日志回传", + "current_state": "ready", + "today": "已具备日志回传开关、关键/全量模式、聚合样本面板与按节点现场日志钻取。", + "target": "后续如规模继续扩大，再补游标续传、流式 tail 与更长窗口归档。", + }, + { + "key": "button_driven_ops", + "title": "后台按钮化运维", + "current_state": "partial", + "today": "已有 runtime actions / ops jobs / ops playbooks，开始收口为统一按钮化编排。", + "target": "继续扩大安装、更新、重启、巡检、诊断包收集与发布回滚 playbook。", + }, + { + "key": "node_agent", + "title": "节点 Agent", + "current_state": "in_progress", + "today": "已具备 agent token / register / heartbeat / pull / complete 协议与本地执行器骨架。", + "target": "继续增强 stdout/stderr 游标回传、流式日志、诊断包与更多动作模板。", + }, + { + "key": "release_management", + "title": "发布包管理", + "current_state": "partial", + "today": "已具备 release / rollout / deploy.release 能力，支持 checksum、切换 current、健康检查、失败回滚，以及 Agent / SSH 双通道单节点发布。", + "target": "继续增强批次策略、灰度放量、发布观测面板与一键回滚编排。", + }, + ], + "action_catalog": [ + {"action": "node.bootstrap", "title": "纳管新节点", "transport": "ops-job -> control-plane -> bootstrap-plan"}, + {"action": "deploy.release", "title": "发布新版本", "transport": "ops-job -> node-agent / ssh-executor"}, + { + "action": "deploy.release.smart_worker", + "title": "按最新包一键 Worker 灰度", + "transport": "ops api -> package release builder -> rollout engine", + }, + { + "action": "deploy.release.smart_control", + "title": "按最新包一键 Control 发布", + "transport": "ops api -> package release builder -> rollout engine", + }, + { + "action": "deploy.rollout.smart_worker", + "title": "对已有 Release 一键 Worker 灰度", + "transport": "ops api -> rollout engine", + }, + { + "action": "deploy.rollout.smart_control", + "title": "对已有 Release 一键 Control 发布", + "transport": "ops api -> rollout engine", + }, + {"action": "service.restart", "title": "重启 API / Worker / Sync-Agent", "transport": "ops-job -> node-agent / ssh-executor"}, + {"action": "logs.collect", "title": "收集远端日志", "transport": "ops-job -> node-agent / ssh-executor"}, + {"action": "diagnostics.collect", "title": "收集诊断包", "transport": "ops-job -> node-agent / ssh-executor"}, + {"action": "ops.playbook.execute", "title": "执行标准 playbook", "transport": "ops api -> playbook expander -> ops-jobs"}, + {"action": "health.check", "title": "执行巡检", "transport": "ops-job -> node-agent"}, + {"action": "deploy.rollout.advance", "title": "推进下一批发布", "transport": "ops api -> rollout engine"}, + {"action": "runtime.restart_api", "title": "本机重启 API", "transport": "ops-job -> local-runtime / ssh-executor"}, + {"action": "runtime.start_worker", "title": "本机启动 Worker", "transport": "ops-job -> local-runtime / ssh-executor"}, + {"action": "runtime.stop_worker", "title": "本机停止 Worker", "transport": "ops-job -> local-runtime / ssh-executor"}, + {"action": "runtime.start_sync_agent", "title": "本机启动 Sync Agent", "transport": "ops-job -> local-runtime / ssh-executor"}, + {"action": "runtime.stop_sync_agent", "title": "本机停止 Sync Agent", "transport": "ops-job -> local-runtime / ssh-executor"}, + {"action": "runtime.pull_tasks", "title": "本机立即拉取任务", "transport": "ops-job -> local-runtime"}, + {"action": "runtime.push_sync", "title": "本机立即推送同步", "transport": "ops-job -> local-runtime"}, + {"action": "health.snapshot", "title": "采集本机运行时快照", "transport": "ops-job -> local-runtime / ssh-executor"}, + ], + } + + +def get_ops_go_live_summary(*, base_url: str = "") -> dict: + normalized_base_url = str(base_url or "").strip() or "http://127.0.0.1:8100" + stack_payload = get_ops_stack_diagnosis(base_url=normalized_base_url) + contracts_payload = get_ops_contract_registry() + launchpad_payload = get_release_launchpad() + build_info_payload = get_runtime_build_info() + nodes_payload = list_managed_nodes_with_agent_state() + overview_payload = get_ops_overview() + + diagnosis = dict(stack_payload.get("diagnosis") or {}) + diagnosis_issues = [dict(item or {}) for item in list(diagnosis.get("issues") or []) if item] + route_surface = dict(build_info_payload.get("route_surface") or {}) + repository_capabilities = dict(build_info_payload.get("repository_capabilities") or {}) + nodes_summary = dict(nodes_payload.get("summary") or {}) + execution_scene = dict(overview_payload.get("execution_scene") or {}) + log_sync = dict(execution_scene.get("log_sync") or overview_payload.get("log_sync") or {}) + recommendation = dict(overview_payload.get("recommendation") or {}) + launchpad_status = dict(launchpad_payload.get("launchpad_status") or {}) + next_step = dict(diagnosis.get("next_step") or {}) + operator_decision = dict(diagnosis.get("operator_decision") or {}) + contracts = [dict(item or {}) for item in list(contracts_payload.get("contracts") or []) if item] + + stack_status = str(diagnosis.get("stack_status") or "").strip() or "unknown" + launchpad_status_code = ( + str(launchpad_status.get("status") or "").strip() + or str((stack_payload.get("release_hub") or {}).get("launchpad_status") or "").strip() + ) + managed_enabled = int(nodes_summary.get("managed_enabled", 0) or 0) + remote_access_ready = int(nodes_summary.get("remote_access_ready", 0) or 0) + queue_dead_letter_nodes = int(nodes_summary.get("queue_dead_letter_nodes", 0) or 0) + participating_nodes_total = int(log_sync.get("participating_node_count", 0) or 0) + log_sync_covered_nodes = int(log_sync.get("covered_participating_node_count", 0) or 0) + launchpad_onboarding_bootstrap_pending_nodes = int( + launchpad_status.get("onboarding_bootstrap_pending_nodes", 0) or 0 + ) + launchpad_onboarding_acceptance_ready_nodes = int( + launchpad_status.get("onboarding_acceptance_ready_nodes", 0) or 0 + ) + start_delivery_issue_total = sum( + 1 + for item in diagnosis_issues + if str(item.get("code") or "").strip() == "ops_job_start_delivery_failed_local" + ) + + blocking_reasons: list[str] = [] + warnings: list[str] = [] + + if stack_status == "blocked": + blocking_reasons.append("stack_diagnosis=blocked") + elif stack_status == "attention": + warnings.append("stack_diagnosis=attention") + + route_surface_complete = bool(route_surface.get("surface_complete", False)) + route_surface_missing_keys = [ + str(item or "").strip() + for item in list(route_surface.get("missing_keys") or []) + if str(item or "").strip() + ] + route_surface_expected_paths = { + str(key).strip(): str(value).strip() + for key, value in dict(route_surface.get("expected_paths") or {}).items() + if str(key).strip() + } + route_surface_declares_bootstrap_plan = "ops_node_handover_bootstrap_plan" in route_surface_expected_paths + runtime_schema_stale = bool( + repository_capabilities.get("supports_install_command_block", False) and not route_surface_declares_bootstrap_plan + ) + if not route_surface_complete: + blocking_reasons.append( + "route_surface_incomplete" + + (f":{','.join(route_surface_missing_keys)}" if route_surface_missing_keys else "") + ) + elif runtime_schema_stale: + warnings.append("runtime_build_schema_stale") + + if not contracts: + blocking_reasons.append("contracts_unavailable") + + if launchpad_status_code == "blocked": + blocking_reasons.append("release_launchpad=blocked") + elif launchpad_status_code == "attention": + warnings.append("release_launchpad=attention") + if launchpad_onboarding_bootstrap_pending_nodes > 0: + warnings.append(f"launchpad_onboarding_bootstrap_pending={launchpad_onboarding_bootstrap_pending_nodes}") + if launchpad_onboarding_acceptance_ready_nodes > 0: + warnings.append(f"launchpad_onboarding_acceptance_ready={launchpad_onboarding_acceptance_ready_nodes}") + + if managed_enabled > 0 and remote_access_ready == 0: + blocking_reasons.append("remote_agent_ready=0") + elif managed_enabled > 0 and remote_access_ready < managed_enabled: + warnings.append(f"remote_agent_ready={remote_access_ready}/{managed_enabled}") + + if queue_dead_letter_nodes > 0: + warnings.append(f"queue_dead_letter_nodes={queue_dead_letter_nodes}") + if start_delivery_issue_total > 0: + warnings.append(f"ops_job_start_delivery_failed_local={start_delivery_issue_total}") + + if bool(log_sync.get("enabled")) and participating_nodes_total > 0: + if log_sync_covered_nodes == 0: + warnings.append(f"log_sync_waiting_sample=0/{participating_nodes_total}") + elif log_sync_covered_nodes < participating_nodes_total: + warnings.append(f"log_sync_partial={log_sync_covered_nodes}/{participating_nodes_total}") + + publish_blocking_reasons: list[str] = [] + publish_warnings: list[str] = [] + if stack_status == "blocked": + publish_blocking_reasons.append("stack_diagnosis=blocked") + elif stack_status == "attention": + publish_warnings.append("stack_diagnosis=attention") + if not route_surface_complete: + publish_blocking_reasons.append( + "route_surface_incomplete" + + (f":{','.join(route_surface_missing_keys)}" if route_surface_missing_keys else "") + ) + elif runtime_schema_stale: + publish_warnings.append("runtime_build_schema_stale") + if not contracts: + publish_blocking_reasons.append("contracts_unavailable") + if launchpad_status_code == "blocked": + publish_blocking_reasons.append("release_launchpad=blocked") + elif launchpad_status_code == "attention": + publish_warnings.append("release_launchpad=attention") + if launchpad_onboarding_bootstrap_pending_nodes > 0: + publish_warnings.append(f"launchpad_onboarding_bootstrap_pending={launchpad_onboarding_bootstrap_pending_nodes}") + if launchpad_onboarding_acceptance_ready_nodes > 0: + publish_warnings.append(f"launchpad_onboarding_acceptance_ready={launchpad_onboarding_acceptance_ready_nodes}") + if managed_enabled > 0 and remote_access_ready == 0: + publish_blocking_reasons.append("remote_agent_ready=0") + elif managed_enabled > 0 and remote_access_ready < managed_enabled: + publish_warnings.append(f"remote_agent_ready={remote_access_ready}/{managed_enabled}") + if queue_dead_letter_nodes > 0: + publish_blocking_reasons.append(f"queue_dead_letter_nodes={queue_dead_letter_nodes}") + if start_delivery_issue_total > 0: + publish_warnings.append(f"ops_job_start_delivery_failed_local={start_delivery_issue_total}") + if bool(log_sync.get("enabled")) and participating_nodes_total > 0: + if log_sync_covered_nodes == 0: + publish_warnings.append(f"log_sync_waiting_sample=0/{participating_nodes_total}") + elif log_sync_covered_nodes < participating_nodes_total: + publish_warnings.append(f"log_sync_partial={log_sync_covered_nodes}/{participating_nodes_total}") + + publish_status = "ready" + if publish_blocking_reasons: + publish_status = "blocked" + elif publish_warnings: + publish_status = "attention" + publish_ready = publish_status == "ready" + if publish_status == "ready": + publish_status_label = "可发布" + publish_summary = "当前执行面、路由面、合同面与发布门禁已收口，可以进入正式发版。" + elif publish_status == "attention": + publish_status_label = "可发布但建议先复核" + publish_summary = "当前没有硬阻断，但仍有上线前关注项，建议先完成复核再正式发版。" + else: + publish_status_label = "暂不可发布" + publish_summary = "当前仍有硬阻断项，不能直接进入正式发版。" + + go_live_status = "ready" + if blocking_reasons: + go_live_status = "blocked" + elif warnings: + go_live_status = "attention" + + next_step_action_code = ( + str(next_step.get("action_code") or "").strip() + or str(recommendation.get("primary_action_code") or "").strip() + or str(launchpad_status.get("recommended_action_code") or "").strip() + ) + next_step_reason = ( + str(next_step.get("reason") or "").strip() + or str(recommendation.get("reason") or "").strip() + or "来自 overview / launchpad fallback" + ) + stack_launchpad_target_node_code = str( + diagnosis.get("launchpad_recommended_target_node_code") + or (dict(next_step.get("focus_ref") or {}).get("node_code") if next_step_action_code in {"bootstrap_run", "run_acceptance"} else "") + or "" + ).strip() + stack_launchpad_recovery_label = str( + diagnosis.get("launchpad_recommended_recovery_label") + or ("签发接入工单" if next_step_action_code == "bootstrap_run" else "执行接管验收" if next_step_action_code == "run_acceptance" else "") + or "" + ).strip() + stack_launchpad_recovery_summary = str( + diagnosis.get("launchpad_recommended_recovery_summary") + or next_step_reason + or "" + ).strip() + launchpad_recommended_target_node_code = str( + launchpad_status.get("recommended_target_node_code") or stack_launchpad_target_node_code or "" + ).strip() + launchpad_recommended_recovery_label = str( + launchpad_status.get("recommended_recovery_label") or stack_launchpad_recovery_label or "" + ).strip() + launchpad_recommended_recovery_summary = str( + launchpad_status.get("recommended_recovery_summary") or stack_launchpad_recovery_summary or "" + ).strip() + operator_lane = str(operator_decision.get("lane") or "").strip() + if not operator_lane: + operator_lane = "recovery" if blocking_reasons else ("verification" if warnings else "go-live") + operator_title = ( + str(operator_decision.get("title") or "").strip() + or str(recommendation.get("title") or "").strip() + or str(launchpad_status.get("status_label") or "").strip() + or ("先处理阻塞项" if blocking_reasons else "可以进入上线复核") + ) + operator_primary_command_key = str(operator_decision.get("primary_command_key") or "").strip() or next_step_action_code + + recommended_commands = { + "stack_summary": build_bash_command("check_ops_center_stack.sh", normalized_base_url, "summary"), + "contracts": build_bash_command("check_ops_contracts.sh", normalized_base_url), + "ops_plane": build_bash_command("check_ops_plane.sh", normalized_base_url), + "release_hub": build_bash_command("check_release_hub.sh", normalized_base_url), + "inspection": build_bash_command("check_ops_inspection.sh", normalized_base_url), + "overview": build_bash_command("drive_ops_center.sh", "overview", normalized_base_url), + "go_live_recover": build_bash_command("drive_ops_center.sh", "go-live-recover", normalized_base_url), + "doctor_export": build_bash_command("drive_ops_center.sh", "doctor-export", "/tmp/domaincheck-go-live", normalized_base_url), + } + if next_step_action_code: + recommended_commands["next_step"] = build_bash_command("drive_ops_center.sh", "driver-resolve", normalized_base_url, next_step_action_code) + if runtime_schema_stale: + recommended_commands["runtime_refresh_recover"] = build_bash_command( + "drive_ops_center.sh", + "runtime-refresh-recover", + normalized_base_url, + ) + recommended_commands["api_restart"] = recommended_commands["runtime_refresh_recover"] + recommended_commands["bootstrap_plan_recheck"] = build_bash_command( + "drive_ops_center.sh", + "node-bootstrap-plan", + normalized_base_url, + "mainland-worker-01", + ) + if bool(log_sync.get("enabled")): + recommended_commands["log_sync_logs"] = build_bash_command("drive_ops_center.sh", "driver-resolve", normalized_base_url, "open_worker_logs_participating") + recommended_commands["log_sync_inspection"] = build_bash_command("drive_ops_center.sh", "driver-resolve", normalized_base_url, "run_inspection_participating") + + return { + "base_url": normalized_base_url, + "generated_at": datetime.now().strftime("%Y-%m-%d %H:%M:%S"), + "go_live_status": go_live_status, + "publish_ready": publish_ready, + "publish_status": publish_status, + "publish_status_label": publish_status_label, + "publish_summary": publish_summary, + "stack_status": stack_status, + "contracts_ready": bool(contracts), + "contracts_total": len(contracts), + "launchpad_status": launchpad_status_code, + "launchpad_status_label": str(launchpad_status.get("status_label") or "").strip(), + "launchpad_recommended_action_code": str(launchpad_status.get("recommended_action_code") or "").strip(), + "launchpad_recommended_target_node_code": launchpad_recommended_target_node_code, + "launchpad_recommended_recovery_label": launchpad_recommended_recovery_label, + "launchpad_recommended_recovery_summary": launchpad_recommended_recovery_summary, + "launchpad_onboarding_bootstrap_pending_nodes": launchpad_onboarding_bootstrap_pending_nodes, + "launchpad_onboarding_acceptance_ready_nodes": launchpad_onboarding_acceptance_ready_nodes, + "route_surface_complete": route_surface_complete, + "route_surface_missing_keys": route_surface_missing_keys, + "route_surface_declares_bootstrap_plan": route_surface_declares_bootstrap_plan, + "runtime_schema_stale": runtime_schema_stale, + "repository_capabilities": repository_capabilities, + "managed_enabled": managed_enabled, + "remote_access_ready": remote_access_ready, + "queue_dead_letter_nodes": queue_dead_letter_nodes, + "activity_start_delivery_issue_total": start_delivery_issue_total, + "participating_nodes_total": participating_nodes_total, + "log_sync_enabled": bool(log_sync.get("enabled")), + "log_sync_state": str(log_sync.get("state") or "").strip(), + "log_sync_mode": str(log_sync.get("mode") or "").strip(), + "log_sync_covered_nodes": log_sync_covered_nodes, + "log_sync_missing_node_codes": [ + str(item or "").strip() + for item in list(log_sync.get("missing_participating_nodes") or []) + if str(item or "").strip() + ], + "next_step_action_code": next_step_action_code, + "next_step_reason": next_step_reason, + "operator_lane": operator_lane, + "operator_title": operator_title, + "operator_primary_command_key": operator_primary_command_key, + "publish_blocking_reasons": publish_blocking_reasons, + "publish_warnings": publish_warnings, + "blocking_reasons": blocking_reasons, + "warnings": warnings, + "recommended_commands": recommended_commands, + "source_refs": { + "stack_diagnosis_contract_key": str(diagnosis.get("contract_key") or "").strip(), + "contracts_registry_version": str(contracts_payload.get("registry_version") or "").strip(), + "runtime_build_commit_sha": str(build_info_payload.get("commit_sha") or "").strip(), + "release_focus_ref": dict(launchpad_status.get("focus_ref") or {}), + }, + } + + +def _build_doctor_preferred_surface(operator_decision: dict) -> str: + next_focus = dict(operator_decision.get("next_focus") or {}) + focus_kind = str(next_focus.get("kind") or "").strip() + lane = str(operator_decision.get("lane") or "").strip() + if focus_kind == "node_scene_log": + return "execution-scene" + if lane == "node_handover": + return "managed-nodes" + if lane == "release": + return "release-hub" + if lane == "observability": + return "execution-scene" + if lane in {"ops_jobs", "steady"}: + return "driver-feed" + return "stack-summary" + + +def _dedupe_command_list(command_candidates: list[object]) -> list[str]: + normalized_commands: list[str] = [] + for candidate in command_candidates: + if isinstance(candidate, (list, tuple)): + for nested in candidate: + nested_command = str(nested or "").strip() + if nested_command and nested_command not in normalized_commands: + normalized_commands.append(nested_command) + continue + command = str(candidate or "").strip() + if command and command not in normalized_commands: + normalized_commands.append(command) + return normalized_commands + + +def _doctor_recommended_command_map( + *, + base_url: str, + manifest_target: str, + raw_recommended_commands: list[str], +) -> dict[str, str]: + target = str(manifest_target or "").strip() or str(base_url or "").strip() or "http://127.0.0.1:8100" + mapped = { + "doctor_decision": build_bash_command("drive_ops_center.sh", "doctor-decision", target), + } + if str(manifest_target or "").strip(): + mapped["go_live_review"] = build_bash_command("drive_ops_center.sh", "go-live-review", manifest_target) + mapped["go_live_signoff"] = build_bash_command("drive_ops_center.sh", "go-live-signoff", manifest_target) + else: + mapped["stack_check"] = build_bash_command("drive_ops_center.sh", "stack-diagnosis", base_url, "summary") + mapped["stack_diagnosis"] = mapped["stack_check"] + mapped["doctor"] = build_bash_command("drive_ops_center.sh", "doctor", base_url) + + for index, command in enumerate(_dedupe_command_list(list(raw_recommended_commands or []))[:4]): + key = "primary_recovery" if index == 0 else f"follow_up_{index}" + mapped[key] = command + return mapped + + +def _doctor_scene_log_problem_node_codes(scene_log_reports: list[dict]) -> list[str]: + problem_nodes: list[str] = [] + for item in scene_log_reports: + if not isinstance(item, dict): + continue + node_code = str(item.get("node_code") or "").strip() + if not node_code: + continue + if item.get("ok") is False: + if node_code not in problem_nodes: + problem_nodes.append(node_code) + continue + status = str(item.get("status") or "").strip().lower() + if status and status not in {"ok", "ready", "success"} and node_code not in problem_nodes: + problem_nodes.append(node_code) + return problem_nodes + + +def _build_live_doctor_payload(base_url: str) -> tuple[dict, dict]: + go_live_summary = get_ops_go_live_summary(base_url=base_url) + stack_payload = get_ops_stack_diagnosis(base_url=base_url) + diagnosis = dict(stack_payload.get("diagnosis") or {}) + operator_decision = dict(diagnosis.get("operator_decision") or {}) + next_step = dict(diagnosis.get("next_step") or {}) + next_actions = dict(diagnosis.get("next_actions") or {}) + recommended_commands = dict(diagnosis.get("recommended_commands") or {}) + stack_status = str(diagnosis.get("stack_status") or "").strip() + go_live_status = str(go_live_summary.get("go_live_status") or "").strip() + publish_status = str(go_live_summary.get("publish_status") or "").strip() + + status = "ready" + if "blocked" in {stack_status, go_live_status, publish_status}: + status = "blocked" + elif "attention" in {stack_status, go_live_status, publish_status}: + status = "attention" + + preferred_surface = _build_doctor_preferred_surface(operator_decision) + headline = ( + str(operator_decision.get("title") or "").strip() + or str(diagnosis.get("headline") or "").strip() + or str(go_live_summary.get("headline") or "").strip() + or "当前总检主决策还没有形成明确 headline。" + ) + detail = ( + str(operator_decision.get("summary") or "").strip() + or str(next_step.get("reason") or "").strip() + or str(go_live_summary.get("next_step_reason") or "").strip() + or str(diagnosis.get("blocked_issue_detail") or "").strip() + ) + next_action_code = ( + str(operator_decision.get("primary_command_key") or "").strip() + or str(next_step.get("action_code") or "").strip() + or str(go_live_summary.get("next_step_action_code") or "").strip() + ) + + raw_commands = _dedupe_command_list( + [ + next_actions.get("primary_command"), + next_actions.get("secondary_command"), + recommended_commands.get(next_action_code), + list(recommended_commands.values())[:3], + ] + ) + + decision = { + "status": status, + "reason_code": str( + operator_decision.get("reason_code") or next_step.get("source") or stack_status or "live_fallback" + ).strip(), + "headline": headline, + "detail": detail, + "preferred_surface": preferred_surface, + "next_action_code": next_action_code, + "recommended_commands": raw_commands, + "evidence": { + "blocking_issue_total": int(diagnosis.get("blocking_issue_total", 0) or 0), + "warning_issue_total": int(diagnosis.get("warning_issue_total", 0) or 0), + "missing_surfaces": list(diagnosis.get("missing_surfaces") or []), + }, + } + summary = { + "ok": status == "ready", + "required_failures": [], + "optional_unavailable": [], + "scene_log_reports_total": 0, + "scene_log_reports_ok": 0, + "scene_log_status_counts": {}, + "scene_log_reports": [], + "contract_surface_gaps": [], + "launchpad_recommended_target_node_code": str( + go_live_summary.get("launchpad_recommended_target_node_code") or "" + ).strip(), + "launchpad_recommended_recovery_label": str( + go_live_summary.get("launchpad_recommended_recovery_label") or "" + ).strip(), + "launchpad_recommended_recovery_summary": str( + go_live_summary.get("launchpad_recommended_recovery_summary") or "" + ).strip(), + "launchpad_onboarding_bootstrap_pending_nodes": int( + go_live_summary.get("launchpad_onboarding_bootstrap_pending_nodes", 0) or 0 + ), + "launchpad_onboarding_acceptance_ready_nodes": int( + go_live_summary.get("launchpad_onboarding_acceptance_ready_nodes", 0) or 0 + ), + } + return decision, summary + + +def get_ops_doctor_decision(*, base_url: str = "", report_dir: str = "") -> dict: + normalized_base_url = str(base_url or "").strip() or "http://127.0.0.1:8100" + normalized_report_dir = str(report_dir or "").strip() + contract_registry = get_ops_contract_registry() + + manifest_path: Path | None = None + resolved_report_dir: Path | None = None + if normalized_report_dir: + candidate_path = Path(normalized_report_dir) + if candidate_path.is_dir(): + resolved_report_dir = candidate_path + nested_manifest = candidate_path / "manifest.json" + if nested_manifest.is_file(): + manifest_path = nested_manifest + elif candidate_path.is_file(): + manifest_path = candidate_path + resolved_report_dir = candidate_path.parent + else: + manifest_path, resolved_report_dir = _find_latest_go_live_bundle_manifest() + + raw_decision: dict = {} + raw_summary: dict = {} + source = "live_fallback" + + if manifest_path is not None and manifest_path.is_file(): + manifest = _load_json_path(manifest_path) + artifacts = list(manifest.get("artifacts") or []) + artifact_by_key = { + str(item.get("key") or "").strip(): dict(item) + for item in artifacts + if isinstance(item, dict) and str(item.get("key") or "").strip() + } + doctor_artifact = artifact_by_key.get("doctor_decision") or {} + doctor_artifact_path = Path(str(doctor_artifact.get("path") or "").strip()) if doctor_artifact else None + doctor_payload = _load_json_path(doctor_artifact_path) if doctor_artifact_path else {} + raw_decision = dict(doctor_payload.get("decision") or manifest.get("decision") or {}) + raw_summary = dict(doctor_payload.get("summary") or manifest.get("summary") or {}) + if raw_decision: + source = "bundle_manifest" if doctor_payload else "doctor_manifest" + + if not raw_decision: + raw_decision, raw_summary = _build_live_doctor_payload(normalized_base_url) + source = "live_fallback" + + status = str(raw_decision.get("status") or "").strip() or ("ready" if bool(raw_summary.get("ok", False)) else "attention") + status_label = { + "ready": "主决策已就绪", + "attention": "主决策待复核", + "blocked": "主决策阻断", + "missing": "待生成", + }.get(status, status or "主决策未知") + summary_scene_log_reports = [ + dict(item) + for item in list(raw_summary.get("scene_log_reports") or []) + if isinstance(item, dict) + ] + scene_log_problem_node_codes = _doctor_scene_log_problem_node_codes(summary_scene_log_reports) + manifest_target = str(manifest_path or normalized_report_dir or "").strip() + recommended_commands = _doctor_recommended_command_map( + base_url=normalized_base_url, + manifest_target=manifest_target, + raw_recommended_commands=[ + str(item).strip() + for item in list(raw_decision.get("recommended_commands") or []) + if str(item).strip() + ], + ) + contract_keys = _normalize_ops_contract_keys( + [ + "ops_doctor_decision_contract", + "ops_stack_diagnosis_contract", + "ops_driver_contract", + "ops_go_live_bundle_contract", + "ops_go_live_signoff_contract", + ] + ) + + return { + "generated_at": datetime.now().isoformat(timespec="seconds"), + "base_url": normalized_base_url, + "contract_key": "ops_doctor_decision_contract", + "contract_version": str( + (contract_registry.get("contracts_by_key") or {}).get("ops_doctor_decision_contract", {}).get("version") + or _OPS_CONTRACT_SCHEMA_VERSION + ).strip(), + "status": status, + "status_label": status_label, + "headline": str(raw_decision.get("headline") or "").strip() or "当前还没有生成总检主决策。", + "detail": str(raw_decision.get("detail") or "").strip(), + "source": source, + "report_dir": str((resolved_report_dir or manifest_path.parent) if manifest_path else normalized_report_dir), + "manifest_path": str(manifest_path or ""), + "decision_available": bool(raw_decision), + "decision": { + "status": status, + "reason_code": str(raw_decision.get("reason_code") or "").strip(), + "headline": str(raw_decision.get("headline") or "").strip(), + "detail": str(raw_decision.get("detail") or "").strip(), + "preferred_surface": str(raw_decision.get("preferred_surface") or "").strip(), + "next_action_code": str(raw_decision.get("next_action_code") or "").strip(), + "recommended_commands": [ + str(item).strip() + for item in list(raw_decision.get("recommended_commands") or []) + if str(item).strip() + ], + "evidence": dict(raw_decision.get("evidence") or {}), + }, + "summary": { + "ok": bool(raw_summary.get("ok", False)), + "required_failures": [ + str(item).strip() + for item in list(raw_summary.get("required_failures") or []) + if str(item).strip() + ], + "optional_unavailable": [ + str(item).strip() + for item in list(raw_summary.get("optional_unavailable") or []) + if str(item).strip() + ], + "scene_log_reports_total": int(raw_summary.get("scene_log_reports_total", len(summary_scene_log_reports)) or 0), + "scene_log_reports_ok": int(raw_summary.get("scene_log_reports_ok", 0) or 0), + "scene_log_status_counts": dict(raw_summary.get("scene_log_status_counts") or {}), + "scene_log_reports": summary_scene_log_reports, + "scene_log_problem_node_codes": scene_log_problem_node_codes, + "contract_surface_gaps": [ + str(item).strip() + for item in list(raw_summary.get("contract_surface_gaps") or []) + if str(item).strip() + ], + "launchpad_recommended_target_node_code": str( + raw_summary.get("launchpad_recommended_target_node_code") or "" + ).strip(), + "launchpad_recommended_recovery_label": str( + raw_summary.get("launchpad_recommended_recovery_label") or "" + ).strip(), + "launchpad_recommended_recovery_summary": str( + raw_summary.get("launchpad_recommended_recovery_summary") or "" + ).strip(), + "launchpad_onboarding_bootstrap_pending_nodes": int( + raw_summary.get("launchpad_onboarding_bootstrap_pending_nodes", 0) or 0 + ), + "launchpad_onboarding_acceptance_ready_nodes": int( + raw_summary.get("launchpad_onboarding_acceptance_ready_nodes", 0) or 0 + ), + }, + "recommended_commands": recommended_commands, + "contract_navigation": _build_ops_contract_navigation( + contract_keys, + primary_contract_key="ops_doctor_decision_contract", + registry=contract_registry, + ), + } + + +def get_ops_go_live_signoff(*, base_url: str = "", report_dir: str = "") -> dict: + normalized_base_url = str(base_url or "").strip() or "http://127.0.0.1:8100" + normalized_report_dir = str(report_dir or "").strip() + go_live_summary = get_ops_go_live_summary(base_url=normalized_base_url) + stack_payload = get_ops_stack_diagnosis(base_url=normalized_base_url) + driver_feed = get_ops_driver_feed() + codex_brief = get_ops_codex_brief() + release_launchpad = get_release_launchpad() + go_live_review = get_ops_go_live_review(base_url=normalized_base_url, report_dir=normalized_report_dir) + doctor_decision_payload = get_ops_doctor_decision(base_url=normalized_base_url, report_dir=normalized_report_dir) + contract_registry = get_ops_contract_registry() + + stack_diagnosis = dict(stack_payload.get("diagnosis") or {}) + driver_summary = dict(driver_feed.get("summary") or {}) + codex_summary = dict(codex_brief.get("summary") or {}) + launchpad_status = dict(release_launchpad.get("launchpad_status") or {}) + operator_decision = dict(stack_diagnosis.get("operator_decision") or {}) + next_step = dict(stack_diagnosis.get("next_step") or {}) + doctor_decision = dict(doctor_decision_payload.get("decision") or {}) + + launchpad_target_node_codes = { + "go_live_summary": str(go_live_summary.get("launchpad_recommended_target_node_code") or "").strip(), + "driver_feed": str(driver_summary.get("launchpad_recommended_target_node_code") or "").strip(), + "codex_brief": str(codex_summary.get("launchpad_recommended_target_node_code") or "").strip(), + "release_launchpad": str(launchpad_status.get("recommended_target_node_code") or "").strip(), + } + launchpad_recovery_labels = { + "go_live_summary": str(go_live_summary.get("launchpad_recommended_recovery_label") or "").strip(), + "driver_feed": str(driver_summary.get("launchpad_recommended_recovery_label") or "").strip(), + "codex_brief": str(codex_summary.get("launchpad_recommended_recovery_label") or "").strip(), + "release_launchpad": str(launchpad_status.get("recommended_recovery_label") or "").strip(), + } + launchpad_recovery_summaries = { + "go_live_summary": str(go_live_summary.get("launchpad_recommended_recovery_summary") or "").strip(), + "driver_feed": str(driver_summary.get("launchpad_recommended_recovery_summary") or "").strip(), + "codex_brief": str(codex_summary.get("launchpad_recommended_recovery_summary") or "").strip(), + "release_launchpad": str(launchpad_status.get("recommended_recovery_summary") or "").strip(), + } + launchpad_bootstrap_pending_nodes = { + "go_live_summary": int(go_live_summary.get("launchpad_onboarding_bootstrap_pending_nodes", 0) or 0), + "driver_feed": int(driver_summary.get("launchpad_onboarding_bootstrap_pending_nodes", 0) or 0), + "codex_brief": int(codex_summary.get("launchpad_onboarding_bootstrap_pending_nodes", 0) or 0), + "release_launchpad": int(launchpad_status.get("onboarding_bootstrap_pending_nodes", 0) or 0), + } + launchpad_acceptance_ready_nodes = { + "go_live_summary": int(go_live_summary.get("launchpad_onboarding_acceptance_ready_nodes", 0) or 0), + "driver_feed": int(driver_summary.get("launchpad_onboarding_acceptance_ready_nodes", 0) or 0), + "codex_brief": int(codex_summary.get("launchpad_onboarding_acceptance_ready_nodes", 0) or 0), + "release_launchpad": int(launchpad_status.get("onboarding_acceptance_ready_nodes", 0) or 0), + } + + def _nonempty_distinct(values: dict[str, str]) -> list[str]: + return sorted({value for value in values.values() if value}) + + def _distinct_ints(values: dict[str, int]) -> list[int]: + return sorted({int(value) for value in values.values()}) + + launchpad_alignment = { + "consistent": True, + "target_node_code_consistent": len(_nonempty_distinct(launchpad_target_node_codes)) <= 1, + "recovery_label_consistent": len(_nonempty_distinct(launchpad_recovery_labels)) <= 1, + "recovery_summary_consistent": len(_nonempty_distinct(launchpad_recovery_summaries)) <= 1, + "bootstrap_pending_consistent": len(_distinct_ints(launchpad_bootstrap_pending_nodes)) <= 1, + "acceptance_ready_consistent": len(_distinct_ints(launchpad_acceptance_ready_nodes)) <= 1, + "available_sources": sorted(launchpad_target_node_codes.keys()), + "target_node_codes": launchpad_target_node_codes, + "recovery_labels": launchpad_recovery_labels, + "recovery_summaries": launchpad_recovery_summaries, + "bootstrap_pending_nodes": launchpad_bootstrap_pending_nodes, + "acceptance_ready_nodes": launchpad_acceptance_ready_nodes, + } + launchpad_alignment["consistent"] = all( + bool(launchpad_alignment.get(key)) + for key in ( + "target_node_code_consistent", + "recovery_label_consistent", + "recovery_summary_consistent", + "bootstrap_pending_consistent", + "acceptance_ready_consistent", + ) + ) + + blocked_reasons: list[str] = [] + attention_reasons: list[str] = [] + + go_live_status = str(go_live_summary.get("go_live_status") or "").strip() + publish_status = str(go_live_summary.get("publish_status") or "").strip() + stack_status = str(stack_diagnosis.get("stack_status") or "").strip() + driver_launch_status = str(driver_summary.get("launch_status") or "").strip() + codex_launch_status = str(codex_summary.get("launch_status") or "").strip() + release_launchpad_status = str(launchpad_status.get("status") or "").strip() + review_status = str(go_live_review.get("status") or "").strip() + doctor_status = str(doctor_decision_payload.get("status") or "").strip() + + if go_live_status == "blocked": + blocked_reasons.append("go_live_summary=blocked") + elif go_live_status == "attention": + attention_reasons.append("go_live_summary=attention") + if publish_status == "blocked": + blocked_reasons.append("publish_status=blocked") + elif publish_status == "attention": + attention_reasons.append("publish_status=attention") + if stack_status == "blocked": + blocked_reasons.append("stack_diagnosis=blocked") + elif stack_status == "attention": + attention_reasons.append("stack_diagnosis=attention") + if driver_launch_status == "blocked": + blocked_reasons.append("driver_feed_launch=blocked") + elif driver_launch_status == "attention": + attention_reasons.append("driver_feed_launch=attention") + if codex_launch_status == "blocked": + blocked_reasons.append("codex_brief_launch=blocked") + elif codex_launch_status == "attention": + attention_reasons.append("codex_brief_launch=attention") + if release_launchpad_status == "blocked": + blocked_reasons.append("release_launchpad=blocked") + elif release_launchpad_status == "attention": + attention_reasons.append("release_launchpad=attention") + if review_status == "blocked": + blocked_reasons.append("go_live_review=blocked") + elif review_status in {"attention", "missing"}: + attention_reasons.append(f"go_live_review={review_status}") + if doctor_status == "blocked": + blocked_reasons.append("doctor_decision=blocked") + elif doctor_status in {"attention", "missing"}: + attention_reasons.append(f"doctor_decision={doctor_status}") + if not bool(launchpad_alignment.get("consistent", False)): + attention_reasons.append("launchpad_alignment=inconsistent") + + signoff_status = "ready" + if blocked_reasons: + signoff_status = "blocked" + elif attention_reasons: + signoff_status = "attention" + + status_label = { + "ready": "可签字上线", + "attention": "待签字复核", + "blocked": "暂不可签字", + }.get(signoff_status, signoff_status or "未知") + if signoff_status == "blocked": + primary_blocked_surface = ( + "正式复核" + if review_status == "blocked" + else "总检主决策" + if doctor_status == "blocked" + else "发布门禁" + ) + headline = ( + f"当前仍存在{primary_blocked_surface}阻断，建议优先处理 " + f"{str(go_live_summary.get('operator_title') or '').strip() or str(operator_decision.get('title') or '').strip() or str(doctor_decision.get('next_action_code') or '').strip() or '阻断项'}。" + ) + elif signoff_status == "attention": + if review_status in {"attention", "missing"}: + headline = "当前已进入最终收口阶段，但正式复核仍待补口，建议先确认 bundle manifest 结论再签字上线。" + elif doctor_status in {"attention", "missing"}: + headline = "当前已进入最终收口阶段，但总检主决策仍待补口，建议先按主建议完成复核再签字上线。" + else: + headline = ( + f"当前已进入最终收口阶段，但仍建议先完成 {str(go_live_summary.get('operator_title') or '').strip() or str(operator_decision.get('title') or '').strip() or 'attention 项'} 再签字上线。" + ) + else: + headline = "当前收口、正式复核、主决策、发布门禁与 launchpad 摘要一致，可以进入最终人工签字或正式发布。" + + contract_keys = _normalize_ops_contract_keys( + [ + "ops_go_live_signoff_contract", + "ops_go_live_review_contract", + "ops_doctor_decision_contract", + "ops_stack_diagnosis_contract", + "ops_driver_contract", + "release_hub_contract", + "ops_observability_contract", + ] + ) + recommended_commands = { + **dict(go_live_summary.get("recommended_commands") or {}), + **dict(go_live_review.get("recommended_commands") or {}), + **dict(doctor_decision_payload.get("recommended_commands") or {}), + "release_launchpad": build_bash_command("drive_ops_center.sh", "release-launchpad", normalized_base_url), + "driver_feed": build_bash_command("drive_ops_center.sh", "driver-feed", normalized_base_url), + "codex_brief": build_bash_command("drive_ops_center.sh", "codex-brief", normalized_base_url), + "go_live_signoff": build_bash_command( + "drive_ops_center.sh", + "go-live-signoff", + normalized_report_dir or normalized_base_url, + ), + } + + return { + "generated_at": datetime.now().isoformat(timespec="seconds"), + "base_url": normalized_base_url, + "contract_key": "ops_go_live_signoff_contract", + "contract_version": str( + (contract_registry.get("contracts_by_key") or {}).get("ops_go_live_signoff_contract", {}).get("version") + or _OPS_CONTRACT_SCHEMA_VERSION + ).strip(), + "signoff_status": signoff_status, + "status_label": status_label, + "headline": headline, + "release_gate": { + "go_live_status": go_live_status, + "publish_status": publish_status, + "stack_status": stack_status, + "driver_launch_status": driver_launch_status, + "codex_launch_status": codex_launch_status, + "release_launchpad_status": release_launchpad_status, + "go_live_review_status": review_status, + "doctor_decision_status": doctor_status, + }, + "blocked_reasons": blocked_reasons, + "attention_reasons": attention_reasons, + "report_dir": str(go_live_review.get("report_dir") or doctor_decision_payload.get("report_dir") or "").strip(), + "manifest_path": str(go_live_review.get("manifest_path") or doctor_decision_payload.get("manifest_path") or "").strip(), + "decision": { + "operator_title": str(go_live_summary.get("operator_title") or "").strip(), + "next_step_action_code": str(go_live_summary.get("next_step_action_code") or "").strip(), + "next_step_reason": str(go_live_summary.get("next_step_reason") or "").strip(), + "preferred_surface": ( + "go-live-review" + if review_status in {"blocked", "attention", "missing"} + else str(doctor_decision.get("preferred_surface") or "").strip() + if doctor_status in {"blocked", "attention", "missing"} and str(doctor_decision.get("preferred_surface") or "").strip() + else "release-launchpad" + if release_launchpad_status in {"blocked", "attention"} + else "codex-brief" + if codex_launch_status in {"blocked", "attention"} + else "go-live-summary" + ), + "doctor_reason_code": str(doctor_decision.get("reason_code") or "").strip(), + "doctor_preferred_surface": str(doctor_decision.get("preferred_surface") or "").strip(), + "doctor_next_action_code": str(doctor_decision.get("next_action_code") or "").strip(), + "review_status": review_status, + "review_headline": str(go_live_review.get("headline") or "").strip(), + "review_report_dir": str(go_live_review.get("report_dir") or "").strip(), + "review_manifest_path": str(go_live_review.get("manifest_path") or "").strip(), + "stack_operator_title": str(operator_decision.get("title") or "").strip(), + "stack_next_action_code": str(next_step.get("action_code") or "").strip(), + "launchpad_recommended_target_node_code": str( + go_live_summary.get("launchpad_recommended_target_node_code") or "" + ).strip(), + "launchpad_recommended_recovery_label": str( + go_live_summary.get("launchpad_recommended_recovery_label") or "" + ).strip(), + "launchpad_recommended_recovery_summary": str( + go_live_summary.get("launchpad_recommended_recovery_summary") or "" + ).strip(), + }, + "launchpad_alignment": launchpad_alignment, + "recommended_commands": recommended_commands, + "contract_navigation": _build_ops_contract_navigation( + contract_keys, + primary_contract_key="ops_go_live_signoff_contract", + registry=contract_registry, + ), + "go_live_summary": go_live_summary, + "go_live_review": { + "status": review_status, + "status_label": str(go_live_review.get("status_label") or "").strip(), + "headline": str(go_live_review.get("headline") or "").strip(), + "report_dir": str(go_live_review.get("report_dir") or "").strip(), + "manifest_path": str(go_live_review.get("manifest_path") or "").strip(), + "critical_failures": list(go_live_review.get("critical_failures") or []), + "noncritical_failures": list(go_live_review.get("noncritical_failures") or []), + }, + "doctor_decision": { + "status": doctor_status, + "headline": str(doctor_decision_payload.get("headline") or "").strip(), + "detail": str(doctor_decision_payload.get("detail") or "").strip(), + "summary": dict(doctor_decision_payload.get("summary") or {}), + "decision": doctor_decision, + "report_dir": str(doctor_decision_payload.get("report_dir") or "").strip(), + "manifest_path": str(doctor_decision_payload.get("manifest_path") or "").strip(), + }, + "stack_diagnosis": { + "stack_status": stack_status, + "operator_decision": operator_decision, + "next_step": next_step, + }, + "driver_feed": { + "headline": str(driver_feed.get("headline") or "").strip(), + "summary": driver_summary, + }, + "codex_brief": { + "headline": str(codex_brief.get("headline") or "").strip(), + "summary": codex_summary, + "focus": dict(codex_brief.get("focus") or {}), + }, + "release_launchpad": { + "status": release_launchpad_status, + "status_label": str(launchpad_status.get("status_label") or "").strip(), + "recommended_action_code": str(launchpad_status.get("recommended_action_code") or "").strip(), + "recommended_execution_mode": str(launchpad_status.get("recommended_execution_mode") or "").strip(), + }, + } + + +def _ops_go_live_bundle_root() -> Path: + return runtime_root() / "ops-center-reports" / "go-live-bundles" + + +def _load_json_path(path: Path) -> dict: + if not path.is_file(): + return {} + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except Exception: + return {} + return payload if isinstance(payload, dict) else {} + + +def _find_latest_go_live_bundle_manifest() -> tuple[Path | None, Path | None]: + bundle_root = _ops_go_live_bundle_root() + if not bundle_root.exists(): + return None, bundle_root + manifest_paths = sorted(bundle_root.glob("*/manifest.json"), key=lambda item: item.stat().st_mtime, reverse=True) + if not manifest_paths: + return None, bundle_root + manifest_path = manifest_paths[0] + return manifest_path, manifest_path.parent + + +def _resolve_go_live_bundle_manifest(report_dir: str) -> tuple[Path | None, Path | None]: + normalized_report_dir = str(report_dir or "").strip() + if normalized_report_dir: + candidate_path = Path(normalized_report_dir) + if candidate_path.is_dir(): + nested_manifest = candidate_path / "manifest.json" + return (nested_manifest if nested_manifest.is_file() else None), candidate_path + if candidate_path.is_file(): + return candidate_path, candidate_path.parent + return None, candidate_path + return _find_latest_go_live_bundle_manifest() + + +def _load_bundle_json_artifact(artifact_by_key: dict[str, dict], key: str) -> dict: + artifact = artifact_by_key.get(str(key or "").strip()) or {} + artifact_path = Path(str(artifact.get("path") or "").strip()) + return _load_json_path(artifact_path) if artifact_path else {} + + +def _load_bundle_env_audit_summary(artifact_by_key: dict[str, dict]) -> dict: + env_audit_artifact = artifact_by_key.get("env_audit") or {} + env_audit_path = Path(str(env_audit_artifact.get("path") or "").strip()) + if not env_audit_path.is_file(): + return {} + raw_text = env_audit_path.read_text(encoding="utf-8", errors="replace") + marker = "[8/8] condensed env audit summary" + candidate_text = raw_text.split(marker, 1)[1].strip() if marker in raw_text else raw_text.strip() + if not candidate_text: + return {} + try: + loaded_env_audit = json.loads(candidate_text) + except Exception: + loaded_env_audit = {} + return loaded_env_audit if isinstance(loaded_env_audit, dict) else {} + + +def get_ops_go_live_bundle(*, base_url: str = "", report_dir: str = "") -> dict: + normalized_base_url = str(base_url or "").strip() or "http://127.0.0.1:8100" + contract_registry = get_ops_contract_registry() + manifest_path, resolved_report_dir = _resolve_go_live_bundle_manifest(report_dir) + + recommended_commands = { + "go_live_export": build_bash_command("drive_ops_center.sh", "go-live-export"), + "go_live_review": build_bash_command("drive_ops_center.sh", "go-live-review", str(report_dir or "").strip() or "/path/to/go-live-bundle"), + "go_live_signoff": build_bash_command("drive_ops_center.sh", "go-live-signoff", str(report_dir or "").strip() or "/path/to/go-live-bundle"), + "doctor_export": build_bash_command("drive_ops_center.sh", "doctor-export", "/tmp/domaincheck-go-live", normalized_base_url), + } + contract_keys = _normalize_ops_contract_keys( + [ + "ops_go_live_bundle_contract", + "ops_go_live_signoff_contract", + "ops_stack_diagnosis_contract", + "ops_driver_contract", + ] + ) + + if manifest_path is None or not manifest_path.is_file(): + bundle_root = resolved_report_dir or _ops_go_live_bundle_root() + headline = ( + "当前还没有可复核的 go-live bundle，建议先导出证据包，再进入 manifest 与最终签收复核。" + ) + return { + "generated_at": datetime.now().isoformat(timespec="seconds"), + "base_url": normalized_base_url, + "contract_key": "ops_go_live_bundle_contract", + "contract_version": str( + (contract_registry.get("contracts_by_key") or {}).get("ops_go_live_bundle_contract", {}).get("version") + or _OPS_CONTRACT_SCHEMA_VERSION + ).strip(), + "status": "missing", + "status_label": "待导出", + "headline": headline, + "bundle_available": False, + "report_dir": str(bundle_root), + "manifest_path": "", + "artifact_total": 0, + "failure_total": 0, + "critical_failures": [], + "noncritical_failures": [], + "env_audit": {}, + "summary": { + "ok": False, + "review_status_hint": "missing", + "review_headline": headline, + "launchpad_alignment": {"consistent": False}, + }, + "recommended_commands": recommended_commands, + "contract_navigation": _build_ops_contract_navigation( + contract_keys, + primary_contract_key="ops_go_live_bundle_contract", + registry=contract_registry, + ), + } + + manifest = _load_json_path(manifest_path) + artifacts = list(manifest.get("artifacts") or []) + failures = [str(item or "").strip() for item in list(manifest.get("failures") or []) if str(item or "").strip()] + artifact_by_key = { + str(item.get("key") or "").strip(): dict(item) + for item in artifacts + if isinstance(item, dict) and str(item.get("key") or "").strip() + } + summary = dict(manifest.get("summary") or {}) + env_audit = _load_bundle_env_audit_summary(artifact_by_key) + + review_status_hint = str(summary.get("review_status_hint") or "").strip() + status = review_status_hint or ("ready" if bool(summary.get("ok", False)) else "attention") + status_label = { + "ready": "证据可交付", + "attention": "证据待复核", + "blocked": "证据阻断", + "missing": "待导出", + }.get(status, status or "未知") + headline = ( + str(summary.get("review_headline") or "").strip() + or "当前已读取最新 go-live bundle，可继续做 manifest 与签收复核。" + ) + + critical_keys = {"go_live_summary", "stack_diagnosis", "driver_feed", "codex_brief", "release_launchpad"} + critical_failures = [item for item in failures if item in critical_keys] + noncritical_failures = [item for item in failures if item not in critical_keys] + launchpad_alignment = dict(summary.get("launchpad_alignment") or {}) + + return { + "generated_at": datetime.now().isoformat(timespec="seconds"), + "base_url": normalized_base_url, + "contract_key": "ops_go_live_bundle_contract", + "contract_version": str( + (contract_registry.get("contracts_by_key") or {}).get("ops_go_live_bundle_contract", {}).get("version") + or _OPS_CONTRACT_SCHEMA_VERSION + ).strip(), + "status": status, + "status_label": status_label, + "headline": headline, + "bundle_available": True, + "report_dir": str(manifest.get("report_dir") or resolved_report_dir or manifest_path.parent), + "manifest_path": str(manifest_path), + "artifact_total": int(summary.get("artifact_total", len(artifacts)) or len(artifacts)), + "failure_total": int(summary.get("failure_total", len(failures)) or len(failures)), + "critical_failures": critical_failures, + "noncritical_failures": noncritical_failures, + "env_audit": { + "status": str(env_audit.get("status") or "").strip(), + "headline": str(env_audit.get("headline") or "").strip(), + "missing_items": list(env_audit.get("missing_items") or []), + "runtime_may_need_restart": bool(((env_audit.get("runtime") or {}).get("runtime_may_need_restart", False))), + "recommended_actions": [ + str(item).strip() + for item in list(env_audit.get("recommended_actions") or []) + if str(item).strip() + ], + }, + "summary": { + "ok": bool(summary.get("ok", False)), + "review_status_hint": review_status_hint or status, + "review_headline": headline, + "launchpad_recommended_target_node_code": str( + summary.get("launchpad_recommended_target_node_code") or "" + ).strip(), + "launchpad_recommended_recovery_label": str( + summary.get("launchpad_recommended_recovery_label") or "" + ).strip(), + "launchpad_recommended_recovery_summary": str( + summary.get("launchpad_recommended_recovery_summary") or "" + ).strip(), + "launchpad_onboarding_bootstrap_pending_nodes": int( + summary.get("launchpad_onboarding_bootstrap_pending_nodes", 0) or 0 + ), + "launchpad_onboarding_acceptance_ready_nodes": int( + summary.get("launchpad_onboarding_acceptance_ready_nodes", 0) or 0 + ), + "launchpad_alignment": launchpad_alignment, + "recommended_reading_order": list(summary.get("recommended_reading_order") or []), + "contract_surface_gaps": list(summary.get("contract_surface_gaps") or []), + "discovered_contract_keys": list(summary.get("discovered_contract_keys") or []), + }, + "artifacts": artifacts, + "failures": failures, + "recommended_commands": { + **recommended_commands, + "go_live_review": build_bash_command("drive_ops_center.sh", "go-live-review", str(manifest_path)), + "go_live_signoff": build_bash_command("drive_ops_center.sh", "go-live-signoff", str(manifest_path)), + }, + "contract_navigation": _build_ops_contract_navigation( + contract_keys, + primary_contract_key="ops_go_live_bundle_contract", + registry=contract_registry, + ), + } + + +def get_ops_go_live_review(*, base_url: str = "", report_dir: str = "") -> dict: + normalized_base_url = str(base_url or "").strip() or "http://127.0.0.1:8100" + normalized_report_dir = str(report_dir or "").strip() + contract_registry = get_ops_contract_registry() + manifest_path, resolved_report_dir = _resolve_go_live_bundle_manifest(report_dir) + contract_keys = _normalize_ops_contract_keys( + [ + "ops_go_live_review_contract", + "ops_go_live_bundle_contract", + "ops_doctor_decision_contract", + "ops_go_live_signoff_contract", + "ops_stack_diagnosis_contract", + "ops_driver_contract", + "release_hub_contract", + ] + ) + recommended_commands = { + "go_live_export": build_bash_command("drive_ops_center.sh", "go-live-export"), + "go_live_review": build_bash_command("drive_ops_center.sh", "go-live-review", normalized_report_dir or "/path/to/go-live-bundle"), + "go_live_signoff": build_bash_command("drive_ops_center.sh", "go-live-signoff", normalized_report_dir or "/path/to/go-live-bundle"), + "doctor_decision": build_bash_command("drive_ops_center.sh", "doctor-decision", normalized_report_dir or normalized_base_url), + } + + if manifest_path is None or not manifest_path.is_file(): + headline = "当前还没有可复核的 go-live bundle，建议先导出证据包，再进入正式复核。" + return { + "generated_at": datetime.now().isoformat(timespec="seconds"), + "base_url": normalized_base_url, + "contract_key": "ops_go_live_review_contract", + "contract_version": str( + (contract_registry.get("contracts_by_key") or {}).get("ops_go_live_review_contract", {}).get("version") + or _OPS_CONTRACT_SCHEMA_VERSION + ).strip(), + "status": "missing", + "status_label": "待导出", + "headline": headline, + "report_dir": str(resolved_report_dir or _ops_go_live_bundle_root()), + "manifest_path": "", + "artifact_total": 0, + "failure_total": 0, + "critical_failures": [], + "noncritical_failures": [], + "env_audit": {}, + "go_live_summary": {}, + "stack_diagnosis": {}, + "driver_feed": {}, + "codex_brief": {}, + "launchpad_alignment": {"consistent": False, "available_sources": []}, + "recommended_reading_order": [], + "recommended_next_steps": [ + "先执行 go-live-export，导出完整上线证据包。", + "导出完成后再执行 go-live-review 或直接打开页面里的证据复核详情。", + ], + "recommended_commands": recommended_commands, + "contract_navigation": _build_ops_contract_navigation( + contract_keys, + primary_contract_key="ops_go_live_review_contract", + registry=contract_registry, + ), + } + + manifest = _load_json_path(manifest_path) + artifacts = list(manifest.get("artifacts") or []) + failures = [str(item or "").strip() for item in list(manifest.get("failures") or []) if str(item or "").strip()] + summary = dict(manifest.get("summary") or {}) + artifact_by_key = { + str(item.get("key") or "").strip(): dict(item) + for item in artifacts + if isinstance(item, dict) and str(item.get("key") or "").strip() + } + env_audit = _load_bundle_env_audit_summary(artifact_by_key) + go_live_summary_payload = _load_bundle_json_artifact(artifact_by_key, "go_live_summary") + stack_diagnosis_payload = _load_bundle_json_artifact(artifact_by_key, "stack_diagnosis") + driver_feed_payload = _load_bundle_json_artifact(artifact_by_key, "driver_feed") + codex_brief_payload = _load_bundle_json_artifact(artifact_by_key, "codex_brief") + + critical_keys = {"go_live_summary", "stack_diagnosis", "driver_feed", "codex_brief", "release_launchpad"} + critical_failures = sorted( + { + key + for key in critical_keys + if key in failures or not bool((artifact_by_key.get(key) or {}).get("ok", False)) + } + ) + noncritical_failures = [key for key in failures if key not in critical_keys] + + go_live_launchpad_target_node_code = str(go_live_summary_payload.get("launchpad_recommended_target_node_code") or "").strip() + go_live_launchpad_recovery_label = str(go_live_summary_payload.get("launchpad_recommended_recovery_label") or "").strip() + go_live_launchpad_recovery_summary = str(go_live_summary_payload.get("launchpad_recommended_recovery_summary") or "").strip() + go_live_launchpad_bootstrap_pending_nodes = int(go_live_summary_payload.get("launchpad_onboarding_bootstrap_pending_nodes", 0) or 0) + go_live_launchpad_acceptance_ready_nodes = int(go_live_summary_payload.get("launchpad_onboarding_acceptance_ready_nodes", 0) or 0) + stack_diagnosis = dict(stack_diagnosis_payload.get("diagnosis") or {}) + driver_feed_summary = dict(driver_feed_payload.get("summary") or {}) + codex_brief_summary = dict(codex_brief_payload.get("summary") or {}) + + launchpad_target_node_codes = { + "go_live_summary": go_live_launchpad_target_node_code, + "stack_diagnosis": str(stack_diagnosis.get("launchpad_recommended_target_node_code") or "").strip(), + "driver_feed": str(driver_feed_summary.get("launchpad_recommended_target_node_code") or "").strip(), + "codex_brief": str(codex_brief_summary.get("launchpad_recommended_target_node_code") or "").strip(), + } + launchpad_recovery_labels = { + "go_live_summary": go_live_launchpad_recovery_label, + "stack_diagnosis": str(stack_diagnosis.get("launchpad_recommended_recovery_label") or "").strip(), + "driver_feed": str(driver_feed_summary.get("launchpad_recommended_recovery_label") or "").strip(), + "codex_brief": str(codex_brief_summary.get("launchpad_recommended_recovery_label") or "").strip(), + } + launchpad_recovery_summaries = { + "go_live_summary": go_live_launchpad_recovery_summary, + "stack_diagnosis": str(stack_diagnosis.get("launchpad_recommended_recovery_summary") or "").strip(), + "driver_feed": str(driver_feed_summary.get("launchpad_recommended_recovery_summary") or "").strip(), + "codex_brief": str(codex_brief_summary.get("launchpad_recommended_recovery_summary") or "").strip(), + } + launchpad_bootstrap_pending_nodes = { + "go_live_summary": go_live_launchpad_bootstrap_pending_nodes, + "stack_diagnosis": int(stack_diagnosis.get("launchpad_onboarding_bootstrap_pending_nodes", 0) or 0), + "driver_feed": int(driver_feed_summary.get("launchpad_onboarding_bootstrap_pending_nodes", 0) or 0), + "codex_brief": int(codex_brief_summary.get("launchpad_onboarding_bootstrap_pending_nodes", 0) or 0), + } + launchpad_acceptance_ready_nodes = { + "go_live_summary": go_live_launchpad_acceptance_ready_nodes, + "stack_diagnosis": int(stack_diagnosis.get("launchpad_onboarding_acceptance_ready_nodes", 0) or 0), + "driver_feed": int(driver_feed_summary.get("launchpad_onboarding_acceptance_ready_nodes", 0) or 0), + "codex_brief": int(codex_brief_summary.get("launchpad_onboarding_acceptance_ready_nodes", 0) or 0), + } + + available_alignment_sources = { + source + for source, payload_value in { + "go_live_summary": go_live_summary_payload, + "stack_diagnosis": stack_diagnosis_payload, + "driver_feed": driver_feed_payload, + "codex_brief": codex_brief_payload, + }.items() + if isinstance(payload_value, dict) and payload_value + } + + def _nonempty_distinct(values: dict[str, str]) -> list[str]: + return sorted({str(value or "").strip() for key, value in values.items() if key in available_alignment_sources and str(value or "").strip()}) + + def _distinct_ints(values: dict[str, int]) -> list[int]: + return sorted({int(value) for key, value in values.items() if key in available_alignment_sources}) + + launchpad_alignment = { + "consistent": True, + "target_node_code_consistent": len(_nonempty_distinct(launchpad_target_node_codes)) <= 1, + "recovery_label_consistent": len(_nonempty_distinct(launchpad_recovery_labels)) <= 1, + "recovery_summary_consistent": len(_nonempty_distinct(launchpad_recovery_summaries)) <= 1, + "bootstrap_pending_consistent": len(_distinct_ints(launchpad_bootstrap_pending_nodes)) <= 1, + "acceptance_ready_consistent": len(_distinct_ints(launchpad_acceptance_ready_nodes)) <= 1, + "available_sources": sorted(available_alignment_sources), + "target_node_codes": launchpad_target_node_codes, + "recovery_labels": launchpad_recovery_labels, + "recovery_summaries": launchpad_recovery_summaries, + "bootstrap_pending_nodes": launchpad_bootstrap_pending_nodes, + "acceptance_ready_nodes": launchpad_acceptance_ready_nodes, + } + launchpad_alignment["consistent"] = all( + bool(launchpad_alignment.get(key)) + for key in ( + "target_node_code_consistent", + "recovery_label_consistent", + "recovery_summary_consistent", + "bootstrap_pending_consistent", + "acceptance_ready_consistent", + ) + ) + + env_audit_status = str(env_audit.get("status") or "").strip() + env_audit_blocking = env_audit_status == "blocked" + env_audit_attention = env_audit_status == "attention" + env_audit_missing_items = [ + str(item).strip() + for item in list(env_audit.get("missing_items") or []) + if str(item).strip() + ] + env_audit_recommended_actions = [ + str(item).strip() + for item in list(env_audit.get("recommended_actions") or []) + if str(item).strip() + ] + env_audit_runtime_may_need_restart = bool(((env_audit.get("runtime") or {}).get("runtime_may_need_restart", False))) + node_agent_env_missing = any(item == "missing_env:/etc/default/domaincheck-node-agent" for item in env_audit_missing_items) + + if critical_failures or env_audit_blocking: + status = "blocked" + if critical_failures and env_audit_blocking: + headline = "关键报告存在缺失，且环境审计已给出阻断结论，当前不适合上线。" + elif critical_failures: + headline = "上线证据包仍缺关键报告，暂不建议直接宣称收口完成。" + else: + headline = "环境审计已给出阻断结论，需先修复现场环境后再进入发布门禁。" + elif noncritical_failures or env_audit_attention or not bool(launchpad_alignment.get("consistent")): + status = "attention" + if not bool(launchpad_alignment.get("consistent")): + headline = "核心报告已齐，但 launchpad 摘要在不同 surface 之间存在漂移，建议先复核再发版。" + elif noncritical_failures and env_audit_attention: + headline = "核心报告已齐，但补充报告和环境审计都提示仍有缺口，建议先复核再发版。" + elif noncritical_failures: + headline = "核心报告已齐，但仍有补充报告超时或失败，建议先复核再发版。" + else: + headline = "核心报告已齐，但环境审计仍提示基础缺口，建议补齐后再发版。" + else: + status = "ready" + headline = "核心上线证据已经齐备，可进入最终人工复核或正式发布门禁。" + + recommended_next_steps = [ + "先看 00_env_audit.txt，确认当前机器环境是否存在阻断或注意项", + "先看 02_go_live_summary.json 和 03_stack_diagnosis.json", + "再看 04_driver_feed.json 和 05_codex_brief.json", + "若涉及发布门禁，再看 06_release_launchpad.json", + "若 manifest 里仍有失败项，再看对应 artifact 原文件", + ] + if node_agent_env_missing: + recommended_next_steps.extend( + [ + "当前缺口指向 Node Agent 未接管：先执行 agent-gap-export，拿到首个缺口节点的接管包", + "随后执行 node-bootstrap-plan，为缺口节点生成可直接落地的接管脚本", + "如果仓库代码已更新但 bootstrap-plan 输出仍偏旧，先重启控制面 API 再重新导出接管计划", + ] + ) + if env_audit_runtime_may_need_restart: + recommended_next_steps.extend( + [ + "env-audit 已识别运行中 API 可能仍是旧代码：先执行 drive_ops_center.sh runtime-refresh-recover，按固定链路完成 API 重启与复检", + "若仍需人工逐条确认，再执行 recommended_actions 里的 API 重启与复检命令", + ] + ) + if go_live_launchpad_target_node_code: + recommended_next_steps.extend( + [ + f"当前 launchpad 缺口已经收敛到节点 {go_live_launchpad_target_node_code}，建议优先执行：{go_live_launchpad_recovery_label or '节点恢复'}", + "先对照 02_go_live_summary.json、04_driver_feed.json、05_codex_brief.json 中的 launchpad 字段，确认三处摘要是否一致", + go_live_launchpad_recovery_summary or "如果 recovery_summary 仍为空，再回到 06_release_launchpad.json 查看 gap rows 详情", + ] + ) + if not bool(launchpad_alignment.get("consistent")): + recommended_next_steps.extend( + [ + "当前 bundle 内的 launchpad 摘要在 go_live_summary / stack_diagnosis / driver_feed / codex_brief 之间存在不一致，先不要直接交付。", + "优先复核 02_go_live_summary.json 与 03_stack_diagnosis.json，再检查 04_driver_feed.json 与 05_codex_brief.json 是否使用了同一版控制面数据。", + "若只有局部摘要偏旧，先执行 drive_ops_center.sh runtime-refresh-recover，并重新导出 go-live bundle。", + ] + ) + + return { + "generated_at": datetime.now().isoformat(timespec="seconds"), + "base_url": normalized_base_url, + "contract_key": "ops_go_live_review_contract", + "contract_version": str( + (contract_registry.get("contracts_by_key") or {}).get("ops_go_live_review_contract", {}).get("version") + or _OPS_CONTRACT_SCHEMA_VERSION + ).strip(), + "status": status, + "status_label": { + "ready": "可复核发布", + "attention": "待复核补口", + "blocked": "暂不可发布", + "missing": "待导出", + }.get(status, status or "未知"), + "headline": headline, + "report_dir": str(manifest.get("report_dir") or resolved_report_dir or manifest_path.parent), + "manifest_path": str(manifest_path), + "artifact_total": int(summary.get("artifact_total", len(artifacts)) or len(artifacts)), + "failure_total": int(summary.get("failure_total", len(failures)) or len(failures)), + "critical_failures": critical_failures, + "noncritical_failures": noncritical_failures, + "env_audit": { + "ok": bool((artifact_by_key.get("env_audit") or {}).get("ok", False)), + "status": env_audit_status, + "headline": str(env_audit.get("headline") or "").strip(), + "missing_items": env_audit_missing_items, + "runtime_may_need_restart": env_audit_runtime_may_need_restart, + "recommended_actions": env_audit_recommended_actions, + }, + "go_live_summary": { + "go_live_status": str(go_live_summary_payload.get("go_live_status") or "").strip(), + "publish_status": str(go_live_summary_payload.get("publish_status") or "").strip(), + "operator_title": str(go_live_summary_payload.get("operator_title") or "").strip(), + "next_step_action_code": str(go_live_summary_payload.get("next_step_action_code") or "").strip(), + "launchpad_recommended_target_node_code": go_live_launchpad_target_node_code, + "launchpad_recommended_recovery_label": go_live_launchpad_recovery_label, + "launchpad_recommended_recovery_summary": go_live_launchpad_recovery_summary, + "launchpad_onboarding_bootstrap_pending_nodes": go_live_launchpad_bootstrap_pending_nodes, + "launchpad_onboarding_acceptance_ready_nodes": go_live_launchpad_acceptance_ready_nodes, + }, + "stack_diagnosis": { + "stack_status": str(stack_diagnosis.get("stack_status") or "").strip(), + "operator_title": str((dict(stack_diagnosis.get("operator_decision") or {})).get("title") or "").strip(), + "next_step_action_code": str((dict(stack_diagnosis.get("next_step") or {})).get("action_code") or "").strip(), + "launchpad_recommended_target_node_code": str(stack_diagnosis.get("launchpad_recommended_target_node_code") or "").strip(), + "launchpad_recommended_recovery_label": str(stack_diagnosis.get("launchpad_recommended_recovery_label") or "").strip(), + "launchpad_recommended_recovery_summary": str(stack_diagnosis.get("launchpad_recommended_recovery_summary") or "").strip(), + "launchpad_onboarding_bootstrap_pending_nodes": int(stack_diagnosis.get("launchpad_onboarding_bootstrap_pending_nodes", 0) or 0), + "launchpad_onboarding_acceptance_ready_nodes": int(stack_diagnosis.get("launchpad_onboarding_acceptance_ready_nodes", 0) or 0), + }, + "driver_feed": { + "headline": str(driver_feed_payload.get("headline") or "").strip(), + "launch_status": str((dict(driver_feed_payload.get("automation_coverage") or {})).get("launch_status") or "").strip(), + "launch_ready": bool((dict(driver_feed_payload.get("automation_coverage") or {})).get("launch_ready", False)), + "launchpad_recommended_target_node_code": str(driver_feed_summary.get("launchpad_recommended_target_node_code") or "").strip(), + "launchpad_recommended_recovery_label": str(driver_feed_summary.get("launchpad_recommended_recovery_label") or "").strip(), + }, + "codex_brief": { + "headline": str(codex_brief_payload.get("headline") or "").strip(), + "focus_entry_key": str((dict(codex_brief_payload.get("focus") or {})).get("entry_key") or "").strip(), + "launch_status": str((dict(codex_brief_payload.get("automation_coverage") or {})).get("launch_status") or "").strip(), + "launch_ready": bool((dict(codex_brief_payload.get("automation_coverage") or {})).get("launch_ready", False)), + "launchpad_recommended_target_node_code": str(codex_brief_summary.get("launchpad_recommended_target_node_code") or "").strip(), + "launchpad_recommended_recovery_label": str(codex_brief_summary.get("launchpad_recommended_recovery_label") or "").strip(), + }, + "launchpad_alignment": launchpad_alignment, + "recommended_reading_order": list(summary.get("recommended_reading_order") or []), + "recommended_next_steps": _dedupe_command_list(recommended_next_steps), + "recommended_commands": { + **recommended_commands, + "go_live_review": build_bash_command("drive_ops_center.sh", "go-live-review", str(manifest_path)), + "go_live_signoff": build_bash_command("drive_ops_center.sh", "go-live-signoff", str(manifest_path)), + "doctor_decision": build_bash_command("drive_ops_center.sh", "doctor-decision", str(manifest_path)), + }, + "contract_navigation": _build_ops_contract_navigation( + contract_keys, + primary_contract_key="ops_go_live_review_contract", + registry=contract_registry, + ), + } + + +def get_ops_blueprint() -> dict: + return { + "contract_registry": get_ops_contract_registry(), + "architecture": { + "control_plane": { + "region": "overseas", + "responsibility": [ + "web backend", + "ops api", + "release registry", + "task scheduler", + "log aggregation entry", + ], + }, + "managed_nodes": { + "region": "mainland", + "responsibility": [ + "worker/controller runtime", + "node agent", + "systemd execution", + "diagnostics collection", + ], + }, + }, + "execution_model": { + "default_transport": "https-polling", + "future_transport": "websocket", + "fallback_transport": "ssh", + "job_result_contract": { + "required_fields": [ + "job_id", + "node_code", + "step", + "started_at", + "finished_at", + "exit_code", + "stdout", + "stderr", + "structured_result", + ] + }, + }, + "mvp_scope": { + "phase_1": [ + "ops overview api", + "action catalog api", + "worker participation / standby visibility", + "remote log sync switch", + ], + "phase_2": [ + "ops jobs table", + "node registration token", + "node heartbeat and pull-task api", + "service restart / health check actions", + ], + "phase_3": [ + "release package distribution", + "one-click bootstrap", + "rollback", + "streaming logs", + ], + }, + "operator_model": { + "with_codex": "海外 Codex 作为智能驾驶员，自动分析现网状态并选择合适动作。", + "without_codex": "后台按钮直接创建 ops job，由控制面和 agent 自动完成执行与回执。", + }, + } + + +def get_ops_runbook() -> dict: + runtime = get_runtime_status() + readiness = runtime.get("readiness") or {} + worker_runtime = runtime.get("worker") or {} + sync_agent_runtime = runtime.get("sync_agent") or {} + managed_nodes_payload = list_managed_nodes_with_agent_state(participation_payload=runtime.get("detect") or {}) + release_launchpad = get_release_launchpad() + control_sequences = _attach_ops_runbook_sequence_resolutions( + _build_ops_runbook_control_sequences( + managed_nodes_payload=managed_nodes_payload, + release_launchpad=release_launchpad, + ), + requested_by="api/runbook", + ) + + return { + "entrypoint": { + "preferred_host": str(settings.sync_target_api_base_url or "").strip() or f"http://127.0.0.1:{settings.api_port}", + "api_prefix": settings.api_prefix, + }, + "services": { + "api": { + "service_name": settings.api_service_name, + "status": "running", + }, + "worker": { + "service_name": settings.worker_service_name, + "status": _bool_label(bool(worker_runtime.get("running", False))), + }, + "sync_agent": { + "service_name": settings.sync_agent_service_name, + "status": _bool_label(bool(sync_agent_runtime.get("running", False))), + }, + }, + "fast_checks": [ + "GET /health", + "GET /api/v1/runtime/status", + "GET /api/v1/runtime/cluster", + "GET /api/v1/runtime/readiness", + "GET /api/v1/ops/stack-diagnosis", + "GET /api/v1/ops/overview", + "GET /api/v1/ops/contracts", + "GET /api/v1/ops/contracts/ops_stack_diagnosis_contract", + "GET /api/v1/ops/releases/launchpad", + "GET /api/v1/ops/activity-stream", + "GET /api/v1/ops/capabilities", + ], + "current_readiness": { + "status": str(readiness.get("status") or ""), + "summary": str(readiness.get("summary") or ""), + }, + "release_launchpad": release_launchpad, + "control_sequences": control_sequences, + } diff --git a/domain-api/app/services/ops_template_service.py b/domain-api/app/services/ops_template_service.py new file mode 100644 index 0000000..ca9ea7b --- /dev/null +++ b/domain-api/app/services/ops_template_service.py @@ -0,0 +1,920 @@ +from __future__ import annotations + +from copy import deepcopy + +from app.core.config import settings +from app.services.ops_execution_mode_service import decorate_execution_mode_fields, execution_mode_label + + +def _split_text_list(raw_value: object) -> list[str]: + if isinstance(raw_value, list): + return [str(item).strip() for item in raw_value if str(item).strip()] + text = str(raw_value or "").replace("\r", "\n") + if not text.strip(): + return [] + normalized = text.replace(",", "\n") + return [item.strip() for item in normalized.split("\n") if item.strip()] + + +def _coerce_bool(raw_value: object) -> bool: + if isinstance(raw_value, bool): + return raw_value + if isinstance(raw_value, (int, float)): + return bool(raw_value) + normalized = str(raw_value or "").strip().lower() + if normalized in {"1", "true", "yes", "y", "on", "enable", "enabled"}: + return True + if normalized in {"0", "false", "no", "n", "off", "disable", "disabled"}: + return False + return bool(normalized) + + +def _is_missing_value(field_type: str, value: object) -> bool: + if field_type == "text_list": + return not list(value or []) + if field_type == "number": + return value is None + if field_type == "boolean": + return value is None + return str(value or "").strip() == "" + + +def _service_options() -> list[dict]: + return [ + {"label": f"API ({settings.api_service_name})", "value": settings.api_service_name}, + {"label": f"Worker ({settings.worker_service_name})", "value": settings.worker_service_name}, + {"label": f"Sync Agent ({settings.sync_agent_service_name})", "value": settings.sync_agent_service_name}, + {"label": "Node Agent (domaincheck-node-agent)", "value": "domaincheck-node-agent"}, + ] + + +def _template_catalog() -> list[dict]: + service_options = _service_options() + return [ + { + "group_key": "runtime", + "group_title": "运行时控制", + "group_description": "适合日常按钮化控制 Worker / API / Sync Agent。", + "items": [ + { + "key": "runtime.start_worker", + "title": "启动 Worker", + "action": "runtime.start_worker", + "description": "启动目标节点的检测 Worker。", + "risk_level": "medium", + "default_execution_mode": "remote-agent", + "execution_modes": ["remote-agent", "local-runtime", "ssh"], + "target_roles": ["control", "worker"], + "default_auto_approve": False, + "fields": [], + }, + { + "key": "runtime.stop_worker", + "title": "停止 Worker", + "action": "runtime.stop_worker", + "description": "停止目标节点的检测 Worker。", + "risk_level": "high", + "default_execution_mode": "remote-agent", + "execution_modes": ["remote-agent", "local-runtime", "ssh"], + "target_roles": ["control", "worker"], + "default_auto_approve": False, + "fields": [], + }, + { + "key": "runtime.restart_api", + "title": "重启 API", + "action": "runtime.restart_api", + "description": "重启目标节点的 domaincheck-api 服务。", + "risk_level": "high", + "default_execution_mode": "remote-agent", + "execution_modes": ["remote-agent", "local-runtime", "ssh"], + "target_roles": ["control"], + "default_auto_approve": False, + "fields": [], + }, + { + "key": "runtime.start_sync_agent", + "title": "启动 Sync Agent", + "action": "runtime.start_sync_agent", + "description": "启动目标节点的同步代理服务。", + "risk_level": "medium", + "default_execution_mode": "remote-agent", + "execution_modes": ["remote-agent", "local-runtime", "ssh"], + "target_roles": ["control"], + "default_auto_approve": False, + "fields": [], + }, + { + "key": "runtime.stop_sync_agent", + "title": "停止 Sync Agent", + "action": "runtime.stop_sync_agent", + "description": "停止目标节点的同步代理服务。", + "risk_level": "high", + "default_execution_mode": "remote-agent", + "execution_modes": ["remote-agent", "local-runtime", "ssh"], + "target_roles": ["control"], + "default_auto_approve": False, + "fields": [], + }, + { + "key": "service.restart", + "title": "重启任意服务", + "action": "service.restart", + "description": "对指定 systemd 服务执行 restart。", + "risk_level": "high", + "default_execution_mode": "remote-agent", + "execution_modes": ["remote-agent", "local-runtime", "ssh"], + "target_roles": ["control", "worker"], + "default_auto_approve": False, + "fields": [ + { + "key": "service_name", + "label": "服务名", + "type": "select", + "required": True, + "default": settings.worker_service_name, + "options": service_options, + "help": "默认提供 API / Worker / Sync Agent / Node Agent。", + } + ], + }, + ], + }, + { + "group_key": "diagnostics", + "group_title": "巡检与取证", + "group_description": "适合做联调、排障、日志回收和节点健康诊断。", + "items": [ + { + "key": "health.snapshot", + "title": "采集健康快照", + "action": "health.snapshot", + "description": "采集目标节点关键 systemd 服务状态。", + "risk_level": "low", + "default_execution_mode": "remote-agent", + "execution_modes": ["remote-agent", "local-runtime", "ssh"], + "target_roles": ["control", "worker"], + "default_auto_approve": True, + "fields": [], + }, + { + "key": "logs.collect", + "title": "收集服务日志", + "action": "logs.collect", + "description": "抓取指定服务最近 journalctl 日志。", + "risk_level": "low", + "default_execution_mode": "remote-agent", + "execution_modes": ["remote-agent", "local-runtime", "ssh"], + "target_roles": ["control", "worker"], + "default_auto_approve": True, + "fields": [ + { + "key": "service_name", + "label": "服务名", + "type": "select", + "required": True, + "default": settings.worker_service_name, + "options": service_options, + "help": "默认抓 Worker；排 API 问题时可切到 domaincheck-api。", + }, + { + "key": "lines", + "label": "日志行数", + "type": "number", + "required": True, + "default": 120, + "min": 20, + "max": 500, + "help": "journalctl -n 的行数，适合短时排障。", + }, + ], + }, + { + "key": "diagnostics.collect", + "title": "收集诊断包", + "action": "diagnostics.collect", + "description": "打包 API / Worker / Sync Agent / Node Agent 的状态与近期日志。", + "risk_level": "low", + "default_execution_mode": "remote-agent", + "execution_modes": ["remote-agent", "local-runtime", "ssh"], + "target_roles": ["control", "worker"], + "default_auto_approve": True, + "fields": [ + { + "key": "lines", + "label": "每个服务日志行数", + "type": "number", + "required": True, + "default": 200, + "min": 20, + "max": 800, + "help": "越大越适合深度排障，但任务返回会更重。", + } + ], + }, + { + "key": "service.status", + "title": "采集服务状态", + "action": "service.status", + "description": "执行 systemctl status --no-pager -l。", + "risk_level": "low", + "default_execution_mode": "remote-agent", + "execution_modes": ["remote-agent", "local-runtime", "ssh"], + "target_roles": ["control", "worker"], + "default_auto_approve": True, + "fields": [ + { + "key": "service_name", + "label": "服务名", + "type": "select", + "required": True, + "default": settings.worker_service_name, + "options": service_options, + "help": "用于查看特定服务当前状态和最近日志片段。", + } + ], + }, + ], + }, + { + "group_key": "delivery_queue", + "group_title": "回执队列治理", + "group_description": "适合统一处理 Node Agent 的 pending / dead-letter 回执队列，而不是手工登机删文件。", + "items": [ + { + "key": "delivery.queue.flush", + "title": "立即冲刷回执队列", + "action": "delivery.queue.flush", + "description": "立即触发目标节点 Node Agent 冲刷 pending 回执队列。", + "risk_level": "low", + "default_execution_mode": "remote-agent", + "execution_modes": ["remote-agent"], + "target_roles": ["control", "worker"], + "default_auto_approve": True, + "fields": [ + { + "key": "limit", + "label": "本次冲刷上限", + "type": "number", + "required": True, + "default": 20, + "min": 1, + "max": 200, + "help": "限制本次最多冲刷的回执数量，避免一次返回过大。", + } + ], + }, + { + "key": "delivery.queue.replay", + "title": "重放死信回执", + "action": "delivery.queue.replay", + "description": "把 dead-letter 记录重新放回 pending，并可立即触发冲刷。", + "risk_level": "medium", + "default_execution_mode": "remote-agent", + "execution_modes": ["remote-agent"], + "target_roles": ["control", "worker"], + "default_auto_approve": False, + "fields": [ + { + "key": "record_id", + "label": "单条记录 ID", + "type": "text", + "required": False, + "default": "", + "help": "留空表示按下面的 selector 条件批量匹配。", + }, + { + "key": "request_kind", + "label": "记录类型", + "type": "text", + "required": False, + "default": "", + "help": "可选 job_complete / job_event。", + }, + { + "key": "detail_code", + "label": "错误码", + "type": "text", + "required": False, + "default": "", + "help": "按 last_detail_code 过滤死信记录。", + }, + { + "key": "limit", + "label": "重放上限", + "type": "number", + "required": True, + "default": 20, + "min": 1, + "max": 200, + "help": "批量重放时最多处理多少条死信。", + }, + { + "key": "flush_after_replay", + "label": "重放后立即冲刷", + "type": "boolean", + "required": True, + "default": True, + "help": "开启后会把重放回 pending 的记录立即补发一次。", + }, + { + "key": "reason", + "label": "重放原因", + "type": "text", + "required": False, + "default": "", + "wide": True, + "help": "可选，便于后续回溯为什么执行此次重放。", + }, + ], + }, + { + "key": "delivery.queue.discard", + "title": "丢弃死信回执", + "action": "delivery.queue.discard", + "description": "把死信移出活动队列，保存到 discarded 归档目录。", + "risk_level": "high", + "default_execution_mode": "remote-agent", + "execution_modes": ["remote-agent"], + "target_roles": ["control", "worker"], + "default_auto_approve": False, + "fields": [ + { + "key": "record_id", + "label": "单条记录 ID", + "type": "text", + "required": False, + "default": "", + "help": "留空表示按下面的 selector 条件批量匹配。", + }, + { + "key": "request_kind", + "label": "记录类型", + "type": "text", + "required": False, + "default": "", + "help": "可选 job_complete / job_event。", + }, + { + "key": "detail_code", + "label": "错误码", + "type": "text", + "required": False, + "default": "", + "help": "按 last_detail_code 过滤死信记录。", + }, + { + "key": "limit", + "label": "丢弃上限", + "type": "number", + "required": True, + "default": 20, + "min": 1, + "max": 200, + "help": "批量丢弃时最多处理多少条死信。", + }, + { + "key": "reason", + "label": "丢弃原因", + "type": "text", + "required": True, + "default": "", + "wide": True, + "help": "必须说明为何确认这些死信可以被放弃。", + }, + ], + }, + ], + }, + { + "group_key": "onboarding", + "group_title": "接管与纳管", + "group_description": "适合在控制面生成节点接入工单、统一沉淀首轮接管动作。", + "items": [ + { + "key": "node.bootstrap", + "title": "生成节点接入工单", + "action": "node.bootstrap", + "description": "在控制面签发 Node Agent Token，并生成 bootstrap env / 脚本 / 一键落地命令。", + "risk_level": "critical", + "default_execution_mode": "control-plane", + "execution_modes": ["control-plane"], + "target_roles": ["control", "worker"], + "default_auto_approve": False, + "fields": [ + { + "key": "control_plane_base_url", + "label": "控制面地址", + "type": "text", + "required": False, + "default": "", + "wide": True, + "help": "可留空。留空时由控制面自动回退到当前 API 地址。", + }, + { + "key": "root_dir", + "label": "目标安装目录", + "type": "text", + "required": True, + "default": "/opt/domaincheck", + "help": "bootstrap_node_agent.sh 安装根目录。", + }, + { + "key": "expires_in_hours", + "label": "Token 有效期（小时）", + "type": "number", + "required": True, + "default": 72, + "min": 1, + "max": 720, + "help": "控制新签发 token 的过期时间。", + }, + { + "key": "node_region", + "label": "节点地域提示", + "type": "text", + "required": False, + "default": "", + "help": "可留空；留空时优先从托管节点或集群快照自动推断。", + }, + { + "key": "node_role", + "label": "节点角色提示", + "type": "text", + "required": False, + "default": "", + "help": "可留空；留空时优先从托管节点或集群快照自动推断。", + }, + ], + }, + ], + }, + { + "group_key": "release", + "group_title": "发布与变更", + "group_description": "适合把单节点灰度、点状修复和手工发布收编到标准 deploy.release 任务。", + "items": [ + { + "key": "deploy.release.control", + "title": "发布控制面节点", + "action": "deploy.release", + "description": "面向 controller 节点的标准发布任务，默认会重启 API / Worker / Sync Agent 并执行 API 健康检查。", + "risk_level": "high", + "default_execution_mode": "remote-agent", + "execution_modes": ["remote-agent", "ssh"], + "target_roles": ["control"], + "default_auto_approve": False, + "fields": [ + { + "key": "release_version", + "label": "版本号", + "type": "text", + "required": True, + "default": "", + "help": "例如 2026.04.18-rc1，用于 releases/ 目录名。", + }, + { + "key": "artifact_url", + "label": "发布包地址", + "type": "textarea", + "required": True, + "default": "", + "wide": True, + "rows": 2, + "help": "Node Agent 会在目标节点直接下载 tar.gz 发布包。", + }, + { + "key": "checksum", + "label": "SHA256 校验", + "type": "text", + "required": False, + "default": "", + "wide": True, + "help": "可留空；填写后会强校验发布包完整性。", + }, + { + "key": "install_root", + "label": "安装根目录", + "type": "text", + "required": True, + "default": "/opt/domaincheck", + "help": "目标节点上的 releases/current 根目录。", + }, + { + "key": "restart_services", + "label": "重启服务", + "type": "text_list", + "required": False, + "default": [ + settings.api_service_name, + settings.worker_service_name, + settings.sync_agent_service_name, + ], + "wide": True, + "rows": 3, + "help": "每行一个服务。控制面默认重启 API / Worker / Sync Agent。", + }, + { + "key": "health_check_urls", + "label": "健康检查 URL", + "type": "text_list", + "required": False, + "default": ["http://127.0.0.1:8100/health"], + "wide": True, + "rows": 2, + "help": "每行一个 URL。控制面默认探活本机 API /health。", + }, + { + "key": "health_check_services", + "label": "健康检查服务", + "type": "text_list", + "required": False, + "default": [ + settings.api_service_name, + settings.worker_service_name, + settings.sync_agent_service_name, + ], + "wide": True, + "rows": 3, + "help": "每行一个 systemd 服务，发布后会检查其 active 状态。", + }, + { + "key": "health_check_timeout_seconds", + "label": "URL 超时", + "type": "number", + "required": True, + "default": 10, + "min": 2, + "max": 120, + "help": "每次 URL 探活超时秒数。", + }, + { + "key": "health_check_retries", + "label": "重试次数", + "type": "number", + "required": True, + "default": 2, + "min": 0, + "max": 10, + "help": "健康检查失败后的额外重试次数。", + }, + { + "key": "health_check_interval_seconds", + "label": "重试间隔", + "type": "number", + "required": True, + "default": 2, + "min": 0, + "max": 60, + "help": "每次重试前的等待秒数。", + }, + { + "key": "rollback_on_failure", + "label": "失败自动回滚", + "type": "boolean", + "required": True, + "default": True, + "help": "健康检查失败时自动切回旧 current 并重启服务。", + }, + { + "key": "switch_current", + "label": "切换 current 软链", + "type": "boolean", + "required": True, + "default": True, + "help": "关闭后只解压版本目录，不切 current。", + }, + ], + }, + { + "key": "deploy.release.worker", + "title": "发布 Worker 节点", + "action": "deploy.release", + "description": "面向独立 Worker 节点的标准发布任务，默认只重启 Worker 并校验 Worker 服务状态。", + "risk_level": "high", + "default_execution_mode": "remote-agent", + "execution_modes": ["remote-agent", "ssh"], + "target_roles": ["worker"], + "default_auto_approve": False, + "fields": [ + { + "key": "release_version", + "label": "版本号", + "type": "text", + "required": True, + "default": "", + "help": "例如 2026.04.18-rc1，用于 releases/ 目录名。", + }, + { + "key": "artifact_url", + "label": "发布包地址", + "type": "textarea", + "required": True, + "default": "", + "wide": True, + "rows": 2, + "help": "Node Agent 会在目标节点直接下载 tar.gz 发布包。", + }, + { + "key": "checksum", + "label": "SHA256 校验", + "type": "text", + "required": False, + "default": "", + "wide": True, + "help": "可留空；填写后会强校验发布包完整性。", + }, + { + "key": "install_root", + "label": "安装根目录", + "type": "text", + "required": True, + "default": "/opt/domaincheck", + "help": "目标节点上的 releases/current 根目录。", + }, + { + "key": "restart_services", + "label": "重启服务", + "type": "text_list", + "required": False, + "default": [settings.worker_service_name], + "wide": True, + "rows": 2, + "help": "每行一个服务。独立 Worker 默认只重启 Worker。", + }, + { + "key": "health_check_urls", + "label": "健康检查 URL", + "type": "text_list", + "required": False, + "default": [], + "wide": True, + "rows": 2, + "help": "Worker 节点通常可留空；为空时不做 URL 探活。", + }, + { + "key": "health_check_services", + "label": "健康检查服务", + "type": "text_list", + "required": False, + "default": [settings.worker_service_name], + "wide": True, + "rows": 2, + "help": "每行一个 systemd 服务，默认只校验 Worker active 状态。", + }, + { + "key": "health_check_timeout_seconds", + "label": "URL 超时", + "type": "number", + "required": True, + "default": 10, + "min": 2, + "max": 120, + "help": "每次 URL 探活超时秒数。", + }, + { + "key": "health_check_retries", + "label": "重试次数", + "type": "number", + "required": True, + "default": 2, + "min": 0, + "max": 10, + "help": "健康检查失败后的额外重试次数。", + }, + { + "key": "health_check_interval_seconds", + "label": "重试间隔", + "type": "number", + "required": True, + "default": 2, + "min": 0, + "max": 60, + "help": "每次重试前的等待秒数。", + }, + { + "key": "rollback_on_failure", + "label": "失败自动回滚", + "type": "boolean", + "required": True, + "default": True, + "help": "健康检查失败时自动切回旧 current 并重启服务。", + }, + { + "key": "switch_current", + "label": "切换 current 软链", + "type": "boolean", + "required": True, + "default": True, + "help": "关闭后只解压版本目录，不切 current。", + }, + ], + }, + { + "key": "deploy.release.custom", + "title": "发布自定义节点", + "action": "deploy.release", + "description": "完全自定义 deploy.release 参数，适合灰度验证、特殊节点或后续扩展场景。", + "risk_level": "high", + "default_execution_mode": "remote-agent", + "execution_modes": ["remote-agent", "ssh"], + "target_roles": ["control", "worker"], + "default_auto_approve": False, + "fields": [ + { + "key": "release_version", + "label": "版本号", + "type": "text", + "required": True, + "default": "", + "help": "例如 2026.04.18-rc1，用于 releases/ 目录名。", + }, + { + "key": "artifact_url", + "label": "发布包地址", + "type": "textarea", + "required": True, + "default": "", + "wide": True, + "rows": 2, + "help": "Node Agent 会在目标节点直接下载 tar.gz 发布包。", + }, + { + "key": "checksum", + "label": "SHA256 校验", + "type": "text", + "required": False, + "default": "", + "wide": True, + "help": "可留空；填写后会强校验发布包完整性。", + }, + { + "key": "install_root", + "label": "安装根目录", + "type": "text", + "required": True, + "default": "/opt/domaincheck", + "help": "目标节点上的 releases/current 根目录。", + }, + { + "key": "restart_services", + "label": "重启服务", + "type": "text_list", + "required": False, + "default": [], + "wide": True, + "rows": 3, + "help": "每行一个服务。留空表示只落盘版本，不主动重启任何服务。", + }, + { + "key": "health_check_urls", + "label": "健康检查 URL", + "type": "text_list", + "required": False, + "default": [], + "wide": True, + "rows": 2, + "help": "每行一个 URL。留空表示不做 URL 探活。", + }, + { + "key": "health_check_services", + "label": "健康检查服务", + "type": "text_list", + "required": False, + "default": [], + "wide": True, + "rows": 3, + "help": "每行一个 systemd 服务。显式留空时不会自动回落到默认 API 检查。", + }, + { + "key": "health_check_timeout_seconds", + "label": "URL 超时", + "type": "number", + "required": True, + "default": 10, + "min": 2, + "max": 120, + "help": "每次 URL 探活超时秒数。", + }, + { + "key": "health_check_retries", + "label": "重试次数", + "type": "number", + "required": True, + "default": 2, + "min": 0, + "max": 10, + "help": "健康检查失败后的额外重试次数。", + }, + { + "key": "health_check_interval_seconds", + "label": "重试间隔", + "type": "number", + "required": True, + "default": 2, + "min": 0, + "max": 60, + "help": "每次重试前的等待秒数。", + }, + { + "key": "rollback_on_failure", + "label": "失败自动回滚", + "type": "boolean", + "required": True, + "default": True, + "help": "健康检查失败时自动切回旧 current 并重启服务。", + }, + { + "key": "switch_current", + "label": "切换 current 软链", + "type": "boolean", + "required": True, + "default": True, + "help": "关闭后只解压版本目录，不切 current。", + }, + ], + }, + ], + }, + ] + + +def get_ops_action_templates() -> dict: + groups = deepcopy(_template_catalog()) + items: list[dict] = [] + for group in groups: + group_key = str(group.get("group_key") or "").strip() + group_title = str(group.get("group_title") or "").strip() + group_description = str(group.get("group_description") or "").strip() + normalized_group_items: list[dict] = [] + for item in list(group.get("items") or []): + normalized_item = decorate_execution_mode_fields(dict(item)) + normalized_item["group_key"] = group_key + normalized_item["group_title"] = group_title + normalized_item["group_description"] = group_description + normalized_group_items.append(normalized_item) + items.append(normalized_item) + group["items"] = normalized_group_items + return { + "groups": groups, + "items": items, + "defaults": { + "requested_by": "web-ui", + "execution_mode": "remote-agent", + "execution_mode_label": execution_mode_label("remote-agent"), + }, + } + + +def get_ops_action_template(template_key: str) -> dict: + normalized_key = str(template_key or "").strip() + if not normalized_key: + return {} + for group in _template_catalog(): + group_key = str(group.get("group_key") or "").strip() + group_title = str(group.get("group_title") or "").strip() + group_description = str(group.get("group_description") or "").strip() + for item in list(group.get("items") or []): + if str(item.get("key") or "").strip() == normalized_key: + normalized_item = decorate_execution_mode_fields(deepcopy(item)) + normalized_item["group_key"] = group_key + normalized_item["group_title"] = group_title + normalized_item["group_description"] = group_description + return normalized_item + return {} + + +def build_ops_template_payload(template_key: str, payload: dict | None = None) -> tuple[bool, str, dict]: + template = get_ops_action_template(template_key) + if not template: + return False, "动作模板不存在", {} + + source_payload = dict(payload or {}) + normalized_payload: dict = {} + for field in list(template.get("fields") or []): + field_key = str(field.get("key") or "").strip() + if not field_key: + continue + field_type = str(field.get("type") or "text").strip() + raw_value = source_payload.get(field_key, field.get("default")) + if field_type == "number": + try: + value = int(raw_value or 0) + except Exception: + return False, f"{field_key} 必须是数字", {} + min_value = field.get("min") + max_value = field.get("max") + if min_value is not None: + value = max(int(min_value), value) + if max_value is not None: + value = min(int(max_value), value) + elif field_type == "boolean": + value = _coerce_bool(raw_value) + elif field_type == "text_list": + value = _split_text_list(raw_value) + else: + value = str(raw_value or "").strip() + if bool(field.get("required", False)) and _is_missing_value(field_type, value): + return False, f"{field_key} 不能为空", {} + normalized_payload[field_key] = value + return True, "ok", normalized_payload diff --git a/domain-api/app/services/runtime_status_service.py b/domain-api/app/services/runtime_status_service.py index 03dac4e..d12231b 100644 --- a/domain-api/app/services/runtime_status_service.py +++ b/domain-api/app/services/runtime_status_service.py @@ -7,6 +7,7 @@ from app.core.config import settings from app.core.db import get_db from app.core.files import read_json from app.core.redis_client import get_redis +from app.services.build_info_service import get_runtime_build_info from app.services.cluster_runtime_service import get_cluster_snapshot from app.services.detect_service import get_detect_status from app.services.detect_job_service import get_detect_capacity_plan, get_detect_queue_health @@ -187,6 +188,89 @@ def _build_multi_region_readiness( } +def _detect_participation_snapshot(*, row: dict) -> dict: + items_running = int(row.get("items_running", 0) or 0) + items_claimed = int(row.get("items_claimed", 0) or 0) + processed_recent = int(row.get("processed_recent", 0) or 0) + current_load = int(row.get("current_load", 0) or 0) + status = str(row.get("status") or "").strip().lower() + + if items_running > 0: + return { + "participation_state": "running", + "participation_label": "执行中", + "participation_reason": f"当前正在执行 {items_running} 项检测任务。", + "is_current_participant": True, + "is_dispatch_active": True, + } + if items_claimed > 0: + return { + "participation_state": "claimed", + "participation_label": "已领待跑", + "participation_reason": f"已领取 {items_claimed} 项任务，等待线程继续执行。", + "is_current_participant": True, + "is_dispatch_active": True, + } + if processed_recent > 0: + return { + "participation_state": "recent_throughput", + "participation_label": "近窗有吞吐", + "participation_reason": f"近 15 分钟内已处理 {processed_recent} 项任务。", + "is_current_participant": True, + "is_dispatch_active": False, + } + if current_load > 0 or status == "busy": + load_value = max(current_load, 1) + return { + "participation_state": "load_syncing", + "participation_label": "负载待确认", + "participation_reason": f"节点当前负载为 {load_value}，但还未观察到已领、执行中或近窗吞吐数据，先归入在线未参与观察。", + "is_current_participant": False, + "is_dispatch_active": False, + } + return { + "participation_state": "standby", + "participation_label": "在线待命", + "participation_reason": "当前未领任务、未执行任务，也没有近窗吞吐。", + "is_current_participant": False, + "is_dispatch_active": False, + } + + +def _build_detect_node_row(*, node_code: str, cluster_node: dict, job_node: dict, queue_node: dict) -> dict: + metadata = cluster_node.get("metadata") or {} + role = str(cluster_node.get("role") or job_node.get("role") or "worker") + region = str(cluster_node.get("region") or settings.node_region) + is_effective_worker = bool(cluster_node.get("is_effective_worker", False) or role == "worker") + items_total = int(job_node.get("items_total", metadata.get("job_items_total", 0)) or 0) + items_claimed = int(job_node.get("items_claimed", metadata.get("job_items_claimed", 0)) or 0) + items_running = int(job_node.get("items_running", metadata.get("job_items_running", 0)) or 0) + items_completed = int(job_node.get("items_completed", metadata.get("job_items_completed", 0)) or 0) + items_failed = int(job_node.get("items_failed", metadata.get("job_items_failed", 0)) or 0) + row = { + "node_code": node_code, + "role": role, + "region": region, + "status": str(cluster_node.get("status") or "unknown"), + "is_effective_worker": is_effective_worker, + "detect_participating": False, + "current_load": int(cluster_node.get("current_load", 0) or 0), + "items_total": items_total, + "items_pending": int(job_node.get("items_pending", max(items_total - items_claimed - items_completed - items_failed, 0)) or 0), + "items_claimed": items_claimed, + "items_running": items_running, + "items_completed": items_completed, + "items_failed": items_failed, + "processed_recent": int(queue_node.get("processed_recent", 0) or 0), + "processed_per_minute": float(queue_node.get("processed_per_minute", 0) or 0), + "last_heartbeat_at": str(cluster_node.get("last_heartbeat_at") or ""), + "metrics_source": "active_job" if bool(job_node) else ("cluster_metadata" if metadata else "derived"), + } + row.update(_detect_participation_snapshot(row=row)) + row["detect_participating"] = bool(row.get("is_current_participant", False)) + return row + + def _build_participating_detect_nodes(*, cluster_snapshot: dict, detect_snapshot: dict, worker_runtime: dict) -> list[dict]: cluster_nodes = list(cluster_snapshot.get("nodes") or []) active_job = detect_snapshot.get("active_job") or {} @@ -204,56 +288,25 @@ def _build_participating_detect_nodes(*, cluster_snapshot: dict, detect_snapshot } merged: list[dict] = [] - seen: set[str] = set() - for item in list(active_job.get("node_stats") or []): - node_code = str(item.get("node_code") or "").strip() - if not node_code or node_code == "unassigned": - continue - seen.add(node_code) + job_map = { + str(item.get("node_code") or "").strip(): item + for item in list(active_job.get("node_stats") or []) + if str(item.get("node_code") or "").strip() and str(item.get("node_code") or "").strip() != "unassigned" + } + candidate_codes = sorted(set(job_map) | set(queue_map) | set(cluster_map)) + for node_code in candidate_codes: cluster_node = cluster_map.get(node_code, {}) - queue_node = queue_map.get(node_code, {}) - merged.append( - { - "node_code": node_code, - "role": str(cluster_node.get("role") or "worker"), - "region": str(cluster_node.get("region") or settings.node_region), - "status": str(cluster_node.get("status") or "unknown"), - "is_effective_worker": bool(cluster_node.get("is_effective_worker", False) or str(cluster_node.get("role") or "") == "worker"), - "detect_participating": True, - "current_load": int(cluster_node.get("current_load", 0) or 0), - "items_total": int(item.get("items_total", 0) or 0), - "items_pending": int(item.get("items_pending", 0) or 0), - "items_claimed": int(item.get("items_claimed", 0) or 0), - "items_running": int(item.get("items_running", 0) or 0), - "items_completed": int(item.get("items_completed", 0) or 0), - "items_failed": int(item.get("items_failed", 0) or 0), - "processed_recent": int(queue_node.get("processed_recent", 0) or 0), - "processed_per_minute": float(queue_node.get("processed_per_minute", 0) or 0), - } + if cluster_node and not bool(cluster_node.get("is_effective_worker", False)): + continue + row = _build_detect_node_row( + node_code=node_code, + cluster_node=cluster_node, + job_node=job_map.get(node_code, {}), + queue_node=queue_map.get(node_code, {}), ) - - if worker_runtime.get("running", False) and settings.node_code not in seen: - cluster_node = cluster_map.get(settings.node_code, {}) - if cluster_node: - merged.append( - { - "node_code": settings.node_code, - "role": str(cluster_node.get("role") or settings.node_role), - "region": str(cluster_node.get("region") or settings.node_region), - "status": str(cluster_node.get("status") or "online"), - "is_effective_worker": True, - "detect_participating": True, - "current_load": int(cluster_node.get("current_load", 0) or 0), - "items_total": 0, - "items_pending": 0, - "items_claimed": 0, - "items_running": 0, - "items_completed": 0, - "items_failed": 0, - "processed_recent": 0, - "processed_per_minute": 0, - } - ) + if not bool(row.get("is_current_participant", False)): + continue + merged.append(row) return sorted( merged, @@ -261,11 +314,143 @@ def _build_participating_detect_nodes(*, cluster_snapshot: dict, detect_snapshot -int(item.get("items_running", 0) or 0), -int(item.get("items_claimed", 0) or 0), -int(item.get("processed_recent", 0) or 0), + -int(item.get("current_load", 0) or 0), str(item.get("node_code") or ""), ), ) +def _build_standby_detect_nodes(*, cluster_snapshot: dict, participating_nodes: list[dict]) -> list[dict]: + cluster_nodes = list(cluster_snapshot.get("nodes") or []) + participating_codes = { + str(item.get("node_code") or "").strip() + for item in list(participating_nodes or []) + if str(item.get("node_code") or "").strip() + } + standby_rows: list[dict] = [] + for node in cluster_nodes: + node_code = str(node.get("node_code") or "").strip() + node_status = str(node.get("status") or "").strip() + node_current_load = int(node.get("current_load", 0) or 0) + metadata = node.get("metadata") or {} + if not node_code: + continue + if not bool(node.get("is_effective_worker", False)): + continue + if node_status not in {"online", "busy"}: + continue + if node_code in participating_codes: + continue + standby_state = "load_syncing" if node_current_load > 0 or node_status == "busy" else "standby" + standby_label = "负载待确认" if standby_state == "load_syncing" else "在线待命" + standby_reason = ( + str(metadata.get("detail") or "").strip() + or str(metadata.get("phase_detail") or "").strip() + or ( + f"当前阶段：{str(metadata.get('phase') or metadata.get('phase_label') or '').strip()}" + if str(metadata.get("phase") or metadata.get("phase_label") or "").strip() + else "" + ) + or ( + f"节点当前负载为 {node_current_load}，但还未观察到已领、执行中或近窗吞吐数据。" + if standby_state == "load_syncing" + else "节点在线，当前未领任务、未执行任务，也没有近窗吞吐。" + ) + ) + standby_rows.append( + { + "node_code": node_code, + "role": str(node.get("role") or "worker"), + "region": str(node.get("region") or settings.node_region), + "status": node_status, + "is_effective_worker": True, + "detect_participating": False, + "participation_state": standby_state, + "participation_label": standby_label, + "participation_reason": standby_reason, + "current_load": node_current_load, + "last_heartbeat_at": str(node.get("last_heartbeat_at") or ""), + "phase": str(metadata.get("phase") or metadata.get("phase_label") or "").strip(), + "detail": str(metadata.get("detail") or metadata.get("phase_detail") or "").strip(), + "worker_online": bool(metadata.get("worker_online", False)), + "standby_reason": standby_reason, + } + ) + return sorted( + standby_rows, + key=lambda item: ( + str(item.get("status") or ""), + str(item.get("role") or ""), + str(item.get("node_code") or ""), + ), + ) + + +def _build_detect_participation_summary( + *, + participating_nodes: list[dict], + standby_nodes: list[dict], + cluster_snapshot: dict, +) -> dict: + cluster_nodes = list(cluster_snapshot.get("nodes") or []) + effective_online_nodes = [ + node + for node in cluster_nodes + if bool(node.get("is_effective_worker", False)) and str(node.get("status") or "").strip() in {"online", "busy"} + ] + dispatch_active_nodes = [ + row for row in participating_nodes + if bool(row.get("is_dispatch_active", False)) + ] + recent_only_nodes = [ + row for row in participating_nodes + if str(row.get("participation_state") or "").strip() == "recent_throughput" + ] + load_syncing_nodes = [ + row for row in standby_nodes + if str(row.get("participation_state") or "").strip() == "load_syncing" + ] + pure_standby_nodes = [ + row for row in standby_nodes + if str(row.get("participation_state") or "").strip() == "standby" + ] + dedicated_worker_nodes = [ + node for node in effective_online_nodes + if str(node.get("role") or "").strip() == "worker" + ] + controller_worker_nodes = [ + node for node in effective_online_nodes + if str(node.get("role") or "").strip() == "control" + ] + + summary_parts = [ + f"有效执行节点 {len(effective_online_nodes)} 台", + f"正在执行/领任务 {len(dispatch_active_nodes)} 台", + f"近窗刚有吞吐 {len(recent_only_nodes)} 台", + f"在线但未参与 {len(standby_nodes)} 台", + ] + if load_syncing_nodes: + summary_parts.append(f"其中负载待确认 {len(load_syncing_nodes)} 台") + + return { + "effective_online_nodes": len(effective_online_nodes), + "participating_nodes": len(participating_nodes), + "dispatch_active_nodes": len(dispatch_active_nodes), + "recent_only_nodes": len(recent_only_nodes), + "non_participating_nodes": len(standby_nodes), + "standby_nodes": len(pure_standby_nodes), + "load_syncing_nodes": len(load_syncing_nodes), + "dedicated_worker_nodes": len(dedicated_worker_nodes), + "controller_worker_nodes": len(controller_worker_nodes), + "dispatch_active_node_codes": [str(item.get("node_code") or "") for item in dispatch_active_nodes], + "recent_only_node_codes": [str(item.get("node_code") or "") for item in recent_only_nodes], + "non_participating_node_codes": [str(item.get("node_code") or "") for item in standby_nodes], + "standby_node_codes": [str(item.get("node_code") or "") for item in pure_standby_nodes], + "load_syncing_node_codes": [str(item.get("node_code") or "") for item in load_syncing_nodes], + "summary": "；".join(summary_parts), + } + + def get_runtime_status() -> dict: runtime_settings = get_runtime_settings() worker_runtime = detect_worker_runtime() @@ -314,6 +499,17 @@ def get_runtime_status() -> dict: "worker_mode": worker_runtime.get("mode", runtime_settings.get("worker_mode", "windows-local")), "queue_health": queue_health, "capacity_plan": capacity_plan, + "log_sync": { + "enabled": bool(runtime_settings.get("worker_log_sync_enabled", False)), + "mode": str(runtime_settings.get("worker_log_sync_mode", "key") or "key"), + "line_count": int(detect_snapshot.get("remote_log_line_count", 0) or 0), + "source_node_count": int(detect_snapshot.get("remote_log_node_count", 0) or 0), + "source_nodes": list(detect_snapshot.get("remote_log_nodes") or []), + "source_node_summaries": list(detect_snapshot.get("remote_log_node_summaries") or []), + "last_at": str(detect_snapshot.get("remote_log_last_at") or ""), + "last_line": str(detect_snapshot.get("remote_log_last_line") or ""), + "preview_lines": list(detect_snapshot.get("remote_log_lines") or [])[-20:], + }, } if not worker_expected_on_this_node: detect_payload.update( @@ -349,6 +545,16 @@ def get_runtime_status() -> dict: detect_snapshot=detect_payload, worker_runtime=worker_runtime, ) + detect_payload["standby_nodes"] = _build_standby_detect_nodes( + cluster_snapshot=cluster_snapshot, + participating_nodes=detect_payload["participating_nodes"], + ) + detect_payload["non_participating_nodes"] = list(detect_payload["standby_nodes"] or []) + detect_payload["participation_summary"] = _build_detect_participation_summary( + participating_nodes=detect_payload["participating_nodes"], + standby_nodes=detect_payload["non_participating_nodes"], + cluster_snapshot=cluster_snapshot, + ) append_runtime_projection_if_changed(detect=detect_payload, cluster=cluster_snapshot) sync_summary = get_sync_summary(record_limit=5) readiness = _build_multi_region_readiness( @@ -357,6 +563,7 @@ def get_runtime_status() -> dict: worker_runtime=worker_runtime, sync_agent_runtime=sync_agent_runtime, ) + build_info = get_runtime_build_info() return { "api": { @@ -371,6 +578,7 @@ def get_runtime_status() -> dict: "health_url": f"http://127.0.0.1:{settings.api_port}/health", "stdout_log": _runtime_log_path("domain-api.stdout.log"), "stderr_log": _runtime_log_path("domain-api.stderr.log"), + "build": build_info, }, "node": { "code": settings.node_code, diff --git a/domain-api/app/services/sync_record_service.py b/domain-api/app/services/sync_record_service.py index 8792dc4..aa75c60 100644 --- a/domain-api/app/services/sync_record_service.py +++ b/domain-api/app/services/sync_record_service.py @@ -450,6 +450,17 @@ def append_runtime_projection_if_changed( continue local_participating = bool(node.get("detect_participating", False) or node.get("current_load", 0)) break + local_job_bucket = {} + for item in list(active_job.get("node_stats") or []): + if str(item.get("node_code") or "").strip() != settings.node_code: + continue + local_job_bucket = item + break + if not local_participating: + local_participating = bool( + int(local_job_bucket.get("items_running", 0) or 0) > 0 + or int(local_job_bucket.get("items_claimed", 0) or 0) > 0 + ) projection = { "node": { "node_code": settings.node_code, diff --git a/domain-api/deploy/multi-region/README.md b/domain-api/deploy/multi-region/README.md index 4380061..61cce35 100644 --- a/domain-api/deploy/multi-region/README.md +++ b/domain-api/deploy/multi-region/README.md @@ -11,6 +11,14 @@ - 检测执行放大陆 - 后续新增大陆 Worker 时，不再改整体部署方式 +如果当前已经不再只是“部署”，而是进入“海外单脑控制面统一驾驶 / 准备上线”阶段，建议先补看： + +- `docs/25_domainCheck_海外单脑控制面上线收口总表.md` +- `docs/23_domainCheck_终局运维架构设计_海外单脑控制面.md` +- `docs/24_domainCheck_NodeAgent协议与ReleaseHub设计.md` +- `docs/schemas/ops_driver_contract.md` +- `docs/schemas/ops_stack_diagnosis_contract.md` + ## 一、目录约定 统一使用： @@ -21,20 +29,612 @@ 仓库内部署入口： -- `deploy/multi-region/bootstrap_overseas.sh` -- `deploy/multi-region/bootstrap_mainland.sh` -- `deploy/multi-region/check_cluster.sh` -- `deploy/multi-region/check_mainland_controller.sh` -- `deploy/multi-region/check_mainland_worker.sh` -- `deploy/multi-region/check_worker_participation.sh` -- `deploy/multi-region/check_temp_overseas_control.sh` -- `deploy/multi-region/check_temp_link.sh` -- `deploy/multi-region/check_temp_topology.sh` -- `deploy/multi-region/simulate_cluster_node.py` -- `deploy/multi-region/simulate_multi_region.sh` -- `deploy/multi-region/prune_cluster_nodes.py` -- `deploy/multi-region/prune_cluster_nodes.sh` -- `deploy/multi-region/templates/*.env.example` +- `domain-api/deploy/multi-region/bootstrap_overseas.sh` +- `domain-api/deploy/multi-region/bootstrap_mainland.sh` +- `domain-api/deploy/multi-region/bootstrap_node_agent.sh` +- `domain-api/deploy/multi-region/build_node_agent_bootstrap_plan.sh` +- `domain-api/deploy/multi-region/init_ops_center_config.sh` +- `domain-api/deploy/multi-region/drive_ops_center.sh` +- `domain-api/deploy/multi-region/drive_release_hub.sh` +- `domain-api/deploy/multi-region/drive_ops_action.sh` +- `domain-api/deploy/multi-region/templates/domaincheck-node-agent.env.example` +- `domain-api/deploy/multi-region/templates/domaincheck-ops-center.env.example` +- `domain-api/deploy/multi-region/check_cluster.sh` +- `domain-api/deploy/multi-region/check_node_agent.sh` +- `domain-api/deploy/multi-region/check_ops_center_stack.sh` +- `domain-api/deploy/multi-region/check_go_live.sh` +- `domain-api/deploy/multi-region/check_release_hub.sh` +- `domain-api/deploy/multi-region/drive_ops_center.sh` +- `domain-api/deploy/multi-region/drive_release_hub.sh` +- `domain-api/deploy/multi-region/drive_ops_action.sh` +- `domain-api/deploy/multi-region/check_mainland_controller.sh` +- `domain-api/deploy/multi-region/check_mainland_worker.sh` +- `domain-api/deploy/multi-region/check_worker_participation.sh` +- `domain-api/deploy/multi-region/check_node_scene_log.sh` +- `domain-api/deploy/multi-region/check_ops_link_snapshot.sh` +- `domain-api/deploy/multi-region/check_temp_overseas_control.sh` +- `domain-api/deploy/multi-region/check_temp_link.sh` +- `domain-api/deploy/multi-region/check_temp_topology.sh` +- `domain-api/deploy/multi-region/simulate_cluster_node.py` +- `domain-api/deploy/multi-region/simulate_multi_region.sh` +- `domain-api/deploy/multi-region/prune_cluster_nodes.py` +- `domain-api/deploy/multi-region/prune_cluster_nodes.sh` +- `domain-api/deploy/multi-region/templates/*.env.example` + +## 一点五、海外单入口现在支持 Contract Preview 和门禁执行 + +从这版开始，海外机的统一入口不只是“直接执行动作”，还支持先看 contract： + +```bash +cd /opt/domaincheck/domain-api + +# 看 contract registry +bash domain-api/deploy/multi-region/drive_ops_center.sh contracts + +# 直接预览某个 driver action +bash domain-api/deploy/multi-region/drive_ops_center.sh driver-preview open_release_dialog + +# 先按统一门禁解析某个 driver action +bash domain-api/deploy/multi-region/drive_ops_center.sh driver-resolve enable_log_sync_key + +# 按统一门禁执行某个 driver action +bash domain-api/deploy/multi-region/drive_ops_center.sh driver-run enable_log_sync_key + +# 单独查看远端日志回传当前状态 +bash domain-api/deploy/multi-region/drive_ops_center.sh log-sync-check + +# 一条命令完成：开启日志回传 + 复核状态 + 给出下一跳建议 +bash domain-api/deploy/multi-region/drive_ops_center.sh log-sync-recover +bash domain-api/deploy/multi-region/drive_ops_center.sh log-sync-recover http://127.0.0.1:8100 full confirm cli/ops + +# 一条命令完成：查看接管缺口 + 选首节点 + 生成接入方案 + 给出下一跳建议 +bash domain-api/deploy/multi-region/drive_ops_center.sh agent-gap-check +bash domain-api/deploy/multi-region/drive_ops_center.sh agent-gap-recover + +# 先读取 stack diagnosis 默认 next_step，再按统一门禁预览 +bash domain-api/deploy/multi-region/drive_ops_center.sh stack-next + +# 显式确认后，直接执行 stack diagnosis 当前默认下一步 +bash domain-api/deploy/multi-region/drive_ops_center.sh stack-next run confirm + +# 先看当前 driver 主线，再直接预览 / 执行 top recommendation +bash domain-api/deploy/multi-region/drive_ops_center.sh driver-feed +bash domain-api/deploy/multi-region/drive_ops_center.sh activity-stream +bash domain-api/deploy/multi-region/drive_ops_center.sh activity-stream kind=ops_job status=running +bash domain-api/deploy/multi-region/drive_ops_center.sh activity-preview +bash domain-api/deploy/multi-region/drive_ops_center.sh driver-focus-preview +bash domain-api/deploy/multi-region/drive_ops_center.sh driver-focus-run + +# 也可以显式指定某个 focus key，例如 runbook +bash domain-api/deploy/multi-region/drive_ops_center.sh driver-focus-preview runbook:release_progression +bash domain-api/deploy/multi-region/drive_ops_center.sh driver-focus-run runbook:release_progression + +# 预览某条标准 runbook path，会自动带出主次动作 +bash domain-api/deploy/multi-region/drive_ops_center.sh runbook-preview release_progression + +# 按统一门禁执行某条标准 runbook path +bash domain-api/deploy/multi-region/drive_ops_center.sh runbook-run release_progression + +# 直接预览 codex-brief 当前焦点动作 +bash domain-api/deploy/multi-region/drive_ops_center.sh codex-focus-preview + +# 按 codex-brief 门禁规则尝试执行当前焦点 +bash domain-api/deploy/multi-region/drive_ops_center.sh codex-focus-run + +# 对 guarded_auto 焦点显式确认后执行 +bash domain-api/deploy/multi-region/drive_ops_center.sh codex-focus-run focus-guarded confirm +``` + +这些命令背后统一走： + +- `GET /api/v1/ops/stack-diagnosis` +- `GET /api/v1/ops/contracts` +- `POST /api/v1/ops/driver-actions/preview` +- `POST /api/v1/ops/driver-actions/resolve` +- `POST /api/v1/ops/driver-actions/execute-resolved` +- `POST /api/v1/ops/codex-actions/resolve` +- `POST /api/v1/ops/codex-actions/execute` + +也就是说： + +- 页面点“看契约” +- 海外 CLI 查“看契约” +- 后续海外 Codex 驾驶员判断“是否执行” + +都开始复用同一套后端解释结果，而不是各自再推导一遍目标 API、执行链、request payload 和门禁规则。 + +当前推荐的海外单脑控制面起手顺序也已经固定： + +```bash +cd /www/wwwroot/getDomain + +# 1. 先看上线收口摘要 +bash domain-api/deploy/multi-region/drive_ops_center.sh go-live-check http://127.0.0.1:8100 http://127.0.0.1:8100 summary + +# 2. 再看总检详情 +bash domain-api/deploy/multi-region/drive_ops_center.sh stack-diagnosis + +# 3. 再看驾驶主线 +bash domain-api/deploy/multi-region/drive_ops_center.sh driver-feed +bash domain-api/deploy/multi-region/drive_ops_center.sh codex-brief +``` + +补充说明： + +- CLI 里 `stack-diagnosis` 和 `stack-check` 现在是同一条总检入口 +- 推荐优先记忆 `stack-diagnosis` + - 因为它和页面 API `/api/v1/ops/stack-diagnosis`、文档名称、值班口径保持一致 + +这 4 步现在分别对应： + +- `go-live-summary` + - 先回答“能不能继续收口 / 能不能继续发布” +- `stack-diagnosis` + - 解释阻断原因、主处理车道、默认下一步、推荐命令 +- `driver-feed` + - 告诉页面 / CLI / Codex 当前最值得处理的驾驶主线 +- `codex-brief` + - 告诉海外 Codex 现在到底该自动执行、确认执行、还是只进 UI + +补充一个非常关键的判读口径： + +- 如果 `env-audit` 已经提示 `runtime.runtime_may_need_restart=true` +- 或 `stack-diagnosis` / `go-live-summary` 已经出现 `runtime_build_schema_stale` +- 但 `stack-next` 仍然优先给出 `node_handover` + +这通常不代表“接管缺口比 API 重启更重要”，而是代表当前运行中的 `domaincheck-api` 还没有切到最新代码。此时应先执行： + +```bash +bash domain-api/deploy/multi-region/drive_ops_center.sh runtime-refresh-recover http://127.0.0.1:8100 +bash domain-api/deploy/multi-region/check_ops_center_stack.sh http://127.0.0.1:8100 summary +bash domain-api/deploy/multi-region/drive_ops_center.sh stack-next +``` + +如果你怀疑当前不是业务链路问题，而是环境本身还有漂移，例如： + +- Python 依赖没装全 +- 当前机器缺 `pytest / fastapi / uvicorn` +- Node Agent env 文件缺失 +- systemd 服务启停口径不一致 +- `runtime/build-info` 的 route surface 还不完整 + +先不要直接进发布链，先跑： + +```bash +bash domain-api/deploy/multi-region/drive_ops_center.sh env-audit +bash domain-api/deploy/multi-region/drive_ops_center.sh runtime-refresh-check +bash domain-api/deploy/multi-region/drive_ops_center.sh runtime-refresh-recover +bash domain-api/deploy/multi-region/drive_ops_center.sh go-live-recover +``` + +它会统一给出： + +- 本机命令可用性 +- Python 模块可用性 +- env 文件存在性 +- systemd 服务状态 +- `runtime/preflight` +- `runtime/readiness` +- `runtime/build-info` +- 一份最终 `status / headline / missing_items / tooling_items` 收口摘要 + +说明： + +- `missing_items` 只保留真正影响部署或运行面的缺口 +- `tooling_items` 用来提示当前机器缺少的本地辅助工具，例如 `pytest` +- Python 依赖会优先按实际运行时 `.venv` 判定，而不是只看当前 shell 的 `python3` +- 现在还会额外识别“本地仓库已支持新能力，但运行中 API 还是旧 build-info / 旧路由面”的部署漂移 +- 重点信号： + - `runtime.repo_capability_drift` + - `runtime.runtime_may_need_restart` + - `recommended_actions` + +推荐理解为两层： + +- `env-audit` + - 看全量环境、依赖、systemd、build-info、route surface 漂移 +- `runtime-refresh-recover` + - 专门收口“仓库代码已更新，但运行中的 API 还是旧版本”这类上线前高频问题 + - 会先打印完整 env audit，再补一段固定的运行时刷新建议 + - 适合值班同事、海外 Codex、后台按钮统一作为“重启前最后确认入口” +- `go-live-recover` + - 在 `runtime-refresh-recover` 之上，再顺序串起： + - `stack-diagnosis summary` + - `stack-next` + - `go-live-check summary` + - `doctor-decision` + - 适合上线前最后一轮“一键复核”，也适合后续后台按钮直接接成固定恢复流 + +如果要把这轮上线前验证直接导出成可交接、可归档、可复核的一整包证据，可以直接执行： + +```bash +cd /www/wwwroot/getDomain +bash domain-api/deploy/multi-region/drive_ops_center.sh go-live-export +``` + +它会在 `runtime/ops-center-reports/go-live-bundles/` 下生成一份 bundle，至少包含： + +- `00_env_audit.txt` +- `01_go_live_check_summary.txt` +- `02_go_live_summary.json` +- `03_stack_diagnosis.json` +- `04_driver_feed.json` +- `05_codex_brief.json` +- `06_release_launchpad.json` +- `07_doctor_decision.json` +- `08_doctor_export_stdout.txt` +- `manifest.json` + +其中 `manifest.json` 会统一标记： + +- 环境审计是否已经给出 `ready / attention / blocked` +- 哪些报告成功 +- 哪些报告超时 +- 哪些接口当前还缺失 +- 推荐优先阅读顺序 + +同时，`bash domain-api/deploy/multi-region/drive_ops_center.sh go-live-review` 现在不再只看 +“关键文件是否齐全”，还会继续核对： + +- `02_go_live_summary.json` +- `03_stack_diagnosis.json` +- `04_driver_feed.json` +- `05_codex_brief.json` + +这四份摘要里的 launchpad 收口字段是否一致，包括： + +- `launchpad_recommended_target_node_code` +- `launchpad_recommended_recovery_label` +- `launchpad_recommended_recovery_summary` +- `launchpad_onboarding_bootstrap_pending_nodes` +- `launchpad_onboarding_acceptance_ready_nodes` + +如果这些字段发生漂移，即使关键文件都存在，`go-live-review.status` 也会至少回到 `attention`， +防止把“局部摘要仍是旧运行态 / 导出时混入不同时间点数据”的 bundle 误判成可直接交付。 + +另外，从这版开始，`driver-run enable_log_sync_key / enable_log_sync_full / disable_log_sync` +执行完成后会自动追加一段 `log sync post-check`，直接回显： + +- 当前是否开启 +- 当前模式 `key/full` +- 当前参与节点数 +- 当前已覆盖节点数 +- 当前样本条数 +- 当前缺口节点 + +也就是说，海外机不需要再手工补 `curl /api/v1/ops/overview` 去确认“按钮到底有没有真生效”。 + +如果你想把这条链路进一步做成“一步恢复流”，现在可以直接用： + +- `bash domain-api/deploy/multi-region/drive_ops_center.sh log-sync-recover` +- `bash domain-api/deploy/multi-region/drive_ops_center.sh log-sync-recover http://127.0.0.1:8100 full confirm cli/ops` + +这个组合入口会依次做三件事： + +1. 执行 `enable_log_sync_key/full` +2. 自动复核当前 `execution_scene.log_sync` +3. 根据当前结果输出下一跳命令 + +下一跳建议会自动带出： + +- `scene-node-log` +- `log-sync-check` +- `doctor` + +也就是说，它已经从“能切开关”升级成“能把恢复动作和现场下钻串起来”的单入口。 + +Node Agent 接管链路现在也有对应的一步恢复流： + +- `bash domain-api/deploy/multi-region/drive_ops_center.sh agent-gap-check` +- `bash domain-api/deploy/multi-region/drive_ops_center.sh agent-gap-recover` +- `bash domain-api/deploy/multi-region/drive_ops_center.sh agent-gap-export` +- `bash domain-api/deploy/multi-region/drive_ops_center.sh node-onboarding http://127.0.0.1:8100 mainland-worker-01` +- `bash domain-api/deploy/multi-region/drive_ops_center.sh node-recover http://127.0.0.1:8100 mainland-worker-01` +- `bash domain-api/deploy/multi-region/drive_ops_center.sh node-recover http://127.0.0.1:8100 mainland-worker-01 confirm ops-center` +- `bash domain-api/deploy/multi-region/drive_ops_center.sh node-bootstrap-preview http://127.0.0.1:8100 mainland-worker-01` +- `bash domain-api/deploy/multi-region/drive_ops_center.sh node-bootstrap-run http://127.0.0.1:8100 mainland-worker-01 ops-center` +- `bash domain-api/deploy/multi-region/drive_ops_center.sh node-acceptance-plan http://127.0.0.1:8100 mainland-worker-01` +- `bash domain-api/deploy/multi-region/drive_ops_center.sh node-acceptance-run http://127.0.0.1:8100 mainland-worker-01 ops-center` + +这个组合入口会依次做四件事： + +1. 从当前 `stack-diagnosis` 找出首个接管缺口节点 +2. 输出该节点当前 handover 状态 +3. 自动生成该节点的 bootstrap plan +4. 再回显当前剩余接管缺口和下一跳建议 + +如果你希望直接把首个缺口节点的接管材料落成一份目录，而不是逐条复制终端输出，现在还可以直接执行： + +- `bash domain-api/deploy/multi-region/drive_ops_center.sh agent-gap-export` + +它会统一导出： + +- `stack-first-gap-handover` +- `agent-gap-check` +- `node-onboarding` +- `node-bootstrap-plan` +- `node-bootstrap-preview` +- `node-acceptance-plan` +- `manifest.json` + +适合三种场景： + +- 需要把首个接管缺口节点的落地材料交给现场同事 +- 需要让海外 Codex / 后台按钮直接读取一份固定目录 +- 需要把“为什么当前还没 ready”收敛成一份节点级证据包 + +其中 `node-onboarding` 是新增的节点级收口视图，它会把： + +- 当前 handover 所处阶段 +- 当前是否已经可以做 `onboarding.acceptance` +- 建议继续 `bootstrap` 还是切到 `acceptance` +- 验收应该走 `remote-agent` 还是 `ssh` + +统一压成一份结果，而不是让值班同事自己在人脑里拼： + +- `node-handover` +- `node-bootstrap-plan` +- `onboarding.acceptance` + +如果节点已经进入 `acceptance_ready`，现在还可以继续走两步： + +- `node-acceptance-plan` + - 预览这次 `onboarding.acceptance` 会创建哪些验收子任务 +- `node-acceptance-run` + - 直接签发这轮接管验收 playbook，进入统一的 ops job / playbook run 轨道 + +而在还没进入 `acceptance_ready` 之前，也不再只有“生成本地 shell 方案”这一条路了： + +- `node-bootstrap-preview` + - 预览这次 `onboarding.bootstrap` 将创建的接入工单 +- `node-bootstrap-run` + - 直接签发接入 playbook，统一收进 ops playbook / ops job 体系 + +这样节点级链路就被拆成三层： + +- `node-bootstrap-plan` + - 适合要拿到落地脚本、env 内容、命令块的时候 +- `node-bootstrap-run` + - 适合要把接入动作纳入标准工单追踪的时候 +- `node-acceptance-run` + - 适合节点已经 ready 后做标准化验收的时候 + +如果你不想再自己判断当前该走哪一步，现在还有一个更高层入口： + +- `node-recover` + - 先读取节点当前 onboarding / handover 状态 + - 如果还在接入阶段，自动选择 `node-bootstrap-run` + - 如果已经进入验收窗口，自动选择 `node-acceptance-run` + - 不带 `confirm` 时只做决策预览；带 `confirm` 时直接执行 + +而且它已经支持兼容降级： + +- 如果当前 API 还没有 `/api/v1/ops/nodes/{node_code}/handover` + - 自动退回 `stack-diagnosis` +- 如果当前 API 还没有 `/api/v1/ops/nodes/{node_code}/handover/bootstrap-plan` + - 自动退回 `/api/v1/ops/agent/bootstrap-plan` +- 如果当前 API 还没有 `/api/v1/ops/nodes/{node_code}/onboarding` + - 自动退回 `handover` 兼容视图，再推导当前 onboarding 阶段 + +另外，Node Agent 安装脚本和 bootstrap 计划现在已经补了多目录口径兼容： + +- 优先尝试 `${ROOT_DIR}/domainCheck/...` +- 再尝试 `${ROOT_DIR}/domain-api/...` +- 最后尝试 `${ROOT_DIR}/deploy/...` + +这样即使线上节点的实际部署根目录是 `domainCheck` 扁平包，而不是保留 `domain-api` 子目录，bootstrap 计划也不会因为安装脚本路径写死而直接失效。 + +也就是说，海外单入口不会再因为控制面版本过渡期少了某个节点级接口，就直接丢失接管能力。 + +另外，`drive_ops_center.sh` 现在输出的 condensed summary 也开始统一带上： + +- payload 顶层 `contract_navigation` +- 当前焦点或当前选中项的 `contract_navigation` +- preview 返回里的 `contract_key / contract_version / contract_primary_endpoint / preview_endpoint / schema_doc_path` + +也就是说海外单入口不只是“能跑命令”，而是已经能把“这一跳到底消费了哪份协议、该去哪个详情接口、该看哪份 schema 文档”一起打印出来，方便后续给 Codex 驾驶员或后台按钮做真正的协议驱动。 + +节点现场日志现在也有独立 drill-down 入口，适合在“参与节点在线但没样本”“日志只覆盖部分节点”“想直接盯某一台 worker/controller 现场”时使用： + +```bash +cd /opt/domaincheck/domain-api + +# 通过单入口调后端节点现场日志 API +bash domain-api/deploy/multi-region/drive_ops_center.sh scene-node-log http://127.0.0.1:8100 mainland-worker-01 120 key + +# 或者直接走独立检查脚本，看原始返回 + 收口摘要 +bash domain-api/deploy/multi-region/check_node_scene_log.sh http://127.0.0.1:8100 mainland-worker-01 120 full +``` + +另外，`check_ops_plane.sh` 现在不再只是顺序打印 overview / nodes / jobs / activity / launchpad 等原始返回，最后还会追加一段 `[13/13] condensed summary`，把下面这些跨面板判断统一收口成一份 JSON： + +- overview 顶部推荐动作 +- 现场日志观察 `scene_log_observation` +- managed nodes 当前需关注节点 +- jobs / activity-stream 的运行态摘要 +- release launchpad 推荐动作与 rollout preview +- latest release 当前状态 +- policy preview sample 的门禁结论 + +而且这一份 condensed summary 现在还会带一组可直接执行的 `recommended_commands`，用于把“总检”直接串到“下一跳动作”： + +- `stack_diagnosis` +- `driver_feed` +- `driver_focus_preview` +- `driver_focus_run` +- `scene_log` +- `scene_log_direct` +- `node_handover` +- `node_bootstrap_plan` +- `delivery_queue` +- `release_launchpad` +- `release_preview_latest` +- `release_preview_latest_worker` +- `release_preview_latest_control` +- `jobs` + +在这之上，`check_ops_plane.sh` 现在还会继续收口两层真正适合自动驾驶消费的字段： + +- `operator_decision` + - 当前主处理车道，例如 `observability / node_handover / release / ops_jobs / steady` + - 优先级 + - `reason_code` + - `summary` + - `next_focus` + - `primary_command_key` + - `secondary_command_key` +- `next_actions` + - 已经按 `primary_command_key / secondary_command_key` 解引用好的下一跳命令 + - 页面按钮、海外 Codex 驾驶员、后续自动编排层可以直接消费，不需要再自己拼接命令 + +也就是说，现在海外单机执行： + +```bash +cd /opt/domaincheck/domain-api +bash domain-api/deploy/multi-region/check_ops_plane.sh http://127.0.0.1:8100 +``` + +不仅能看原始面板，也能在最后直接拿到一份适合： + +- 人工值守快速判断 +- 海外 Codex 驾驶员自动消费 +- 后台后续接“一键总检 / 一键诊断”按钮 + +的总控摘要。 + +补充一点： + +- 如果当前 `overview` 还没有直接带 `driver_feed.scene_log_observation` +- `check_ops_plane.sh` 也会自动从 `execution_scene.log_sync + participation_summary` 推导现场日志观察状态 + +所以即便线上还处在“overview 只有 execution_scene，没有 driver_feed”的阶段， +总检摘要里也依然会稳定给出： + +- `scene_log_observation.status` +- `scene_log_observation.status_label` +- `scene_log_observation.summary` +- `scene_log_observation.target_node_code` +- `scene_log_observation.focus_ref` + +避免海外单脑控制面因为接口层级差异而出现空白状态。 + +当前推荐的消费顺序也固定下来了： + +1. 先看 `operator_decision` +2. 再看 `next_actions.primary_command` +3. 如果需要人工复核，再看 `next_actions.secondary_command` +4. 仍需下钻时，再结合 `focus_refs` 和 `recommended_commands` 进入细分面板 + +这样后续不管是： + +- 海外单机人工值守 +- 后台“一键诊断 / 一键处理”按钮 +- 海外 Codex 作为智能驾驶员 + +都可以先消费同一份“主决策 + 下一跳”摘要，而不是每个入口各写一套判断逻辑。 + +同样的口径现在也已经补到了： + +```bash +bash domain-api/deploy/multi-region/check_ops_center_stack.sh http://127.0.0.1:8100 summary +``` + +也就是说，`check_ops_center_stack.sh` 在以下两种情况下都会输出同一类增强摘要： + +- 后端 `/api/v1/ops/stack-diagnosis` 已经可用 +- 后端还没完全切到正式 contract，脚本退回本地兜底汇总 + +无论走哪条路径，最终 condensed stack summary 里现在都会补上： + +- `diagnosis.operator_decision` +- `diagnosis.next_actions` +- `diagnosis.recommended_commands` + +这样海外单脑控制面现在已经具备： + +- `check_ops_plane.sh` +- `check_ops_center_stack.sh` + +两条总检入口的统一主决策口径，后续页面、按钮、Codex 驾驶员都可以围绕这三层字段继续收口，不需要分别适配两套不同输出。 + +现在这套统一解释结果里，最推荐先消费的就是： + +- `GET /api/v1/ops/stack-diagnosis` + +因为它会先把： + +- contract registry +- link snapshot +- overview +- managed nodes +- release launchpad +- recent playbook runs +- recent activity stream + +收口成一份正式 `diagnosis`，再由 CLI / Codex / 页面继续下钻。 + +其中 `stack-next` 的定位是： + +- 先读取 `GET /api/v1/ops/stack-diagnosis` +- 直接消费 `diagnosis.next_step` +- 再走 `POST /api/v1/ops/driver-actions/resolve` +- 或 `POST /api/v1/ops/driver-actions/execute-resolved` + +这样海外单入口就不需要再人工从总检结果里抄 `action_code`，而是可以把“当前默认下一步”直接贯通到预览和执行。 + +其中： + +- `driver-preview` / `runbook-preview` + - 继续直接走 `driver-actions/preview` +- `driver-resolve` / `driver-run` / `runbook-run` + - 现在走 `driver-actions/resolve` / `driver-actions/execute-resolved` + - 对普通 driver action 和 runbook sequence 也复用同一套 `gate` + - CLI 不再自己推断 `直接执行 / 等确认 / 进入 UI / 阻断` +- `driver-focus-preview` / `driver-focus-run` + - 先读取 `GET /api/v1/ops/driver-feed` + - 默认优先消费 `top_recommendation` + - 也可显式指定 `entry_key` + - 再统一走 `driver-actions/resolve` / `driver-actions/execute-resolved` + - 会把 `focus_ref / primary_focus_ref / secondary_focus_ref` 一起透传到 resolve / execute-resolved + - condensed summary 也会直接打印焦点落点，不需要再人工猜“当前该看哪个 release / execution scene” + - 这样海外单入口已经能直接围绕“当前最该处理什么”工作，而不是先人工抄 action code +- 页面里的 `overview driver recommendation` / `driver-feed` + - 现在优先走 `driver-actions/resolve` + - 执行时走 `driver-actions/execute-resolved` + - 由后端统一决定 `直接执行 / 等确认 / 先复核 / 进入 UI / 阻断` +- `codex-focus-preview` + - 现在走 `codex-actions/resolve` + - 由后端一次性返回 `selected_entry + preview + gate` + - CLI 会保留并展示 `selected_entry.focus_ref / gate.focus_ref / 顶层 focus_ref` +- `codex-focus-run` + - 现在走 `codex-actions/execute` + - 由后端统一决定是 `直接执行 / 等确认 / 先复核 / 进入 UI / 阻断` + +当前 `codex-focus-run` 的门禁规则也已经固定： + +- `safe_auto` + - 直接执行 +- `guarded_auto` + - 默认只输出审阅结果 + - 只有显式传 `confirm` 才执行 +- `resolve_first` + - 只给出阻断说明，不执行 +- `open_ui` + - 只给出 UI-only 提示，不执行 +- `blocked` + - 明确阻断，不执行 + +补充说明： + +- `driver` / `runbook-sequence` + - 仍然保留，作为低层直打执行接口的 legacy 调试入口 + - 更适合排查后端执行器本身，不适合作为日常自动驾驶入口 +- 日常推荐优先使用： + - `driver-resolve` + - `driver-run` + - `runbook-run` + - `codex-focus-preview` + - `codex-focus-run` ## 二、国外机器部署 @@ -42,7 +642,7 @@ ```bash cd /opt/domaincheck/domain-api -bash deploy/multi-region/bootstrap_overseas.sh /opt/domaincheck +bash domain-api/deploy/multi-region/bootstrap_overseas.sh /opt/domaincheck ``` 脚本会： @@ -60,14 +660,14 @@ bash deploy/multi-region/bootstrap_overseas.sh /opt/domaincheck ```bash cd /opt/domaincheck/domain-api -bash deploy/multi-region/bootstrap_mainland.sh /opt/domaincheck controller +bash domain-api/deploy/multi-region/bootstrap_mainland.sh /opt/domaincheck controller ``` 如果后面新增纯 Worker 节点： ```bash cd /opt/domaincheck/domain-api -bash deploy/multi-region/bootstrap_mainland.sh /opt/domaincheck worker +bash domain-api/deploy/multi-region/bootstrap_mainland.sh /opt/domaincheck worker ``` 其中： @@ -111,6 +711,16 @@ bash deploy/multi-region/bootstrap_mainland.sh /opt/domaincheck worker - 节点环境变量入口统一 - 新增节点时操作步骤统一 +当前仓库也已经补出了 Node Agent 的正式运行骨架： + +- `app.node_agent` +- `deploy/systemd/domain-node-agent.service` +- `domain-api/deploy/multi-region/bootstrap_node_agent.sh` +- `domain-api/deploy/multi-region/build_node_agent_bootstrap_plan.sh` +- `domain-api/deploy/multi-region/check_node_agent.sh` + +它的定位不是替代现有 bootstrap，而是为下一阶段“海外控制面统一接管大陆节点”提供正式执行器入口。 + 当前大陆 bootstrap 还会自动补齐一组最小 Python 依赖： - `fastapi` @@ -121,6 +731,54 @@ bash deploy/multi-region/bootstrap_mainland.sh /opt/domaincheck worker 这样 `domaincheck-sync-agent` 在大陆 controller 节点上可以直接启动，不会因为 `app.sync_agent` 缺少 API 侧依赖而失败。 +如果要在海外控制面直接为某台节点生成一份可复制执行的 Agent 接入方案，可以执行： + +```bash +cd /opt/domaincheck/domain-api +bash domain-api/deploy/multi-region/build_node_agent_bootstrap_plan.sh \ + http://127.0.0.1:8100 \ + mainland-worker-02 \ + mainland \ + worker \ + https://ops.example.com \ + /opt/domaincheck +``` + +它会直接请求： + +- `POST /api/v1/ops/agent/bootstrap-plan` +- 现在也可以先用 `GET /api/v1/ops/nodes/{node_code}/handover` 看单节点接管详情 +- 再通过 `POST /api/v1/ops/nodes/{node_code}/handover/bootstrap-plan` 生成节点级 bootstrap 方案 + +并输出： + +- token 摘要 +- `/etc/default/domaincheck-node-agent` 的完整 env 内容 +- 一段可直接在目标节点执行的安装/启动命令块 +- 一份完整可执行的 `bootstrap-node-agent-.sh` 脚本内容 +- 一段“写入 `/tmp` 并立即执行”的一键落地命令块 +- 基础健康检查命令 + +也就是说，现在这份接入方案已经不是“给你一枚 token 自己拼命令”，而是标准化输出： + +- `env_content` +- `command_block` +- `bootstrap_script_content` +- `bootstrap_run_script_block` +- `health_check_block` + +后续无论是人工 SSH 粘贴执行、海外 Codex 远程接管，还是后台按钮化接入，都可以直接复用这一套产物。 + +如果当前节点已经能在集群心跳里看到，但还没有被正式纳管，现在还可以直接走控制面页面里的新闭环： + +1. 打开 `运维中枢` +2. 在“托管节点”里找到状态为 `仅运行态在线 / 未纳管` 的节点 +3. 点击 `纳管接入` +4. 补齐 SSH 信息后保存 +5. 控制面会立刻生成 Node Agent 接入方案 + +这样“先看到节点，再纳管节点，再生成接入命令”已经收敛成一条固定流程，不需要再在多处入口之间来回切换。 + 当前脚手架还额外区分了“自动同步由谁跑”： - 海外控制面： @@ -178,7 +836,7 @@ curl http://127.0.0.1:8100/api/v1/detect/jobs ```bash cd /opt/domaincheck/domain-api -bash deploy/multi-region/check_cluster.sh http://127.0.0.1:8100 +bash domain-api/deploy/multi-region/check_cluster.sh http://127.0.0.1:8100 ``` 这条命令当前除了原始接口输出，还会额外给出一段压缩摘要，直接汇总： @@ -199,7 +857,7 @@ bash deploy/multi-region/check_cluster.sh http://127.0.0.1:8100 ```bash cd /opt/domaincheck/domain-api -bash deploy/multi-region/check_mainland_controller.sh +bash domain-api/deploy/multi-region/check_mainland_controller.sh ``` 这条命令会直接检查： @@ -216,27 +874,47 @@ bash deploy/multi-region/check_mainland_controller.sh ```bash cd /opt/domaincheck/domain-api -bash deploy/multi-region/check_worker_participation.sh +bash domain-api/deploy/multi-region/check_worker_participation.sh ``` 这条命令会直接输出： - `detect_worker_nodes` 当前在线节点概览 - 最新活跃任务的 `claimed_by / status / count` -- 一段压缩摘要 JSON +- `runtime/status` 里的参与度摘要： + - `participation_summary` + - `participating_nodes` + - `non_participating_nodes` + - `log_sync` +- 一段压缩摘要 JSON，直接告诉你： + - 真正执行/领任务的是哪些节点 + - 近窗刚有吞吐的是哪些节点 + - 在线但未参与的是哪些节点 + - 远端日志回传是否开启、来源节点有哪些、最近一次回传时间是什么 + +建议这样理解： + +- `有效执行节点` + 代表当前在线、理论上可承接检测任务的节点，包含独立 worker，也可能包含 controller 兼跑 +- `真正执行/领任务` + 才是当前这一刻真的在拿任务、跑任务的节点 +- `在线但未参与` + 代表节点在线、可用，但这轮任务还没落到它身上 +- `负载待确认` + 代表节点上报了负载或 `busy`，但还没观察到明确的已领/执行中/近窗吞吐证据，通常是快照同步窗口问题，不要直接判断为“正在干活” 如果要检查纯大陆 worker 节点自己的配置和服务状态，可以执行： ```bash cd /opt/domaincheck/domain-api -bash deploy/multi-region/check_mainland_worker.sh +bash domain-api/deploy/multi-region/check_mainland_worker.sh ``` 如果当前是“大陆 controller + 大陆 worker + 临时海外控制面”的联调拓扑，还可以执行： ```bash cd /opt/domaincheck/domain-api -bash deploy/multi-region/check_temp_topology.sh http://127.0.0.1:8100 http://152.53.37.118:8100 +bash domain-api/deploy/multi-region/check_temp_topology.sh http://127.0.0.1:8100 http://152.53.37.118:8100 ``` 这条命令会同时汇总： @@ -246,11 +924,821 @@ bash deploy/multi-region/check_temp_topology.sh http://127.0.0.1:8100 http://152 - 最新活跃任务到底由谁 `claimed_by` - 临时海外控制面的 `runtime/cluster` +如果要专门检查新的 Release Hub / Rollout 引擎，可以执行： + +```bash +cd /opt/domaincheck/domain-api +bash domain-api/deploy/multi-region/check_release_hub.sh http://127.0.0.1:8100 +``` + +它会输出： + +- `release launchpad` +- 最新激活 release +- 最近 release 列表 +- `deploy.release.control / worker / custom` 模板摘要 +- 控制面 / Worker / 混合节点批量预检结果 +- 当前 release 的 rollout 列表 +- 主 rollout 详情 +- 主 rollout 对应的 job 列表 + +如果你想直接从海外控制面命令行触发 Release Hub 的关键动作，而不是先打开页面，可以执行： + +```bash +cd /opt/domaincheck/domain-api +bash domain-api/deploy/multi-region/drive_release_hub.sh http://127.0.0.1:8100 launchpad +bash domain-api/deploy/multi-region/drive_release_hub.sh http://127.0.0.1:8100 preview-latest +bash domain-api/deploy/multi-region/drive_release_hub.sh http://127.0.0.1:8100 preview-latest-smart worker +bash domain-api/deploy/multi-region/drive_release_hub.sh http://127.0.0.1:8100 latest-package +bash domain-api/deploy/multi-region/drive_release_hub.sh http://127.0.0.1:8100 smart-latest worker +bash domain-api/deploy/multi-region/drive_release_hub.sh http://127.0.0.1:8100 preview-release 12 control +bash domain-api/deploy/multi-region/drive_release_hub.sh http://127.0.0.1:8100 smart-release 12 control confirm +``` + +这条脚本会统一走当前标准接口： + +- `GET /api/v1/ops/releases/launchpad` +- `POST /api/v1/ops/releases/from-package/latest/preview` +- `POST /api/v1/ops/releases/from-package/latest/smart-rollout-preview` +- `GET /api/v1/ops/releases/package-metadata/latest` +- `POST /api/v1/ops/releases/from-package/latest/smart-rollout` +- `POST /api/v1/ops/releases/{release_id}/smart-rollout-preview` +- `POST /api/v1/ops/releases/{release_id}/smart-rollout` + +并输出三段固定内容： + +- 请求摘要 +- 原始响应 +- 压缩后的关键信息 + +另外现在 `ops overview` 也已经直接内嵌了： + +- `release_hub.launchpad` + +也就是说，页面里的运维中枢、`check_ops_plane.sh`、`check_release_hub.sh`、`drive_release_hub.sh launchpad` 已经统一复用同一套发布驾驶舱结论，不需要再分别猜“当前到底该发 Worker、补接管，还是先看 Rollout 阻断”。 + +如果你想进一步复用 `OpsCenter` 里的“驾驶动作”，而不是只限于 Release Hub，可以直接执行： + +```bash +cd /opt/domaincheck/domain-api +bash domain-api/deploy/multi-region/drive_ops_action.sh \ + http://127.0.0.1:8100 \ + create_release_rollout_worker \ + '{"release_id":12}' +``` + +它会统一调用： + +- `POST /api/v1/ops/driver-actions/execute` + +所以后续无论是： + +- 海外主机手工执行 +- 海外 Codex 自动驾驶 +- 后台按钮化封装 + +都可以复用同一套 driver action，而不是再写多份分叉逻辑。 + +当前 Web 后台也已经新增： + +- `/#/ops-center` + +这页就是海外控制面的运维中枢，后续节点纳管、任务审批、release rollout 推进都优先从这里进入。 + +如果想在命令行里快速看当前运维控制面的“总览 + 活动流 + Release 状态”，可以直接执行： + +```bash +cd /opt/domaincheck/domain-api +bash domain-api/deploy/multi-region/check_ops_plane.sh http://127.0.0.1:8100 +``` + +这条脚本现在会一并输出： + +- `top recommendation` +- `driver recommendations` +- `ops overview` +- `ops driver feed` +- `ops capabilities` +- `ops blueprint` +- `ops runbook` +- `ops nodes` +- `ops jobs` +- `ops activity stream` +- `release launchpad` +- `latest release` +- `policy preview sample` + +其中 `ops runbook` 里的 `release_progression` 现在也已经直接复用 `release launchpad` 结论，不再是另一套独立的默认 Release 判断。 + +另外，标准作业路径也已经有独立执行入口： + +- `POST /api/v1/ops/runbook/sequences/{sequence_key}/execute` + +这样页面里的“标准作业路径”不再直接自己拼 driver action，而是先走统一的 runbook sequence，再由后端决定当前该落到哪个具体动作。 + +如果只想先看“这条标准作业路径此刻会解析成什么动作、哪些节点、什么 payload”，现在也有独立解析入口： + +- `POST /api/v1/ops/runbook/sequences/{sequence_key}/resolve` +- `bash domain-api/deploy/multi-region/drive_ops_center.sh runbook-resolve release_progression` + +这样页面按钮、CLI、海外 Codex 驾驶员和未来自动调度器，都可以先 `resolve` 再决定是否 `execute`，不需要各自复制一套判断逻辑。 + +另外，当前 `GET /api/v1/ops/runbook` 返回的每条 `control_sequence` 里，也会直接附带： + +- `primary_resolution` +- `secondary_resolution` + +也就是说，海外控制面页面现在只拉一次 `runbook`，就能同时看到： + +- 这条路径原本定义的标准动作 +- 当前后端实际解析后的动作 +- 当前目标节点 +- 解析时间 + +这样前端不需要再为每条路径额外发一次 `resolve` 请求，标准作业路径的“展示”和“解析结果”也终于收成一份数据。 + +最近还补了一层“活动流和 runbook 打通”： + +- `GET /api/v1/ops/activity-stream` + +现在除了： + +- `playbook_run` +- `ops_job` +- `rollout` + +活动流里还会自动附带： + +- `runbook_sequence` + +它表示“当前标准作业路径此刻解析出的下一步动作快照”。 + +这样海外控制面在统一活动流里，不仅能看到最近执行回执，也能直接看到： + +- 哪条固定路径当前最该进 +- 它此刻收口成了什么动作 +- 点进去会直接定位到哪张标准作业路径卡片 + +活动流里的 `runbook_sequence` 不是新的执行任务，而是驾驶层的“当前路径快照”，用于把“最近发生了什么”和“现在应该走哪条路径”收进同一条观察链路。 + +现在又继续往前收了一层正式入口： + +- `GET /api/v1/ops/driver-feed` +- `GET /api/v1/ops/codex-brief` +- `bash domain-api/deploy/multi-region/drive_ops_center.sh driver-feed` +- `bash domain-api/deploy/multi-region/drive_ops_center.sh driver-focus-preview` +- `bash domain-api/deploy/multi-region/drive_ops_center.sh driver-focus-run` +- `bash domain-api/deploy/multi-region/drive_ops_center.sh codex-brief` + +它会把三类信息统一压成一份“驾驶主线”： + +- 当前优先建议 +- 其余驾驶建议 +- 当前标准作业路径 + +并明确区分两种执行器： + +- `executor_kind=driver_action` +- `executor_kind=runbook_sequence` + +这样页面、CLI 和未来海外 Codex 都不需要自己再拼“这是一条普通 driver action，还是应该重新走 runbook sequence 执行”，而是直接消费统一 contract。 + +其中 `driver-feed` 现在已经有稳定的消费口径： + +- `top_recommendation` + - 当前默认最该处理的一条驾驶建议 +- `driver_recommendations` + - 其余普通驾驶建议 +- `runbook_sequences` + - 当前适合进入的标准作业路径 +- `activity_focus` + - 当前值得优先关注的活动流焦点 +- `scene_log_observation` + - 当前现场日志回传是否已经能直接下钻到某台节点现场日志的统一观察摘要 + +同时，`driver-feed.summary` 现在也应继续固定带出： + +- `publish_ready` +- `publish_status` +- `publish_status_label` + +这样页面、CLI、海外 Codex 在只消费驾驶摘要时，也能直接区分： + +- 当前只是“可继续收口 / 可继续联调” +- 还是“已经满足正式发布门禁” + +而 `driver-focus-preview` / `driver-focus-run` 会按这套顺序自动选择目标： + +1. 如果命令显式传了 `entry_key`，优先命中该项 +2. 否则优先消费 `top_recommendation` +3. 如果 `top_recommendation` 为空，再退回 `entries[0]` + +这样海外主机上的统一 CLI 已经不需要先人工抄： + +- `action_code` +- `sequence_key` +- `node_codes` + +只要盯住“当前最该处理的一条主线”，就能先预览 contract，再按门禁执行。 + +其中 `scene_log_observation` 这一块现在固定会给出： + +- `status` +- `status_label` +- `summary` +- `target_node_code` +- `focus_ref` + +这意味着海外 Codex、后台自动驾驶和人工值班都不需要再各自重算“当前该去看哪台节点的 scene log”， +直接消费这一个块即可。 + +同时，`bash domain-api/deploy/multi-region/drive_ops_center.sh driver-feed` 和 +`bash domain-api/deploy/multi-region/drive_ops_center.sh codex-brief` +的 condensed summary 现在也会直接带出： + +- `summary.scene_log_status` +- `scene_log_observation.status` +- `scene_log_observation.target_node_code` +- `scene_log_observation.focus_ref` + +也就是说，就算不打开页面、不翻 raw JSON，海外机只看 CLI 摘要也能直接知道： + +- 现场日志有没有开 +- 是不是还在等样本 +- 当前应该下钻哪台节点 + +在 `driver-feed` 之上，现在又补了一层专门给“海外 Codex / 自动驾驶器”消费的判断结果： + +- `codex-brief` + +它不会重新发动作，只负责把每条驾驶建议进一步分级为： + +- `auto_execute` +- `confirm_then_execute` +- `resolve_first` +- `open_ui` +- `blocked` + +并明确告诉消费方： + +- 当前动作最终应打哪个 API +- 是直接执行 `driver-action` + 还是走 `runbook-sequence` +- 当前是低风险设置动作、标准巡检动作、发布推进动作，还是只能进 UI 的阅读动作 + +同时，`codex-brief.summary` 现在也应继续固定返回： + +- `go_live_status` +- `publish_ready` +- `publish_status` +- `publish_status_label` + +这样海外 Codex 的第一轮判断不需要再自己把“能不能继续执行”和“能不能正式发版”混在一起。 + +这样海外 Codex、CLI 和后续后台“一键处理”都能共享同一份自动化判断，而不用各自再维护一套风险口径。 + +另外，`codex-brief.focus` 现在除了能回退到首条 `entry`、首条 `activity_focus`， +也能在“当前没有动作主线，但现场日志已经识别出明确节点目标”时， +直接回退到 `scene_log_observation.focus_ref`。 + +这样在“日志回传已开但样本还没形成”的联调现场里， +海外 Codex 不会卡在“没有动作可执行，也没有明确落点可看”的中间态。 + +如果想专门从命令行盯“哪些节点巡检没收口、谁该先处理”，可以直接执行： + +```bash +cd /opt/domaincheck/domain-api +bash domain-api/deploy/multi-region/check_ops_inspection.sh http://127.0.0.1:8100 +``` + +如果你不想记这么多脚本名，现在也可以直接只记一个统一入口： + +```bash +cd /opt/domaincheck/domain-api +bash domain-api/deploy/multi-region/drive_ops_center.sh help +``` + +它是当前推荐的“海外控制面统一命令入口”，主要负责把多机联调、Release Hub 和 Driver 动作收敛成一套命令面。 + +常见用法： + +```bash +# 在海外控制面本机打包/验包/查看最新发布包 +bash domain-api/deploy/multi-region/drive_ops_center.sh release-package +bash domain-api/deploy/multi-region/drive_ops_center.sh release-verify +bash domain-api/deploy/multi-region/drive_ops_center.sh release-show +bash domain-api/deploy/multi-region/drive_ops_center.sh release-launchpad +bash domain-api/deploy/multi-region/drive_ops_center.sh release-preview +bash domain-api/deploy/multi-region/drive_ops_center.sh release-preview-smart worker + +# 一次性看控制面总览 + release hub + inspection + participation +bash domain-api/deploy/multi-region/drive_ops_center.sh doctor http://127.0.0.1:8100 http://152.53.37.118:8100 + +# 只看海外单入口总检摘要 +bash domain-api/deploy/multi-region/drive_ops_center.sh stack-diagnosis http://127.0.0.1:8100 summary + +# 直接落到首个接管缺口节点 +bash domain-api/deploy/multi-region/drive_ops_center.sh stack-first-gap-handover http://127.0.0.1:8100 + +# 查看单节点 handover 详情 / 直接生成节点级 bootstrap 方案 +bash domain-api/deploy/multi-region/drive_ops_center.sh node-handover http://127.0.0.1:8100 mainland-worker-01 +bash domain-api/deploy/multi-region/drive_ops_center.sh node-onboarding http://127.0.0.1:8100 mainland-worker-01 +bash domain-api/deploy/multi-region/drive_ops_center.sh scene-node-log http://127.0.0.1:8100 mainland-worker-01 120 +bash domain-api/deploy/multi-region/drive_ops_center.sh node-bootstrap-plan http://127.0.0.1:8100 mainland-worker-01 + +# 治理 Node Agent 回执队列 +bash domain-api/deploy/multi-region/drive_ops_center.sh queue-status mainland-worker-01 +bash domain-api/deploy/multi-region/drive_ops_center.sh queue-records mainland-worker-01 dead_letter +bash domain-api/deploy/multi-region/drive_ops_center.sh queue-flush mainland-worker-01 20 +bash domain-api/deploy/multi-region/drive_ops_center.sh queue-replay mainland-worker-01 20 +bash domain-api/deploy/multi-region/drive_ops_center.sh queue-replay-record mainland-worker-01 event-ops-123-2 +bash domain-api/deploy/multi-region/drive_ops_center.sh queue-discard-record mainland-worker-01 event-ops-123-2 "确认已人工归档" + +# 走 Release Hub 标准接口 +bash domain-api/deploy/multi-region/drive_ops_center.sh release-hub http://127.0.0.1:8100 latest-package +bash domain-api/deploy/multi-region/drive_ops_center.sh smart-latest http://127.0.0.1:8100 worker +bash domain-api/deploy/multi-region/drive_ops_center.sh smart-release http://127.0.0.1:8100 12 control confirm + +# 走 Driver Feed 主线入口 +bash domain-api/deploy/multi-region/drive_ops_center.sh driver-feed http://127.0.0.1:8100 +bash domain-api/deploy/multi-region/drive_ops_center.sh driver-focus-preview http://127.0.0.1:8100 +bash domain-api/deploy/multi-region/drive_ops_center.sh driver-focus-run http://127.0.0.1:8100 +bash domain-api/deploy/multi-region/drive_ops_center.sh driver-focus-run http://127.0.0.1:8100 runbook:release_progression + +# 走 Driver Action 标准接口 +bash domain-api/deploy/multi-region/drive_ops_center.sh driver http://127.0.0.1:8100 open_release_dialog +bash domain-api/deploy/multi-region/drive_ops_center.sh driver http://127.0.0.1:8100 create_release_rollout_worker '{"release_id":12}' +``` + +从这版开始，Node Agent 这组命令的 condensed summary 也开始直接给出“下一步标准命令”： + +- `node-handover` + - 会附带： + - `recommended_commands.bootstrap_plan` + - `recommended_commands.delivery_queue` + - `recommended_commands.node_agent_check` + - `recommended_commands.ssh_hint` +- `node-bootstrap-plan` + - 会附带： + - env / service / bootstrap script 路径 + - install command block 是否已生成 + - bootstrap run script block 是否已生成 + - `repo_capability.supports_install_command_block / runtime_may_need_restart` + - 如果仓库代码已支持但接口返回仍旧，summary 会直接提示先执行 `runtime-refresh-recover` + - 回头看 handover / delivery queue 的推荐命令 +- `node-onboarding` + - 会附带： + - `onboarding_stage` + - `acceptance.status_code / execution_mode` + - 当前该继续 `bootstrap` 还是切到 `onboarding.acceptance` +- `queue-status` + - 会附带： + - `recommended_commands.records` + - `recommended_commands.flush` + - `recommended_commands.replay` +- `queue-records` + - 会附带： + - `recommended_commands.queue_status` + - `recommended_commands.replay` + +也就是说，海外主机现在不只是“能看到节点卡在哪”，而是看完摘要就能直接知道下一条标准命令该跑什么。 + +它的边界也刻意保持得很清楚： + +- 不直接 SSH 执行远端 shell +- 不自己拼多套业务逻辑 +- 只复用： + - 现有检查脚本 + - `drive_release_hub.sh` + - `drive_ops_action.sh` + - 后端标准 API + +这样后续无论是： + +- 海外主机人工执行 +- 海外 Codex 自动驾驶 +- 后台按钮封装 + +都能共用一套动作语义，而不是各写各的。 + +回执队列这条线现在也已经正式纳入这套统一入口： + +- 页面可以打开“回执队列治理”抽屉 +- driver recommendation 遇到 `dead_letter / retrying` 会优先打开标准动作模板 +- CLI 可以直接执行 `queue-status / queue-flush / queue-replay / queue-discard-record` + +当前这组命令的设计边界也要明确： + +- 所有动作都通过正式 `ops job` 下发 +- 当前单条动作只面向 `record_visibility=head_only` 的可见头部记录 +- CLI 不直接 SSH 到节点修改 `pending / dead-letter` 文件 + +如果希望海外控制面长期固定使用同一套默认地址和身份信息，推荐额外放一份配置文件： + +```bash +bash domain-api/deploy/multi-region/init_ops_center_config.sh \ + /etc/default/domaincheck-ops-center \ + http://121.204.244.188:8100 \ + http://152.53.37.118:8100 \ + https://api.example.com +``` + +然后按实际情况改： + +- `OPS_ACTIVE_PROFILE` +- `OPS_PROFILE_NAMES` +- `OPS_MAINLAND_API_BASE_URL` +- `OPS_OVERSEAS_API_BASE_URL` +- `OPS_DEFAULT_CONTROL_PLANE_BASE_URL` +- `OPS_DEFAULT_REQUESTED_BY` +- `OPS_DEFAULT_ISSUED_BY` +- `OPS_REPORT_ROOT` + +如果要按环境切换，也可以在同一份文件里追加 profile 覆盖，例如： + +- `OPS_PROFILE_PROD_MAINLAND_API_BASE_URL` +- `OPS_PROFILE_PROD_OVERSEAS_API_BASE_URL` +- `OPS_PROFILE_PROD_DEFAULT_CONTROL_PLANE_BASE_URL` + +然后执行： + +```bash +OPS_ACTIVE_PROFILE=prod bash domain-api/deploy/multi-region/drive_ops_center.sh config +``` + +配好以后，`drive_ops_center.sh / drive_release_hub.sh / drive_ops_action.sh` 都会自动读取它，不需要每次再手传 URL。 + +如果要把“发布打包链”也收进统一入口，不再额外记根目录脚本名，现在可以直接用： + +```bash +cd /opt/domaincheck/domain-api +bash domain-api/deploy/multi-region/drive_ops_center.sh release-package +bash domain-api/deploy/multi-region/drive_ops_center.sh release-verify +bash domain-api/deploy/multi-region/drive_ops_center.sh release-show +bash domain-api/deploy/multi-region/drive_ops_center.sh release-prepare +``` + +其中： + +- `release-package` + 在当前控制面主机本机生成新的 `release/latest_release.json` +- `release-verify` + 校验最新发布包与 `.sha256.txt` +- `release-show` + 查看当前最新发布包摘要 +- `release-launchpad` + 一次性查看“最新包 / 最新 Release / Worker 与 Control 预检 / 推荐下一步” +- `release-preview` + 仅预览“最新发布包如果导入为 Release，会是什么样” +- `release-preview-smart` + 仅预览“最新发布包如果走 Worker / Control 智能 Rollout，会命中哪些节点、是否有阻断或待确认” +- `release-prepare` + 打包并生成 `final_release_report.json` + +如果只是为了快速生成契约文件、不想等 smoke 全量检查，也可以临时执行： + +```bash +bash domain-api/deploy/multi-region/drive_ops_center.sh release-package skip-smoke +``` + +如果你准备真正发版，现在推荐先预览、再发布： + +```bash +bash domain-api/deploy/multi-region/drive_ops_center.sh release-preview +bash domain-api/deploy/multi-region/drive_ops_center.sh release-preview-smart worker +bash domain-api/deploy/multi-region/drive_ops_center.sh publish-latest worker +``` + +`publish-latest` 现在也会先自动打印一次 `preview-latest-smart` 的预检结果，再继续走正式智能 Rollout。 +如果只是想快速判断“现在是不是已经具备发版条件”，优先看 `release-launchpad` 就够了。 + +从这版开始，`bash domain-api/deploy/multi-region/drive_release_hub.sh launchpad` +输出的 condensed summary 也会直接附带： + +- `launchpad.recommended_action_code` +- `launchpad.recommended_execution_mode` +- `launchpad.recommended_execution_mode_label` +- `launchpad.focus_ref` +- `worker_rollout_preview.execution_mode` +- `control_rollout_preview.execution_mode` + +也就是说，海外机只看摘要就能直接知道： + +- 当前推荐先补什么动作 +- 这次默认应该走 `remote-agent`、`ssh` 还是别的执行模式 +- 焦点应该落在 `release_launchpad`、`default_rollout_gate` 还是后续 rollout + +而 `bash domain-api/deploy/multi-region/check_release_hub.sh` 现在在最后也会多打印一段 condensed summary， +把： + +- 当前 release +- launchpad 状态 +- worker/control preview 可用性 +- 在线 control/worker 节点 +- rollout 数量 + +统一压成一份简报，方便海外值班和 Codex 自动驾驶先读摘要，再决定是否深入看 raw 输出。 + +如果你想把当前海外控制面的现场快照直接导出成一份交接包，而不是盯着终端刷大段 JSON，可以执行： + +```bash +cd /opt/domaincheck/domain-api +bash domain-api/deploy/multi-region/drive_ops_center.sh doctor-export +``` + +导出命令会安静落盘，并最终返回： + +- `report_dir` +- `meta_path` +- `manifest_path` +- `readme_path` +- `summary` +- `decision` + +导出目录默认包含： + +- `00_ops_center_stack_summary.txt` +- `01_ops_plane.txt` +- `02_release_hub.txt` +- `03_inspection.txt` +- `04_participation.txt` +- `05_topology.txt` +- `06_contracts_registry.txt` +- `07_driver_feed.txt` +- `08_activity_stream.txt` +- `09_codex_brief.txt` +- `10_stack_next_preview.txt` +- `11_driver_focus_preview.txt` +- `12_activity_preview.txt` +- `13_codex_focus_preview.txt` +- `15_scene_node_log_.txt` +- `meta.json` +- `manifest.json` +- `README.txt` + +其中 `15_scene_node_log_.txt` 不是固定必有文件。 +只有当 `stack-diagnosis` 已经识别出明确的 `node_scene_log` 焦点时，doctor export 才会自动追加这些节点级现场日志报告。 + +这样交接包不只是告诉你“该去看哪台节点”，而是会顺手把这些重点节点的现场日志摘要一起带出来。 + +其中最值得先看的就是 `00_ops_center_stack_summary.txt`，因为它会直接给出： + +- `stack_status` +- `issues` +- `next_step` +- `quick_commands` + +也就是“当前先处理什么、为什么、下一条命令是什么”。 + +如果你后面准备让海外 Codex 或后台自动驾驶先读“机器摘要”再决定下一步，优先看： + +- `manifest.json` + +而且现在 `doctor-export` 的终端返回里也会直接附带精简版： + +- `summary.ok` +- `summary.required_failures` +- `summary.required_failure_details` +- `summary.optional_unavailable` +- `summary.optional_unavailable_details` +- `summary.scene_log_reports` +- `decision.status` +- `decision.headline` +- `decision.recommended_commands` + +也就是说，很多场景下你连打开文件都不用，直接看命令返回就知道当前该先修什么。 + +它会先把整包报告压成一份统一摘要，包括： + +- 哪些报告成功 +- 哪些 required report 失败 +- 哪些 optional preview 当前不可用 +- 当前现场一共暴露出哪些 contract key +- 哪些 preview 已经真正落到了 contract preview + +如果海外 Codex、未来后台自动驾驶、或者值班同事不想先翻整包文件，而是希望先拿一份“机器先判断后的结论”，现在可以直接执行： + +```bash +cd /opt/domaincheck/domain-api +bash domain-api/deploy/multi-region/drive_ops_center.sh doctor-decision +``` + +它会自动先生成一份临时 doctor export，再只回传精简后的： + +- `report_dir` +- `manifest_path` +- `summary` +- `decision` + +其中 `summary` 现在除了 surface 可用性，也会继续带出 launchpad 收口口径： + +- `launchpad_recommended_target_node_code` +- `launchpad_recommended_recovery_label` +- `launchpad_recommended_recovery_summary` +- `launchpad_onboarding_bootstrap_pending_nodes` +- `launchpad_onboarding_acceptance_ready_nodes` + +而 `decision.evidence` 也会同步附带这 5 个字段，保证海外 Codex、后台自动驾驶、值班同事 +在只消费 `doctor-decision` 的情况下，也能直接判断： + +- 当前该优先收哪台节点 +- 当前更像“还没 bootstrap 接入”还是“已经接入、等待 acceptance 收口” +- 是否还需要继续打开 `driver-feed / codex-brief / release-launchpad` 深挖 + +也可以直接对已经生成好的导出目录或 `manifest.json` 二次读取： + +```bash +cd /opt/domaincheck/domain-api +bash domain-api/deploy/multi-region/drive_ops_center.sh doctor-decision /tmp/domaincheck-doctor-export-XXXXXX +``` + +如果当前只想诊断 mainland 控制面，不想因为默认 overseas 地址不可达而拖慢整次分析，也可以显式关闭 overseas 拓扑： + +```bash +cd /opt/domaincheck/domain-api +bash domain-api/deploy/multi-region/drive_ops_center.sh doctor-decision http://127.0.0.1:8100 - +``` + +同样，`doctor-export` 和 `doctor` 也支持把最后一个 overseas 参数写成 `-`，表示本次跳过 `topology`。 + +另外，从这一版开始，控制面自己的 `/health` 和 `runtime/status` 也会直接带上运行指纹： + +- `build.source` +- `build.package_name` +- `build.generated_at` +- `build.commit_sha` +- `build.commit_ref` +- `build.manifest_path` +- `build.route_surface.surface_complete` +- `build.route_surface.missing_paths` + +这层不是为了展示漂亮，而是专门用来判断： + +- 当前运行中的 API 到底是不是最新部署内容 +- 当前实例到底有没有真正挂上 `ops/contracts / stack-diagnosis / driver-feed / codex-brief` +- 现在遇到的 404 到底是代码没写，还是服务实例没切到当前版本 + +这个入口的定位很明确： + +- `doctor-export` + - 面向“完整交接包 / 现场归档 / 人工深挖” +- `doctor-decision` + - 面向“先让海外 Codex / 按钮编排 / 自动驾驶做第一轮分流判断” + +现在 doctor 导出链本身也带了步骤级超时保护： + +- 单个子报告卡住时，不会把整包导出拖死 +- 报告文件仍会落盘 +- `manifest.json.summary.required_failures` / `optional_unavailable` 会把超时或失败记录进去 +- 这意味着海外 Codex 拿到的永远是“可判断的现场”，而不是“无结果的悬挂进程” + +从现在开始，`manifest.json` 里不再只有“哪个文件失败了”，还会补一层统一故障语义： + +- `summary.required_failure_details[].reason_code` +- `summary.required_failure_details[].label` +- `summary.required_failure_details[].detail` +- `summary.optional_unavailable_details[].reason_code` +- `reports[].failure.reason_code` +- `reports[].failure.timeout` + +当前固定口径如下： + +- `ok` + - 该报告命令执行成功，且接口状态与结构化结果正常 +- `command_timeout` + - 子命令被步骤级超时保护终止 +- `command_failed` + - 子命令本身退出失败 +- `http_unreachable` + - curl 没拿到有效 HTTP 状态，通常是超时、连接失败、网络不通 +- `http_error` + - 接口返回了明确的 `4xx / 5xx` +- `result_unparsed` + - 命令成功返回，但没有形成可消费的结构化 JSON +- `report_missing` + - 导出阶段甚至没有落盘该报告文件 + +这样后面无论是海外 Codex、后台自动驾驶，还是人工值班，都不需要再靠猜： + +- 是 API 真挂了 +- 还是 contract route 没挂齐 +- 还是 preview 本来就没有焦点 +- 还是单个 report 只是超时 + +都能直接从 `manifest.json` 读出来。 + +同时，`manifest.json.summary` 也不再只是 surface 成功失败统计，而是固定继续保留 launchpad +的接管缺口摘要： + +- `review_status_hint` +- `review_headline` +- `env_audit_status` +- `launchpad_recommended_target_node_code` +- `launchpad_recommended_recovery_label` +- `launchpad_recommended_recovery_summary` +- `launchpad_onboarding_bootstrap_pending_nodes` +- `launchpad_onboarding_acceptance_ready_nodes` +- `launchpad_alignment.consistent` +- `launchpad_alignment.target_node_code_consistent` +- `launchpad_alignment.bootstrap_pending_consistent` + +这样导出包一旦落盘，后续无论是海外 Codex 二次读取、后台自动驾驶回放，还是人工交班， +都不需要重新翻 `07_driver_feed.txt / 09_codex_brief.txt` 才知道“下一台该接谁”。 + +如果 `launchpad_alignment.consistent=false`，说明： + +- `go_live_summary` +- `stack_diagnosis` +- `driver_feed` +- `codex_brief` + +这几份摘要里至少有一层还在消费偏旧或偏离的运行态；此时即使文件都齐，也不应把 bundle 直接当成可交付终稿。 + +因此后续海外 Codex / 后台自动驾驶如果只读取 `manifest.json`，也应该优先先看： + +- `manifest.json.summary.review_status_hint` +- `manifest.json.summary.review_headline` +- `manifest.json.summary.launchpad_alignment` + +这里的 `required report 失败` 现在口径也固定了： + +- 不只是脚本 exit code 非 `0` +- 如果报告自身已经解析出了 `http_status`，但接口返回 `4xx / 5xx` +- 也会在 `manifest.json.summary.required_failures` 里记为失败 + +第二批最值得看的文件现在变成： + +- `06_contracts_registry.txt` + - 看当前控制面到底暴露了哪些 contract +- `07_driver_feed.txt` + - 看当前主驾驶主线和每条 entry 对应哪份 contract +- `08_activity_stream.txt` + - 看当前活动流、筛选口径和每条 activity 对应哪份 contract +- `10_stack_next_preview.txt` +- `11_driver_focus_preview.txt` +- `12_activity_preview.txt` +- `13_codex_focus_preview.txt` + - 看 preview 最终落到哪份 contract、哪个 endpoint、哪份 schema 文档 + +需要注意的是，后面这几份 preview 文件现在按“可选导出”执行： + +- 即使当前没有可预览焦点，也会保留报告文件 +- 文件末尾会带 `command_exit_code=1` +- 这不是导出失败，而是现场当时没有对应 focus / entry +- `manifest.json.summary.optional_unavailable` 里也会同步记录这些不可用预览 + +如果要从海外控制面直接为一台新节点生成 Node Agent 接入方案，现在也可以走统一入口： + +```bash +cd /opt/domaincheck/domain-api +bash domain-api/deploy/multi-region/drive_ops_center.sh agent-plan \ + http://127.0.0.1:8100 \ + mainland-worker-02 \ + mainland \ + worker \ + https://api.example.com \ + /opt/domaincheck +``` + +如果希望直接把接管计划导出到文件，留给异地同事执行或后续自动化系统消费，也可以执行： + +```bash +cd /opt/domaincheck/domain-api +bash domain-api/deploy/multi-region/drive_ops_center.sh agent-plan-export \ + /opt/domaincheck/domain-api/runtime/ops-center-reports/agent-plans \ + http://127.0.0.1:8100 \ + mainland-worker-02 \ + mainland \ + worker \ + https://api.example.com \ + /opt/domaincheck +``` + +它会返回： + +- `output_path` +- `meta_path` + +如果只是想在当前机器上快速检查本机 Node Agent 是否已安装、环境文件是否存在，也可以直接执行： + +```bash +cd /opt/domaincheck/domain-api +bash domain-api/deploy/multi-region/drive_ops_center.sh agent-check +``` + +这条脚本会重点输出： + +- `inspection overview` +- `condensed summary` +- `priority queue` +- `attention only` +- `participating problems only` + 如果当前还无法真正部署到大陆机器，也可以先在国外测试机上做“单机模拟多节点联调”： ```bash cd /opt/domaincheck/domain-api -bash deploy/multi-region/simulate_multi_region.sh http://127.0.0.1:8100 +bash domain-api/deploy/multi-region/simulate_multi_region.sh http://127.0.0.1:8100 ``` 这条命令会临时模拟： @@ -270,14 +1758,14 @@ bash deploy/multi-region/simulate_multi_region.sh http://127.0.0.1:8100 ```bash cd /opt/domaincheck/domain-api -bash deploy/multi-region/prune_cluster_nodes.sh --minutes 30 --dry-run -bash deploy/multi-region/prune_cluster_nodes.sh --minutes 30 +bash domain-api/deploy/multi-region/prune_cluster_nodes.sh --minutes 30 --dry-run +bash domain-api/deploy/multi-region/prune_cluster_nodes.sh --minutes 30 ``` 如果只想清理某个确定已经废弃的节点： ```bash -bash deploy/multi-region/prune_cluster_nodes.sh --node-code mainland-worker-01 +bash domain-api/deploy/multi-region/prune_cluster_nodes.sh --node-code mainland-worker-01 ``` 如果要判断大陆 Worker 是否已经真正接入，不要只看 `systemctl`，还要看： @@ -303,7 +1791,7 @@ bash deploy/multi-region/prune_cluster_nodes.sh --node-code mainland-worker-01 ```bash cd /opt/domaincheck/domain-api -bash deploy/multi-region/bootstrap_mainland.sh /opt/domaincheck worker +bash domain-api/deploy/multi-region/bootstrap_mainland.sh /opt/domaincheck worker ``` 5. 启动后在国外控制面检查： @@ -339,9 +1827,94 @@ curl http://127.0.0.1:8100/api/v1/detect/jobs?limit=5 ```bash cd /opt/domaincheck/domain-api -bash deploy/multi-region/check_cluster.sh http://127.0.0.1:8100 +bash domain-api/deploy/multi-region/check_cluster.sh http://127.0.0.1:8100 ``` +如需直接拿到“控制面 / 执行现场 / 远端日志回传 / 同步 / 下一步动作”的统一快照，可以执行： + +```bash +cd /opt/domaincheck/domain-api +bash domain-api/deploy/multi-region/check_ops_link_snapshot.sh http://127.0.0.1:8100 +``` + +如果要把“协议版本 / 控制面驾驶建议 / 节点托管状态 / ReleaseHub 门禁 / 最近 playbook run / 最近活动流”一次性收口成海外单入口总检，可以直接执行： + +```bash +cd /opt/domaincheck/domain-api +bash domain-api/deploy/multi-region/check_ops_center_stack.sh http://127.0.0.1:8100 + +# 如果只想看最终收口，不想展开全量 JSON +bash domain-api/deploy/multi-region/check_ops_center_stack.sh http://127.0.0.1:8100 summary +``` + +这条总检入口的定位不是替代各类细分检查脚本，而是给海外控制面提供一个固定起手式： + +1. 先确认 API 与 `ops/contracts` 已加载的是哪一版 contract +2. 再确认 `link-snapshot / overview` 当前推荐你先处理什么 +3. 再看托管节点、Node Agent、回执队列是否健康 +4. 最后确认 ReleaseHub 当前能不能进入正式 Rollout + +当总检发现某一层异常时，再继续下钻： + +- 协议注册表异常：`check_ops_contracts.sh` +- 控制面驾驶面异常：`check_ops_plane.sh` +- 节点托管 / 回执队列异常：`check_node_agent.sh` +- 发布门禁 / Launchpad 异常：`check_release_hub.sh` + +`check_ops_center_stack.sh` 现在在最终 `condensed stack summary` 中还会额外产出统一诊断层： + +- `diagnosis.surface_status` + 当前总检表面层是否完整联通，典型值为 `healthy / partial / broken` +- `diagnosis.automation_status` + 当前海外单脑自动化执行面是否真正可接管，典型值为 `ready / attention / blocked` +- `diagnosis.stack_status` + 面向人和 Codex 的最终收口结论 +- `diagnosis.issues` + 已排序的问题清单，附带 `code / severity / layer / commands` +- `diagnosis.next_step` + 默认下一步动作码、来源和推荐命令 +- `diagnosis.quick_commands` + 一组可以直接复制执行的收口命令 + +这意味着后续无论是： + +- 海外 Codex 驾驶员 +- 海外控制面按钮 +- 人工终端排障 + +都可以先消费同一份 `diagnosis`，而不是再先人工判断“这次先跑哪个脚本”。 + +如果当前后端已经升级到新 contract，`check_ops_center_stack.sh` 会优先直接调用： + +- `GET /api/v1/ops/stack-diagnosis` + +只有当这个 endpoint 还未上线时，才会自动回退到本地聚合模式。 + +而 `drive_ops_center.sh stack-next` 不会回退到旧逻辑。 + +原因是它的职责不是“猜一个差不多的下一步”，而是严格消费 `diagnosis.next_step` 这份正式 contract。 + +所以如果你执行： + +```bash +bash domain-api/deploy/multi-region/drive_ops_center.sh stack-next +``` + +返回的是 `404 Not Found`，应直接理解为： + +- 当前运行中的控制面 API 版本还没升级到支持 `GET /api/v1/ops/stack-diagnosis` +- 或代码已同步，但 `domaincheck-api` 还没重启 + +此时优先执行： + +```bash +git status +bash domain-api/deploy/multi-region/drive_ops_center.sh runtime-refresh-recover http://127.0.0.1:8100 +curl -sS http://127.0.0.1:8100/api/v1/ops/stack-diagnosis +``` + +确认 endpoint 已存在后，再继续用 `stack-next`。 + 观察重点： - 控制面节点心跳应持续刷新，而不是只在 API 启动时更新一次 @@ -380,6 +1953,67 @@ systemctl status domaincheck-sync-agent --no-pager -l - `runtime_push` - `runtime_ingest` +## 七点五、上线前统一总检 + +如果当前已经进入“海外单脑控制面 + Node Agent + ReleaseHub”这条正式链路，建议在发版或切正式流量前固定先跑： + +```bash +bash domain-api/deploy/multi-region/drive_ops_center.sh go-live-check http://127.0.0.1:8100 http://152.53.37.118:8100 summary +``` + +对应的正式 API 汇总入口是： + +```bash +curl -s "http://127.0.0.1:8100/api/v1/ops/go-live-summary?base_url=http://127.0.0.1:8100" +``` + +CLI 脚本会优先读取这条 API；如果当前运行中的服务还没重启到最新版本，才会自动回退到本地聚合逻辑。 + +它会统一收口这些层面： + +- `stack-diagnosis` +- `contracts` +- `runtime-build-info / route_surface` +- `release-launchpad` +- `managed nodes / remote-agent ready` +- `overview / execution_scene / log_sync` +- 本机 `check_node_agent.sh` 探针 + +返回里至少会给出： + +- `go_live_status` +- `blocking_reasons` +- `warnings` +- `launchpad_recommended_action_code` +- `launchpad_recommended_target_node_code` +- `launchpad_recommended_recovery_label` +- `launchpad_recommended_recovery_summary` +- `launchpad_onboarding_bootstrap_pending_nodes` +- `launchpad_onboarding_acceptance_ready_nodes` +- `next_step_action_code` +- `operator_lane` +- `operator_title` +- `log_sync_state` +- `log_sync_covered_nodes` +- `recommended_commands` + +现在推荐这样理解： + +- `blocking_reasons` + 只看真正阻断上线的项 +- `warnings` + 代表还能继续联调，但当前还不建议直接宣称“全部收口” +- `launchpad_recommended_target_node_code` / `launchpad_recommended_recovery_label` + 代表 launchpad 已经明确告诉你“该收哪台节点、优先补哪一步” +- `launchpad_onboarding_bootstrap_pending_nodes` / `launchpad_onboarding_acceptance_ready_nodes` + 用来快速判断当前是“还没接入”还是“已经接入但还没跑完验收” +- `recommended_commands.next_step` + 就是当前最值得先跑的一条命令 +- `recommended_commands.log_sync_logs` / `recommended_commands.log_sync_inspection` + 用来补现场日志样本与参与节点观测闭环 + +建议把它当成正式上线前的固定起手式，而不是继续手动拼一组零散检查命令。 + ## 八、后续演进 后续会继续补： diff --git a/domain-api/deploy/multi-region/bootstrap_mainland.sh b/domain-api/deploy/multi-region/bootstrap_mainland.sh index c6bdc3b..0188bda 100755 --- a/domain-api/deploy/multi-region/bootstrap_mainland.sh +++ b/domain-api/deploy/multi-region/bootstrap_mainland.sh @@ -12,7 +12,7 @@ TEMPLATE_DIR="$API_DIR/deploy/multi-region/templates" if [ "$ROLE" != "controller" ] && [ "$ROLE" != "worker" ]; then echo "invalid role: $ROLE" - echo "usage: bash deploy/multi-region/bootstrap_mainland.sh /opt/domaincheck [controller|worker]" + echo "usage: bash domain-api/deploy/multi-region/bootstrap_mainland.sh /opt/domaincheck [controller|worker]" exit 1 fi diff --git a/domain-api/deploy/multi-region/bootstrap_node_agent.sh b/domain-api/deploy/multi-region/bootstrap_node_agent.sh new file mode 100755 index 0000000..551c3bc --- /dev/null +++ b/domain-api/deploy/multi-region/bootstrap_node_agent.sh @@ -0,0 +1,61 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR="${1:-/opt/domaincheck}" +ENV_FILE="/etc/default/domaincheck-node-agent" +SERVICE_FILE="/etc/systemd/system/domaincheck-node-agent.service" +SERVICE_NAME="domaincheck-node-agent" + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" + +resolve_project_root() { + local candidate + for candidate in \ + "${ROOT_DIR}/domainCheck" \ + "${ROOT_DIR}/domain-api" \ + "${ROOT_DIR}" \ + "$(cd "${SCRIPT_DIR}/../.." && pwd)"; do + if [[ -f "${candidate}/deploy/systemd/domain-node-agent.service" && -f "${candidate}/deploy/multi-region/templates/domaincheck-node-agent.env.example" ]]; then + printf '%s' "${candidate}" + return 0 + fi + done + return 1 +} + +PROJECT_ROOT="$(resolve_project_root || true)" +if [[ -z "${PROJECT_ROOT}" ]]; then + echo "unable to resolve project root under ${ROOT_DIR} or ${SCRIPT_DIR}" >&2 + exit 1 +fi + +TEMPLATE_FILE="${PROJECT_ROOT}/deploy/multi-region/templates/domaincheck-node-agent.env.example" +UNIT_TEMPLATE="${PROJECT_ROOT}/deploy/systemd/domain-node-agent.service" + +echo "[1/5] install systemd unit" +install -m 0644 "${UNIT_TEMPLATE}" "${SERVICE_FILE}" + +echo "[2/5] install env template if absent" +if [[ ! -f "${ENV_FILE}" ]]; then + install -m 0644 "${TEMPLATE_FILE}" "${ENV_FILE}" + echo "created ${ENV_FILE}" +else + echo "keep existing ${ENV_FILE}" +fi + +echo "[3/5] reload systemd" +systemctl daemon-reload + +echo "[4/5] enable service" +systemctl enable "${SERVICE_NAME}" >/dev/null + +echo "[5/5] next steps" +echo "project_root=${PROJECT_ROOT}" +echo "edit ${ENV_FILE} and set:" +echo " OPS_CONTROL_PLANE_BASE_URL" +echo " OPS_AGENT_TOKEN" +echo " NODE_CODE / NODE_REGION / NODE_ROLE" +echo +echo "then run:" +echo " systemctl restart ${SERVICE_NAME}" +echo " systemctl status ${SERVICE_NAME} --no-pager -l" diff --git a/domain-api/deploy/multi-region/build_node_agent_bootstrap_plan.sh b/domain-api/deploy/multi-region/build_node_agent_bootstrap_plan.sh new file mode 100755 index 0000000..6882142 --- /dev/null +++ b/domain-api/deploy/multi-region/build_node_agent_bootstrap_plan.sh @@ -0,0 +1,91 @@ +#!/usr/bin/env bash +set -euo pipefail + +API_BASE_URL="${1:-http://127.0.0.1:8100}" +NODE_CODE="${2:-}" +NODE_REGION="${3:-mainland}" +NODE_ROLE="${4:-worker}" +CONTROL_PLANE_BASE_URL="${5:-${API_BASE_URL}}" +ROOT_DIR="${6:-/opt/domaincheck}" +ISSUED_BY="${ISSUED_BY:-cli-bootstrap}" +EXPIRES_IN_HOURS="${EXPIRES_IN_HOURS:-72}" + +if [[ -z "${NODE_CODE}" ]]; then + echo "usage: $0 [node_region] [node_role] [control_plane_base_url] [root_dir]" >&2 + exit 1 +fi + +NORMALIZED_API_BASE_URL="${API_BASE_URL%/}" +NORMALIZED_CONTROL_PLANE_BASE_URL="${CONTROL_PLANE_BASE_URL%/}" +if [[ "${NORMALIZED_API_BASE_URL}" != */api/v1 ]]; then + API_ENDPOINT="${NORMALIZED_API_BASE_URL}/api/v1/ops/agent/bootstrap-plan" +else + API_ENDPOINT="${NORMALIZED_API_BASE_URL}/ops/agent/bootstrap-plan" +fi + +TMP_RESPONSE_FILE="$(mktemp)" +cleanup() { + rm -f "${TMP_RESPONSE_FILE}" +} +trap cleanup EXIT + +python3 - "${NODE_CODE}" "${NODE_REGION}" "${NODE_ROLE}" "${ISSUED_BY}" "${EXPIRES_IN_HOURS}" "${NORMALIZED_CONTROL_PLANE_BASE_URL}" "${ROOT_DIR}" >"${TMP_RESPONSE_FILE}.payload" <<'PY' +import json +import sys + +payload = { + "node_code": sys.argv[1], + "node_region": sys.argv[2], + "node_role": sys.argv[3], + "issued_by": sys.argv[4], + "expires_in_hours": int(sys.argv[5]), + "control_plane_base_url": sys.argv[6], + "root_dir": sys.argv[7], + "metadata": { + "issued_from": "build_node_agent_bootstrap_plan.sh", + }, +} +print(json.dumps(payload, ensure_ascii=False)) +PY + +curl -fsS -X POST \ + -H 'Content-Type: application/json' \ + --data-binary @"${TMP_RESPONSE_FILE}.payload" \ + "${API_ENDPOINT}" >"${TMP_RESPONSE_FILE}" + +python3 - "${TMP_RESPONSE_FILE}" <<'PY' +import json +import sys +from pathlib import Path + +response = json.loads(Path(sys.argv[1]).read_text(encoding="utf-8")) +response_code = response.get("code", 0) +if int(response_code if response_code is not None else 0) != 0: + raise SystemExit(response.get("message") or "bootstrap plan request failed") + +data = response.get("data") or {} +plan = data.get("bootstrap_plan") or {} + +print("[token]") +print(f"node_code={data.get('node_code', '')}") +print(f"token_preview={data.get('token_preview', '')}") +print(f"expires_at={data.get('expires_at', '')}") +print() +print("[env]") +print(plan.get("env_content") or "") +print() +print("[commands]") +print(plan.get("command_block") or "") +print() +print("[bootstrap-script]") +print(plan.get("bootstrap_script_content") or "") +print() +print("[bootstrap-run-block]") +print(plan.get("bootstrap_run_script_block") or "") +print() +print("[health-checks]") +for item in plan.get("health_checks") or []: + print(item) +PY + +rm -f "${TMP_RESPONSE_FILE}.payload" diff --git a/domain-api/deploy/multi-region/check_env_drift.sh b/domain-api/deploy/multi-region/check_env_drift.sh new file mode 100644 index 0000000..0a951c6 --- /dev/null +++ b/domain-api/deploy/multi-region/check_env_drift.sh @@ -0,0 +1,345 @@ +#!/usr/bin/env bash +set -euo pipefail + +BASE_URL="${1:-http://127.0.0.1:8100}" +API_SERVICE="${API_SERVICE:-domaincheck-api}" +WORKER_SERVICE="${WORKER_SERVICE:-domaincheck-worker}" +SYNC_AGENT_SERVICE="${SYNC_AGENT_SERVICE:-domaincheck-sync-agent}" +NODE_AGENT_SERVICE="${NODE_AGENT_SERVICE:-domaincheck-node-agent}" +API_ENV_FILE="${API_ENV_FILE:-/etc/default/domaincheck-api}" +WORKER_ENV_FILE="${WORKER_ENV_FILE:-/etc/default/domaincheck-worker}" +NODE_AGENT_ENV_FILE="${NODE_AGENT_ENV_FILE:-/etc/default/domaincheck-node-agent}" +PROJECT_ROOT="$(cd "$(dirname "${BASH_SOURCE[0]}")/../../.." && pwd)" +API_DIR="${PROJECT_ROOT}/domain-api" +WEB_DIR="${PROJECT_ROOT}/domain-web" +PYTHON_BIN="${PYTHON_BIN:-python3}" + +FETCH_BODY="" +FETCH_STATUS="000" + +fetch_json() { + local url="${1:-}" + local raw_text + raw_text="$(curl -sS \ + --connect-timeout "${CURL_CONNECT_TIMEOUT:-3}" \ + --max-time "${CURL_MAX_TIME:-10}" \ + "${url}" \ + -w $'\nHTTP_STATUS=%{http_code}' || true)" + FETCH_BODY="${raw_text%$'\n'HTTP_STATUS=*}" + FETCH_STATUS="${raw_text##*$'\n'HTTP_STATUS=}" +} + +echo "[1/8] local command availability" +for cmd in bash curl systemctl "${PYTHON_BIN}" node npm; do + if command -v "${cmd}" >/dev/null 2>&1; then + printf '%s: ok -> %s\n' "${cmd}" "$(command -v "${cmd}")" + else + printf '%s: missing\n' "${cmd}" + fi +done +echo + +echo "[2/8] python environment" +"${PYTHON_BIN}" - <<'PY' "${API_DIR}" "${WEB_DIR}" +import importlib.util +import json +import pathlib +import sys + +api_dir = pathlib.Path(sys.argv[1]) +web_dir = pathlib.Path(sys.argv[2]) + +def has_module(name: str) -> bool: + return importlib.util.find_spec(name) is not None + +payload = { + "python_executable": sys.executable, + "python_version": sys.version.split()[0], + "pytest_installed": has_module("pytest"), + "uvicorn_installed": has_module("uvicorn"), + "fastapi_installed": has_module("fastapi"), + "api_requirements_exists": (api_dir / "requirements.txt").exists(), + "web_package_json_exists": (web_dir / "package.json").exists(), +} +print(json.dumps(payload, ensure_ascii=False, indent=2)) +PY +echo + +echo "[3/8] env files" +for env_file in "${API_ENV_FILE}" "${WORKER_ENV_FILE}" "${NODE_AGENT_ENV_FILE}"; do + if [[ -f "${env_file}" ]]; then + echo "exists: ${env_file}" + else + echo "missing: ${env_file}" + fi +done +echo + +echo "[4/8] systemd service states" +for service in "${API_SERVICE}" "${WORKER_SERVICE}" "${SYNC_AGENT_SERVICE}" "${NODE_AGENT_SERVICE}"; do + if systemctl list-unit-files "${service}.service" --no-pager >/dev/null 2>&1; then + printf '%s: enabled=%s active=%s\n' \ + "${service}" \ + "$(systemctl is-enabled "${service}" 2>/dev/null || true)" \ + "$(systemctl is-active "${service}" 2>/dev/null || true)" + else + printf '%s: not-installed\n' "${service}" + fi +done +echo + +echo "[5/8] runtime preflight" +fetch_json "${BASE_URL}/api/v1/runtime/preflight" +printf 'status=%s\n%s\n\n' "${FETCH_STATUS}" "${FETCH_BODY}" + +echo "[6/8] runtime readiness" +fetch_json "${BASE_URL}/api/v1/runtime/readiness" +printf 'status=%s\n%s\n\n' "${FETCH_STATUS}" "${FETCH_BODY}" + +echo "[7/8] runtime build-info" +fetch_json "${BASE_URL}/api/v1/runtime/build-info" +printf 'status=%s\n%s\n\n' "${FETCH_STATUS}" "${FETCH_BODY}" + +echo "[8/8] condensed env audit summary" +"${PYTHON_BIN}" - <<'PY' \ + "${BASE_URL}" \ + "${API_SERVICE}" \ + "${WORKER_SERVICE}" \ + "${SYNC_AGENT_SERVICE}" \ + "${NODE_AGENT_SERVICE}" \ + "${API_ENV_FILE}" \ + "${WORKER_ENV_FILE}" \ + "${NODE_AGENT_ENV_FILE}" \ + "${PROJECT_ROOT}" +import importlib.util +import json +import pathlib +import subprocess +import sys +import urllib.request +from typing import Dict, List + +base_url, api_service, worker_service, sync_agent_service, node_agent_service, api_env, worker_env, node_agent_env, project_root = sys.argv[1:10] +project_root_path = pathlib.Path(project_root) + +def has_module(name: str) -> bool: + return importlib.util.find_spec(name) is not None + +def inspect_python_runtime(path: str) -> Dict[str, object]: + runtime_path = pathlib.Path(path) + if not runtime_path.exists(): + return {} + probe = r""" +import importlib.util +import json +import sys + +mods = ["pytest", "fastapi", "uvicorn"] +print(json.dumps({ + "python_executable": sys.executable, + "python_version": sys.version.split()[0], + "modules": {name: importlib.util.find_spec(name) is not None for name in mods}, +}, ensure_ascii=False)) +""" + try: + raw = subprocess.check_output([str(runtime_path), "-c", probe], text=True, stderr=subprocess.DEVNULL) + except Exception: + return {} + try: + payload = json.loads(raw) + except Exception: + return {} + payload["path"] = str(runtime_path) + return payload + +def systemd_state(service: str) -> dict: + try: + enabled = subprocess.check_output( + ["systemctl", "is-enabled", service], + text=True, + stderr=subprocess.DEVNULL, + ).strip() + except Exception: + enabled = "" + try: + active = subprocess.check_output( + ["systemctl", "is-active", service], + text=True, + stderr=subprocess.DEVNULL, + ).strip() + except Exception: + active = "" + return {"enabled": enabled, "active": active} + +def fetch_json(url: str) -> tuple[str, dict]: + try: + with urllib.request.urlopen(url, timeout=10) as resp: + status = str(resp.status) + raw = resp.read().decode("utf-8", errors="replace") + except Exception: + return "000", {} + try: + payload = json.loads(raw) + except Exception: + return status, {} + return status, payload.get("data") if isinstance(payload, dict) else {} + +def read_git_value(*args: str) -> str: + try: + result = subprocess.check_output( + ["git", "-C", str(project_root_path), *args], + text=True, + stderr=subprocess.DEVNULL, + ).strip() + except Exception: + return "" + return result + +def read_text(path: pathlib.Path) -> str: + try: + return path.read_text(encoding="utf-8", errors="replace") + except Exception: + return "" + +local_commit_sha = read_git_value("rev-parse", "--short=12", "HEAD") +local_commit_ref = read_git_value("rev-parse", "--abbrev-ref", "HEAD") +ops_agent_service_text = read_text(project_root_path / "domain-api" / "app" / "services" / "ops_agent_service.py") +bootstrap_node_agent_text = read_text(project_root_path / "domain-api" / "deploy" / "multi-region" / "bootstrap_node_agent.sh") + +repo_supports_install_command_block = ( + "install_command_block" in ops_agent_service_text + and "_candidate_node_agent_install_paths" in ops_agent_service_text +) +repo_supports_multi_layout_bootstrap = "resolve_project_root" in bootstrap_node_agent_text + +preflight_status, preflight = fetch_json(f"{base_url}/api/v1/runtime/preflight") +readiness_status, readiness = fetch_json(f"{base_url}/api/v1/runtime/readiness") +build_status, build = fetch_json(f"{base_url}/api/v1/runtime/build-info") +route_surface = build.get("route_surface") if isinstance(build, dict) else {} + +missing = [] +tooling_items: List[str] = [] +for env_file in [api_env, worker_env, node_agent_env]: + if not pathlib.Path(env_file).exists(): + missing.append(f"missing_env:{env_file}") + +python_candidates: List[str] = [] +seen_candidates = set() +for candidate in [ + "/opt/domaincheck/domainCheck/.venv/bin/python", + str(pathlib.Path.cwd() / ".venv" / "bin" / "python"), + str(pathlib.Path.cwd() / "domain-api" / ".venv" / "bin" / "python"), + pathlib.Path(sys.executable).as_posix(), +]: + if candidate and candidate not in seen_candidates: + seen_candidates.add(candidate) + python_candidates.append(candidate) + +python_runtimes = [item for item in (inspect_python_runtime(candidate) for candidate in python_candidates) if item] + +def runtime_has_module(module_name: str) -> bool: + for runtime in python_runtimes: + modules = runtime.get("modules") or {} + if bool(modules.get(module_name)): + return True + return False + +if not runtime_has_module("fastapi"): + missing.append("python_module:fastapi") +if not runtime_has_module("uvicorn"): + missing.append("python_module:uvicorn") +if not runtime_has_module("pytest"): + tooling_items.append("python_module:pytest") + +runtime_commit_sha = str(build.get("commit_sha") or "").strip() if isinstance(build, dict) else "" +runtime_commit_ref = str(build.get("commit_ref") or "").strip() if isinstance(build, dict) else "" +runtime_route_missing_keys = [str(item).strip() for item in list((route_surface or {}).get("missing_keys") or []) if str(item).strip()] +runtime_expected_paths = { + str(key).strip(): str(value).strip() + for key, value in dict((route_surface or {}).get("expected_paths") or {}).items() + if str(key).strip() +} +runtime_bootstrap_plan_route_missing = "ops_node_handover_bootstrap_plan" in runtime_route_missing_keys +runtime_declares_bootstrap_plan_route = "ops_node_handover_bootstrap_plan" in runtime_expected_paths +runtime_repo_commit_drift = bool(local_commit_sha and runtime_commit_sha and local_commit_sha != runtime_commit_sha) +runtime_repo_capability_drift = bool( + repo_supports_install_command_block + and (runtime_bootstrap_plan_route_missing or not runtime_declares_bootstrap_plan_route) +) + +recommended_actions: List[str] = [] +if runtime_repo_capability_drift: + recommended_actions.append("systemctl restart domaincheck-api") + recommended_actions.append("bash domain-api/deploy/multi-region/drive_ops_center.sh stack-diagnosis http://127.0.0.1:8100 summary") + recommended_actions.append("bash domain-api/deploy/multi-region/drive_ops_center.sh node-bootstrap-plan http://127.0.0.1:8100 mainland-worker-01") +elif runtime_repo_commit_drift: + recommended_actions.append("systemctl restart domaincheck-api") + recommended_actions.append("bash domain-api/deploy/multi-region/drive_ops_center.sh doctor http://127.0.0.1:8100") + +payload = { + "status": "ready", + "headline": "当前环境没有发现新的基础阻断，可继续做上线复核。", + "repo": { + "project_root": str(project_root_path), + "commit_sha": local_commit_sha, + "commit_ref": local_commit_ref, + "supports_install_command_block": repo_supports_install_command_block, + "supports_multi_layout_bootstrap": repo_supports_multi_layout_bootstrap, + }, + "python": { + "pytest_installed": runtime_has_module("pytest"), + "fastapi_installed": runtime_has_module("fastapi"), + "uvicorn_installed": runtime_has_module("uvicorn"), + "runtimes": python_runtimes, + }, + "services": { + api_service: systemd_state(api_service), + worker_service: systemd_state(worker_service), + sync_agent_service: systemd_state(sync_agent_service), + node_agent_service: systemd_state(node_agent_service), + }, + "runtime": { + "preflight_status_code": preflight_status, + "preflight_ok": bool((preflight or {}).get("ok", False)), + "readiness_status_code": readiness_status, + "readiness_status": str((readiness or {}).get("status") or ""), + "readiness_summary": str((readiness or {}).get("summary") or ""), + "build_status_code": build_status, + "build_commit_sha": runtime_commit_sha, + "build_commit_ref": runtime_commit_ref, + "route_surface_complete": bool((route_surface or {}).get("surface_complete", False)), + "route_surface_missing_keys": runtime_route_missing_keys, + "route_surface_declares_bootstrap_plan": runtime_declares_bootstrap_plan_route, + "repo_commit_drift": runtime_repo_commit_drift, + "repo_capability_drift": runtime_repo_capability_drift, + "runtime_may_need_restart": runtime_repo_commit_drift or runtime_repo_capability_drift, + }, + "missing_items": missing, + "tooling_items": tooling_items, + "recommended_actions": recommended_actions, +} + +if not bool((preflight or {}).get("ok", False)): + payload["status"] = "blocked" + payload["headline"] = "runtime/preflight 未通过，先修环境与依赖，再继续上线复核。" +elif str((readiness or {}).get("status") or "") == "blocking": + payload["status"] = "blocked" + payload["headline"] = "runtime/readiness 当前为 blocking，现场仍不适合进入正式发布。" +elif runtime_repo_capability_drift: + payload["status"] = "attention" + payload["headline"] = "本地仓库已经具备新的 Node Agent 接管能力，但运行中的 API 路由面仍旧，当前更像是服务未重启到最新代码。" +elif runtime_repo_commit_drift: + payload["status"] = "attention" + payload["headline"] = "本地仓库 commit 与运行中 API build-info 不一致，建议先重启或重发当前版本，再继续上线复核。" +elif not bool((route_surface or {}).get("surface_complete", False)): + payload["status"] = "attention" + payload["headline"] = "运行中 API 路由面还不完整，建议先确认当前实例是否已经吃到最新版本。" +elif missing: + payload["status"] = "attention" + payload["headline"] = "基础环境存在缺口，虽然未必阻断运行，但建议先补齐后再交付。" +elif tooling_items: + payload["status"] = "ready" + payload["headline"] = "运行环境可继续上线复核，但本机工具链仍有缺口，建议后续补齐。" + +print(json.dumps(payload, ensure_ascii=False, indent=2)) +PY diff --git a/domain-api/deploy/multi-region/check_go_live.sh b/domain-api/deploy/multi-region/check_go_live.sh new file mode 100644 index 0000000..087b77b --- /dev/null +++ b/domain-api/deploy/multi-region/check_go_live.sh @@ -0,0 +1,423 @@ +#!/usr/bin/env bash +set -euo pipefail + +BASE_URL="${1:-http://127.0.0.1:8100}" +OVERSEAS_BASE_URL="${2:-}" +VIEW_MODE="${3:-full}" +CURL_CONNECT_TIMEOUT="${CURL_CONNECT_TIMEOUT:-3}" +CURL_MAX_TIME="${CURL_MAX_TIME:-20}" +LOCAL_NODE_AGENT_SERVICE_NAME="${LOCAL_NODE_AGENT_SERVICE_NAME:-domaincheck-node-agent}" +LOCAL_NODE_AGENT_ENV_FILE="${LOCAL_NODE_AGENT_ENV_FILE:-/etc/default/domaincheck-node-agent}" + +FETCH_BODY="" +FETCH_STATUS="000" + +fetch_json() { + local url="${1:-}" + local raw_text + raw_text="$(curl -sS \ + --connect-timeout "${CURL_CONNECT_TIMEOUT}" \ + --max-time "${CURL_MAX_TIME}" \ + "${url}" \ + -w $'\nHTTP_STATUS=%{http_code}' || true)" + FETCH_BODY="${raw_text%$'\n'HTTP_STATUS=*}" + FETCH_STATUS="${raw_text##*$'\n'HTTP_STATUS=}" +} + +run_local_node_agent_probe() { + SERVICE_NAME="${LOCAL_NODE_AGENT_SERVICE_NAME}" \ + ENV_FILE="${LOCAL_NODE_AGENT_ENV_FILE}" \ + bash "$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)/check_node_agent.sh" +} + +fetch_json "${BASE_URL}/api/v1/ops/stack-diagnosis?base_url=${BASE_URL}" +STACK_JSON="${FETCH_BODY}" +STACK_STATUS="${FETCH_STATUS}" + +fetch_json "${BASE_URL}/api/v1/ops/go-live-summary?base_url=${BASE_URL}" +GO_LIVE_API_JSON="${FETCH_BODY}" +GO_LIVE_API_STATUS="${FETCH_STATUS}" + +if [[ "${GO_LIVE_API_STATUS}" == 2* ]]; then + if [[ "${VIEW_MODE}" != "summary" ]]; then + echo "[1/1] go-live summary (api)" + fi + printf '%s\n' "${GO_LIVE_API_JSON}" + exit 0 +fi + +fetch_json "${BASE_URL}/api/v1/ops/contracts" +CONTRACTS_JSON="${FETCH_BODY}" +CONTRACTS_STATUS="${FETCH_STATUS}" + +fetch_json "${BASE_URL}/api/v1/ops/releases/launchpad" +LAUNCHPAD_JSON="${FETCH_BODY}" +LAUNCHPAD_STATUS="${FETCH_STATUS}" + +fetch_json "${BASE_URL}/api/v1/runtime/build-info" +BUILD_INFO_JSON="${FETCH_BODY}" +BUILD_INFO_STATUS="${FETCH_STATUS}" + +fetch_json "${BASE_URL}/api/v1/ops/nodes" +NODES_JSON="${FETCH_BODY}" +NODES_STATUS="${FETCH_STATUS}" + +fetch_json "${BASE_URL}/api/v1/ops/overview" +OVERVIEW_JSON="${FETCH_BODY}" +OVERVIEW_STATUS="${FETCH_STATUS}" + +LOCAL_NODE_AGENT_RAW="$(run_local_node_agent_probe 2>&1 || true)" + +if [[ "${VIEW_MODE}" != "summary" ]]; then + echo "[1/7] ops stack diagnosis" + printf 'status=%s\n%s\n\n' "${STACK_STATUS}" "${STACK_JSON}" + + echo "[2/7] ops contracts" + printf 'status=%s\n%s\n\n' "${CONTRACTS_STATUS}" "${CONTRACTS_JSON}" + + echo "[3/7] release launchpad" + printf 'status=%s\n%s\n\n' "${LAUNCHPAD_STATUS}" "${LAUNCHPAD_JSON}" + + echo "[4/7] runtime build info" + printf 'status=%s\n%s\n\n' "${BUILD_INFO_STATUS}" "${BUILD_INFO_JSON}" + + echo "[5/7] managed nodes" + printf 'status=%s\n%s\n\n' "${NODES_STATUS}" "${NODES_JSON}" + + echo "[6/7] ops overview" + printf 'status=%s\n%s\n\n' "${OVERVIEW_STATUS}" "${OVERVIEW_JSON}" + + echo "[7/7] local node agent probe" + printf '%s\n\n' "${LOCAL_NODE_AGENT_RAW}" +fi + +echo "[8/8] go-live summary" +python3 - <<'PY' \ + "${BASE_URL}" \ + "${OVERSEAS_BASE_URL}" \ + "${STACK_JSON}" \ + "${STACK_STATUS}" \ + "${CONTRACTS_JSON}" \ + "${CONTRACTS_STATUS}" \ + "${LAUNCHPAD_JSON}" \ + "${LAUNCHPAD_STATUS}" \ + "${BUILD_INFO_JSON}" \ + "${BUILD_INFO_STATUS}" \ + "${NODES_JSON}" \ + "${NODES_STATUS}" \ + "${OVERVIEW_JSON}" \ + "${OVERVIEW_STATUS}" \ + "${LOCAL_NODE_AGENT_RAW}" \ + "${LOCAL_NODE_AGENT_SERVICE_NAME}" +import json +import sys + + +def load_payload(raw_text: str) -> dict: + try: + payload = json.loads(str(raw_text or "")) + except Exception: + return {} + return payload if isinstance(payload, dict) else {} + + +def load_data(raw_text: str) -> dict: + payload = load_payload(raw_text) + data = payload.get("data") + return data if isinstance(data, dict) else {} + + +def ensure_dict(value): + return value if isinstance(value, dict) else {} + + +def ensure_list(value): + return value if isinstance(value, list) else [] + + +def ensure_str(value): + return value if isinstance(value, str) else "" + + +def truthy_http(status_code: str) -> bool: + return str(status_code or "").startswith("2") + + +base_url = str(sys.argv[1] or "http://127.0.0.1:8100").strip() +overseas_base_url = str(sys.argv[2] or "").strip() +stack_payload = load_data(sys.argv[3]) +stack_status_code = str(sys.argv[4] or "000") +contracts_payload = load_data(sys.argv[5]) +contracts_status_code = str(sys.argv[6] or "000") +launchpad_payload = load_data(sys.argv[7]) +launchpad_status_code = str(sys.argv[8] or "000") +build_payload = load_data(sys.argv[9]) +build_status_code = str(sys.argv[10] or "000") +nodes_payload = load_data(sys.argv[11]) +nodes_status_code = str(sys.argv[12] or "000") +overview_payload = load_data(sys.argv[13]) +overview_status_code = str(sys.argv[14] or "000") +local_node_agent_raw = str(sys.argv[15] or "") +local_node_agent_service_name = str(sys.argv[16] or "domaincheck-node-agent").strip() + +diagnosis = ensure_dict(stack_payload.get("diagnosis")) +release_hub = ensure_dict(stack_payload.get("release_hub")) +managed_nodes = ensure_dict(stack_payload.get("managed_nodes")) +runtime_build_info = ensure_dict(stack_payload.get("runtime_build_info")) or build_payload +launchpad_status = ensure_dict(launchpad_payload.get("launchpad_status")) +launchpad_recommended_target_node_code = ensure_str(launchpad_status.get("recommended_target_node_code")) +launchpad_recommended_recovery_label = ensure_str(launchpad_status.get("recommended_recovery_label")) +launchpad_recommended_recovery_summary = ensure_str(launchpad_status.get("recommended_recovery_summary")) +launchpad_onboarding_bootstrap_pending_nodes = int( + launchpad_status.get("onboarding_bootstrap_pending_nodes", 0) or 0 +) +launchpad_onboarding_acceptance_ready_nodes = int( + launchpad_status.get("onboarding_acceptance_ready_nodes", 0) or 0 +) +contracts = ensure_list(contracts_payload.get("contracts")) +nodes_summary = ensure_dict(nodes_payload.get("summary")) or ensure_dict(managed_nodes.get("summary")) +route_surface = ensure_dict(runtime_build_info.get("route_surface")) +next_step = ensure_dict(diagnosis.get("next_step")) +operator_decision = ensure_dict(diagnosis.get("operator_decision")) +overview_recommendation = ensure_dict(overview_payload.get("recommendation")) +execution_scene = ensure_dict(overview_payload.get("execution_scene")) +log_sync = ensure_dict(execution_scene.get("log_sync")) + +local_agent_enabled = "" +local_agent_active = "" +local_agent_installed = False +for line in [item.strip() for item in local_node_agent_raw.splitlines() if item.strip()]: + if line == "not-installed": + local_agent_installed = False + if line == "enabled": + local_agent_enabled = "enabled" + local_agent_installed = True + elif line == "disabled": + local_agent_enabled = "disabled" + local_agent_installed = True + elif line in {"active", "inactive", "failed"}: + local_agent_active = line + local_agent_installed = True + +blocking_reasons = [] +warnings = [] +publish_blocking_reasons = [] +publish_warnings = [] + +stack_status = ensure_str(diagnosis.get("stack_status")) or "unknown" +if stack_status == "blocked": + blocking_reasons.append("stack_diagnosis=blocked") + publish_blocking_reasons.append("stack_diagnosis=blocked") +elif stack_status == "attention": + warnings.append("stack_diagnosis=attention") + publish_warnings.append("stack_diagnosis=attention") +elif not truthy_http(stack_status_code): + blocking_reasons.append("stack_diagnosis_unavailable") + publish_blocking_reasons.append("stack_diagnosis_unavailable") + +surface_complete = bool(route_surface.get("surface_complete", False)) +if not surface_complete: + missing_keys = [ensure_str(item) for item in ensure_list(route_surface.get("missing_keys")) if ensure_str(item)] + blocking_reasons.append( + "route_surface_incomplete" + (f":{','.join(missing_keys)}" if missing_keys else "") + ) + publish_blocking_reasons.append( + "route_surface_incomplete" + (f":{','.join(missing_keys)}" if missing_keys else "") + ) + +if not truthy_http(contracts_status_code) or len(contracts) <= 0: + blocking_reasons.append("contracts_unavailable") + publish_blocking_reasons.append("contracts_unavailable") + +launchpad_status_code_value = ensure_str(launchpad_status.get("status")) or ensure_str(release_hub.get("launchpad_status")) or "" +if launchpad_status_code_value == "blocked": + blocking_reasons.append("release_launchpad=blocked") + publish_blocking_reasons.append("release_launchpad=blocked") +elif launchpad_status_code_value == "attention": + warnings.append("release_launchpad=attention") + publish_warnings.append("release_launchpad=attention") +elif not truthy_http(launchpad_status_code): + blocking_reasons.append("release_launchpad_unavailable") + publish_blocking_reasons.append("release_launchpad_unavailable") + +if launchpad_onboarding_bootstrap_pending_nodes > 0: + warnings.append(f"launchpad_onboarding_bootstrap_pending={launchpad_onboarding_bootstrap_pending_nodes}") + publish_warnings.append(f"launchpad_onboarding_bootstrap_pending={launchpad_onboarding_bootstrap_pending_nodes}") +if launchpad_onboarding_acceptance_ready_nodes > 0: + warnings.append(f"launchpad_onboarding_acceptance_ready={launchpad_onboarding_acceptance_ready_nodes}") + publish_warnings.append(f"launchpad_onboarding_acceptance_ready={launchpad_onboarding_acceptance_ready_nodes}") + +managed_enabled = int(nodes_summary.get("managed_enabled", 0) or 0) +remote_access_ready = int(nodes_summary.get("remote_access_ready", 0) or 0) +queue_dead_letter_nodes = int(nodes_summary.get("queue_dead_letter_nodes", 0) or 0) +if managed_enabled > 0 and remote_access_ready == 0: + blocking_reasons.append("remote_agent_ready=0") + publish_blocking_reasons.append("remote_agent_ready=0") +elif managed_enabled > 0 and remote_access_ready < managed_enabled: + warnings.append(f"remote_agent_ready={remote_access_ready}/{managed_enabled}") + publish_warnings.append(f"remote_agent_ready={remote_access_ready}/{managed_enabled}") + +if queue_dead_letter_nodes > 0: + warnings.append(f"queue_dead_letter_nodes={queue_dead_letter_nodes}") + publish_warnings.append(f"queue_dead_letter_nodes={queue_dead_letter_nodes}") + +if local_agent_installed and local_agent_active not in {"", "active"}: + warnings.append(f"{local_node_agent_service_name}={local_agent_active}") + +log_sync_enabled = bool(log_sync.get("enabled")) +log_sync_state = ensure_str(log_sync.get("state")) +log_sync_mode = ensure_str(log_sync.get("mode")) +participating_nodes_total = int(log_sync.get("participating_node_count", 0) or 0) +covered_participating_nodes = int(log_sync.get("covered_participating_node_count", 0) or 0) +missing_participating_node_codes = [ + ensure_str(item) + for item in ensure_list(log_sync.get("missing_participating_nodes")) + if ensure_str(item) +] +if log_sync_enabled and participating_nodes_total > 0 and covered_participating_nodes == 0: + warnings.append(f"log_sync_waiting_sample=0/{participating_nodes_total}") + publish_warnings.append(f"log_sync_waiting_sample=0/{participating_nodes_total}") +elif log_sync_enabled and participating_nodes_total > 0 and covered_participating_nodes < participating_nodes_total: + warnings.append(f"log_sync_partial={covered_participating_nodes}/{participating_nodes_total}") + publish_warnings.append(f"log_sync_partial={covered_participating_nodes}/{participating_nodes_total}") + +operator_lane = ensure_str(operator_decision.get("lane")) +operator_title = ensure_str(operator_decision.get("title")) +operator_primary_command_key = ensure_str(operator_decision.get("primary_command_key")) +if not operator_lane: + if blocking_reasons: + operator_lane = "recovery" + elif warnings: + operator_lane = "verification" + else: + operator_lane = "go-live" +if not operator_title: + operator_title = ( + ensure_str(overview_recommendation.get("title")) + or ensure_str(launchpad_status.get("status_label")) + or ("先处理阻塞项" if blocking_reasons else "可以进入上线复核") + ) + +next_step_action_code = ( + ensure_str(next_step.get("action_code")) + or ensure_str(overview_recommendation.get("primary_action_code")) + or ensure_str(launchpad_status.get("recommended_action_code")) +) +next_step_reason = ( + ensure_str(next_step.get("reason")) + or ensure_str(overview_recommendation.get("reason")) + or "来自 launchpad/overview fallback" +) +if not operator_primary_command_key: + operator_primary_command_key = next_step_action_code + +go_live_status = "ready" +if blocking_reasons: + go_live_status = "blocked" +elif warnings: + go_live_status = "attention" + +publish_status = "ready" +if publish_blocking_reasons: + publish_status = "blocked" +elif publish_warnings: + publish_status = "attention" +publish_ready = publish_status == "ready" +if publish_status == "ready": + publish_status_label = "可发布" +elif publish_status == "attention": + publish_status_label = "可发布但建议先复核" +else: + publish_status_label = "暂不可发布" + +action_commands = { + "handover_first_gap": f"bash domain-api/deploy/multi-region/drive_ops_center.sh driver-resolve {base_url} handover_first_gap", + "view_first_gap": f"bash domain-api/deploy/multi-region/drive_ops_center.sh driver-resolve {base_url} view_first_gap", + "bootstrap_run": ( + f"bash domain-api/deploy/multi-region/drive_ops_center.sh node-bootstrap-run {base_url} {launchpad_recommended_target_node_code} cli" + if launchpad_recommended_target_node_code + else f"bash domain-api/deploy/multi-region/drive_ops_center.sh driver-resolve {base_url} bootstrap_run" + ), + "run_acceptance": ( + f"bash domain-api/deploy/multi-region/drive_ops_center.sh node-acceptance-run {base_url} {launchpad_recommended_target_node_code} cli" + if launchpad_recommended_target_node_code + else f"bash domain-api/deploy/multi-region/drive_ops_center.sh driver-resolve {base_url} run_acceptance" + ), + "fix_managed_nodes": f"bash domain-api/deploy/multi-region/drive_ops_center.sh driver-resolve {base_url} fix_managed_nodes", + "open_worker_logs_participating": f"bash domain-api/deploy/multi-region/drive_ops_center.sh driver-resolve {base_url} open_worker_logs_participating", + "run_inspection_participating": f"bash domain-api/deploy/multi-region/drive_ops_center.sh driver-resolve {base_url} run_inspection_participating", +} + +recommended_commands = { + "stack_summary": f"bash domain-api/deploy/multi-region/check_ops_center_stack.sh {base_url} summary", + "contracts": f"bash domain-api/deploy/multi-region/check_ops_contracts.sh {base_url}", + "ops_plane": f"bash domain-api/deploy/multi-region/check_ops_plane.sh {base_url}", + "release_hub": f"bash domain-api/deploy/multi-region/check_release_hub.sh {base_url}", + "inspection": f"bash domain-api/deploy/multi-region/check_ops_inspection.sh {base_url}", + "node_agent": "bash domain-api/deploy/multi-region/check_node_agent.sh", + "overview": f"bash domain-api/deploy/multi-region/drive_ops_center.sh overview {base_url}", + "doctor_export": ( + f"bash domain-api/deploy/multi-region/drive_ops_center.sh doctor-export /tmp/domaincheck-go-live {base_url} {overseas_base_url}" + if overseas_base_url + else f"bash domain-api/deploy/multi-region/drive_ops_center.sh doctor-export /tmp/domaincheck-go-live {base_url}" + ), +} +if next_step_action_code and next_step_action_code in action_commands: + recommended_commands["next_step"] = action_commands[next_step_action_code] +if log_sync_enabled: + if "open_worker_logs_participating" in action_commands: + recommended_commands["log_sync_logs"] = action_commands["open_worker_logs_participating"] + if "run_inspection_participating" in action_commands: + recommended_commands["log_sync_inspection"] = action_commands["run_inspection_participating"] + +summary = { + "base_url": base_url, + "overseas_base_url": overseas_base_url, + "go_live_status": go_live_status, + "publish_ready": publish_ready, + "publish_status": publish_status, + "publish_status_label": publish_status_label, + "stack_status": stack_status, + "contracts_ready": truthy_http(contracts_status_code) and len(contracts) > 0, + "contracts_total": len(contracts), + "launchpad_status": launchpad_status_code_value, + "launchpad_status_label": ensure_str(launchpad_status.get("status_label")), + "launchpad_recommended_action_code": ensure_str(launchpad_status.get("recommended_action_code")), + "launchpad_recommended_target_node_code": launchpad_recommended_target_node_code, + "launchpad_recommended_recovery_label": launchpad_recommended_recovery_label, + "launchpad_recommended_recovery_summary": launchpad_recommended_recovery_summary, + "launchpad_onboarding_bootstrap_pending_nodes": launchpad_onboarding_bootstrap_pending_nodes, + "launchpad_onboarding_acceptance_ready_nodes": launchpad_onboarding_acceptance_ready_nodes, + "route_surface_complete": surface_complete, + "route_surface_missing_keys": [ + ensure_str(item) for item in ensure_list(route_surface.get("missing_keys")) if ensure_str(item) + ], + "managed_enabled": managed_enabled, + "remote_access_ready": remote_access_ready, + "queue_dead_letter_nodes": queue_dead_letter_nodes, + "participating_nodes_total": participating_nodes_total, + "log_sync_enabled": log_sync_enabled, + "log_sync_state": log_sync_state, + "log_sync_mode": log_sync_mode, + "log_sync_covered_nodes": covered_participating_nodes, + "log_sync_missing_node_codes": missing_participating_node_codes, + "next_step_action_code": next_step_action_code, + "next_step_reason": next_step_reason, + "operator_lane": operator_lane, + "operator_title": operator_title, + "operator_primary_command_key": operator_primary_command_key, + "local_node_agent": { + "service_name": local_node_agent_service_name, + "installed": local_agent_installed, + "enabled": local_agent_enabled, + "active": local_agent_active, + }, + "blocking_reasons": blocking_reasons, + "warnings": warnings, + "publish_blocking_reasons": publish_blocking_reasons, + "publish_warnings": publish_warnings, + "recommended_commands": recommended_commands, +} + +print(json.dumps(summary, ensure_ascii=False, indent=2)) +PY diff --git a/domain-api/deploy/multi-region/check_go_live_signoff.sh b/domain-api/deploy/multi-region/check_go_live_signoff.sh new file mode 100644 index 0000000..99478fa --- /dev/null +++ b/domain-api/deploy/multi-region/check_go_live_signoff.sh @@ -0,0 +1,182 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck disable=SC1091 +source "${SCRIPT_DIR}/lib_ops_center.sh" +ops_center_load_config "${SCRIPT_DIR}" + +TARGET_OR_BASE_URL="${1:-}" +SECOND_ARG="${2:-}" + +REPORT_DIR="" +MANIFEST_PATH="" +GENERATED_BUNDLE="false" +MAINLAND_BASE_URL="" +OVERSEAS_BASE_URL="" + +if [[ -n "${TARGET_OR_BASE_URL}" && ( -f "${TARGET_OR_BASE_URL}" || -d "${TARGET_OR_BASE_URL}" ) ]]; then + if [[ -d "${TARGET_OR_BASE_URL}" ]]; then + REPORT_DIR="${TARGET_OR_BASE_URL%/}" + MANIFEST_PATH="${REPORT_DIR}/manifest.json" + else + MANIFEST_PATH="${TARGET_OR_BASE_URL}" + REPORT_DIR="$(cd "$(dirname "${MANIFEST_PATH}")" && pwd)" + fi +else + MAINLAND_BASE_URL="${TARGET_OR_BASE_URL:-$(ops_center_resolve_mainland_api_base_url)}" + OVERSEAS_BASE_URL="$(normalize_optional_base_url_arg "${SECOND_ARG:-}" "$(ops_center_resolve_overseas_api_base_url)")" + REPORT_DIR="${OPS_CENTER_REPORT_ROOT}/go-live-bundles/go-live-signoff-$(timestamp_now)" + ops_center_ensure_report_dir "${REPORT_DIR}" + bash "${SCRIPT_DIR}/export_go_live_bundle.sh" "${REPORT_DIR}" "${MAINLAND_BASE_URL}" "${OVERSEAS_BASE_URL}" >/dev/null + MANIFEST_PATH="${REPORT_DIR}/manifest.json" + GENERATED_BUNDLE="true" +fi + +if [[ ! -f "${MANIFEST_PATH}" ]]; then + echo "manifest not found: ${MANIFEST_PATH}" >&2 + exit 1 +fi + +REVIEW_JSON="$(bash "${SCRIPT_DIR}/review_go_live_bundle.sh" "${MANIFEST_PATH}")" +DOCTOR_JSON="$(bash "${SCRIPT_DIR}/drive_ops_center.sh" doctor-decision "${MANIFEST_PATH}")" + +python3 - <<'PY' \ + "${REVIEW_JSON}" \ + "${DOCTOR_JSON}" \ + "${GENERATED_BUNDLE}" \ + "${REPORT_DIR}" \ + "${MANIFEST_PATH}" \ + "${MAINLAND_BASE_URL}" \ + "${OVERSEAS_BASE_URL}" +import json +import sys + +review = json.loads(sys.argv[1]) +doctor = json.loads(sys.argv[2]) +generated_bundle = str(sys.argv[3]).strip().lower() == "true" +report_dir = str(sys.argv[4] or "").strip() +manifest_path = str(sys.argv[5] or "").strip() +mainland_base_url = str(sys.argv[6] or "").strip() +overseas_base_url = str(sys.argv[7] or "").strip() + +go_live_summary = dict(review.get("go_live_summary") or {}) +stack_diagnosis = dict(review.get("stack_diagnosis") or {}) +driver_feed = dict(review.get("driver_feed") or {}) +codex_brief = dict(review.get("codex_brief") or {}) +launchpad_alignment = dict(review.get("launchpad_alignment") or {}) +doctor_decision = dict(doctor.get("decision") or {}) +doctor_summary = dict(doctor.get("summary") or {}) + +review_status = str(review.get("status") or "").strip() +doctor_status = str(doctor_decision.get("status") or "").strip() +go_live_status = str(go_live_summary.get("go_live_status") or "").strip() +publish_status = str(go_live_summary.get("publish_status") or "").strip() +stack_status = str(stack_diagnosis.get("stack_status") or "").strip() +driver_launch_status = str(driver_feed.get("launch_status") or "").strip() +codex_launch_status = str(codex_brief.get("launch_status") or "").strip() + + +def dedupe_commands(values): + result = [] + seen = set() + for item in values: + text = str(item or "").strip() + if not text or text in seen: + continue + seen.add(text) + result.append(text) + return result + + +blocked_reasons = dedupe_commands( + [] + + ([f"review:{review_status}"] if review_status == "blocked" else []) + + ([f"doctor:{doctor_status}"] if doctor_status == "blocked" else []) + + ([f"go_live:{go_live_status}"] if go_live_status == "blocked" else []) + + ([f"publish:{publish_status}"] if publish_status == "blocked" else []) + + ([f"stack:{stack_status}"] if stack_status == "blocked" else []) + + ([f"driver_launch:{driver_launch_status}"] if driver_launch_status == "blocked" else []) + + ([f"codex_launch:{codex_launch_status}"] if codex_launch_status == "blocked" else []) +) + +attention_reasons = dedupe_commands( + [] + + ([f"review:{review_status}"] if review_status == "attention" else []) + + ([f"doctor:{doctor_status}"] if doctor_status == "attention" else []) + + ([f"go_live:{go_live_status}"] if go_live_status == "attention" else []) + + ([f"publish:{publish_status}"] if publish_status == "attention" else []) + + ([f"stack:{stack_status}"] if stack_status == "attention" else []) + + ([f"driver_launch:{driver_launch_status}"] if driver_launch_status == "attention" else []) + + ([f"codex_launch:{codex_launch_status}"] if codex_launch_status == "attention" else []) + + (["launchpad_alignment:inconsistent"] if not bool(launchpad_alignment.get("consistent", False)) else []) +) + +if blocked_reasons: + signoff_status = "blocked" + headline = ( + str(review.get("headline") or "").strip() + or str(doctor_decision.get("headline") or "").strip() + or "当前发布前验证仍存在阻断项，不应直接签字上线。" + ) +elif attention_reasons: + signoff_status = "attention" + headline = ( + str(doctor_decision.get("headline") or "").strip() + or str(review.get("headline") or "").strip() + or "当前现场已进入收口区间，但仍建议先完成 attention 项再正式签字。" + ) +else: + signoff_status = "ready" + headline = "当前证据包、doctor 结论与发布门禁一致，可进入最终人工签字或正式发布。" + +recommended_commands = dedupe_commands( + [] + + list(doctor_decision.get("recommended_commands") or []) + + [ + f"bash domain-api/deploy/multi-region/drive_ops_center.sh go-live-review {manifest_path}", + f"bash domain-api/deploy/multi-region/drive_ops_center.sh doctor-decision {manifest_path}", + ] +) + +payload = { + "signoff_status": signoff_status, + "headline": headline, + "generated_bundle": generated_bundle, + "report_dir": report_dir or str(doctor.get("report_dir") or review.get("report_dir") or ""), + "manifest_path": manifest_path or str(doctor.get("manifest_path") or ""), + "base_urls": { + "mainland": mainland_base_url, + "overseas": overseas_base_url, + }, + "release_gate": { + "go_live_status": go_live_status, + "publish_status": publish_status, + "stack_status": stack_status, + "driver_launch_status": driver_launch_status, + "codex_launch_status": codex_launch_status, + "bundle_review_status": review_status, + "doctor_status": doctor_status, + }, + "blocked_reasons": blocked_reasons, + "attention_reasons": attention_reasons, + "decision": { + "operator_title": str(go_live_summary.get("operator_title") or "").strip(), + "next_step_action_code": str(go_live_summary.get("next_step_action_code") or "").strip(), + "doctor_reason_code": str(doctor_decision.get("reason_code") or "").strip(), + "doctor_preferred_surface": str(doctor_decision.get("preferred_surface") or "").strip(), + "launchpad_recommended_target_node_code": str(go_live_summary.get("launchpad_recommended_target_node_code") or "").strip(), + "launchpad_recommended_recovery_label": str(go_live_summary.get("launchpad_recommended_recovery_label") or "").strip(), + "launchpad_recommended_recovery_summary": str(go_live_summary.get("launchpad_recommended_recovery_summary") or "").strip(), + }, + "launchpad_alignment": launchpad_alignment, + "recommended_commands": recommended_commands, + "bundle_review": review, + "doctor_decision": { + "summary": doctor_summary, + "decision": doctor_decision, + }, +} + +print(json.dumps(payload, ensure_ascii=False, indent=2)) +PY diff --git a/domain-api/deploy/multi-region/check_node_agent.sh b/domain-api/deploy/multi-region/check_node_agent.sh new file mode 100755 index 0000000..5686624 --- /dev/null +++ b/domain-api/deploy/multi-region/check_node_agent.sh @@ -0,0 +1,38 @@ +#!/usr/bin/env bash +set -euo pipefail + +SERVICE_NAME="${SERVICE_NAME:-domaincheck-node-agent}" +ENV_FILE="${ENV_FILE:-/etc/default/domaincheck-node-agent}" + +echo "[1/4] environment file" +if [[ -f "${ENV_FILE}" ]]; then + echo "env file: ${ENV_FILE}" +else + echo "missing env file: ${ENV_FILE}" +fi +echo + +echo "[2/4] key env summary" +if [[ -f "${ENV_FILE}" ]]; then + grep -E '^(OPS_CONTROL_PLANE_BASE_URL|NODE_CODE|NODE_REGION|NODE_ROLE|OPS_AGENT_POLL_INTERVAL_SECONDS|WORKER_SERVICE_NAME|API_SERVICE_NAME|SYNC_AGENT_SERVICE_NAME)=' "${ENV_FILE}" || true +else + echo "skip" +fi +echo + +echo "[3/4] systemd status" +if systemctl list-unit-files "${SERVICE_NAME}.service" --no-pager >/dev/null 2>&1; then + systemctl is-enabled "${SERVICE_NAME}" || true + systemctl is-active "${SERVICE_NAME}" || true +else + echo "not-installed" + echo "inactive" +fi +echo + +echo "[4/4] service detail" +if systemctl list-unit-files "${SERVICE_NAME}.service" --no-pager >/dev/null 2>&1; then + systemctl status "${SERVICE_NAME}" --no-pager -l | sed -n '1,25p' || true +else + echo "service not found: ${SERVICE_NAME}" +fi diff --git a/domain-api/deploy/multi-region/check_node_scene_log.sh b/domain-api/deploy/multi-region/check_node_scene_log.sh new file mode 100644 index 0000000..b05d4a5 --- /dev/null +++ b/domain-api/deploy/multi-region/check_node_scene_log.sh @@ -0,0 +1,65 @@ +#!/usr/bin/env bash +set -euo pipefail + +BASE_URL="${1:-http://127.0.0.1:8100}" +NODE_CODE="${2:-}" +LIMIT="${3:-80}" +MODE="${4:-key}" +CURL_CONNECT_TIMEOUT="${CURL_CONNECT_TIMEOUT:-3}" +CURL_MAX_TIME="${CURL_MAX_TIME:-8}" + +if [[ -z "${NODE_CODE}" ]]; then + echo "usage: bash domain-api/deploy/multi-region/check_node_scene_log.sh [base_url] [limit] [key|full]" >&2 + exit 1 +fi + +RAW_RESPONSE="$(curl -sS \ + --connect-timeout "${CURL_CONNECT_TIMEOUT}" \ + --max-time "${CURL_MAX_TIME}" \ + "${BASE_URL}/api/v1/ops/nodes/${NODE_CODE}/scene-log?limit=${LIMIT}&mode=${MODE}")" + +echo "[1/2] raw response" +printf '%s\n\n' "${RAW_RESPONSE}" + +echo "[2/2] condensed scene-log summary" +python3 - "${RAW_RESPONSE}" <<'PY' +import json +import sys + +raw = str(sys.argv[1] or "") +payload = json.loads(raw) +data = dict(payload.get("data") or {}) +participation = dict(data.get("participation") or {}) +source_summary = dict(data.get("source_summary") or {}) +records = list(data.get("records") or []) +latest = dict(data.get("latest_record") or {}) + +summary = { + "node_code": str(data.get("node_code") or ""), + "status": str(data.get("status") or ""), + "status_label": str(data.get("status_label") or ""), + "mode": str(data.get("mode") or ""), + "summary": str(data.get("summary") or ""), + "log_sync_enabled": bool(data.get("log_sync_enabled", False)), + "records_total": int(data.get("records_total", len(records)) or 0), + "missing_reason_code": str(data.get("missing_reason_code") or ""), + "participation": { + "detect_participating": bool(participation.get("detect_participating", False)), + "participation_state": str(participation.get("participation_state") or ""), + "participation_label": str(participation.get("participation_label") or ""), + "participation_reason": str(participation.get("participation_reason") or ""), + "current_load": int(participation.get("current_load", 0) or 0), + }, + "source_summary": { + "line_count": int(source_summary.get("line_count", 0) or 0), + "key_line_count": int(source_summary.get("key_line_count", 0) or 0), + "full_line_count": int(source_summary.get("full_line_count", 0) or 0), + "last_at": str(source_summary.get("last_at") or ""), + "last_line": str(source_summary.get("last_line") or ""), + }, + "latest_record": latest, + "sample_messages": [str(item.get("message") or "") for item in records[:8]], +} + +print(json.dumps(summary, ensure_ascii=False, indent=2)) +PY diff --git a/domain-api/deploy/multi-region/check_ops_center_stack.sh b/domain-api/deploy/multi-region/check_ops_center_stack.sh new file mode 100644 index 0000000..f0da875 --- /dev/null +++ b/domain-api/deploy/multi-region/check_ops_center_stack.sh @@ -0,0 +1,1272 @@ +#!/usr/bin/env bash +set -euo pipefail + +BASE_URL="${1:-http://127.0.0.1:8100}" +VIEW_MODE="${2:-full}" +CURL_CONNECT_TIMEOUT="${CURL_CONNECT_TIMEOUT:-3}" +CURL_MAX_TIME="${CURL_MAX_TIME:-8}" + +FETCH_BODY="" +FETCH_STATUS="000" + +extract_api_data_json() { + local raw_json="${1:-}" + python3 - "${raw_json}" <<'PY' +import json +import sys + +raw_json = str(sys.argv[1] or "") +try: + payload = json.loads(raw_json) +except Exception: + raise SystemExit(1) + +data = payload.get("data") +if not isinstance(data, dict): + raise SystemExit(1) + +print(json.dumps(data, ensure_ascii=False, indent=2)) +PY +} + +fetch_json() { + local url="${1:-}" + local raw_text + raw_text="$(curl -sS \ + --connect-timeout "${CURL_CONNECT_TIMEOUT}" \ + --max-time "${CURL_MAX_TIME}" \ + "${url}" \ + -w $'\nHTTP_STATUS=%{http_code}' || true)" + FETCH_BODY="${raw_text%$'\n'HTTP_STATUS=*}" + FETCH_STATUS="${raw_text##*$'\n'HTTP_STATUS=}" +} + +fetch_json "${BASE_URL}/api/v1/ops/stack-diagnosis?base_url=${BASE_URL}" +STACK_DIAGNOSIS_JSON="${FETCH_BODY}" +STACK_DIAGNOSIS_STATUS="${FETCH_STATUS}" +if [[ "${STACK_DIAGNOSIS_STATUS}" == 2* ]]; then + if STACK_DIAGNOSIS_DATA_JSON="$(extract_api_data_json "${STACK_DIAGNOSIS_JSON}" 2>/dev/null)"; then + if [[ "${VIEW_MODE}" != "summary" ]]; then + echo "[1/1] ops stack diagnosis (api contract)" + printf 'status=%s\n%s\n\n' "${STACK_DIAGNOSIS_STATUS}" "${STACK_DIAGNOSIS_JSON}" + fi + echo "[9/9] condensed stack summary" + STACK_BASE_URL="${BASE_URL}" python3 - <<'PY' "${STACK_DIAGNOSIS_DATA_JSON}" +import json +import os +import sys + + +def ensure_dict(value): + return value if isinstance(value, dict) else {} + + +def ensure_list(value): + return value if isinstance(value, list) else [] + + +def ensure_str(value): + return value if isinstance(value, str) else "" + + +def command(*parts: str) -> str: + tokens = ["bash", "domain-api/deploy/multi-region/drive_ops_center.sh"] + tokens.extend([str(part).strip() for part in parts if str(part).strip()]) + return " ".join(tokens) + + +def release_command(*parts: str) -> str: + tokens = ["bash", "domain-api/deploy/multi-region/drive_release_hub.sh"] + tokens.extend([str(part).strip() for part in parts if str(part).strip()]) + return " ".join(tokens) + + +payload = json.loads(str(sys.argv[1] or "{}")) +base_url = ensure_str(os.environ.get("STACK_BASE_URL")) or "http://127.0.0.1:8100" +diagnosis = ensure_dict(payload.get("diagnosis")) +overview = ensure_dict(payload.get("overview")) +managed_nodes = ensure_dict(payload.get("managed_nodes")) +release_hub = ensure_dict(payload.get("release_hub")) + +issues = ensure_list(diagnosis.get("issues")) +recommended_actions = ensure_list(diagnosis.get("recommended_actions")) +next_step = ensure_dict(diagnosis.get("next_step")) +backend_operator_decision = ensure_dict(diagnosis.get("operator_decision")) +backend_next_actions = ensure_dict(diagnosis.get("next_actions")) +backend_recommended_commands = ensure_dict(diagnosis.get("recommended_commands")) +log_sync = ensure_dict(overview.get("log_sync")) +problem_nodes = ensure_list(managed_nodes.get("problem_nodes")) +next_step_focus_ref = ensure_dict(next_step.get("focus_ref")) +launchpad_target_node_code = ensure_str(diagnosis.get("launchpad_recommended_target_node_code")) or ensure_str(next_step_focus_ref.get("node_code")) +launchpad_recovery_label = ensure_str(diagnosis.get("launchpad_recommended_recovery_label")) +launchpad_recovery_summary = ensure_str(diagnosis.get("launchpad_recommended_recovery_summary")) or ensure_str(next_step.get("reason")) +handover_next_action_code = ensure_str(next_step.get("action_code")) + +scene_log_target_node_code = "" +missing_nodes = ensure_list(log_sync.get("missing_sample_node_codes")) +if missing_nodes: + scene_log_target_node_code = ensure_str(missing_nodes[0]) +elif problem_nodes: + scene_log_target_node_code = ensure_str(ensure_dict(problem_nodes[0]).get("node_code")) + +recommended_commands = { + "stack_summary": f"bash domain-api/deploy/multi-region/check_ops_center_stack.sh {base_url} summary", + "doctor": command("doctor", base_url), + "driver_feed": command("driver-feed", base_url), + "driver_focus_preview": command("driver-focus-preview", base_url), + "driver_focus_run": command("driver-focus-run", base_url), + "go_live_recover": command("go-live-recover", base_url), + "log_sync_recover": command("log-sync-recover", base_url, "key"), + "log_sync_recover_full": command("log-sync-recover", base_url, "full"), + "agent_gap_check": command("agent-gap-check", base_url), + "agent_gap_recover": command("agent-gap-recover", base_url), + "api_restart": command("runtime-refresh-recover", base_url), + "node_onboarding": ( + command("node-onboarding", base_url, ensure_str(ensure_dict(problem_nodes[0]).get("node_code"))) + if problem_nodes and ensure_str(ensure_dict(problem_nodes[0]).get("node_code")) + else "" + ), + "node_recover": ( + command("node-recover", base_url, ensure_str(ensure_dict(problem_nodes[0]).get("node_code"))) + if problem_nodes and ensure_str(ensure_dict(problem_nodes[0]).get("node_code")) + else "" + ), + "node_bootstrap_preview": ( + command("node-bootstrap-preview", base_url, ensure_str(ensure_dict(problem_nodes[0]).get("node_code"))) + if problem_nodes and ensure_str(ensure_dict(problem_nodes[0]).get("node_code")) + else "" + ), + "node_bootstrap_run": ( + command("node-bootstrap-run", base_url, launchpad_target_node_code, "cli") + if launchpad_target_node_code + else command("node-bootstrap-run", base_url, ensure_str(ensure_dict(problem_nodes[0]).get("node_code")), "cli") + if problem_nodes and ensure_str(ensure_dict(problem_nodes[0]).get("node_code")) + else "" + ), + "node_acceptance_run": ( + command("node-acceptance-run", base_url, launchpad_target_node_code, "cli") + if launchpad_target_node_code + else command("node-acceptance-run", base_url, ensure_str(ensure_dict(problem_nodes[0]).get("node_code")), "cli") + if launchpad_target_node_code or (problem_nodes and ensure_str(ensure_dict(problem_nodes[0]).get("node_code"))) + else "" + ), + "node_acceptance_plan": ( + command("node-acceptance-plan", base_url, ensure_str(ensure_dict(problem_nodes[0]).get("node_code"))) + if problem_nodes and ensure_str(ensure_dict(problem_nodes[0]).get("node_code")) + else "" + ), + "scene_log": ( + command("scene-node-log", base_url, scene_log_target_node_code, "120", "key") + if scene_log_target_node_code + else "" + ), + "release_launchpad": release_command(base_url, "launchpad"), + "release_preview_latest": release_command(base_url, "preview-latest"), + "jobs": command("activity-stream", base_url, "kind=ops_job"), + "node_handover": ( + command("node-handover", base_url, ensure_str(ensure_dict(problem_nodes[0]).get("node_code"))) + if problem_nodes and ensure_str(ensure_dict(problem_nodes[0]).get("node_code")) + else command("stack-first-gap-handover", base_url) + ), +} + +operator_decision = { + "lane": "steady", + "priority": "low", + "reason_code": "steady_state", + "title": "当前进入稳定观察态", + "summary": "当前总检没有发现新的阻断问题，可继续按既有链路观察与推进。", + "next_focus": ensure_dict(next_step.get("focus_ref")), + "primary_command_key": "driver_feed", + "secondary_command_key": "jobs", +} + +participating_total = int(log_sync.get("participating_nodes_total", 0) or 0) +remote_access_ready = int(managed_nodes.get("remote_access_ready", 0) or 0) +managed_enabled = int(managed_nodes.get("managed_enabled", 0) or 0) +default_rollout_gate_status = ensure_str(release_hub.get("default_rollout_gate_status")) + +if not bool(log_sync.get("enabled")) and participating_total > 0: + operator_decision = { + "lane": "observability", + "priority": "high", + "reason_code": "scene_log_disabled", + "title": "先恢复远端现场日志可见性", + "summary": "当前已有参与检测节点，但远端日志回传仍关闭，海外控制面仍处于半盲态。", + "next_focus": {"kind": "node_scene_log", "node_code": scene_log_target_node_code, "mode": "key", "limit": 80} + if scene_log_target_node_code + else {}, + "primary_command_key": "log_sync_recover", + "secondary_command_key": "scene_log", + } +elif managed_enabled > 0 and remote_access_ready == 0: + primary_command_key = "agent_gap_recover" + secondary_command_key = "node_handover" + title = "先补齐节点接管与 remote-agent 可达性" + summary = ( + f"managed_enabled={managed_enabled}，remote_access_ready={remote_access_ready}；" + "当前还不能从海外控制面统一下发标准动作。" + ) + if handover_next_action_code == "bootstrap_run" and launchpad_target_node_code: + primary_command_key = "node_bootstrap_run" + secondary_command_key = "node_handover" + title = "先跑首台节点接入收口" + summary = ( + launchpad_recovery_summary + or join_text( + launchpad_recovery_label, + launchpad_target_node_code, + "当前真正阻断已经收敛到首台节点接入收口。" + ) + ) + elif handover_next_action_code == "run_acceptance" and launchpad_target_node_code: + primary_command_key = "node_acceptance_run" + secondary_command_key = "node_handover" + title = "先跑首台节点接管验收" + summary = ( + launchpad_recovery_summary + or join_text( + launchpad_recovery_label, + launchpad_target_node_code, + "当前真正阻断已经收敛到首台节点接管验收。" + ) + ) + operator_decision = { + "lane": "node_handover", + "priority": "high", + "reason_code": "managed_nodes_agent_pending", + "title": title, + "summary": summary, + "next_focus": next_step_focus_ref, + "primary_command_key": primary_command_key, + "secondary_command_key": secondary_command_key, + } +elif default_rollout_gate_status == "blocked": + operator_decision = { + "lane": "release", + "priority": "medium", + "reason_code": "release_rollout_gate_blocked", + "title": "先检查发布门禁与 rollout 阻断", + "summary": ensure_str(release_hub.get("default_rollout_gate_summary")), + "next_focus": ensure_dict(release_hub.get("default_rollout_gate_focus_ref") or release_hub.get("focus_ref")), + "primary_command_key": "release_launchpad", + "secondary_command_key": "release_preview_latest", + } +elif any(ensure_str(ensure_dict(item).get("code")) == "runtime_build_schema_stale" for item in issues): + operator_decision = { + "lane": "runtime_recovery", + "priority": "high", + "reason_code": "runtime_build_schema_stale", + "title": "先执行运行时刷新与上线复核", + "summary": "当前已经识别到仓库代码与运行中 API schema 之间存在漂移，先走统一恢复链，再继续节点接管或发布判断。", + "next_focus": ensure_dict(next_step.get("focus_ref")), + "primary_command_key": "go_live_recover", + "secondary_command_key": "api_restart", + } +elif recommended_actions or issues: + operator_decision = { + "lane": "ops_jobs", + "priority": "medium", + "reason_code": "issues_need_attention", + "title": "先按总检建议逐项收口当前问题", + "summary": ( + f"recommended_actions={len(recommended_actions)}，issues={len(issues)}，" + f"stack_status={ensure_str(diagnosis.get('stack_status')) or 'unknown'}。" + ), + "next_focus": ensure_dict(next_step.get("focus_ref")), + "primary_command_key": "doctor", + "secondary_command_key": "jobs", + } + +if backend_recommended_commands: + merged_recommended_commands = dict(recommended_commands) + merged_recommended_commands.update({ + key: ensure_str(value) + for key, value in backend_recommended_commands.items() + if ensure_str(key) and ensure_str(value) + }) + recommended_commands = merged_recommended_commands + +if backend_operator_decision: + operator_decision = { + "lane": ensure_str(backend_operator_decision.get("lane")) or ensure_str(operator_decision.get("lane")), + "priority": ensure_str(backend_operator_decision.get("priority")) or ensure_str(operator_decision.get("priority")), + "reason_code": ensure_str(backend_operator_decision.get("reason_code")) or ensure_str(operator_decision.get("reason_code")), + "title": ensure_str(backend_operator_decision.get("title")) or ensure_str(operator_decision.get("title")), + "summary": ensure_str(backend_operator_decision.get("summary")) or ensure_str(operator_decision.get("summary")), + "next_focus": ensure_dict(backend_operator_decision.get("next_focus")) or ensure_dict(operator_decision.get("next_focus")), + "primary_command_key": ensure_str(backend_operator_decision.get("primary_command_key")) or ensure_str(operator_decision.get("primary_command_key")), + "secondary_command_key": ensure_str(backend_operator_decision.get("secondary_command_key")) or ensure_str(operator_decision.get("secondary_command_key")), + } + +next_actions = { + "primary_command_key": ensure_str(operator_decision.get("primary_command_key")), + "primary_command": ensure_str(recommended_commands.get(ensure_str(operator_decision.get("primary_command_key")))), + "secondary_command_key": ensure_str(operator_decision.get("secondary_command_key")), + "secondary_command": ensure_str(recommended_commands.get(ensure_str(operator_decision.get("secondary_command_key")))), +} + +if backend_next_actions: + next_actions["primary_command_key"] = ensure_str(backend_next_actions.get("primary_command_key")) or next_actions["primary_command_key"] + next_actions["secondary_command_key"] = ensure_str(backend_next_actions.get("secondary_command_key")) or next_actions["secondary_command_key"] + next_actions["primary_command"] = ( + ensure_str(backend_next_actions.get("primary_command")) + or ensure_str(recommended_commands.get(next_actions["primary_command_key"])) + or next_actions["primary_command"] + ) + next_actions["secondary_command"] = ( + ensure_str(backend_next_actions.get("secondary_command")) + or ensure_str(recommended_commands.get(next_actions["secondary_command_key"])) + or next_actions["secondary_command"] + ) + +diagnosis["operator_decision"] = operator_decision +diagnosis["next_actions"] = next_actions +diagnosis["recommended_commands"] = recommended_commands +payload["diagnosis"] = diagnosis +print(json.dumps(payload, ensure_ascii=False, indent=2)) +PY + exit 0 + fi +fi + +fetch_json "${BASE_URL}/health" +HEALTH_JSON="${FETCH_BODY}" +HEALTH_STATUS="${FETCH_STATUS}" + +fetch_json "${BASE_URL}/api/v1/ops/contracts" +CONTRACTS_JSON="${FETCH_BODY}" +CONTRACTS_STATUS="${FETCH_STATUS}" + +fetch_json "${BASE_URL}/api/v1/ops/link-snapshot" +LINK_SNAPSHOT_JSON="${FETCH_BODY}" +LINK_SNAPSHOT_STATUS="${FETCH_STATUS}" + +fetch_json "${BASE_URL}/api/v1/ops/overview" +OVERVIEW_JSON="${FETCH_BODY}" +OVERVIEW_STATUS="${FETCH_STATUS}" + +fetch_json "${BASE_URL}/api/v1/ops/nodes" +NODES_JSON="${FETCH_BODY}" +NODES_STATUS="${FETCH_STATUS}" + +fetch_json "${BASE_URL}/api/v1/ops/releases/launchpad" +LAUNCHPAD_JSON="${FETCH_BODY}" +LAUNCHPAD_STATUS="${FETCH_STATUS}" + +fetch_json "${BASE_URL}/api/v1/ops/playbook-runs?limit=6" +PLAYBOOK_RUNS_JSON="${FETCH_BODY}" +PLAYBOOK_RUNS_STATUS="${FETCH_STATUS}" + +fetch_json "${BASE_URL}/api/v1/ops/activity-stream?limit=8" +ACTIVITY_JSON="${FETCH_BODY}" +ACTIVITY_STATUS="${FETCH_STATUS}" + +if [[ "${VIEW_MODE}" != "summary" ]]; then + echo "[1/9] api health" + printf 'status=%s\n%s\n\n' "${HEALTH_STATUS}" "${HEALTH_JSON}" + + echo "[2/9] ops contracts" + printf 'status=%s\n%s\n\n' "${CONTRACTS_STATUS}" "${CONTRACTS_JSON}" + + echo "[3/9] ops link snapshot" + printf 'status=%s\n%s\n\n' "${LINK_SNAPSHOT_STATUS}" "${LINK_SNAPSHOT_JSON}" + + echo "[4/9] ops overview" + printf 'status=%s\n%s\n\n' "${OVERVIEW_STATUS}" "${OVERVIEW_JSON}" + + echo "[5/9] managed nodes" + printf 'status=%s\n%s\n\n' "${NODES_STATUS}" "${NODES_JSON}" + + echo "[6/9] release launchpad" + printf 'status=%s\n%s\n\n' "${LAUNCHPAD_STATUS}" "${LAUNCHPAD_JSON}" + + echo "[7/9] recent playbook runs" + printf 'status=%s\n%s\n\n' "${PLAYBOOK_RUNS_STATUS}" "${PLAYBOOK_RUNS_JSON}" + + echo "[8/9] recent activity stream" + printf 'status=%s\n%s\n\n' "${ACTIVITY_STATUS}" "${ACTIVITY_JSON}" +fi + +echo "[9/9] condensed stack summary" +STACK_BASE_URL="${BASE_URL}" python3 - <<'PY' \ + "${STACK_DIAGNOSIS_JSON}" \ + "${STACK_DIAGNOSIS_STATUS}" \ + "${HEALTH_JSON}" \ + "${HEALTH_STATUS}" \ + "${CONTRACTS_JSON}" \ + "${CONTRACTS_STATUS}" \ + "${LINK_SNAPSHOT_JSON}" \ + "${LINK_SNAPSHOT_STATUS}" \ + "${OVERVIEW_JSON}" \ + "${OVERVIEW_STATUS}" \ + "${NODES_JSON}" \ + "${NODES_STATUS}" \ + "${LAUNCHPAD_JSON}" \ + "${LAUNCHPAD_STATUS}" \ + "${PLAYBOOK_RUNS_JSON}" \ + "${PLAYBOOK_RUNS_STATUS}" \ + "${ACTIVITY_JSON}" \ + "${ACTIVITY_STATUS}" +import json +import os +import sys + + +def ensure_str(value): + return value if isinstance(value, str) else "" + + +def load_payload(raw_text: str) -> dict: + try: + return json.loads(raw_text) + except Exception: + return {} + + +def as_data(payload: dict) -> dict: + if isinstance(payload.get("data"), dict): + return dict(payload.get("data") or {}) + return {} + + +def issue_rank(severity: str) -> int: + return {"blocked": 3, "warning": 2, "info": 1}.get(str(severity or "").strip(), 0) + + +def command(*parts: str) -> str: + tokens = ["bash", "domain-api/deploy/multi-region/drive_ops_center.sh"] + tokens.extend([str(part).strip() for part in parts if str(part).strip()]) + return " ".join(tokens) + + +def release_command(*parts: str) -> str: + tokens = ["bash", "domain-api/deploy/multi-region/drive_release_hub.sh"] + tokens.extend([str(part).strip() for part in parts if str(part).strip()]) + return " ".join(tokens) + + +stack_diagnosis_payload = load_payload(sys.argv[1]) +STACK_DIAGNOSIS_STATUS = str(sys.argv[2] or "000") +health = load_payload(sys.argv[3]) +health_status = str(sys.argv[4] or "000") +contracts_payload = load_payload(sys.argv[5]) +contracts_status = str(sys.argv[6] or "000") +link_snapshot_payload = load_payload(sys.argv[7]) +link_snapshot_status = str(sys.argv[8] or "000") +overview_payload = load_payload(sys.argv[9]) +overview_status = str(sys.argv[10] or "000") +nodes_root_payload = load_payload(sys.argv[11]) +nodes_status = str(sys.argv[12] or "000") +launchpad_payload = load_payload(sys.argv[13]) +launchpad_status_code = str(sys.argv[14] or "000") +playbook_runs_root_payload = load_payload(sys.argv[15]) +playbook_runs_status = str(sys.argv[16] or "000") +activity_root_payload = load_payload(sys.argv[17]) +activity_status = str(sys.argv[18] or "000") + +contracts = as_data(contracts_payload) +link_snapshot = as_data(link_snapshot_payload) +overview = as_data(overview_payload) +nodes_payload = as_data(nodes_root_payload) +launchpad = as_data(launchpad_payload) +playbook_runs_payload = as_data(playbook_runs_root_payload) +activity_payload = as_data(activity_root_payload) +stack_base_url = str(os.environ.get("STACK_BASE_URL") or "").strip() +health_build = dict(health.get("build") or {}) +health_route_surface = dict(health_build.get("route_surface") or {}) +stack_diagnosis = as_data(stack_diagnosis_payload) +diagnosis = dict(stack_diagnosis.get("diagnosis") or {}) + +contracts_rows = list(contracts.get("contracts") or []) +nodes = list(nodes_payload.get("nodes") or []) +nodes_summary = dict(nodes_payload.get("summary") or {}) +playbook_runs = list(playbook_runs_payload.get("runs") or playbook_runs_payload.get("playbook_runs") or []) +activities = list(activity_payload.get("items") or activity_payload.get("activities") or []) + +recommendation = dict(overview.get("recommendation") or {}) +execution_scene = dict(overview.get("execution_scene") or {}) +log_sync = dict(execution_scene.get("log_sync") or overview.get("log_sync") or {}) +release_hub = dict(overview.get("release_hub") or {}) +default_rollout_gate = dict(release_hub.get("default_rollout_gate") or {}) +launchpad_status = dict(launchpad.get("launchpad_status") or {}) +latest_release = dict(launchpad.get("latest_release") or release_hub.get("latest_release") or {}) + +problem_nodes = [] +for node in nodes: + node_code = str(node.get("node_code") or "").strip() + if not node_code: + continue + queue_state = str(node.get("delivery_queue_state") or "").strip() + agent_state = str(node.get("agent_state") or "").strip() + remote_access_state = str(node.get("remote_access_state") or "").strip() + if queue_state == "dead_letter" or agent_state not in {"online", "online_busy"} or remote_access_state in { + "agent_pending", + "disabled", + "unmanaged", + }: + problem_nodes.append( + { + "node_code": node_code, + "agent_state": agent_state, + "remote_access_state": remote_access_state, + "delivery_queue_state": queue_state, + "participation_state": str(node.get("participation_state") or "").strip(), + } + ) + +problem_runs = [ + { + "run_code": str(item.get("run_code") or "").strip(), + "status": str(item.get("status") or "").strip(), + "status_label": str(item.get("status_label") or "").strip(), + "focus_step_key": str(item.get("focus_step_key") or "").strip(), + "focus_summary": str(item.get("focus_summary") or item.get("summary_text") or "").strip(), + } + for item in playbook_runs + if str(item.get("status") or "").strip() not in {"success", "completed", "healthy"} +] + +activity_counts = {} +for item in activities: + status_key = str(item.get("status") or item.get("job_status") or "unknown").strip() or "unknown" + activity_counts[status_key] = int(activity_counts.get(status_key, 0) or 0) + 1 + +issues = [] + + +def add_issue( + *, + code: str, + severity: str, + layer: str, + summary: str, + detail: str = "", + action_code: str = "", + focus_ref=None, + commands=None, +) -> None: + issues.append( + { + "code": str(code or "").strip(), + "severity": str(severity or "").strip() or "warning", + "layer": str(layer or "").strip(), + "summary": str(summary or "").strip(), + "detail": str(detail or "").strip(), + "action_code": str(action_code or "").strip(), + "focus_ref": dict(focus_ref or {}), + "commands": [str(item).strip() for item in list(commands or []) if str(item).strip()], + } + ) + + +surface_matrix = [ + ("stack_diagnosis", STACK_DIAGNOSIS_STATUS.startswith("2")), + ("api", health_status.startswith("2")), + ("contracts", contracts_status.startswith("2")), + ("link_snapshot", link_snapshot_status.startswith("2")), + ("overview", overview_status.startswith("2")), + ("managed_nodes", nodes_status.startswith("2")), + ("release_hub", launchpad_status_code.startswith("2")), + ("playbook_runs", playbook_runs_status.startswith("2")), + ("activity_stream", activity_status.startswith("2")), +] +surface_status_counts = { + "total": len(surface_matrix), + "available": sum(1 for _, available in surface_matrix if available), + "missing": sum(1 for _, available in surface_matrix if not available), +} +missing_surfaces = [name for name, available in surface_matrix if not available] + +if not health_status.startswith("2"): + add_issue( + code="api_unreachable", + severity="blocked", + layer="api", + summary="控制面 API 当前不可用，总检无法进入正式判断链路。", + detail=f"/health 返回 HTTP {health_status}。", + commands=[ + f"curl -s {stack_base_url}/health", + "systemctl status domaincheck-api --no-pager -l", + ], + ) + +stack_diagnosis_hint = ( + "当前 stack-diagnosis 没有成功返回，总检已退化为多 surface 拼装视图。" +) +if not STACK_DIAGNOSIS_STATUS.startswith("2"): + stack_diag_detail = ( + f"/api/v1/ops/stack-diagnosis 返回 HTTP {STACK_DIAGNOSIS_STATUS}。" + if str(STACK_DIAGNOSIS_STATUS).isdigit() and str(STACK_DIAGNOSIS_STATUS) != "000" + else "/api/v1/ops/stack-diagnosis 当前不可达或请求超时。" + ) + add_issue( + code="stack_diagnosis_unavailable", + severity="warning" if health_status.startswith("2") else "blocked", + layer="stack_diagnosis", + summary=stack_diagnosis_hint, + detail=( + stack_diag_detail + + " 建议先单独复跑 stack-diagnosis / stack-next，确认是不是 API 慢、路由未挂齐，或者服务未切到最新版本。" + ), + commands=[ + f"curl -s {stack_base_url}/api/v1/ops/stack-diagnosis?base_url={stack_base_url}", + f"bash domain-api/deploy/multi-region/drive_ops_center.sh stack-next {stack_base_url}", + "systemctl status domaincheck-api --no-pager -l", + ], + ) + +if health_status.startswith("2") and not contracts_status.startswith("2"): + add_issue( + code="ops_contracts_endpoint_missing", + severity="warning", + layer="contracts", + summary="控制面 API 已在线，但 ops/contracts 未正常返回，存在运行实例与当前代码契约不一致的风险。", + detail=( + f"/api/v1/ops/contracts 返回 HTTP {contracts_status}；" + "这通常意味着当前 systemd 进程没有跑到最新接口集，或部署包/服务重启未完全对齐。" + ), + commands=[ + f"curl -s {stack_base_url}/api/v1/ops/contracts", + "systemctl status domaincheck-api --no-pager -l", + f"bash domain-api/deploy/multi-region/drive_ops_center.sh runtime-refresh-recover {stack_base_url}", + ], + ) + +if health_status.startswith("2") and not health_build: + add_issue( + code="api_build_identity_missing", + severity="warning", + layer="api", + summary="控制面 health 还没有暴露 build 指纹，当前不利于快速判断服务是否真的切到最新部署内容。", + detail="建议升级到包含 build + route_surface 指纹的新版本后再继续多机联调。", + commands=[ + f"curl -s {stack_base_url}/health", + f"bash domain-api/deploy/multi-region/drive_ops_center.sh runtime-refresh-recover {stack_base_url}", + ], + ) + +if health_status.startswith("2") and health_route_surface and not bool(health_route_surface.get("surface_complete", True)): + add_issue( + code="api_route_surface_incomplete", + severity="warning", + layer="api.routes", + summary="当前运行中的控制面实例没有挂齐关键协议路由，存在代码与服务实例不一致的风险。", + detail=( + "缺少路由: " + + ", ".join([str(item).strip() for item in list(health_route_surface.get("missing_paths") or []) if str(item).strip()]) + ), + commands=[ + f"curl -s {stack_base_url}/health", + f"bash domain-api/deploy/multi-region/drive_ops_center.sh runtime-refresh-recover {stack_base_url}", + ], + ) + +if health_status.startswith("2") and missing_surfaces and any( + name in missing_surfaces for name in ("link_snapshot", "overview", "managed_nodes", "release_hub") +): + add_issue( + code="ops_surface_partial", + severity="warning", + layer="stack", + summary="海外单脑控制面关键观察层没有全部联通，总检已退化为部分收口。", + detail=f"当前缺失层: {', '.join(missing_surfaces)}", + commands=[ + f"bash domain-api/deploy/multi-region/drive_ops_center.sh stack-diagnosis {stack_base_url} full", + f"bash domain-api/deploy/multi-region/drive_ops_center.sh doctor {stack_base_url}", + ], + ) + +participating_nodes_total = int(log_sync.get("participating_nodes_total", log_sync.get("participating_node_count", 0)) or 0) +if bool(log_sync.get("enabled", False)) is False and participating_nodes_total > 0: + log_sync_action = "" + next_action_codes = list(link_snapshot.get("next_actions") or []) + for item in next_action_codes: + action_code = str(item.get("action_code") or "").strip() + if action_code.startswith("enable_log_sync"): + log_sync_action = action_code + break + if not log_sync_action: + log_sync_action = "enable_log_sync_key" + add_issue( + code="remote_log_sync_disabled", + severity="warning", + layer="overview.log_sync", + summary="现场已有参与检测节点，但远端日志回传仍关闭，海外控制面对执行细节仍处于半盲态。", + detail=( + f"参与检测节点 {participating_nodes_total} 台，" + f"已覆盖 {int(log_sync.get('covered_participating_nodes', log_sync.get('covered_participating_node_count', 0)) or 0)} 台。" + ), + action_code=log_sync_action, + commands=[ + f"bash domain-api/deploy/multi-region/drive_ops_center.sh driver-resolve {stack_base_url} {log_sync_action}", + ], + ) + +managed_enabled = int(nodes_summary.get("managed_enabled", 0) or 0) +remote_access_ready = int(nodes_summary.get("remote_access_ready", 0) or 0) +if managed_enabled > 0 and remote_access_ready == 0: + add_issue( + code="managed_nodes_agent_pending", + severity="blocked", + layer="managed_nodes", + summary="托管节点虽然已经纳入 Ops Center，但 0 台 remote-agent 就绪，海外主控还不能统一下发标准执行动作。", + detail=( + f"managed_enabled={managed_enabled}，remote_access_ready={remote_access_ready}；" + "当前节点多数仍处于 agent_pending / runtime_only / pending_bootstrap。" + ), + action_code=str(launchpad_status.get("recommended_action_code") or "fix_managed_nodes"), + focus_ref=dict(launchpad_status.get("focus_ref") or {}), + commands=[ + f"bash domain-api/deploy/multi-region/drive_ops_center.sh driver-resolve {stack_base_url} fix_managed_nodes", + f"bash domain-api/deploy/multi-region/drive_ops_center.sh doctor {stack_base_url}", + ], + ) + +queue_dead_letter_nodes = int(nodes_summary.get("queue_dead_letter_nodes", 0) or 0) +queue_dead_letter_records = int(nodes_summary.get("queue_dead_letter_records", 0) or 0) +if queue_dead_letter_nodes > 0 or queue_dead_letter_records > 0: + add_issue( + code="delivery_queue_dead_letter", + severity="warning", + layer="managed_nodes.queue", + summary="存在 Node Agent 回执死信，后续自动化动作可能反复失败或无法收敛。", + detail=( + f"queue_dead_letter_nodes={queue_dead_letter_nodes}，" + f"queue_dead_letter_records={queue_dead_letter_records}" + ), + commands=[ + f"bash domain-api/deploy/multi-region/drive_ops_center.sh doctor {stack_base_url}", + ], + ) + +if str(default_rollout_gate.get("status") or "").strip() == "blocked": + add_issue( + code="release_rollout_gate_blocked", + severity="warning", + layer="release_hub", + summary="ReleaseHub 默认门禁当前阻断，说明虽然发布包已准备好，但正式 Rollout 还不具备条件。", + detail=str(default_rollout_gate.get("summary_text") or default_rollout_gate.get("summary") or "").strip(), + action_code=str(launchpad_status.get("recommended_action_code") or "").strip(), + focus_ref=dict(default_rollout_gate.get("focus_ref") or launchpad_status.get("focus_ref") or {}), + commands=[ + f"bash domain-api/deploy/multi-region/drive_ops_center.sh release-launchpad {stack_base_url}", + f"bash domain-api/deploy/multi-region/drive_ops_center.sh driver-resolve {stack_base_url} {str(launchpad_status.get('recommended_action_code') or 'fix_managed_nodes').strip()}", + ], + ) + +if problem_runs: + add_issue( + code="playbook_runs_need_attention", + severity="warning", + layer="playbook_runs", + summary="最近存在未完全收口的 playbook run，建议先确认执行序列是否停在中间步骤。", + detail=f"problem_runs_total={len(problem_runs)}", + commands=[ + f"bash domain-api/deploy/multi-region/drive_ops_center.sh doctor {stack_base_url}", + ], + ) + +ordered_action_candidates = [ + { + "source": "overview.primary", + "action_code": str(recommendation.get("primary_action_code") or "").strip(), + "title": str(recommendation.get("title") or "").strip(), + "reason": "来自 overview.recommendation.primary_action_code", + "focus_ref": dict(recommendation.get("focus_ref") or {}), + }, + { + "source": "release_hub.launchpad", + "action_code": str(launchpad_status.get("recommended_action_code") or "").strip(), + "title": str(launchpad_status.get("status_label") or "").strip(), + "reason": "来自 release_launchpad.launchpad_status.recommended_action_code", + "focus_ref": dict(launchpad_status.get("focus_ref") or {}), + }, + { + "source": "overview.secondary", + "action_code": str(recommendation.get("secondary_action_code") or "").strip(), + "title": str(recommendation.get("title") or "").strip(), + "reason": "来自 overview.recommendation.secondary_action_code", + "focus_ref": dict(recommendation.get("focus_ref") or {}), + }, +] +for issue in issues: + issue_action_code = str(issue.get("action_code") or "").strip() + if issue_action_code: + ordered_action_candidates.append( + { + "source": f"issue:{issue.get('code')}", + "action_code": issue_action_code, + "title": str(issue.get("summary") or "").strip(), + "reason": str(issue.get("detail") or issue.get("summary") or "").strip(), + "focus_ref": dict(issue.get("focus_ref") or {}), + } + ) + +recommended_actions = [] +seen_action_codes = set() +for item in ordered_action_candidates: + action_code = str(item.get("action_code") or "").strip() + if not action_code or action_code in seen_action_codes: + continue + seen_action_codes.add(action_code) + recommended_actions.append( + { + "action_code": action_code, + "source": str(item.get("source") or "").strip(), + "title": str(item.get("title") or "").strip(), + "reason": str(item.get("reason") or "").strip(), + "focus_ref": dict(item.get("focus_ref") or {}), + "command": f"bash domain-api/deploy/multi-region/drive_ops_center.sh driver-resolve {stack_base_url} {action_code}", + } + ) + +blocking_issue_total = sum(1 for item in issues if str(item.get("severity") or "").strip() == "blocked") +warning_issue_total = sum(1 for item in issues if str(item.get("severity") or "").strip() == "warning") +info_issue_total = sum(1 for item in issues if str(item.get("severity") or "").strip() == "info") + +surface_status = "healthy" +if not health_status.startswith("2"): + surface_status = "broken" +elif missing_surfaces: + surface_status = "partial" + +automation_status = "ready" +if blocking_issue_total > 0: + automation_status = "blocked" +elif warning_issue_total > 0: + automation_status = "attention" + +stack_status = "ready" +if surface_status == "broken" or automation_status == "blocked": + stack_status = "blocked" +elif surface_status != "healthy" or automation_status != "ready": + stack_status = "attention" + +top_issue = None +if issues: + top_issue = sorted( + issues, + key=lambda item: ( + -issue_rank(str(item.get("severity") or "").strip()), + str(item.get("code") or "").strip(), + ), + )[0] + +next_step = {} +if recommended_actions: + first_action = dict(recommended_actions[0] or {}) + next_step = { + "action_code": str(first_action.get("action_code") or "").strip(), + "source": str(first_action.get("source") or "").strip(), + "reason": str(first_action.get("reason") or "").strip(), + "command": str(first_action.get("command") or "").strip(), + "focus_ref": dict(first_action.get("focus_ref") or {}), + } +elif top_issue: + next_step = { + "action_code": "", + "source": f"issue:{str(top_issue.get('code') or '').strip()}", + "reason": str(top_issue.get("summary") or "").strip(), + "command": "", + "focus_ref": dict(top_issue.get("focus_ref") or {}), + } + +next_step_focus_ref = dict(next_step.get("focus_ref") or {}) +launchpad_target_node_code = str( + diagnosis.get("launchpad_recommended_target_node_code") + or next_step_focus_ref.get("node_code") + or "" +).strip() +launchpad_recovery_label = str(diagnosis.get("launchpad_recommended_recovery_label") or "").strip() +launchpad_recovery_summary = str(diagnosis.get("launchpad_recommended_recovery_summary") or next_step.get("reason") or "").strip() +handover_next_action_code = str(next_step.get("action_code") or "").strip() + +scene_log_missing_nodes = list( + log_sync.get("missing_sample_node_codes") + or log_sync.get("missing_participating_nodes") + or [] +) +scene_log_target_node_code = "" +if scene_log_missing_nodes: + scene_log_target_node_code = str(scene_log_missing_nodes[0] or "").strip() +elif problem_nodes: + scene_log_target_node_code = str(problem_nodes[0].get("node_code") or "").strip() + +recommended_commands = { + "stack_summary": f"bash domain-api/deploy/multi-region/check_ops_center_stack.sh {stack_base_url} summary", + "doctor": command("doctor", stack_base_url), + "driver_feed": command("driver-feed", stack_base_url), + "driver_focus_preview": command("driver-focus-preview", stack_base_url), + "driver_focus_run": command("driver-focus-run", stack_base_url), + "go_live_recover": command("go-live-recover", stack_base_url), + "log_sync_recover": command("log-sync-recover", stack_base_url, "key"), + "log_sync_recover_full": command("log-sync-recover", stack_base_url, "full"), + "agent_gap_check": command("agent-gap-check", stack_base_url), + "agent_gap_recover": command("agent-gap-recover", stack_base_url), + "api_restart": command("runtime-refresh-recover", stack_base_url), + "node_onboarding": ( + command("node-onboarding", stack_base_url, str(problem_nodes[0].get("node_code") or "").strip()) + if problem_nodes and str(problem_nodes[0].get("node_code") or "").strip() + else "" + ), + "node_recover": ( + command("node-recover", stack_base_url, str(problem_nodes[0].get("node_code") or "").strip()) + if problem_nodes and str(problem_nodes[0].get("node_code") or "").strip() + else "" + ), + "node_bootstrap_preview": ( + command("node-bootstrap-preview", stack_base_url, str(problem_nodes[0].get("node_code") or "").strip()) + if problem_nodes and str(problem_nodes[0].get("node_code") or "").strip() + else "" + ), + "node_bootstrap_run": ( + command("node-bootstrap-run", stack_base_url, launchpad_target_node_code, "cli") + if launchpad_target_node_code + else command("node-bootstrap-run", stack_base_url, str(problem_nodes[0].get("node_code") or "").strip(), "cli") + if launchpad_target_node_code or (problem_nodes and str(problem_nodes[0].get("node_code") or "").strip()) + else "" + ), + "node_acceptance_run": ( + command("node-acceptance-run", stack_base_url, launchpad_target_node_code, "cli") + if launchpad_target_node_code + else command("node-acceptance-run", stack_base_url, str(problem_nodes[0].get("node_code") or "").strip(), "cli") + if launchpad_target_node_code or (problem_nodes and str(problem_nodes[0].get("node_code") or "").strip()) + else "" + ), + "node_acceptance_plan": ( + command("node-acceptance-plan", stack_base_url, str(problem_nodes[0].get("node_code") or "").strip()) + if problem_nodes and str(problem_nodes[0].get("node_code") or "").strip() + else "" + ), + "scene_log": ( + command("scene-node-log", stack_base_url, scene_log_target_node_code, "120", "key") + if scene_log_target_node_code + else "" + ), + "release_launchpad": release_command(stack_base_url, "launchpad"), + "release_preview_latest": release_command(stack_base_url, "preview-latest"), + "jobs": command("activity-stream", stack_base_url, "kind=ops_job"), + "node_handover": ( + command("node-handover", stack_base_url, str(problem_nodes[0].get("node_code") or "").strip()) + if problem_nodes and str(problem_nodes[0].get("node_code") or "").strip() + else command("stack-first-gap-handover", stack_base_url) + ), +} + +operator_decision = { + "lane": "steady", + "priority": "low", + "reason_code": "steady_state", + "title": "当前进入稳定观察态", + "summary": "当前总检没有发现新的阻断问题，可继续按既有链路观察与推进。", + "next_focus": next_step_focus_ref, + "primary_command_key": "driver_feed", + "secondary_command_key": "jobs", +} + +if str(log_sync.get("enabled", False)).lower() in {"false", ""} and participating_nodes_total > 0: + operator_decision = { + "lane": "observability", + "priority": "high", + "reason_code": "scene_log_disabled", + "title": "先恢复远端现场日志可见性", + "summary": "当前已有参与检测节点，但远端日志回传仍关闭，海外控制面仍处于半盲态。", + "next_focus": {"kind": "node_scene_log", "node_code": scene_log_target_node_code, "mode": "key", "limit": 80} + if scene_log_target_node_code + else {}, + "primary_command_key": "log_sync_recover", + "secondary_command_key": "scene_log", + } +elif managed_enabled > 0 and remote_access_ready == 0: + primary_command_key = "agent_gap_recover" + secondary_command_key = "node_handover" + title = "先补齐节点接管与 remote-agent 可达性" + summary = ( + f"managed_enabled={managed_enabled}，remote_access_ready={remote_access_ready}；" + "当前还不能从海外控制面统一下发标准动作。" + ) + if handover_next_action_code == "bootstrap_run" and launchpad_target_node_code: + primary_command_key = "node_bootstrap_run" + secondary_command_key = "node_handover" + title = "先跑首台节点接入收口" + summary = ( + launchpad_recovery_summary + or join_text( + launchpad_recovery_label, + launchpad_target_node_code, + "当前真正阻断已经收敛到首台节点接入收口。" + ) + ) + elif handover_next_action_code == "run_acceptance" and launchpad_target_node_code: + primary_command_key = "node_acceptance_run" + secondary_command_key = "node_handover" + title = "先跑首台节点接管验收" + summary = ( + launchpad_recovery_summary + or join_text( + launchpad_recovery_label, + launchpad_target_node_code, + "当前真正阻断已经收敛到首台节点接管验收。" + ) + ) + operator_decision = { + "lane": "node_handover", + "priority": "high", + "reason_code": "managed_nodes_agent_pending", + "title": title, + "summary": summary, + "next_focus": next_step_focus_ref, + "primary_command_key": primary_command_key, + "secondary_command_key": secondary_command_key, + } +elif str(default_rollout_gate.get("status") or "").strip() == "blocked": + operator_decision = { + "lane": "release", + "priority": "medium", + "reason_code": "release_rollout_gate_blocked", + "title": "先检查发布门禁与 rollout 阻断", + "summary": str(default_rollout_gate.get("summary_text") or default_rollout_gate.get("summary") or "").strip(), + "next_focus": dict(default_rollout_gate.get("focus_ref") or launchpad_status.get("focus_ref") or {}), + "primary_command_key": "release_launchpad", + "secondary_command_key": "release_preview_latest", + } +elif any(str(item.get("code") or "").strip() == "runtime_build_schema_stale" for item in issues): + operator_decision = { + "lane": "runtime_recovery", + "priority": "high", + "reason_code": "runtime_build_schema_stale", + "title": "先执行运行时刷新与上线复核", + "summary": "当前已经识别到仓库代码与运行中 API schema 之间存在漂移，先走统一恢复链，再继续节点接管或发布判断。", + "next_focus": dict(next_step.get("focus_ref") or {}), + "primary_command_key": "go_live_recover", + "secondary_command_key": "api_restart", + } +elif problem_runs: + operator_decision = { + "lane": "ops_jobs", + "priority": "medium", + "reason_code": "playbook_runs_need_attention", + "title": "先确认未收口的 playbook run", + "summary": f"最近存在 {len(problem_runs)} 个未完全收口的 playbook run。", + "next_focus": dict(next_step.get("focus_ref") or {}), + "primary_command_key": "doctor", + "secondary_command_key": "jobs", + } + +next_actions = { + "primary_command": str(recommended_commands.get(str(operator_decision.get("primary_command_key") or "").strip()) or "").strip(), + "secondary_command": str(recommended_commands.get(str(operator_decision.get("secondary_command_key") or "").strip()) or "").strip(), +} + +quick_commands = [ + f"bash domain-api/deploy/multi-region/drive_ops_center.sh stack-diagnosis {stack_base_url} summary", + f"bash domain-api/deploy/multi-region/drive_ops_center.sh doctor {stack_base_url}", +] +for action in recommended_actions[:3]: + command = str(action.get("command") or "").strip() + if command and command not in quick_commands: + quick_commands.append(command) +if not contracts_status.startswith("2"): + contract_check_command = f"curl -s {stack_base_url}/api/v1/ops/contracts" + if contract_check_command not in quick_commands: + quick_commands.append(contract_check_command) + +operator_hints = [] +if stack_status == "blocked": + operator_hints.append("当前总检已经能看到现场，但海外单脑自动化执行链路仍未正式打通，先处理阻断问题再继续 Rollout 或批量运维。") +elif stack_status == "attention": + operator_hints.append("当前总检可用，但仍有缺口；建议先按 next_step 收口，再进入更细粒度脚本。") +else: + operator_hints.append("当前海外单脑控制面总检已基本就绪，可以把它作为后续所有联调和发布的固定起手式。") +if not contracts_status.startswith("2") and health_status.startswith("2"): + operator_hints.append("health 正常但 contracts 缺失，优先怀疑运行中的 API 实例不是这份最新代码，或服务重启后未真正切到最新部署内容。") +if bool(log_sync.get("enabled", False)) is False and participating_nodes_total > 0: + operator_hints.append("远端日志回传当前关闭，后续如果继续多机联调，海外控制面会继续处于半盲态。") + +summary = { + "diagnosis": { + "base_url": stack_base_url, + "surface_status": surface_status, + "automation_status": automation_status, + "stack_status": stack_status, + "issue_total": len(issues), + "blocking_issue_total": blocking_issue_total, + "warning_issue_total": warning_issue_total, + "info_issue_total": info_issue_total, + "missing_surfaces": missing_surfaces, + "launchpad_recommended_target_node_code": ensure_str(diagnosis.get("launchpad_recommended_target_node_code")), + "launchpad_recommended_recovery_label": ensure_str(diagnosis.get("launchpad_recommended_recovery_label")), + "launchpad_recommended_recovery_summary": ensure_str(diagnosis.get("launchpad_recommended_recovery_summary")), + "launchpad_onboarding_bootstrap_pending_nodes": int(diagnosis.get("launchpad_onboarding_bootstrap_pending_nodes", 0) or 0), + "launchpad_onboarding_acceptance_ready_nodes": int(diagnosis.get("launchpad_onboarding_acceptance_ready_nodes", 0) or 0), + "next_step": next_step, + "recommended_actions": recommended_actions[:5], + "operator_decision": operator_decision, + "next_actions": next_actions, + "issues": issues[:8], + "operator_hints": operator_hints, + "quick_commands": quick_commands, + "recommended_commands": recommended_commands, + }, + "api": { + "stack_diagnosis_http_status": STACK_DIAGNOSIS_STATUS, + "stack_diagnosis_available": STACK_DIAGNOSIS_STATUS.startswith("2"), + "http_status": health_status, + "available": health_status.startswith("2"), + "service": str(health.get("service") or ""), + "version": str(health.get("version") or ""), + "api_prefix": str(health.get("api_prefix") or ""), + "worker_mode": str(health.get("worker_mode") or ""), + "build": { + "source": str(health_build.get("source") or ""), + "package_name": str(health_build.get("package_name") or ""), + "generated_at": str(health_build.get("generated_at") or ""), + "commit_sha": str(health_build.get("commit_sha") or ""), + "commit_ref": str(health_build.get("commit_ref") or ""), + "manifest_path": str(health_build.get("manifest_path") or ""), + "route_surface_complete": bool(health_route_surface.get("surface_complete", False)) if health_route_surface else False, + "route_surface_missing": [ + str(item).strip() + for item in list(health_route_surface.get("missing_paths") or []) + if str(item).strip() + ], + }, + }, + "surface_matrix": { + "status_counts": surface_status_counts, + "items": [ + {"name": name, "available": bool(available)} + for name, available in surface_matrix + ], + }, + "contracts": { + "http_status": contracts_status, + "available": contracts_status.startswith("2"), + "registry_version": str(contracts.get("registry_version") or ""), + "schema_version": str(contracts.get("schema_version") or ""), + "contracts_total": len(contracts_rows), + "contract_keys": [str(item.get("key") or "").strip() for item in contracts_rows if str(item.get("key") or "").strip()], + "error": str(contracts_payload.get("detail") or contracts_payload.get("message") or ""), + }, + "link_snapshot": { + "http_status": link_snapshot_status, + "available": link_snapshot_status.startswith("2"), + "status": str(link_snapshot.get("status") or ""), + "headline": str(link_snapshot.get("headline") or ""), + "next_action_codes": [ + str(item.get("action_code") or "").strip() + for item in list(link_snapshot.get("next_actions") or []) + if str(item.get("action_code") or "").strip() + ], + "error": str(link_snapshot_payload.get("detail") or link_snapshot_payload.get("message") or ""), + }, + "overview": { + "http_status": overview_status, + "available": overview_status.startswith("2"), + "recommendation_key": str(recommendation.get("key") or ""), + "recommendation_title": str(recommendation.get("title") or ""), + "primary_action_code": str(recommendation.get("primary_action_code") or ""), + "secondary_action_code": str(recommendation.get("secondary_action_code") or ""), + "focus_ref": dict(recommendation.get("focus_ref") or {}), + "execution_scene_summary": str( + execution_scene.get("summary") + or (execution_scene.get("participation_summary") or {}).get("summary") + or execution_scene.get("phase_detail") + or "" + ), + "execution_scene_counts": dict(execution_scene.get("counts") or {}), + "log_sync": { + "status": str(log_sync.get("status") or log_sync.get("state") or ""), + "status_label": str(log_sync.get("status_label") or ""), + "mode": str(log_sync.get("mode") or ""), + "mode_label": str(log_sync.get("mode_label") or ""), + "enabled": bool(log_sync.get("enabled", False)), + "covered_participating_nodes": int( + log_sync.get("covered_participating_nodes", log_sync.get("covered_participating_node_count", 0)) or 0 + ), + "participating_nodes_total": int( + log_sync.get("participating_nodes_total", log_sync.get("participating_node_count", 0)) or 0 + ), + "missing_sample_node_codes": list( + log_sync.get("missing_sample_node_codes") + or log_sync.get("missing_participating_nodes") + or [] + ), + }, + "error": str(overview_payload.get("detail") or overview_payload.get("message") or ""), + }, + "managed_nodes": { + "http_status": nodes_status, + "available": nodes_status.startswith("2"), + "total": int(nodes_summary.get("total", len(nodes)) or 0), + "managed_total": int(nodes_summary.get("managed_total", 0) or 0), + "managed_enabled": int(nodes_summary.get("managed_enabled", 0) or 0), + "online": int(nodes_summary.get("online", 0) or 0), + "agent_ready": int(nodes_summary.get("agent_ready", 0) or 0), + "ssh_ready": int(nodes_summary.get("ssh_ready", 0) or 0), + "remote_access_ready": int(nodes_summary.get("remote_access_ready", 0) or 0), + "participating": int(nodes_summary.get("participating", 0) or 0), + "dispatch_active": int(nodes_summary.get("dispatch_active", 0) or 0), + "standby": int(nodes_summary.get("standby", 0) or 0), + "load_syncing": int(nodes_summary.get("load_syncing", 0) or 0), + "queue_retrying_nodes": int(nodes_summary.get("queue_retrying_nodes", 0) or 0), + "queue_dead_letter_nodes": int(nodes_summary.get("queue_dead_letter_nodes", 0) or 0), + "queue_pending_records": int(nodes_summary.get("queue_pending_records", 0) or 0), + "queue_dead_letter_records": int(nodes_summary.get("queue_dead_letter_records", 0) or 0), + "agent_state_counts": dict(nodes_summary.get("status_counts") or {}), + "remote_access_state_counts": dict(nodes_summary.get("remote_access_state_counts") or {}), + "delivery_queue_state_counts": dict(nodes_summary.get("delivery_queue_state_counts") or {}), + "problem_nodes": problem_nodes[:8], + "error": str(nodes_root_payload.get("detail") or nodes_root_payload.get("message") or ""), + }, + "release_hub": { + "http_status": launchpad_status_code, + "available": launchpad_status_code.startswith("2"), + "latest_release_version": str(latest_release.get("release_version") or ""), + "latest_release_status": str(latest_release.get("status") or ""), + "default_rollout_gate_status": str(default_rollout_gate.get("status") or ""), + "default_rollout_gate_label": str(default_rollout_gate.get("status_label") or ""), + "default_rollout_gate_summary": str(default_rollout_gate.get("summary_text") or default_rollout_gate.get("summary") or ""), + "default_rollout_gate_focus_ref": dict(default_rollout_gate.get("focus_ref") or {}), + "launchpad_status": str(launchpad_status.get("status") or ""), + "launchpad_status_label": str(launchpad_status.get("status_label") or ""), + "launchpad_summary": str(launchpad_status.get("summary_text") or launchpad_status.get("summary") or ""), + "recommended_action_code": str(launchpad_status.get("recommended_action_code") or ""), + "recommended_target_node_code": str(launchpad_status.get("recommended_target_node_code") or ""), + "recommended_recovery_label": str(launchpad_status.get("recommended_recovery_label") or ""), + "recommended_recovery_summary": str(launchpad_status.get("recommended_recovery_summary") or ""), + "onboarding_bootstrap_pending_nodes": int(launchpad_status.get("onboarding_bootstrap_pending_nodes", 0) or 0), + "onboarding_acceptance_ready_nodes": int(launchpad_status.get("onboarding_acceptance_ready_nodes", 0) or 0), + "recommended_execution_mode": str(launchpad_status.get("recommended_execution_mode") or ""), + "focus_ref": dict(launchpad_status.get("focus_ref") or {}), + "error": str(launchpad_payload.get("detail") or launchpad_payload.get("message") or ""), + }, + "playbook_runs": { + "http_status": playbook_runs_status, + "available": playbook_runs_status.startswith("2"), + "recent_total": len(playbook_runs), + "problem_runs_total": len(problem_runs), + "problem_runs": problem_runs[:5], + "error": str(playbook_runs_root_payload.get("detail") or playbook_runs_root_payload.get("message") or ""), + }, + "activity_stream": { + "http_status": activity_status, + "available": activity_status.startswith("2"), + "recent_total": len(activities), + "status_counts": activity_counts, + "top_items": [ + { + "kind": str(item.get("kind") or "").strip(), + "status": str(item.get("status") or item.get("job_status") or "").strip(), + "summary": str(item.get("summary_text") or item.get("summary") or "").strip(), + "focus_ref": dict(item.get("focus_ref") or {}), + } + for item in activities[:5] + ], + "error": str(activity_root_payload.get("detail") or activity_root_payload.get("message") or ""), + }, +} + +print(json.dumps(summary, ensure_ascii=False, indent=2)) +PY diff --git a/domain-api/deploy/multi-region/check_ops_contracts.sh b/domain-api/deploy/multi-region/check_ops_contracts.sh new file mode 100755 index 0000000..957bcc1 --- /dev/null +++ b/domain-api/deploy/multi-region/check_ops_contracts.sh @@ -0,0 +1,55 @@ +#!/usr/bin/env bash +set -euo pipefail + +BASE_URL="${1:-http://127.0.0.1:8100}" + +contracts_json="$(curl -fsS "${BASE_URL}/api/v1/ops/contracts")" + +echo "[1/3] ops contracts registry" +printf '%s\n\n' "${contracts_json}" + +echo "[2/3] condensed summary" +python3 - <<'PY' "${contracts_json}" +import json +import sys + +payload = json.loads(sys.argv[1]) +data = payload.get("data") or {} +contracts = list(data.get("contracts") or []) + +summary = { + "registry_version": str(data.get("registry_version") or ""), + "schema_version": str(data.get("schema_version") or ""), + "contracts_total": len(contracts), + "contract_keys": [str(item.get("key") or "") for item in contracts if str(item.get("key") or "")], + "docs_root": str(data.get("docs_root") or ""), + "discovery_entrypoints": list(data.get("discovery_entrypoints") or []), +} + +print(json.dumps(summary, ensure_ascii=False, indent=2)) +PY +echo + +echo "[3/3] contract rows" +python3 - <<'PY' "${contracts_json}" +import json +import sys + +payload = json.loads(sys.argv[1]) +data = payload.get("data") or {} +contracts = list(data.get("contracts") or []) + +rows = [ + { + "key": str(item.get("key") or ""), + "version": str(item.get("version") or ""), + "status": str(item.get("status") or ""), + "primary_endpoint": str(item.get("primary_endpoint") or ""), + "schema_doc_path": str(item.get("schema_doc_path") or ""), + "consumers": list(item.get("consumers") or []), + } + for item in contracts +] + +print(json.dumps(rows, ensure_ascii=False, indent=2)) +PY diff --git a/domain-api/deploy/multi-region/check_ops_inspection.sh b/domain-api/deploy/multi-region/check_ops_inspection.sh new file mode 100755 index 0000000..44a9865 --- /dev/null +++ b/domain-api/deploy/multi-region/check_ops_inspection.sh @@ -0,0 +1,72 @@ +#!/usr/bin/env bash +set -euo pipefail + +BASE_URL="${1:-http://127.0.0.1:8100}" +CURL_CONNECT_TIMEOUT="${CURL_CONNECT_TIMEOUT:-3}" +CURL_MAX_TIME="${CURL_MAX_TIME:-10}" + +curl_json() { + curl -fsS \ + --connect-timeout "${CURL_CONNECT_TIMEOUT}" \ + --max-time "${CURL_MAX_TIME}" \ + "$1" +} + +INSPECTION_JSON="$(curl_json "${BASE_URL}/api/v1/ops/inspection-overview?limit=120")" + +echo "[1/5] inspection overview" +printf '%s\n\n' "${INSPECTION_JSON}" + +echo "[2/5] condensed summary" +python3 - <<'PY' "${INSPECTION_JSON}" +import json +import sys + +payload = json.loads(sys.argv[1]).get("data") or {} +print(json.dumps({ + "rows_total": int(payload.get("rows_total") or 0), + "filtered_rows_total": int(payload.get("filtered_rows_total") or 0), + "visible_nodes_total": int(payload.get("visible_nodes_total") or 0), + "eligible_nodes_total": int(payload.get("eligible_nodes_total") or 0), + "handover_gap_nodes_total": int(payload.get("handover_gap_nodes_total") or 0), + "participating_nodes_total": int(payload.get("participating_nodes_total") or 0), + "summary": dict(payload.get("summary") or {}), + "available_problem_kind_counts": dict(payload.get("available_problem_kind_counts") or {}), + "filters": dict(payload.get("filters") or {}), +}, ensure_ascii=False, indent=2)) +PY +echo + +echo "[3/5] priority queue" +python3 - <<'PY' "${INSPECTION_JSON}" +import json +import sys + +payload = json.loads(sys.argv[1]).get("data") or {} +rows = list(payload.get("priority_queue") or []) +print(json.dumps({ + "total": len(rows), + "items": [ + { + "node_code": str(item.get("node_code") or ""), + "overall_status": str(item.get("overall_status") or ""), + "problem_kind": str(item.get("problem_kind") or ""), + "problem_label": str(item.get("problem_label") or ""), + "recommended_action_code": str(item.get("recommended_action_code") or ""), + "last_updated_at": str(item.get("last_updated_at") or ""), + "issue_summary": str(item.get("issue_summary") or ""), + } + for item in rows[:12] + ], +}, ensure_ascii=False, indent=2)) +PY +echo + +echo "[4/5] attention only" +curl_json "${BASE_URL}/api/v1/ops/inspection-overview?limit=80&status=attention&only_problem=true" +echo +echo + +echo "[5/5] participating problems only" +curl_json "${BASE_URL}/api/v1/ops/inspection-overview?limit=80&only_problem=true&only_participating=true" +echo diff --git a/domain-api/deploy/multi-region/check_ops_link_snapshot.sh b/domain-api/deploy/multi-region/check_ops_link_snapshot.sh new file mode 100644 index 0000000..9047c1b --- /dev/null +++ b/domain-api/deploy/multi-region/check_ops_link_snapshot.sh @@ -0,0 +1,35 @@ +#!/usr/bin/env bash +set -euo pipefail + +BASE_URL="${1:-http://127.0.0.1:8100}" +PYTHON_BIN="${PYTHON_BIN:-python3}" + +echo "[1/2] ops link snapshot" +SNAPSHOT_JSON="$(curl -fsS "${BASE_URL}/api/v1/ops/link-snapshot")" +echo "${SNAPSHOT_JSON}" +echo +echo + +echo "[2/2] condensed summary" +"${PYTHON_BIN}" - <<'PY' "$SNAPSHOT_JSON" +import json +import sys + +payload = json.loads(sys.argv[1]).get("data", {}) +print(f"status: {payload.get('status', '-')}") +print(f"headline: {payload.get('headline', '-')}") +print() +print("summary:") +for line in payload.get("summary_lines", []) or []: + print(f"- {line}") + +actions = payload.get("next_actions", []) or [] +if actions: + print() + print("next_actions:") + for item in actions: + label = str(item.get("label", "") or "").strip() or "-" + action_code = str(item.get("action_code", "") or "").strip() or "-" + node_codes = "、".join(str(x).strip() for x in (item.get("node_codes", []) or []) if str(x).strip()) or "-" + print(f"- {label} / {action_code} / nodes: {node_codes}") +PY diff --git a/domain-api/deploy/multi-region/check_ops_plane.sh b/domain-api/deploy/multi-region/check_ops_plane.sh new file mode 100644 index 0000000..17f5dc7 --- /dev/null +++ b/domain-api/deploy/multi-region/check_ops_plane.sh @@ -0,0 +1,609 @@ +#!/usr/bin/env bash +set -euo pipefail + +BASE_URL="${1:-http://127.0.0.1:8100}" +CURL_CONNECT_TIMEOUT="${CURL_CONNECT_TIMEOUT:-3}" +CURL_MAX_TIME="${CURL_MAX_TIME:-10}" + +curl_json() { + curl -fsS \ + --connect-timeout "${CURL_CONNECT_TIMEOUT}" \ + --max-time "${CURL_MAX_TIME}" \ + "$1" +} + +curl_post_json() { + local url="$1" + local body="$2" + curl -fsS \ + --connect-timeout "${CURL_CONNECT_TIMEOUT}" \ + --max-time "${CURL_MAX_TIME}" \ + -X POST "${url}" \ + -H 'Content-Type: application/json' \ + -d "${body}" +} + +OVERVIEW_JSON="$(curl_json "${BASE_URL}/api/v1/ops/overview")" +NODES_JSON="$(curl_json "${BASE_URL}/api/v1/ops/nodes")" +JOBS_JSON="$(curl_json "${BASE_URL}/api/v1/ops/jobs")" +ACTIVITY_JSON="$(curl_json "${BASE_URL}/api/v1/ops/activity-stream")" +LAUNCHPAD_JSON="$(curl_json "${BASE_URL}/api/v1/ops/releases/launchpad")" +LATEST_RELEASE_JSON="$(curl_json "${BASE_URL}/api/v1/ops/releases/latest")" +POLICY_PREVIEW_JSON="$( + curl_post_json "${BASE_URL}/api/v1/ops/policy/preview" \ + '{"action":"runtime.restart_api","target_type":"node","target_node_code":"mainland-controller-01"}' +)" + +echo "[1/12] ops overview" +printf '%s\n\n' "${OVERVIEW_JSON}" + +echo "[2/12] top recommendation" +python3 - <<'PY' "${OVERVIEW_JSON}" +import json +import sys + +payload = json.loads(sys.argv[1]) +data = payload.get("data") or {} +recommendation = dict(data.get("recommendation") or {}) +print(json.dumps({ + "source": str(recommendation.get("source") or ""), + "key": str(recommendation.get("key") or ""), + "priority": str(recommendation.get("priority") or ""), + "summary": str(recommendation.get("summary") or ""), + "primary_action_code": str(recommendation.get("primary_action_code") or ""), + "secondary_action_code": str(recommendation.get("secondary_action_code") or ""), + "node_codes": list(recommendation.get("node_codes") or []), +}, ensure_ascii=False, indent=2)) +PY +echo + +echo "[3/12] driver recommendations (condensed)" +python3 - <<'PY' "${OVERVIEW_JSON}" +import json +import sys + +payload = json.loads(sys.argv[1]) +data = payload.get("data") or {} +recommendations = list(data.get("driver_recommendations") or []) +summary = { + "total": len(recommendations), + "items": [ + { + "key": str(item.get("key") or ""), + "title": str(item.get("title") or ""), + "level_label": str(item.get("level_label") or ""), + "primary_action_code": str(item.get("primary_action_code") or ""), + "secondary_action_code": str(item.get("secondary_action_code") or ""), + "meta_text": str(item.get("meta_text") or ""), + } + for item in recommendations[:8] + ], +} +print(json.dumps(summary, ensure_ascii=False, indent=2)) +PY +echo + +echo "[4/12] ops capabilities" +curl_json "${BASE_URL}/api/v1/ops/capabilities" +echo +echo + +echo "[5/12] ops blueprint" +curl_json "${BASE_URL}/api/v1/ops/blueprint" +echo +echo + +echo "[6/12] ops runbook" +curl_json "${BASE_URL}/api/v1/ops/runbook" +echo +echo + +echo "[7/12] ops nodes" +printf '%s\n' "${NODES_JSON}" +echo +echo + +echo "[8/12] ops jobs" +printf '%s\n' "${JOBS_JSON}" +echo +echo + +echo "[9/12] ops activity stream" +printf '%s\n' "${ACTIVITY_JSON}" +echo +echo + +echo "[10/12] release launchpad" +printf '%s\n' "${LAUNCHPAD_JSON}" +echo +echo + +echo "[11/12] latest release" +printf '%s\n' "${LATEST_RELEASE_JSON}" +echo +echo + +echo "[12/12] policy preview sample" +printf '%s\n' "${POLICY_PREVIEW_JSON}" +echo +echo + +echo "[13/13] condensed summary" +python3 - <<'PY' \ + "${BASE_URL}" \ + "${OVERVIEW_JSON}" \ + "${NODES_JSON}" \ + "${JOBS_JSON}" \ + "${ACTIVITY_JSON}" \ + "${LAUNCHPAD_JSON}" \ + "${LATEST_RELEASE_JSON}" \ + "${POLICY_PREVIEW_JSON}" +import json +import sys + + +def as_dict(payload): + try: + raw = json.loads(payload) + except Exception: + return {} + return raw.get("data") or {} + + +def ensure_dict(value): + return value if isinstance(value, dict) else {} + + +def ensure_list(value): + return value if isinstance(value, list) else [] + + +def ensure_str(value): + return value if isinstance(value, str) else "" + + +def command(name, *parts): + tokens = [ + "bash", + "domain-api/deploy/multi-region/drive_ops_center.sh", + name, + base_url, + ] + tokens.extend([str(part) for part in parts if str(part or "").strip()]) + return " ".join(tokens) + + +def release_command(name, *parts): + tokens = [ + "bash", + "domain-api/deploy/multi-region/drive_release_hub.sh", + base_url, + name, + ] + tokens.extend([str(part) for part in parts if str(part or "").strip()]) + return " ".join(tokens) + + +def build_operator_decision(): + remote_access_state_counts = ensure_dict(node_summary.get("remote_access_state_counts")) + pending_bootstrap = int(node_summary.get("pending_bootstrap", 0) or 0) + runtime_only = int(node_summary.get("runtime_only", 0) or 0) + online_total = int(node_summary.get("online", 0) or 0) + participating_total = int(scene_log.get("participating_node_count", 0) or 0) + launchpad_blocked = bool(launchpad_workers.get("blocked")) or bool(launchpad_controls.get("blocked")) + running_total = int(jobs_summary.get("running_count", 0) or 0) + queued_total = int(jobs_summary.get("queued_count", 0) or 0) + + if scene_log.get("status") == "disabled" and participating_total > 0: + return { + "lane": "observability", + "priority": "high", + "reason_code": "scene_log_disabled", + "title": "先恢复远端现场日志可见性", + "summary": ensure_str(scene_log.get("summary")), + "next_focus": ensure_dict(scene_log.get("focus_ref")), + "primary_command_key": "scene_log", + "secondary_command_key": "driver_focus_preview", + } + + if pending_bootstrap > 0 or remote_access_state_counts: + return { + "lane": "node_handover", + "priority": "high", + "reason_code": "managed_nodes_not_ready", + "title": "先补齐节点接管与远程可达性", + "summary": ( + f"当前 managed node 共 {int(node_summary.get('managed_total', 0) or 0)} 台，" + f"pending_bootstrap={pending_bootstrap}，runtime_only={runtime_only}。" + ), + "next_focus": { + "kind": "managed_node_handover", + "node_code": first_attention_node_code, + } if first_attention_node_code else {}, + "primary_command_key": "node_handover", + "secondary_command_key": "node_bootstrap_plan", + } + + if launchpad_blocked or str(launchpad_status.get("status") or "") == "attention": + return { + "lane": "release", + "priority": "medium", + "reason_code": "release_launchpad_attention", + "title": "检查发布驾驶舱与 rollout 阻断", + "summary": ensure_str(launchpad_status.get("summary")), + "next_focus": release_focus_ref, + "primary_command_key": "release_launchpad", + "secondary_command_key": "release_preview_latest", + } + + if running_total > 0 or queued_total > 0: + return { + "lane": "ops_jobs", + "priority": "medium", + "reason_code": "jobs_active", + "title": "继续盯运行中任务与活动流", + "summary": ( + f"当前 running={running_total}，queued={queued_total}，" + f"activity_status={ensure_str(activity_summary.get('status')) or 'unknown'}。" + ), + "next_focus": { + "kind": "ops_jobs", + "section": "activity_stream", + }, + "primary_command_key": "jobs", + "secondary_command_key": "activity_preview", + } + + return { + "lane": "steady", + "priority": "low", + "reason_code": "steady_state", + "title": "当前进入稳定观察态", + "summary": ( + f"online={online_total}，attention_nodes={len(attention_nodes)}，" + f"scene_log_status={ensure_str(scene_log.get('status')) or 'standby'}。" + ), + "next_focus": recommendation_focus_ref or ensure_dict(scene_log.get("focus_ref")), + "primary_command_key": "driver_feed", + "secondary_command_key": "activity_preview", + } + + +def normalize_scene_log_observation(overview_data): + driver_feed = ensure_dict(overview_data.get("driver_feed")) + scene_log_data = ensure_dict(driver_feed.get("scene_log_observation")) + if scene_log_data: + normalized = dict(scene_log_data) + normalized["status"] = ensure_str(normalized.get("status")) or "standby" + normalized["status_label"] = ensure_str(normalized.get("status_label")) or "待命" + normalized["summary"] = ensure_str(normalized.get("summary")) or "当前现场日志观察暂无额外结论。" + normalized["focus_ref"] = ensure_dict(normalized.get("focus_ref")) + normalized["target_node_code"] = ensure_str(normalized.get("target_node_code")) + normalized["participating_node_count"] = int(normalized.get("participating_node_count", 0) or 0) + normalized["covered_participating_node_count"] = int(normalized.get("covered_participating_node_count", 0) or 0) + normalized["line_count"] = int(normalized.get("line_count", 0) or 0) + return normalized + + execution_scene = ensure_dict(overview_data.get("execution_scene")) + log_sync = ensure_dict(execution_scene.get("log_sync")) + participation = ensure_dict(execution_scene.get("participation_summary")) + missing_nodes = ensure_list(log_sync.get("missing_participating_nodes")) + participating_nodes = ensure_list(execution_scene.get("participating_nodes")) + target_node_code = "" + if missing_nodes: + target_node_code = ensure_str(missing_nodes[0]) + elif participating_nodes: + target_node_code = ensure_str(ensure_dict(participating_nodes[0]).get("node_code")) + + participating_total = int( + log_sync.get("participating_node_count", participation.get("participating_nodes", len(participating_nodes))) + or participation.get("participating_nodes", len(participating_nodes)) + or len(participating_nodes) + or 0 + ) + covered_total = int(log_sync.get("covered_participating_node_count", 0) or 0) + line_count = int(log_sync.get("line_count", 0) or 0) + enabled = bool(log_sync.get("enabled", False)) + mode = ensure_str(log_sync.get("mode")) or "key" + status = "standby" + status_label = "待命" + summary = "当前没有参与检测节点，现场日志观察处于待命状态。" + if participating_total > 0 and not enabled: + status = "disabled" + status_label = "未开启" + summary = ( + f"当前有 {participating_total} 台参与节点，但远端日志回传仍关闭，" + "海外控制面还看不到节点级现场日志。" + ) + elif participating_total > 0 and ( + line_count <= 0 or covered_total <= 0 or int(log_sync.get("missing_participating_node_count", len(missing_nodes)) or len(missing_nodes)) > 0 + ): + status = "waiting_sample" + status_label = "等待样本" + summary = ( + f"远端日志回传已经开启，但参与节点只覆盖 {covered_total}/{participating_total}，" + f"样本 {line_count} 条，仍需继续观察或下钻节点现场日志。" + ) + elif participating_total > 0: + status = "ready" + status_label = "可下钻" + summary = ( + f"远端日志回传已经形成样本，当前覆盖 {covered_total}/{participating_total} 台参与节点，" + "可以直接下钻节点现场日志。" + ) + + focus_ref = {} + if target_node_code: + focus_ref = { + "kind": "node_scene_log", + "node_code": target_node_code, + "mode": mode, + "limit": 120 if status == "waiting_sample" else 80, + } + return { + "status": status, + "status_label": status_label, + "summary": summary, + "focus_ref": focus_ref, + "target_node_code": target_node_code, + "participating_node_count": participating_total, + "covered_participating_node_count": covered_total, + "line_count": line_count, + } + + +base_url = ensure_str(sys.argv[1]) or "http://127.0.0.1:8100" +overview = as_dict(sys.argv[2]) +nodes = as_dict(sys.argv[3]) +jobs = as_dict(sys.argv[4]) +activity = as_dict(sys.argv[5]) +launchpad = as_dict(sys.argv[6]) +latest_release = as_dict(sys.argv[7]) +policy_preview = as_dict(sys.argv[8]) + +recommendation = ensure_dict(overview.get("recommendation")) +scene_log = normalize_scene_log_observation(overview) +managed_nodes = ensure_list(nodes.get("managed_nodes")) +node_summary = ensure_dict(nodes.get("summary")) +jobs_summary = ensure_dict(jobs.get("summary")) +activity_summary = ensure_dict(activity.get("summary")) +launchpad_release = ensure_dict(launchpad.get("release")) +launchpad_rec = ensure_dict(launchpad.get("top_recommendation")) +launchpad_workers = ensure_dict(launchpad.get("worker_rollout_preview")) +launchpad_controls = ensure_dict(launchpad.get("control_rollout_preview")) +launchpad_status = ensure_dict(launchpad.get("launchpad_status")) +latest_release_entry = latest_release if isinstance(latest_release, dict) else {} +latest_package_preview = ensure_dict(launchpad.get("latest_package_preview")) + +online_nodes = [ + node for node in managed_nodes + if str(node.get("status") or "").lower() in {"online", "busy"} +] +attention_nodes = [ + { + "node_code": str(node.get("node_code") or ""), + "status": str(node.get("status") or ""), + "role": str(node.get("role") or ""), + "stack_status_label": str(node.get("stack_status_label") or ""), + "recommended_action_code": str(node.get("recommended_action_code") or ""), + "recommended_action_label": str(node.get("recommended_action_label") or ""), + "primary_issue": str(node.get("primary_issue") or ""), + } + for node in managed_nodes + if str(node.get("status") or "").lower() not in {"online", "busy"} + or str(node.get("stack_status") or "").lower() not in {"healthy", "ready", ""} +] + +activity_items = ensure_list(activity.get("items")) +recent_activity = [ + { + "kind": str(item.get("kind") or ""), + "title": str(item.get("title") or ""), + "level": str(item.get("level") or ""), + "created_at": str(item.get("created_at") or ""), + } + for item in activity_items[:5] +] + +job_items = ensure_list(jobs.get("items")) +running_jobs = [ + { + "job_key": str(item.get("job_key") or ""), + "status": str(item.get("status") or ""), + "title": str(item.get("title") or ""), + "progress_text": str(item.get("progress_text") or ""), + } + for item in job_items + if str(item.get("status") or "").lower() in {"running", "queued", "waiting"} +][:5] + +recommendation_focus_ref = ensure_dict(recommendation.get("focus_ref")) +first_attention = attention_nodes[0] if attention_nodes else {} +first_attention_node_code = ensure_str(first_attention.get("node_code")) +scene_log_target_node_code = ensure_str(scene_log.get("target_node_code")) +latest_release_version = str(latest_release_entry.get("version") or latest_release_entry.get("release_version") or "") +release_focus_ref = ensure_dict(launchpad_rec.get("focus_ref")) +release_id = str(release_focus_ref.get("release_id") or "") +operator_decision = build_operator_decision() +primary_command_key = ensure_str(operator_decision.get("primary_command_key")) +secondary_command_key = ensure_str(operator_decision.get("secondary_command_key")) + +result = { + "overview": { + "top_recommendation": { + "source": str(recommendation.get("source") or ""), + "key": str(recommendation.get("key") or ""), + "priority": str(recommendation.get("priority") or ""), + "summary": str(recommendation.get("summary") or ""), + "primary_action_code": str(recommendation.get("primary_action_code") or ""), + "secondary_action_code": str(recommendation.get("secondary_action_code") or ""), + "focus_ref": recommendation.get("focus_ref") or {}, + "node_codes": list(recommendation.get("node_codes") or []), + }, + "scene_log_observation": { + "status": str(scene_log.get("status") or ""), + "status_label": str(scene_log.get("status_label") or ""), + "summary": str(scene_log.get("summary") or ""), + "target_node_code": str(scene_log.get("target_node_code") or ""), + "participating_node_count": int(scene_log.get("participating_node_count") or 0), + "covered_participating_node_count": int(scene_log.get("covered_participating_node_count") or 0), + "line_count": int(scene_log.get("line_count") or 0), + "focus_ref": scene_log.get("focus_ref") or {}, + }, + }, + "nodes": { + "managed_total": int(node_summary.get("managed_total") or len(managed_nodes)), + "online_or_busy_total": len(online_nodes), + "offline_total": int(node_summary.get("offline_count") or 0), + "attention_total": len(attention_nodes), + "attention_nodes": attention_nodes[:8], + }, + "jobs": { + "running_total": int(jobs_summary.get("running_count") or 0), + "queued_total": int(jobs_summary.get("queued_count") or 0), + "failed_total": int(jobs_summary.get("failed_count") or 0), + "recent_running_jobs": running_jobs, + }, + "activity_stream": { + "total": int(activity_summary.get("total") or len(activity_items)), + "error_count": int(activity_summary.get("error_count") or 0), + "warning_count": int(activity_summary.get("warning_count") or 0), + "running_job_count": int(activity_summary.get("running_job_count") or 0), + "recent_items": recent_activity, + }, + "operator_decision": { + "lane": ensure_str(operator_decision.get("lane")), + "priority": ensure_str(operator_decision.get("priority")), + "reason_code": ensure_str(operator_decision.get("reason_code")), + "title": ensure_str(operator_decision.get("title")), + "summary": ensure_str(operator_decision.get("summary")), + "next_focus": ensure_dict(operator_decision.get("next_focus")), + "primary_command_key": primary_command_key, + "secondary_command_key": secondary_command_key, + }, + "recommended_commands": { + "stack_diagnosis": "bash domain-api/deploy/multi-region/check_ops_center_stack.sh", + "driver_feed": command("driver-feed"), + "activity_preview": command("activity-preview"), + "driver_focus_preview": command("driver-focus-preview"), + "driver_focus_run": command("driver-focus-run"), + "top_recommendation_preview": ( + command("driver-preview", str(recommendation.get("primary_action_code") or "")) + if str(recommendation.get("primary_action_code") or "").strip() + else "" + ), + "top_recommendation_focus_preview": ( + command("driver-focus-preview", f"entry:{str(recommendation.get('key') or '')}") + if str(recommendation.get("key") or "").strip() + else "" + ), + "scene_log": ( + command("scene-node-log", scene_log_target_node_code, "120", "key") + if scene_log_target_node_code + else "" + ), + "scene_log_direct": ( + f"bash domain-api/deploy/multi-region/check_node_scene_log.sh {base_url} {scene_log_target_node_code} 120 key" + if scene_log_target_node_code + else "" + ), + "node_handover": ( + command("node-handover", first_attention_node_code) + if first_attention_node_code + else command("stack-first-gap-handover") + ), + "node_bootstrap_plan": ( + command("node-bootstrap-plan", first_attention_node_code) + if first_attention_node_code + else "" + ), + "delivery_queue": ( + command("delivery-queue", first_attention_node_code) + if first_attention_node_code + else "" + ), + "release_launchpad": release_command("launchpad"), + "release_preview_latest": release_command("preview-latest"), + "release_preview_latest_worker": release_command("preview-latest-smart", "worker"), + "release_preview_latest_control": release_command("preview-latest-smart", "control"), + "release_focus": ( + command("driver-preview", "focus_release_hub") + if str(launchpad_rec.get("action_code") or "").strip() + else "" + ), + "release_preview_selected": ( + release_command("preview-release", release_id, "worker") + if release_id + else "" + ), + "jobs": command("activity-stream", "kind=ops_job"), + }, + "next_actions": { + "primary_command": "", + "secondary_command": "", + }, + "release_launchpad": { + "status": str(launchpad_status.get("status") or ""), + "status_label": str(launchpad_status.get("status_label") or ""), + "summary": str(launchpad_status.get("summary") or ""), + "recommended_action_code": str(launchpad_rec.get("action_code") or ""), + "recommended_action_label": str(launchpad_rec.get("action_label") or ""), + "recommended_execution_mode": str(launchpad_rec.get("execution_mode") or ""), + "recommended_execution_mode_label": str(launchpad_rec.get("execution_mode_label") or ""), + "focus_ref": launchpad_rec.get("focus_ref") or {}, + "release_version": str(launchpad_release.get("version") or launchpad_release.get("release_version") or ""), + "worker_rollout_preview": { + "available": bool(launchpad_workers.get("available")), + "blocked": bool(launchpad_workers.get("blocked")), + "execution_mode": str(launchpad_workers.get("execution_mode") or ""), + "execution_mode_label": str(launchpad_workers.get("execution_mode_label") or ""), + "target_node_codes": list(launchpad_workers.get("target_node_codes") or []), + }, + "control_rollout_preview": { + "available": bool(launchpad_controls.get("available")), + "blocked": bool(launchpad_controls.get("blocked")), + "execution_mode": str(launchpad_controls.get("execution_mode") or ""), + "execution_mode_label": str(launchpad_controls.get("execution_mode_label") or ""), + "target_node_codes": list(launchpad_controls.get("target_node_codes") or []), + }, + }, + "latest_release": { + "version": str(latest_release_entry.get("version") or latest_release_entry.get("release_version") or ""), + "status": ensure_str(latest_release_entry.get("status")), + "status_label": ensure_str(latest_release_entry.get("status_label")), + "summary": ensure_str(latest_release_entry.get("summary") or latest_release_entry.get("summary_text")), + }, + "policy_preview_sample": { + "execution_mode": str(policy_preview.get("execution_mode") or ""), + "risk_level": str(policy_preview.get("risk_level") or ""), + "approval_required": bool(policy_preview.get("approval_required")), + "blocked": bool(policy_preview.get("blocked")), + "blocking_reasons": ensure_list(policy_preview.get("blocking_reasons"))[:5], + "warnings": ensure_list(policy_preview.get("warnings"))[:5], + }, + "focus_refs": { + "top_recommendation": recommendation_focus_ref, + "scene_log": ensure_dict(scene_log.get("focus_ref")), + "release_hub": release_focus_ref, + }, + "rollout_targets": { + "worker": list(launchpad_workers.get("target_node_codes") or []), + "control": list(launchpad_controls.get("target_node_codes") or []), + }, + "package_preview": { + "available": bool(latest_package_preview.get("release")), + "preview_source": ensure_str(latest_package_preview.get("preview_source")), + "message": ensure_str(latest_package_preview.get("message")), + }, +} + +result["next_actions"]["primary_command"] = ensure_str( + ensure_dict(result.get("recommended_commands")).get(primary_command_key) +) +result["next_actions"]["secondary_command"] = ensure_str( + ensure_dict(result.get("recommended_commands")).get(secondary_command_key) +) + +print(json.dumps(result, ensure_ascii=False, indent=2)) +PY +echo diff --git a/domain-api/deploy/multi-region/check_release_hub.sh b/domain-api/deploy/multi-region/check_release_hub.sh new file mode 100644 index 0000000..98c38fe --- /dev/null +++ b/domain-api/deploy/multi-region/check_release_hub.sh @@ -0,0 +1,373 @@ +#!/usr/bin/env bash +set -euo pipefail + +BASE_URL="${1:-http://127.0.0.1:8100}" +INPUT_RELEASE_ID="${2:-}" +CURL_CONNECT_TIMEOUT="${CURL_CONNECT_TIMEOUT:-3}" +CURL_MAX_TIME="${CURL_MAX_TIME:-10}" + +curl_json() { + curl -fsS \ + --connect-timeout "${CURL_CONNECT_TIMEOUT}" \ + --max-time "${CURL_MAX_TIME}" \ + "$1" +} + +curl_post_json() { + local url="$1" + local body="$2" + curl -fsS \ + --connect-timeout "${CURL_CONNECT_TIMEOUT}" \ + --max-time "${CURL_MAX_TIME}" \ + -X POST "${url}" \ + -H 'Content-Type: application/json' \ + -d "${body}" +} + +launchpad="$(curl_json "${BASE_URL}/api/v1/ops/releases/launchpad")" +latest_release="$(curl_json "${BASE_URL}/api/v1/ops/releases/latest")" +action_templates="$(curl_json "${BASE_URL}/api/v1/ops/action-templates")" +cluster_json="$(curl_json "${BASE_URL}/api/v1/runtime/cluster")" +release_id="${INPUT_RELEASE_ID}" + +if [[ -z "${release_id}" ]]; then + release_id="$(python3 -c ' +import json +import sys + +payload = json.loads(sys.argv[1]) +data = payload.get("data") or {} +value = data.get("id") or 0 +print(value) +' "${latest_release}")" +fi + +selected_release="${latest_release}" +if [[ -n "${release_id}" && "${release_id}" != "0" ]]; then + selected_release="$(curl_json "${BASE_URL}/api/v1/ops/releases/${release_id}")" +fi + +build_release_preview_payload() { + local preview_mode="$1" + python3 - <<'PY' "${selected_release}" "${cluster_json}" "${preview_mode}" +import json +import sys + +release_payload = json.loads(sys.argv[1]).get("data") or {} +cluster_payload = json.loads(sys.argv[2]).get("data") or {} +preview_mode = str(sys.argv[3] or "").strip().lower() + +nodes = list(cluster_payload.get("nodes") or []) + +def is_online(node): + return str(node.get("status") or "").strip() in {"online", "busy"} + +control_nodes = [ + str(node.get("node_code") or "").strip() + for node in nodes + if is_online(node) and str(node.get("role") or "").strip() == "control" +] +worker_nodes = [ + str(node.get("node_code") or "").strip() + for node in nodes + if is_online(node) + and str(node.get("role") or "").strip() == "worker" + and bool(node.get("is_effective_worker", False)) +] +online_nodes = [ + str(node.get("node_code") or "").strip() + for node in nodes + if is_online(node) +] + +payload = { + "release_version": str(release_payload.get("release_version") or "").strip(), + "artifact_url": str(release_payload.get("artifact_url") or "").strip(), + "checksum": str(release_payload.get("checksum") or "").strip(), + "install_root": "/opt/domaincheck", + "health_check_timeout_seconds": 10, + "health_check_retries": 2, + "health_check_interval_seconds": 2, + "rollback_on_failure": True, + "switch_current": True, +} + +if preview_mode == "control": + template_key = "deploy.release.control" + target_node_codes = control_nodes + payload.update( + { + "restart_services": ["domaincheck-api", "domaincheck-worker", "domaincheck-sync-agent"], + "health_check_urls": ["http://127.0.0.1:8100/health"], + "health_check_services": ["domaincheck-api", "domaincheck-worker", "domaincheck-sync-agent"], + } + ) +elif preview_mode == "worker": + template_key = "deploy.release.worker" + target_node_codes = worker_nodes + payload.update( + { + "restart_services": ["domaincheck-worker"], + "health_check_urls": [], + "health_check_services": ["domaincheck-worker"], + } + ) +else: + template_key = "deploy.release.custom" + target_node_codes = (control_nodes[:1] + worker_nodes[:2]) or online_nodes[:3] + payload.update( + { + "restart_services": [], + "health_check_urls": [], + "health_check_services": [], + } + ) + +request_payload = { + "template_key": template_key, + "action": "deploy.release", + "target_type": "batch", + "execution_mode": "remote-agent", + "target_node_codes": [item for item in target_node_codes if item], + "payload": payload, +} + +print(json.dumps(request_payload, ensure_ascii=False)) +PY +} + +print_preview_request_summary() { + local preview_payload="$1" + python3 - <<'PY' "${preview_payload}" +import json +import sys + +payload = json.loads(sys.argv[1]) +body = payload.get("payload") or {} +print( + json.dumps( + { + "template_key": payload.get("template_key") or "", + "action": payload.get("action") or "", + "execution_mode": payload.get("execution_mode") or "", + "target_node_codes": payload.get("target_node_codes") or [], + "release_version": body.get("release_version") or "", + "artifact_url_present": bool(str(body.get("artifact_url") or "").strip()), + "checksum_present": bool(str(body.get("checksum") or "").strip()), + "restart_services": body.get("restart_services") or [], + "health_check_urls": body.get("health_check_urls") or [], + "health_check_services": body.get("health_check_services") or [], + }, + ensure_ascii=False, + indent=2, + ) +) +PY +} + +run_release_preview() { + local step_no="$1" + local preview_mode="$2" + local section_title="$3" + local preview_payload + preview_payload="$(build_release_preview_payload "${preview_mode}")" + + echo "[${step_no}/10] ${section_title}" + print_preview_request_summary "${preview_payload}" + echo + curl_post_json "${BASE_URL}/api/v1/ops/policy/preview" "${preview_payload}" + echo + echo +} + +echo "[1/11] release launchpad" +printf '%s\n\n' "${launchpad}" + +echo "[2/11] latest release" +printf '%s\n\n' "${selected_release}" + +echo "[3/11] recent releases" +curl_json "${BASE_URL}/api/v1/ops/releases?limit=8" +echo +echo + +echo "[4/11] release deploy templates" +python3 - <<'PY' "${action_templates}" +import json +import sys + +payload = json.loads(sys.argv[1]) +groups = (payload.get("data") or {}).get("groups") or [] +release_group = next((item for item in groups if str(item.get("group_key") or "") == "release"), {}) +items = list(release_group.get("items") or []) +summary = { + "group_key": release_group.get("group_key") or "", + "group_title": release_group.get("group_title") or "", + "template_keys": [str(item.get("key") or "") for item in items], + "field_types": sorted( + { + str(field.get("type") or "") + for item in items + for field in list(item.get("fields") or []) + if str(field.get("type") or "") + } + ), +} +print(json.dumps(summary, ensure_ascii=False, indent=2)) +PY +echo + +echo "[5/11] online release target summary" +python3 - <<'PY' "${cluster_json}" +import json +import sys + +payload = json.loads(sys.argv[1]).get("data") or {} +nodes = list(payload.get("nodes") or []) + +def is_online(node): + return str(node.get("status") or "").strip() in {"online", "busy"} + +summary = { + "online_control_nodes": [ + str(node.get("node_code") or "").strip() + for node in nodes + if is_online(node) and str(node.get("role") or "").strip() == "control" + ], + "online_worker_nodes": [ + str(node.get("node_code") or "").strip() + for node in nodes + if is_online(node) and str(node.get("role") or "").strip() == "worker" and bool(node.get("is_effective_worker", False)) + ], + "all_online_nodes": [ + { + "node_code": str(node.get("node_code") or "").strip(), + "role": str(node.get("role") or "").strip(), + "status": str(node.get("status") or "").strip(), + "is_effective_worker": bool(node.get("is_effective_worker", False)), + } + for node in nodes + if is_online(node) + ], +} +print(json.dumps(summary, ensure_ascii=False, indent=2)) +PY +echo +echo + +run_release_preview "6" "control" "deploy.release.control preview" +run_release_preview "7" "worker" "deploy.release.worker preview" +run_release_preview "8" "custom" "deploy.release.custom preview" + +rollout_list='{"code":0,"message":"no release selected","data":{"rollouts":[]}}' +rollout_id="0" +if [[ -n "${release_id}" && "${release_id}" != "0" ]]; then + rollout_list="$(curl_json "${BASE_URL}/api/v1/ops/releases/${release_id}/rollouts?limit=10")" + rollout_id="$(python3 -c ' +import json +import sys + +payload = json.loads(sys.argv[1]) +rollouts = (payload.get("data") or {}).get("rollouts") or [] +print((rollouts[0] or {}).get("id") or 0 if rollouts else 0) +' "${rollout_list}")" +fi + +echo "[9/11] rollout list for release ${release_id:-0}" +printf '%s\n\n' "${rollout_list}" + +echo "[10/11] primary rollout detail" +if [[ "${rollout_id}" != "0" ]]; then + curl_json "${BASE_URL}/api/v1/ops/rollouts/${rollout_id}" +else + echo '{"code":0,"message":"no rollout","data":{}}' +fi +echo +echo + +echo "[11/12] primary rollout jobs" +if [[ "${rollout_id}" != "0" ]]; then + curl_json "${BASE_URL}/api/v1/ops/rollouts/${rollout_id}/jobs?limit=50" +else + echo '{"code":0,"message":"no rollout","data":{"jobs":[]}}' +fi +echo + +echo +echo "[12/12] condensed summary" +python3 - <<'PY' "${launchpad}" "${selected_release}" "${cluster_json}" "${rollout_list}" "${rollout_id}" +import json +import sys + +launchpad_payload = json.loads(sys.argv[1] or "{}").get("data") or {} +selected_release = json.loads(sys.argv[2] or "{}").get("data") or {} +cluster_payload = json.loads(sys.argv[3] or "{}").get("data") or {} +rollout_list_payload = json.loads(sys.argv[4] or "{}").get("data") or {} +rollout_id = int(sys.argv[5] or 0) + +launchpad_status = dict(launchpad_payload.get("launchpad_status") or {}) +worker_preview = dict(launchpad_payload.get("worker_rollout_preview") or {}) +worker_smart_rollout = dict(worker_preview.get("smart_rollout") or {}) +control_preview = dict(launchpad_payload.get("control_rollout_preview") or {}) +control_smart_rollout = dict(control_preview.get("smart_rollout") or {}) +nodes = list(cluster_payload.get("nodes") or []) +rollouts = list(rollout_list_payload.get("rollouts") or []) + +def is_online(node): + return str(node.get("status") or "").strip() in {"online", "busy"} + +summary = { + "release": { + "id": int(selected_release.get("id") or 0), + "release_version": str(selected_release.get("release_version") or ""), + "channel": str(selected_release.get("channel") or ""), + "status": str(selected_release.get("status") or ""), + }, + "launchpad": { + "status": str(launchpad_status.get("status") or ""), + "status_label": str(launchpad_status.get("status_label") or ""), + "summary": str(launchpad_status.get("summary") or ""), + "recommended_action_code": str(launchpad_status.get("recommended_action_code") or ""), + "recommended_target_node_code": str(launchpad_status.get("recommended_target_node_code") or ""), + "recommended_recovery_label": str(launchpad_status.get("recommended_recovery_label") or ""), + "recommended_recovery_summary": str(launchpad_status.get("recommended_recovery_summary") or ""), + "onboarding_bootstrap_pending_nodes": int(launchpad_status.get("onboarding_bootstrap_pending_nodes", 0) or 0), + "onboarding_acceptance_ready_nodes": int(launchpad_status.get("onboarding_acceptance_ready_nodes", 0) or 0), + "recommended_execution_mode": str(launchpad_status.get("recommended_execution_mode") or ""), + "recommended_execution_mode_label": str(launchpad_status.get("recommended_execution_mode_label") or ""), + "focus_ref": dict(launchpad_status.get("focus_ref") or {}), + }, + "worker_rollout_preview": { + "available": bool(worker_smart_rollout.get("available", False)), + "blocked": bool(worker_preview.get("blocked", False)), + "requires_confirmation": bool(worker_preview.get("requires_confirmation", False)), + "execution_mode": str(worker_smart_rollout.get("execution_mode") or ""), + "target_node_codes": list(worker_smart_rollout.get("target_node_codes") or []), + }, + "control_rollout_preview": { + "available": bool(control_smart_rollout.get("available", False)), + "blocked": bool(control_preview.get("blocked", False)), + "requires_confirmation": bool(control_preview.get("requires_confirmation", False)), + "execution_mode": str(control_smart_rollout.get("execution_mode") or ""), + "target_node_codes": list(control_smart_rollout.get("target_node_codes") or []), + }, + "cluster": { + "online_control_nodes": [ + str(node.get("node_code") or "").strip() + for node in nodes + if is_online(node) and str(node.get("role") or "").strip() == "control" + ], + "online_worker_nodes": [ + str(node.get("node_code") or "").strip() + for node in nodes + if is_online(node) and str(node.get("role") or "").strip() == "worker" and bool(node.get("is_effective_worker", False)) + ], + }, + "rollouts": { + "count": len(rollouts), + "selected_rollout_id": rollout_id, + }, +} +print(json.dumps(summary, ensure_ascii=False, indent=2)) +PY +echo diff --git a/domain-api/deploy/multi-region/check_temp_topology.sh b/domain-api/deploy/multi-region/check_temp_topology.sh index 976114d..087d649 100755 --- a/domain-api/deploy/multi-region/check_temp_topology.sh +++ b/domain-api/deploy/multi-region/check_temp_topology.sh @@ -4,15 +4,24 @@ set -euo pipefail MAINLAND_BASE_URL="${1:-http://127.0.0.1:8100}" OVERSEAS_BASE_URL="${2:-http://152.53.37.118:8100}" PYTHON_BIN="${PYTHON_BIN:-python3}" +CURL_CONNECT_TIMEOUT="${CURL_CONNECT_TIMEOUT:-3}" +CURL_MAX_TIME="${CURL_MAX_TIME:-10}" + +curl_json() { + curl -fsS \ + --connect-timeout "${CURL_CONNECT_TIMEOUT}" \ + --max-time "${CURL_MAX_TIME}" \ + "$1" +} echo "[1/5] mainland runtime cluster" -MAINLAND_CLUSTER="$(curl -fsS "${MAINLAND_BASE_URL}/api/v1/runtime/cluster")" +MAINLAND_CLUSTER="$(curl_json "${MAINLAND_BASE_URL}/api/v1/runtime/cluster")" echo "${MAINLAND_CLUSTER}" echo echo echo "[2/5] mainland runtime readiness" -MAINLAND_READINESS="$(curl -fsS "${MAINLAND_BASE_URL}/api/v1/runtime/readiness")" +MAINLAND_READINESS="$(curl_json "${MAINLAND_BASE_URL}/api/v1/runtime/readiness")" echo "${MAINLAND_READINESS}" echo echo @@ -79,7 +88,7 @@ echo echo echo "[4/5] overseas runtime cluster" -OVERSEAS_CLUSTER="$(curl -fsS "${OVERSEAS_BASE_URL}/api/v1/runtime/cluster")" +OVERSEAS_CLUSTER="$(curl_json "${OVERSEAS_BASE_URL}/api/v1/runtime/cluster")" echo "${OVERSEAS_CLUSTER}" echo echo diff --git a/domain-api/deploy/multi-region/check_worker_participation.sh b/domain-api/deploy/multi-region/check_worker_participation.sh index e72693c..c798572 100755 --- a/domain-api/deploy/multi-region/check_worker_participation.sh +++ b/domain-api/deploy/multi-region/check_worker_participation.sh @@ -4,23 +4,44 @@ set -euo pipefail DB_NAME="${DB_NAME:-domain}" DB_USER="${DB_USER:-postgres}" PSQL_BIN="${PSQL_BIN:-/www/server/pgsql/bin/psql}" +API_BASE_URL="${1:-${API_BASE_URL:-http://127.0.0.1:8100}}" +CURL_CONNECT_TIMEOUT="${CURL_CONNECT_TIMEOUT:-3}" +CURL_MAX_TIME="${CURL_MAX_TIME:-10}" -echo "[1/3] cluster nodes" +curl_json() { + curl -fsS \ + --connect-timeout "${CURL_CONNECT_TIMEOUT}" \ + --max-time "${CURL_MAX_TIME}" \ + "$1" +} + +RUNTIME_STATUS_JSON="$(curl_json "${API_BASE_URL}/api/v1/runtime/status")" + +echo "[1/4] cluster nodes" "${PSQL_BIN}" -U "${DB_USER}" -d "${DB_NAME}" -Atc " select node_code || '|' || role || '|' || status || '|' || coalesce(current_load, 0) || '|' || + coalesce(metadata_json->>'job_items_claimed', '0') || '|' || + coalesce(metadata_json->>'job_items_running', '0') || '|' || + coalesce(metadata_json->>'job_items_completed', '0') || '|' || to_char(last_heartbeat_at, 'YYYY-MM-DD HH24:MI:SS') || '|' || coalesce(metadata_json->>'worker_online', '') || '|' || - coalesce(metadata_json->>'detect_participating', '') + ( + case + when coalesce(current_load, 0) > 0 then 'true' + when coalesce(metadata_json->>'detect_participating', '') in ('true', 'false') then metadata_json->>'detect_participating' + else '' + end + ) from detect_worker_nodes order by node_code; " echo -echo "[2/3] active job distribution" +echo "[2/4] active job distribution" "${PSQL_BIN}" -U "${DB_USER}" -d "${DB_NAME}" -Atc " select coalesce(claimed_by, '') || '|' || @@ -39,59 +60,43 @@ order by claimed_by, status; " echo -echo "[3/3] condensed summary" -"${PSQL_BIN}" -U "${DB_USER}" -d "${DB_NAME}" -At <<'SQL' -with latest_job as ( - select id, job_code, status - from detect_jobs - where status in ('pending','running') - order by id desc - limit 1 -), -node_stats as ( - select - node_code, - role, - status, - coalesce(metadata_json->>'worker_online', '') as worker_online, - coalesce(metadata_json->>'detect_participating', '') as detect_participating - from detect_worker_nodes -), -item_stats as ( - select - coalesce(claimed_by, '') as claimed_by, - status, - count(*) as cnt - from detect_job_items - where job_id = (select id from latest_job) - group by claimed_by, status -) -select json_build_object( - 'job', ( - select json_build_object( - 'job_code', coalesce(job_code, ''), - 'status', coalesce(status, '') - ) - from latest_job - ), - 'online_nodes', ( - select coalesce(json_agg(json_build_object( - 'node_code', node_code, - 'role', role, - 'status', status, - 'worker_online', worker_online, - 'detect_participating', detect_participating - ) order by node_code), '[]'::json) - from node_stats - where status in ('online', 'busy') - ), - 'claimed_distribution', ( - select coalesce(json_agg(json_build_object( - 'claimed_by', claimed_by, - 'status', status, - 'count', cnt - ) order by claimed_by, status), '[]'::json) - from item_stats - ) -); -SQL +echo "[3/4] runtime participation snapshot" +printf "%s" "${RUNTIME_STATUS_JSON}" | python3 -c ' +import json, sys +payload = json.load(sys.stdin).get("data", {}) +detect = payload.get("detect", {}) +print(json.dumps({ + "node": payload.get("node", {}), + "participation_summary": detect.get("participation_summary", {}), + "participating_nodes": detect.get("participating_nodes", []), + "non_participating_nodes": detect.get("non_participating_nodes", detect.get("standby_nodes", [])), + "log_sync": detect.get("log_sync", {}), +}, ensure_ascii=False, indent=2)) +' +echo + +echo "[4/4] condensed summary" +printf "%s" "${RUNTIME_STATUS_JSON}" | python3 -c ' +import json, sys +payload = json.load(sys.stdin).get("data", {}) +detect = payload.get("detect", {}) +summary = detect.get("participation_summary", {}) or {} +print(json.dumps({ + "node": payload.get("node", {}), + "active_job": { + "job_code": (detect.get("active_job") or {}).get("job_code", ""), + "status": (detect.get("active_job") or {}).get("status", ""), + }, + "participation_summary": summary, + "dispatch_active_nodes": summary.get("dispatch_active_node_codes", []), + "recent_only_nodes": summary.get("recent_only_node_codes", []), + "non_participating_nodes": summary.get("non_participating_node_codes", []), + "log_sync": { + "enabled": bool((detect.get("log_sync") or {}).get("enabled", False)), + "mode": (detect.get("log_sync") or {}).get("mode", ""), + "line_count": (detect.get("log_sync") or {}).get("line_count", 0), + "source_nodes": (detect.get("log_sync") or {}).get("source_nodes", []), + "last_at": (detect.get("log_sync") or {}).get("last_at", ""), + }, +}, ensure_ascii=False, indent=2)) +' diff --git a/domain-api/deploy/multi-region/drive_ops_action.sh b/domain-api/deploy/multi-region/drive_ops_action.sh new file mode 100755 index 0000000..320a8d9 --- /dev/null +++ b/domain-api/deploy/multi-region/drive_ops_action.sh @@ -0,0 +1,447 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck disable=SC1091 +source "${SCRIPT_DIR}/lib_ops_center.sh" +ops_center_load_config "${SCRIPT_DIR}" + +usage() { + cat <<'EOF' +usage: + bash domain-api/deploy/multi-region/drive_ops_action.sh [action_payload_json|@payload.json] [requested_by] + bash domain-api/deploy/multi-region/drive_ops_action.sh [action_payload_json|@payload.json] [requested_by] + +examples: + bash domain-api/deploy/multi-region/drive_ops_action.sh \ + http://127.0.0.1:8100 \ + preview \ + create_release_rollout_worker \ + '{"release_id":12}' + + bash domain-api/deploy/multi-region/drive_ops_action.sh \ + http://127.0.0.1:8100 \ + create_release_rollout_worker \ + '{"release_id":12}' + + NODE_CODES="mainland-worker-01,mainland-worker-02" \ + bash domain-api/deploy/multi-region/drive_ops_action.sh \ + http://127.0.0.1:8100 \ + run_standard_inspection \ + '{}' + +notes: + - preview/execute 双模式共用同一份 action payload 和 node_codes + - 旧用法仍默认按 execute 处理 + - NODE_CODES can be passed through env as comma/newline separated node_code list + - action_payload_json can also use @/path/to/file.json + - preview metadata can be passed through env: + SOURCE_LABEL / ACTION_TITLE / ACTION_SUMMARY / ACTION_REASON / ACTION_META_TEXT / PRIMARY_LABEL +EOF +} + +BASE_URL="${1:-$(ops_center_resolve_mainland_api_base_url)}" +SECOND_ARG="${2:-}" +THIRD_ARG="${3:-}" +FOURTH_ARG="${4:-}" +FIFTH_ARG="${5:-}" + +if [[ "${SECOND_ARG}" == "preview" || "${SECOND_ARG}" == "execute" ]]; then + OPERATION="${SECOND_ARG}" + ACTION_CODE="${THIRD_ARG:-}" + RAW_ACTION_PAYLOAD="${FOURTH_ARG:-}" + REQUESTED_BY="${FIFTH_ARG:-$(ops_center_resolve_requested_by)}" +else + OPERATION="execute" + ACTION_CODE="${SECOND_ARG:-}" + RAW_ACTION_PAYLOAD="${THIRD_ARG:-}" + REQUESTED_BY="${FOURTH_ARG:-$(ops_center_resolve_requested_by)}" +fi + +if [[ -z "${RAW_ACTION_PAYLOAD}" ]]; then + RAW_ACTION_PAYLOAD='{}' +fi + +NODE_CODES_INPUT="${NODE_CODES:-}" +SOURCE_LABEL="${SOURCE_LABEL:-cli/drive_ops_action}" +ACTION_TITLE="${ACTION_TITLE:-}" +ACTION_SUMMARY="${ACTION_SUMMARY:-}" +ACTION_REASON="${ACTION_REASON:-}" +ACTION_META_TEXT="${ACTION_META_TEXT:-}" +PRIMARY_LABEL="${PRIMARY_LABEL:-}" + +print_json_or_raw() { + local raw_text="${1:-}" + python3 - "${raw_text}" <<'PY' +import json +import sys + +raw_text = str(sys.argv[1] or "") +try: + payload = json.loads(raw_text) +except Exception: + print(raw_text) +else: + print(json.dumps(payload, ensure_ascii=False, indent=2)) +PY +} + +request_with_status() { + local method="$1" + local url="$2" + local body="${3:-}" + if [[ "${method}" == "POST" ]]; then + curl -sS -X POST "${url}" \ + -H 'Content-Type: application/json' \ + -d "${body}" \ + -w $'\nHTTP_STATUS=%{http_code}' + else + curl -sS "${url}" -w $'\nHTTP_STATUS=%{http_code}' + fi +} + +if [[ -z "${ACTION_CODE}" ]]; then + usage + exit 1 +fi + +if [[ "${OPERATION}" != "preview" && "${OPERATION}" != "execute" ]]; then + echo "operation must be preview or execute" >&2 + usage + exit 1 +fi + +ACTION_PAYLOAD_JSON="${RAW_ACTION_PAYLOAD}" +if [[ "${ACTION_PAYLOAD_JSON}" == @* ]]; then + ACTION_PAYLOAD_PATH="${ACTION_PAYLOAD_JSON#@}" + if [[ ! -f "${ACTION_PAYLOAD_PATH}" ]]; then + echo "payload file not found: ${ACTION_PAYLOAD_PATH}" >&2 + exit 1 + fi + ACTION_PAYLOAD_JSON="$(cat "${ACTION_PAYLOAD_PATH}")" +fi + +if [[ -z "${NODE_CODES_INPUT}" ]]; then + NODE_CODES_INPUT="$(python3 - <<'PY' "${BASE_URL}" "${ACTION_CODE}" "${ACTION_PAYLOAD_JSON}" +import json +import sys +import urllib.request + + +def normalize_codes(*groups): + result = [] + for group in groups: + for item in list(group or []): + node_code = str(item or "").strip() + if node_code and node_code not in result: + result.append(node_code) + return result + + +base_url = str(sys.argv[1] or "").rstrip("/") +action_code = str(sys.argv[2] or "").strip() +payload_raw = str(sys.argv[3] or "").strip() or "{}" + +try: + payload = json.loads(payload_raw) +except Exception: + payload = {} +if not isinstance(payload, dict): + payload = {} + +payload_node_codes = normalize_codes(payload.get("node_codes") or []) +if payload_node_codes: + print(",".join(payload_node_codes)) + raise SystemExit(0) + +supported_actions = { + "open_worker_logs_participating", + "run_inspection_participating", + "open_worker_logs_standby", + "run_inspection_standby", + "open_worker_logs", + "run_standard_inspection", +} +if action_code not in supported_actions or not base_url: + print("") + raise SystemExit(0) + +try: + with urllib.request.urlopen(f"{base_url}/api/v1/ops/overview", timeout=20) as response: + overview_payload = json.loads(response.read().decode("utf-8")) + overview = dict((overview_payload or {}).get("data") or {}) +except Exception: + print("") + raise SystemExit(0) + +execution_scene = dict(overview.get("execution_scene") or {}) +participation_summary = dict(execution_scene.get("participation_summary") or {}) +log_sync = dict(execution_scene.get("log_sync") or {}) + +dispatch_active = normalize_codes(participation_summary.get("dispatch_active_node_codes") or []) +recent_only = normalize_codes(participation_summary.get("recent_only_node_codes") or []) +standby = normalize_codes( + participation_summary.get("standby_node_codes") or [], + participation_summary.get("non_participating_node_codes") or [], +) +missing_logs = normalize_codes(log_sync.get("missing_participating_nodes") or []) + +resolved_node_codes = [] +if action_code in {"open_worker_logs_participating", "run_inspection_participating"}: + resolved_node_codes = normalize_codes(missing_logs, dispatch_active, recent_only) +elif action_code in {"open_worker_logs_standby", "run_inspection_standby"}: + resolved_node_codes = normalize_codes(standby) +elif action_code == "open_worker_logs": + resolved_node_codes = normalize_codes(missing_logs, dispatch_active, recent_only, standby) +elif action_code == "run_standard_inspection": + resolved_node_codes = normalize_codes(dispatch_active, recent_only, standby) + +print(",".join(resolved_node_codes)) +PY +)" +fi + +REQUEST_JSON="$(python3 - <<'PY' "${ACTION_CODE}" "${ACTION_PAYLOAD_JSON}" "${REQUESTED_BY}" "${NODE_CODES_INPUT}" +import json +import sys + +action_code = str(sys.argv[1] or "").strip() +payload_raw = str(sys.argv[2] or "").strip() or "{}" +requested_by = str(sys.argv[3] or "").strip() or "cli/drive_ops_action" +node_codes_raw = str(sys.argv[4] or "") + +try: + action_payload = json.loads(payload_raw) +except Exception as exc: # pragma: no cover - cli validation + raise SystemExit(f"invalid action payload json: {exc}") + +if not isinstance(action_payload, dict): + raise SystemExit("action payload must be a json object") + +node_codes = [] +seen = set() +for item in node_codes_raw.replace("\n", ",").split(","): + value = str(item or "").strip() + if not value or value in seen: + continue + seen.add(value) + node_codes.append(value) + +request_payload = { + "action_code": action_code, + "node_codes": node_codes, + "action_payload": action_payload, + "requested_by": requested_by, +} +print(json.dumps(request_payload, ensure_ascii=False)) +PY +)" + +PREVIEW_REQUEST_JSON="$(python3 - <<'PY' "${ACTION_CODE}" "${ACTION_PAYLOAD_JSON}" "${REQUESTED_BY}" "${NODE_CODES_INPUT}" "${SOURCE_LABEL}" "${ACTION_TITLE}" "${ACTION_SUMMARY}" "${ACTION_REASON}" "${ACTION_META_TEXT}" "${PRIMARY_LABEL}" +import json +import sys + +action_code = str(sys.argv[1] or "").strip() +payload_raw = str(sys.argv[2] or "").strip() or "{}" +requested_by = str(sys.argv[3] or "").strip() or "cli/drive_ops_action" +node_codes_raw = str(sys.argv[4] or "") +source_label = str(sys.argv[5] or "").strip() or "cli/drive_ops_action" +action_title = str(sys.argv[6] or "").strip() +action_summary = str(sys.argv[7] or "").strip() +action_reason = str(sys.argv[8] or "").strip() +action_meta_text = str(sys.argv[9] or "").strip() +primary_label = str(sys.argv[10] or "").strip() + +try: + action_payload = json.loads(payload_raw) +except Exception as exc: + raise SystemExit(f"invalid action payload json: {exc}") + +if not isinstance(action_payload, dict): + raise SystemExit("action payload must be a json object") + +node_codes = [] +seen = set() +for item in node_codes_raw.replace("\n", ",").split(","): + value = str(item or "").strip() + if not value or value in seen: + continue + seen.add(value) + node_codes.append(value) + +request_payload = { + "source_label": source_label, + "title": action_title or primary_label or action_code, + "summary": action_summary, + "reason": action_reason, + "meta_text": action_meta_text, + "executor_kind": "driver_action", + "primary_label": primary_label or action_code, + "primary_action_code": action_code, + "primary_node_codes": node_codes, + "primary_action_payload": action_payload, + "requested_by": requested_by, +} +print(json.dumps(request_payload, ensure_ascii=False)) +PY +)" + +if [[ "${OPERATION}" == "preview" ]]; then + echo "[1/3] driver action preview request" + print_json_or_raw "${PREVIEW_REQUEST_JSON}" + echo + echo + + RAW_HTTP_RESPONSE="$(request_with_status "POST" "${BASE_URL}/api/v1/ops/driver-actions/preview" "${PREVIEW_REQUEST_JSON}")" + HTTP_STATUS="${RAW_HTTP_RESPONSE##*$'\n'HTTP_STATUS=}" + RESPONSE_JSON="${RAW_HTTP_RESPONSE%$'\n'HTTP_STATUS=*}" + + echo "[2/3] raw response" + printf '%s\n' "http_status=${HTTP_STATUS}" + print_json_or_raw "${RESPONSE_JSON}" + echo + echo + + echo "[3/3] condensed summary" + python3 - "${RESPONSE_JSON}" "${HTTP_STATUS}" <<'PY' +import json +import sys + +payload = json.loads(sys.argv[1]) +http_status = int(sys.argv[2] or 0) +data = payload.get("data") or {} +primary = data.get("primary") or {} +secondary = data.get("secondary") or {} + +raw_code = payload.get("code", 1) +if raw_code in (0, "0", False): + normalized_code = 0 +else: + normalized_code = int(raw_code or 1) + +summary = { + "http_status": http_status, + "code": normalized_code, + "message": str(payload.get("message") or ""), + "contract_key": str(data.get("contract_key") or ""), + "contract_version": str(data.get("contract_version") or ""), + "preview_endpoint": str(data.get("preview_endpoint") or ""), + "source_label": str(data.get("source_label") or ""), + "title": str(data.get("title") or ""), + "executor_kind": str(data.get("executor_kind") or ""), + "execution_chain": str(data.get("execution_chain") or ""), + "primary": { + "label": str(primary.get("label") or ""), + "action_code": str(primary.get("action_code") or ""), + "target_method": str(primary.get("target_method") or ""), + "target_api": str(primary.get("target_api") or ""), + "recommendation_label": str(primary.get("recommendation_label") or ""), + "automation_level_label": str(primary.get("automation_level_label") or ""), + "risk_level_label": str(primary.get("risk_level_label") or ""), + "node_codes": list(primary.get("node_codes") or []), + "request_payload": primary.get("request_payload") or {}, + }, +} + +if secondary: + summary["secondary"] = { + "label": str(secondary.get("label") or ""), + "action_code": str(secondary.get("action_code") or ""), + "target_method": str(secondary.get("target_method") or ""), + "target_api": str(secondary.get("target_api") or ""), + "recommendation_label": str(secondary.get("recommendation_label") or ""), + "automation_level_label": str(secondary.get("automation_level_label") or ""), + "risk_level_label": str(secondary.get("risk_level_label") or ""), + "node_codes": list(secondary.get("node_codes") or []), + "request_payload": secondary.get("request_payload") or {}, + } + +print(json.dumps(summary, ensure_ascii=False, indent=2)) +PY +else + echo "[1/3] driver action request" + print_json_or_raw "${REQUEST_JSON}" + echo + echo + + RAW_HTTP_RESPONSE="$(request_with_status "POST" "${BASE_URL}/api/v1/ops/driver-actions/execute" "${REQUEST_JSON}")" + HTTP_STATUS="${RAW_HTTP_RESPONSE##*$'\n'HTTP_STATUS=}" + RESPONSE_JSON="${RAW_HTTP_RESPONSE%$'\n'HTTP_STATUS=*}" + + echo "[2/3] raw response" + printf '%s\n' "http_status=${HTTP_STATUS}" + print_json_or_raw "${RESPONSE_JSON}" + echo + echo + + echo "[3/3] condensed summary" + python3 - "${RESPONSE_JSON}" "${HTTP_STATUS}" <<'PY' +import json +import sys + +payload = json.loads(sys.argv[1]) +http_status = int(sys.argv[2] or 0) +data = payload.get("data") or {} +smart_release_result = data.get("smart_release_result") or {} +ui_intent = data.get("ui_intent") or {} +runtime_settings = data.get("runtime_settings") or {} + +raw_code = payload.get("code", 1) +if raw_code in (0, "0", False): + normalized_code = 0 +else: + normalized_code = int(raw_code or 1) + +summary = { + "http_status": http_status, + "code": normalized_code, + "message": str(payload.get("message") or ""), + "handled": bool(data.get("handled", False)), + "action_code": str(data.get("action_code") or ""), + "mode": str(data.get("mode") or ""), +} + +if smart_release_result: + release = smart_release_result.get("release") or {} + rollout = smart_release_result.get("rollout") or {} + smart_plan = smart_release_result.get("smart_rollout") or {} + preview = smart_release_result.get("preview") or {} + summary["smart_release_result"] = { + "release_id": int(release.get("id") or 0), + "release_version": str(release.get("release_version") or ""), + "release_created": bool(smart_release_result.get("release_created", False)), + "release_deduplicated": bool(smart_release_result.get("release_deduplicated", False)), + "rollout_created": bool(smart_release_result.get("rollout_created", False)), + "requires_confirmation": bool(smart_release_result.get("requires_confirmation", False)), + "blocked": bool(smart_release_result.get("blocked", False)), + "rollout_id": int(rollout.get("id") or 0), + "rollout_code": str(rollout.get("rollout_code") or ""), + "smart_mode": str(smart_plan.get("mode") or ""), + "smart_available": bool(smart_plan.get("available", False)), + "target_node_codes": list((smart_plan.get("target_selector") or {}).get("node_codes") or []), + "preview_warnings": list(preview.get("warnings") or []), + "preview_approval_required": bool(preview.get("approval_required", False)), + } + +if ui_intent: + summary["ui_intent"] = { + "kind": str(ui_intent.get("kind") or ""), + "release_id": int(ui_intent.get("release_id") or 0), + "template_key": str(ui_intent.get("template_key") or ""), + "playbook_key": str(ui_intent.get("playbook_key") or ""), + "target_node_codes": list(ui_intent.get("target_node_codes") or []), + } + +if runtime_settings: + summary["runtime_settings"] = { + "worker_log_sync_enabled": bool(runtime_settings.get("worker_log_sync_enabled", False)), + "worker_log_sync_mode": str(runtime_settings.get("worker_log_sync_mode") or ""), + } + +print(json.dumps(summary, ensure_ascii=False, indent=2)) +PY +fi + +if [[ "${HTTP_STATUS}" -lt 200 || "${HTTP_STATUS}" -ge 300 ]]; then + exit 1 +fi diff --git a/domain-api/deploy/multi-region/drive_ops_center.sh b/domain-api/deploy/multi-region/drive_ops_center.sh new file mode 100755 index 0000000..16f74b6 --- /dev/null +++ b/domain-api/deploy/multi-region/drive_ops_center.sh @@ -0,0 +1,7776 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck disable=SC1091 +source "${SCRIPT_DIR}/lib_ops_center.sh" +ops_center_load_config "${SCRIPT_DIR}" + +OPS_CENTER_CURL_CONNECT_TIMEOUT="${OPS_CENTER_CURL_CONNECT_TIMEOUT:-3}" +OPS_CENTER_CURL_MAX_TIME="${OPS_CENTER_CURL_MAX_TIME:-20}" +OPS_CENTER_CURL_MAX_TIME_CODEX_BRIEF="${OPS_CENTER_CURL_MAX_TIME_CODEX_BRIEF:-45}" +OPS_CENTER_REPORT_COMMAND_TIMEOUT_SECONDS="${OPS_CENTER_REPORT_COMMAND_TIMEOUT_SECONDS:-25}" +OPS_CENTER_DOCTOR_REPORT_TIMEOUT_SECONDS="${OPS_CENTER_DOCTOR_REPORT_TIMEOUT_SECONDS:-${OPS_CENTER_REPORT_COMMAND_TIMEOUT_SECONDS}}" + +usage() { + cat <<'EOF' +usage: + bash domain-api/deploy/multi-region/drive_ops_center.sh [args...] + +commands: + help + config + contracts [mainland_base_url] [contract_key] + env-audit [mainland_base_url] + runtime-refresh-check [mainland_base_url] + runtime-refresh-recover [mainland_base_url] + stack-diagnosis [mainland_base_url] [summary|full] + stack-check [mainland_base_url] [summary|full] (backward-compatible alias of stack-diagnosis) + go-live-check [mainland_base_url] [overseas_base_url] [summary|full] + go-live-recover [mainland_base_url] [overseas_base_url] + go-live-export [report_dir] [mainland_base_url] [overseas_base_url] + go-live-review [manifest.json|report_dir] + go-live-signoff [manifest.json|report_dir|mainland_base_url] [overseas_base_url] + stack-next [mainland_base_url] [preview|run] [confirm] [requested_by] + release-package [skip-smoke] + release-verify [archive_path] [hash_path] + release-prepare + release-show + release-launchpad [mainland_base_url] [channel] + release-preview [mainland_base_url] + release-preview-smart [mainland_base_url] + release-import-latest [mainland_base_url] + publish-latest [mainland_base_url] [confirm] [skip-smoke] + doctor-export [report_dir] [mainland_base_url] [overseas_base_url] + doctor-decision [manifest_path|report_dir|mainland_base_url] [overseas_base_url] + doctor [mainland_base_url] [overseas_base_url] + ops-plane [mainland_base_url] + inspection [mainland_base_url] + participation [mainland_base_url] + topology [mainland_base_url] [overseas_base_url] + nodes [mainland_base_url] + stack-first-gap-handover [mainland_base_url] + node-bind-ssh [mainland_base_url] [ssh_port] [title] + node-handover [mainland_base_url] + node-onboarding [mainland_base_url] + node-recover [mainland_base_url] [confirm] [requested_by] + node-bootstrap-preview [mainland_base_url] [requested_by] + node-bootstrap-run [mainland_base_url] [requested_by] + node-acceptance-plan [mainland_base_url] [requested_by] + node-acceptance-run [mainland_base_url] [requested_by] + scene-node-log [mainland_base_url] [limit] [key|full] + log-sync-check [mainland_base_url] + log-sync-recover [mainland_base_url] [key|full] [confirm] [requested_by] + agent-gap-check [mainland_base_url] + agent-gap-recover [mainland_base_url] [confirm] [requested_by] + agent-gap-export [report_dir] [mainland_base_url] [requested_by] + node-bootstrap-plan [mainland_base_url] [issued_by] [expires_in_hours] [control_plane_base_url] [root_dir] + queue-status [mainland_base_url] + queue-records [mainland_base_url] [state] + queue-flush [mainland_base_url] [limit] [requested_by] + queue-replay [mainland_base_url] [limit] [requested_by] + queue-replay-record [mainland_base_url] [requested_by] + queue-discard-record [mainland_base_url] [requested_by] + driver-feed [mainland_base_url] + activity-stream [mainland_base_url] [key=value ...] + activity-preview [mainland_base_url] [activity_key] [requested_by] + driver-focus-preview [mainland_base_url] [entry_key] [requested_by] + driver-focus-run [mainland_base_url] [entry_key] [primary|secondary] [confirm] [requested_by] + codex-brief [mainland_base_url] + codex-focus-preview [mainland_base_url] [entry_key] [requested_by] + codex-focus-run [mainland_base_url] [entry_key] [confirm] [requested_by] + release-hub [mainland_base_url] [args...] + driver-preview [mainland_base_url] [action_payload_json|@payload.json] [requested_by] + driver-resolve [mainland_base_url] [action_payload_json|@payload.json] [requested_by] + driver-run [mainland_base_url] [action_payload_json|@payload.json] [confirm] [requested_by] + driver [mainland_base_url] [action_payload_json|@payload.json] [requested_by] + runbook-preview [mainland_base_url] [action_payload_json|@payload.json] [requested_by] + runbook-run [mainland_base_url] [primary|secondary] [action_payload_json|@payload.json] [confirm] [requested_by] + runbook-resolve [mainland_base_url] [primary|secondary] [action_payload_json|@payload.json] [requested_by] + runbook-sequence [mainland_base_url] [primary|secondary] [action_payload_json|@payload.json] [requested_by] + smart-latest [mainland_base_url] [confirm] + smart-release [mainland_base_url] [confirm] + agent-plan [mainland_base_url] [node_region] [node_role] [control_plane_base_url] [root_dir] + agent-plan-export [report_dir] [mainland_base_url] [node_region] [node_role] [control_plane_base_url] [root_dir] + agent-check [service_name] [env_file] + +examples: + bash domain-api/deploy/multi-region/drive_ops_center.sh config + bash domain-api/deploy/multi-region/drive_ops_center.sh contracts + bash domain-api/deploy/multi-region/drive_ops_center.sh env-audit + bash domain-api/deploy/multi-region/drive_ops_center.sh runtime-refresh-check + bash domain-api/deploy/multi-region/drive_ops_center.sh runtime-refresh-recover + bash domain-api/deploy/multi-region/drive_ops_center.sh stack-diagnosis + bash domain-api/deploy/multi-region/drive_ops_center.sh contracts ops_stack_diagnosis_contract + bash domain-api/deploy/multi-region/drive_ops_center.sh contracts http://127.0.0.1:8100 release_hub_contract + bash domain-api/deploy/multi-region/drive_ops_center.sh go-live-recover + bash domain-api/deploy/multi-region/drive_ops_center.sh go-live-check + bash domain-api/deploy/multi-region/drive_ops_center.sh go-live-export + bash domain-api/deploy/multi-region/drive_ops_center.sh go-live-review + bash domain-api/deploy/multi-region/drive_ops_center.sh go-live-signoff + bash domain-api/deploy/multi-region/drive_ops_center.sh go-live-signoff /path/to/go-live-bundle + bash domain-api/deploy/multi-region/drive_ops_center.sh stack-next + bash domain-api/deploy/multi-region/drive_ops_center.sh stack-next run confirm + bash domain-api/deploy/multi-region/drive_ops_center.sh release-package + bash domain-api/deploy/multi-region/drive_ops_center.sh release-show + bash domain-api/deploy/multi-region/drive_ops_center.sh release-launchpad + bash domain-api/deploy/multi-region/drive_ops_center.sh release-preview + bash domain-api/deploy/multi-region/drive_ops_center.sh release-preview-smart worker + bash domain-api/deploy/multi-region/drive_ops_center.sh release-import-latest + bash domain-api/deploy/multi-region/drive_ops_center.sh publish-latest worker + bash domain-api/deploy/multi-region/drive_ops_center.sh doctor-export + bash domain-api/deploy/multi-region/drive_ops_center.sh doctor-decision + bash domain-api/deploy/multi-region/drive_ops_center.sh doctor http://127.0.0.1:8100 http://152.53.37.118:8100 + bash domain-api/deploy/multi-region/drive_ops_center.sh nodes + bash domain-api/deploy/multi-region/drive_ops_center.sh stack-first-gap-handover + bash domain-api/deploy/multi-region/drive_ops_center.sh node-bind-ssh mainland-worker-01 121.204.244.248 root + bash domain-api/deploy/multi-region/drive_ops_center.sh node-bind-ssh http://127.0.0.1:8100 mainland-controller-01 121.204.244.188 root 22 mainland-controller + bash domain-api/deploy/multi-region/drive_ops_center.sh node-handover mainland-worker-01 + bash domain-api/deploy/multi-region/drive_ops_center.sh node-onboarding mainland-worker-01 + bash domain-api/deploy/multi-region/drive_ops_center.sh node-recover mainland-worker-01 + bash domain-api/deploy/multi-region/drive_ops_center.sh node-recover mainland-worker-01 confirm ops-center + bash domain-api/deploy/multi-region/drive_ops_center.sh node-bootstrap-preview mainland-worker-01 + bash domain-api/deploy/multi-region/drive_ops_center.sh node-bootstrap-run mainland-worker-01 ops-center + bash domain-api/deploy/multi-region/drive_ops_center.sh node-acceptance-plan mainland-worker-01 + bash domain-api/deploy/multi-region/drive_ops_center.sh node-acceptance-run mainland-worker-01 ops-center + bash domain-api/deploy/multi-region/drive_ops_center.sh scene-node-log mainland-worker-01 + bash domain-api/deploy/multi-region/drive_ops_center.sh scene-node-log http://127.0.0.1:8100 mainland-worker-01 120 full + bash domain-api/deploy/multi-region/drive_ops_center.sh log-sync-check + bash domain-api/deploy/multi-region/drive_ops_center.sh log-sync-recover + bash domain-api/deploy/multi-region/drive_ops_center.sh log-sync-recover http://127.0.0.1:8100 full confirm cli/ops + bash domain-api/deploy/multi-region/drive_ops_center.sh agent-gap-check + bash domain-api/deploy/multi-region/drive_ops_center.sh agent-gap-recover + bash domain-api/deploy/multi-region/drive_ops_center.sh agent-gap-export + bash domain-api/deploy/multi-region/drive_ops_center.sh node-bootstrap-plan mainland-worker-01 + bash domain-api/deploy/multi-region/drive_ops_center.sh queue-status mainland-worker-01 + bash domain-api/deploy/multi-region/drive_ops_center.sh queue-records mainland-worker-01 dead_letter + bash domain-api/deploy/multi-region/drive_ops_center.sh queue-flush mainland-worker-01 20 + bash domain-api/deploy/multi-region/drive_ops_center.sh queue-replay mainland-worker-01 20 + bash domain-api/deploy/multi-region/drive_ops_center.sh queue-replay-record mainland-worker-01 event-ops-123-2 + bash domain-api/deploy/multi-region/drive_ops_center.sh queue-discard-record mainland-worker-01 event-ops-123-2 "确认已人工归档" + bash domain-api/deploy/multi-region/drive_ops_center.sh driver-feed + bash domain-api/deploy/multi-region/drive_ops_center.sh activity-stream + bash domain-api/deploy/multi-region/drive_ops_center.sh activity-stream kind=ops_job status=running + bash domain-api/deploy/multi-region/drive_ops_center.sh activity-preview + bash domain-api/deploy/multi-region/drive_ops_center.sh driver-focus-preview + bash domain-api/deploy/multi-region/drive_ops_center.sh driver-focus-run + bash domain-api/deploy/multi-region/drive_ops_center.sh codex-brief + bash domain-api/deploy/multi-region/drive_ops_center.sh codex-focus-preview + bash domain-api/deploy/multi-region/drive_ops_center.sh codex-focus-run + bash domain-api/deploy/multi-region/drive_ops_center.sh release-hub http://127.0.0.1:8100 latest-package + bash domain-api/deploy/multi-region/drive_ops_center.sh smart-release http://127.0.0.1:8100 12 worker + bash domain-api/deploy/multi-region/drive_ops_center.sh driver-preview http://127.0.0.1:8100 open_release_dialog + bash domain-api/deploy/multi-region/drive_ops_center.sh driver-resolve http://127.0.0.1:8100 enable_log_sync_key + bash domain-api/deploy/multi-region/drive_ops_center.sh driver-run http://127.0.0.1:8100 enable_log_sync_key + bash domain-api/deploy/multi-region/drive_ops_center.sh driver http://127.0.0.1:8100 open_release_dialog + bash domain-api/deploy/multi-region/drive_ops_center.sh runbook-preview release_progression + bash domain-api/deploy/multi-region/drive_ops_center.sh runbook-run release_progression + bash domain-api/deploy/multi-region/drive_ops_center.sh runbook-resolve release_progression + bash domain-api/deploy/multi-region/drive_ops_center.sh runbook-sequence release_progression + +design: + - this is the single overseas control-plane entrypoint + - it does not execute remote shell directly + - it first delegates to contract preview APIs, then to ops/release execute APIs +EOF +} + +COMMAND="${1:-help}" +shift || true + +run_step() { + local title="$1" + shift + echo "============================================================" + echo "${title}" + echo "============================================================" + "$@" + echo +} + +print_json_or_raw() { + local raw_text="${1:-}" + python3 - "${raw_text}" <<'PY' +import json +import sys + +raw_text = str(sys.argv[1] or "") +try: + payload = json.loads(raw_text) +except Exception: + print(raw_text) +else: + print(json.dumps(payload, ensure_ascii=False, indent=2)) +PY +} + +request_with_status_custom() { + local method="$1" + local url="$2" + local body="${3:-}" + local connect_timeout="${4:-${OPS_CENTER_CURL_CONNECT_TIMEOUT}}" + local max_time="${5:-${OPS_CENTER_CURL_MAX_TIME}}" + if [[ "${method}" == "POST" ]]; then + curl -sS -X POST "${url}" \ + --connect-timeout "${connect_timeout}" \ + --max-time "${max_time}" \ + -H 'Content-Type: application/json' \ + -d "${body}" \ + -w $'\nHTTP_STATUS=%{http_code}' + else + curl -sS \ + --connect-timeout "${connect_timeout}" \ + --max-time "${max_time}" \ + "${url}" \ + -w $'\nHTTP_STATUS=%{http_code}' + fi +} + +request_with_status() { + local method="$1" + local url="$2" + local body="${3:-}" + request_with_status_custom "${method}" "${url}" "${body}" \ + "${OPS_CENTER_CURL_CONNECT_TIMEOUT}" "${OPS_CENTER_CURL_MAX_TIME}" +} + +print_config() { + local requested_profile="${1:-${OPS_CENTER_ACTIVE_PROFILE}}" + local saved_profile="${OPS_CENTER_ACTIVE_PROFILE}" + local saved_profile_key="${OPS_CENTER_ACTIVE_PROFILE_KEY}" + OPS_CENTER_ACTIVE_PROFILE="${requested_profile}" + OPS_CENTER_ACTIVE_PROFILE_KEY="$(ops_center_normalize_profile_key "${OPS_CENTER_ACTIVE_PROFILE}")" + python3 - <<'PY' "${OPS_CENTER_CONFIG_FILE}" \ + "${OPS_CENTER_ACTIVE_PROFILE}" \ + "${OPS_CENTER_PROFILE_NAMES}" \ + "$(ops_center_resolve_mainland_api_base_url)" \ + "$(ops_center_resolve_overseas_api_base_url)" \ + "$(ops_center_resolve_control_plane_base_url)" \ + "$(ops_center_resolve_requested_by)" \ + "$(ops_center_resolve_issued_by)" \ + "$(ops_center_resolve_expires_in_hours)" \ + "$(ops_center_resolve_root_dir)" \ + "$(ops_center_resolve_node_agent_service_name)" \ + "$(ops_center_resolve_node_agent_env_file)" \ + "${OPS_CENTER_REPORT_ROOT}" +import json +import sys +from pathlib import Path + +config_path = Path(sys.argv[1]) +print(json.dumps({ + "config_file": str(config_path), + "config_exists": config_path.exists(), + "active_profile": sys.argv[2], + "available_profiles": [item.strip() for item in str(sys.argv[3]).split(",") if item.strip()], + "ops_mainland_api_base_url": sys.argv[4], + "ops_overseas_api_base_url": sys.argv[5], + "ops_default_control_plane_base_url": sys.argv[6], + "ops_default_requested_by": sys.argv[7], + "ops_default_issued_by": sys.argv[8], + "ops_default_expires_in_hours": int(sys.argv[9] or 72), + "ops_default_root_dir": sys.argv[10], + "ops_node_agent_service_name": sys.argv[11], + "ops_node_agent_env_file": sys.argv[12], + "ops_report_root": sys.argv[13], +}, ensure_ascii=False, indent=2)) +PY + OPS_CENTER_ACTIVE_PROFILE="${saved_profile}" + OPS_CENTER_ACTIVE_PROFILE_KEY="${saved_profile_key}" +} + +timestamp_now() { + date '+%Y%m%d_%H%M%S' +} + +ops_center_repo_root() { + printf '%s' "${OPS_CENTER_PROJECT_ROOT}" +} + +confirm_to_bool() { + local raw_value="${1:-}" + case "$(printf '%s' "${raw_value}" | tr '[:upper:]' '[:lower:]')" in + 1|true|yes|y|confirm|confirmed|skip|skip-smoke) + printf 'true' + ;; + *) + printf 'false' + ;; + esac +} + +is_confirm_token() { + local raw_value="${1:-}" + case "$(printf '%s' "${raw_value}" | tr '[:upper:]' '[:lower:]')" in + 1|true|yes|y|confirm|confirmed) + return 0 + ;; + *) + return 1 + ;; + esac +} + +is_secondary_token() { + local raw_value="${1:-}" + case "$(printf '%s' "${raw_value}" | tr '[:upper:]' '[:lower:]')" in + secondary|second|2|true|yes|y) + return 0 + ;; + *) + return 1 + ;; + esac +} + +normalize_payload_json_input() { + local raw_value="${1:-}" + if [[ -z "${raw_value}" ]]; then + printf '{}' + return 0 + fi + if [[ "${raw_value}" == @* ]]; then + local payload_path="${raw_value#@}" + if [[ ! -f "${payload_path}" ]]; then + echo "payload file not found: ${payload_path}" >&2 + return 1 + fi + cat "${payload_path}" + return 0 + fi + printf '%s' "${raw_value}" +} + +enrich_driver_action_payload_json() { + local mainland_base_url="$1" + local action_code="$2" + local payload_json="$3" + + python3 - <<'PY' "${mainland_base_url}" "${action_code}" "${payload_json}" +import json +import sys +import urllib.request + + +def normalize_codes(*groups): + result = [] + for group in groups: + for item in list(group or []): + node_code = str(item or "").strip() + if node_code and node_code not in result: + result.append(node_code) + return result + + +base_url = str(sys.argv[1] or "").rstrip("/") +action_code = str(sys.argv[2] or "").strip() +payload_raw = str(sys.argv[3] or "").strip() or "{}" + +try: + payload = json.loads(payload_raw) +except Exception: + payload = {} +if not isinstance(payload, dict): + payload = {} + +existing_node_codes = normalize_codes(payload.get("node_codes") or []) +if existing_node_codes: + print(json.dumps(payload, ensure_ascii=False)) + raise SystemExit(0) + +supported_actions = { + "open_worker_logs_participating", + "run_inspection_participating", + "open_worker_logs_standby", + "run_inspection_standby", + "open_worker_logs", + "run_standard_inspection", +} +if action_code not in supported_actions or not base_url: + print(json.dumps(payload, ensure_ascii=False)) + raise SystemExit(0) + +overview = {} +try: + with urllib.request.urlopen(f"{base_url}/api/v1/ops/overview", timeout=20) as response: + overview_payload = json.loads(response.read().decode("utf-8")) + overview = dict((overview_payload or {}).get("data") or {}) +except Exception: + print(json.dumps(payload, ensure_ascii=False)) + raise SystemExit(0) + +execution_scene = dict(overview.get("execution_scene") or {}) +participation_summary = dict(execution_scene.get("participation_summary") or {}) +log_sync = dict(execution_scene.get("log_sync") or {}) + +dispatch_active = normalize_codes(participation_summary.get("dispatch_active_node_codes") or []) +recent_only = normalize_codes(participation_summary.get("recent_only_node_codes") or []) +standby = normalize_codes( + participation_summary.get("standby_node_codes") or [], + participation_summary.get("non_participating_node_codes") or [], +) +missing_logs = normalize_codes(log_sync.get("missing_participating_nodes") or []) + +derived_node_codes = [] +if action_code in {"open_worker_logs_participating", "run_inspection_participating"}: + derived_node_codes = normalize_codes(missing_logs, dispatch_active, recent_only) +elif action_code in {"open_worker_logs_standby", "run_inspection_standby"}: + derived_node_codes = normalize_codes(standby) +elif action_code == "open_worker_logs": + derived_node_codes = normalize_codes(missing_logs, dispatch_active, recent_only, standby) +elif action_code == "run_standard_inspection": + derived_node_codes = normalize_codes(dispatch_active, recent_only, standby) + +if derived_node_codes: + payload["node_codes"] = derived_node_codes + payload.setdefault("_auto_context", {}) + payload["_auto_context"]["resolved_by"] = "drive_ops_center" + payload["_auto_context"]["resolved_from"] = "ops.overview" + payload["_auto_context"]["action_code"] = action_code + +print(json.dumps(payload, ensure_ascii=False)) +PY +} + +normalize_optional_base_url_arg() { + local raw_value="${1:-}" + local default_value="${2:-}" + case "$(printf '%s' "${raw_value}" | tr '[:upper:]' '[:lower:]')" in + "" ) + printf '%s' "${default_value}" + ;; + -|none|null|disabled|off) + printf '' + ;; + *) + printf '%s' "${raw_value}" + ;; + esac +} + +write_json_file() { + local output_path="$1" + local json_payload="$2" + python3 - "${output_path}" "${json_payload}" <<'PY' +import json +import sys +from pathlib import Path + +path = Path(sys.argv[1]) +payload = json.loads(sys.argv[2]) +path.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8") +PY +} + +run_and_capture_report() { + local title="$1" + local output_path="$2" + shift 2 + { + echo "============================================================" + echo "${title}" + echo "============================================================" + "$@" + echo + } | tee "${output_path}" +} + +capture_report_quiet() { + local title="$1" + local output_path="$2" + shift 2 + local command_exit_code=0 + echo "============================================================" + echo "${title}" + echo "============================================================" + { + echo "============================================================" + echo "${title}" + echo "============================================================" + set +e + timeout "${OPS_CENTER_DOCTOR_REPORT_TIMEOUT_SECONDS}" "$@" + command_exit_code=$? + set -e + echo + echo "command_exit_code=${command_exit_code}" + } >"${output_path}" + echo "saved_report=${output_path}" + echo "command_exit_code=${command_exit_code}" + echo +} + +capture_report_optional() { + local title="$1" + local output_path="$2" + shift 2 + local command_exit_code=0 + + echo "============================================================" + echo "${title}" + echo "============================================================" + { + echo "============================================================" + echo "${title}" + echo "============================================================" + set +e + timeout "${OPS_CENTER_DOCTOR_REPORT_TIMEOUT_SECONDS}" "$@" + command_exit_code=$? + set -e + echo + echo "command_exit_code=${command_exit_code}" + } >"${output_path}" 2>&1 + echo "saved_report=${output_path}" + echo "command_exit_code=${command_exit_code}" + echo +} + +run_doctor_report_export() { + local report_dir="$1" + local mainland_base_url="$2" + local overseas_base_url="$3" + ops_center_ensure_report_dir "${report_dir}" + local -a report_files=( + "00_ops_center_stack_summary.txt" + "01_ops_plane.txt" + "02_release_hub.txt" + "03_inspection.txt" + "04_participation.txt" + ) + if [[ -n "${overseas_base_url}" ]]; then + report_files+=("05_topology.txt") + fi + report_files+=( + "06_contracts_registry.txt" + "07_driver_feed.txt" + "08_activity_stream.txt" + "09_codex_brief.txt" + "10_stack_next_preview.txt" + "11_driver_focus_preview.txt" + "12_activity_preview.txt" + "13_codex_focus_preview.txt" + "14_first_handover_gap.txt" + ) + + capture_report_quiet "[doctor/0] ops center stack" "${report_dir}/00_ops_center_stack_summary.txt" \ + bash "${SCRIPT_DIR}/check_ops_center_stack.sh" "${mainland_base_url}" summary + capture_report_quiet "[doctor/1] ops plane" "${report_dir}/01_ops_plane.txt" \ + bash "${SCRIPT_DIR}/check_ops_plane.sh" "${mainland_base_url}" + capture_report_quiet "[doctor/2] release hub" "${report_dir}/02_release_hub.txt" \ + bash "${SCRIPT_DIR}/check_release_hub.sh" "${mainland_base_url}" + capture_report_quiet "[doctor/3] inspection" "${report_dir}/03_inspection.txt" \ + bash "${SCRIPT_DIR}/check_ops_inspection.sh" "${mainland_base_url}" + capture_report_quiet "[doctor/4] participation" "${report_dir}/04_participation.txt" \ + bash "${SCRIPT_DIR}/check_worker_participation.sh" "${mainland_base_url}" + if [[ -n "${overseas_base_url}" ]]; then + capture_report_quiet "[doctor/5] temp topology" "${report_dir}/05_topology.txt" \ + bash "${SCRIPT_DIR}/check_temp_topology.sh" "${mainland_base_url}" "${overseas_base_url}" + fi + capture_report_quiet "[doctor/6] contracts registry" "${report_dir}/06_contracts_registry.txt" \ + bash "${SCRIPT_DIR}/drive_ops_center.sh" contracts "${mainland_base_url}" + capture_report_quiet "[doctor/7] driver feed" "${report_dir}/07_driver_feed.txt" \ + bash "${SCRIPT_DIR}/drive_ops_center.sh" driver-feed "${mainland_base_url}" + capture_report_quiet "[doctor/8] activity stream" "${report_dir}/08_activity_stream.txt" \ + bash "${SCRIPT_DIR}/drive_ops_center.sh" activity-stream "${mainland_base_url}" + capture_report_quiet "[doctor/9] codex brief" "${report_dir}/09_codex_brief.txt" \ + bash "${SCRIPT_DIR}/drive_ops_center.sh" codex-brief "${mainland_base_url}" + capture_report_optional "[doctor/10] stack next preview" "${report_dir}/10_stack_next_preview.txt" \ + bash "${SCRIPT_DIR}/drive_ops_center.sh" stack-next "${mainland_base_url}" preview + capture_report_optional "[doctor/11] driver focus preview" "${report_dir}/11_driver_focus_preview.txt" \ + bash "${SCRIPT_DIR}/drive_ops_center.sh" driver-focus-preview "${mainland_base_url}" + capture_report_optional "[doctor/12] activity preview" "${report_dir}/12_activity_preview.txt" \ + bash "${SCRIPT_DIR}/drive_ops_center.sh" activity-preview "${mainland_base_url}" + capture_report_optional "[doctor/13] codex focus preview" "${report_dir}/13_codex_focus_preview.txt" \ + bash "${SCRIPT_DIR}/drive_ops_center.sh" codex-focus-preview "${mainland_base_url}" + capture_report_optional "[doctor/14] first handover gap" "${report_dir}/14_first_handover_gap.txt" \ + bash "${SCRIPT_DIR}/drive_ops_center.sh" stack-first-gap-handover "${mainland_base_url}" + + local scene_log_targets_json + scene_log_targets_json="$(python3 - <<'PY' "${report_dir}/00_ops_center_stack_summary.txt" +import json +import re +import sys +from pathlib import Path + +target = Path(str(sys.argv[1] or "").strip()) +if not target.exists(): + print("[]") + raise SystemExit(0) + +text = target.read_text(encoding="utf-8", errors="replace") +decoder = json.JSONDecoder() +best = None +for index, ch in enumerate(text): + if ch not in "{[": + continue + try: + value, end = decoder.raw_decode(text[index:]) + except Exception: + continue + absolute_end = index + end + if best is None or absolute_end >= best[0]: + best = (absolute_end, value) + +payload = best[1] if best else {} +diagnosis = dict((payload or {}).get("diagnosis") or {}) +issues = list(diagnosis.get("issues") or []) +next_step = dict(diagnosis.get("next_step") or {}) +operator_decision = dict(diagnosis.get("operator_decision") or {}) + +targets = [] +seen = set() + +def add_focus_ref(focus_ref): + if not isinstance(focus_ref, dict): + return + if str(focus_ref.get("kind") or "").strip() != "node_scene_log": + return + node_code = str(focus_ref.get("node_code") or "").strip() + if not node_code or node_code in seen: + return + seen.add(node_code) + mode = str(focus_ref.get("mode") or "key").strip().lower() + if mode not in {"key", "full"}: + mode = "key" + try: + limit = int(focus_ref.get("limit") or 80) + except Exception: + limit = 80 + limit = max(20, min(limit, 300)) + targets.append({ + "node_code": node_code, + "mode": mode, + "limit": limit, + "source": str(focus_ref.get("source") or "").strip(), + }) + +for issue in issues: + add_focus_ref((issue or {}).get("focus_ref")) +add_focus_ref(next_step.get("focus_ref")) +add_focus_ref(operator_decision.get("next_focus")) + +print(json.dumps(targets[:3], ensure_ascii=False)) +PY +)" + + while IFS=$'\t' read -r scene_node_code scene_mode scene_limit scene_source; do + [[ -z "${scene_node_code}" ]] && continue + local safe_node_code + safe_node_code="$(printf '%s' "${scene_node_code}" | tr -c 'A-Za-z0-9._-' '_')" + local scene_report_name="15_scene_node_log_${safe_node_code}.txt" + report_files+=("${scene_report_name}") + capture_report_optional "[doctor/15] scene node log ${scene_node_code}" "${report_dir}/${scene_report_name}" \ + bash "${SCRIPT_DIR}/drive_ops_center.sh" scene-node-log "${mainland_base_url}" "${scene_node_code}" "${scene_limit:-80}" "${scene_mode:-key}" + done < <( + python3 - <<'PY' "${scene_log_targets_json}" +import json +import sys + +targets = json.loads(str(sys.argv[1] or "[]")) +for item in targets: + print( + "\t".join( + [ + str(item.get("node_code") or "").strip(), + str(item.get("mode") or "key").strip(), + str(item.get("limit") or 80).strip(), + str(item.get("source") or "").strip(), + ] + ) + ) +PY + ) + + write_json_file "${report_dir}/meta.json" "$(python3 - \ + "${OPS_CENTER_CONFIG_FILE}" \ + "${OPS_CENTER_ACTIVE_PROFILE}" \ + "${mainland_base_url}" \ + "${overseas_base_url}" \ + "${report_dir}" \ + "${report_files[@]}" <<'PY' +import json +import sys +from datetime import datetime + +files = list(sys.argv[6:]) + +print(json.dumps({ + "generated_at": datetime.now().isoformat(timespec="seconds"), + "config_file": sys.argv[1], + "active_profile": sys.argv[2], + "mainland_api_base_url": sys.argv[3], + "overseas_api_base_url": sys.argv[4], + "report_dir": sys.argv[5], + "files": files, +}, ensure_ascii=False)) +PY +)" + + write_json_file "${report_dir}/manifest.json" "$(python3 - \ + "${OPS_CENTER_CONFIG_FILE}" \ + "${OPS_CENTER_ACTIVE_PROFILE}" \ + "${mainland_base_url}" \ + "${overseas_base_url}" \ + "${report_dir}" \ + "${report_files[@]}" <<'PY' +import json +import re +import sys +from datetime import datetime +from pathlib import Path + +config_file = str(sys.argv[1] or "").strip() +active_profile = str(sys.argv[2] or "").strip() +mainland_api_base_url = str(sys.argv[3] or "").strip() +overseas_api_base_url = str(sys.argv[4] or "").strip() +report_dir = Path(str(sys.argv[5] or "").strip()) +report_files = [str(item or "").strip() for item in sys.argv[6:] if str(item or "").strip()] +optional_preview_files = { + "10_stack_next_preview.txt", + "11_driver_focus_preview.txt", + "12_activity_preview.txt", + "13_codex_focus_preview.txt", +} + + +def extract_exit_code(text: str) -> int: + matches = re.findall(r"(?m)^command_exit_code=(\d+)\s*$", text) + if matches: + try: + return int(matches[-1]) + except Exception: + return 1 + return 0 + + +def extract_last_json_value(text: str): + decoder = json.JSONDecoder() + best = None + for index, ch in enumerate(text): + if ch not in "{[": + continue + try: + value, end = decoder.raw_decode(text[index:]) + except Exception: + continue + absolute_end = index + end + if best is None or absolute_end >= best[0]: + best = (absolute_end, value) + return None if best is None else best[1] + + +def last_nonempty_line(text: str) -> str: + for line in reversed(text.splitlines()): + stripped = line.strip() + if stripped: + return stripped[:300] + return "" + + +def collect_contract_keys(value, bucket: set) -> None: + if isinstance(value, dict): + for key, child in value.items(): + if key in {"contract_key", "primary_contract_key"}: + contract_key = str(child or "").strip() + if contract_key: + bucket.add(contract_key) + continue + if key in {"contract_keys", "related_contract_keys"} and isinstance(child, list): + for item in child: + contract_key = str(item or "").strip() + if contract_key: + bucket.add(contract_key) + continue + collect_contract_keys(child, bucket) + elif isinstance(value, list): + for item in value: + collect_contract_keys(item, bucket) + + +def summarize_report(name: str, payload, exit_code: int, text: str) -> dict: + if not isinstance(payload, dict): + return { + "parsed_json": False, + "available": exit_code == 0, + "text_tail": last_nonempty_line(text), + } + + if name == "00_ops_center_stack_summary.txt": + diagnosis = dict(payload.get("diagnosis") or {}) + api = dict(payload.get("api") or {}) + issues = list(diagnosis.get("issues") or payload.get("issues") or []) + next_step = dict(diagnosis.get("next_step") or payload.get("next_step") or {}) + operator_decision = dict(diagnosis.get("operator_decision") or {}) + next_actions = dict(diagnosis.get("next_actions") or {}) + recommended_commands = dict(diagnosis.get("recommended_commands") or {}) + stack_status = str(diagnosis.get("stack_status") or payload.get("stack_status") or payload.get("status") or "") + blocked_issue = next( + (dict(item or {}) for item in issues if str((item or {}).get("severity") or "") == "blocked"), + {}, + ) + headline = "" + if stack_status == "blocked": + headline = str(blocked_issue.get("summary") or "").strip() + if not headline: + headline = str( + diagnosis.get("headline") + or (payload.get("link_snapshot") or {}).get("headline") + or "" + ).strip() + return { + "parsed_json": True, + "available": exit_code == 0, + "http_status": int(api.get("http_status", 0) or 0), + "stack_status": stack_status, + "surface_status": str(diagnosis.get("surface_status") or ""), + "automation_status": str(diagnosis.get("automation_status") or ""), + "issues_total": len(issues), + "blocked_issues": sum(1 for item in issues if str((item or {}).get("severity") or "") == "blocked"), + "headline": headline, + "blocked_issue_detail": str(blocked_issue.get("detail") or "").strip(), + "launchpad_recommended_target_node_code": str(diagnosis.get("launchpad_recommended_target_node_code") or ""), + "launchpad_recommended_recovery_label": str(diagnosis.get("launchpad_recommended_recovery_label") or ""), + "launchpad_recommended_recovery_summary": str(diagnosis.get("launchpad_recommended_recovery_summary") or ""), + "launchpad_onboarding_bootstrap_pending_nodes": int(diagnosis.get("launchpad_onboarding_bootstrap_pending_nodes", 0) or 0), + "launchpad_onboarding_acceptance_ready_nodes": int(diagnosis.get("launchpad_onboarding_acceptance_ready_nodes", 0) or 0), + "next_step_action_code": str(next_step.get("action_code") or ""), + "next_step_summary": str(next_step.get("summary") or next_step.get("title") or next_step.get("reason") or ""), + "operator_lane": str(operator_decision.get("lane") or "").strip(), + "operator_priority": str(operator_decision.get("priority") or "").strip(), + "operator_reason_code": str(operator_decision.get("reason_code") or "").strip(), + "operator_title": str(operator_decision.get("title") or "").strip(), + "operator_summary": str(operator_decision.get("summary") or "").strip(), + "operator_primary_command_key": str(operator_decision.get("primary_command_key") or "").strip(), + "operator_secondary_command_key": str(operator_decision.get("secondary_command_key") or "").strip(), + "operator_primary_command": str( + next_actions.get("primary_command") + or recommended_commands.get(str(operator_decision.get("primary_command_key") or "").strip()) + or "" + ).strip(), + "operator_secondary_command": str( + next_actions.get("secondary_command") + or recommended_commands.get(str(operator_decision.get("secondary_command_key") or "").strip()) + or "" + ).strip(), + "operator_focus_kind": str((operator_decision.get("next_focus") or {}).get("kind") or "").strip(), + "operator_focus_node_code": str((operator_decision.get("next_focus") or {}).get("node_code") or "").strip(), + } + + if name == "06_contracts_registry.txt": + contracts = list(payload.get("contracts") or []) + return { + "parsed_json": True, + "available": exit_code == 0, + "http_status": int(payload.get("http_status", 0) or 0), + "registry_version": str(payload.get("registry_version") or ""), + "schema_version": str(payload.get("schema_version") or ""), + "contracts_total": int(payload.get("contracts_total", 0) or len(contracts)), + } + + if name == "07_driver_feed.txt": + summary = dict(payload.get("summary") or {}) + top_recommendation = dict(payload.get("top_recommendation") or {}) + contract_navigation = dict(payload.get("contract_navigation") or {}) + return { + "parsed_json": True, + "available": exit_code == 0, + "http_status": int(payload.get("http_status", 0) or 0), + "headline": str(payload.get("headline") or ""), + "primary_contract_key": str(contract_navigation.get("primary_contract_key") or ""), + "entry_count": len(list(payload.get("entries") or [])), + "activity_focus_count": len(list(payload.get("activity_focus") or [])), + "top_recommendation_key": str(top_recommendation.get("key") or ""), + "top_recommendation_contract_key": str(top_recommendation.get("primary_contract_key") or ""), + "do_now_total": int(summary.get("do_now_total", 0) or 0), + "launchpad_recommended_target_node_code": str(summary.get("launchpad_recommended_target_node_code") or ""), + "launchpad_recommended_recovery_label": str(summary.get("launchpad_recommended_recovery_label") or ""), + "launchpad_recommended_recovery_summary": str(summary.get("launchpad_recommended_recovery_summary") or ""), + "launchpad_onboarding_bootstrap_pending_nodes": int(summary.get("launchpad_onboarding_bootstrap_pending_nodes", 0) or 0), + "launchpad_onboarding_acceptance_ready_nodes": int(summary.get("launchpad_onboarding_acceptance_ready_nodes", 0) or 0), + } + + if name == "08_activity_stream.txt": + summary = dict(payload.get("summary") or {}) + contract_navigation = dict(payload.get("contract_navigation") or {}) + items = list(payload.get("items") or []) + first_item = dict(items[0] or {}) if items else {} + return { + "parsed_json": True, + "available": exit_code == 0, + "http_status": int(payload.get("http_status", 0) or 0), + "primary_contract_key": str(contract_navigation.get("primary_contract_key") or ""), + "filtered_total": int(summary.get("filtered_total", 0) or 0), + "item_count": len(items), + "first_item_key": str(first_item.get("activity_key") or ""), + "first_item_contract_key": str(first_item.get("primary_contract_key") or ""), + } + + if name == "09_codex_brief.txt": + summary = dict(payload.get("summary") or {}) + focus = dict(payload.get("focus") or {}) + contract_navigation = dict(payload.get("contract_navigation") or {}) + return { + "parsed_json": True, + "available": exit_code == 0, + "http_status": int(payload.get("http_status", 0) or 0), + "headline": str(payload.get("headline") or ""), + "primary_contract_key": str(contract_navigation.get("primary_contract_key") or ""), + "focus_entry_key": str(focus.get("entry_key") or ""), + "focus_contract_key": str(focus.get("primary_contract_key") or ""), + "entry_count": len(list(payload.get("entries") or [])), + "activity_focus_count": len(list(payload.get("activity_focus") or [])), + "blocked_total": int(summary.get("blocked_total", 0) or 0), + "launchpad_recommended_target_node_code": str(summary.get("launchpad_recommended_target_node_code") or ""), + "launchpad_recommended_recovery_label": str(summary.get("launchpad_recommended_recovery_label") or ""), + "launchpad_recommended_recovery_summary": str(summary.get("launchpad_recommended_recovery_summary") or ""), + "launchpad_onboarding_bootstrap_pending_nodes": int(summary.get("launchpad_onboarding_bootstrap_pending_nodes", 0) or 0), + "launchpad_onboarding_acceptance_ready_nodes": int(summary.get("launchpad_onboarding_acceptance_ready_nodes", 0) or 0), + } + + if name in optional_preview_files: + contract = dict(payload.get("contract") or {}) + gate = dict(payload.get("gate") or {}) + selected = dict(payload.get("selected_entry") or {}) + selected_key = ( + str(selected.get("key") or "").strip() + or str(selected.get("activity_key") or "").strip() + or str(selected.get("action_code") or "").strip() + ) + return { + "parsed_json": True, + "available": exit_code == 0, + "http_status": int(payload.get("http_status", 0) or 0), + "contract_key": str(contract.get("contract_key") or ""), + "contract_version": str(contract.get("contract_version") or ""), + "preview_endpoint": str(contract.get("preview_endpoint") or ""), + "gate_decision": str(gate.get("decision") or ""), + "selected_key": selected_key, + "selected_title": str(selected.get("title") or ""), + } + + if name.startswith("15_scene_node_log_"): + participation = dict(payload.get("participation") or {}) + source_summary = dict(payload.get("source_summary") or {}) + return { + "parsed_json": True, + "available": exit_code == 0, + "http_status": int(payload.get("http_status", 0) or 0), + "node_code": str(payload.get("node_code") or ""), + "status": str(payload.get("status") or ""), + "status_label": str(payload.get("status_label") or ""), + "mode": str(payload.get("mode") or ""), + "records_total": int(payload.get("records_total", 0) or 0), + "detect_participating": bool(participation.get("detect_participating", False)), + "participation_label": str(participation.get("participation_label") or ""), + "source_line_count": int(source_summary.get("line_count", 0) or 0), + "missing_reason_code": str(payload.get("missing_reason_code") or ""), + } + + return { + "parsed_json": True, + "available": exit_code == 0, + "top_level_keys": list(payload.keys())[:12], + } + + +def classify_report_failure(name: str, exists: bool, payload, summary: dict, exit_code: int, text: str) -> dict: + http_status_raw = "" + if isinstance(summary, dict): + http_status_raw = str(summary.get("http_status") or "").strip() + text_tail = last_nonempty_line(text) + has_payload = isinstance(payload, (dict, list)) + detail = "" + reason_code = "" + label = "" + if not exists: + reason_code = "report_missing" + label = "未生成报告" + detail = "导出阶段没有落盘该 report 文件。" + elif exit_code == 124: + reason_code = "command_timeout" + label = "执行超时" + detail = "报告命令超过超时阈值后被终止，建议先单独复跑对应子命令。" + elif exit_code != 0: + reason_code = "command_failed" + label = "命令失败" + detail = f"报告命令退出码为 {exit_code}。" + elif http_status_raw and http_status_raw.isdigit() and not (200 <= int(http_status_raw) < 300): + http_status = int(http_status_raw) + if http_status == 0: + reason_code = "http_unreachable" + label = "接口不可达" + detail = "curl 未拿到有效 HTTP 状态，通常表示超时、连接失败或被中间网络丢弃。" + else: + reason_code = "http_error" + label = "接口异常" + detail = f"目标接口返回 HTTP {http_status}。" + elif not has_payload: + reason_code = "result_unparsed" + label = "结果不可解析" + detail = "命令成功返回，但未解析出结构化 JSON 结果。" + else: + reason_code = "ok" + label = "正常" + detail = "" + + if text_tail and reason_code != "ok": + detail = (detail + f" 最近输出: {text_tail}")[:500].strip() + + return { + "reason_code": reason_code, + "label": label, + "detail": detail, + "timeout": reason_code == "command_timeout", + "http_status": int(http_status_raw) if http_status_raw.isdigit() else 0, + "text_tail": text_tail, + "parsed_json": bool(isinstance(summary, dict) and summary.get("parsed_json")), + } + + +reports = [] +all_contract_keys = set() +required_failures = [] +optional_unavailable = [] + +for name in report_files: + path = report_dir / name + exists = path.exists() + text = path.read_text(encoding="utf-8", errors="replace") if exists else "" + exit_code = extract_exit_code(text) if exists else 1 + parsed_payload = extract_last_json_value(text) if exists else None + summary = summarize_report(name, parsed_payload, exit_code, text) + if parsed_payload is not None: + collect_contract_keys(parsed_payload, all_contract_keys) + http_status = summary.get("http_status") if isinstance(summary, dict) else None + http_ok = True + if isinstance(http_status, int): + http_ok = 200 <= http_status < 300 + report_ok = exists and exit_code == 0 and http_ok + if isinstance(summary, dict) and "available" in summary: + summary["available"] = report_ok + report_entry = { + "name": name, + "path": str(path), + "exists": exists, + "size_bytes": int(path.stat().st_size) if exists else 0, + "optional": name in optional_preview_files, + "command_exit_code": exit_code, + "ok": report_ok, + "summary": summary, + "failure": classify_report_failure(name, exists, parsed_payload, summary, exit_code, text), + } + reports.append(report_entry) + + if name in optional_preview_files: + if not report_entry["ok"]: + optional_unavailable.append(name) + else: + if not report_entry["ok"]: + required_failures.append(name) + +preview_contract_keys = sorted( + { + str((report.get("summary") or {}).get("contract_key") or "").strip() + for report in reports + if report.get("name") in optional_preview_files + and str((report.get("summary") or {}).get("contract_key") or "").strip() + } +) +primary_contract_keys = sorted( + { + str((report.get("summary") or {}).get("primary_contract_key") or "").strip() + for report in reports + if str((report.get("summary") or {}).get("primary_contract_key") or "").strip() + } +) +reports_by_name = {str(report.get("name") or ""): report for report in reports} +scene_log_reports = [ + report + for report in reports + if str(report.get("name") or "").startswith("15_scene_node_log_") +] +scene_log_report_summaries = [dict(report.get("summary") or {}) for report in scene_log_reports] +scene_log_reports_total = len(scene_log_reports) +scene_log_reports_ok = sum(1 for report in scene_log_reports if bool(report.get("ok"))) +scene_log_status_counts = {} +for summary in scene_log_report_summaries: + status_key = str(summary.get("status") or "unknown").strip() or "unknown" + scene_log_status_counts[status_key] = int(scene_log_status_counts.get(status_key, 0) or 0) + 1 +scene_log_problem_reports = [ + summary + for summary in scene_log_report_summaries + if str(summary.get("status") or "").strip() in {"waiting_sample", "missing_sample", "disabled", "standby", "unknown"} +] +scene_log_problem_node_codes = [ + str(summary.get("node_code") or "").strip() + for summary in scene_log_problem_reports + if str(summary.get("node_code") or "").strip() +] +scene_log_healthy_reports = [ + summary + for summary in scene_log_report_summaries + if str(summary.get("status") or "").strip() in {"healthy", "full_capture", "historical_sample"} +] + +required_failure_details = [ + { + "name": str(report.get("name") or ""), + "reason_code": str((report.get("failure") or {}).get("reason_code") or ""), + "label": str((report.get("failure") or {}).get("label") or ""), + "detail": str((report.get("failure") or {}).get("detail") or ""), + } + for report in reports + if not bool(report.get("optional")) and not bool(report.get("ok")) +] +optional_unavailable_details = [ + { + "name": str(report.get("name") or ""), + "reason_code": str((report.get("failure") or {}).get("reason_code") or ""), + "label": str((report.get("failure") or {}).get("label") or ""), + "detail": str((report.get("failure") or {}).get("detail") or ""), + } + for report in reports + if bool(report.get("optional")) and not bool(report.get("ok")) +] + + +def report_summary(name: str) -> dict: + return dict((reports_by_name.get(name) or {}).get("summary") or {}) + + +def report_ok(name: str) -> bool: + return bool((reports_by_name.get(name) or {}).get("ok")) + + +def report_http_status(name: str) -> int: + value = report_summary(name).get("http_status") + return int(value or 0) if isinstance(value, int) or str(value).isdigit() else 0 + + +contracts_summary = report_summary("06_contracts_registry.txt") +driver_feed_summary = report_summary("07_driver_feed.txt") +activity_stream_summary = report_summary("08_activity_stream.txt") +codex_brief_summary = report_summary("09_codex_brief.txt") +stack_summary = report_summary("00_ops_center_stack_summary.txt") +launchpad_recommended_target_node_code = next( + ( + str(source.get("launchpad_recommended_target_node_code") or "").strip() + for source in [driver_feed_summary, codex_brief_summary, stack_summary] + if str(source.get("launchpad_recommended_target_node_code") or "").strip() + ), + "", +) +launchpad_recommended_recovery_label = next( + ( + str(source.get("launchpad_recommended_recovery_label") or "").strip() + for source in [driver_feed_summary, codex_brief_summary, stack_summary] + if str(source.get("launchpad_recommended_recovery_label") or "").strip() + ), + "", +) +launchpad_recommended_recovery_summary = next( + ( + str(source.get("launchpad_recommended_recovery_summary") or "").strip() + for source in [driver_feed_summary, codex_brief_summary, stack_summary] + if str(source.get("launchpad_recommended_recovery_summary") or "").strip() + ), + "", +) +launchpad_onboarding_bootstrap_pending_nodes = max( + int(driver_feed_summary.get("launchpad_onboarding_bootstrap_pending_nodes", 0) or 0), + int(codex_brief_summary.get("launchpad_onboarding_bootstrap_pending_nodes", 0) or 0), + int(stack_summary.get("launchpad_onboarding_bootstrap_pending_nodes", 0) or 0), +) +launchpad_onboarding_acceptance_ready_nodes = max( + int(driver_feed_summary.get("launchpad_onboarding_acceptance_ready_nodes", 0) or 0), + int(codex_brief_summary.get("launchpad_onboarding_acceptance_ready_nodes", 0) or 0), + int(stack_summary.get("launchpad_onboarding_acceptance_ready_nodes", 0) or 0), +) +stack_status = str(stack_summary.get("stack_status") or "").strip() +stack_headline = str(stack_summary.get("headline") or "").strip() +stack_blocked_issue_detail = str(stack_summary.get("blocked_issue_detail") or "").strip() +stack_next_summary = str(stack_summary.get("next_step_summary") or "").strip() +stack_next_action_code = str(stack_summary.get("next_step_action_code") or "").strip() +stack_blocked_issues = int(stack_summary.get("blocked_issues", 0) or 0) +stack_operator_lane = str(stack_summary.get("operator_lane") or "").strip() +stack_operator_priority = str(stack_summary.get("operator_priority") or "").strip() +stack_operator_reason_code = str(stack_summary.get("operator_reason_code") or "").strip() +stack_operator_title = str(stack_summary.get("operator_title") or "").strip() +stack_operator_summary = str(stack_summary.get("operator_summary") or "").strip() +stack_operator_primary_command_key = str(stack_summary.get("operator_primary_command_key") or "").strip() +stack_operator_secondary_command_key = str(stack_summary.get("operator_secondary_command_key") or "").strip() +stack_operator_primary_command = str(stack_summary.get("operator_primary_command") or "").strip() +stack_operator_secondary_command = str(stack_summary.get("operator_secondary_command") or "").strip() +stack_operator_focus_kind = str(stack_summary.get("operator_focus_kind") or "").strip() +stack_operator_focus_node_code = str(stack_summary.get("operator_focus_node_code") or "").strip() + +stack_primary_commands = [] +for candidate in [ + stack_operator_primary_command, + stack_operator_secondary_command, +]: + normalized_candidate = str(candidate or "").strip() + if normalized_candidate and normalized_candidate not in stack_primary_commands: + stack_primary_commands.append(normalized_candidate) + +stack_preferred_surface = "stack-summary" +if stack_operator_focus_kind == "node_scene_log": + stack_preferred_surface = "execution-scene" +elif stack_operator_lane == "node_handover": + stack_preferred_surface = "managed-nodes" +elif stack_operator_lane == "release": + stack_preferred_surface = "release-hub" +elif stack_operator_lane == "observability": + stack_preferred_surface = "execution-scene" +elif stack_operator_lane in {"ops_jobs", "steady"}: + stack_preferred_surface = "driver-feed" + +contract_surface_gaps = [] +if report_ok("07_driver_feed.txt") and not str(driver_feed_summary.get("primary_contract_key") or "").strip(): + contract_surface_gaps.append("driver-feed") +if report_ok("08_activity_stream.txt") and not str(activity_stream_summary.get("primary_contract_key") or "").strip(): + contract_surface_gaps.append("activity-stream") +if report_ok("09_codex_brief.txt") and not str(codex_brief_summary.get("primary_contract_key") or "").strip(): + contract_surface_gaps.append("codex-brief") + +surface_http_status = { + "stack_summary": report_http_status("00_ops_center_stack_summary.txt"), + "contracts_registry": report_http_status("06_contracts_registry.txt"), + "driver_feed": report_http_status("07_driver_feed.txt"), + "activity_stream": report_http_status("08_activity_stream.txt"), + "codex_brief": report_http_status("09_codex_brief.txt"), + "stack_next_preview": report_http_status("10_stack_next_preview.txt"), + "driver_focus_preview": report_http_status("11_driver_focus_preview.txt"), + "activity_preview": report_http_status("12_activity_preview.txt"), + "codex_focus_preview": report_http_status("13_codex_focus_preview.txt"), +} + +doctor_command = f"bash domain-api/deploy/multi-region/drive_ops_center.sh doctor {mainland_api_base_url}" +if overseas_api_base_url: + doctor_command += f" {overseas_api_base_url}" + +decision = { + "status": "ready", + "reason_code": "ops_center_ready", + "headline": "海外控制面已经具备协议驱动的观察与预览能力。", + "detail": "", + "preferred_surface": "driver-feed", + "next_action_code": "review_driver_feed", + "recommended_commands": [ + f"bash domain-api/deploy/multi-region/drive_ops_center.sh driver-feed {mainland_api_base_url}", + f"bash domain-api/deploy/multi-region/drive_ops_center.sh activity-stream {mainland_api_base_url}", + f"bash domain-api/deploy/multi-region/drive_ops_center.sh codex-brief {mainland_api_base_url}", + ], + "evidence": { + "required_failures": required_failures, + "optional_unavailable": optional_unavailable, + "contract_surface_gaps": contract_surface_gaps, + "surface_http_status": surface_http_status, + }, +} + +if "06_contracts_registry.txt" in required_failures: + http_status = int(contracts_summary.get("http_status", 0) or 0) + contracts_failure = next( + ( + item for item in required_failure_details + if str(item.get("name") or "") == "06_contracts_registry.txt" + ), + {}, + ) + contracts_reason_code = str(contracts_failure.get("reason_code") or "").strip() + contracts_failure_detail = str(contracts_failure.get("detail") or "").strip() + decision = { + "status": "blocked", + "reason_code": contracts_reason_code or "contracts_registry_unavailable", + "headline": ( + "控制面 contract registry 拉取超时，海外驾驶链路当前无法确认协议面是否完整。 " + if contracts_reason_code == "command_timeout" + else "控制面还没暴露 ops/contracts，海外驾驶链路当前不完整。" + ), + "detail": contracts_failure_detail or f"06_contracts_registry.txt http_status={http_status}，先恢复 contract registry，再继续 preview / execute。", + "preferred_surface": "contracts", + "next_action_code": "restore_contract_registry", + "recommended_commands": [ + f"bash domain-api/deploy/multi-region/drive_ops_center.sh contracts {mainland_api_base_url}", + f"bash domain-api/deploy/multi-region/drive_ops_center.sh runtime-refresh-recover {mainland_api_base_url}", + f"bash domain-api/deploy/multi-region/check_ops_center_stack.sh {mainland_api_base_url} summary", + ], + "evidence": { + "required_failures": required_failures, + "required_failure_details": required_failure_details, + "optional_unavailable": optional_unavailable, + "optional_unavailable_details": optional_unavailable_details, + "contract_surface_gaps": contract_surface_gaps, + "surface_http_status": surface_http_status, + }, + } +elif required_failures: + scene_log_required_failures = [ + name for name in required_failures + if str(name or "").startswith("15_scene_node_log_") + ] + if scene_log_required_failures and stack_primary_commands: + decision = { + "status": "blocked" if stack_status == "blocked" else "attention", + "reason_code": stack_operator_reason_code or "scene_log_surface_unavailable", + "headline": stack_operator_title or "执行现场观察链路还没拿到可用日志样本。", + "detail": stack_operator_summary or " / ".join(scene_log_required_failures), + "preferred_surface": stack_preferred_surface, + "next_action_code": stack_operator_primary_command_key or "inspect_scene_log_reports", + "recommended_commands": [ + *stack_primary_commands, + f"bash domain-api/deploy/multi-region/check_ops_center_stack.sh {mainland_api_base_url} summary", + doctor_command, + ], + "evidence": { + "required_failures": required_failures, + "required_failure_details": required_failure_details, + "optional_unavailable": optional_unavailable, + "optional_unavailable_details": optional_unavailable_details, + "contract_surface_gaps": contract_surface_gaps, + "surface_http_status": surface_http_status, + "scene_log_status_counts": scene_log_status_counts, + }, + } + else: + decision = { + "status": "blocked", + "reason_code": "required_surfaces_unavailable", + "headline": "海外控制面仍有必需 surface 缺失，先修 surface 再谈自动驾驶。", + "detail": " / ".join(required_failures), + "preferred_surface": stack_preferred_surface if stack_primary_commands else "stack-summary", + "next_action_code": stack_operator_primary_command_key or "repair_required_surfaces", + "recommended_commands": [ + *stack_primary_commands, + doctor_command, + f"bash domain-api/deploy/multi-region/check_ops_center_stack.sh {mainland_api_base_url} summary", + ], + "evidence": { + "required_failures": required_failures, + "required_failure_details": required_failure_details, + "optional_unavailable": optional_unavailable, + "optional_unavailable_details": optional_unavailable_details, + "contract_surface_gaps": contract_surface_gaps, + "surface_http_status": surface_http_status, + }, + } +elif contract_surface_gaps: + decision = { + "status": "attention", + "reason_code": "contract_navigation_incomplete", + "headline": "核心观察面已在线，但 contract_navigation 还没完全补齐。", + "detail": "缺少主协议标记的 surface: " + " / ".join(contract_surface_gaps), + "preferred_surface": contract_surface_gaps[0], + "next_action_code": "fill_contract_navigation", + "recommended_commands": [ + f"bash domain-api/deploy/multi-region/drive_ops_center.sh driver-feed {mainland_api_base_url}", + f"bash domain-api/deploy/multi-region/drive_ops_center.sh activity-stream {mainland_api_base_url}", + f"bash domain-api/deploy/multi-region/drive_ops_center.sh codex-brief {mainland_api_base_url}", + ], + "evidence": { + "required_failures": required_failures, + "required_failure_details": required_failure_details, + "optional_unavailable": optional_unavailable, + "optional_unavailable_details": optional_unavailable_details, + "contract_surface_gaps": contract_surface_gaps, + "surface_http_status": surface_http_status, + }, + } +elif stack_status == "blocked": + decision = { + "status": "blocked", + "reason_code": stack_operator_reason_code or "stack_diagnosis_blocked", + "headline": stack_operator_title or stack_headline or "控制面总检已判定当前现场存在阻断问题，暂不应继续执行自动化动作。", + "detail": stack_operator_summary or stack_blocked_issue_detail or stack_next_summary or f"stack_diagnosis.blocked_issues={stack_blocked_issues}", + "preferred_surface": stack_preferred_surface, + "next_action_code": stack_operator_primary_command_key or stack_next_action_code or "resolve_stack_blockers", + "recommended_commands": [ + *stack_primary_commands, + f"bash domain-api/deploy/multi-region/drive_ops_center.sh stack-next {mainland_api_base_url}", + f"bash domain-api/deploy/multi-region/check_ops_center_stack.sh {mainland_api_base_url} summary", + doctor_command, + ], + "evidence": { + "required_failures": required_failures, + "required_failure_details": required_failure_details, + "optional_unavailable": optional_unavailable, + "optional_unavailable_details": optional_unavailable_details, + "contract_surface_gaps": contract_surface_gaps, + "surface_http_status": surface_http_status, + }, + } +elif stack_status == "attention": + if stack_operator_reason_code == "runtime_build_schema_stale": + restart_commands = [] + for candidate in [ + stack_operator_primary_command, + stack_operator_secondary_command, + f"bash domain-api/deploy/multi-region/drive_ops_center.sh runtime-refresh-recover {mainland_api_base_url}", + f"bash domain-api/deploy/multi-region/check_ops_center_stack.sh {mainland_api_base_url} summary", + f"bash domain-api/deploy/multi-region/drive_ops_center.sh stack-next {mainland_api_base_url}", + doctor_command, + ]: + normalized_candidate = str(candidate or "").strip() + if normalized_candidate and normalized_candidate not in restart_commands: + restart_commands.append(normalized_candidate) + decision = { + "status": "attention", + "reason_code": "runtime_build_schema_stale", + "headline": stack_operator_title or "仓库代码已更新，但运行中的控制面 API 还是旧 schema。", + "detail": stack_operator_summary or stack_next_summary or "先执行 runtime-refresh-recover，让控制面 API 切到最新代码；随后重新执行 stack-diagnosis / stack-next，再继续节点接管。", + "preferred_surface": "stack-summary", + "next_action_code": stack_operator_primary_command_key or "api-restart", + "recommended_commands": restart_commands, + "evidence": { + "required_failures": required_failures, + "required_failure_details": required_failure_details, + "optional_unavailable": optional_unavailable, + "optional_unavailable_details": optional_unavailable_details, + "contract_surface_gaps": contract_surface_gaps, + "surface_http_status": surface_http_status, + }, + } + else: + decision = { + "status": "attention", + "reason_code": stack_operator_reason_code or "stack_diagnosis_attention", + "headline": stack_operator_title or stack_headline or "控制面总检已发现待收口事项，建议先顺着 next_step 收口后再继续预览或执行。", + "detail": stack_operator_summary or stack_next_summary or "请先处理 stack-diagnosis 提示的 attention 项。", + "preferred_surface": stack_preferred_surface, + "next_action_code": stack_operator_primary_command_key or stack_next_action_code or "review_stack_next", + "recommended_commands": [ + *stack_primary_commands, + f"bash domain-api/deploy/multi-region/drive_ops_center.sh stack-next {mainland_api_base_url}", + f"bash domain-api/deploy/multi-region/check_ops_center_stack.sh {mainland_api_base_url} summary", + doctor_command, + ], + "evidence": { + "required_failures": required_failures, + "required_failure_details": required_failure_details, + "optional_unavailable": optional_unavailable, + "optional_unavailable_details": optional_unavailable_details, + "contract_surface_gaps": contract_surface_gaps, + "surface_http_status": surface_http_status, + }, + } +elif optional_unavailable: + focus_entry = str(codex_brief_summary.get("focus_entry_key") or "").strip() + decision = { + "status": "attention", + "reason_code": "preview_unavailable", + "headline": "核心观察面已可用，但当前 preview 入口还没形成可执行焦点。", + "detail": "当前不可用 preview: " + " / ".join(optional_unavailable), + "preferred_surface": "codex-brief" if focus_entry else "driver-feed", + "next_action_code": "inspect_focus_surfaces", + "recommended_commands": [ + f"bash domain-api/deploy/multi-region/drive_ops_center.sh driver-feed {mainland_api_base_url}", + f"bash domain-api/deploy/multi-region/drive_ops_center.sh activity-stream {mainland_api_base_url}", + f"bash domain-api/deploy/multi-region/drive_ops_center.sh codex-brief {mainland_api_base_url}", + ], + "evidence": { + "required_failures": required_failures, + "required_failure_details": required_failure_details, + "optional_unavailable": optional_unavailable, + "optional_unavailable_details": optional_unavailable_details, + "contract_surface_gaps": contract_surface_gaps, + "surface_http_status": surface_http_status, + }, + } +elif scene_log_reports_total > 0 and scene_log_reports_ok == 0: + first_scene_log = dict(scene_log_report_summaries[0] or {}) if scene_log_report_summaries else {} + scene_log_node_code = str(first_scene_log.get("node_code") or "").strip() + scene_log_mode = str(first_scene_log.get("mode") or "key").strip() or "key" + decision = { + "status": "attention", + "reason_code": "scene_log_reports_unavailable", + "headline": "总检已经识别出关键节点现场，但节点现场日志附件还没有成功形成可用样本。", + "detail": ( + f"scene_log_reports_total={scene_log_reports_total}，全部附件当前不可用；" + f"缺口节点 {', '.join(scene_log_problem_node_codes) or scene_log_node_code or '-'}" + ), + "preferred_surface": "execution-scene", + "next_action_code": "inspect_scene_log_reports", + "recommended_commands": [ + *( + [ + f"bash domain-api/deploy/multi-region/drive_ops_center.sh scene-node-log {mainland_api_base_url} {scene_log_node_code} 120 {scene_log_mode}" + ] + if scene_log_node_code + else [] + ), + f"bash domain-api/deploy/multi-region/check_ops_center_stack.sh {mainland_api_base_url} summary", + doctor_command, + ], + "evidence": { + "required_failures": required_failures, + "required_failure_details": required_failure_details, + "optional_unavailable": optional_unavailable, + "optional_unavailable_details": optional_unavailable_details, + "contract_surface_gaps": contract_surface_gaps, + "surface_http_status": surface_http_status, + "scene_log_status_counts": scene_log_status_counts, + }, + } +elif scene_log_problem_reports: + first_scene_log = dict(scene_log_problem_reports[0] or {}) + scene_log_node_code = str(first_scene_log.get("node_code") or "").strip() + scene_log_mode = str(first_scene_log.get("mode") or "key").strip() or "key" + decision = { + "status": "attention", + "reason_code": "scene_log_reports_need_attention", + "headline": "总检已经带回节点现场日志附件，但仍有参与节点的现场样本没有完全收口。", + "detail": ( + f"问题节点 {', '.join(scene_log_problem_node_codes) or scene_log_node_code or '-'}；" + f" 当前状态分布 {scene_log_status_counts}" + ), + "preferred_surface": "execution-scene", + "next_action_code": "inspect_scene_log_reports", + "recommended_commands": [ + *( + [ + f"bash domain-api/deploy/multi-region/drive_ops_center.sh scene-node-log {mainland_api_base_url} {scene_log_node_code} 120 {scene_log_mode}" + ] + if scene_log_node_code + else [] + ), + f"bash domain-api/deploy/multi-region/drive_ops_center.sh stack-next {mainland_api_base_url}", + doctor_command, + ], + "evidence": { + "required_failures": required_failures, + "required_failure_details": required_failure_details, + "optional_unavailable": optional_unavailable, + "optional_unavailable_details": optional_unavailable_details, + "contract_surface_gaps": contract_surface_gaps, + "surface_http_status": surface_http_status, + "scene_log_status_counts": scene_log_status_counts, + }, + } +else: + codex_headline = str(codex_brief_summary.get("headline") or "").strip() + driver_headline = str(driver_feed_summary.get("headline") or "").strip() + decision["headline"] = stack_operator_title or codex_headline or driver_headline or decision["headline"] + decision["detail"] = stack_operator_summary or stack_next_summary or "先看 driver-feed，再按 manifest 已准备好的 preview surface 继续收敛。" + decision["preferred_surface"] = ( + stack_preferred_surface + if stack_operator_primary_command_key or stack_operator_focus_kind + else ("codex-brief" if str(codex_brief_summary.get("focus_entry_key") or "").strip() else "driver-feed") + ) + decision["next_action_code"] = stack_operator_primary_command_key or stack_next_action_code or "preview_primary_focus" + decision["recommended_commands"] = [ + *stack_primary_commands, + f"bash domain-api/deploy/multi-region/drive_ops_center.sh stack-next {mainland_api_base_url}", + f"bash domain-api/deploy/multi-region/drive_ops_center.sh driver-focus-preview {mainland_api_base_url}", + f"bash domain-api/deploy/multi-region/drive_ops_center.sh codex-focus-preview {mainland_api_base_url}", + ] + +if isinstance(decision.get("evidence"), dict): + decision["evidence"].update({ + "launchpad_recommended_target_node_code": launchpad_recommended_target_node_code, + "launchpad_recommended_recovery_label": launchpad_recommended_recovery_label, + "launchpad_recommended_recovery_summary": launchpad_recommended_recovery_summary, + "launchpad_onboarding_bootstrap_pending_nodes": launchpad_onboarding_bootstrap_pending_nodes, + "launchpad_onboarding_acceptance_ready_nodes": launchpad_onboarding_acceptance_ready_nodes, + }) + +manifest = { + "generated_at": datetime.now().isoformat(timespec="seconds"), + "config_file": config_file, + "active_profile": active_profile, + "mainland_api_base_url": mainland_api_base_url, + "overseas_api_base_url": overseas_api_base_url, + "report_dir": str(report_dir), + "reports_total": len(reports), + "reports": reports, + "summary": { + "ok": len(required_failures) == 0, + "required_failures": required_failures, + "required_failure_details": required_failure_details, + "optional_unavailable": optional_unavailable, + "optional_unavailable_details": optional_unavailable_details, + "scene_log_reports_total": scene_log_reports_total, + "scene_log_reports_ok": scene_log_reports_ok, + "scene_log_status_counts": scene_log_status_counts, + "scene_log_reports": [ + { + "name": str(report.get("name") or ""), + "node_code": str((report.get("summary") or {}).get("node_code") or ""), + "status": str((report.get("summary") or {}).get("status") or ""), + "status_label": str((report.get("summary") or {}).get("status_label") or ""), + "mode": str((report.get("summary") or {}).get("mode") or ""), + "records_total": int((report.get("summary") or {}).get("records_total", 0) or 0), + "ok": bool(report.get("ok")), + } + for report in scene_log_reports + ], + "available_preview_reports": sum( + 1 for report in reports if report.get("name") in optional_preview_files and report.get("ok") + ), + "discovered_contract_keys": sorted(all_contract_keys), + "primary_contract_keys": primary_contract_keys, + "preview_contract_keys": preview_contract_keys, + "contract_surface_gaps": contract_surface_gaps, + "surface_http_status": surface_http_status, + "launchpad_recommended_target_node_code": launchpad_recommended_target_node_code, + "launchpad_recommended_recovery_label": launchpad_recommended_recovery_label, + "launchpad_recommended_recovery_summary": launchpad_recommended_recovery_summary, + "launchpad_onboarding_bootstrap_pending_nodes": launchpad_onboarding_bootstrap_pending_nodes, + "launchpad_onboarding_acceptance_ready_nodes": launchpad_onboarding_acceptance_ready_nodes, + }, + "decision": decision, +} + +print(json.dumps(manifest, ensure_ascii=False)) +PY +)" + + cat >"${report_dir}/README.txt" </dev/null + print_doctor_manifest_summary "${report_dir}/manifest.json" +} + +export_agent_plan() { + local report_dir="$1" + local mainland_base_url="$2" + local node_code="$3" + local node_region="$4" + local node_role="$5" + local control_plane_base_url="$6" + local root_dir="$7" + local timestamp + timestamp="$(timestamp_now)" + local safe_node_code + safe_node_code="$(printf '%s' "${node_code}" | tr '/:' '__')" + ops_center_ensure_report_dir "${report_dir}" + local output_path="${report_dir}/agent-plan-${safe_node_code}-${timestamp}.txt" + ISSUED_BY="$(ops_center_resolve_issued_by)" \ + EXPIRES_IN_HOURS="$(ops_center_resolve_expires_in_hours)" \ + bash "${SCRIPT_DIR}/build_node_agent_bootstrap_plan.sh" \ + "${mainland_base_url}" \ + "${node_code}" \ + "${node_region}" \ + "${node_role}" \ + "${control_plane_base_url}" \ + "${root_dir}" | tee "${output_path}" + + write_json_file "${output_path}.meta.json" "$(python3 - \ + "${node_code}" "${node_region}" "${node_role}" "${mainland_base_url}" "${control_plane_base_url}" "${root_dir}" "${output_path}" <<'PY' +import json +import sys +from datetime import datetime + +print(json.dumps({ + "generated_at": datetime.now().isoformat(timespec="seconds"), + "node_code": sys.argv[1], + "node_region": sys.argv[2], + "node_role": sys.argv[3], + "mainland_api_base_url": sys.argv[4], + "control_plane_base_url": sys.argv[5], + "root_dir": sys.argv[6], + "output_path": sys.argv[7], +}, ensure_ascii=False)) +PY +)" + + python3 - <<'PY' "${output_path}" "${output_path}.meta.json" +import json +import sys + +print(json.dumps({ + "ok": True, + "output_path": sys.argv[1], + "meta_path": sys.argv[2], +}, ensure_ascii=False, indent=2)) +PY +} + +run_publish_latest() { + local mainland_base_url="$1" + local mode="$2" + local confirm_flag="$3" + local skip_smoke_flag="$4" + local repo_root + repo_root="$(ops_center_repo_root)" + local skip_smoke + skip_smoke="$(confirm_to_bool "${skip_smoke_flag}")" + + if [[ "${mode}" != "worker" && "${mode}" != "control" ]]; then + echo "mode must be worker or control" >&2 + usage + exit 1 + fi + + echo "============================================================" + echo "[publish/1] package latest" + echo "============================================================" + if [[ "${skip_smoke}" == "true" ]]; then + DOMAINCHECK_SKIP_SMOKE_TEST=1 bash "${repo_root}/package_domain_release.sh" + else + bash "${repo_root}/package_domain_release.sh" + fi + echo + + echo "============================================================" + echo "[publish/2] preview smart rollout from latest package" + echo "============================================================" + bash "${SCRIPT_DIR}/drive_release_hub.sh" "${mainland_base_url}" preview-latest-smart "${mode}" + echo + + echo "============================================================" + echo "[publish/3] smart rollout from latest package" + echo "============================================================" + bash "${SCRIPT_DIR}/drive_release_hub.sh" "${mainland_base_url}" smart-latest "${mode}" "${confirm_flag}" +} + +show_ops_contracts() { + local mainland_base_url="$1" + local contract_key="${2:-}" + local raw_http_response + if [[ -n "${contract_key}" ]]; then + raw_http_response="$(request_with_status "GET" "${mainland_base_url}/api/v1/ops/contracts/${contract_key}")" + else + raw_http_response="$(request_with_status "GET" "${mainland_base_url}/api/v1/ops/contracts")" + fi + local http_status="${raw_http_response##*$'\n'HTTP_STATUS=}" + local response_json="${raw_http_response%$'\n'HTTP_STATUS=*}" + + echo "[1/2] raw response" + printf '%s\n' "http_status=${http_status}" + print_json_or_raw "${response_json}" + echo + echo "[2/2] condensed summary" + python3 - <<'PY' "${response_json}" "${http_status}" "${contract_key}" +import json +import sys + +payload = json.loads(str(sys.argv[1] or "{}")) +http_status = int(sys.argv[2] or 0) +contract_key = str(sys.argv[3] or "").strip() +data = payload.get("data") or {} + +if contract_key: + contract = data.get("contract") or {} + related_contracts = list(data.get("related_contracts") or []) + print(json.dumps({ + "http_status": http_status, + "requested_contract_key": contract_key, + "registry_version": str(data.get("registry_version") or ""), + "schema_version": str(data.get("schema_version") or ""), + "docs_root": str(data.get("docs_root") or ""), + "detail_endpoint": str(data.get("detail_endpoint") or ""), + "contract": { + "key": str(contract.get("key") or ""), + "title": str(contract.get("title") or ""), + "status": str(contract.get("status") or ""), + "version": str(contract.get("version") or ""), + "primary_endpoint": str(contract.get("primary_endpoint") or ""), + "schema_doc_path": str(contract.get("schema_doc_path") or ""), + "summary": str(contract.get("summary") or ""), + "consumers": list(contract.get("consumers") or []), + "service_keys": list(contract.get("service_keys") or []), + "related_contract_keys": list(contract.get("related_contract_keys") or data.get("related_contract_keys") or []), + "discovery_endpoints": list(contract.get("discovery_endpoints") or []), + }, + "related_contracts": [ + { + "key": str(item.get("key") or ""), + "title": str(item.get("title") or ""), + "version": str(item.get("version") or ""), + "primary_endpoint": str(item.get("primary_endpoint") or ""), + } + for item in related_contracts + ], + }, ensure_ascii=False, indent=2)) +else: + contracts = list(data.get("contracts") or []) + print(json.dumps({ + "http_status": http_status, + "registry_version": str(data.get("registry_version") or ""), + "schema_version": str(data.get("schema_version") or ""), + "contracts_total": int(data.get("contracts_total", 0) or 0), + "docs_root": str(data.get("docs_root") or ""), + "discovery_entrypoints": list(data.get("discovery_entrypoints") or []), + "contracts": [ + { + "key": str(item.get("key") or ""), + "version": str(item.get("version") or ""), + "primary_endpoint": str(item.get("primary_endpoint") or ""), + "schema_doc_path": str(item.get("schema_doc_path") or ""), + "related_contract_keys": list(item.get("related_contract_keys") or []), + } + for item in contracts + ], + }, ensure_ascii=False, indent=2)) +PY +} + +show_activity_stream() { + local mainland_base_url="$1" + shift || true + + local request_url + request_url="$(python3 - <<'PY' "${mainland_base_url}" "$@" +import sys +from urllib.parse import urlencode + +base_url = str(sys.argv[1] or "").rstrip("/") +tokens = [str(item or "").strip() for item in sys.argv[2:] if str(item or "").strip()] +params = {} +allowed = {"limit", "kind", "status", "execution_mode", "query"} +for token in tokens: + if "=" in token: + key, value = token.split("=", 1) + key = str(key or "").strip() + value = str(value or "").strip() + if key in allowed and value: + params[key] = value + elif "query" not in params: + params["query"] = token + +query_string = urlencode(params) +endpoint = f"{base_url}/api/v1/ops/activity-stream" +print(f"{endpoint}?{query_string}" if query_string else endpoint) +PY +)" + + local raw_http_response + raw_http_response="$(request_with_status "GET" "${request_url}")" + local http_status="${raw_http_response##*$'\n'HTTP_STATUS=}" + local response_json="${raw_http_response%$'\n'HTTP_STATUS=*}" + + echo "[1/2] raw response" + printf '%s\n' "http_status=${http_status}" + printf '%s\n' "request_url=${request_url}" + print_json_or_raw "${response_json}" + echo + echo "[2/2] condensed summary" + python3 - <<'PY' "${response_json}" "${http_status}" "${request_url}" +import json +import sys + +payload = json.loads(str(sys.argv[1] or "{}")) +http_status = int(sys.argv[2] or 0) +request_url = str(sys.argv[3] or "").strip() +data = payload.get("data") or {} +summary = data.get("summary") or {} +items = list(data.get("items") or []) +contract_navigation = data.get("contract_navigation") or {} + +print(json.dumps({ + "http_status": http_status, + "request_url": request_url, + "contract_navigation": { + "primary_contract_key": str(contract_navigation.get("primary_contract_key") or ""), + "contract_keys": list(contract_navigation.get("contract_keys") or []), + "detail_endpoint_pattern": str(contract_navigation.get("detail_endpoint_pattern") or ""), + }, + "summary": { + "status": str(summary.get("status") or ""), + "status_label": str(summary.get("status_label") or ""), + "summary_text": str(summary.get("summary_text") or ""), + "total": int(summary.get("total", 0) or 0), + "filtered_total": int(summary.get("filtered_total", 0) or 0), + "unfiltered_total": int(summary.get("unfiltered_total", 0) or 0), + "latest_at": str(summary.get("latest_at") or ""), + "filters": dict(summary.get("filters") or {}), + "kind_counts": dict(summary.get("kind_counts") or {}), + "status_counts": dict(summary.get("status_counts") or {}), + }, + "items": [ + { + "activity_key": str(item.get("activity_key") or ""), + "kind": str(item.get("kind") or ""), + "title": str(item.get("title") or ""), + "status": str(item.get("status") or ""), + "status_label": str(item.get("status_label") or ""), + "execution_mode": str(item.get("execution_mode") or ""), + "occurred_at": str(item.get("occurred_at") or ""), + "primary_contract_key": str((item.get("contract_navigation") or {}).get("primary_contract_key") or ""), + "contract_keys": list((item.get("contract_navigation") or {}).get("contract_keys") or []), + "focus_ref": item.get("focus_ref") or {}, + "ui_intent": item.get("ui_intent") or {}, + } + for item in items + ], +}, ensure_ascii=False, indent=2)) +PY +} + +build_activity_preview_bundle() { + local activity_response_json="$1" + local activity_key="$2" + local requested_by="$3" + local source_label="$4" + python3 - <<'PY' "${activity_response_json}" "${activity_key}" "${requested_by}" "${source_label}" +import json +import sys + +payload = json.loads(str(sys.argv[1] or "{}")) +requested_activity_key = str(sys.argv[2] or "").strip() +requested_by = str(sys.argv[3] or "").strip() or "cli/ops-center/activity-preview" +source_label = str(sys.argv[4] or "").strip() or "cli/activity-preview" +data = payload.get("data") or {} +items = [dict(item or {}) for item in list(data.get("items") or []) if dict(item or {})] + +target_item = {} +if requested_activity_key: + target_item = next((item for item in items if str(item.get("activity_key") or "").strip() == requested_activity_key), {}) +if not target_item: + target_item = next((item for item in items if str((item.get("ui_intent") or {}).get("kind") or "").strip()), {}) +if not target_item: + raise SystemExit("activity-stream 当前没有可预览的活动项") + +ui_intent = dict(target_item.get("ui_intent") or {}) +if not str(ui_intent.get("kind") or "").strip(): + raise SystemExit("目标活动项没有 ui_intent，当前不能生成 preview") + +kind = str(target_item.get("kind") or "").strip() +if kind == "playbook_run": + primary_label = "查看编排" +elif kind == "rollout": + primary_label = "查看任务" +elif kind == "runbook_sequence": + primary_label = "查看路径" +elif kind in {"execution_scene", "log_sync"}: + primary_label = "查看现场" +elif str(ui_intent.get("kind") or "").strip() == "job_detail": + primary_label = "看详情" +else: + primary_label = "看事件" + +focus_ref = dict(target_item.get("focus_ref") or {}) +request_payload = { + "source_label": source_label, + "title": str(target_item.get("title") or "").strip() or str(target_item.get("activity_key") or "").strip() or "activity-preview", + "summary": str(target_item.get("summary_text") or target_item.get("summary") or "").strip(), + "reason": str(target_item.get("summary_text") or target_item.get("summary") or "").strip() or "建议先进入该活动项对应的工作区。", + "meta_text": str(target_item.get("meta_text") or "").strip(), + "executor_kind": "driver_action", + "focus_ref": focus_ref, + "primary_focus_ref": focus_ref, + "primary_label": primary_label, + "primary_action_code": "focus_activity_item", + "primary_action_payload": { + "activity_key": str(target_item.get("activity_key") or "").strip(), + "kind": kind, + "status": str(target_item.get("status") or "").strip(), + "ui_intent": ui_intent, + "focus_ref": focus_ref, + }, + "include_secondary": False, + "requested_by": requested_by, +} + +print(json.dumps({ + "selected_entry": { + "activity_key": str(target_item.get("activity_key") or "").strip(), + "kind": kind, + "title": str(target_item.get("title") or "").strip(), + "status": str(target_item.get("status") or "").strip(), + "status_label": str(target_item.get("status_label") or "").strip(), + "summary": str(target_item.get("summary_text") or target_item.get("summary") or "").strip(), + "focus_ref": focus_ref, + "ui_intent": ui_intent, + "contract_navigation": dict(target_item.get("contract_navigation") or {}), + }, + "request": request_payload, +}, ensure_ascii=False)) +PY +} + +preview_activity_item() { + local mainland_base_url="$1" + local activity_key="$2" + local requested_by="$3" + + local raw_activity_response + raw_activity_response="$(request_with_status "GET" "${mainland_base_url}/api/v1/ops/activity-stream?limit=50")" + local activity_http_status="${raw_activity_response##*$'\n'HTTP_STATUS=}" + local activity_response_json="${raw_activity_response%$'\n'HTTP_STATUS=*}" + + if [[ "${activity_http_status}" -lt 200 || "${activity_http_status}" -ge 300 ]]; then + echo "[1/3] activity stream raw response" + printf '%s\n' "http_status=${activity_http_status}" + print_json_or_raw "${activity_response_json}" + exit 1 + fi + + local bundle_json + bundle_json="$(build_activity_preview_bundle "${activity_response_json}" "${activity_key}" "${requested_by}" "cli/activity-preview")" + local selected_entry_json + selected_entry_json="$(python3 - <<'PY' "${bundle_json}" +import json +import sys +payload = json.loads(str(sys.argv[1] or "{}")) +print(json.dumps(payload.get("selected_entry") or {}, ensure_ascii=False)) +PY +)" + local request_json + request_json="$(python3 - <<'PY' "${bundle_json}" +import json +import sys +payload = json.loads(str(sys.argv[1] or "{}")) +print(json.dumps(payload.get("request") or {}, ensure_ascii=False)) +PY +)" + + echo "[1/4] selected activity" + print_json_or_raw "${selected_entry_json}" + echo + echo + + echo "[2/4] request" + print_json_or_raw "${request_json}" + echo + echo + + local raw_http_response + raw_http_response="$(request_with_status "POST" "${mainland_base_url}/api/v1/ops/driver-actions/resolve" "${request_json}")" + local http_status="${raw_http_response##*$'\n'HTTP_STATUS=}" + local response_json="${raw_http_response%$'\n'HTTP_STATUS=*}" + + echo "[3/4] raw response" + printf '%s\n' "http_status=${http_status}" + print_json_or_raw "${response_json}" + echo + echo + + echo "[4/4] condensed summary" + python3 - <<'PY' "${response_json}" "${http_status}" "${selected_entry_json}" +import json +import sys + +payload = json.loads(str(sys.argv[1] or "{}")) +http_status = int(sys.argv[2] or 0) +selected = json.loads(str(sys.argv[3] or "{}")) +data = payload.get("data") or {} +preview = data.get("preview") or {} +primary = preview.get("primary") or {} +gate = data.get("gate") or {} + +print(json.dumps({ + "http_status": http_status, + "message": str(payload.get("message") or ""), + "selected_entry": selected, + "contract": { + "contract_key": str(preview.get("contract_key") or ""), + "contract_version": str(preview.get("contract_version") or ""), + "contract_primary_endpoint": str(preview.get("contract_primary_endpoint") or ""), + "preview_endpoint": str(preview.get("preview_endpoint") or ""), + "schema_doc_path": str(preview.get("contract_schema_doc_path") or ""), + }, + "execution_chain": str(preview.get("execution_chain") or ""), + "focus_ref": data.get("focus_ref") or {}, + "gate": { + "decision": str(gate.get("decision") or ""), + "decision_label": str(gate.get("decision_label") or ""), + "recommendation": str(gate.get("recommendation") or ""), + "recommendation_label": str(gate.get("recommendation_label") or ""), + "will_execute": bool(gate.get("will_execute", False)), + "confirm_required": bool(gate.get("confirm_required", False)), + "target_api": str(gate.get("target_api") or ""), + "focus_ref": gate.get("focus_ref") or {}, + }, + "primary": { + "label": str(primary.get("label") or ""), + "action_code": str(primary.get("action_code") or ""), + "target_api": str(primary.get("target_api") or ""), + "recommendation_label": str(primary.get("recommendation_label") or ""), + "focus_ref": primary.get("focus_ref") or {}, + "request_payload": primary.get("request_payload") or {}, + }, +}, ensure_ascii=False, indent=2)) +PY + + if [[ "${http_status}" -lt 200 || "${http_status}" -ge 300 ]]; then + exit 1 + fi +} + +show_delivery_queue_status() { + local mainland_base_url="$1" + local node_code="$2" + if [[ -z "${node_code}" ]]; then + echo "node_code is required" >&2 + exit 1 + fi + local raw_http_response + raw_http_response="$(request_with_status "GET" "${mainland_base_url}/api/v1/ops/nodes/${node_code}/delivery-queue")" + local http_status="${raw_http_response##*$'\n'HTTP_STATUS=}" + local response_json="${raw_http_response%$'\n'HTTP_STATUS=*}" + echo "[1/2] raw response" + printf '%s\n' "http_status=${http_status}" + print_json_or_raw "${response_json}" + echo + echo "[2/2] condensed summary" + python3 - <<'PY' "${response_json}" "${http_status}" +import json +import sys + +payload = json.loads(str(sys.argv[1] or "{}")) +http_status = int(sys.argv[2] or 0) +data = payload.get("data") or {} +summary = data.get("summary") or {} +node = data.get("node") or {} +records_capability = data.get("records_capability") or {} +print(json.dumps({ + "http_status": http_status, + "node": { + "node_code": str(node.get("node_code") or ""), + "region": str(node.get("region") or ""), + "role": str(node.get("role") or ""), + "agent_state_label": str(node.get("agent_state_label") or ""), + "remote_access_label": str(node.get("remote_access_label") or ""), + }, + "summary": { + "state": str(summary.get("state") or ""), + "label": str(summary.get("label") or ""), + "reason": str(summary.get("reason") or ""), + "pending_count": int(summary.get("pending_count", 0) or 0), + "dead_letter_count": int(summary.get("dead_letter_count", 0) or 0), + "last_flush_at": str(summary.get("last_flush_at") or ""), + "oldest_pending_at": str(summary.get("oldest_pending_at") or ""), + "oldest_dead_letter_at": str(summary.get("oldest_dead_letter_at") or ""), + }, + "records_capability": { + "record_visibility": str(records_capability.get("record_visibility") or ""), + "record_source": str(records_capability.get("record_source") or ""), + "full_records_available": bool(records_capability.get("full_records_available", False)), + }, + "recommended_commands": { + "records": ( + f"bash domain-api/deploy/multi-region/drive_ops_center.sh delivery-queue-records http://127.0.0.1:8100 {str(node.get('node_code') or '')}" + if str(node.get("node_code") or "").strip() + else "" + ), + "flush": ( + f"bash domain-api/deploy/multi-region/drive_ops_center.sh delivery-queue-flush http://127.0.0.1:8100 {str(node.get('node_code') or '')}" + if str(node.get("node_code") or "").strip() + else "" + ), + "replay": ( + f"bash domain-api/deploy/multi-region/drive_ops_center.sh delivery-queue-replay http://127.0.0.1:8100 {str(node.get('node_code') or '')}" + if str(node.get("node_code") or "").strip() + else "" + ), + }, + "head_records": data.get("head_records") or {}, +}, ensure_ascii=False, indent=2)) +PY +} + +show_node_handover() { + local mainland_base_url="$1" + local node_code="$2" + if [[ -z "${node_code}" ]]; then + echo "node_code is required" >&2 + exit 1 + fi + local raw_http_response + raw_http_response="$(request_with_status "GET" "${mainland_base_url}/api/v1/ops/nodes/${node_code}/handover")" + local http_status="${raw_http_response##*$'\n'HTTP_STATUS=}" + local response_json="${raw_http_response%$'\n'HTTP_STATUS=*}" + local effective_http_status="${http_status}" + local fallback_json="" + + if [[ "${http_status}" == "404" ]]; then + local stack_http_response stack_http_status stack_response_json + stack_http_response="$(request_with_status "GET" "${mainland_base_url}/api/v1/ops/stack-diagnosis?base_url=${mainland_base_url}")" + stack_http_status="${stack_http_response##*$'\n'HTTP_STATUS=}" + stack_response_json="${stack_http_response%$'\n'HTTP_STATUS=*}" + if [[ "${stack_http_status}" -ge 200 && "${stack_http_status}" -lt 300 ]]; then + fallback_json="$(python3 - <<'PY' "${stack_response_json}" "${node_code}" "${http_status}" +import json +import sys + +payload = json.loads(str(sys.argv[1] or "{}")) +target_node_code = str(sys.argv[2] or "").strip() +backend_http_status = int(sys.argv[3] or 404) +data = payload.get("data") or {} +managed_nodes = data.get("managed_nodes") or {} +problem_nodes = [dict(item or {}) for item in list(managed_nodes.get("problem_nodes") or []) if isinstance(item, dict)] + +node = next( + (item for item in problem_nodes if str(item.get("node_code") or "").strip() == target_node_code), + {}, +) +first_gap = dict(managed_nodes.get("first_handover_gap") or {}) +if not node and str(first_gap.get("node_code") or "").strip() == target_node_code: + node = dict(first_gap.get("node") or {}) + if not node: + node = {"node_code": target_node_code} + +agent_state = str(node.get("agent_state") or "").strip() +remote_access_state = str(node.get("remote_access_state") or "").strip() +summary = "当前节点存在接管缺口，建议先生成 bootstrap 方案并检查 Node Agent 接入。" +stage_code = "unknown" +stage_label = "待判断" +stage_summary = summary +next_step = { + "code": "managed_node_handover", + "label": "先看节点接管详情", + "summary": "建议先确认节点为什么还没进入 remote-agent 就绪状态。", +} + +if agent_state == "pending_bootstrap": + stage_code = "pending_bootstrap" + stage_label = "待执行接入" + stage_summary = "已签发有效 Token，但目标节点尚未执行 bootstrap 片段。" + next_step = { + "code": "execute_bootstrap_plan", + "label": "去目标机执行接入片段", + "summary": "把 bootstrap env 和一键落地片段放到目标机执行，然后观察 register / heartbeat 是否建立。", + } +elif agent_state == "runtime_only": + stage_code = "runtime_only" + stage_label = "仅运行态在线" + stage_summary = "控制面能看到 runtime 心跳，但 Node Agent 尚未接管。" + next_step = { + "code": "issue_bootstrap_plan", + "label": "生成接入方案", + "summary": "下一步应生成 bootstrap env、脚本和一键落地命令，再在目标节点启动 domaincheck-node-agent。", + } +elif agent_state == "stale": + stage_code = "stale" + stage_label = "心跳过期" + stage_summary = "Node Agent 曾接入过，但心跳已过期，需先恢复服务或重新接入。" + next_step = { + "code": "recover_agent", + "label": "优先恢复 Agent", + "summary": "先检查 domaincheck-node-agent 服务、网络连通性和控制面地址，再决定是否重新签发接入方案。", + } +elif remote_access_state == "agent_pending": + stage_code = "agent_pending" + stage_label = "待接入" + stage_summary = "当前 remote-agent 仍未就绪，先补齐 bootstrap 接入链路。" + +result = { + "http_status": 200, + "backend_http_status": backend_http_status, + "fallback_used": True, + "fallback_source": "stack_diagnosis", + "compatibility_warning": "当前运行中的 API 尚未暴露 /api/v1/ops/nodes/{node_code}/handover，已降级为 stack-diagnosis 兼容视图。", + "node_code": target_node_code, + "summary": stage_summary, + "stage": { + "code": stage_code, + "label": stage_label, + "summary": stage_summary, + "next_step": next_step, + }, + "readiness": { + "is_managed": True, + "is_enabled": True, + "has_ssh_access": False, + "is_agent_online": agent_state in {"online", "online_busy"}, + "is_remote_access_ready": False, + }, + "ssh_profile": { + "host": "", + "port": 22, + "user": "", + "label": "", + "reason": "兼容视图未携带 SSH 资料，请到托管节点详情页补看。", + "ssh_command_hint": "", + }, + "cluster": { + "status": "", + "current_load": 0, + "last_heartbeat_at": "", + }, + "blocking_items": [stage_summary] if stage_code in {"pending_bootstrap", "runtime_only", "stale", "agent_pending"} else [], + "attention_items": [ + "当前是兼容降级视图，如需完整 SSH / Token / delivery queue 详情，请升级控制面 API 到包含 handover 路由的版本。" + ], + "bootstrap_defaults": { + "control_plane_base_url": "", + "root_dir": "/opt/domaincheck", + "node_region": "", + "node_role": "", + }, + "endpoints": { + "detail": f"/api/v1/ops/nodes/{target_node_code}/handover", + "bootstrap_plan": f"/api/v1/ops/nodes/{target_node_code}/handover/bootstrap-plan", + }, +} + +print(json.dumps(result, ensure_ascii=False)) +PY +)" + if [[ -n "${fallback_json}" ]]; then + effective_http_status="200" + fi + fi + fi + echo "[1/2] raw response" + printf '%s\n' "http_status=${http_status}" + print_json_or_raw "${response_json}" + if [[ -n "${fallback_json}" ]]; then + echo + printf '%s\n' "fallback=stack_diagnosis" + print_json_or_raw "${fallback_json}" + fi + echo + echo "[2/2] condensed summary" + python3 - <<'PY' "${response_json}" "${effective_http_status}" "${fallback_json}" +import json +import pathlib +import sys + +payload = json.loads(str(sys.argv[1] or "{}")) +http_status = int(sys.argv[2] or 0) +fallback_json = str(sys.argv[3] or "").strip() +if fallback_json: + data = json.loads(fallback_json) +else: + data = payload.get("data") or {} +stage = data.get("stage") or {} +readiness = data.get("readiness") or {} +ssh_profile = data.get("ssh_profile") or {} +cluster = data.get("cluster") or {} +bootstrap_defaults = data.get("bootstrap_defaults") or {} +node_code = str(data.get("node_code") or "").strip() +next_step = stage.get("next_step") or {} +print(json.dumps({ + "http_status": http_status, + "backend_http_status": int(data.get("backend_http_status", 0) or 0) if fallback_json else http_status, + "fallback_used": bool(data.get("fallback_used", False)), + "fallback_source": str(data.get("fallback_source") or ""), + "compatibility_warning": str(data.get("compatibility_warning") or ""), + "node_code": node_code, + "summary": str(data.get("summary") or ""), + "stage": { + "code": str(stage.get("code") or ""), + "label": str(stage.get("label") or ""), + "summary": str(stage.get("summary") or ""), + "next_step": next_step, + }, + "readiness": { + "is_managed": bool(readiness.get("is_managed", False)), + "is_enabled": bool(readiness.get("is_enabled", False)), + "has_ssh_access": bool(readiness.get("has_ssh_access", False)), + "is_agent_online": bool(readiness.get("is_agent_online", False)), + "is_remote_access_ready": bool(readiness.get("is_remote_access_ready", False)), + }, + "ssh_profile": { + "host": str(ssh_profile.get("host") or ""), + "port": int(ssh_profile.get("port", 22) or 22), + "user": str(ssh_profile.get("user") or ""), + "label": str(ssh_profile.get("label") or ""), + "reason": str(ssh_profile.get("reason") or ""), + "ssh_command_hint": str(ssh_profile.get("ssh_command_hint") or ""), + }, + "cluster": { + "status": str(cluster.get("status") or ""), + "current_load": int(cluster.get("current_load", 0) or 0), + "last_heartbeat_at": str(cluster.get("last_heartbeat_at") or ""), + }, + "blocking_items": list(data.get("blocking_items") or []), + "attention_items": list(data.get("attention_items") or []), + "bootstrap_defaults": { + "control_plane_base_url": str(bootstrap_defaults.get("control_plane_base_url") or ""), + "root_dir": str(bootstrap_defaults.get("root_dir") or ""), + "node_region": str(bootstrap_defaults.get("node_region") or ""), + "node_role": str(bootstrap_defaults.get("node_role") or ""), + }, + "endpoints": data.get("endpoints") or {}, + "recommended_commands": { + "bootstrap_plan": ( + f"bash domain-api/deploy/multi-region/drive_ops_center.sh node-bootstrap-plan http://127.0.0.1:8100 {node_code}" + if node_code + else "" + ), + "delivery_queue": ( + f"bash domain-api/deploy/multi-region/drive_ops_center.sh delivery-queue http://127.0.0.1:8100 {node_code}" + if node_code + else "" + ), + "node_agent_check": "bash domain-api/deploy/multi-region/check_node_agent.sh", + "ssh_hint": str(ssh_profile.get("ssh_command_hint") or ""), + }, +}, ensure_ascii=False, indent=2)) +PY +} + +show_ops_nodes() { + local mainland_base_url="$1" + local raw_http_response + raw_http_response="$(request_with_status "GET" "${mainland_base_url}/api/v1/ops/nodes")" + local http_status="${raw_http_response##*$'\n'HTTP_STATUS=}" + local response_json="${raw_http_response%$'\n'HTTP_STATUS=*}" + echo "[1/2] raw response" + printf '%s\n' "http_status=${http_status}" + print_json_or_raw "${response_json}" + echo + echo "[2/2] condensed summary" + python3 - <<'PY' "${response_json}" "${http_status}" +import json +import sys + +payload = json.loads(str(sys.argv[1] or "{}")) +http_status = int(sys.argv[2] or 0) +data = payload.get("data") or {} +summary = data.get("summary") or {} +nodes = [dict(item or {}) for item in list(data.get("nodes") or []) if isinstance(item, dict)] + +rows = [] +for node in nodes: + rows.append({ + "node_code": str(node.get("node_code") or "").strip(), + "region": str(node.get("region") or "").strip(), + "role": str(node.get("role") or "").strip(), + "agent_state": str(node.get("agent_state") or "").strip(), + "remote_access_state": str(node.get("remote_access_state") or "").strip(), + "ssh_access_label": str(node.get("ssh_access_label") or "").strip(), + "ssh_entry": ( + f"{str(node.get('ssh_user') or '').strip()}@{str(node.get('ssh_host') or '').strip()}:{int(node.get('ssh_port') or 22)}" + if str(node.get("ssh_host") or "").strip() and str(node.get("ssh_user") or "").strip() + else "" + ), + "participation_state": str(node.get("participation_state") or "").strip(), + "cluster_status": str(node.get("cluster_status") or "").strip(), + "current_load": int(node.get("cluster_current_load", 0) or 0), + }) + +print(json.dumps({ + "http_status": http_status, + "summary": { + "total": int(summary.get("total", len(nodes)) or 0), + "managed_enabled": int(summary.get("managed_enabled", 0) or 0), + "agent_ready": int(summary.get("agent_ready", 0) or 0), + "ssh_ready": int(summary.get("ssh_ready", 0) or 0), + "remote_access_ready": int(summary.get("remote_access_ready", 0) or 0), + "participating": int(summary.get("participating", 0) or 0), + "queue_dead_letter_nodes": int(summary.get("queue_dead_letter_nodes", 0) or 0), + "remote_access_state_counts": dict(summary.get("remote_access_state_counts") or {}), + }, + "nodes": rows, +}, ensure_ascii=False, indent=2)) +PY +} + +bind_node_ssh_profile() { + local mainland_base_url="$1" + local node_code="$2" + local ssh_host="$3" + local ssh_user="$4" + local ssh_port="${5:-22}" + local title="${6:-}" + if [[ -z "${node_code}" || -z "${ssh_host}" || -z "${ssh_user}" ]]; then + echo "usage: node-bind-ssh [mainland_base_url] [ssh_port] [title]" >&2 + exit 1 + fi + + local request_json + request_json="$(python3 - <<'PY' "${node_code}" "${ssh_host}" "${ssh_user}" "${ssh_port}" "${title}" +import json +import sys + +node_code = str(sys.argv[1] or "").strip() +ssh_host = str(sys.argv[2] or "").strip() +ssh_user = str(sys.argv[3] or "").strip() +ssh_port = int(sys.argv[4] or 22) +title = str(sys.argv[5] or "").strip() + +payload = { + "node_code": node_code, + "ssh_host": ssh_host, + "ssh_user": ssh_user, + "ssh_port": ssh_port, + "auth_mode": "key", + "metadata": { + "ssh_bound_via": "drive_ops_center.sh/node-bind-ssh", + }, +} +if title: + payload["title"] = title + +print(json.dumps(payload, ensure_ascii=False)) +PY +)" + + local raw_http_response + raw_http_response="$(request_with_status "POST" "${mainland_base_url}/api/v1/ops/nodes" "${request_json}")" + local http_status="${raw_http_response##*$'\n'HTTP_STATUS=}" + local response_json="${raw_http_response%$'\n'HTTP_STATUS=*}" + + echo "[1/3] save managed node ssh profile" + printf '%s\n' "http_status=${http_status}" + print_json_or_raw "${response_json}" + echo + + echo "[2/3] node handover after binding" + show_node_handover "${mainland_base_url}" "${node_code}" + echo + + echo "[3/3] condensed ssh binding" + python3 - <<'PY' "${response_json}" "${http_status}" "${node_code}" "${ssh_host}" "${ssh_user}" "${ssh_port}" "${mainland_base_url}" +import json +import sys + +payload = json.loads(str(sys.argv[1] or "{}")) +http_status = int(sys.argv[2] or 0) +node_code = str(sys.argv[3] or "").strip() +ssh_host = str(sys.argv[4] or "").strip() +ssh_user = str(sys.argv[5] or "").strip() +ssh_port = int(sys.argv[6] or 22) +base_url = str(sys.argv[7] or "http://127.0.0.1:8100").strip() or "http://127.0.0.1:8100" +data = payload.get("data") or {} +node = dict(data.get("node") or {}) + +print(json.dumps({ + "http_status": http_status, + "message": str(payload.get("message") or ""), + "node_code": node_code, + "ssh_entry": f"{ssh_user}@{ssh_host}:{ssh_port}", + "saved_title": str(node.get("title") or "").strip(), + "saved_region": str(node.get("region") or "").strip(), + "saved_role": str(node.get("role") or "").strip(), + "next_recommended_commands": { + "nodes": f"bash domain-api/deploy/multi-region/drive_ops_center.sh nodes {base_url}", + "handover": f"bash domain-api/deploy/multi-region/drive_ops_center.sh node-handover {base_url} {node_code}", + "bootstrap_plan": f"bash domain-api/deploy/multi-region/drive_ops_center.sh node-bootstrap-plan {base_url} {node_code}", + }, +}, ensure_ascii=False, indent=2)) +PY +} + +show_node_scene_log() { + local mainland_base_url="$1" + local node_code="$2" + local limit="${3:-80}" + local mode="${4:-}" + if [[ -z "${node_code}" ]]; then + echo "node_code is required" >&2 + exit 1 + fi + + local request_url + request_url="$(python3 - <<'PY' "${mainland_base_url}" "${node_code}" "${limit}" "${mode}" +import sys +from urllib.parse import quote, urlencode + +base_url = str(sys.argv[1] or "").rstrip("/") +node_code = str(sys.argv[2] or "").strip() +limit = max(1, min(int(sys.argv[3] or 80), 400)) +mode = str(sys.argv[4] or "").strip().lower() +params = {"limit": limit} +if mode in {"key", "full"}: + params["mode"] = mode +endpoint = f"{base_url}/api/v1/ops/nodes/{quote(node_code, safe='')}/scene-log" +query_string = urlencode(params) +print(f"{endpoint}?{query_string}" if query_string else endpoint) +PY +)" + + local raw_http_response + raw_http_response="$(request_with_status "GET" "${request_url}")" + local http_status="${raw_http_response##*$'\n'HTTP_STATUS=}" + local response_json="${raw_http_response%$'\n'HTTP_STATUS=*}" + local effective_http_status="${http_status}" + local fallback_json="" + + if [[ "${http_status}" == "404" ]]; then + local runtime_http_response runtime_http_status runtime_response_json + runtime_http_response="$(request_with_status "GET" "${mainland_base_url}/api/v1/runtime/status")" + runtime_http_status="${runtime_http_response##*$'\n'HTTP_STATUS=}" + runtime_response_json="${runtime_http_response%$'\n'HTTP_STATUS=*}" + if [[ "${runtime_http_status}" -ge 200 && "${runtime_http_status}" -lt 300 ]]; then + fallback_json="$(python3 - <<'PY' "${runtime_response_json}" "${node_code}" "${limit}" "${mode}" "${http_status}" +import json +import re +import sys + +payload = json.loads(str(sys.argv[1] or "{}")) +node_code = str(sys.argv[2] or "").strip() +limit = max(1, min(int(sys.argv[3] or 80), 400)) +requested_mode = str(sys.argv[4] or "").strip().lower() +backend_http_status = int(sys.argv[5] or 404) + +data = dict(payload.get("data") or {}) +detect = dict(data.get("detect") or {}) +cluster = dict(data.get("cluster") or {}) +log_sync = dict(detect.get("log_sync") or {}) + +mode = "full" if requested_mode == "full" else ("full" if str(log_sync.get("mode") or "").strip().lower() == "full" else "key") + +cluster_nodes = [dict(item or {}) for item in list(cluster.get("nodes") or []) if isinstance(item, dict)] +cluster_map = {str(item.get("node_code") or "").strip(): item for item in cluster_nodes if str(item.get("node_code") or "").strip()} + +participating_nodes = [dict(item or {}) for item in list(detect.get("participating_nodes") or []) if isinstance(item, dict)] +standby_nodes = [dict(item or {}) for item in list(detect.get("non_participating_nodes") or []) if isinstance(item, dict)] +participating_map = {str(item.get("node_code") or "").strip(): item for item in participating_nodes if str(item.get("node_code") or "").strip()} +standby_map = {str(item.get("node_code") or "").strip(): item for item in standby_nodes if str(item.get("node_code") or "").strip()} + +preview_lines = [str(item or "").strip() for item in list(log_sync.get("preview_lines") or []) if str(item or "").strip()] +pattern = re.compile(r"^\[(?P[^\]]+)\]\s+\[(?P[^\]]+)\]\s+(?P.+)$") +node_preview_lines = [] +for line in preview_lines: + match = pattern.match(line) + if not match: + continue + if str(match.group("node_code") or "").strip() != node_code: + continue + node_preview_lines.append({ + "created_at": str(match.group("created_at") or "").strip(), + "node_code": node_code, + "message": str(match.group("message") or "").strip(), + "line": line, + "mode": "key", + }) +node_preview_lines = node_preview_lines[-limit:] + +source_summary_map = { + str(item.get("node_code") or "").strip(): dict(item or {}) + for item in list(log_sync.get("source_node_summaries") or []) + if isinstance(item, dict) and str(item.get("node_code") or "").strip() +} +missing_summary_map = { + str(item.get("node_code") or "").strip(): dict(item or {}) + for item in list(log_sync.get("missing_participating_node_summaries") or []) + if isinstance(item, dict) and str(item.get("node_code") or "").strip() +} +covered_summary_map = { + str(item.get("node_code") or "").strip(): dict(item or {}) + for item in list(log_sync.get("covered_participating_node_summaries") or []) + if isinstance(item, dict) and str(item.get("node_code") or "").strip() +} + +node_meta = dict(participating_map.get(node_code) or standby_map.get(node_code) or {}) +cluster_meta = dict(cluster_map.get(node_code) or {}) +source_summary = dict(source_summary_map.get(node_code) or covered_summary_map.get(node_code) or {}) +missing_summary = dict(missing_summary_map.get(node_code) or {}) + +derived_line_count = len(node_preview_lines) +derived_last_at = str((node_preview_lines[-1] if node_preview_lines else {}).get("created_at") or "").strip() +derived_last_line = str((node_preview_lines[-1] if node_preview_lines else {}).get("line") or "").strip() + +source_summary = { + "node_code": node_code, + "line_count": max(int(source_summary.get("line_count", 0) or 0), derived_line_count), + "key_line_count": max(int(source_summary.get("key_line_count", 0) or 0), derived_line_count), + "full_line_count": int(source_summary.get("full_line_count", 0) or 0), + "last_at": derived_last_at or str(source_summary.get("last_at") or "").strip(), + "last_line": derived_last_line or str(source_summary.get("last_line") or "").strip(), +} + +log_sync_enabled = bool(log_sync.get("enabled", False)) +detect_participating = node_code in participating_map +standby_visible = node_code in standby_map +record_available = bool(source_summary.get("line_count", 0) or node_preview_lines) + +if record_available: + if detect_participating: + status = "full_capture" if mode == "full" else "healthy" + else: + status = "historical_sample" +elif missing_summary: + missing_reason_code = str(missing_summary.get("missing_reason_code") or "").strip() + if missing_reason_code == "disabled": + status = "disabled" + elif missing_reason_code == "waiting_sample": + status = "waiting_sample" + else: + status = "missing_sample" +elif detect_participating: + status = "waiting_sample" if log_sync_enabled else "disabled" +elif standby_visible: + status = "standby" +else: + status = "unknown" + +status_labels = { + "healthy": ("关键覆盖", "success"), + "full_capture": ("全量观察", "success"), + "historical_sample": ("历史样本", "success"), + "waiting_sample": ("等待样本", "warning"), + "missing_sample": ("缺少样本", "warning"), + "disabled": ("已关闭", "info"), + "standby": ("在线待命", "info"), + "unknown": ("未发现", "info"), +} +status_label, status_type = status_labels.get(status, ("待确认", "info")) + +if record_available and detect_participating: + summary = f"节点当前正在参与检测，已通过兼容降级视图识别到 {int(source_summary.get('line_count', 0) or 0)} 条现场日志样本。" +elif record_available: + summary = f"节点已通过兼容降级视图识别到 {int(source_summary.get('line_count', 0) or 0)} 条现场日志样本。" +elif missing_summary: + summary = str(missing_summary.get("missing_reason") or "").strip() or "当前还没有收到该节点的现场日志样本。" +elif detect_participating and log_sync_enabled: + summary = "节点正在参与检测，但当前现场日志样本仍未形成。" +elif detect_participating: + summary = "节点正在参与检测，但远端日志回传当前关闭。" +elif standby_visible: + summary = "节点当前在线但未参与本轮检测，暂时没有新的现场日志样本。" +else: + summary = "当前没有找到该节点的现场日志记录。" + +latest_record = dict(node_preview_lines[-1] or {}) if node_preview_lines else {} +missing_reason_code = str(missing_summary.get("missing_reason_code") or "").strip() +missing_reason = str(missing_summary.get("missing_reason") or "").strip() + +result = { + "http_status": 200, + "backend_http_status": backend_http_status, + "fallback_used": True, + "fallback_source": "runtime_status", + "fallback_reason": "running_api_missing_scene_log_route", + "compatibility_warning": "当前运行中的 API 尚未暴露 /api/v1/ops/nodes/{node_code}/scene-log，已降级为 runtime/status 兼容视图。", + "node_code": node_code, + "status": status, + "status_label": status_label, + "status_type": status_type, + "mode": mode, + "mode_label": "全量回传" if mode == "full" else "关键回传", + "summary": summary, + "node": { + "region": str(node_meta.get("region") or cluster_meta.get("region") or missing_summary.get("region") or "").strip(), + "role": str(node_meta.get("role") or cluster_meta.get("role") or missing_summary.get("role") or "").strip(), + "status": str(node_meta.get("status") or cluster_meta.get("status") or "").strip(), + "current_load": int(node_meta.get("current_load", cluster_meta.get("current_load", 0)) or 0), + "last_heartbeat_at": str(node_meta.get("last_heartbeat_at") or cluster_meta.get("last_heartbeat_at") or "").strip(), + }, + "participation": { + "detect_participating": detect_participating, + "participation_label": str(node_meta.get("participation_label") or "").strip(), + "participation_bucket": str(node_meta.get("participation_bucket") or "").strip(), + "participation_reason": str(node_meta.get("participation_reason") or "").strip(), + }, + "source_summary": { + "line_count": int(source_summary.get("line_count", 0) or 0), + "key_line_count": int(source_summary.get("key_line_count", 0) or 0), + "full_line_count": int(source_summary.get("full_line_count", 0) or 0), + "last_at": str(source_summary.get("last_at") or "").strip(), + "last_line": str(source_summary.get("last_line") or "").strip(), + }, + "missing_reason_code": missing_reason_code, + "missing_reason": missing_reason, + "records_total": int(source_summary.get("line_count", 0) or 0), + "records_visible": len(node_preview_lines), + "records_truncated": False, + "latest_record": latest_record, + "preview_lines": [str(item.get("line") or "").strip() for item in node_preview_lines[-8:]], + } +print(json.dumps(result, ensure_ascii=False)) +PY +)" + if [[ -n "${fallback_json}" ]]; then + effective_http_status="200" + fi + fi + fi + + echo "[1/2] raw response" + printf '%s\n' "http_status=${http_status}" + printf '%s\n' "request_url=${request_url}" + print_json_or_raw "${response_json}" + if [[ -n "${fallback_json}" ]]; then + echo + printf '%s\n' "fallback=runtime_status" + print_json_or_raw "${fallback_json}" + fi + echo + echo "[2/2] condensed summary" + python3 - <<'PY' "${response_json}" "${effective_http_status}" "${fallback_json}" +import json +import pathlib +import sys + +payload = json.loads(str(sys.argv[1] or "{}")) +http_status = int(sys.argv[2] or 0) +fallback_json = str(sys.argv[3] or "").strip() + +if fallback_json: + data = json.loads(fallback_json) + node = data.get("node") or {} + participation = data.get("participation") or {} + source_summary = data.get("source_summary") or {} + latest_record = data.get("latest_record") or {} + preview_lines = list(data.get("preview_lines") or []) +else: + data = payload.get("data") or {} + node = data.get("node") or {} + participation = data.get("participation") or {} + source_summary = data.get("source_summary") or {} + records = [dict(item or {}) for item in list(data.get("records") or []) if dict(item or {})] + latest_record = data.get("latest_record") or {} + preview_lines = [str(item.get("line") or "") for item in records[-8:]] + +print(json.dumps({ + "http_status": http_status, + "backend_http_status": int(data.get("backend_http_status", 0) or 0) if fallback_json else http_status, + "fallback_used": bool(data.get("fallback_used", False)), + "fallback_source": str(data.get("fallback_source") or ""), + "compatibility_warning": str(data.get("compatibility_warning") or ""), + "node_code": str(data.get("node_code") or ""), + "status": str(data.get("status") or ""), + "status_label": str(data.get("status_label") or ""), + "mode": str(data.get("mode") or ""), + "mode_label": str(data.get("mode_label") or ""), + "summary": str(data.get("summary") or ""), + "node": { + "region": str(node.get("region") or ""), + "role": str(node.get("role") or ""), + "status": str(node.get("status") or ""), + "current_load": int(node.get("current_load", 0) or 0), + "last_heartbeat_at": str(node.get("last_heartbeat_at") or ""), + }, + "participation": { + "detect_participating": bool(participation.get("detect_participating", False)), + "participation_label": str(participation.get("participation_label") or ""), + "participation_bucket": str(participation.get("participation_bucket") or ""), + "participation_reason": str(participation.get("participation_reason") or ""), + }, + "source_summary": { + "line_count": int(source_summary.get("line_count", 0) or 0), + "key_line_count": int(source_summary.get("key_line_count", 0) or 0), + "full_line_count": int(source_summary.get("full_line_count", 0) or 0), + "last_at": str(source_summary.get("last_at") or ""), + "last_line": str(source_summary.get("last_line") or ""), + }, + "missing_reason_code": str(data.get("missing_reason_code") or ""), + "missing_reason": str(data.get("missing_reason") or ""), + "records_total": int(data.get("records_total", 0) or 0), + "records_visible": int(data.get("records_visible", 0) or 0), + "records_truncated": bool(data.get("records_truncated", False)), + "latest_record": latest_record, + "preview_lines": [str(item or "") for item in preview_lines], +}, ensure_ascii=False, indent=2)) +PY +} + +build_node_handover_bootstrap_plan() { + local mainland_base_url="$1" + local node_code="$2" + local issued_by="$3" + local expires_in_hours="$4" + local control_plane_base_url="$5" + local root_dir="$6" + if [[ -z "${node_code}" ]]; then + echo "node_code is required" >&2 + exit 1 + fi + local request_json + request_json="$(python3 - <<'PY' "${issued_by}" "${expires_in_hours}" "${control_plane_base_url}" "${root_dir}" +import json +import sys + +issued_by = str(sys.argv[1] or "cli/ops-center/node-bootstrap-plan").strip() or "cli/ops-center/node-bootstrap-plan" +expires_in_hours = max(1, min(int(sys.argv[2] or 72), 24 * 30)) +control_plane_base_url = str(sys.argv[3] or "").strip() +root_dir = str(sys.argv[4] or "/opt/domaincheck").strip() or "/opt/domaincheck" +print(json.dumps({ + "issued_by": issued_by, + "expires_in_hours": expires_in_hours, + "control_plane_base_url": control_plane_base_url, + "root_dir": root_dir, + "metadata": { + "issued_from": "cli-node-handover-bootstrap-plan", + }, +}, ensure_ascii=False)) +PY +)" + local raw_http_response + raw_http_response="$(request_with_status "POST" "${mainland_base_url}/api/v1/ops/nodes/${node_code}/handover/bootstrap-plan" "${request_json}")" + local http_status="${raw_http_response##*$'\n'HTTP_STATUS=}" + local response_json="${raw_http_response%$'\n'HTTP_STATUS=*}" + local effective_http_status="${http_status}" + local fallback_json="" + + if [[ "${http_status}" == "404" ]]; then + local generic_request_json generic_http_response generic_http_status generic_response_json + generic_request_json="$(python3 - <<'PY' "${node_code}" "${issued_by}" "${expires_in_hours}" "${control_plane_base_url}" "${root_dir}" +import json +import sys + +node_code = str(sys.argv[1] or "").strip() +issued_by = str(sys.argv[2] or "cli/ops-center/node-bootstrap-plan").strip() or "cli/ops-center/node-bootstrap-plan" +expires_in_hours = max(1, min(int(sys.argv[3] or 72), 24 * 30)) +control_plane_base_url = str(sys.argv[4] or "").strip() +root_dir = str(sys.argv[5] or "/opt/domaincheck").strip() or "/opt/domaincheck" + +normalized = node_code.lower() +node_region = "overseas" if normalized.startswith("overseas-") else "mainland" +node_role = "control" if ("controller" in normalized or "-control" in normalized) else "worker" + +print(json.dumps({ + "node_code": node_code, + "node_region": node_region, + "node_role": node_role, + "issued_by": issued_by, + "expires_in_hours": expires_in_hours, + "control_plane_base_url": control_plane_base_url, + "root_dir": root_dir, + "metadata": { + "issued_from": "cli-node-handover-bootstrap-plan-fallback", + "fallback_used": True, + }, +}, ensure_ascii=False)) +PY +)" + generic_http_response="$(request_with_status "POST" "${mainland_base_url}/api/v1/ops/agent/bootstrap-plan" "${generic_request_json}")" + generic_http_status="${generic_http_response##*$'\n'HTTP_STATUS=}" + generic_response_json="${generic_http_response%$'\n'HTTP_STATUS=*}" + if [[ "${generic_http_status}" -ge 200 && "${generic_http_status}" -lt 300 ]]; then + fallback_json="$(python3 - <<'PY' "${generic_response_json}" "${http_status}" "${node_code}" +import json +import sys + +payload = json.loads(str(sys.argv[1] or "{}")) +backend_http_status = int(sys.argv[2] or 404) +node_code = str(sys.argv[3] or "").strip() +data = payload.get("data") or {} +bootstrap_plan = dict(data.get("bootstrap_plan") or {}) + +print(json.dumps({ + "http_status": 200, + "backend_http_status": backend_http_status, + "fallback_used": True, + "fallback_source": "ops_agent_bootstrap_plan", + "compatibility_warning": "当前运行中的 API 尚未暴露节点级 /handover/bootstrap-plan，已降级为通用 /ops/agent/bootstrap-plan。", + "message": str(payload.get("message") or ""), + "node_code": node_code, + "token_preview": str(data.get("token_preview") or ""), + "expires_at": str(data.get("expires_at") or ""), + "control_plane_base_url": str(data.get("control_plane_base_url") or ""), + "bootstrap_plan": bootstrap_plan, + "handover_stage": { + "code": "bootstrap_plan_issued", + "label": "已生成接入方案", + "summary": "当前已通过通用 bootstrap-plan 接口生成接入方案，可直接去目标节点执行。", + }, +}, ensure_ascii=False)) +PY +)" + effective_http_status="200" + fi + fi + echo "[1/2] raw response" + printf '%s\n' "http_status=${http_status}" + print_json_or_raw "${response_json}" + if [[ -n "${fallback_json}" ]]; then + echo + printf '%s\n' "fallback=ops_agent_bootstrap_plan" + print_json_or_raw "${fallback_json}" + fi + echo + echo "[2/2] condensed summary" + python3 - <<'PY' "${response_json}" "${effective_http_status}" "${fallback_json}" +import json +import pathlib +import sys + +payload = json.loads(str(sys.argv[1] or "{}")) +http_status = int(sys.argv[2] or 0) +fallback_json = str(sys.argv[3] or "").strip() +if fallback_json: + data = json.loads(fallback_json) + plan = data.get("bootstrap_plan") or {} + stage = data.get("handover_stage") or {} + node_code = str(data.get("node_code") or "").strip() +else: + data = payload.get("data") or {} + plan = data.get("bootstrap_plan") or {} + handover = data.get("handover") or {} + stage = handover.get("stage") or {} + node_code = str(data.get("node_code") or handover.get("node_code") or "").strip() +workspace_root = pathlib.Path.cwd() +ops_agent_service_path = workspace_root / "domain-api" / "app" / "services" / "ops_agent_service.py" +bootstrap_script_path = workspace_root / "domain-api" / "deploy" / "multi-region" / "bootstrap_node_agent.sh" +ops_agent_service_text = "" +bootstrap_script_text = "" +try: + ops_agent_service_text = ops_agent_service_path.read_text(encoding="utf-8", errors="replace") +except Exception: + ops_agent_service_text = "" +try: + bootstrap_script_text = bootstrap_script_path.read_text(encoding="utf-8", errors="replace") +except Exception: + bootstrap_script_text = "" + +repo_supports_install_command_block = ( + "install_command_block" in ops_agent_service_text + and "_candidate_node_agent_install_paths" in ops_agent_service_text +) +repo_supports_multi_layout_bootstrap = "resolve_project_root" in bootstrap_script_text +install_command_block_present = bool(str(plan.get("install_command_block") or "").strip()) +runtime_may_need_restart = bool(repo_supports_install_command_block and not install_command_block_present) +runtime_restart_hint = ( + "仓库代码已支持 install_command_block，但本次接口响应里仍未返回，当前运行中的控制面 API 可能还没重启到最新代码。" + if runtime_may_need_restart + else "" +) +print(json.dumps({ + "http_status": http_status, + "backend_http_status": int(data.get("backend_http_status", 0) or 0) if fallback_json else http_status, + "fallback_used": bool(data.get("fallback_used", False)), + "fallback_source": str(data.get("fallback_source") or ""), + "compatibility_warning": str(data.get("compatibility_warning") or ""), + "runtime_restart_hint": runtime_restart_hint, + "message": str(payload.get("message") or ""), + "node_code": node_code, + "token_preview": str(data.get("token_preview") or ""), + "expires_at": str(data.get("expires_at") or ""), + "control_plane_base_url": str(data.get("control_plane_base_url") or ""), + "bootstrap_plan": { + "env_file": str(plan.get("env_file") or ""), + "service_name": str(plan.get("service_name") or ""), + "bootstrap_script_name": str(plan.get("bootstrap_script_name") or ""), + "bootstrap_script_path": str(plan.get("bootstrap_script_path") or ""), + "install_command_block_present": install_command_block_present, + "bootstrap_run_script_block_present": bool(str(plan.get("bootstrap_run_script_block") or "").strip()), + }, + "repo_capability": { + "supports_install_command_block": repo_supports_install_command_block, + "supports_multi_layout_bootstrap": repo_supports_multi_layout_bootstrap, + "runtime_may_need_restart": runtime_may_need_restart, + }, + "handover_stage": { + "code": str(stage.get("code") or ""), + "label": str(stage.get("label") or ""), + "summary": str(stage.get("summary") or ""), + }, + "recommended_commands": { + "node_handover": ( + f"bash domain-api/deploy/multi-region/drive_ops_center.sh node-handover http://127.0.0.1:8100 {node_code}" + if node_code + else "" + ), + "delivery_queue": ( + f"bash domain-api/deploy/multi-region/drive_ops_center.sh delivery-queue http://127.0.0.1:8100 {node_code}" + if node_code + else "" + ), + "restart_api_if_runtime_stale": ( + "bash domain-api/deploy/multi-region/drive_ops_center.sh runtime-refresh-recover http://127.0.0.1:8100" + if runtime_may_need_restart + else "" + ), + }, +}, ensure_ascii=False, indent=2)) +PY +} + +show_stack_first_gap_handover() { + local mainland_base_url="$1" + local raw_http_response + raw_http_response="$(request_with_status "GET" "${mainland_base_url}/api/v1/ops/stack-diagnosis?base_url=${mainland_base_url}")" + local http_status="${raw_http_response##*$'\n'HTTP_STATUS=}" + local response_json="${raw_http_response%$'\n'HTTP_STATUS=*}" + local first_gap_node_code + first_gap_node_code="$(python3 - <<'PY' "${response_json}" +import json +import sys + +payload = json.loads(str(sys.argv[1] or "{}")) +data = payload.get("data") or {} +managed_nodes = data.get("managed_nodes") or {} +first_gap = managed_nodes.get("first_handover_gap") or {} +problem_nodes = [dict(item or {}) for item in list(managed_nodes.get("problem_nodes") or []) if isinstance(item, dict)] +node_code = str( + first_gap.get("node_code") + or managed_nodes.get("first_handover_gap_node_code") + or ((problem_nodes[0] or {}).get("node_code") if problem_nodes else "") + or "" +).strip() +print(node_code) +PY +)" + if [[ -z "${first_gap_node_code}" ]]; then + python3 - <<'PY' "${http_status}" "${response_json}" +import json +import sys + +http_status = int(sys.argv[1] or 0) +payload = json.loads(str(sys.argv[2] or "{}")) +data = payload.get("data") or {} +managed_nodes = data.get("managed_nodes") or {} +print(json.dumps({ + "http_status": http_status, + "message": "当前 stack diagnosis 没有首个 handover gap 节点。", + "managed_nodes": { + "managed_enabled": int(managed_nodes.get("managed_enabled", 0) or 0), + "remote_access_ready": int(managed_nodes.get("remote_access_ready", 0) or 0), + "problem_nodes": list(managed_nodes.get("problem_nodes") or [])[:5], + }, +}, ensure_ascii=False, indent=2)) +PY + return 0 + fi + show_node_handover "${mainland_base_url}" "${first_gap_node_code}" +} + +show_node_onboarding() { + local mainland_base_url="$1" + local node_code="$2" + if [[ -z "${node_code}" ]]; then + echo "node_code is required" >&2 + exit 1 + fi + local raw_http_response + raw_http_response="$(request_with_status "GET" "${mainland_base_url}/api/v1/ops/nodes/${node_code}/onboarding")" + local http_status="${raw_http_response##*$'\n'HTTP_STATUS=}" + local response_json="${raw_http_response%$'\n'HTTP_STATUS=*}" + local effective_http_status="${http_status}" + local fallback_json="" + + if [[ "${http_status}" == "404" ]]; then + local handover_http_response handover_http_status handover_response_json + handover_http_response="$(request_with_status "GET" "${mainland_base_url}/api/v1/ops/nodes/${node_code}/handover")" + handover_http_status="${handover_http_response##*$'\n'HTTP_STATUS=}" + handover_response_json="${handover_http_response%$'\n'HTTP_STATUS=*}" + if [[ "${handover_http_status}" == "404" ]]; then + local stack_http_response stack_http_status stack_response_json + stack_http_response="$(request_with_status "GET" "${mainland_base_url}/api/v1/ops/stack-diagnosis?base_url=${mainland_base_url}")" + stack_http_status="${stack_http_response##*$'\n'HTTP_STATUS=}" + stack_response_json="${stack_http_response%$'\n'HTTP_STATUS=*}" + if [[ "${stack_http_status}" -ge 200 && "${stack_http_status}" -lt 300 ]]; then + handover_response_json="$(python3 - <<'PY' "${stack_response_json}" "${node_code}" +import json +import sys + +payload = json.loads(str(sys.argv[1] or "{}")) +target_node_code = str(sys.argv[2] or "").strip() +data = payload.get("data") or {} +managed_nodes = data.get("managed_nodes") or {} +problem_nodes = [dict(item or {}) for item in list(managed_nodes.get("problem_nodes") or []) if isinstance(item, dict)] +first_gap = dict(managed_nodes.get("first_handover_gap") or {}) + +node = next((item for item in problem_nodes if str(item.get("node_code") or "").strip() == target_node_code), {}) +if not node and str(first_gap.get("node_code") or "").strip() == target_node_code: + node = dict(first_gap.get("node") or {}) +if not node: + node = {"node_code": target_node_code} + +agent_state = str(node.get("agent_state") or "").strip() +remote_access_state = str(node.get("remote_access_state") or "").strip() +stage_code = "unknown" +stage_label = "待判断" +stage_summary = "当前节点存在接管缺口，建议先生成 bootstrap 方案并检查 Node Agent 接入。" +if agent_state == "pending_bootstrap": + stage_code = "pending_bootstrap" + stage_label = "待执行接入" + stage_summary = "已签发有效 Token，但目标节点尚未执行 bootstrap 片段。" +elif agent_state == "runtime_only": + stage_code = "runtime_only" + stage_label = "仅运行态在线" + stage_summary = "控制面能看到 runtime 心跳，但 Node Agent 尚未接管。" +elif agent_state == "stale": + stage_code = "stale" + stage_label = "心跳过期" + stage_summary = "Node Agent 曾接入过，但心跳已过期，需先恢复服务或重新接入。" +elif remote_access_state == "agent_pending": + stage_code = "agent_pending" + stage_label = "待接入" + stage_summary = "当前 remote-agent 仍未就绪，先补齐 bootstrap 接入链路。" + +print(json.dumps({ + "data": { + "node_code": target_node_code, + "summary": stage_summary, + "stage": { + "code": stage_code, + "label": stage_label, + "summary": stage_summary, + }, + "readiness": { + "is_managed": True, + "is_enabled": True, + "has_ssh_access": False, + "is_agent_online": agent_state in {"online", "online_busy"}, + "is_remote_access_ready": False, + }, + } +}, ensure_ascii=False)) +PY +)" + handover_http_status="200" + fi + fi + if [[ "${handover_http_status}" -ge 200 && "${handover_http_status}" -lt 300 ]]; then + fallback_json="$(python3 - <<'PY' "${handover_response_json}" "${node_code}" "${http_status}" +import json +import sys + +payload = json.loads(str(sys.argv[1] or "{}")) +node_code = str(sys.argv[2] or "").strip() +backend_http_status = int(sys.argv[3] or 404) +data = payload.get("data") or {} +readiness = data.get("readiness") or {} +stage = data.get("stage") or {} +is_agent_online = bool(readiness.get("is_agent_online", False)) +has_ssh_access = bool(readiness.get("has_ssh_access", False)) +is_remote_access_ready = bool(readiness.get("is_remote_access_ready", False)) + +acceptance_execution_mode = "" +acceptance_status_code = "blocked" +acceptance_status_label = "暂不可验收" +acceptance_summary = "当前还没进入标准接管就绪状态，暂不建议直接跑 onboarding.acceptance。" +if is_agent_online: + acceptance_execution_mode = "remote-agent" + acceptance_status_code = "ready" + acceptance_status_label = "可远端验收" + acceptance_summary = "Node Agent 已在线，可以直接跑 onboarding.acceptance 做标准接管验收。" +elif has_ssh_access: + acceptance_execution_mode = "ssh" + acceptance_status_code = "ssh_only" + acceptance_status_label = "可走 SSH 验收" + acceptance_summary = "当前 Agent 还没在线，但 SSH 已备好，可先走 SSH 模式做接管验收。" + +onboarding_stage_code = str(stage.get("code") or "").strip() or "unknown" +onboarding_stage_label = str(stage.get("label") or "").strip() or "待判断" +onboarding_summary = str(data.get("summary") or stage.get("summary") or "").strip() +if acceptance_status_code in {"ready", "ssh_only"} and onboarding_stage_code == "ready": + onboarding_stage_code = "acceptance_ready" + onboarding_stage_label = "待接管验收" + onboarding_summary = acceptance_summary + +actions = [] +if onboarding_stage_code in {"pending_bootstrap", "runtime_only", "profile_incomplete", "ssh_ready", "token_issue", "stale"}: + actions.append({ + "key": "bootstrap_plan", + "label": "生成接入方案", + "summary": "先生成并执行 Node Agent bootstrap 方案，再观察 register / heartbeat 是否建立。", + }) +if acceptance_status_code in {"ready", "ssh_only"}: + actions.append({ + "key": "run_acceptance", + "label": "执行接管验收", + "summary": acceptance_summary, + }) + +recovery_decision = { + "action": "noop", + "label": "当前无需额外恢复动作", + "summary": onboarding_summary or "当前节点暂无需要执行的接管恢复动作。", + "command_hint": "", + "needs_confirmation": True, + "window": "none", + "stage_code": onboarding_stage_code, + "acceptance_status_code": acceptance_status_code, +} +if onboarding_stage_code == "acceptance_ready" or is_agent_online: + recovery_decision = { + "action": "run_acceptance", + "label": "执行接管验收", + "summary": acceptance_summary, + "command_hint": ( + f"bash domain-api/deploy/multi-region/drive_ops_center.sh node-acceptance-run http://127.0.0.1:8100 {node_code} cli" + if node_code else "" + ), + "needs_confirmation": True, + "window": "acceptance", + "stage_code": onboarding_stage_code, + "acceptance_status_code": acceptance_status_code, + } +elif onboarding_stage_code in {"pending_bootstrap", "runtime_only", "profile_incomplete", "ssh_ready", "token_issue", "stale", "agent_pending", "unknown", ""}: + recovery_decision = { + "action": "bootstrap_run", + "label": "签发接入工单", + "summary": "当前节点仍处于接入阶段，建议先签发 onboarding.bootstrap 并等待 register / heartbeat 建立。", + "command_hint": ( + f"bash domain-api/deploy/multi-region/drive_ops_center.sh node-bootstrap-run http://127.0.0.1:8100 {node_code} cli" + if node_code else "" + ), + "needs_confirmation": True, + "window": "bootstrap", + "stage_code": onboarding_stage_code, + "acceptance_status_code": acceptance_status_code, + } + +print(json.dumps({ + "http_status": 200, + "backend_http_status": backend_http_status, + "fallback_used": True, + "fallback_source": "node_handover", + "compatibility_warning": "当前运行中的 API 尚未暴露 /api/v1/ops/nodes/{node_code}/onboarding，已降级为 handover 兼容视图。", + "node_code": node_code, + "summary": onboarding_summary, + "onboarding_stage": { + "code": onboarding_stage_code, + "label": onboarding_stage_label, + "summary": onboarding_summary, + }, + "handover": data, + "acceptance": { + "status_code": acceptance_status_code, + "status_label": acceptance_status_label, + "summary": acceptance_summary, + "execution_mode": acceptance_execution_mode, + "is_agent_online": is_agent_online, + "has_ssh_access": has_ssh_access, + "is_remote_access_ready": is_remote_access_ready, + "playbook": { + "playbook_key": "onboarding.acceptance", + "target_node_codes": [node_code] if node_code else [], + "execution_mode": acceptance_execution_mode or "remote-agent", + "auto_approve": True, + }, + }, + "recommended_actions": actions, + "recovery_decision": recovery_decision, +}, ensure_ascii=False)) +PY +)" + effective_http_status="200" + fi + fi + + echo "[1/2] raw response" + printf '%s\n' "http_status=${http_status}" + print_json_or_raw "${response_json}" + if [[ -n "${fallback_json}" ]]; then + echo + printf '%s\n' "fallback=node_handover" + print_json_or_raw "${fallback_json}" + fi + echo + echo "[2/2] condensed summary" + python3 - <<'PY' "${response_json}" "${effective_http_status}" "${fallback_json}" +import json +import sys + +payload = json.loads(str(sys.argv[1] or "{}")) +http_status = int(sys.argv[2] or 0) +fallback_json = str(sys.argv[3] or "").strip() +data = json.loads(fallback_json) if fallback_json else (payload.get("data") or {}) +onboarding_stage = data.get("onboarding_stage") or {} +acceptance = data.get("acceptance") or {} +recovery_decision = data.get("recovery_decision") or {} +playbook = acceptance.get("playbook") or {} +node_code = str(data.get("node_code") or "").strip() +print(json.dumps({ + "http_status": http_status, + "backend_http_status": int(data.get("backend_http_status", 0) or 0) if fallback_json else http_status, + "fallback_used": bool(data.get("fallback_used", False)), + "fallback_source": str(data.get("fallback_source") or ""), + "compatibility_warning": str(data.get("compatibility_warning") or ""), + "node_code": node_code, + "summary": str(data.get("summary") or ""), + "onboarding_stage": { + "code": str(onboarding_stage.get("code") or ""), + "label": str(onboarding_stage.get("label") or ""), + "summary": str(onboarding_stage.get("summary") or ""), + }, + "acceptance": { + "status_code": str(acceptance.get("status_code") or ""), + "status_label": str(acceptance.get("status_label") or ""), + "summary": str(acceptance.get("summary") or ""), + "execution_mode": str(acceptance.get("execution_mode") or ""), + "is_agent_online": bool(acceptance.get("is_agent_online", False)), + "has_ssh_access": bool(acceptance.get("has_ssh_access", False)), + "is_remote_access_ready": bool(acceptance.get("is_remote_access_ready", False)), + "playbook": playbook, + }, + "recommended_actions": list(data.get("recommended_actions") or []), + "recovery_decision": { + "action": str(recovery_decision.get("action") or ""), + "label": str(recovery_decision.get("label") or ""), + "summary": str(recovery_decision.get("summary") or ""), + "command_hint": str(recovery_decision.get("command_hint") or ""), + "needs_confirmation": bool(recovery_decision.get("needs_confirmation", False)), + "window": str(recovery_decision.get("window") or ""), + "stage_code": str(recovery_decision.get("stage_code") or ""), + "acceptance_status_code": str(recovery_decision.get("acceptance_status_code") or ""), + }, + "recommended_commands": { + "node_handover": ( + f"bash domain-api/deploy/multi-region/drive_ops_center.sh node-handover http://127.0.0.1:8100 {node_code}" + if node_code else "" + ), + "node_bootstrap_preview": ( + f"bash domain-api/deploy/multi-region/drive_ops_center.sh node-bootstrap-preview http://127.0.0.1:8100 {node_code}" + if node_code else "" + ), + "node_bootstrap_run": ( + f"bash domain-api/deploy/multi-region/drive_ops_center.sh node-bootstrap-run http://127.0.0.1:8100 {node_code} cli" + if node_code else "" + ), + "node_bootstrap_plan": ( + f"bash domain-api/deploy/multi-region/drive_ops_center.sh node-bootstrap-plan http://127.0.0.1:8100 {node_code}" + if node_code else "" + ), + "node_acceptance_plan": ( + f"bash domain-api/deploy/multi-region/drive_ops_center.sh node-acceptance-plan http://127.0.0.1:8100 {node_code}" + if node_code else "" + ), + "node_acceptance_run": ( + f"bash domain-api/deploy/multi-region/drive_ops_center.sh node-acceptance-run http://127.0.0.1:8100 {node_code} cli" + if node_code else "" + ), + "agent_gap_check": "bash domain-api/deploy/multi-region/drive_ops_center.sh agent-gap-check http://127.0.0.1:8100", + }, +}, ensure_ascii=False, indent=2)) +PY +} + +recover_node_onboarding() { + local mainland_base_url="$1" + local node_code="$2" + local confirm_flag="${3:-}" + local requested_by="${4:-cli}" + if [[ -z "${node_code}" ]]; then + echo "node_code is required" >&2 + exit 1 + fi + + local recovery_endpoint + if [[ "${confirm_flag}" == "confirm" ]]; then + recovery_endpoint="${mainland_base_url}/api/v1/ops/nodes/${node_code}/onboarding/recovery/execute" + else + recovery_endpoint="${mainland_base_url}/api/v1/ops/nodes/${node_code}/onboarding/recovery/preview" + fi + local recovery_request_json + recovery_request_json="$(python3 - <<'PY' "${requested_by}" +import json +import sys +print(json.dumps({"requested_by": str(sys.argv[1] or "cli").strip() or "cli"}, ensure_ascii=False)) +PY +)" + + local raw_http_response + local source_mode="recovery_endpoint" + raw_http_response="$(request_with_status "POST" "${recovery_endpoint}" "${recovery_request_json}")" + local http_status="${raw_http_response##*$'\n'HTTP_STATUS=}" + local response_json="${raw_http_response%$'\n'HTTP_STATUS=*}" + + if [[ "${http_status}" == "404" ]]; then + source_mode="onboarding" + raw_http_response="$(request_with_status "GET" "${mainland_base_url}/api/v1/ops/nodes/${node_code}/onboarding")" + http_status="${raw_http_response##*$'\n'HTTP_STATUS=}" + response_json="${raw_http_response%$'\n'HTTP_STATUS=*}" + fi + + if [[ "${http_status}" == "404" ]]; then + source_mode="handover" + raw_http_response="$(request_with_status "GET" "${mainland_base_url}/api/v1/ops/nodes/${node_code}/handover")" + http_status="${raw_http_response##*$'\n'HTTP_STATUS=}" + response_json="${raw_http_response%$'\n'HTTP_STATUS=*}" + if [[ "${http_status}" == "404" ]]; then + source_mode="stack_diagnosis" + raw_http_response="$(request_with_status "GET" "${mainland_base_url}/api/v1/ops/stack-diagnosis?base_url=${mainland_base_url}")" + http_status="${raw_http_response##*$'\n'HTTP_STATUS=}" + response_json="${raw_http_response%$'\n'HTTP_STATUS=*}" + if [[ "${http_status}" -ge 200 && "${http_status}" -lt 300 ]]; then + response_json="$(python3 - <<'PY' "${response_json}" "${node_code}" +import json +import sys + +payload = json.loads(str(sys.argv[1] or "{}")) +target_node_code = str(sys.argv[2] or "").strip() +data = payload.get("data") or {} +managed_nodes = data.get("managed_nodes") or {} +problem_nodes = [dict(item or {}) for item in list(managed_nodes.get("problem_nodes") or []) if isinstance(item, dict)] +node = next((item for item in problem_nodes if str(item.get("node_code") or "").strip() == target_node_code), {}) +first_gap = dict(managed_nodes.get("first_handover_gap") or {}) +if not node and str(first_gap.get("node_code") or "").strip() == target_node_code: + node = dict(first_gap.get("node") or {}) +agent_state = str(node.get("agent_state") or "").strip() +stage_code = "unknown" +stage_summary = "当前节点存在接管缺口，建议先生成 bootstrap 方案并检查 Node Agent 接入。" +if agent_state == "pending_bootstrap": + stage_code = "pending_bootstrap" + stage_summary = "已签发有效 Token，但目标节点尚未执行 bootstrap 片段。" +elif agent_state == "runtime_only": + stage_code = "runtime_only" + stage_summary = "控制面能看到 runtime 心跳，但 Node Agent 尚未接管。" +elif agent_state == "stale": + stage_code = "stale" + stage_summary = "Node Agent 曾接入过，但心跳已过期，需先恢复服务或重新接入。" +elif str(node.get("remote_access_state") or "").strip() == "agent_pending": + stage_code = "agent_pending" + stage_summary = "当前 remote-agent 仍未就绪，先补齐 bootstrap 接入链路。" + +print(json.dumps({ + "data": { + "node_code": target_node_code, + "summary": stage_summary, + "onboarding_stage": { + "code": stage_code, + "summary": stage_summary, + }, + "acceptance": { + "status_code": "blocked", + "summary": "当前还没进入标准接管就绪状态，暂不建议直接跑 onboarding.acceptance。", + }, + } +}, ensure_ascii=False)) +PY +)" + http_status="200" + fi + fi + fi + + python3 - <<'PY' "${response_json}" "${node_code}" "${mainland_base_url}" "${confirm_flag}" "${requested_by}" "${http_status}" "${source_mode}" +import json +import sys + +payload = json.loads(str(sys.argv[1] or "{}")) +node_code = str(sys.argv[2] or "").strip() +base_url = str(sys.argv[3] or "").strip() +confirm_flag = str(sys.argv[4] or "").strip() +requested_by = str(sys.argv[5] or "cli").strip() or "cli" +backend_http_status = int(sys.argv[6] or 0) +source_mode = str(sys.argv[7] or "").strip() + +data = payload.get("data") or {} +if str(data.get("recovery_action") or "").strip(): + recovery_decision = (data.get("recovery_decision") or {}) + selected_preview = (data.get("selected_preview") or {}) + selected_run = (data.get("selected_run") or {}) + selected_payload = selected_run or selected_preview + playbook_preview = (selected_payload.get("playbook_preview") or {}) + playbook_run = (selected_payload.get("playbook_run") or {}) + follow_up = (selected_payload.get("follow_up") or {}) + result = { + "node_code": node_code, + "source_mode": source_mode, + "backend_http_status": backend_http_status, + "stage_code": str((recovery_decision.get("stage_code") or "")).strip(), + "acceptance_status_code": str((recovery_decision.get("acceptance_status_code") or "")).strip(), + "recovery_action": str(data.get("recovery_action") or "").strip(), + "selected_action": "node_acceptance_run" if str(data.get("recovery_action") or "").strip() == "run_acceptance" else "node_bootstrap_run", + "summary": str((recovery_decision.get("summary") or payload.get("message") or "")).strip(), + "command": str((recovery_decision.get("command_hint") or "")).strip(), + "execution_mode": str((selected_payload.get("execution_mode") or "")).strip(), + "playbook_preview_key": str(((playbook_preview.get("playbook") or {}).get("key") or "")).strip(), + "playbook_run_code": str((playbook_run.get("run_code") or "")).strip(), + "follow_up_next_stage_code": str((follow_up.get("next_stage_code") or "")).strip(), + "follow_up_summary": str((follow_up.get("summary") or "")).strip(), + "follow_up_commands": list(follow_up.get("recommended_commands") or []), + "confirm_required": True, + "confirmed": confirm_flag == "confirm", + "handled_by_recovery_endpoint": source_mode == "recovery_endpoint", + } + print(json.dumps(result, ensure_ascii=False, indent=2)) + raise SystemExit(0) + +stage = data.get("onboarding_stage") or data.get("stage") or {} +acceptance = data.get("acceptance") or {} +recovery = data.get("recovery_decision") or {} +stage_code = str(stage.get("code") or "").strip() +acceptance_status_code = str(acceptance.get("status_code") or "").strip() +stage_summary = str(stage.get("summary") or data.get("summary") or "").strip() + +if not stage_code: + handover_stage = data.get("stage") or {} + stage_code = str(handover_stage.get("code") or "").strip() + stage_summary = str(handover_stage.get("summary") or stage_summary).strip() + +action = "noop" +reason = stage_summary or "当前节点暂无可执行恢复动作。" +command = "" +recovery_action = str(recovery.get("action") or "").strip() +recovery_command_hint = str(recovery.get("command_hint") or "").strip() +recovery_summary = str(recovery.get("summary") or "").strip() +if recovery_action == "run_acceptance": + action = "node_acceptance_run" + reason = recovery_summary or "当前节点已进入验收窗口，建议直接跑 onboarding.acceptance。" + command = recovery_command_hint or f"bash domain-api/deploy/multi-region/drive_ops_center.sh node-acceptance-run {base_url} {node_code} {requested_by}" +elif recovery_action == "bootstrap_run": + action = "node_bootstrap_run" + reason = recovery_summary or "当前节点仍处于接入阶段，建议先签发 onboarding.bootstrap。" + command = recovery_command_hint or f"bash domain-api/deploy/multi-region/drive_ops_center.sh node-bootstrap-run {base_url} {node_code} {requested_by}" +elif acceptance_status_code in {"ready", "ssh_only"} or stage_code == "acceptance_ready": + action = "node_acceptance_run" + reason = "当前节点已进入验收窗口，建议直接跑 onboarding.acceptance。" + command = f"bash domain-api/deploy/multi-region/drive_ops_center.sh node-acceptance-run {base_url} {node_code} {requested_by}" +elif stage_code in {"pending_bootstrap", "runtime_only", "profile_incomplete", "ssh_ready", "token_issue", "stale", "agent_pending", "unknown", ""}: + action = "node_bootstrap_run" + reason = "当前节点仍处于接入阶段，建议先签发 onboarding.bootstrap。" + command = f"bash domain-api/deploy/multi-region/drive_ops_center.sh node-bootstrap-run {base_url} {node_code} {requested_by}" + +result = { + "node_code": node_code, + "source_mode": source_mode, + "backend_http_status": backend_http_status, + "stage_code": stage_code, + "acceptance_status_code": acceptance_status_code, + "recovery_action": recovery_action, + "selected_action": action, + "summary": reason, + "command": command, + "confirm_required": True, + "confirmed": confirm_flag == "confirm", +} +print(json.dumps(result, ensure_ascii=False, indent=2)) +PY + + if [[ "${confirm_flag}" != "confirm" ]]; then + return 0 + fi + + if [[ "${source_mode}" == "recovery_endpoint" ]]; then + return 0 + fi + + local selected_action + selected_action="$(python3 - <<'PY' "${response_json}" +import json +import sys + +payload = json.loads(str(sys.argv[1] or "{}")) +data = payload.get("data") or {} +stage = data.get("onboarding_stage") or data.get("stage") or {} +acceptance = data.get("acceptance") or {} +recovery = data.get("recovery_decision") or {} +stage_code = str(stage.get("code") or "").strip() +acceptance_status_code = str(acceptance.get("status_code") or "").strip() +recovery_action = str(recovery.get("action") or "").strip() +if recovery_action == "run_acceptance": + print("node_acceptance_run") +elif recovery_action == "bootstrap_run": + print("node_bootstrap_run") +elif acceptance_status_code in {"ready", "ssh_only"} or stage_code == "acceptance_ready": + print("node_acceptance_run") +elif stage_code in {"pending_bootstrap", "runtime_only", "profile_incomplete", "ssh_ready", "token_issue", "stale", "agent_pending", "unknown", ""}: + print("node_bootstrap_run") +else: + print("") +PY +)" + + case "${selected_action}" in + node_bootstrap_run) + run_node_bootstrap "${mainland_base_url}" "${node_code}" "${requested_by}" + ;; + node_acceptance_run) + run_node_acceptance "${mainland_base_url}" "${node_code}" "${requested_by}" + ;; + *) + printf '%s\n' '{"message":"当前节点没有可自动执行的恢复动作。"}' + ;; + esac +} + +show_node_acceptance_plan() { + local mainland_base_url="$1" + local node_code="$2" + local requested_by="${3:-cli}" + if [[ -z "${node_code}" ]]; then + echo "node_code is required" >&2 + exit 1 + fi + local request_json + request_json="$(python3 - <<'PY' "${requested_by}" +import json +import sys +print(json.dumps({"requested_by": str(sys.argv[1] or "cli").strip() or "cli"}, ensure_ascii=False)) +PY +)" + local raw_http_response + raw_http_response="$(request_with_status "POST" "${mainland_base_url}/api/v1/ops/nodes/${node_code}/onboarding/acceptance/preview" "${request_json}")" + local http_status="${raw_http_response##*$'\n'HTTP_STATUS=}" + local response_json="${raw_http_response%$'\n'HTTP_STATUS=*}" + if [[ "${http_status}" == "404" ]]; then + request_json="$(python3 - <<'PY' "${node_code}" "${requested_by}" +import json +import sys +node_code = str(sys.argv[1] or "").strip() +requested_by = str(sys.argv[2] or "cli").strip() or "cli" +print(json.dumps({ + "playbook_key": "onboarding.acceptance", + "node_codes": [node_code] if node_code else [], + "execution_mode": "remote-agent", + "auto_approve": True, + "requested_by": requested_by, +}, ensure_ascii=False)) +PY +)" + raw_http_response="$(request_with_status "POST" "${mainland_base_url}/api/v1/ops/playbooks/preview" "${request_json}")" + response_json="${raw_http_response%$'\n'HTTP_STATUS=*}" + fi + printf '%s\n' "${response_json}" +} + +show_node_bootstrap_preview() { + local mainland_base_url="$1" + local node_code="$2" + local requested_by="${3:-cli}" + if [[ -z "${node_code}" ]]; then + echo "node_code is required" >&2 + exit 1 + fi + local request_json + request_json="$(python3 - <<'PY' "${requested_by}" +import json +import sys +print(json.dumps({"requested_by": str(sys.argv[1] or "cli").strip() or "cli"}, ensure_ascii=False)) +PY +)" + local raw_http_response + raw_http_response="$(request_with_status "POST" "${mainland_base_url}/api/v1/ops/nodes/${node_code}/onboarding/bootstrap/preview" "${request_json}")" + local http_status="${raw_http_response##*$'\n'HTTP_STATUS=}" + local response_json="${raw_http_response%$'\n'HTTP_STATUS=*}" + if [[ "${http_status}" == "404" ]]; then + request_json="$(python3 - <<'PY' "${node_code}" "${requested_by}" +import json +import sys +node_code = str(sys.argv[1] or "").strip() +requested_by = str(sys.argv[2] or "cli").strip() or "cli" +print(json.dumps({ + "playbook_key": "onboarding.bootstrap", + "node_codes": [node_code] if node_code else [], + "execution_mode": "control-plane", + "auto_approve": True, + "requested_by": requested_by, +}, ensure_ascii=False)) +PY +)" + raw_http_response="$(request_with_status "POST" "${mainland_base_url}/api/v1/ops/playbooks/preview" "${request_json}")" + response_json="${raw_http_response%$'\n'HTTP_STATUS=*}" + fi + printf '%s\n' "${response_json}" +} + +run_node_bootstrap() { + local mainland_base_url="$1" + local node_code="$2" + local requested_by="${3:-cli}" + if [[ -z "${node_code}" ]]; then + echo "node_code is required" >&2 + exit 1 + fi + local request_json + request_json="$(python3 - <<'PY' "${requested_by}" +import json +import sys +print(json.dumps({"requested_by": str(sys.argv[1] or "cli").strip() or "cli"}, ensure_ascii=False)) +PY +)" + local raw_http_response + raw_http_response="$(request_with_status "POST" "${mainland_base_url}/api/v1/ops/nodes/${node_code}/onboarding/bootstrap/execute" "${request_json}")" + local http_status="${raw_http_response##*$'\n'HTTP_STATUS=}" + local response_json="${raw_http_response%$'\n'HTTP_STATUS=*}" + if [[ "${http_status}" == "404" ]]; then + request_json="$(python3 - <<'PY' "${node_code}" "${requested_by}" +import json +import sys +node_code = str(sys.argv[1] or "").strip() +requested_by = str(sys.argv[2] or "cli").strip() or "cli" +print(json.dumps({ + "playbook_key": "onboarding.bootstrap", + "node_codes": [node_code] if node_code else [], + "execution_mode": "control-plane", + "auto_approve": True, + "requested_by": requested_by, +}, ensure_ascii=False)) +PY +)" + raw_http_response="$(request_with_status "POST" "${mainland_base_url}/api/v1/ops/playbooks/execute" "${request_json}")" + response_json="${raw_http_response%$'\n'HTTP_STATUS=*}" + fi + printf '%s\n' "${response_json}" + echo + echo "[follow-up]" + python3 - <<'PY' "${response_json}" "${node_code}" +import json +import sys + +payload = json.loads(str(sys.argv[1] or "{}")) +node_code = str(sys.argv[2] or "").strip() +data = payload.get("data") or {} +follow_up = dict(data.get("follow_up") or {}) +playbook_run = dict(data.get("playbook_run") or {}) +follow_up_summary = str(follow_up.get("summary") or "").strip() or str(playbook_run.get("focus_summary") or "").strip() +print(json.dumps({ + "node_code": node_code, + "playbook_run_code": str(playbook_run.get("run_code") or "").strip(), + "follow_up_next_stage_code": str(follow_up.get("next_stage_code") or "").strip(), + "follow_up_summary": follow_up_summary, + "follow_up_commands": list(follow_up.get("recommended_commands") or []), +}, ensure_ascii=False, indent=2)) +PY +} + +run_node_acceptance() { + local mainland_base_url="$1" + local node_code="$2" + local requested_by="${3:-cli}" + if [[ -z "${node_code}" ]]; then + echo "node_code is required" >&2 + exit 1 + fi + local request_json + request_json="$(python3 - <<'PY' "${requested_by}" +import json +import sys +print(json.dumps({"requested_by": str(sys.argv[1] or "cli").strip() or "cli"}, ensure_ascii=False)) +PY +)" + local raw_http_response + raw_http_response="$(request_with_status "POST" "${mainland_base_url}/api/v1/ops/nodes/${node_code}/onboarding/acceptance/execute" "${request_json}")" + local http_status="${raw_http_response##*$'\n'HTTP_STATUS=}" + local response_json="${raw_http_response%$'\n'HTTP_STATUS=*}" + if [[ "${http_status}" == "404" ]]; then + request_json="$(python3 - <<'PY' "${node_code}" "${requested_by}" +import json +import sys +node_code = str(sys.argv[1] or "").strip() +requested_by = str(sys.argv[2] or "cli").strip() or "cli" +print(json.dumps({ + "playbook_key": "onboarding.acceptance", + "node_codes": [node_code] if node_code else [], + "execution_mode": "remote-agent", + "auto_approve": True, + "requested_by": requested_by, +}, ensure_ascii=False)) +PY +)" + raw_http_response="$(request_with_status "POST" "${mainland_base_url}/api/v1/ops/playbooks/execute" "${request_json}")" + response_json="${raw_http_response%$'\n'HTTP_STATUS=*}" + fi + printf '%s\n' "${response_json}" + echo + echo "[follow-up]" + python3 - <<'PY' "${response_json}" "${node_code}" +import json +import sys + +payload = json.loads(str(sys.argv[1] or "{}")) +node_code = str(sys.argv[2] or "").strip() +data = payload.get("data") or {} +follow_up = dict(data.get("follow_up") or {}) +playbook_run = dict(data.get("playbook_run") or {}) +follow_up_summary = str(follow_up.get("summary") or "").strip() or str(playbook_run.get("focus_summary") or "").strip() +print(json.dumps({ + "node_code": node_code, + "playbook_run_code": str(playbook_run.get("run_code") or "").strip(), + "follow_up_next_stage_code": str(follow_up.get("next_stage_code") or "").strip(), + "follow_up_summary": follow_up_summary, + "follow_up_commands": list(follow_up.get("recommended_commands") or []), +}, ensure_ascii=False, indent=2)) +PY +} + +show_delivery_queue_records() { + local mainland_base_url="$1" + local node_code="$2" + local state="${3:-}" + if [[ -z "${node_code}" ]]; then + echo "node_code is required" >&2 + exit 1 + fi + local url="${mainland_base_url}/api/v1/ops/nodes/${node_code}/delivery-queue/records" + if [[ -n "${state}" ]]; then + url="${url}?state=${state}" + fi + local raw_http_response + raw_http_response="$(request_with_status "GET" "${url}")" + local http_status="${raw_http_response##*$'\n'HTTP_STATUS=}" + local response_json="${raw_http_response%$'\n'HTTP_STATUS=*}" + echo "[1/2] raw response" + printf '%s\n' "http_status=${http_status}" + print_json_or_raw "${response_json}" + echo + echo "[2/2] condensed summary" + python3 - <<'PY' "${response_json}" "${http_status}" +import json +import sys + +payload = json.loads(str(sys.argv[1] or "{}")) +http_status = int(sys.argv[2] or 0) +data = payload.get("data") or {} +summary = data.get("summary") or {} +records = list(data.get("records") or []) +node_code = str(data.get("node_code") or "").strip() +print(json.dumps({ + "http_status": http_status, + "node_code": node_code, + "summary": { + "total": int(summary.get("total", 0) or 0), + "available_total": int(summary.get("available_total", 0) or 0), + "record_visibility": str(summary.get("record_visibility") or ""), + "available_state_counts": dict(summary.get("available_state_counts") or {}), + }, + "recommended_commands": { + "queue_status": ( + f"bash domain-api/deploy/multi-region/drive_ops_center.sh delivery-queue http://127.0.0.1:8100 {node_code}" + if node_code + else "" + ), + "replay": ( + f"bash domain-api/deploy/multi-region/drive_ops_center.sh delivery-queue-replay http://127.0.0.1:8100 {node_code}" + if node_code + else "" + ), + }, + "records": [ + { + "record_id": str(item.get("record_id") or item.get("client_request_id") or item.get("record_key") or ""), + "state": str(item.get("state") or ""), + "request_kind": str(item.get("request_kind") or ""), + "created_at": str(item.get("created_at") or ""), + "dead_letter_at": str(item.get("dead_letter_at") or ""), + "label": str(item.get("label") or ""), + "record_source": str(item.get("record_source") or ""), + } + for item in records + ], +}, ensure_ascii=False, indent=2)) +PY +} + +run_delivery_queue_action() { + local mainland_base_url="$1" + local node_code="$2" + local action_path="$3" + local body_json="$4" + if [[ -z "${node_code}" ]]; then + echo "node_code is required" >&2 + exit 1 + fi + local raw_http_response + raw_http_response="$(request_with_status "POST" "${mainland_base_url}/api/v1/ops/nodes/${node_code}${action_path}" "${body_json}")" + local http_status="${raw_http_response##*$'\n'HTTP_STATUS=}" + local response_json="${raw_http_response%$'\n'HTTP_STATUS=*}" + printf '%s\n' "http_status=${http_status}" + print_json_or_raw "${response_json}" +} + +preview_runbook_sequence() { + local mainland_base_url="$1" + local sequence_key="$2" + local action_payload_json="$3" + local requested_by="$4" + + if [[ "${action_payload_json}" == @* ]]; then + local action_payload_path="${action_payload_json#@}" + if [[ ! -f "${action_payload_path}" ]]; then + echo "payload file not found: ${action_payload_path}" >&2 + exit 1 + fi + action_payload_json="$(cat "${action_payload_path}")" + fi + + local request_json + request_json="$(python3 - <<'PY' "${sequence_key}" "${action_payload_json}" "${requested_by}" +import json +import sys + +sequence_key = str(sys.argv[1] or "").strip() +payload_raw = str(sys.argv[2] or "").strip() or "{}" +requested_by = str(sys.argv[3] or "").strip() or "cli/ops-center/runbook-preview" + +try: + action_payload = json.loads(payload_raw) +except Exception as exc: + raise SystemExit(f"invalid action payload json: {exc}") + +if not isinstance(action_payload, dict): + raise SystemExit("action payload must be a json object") + +request_payload = { + "source_label": "cli/runbook-preview", + "title": sequence_key, + "executor_kind": "runbook_sequence", + "sequence_key": sequence_key, + "requested_by": requested_by, + "include_secondary": True, + "primary_action_payload": action_payload, + "secondary_action_payload": action_payload, +} +print(json.dumps(request_payload, ensure_ascii=False)) +PY +)" + + echo "[1/3] runbook preview request" + print_json_or_raw "${request_json}" + echo + echo + + local raw_http_response + raw_http_response="$(request_with_status "POST" "${mainland_base_url}/api/v1/ops/driver-actions/preview" "${request_json}")" + local http_status="${raw_http_response##*$'\n'HTTP_STATUS=}" + local response_json="${raw_http_response%$'\n'HTTP_STATUS=*}" + + echo "[2/3] raw response" + printf '%s\n' "http_status=${http_status}" + print_json_or_raw "${response_json}" + echo + echo + + echo "[3/3] condensed summary" + python3 - <<'PY' "${response_json}" "${http_status}" +import json +import sys + +payload = json.loads(str(sys.argv[1] or "{}")) +http_status = int(sys.argv[2] or 0) +data = payload.get("data") or {} +primary = data.get("primary") or {} +secondary = data.get("secondary") or {} + +print(json.dumps({ + "http_status": http_status, + "message": str(payload.get("message") or ""), + "sequence_key": str(data.get("sequence_key") or ""), + "execution_chain": str(data.get("execution_chain") or ""), + "focus_ref": data.get("focus_ref") or {}, + "primary_focus_ref": data.get("primary_focus_ref") or {}, + "secondary_focus_ref": data.get("secondary_focus_ref") or {}, + "primary": { + "label": str(primary.get("label") or ""), + "action_code": str(primary.get("action_code") or ""), + "target_api": str(primary.get("target_api") or ""), + "recommendation_label": str(primary.get("recommendation_label") or ""), + "focus_ref": primary.get("focus_ref") or {}, + "request_payload": primary.get("request_payload") or {}, + }, + "secondary": ( + { + "label": str(secondary.get("label") or ""), + "action_code": str(secondary.get("action_code") or ""), + "target_api": str(secondary.get("target_api") or ""), + "recommendation_label": str(secondary.get("recommendation_label") or ""), + "focus_ref": secondary.get("focus_ref") or {}, + "request_payload": secondary.get("request_payload") or {}, + } + if secondary else {} + ), +}, ensure_ascii=False, indent=2)) +PY + + if [[ "${http_status}" -lt 200 || "${http_status}" -ge 300 ]]; then + exit 1 + fi +} + +build_codex_focus_bundle() { + local codex_response_json="$1" + local entry_key="$2" + local requested_by="$3" + local source_label="$4" + python3 - <<'PY' "${codex_response_json}" "${entry_key}" "${requested_by}" "${source_label}" +import json +import sys + +payload = json.loads(str(sys.argv[1] or "{}")) +requested_entry_key = str(sys.argv[2] or "").strip() +requested_by = str(sys.argv[3] or "").strip() or "cli/ops-center/codex-focus-preview" +source_label = str(sys.argv[4] or "").strip() or "cli/codex-focus-preview" +data = payload.get("data") or {} +focus = data.get("focus") or {} +entries = [dict(item or {}) for item in list(data.get("entries") or []) if dict(item or {})] +activity_focus = [dict(item or {}) for item in list(data.get("activity_focus") or []) if dict(item or {})] + +def build_activity_focus_entry(item): + normalized = dict(item or {}) + focus_action_code = str(normalized.get("focus_action_code") or "").strip() + if not str(normalized.get("key") or "").strip() or not focus_action_code: + return {} + + focus_ref = dict(normalized.get("focus_ref") or {}) + target_node_codes = list(normalized.get("target_node_codes") or []) + focus_action_payload = dict(normalized.get("focus_action_payload") or {}) + summary = str(normalized.get("summary") or "").strip() + reason = summary or "该焦点属于运行态观察项，建议进入对应工作区查看上下文。" + return { + "key": str(normalized.get("key") or "").strip(), + "title": str(normalized.get("title") or "").strip(), + "summary": summary, + "reason": reason, + "meta_text": str(normalized.get("meta_text") or "").strip(), + "executor_kind": "driver_action", + "primary_action_code": focus_action_code, + "secondary_action_code": "", + "focus_ref": focus_ref, + "primary_focus_ref": focus_ref, + "secondary_focus_ref": {}, + "primary_label": "进入工作区", + "secondary_label": "", + "primary_execution": { + "label": "进入工作区", + "node_codes": target_node_codes, + "action_payload": focus_action_payload, + }, + "secondary_execution": {}, + "autopilot": { + "recommendation": "open_ui", + "automation_level": "ui_only", + "status_text": str(normalized.get("status_label") or normalized.get("status") or "").strip(), + "executor_mode_hint": "ui-intent", + "target_api": "/api/v1/ops/driver-actions/execute", + "target_method": "POST", + "reason": reason, + "confirm_required": False, + "ui_only": True, + "blocked": False, + }, + "observation_only": True, + "activity_key": str(normalized.get("activity_key") or "").strip(), + "activity_kind": str(normalized.get("activity_kind") or "").strip(), + "ui_intent_kind": str(normalized.get("ui_intent_kind") or "").strip(), + "contract_navigation": dict(normalized.get("contract_navigation") or {}), + } + +activity_focus_entries = [entry for entry in (build_activity_focus_entry(item) for item in activity_focus) if entry] + +target_entry_key = requested_entry_key or str(focus.get("entry_key") or "").strip() +target_entry = {} +if target_entry_key: + target_entry = next((item for item in entries if str(item.get("key") or "").strip() == target_entry_key), {}) +if not target_entry and target_entry_key: + target_entry = next((item for item in activity_focus_entries if str(item.get("key") or "").strip() == target_entry_key), {}) +if not target_entry and entries: + target_entry = dict(entries[0] or {}) + target_entry_key = str(target_entry.get("key") or "").strip() +if not target_entry and activity_focus_entries: + target_entry = dict(activity_focus_entries[0] or {}) + target_entry_key = str(target_entry.get("key") or "").strip() + +if not target_entry: + raise SystemExit("codex brief 当前没有可预览的 entry") + +autopilot = dict(target_entry.get("autopilot") or {}) +primary_execution = dict(target_entry.get("primary_execution") or {}) +secondary_execution = dict(target_entry.get("secondary_execution") or {}) +request_payload = { + "source_label": source_label, + "title": str(target_entry.get("title") or "").strip() or target_entry_key or "codex-focus", + "summary": str(target_entry.get("summary") or "").strip(), + "reason": str(target_entry.get("reason") or "").strip(), + "meta_text": str(target_entry.get("meta_text") or "").strip(), + "executor_kind": str(target_entry.get("executor_kind") or "driver_action").strip() or "driver_action", + "sequence_key": str(target_entry.get("sequence_key") or "").strip(), + "focus_ref": dict(target_entry.get("focus_ref") or {}), + "primary_focus_ref": dict(target_entry.get("primary_focus_ref") or {}), + "secondary_focus_ref": dict(target_entry.get("secondary_focus_ref") or {}), + "primary_label": str(primary_execution.get("label") or target_entry.get("primary_label") or "").strip(), + "secondary_label": str(secondary_execution.get("label") or target_entry.get("secondary_label") or "").strip(), + "primary_action_code": str(target_entry.get("primary_action_code") or "").strip(), + "secondary_action_code": str(target_entry.get("secondary_action_code") or "").strip(), + "primary_node_codes": list(primary_execution.get("node_codes") or target_entry.get("node_codes") or []), + "secondary_node_codes": list(secondary_execution.get("node_codes") or target_entry.get("node_codes") or []), + "primary_action_payload": dict(primary_execution.get("action_payload") or {}), + "secondary_action_payload": dict(secondary_execution.get("action_payload") or {}), + "include_secondary": True, + "requested_by": requested_by, +} + +print(json.dumps({ + "selected_entry": { + "key": target_entry_key, + "title": str(target_entry.get("title") or "").strip(), + "executor_kind": str(target_entry.get("executor_kind") or "").strip(), + "recommendation": str(autopilot.get("recommendation") or "").strip(), + "automation_level": str(autopilot.get("automation_level") or "").strip(), + "status_text": str(autopilot.get("status_text") or "").strip(), + "executor_mode_hint": str(autopilot.get("executor_mode_hint") or "").strip(), + "target_api": str(autopilot.get("target_api") or "").strip(), + "target_method": str(autopilot.get("target_method") or "").strip(), + "reason": str(autopilot.get("reason") or "").strip(), + "confirm_required": bool(autopilot.get("confirm_required", False)), + "ui_only": bool(autopilot.get("ui_only", False)), + "blocked": bool(autopilot.get("blocked", False)), + "primary_action_code": str(target_entry.get("primary_action_code") or "").strip(), + "sequence_key": str(target_entry.get("sequence_key") or "").strip(), + "focus_ref": dict(target_entry.get("focus_ref") or {}), + "primary_focus_ref": dict(target_entry.get("primary_focus_ref") or {}), + "secondary_focus_ref": dict(target_entry.get("secondary_focus_ref") or {}), + "observation_only": bool(target_entry.get("observation_only", False)), + "activity_key": str(target_entry.get("activity_key") or ""), + "activity_kind": str(target_entry.get("activity_kind") or ""), + "ui_intent_kind": str(target_entry.get("ui_intent_kind") or ""), + "contract_navigation": dict(target_entry.get("contract_navigation") or {}), + }, + "preview_request": request_payload, +}, ensure_ascii=False)) +PY +} + +preview_codex_focus_entry() { + local mainland_base_url="$1" + local entry_key="$2" + local requested_by="$3" + local request_json + request_json="$(python3 - <<'PY' "${entry_key}" "${requested_by}" +import json +import sys +print(json.dumps({ + "entry_key": str(sys.argv[1] or "").strip(), + "requested_by": str(sys.argv[2] or "").strip() or "cli/ops-center/codex-focus-preview", + "source_label": "cli/codex-focus-preview", + "include_secondary": True, +}, ensure_ascii=False)) +PY +)" + + echo "[1/3] request" + print_json_or_raw "${request_json}" + echo + echo + + local raw_http_response + raw_http_response="$(request_with_status "POST" "${mainland_base_url}/api/v1/ops/codex-actions/resolve" "${request_json}")" + local http_status="${raw_http_response##*$'\n'HTTP_STATUS=}" + local response_json="${raw_http_response%$'\n'HTTP_STATUS=*}" + + echo "[2/3] raw response" + printf '%s\n' "http_status=${http_status}" + print_json_or_raw "${response_json}" + echo + echo + + echo "[3/3] condensed summary" + python3 - <<'PY' "${response_json}" "${http_status}" +import json +import sys + +payload = json.loads(str(sys.argv[1] or "{}")) +http_status = int(sys.argv[2] or 0) +data = payload.get("data") or {} +selected = data.get("selected_entry") or {} +preview = data.get("preview") or {} +primary = preview.get("primary") or {} +secondary = preview.get("secondary") or {} +gate = data.get("gate") or {} + +print(json.dumps({ + "http_status": http_status, + "message": str(payload.get("message") or ""), + "entry_key": str(data.get("entry_key") or ""), + "title": str(selected.get("title") or ""), + "executor_kind": str(preview.get("executor_kind") or selected.get("executor_kind") or ""), + "contract": { + "contract_key": str(preview.get("contract_key") or ""), + "contract_version": str(preview.get("contract_version") or ""), + "contract_primary_endpoint": str(preview.get("contract_primary_endpoint") or ""), + "preview_endpoint": str(preview.get("preview_endpoint") or ""), + "schema_doc_path": str(preview.get("contract_schema_doc_path") or ""), + }, + "execution_chain": str(preview.get("execution_chain") or ""), + "focus_ref": data.get("focus_ref") or {}, + "gate": { + "decision": str(gate.get("decision") or ""), + "decision_label": str(gate.get("decision_label") or ""), + "recommendation": str(gate.get("recommendation") or ""), + "recommendation_label": str(gate.get("recommendation_label") or ""), + "will_execute": bool(gate.get("will_execute", False)), + "confirm_required": bool(gate.get("confirm_required", False)), + "target_api": str(gate.get("target_api") or ""), + "focus_ref": gate.get("focus_ref") or {}, + }, + "selected_entry": { + "key": str(selected.get("key") or ""), + "contract_navigation": selected.get("contract_navigation") or {}, + "focus_ref": selected.get("focus_ref") or {}, + "primary_focus_ref": selected.get("primary_focus_ref") or {}, + "secondary_focus_ref": selected.get("secondary_focus_ref") or {}, + }, + "primary": { + "label": str(primary.get("label") or ""), + "action_code": str(primary.get("action_code") or ""), + "target_api": str(primary.get("target_api") or ""), + "recommendation_label": str(primary.get("recommendation_label") or ""), + "focus_ref": primary.get("focus_ref") or {}, + }, + "secondary": ( + { + "label": str(secondary.get("label") or ""), + "action_code": str(secondary.get("action_code") or ""), + "target_api": str(secondary.get("target_api") or ""), + "recommendation_label": str(secondary.get("recommendation_label") or ""), + "focus_ref": secondary.get("focus_ref") or {}, + } + if secondary else {} + ), +}, ensure_ascii=False, indent=2)) +PY + + if [[ "${http_status}" -lt 200 || "${http_status}" -ge 300 ]]; then + exit 1 + fi +} + +run_codex_focus_entry() { + local mainland_base_url="$1" + local entry_key="$2" + local confirm_flag="$3" + local requested_by="$4" + local confirm_risky + confirm_risky="$(confirm_to_bool "${confirm_flag}")" + + local request_json + request_json="$(python3 - <<'PY' "${entry_key}" "${requested_by}" "${confirm_risky}" +import json +import sys +print(json.dumps({ + "entry_key": str(sys.argv[1] or "").strip(), + "requested_by": str(sys.argv[2] or "").strip() or "cli/ops-center/codex-focus-run", + "source_label": "cli/codex-focus-run", + "include_secondary": True, + "confirm": str(sys.argv[3] or "").strip().lower() == "true", +}, ensure_ascii=False)) +PY +)" + + echo "[1/3] request" + print_json_or_raw "${request_json}" + echo + echo + + local execute_http_response + execute_http_response="$(request_with_status "POST" "${mainland_base_url}/api/v1/ops/codex-actions/execute" "${request_json}")" + local execute_http_status="${execute_http_response##*$'\n'HTTP_STATUS=}" + local execute_response_json="${execute_http_response%$'\n'HTTP_STATUS=*}" + + echo "[2/3] raw response" + printf '%s\n' "http_status=${execute_http_status}" + print_json_or_raw "${execute_response_json}" + echo + echo + + echo "[3/3] execution result" + python3 - <<'PY' "${execute_response_json}" "${execute_http_status}" +import json +import sys + +payload = json.loads(str(sys.argv[1] or "{}")) +http_status = int(sys.argv[2] or 0) +data = payload.get("data") or {} +gate = data.get("gate") or {} +execution_result = data.get("execution_result") or {} +print(json.dumps({ + "http_status": http_status, + "entry_key": str(data.get("entry_key") or ""), + "code": int((0 if payload.get("code", 1) in (0, "0", False) else payload.get("code", 1)) or 0), + "message": str(payload.get("message") or ""), + "executed": bool(data.get("executed", False)), + "focus_ref": data.get("focus_ref") or {}, + "decision": str(gate.get("decision") or ""), + "decision_label": str(gate.get("decision_label") or ""), + "recommendation": str(gate.get("recommendation") or ""), + "target_api": str(gate.get("target_api") or ""), + "target_method": str(gate.get("target_method") or ""), + "gate_focus_ref": gate.get("focus_ref") or {}, + "handled": bool(execution_result.get("handled", False)), + "mode": str(execution_result.get("mode") or ""), + "action_code": str(gate.get("action_code") or execution_result.get("action_code") or execution_result.get("driver_action_code") or ""), + "sequence_key": str(gate.get("sequence_key") or execution_result.get("sequence_key") or ""), +}, ensure_ascii=False, indent=2)) +PY + + if [[ "${execute_http_status}" -lt 200 || "${execute_http_status}" -ge 300 ]]; then + exit 1 + fi +} + +build_driver_focus_bundle() { + local driver_feed_response_json="$1" + local entry_key="$2" + local requested_by="$3" + local source_label="$4" + local secondary_flag="$5" + local confirm_flag="$6" + python3 - <<'PY' "${driver_feed_response_json}" "${entry_key}" "${requested_by}" "${source_label}" "${secondary_flag}" "${confirm_flag}" +import json +import sys + +payload = json.loads(str(sys.argv[1] or "{}")) +requested_entry_key = str(sys.argv[2] or "").strip() +requested_by = str(sys.argv[3] or "").strip() or "cli/ops-center/driver-focus-preview" +source_label = str(sys.argv[4] or "").strip() or "cli/driver-focus-preview" +secondary_flag = str(sys.argv[5] or "primary").strip().lower() +confirm_flag = str(sys.argv[6] or "").strip().lower() +data = payload.get("data") or {} +top_recommendation = dict(data.get("top_recommendation") or {}) +entries = [dict(item or {}) for item in list(data.get("entries") or []) if dict(item or {})] +activity_focus = [dict(item or {}) for item in list(data.get("activity_focus") or []) if dict(item or {})] + +secondary = secondary_flag in {"secondary", "second", "2", "true", "yes", "y"} +confirm = confirm_flag in {"1", "true", "yes", "y", "confirm", "confirmed"} + +def build_activity_focus_entry(item): + normalized = dict(item or {}) + focus_action_code = str(normalized.get("focus_action_code") or "").strip() + if not str(normalized.get("key") or "").strip() or not focus_action_code: + return {} + + focus_ref = dict(normalized.get("focus_ref") or {}) + target_node_codes = list(normalized.get("target_node_codes") or []) + focus_action_payload = dict(normalized.get("focus_action_payload") or {}) + summary = str(normalized.get("summary") or "").strip() + reason = summary or "该焦点属于运行态观察项，建议进入对应工作区查看上下文。" + return { + "key": str(normalized.get("key") or "").strip(), + "kind": "activity_focus", + "lane": "activity_focus", + "title": str(normalized.get("title") or "").strip(), + "summary": summary, + "reason": reason, + "meta_text": str(normalized.get("meta_text") or "").strip(), + "status": str(normalized.get("status") or "").strip(), + "executor_kind": "driver_action", + "sequence_key": "", + "primary_label": "进入工作区", + "secondary_label": "", + "primary_action_code": focus_action_code, + "secondary_action_code": "", + "node_codes": target_node_codes, + "primary_node_codes": target_node_codes, + "secondary_node_codes": [], + "focus_ref": focus_ref, + "primary_focus_ref": focus_ref, + "secondary_focus_ref": {}, + "primary_action_payload": focus_action_payload, + "secondary_action_payload": {}, + "observation_only": True, + "activity_key": str(normalized.get("activity_key") or "").strip(), + "activity_kind": str(normalized.get("activity_kind") or "").strip(), + "ui_intent_kind": str(normalized.get("ui_intent_kind") or "").strip(), + "contract_navigation": dict(normalized.get("contract_navigation") or {}), + } + +activity_focus_entries = [entry for entry in (build_activity_focus_entry(item) for item in activity_focus) if entry] + +target_entry = {} +target_entry_key = requested_entry_key +if requested_entry_key: + target_entry = next((item for item in entries if str(item.get("key") or "").strip() == requested_entry_key), {}) +if not target_entry and requested_entry_key: + target_entry = next((item for item in activity_focus_entries if str(item.get("key") or "").strip() == requested_entry_key), {}) +if not target_entry and requested_entry_key and str(top_recommendation.get("key") or "").strip() == requested_entry_key: + target_entry = dict(top_recommendation or {}) +if not target_entry: + target_entry = dict(top_recommendation or {}) + target_entry_key = str(target_entry.get("key") or "").strip() +if not target_entry and entries: + target_entry = dict(entries[0] or {}) + target_entry_key = str(target_entry.get("key") or "").strip() +if not target_entry and activity_focus_entries: + target_entry = dict(activity_focus_entries[0] or {}) + target_entry_key = str(target_entry.get("key") or "").strip() + +if not target_entry: + raise SystemExit("driver feed 当前没有可执行的焦点动作") + +request_payload = { + "source_label": source_label, + "title": str(target_entry.get("title") or "").strip() or target_entry_key or "driver-focus", + "summary": str(target_entry.get("summary") or "").strip(), + "reason": str(target_entry.get("reason") or "").strip(), + "meta_text": str(target_entry.get("meta_text") or "").strip(), + "executor_kind": str(target_entry.get("executor_kind") or "driver_action").strip() or "driver_action", + "sequence_key": str(target_entry.get("sequence_key") or "").strip(), + "focus_ref": dict(target_entry.get("focus_ref") or {}), + "primary_focus_ref": dict(target_entry.get("primary_focus_ref") or {}), + "secondary_focus_ref": dict(target_entry.get("secondary_focus_ref") or {}), + "primary_label": str(target_entry.get("primary_label") or "").strip(), + "secondary_label": str(target_entry.get("secondary_label") or "").strip(), + "primary_action_code": str(target_entry.get("primary_action_code") or "").strip(), + "secondary_action_code": str(target_entry.get("secondary_action_code") or "").strip(), + "primary_node_codes": list(target_entry.get("primary_node_codes") or target_entry.get("node_codes") or []), + "secondary_node_codes": list(target_entry.get("secondary_node_codes") or target_entry.get("node_codes") or []), + "primary_action_payload": dict(target_entry.get("primary_action_payload") or {}), + "secondary_action_payload": dict(target_entry.get("secondary_action_payload") or {}), + "include_secondary": True, + "requested_by": requested_by, + "secondary": secondary, + "confirm": confirm, +} + +print(json.dumps({ + "selected_entry": { + "key": target_entry_key, + "kind": str(target_entry.get("kind") or "").strip(), + "lane": str(target_entry.get("lane") or "").strip(), + "title": str(target_entry.get("title") or "").strip(), + "status": str(target_entry.get("status") or "").strip(), + "executor_kind": str(target_entry.get("executor_kind") or "").strip(), + "sequence_key": str(target_entry.get("sequence_key") or "").strip(), + "primary_action_code": str(target_entry.get("primary_action_code") or "").strip(), + "secondary_action_code": str(target_entry.get("secondary_action_code") or "").strip(), + "focus_ref": dict(target_entry.get("focus_ref") or {}), + "primary_focus_ref": dict(target_entry.get("primary_focus_ref") or {}), + "secondary_focus_ref": dict(target_entry.get("secondary_focus_ref") or {}), + "observation_only": bool(target_entry.get("observation_only", False)), + "activity_key": str(target_entry.get("activity_key") or ""), + "activity_kind": str(target_entry.get("activity_kind") or ""), + "ui_intent_kind": str(target_entry.get("ui_intent_kind") or ""), + "contract_navigation": dict(target_entry.get("contract_navigation") or {}), + }, + "request": request_payload, +}, ensure_ascii=False)) +PY +} + +preview_driver_focus_entry() { + local mainland_base_url="$1" + local entry_key="$2" + local requested_by="$3" + + local raw_driver_feed_response + raw_driver_feed_response="$(request_with_status "GET" "${mainland_base_url}/api/v1/ops/driver-feed")" + local driver_feed_http_status="${raw_driver_feed_response##*$'\n'HTTP_STATUS=}" + local driver_feed_response_json="${raw_driver_feed_response%$'\n'HTTP_STATUS=*}" + + if [[ "${driver_feed_http_status}" -lt 200 || "${driver_feed_http_status}" -ge 300 ]]; then + echo "[1/3] driver feed raw response" + printf '%s\n' "http_status=${driver_feed_http_status}" + print_json_or_raw "${driver_feed_response_json}" + exit 1 + fi + + local bundle_json + bundle_json="$(build_driver_focus_bundle "${driver_feed_response_json}" "${entry_key}" "${requested_by}" "cli/driver-focus-preview" "primary" "false")" + local selected_entry_json + selected_entry_json="$(python3 - <<'PY' "${bundle_json}" +import json +import sys +payload = json.loads(str(sys.argv[1] or "{}")) +print(json.dumps(payload.get("selected_entry") or {}, ensure_ascii=False)) +PY +)" + local request_json + request_json="$(python3 - <<'PY' "${bundle_json}" +import json +import sys +payload = json.loads(str(sys.argv[1] or "{}")) +print(json.dumps(payload.get("request") or {}, ensure_ascii=False)) +PY +)" + + echo "[1/4] selected focus" + print_json_or_raw "${selected_entry_json}" + echo + echo + + echo "[2/4] request" + print_json_or_raw "${request_json}" + echo + echo + + local raw_http_response + raw_http_response="$(request_with_status "POST" "${mainland_base_url}/api/v1/ops/driver-actions/resolve" "${request_json}")" + local http_status="${raw_http_response##*$'\n'HTTP_STATUS=}" + local response_json="${raw_http_response%$'\n'HTTP_STATUS=*}" + + echo "[3/4] raw response" + printf '%s\n' "http_status=${http_status}" + print_json_or_raw "${response_json}" + echo + echo + + echo "[4/4] condensed summary" + python3 - <<'PY' "${response_json}" "${http_status}" "${selected_entry_json}" +import json +import sys + +payload = json.loads(str(sys.argv[1] or "{}")) +http_status = int(sys.argv[2] or 0) +selected = json.loads(str(sys.argv[3] or "{}")) +data = payload.get("data") or {} +preview = data.get("preview") or {} +primary = preview.get("primary") or {} +secondary = preview.get("secondary") or {} +gate = data.get("gate") or {} + +print(json.dumps({ + "http_status": http_status, + "message": str(payload.get("message") or ""), + "selected_entry": selected, + "selected_section": str(data.get("selected_section") or ""), + "contract": { + "contract_key": str(preview.get("contract_key") or ""), + "contract_version": str(preview.get("contract_version") or ""), + "contract_primary_endpoint": str(preview.get("contract_primary_endpoint") or ""), + "preview_endpoint": str(preview.get("preview_endpoint") or ""), + "schema_doc_path": str(preview.get("contract_schema_doc_path") or ""), + }, + "execution_chain": str(preview.get("execution_chain") or ""), + "focus_ref": data.get("focus_ref") or {}, + "gate": { + "decision": str(gate.get("decision") or ""), + "decision_label": str(gate.get("decision_label") or ""), + "recommendation": str(gate.get("recommendation") or ""), + "recommendation_label": str(gate.get("recommendation_label") or ""), + "will_execute": bool(gate.get("will_execute", False)), + "confirm_required": bool(gate.get("confirm_required", False)), + "target_api": str(gate.get("target_api") or ""), + "focus_ref": gate.get("focus_ref") or {}, + }, + "primary": { + "label": str(primary.get("label") or ""), + "action_code": str(primary.get("action_code") or ""), + "target_api": str(primary.get("target_api") or ""), + "recommendation_label": str(primary.get("recommendation_label") or ""), + "focus_ref": primary.get("focus_ref") or {}, + "request_payload": primary.get("request_payload") or {}, + }, + "secondary": ( + { + "label": str(secondary.get("label") or ""), + "action_code": str(secondary.get("action_code") or ""), + "target_api": str(secondary.get("target_api") or ""), + "recommendation_label": str(secondary.get("recommendation_label") or ""), + "focus_ref": secondary.get("focus_ref") or {}, + "request_payload": secondary.get("request_payload") or {}, + } + if secondary else {} + ), +}, ensure_ascii=False, indent=2)) +PY + + if [[ "${http_status}" -lt 200 || "${http_status}" -ge 300 ]]; then + exit 1 + fi +} + +run_driver_focus_entry() { + local mainland_base_url="$1" + local entry_key="$2" + local secondary_flag="$3" + local confirm_flag="$4" + local requested_by="$5" + + local raw_driver_feed_response + raw_driver_feed_response="$(request_with_status "GET" "${mainland_base_url}/api/v1/ops/driver-feed")" + local driver_feed_http_status="${raw_driver_feed_response##*$'\n'HTTP_STATUS=}" + local driver_feed_response_json="${raw_driver_feed_response%$'\n'HTTP_STATUS=*}" + + if [[ "${driver_feed_http_status}" -lt 200 || "${driver_feed_http_status}" -ge 300 ]]; then + echo "[1/3] driver feed raw response" + printf '%s\n' "http_status=${driver_feed_http_status}" + print_json_or_raw "${driver_feed_response_json}" + exit 1 + fi + + local bundle_json + bundle_json="$(build_driver_focus_bundle "${driver_feed_response_json}" "${entry_key}" "${requested_by}" "cli/driver-focus-run" "${secondary_flag}" "${confirm_flag}")" + local selected_entry_json + selected_entry_json="$(python3 - <<'PY' "${bundle_json}" +import json +import sys +payload = json.loads(str(sys.argv[1] or "{}")) +print(json.dumps(payload.get("selected_entry") or {}, ensure_ascii=False)) +PY +)" + local request_json + request_json="$(python3 - <<'PY' "${bundle_json}" +import json +import sys +payload = json.loads(str(sys.argv[1] or "{}")) +print(json.dumps(payload.get("request") or {}, ensure_ascii=False)) +PY +)" + + echo "[1/4] selected focus" + print_json_or_raw "${selected_entry_json}" + echo + echo + + echo "[2/4] request" + print_json_or_raw "${request_json}" + echo + echo + + local raw_http_response + raw_http_response="$(request_with_status "POST" "${mainland_base_url}/api/v1/ops/driver-actions/execute-resolved" "${request_json}")" + local http_status="${raw_http_response##*$'\n'HTTP_STATUS=}" + local response_json="${raw_http_response%$'\n'HTTP_STATUS=*}" + + echo "[3/4] raw response" + printf '%s\n' "http_status=${http_status}" + print_json_or_raw "${response_json}" + echo + echo + + echo "[4/4] execution result" + python3 - <<'PY' "${response_json}" "${http_status}" "${selected_entry_json}" +import json +import sys + +payload = json.loads(str(sys.argv[1] or "{}")) +http_status = int(sys.argv[2] or 0) +selected = json.loads(str(sys.argv[3] or "{}")) +data = payload.get("data") or {} +gate = data.get("gate") or {} +execution_result = data.get("execution_result") or {} + +raw_code = payload.get("code", 1) +if raw_code in (0, "0", False): + normalized_code = 0 +else: + normalized_code = int(raw_code or 1) + +print(json.dumps({ + "http_status": http_status, + "code": normalized_code, + "message": str(payload.get("message") or ""), + "selected_entry": selected, + "executed": bool(data.get("executed", False)), + "selected_section": str(data.get("selected_section") or ""), + "focus_ref": data.get("focus_ref") or {}, + "decision": str(gate.get("decision") or ""), + "decision_label": str(gate.get("decision_label") or ""), + "recommendation": str(gate.get("recommendation") or ""), + "target_api": str(gate.get("target_api") or ""), + "target_method": str(gate.get("target_method") or ""), + "gate_focus_ref": gate.get("focus_ref") or {}, + "action_code": str(gate.get("action_code") or execution_result.get("action_code") or execution_result.get("driver_action_code") or ""), + "sequence_key": str(gate.get("sequence_key") or execution_result.get("sequence_key") or ""), + "handled": bool(execution_result.get("handled", False)), + "mode": str(execution_result.get("mode") or ""), +}, ensure_ascii=False, indent=2)) +PY + + if [[ "${http_status}" -lt 200 || "${http_status}" -ge 300 ]]; then + exit 1 + fi +} + +build_stack_next_bundle() { + local stack_diagnosis_response_json="$1" + local requested_by="$2" + local source_label="$3" + local confirm_flag="$4" + python3 - <<'PY' "${stack_diagnosis_response_json}" "${requested_by}" "${source_label}" "${confirm_flag}" +import json +import sys + +payload = json.loads(str(sys.argv[1] or "{}")) +requested_by = str(sys.argv[2] or "").strip() or "cli/ops-center/stack-next" +source_label = str(sys.argv[3] or "").strip() or "cli/stack-next" +confirm_flag = str(sys.argv[4] or "").strip().lower() + +data = payload.get("data") or {} +diagnosis = dict(data.get("diagnosis") or {}) +operator_decision = dict(diagnosis.get("operator_decision") or {}) +next_actions = dict(diagnosis.get("next_actions") or {}) +recommended_commands = dict(diagnosis.get("recommended_commands") or {}) +next_step = dict(diagnosis.get("next_step") or {}) +recommended_actions = [dict(item or {}) for item in list(diagnosis.get("recommended_actions") or []) if dict(item or {})] +recommended_action_codes = { + str(item.get("action_code") or "").strip() + for item in recommended_actions + if str(item.get("action_code") or "").strip() +} +overview = dict(data.get("overview") or {}) +managed_nodes = dict(data.get("managed_nodes") or {}) +release_hub = dict(data.get("release_hub") or {}) +issues = [dict(item or {}) for item in list(diagnosis.get("issues") or []) if dict(item or {})] + +def normalize_node_codes(value): + result = [] + seen = set() + if not isinstance(value, list): + return result + for item in value: + if isinstance(item, dict): + node_code = str(item.get("node_code") or "").strip() + else: + node_code = str(item or "").strip() + if not node_code or node_code in seen: + continue + seen.add(node_code) + result.append(node_code) + return result + + +def scene_node_log_command(node_code, limit=120, mode="key"): + node_code = str(node_code or "").strip() + if not node_code: + return "" + return ( + "bash domain-api/deploy/multi-region/drive_ops_center.sh " + f"scene-node-log http://127.0.0.1:8100 {node_code} {int(limit or 120)} {str(mode or 'key').strip() or 'key'}" + ) + + +if not operator_decision or not next_actions or not recommended_commands: + log_sync = dict((overview.get("log_sync") or {})) + problem_nodes = [dict(item or {}) for item in list(managed_nodes.get("problem_nodes") or []) if dict(item or {})] + launchpad_focus_ref = dict(release_hub.get("focus_ref") or release_hub.get("default_rollout_gate_focus_ref") or {}) + default_rollout_gate_status = str(release_hub.get("default_rollout_gate_status") or "").strip() + first_problem_node_code = str((problem_nodes[0] if problem_nodes else {}).get("node_code") or "").strip() + missing_sample_node_codes = normalize_node_codes(log_sync.get("missing_sample_node_codes") or []) + scene_log_target_node_code = missing_sample_node_codes[0] if missing_sample_node_codes else first_problem_node_code + + synthesized_recommended_commands = { + "api-restart": "bash domain-api/deploy/multi-region/drive_ops_center.sh runtime-refresh-recover http://127.0.0.1:8100", + "stack-diagnosis": "bash domain-api/deploy/multi-region/drive_ops_center.sh stack-diagnosis http://127.0.0.1:8100 summary", + "stack-check": "bash domain-api/deploy/multi-region/drive_ops_center.sh stack-diagnosis http://127.0.0.1:8100 summary", + "managed-node-bootstrap-plan": ( + "bash domain-api/deploy/multi-region/drive_ops_center.sh " + f"node-bootstrap-plan http://127.0.0.1:8100 {first_problem_node_code}" + if first_problem_node_code + else "" + ), + "scene_log": scene_node_log_command(scene_log_target_node_code, 120, "key"), + "driver_focus_preview": "bash domain-api/deploy/multi-region/drive_ops_center.sh driver-focus-preview http://127.0.0.1:8100", + "doctor": "bash domain-api/deploy/multi-region/drive_ops_center.sh doctor http://127.0.0.1:8100", + "node_handover": ( + f"bash domain-api/deploy/multi-region/drive_ops_center.sh node-handover http://127.0.0.1:8100 {first_problem_node_code}" + if first_problem_node_code + else "" + ), + "release_launchpad": "bash domain-api/deploy/multi-region/drive_ops_center.sh release-launchpad http://127.0.0.1:8100", + } + synthesized_operator_decision = {} + if any(str(item.get("code") or "").strip() in {"remote_log_sync_disabled", "remote_log_sync_waiting_sample"} for item in issues): + synthesized_operator_decision = { + "lane": "observability", + "priority": "high", + "reason_code": "scene_log_attention", + "title": "先看执行现场日志", + "summary": "当前总检优先建议先补现场可见性，再决定是否继续接管或推进发布。", + "next_focus": { + "kind": "node_scene_log", + "node_code": scene_log_target_node_code, + "mode": "key", + "limit": 120, + } if scene_log_target_node_code else {}, + "primary_command_key": "scene_log", + "secondary_command_key": "driver_focus_preview", + } + elif any(str(item.get("code") or "").strip() == "runtime_build_schema_stale" for item in issues): + synthesized_operator_decision = { + "lane": "runtime_recovery", + "priority": "high", + "reason_code": "runtime_build_schema_stale", + "title": "先重启控制面 API，让运行时切到最新 schema", + "summary": "当前属于仓库代码已更新，但运行中的 API 仍旧停留在旧 schema 的场景。先重启 domaincheck-api，再重新看 stack-diagnosis 与 node-bootstrap-plan。", + "next_focus": { + "kind": "runtime_build_info", + "node_code": "", + }, + "primary_command_key": "api-restart", + "secondary_command_key": "managed-node-bootstrap-plan" if first_problem_node_code else "stack-diagnosis", + } + elif any(str(item.get("code") or "").strip() == "managed_nodes_agent_pending" for item in issues): + synthesized_operator_decision = { + "lane": "node_handover", + "priority": "high", + "reason_code": "managed_nodes_agent_pending", + "title": "先补接管缺口", + "summary": "当前真正阻断是 remote-agent / 接管未就绪，应先把节点接入链路打通。", + "next_focus": {}, + "primary_command_key": "node_handover", + "secondary_command_key": "doctor", + } + elif default_rollout_gate_status == "blocked": + synthesized_operator_decision = { + "lane": "release", + "priority": "medium", + "reason_code": "release_rollout_gate_blocked", + "title": "先看发布门禁", + "summary": str(release_hub.get("default_rollout_gate_summary") or "").strip(), + "next_focus": launchpad_focus_ref, + "primary_command_key": "release_launchpad", + "secondary_command_key": "doctor", + } + + if synthesized_operator_decision: + operator_decision = synthesized_operator_decision + if synthesized_recommended_commands: + merged_recommended_commands = dict(synthesized_recommended_commands) + merged_recommended_commands.update({ + str(key or "").strip(): str(value or "").strip() + for key, value in recommended_commands.items() + if str(key or "").strip() and str(value or "").strip() + }) + recommended_commands = merged_recommended_commands + if operator_decision: + primary_key = str(operator_decision.get("primary_command_key") or "").strip() + secondary_key = str(operator_decision.get("secondary_command_key") or "").strip() + next_actions = { + "primary_command_key": primary_key, + "primary_command": str(recommended_commands.get(primary_key) or "").strip(), + "secondary_command_key": secondary_key, + "secondary_command": str(recommended_commands.get(secondary_key) or "").strip(), + } + +primary_command_key = str(operator_decision.get("primary_command_key") or "").strip() +primary_command = str( + next_actions.get("primary_command") + or recommended_commands.get(primary_command_key) + or "" +).strip() +operator_focus_ref = dict(operator_decision.get("next_focus") or {}) +action_code = "" +selection_mode = "driver_action" + +if primary_command_key and primary_command_key in recommended_action_codes: + action_code = primary_command_key +elif primary_command_key and primary_command: + selection_mode = "command_only" +elif str(next_step.get("action_code") or "").strip(): + action_code = str(next_step.get("action_code") or "").strip() +elif primary_command: + selection_mode = "command_only" +else: + raise SystemExit("stack diagnosis 当前既没有可执行 driver action，也没有可下钻命令") + +matched_action = next( + (item for item in recommended_actions if str(item.get("action_code") or "").strip() == action_code), + {}, +) +focus_ref = dict(operator_focus_ref or next_step.get("focus_ref") or matched_action.get("focus_ref") or {}) +confirm = confirm_flag in {"1", "true", "yes", "y", "confirm", "confirmed"} + +request_payload = { + "source_label": source_label, + "title": str(matched_action.get("title") or action_code).strip() or action_code, + "summary": str(next_step.get("reason") or matched_action.get("reason") or "").strip(), + "reason": str(next_step.get("reason") or matched_action.get("reason") or "").strip(), + "meta_text": str(next_step.get("source") or matched_action.get("source") or "stack-diagnosis").strip(), + "executor_kind": "driver_action", + "focus_ref": focus_ref, + "primary_focus_ref": focus_ref, + "primary_label": str(matched_action.get("title") or action_code).strip() or action_code, + "primary_action_code": action_code, + "primary_action_payload": {}, + "include_secondary": True, + "requested_by": requested_by, + "confirm": confirm, +} + +if selection_mode == "command_only": + print(json.dumps({ + "http_status": 200, + "mode": selection_mode, + "message": "当前主决策优先指向命令型下钻动作，不进入 driver-actions/resolve。", + "contract": { + "contract_key": str((diagnosis.get("contract_navigation") or {}).get("primary_contract_key") or "ops_stack_diagnosis_contract").strip(), + "contract_version": str(diagnosis.get("contract_version") or "").strip(), + "contract_primary_endpoint": "/api/v1/ops/stack-diagnosis", + "preview_endpoint": "", + "schema_doc_path": "docs/schemas/ops_stack_diagnosis_contract.md", + }, + "selected_entry": { + "action_code": primary_command_key, + "source": str(operator_decision.get("reason_code") or next_step.get("source") or "stack-diagnosis").strip(), + "title": str(operator_decision.get("title") or primary_command_key or "stack-next").strip(), + "reason": str(operator_decision.get("summary") or next_step.get("reason") or "").strip(), + "command": primary_command, + "focus_ref": focus_ref, + "contract_navigation": dict( + diagnosis.get("contract_navigation") + or next_step.get("contract_navigation") + or {} + ), + }, + "gate": { + "decision": "command_only", + "decision_label": "命令下钻", + "recommendation": "open_ui", + "recommendation_label": "先下钻", + "will_execute": False, + "confirm_required": False, + "target_api": "", + "focus_ref": focus_ref, + }, + "execution_chain": "stack-diagnosis -> command-only guidance", + "focus_ref": focus_ref, + "request": {}, + }, ensure_ascii=False)) + raise SystemExit(0) + +print(json.dumps({ + "mode": selection_mode, + "selected_entry": { + "action_code": action_code, + "source": str(next_step.get("source") or matched_action.get("source") or "stack-diagnosis").strip(), + "title": str(matched_action.get("title") or action_code).strip() or action_code, + "reason": str(next_step.get("reason") or matched_action.get("reason") or "").strip(), + "command": str(next_step.get("command") or matched_action.get("command") or "").strip(), + "focus_ref": focus_ref, + "contract_navigation": dict(next_step.get("contract_navigation") or matched_action.get("contract_navigation") or {}), + }, + "request": request_payload, +}, ensure_ascii=False)) +PY +} + +print_stack_diagnosis_failure_and_hint() { + local mainland_base_url="$1" + local stack_http_status="$2" + local stack_response_json="$3" + + echo "[1/3] stack diagnosis raw response" + printf '%s\n' "http_status=${stack_http_status}" + print_json_or_raw "${stack_response_json}" + echo + echo + + echo "[2/3] compatibility hint" + python3 - <<'PY' "${mainland_base_url}" "${stack_http_status}" "${stack_response_json}" +import json +import sys + +base_url = str(sys.argv[1] or "").strip() +http_status = int(sys.argv[2] or 0) +raw_text = str(sys.argv[3] or "") + +detail = "" +try: + payload = json.loads(raw_text) +except Exception: + payload = {} +else: + detail = str(payload.get("detail") or payload.get("message") or "").strip() + +message = "stack-diagnosis 请求失败，请先检查控制面 API 连通性。" +commands = [ + f"curl -sS {base_url}/api/v1/ops/stack-diagnosis", +] + +if http_status == 404: + message = "当前控制面 API 还未暴露 /api/v1/ops/stack-diagnosis，通常表示运行中的后端版本还没升级或升级后未重启。" + commands = [ + "git status", + f"bash domain-api/deploy/multi-region/drive_ops_center.sh runtime-refresh-recover {base_url}", + f"curl -sS {base_url}/api/v1/ops/stack-diagnosis", + ] + +print(json.dumps({ + "http_status": http_status, + "detail": detail, + "message": message, + "recommended_commands": commands, +}, ensure_ascii=False, indent=2)) +PY + echo + echo + + echo "[3/3] next action" + if [[ "${stack_http_status}" == "404" ]]; then + printf '%s\n' "先把控制面后端升级到包含 /api/v1/ops/stack-diagnosis 的版本，并重启 domaincheck-api；随后再重试 stack-next。" + else + printf '%s\n' "先修复控制面 API / 网络问题，再重试 stack-next。" + fi +} + +preview_stack_next_entry() { + local mainland_base_url="$1" + local requested_by="$2" + + local raw_stack_response + raw_stack_response="$(request_with_status "GET" "${mainland_base_url}/api/v1/ops/stack-diagnosis?base_url=${mainland_base_url}")" + local stack_http_status="${raw_stack_response##*$'\n'HTTP_STATUS=}" + local stack_response_json="${raw_stack_response%$'\n'HTTP_STATUS=*}" + + if [[ "${stack_http_status}" -lt 200 || "${stack_http_status}" -ge 300 ]]; then + print_stack_diagnosis_failure_and_hint "${mainland_base_url}" "${stack_http_status}" "${stack_response_json}" + exit 1 + fi + + local bundle_json + bundle_json="$(build_stack_next_bundle "${stack_response_json}" "${requested_by}" "cli/stack-next-preview" "false")" + local bundle_mode + bundle_mode="$(python3 - <<'PY' "${bundle_json}" +import json +import sys +payload = json.loads(str(sys.argv[1] or "{}")) +print(str(payload.get("mode") or "driver_action").strip()) +PY +)" + local selected_entry_json + selected_entry_json="$(python3 - <<'PY' "${bundle_json}" +import json +import sys +payload = json.loads(str(sys.argv[1] or "{}")) +print(json.dumps(payload.get("selected_entry") or {}, ensure_ascii=False)) +PY +)" + if [[ "${bundle_mode}" == "command_only" ]]; then + echo "[1/2] selected next step" + print_json_or_raw "${selected_entry_json}" + echo + echo + echo "[2/2] command-only guidance" + python3 - <<'PY' "${selected_entry_json}" +import json +import sys + +selected = json.loads(str(sys.argv[1] or "{}")) +print(json.dumps({ + "http_status": 200, + "mode": "command_only", + "message": "当前主决策优先指向命令型下钻动作，不进入 driver-actions/resolve。", + "contract": { + "contract_key": str((selected.get("contract_navigation") or {}).get("primary_contract_key") or "ops_stack_diagnosis_contract"), + "contract_version": "", + "contract_primary_endpoint": "/api/v1/ops/stack-diagnosis", + "preview_endpoint": "", + "schema_doc_path": "docs/schemas/ops_stack_diagnosis_contract.md", + }, + "selected_entry": selected, + "gate": { + "decision": "command_only", + "decision_label": "命令下钻", + "recommendation": "open_ui", + "recommendation_label": "先下钻", + "will_execute": False, + "confirm_required": False, + "target_api": "", + "focus_ref": selected.get("focus_ref") or {}, + }, + "execution_chain": "stack-diagnosis -> command-only guidance", + "focus_ref": selected.get("focus_ref") or {}, + "recommended_next_command": str(selected.get("command") or ""), +}, ensure_ascii=False, indent=2)) +PY + return 0 + fi + local request_json + request_json="$(python3 - <<'PY' "${bundle_json}" +import json +import sys +payload = json.loads(str(sys.argv[1] or "{}")) +print(json.dumps(payload.get("request") or {}, ensure_ascii=False)) +PY +)" + + echo "[1/4] selected next step" + print_json_or_raw "${selected_entry_json}" + echo + echo + + echo "[2/4] request" + print_json_or_raw "${request_json}" + echo + echo + + local raw_http_response + raw_http_response="$(request_with_status "POST" "${mainland_base_url}/api/v1/ops/driver-actions/resolve" "${request_json}")" + local http_status="${raw_http_response##*$'\n'HTTP_STATUS=}" + local response_json="${raw_http_response%$'\n'HTTP_STATUS=*}" + + echo "[3/4] raw response" + printf '%s\n' "http_status=${http_status}" + print_json_or_raw "${response_json}" + echo + echo + + echo "[4/4] condensed summary" + python3 - <<'PY' "${response_json}" "${http_status}" "${selected_entry_json}" +import json +import sys + +payload = json.loads(str(sys.argv[1] or "{}")) +http_status = int(sys.argv[2] or 0) +selected = json.loads(str(sys.argv[3] or "{}")) +data = payload.get("data") or {} +preview = data.get("preview") or {} +primary = preview.get("primary") or {} +secondary = preview.get("secondary") or {} +gate = data.get("gate") or {} + +print(json.dumps({ + "http_status": http_status, + "message": str(payload.get("message") or ""), + "selected_entry": selected, + "selected_section": str(data.get("selected_section") or ""), + "contract": { + "contract_key": str(preview.get("contract_key") or ""), + "contract_version": str(preview.get("contract_version") or ""), + "contract_primary_endpoint": str(preview.get("contract_primary_endpoint") or ""), + "preview_endpoint": str(preview.get("preview_endpoint") or ""), + "schema_doc_path": str(preview.get("contract_schema_doc_path") or ""), + }, + "execution_chain": str(preview.get("execution_chain") or ""), + "focus_ref": data.get("focus_ref") or {}, + "gate": { + "decision": str(gate.get("decision") or ""), + "decision_label": str(gate.get("decision_label") or ""), + "recommendation": str(gate.get("recommendation") or ""), + "recommendation_label": str(gate.get("recommendation_label") or ""), + "will_execute": bool(gate.get("will_execute", False)), + "confirm_required": bool(gate.get("confirm_required", False)), + "target_api": str(gate.get("target_api") or ""), + "focus_ref": gate.get("focus_ref") or {}, + }, + "primary": { + "label": str(primary.get("label") or ""), + "action_code": str(primary.get("action_code") or ""), + "target_api": str(primary.get("target_api") or ""), + "recommendation_label": str(primary.get("recommendation_label") or ""), + "focus_ref": primary.get("focus_ref") or {}, + "request_payload": primary.get("request_payload") or {}, + }, + "secondary": ( + { + "label": str(secondary.get("label") or ""), + "action_code": str(secondary.get("action_code") or ""), + "target_api": str(secondary.get("target_api") or ""), + "recommendation_label": str(secondary.get("recommendation_label") or ""), + "focus_ref": secondary.get("focus_ref") or {}, + "request_payload": secondary.get("request_payload") or {}, + } + if secondary else {} + ), +}, ensure_ascii=False, indent=2)) +PY + + if [[ "${http_status}" -lt 200 || "${http_status}" -ge 300 ]]; then + exit 1 + fi +} + +run_stack_next_entry() { + local mainland_base_url="$1" + local confirm_flag="$2" + local requested_by="$3" + + local raw_stack_response + raw_stack_response="$(request_with_status "GET" "${mainland_base_url}/api/v1/ops/stack-diagnosis?base_url=${mainland_base_url}")" + local stack_http_status="${raw_stack_response##*$'\n'HTTP_STATUS=}" + local stack_response_json="${raw_stack_response%$'\n'HTTP_STATUS=*}" + + if [[ "${stack_http_status}" -lt 200 || "${stack_http_status}" -ge 300 ]]; then + print_stack_diagnosis_failure_and_hint "${mainland_base_url}" "${stack_http_status}" "${stack_response_json}" + exit 1 + fi + + local bundle_json + bundle_json="$(build_stack_next_bundle "${stack_response_json}" "${requested_by}" "cli/stack-next-run" "${confirm_flag}")" + local bundle_mode + bundle_mode="$(python3 - <<'PY' "${bundle_json}" +import json +import sys +payload = json.loads(str(sys.argv[1] or "{}")) +print(str(payload.get("mode") or "driver_action").strip()) +PY +)" + local selected_entry_json + selected_entry_json="$(python3 - <<'PY' "${bundle_json}" +import json +import sys +payload = json.loads(str(sys.argv[1] or "{}")) +print(json.dumps(payload.get("selected_entry") or {}, ensure_ascii=False)) +PY +)" + if [[ "${bundle_mode}" == "command_only" ]]; then + echo "[1/2] selected next step" + print_json_or_raw "${selected_entry_json}" + echo + echo + echo "[2/2] command-only guidance" + python3 - <<'PY' "${selected_entry_json}" +import json +import sys + +selected = json.loads(str(sys.argv[1] or "{}")) +print(json.dumps({ + "http_status": 200, + "mode": "command_only", + "executed": False, + "message": "当前主决策属于命令型下钻动作，stack-next run 不会代替你执行本地 shell。", + "contract": { + "contract_key": str((selected.get("contract_navigation") or {}).get("primary_contract_key") or "ops_stack_diagnosis_contract"), + "contract_version": "", + "contract_primary_endpoint": "/api/v1/ops/stack-diagnosis", + "preview_endpoint": "", + "schema_doc_path": "docs/schemas/ops_stack_diagnosis_contract.md", + }, + "selected_entry": selected, + "gate": { + "decision": "command_only", + "decision_label": "命令下钻", + "recommendation": "open_ui", + "recommendation_label": "先下钻", + "will_execute": False, + "confirm_required": False, + "target_api": "", + "focus_ref": selected.get("focus_ref") or {}, + }, + "execution_chain": "stack-diagnosis -> command-only guidance", + "focus_ref": selected.get("focus_ref") or {}, + "recommended_next_command": str(selected.get("command") or ""), +}, ensure_ascii=False, indent=2)) +PY + return 0 + fi + local request_json + request_json="$(python3 - <<'PY' "${bundle_json}" +import json +import sys +payload = json.loads(str(sys.argv[1] or "{}")) +print(json.dumps(payload.get("request") or {}, ensure_ascii=False)) +PY +)" + + echo "[1/4] selected next step" + print_json_or_raw "${selected_entry_json}" + echo + echo + + echo "[2/4] request" + print_json_or_raw "${request_json}" + echo + echo + + local raw_http_response + raw_http_response="$(request_with_status "POST" "${mainland_base_url}/api/v1/ops/driver-actions/execute-resolved" "${request_json}")" + local http_status="${raw_http_response##*$'\n'HTTP_STATUS=}" + local response_json="${raw_http_response%$'\n'HTTP_STATUS=*}" + + echo "[3/4] raw response" + printf '%s\n' "http_status=${http_status}" + print_json_or_raw "${response_json}" + echo + echo + + echo "[4/4] execution result" + python3 - <<'PY' "${response_json}" "${http_status}" "${selected_entry_json}" +import json +import sys + +payload = json.loads(str(sys.argv[1] or "{}")) +http_status = int(sys.argv[2] or 0) +selected = json.loads(str(sys.argv[3] or "{}")) +data = payload.get("data") or {} +gate = data.get("gate") or {} +execution_result = data.get("execution_result") or {} + +raw_code = payload.get("code", 1) +if raw_code in (0, "0", False): + normalized_code = 0 +else: + normalized_code = int(raw_code or 1) + +print(json.dumps({ + "http_status": http_status, + "code": normalized_code, + "message": str(payload.get("message") or ""), + "selected_entry": selected, + "executed": bool(data.get("executed", False)), + "selected_section": str(data.get("selected_section") or ""), + "focus_ref": data.get("focus_ref") or {}, + "decision": str(gate.get("decision") or ""), + "decision_label": str(gate.get("decision_label") or ""), + "recommendation": str(gate.get("recommendation") or ""), + "target_api": str(gate.get("target_api") or ""), + "target_method": str(gate.get("target_method") or ""), + "gate_focus_ref": gate.get("focus_ref") or {}, + "action_code": str(gate.get("action_code") or execution_result.get("action_code") or execution_result.get("driver_action_code") or ""), + "sequence_key": str(gate.get("sequence_key") or execution_result.get("sequence_key") or ""), + "handled": bool(execution_result.get("handled", False)), + "mode": str(execution_result.get("mode") or ""), +}, ensure_ascii=False, indent=2)) +PY + + if [[ "${http_status}" -lt 200 || "${http_status}" -ge 300 ]]; then + exit 1 + fi +} + +resolve_driver_action_entry() { + local mainland_base_url="$1" + local action_code="$2" + local action_payload_json="$3" + local requested_by="$4" + local source_label="$5" + + action_payload_json="$(normalize_payload_json_input "${action_payload_json}")" + action_payload_json="$(enrich_driver_action_payload_json "${mainland_base_url}" "${action_code}" "${action_payload_json}")" + + local request_json + request_json="$(python3 - <<'PY' "${action_code}" "${action_payload_json}" "${requested_by}" "${source_label}" +import json +import sys + +action_code = str(sys.argv[1] or "").strip() +payload_raw = str(sys.argv[2] or "").strip() or "{}" +requested_by = str(sys.argv[3] or "").strip() or "cli/ops-center/driver-resolve" +source_label = str(sys.argv[4] or "").strip() or "cli/driver-resolve" + +try: + action_payload = json.loads(payload_raw) +except Exception as exc: + raise SystemExit(f"invalid action payload json: {exc}") + +if not isinstance(action_payload, dict): + raise SystemExit("action payload must be a json object") + +node_codes = [] +seen = set() +for item in list(action_payload.get("node_codes") or []): + value = str(item or "").strip() + if not value or value in seen: + continue + seen.add(value) + node_codes.append(value) + +print(json.dumps({ + "source_label": source_label, + "title": action_code, + "executor_kind": "driver_action", + "primary_label": action_code, + "primary_action_code": action_code, + "primary_node_codes": node_codes, + "primary_action_payload": action_payload, + "include_secondary": True, + "requested_by": requested_by, +}, ensure_ascii=False)) +PY +)" + + echo "[1/3] request" + print_json_or_raw "${request_json}" + echo + echo + + local raw_http_response + raw_http_response="$(request_with_status "POST" "${mainland_base_url}/api/v1/ops/driver-actions/resolve" "${request_json}")" + local http_status="${raw_http_response##*$'\n'HTTP_STATUS=}" + local response_json="${raw_http_response%$'\n'HTTP_STATUS=*}" + + echo "[2/3] raw response" + printf '%s\n' "http_status=${http_status}" + print_json_or_raw "${response_json}" + echo + echo + + echo "[3/3] condensed summary" + python3 - <<'PY' "${response_json}" "${http_status}" +import json +import sys + +payload = json.loads(str(sys.argv[1] or "{}")) +http_status = int(sys.argv[2] or 0) +data = payload.get("data") or {} +preview = data.get("preview") or {} +primary = preview.get("primary") or {} +secondary = preview.get("secondary") or {} +gate = data.get("gate") or {} + +print(json.dumps({ + "http_status": http_status, + "message": str(payload.get("message") or ""), + "selected_section": str(data.get("selected_section") or ""), + "execution_chain": str(preview.get("execution_chain") or ""), + "focus_ref": data.get("focus_ref") or {}, + "gate": { + "decision": str(gate.get("decision") or ""), + "decision_label": str(gate.get("decision_label") or ""), + "recommendation": str(gate.get("recommendation") or ""), + "recommendation_label": str(gate.get("recommendation_label") or ""), + "will_execute": bool(gate.get("will_execute", False)), + "confirm_required": bool(gate.get("confirm_required", False)), + "target_api": str(gate.get("target_api") or ""), + "focus_ref": gate.get("focus_ref") or {}, + }, + "primary": { + "label": str(primary.get("label") or ""), + "action_code": str(primary.get("action_code") or ""), + "target_api": str(primary.get("target_api") or ""), + "recommendation_label": str(primary.get("recommendation_label") or ""), + "focus_ref": primary.get("focus_ref") or {}, + "request_payload": primary.get("request_payload") or {}, + }, + "secondary": ( + { + "label": str(secondary.get("label") or ""), + "action_code": str(secondary.get("action_code") or ""), + "target_api": str(secondary.get("target_api") or ""), + "recommendation_label": str(secondary.get("recommendation_label") or ""), + "focus_ref": secondary.get("focus_ref") or {}, + "request_payload": secondary.get("request_payload") or {}, + } + if secondary else {} + ), +}, ensure_ascii=False, indent=2)) +PY + + if [[ "${http_status}" -lt 200 || "${http_status}" -ge 300 ]]; then + exit 1 + fi +} + +run_driver_action_entry() { + local mainland_base_url="$1" + local action_code="$2" + local action_payload_json="$3" + local confirm_flag="$4" + local requested_by="$5" + local source_label="$6" + local confirm_value + confirm_value="$(confirm_to_bool "${confirm_flag}")" + + action_payload_json="$(normalize_payload_json_input "${action_payload_json}")" + action_payload_json="$(enrich_driver_action_payload_json "${mainland_base_url}" "${action_code}" "${action_payload_json}")" + + local request_json + request_json="$(python3 - <<'PY' "${action_code}" "${action_payload_json}" "${requested_by}" "${source_label}" "${confirm_value}" +import json +import sys + +action_code = str(sys.argv[1] or "").strip() +payload_raw = str(sys.argv[2] or "").strip() or "{}" +requested_by = str(sys.argv[3] or "").strip() or "cli/ops-center/driver-run" +source_label = str(sys.argv[4] or "").strip() or "cli/driver-run" +confirm = str(sys.argv[5] or "").strip().lower() == "true" + +try: + action_payload = json.loads(payload_raw) +except Exception as exc: + raise SystemExit(f"invalid action payload json: {exc}") + +if not isinstance(action_payload, dict): + raise SystemExit("action payload must be a json object") + +node_codes = [] +seen = set() +for item in list(action_payload.get("node_codes") or []): + value = str(item or "").strip() + if not value or value in seen: + continue + seen.add(value) + node_codes.append(value) + +print(json.dumps({ + "source_label": source_label, + "title": action_code, + "executor_kind": "driver_action", + "primary_label": action_code, + "primary_action_code": action_code, + "primary_node_codes": node_codes, + "primary_action_payload": action_payload, + "include_secondary": True, + "requested_by": requested_by, + "confirm": confirm, +}, ensure_ascii=False)) +PY +)" + + echo "[1/3] request" + print_json_or_raw "${request_json}" + echo + echo + + local raw_http_response + raw_http_response="$(request_with_status "POST" "${mainland_base_url}/api/v1/ops/driver-actions/execute-resolved" "${request_json}")" + local http_status="${raw_http_response##*$'\n'HTTP_STATUS=}" + local response_json="${raw_http_response%$'\n'HTTP_STATUS=*}" + + echo "[2/3] raw response" + printf '%s\n' "http_status=${http_status}" + print_json_or_raw "${response_json}" + echo + echo + + echo "[3/3] execution result" + python3 - <<'PY' "${response_json}" "${http_status}" +import json +import sys + +payload = json.loads(str(sys.argv[1] or "{}")) +http_status = int(sys.argv[2] or 0) +data = payload.get("data") or {} +gate = data.get("gate") or {} +execution_result = data.get("execution_result") or {} + +raw_code = payload.get("code", 1) +if raw_code in (0, "0", False): + normalized_code = 0 +else: + normalized_code = int(raw_code or 1) + +print(json.dumps({ + "http_status": http_status, + "code": normalized_code, + "message": str(payload.get("message") or ""), + "executed": bool(data.get("executed", False)), + "selected_section": str(data.get("selected_section") or ""), + "focus_ref": data.get("focus_ref") or {}, + "decision": str(gate.get("decision") or ""), + "decision_label": str(gate.get("decision_label") or ""), + "recommendation": str(gate.get("recommendation") or ""), + "target_api": str(gate.get("target_api") or ""), + "target_method": str(gate.get("target_method") or ""), + "gate_focus_ref": gate.get("focus_ref") or {}, + "action_code": str(gate.get("action_code") or execution_result.get("action_code") or execution_result.get("driver_action_code") or ""), + "handled": bool(execution_result.get("handled", False)), + "mode": str(execution_result.get("mode") or ""), +}, ensure_ascii=False, indent=2)) +PY + + if [[ "${http_status}" -lt 200 || "${http_status}" -ge 300 ]]; then + exit 1 + fi + + if [[ "${action_code}" == "enable_log_sync_key" || "${action_code}" == "enable_log_sync_full" || "${action_code}" == "disable_log_sync" ]]; then + echo + echo "[4/4] log sync post-check" + print_log_sync_post_check "${mainland_base_url}" + fi +} + +print_log_sync_post_check() { + local mainland_base_url="$1" + local raw_http_response + raw_http_response="$(request_with_status "GET" "${mainland_base_url}/api/v1/ops/overview")" + local http_status="${raw_http_response##*$'\n'HTTP_STATUS=}" + local response_json="${raw_http_response%$'\n'HTTP_STATUS=*}" + + python3 - <<'PY' "${response_json}" "${http_status}" +import json +import sys + +payload = json.loads(str(sys.argv[1] or "{}")) +http_status = int(sys.argv[2] or 0) +data = payload.get("data") or {} +execution_scene = data.get("execution_scene") or {} +log_sync = execution_scene.get("log_sync") or {} +participation_summary = execution_scene.get("participation_summary") or {} + +participating_nodes = int( + log_sync.get("participating_nodes_total", log_sync.get("participating_node_count", 0)) + or participation_summary.get("participating_nodes", 0) + or 0 +) +covered_nodes = int( + log_sync.get("covered_participating_nodes", log_sync.get("covered_participating_node_count", 0)) + or 0 +) +line_count = int(log_sync.get("line_count", 0) or 0) +source_node_count = int(log_sync.get("source_node_count", 0) or 0) +missing_nodes = list(log_sync.get("missing_participating_nodes") or []) +mode = str(log_sync.get("mode") or "key").strip() or "key" +enabled = bool(log_sync.get("enabled", False)) +last_at = str(log_sync.get("last_at") or "").strip() +preview_lines = [ + str(item or "").strip() + for item in list(log_sync.get("preview_lines") or []) + if str(item or "").strip() +][:5] + +status = "standby" +summary = "当前没有参与检测节点，远端日志回传处于待命状态。" +if participating_nodes > 0 and not enabled: + status = "disabled" + summary = "当前已有参与检测节点，但远端日志回传仍关闭。" +elif participating_nodes > 0 and (line_count <= 0 or source_node_count <= 0 or missing_nodes): + status = "waiting_sample" + summary = "远端日志回传已经开启，但样本或覆盖仍未形成完整闭环。" +elif participating_nodes > 0: + status = "ready" + summary = "远端日志回传已经形成样本，可以继续下钻节点现场日志。" + +result = { + "http_status": http_status, + "code": int(payload.get("code", 0) or 0), + "message": str(payload.get("message") or ""), + "status": status, + "summary": summary, + "log_sync": { + "enabled": enabled, + "mode": mode, + "participating_nodes": participating_nodes, + "covered_participating_nodes": covered_nodes, + "line_count": line_count, + "source_node_count": source_node_count, + "missing_participating_nodes": missing_nodes, + "last_at": last_at, + "preview_lines": preview_lines, + }, +} + +if http_status < 200 or http_status >= 300: + result["warning"] = "post-check 未能成功读取 /api/v1/ops/overview，请确认当前 API 版本是否已包含 ops overview。" + +print(json.dumps(result, ensure_ascii=False, indent=2)) +PY +} + +run_log_sync_recover_flow() { + local mainland_base_url="$1" + local desired_mode_raw="$2" + local confirm_flag="$3" + local requested_by="$4" + local desired_mode + desired_mode="$(printf '%s' "${desired_mode_raw:-key}" | tr '[:upper:]' '[:lower:]')" + if [[ "${desired_mode}" != "full" ]]; then + desired_mode="key" + fi + + local action_code="enable_log_sync_key" + if [[ "${desired_mode}" == "full" ]]; then + action_code="enable_log_sync_full" + fi + + echo "[1/2] execute log sync recovery action" + run_driver_action_entry "${mainland_base_url}" "${action_code}" '{}' "${confirm_flag}" "${requested_by}" "cli/log-sync-recover" + echo + echo "[2/2] recovery guidance" + build_log_sync_guidance_json "${mainland_base_url}" +} + +build_log_sync_guidance_json() { + local mainland_base_url="$1" + local raw_http_response + raw_http_response="$(request_with_status "GET" "${mainland_base_url}/api/v1/ops/overview")" + local http_status="${raw_http_response##*$'\n'HTTP_STATUS=}" + local response_json="${raw_http_response%$'\n'HTTP_STATUS=*}" + + python3 - <<'PY' "${response_json}" "${http_status}" "${mainland_base_url}" +import json +import sys + +payload = json.loads(str(sys.argv[1] or "{}")) +http_status = int(sys.argv[2] or 0) +base_url = str(sys.argv[3] or "").strip() +data = payload.get("data") or {} +execution_scene = data.get("execution_scene") or {} +log_sync = execution_scene.get("log_sync") or {} +participation_summary = execution_scene.get("participation_summary") or {} + +state = str(log_sync.get("state") or "").strip() or "unknown" +description = str(log_sync.get("description") or "").strip() +enabled = bool(log_sync.get("enabled", False)) +mode = str(log_sync.get("mode") or "key").strip() or "key" +source_nodes = [str(item or "").strip() for item in list(log_sync.get("source_nodes") or []) if str(item or "").strip()] +missing_nodes = [str(item or "").strip() for item in list(log_sync.get("missing_participating_nodes") or []) if str(item or "").strip()] +covered_nodes = [str(item or "").strip() for item in list(log_sync.get("covered_participating_nodes") or []) if str(item or "").strip()] +preview_lines = [str(item or "").strip() for item in list(log_sync.get("preview_lines") or []) if str(item or "").strip()][:5] +participating_nodes = int( + log_sync.get("participating_node_count", participation_summary.get("participating_nodes", 0)) + or participation_summary.get("participating_nodes", 0) + or 0 +) +covered_count = int(log_sync.get("covered_participating_node_count", len(covered_nodes)) or len(covered_nodes)) +line_count = int(log_sync.get("line_count", 0) or 0) + +recommended_commands = [] +primary_target = "" +if source_nodes: + primary_target = source_nodes[0] +elif covered_nodes: + primary_target = covered_nodes[0] +elif missing_nodes: + primary_target = missing_nodes[0] + +if primary_target: + recommended_commands.append( + f"bash domain-api/deploy/multi-region/drive_ops_center.sh scene-node-log {base_url} {primary_target} 120 {mode}" + ) +if missing_nodes: + recommended_commands.extend( + f"bash domain-api/deploy/multi-region/drive_ops_center.sh scene-node-log {base_url} {node_code} 120 {mode}" + for node_code in missing_nodes[:2] + if node_code != primary_target + ) +recommended_commands.append( + f"bash domain-api/deploy/multi-region/drive_ops_center.sh log-sync-check {base_url}" +) +recommended_commands.append( + f"bash domain-api/deploy/multi-region/drive_ops_center.sh doctor {base_url}" +) + +next_step = "继续观察" +if state in {"disabled", "unknown"}: + next_step = "开关仍未真正生效，先确认当前 API 是否已加载最新 runtime_settings，并复查执行链。" +elif state in {"waiting_sample", "partial_coverage"}: + next_step = "开关已经生效，但样本或覆盖还不完整，优先下钻参与节点现场日志。" +elif state in {"healthy", "full_capture"}: + next_step = "远端日志回传已进入可用状态，可以继续从海外控制面盯现场。" +elif state == "idle": + next_step = "当前没有参与检测节点，日志回传已就绪，等任务进入执行即可。" + +print(json.dumps({ + "http_status": http_status, + "code": int(payload.get("code", 0) or 0), + "message": str(payload.get("message") or ""), + "status": state, + "summary": description, + "next_step": next_step, + "log_sync": { + "enabled": enabled, + "mode": mode, + "participating_nodes": participating_nodes, + "covered_participating_nodes": covered_count, + "line_count": line_count, + "source_nodes": source_nodes, + "missing_participating_nodes": missing_nodes, + "preview_lines": preview_lines, + }, + "recommended_commands": recommended_commands[:5], +}, ensure_ascii=False, indent=2)) +PY +} + +run_runtime_refresh_recover() { + local mainland_base_url="$1" + local audit_output + audit_output="$(bash "${SCRIPT_DIR}/check_env_drift.sh" "${mainland_base_url}")" + + echo "[1/2] env audit" + printf '%s\n' "${audit_output}" + echo + echo + + echo "[2/2] runtime refresh guidance" + python3 - <<'PY' "${audit_output}" "${mainland_base_url}" +import json +import sys + +raw_text = str(sys.argv[1] or "") +base_url = str(sys.argv[2] or "").strip() +marker = "[8/8] condensed env audit summary" +candidate_text = raw_text.split(marker, 1)[1].strip() if marker in raw_text else raw_text.strip() + +payload = {} +if candidate_text: + try: + payload = json.loads(candidate_text) + except Exception: + payload = {} + +runtime = dict(payload.get("runtime") or {}) +recommended_actions = [ + str(item).strip() + for item in list(payload.get("recommended_actions") or []) + if str(item).strip() +] + +recommended_commands = [] +for candidate in recommended_actions: + if candidate not in recommended_commands: + recommended_commands.append(candidate) + +for candidate in [ + f"bash domain-api/deploy/multi-region/drive_ops_center.sh stack-diagnosis {base_url} summary", + f"bash domain-api/deploy/multi-region/drive_ops_center.sh stack-next {base_url}", + f"bash domain-api/deploy/multi-region/drive_ops_center.sh doctor-decision {base_url} -", +]: + normalized_candidate = str(candidate or "").strip() + if normalized_candidate and normalized_candidate not in recommended_commands: + recommended_commands.append(normalized_candidate) + +status = str(payload.get("status") or "").strip() or "unknown" +headline = str(payload.get("headline") or "").strip() +runtime_may_need_restart = bool(runtime.get("runtime_may_need_restart", False)) +repo_capability_drift = bool(runtime.get("repo_capability_drift", False)) +repo_commit_drift = bool(runtime.get("repo_commit_drift", False)) + +summary = "当前没有识别到明显的运行时版本漂移。" +if runtime_may_need_restart: + summary = "已经识别到运行时版本漂移，先让 domaincheck-api 切到最新代码，再继续节点接管与上线复核。" +elif status == "blocked": + summary = "当前环境本身仍有阻断项，先修环境，再谈运行时刷新。" +elif status == "attention": + summary = "当前环境存在注意项，但不一定都是运行时版本问题。" + +print(json.dumps({ + "status": status, + "headline": headline, + "summary": summary, + "runtime_refresh": { + "runtime_may_need_restart": runtime_may_need_restart, + "repo_capability_drift": repo_capability_drift, + "repo_commit_drift": repo_commit_drift, + "build_commit_sha": str(runtime.get("build_commit_sha") or "").strip(), + "build_commit_ref": str(runtime.get("build_commit_ref") or "").strip(), + "route_surface_declares_bootstrap_plan": bool(runtime.get("route_surface_declares_bootstrap_plan", False)), + "route_surface_missing_keys": [ + str(item).strip() + for item in list(runtime.get("route_surface_missing_keys") or []) + if str(item).strip() + ], + }, + "recommended_commands": recommended_commands[:6], +}, ensure_ascii=False, indent=2)) +PY +} + +run_go_live_recover() { + local mainland_base_url="$1" + local overseas_base_url="$2" + + echo "[1/5] runtime refresh recover" + run_runtime_refresh_recover "${mainland_base_url}" + echo + echo + + echo "[2/5] stack check summary" + bash "${SCRIPT_DIR}/check_ops_center_stack.sh" "${mainland_base_url}" "summary" + echo + echo + + echo "[3/5] stack next preview" + preview_stack_next_entry "${mainland_base_url}" "$(ops_center_resolve_requested_by)" + echo + echo + + echo "[4/5] go live check summary" + bash "${SCRIPT_DIR}/check_go_live.sh" "${mainland_base_url}" "${overseas_base_url}" "summary" + echo + echo + + echo "[5/5] doctor decision" + run_doctor_decision "${mainland_base_url}" "${overseas_base_url}" +} + +show_agent_gap_check() { + local mainland_base_url="$1" + local raw_http_response + raw_http_response="$(request_with_status "GET" "${mainland_base_url}/api/v1/ops/stack-diagnosis?base_url=${mainland_base_url}")" + local http_status="${raw_http_response##*$'\n'HTTP_STATUS=}" + local response_json="${raw_http_response%$'\n'HTTP_STATUS=*}" + + python3 - <<'PY' "${response_json}" "${http_status}" "${mainland_base_url}" +import json +import sys + +payload = json.loads(str(sys.argv[1] or "{}")) +http_status = int(sys.argv[2] or 0) +base_url = str(sys.argv[3] or "").strip() +data = payload.get("data") or {} +managed_nodes = data.get("managed_nodes") or {} +problem_nodes = [dict(item or {}) for item in list(managed_nodes.get("problem_nodes") or []) if isinstance(item, dict)] +first_gap = dict(managed_nodes.get("first_handover_gap") or {}) +managed_enabled = int(managed_nodes.get("managed_enabled", 0) or 0) +remote_access_ready = int(managed_nodes.get("remote_access_ready", 0) or 0) +first_node_code = str( + first_gap.get("node_code") + or managed_nodes.get("first_handover_gap_node_code") + or ((problem_nodes[0] or {}).get("node_code") if problem_nodes else "") + or "" +).strip() +summary = "当前没有启用托管节点。" +if problem_nodes and managed_enabled > 0 and remote_access_ready <= 0: + summary = "当前已有托管节点，但 remote-agent 仍未就绪，优先补齐节点接管链路。" +elif problem_nodes and managed_enabled > 0 and remote_access_ready < managed_enabled: + summary = ( + f"当前仍有 {len(problem_nodes)} 台托管节点未进入 remote-agent 就绪状态，" + "优先补齐剩余节点接管。" + ) +elif managed_enabled > 0: + summary = "当前托管节点均已进入 remote-agent 就绪状态。" + +recommended_commands = [] +if first_node_code: + recommended_commands.extend([ + f"bash domain-api/deploy/multi-region/drive_ops_center.sh node-recover {base_url} {first_node_code}", + f"bash domain-api/deploy/multi-region/drive_ops_center.sh node-onboarding {base_url} {first_node_code}", + f"bash domain-api/deploy/multi-region/drive_ops_center.sh node-handover {base_url} {first_node_code}", + f"bash domain-api/deploy/multi-region/drive_ops_center.sh node-bootstrap-plan {base_url} {first_node_code}", + ]) +recommended_commands.extend([ + f"bash domain-api/deploy/multi-region/drive_ops_center.sh agent-gap-export /tmp/domaincheck-agent-gap {base_url}", + f"bash domain-api/deploy/multi-region/drive_ops_center.sh stack-first-gap-handover {base_url}", + f"bash domain-api/deploy/multi-region/drive_ops_center.sh doctor {base_url}", +]) + +print(json.dumps({ + "http_status": http_status, + "code": int(payload.get("code", 0) or 0), + "message": str(payload.get("message") or ""), + "summary": summary, + "managed_nodes": { + "managed_enabled": managed_enabled, + "remote_access_ready": remote_access_ready, + "first_gap_node_code": first_node_code, + "problem_nodes": [ + { + "node_code": str(item.get("node_code") or "").strip(), + "agent_state": str(item.get("agent_state") or "").strip(), + "remote_access_state": str(item.get("remote_access_state") or "").strip(), + "delivery_queue_state": str(item.get("delivery_queue_state") or "").strip(), + "participation_state": str(item.get("participation_state") or "").strip(), + } + for item in problem_nodes[:5] + ], + }, + "recommended_commands": recommended_commands[:5], +}, ensure_ascii=False, indent=2)) +PY +} + +run_agent_gap_recover_flow() { + local mainland_base_url="$1" + local confirm_flag="$2" + local requested_by="$3" + local normalized_confirm + normalized_confirm="$(confirm_to_bool "${confirm_flag}")" + + echo "[1/4] stack first gap handover" + show_stack_first_gap_handover "${mainland_base_url}" + echo + echo "[2/4] first node bootstrap plan" + local first_gap_node_code + first_gap_node_code="$(python3 - <<'PY' "${mainland_base_url}" "${OPS_CENTER_CURL_CONNECT_TIMEOUT}" "${OPS_CENTER_CURL_MAX_TIME}" +import json +import subprocess +import sys + +base_url = str(sys.argv[1] or "").strip() +connect_timeout = str(sys.argv[2] or "3").strip() +max_time = str(sys.argv[3] or "10").strip() +proc = subprocess.run( + [ + "curl", + "-sS", + "--connect-timeout", + connect_timeout, + "--max-time", + max_time, + f"{base_url}/api/v1/ops/stack-diagnosis?base_url={base_url}", + ], + capture_output=True, + text=True, + check=False, +) +raw = (proc.stdout or "").strip() +try: + payload = json.loads(raw) +except Exception: + print("") + raise SystemExit(0) +data = payload.get("data") or {} +managed_nodes = data.get("managed_nodes") or {} +problem_nodes = [dict(item or {}) for item in list(managed_nodes.get("problem_nodes") or []) if isinstance(item, dict)] +first_gap = dict(managed_nodes.get("first_handover_gap") or {}) +node_code = str( + first_gap.get("node_code") + or managed_nodes.get("first_handover_gap_node_code") + or ((problem_nodes[0] or {}).get("node_code") if problem_nodes else "") + or "" +).strip() +print(node_code) +PY +)" + if [[ -n "${first_gap_node_code}" ]]; then + build_node_handover_bootstrap_plan \ + "${mainland_base_url}" \ + "${first_gap_node_code}" \ + "${requested_by}" \ + "72" \ + "${mainland_base_url}" \ + "/opt/domaincheck" + else + echo '{"message":"当前没有可生成 bootstrap-plan 的缺口节点。"}' + fi + echo + echo "[3/4] node onboarding view" + if [[ -n "${first_gap_node_code}" ]]; then + show_node_onboarding "${mainland_base_url}" "${first_gap_node_code}" + else + echo '{"message":"当前没有可查看 onboarding 的缺口节点。"}' + fi + echo + echo "[4/4] first node recovery decision" + if [[ -n "${first_gap_node_code}" ]]; then + bash "${SCRIPT_DIR}/drive_ops_center.sh" node-recover "${mainland_base_url}" "${first_gap_node_code}" "" "${requested_by}" + echo + echo "[follow-up] gap recovery guidance" + show_agent_gap_check "${mainland_base_url}" + else + echo '{"message":"当前没有可生成恢复决策的缺口节点。"}' + echo + echo "[follow-up] gap recovery guidance" + show_agent_gap_check "${mainland_base_url}" + fi + if [[ "${normalized_confirm}" == "true" ]]; then + echo + echo '{"confirm":"true","note":"当前 agent-gap-recover 仍是结构化恢复流，不会直接 SSH 落地远端 bootstrap。"}' + fi +} + +export_agent_gap_bundle() { + local report_dir="$1" + local mainland_base_url="$2" + local requested_by="$3" + ops_center_ensure_report_dir "${report_dir}" + + local stack_json_path="${report_dir}/00_stack_first_gap_handover.json" + local check_json_path="${report_dir}/01_agent_gap_check.json" + local onboarding_json_path="${report_dir}/02_node_onboarding.json" + local bootstrap_plan_txt_path="${report_dir}/03_node_bootstrap_plan.txt" + local bootstrap_preview_json_path="${report_dir}/04_node_bootstrap_preview.json" + local acceptance_plan_json_path="${report_dir}/05_node_acceptance_plan.json" + local manifest_path="${report_dir}/manifest.json" + local nodes_dir="${report_dir}/nodes" + + bash "${SCRIPT_DIR}/drive_ops_center.sh" stack-first-gap-handover "${mainland_base_url}" >"${stack_json_path}" + bash "${SCRIPT_DIR}/drive_ops_center.sh" agent-gap-check "${mainland_base_url}" >"${check_json_path}" + + local first_gap_node_code + local gap_nodes_json + gap_nodes_json="$(python3 - <<'PY' "${check_json_path}" +import json +import sys +from pathlib import Path + +text = Path(sys.argv[1]).read_text(encoding="utf-8") +decoder = json.JSONDecoder() +best = None +for index, ch in enumerate(text): + if ch not in "{[": + continue + try: + value, end = decoder.raw_decode(text[index:]) + except Exception: + continue + absolute_end = index + end + if best is None or absolute_end >= best[0]: + best = (absolute_end, value) +payload = best[1] if best else {} +managed = payload.get("managed_nodes") or {} +problem_nodes = [dict(item or {}) for item in list(managed.get("problem_nodes") or []) if isinstance(item, dict)] +first_gap_node_code = str(managed.get("first_gap_node_code") or "").strip() +node_codes = [] +for item in problem_nodes: + node_code = str(item.get("node_code") or "").strip() + if node_code and node_code not in node_codes: + node_codes.append(node_code) +if first_gap_node_code and first_gap_node_code not in node_codes: + node_codes.insert(0, first_gap_node_code) +print(json.dumps({ + "first_gap_node_code": first_gap_node_code, + "gap_node_codes": node_codes, +}, ensure_ascii=False)) +PY +)" + first_gap_node_code="$(python3 - <<'PY' "${gap_nodes_json}" +import json +import sys + +payload = json.loads(str(sys.argv[1] or "{}")) +print(str(payload.get("first_gap_node_code") or "").strip()) +PY +)" + mapfile -t gap_node_codes < <(python3 - <<'PY' "${gap_nodes_json}" +import json +import sys + +payload = json.loads(str(sys.argv[1] or "{}")) +for item in list(payload.get("gap_node_codes") or []): + node_code = str(item or "").strip() + if node_code: + print(node_code) +PY +) + + if [[ -n "${first_gap_node_code}" ]]; then + bash "${SCRIPT_DIR}/drive_ops_center.sh" node-onboarding "${mainland_base_url}" "${first_gap_node_code}" >"${onboarding_json_path}" + bash "${SCRIPT_DIR}/drive_ops_center.sh" node-bootstrap-plan "${mainland_base_url}" "${first_gap_node_code}" "${requested_by}" "72" "${mainland_base_url}" "/opt/domaincheck" >"${bootstrap_plan_txt_path}" + bash "${SCRIPT_DIR}/drive_ops_center.sh" node-bootstrap-preview "${mainland_base_url}" "${first_gap_node_code}" "${requested_by}" >"${bootstrap_preview_json_path}" + bash "${SCRIPT_DIR}/drive_ops_center.sh" node-acceptance-plan "${mainland_base_url}" "${first_gap_node_code}" "${requested_by}" >"${acceptance_plan_json_path}" || true + fi + + mkdir -p "${nodes_dir}" + local node_code safe_node_code node_dir + for node_code in "${gap_node_codes[@]}"; do + [[ -n "${node_code}" ]] || continue + safe_node_code="$(printf '%s' "${node_code}" | tr '/:' '__')" + safe_node_code="${safe_node_code// /_}" + node_dir="${nodes_dir}/${safe_node_code}" + mkdir -p "${node_dir}" + bash "${SCRIPT_DIR}/drive_ops_center.sh" node-handover "${mainland_base_url}" "${node_code}" >"${node_dir}/00_node_handover.json" + bash "${SCRIPT_DIR}/drive_ops_center.sh" node-onboarding "${mainland_base_url}" "${node_code}" >"${node_dir}/01_node_onboarding.json" + bash "${SCRIPT_DIR}/drive_ops_center.sh" node-bootstrap-plan "${mainland_base_url}" "${node_code}" "${requested_by}" "72" "${mainland_base_url}" "/opt/domaincheck" >"${node_dir}/02_node_bootstrap_plan.txt" + bash "${SCRIPT_DIR}/drive_ops_center.sh" node-bootstrap-preview "${mainland_base_url}" "${node_code}" "${requested_by}" >"${node_dir}/03_node_bootstrap_preview.json" + bash "${SCRIPT_DIR}/drive_ops_center.sh" node-acceptance-plan "${mainland_base_url}" "${node_code}" "${requested_by}" >"${node_dir}/04_node_acceptance_plan.json" || true + done + + python3 - <<'PY' \ + "${manifest_path}" \ + "${report_dir}" \ + "${mainland_base_url}" \ + "${requested_by}" \ + "${first_gap_node_code}" \ + "${gap_nodes_json}" \ + "${nodes_dir}" \ + "${stack_json_path}" \ + "${check_json_path}" \ + "${onboarding_json_path}" \ + "${bootstrap_plan_txt_path}" \ + "${bootstrap_preview_json_path}" \ + "${acceptance_plan_json_path}" +import json +import sys +from pathlib import Path + +manifest_path = Path(sys.argv[1]) +report_dir = Path(sys.argv[2]) +base_url = sys.argv[3] +requested_by = sys.argv[4] +first_gap_node_code = sys.argv[5] +gap_nodes_payload = json.loads(str(sys.argv[6] or "{}")) +nodes_dir = Path(sys.argv[7]) +paths = { + "stack_first_gap_handover": Path(sys.argv[8]), + "agent_gap_check": Path(sys.argv[9]), + "node_onboarding": Path(sys.argv[10]), + "node_bootstrap_plan": Path(sys.argv[11]), + "node_bootstrap_preview": Path(sys.argv[12]), + "node_acceptance_plan": Path(sys.argv[13]), +} + +artifacts = [] +for key, path in paths.items(): + artifacts.append({ + "key": key, + "path": str(path), + "exists": path.exists(), + "size": path.stat().st_size if path.exists() else 0, + }) + +node_bundles = [] +for node_code in list(gap_nodes_payload.get("gap_node_codes") or []): + normalized_node_code = str(node_code or "").strip() + if not normalized_node_code: + continue + normalized_dir_name = normalized_node_code.replace("/", "_").replace(":", "_").replace(" ", "_") + node_dir = nodes_dir / normalized_dir_name + bundle_paths = { + "node_handover": node_dir / "00_node_handover.json", + "node_onboarding": node_dir / "01_node_onboarding.json", + "node_bootstrap_plan": node_dir / "02_node_bootstrap_plan.txt", + "node_bootstrap_preview": node_dir / "03_node_bootstrap_preview.json", + "node_acceptance_plan": node_dir / "04_node_acceptance_plan.json", + } + node_bundles.append({ + "node_code": normalized_node_code, + "bundle_dir": str(node_dir), + "artifacts": [ + { + "key": key, + "path": str(path), + "exists": path.exists(), + "size": path.stat().st_size if path.exists() else 0, + } + for key, path in bundle_paths.items() + ], + }) + +payload = { + "report_dir": str(report_dir), + "mainland_base_url": base_url, + "requested_by": requested_by, + "first_gap_node_code": first_gap_node_code, + "gap_node_codes": list(gap_nodes_payload.get("gap_node_codes") or []), + "gap_node_total": len(list(gap_nodes_payload.get("gap_node_codes") or [])), + "artifacts": artifacts, + "node_bundles": node_bundles, + "summary": { + "ok": bool(node_bundles), + "headline": ( + f"已为 {len(node_bundles)} 台接管缺口节点导出接管材料；首个缺口节点为 {first_gap_node_code}。" + if node_bundles and first_gap_node_code + else ( + f"已为 {len(node_bundles)} 台接管缺口节点导出接管材料。" + if node_bundles + else "当前没有可导出的接管缺口节点。" + ) + ), + }, +} +manifest_path.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8") +print(json.dumps(payload, ensure_ascii=False, indent=2)) +PY +} + +run_resolved_runbook_sequence() { + local mainland_base_url="$1" + local sequence_key="$2" + local secondary_flag="$3" + local action_payload_json="$4" + local confirm_flag="$5" + local requested_by="$6" + local source_label="$7" + local confirm_value + confirm_value="$(confirm_to_bool "${confirm_flag}")" + + action_payload_json="$(normalize_payload_json_input "${action_payload_json}")" + + local request_json + request_json="$(python3 - <<'PY' "${sequence_key}" "${secondary_flag}" "${action_payload_json}" "${requested_by}" "${source_label}" "${confirm_value}" +import json +import sys + +sequence_key = str(sys.argv[1] or "").strip() +secondary_flag = str(sys.argv[2] or "primary").strip().lower() +payload_raw = str(sys.argv[3] or "").strip() or "{}" +requested_by = str(sys.argv[4] or "").strip() or "cli/ops-center/runbook-run" +source_label = str(sys.argv[5] or "").strip() or "cli/runbook-run" +confirm = str(sys.argv[6] or "").strip().lower() == "true" + +try: + action_payload = json.loads(payload_raw) +except Exception as exc: + raise SystemExit(f"invalid action payload json: {exc}") + +if not isinstance(action_payload, dict): + raise SystemExit("action payload must be a json object") + +secondary = secondary_flag in {"secondary", "second", "2", "true", "yes", "y"} + +print(json.dumps({ + "source_label": source_label, + "title": sequence_key, + "executor_kind": "runbook_sequence", + "sequence_key": sequence_key, + "include_secondary": True, + "requested_by": requested_by, + "secondary": secondary, + "confirm": confirm, + "primary_action_payload": action_payload, + "secondary_action_payload": action_payload, +}, ensure_ascii=False)) +PY +)" + + echo "[1/3] request" + print_json_or_raw "${request_json}" + echo + echo + + local raw_http_response + raw_http_response="$(request_with_status "POST" "${mainland_base_url}/api/v1/ops/driver-actions/execute-resolved" "${request_json}")" + local http_status="${raw_http_response##*$'\n'HTTP_STATUS=}" + local response_json="${raw_http_response%$'\n'HTTP_STATUS=*}" + + echo "[2/3] raw response" + printf '%s\n' "http_status=${http_status}" + print_json_or_raw "${response_json}" + echo + echo + + echo "[3/3] execution result" + python3 - <<'PY' "${response_json}" "${http_status}" +import json +import sys + +payload = json.loads(str(sys.argv[1] or "{}")) +http_status = int(sys.argv[2] or 0) +data = payload.get("data") or {} +gate = data.get("gate") or {} +execution_result = data.get("execution_result") or {} + +raw_code = payload.get("code", 1) +if raw_code in (0, "0", False): + normalized_code = 0 +else: + normalized_code = int(raw_code or 1) + +print(json.dumps({ + "http_status": http_status, + "code": normalized_code, + "message": str(payload.get("message") or ""), + "executed": bool(data.get("executed", False)), + "selected_section": str(data.get("selected_section") or ""), + "decision": str(gate.get("decision") or ""), + "decision_label": str(gate.get("decision_label") or ""), + "recommendation": str(gate.get("recommendation") or ""), + "sequence_key": str(gate.get("sequence_key") or execution_result.get("sequence_key") or ""), + "action_code": str(gate.get("action_code") or execution_result.get("action_code") or execution_result.get("driver_action_code") or ""), + "handled": bool(execution_result.get("handled", False)), + "mode": str(execution_result.get("mode") or ""), +}, ensure_ascii=False, indent=2)) +PY + + if [[ "${http_status}" -lt 200 || "${http_status}" -ge 300 ]]; then + exit 1 + fi +} + +case "${COMMAND}" in + help|-h|--help) + usage + ;; + config) + print_config "${1:-${OPS_CENTER_ACTIVE_PROFILE}}" + ;; + contracts) + MAINLAND_BASE_URL="${1:-$(ops_center_resolve_mainland_api_base_url)}" + CONTRACT_KEY="${2:-}" + if [[ "${MAINLAND_BASE_URL}" != http://* && "${MAINLAND_BASE_URL}" != https://* ]]; then + CONTRACT_KEY="${MAINLAND_BASE_URL}" + MAINLAND_BASE_URL="$(ops_center_resolve_mainland_api_base_url)" + fi + show_ops_contracts "${MAINLAND_BASE_URL}" "${CONTRACT_KEY}" + ;; + env-audit) + MAINLAND_BASE_URL="${1:-$(ops_center_resolve_mainland_api_base_url)}" + exec bash "${SCRIPT_DIR}/check_env_drift.sh" "${MAINLAND_BASE_URL}" + ;; + runtime-refresh-check) + MAINLAND_BASE_URL="${1:-$(ops_center_resolve_mainland_api_base_url)}" + exec bash "${SCRIPT_DIR}/check_env_drift.sh" "${MAINLAND_BASE_URL}" + ;; + runtime-refresh-recover) + MAINLAND_BASE_URL="${1:-$(ops_center_resolve_mainland_api_base_url)}" + run_runtime_refresh_recover "${MAINLAND_BASE_URL}" + ;; + stack-diagnosis|stack-check) + MAINLAND_BASE_URL="${1:-$(ops_center_resolve_mainland_api_base_url)}" + VIEW_MODE="${2:-summary}" + if [[ "${MAINLAND_BASE_URL}" != http://* && "${MAINLAND_BASE_URL}" != https://* ]]; then + VIEW_MODE="${MAINLAND_BASE_URL}" + MAINLAND_BASE_URL="$(ops_center_resolve_mainland_api_base_url)" + fi + exec bash "${SCRIPT_DIR}/check_ops_center_stack.sh" "${MAINLAND_BASE_URL}" "${VIEW_MODE}" + ;; + go-live-check) + MAINLAND_BASE_URL="${1:-$(ops_center_resolve_mainland_api_base_url)}" + OVERSEAS_BASE_URL="${2:-$(ops_center_resolve_overseas_api_base_url)}" + VIEW_MODE="${3:-summary}" + if [[ "${MAINLAND_BASE_URL}" != http://* && "${MAINLAND_BASE_URL}" != https://* ]]; then + VIEW_MODE="${OVERSEAS_BASE_URL:-summary}" + OVERSEAS_BASE_URL="${MAINLAND_BASE_URL}" + MAINLAND_BASE_URL="$(ops_center_resolve_mainland_api_base_url)" + fi + exec bash "${SCRIPT_DIR}/check_go_live.sh" "${MAINLAND_BASE_URL}" "${OVERSEAS_BASE_URL}" "${VIEW_MODE}" + ;; + go-live-recover) + MAINLAND_BASE_URL="${1:-$(ops_center_resolve_mainland_api_base_url)}" + OVERSEAS_BASE_URL="$(normalize_optional_base_url_arg "${2:-}" "$(ops_center_resolve_overseas_api_base_url)")" + if [[ "${MAINLAND_BASE_URL}" != http://* && "${MAINLAND_BASE_URL}" != https://* ]]; then + OVERSEAS_BASE_URL="$(normalize_optional_base_url_arg "${MAINLAND_BASE_URL}" "$(ops_center_resolve_overseas_api_base_url)")" + MAINLAND_BASE_URL="$(ops_center_resolve_mainland_api_base_url)" + fi + run_go_live_recover "${MAINLAND_BASE_URL}" "${OVERSEAS_BASE_URL}" + ;; + go-live-export) + REPORT_DIR="${1:-${OPS_CENTER_REPORT_ROOT}/go-live-bundles/go-live-bundle-$(timestamp_now)}" + MAINLAND_BASE_URL="${2:-$(ops_center_resolve_mainland_api_base_url)}" + OVERSEAS_BASE_URL="$(normalize_optional_base_url_arg "${3:-}" "$(ops_center_resolve_overseas_api_base_url)")" + if [[ "${REPORT_DIR}" == http://* || "${REPORT_DIR}" == https://* ]]; then + OVERSEAS_BASE_URL="$(normalize_optional_base_url_arg "${MAINLAND_BASE_URL:-}" "$(ops_center_resolve_overseas_api_base_url)")" + MAINLAND_BASE_URL="${REPORT_DIR}" + REPORT_DIR="${OPS_CENTER_REPORT_ROOT}/go-live-bundles/go-live-bundle-$(timestamp_now)}" + fi + exec bash "${SCRIPT_DIR}/export_go_live_bundle.sh" "${REPORT_DIR}" "${MAINLAND_BASE_URL}" "${OVERSEAS_BASE_URL}" + ;; + go-live-review) + TARGET_PATH="${1:-${OPS_CENTER_REPORT_ROOT}/go-live-bundles}" + if [[ -d "${TARGET_PATH}" && ! -f "${TARGET_PATH}/manifest.json" ]]; then + TARGET_PATH="$(find "${TARGET_PATH}" -mindepth 1 -maxdepth 1 -type d | sort | tail -n 1)" + fi + exec bash "${SCRIPT_DIR}/review_go_live_bundle.sh" "${TARGET_PATH}" + ;; + go-live-signoff) + FIRST_ARG="${1:-}" + SECOND_ARG="${2:-}" + if [[ -n "${FIRST_ARG}" && -d "${FIRST_ARG}" && ! -f "${FIRST_ARG}/manifest.json" ]]; then + FIRST_ARG="$(find "${FIRST_ARG}" -mindepth 1 -maxdepth 1 -type d | sort | tail -n 1)" + fi + exec bash "${SCRIPT_DIR}/check_go_live_signoff.sh" "${FIRST_ARG}" "${SECOND_ARG}" + ;; + stack-next) + MAINLAND_BASE_URL="${1:-$(ops_center_resolve_mainland_api_base_url)}" + STACK_MODE="${2:-preview}" + CONFIRM_FLAG="${3:-}" + REQUESTED_BY="${4:-$(ops_center_resolve_requested_by)}" + if [[ "${MAINLAND_BASE_URL}" != http://* && "${MAINLAND_BASE_URL}" != https://* ]]; then + REQUESTED_BY="${CONFIRM_FLAG:-$(ops_center_resolve_requested_by)}" + CONFIRM_FLAG="${STACK_MODE:-}" + STACK_MODE="${MAINLAND_BASE_URL}" + MAINLAND_BASE_URL="$(ops_center_resolve_mainland_api_base_url)" + fi + if is_confirm_token "${STACK_MODE}" && [[ -z "${CONFIRM_FLAG}" ]]; then + CONFIRM_FLAG="${STACK_MODE}" + STACK_MODE="run" + fi + case "$(printf '%s' "${STACK_MODE}" | tr '[:upper:]' '[:lower:]')" in + preview|show|resolve|"") + preview_stack_next_entry "${MAINLAND_BASE_URL}" "${REQUESTED_BY}" + ;; + run|execute) + run_stack_next_entry "${MAINLAND_BASE_URL}" "${CONFIRM_FLAG}" "${REQUESTED_BY}" + ;; + *) + echo "stack-next mode must be preview or run" >&2 + exit 1 + ;; + esac + ;; + release-package) + REPO_ROOT="$(ops_center_repo_root)" + SKIP_SMOKE="$(confirm_to_bool "${1:-}")" + if [[ "${SKIP_SMOKE}" == "true" ]]; then + DOMAINCHECK_SKIP_SMOKE_TEST=1 exec bash "${REPO_ROOT}/package_domain_release.sh" + fi + exec bash "${REPO_ROOT}/package_domain_release.sh" + ;; + release-verify) + REPO_ROOT="$(ops_center_repo_root)" + exec bash "${REPO_ROOT}/verify_domain_release.sh" "${1:-}" "${2:-}" + ;; + release-prepare) + REPO_ROOT="$(ops_center_repo_root)" + exec bash "${REPO_ROOT}/prepare_final_release.sh" + ;; + release-show) + REPO_ROOT="$(ops_center_repo_root)" + exec bash "${REPO_ROOT}/show_latest_release.sh" + ;; + release-launchpad) + MAINLAND_BASE_URL="${1:-$(ops_center_resolve_mainland_api_base_url)}" + CHANNEL="${2:-stable}" + exec bash "${SCRIPT_DIR}/drive_release_hub.sh" "${MAINLAND_BASE_URL}" launchpad "${CHANNEL}" + ;; + release-preview) + MAINLAND_BASE_URL="${1:-$(ops_center_resolve_mainland_api_base_url)}" + exec bash "${SCRIPT_DIR}/drive_release_hub.sh" "${MAINLAND_BASE_URL}" preview-latest + ;; + release-preview-smart) + MAINLAND_BASE_URL="${1:-$(ops_center_resolve_mainland_api_base_url)}" + MODE="${2:-}" + if [[ -z "${MODE}" && ( "${MAINLAND_BASE_URL}" == "worker" || "${MAINLAND_BASE_URL}" == "control" ) ]]; then + MODE="${MAINLAND_BASE_URL}" + MAINLAND_BASE_URL="$(ops_center_resolve_mainland_api_base_url)" + fi + if [[ "${MODE}" != "worker" && "${MODE}" != "control" ]]; then + echo "mode must be worker or control" >&2 + usage + exit 1 + fi + exec bash "${SCRIPT_DIR}/drive_release_hub.sh" "${MAINLAND_BASE_URL}" preview-latest-smart "${MODE}" + ;; + release-import-latest) + MAINLAND_BASE_URL="${1:-$(ops_center_resolve_mainland_api_base_url)}" + exec bash "${SCRIPT_DIR}/drive_release_hub.sh" "${MAINLAND_BASE_URL}" create-latest-release + ;; + publish-latest) + MAINLAND_BASE_URL="${1:-$(ops_center_resolve_mainland_api_base_url)}" + MODE="${2:-}" + CONFIRM_FLAG="${3:-}" + SKIP_SMOKE_FLAG="${4:-}" + if [[ -z "${MODE}" && ( "${MAINLAND_BASE_URL}" == "worker" || "${MAINLAND_BASE_URL}" == "control" ) ]]; then + MODE="${MAINLAND_BASE_URL}" + MAINLAND_BASE_URL="$(ops_center_resolve_mainland_api_base_url)" + CONFIRM_FLAG="${2:-}" + SKIP_SMOKE_FLAG="${3:-}" + fi + run_publish_latest "${MAINLAND_BASE_URL}" "${MODE}" "${CONFIRM_FLAG}" "${SKIP_SMOKE_FLAG}" + ;; + doctor-export) + REPORT_DIR="${1:-${OPS_CENTER_REPORT_ROOT}/doctor-$(timestamp_now)}" + MAINLAND_BASE_URL="${2:-$(ops_center_resolve_mainland_api_base_url)}" + OVERSEAS_BASE_URL="$(normalize_optional_base_url_arg "${3:-}" "$(ops_center_resolve_overseas_api_base_url)")" + run_doctor_report_export "${REPORT_DIR}" "${MAINLAND_BASE_URL}" "${OVERSEAS_BASE_URL}" + ;; + doctor-decision) + FIRST_ARG="${1:-}" + SECOND_ARG="${2:-}" + run_doctor_decision "${FIRST_ARG}" "${SECOND_ARG}" + ;; + doctor) + MAINLAND_BASE_URL="${1:-$(ops_center_resolve_mainland_api_base_url)}" + OVERSEAS_BASE_URL="$(normalize_optional_base_url_arg "${2:-}" "$(ops_center_resolve_overseas_api_base_url)")" + run_step "[doctor/0] ops center stack" bash "${SCRIPT_DIR}/check_ops_center_stack.sh" "${MAINLAND_BASE_URL}" summary + run_step "[doctor/1] ops plane" bash "${SCRIPT_DIR}/check_ops_plane.sh" "${MAINLAND_BASE_URL}" + run_step "[doctor/2] release hub" bash "${SCRIPT_DIR}/check_release_hub.sh" "${MAINLAND_BASE_URL}" + run_step "[doctor/3] inspection" bash "${SCRIPT_DIR}/check_ops_inspection.sh" "${MAINLAND_BASE_URL}" + run_step "[doctor/4] participation" bash "${SCRIPT_DIR}/check_worker_participation.sh" "${MAINLAND_BASE_URL}" + if [[ -n "${OVERSEAS_BASE_URL}" ]]; then + run_step "[doctor/5] temp topology" bash "${SCRIPT_DIR}/check_temp_topology.sh" "${MAINLAND_BASE_URL}" "${OVERSEAS_BASE_URL}" + fi + ;; + ops-plane) + MAINLAND_BASE_URL="${1:-$(ops_center_resolve_mainland_api_base_url)}" + exec bash "${SCRIPT_DIR}/check_ops_plane.sh" "${MAINLAND_BASE_URL}" + ;; + inspection) + MAINLAND_BASE_URL="${1:-$(ops_center_resolve_mainland_api_base_url)}" + exec bash "${SCRIPT_DIR}/check_ops_inspection.sh" "${MAINLAND_BASE_URL}" + ;; + participation) + MAINLAND_BASE_URL="${1:-$(ops_center_resolve_mainland_api_base_url)}" + exec bash "${SCRIPT_DIR}/check_worker_participation.sh" "${MAINLAND_BASE_URL}" + ;; + topology) + MAINLAND_BASE_URL="${1:-$(ops_center_resolve_mainland_api_base_url)}" + OVERSEAS_BASE_URL="${2:-$(ops_center_resolve_overseas_api_base_url)}" + exec bash "${SCRIPT_DIR}/check_temp_topology.sh" "${MAINLAND_BASE_URL}" "${OVERSEAS_BASE_URL}" + ;; + nodes) + MAINLAND_BASE_URL="${1:-$(ops_center_resolve_mainland_api_base_url)}" + show_ops_nodes "${MAINLAND_BASE_URL}" + ;; + stack-first-gap-handover) + MAINLAND_BASE_URL="${1:-$(ops_center_resolve_mainland_api_base_url)}" + if [[ "${MAINLAND_BASE_URL}" != http://* && "${MAINLAND_BASE_URL}" != https://* ]]; then + MAINLAND_BASE_URL="$(ops_center_resolve_mainland_api_base_url)" + fi + show_stack_first_gap_handover "${MAINLAND_BASE_URL}" + ;; + node-bind-ssh) + ARG1="${1:-}" + ARG2="${2:-}" + ARG3="${3:-}" + ARG4="${4:-}" + ARG5="${5:-}" + ARG6="${6:-}" + if [[ -n "${ARG1}" && ( "${ARG1}" == http://* || "${ARG1}" == https://* ) ]]; then + MAINLAND_BASE_URL="${ARG1}" + NODE_CODE="${ARG2:-}" + SSH_HOST="${ARG3:-}" + SSH_USER="${ARG4:-}" + SSH_PORT="${ARG5:-22}" + TITLE="${ARG6:-}" + else + MAINLAND_BASE_URL="$(ops_center_resolve_mainland_api_base_url)" + NODE_CODE="${ARG1:-}" + SSH_HOST="${ARG2:-}" + SSH_USER="${ARG3:-}" + SSH_PORT="${ARG4:-22}" + TITLE="${ARG5:-}" + fi + bind_node_ssh_profile "${MAINLAND_BASE_URL}" "${NODE_CODE}" "${SSH_HOST}" "${SSH_USER}" "${SSH_PORT}" "${TITLE}" + ;; + node-handover) + MAINLAND_BASE_URL="${1:-$(ops_center_resolve_mainland_api_base_url)}" + NODE_CODE="${2:-}" + if [[ "${MAINLAND_BASE_URL}" != http://* && "${MAINLAND_BASE_URL}" != https://* ]]; then + NODE_CODE="${MAINLAND_BASE_URL}" + MAINLAND_BASE_URL="$(ops_center_resolve_mainland_api_base_url)" + fi + show_node_handover "${MAINLAND_BASE_URL}" "${NODE_CODE}" + ;; + node-onboarding) + MAINLAND_BASE_URL="${1:-$(ops_center_resolve_mainland_api_base_url)}" + NODE_CODE="${2:-}" + if [[ "${MAINLAND_BASE_URL}" != http://* && "${MAINLAND_BASE_URL}" != https://* ]]; then + NODE_CODE="${MAINLAND_BASE_URL}" + MAINLAND_BASE_URL="$(ops_center_resolve_mainland_api_base_url)" + fi + show_node_onboarding "${MAINLAND_BASE_URL}" "${NODE_CODE}" + ;; + node-recover) + MAINLAND_BASE_URL="${1:-$(ops_center_resolve_mainland_api_base_url)}" + NODE_CODE="${2:-}" + CONFIRM_FLAG="${3:-}" + REQUESTED_BY="${4:-cli}" + if [[ "${MAINLAND_BASE_URL}" != http://* && "${MAINLAND_BASE_URL}" != https://* ]]; then + REQUESTED_BY="${CONFIRM_FLAG:-cli}" + CONFIRM_FLAG="${NODE_CODE:-}" + NODE_CODE="${MAINLAND_BASE_URL}" + MAINLAND_BASE_URL="$(ops_center_resolve_mainland_api_base_url)" + fi + recover_node_onboarding "${MAINLAND_BASE_URL}" "${NODE_CODE}" "${CONFIRM_FLAG}" "${REQUESTED_BY}" + ;; + node-bootstrap-preview) + MAINLAND_BASE_URL="${1:-$(ops_center_resolve_mainland_api_base_url)}" + NODE_CODE="${2:-}" + REQUESTED_BY="${3:-cli}" + if [[ "${MAINLAND_BASE_URL}" != http://* && "${MAINLAND_BASE_URL}" != https://* ]]; then + REQUESTED_BY="${NODE_CODE:-cli}" + NODE_CODE="${MAINLAND_BASE_URL}" + MAINLAND_BASE_URL="$(ops_center_resolve_mainland_api_base_url)" + fi + show_node_bootstrap_preview "${MAINLAND_BASE_URL}" "${NODE_CODE}" "${REQUESTED_BY}" + ;; + node-bootstrap-run) + MAINLAND_BASE_URL="${1:-$(ops_center_resolve_mainland_api_base_url)}" + NODE_CODE="${2:-}" + REQUESTED_BY="${3:-cli}" + if [[ "${MAINLAND_BASE_URL}" != http://* && "${MAINLAND_BASE_URL}" != https://* ]]; then + REQUESTED_BY="${NODE_CODE:-cli}" + NODE_CODE="${MAINLAND_BASE_URL}" + MAINLAND_BASE_URL="$(ops_center_resolve_mainland_api_base_url)" + fi + run_node_bootstrap "${MAINLAND_BASE_URL}" "${NODE_CODE}" "${REQUESTED_BY}" + ;; + node-acceptance-plan) + MAINLAND_BASE_URL="${1:-$(ops_center_resolve_mainland_api_base_url)}" + NODE_CODE="${2:-}" + REQUESTED_BY="${3:-cli}" + if [[ "${MAINLAND_BASE_URL}" != http://* && "${MAINLAND_BASE_URL}" != https://* ]]; then + REQUESTED_BY="${NODE_CODE:-cli}" + NODE_CODE="${MAINLAND_BASE_URL}" + MAINLAND_BASE_URL="$(ops_center_resolve_mainland_api_base_url)" + fi + show_node_acceptance_plan "${MAINLAND_BASE_URL}" "${NODE_CODE}" "${REQUESTED_BY}" + ;; + node-acceptance-run) + MAINLAND_BASE_URL="${1:-$(ops_center_resolve_mainland_api_base_url)}" + NODE_CODE="${2:-}" + REQUESTED_BY="${3:-cli}" + if [[ "${MAINLAND_BASE_URL}" != http://* && "${MAINLAND_BASE_URL}" != https://* ]]; then + REQUESTED_BY="${NODE_CODE:-cli}" + NODE_CODE="${MAINLAND_BASE_URL}" + MAINLAND_BASE_URL="$(ops_center_resolve_mainland_api_base_url)" + fi + run_node_acceptance "${MAINLAND_BASE_URL}" "${NODE_CODE}" "${REQUESTED_BY}" + ;; + scene-node-log) + MAINLAND_BASE_URL="${1:-$(ops_center_resolve_mainland_api_base_url)}" + NODE_CODE="${2:-}" + LIMIT="${3:-80}" + MODE="${4:-}" + if [[ "${MAINLAND_BASE_URL}" != http://* && "${MAINLAND_BASE_URL}" != https://* ]]; then + MODE="${LIMIT:-}" + LIMIT="${NODE_CODE:-80}" + NODE_CODE="${MAINLAND_BASE_URL}" + MAINLAND_BASE_URL="$(ops_center_resolve_mainland_api_base_url)" + fi + show_node_scene_log "${MAINLAND_BASE_URL}" "${NODE_CODE}" "${LIMIT}" "${MODE}" + ;; + log-sync-check) + MAINLAND_BASE_URL="${1:-$(ops_center_resolve_mainland_api_base_url)}" + print_log_sync_post_check "${MAINLAND_BASE_URL}" + ;; + log-sync-recover) + MAINLAND_BASE_URL="${1:-$(ops_center_resolve_mainland_api_base_url)}" + MODE_ARG="${2:-key}" + CONFIRM_FLAG="${3:-}" + REQUESTED_BY="${4:-$(ops_center_resolve_requested_by)}" + if [[ "${MAINLAND_BASE_URL}" != http://* && "${MAINLAND_BASE_URL}" != https://* ]]; then + REQUESTED_BY="${CONFIRM_FLAG:-$(ops_center_resolve_requested_by)}" + CONFIRM_FLAG="${MODE_ARG:-}" + MODE_ARG="${MAINLAND_BASE_URL:-key}" + MAINLAND_BASE_URL="$(ops_center_resolve_mainland_api_base_url)" + fi + run_log_sync_recover_flow "${MAINLAND_BASE_URL}" "${MODE_ARG}" "${CONFIRM_FLAG}" "${REQUESTED_BY}" + ;; + agent-gap-check) + MAINLAND_BASE_URL="${1:-$(ops_center_resolve_mainland_api_base_url)}" + show_agent_gap_check "${MAINLAND_BASE_URL}" + ;; + agent-gap-recover) + MAINLAND_BASE_URL="${1:-$(ops_center_resolve_mainland_api_base_url)}" + CONFIRM_FLAG="${2:-}" + REQUESTED_BY="${3:-$(ops_center_resolve_requested_by)}" + if [[ "${MAINLAND_BASE_URL}" != http://* && "${MAINLAND_BASE_URL}" != https://* ]]; then + REQUESTED_BY="${CONFIRM_FLAG:-$(ops_center_resolve_requested_by)}" + CONFIRM_FLAG="${MAINLAND_BASE_URL:-}" + MAINLAND_BASE_URL="$(ops_center_resolve_mainland_api_base_url)" + fi + run_agent_gap_recover_flow "${MAINLAND_BASE_URL}" "${CONFIRM_FLAG}" "${REQUESTED_BY}" + ;; + agent-gap-export) + ARG1="${1:-}" + ARG2="${2:-}" + ARG3="${3:-$(ops_center_resolve_requested_by)}" + if [[ -n "${ARG1}" && "${ARG1}" != http://* && "${ARG1}" != https://* ]]; then + REPORT_DIR="${ARG1}" + MAINLAND_BASE_URL="${ARG2:-$(ops_center_resolve_mainland_api_base_url)}" + REQUESTED_BY="${ARG3:-$(ops_center_resolve_requested_by)}" + else + REPORT_DIR="${OPS_CENTER_REPORT_ROOT}/agent-gap-export-$(timestamp_now)" + MAINLAND_BASE_URL="${ARG1:-$(ops_center_resolve_mainland_api_base_url)}" + REQUESTED_BY="${ARG2:-$(ops_center_resolve_requested_by)}" + fi + export_agent_gap_bundle "${REPORT_DIR}" "${MAINLAND_BASE_URL}" "${REQUESTED_BY}" + ;; + node-bootstrap-plan) + ARG1="${1:-}" + ARG2="${2:-}" + ARG3="${3:-}" + ARG4="${4:-}" + ARG5="${5:-}" + ARG6="${6:-}" + if [[ -n "${ARG1}" && ( "${ARG1}" == http://* || "${ARG1}" == https://* ) ]]; then + MAINLAND_BASE_URL="${ARG1}" + NODE_CODE="${ARG2:-}" + ISSUED_BY="${ARG3:-cli/ops-center/node-bootstrap-plan}" + EXPIRES_IN_HOURS="${ARG4:-72}" + CONTROL_PLANE_BASE_URL="${ARG5:-${MAINLAND_BASE_URL}}" + ROOT_DIR="${ARG6:-/opt/domaincheck}" + else + MAINLAND_BASE_URL="$(ops_center_resolve_mainland_api_base_url)" + NODE_CODE="${ARG1:-}" + ISSUED_BY="${ARG2:-cli/ops-center/node-bootstrap-plan}" + EXPIRES_IN_HOURS="${ARG3:-72}" + CONTROL_PLANE_BASE_URL="${ARG4:-${MAINLAND_BASE_URL}}" + ROOT_DIR="${ARG5:-/opt/domaincheck}" + fi + build_node_handover_bootstrap_plan \ + "${MAINLAND_BASE_URL}" \ + "${NODE_CODE}" \ + "${ISSUED_BY}" \ + "${EXPIRES_IN_HOURS}" \ + "${CONTROL_PLANE_BASE_URL}" \ + "${ROOT_DIR}" + ;; + queue-status) + MAINLAND_BASE_URL="${1:-$(ops_center_resolve_mainland_api_base_url)}" + NODE_CODE="${2:-}" + if [[ "${MAINLAND_BASE_URL}" != http://* && "${MAINLAND_BASE_URL}" != https://* ]]; then + NODE_CODE="${MAINLAND_BASE_URL}" + MAINLAND_BASE_URL="$(ops_center_resolve_mainland_api_base_url)" + fi + show_delivery_queue_status "${MAINLAND_BASE_URL}" "${NODE_CODE}" + ;; + queue-records) + MAINLAND_BASE_URL="${1:-$(ops_center_resolve_mainland_api_base_url)}" + NODE_CODE="${2:-}" + STATE_FILTER="${3:-}" + if [[ "${MAINLAND_BASE_URL}" != http://* && "${MAINLAND_BASE_URL}" != https://* ]]; then + STATE_FILTER="${NODE_CODE:-}" + NODE_CODE="${MAINLAND_BASE_URL}" + MAINLAND_BASE_URL="$(ops_center_resolve_mainland_api_base_url)" + fi + show_delivery_queue_records "${MAINLAND_BASE_URL}" "${NODE_CODE}" "${STATE_FILTER}" + ;; + queue-flush) + MAINLAND_BASE_URL="${1:-$(ops_center_resolve_mainland_api_base_url)}" + NODE_CODE="${2:-}" + LIMIT="${3:-20}" + REQUESTED_BY="${4:-$(ops_center_resolve_requested_by)}" + if [[ "${MAINLAND_BASE_URL}" != http://* && "${MAINLAND_BASE_URL}" != https://* ]]; then + REQUESTED_BY="${LIMIT:-$(ops_center_resolve_requested_by)}" + LIMIT="${NODE_CODE:-20}" + NODE_CODE="${MAINLAND_BASE_URL}" + MAINLAND_BASE_URL="$(ops_center_resolve_mainland_api_base_url)" + fi + BODY_JSON="$(python3 - <<'PY' "${LIMIT}" "${REQUESTED_BY}" +import json +import sys +print(json.dumps({ + "limit": max(1, min(int(sys.argv[1] or 20), 200)), + "requested_by": str(sys.argv[2] or "cli/ops-center/queue-flush"), +}, ensure_ascii=False)) +PY +)" + run_delivery_queue_action "${MAINLAND_BASE_URL}" "${NODE_CODE}" "/delivery-queue/flush" "${BODY_JSON}" + ;; + queue-replay) + MAINLAND_BASE_URL="${1:-$(ops_center_resolve_mainland_api_base_url)}" + NODE_CODE="${2:-}" + LIMIT="${3:-20}" + REQUESTED_BY="${4:-$(ops_center_resolve_requested_by)}" + if [[ "${MAINLAND_BASE_URL}" != http://* && "${MAINLAND_BASE_URL}" != https://* ]]; then + REQUESTED_BY="${LIMIT:-$(ops_center_resolve_requested_by)}" + LIMIT="${NODE_CODE:-20}" + NODE_CODE="${MAINLAND_BASE_URL}" + MAINLAND_BASE_URL="$(ops_center_resolve_mainland_api_base_url)" + fi + BODY_JSON="$(python3 - <<'PY' "${LIMIT}" "${REQUESTED_BY}" +import json +import sys +print(json.dumps({ + "limit": max(1, min(int(sys.argv[1] or 20), 200)), + "flush_after_replay": True, + "reason": "cli quick replay from drive_ops_center.sh", + "requested_by": str(sys.argv[2] or "cli/ops-center/queue-replay"), +}, ensure_ascii=False)) +PY +)" + run_delivery_queue_action "${MAINLAND_BASE_URL}" "${NODE_CODE}" "/delivery-queue/replay" "${BODY_JSON}" + ;; + queue-replay-record) + MAINLAND_BASE_URL="${1:-$(ops_center_resolve_mainland_api_base_url)}" + NODE_CODE="${2:-}" + RECORD_ID="${3:-}" + REQUESTED_BY="${4:-$(ops_center_resolve_requested_by)}" + if [[ "${MAINLAND_BASE_URL}" != http://* && "${MAINLAND_BASE_URL}" != https://* ]]; then + REQUESTED_BY="${RECORD_ID:-$(ops_center_resolve_requested_by)}" + RECORD_ID="${NODE_CODE:-}" + NODE_CODE="${MAINLAND_BASE_URL}" + MAINLAND_BASE_URL="$(ops_center_resolve_mainland_api_base_url)" + fi + if [[ -z "${RECORD_ID}" ]]; then + echo "record_id is required" >&2 + exit 1 + fi + BODY_JSON="$(python3 - <<'PY' "${REQUESTED_BY}" +import json +import sys +print(json.dumps({ + "limit": 1, + "flush_after_replay": True, + "reason": "cli replay visible head record", + "requested_by": str(sys.argv[1] or "cli/ops-center/queue-replay-record"), +}, ensure_ascii=False)) +PY +)" + run_delivery_queue_action "${MAINLAND_BASE_URL}" "${NODE_CODE}" "/delivery-queue/records/${RECORD_ID}/replay" "${BODY_JSON}" + ;; + queue-discard-record) + MAINLAND_BASE_URL="${1:-$(ops_center_resolve_mainland_api_base_url)}" + NODE_CODE="${2:-}" + RECORD_ID="${3:-}" + DISCARD_REASON="${4:-}" + REQUESTED_BY="${5:-$(ops_center_resolve_requested_by)}" + if [[ "${MAINLAND_BASE_URL}" != http://* && "${MAINLAND_BASE_URL}" != https://* ]]; then + REQUESTED_BY="${DISCARD_REASON:-$(ops_center_resolve_requested_by)}" + DISCARD_REASON="${RECORD_ID:-}" + RECORD_ID="${NODE_CODE:-}" + NODE_CODE="${MAINLAND_BASE_URL}" + MAINLAND_BASE_URL="$(ops_center_resolve_mainland_api_base_url)" + fi + if [[ -z "${RECORD_ID}" || -z "${DISCARD_REASON}" ]]; then + echo "record_id and reason are required" >&2 + exit 1 + fi + BODY_JSON="$(python3 - <<'PY' "${DISCARD_REASON}" "${REQUESTED_BY}" +import json +import sys +print(json.dumps({ + "limit": 1, + "reason": str(sys.argv[1] or "").strip(), + "discarded_by": "cli", + "requested_by": str(sys.argv[2] or "cli/ops-center/queue-discard-record"), +}, ensure_ascii=False)) +PY +)" + run_delivery_queue_action "${MAINLAND_BASE_URL}" "${NODE_CODE}" "/delivery-queue/records/${RECORD_ID}/discard" "${BODY_JSON}" + ;; + activity-stream) + if [[ $# -gt 0 && ( "$1" == http://* || "$1" == https://* ) ]]; then + MAINLAND_BASE_URL="$1" + shift || true + else + MAINLAND_BASE_URL="$(ops_center_resolve_mainland_api_base_url)" + fi + show_activity_stream "${MAINLAND_BASE_URL}" "$@" + ;; + activity-preview) + MAINLAND_BASE_URL="${1:-$(ops_center_resolve_mainland_api_base_url)}" + ACTIVITY_KEY="${2:-}" + REQUESTED_BY="${3:-$(ops_center_resolve_requested_by)}" + if [[ "${MAINLAND_BASE_URL}" != http://* && "${MAINLAND_BASE_URL}" != https://* ]]; then + ACTIVITY_KEY="${MAINLAND_BASE_URL}" + MAINLAND_BASE_URL="$(ops_center_resolve_mainland_api_base_url)" + REQUESTED_BY="${2:-$(ops_center_resolve_requested_by)}" + fi + preview_activity_item "${MAINLAND_BASE_URL}" "${ACTIVITY_KEY}" "${REQUESTED_BY}" + ;; + driver-feed) + MAINLAND_BASE_URL="${1:-$(ops_center_resolve_mainland_api_base_url)}" + RAW_HTTP_RESPONSE="$(request_with_status "GET" "${MAINLAND_BASE_URL}/api/v1/ops/driver-feed")" + HTTP_STATUS="${RAW_HTTP_RESPONSE##*$'\n'HTTP_STATUS=}" + RESPONSE_JSON="${RAW_HTTP_RESPONSE%$'\n'HTTP_STATUS=*}" + RESPONSE_JSON_FILE="$(mktemp)" + printf '%s' "${RESPONSE_JSON}" > "${RESPONSE_JSON_FILE}" + echo "[1/2] raw response" + printf '%s\n' "http_status=${HTTP_STATUS}" + python3 - <<'PY' "${RESPONSE_JSON_FILE}" +import json +import sys +with open(sys.argv[1], "r", encoding="utf-8") as fh: + payload = json.loads(fh.read() or "{}") +print(json.dumps(payload, ensure_ascii=False, indent=2)) +PY + echo + echo "[2/2] condensed summary" + python3 - <<'PY' "${RESPONSE_JSON_FILE}" "${HTTP_STATUS}" +import json +import sys + +with open(sys.argv[1], "r", encoding="utf-8") as fh: + payload = json.loads(fh.read() or "{}") +http_status = int(sys.argv[2] or 0) +data = payload.get("data") or {} +summary = data.get("summary") or {} +entries = list(data.get("entries") or []) +activity_focus = list(data.get("activity_focus") or []) +top_recommendation = data.get("top_recommendation") or {} +scene_log_observation = data.get("scene_log_observation") or {} +contract_navigation = data.get("contract_navigation") or {} +print(json.dumps({ + "http_status": http_status, + "headline": str(data.get("headline") or ""), + "generated_at": str(data.get("generated_at") or ""), + "contract_navigation": { + "primary_contract_key": str(contract_navigation.get("primary_contract_key") or ""), + "contract_keys": list(contract_navigation.get("contract_keys") or []), + "detail_endpoint_pattern": str(contract_navigation.get("detail_endpoint_pattern") or ""), + }, + "top_recommendation": { + "key": str(top_recommendation.get("key") or ""), + "kind": str(top_recommendation.get("kind") or ""), + "lane": str(top_recommendation.get("lane") or ""), + "title": str(top_recommendation.get("title") or ""), + "primary_contract_key": str((top_recommendation.get("contract_navigation") or {}).get("primary_contract_key") or ""), + "contract_keys": list((top_recommendation.get("contract_navigation") or {}).get("contract_keys") or []), + }, + "summary": { + "total": int(summary.get("total", 0) or 0), + "do_now_total": int(summary.get("do_now_total", 0) or 0), + "standard_path_total": int(summary.get("standard_path_total", 0) or 0), + "publish_ready": bool(summary.get("publish_ready", False)), + "publish_status": str(summary.get("publish_status") or ""), + "publish_status_label": str(summary.get("publish_status_label") or ""), + "launchpad_recommended_target_node_code": str(summary.get("launchpad_recommended_target_node_code") or ""), + "launchpad_recommended_recovery_label": str(summary.get("launchpad_recommended_recovery_label") or ""), + "launchpad_recommended_recovery_summary": str(summary.get("launchpad_recommended_recovery_summary") or ""), + "launchpad_onboarding_bootstrap_pending_nodes": int(summary.get("launchpad_onboarding_bootstrap_pending_nodes", 0) or 0), + "launchpad_onboarding_acceptance_ready_nodes": int(summary.get("launchpad_onboarding_acceptance_ready_nodes", 0) or 0), + "scene_log_status": str(summary.get("scene_log_status") or ""), + "kind_counts": dict(summary.get("kind_counts") or {}), + }, + "scene_log_observation": { + "status": str(scene_log_observation.get("status") or ""), + "status_label": str(scene_log_observation.get("status_label") or ""), + "summary": str(scene_log_observation.get("summary") or ""), + "target_node_code": str(scene_log_observation.get("target_node_code") or ""), + "participating_node_count": int(scene_log_observation.get("participating_node_count", 0) or 0), + "covered_participating_node_count": int(scene_log_observation.get("covered_participating_node_count", 0) or 0), + "line_count": int(scene_log_observation.get("line_count", 0) or 0), + "focus_ref": scene_log_observation.get("focus_ref") or {}, + "primary_contract_key": str((scene_log_observation.get("contract_navigation") or {}).get("primary_contract_key") or ""), + "contract_keys": list((scene_log_observation.get("contract_navigation") or {}).get("contract_keys") or []), + }, + "entries": [ + { + "key": str(item.get("key") or ""), + "kind": str(item.get("kind") or ""), + "lane": str(item.get("lane") or ""), + "title": str(item.get("title") or ""), + "status": str(item.get("status") or ""), + "primary_label": str(item.get("primary_label") or ""), + "secondary_label": str(item.get("secondary_label") or ""), + "primary_contract_key": str((item.get("contract_navigation") or {}).get("primary_contract_key") or ""), + "contract_keys": list((item.get("contract_navigation") or {}).get("contract_keys") or []), + } + for item in entries + ], + "activity_focus": [ + { + "key": str(item.get("key") or ""), + "title": str(item.get("title") or ""), + "status": str(item.get("status") or ""), + "primary_contract_key": str((item.get("contract_navigation") or {}).get("primary_contract_key") or ""), + "contract_keys": list((item.get("contract_navigation") or {}).get("contract_keys") or []), + } + for item in activity_focus + ], +}, ensure_ascii=False, indent=2)) +PY + rm -f "${RESPONSE_JSON_FILE}" + ;; + driver-focus-preview) + MAINLAND_BASE_URL="${1:-$(ops_center_resolve_mainland_api_base_url)}" + ENTRY_KEY="${2:-}" + REQUESTED_BY="${3:-$(ops_center_resolve_requested_by)}" + if [[ "${MAINLAND_BASE_URL}" != http://* && "${MAINLAND_BASE_URL}" != https://* ]]; then + ENTRY_KEY="${MAINLAND_BASE_URL}" + MAINLAND_BASE_URL="$(ops_center_resolve_mainland_api_base_url)" + REQUESTED_BY="${2:-$(ops_center_resolve_requested_by)}" + fi + preview_driver_focus_entry "${MAINLAND_BASE_URL}" "${ENTRY_KEY}" "${REQUESTED_BY}" + ;; + driver-focus-run) + MAINLAND_BASE_URL="${1:-$(ops_center_resolve_mainland_api_base_url)}" + ENTRY_KEY="${2:-}" + SECONDARY_FLAG="${3:-primary}" + CONFIRM_FLAG="${4:-}" + REQUESTED_BY="${5:-$(ops_center_resolve_requested_by)}" + if [[ "${MAINLAND_BASE_URL}" != http://* && "${MAINLAND_BASE_URL}" != https://* ]]; then + ENTRY_KEY="${MAINLAND_BASE_URL}" + MAINLAND_BASE_URL="$(ops_center_resolve_mainland_api_base_url)" + SECONDARY_FLAG="${2:-primary}" + CONFIRM_FLAG="${3:-}" + REQUESTED_BY="${4:-$(ops_center_resolve_requested_by)}" + fi + if is_secondary_token "${ENTRY_KEY}"; then + REQUESTED_BY="${CONFIRM_FLAG:-$(ops_center_resolve_requested_by)}" + CONFIRM_FLAG="${SECONDARY_FLAG:-}" + SECONDARY_FLAG="${ENTRY_KEY}" + ENTRY_KEY="" + fi + if is_confirm_token "${SECONDARY_FLAG}" && [[ -z "${CONFIRM_FLAG}" ]]; then + CONFIRM_FLAG="${SECONDARY_FLAG}" + SECONDARY_FLAG="primary" + fi + run_driver_focus_entry "${MAINLAND_BASE_URL}" "${ENTRY_KEY}" "${SECONDARY_FLAG}" "${CONFIRM_FLAG}" "${REQUESTED_BY}" + ;; + codex-brief) + MAINLAND_BASE_URL="${1:-$(ops_center_resolve_mainland_api_base_url)}" + RAW_HTTP_RESPONSE="$( + request_with_status_custom \ + "GET" \ + "${MAINLAND_BASE_URL}/api/v1/ops/codex-brief" \ + "" \ + "${OPS_CENTER_CURL_CONNECT_TIMEOUT}" \ + "${OPS_CENTER_CURL_MAX_TIME_CODEX_BRIEF}" + )" + HTTP_STATUS="${RAW_HTTP_RESPONSE##*$'\n'HTTP_STATUS=}" + RESPONSE_JSON="${RAW_HTTP_RESPONSE%$'\n'HTTP_STATUS=*}" + RESPONSE_JSON_FILE="$(mktemp)" + printf '%s' "${RESPONSE_JSON}" > "${RESPONSE_JSON_FILE}" + echo "[1/2] raw response" + printf '%s\n' "http_status=${HTTP_STATUS}" + python3 - <<'PY' "${RESPONSE_JSON_FILE}" +import json +import sys +with open(sys.argv[1], "r", encoding="utf-8") as fh: + payload = json.loads(fh.read() or "{}") +print(json.dumps(payload, ensure_ascii=False, indent=2)) +PY + echo + echo "[2/2] condensed summary" + python3 - <<'PY' "${RESPONSE_JSON_FILE}" "${HTTP_STATUS}" +import json +import sys + +with open(sys.argv[1], "r", encoding="utf-8") as fh: + payload = json.loads(fh.read() or "{}") +http_status = int(sys.argv[2] or 0) +data = payload.get("data") or {} +summary = data.get("summary") or {} +focus = data.get("focus") or {} +entries = list(data.get("entries") or []) +activity_focus = list(data.get("activity_focus") or []) +scene_log_observation = data.get("scene_log_observation") or {} +contract_navigation = data.get("contract_navigation") or {} +print(json.dumps({ + "http_status": http_status, + "headline": str(data.get("headline") or ""), + "generated_at": str(data.get("generated_at") or ""), + "contract_navigation": { + "primary_contract_key": str(contract_navigation.get("primary_contract_key") or ""), + "contract_keys": list(contract_navigation.get("contract_keys") or []), + "detail_endpoint_pattern": str(contract_navigation.get("detail_endpoint_pattern") or ""), + }, + "focus": { + "entry_key": str(focus.get("entry_key") or ""), + "title": str(focus.get("title") or ""), + "recommendation": str(focus.get("recommendation") or ""), + "status_text": str(focus.get("status_text") or ""), + "target_method": str(focus.get("target_method") or ""), + "target_api": str(focus.get("target_api") or ""), + "focus_ref": focus.get("focus_ref") or {}, + "primary_focus_ref": focus.get("primary_focus_ref") or {}, + "secondary_focus_ref": focus.get("secondary_focus_ref") or {}, + "primary_contract_key": str((focus.get("contract_navigation") or {}).get("primary_contract_key") or ""), + "contract_keys": list((focus.get("contract_navigation") or {}).get("contract_keys") or []), + }, + "summary": { + "total": int(summary.get("total", 0) or 0), + "go_live_status": str(summary.get("go_live_status") or ""), + "publish_ready": bool(summary.get("publish_ready", False)), + "publish_status": str(summary.get("publish_status") or ""), + "publish_status_label": str(summary.get("publish_status_label") or ""), + "launchpad_recommended_target_node_code": str(summary.get("launchpad_recommended_target_node_code") or ""), + "launchpad_recommended_recovery_label": str(summary.get("launchpad_recommended_recovery_label") or ""), + "launchpad_recommended_recovery_summary": str(summary.get("launchpad_recommended_recovery_summary") or ""), + "launchpad_onboarding_bootstrap_pending_nodes": int(summary.get("launchpad_onboarding_bootstrap_pending_nodes", 0) or 0), + "launchpad_onboarding_acceptance_ready_nodes": int(summary.get("launchpad_onboarding_acceptance_ready_nodes", 0) or 0), + "auto_execute_total": int(summary.get("auto_execute_total", 0) or 0), + "confirm_then_execute_total": int(summary.get("confirm_then_execute_total", 0) or 0), + "resolve_first_total": int(summary.get("resolve_first_total", 0) or 0), + "open_ui_total": int(summary.get("open_ui_total", 0) or 0), + "blocked_total": int(summary.get("blocked_total", 0) or 0), + "automation_level_counts": dict(summary.get("automation_level_counts") or {}), + }, + "scene_log_observation": { + "status": str(scene_log_observation.get("status") or ""), + "status_label": str(scene_log_observation.get("status_label") or ""), + "summary": str(scene_log_observation.get("summary") or ""), + "target_node_code": str(scene_log_observation.get("target_node_code") or ""), + "participating_node_count": int(scene_log_observation.get("participating_node_count", 0) or 0), + "covered_participating_node_count": int(scene_log_observation.get("covered_participating_node_count", 0) or 0), + "line_count": int(scene_log_observation.get("line_count", 0) or 0), + "focus_ref": scene_log_observation.get("focus_ref") or {}, + "primary_contract_key": str((scene_log_observation.get("contract_navigation") or {}).get("primary_contract_key") or ""), + "contract_keys": list((scene_log_observation.get("contract_navigation") or {}).get("contract_keys") or []), + }, + "entries": [ + { + "key": str(item.get("key") or ""), + "kind": str(item.get("kind") or ""), + "title": str(item.get("title") or ""), + "executor_kind": str(item.get("executor_kind") or ""), + "recommendation": str((item.get("autopilot") or {}).get("recommendation") or ""), + "automation_level": str((item.get("autopilot") or {}).get("automation_level") or ""), + "executor_mode_hint": str((item.get("autopilot") or {}).get("executor_mode_hint") or ""), + "primary_action_code": str(item.get("primary_action_code") or ""), + "focus_ref": item.get("focus_ref") or {}, + "primary_focus_ref": item.get("primary_focus_ref") or {}, + "secondary_focus_ref": item.get("secondary_focus_ref") or {}, + "primary_contract_key": str((item.get("contract_navigation") or {}).get("primary_contract_key") or ""), + "contract_keys": list((item.get("contract_navigation") or {}).get("contract_keys") or []), + } + for item in entries + ], + "activity_focus": [ + { + "key": str(item.get("key") or ""), + "title": str(item.get("title") or ""), + "status": str(item.get("status") or ""), + "primary_contract_key": str((item.get("contract_navigation") or {}).get("primary_contract_key") or ""), + "contract_keys": list((item.get("contract_navigation") or {}).get("contract_keys") or []), + } + for item in activity_focus + ], +}, ensure_ascii=False, indent=2)) +PY + rm -f "${RESPONSE_JSON_FILE}" + ;; + codex-focus-preview) + MAINLAND_BASE_URL="${1:-$(ops_center_resolve_mainland_api_base_url)}" + ENTRY_KEY="${2:-}" + REQUESTED_BY="${3:-$(ops_center_resolve_requested_by)}" + if [[ "${MAINLAND_BASE_URL}" != http://* && "${MAINLAND_BASE_URL}" != https://* ]]; then + ENTRY_KEY="${MAINLAND_BASE_URL}" + MAINLAND_BASE_URL="$(ops_center_resolve_mainland_api_base_url)" + REQUESTED_BY="${2:-$(ops_center_resolve_requested_by)}" + fi + preview_codex_focus_entry "${MAINLAND_BASE_URL}" "${ENTRY_KEY}" "${REQUESTED_BY}" + ;; + codex-focus-run) + MAINLAND_BASE_URL="${1:-$(ops_center_resolve_mainland_api_base_url)}" + ENTRY_KEY="${2:-}" + CONFIRM_FLAG="${3:-}" + REQUESTED_BY="${4:-$(ops_center_resolve_requested_by)}" + if [[ "${MAINLAND_BASE_URL}" != http://* && "${MAINLAND_BASE_URL}" != https://* ]]; then + ENTRY_KEY="${MAINLAND_BASE_URL}" + MAINLAND_BASE_URL="$(ops_center_resolve_mainland_api_base_url)" + CONFIRM_FLAG="${2:-}" + REQUESTED_BY="${3:-$(ops_center_resolve_requested_by)}" + fi + run_codex_focus_entry "${MAINLAND_BASE_URL}" "${ENTRY_KEY}" "${CONFIRM_FLAG}" "${REQUESTED_BY}" + ;; + release-hub) + MAINLAND_BASE_URL="${1:-$(ops_center_resolve_mainland_api_base_url)}" + shift || true + exec bash "${SCRIPT_DIR}/drive_release_hub.sh" "${MAINLAND_BASE_URL}" "$@" + ;; + driver-preview) + MAINLAND_BASE_URL="${1:-$(ops_center_resolve_mainland_api_base_url)}" + ACTION_CODE="${2:-}" + ACTION_PAYLOAD_JSON="${3:-}" + REQUESTED_BY="${4:-$(ops_center_resolve_requested_by)}" + if [[ -z "${ACTION_CODE}" && "${MAINLAND_BASE_URL}" != http://* && "${MAINLAND_BASE_URL}" != https://* ]]; then + ACTION_CODE="${MAINLAND_BASE_URL}" + MAINLAND_BASE_URL="$(ops_center_resolve_mainland_api_base_url)" + ACTION_PAYLOAD_JSON="${2:-}" + REQUESTED_BY="${3:-$(ops_center_resolve_requested_by)}" + fi + if [[ -z "${ACTION_PAYLOAD_JSON}" ]]; then + ACTION_PAYLOAD_JSON='{}' + fi + if [[ -z "${ACTION_CODE}" ]]; then + usage + exit 1 + fi + exec bash "${SCRIPT_DIR}/drive_ops_action.sh" "${MAINLAND_BASE_URL}" preview "${ACTION_CODE}" "${ACTION_PAYLOAD_JSON}" "${REQUESTED_BY}" + ;; + driver-resolve) + MAINLAND_BASE_URL="${1:-$(ops_center_resolve_mainland_api_base_url)}" + ACTION_CODE="${2:-}" + ACTION_PAYLOAD_JSON="${3:-}" + REQUESTED_BY="${4:-$(ops_center_resolve_requested_by)}" + if [[ -z "${ACTION_CODE}" && "${MAINLAND_BASE_URL}" != http://* && "${MAINLAND_BASE_URL}" != https://* ]]; then + ACTION_CODE="${MAINLAND_BASE_URL}" + MAINLAND_BASE_URL="$(ops_center_resolve_mainland_api_base_url)" + ACTION_PAYLOAD_JSON="${2:-}" + REQUESTED_BY="${3:-$(ops_center_resolve_requested_by)}" + fi + if [[ -z "${ACTION_CODE}" ]]; then + usage + exit 1 + fi + if [[ -z "${ACTION_PAYLOAD_JSON}" ]]; then + ACTION_PAYLOAD_JSON='{}' + fi + resolve_driver_action_entry "${MAINLAND_BASE_URL}" "${ACTION_CODE}" "${ACTION_PAYLOAD_JSON}" "${REQUESTED_BY}" "cli/driver-resolve" + ;; + driver-run) + MAINLAND_BASE_URL="${1:-$(ops_center_resolve_mainland_api_base_url)}" + ACTION_CODE="${2:-}" + ACTION_PAYLOAD_JSON="${3:-}" + CONFIRM_FLAG="${4:-}" + REQUESTED_BY="${5:-$(ops_center_resolve_requested_by)}" + if [[ -z "${ACTION_CODE}" && "${MAINLAND_BASE_URL}" != http://* && "${MAINLAND_BASE_URL}" != https://* ]]; then + ACTION_CODE="${MAINLAND_BASE_URL}" + MAINLAND_BASE_URL="$(ops_center_resolve_mainland_api_base_url)" + ACTION_PAYLOAD_JSON="${2:-}" + CONFIRM_FLAG="${3:-}" + REQUESTED_BY="${4:-$(ops_center_resolve_requested_by)}" + fi + if is_confirm_token "${ACTION_PAYLOAD_JSON}" && [[ -z "${CONFIRM_FLAG}" ]]; then + CONFIRM_FLAG="${ACTION_PAYLOAD_JSON}" + ACTION_PAYLOAD_JSON='{}' + fi + if [[ -z "${ACTION_CODE}" ]]; then + usage + exit 1 + fi + if [[ -z "${ACTION_PAYLOAD_JSON}" ]]; then + ACTION_PAYLOAD_JSON='{}' + fi + run_driver_action_entry "${MAINLAND_BASE_URL}" "${ACTION_CODE}" "${ACTION_PAYLOAD_JSON}" "${CONFIRM_FLAG}" "${REQUESTED_BY}" "cli/driver-run" + ;; + driver) + MAINLAND_BASE_URL="${1:-$(ops_center_resolve_mainland_api_base_url)}" + ACTION_CODE="${2:-}" + ACTION_PAYLOAD_JSON="${3:-}" + REQUESTED_BY="${4:-$(ops_center_resolve_requested_by)}" + if [[ -z "${ACTION_CODE}" && "${MAINLAND_BASE_URL}" != http://* && "${MAINLAND_BASE_URL}" != https://* ]]; then + ACTION_CODE="${MAINLAND_BASE_URL}" + MAINLAND_BASE_URL="$(ops_center_resolve_mainland_api_base_url)" + ACTION_PAYLOAD_JSON="${2:-}" + REQUESTED_BY="${3:-$(ops_center_resolve_requested_by)}" + fi + if [[ -z "${ACTION_PAYLOAD_JSON}" ]]; then + ACTION_PAYLOAD_JSON='{}' + fi + if [[ -z "${ACTION_CODE}" ]]; then + usage + exit 1 + fi + exec bash "${SCRIPT_DIR}/drive_ops_action.sh" "${MAINLAND_BASE_URL}" "${ACTION_CODE}" "${ACTION_PAYLOAD_JSON}" "${REQUESTED_BY}" + ;; + runbook-preview) + MAINLAND_BASE_URL="${1:-$(ops_center_resolve_mainland_api_base_url)}" + SEQUENCE_KEY="${2:-}" + ACTION_PAYLOAD_JSON="${3:-}" + REQUESTED_BY="${4:-$(ops_center_resolve_requested_by)}" + if [[ -z "${SEQUENCE_KEY}" && "${MAINLAND_BASE_URL}" != http://* && "${MAINLAND_BASE_URL}" != https://* ]]; then + SEQUENCE_KEY="${MAINLAND_BASE_URL}" + MAINLAND_BASE_URL="$(ops_center_resolve_mainland_api_base_url)" + ACTION_PAYLOAD_JSON="${2:-}" + REQUESTED_BY="${3:-$(ops_center_resolve_requested_by)}" + fi + if [[ -z "${ACTION_PAYLOAD_JSON}" ]]; then + ACTION_PAYLOAD_JSON='{}' + fi + if [[ -z "${SEQUENCE_KEY}" ]]; then + echo "sequence_key is required" >&2 + usage + exit 1 + fi + preview_runbook_sequence "${MAINLAND_BASE_URL}" "${SEQUENCE_KEY}" "${ACTION_PAYLOAD_JSON}" "${REQUESTED_BY}" + ;; + runbook-run) + MAINLAND_BASE_URL="${1:-$(ops_center_resolve_mainland_api_base_url)}" + SEQUENCE_KEY="${2:-}" + SECONDARY_FLAG="${3:-primary}" + ACTION_PAYLOAD_JSON="${4:-}" + CONFIRM_FLAG="${5:-}" + REQUESTED_BY="${6:-$(ops_center_resolve_requested_by)}" + if [[ -z "${SEQUENCE_KEY}" && "${MAINLAND_BASE_URL}" != http://* && "${MAINLAND_BASE_URL}" != https://* ]]; then + SEQUENCE_KEY="${MAINLAND_BASE_URL}" + MAINLAND_BASE_URL="$(ops_center_resolve_mainland_api_base_url)" + SECONDARY_FLAG="${2:-primary}" + ACTION_PAYLOAD_JSON="${3:-}" + CONFIRM_FLAG="${4:-}" + REQUESTED_BY="${5:-$(ops_center_resolve_requested_by)}" + fi + if is_confirm_token "${ACTION_PAYLOAD_JSON}" && [[ -z "${CONFIRM_FLAG}" ]]; then + CONFIRM_FLAG="${ACTION_PAYLOAD_JSON}" + ACTION_PAYLOAD_JSON='{}' + fi + if [[ -z "${SEQUENCE_KEY}" ]]; then + echo "sequence_key is required" >&2 + usage + exit 1 + fi + if [[ -z "${ACTION_PAYLOAD_JSON}" ]]; then + ACTION_PAYLOAD_JSON='{}' + fi + run_resolved_runbook_sequence "${MAINLAND_BASE_URL}" "${SEQUENCE_KEY}" "${SECONDARY_FLAG}" "${ACTION_PAYLOAD_JSON}" "${CONFIRM_FLAG}" "${REQUESTED_BY}" "cli/runbook-run" + ;; + runbook-resolve) + MAINLAND_BASE_URL="${1:-$(ops_center_resolve_mainland_api_base_url)}" + SEQUENCE_KEY="${2:-}" + SECONDARY_FLAG="${3:-primary}" + ACTION_PAYLOAD_JSON="${4:-}" + REQUESTED_BY="${5:-$(ops_center_resolve_requested_by)}" + if [[ -z "${SEQUENCE_KEY}" && "${MAINLAND_BASE_URL}" != http://* && "${MAINLAND_BASE_URL}" != https://* ]]; then + SEQUENCE_KEY="${MAINLAND_BASE_URL}" + MAINLAND_BASE_URL="$(ops_center_resolve_mainland_api_base_url)" + SECONDARY_FLAG="${2:-primary}" + ACTION_PAYLOAD_JSON="${3:-}" + REQUESTED_BY="${4:-$(ops_center_resolve_requested_by)}" + fi + if [[ -z "${ACTION_PAYLOAD_JSON}" ]]; then + ACTION_PAYLOAD_JSON='{}' + fi + if [[ -z "${SEQUENCE_KEY}" ]]; then + echo "sequence_key is required" >&2 + usage + exit 1 + fi + if [[ "${ACTION_PAYLOAD_JSON}" == @* ]]; then + ACTION_PAYLOAD_PATH="${ACTION_PAYLOAD_JSON#@}" + if [[ ! -f "${ACTION_PAYLOAD_PATH}" ]]; then + echo "payload file not found: ${ACTION_PAYLOAD_PATH}" >&2 + exit 1 + fi + ACTION_PAYLOAD_JSON="$(cat "${ACTION_PAYLOAD_PATH}")" + fi + REQUEST_JSON="$(python3 - <<'PY' "${SECONDARY_FLAG}" "${ACTION_PAYLOAD_JSON}" "${REQUESTED_BY}" +import json +import sys + +secondary_flag = str(sys.argv[1] or "primary").strip().lower() +payload_raw = str(sys.argv[2] or "").strip() or "{}" +requested_by = str(sys.argv[3] or "").strip() or "cli/drive_ops_center" + +try: + action_payload = json.loads(payload_raw) +except Exception as exc: + raise SystemExit(f"invalid action payload json: {exc}") + +if not isinstance(action_payload, dict): + raise SystemExit("action payload must be a json object") + +request_payload = { + "secondary": secondary_flag in {"secondary", "second", "2", "true", "yes", "y"}, + "action_payload": action_payload, + "requested_by": requested_by, +} +print(json.dumps(request_payload, ensure_ascii=False)) +PY +)" + echo "[1/3] runbook resolve request" + python3 - <<'PY' "${MAINLAND_BASE_URL}" "${SEQUENCE_KEY}" "${REQUEST_JSON}" +import json +import sys +print(json.dumps({ + "base_url": str(sys.argv[1] or ""), + "sequence_key": str(sys.argv[2] or ""), + "request": json.loads(str(sys.argv[3] or "{}")), +}, ensure_ascii=False, indent=2)) +PY + echo + echo + RAW_HTTP_RESPONSE="$(request_with_status "POST" "${MAINLAND_BASE_URL}/api/v1/ops/runbook/sequences/${SEQUENCE_KEY}/resolve" "${REQUEST_JSON}")" + HTTP_STATUS="${RAW_HTTP_RESPONSE##*$'\n'HTTP_STATUS=}" + RESPONSE_JSON="${RAW_HTTP_RESPONSE%$'\n'HTTP_STATUS=*}" + echo "[2/3] raw response" + printf '%s\n' "http_status=${HTTP_STATUS}" + python3 - <<'PY' "${RESPONSE_JSON}" +import json +import sys +payload = json.loads(str(sys.argv[1] or "{}")) +print(json.dumps(payload, ensure_ascii=False, indent=2)) +PY + echo + echo + echo "[3/3] condensed summary" + python3 - <<'PY' "${RESPONSE_JSON}" "${HTTP_STATUS}" +import json +import sys + +payload = json.loads(str(sys.argv[1] or "{}")) +http_status = int(sys.argv[2] or 0) +code = payload.get("code", 1) +if code in (None, ""): + code = 0 if http_status == 200 else 1 +data = payload.get("data") or {} +print(json.dumps({ + "http_status": http_status, + "code": int(code or 0), + "message": str(payload.get("message") or ""), + "sequence_key": str(data.get("sequence_key") or ""), + "driver_action_code": str(data.get("driver_action_code") or ""), + "driver_action_label": str(data.get("driver_action_label") or ""), + "driver_node_codes": list(data.get("driver_node_codes") or []), + "driver_requested_by": str(data.get("driver_requested_by") or ""), + "resolved_at": str(data.get("resolved_at") or ""), + "sequence_focus_ref": ((data.get("sequence") or {}).get("focus_ref") or {}), + "action_focus_ref": ((data.get("sequence") or {}).get("action_focus_ref") or {}), +}, ensure_ascii=False, indent=2)) +PY + ;; + runbook-sequence) + MAINLAND_BASE_URL="${1:-$(ops_center_resolve_mainland_api_base_url)}" + SEQUENCE_KEY="${2:-}" + SECONDARY_FLAG="${3:-primary}" + ACTION_PAYLOAD_JSON="${4:-}" + REQUESTED_BY="${5:-$(ops_center_resolve_requested_by)}" + if [[ -z "${SEQUENCE_KEY}" && "${MAINLAND_BASE_URL}" != http://* && "${MAINLAND_BASE_URL}" != https://* ]]; then + SEQUENCE_KEY="${MAINLAND_BASE_URL}" + MAINLAND_BASE_URL="$(ops_center_resolve_mainland_api_base_url)" + SECONDARY_FLAG="${2:-primary}" + ACTION_PAYLOAD_JSON="${3:-}" + REQUESTED_BY="${4:-$(ops_center_resolve_requested_by)}" + fi + if [[ -z "${ACTION_PAYLOAD_JSON}" ]]; then + ACTION_PAYLOAD_JSON='{}' + fi + if [[ -z "${SEQUENCE_KEY}" ]]; then + echo "sequence_key is required" >&2 + usage + exit 1 + fi + if [[ "${ACTION_PAYLOAD_JSON}" == @* ]]; then + ACTION_PAYLOAD_PATH="${ACTION_PAYLOAD_JSON#@}" + if [[ ! -f "${ACTION_PAYLOAD_PATH}" ]]; then + echo "payload file not found: ${ACTION_PAYLOAD_PATH}" >&2 + exit 1 + fi + ACTION_PAYLOAD_JSON="$(cat "${ACTION_PAYLOAD_PATH}")" + fi + REQUEST_JSON="$(python3 - <<'PY' "${SECONDARY_FLAG}" "${ACTION_PAYLOAD_JSON}" "${REQUESTED_BY}" +import json +import sys + +secondary_flag = str(sys.argv[1] or "primary").strip().lower() +payload_raw = str(sys.argv[2] or "").strip() or "{}" +requested_by = str(sys.argv[3] or "").strip() or "cli/drive_ops_center" + +try: + action_payload = json.loads(payload_raw) +except Exception as exc: + raise SystemExit(f"invalid action payload json: {exc}") + +if not isinstance(action_payload, dict): + raise SystemExit("action payload must be a json object") + +request_payload = { + "secondary": secondary_flag in {"secondary", "second", "2", "true", "yes", "y"}, + "action_payload": action_payload, + "requested_by": requested_by, +} +print(json.dumps(request_payload, ensure_ascii=False)) +PY +)" + echo "[1/3] runbook sequence request" + python3 - <<'PY' "${MAINLAND_BASE_URL}" "${SEQUENCE_KEY}" "${REQUEST_JSON}" +import json +import sys +print(json.dumps({ + "base_url": str(sys.argv[1] or ""), + "sequence_key": str(sys.argv[2] or ""), + "request": json.loads(str(sys.argv[3] or "{}")), +}, ensure_ascii=False, indent=2)) +PY + echo + echo + RAW_HTTP_RESPONSE="$(request_with_status "POST" "${MAINLAND_BASE_URL}/api/v1/ops/runbook/sequences/${SEQUENCE_KEY}/execute" "${REQUEST_JSON}")" + HTTP_STATUS="${RAW_HTTP_RESPONSE##*$'\n'HTTP_STATUS=}" + RESPONSE_JSON="${RAW_HTTP_RESPONSE%$'\n'HTTP_STATUS=*}" + echo "[2/3] raw response" + printf '%s\n' "http_status=${HTTP_STATUS}" + python3 - <<'PY' "${RESPONSE_JSON}" +import json +import sys +payload = json.loads(str(sys.argv[1] or "{}")) +print(json.dumps(payload, ensure_ascii=False, indent=2)) +PY + echo + echo + echo "[3/3] condensed summary" + python3 - <<'PY' "${RESPONSE_JSON}" "${HTTP_STATUS}" +import json +import sys + +payload = json.loads(str(sys.argv[1] or "{}")) +http_status = int(sys.argv[2] or 0) +data = payload.get("data") or {} +raw_code = payload.get("code", 1) +if raw_code in (0, "0", False): + normalized_code = 0 +else: + normalized_code = int(raw_code or 1) + +summary = { + "http_status": http_status, + "code": normalized_code, + "message": str(payload.get("message") or ""), + "sequence_key": str(data.get("sequence_key") or ""), + "driver_action_code": str(data.get("driver_action_code") or ""), + "handled": bool(data.get("handled", False)), + "mode": str(data.get("mode") or ""), + "sequence": data.get("sequence") or {}, +} +print(json.dumps(summary, ensure_ascii=False, indent=2)) +PY + if [[ "${HTTP_STATUS}" -lt 200 || "${HTTP_STATUS}" -ge 300 ]]; then + exit 1 + fi + ;; + smart-latest) + MAINLAND_BASE_URL="${1:-$(ops_center_resolve_mainland_api_base_url)}" + MODE="${2:-}" + CONFIRM_FLAG="${3:-}" + if [[ "${MODE}" != "worker" && "${MODE}" != "control" ]]; then + echo "mode must be worker or control" >&2 + usage + exit 1 + fi + exec bash "${SCRIPT_DIR}/drive_release_hub.sh" "${MAINLAND_BASE_URL}" smart-latest "${MODE}" "${CONFIRM_FLAG}" + ;; + smart-release) + MAINLAND_BASE_URL="${1:-$(ops_center_resolve_mainland_api_base_url)}" + RELEASE_ID="${2:-}" + MODE="${3:-}" + CONFIRM_FLAG="${4:-}" + if [[ -z "${RELEASE_ID}" ]]; then + echo "release_id is required" >&2 + usage + exit 1 + fi + if [[ "${MODE}" != "worker" && "${MODE}" != "control" ]]; then + echo "mode must be worker or control" >&2 + usage + exit 1 + fi + exec bash "${SCRIPT_DIR}/drive_release_hub.sh" "${MAINLAND_BASE_URL}" smart-release "${RELEASE_ID}" "${MODE}" "${CONFIRM_FLAG}" + ;; + agent-plan) + MAINLAND_BASE_URL="${1:-$(ops_center_resolve_mainland_api_base_url)}" + NODE_CODE="${2:-}" + NODE_REGION="${3:-mainland}" + NODE_ROLE="${4:-worker}" + CONTROL_PLANE_BASE_URL="${5:-$(ops_center_resolve_control_plane_base_url)}" + if [[ -z "${CONTROL_PLANE_BASE_URL}" ]]; then + CONTROL_PLANE_BASE_URL="$(ops_center_resolve_overseas_api_base_url)" + fi + ROOT_DIR="${6:-$(ops_center_resolve_root_dir)}" + if [[ -z "${NODE_CODE}" ]]; then + echo "node_code is required" >&2 + usage + exit 1 + fi + ISSUED_BY="$(ops_center_resolve_issued_by)" \ + EXPIRES_IN_HOURS="$(ops_center_resolve_expires_in_hours)" \ + exec bash "${SCRIPT_DIR}/build_node_agent_bootstrap_plan.sh" \ + "${MAINLAND_BASE_URL}" \ + "${NODE_CODE}" \ + "${NODE_REGION}" \ + "${NODE_ROLE}" \ + "${CONTROL_PLANE_BASE_URL}" \ + "${ROOT_DIR}" + ;; + agent-plan-export) + REPORT_DIR="${1:-${OPS_CENTER_REPORT_ROOT}/agent-plans}" + MAINLAND_BASE_URL="${2:-$(ops_center_resolve_mainland_api_base_url)}" + NODE_CODE="${3:-}" + NODE_REGION="${4:-mainland}" + NODE_ROLE="${5:-worker}" + CONTROL_PLANE_BASE_URL="${6:-$(ops_center_resolve_control_plane_base_url)}" + if [[ -z "${CONTROL_PLANE_BASE_URL}" ]]; then + CONTROL_PLANE_BASE_URL="$(ops_center_resolve_overseas_api_base_url)" + fi + ROOT_DIR="${7:-$(ops_center_resolve_root_dir)}" + if [[ -z "${NODE_CODE}" ]]; then + echo "node_code is required" >&2 + usage + exit 1 + fi + export_agent_plan "${REPORT_DIR}" "${MAINLAND_BASE_URL}" "${NODE_CODE}" "${NODE_REGION}" "${NODE_ROLE}" "${CONTROL_PLANE_BASE_URL}" "${ROOT_DIR}" + ;; + agent-check) + SERVICE_NAME="${1:-$(ops_center_resolve_node_agent_service_name)}" + ENV_FILE="${2:-$(ops_center_resolve_node_agent_env_file)}" + SERVICE_NAME="${SERVICE_NAME}" ENV_FILE="${ENV_FILE}" exec bash "${SCRIPT_DIR}/check_node_agent.sh" + ;; + *) + echo "unknown command: ${COMMAND}" >&2 + usage + exit 1 + ;; +esac diff --git a/domain-api/deploy/multi-region/drive_release_hub.sh b/domain-api/deploy/multi-region/drive_release_hub.sh new file mode 100755 index 0000000..9c86253 --- /dev/null +++ b/domain-api/deploy/multi-region/drive_release_hub.sh @@ -0,0 +1,476 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck disable=SC1091 +source "${SCRIPT_DIR}/lib_ops_center.sh" +ops_center_load_config "${SCRIPT_DIR}" + +usage() { + cat <<'EOF' +usage: + bash domain-api/deploy/multi-region/drive_release_hub.sh [args...] + +commands: + launchpad + latest-package + latest-release + preview-latest + preview-latest-smart + preview-release + create-latest-release + list-releases [limit] + list-rollouts [limit] + smart-latest [confirm] + smart-release [confirm] + +examples: + bash domain-api/deploy/multi-region/drive_release_hub.sh http://127.0.0.1:8100 launchpad + bash domain-api/deploy/multi-region/drive_release_hub.sh http://127.0.0.1:8100 latest-package + bash domain-api/deploy/multi-region/drive_release_hub.sh http://127.0.0.1:8100 preview-latest + bash domain-api/deploy/multi-region/drive_release_hub.sh http://127.0.0.1:8100 preview-latest-smart worker + bash domain-api/deploy/multi-region/drive_release_hub.sh http://127.0.0.1:8100 create-latest-release + bash domain-api/deploy/multi-region/drive_release_hub.sh http://127.0.0.1:8100 smart-latest worker + bash domain-api/deploy/multi-region/drive_release_hub.sh http://127.0.0.1:8100 preview-release 18 control + bash domain-api/deploy/multi-region/drive_release_hub.sh http://127.0.0.1:8100 smart-release 18 control confirm +EOF +} + +BASE_URL="${1:-$(ops_center_resolve_mainland_api_base_url)}" +COMMAND="${2:-}" +ARG1="${3:-}" +ARG2="${4:-}" +ARG3="${5:-}" +REQUESTED_BY="${REQUESTED_BY:-$(ops_center_resolve_requested_by)}" + +print_json_or_raw() { + local raw_text="${1:-}" + python3 - "${raw_text}" <<'PY' +import json +import sys + +raw_text = str(sys.argv[1] or "") +try: + payload = json.loads(raw_text) +except Exception: + print(raw_text) +else: + print(json.dumps(payload, ensure_ascii=False, indent=2)) +PY +} + +request_with_status() { + local method="$1" + local url="$2" + local body="${3:-}" + if [[ "${method}" == "POST" ]]; then + curl -sS -X POST "${url}" \ + -H 'Content-Type: application/json' \ + -d "${body}" \ + -w $'\nHTTP_STATUS=%{http_code}' + else + curl -sS "${url}" -w $'\nHTTP_STATUS=%{http_code}' + fi +} + +if [[ -z "${COMMAND}" ]]; then + usage + exit 1 +fi + +METHOD="GET" +PATH_SUFFIX="" +REQUEST_JSON="" + +confirm_to_bool() { + local raw_value="${1:-}" + case "$(printf '%s' "${raw_value}" | tr '[:upper:]' '[:lower:]')" in + 1|true|yes|y|confirm|confirmed) + printf 'true' + ;; + *) + printf 'false' + ;; + esac +} + +case "${COMMAND}" in + launchpad) + CHANNEL="${ARG1:-stable}" + PATH_SUFFIX="/api/v1/ops/releases/launchpad?channel=${CHANNEL}" + ;; + latest-package) + PATH_SUFFIX="/api/v1/ops/releases/package-metadata/latest" + ;; + latest-release) + PATH_SUFFIX="/api/v1/ops/releases/latest" + ;; + preview-latest) + METHOD="POST" + PATH_SUFFIX="/api/v1/ops/releases/from-package/latest/preview" + REQUEST_JSON="$(python3 - <<'PY' "${REQUESTED_BY}" +import json +import sys + +requested_by = str(sys.argv[1] or "").strip() or "cli/drive_release_hub" +payload = { + "created_by": f"{requested_by}/latest-package-preview", +} +print(json.dumps(payload, ensure_ascii=False)) +PY +)" + ;; + preview-latest-smart) + MODE="${ARG1:-}" + if [[ "${MODE}" != "worker" && "${MODE}" != "control" ]]; then + echo "mode must be worker or control" >&2 + usage + exit 1 + fi + METHOD="POST" + PATH_SUFFIX="/api/v1/ops/releases/from-package/latest/smart-rollout-preview" + REQUEST_JSON="$(python3 - <<'PY' "${MODE}" "${REQUESTED_BY}" +import json +import sys + +mode = str(sys.argv[1] or "").strip() +requested_by = str(sys.argv[2] or "").strip() or "cli/drive_release_hub" + +payload = { + "mode": mode, + "created_by": f"{requested_by}/latest-package-preview", + "rollout_created_by": f"{requested_by}/latest-package-preview/rollout", +} +print(json.dumps(payload, ensure_ascii=False)) +PY +)" + ;; + create-latest-release) + METHOD="POST" + PATH_SUFFIX="/api/v1/ops/releases/from-package/latest" + REQUEST_JSON="$(python3 - <<'PY' "${REQUESTED_BY}" +import json +import sys + +requested_by = str(sys.argv[1] or "").strip() or "cli/drive_release_hub" +payload = { + "created_by": f"{requested_by}/latest-package", +} +print(json.dumps(payload, ensure_ascii=False)) +PY +)" + ;; + list-releases) + RELEASE_LIMIT="${ARG1:-10}" + PATH_SUFFIX="/api/v1/ops/releases?limit=${RELEASE_LIMIT}" + ;; + list-rollouts) + RELEASE_ID="${ARG1:-}" + ROLLOUT_LIMIT="${ARG2:-10}" + if [[ -z "${RELEASE_ID}" ]]; then + echo "release_id is required for list-rollouts" >&2 + usage + exit 1 + fi + PATH_SUFFIX="/api/v1/ops/releases/${RELEASE_ID}/rollouts?limit=${ROLLOUT_LIMIT}" + ;; + smart-latest) + MODE="${ARG1:-}" + if [[ "${MODE}" != "worker" && "${MODE}" != "control" ]]; then + echo "mode must be worker or control" >&2 + usage + exit 1 + fi + METHOD="POST" + PATH_SUFFIX="/api/v1/ops/releases/from-package/latest/smart-rollout" + CONFIRM_RISKY="$(confirm_to_bool "${ARG2:-}")" + REQUEST_JSON="$(python3 - <<'PY' "${MODE}" "${CONFIRM_RISKY}" "${REQUESTED_BY}" +import json +import sys + +mode = str(sys.argv[1] or "").strip() +confirm_risky = str(sys.argv[2] or "").strip().lower() == "true" +requested_by = str(sys.argv[3] or "").strip() or "cli/drive_release_hub" + +payload = { + "mode": mode, + "confirm_risky": confirm_risky, + "created_by": f"{requested_by}/latest-package", + "rollout_created_by": f"{requested_by}/latest-package/rollout", +} +print(json.dumps(payload, ensure_ascii=False)) +PY +)" + ;; + preview-release) + RELEASE_ID="${ARG1:-}" + MODE="${ARG2:-}" + if [[ -z "${RELEASE_ID}" ]]; then + echo "release_id is required for preview-release" >&2 + usage + exit 1 + fi + if [[ "${MODE}" != "worker" && "${MODE}" != "control" ]]; then + echo "mode must be worker or control" >&2 + usage + exit 1 + fi + METHOD="POST" + PATH_SUFFIX="/api/v1/ops/releases/${RELEASE_ID}/smart-rollout-preview" + REQUEST_JSON="$(python3 - <<'PY' "${MODE}" "${REQUESTED_BY}" +import json +import sys + +mode = str(sys.argv[1] or "").strip() +requested_by = str(sys.argv[2] or "").strip() or "cli/drive_release_hub" + +payload = { + "mode": mode, + "created_by": f"{requested_by}/existing-release-preview", + "rollout_created_by": f"{requested_by}/existing-release-preview/rollout", +} +print(json.dumps(payload, ensure_ascii=False)) +PY +)" + ;; + smart-release) + RELEASE_ID="${ARG1:-}" + MODE="${ARG2:-}" + if [[ -z "${RELEASE_ID}" ]]; then + echo "release_id is required for smart-release" >&2 + usage + exit 1 + fi + if [[ "${MODE}" != "worker" && "${MODE}" != "control" ]]; then + echo "mode must be worker or control" >&2 + usage + exit 1 + fi + METHOD="POST" + PATH_SUFFIX="/api/v1/ops/releases/${RELEASE_ID}/smart-rollout" + CONFIRM_RISKY="$(confirm_to_bool "${ARG3:-}")" + REQUEST_JSON="$(python3 - <<'PY' "${MODE}" "${CONFIRM_RISKY}" "${REQUESTED_BY}" +import json +import sys + +mode = str(sys.argv[1] or "").strip() +confirm_risky = str(sys.argv[2] or "").strip().lower() == "true" +requested_by = str(sys.argv[3] or "").strip() or "cli/drive_release_hub" + +payload = { + "mode": mode, + "confirm_risky": confirm_risky, + "created_by": f"{requested_by}/existing-release", + "rollout_created_by": f"{requested_by}/existing-release/rollout", +} +print(json.dumps(payload, ensure_ascii=False)) +PY +)" + ;; + *) + echo "unknown command: ${COMMAND}" >&2 + usage + exit 1 + ;; +esac + +echo "[1/3] request" +printf '%s\n' "method=${METHOD}" +printf '%s\n' "url=${BASE_URL}${PATH_SUFFIX}" +if [[ -n "${REQUEST_JSON}" ]]; then + print_json_or_raw "${REQUEST_JSON}" +else + echo "(no request body)" +fi +echo +echo + +RAW_HTTP_RESPONSE="$(request_with_status "${METHOD}" "${BASE_URL}${PATH_SUFFIX}" "${REQUEST_JSON}")" +HTTP_STATUS="${RAW_HTTP_RESPONSE##*$'\n'HTTP_STATUS=}" +RESPONSE_JSON="${RAW_HTTP_RESPONSE%$'\n'HTTP_STATUS=*}" + +echo "[2/3] raw response" +printf '%s\n' "http_status=${HTTP_STATUS}" +print_json_or_raw "${RESPONSE_JSON}" +echo +echo + +echo "[3/3] condensed summary" +python3 - "${RESPONSE_JSON}" "${HTTP_STATUS}" <<'PY' +import json +import sys + +raw_response = str(sys.argv[1] or "") +http_status = int(sys.argv[2] or 0) +try: + payload = json.loads(raw_response) +except Exception: + print(json.dumps({ + "http_status": http_status, + "code": 1, + "message": "response is not valid json", + "raw_response": raw_response[:500], + }, ensure_ascii=False, indent=2)) + sys.exit(0 if 200 <= http_status < 300 else 1) + +data = payload.get("data") or {} + +raw_code = payload.get("code", 1) +if raw_code in (0, "0", False): + normalized_code = 0 +else: + normalized_code = int(raw_code or 1) + +summary = { + "http_status": http_status, + "code": normalized_code, + "message": str(payload.get("message") or ""), +} + +if isinstance(data, dict) and "package" in data: + package = dict(data.get("package") or {}) + summary["package"] = { + "available": bool(package.get("available", False)), + "package_name": str(package.get("package_name") or ""), + "package_type": str(package.get("package_type") or ""), + "generated_at": str(package.get("generated_at") or ""), + "smoke_test_ok": bool(package.get("smoke_test_ok", False)), + "smoke_test_message": str(package.get("smoke_test_message") or ""), +} + +if isinstance(data, dict) and "launchpad_status" in data: + launchpad_status = dict(data.get("launchpad_status") or {}) + worker_preview = dict(data.get("worker_rollout_preview") or {}) + worker_smart_rollout = dict(worker_preview.get("smart_rollout") or {}) + control_preview = dict(data.get("control_rollout_preview") or {}) + control_smart_rollout = dict(control_preview.get("smart_rollout") or {}) + summary["launchpad"] = { + "channel": str(data.get("channel") or ""), + "status": str(launchpad_status.get("status") or ""), + "status_label": str(launchpad_status.get("status_label") or ""), + "summary": str(launchpad_status.get("summary") or ""), + "recommended_action_code": str(launchpad_status.get("recommended_action_code") or ""), + "recommended_target_node_code": str(launchpad_status.get("recommended_target_node_code") or ""), + "recommended_recovery_label": str(launchpad_status.get("recommended_recovery_label") or ""), + "recommended_recovery_summary": str(launchpad_status.get("recommended_recovery_summary") or ""), + "onboarding_bootstrap_pending_nodes": int(launchpad_status.get("onboarding_bootstrap_pending_nodes", 0) or 0), + "onboarding_acceptance_ready_nodes": int(launchpad_status.get("onboarding_acceptance_ready_nodes", 0) or 0), + "recommended_command": str(launchpad_status.get("recommended_command") or ""), + "recommended_execution_mode": str(launchpad_status.get("recommended_execution_mode") or ""), + "recommended_execution_mode_label": str(launchpad_status.get("recommended_execution_mode_label") or ""), + "focus_ref": dict(launchpad_status.get("focus_ref") or {}), + "latest_package": dict(data.get("latest_package") or {}), + "latest_release": { + "id": int(((data.get("latest_release") or {}).get("id") or 0)), + "release_version": str((data.get("latest_release") or {}).get("release_version") or ""), + "status": str((data.get("latest_release") or {}).get("status") or ""), + }, + "worker_rollout_preview": { + "message": str(worker_preview.get("message") or ""), + "available": bool(worker_smart_rollout.get("available", False)), + "requires_confirmation": bool(worker_preview.get("requires_confirmation", False)), + "blocked": bool(worker_preview.get("blocked", False)), + "execution_mode": str(worker_smart_rollout.get("execution_mode") or ""), + "execution_mode_label": str(worker_smart_rollout.get("execution_mode_label") or ""), + "target_node_codes": list(worker_smart_rollout.get("target_node_codes") or []), + }, + "control_rollout_preview": { + "message": str(control_preview.get("message") or ""), + "available": bool(control_smart_rollout.get("available", False)), + "requires_confirmation": bool(control_preview.get("requires_confirmation", False)), + "blocked": bool(control_preview.get("blocked", False)), + "execution_mode": str(control_smart_rollout.get("execution_mode") or ""), + "execution_mode_label": str(control_smart_rollout.get("execution_mode_label") or ""), + "target_node_codes": list(control_smart_rollout.get("target_node_codes") or []), + }, + } + print(json.dumps(summary, ensure_ascii=False, indent=2)) + sys.exit(0) + +if isinstance(data, dict) and "available" in data: + summary["latest_package"] = { + "available": bool(data.get("available", False)), + "package_name": str(data.get("package_name") or ""), + "package_type": str(data.get("package_type") or ""), + "generated_at": str(data.get("generated_at") or ""), + "commit_sha": str(data.get("commit_sha") or ""), + "smoke_test_ok": bool(data.get("smoke_test_ok", False)), + "smoke_test_message": str(data.get("smoke_test_message") or ""), + "reason": str(data.get("reason") or ""), + } +elif isinstance(data, dict) and "releases" in data: + releases = list(data.get("releases") or []) + summary["releases"] = { + "count": len(releases), + "items": [ + { + "id": int(item.get("id") or 0), + "release_version": str(item.get("release_version") or ""), + "channel": str(item.get("channel") or ""), + "status": str(item.get("status") or ""), + } + for item in releases[:5] + ], + } +elif isinstance(data, dict) and "rollouts" in data: + rollouts = list(data.get("rollouts") or []) + summary["rollouts"] = { + "count": len(rollouts), + "items": [ + { + "id": int(item.get("id") or 0), + "rollout_code": str(item.get("rollout_code") or ""), + "status": str(item.get("status") or ""), + "jobs_total": int(item.get("jobs_total") or 0), + "jobs_created": int(item.get("jobs_created") or 0), + } + for item in rollouts[:5] + ], + } +else: + release = data.get("release") or data + rollout = data.get("rollout") or {} + smart_rollout = data.get("smart_rollout") or {} + preview = data.get("preview") or {} + summary["preview_source"] = str(data.get("preview_source") or "") + summary["release"] = { + "id": int(release.get("id") or 0), + "release_version": str(release.get("release_version") or ""), + "channel": str(release.get("channel") or ""), + "status": str(release.get("status") or ""), + "is_preview": bool(release.get("is_preview", False)), + } + if rollout: + summary["rollout"] = { + "id": int(rollout.get("id") or 0), + "rollout_code": str(rollout.get("rollout_code") or ""), + "status": str(rollout.get("status") or ""), + } + if smart_rollout: + summary["smart_rollout"] = { + "available": bool(smart_rollout.get("available", False)), + "mode": str(smart_rollout.get("mode") or ""), + "mode_label": str(smart_rollout.get("mode_label") or ""), + "target_node_codes": list((smart_rollout.get("target_selector") or {}).get("node_codes") or []), + "summary": str(smart_rollout.get("summary") or smart_rollout.get("reason") or ""), + } + if preview: + summary["preview"] = { + "blocked": bool(preview.get("blocked", False)), + "approval_required": bool(preview.get("approval_required", False)), + "warnings": list(preview.get("warnings") or []), + } + summary["result_flags"] = { + "release_created": bool(data.get("release_created", False)), + "release_deduplicated": bool(data.get("release_deduplicated", False)), + "rollout_created": bool(data.get("rollout_created", False)), + "requires_confirmation": bool(data.get("requires_confirmation", False)), + "blocked": bool(data.get("blocked", False)), + } + +print(json.dumps(summary, ensure_ascii=False, indent=2)) +PY + +if [[ "${HTTP_STATUS}" -lt 200 || "${HTTP_STATUS}" -ge 300 ]]; then + exit 1 +fi diff --git a/domain-api/deploy/multi-region/export_go_live_bundle.sh b/domain-api/deploy/multi-region/export_go_live_bundle.sh new file mode 100755 index 0000000..baa1151 --- /dev/null +++ b/domain-api/deploy/multi-region/export_go_live_bundle.sh @@ -0,0 +1,556 @@ +#!/usr/bin/env bash +set -uo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +# shellcheck disable=SC1091 +source "${SCRIPT_DIR}/lib_ops_center.sh" +ops_center_load_config "${SCRIPT_DIR}" + +MAINLAND_BASE_URL_DEFAULT="$(ops_center_resolve_mainland_api_base_url)" +OVERSEAS_BASE_URL_DEFAULT="$(ops_center_resolve_overseas_api_base_url)" +REPORT_ROOT_DEFAULT="${OPS_CENTER_REPORT_ROOT}/go-live-bundles" + +REPORT_DIR_INPUT="${1:-}" +MAINLAND_BASE_URL_INPUT="${2:-}" +OVERSEAS_BASE_URL_INPUT="${3:-}" + +timestamp_now() { + date '+%Y%m%d_%H%M%S' +} + +normalize_optional_base_url() { + local raw_value="${1:-}" + local fallback_value="${2:-}" + if [[ -z "${raw_value}" ]]; then + printf '%s' "${fallback_value}" + return + fi + case "${raw_value}" in + -|none|null|disabled) + printf '' + ;; + *) + printf '%s' "${raw_value}" + ;; + esac +} + +if [[ -n "${REPORT_DIR_INPUT}" && "${REPORT_DIR_INPUT}" != http://* && "${REPORT_DIR_INPUT}" != https://* ]]; then + REPORT_DIR="${REPORT_DIR_INPUT}" + MAINLAND_BASE_URL="${MAINLAND_BASE_URL_INPUT:-${MAINLAND_BASE_URL_DEFAULT}}" + OVERSEAS_BASE_URL="$(normalize_optional_base_url "${OVERSEAS_BASE_URL_INPUT:-}" "${OVERSEAS_BASE_URL_DEFAULT}")" +else + REPORT_DIR="${REPORT_ROOT_DEFAULT}/go-live-bundle-$(timestamp_now)" + MAINLAND_BASE_URL="${REPORT_DIR_INPUT:-${MAINLAND_BASE_URL_DEFAULT}}" + OVERSEAS_BASE_URL="$(normalize_optional_base_url "${MAINLAND_BASE_URL_INPUT:-}" "${OVERSEAS_BASE_URL_DEFAULT}")" +fi + +mkdir -p "${REPORT_DIR}" + +CURL_CONNECT_TIMEOUT="${OPS_CENTER_CURL_CONNECT_TIMEOUT:-3}" +CURL_MAX_TIME="${OPS_CENTER_CURL_MAX_TIME:-15}" +STEP_COMMAND_TIMEOUT_SECONDS="${OPS_CENTER_REPORT_COMMAND_TIMEOUT_SECONDS:-45}" +DOCTOR_DECISION_TIMEOUT_SECONDS="${OPS_CENTER_REPORT_DOCTOR_DECISION_TIMEOUT_SECONDS:-120}" +DOCTOR_EXPORT_TIMEOUT_SECONDS="${OPS_CENTER_REPORT_DOCTOR_EXPORT_TIMEOUT_SECONDS:-180}" +DOCTOR_INNER_REPORT_TIMEOUT_SECONDS="${OPS_CENTER_DOCTOR_INNER_REPORT_TIMEOUT_SECONDS:-60}" + +ENV_AUDIT_TXT_PATH="${REPORT_DIR}/00_env_audit.txt" +SUMMARY_TXT_PATH="${REPORT_DIR}/01_go_live_check_summary.txt" +GO_LIVE_JSON_PATH="${REPORT_DIR}/02_go_live_summary.json" +STACK_JSON_PATH="${REPORT_DIR}/03_stack_diagnosis.json" +DRIVER_FEED_JSON_PATH="${REPORT_DIR}/04_driver_feed.json" +CODEX_BRIEF_JSON_PATH="${REPORT_DIR}/05_codex_brief.json" +RELEASE_LAUNCHPAD_JSON_PATH="${REPORT_DIR}/06_release_launchpad.json" +DOCTOR_DECISION_JSON_PATH="${REPORT_DIR}/07_doctor_decision.json" +DOCTOR_EXPORT_STDOUT_PATH="${REPORT_DIR}/08_doctor_export_stdout.txt" +MANIFEST_PATH="${REPORT_DIR}/manifest.json" + +declare -a STEP_RESULTS=() + +pretty_print_json_file() { + local source_path="${1:-}" + python3 - "${source_path}" <<'PY' +import json +import pathlib +import sys + +path = pathlib.Path(sys.argv[1]) +raw_text = path.read_text(encoding="utf-8") if path.exists() else "" +try: + payload = json.loads(raw_text) +except Exception: + print(raw_text) +else: + print(json.dumps(payload, ensure_ascii=False, indent=2)) +PY +} + +record_step_result() { + local key="${1:-}" + local path="${2:-}" + local ok="${3:-false}" + local http_status="${4:-}" + local exit_code="${5:-0}" + STEP_RESULTS+=("${key}|${path}|${ok}|${http_status}|${exit_code}") +} + +fetch_json_to_file() { + local key="${1:-}" + local url="${2:-}" + local output_path="${3:-}" + local tmp_path + tmp_path="$(mktemp)" + local raw_text http_status + raw_text="$(curl -sS \ + --connect-timeout "${CURL_CONNECT_TIMEOUT}" \ + --max-time "${CURL_MAX_TIME}" \ + "${url}" \ + -w $'\nHTTP_STATUS=%{http_code}' || true)" + http_status="${raw_text##*$'\n'HTTP_STATUS=}" + printf '%s' "${raw_text%$'\n'HTTP_STATUS=*}" > "${tmp_path}" + pretty_print_json_file "${tmp_path}" > "${output_path}" + rm -f "${tmp_path}" + if [[ "${http_status}" == 2* ]]; then + record_step_result "${key}" "${output_path}" "true" "${http_status}" "0" + return 0 + fi + record_step_result "${key}" "${output_path}" "false" "${http_status}" "1" + return 1 +} + +capture_command_to_file() { + local key="${1:-}" + local output_path="${2:-}" + shift 2 || true + capture_command_to_file_with_timeout "${STEP_COMMAND_TIMEOUT_SECONDS}" "${key}" "${output_path}" "$@" +} + +capture_command_to_file_with_timeout() { + local timeout_seconds="${1:-45}" + local key="${2:-}" + local output_path="${3:-}" + shift 3 || true + local -a wrapped_command=("$@") + local had_errexit="false" + case "$-" in + *e*) had_errexit="true" ;; + esac + if command -v timeout >/dev/null 2>&1; then + wrapped_command=(timeout "${timeout_seconds}" "$@") + fi + set +e + "${wrapped_command[@]}" > "${output_path}" 2>&1 + local exit_code=$? + if [[ "${had_errexit}" == "true" ]]; then + set -e + else + set +e + fi + if [[ "${exit_code}" -eq 0 ]]; then + record_step_result "${key}" "${output_path}" "true" "" "${exit_code}" + return 0 + fi + record_step_result "${key}" "${output_path}" "false" "" "${exit_code}" + return "${exit_code}" +} + +materialize_go_live_summary_fallback() { + local summary_path="${1:-}" + local output_path="${2:-}" + python3 - "${summary_path}" "${output_path}" <<'PY' +import json +import pathlib +import sys + +summary_path = pathlib.Path(sys.argv[1]) +output_path = pathlib.Path(sys.argv[2]) +raw_text = summary_path.read_text(encoding="utf-8") if summary_path.exists() else "" +start = raw_text.find("{") +end = raw_text.rfind("}") +if start < 0 or end < start: + sys.exit(1) +json_text = raw_text[start:end + 1] +payload = json.loads(json_text) +if not isinstance(payload, dict): + sys.exit(1) +output_path.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8") +PY +} + +override_step_result() { + local key="${1:-}" + local path="${2:-}" + local ok="${3:-false}" + local http_status="${4:-}" + local exit_code="${5:-0}" + local -a updated_results=() + local raw_item existing_key + for raw_item in "${STEP_RESULTS[@]}"; do + existing_key="${raw_item%%|*}" + if [[ "${existing_key}" == "${key}" ]]; then + continue + fi + updated_results+=("${raw_item}") + done + STEP_RESULTS=("${updated_results[@]}") + record_step_result "${key}" "${path}" "${ok}" "${http_status}" "${exit_code}" +} + +set +e +capture_command_to_file \ + "env_audit" \ + "${ENV_AUDIT_TXT_PATH}" \ + bash "${SCRIPT_DIR}/drive_ops_center.sh" env-audit "${MAINLAND_BASE_URL}" + +capture_command_to_file \ + "go_live_check_summary" \ + "${SUMMARY_TXT_PATH}" \ + bash "${SCRIPT_DIR}/drive_ops_center.sh" go-live-check "${MAINLAND_BASE_URL}" "${OVERSEAS_BASE_URL}" summary + +fetch_json_to_file \ + "go_live_summary" \ + "${MAINLAND_BASE_URL}/api/v1/ops/go-live-summary?base_url=${MAINLAND_BASE_URL}" \ + "${GO_LIVE_JSON_PATH}" + +if [[ ! -s "${GO_LIVE_JSON_PATH}" || "$(sed -n '1p' "${GO_LIVE_JSON_PATH}" 2>/dev/null)" != "{"* ]]; then + if materialize_go_live_summary_fallback "${SUMMARY_TXT_PATH}" "${GO_LIVE_JSON_PATH}" 2>/dev/null; then + override_step_result "go_live_summary" "${GO_LIVE_JSON_PATH}" "true" "fallback" "0" + fi +fi + +if [[ "$(python3 - "${GO_LIVE_JSON_PATH}" <<'PY' +import json +import pathlib +import sys + +path = pathlib.Path(sys.argv[1]) +try: + payload = json.loads(path.read_text(encoding="utf-8")) +except Exception: + print("invalid") + raise SystemExit(0) + +if isinstance(payload, dict) and payload.get("detail") == "Not Found": + print("not_found") +else: + print("ok") +PY +)" == "not_found" ]]; then + if materialize_go_live_summary_fallback "${SUMMARY_TXT_PATH}" "${GO_LIVE_JSON_PATH}" 2>/dev/null; then + override_step_result "go_live_summary" "${GO_LIVE_JSON_PATH}" "true" "fallback" "0" + fi +fi + +fetch_json_to_file \ + "stack_diagnosis" \ + "${MAINLAND_BASE_URL}/api/v1/ops/stack-diagnosis?base_url=${MAINLAND_BASE_URL}" \ + "${STACK_JSON_PATH}" + +fetch_json_to_file \ + "driver_feed" \ + "${MAINLAND_BASE_URL}/api/v1/ops/driver-feed" \ + "${DRIVER_FEED_JSON_PATH}" + +fetch_json_to_file \ + "codex_brief" \ + "${MAINLAND_BASE_URL}/api/v1/ops/codex-brief" \ + "${CODEX_BRIEF_JSON_PATH}" + +fetch_json_to_file \ + "release_launchpad" \ + "${MAINLAND_BASE_URL}/api/v1/ops/releases/launchpad" \ + "${RELEASE_LAUNCHPAD_JSON_PATH}" + +capture_command_to_file_with_timeout \ + "${DOCTOR_EXPORT_TIMEOUT_SECONDS}" \ + "doctor_export" \ + "${DOCTOR_EXPORT_STDOUT_PATH}" \ + env "OPS_CENTER_DOCTOR_REPORT_TIMEOUT_SECONDS=${DOCTOR_INNER_REPORT_TIMEOUT_SECONDS}" \ + bash "${SCRIPT_DIR}/drive_ops_center.sh" doctor-export "${REPORT_DIR}/doctor-export" "${MAINLAND_BASE_URL}" "${OVERSEAS_BASE_URL}" + +capture_command_to_file_with_timeout \ + "${DOCTOR_DECISION_TIMEOUT_SECONDS}" \ + "doctor_decision" \ + "${DOCTOR_DECISION_JSON_PATH}" \ + bash "${SCRIPT_DIR}/drive_ops_center.sh" doctor-decision "${REPORT_DIR}/doctor-export" +set -e + +python3 - <<'PY' \ + "${MANIFEST_PATH}" \ + "${REPORT_DIR}" \ + "${MAINLAND_BASE_URL}" \ + "${OVERSEAS_BASE_URL}" \ + "${ENV_AUDIT_TXT_PATH}" \ + "${SUMMARY_TXT_PATH}" \ + "${GO_LIVE_JSON_PATH}" \ + "${STACK_JSON_PATH}" \ + "${DRIVER_FEED_JSON_PATH}" \ + "${CODEX_BRIEF_JSON_PATH}" \ + "${RELEASE_LAUNCHPAD_JSON_PATH}" \ + "${DOCTOR_DECISION_JSON_PATH}" \ + "${DOCTOR_EXPORT_STDOUT_PATH}" \ + "${STEP_RESULTS[@]}" +import json +import pathlib +import sys + +manifest_path = pathlib.Path(sys.argv[1]) +report_dir = pathlib.Path(sys.argv[2]) +mainland_base_url = sys.argv[3] +overseas_base_url = sys.argv[4] + +file_keys = [ + "env_audit", + "go_live_check_summary", + "go_live_summary", + "stack_diagnosis", + "driver_feed", + "codex_brief", + "release_launchpad", + "doctor_decision", + "doctor_export", +] +file_paths = { + "env_audit": pathlib.Path(sys.argv[5]), + "go_live_check_summary": pathlib.Path(sys.argv[6]), + "go_live_summary": pathlib.Path(sys.argv[7]), + "stack_diagnosis": pathlib.Path(sys.argv[8]), + "driver_feed": pathlib.Path(sys.argv[9]), + "codex_brief": pathlib.Path(sys.argv[10]), + "release_launchpad": pathlib.Path(sys.argv[11]), + "doctor_decision": pathlib.Path(sys.argv[12]), + "doctor_export": pathlib.Path(sys.argv[13]), +} + +step_results = {} +for raw_item in sys.argv[14:]: + parts = raw_item.split("|", 4) + if len(parts) != 5: + continue + key, path_value, ok_value, http_status, exit_code = parts + step_results[key] = { + "path": path_value, + "ok": ok_value == "true", + "http_status": http_status, + "exit_code": int(exit_code or 0), + } + + +def load_json_file(path: pathlib.Path) -> dict: + if not path.is_file(): + return {} + try: + payload = json.loads(path.read_text(encoding="utf-8")) + except Exception: + return {} + return payload if isinstance(payload, dict) else {} + + +def load_env_audit_summary(path: pathlib.Path) -> dict: + if not path.is_file(): + return {} + raw_text = path.read_text(encoding="utf-8", errors="replace") + marker = "[8/8] condensed env audit summary" + candidate_text = raw_text.split(marker, 1)[1].strip() if marker in raw_text else raw_text.strip() + if not candidate_text: + return {} + try: + payload = json.loads(candidate_text) + except Exception: + return {} + return payload if isinstance(payload, dict) else {} + +failures = [] +artifacts = [] +for key in file_keys: + artifact_path = file_paths[key] + result = dict(step_results.get(key) or {}) + if not result: + result = { + "path": str(artifact_path), + "ok": artifact_path.exists(), + "http_status": "", + "exit_code": 0 if artifact_path.exists() else 1, + } + entry = { + "key": key, + "path": str(artifact_path), + "exists": artifact_path.exists(), + "ok": bool(result.get("ok", False)), + "http_status": str(result.get("http_status") or ""), + "exit_code": int(result.get("exit_code", 0) or 0), + } + artifacts.append(entry) + if not entry["ok"]: + failures.append(key) + +go_live_summary_payload = load_json_file(file_paths["go_live_summary"]) +stack_diagnosis_payload = load_json_file(file_paths["stack_diagnosis"]) +driver_feed_payload = load_json_file(file_paths["driver_feed"]) +codex_brief_payload = load_json_file(file_paths["codex_brief"]) +env_audit_summary = load_env_audit_summary(file_paths["env_audit"]) + +stack_diagnosis = dict(stack_diagnosis_payload.get("diagnosis") or {}) +driver_feed_summary = dict(driver_feed_payload.get("summary") or {}) +codex_brief_summary = dict(codex_brief_payload.get("summary") or {}) + + +def first_nonempty(*values: object) -> str: + for value in values: + normalized = str(value or "").strip() + if normalized: + return normalized + return "" + + +def nonempty_distinct(values: dict[str, str]) -> list[str]: + return sorted({str(value or "").strip() for value in values.values() if str(value or "").strip()}) + + +def distinct_ints(values: dict[str, int], available_sources: set[str]) -> list[int]: + return sorted({int(values[source]) for source in values if source in available_sources}) + + +launchpad_target_node_codes = { + "go_live_summary": str(go_live_summary_payload.get("launchpad_recommended_target_node_code") or "").strip(), + "stack_diagnosis": str(stack_diagnosis.get("launchpad_recommended_target_node_code") or "").strip(), + "driver_feed": str(driver_feed_summary.get("launchpad_recommended_target_node_code") or "").strip(), + "codex_brief": str(codex_brief_summary.get("launchpad_recommended_target_node_code") or "").strip(), +} +launchpad_recovery_labels = { + "go_live_summary": str(go_live_summary_payload.get("launchpad_recommended_recovery_label") or "").strip(), + "stack_diagnosis": str(stack_diagnosis.get("launchpad_recommended_recovery_label") or "").strip(), + "driver_feed": str(driver_feed_summary.get("launchpad_recommended_recovery_label") or "").strip(), + "codex_brief": str(codex_brief_summary.get("launchpad_recommended_recovery_label") or "").strip(), +} +launchpad_recovery_summaries = { + "go_live_summary": str(go_live_summary_payload.get("launchpad_recommended_recovery_summary") or "").strip(), + "stack_diagnosis": str(stack_diagnosis.get("launchpad_recommended_recovery_summary") or "").strip(), + "driver_feed": str(driver_feed_summary.get("launchpad_recommended_recovery_summary") or "").strip(), + "codex_brief": str(codex_brief_summary.get("launchpad_recommended_recovery_summary") or "").strip(), +} +launchpad_bootstrap_pending_nodes = { + "go_live_summary": int(go_live_summary_payload.get("launchpad_onboarding_bootstrap_pending_nodes", 0) or 0), + "stack_diagnosis": int(stack_diagnosis.get("launchpad_onboarding_bootstrap_pending_nodes", 0) or 0), + "driver_feed": int(driver_feed_summary.get("launchpad_onboarding_bootstrap_pending_nodes", 0) or 0), + "codex_brief": int(codex_brief_summary.get("launchpad_onboarding_bootstrap_pending_nodes", 0) or 0), +} +launchpad_acceptance_ready_nodes = { + "go_live_summary": int(go_live_summary_payload.get("launchpad_onboarding_acceptance_ready_nodes", 0) or 0), + "stack_diagnosis": int(stack_diagnosis.get("launchpad_onboarding_acceptance_ready_nodes", 0) or 0), + "driver_feed": int(driver_feed_summary.get("launchpad_onboarding_acceptance_ready_nodes", 0) or 0), + "codex_brief": int(codex_brief_summary.get("launchpad_onboarding_acceptance_ready_nodes", 0) or 0), +} +available_alignment_sources = { + source + for source, payload_value in { + "go_live_summary": go_live_summary_payload, + "stack_diagnosis": stack_diagnosis_payload, + "driver_feed": driver_feed_payload, + "codex_brief": codex_brief_payload, + }.items() + if isinstance(payload_value, dict) and payload_value +} +launchpad_alignment = { + "consistent": True, + "target_node_code_consistent": True, + "recovery_label_consistent": True, + "recovery_summary_consistent": True, + "bootstrap_pending_consistent": True, + "acceptance_ready_consistent": True, + "available_sources": sorted(available_alignment_sources), + "target_node_codes": launchpad_target_node_codes, + "recovery_labels": launchpad_recovery_labels, + "recovery_summaries": launchpad_recovery_summaries, + "bootstrap_pending_nodes": launchpad_bootstrap_pending_nodes, + "acceptance_ready_nodes": launchpad_acceptance_ready_nodes, +} +launchpad_alignment["target_node_code_consistent"] = len(nonempty_distinct({ + source: value for source, value in launchpad_target_node_codes.items() if source in available_alignment_sources +})) <= 1 +launchpad_alignment["recovery_label_consistent"] = len(nonempty_distinct({ + source: value for source, value in launchpad_recovery_labels.items() if source in available_alignment_sources +})) <= 1 +launchpad_alignment["recovery_summary_consistent"] = len(nonempty_distinct({ + source: value for source, value in launchpad_recovery_summaries.items() if source in available_alignment_sources +})) <= 1 +launchpad_alignment["bootstrap_pending_consistent"] = len( + distinct_ints(launchpad_bootstrap_pending_nodes, available_alignment_sources) +) <= 1 +launchpad_alignment["acceptance_ready_consistent"] = len( + distinct_ints(launchpad_acceptance_ready_nodes, available_alignment_sources) +) <= 1 +launchpad_alignment["consistent"] = all([ + bool(launchpad_alignment["target_node_code_consistent"]), + bool(launchpad_alignment["recovery_label_consistent"]), + bool(launchpad_alignment["recovery_summary_consistent"]), + bool(launchpad_alignment["bootstrap_pending_consistent"]), + bool(launchpad_alignment["acceptance_ready_consistent"]), +]) + +env_audit_status = str(env_audit_summary.get("status") or "").strip() +env_audit_runtime_may_need_restart = bool(((env_audit_summary.get("runtime") or {}).get("runtime_may_need_restart", False))) +review_status_hint = "ready" +review_headline = "核心上线证据已经齐备，可进入最终人工复核或正式发布门禁。" +if failures or env_audit_status == "blocked": + review_status_hint = "blocked" + review_headline = "当前导出包仍有关键阻断项，不能直接作为正式上线终稿。" +elif env_audit_status == "attention" or not launchpad_alignment["consistent"]: + review_status_hint = "attention" + if not launchpad_alignment["consistent"]: + review_headline = "核心报告虽已导出，但 launchpad 摘要在不同 surface 之间存在漂移，建议先重新复核。" + else: + review_headline = "核心报告已齐，但环境审计仍提示存在待收口事项。" + +payload = { + "report_dir": str(report_dir), + "mainland_base_url": mainland_base_url, + "overseas_base_url": overseas_base_url, + "artifacts": artifacts, + "failures": failures, + "summary": { + "ok": len(failures) == 0, + "artifact_total": len(artifacts), + "failure_total": len(failures), + "review_status_hint": review_status_hint, + "review_headline": review_headline, + "env_audit_status": env_audit_status, + "env_audit_runtime_may_need_restart": env_audit_runtime_may_need_restart, + "launchpad_recommended_target_node_code": first_nonempty( + go_live_summary_payload.get("launchpad_recommended_target_node_code"), + stack_diagnosis.get("launchpad_recommended_target_node_code"), + driver_feed_summary.get("launchpad_recommended_target_node_code"), + codex_brief_summary.get("launchpad_recommended_target_node_code"), + ), + "launchpad_recommended_recovery_label": first_nonempty( + go_live_summary_payload.get("launchpad_recommended_recovery_label"), + stack_diagnosis.get("launchpad_recommended_recovery_label"), + driver_feed_summary.get("launchpad_recommended_recovery_label"), + codex_brief_summary.get("launchpad_recommended_recovery_label"), + ), + "launchpad_recommended_recovery_summary": first_nonempty( + go_live_summary_payload.get("launchpad_recommended_recovery_summary"), + stack_diagnosis.get("launchpad_recommended_recovery_summary"), + driver_feed_summary.get("launchpad_recommended_recovery_summary"), + codex_brief_summary.get("launchpad_recommended_recovery_summary"), + ), + "launchpad_onboarding_bootstrap_pending_nodes": max(launchpad_bootstrap_pending_nodes.values() or [0]), + "launchpad_onboarding_acceptance_ready_nodes": max(launchpad_acceptance_ready_nodes.values() or [0]), + "launchpad_alignment": launchpad_alignment, + "recommended_reading_order": [ + "00_env_audit.txt", + "01_go_live_check_summary.txt", + "02_go_live_summary.json", + "03_stack_diagnosis.json", + "04_driver_feed.json", + "05_codex_brief.json", + "06_release_launchpad.json", + "07_doctor_decision.json", + "08_doctor_export_stdout.txt", + ], + }, +} + +manifest_path.write_text(json.dumps(payload, ensure_ascii=False, indent=2), encoding="utf-8") +print(json.dumps(payload, ensure_ascii=False, indent=2)) +PY diff --git a/domain-api/deploy/multi-region/init_ops_center_config.sh b/domain-api/deploy/multi-region/init_ops_center_config.sh new file mode 100755 index 0000000..e16d6e5 --- /dev/null +++ b/domain-api/deploy/multi-region/init_ops_center_config.sh @@ -0,0 +1,135 @@ +#!/usr/bin/env bash +set -euo pipefail + +SCRIPT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +TEMPLATE_PATH="${SCRIPT_DIR}/templates/domaincheck-ops-center.env.example" + +usage() { + cat <<'EOF' +usage: + bash domain-api/deploy/multi-region/init_ops_center_config.sh \ + [config_path] [mainland_api_base_url] [overseas_api_base_url] [control_plane_base_url] + +examples: + bash domain-api/deploy/multi-region/init_ops_center_config.sh + bash domain-api/deploy/multi-region/init_ops_center_config.sh /etc/default/domaincheck-ops-center \ + http://121.204.244.188:8100 \ + http://152.53.37.118:8100 \ + https://api.example.com + +notes: + - existing config files are preserved unless FORCE=true + - advanced fields can be overridden with env vars: + OPS_ACTIVE_PROFILE + OPS_PROFILE_NAMES + OPS_DEFAULT_REQUESTED_BY + OPS_DEFAULT_ISSUED_BY + OPS_DEFAULT_EXPIRES_IN_HOURS + OPS_DEFAULT_ROOT_DIR + OPS_REPORT_ROOT +EOF +} + +if [[ "${1:-}" == "-h" || "${1:-}" == "--help" ]]; then + usage + exit 0 +fi + +TARGET_PATH="${1:-/etc/default/domaincheck-ops-center}" +MAINLAND_API_BASE_URL="${2:-${OPS_MAINLAND_API_BASE_URL:-http://127.0.0.1:8100}}" +OVERSEAS_API_BASE_URL="${3:-${OPS_OVERSEAS_API_BASE_URL:-http://127.0.0.1:8100}}" +CONTROL_PLANE_BASE_URL="${4:-${OPS_DEFAULT_CONTROL_PLANE_BASE_URL:-}}" + +ACTIVE_PROFILE="${OPS_ACTIVE_PROFILE:-default}" +PROFILE_NAMES="${OPS_PROFILE_NAMES:-${ACTIVE_PROFILE}}" +DEFAULT_REQUESTED_BY="${OPS_DEFAULT_REQUESTED_BY:-cli/ops-center}" +DEFAULT_ISSUED_BY="${OPS_DEFAULT_ISSUED_BY:-ops-center}" +DEFAULT_EXPIRES_IN_HOURS="${OPS_DEFAULT_EXPIRES_IN_HOURS:-72}" +DEFAULT_ROOT_DIR="${OPS_DEFAULT_ROOT_DIR:-/opt/domaincheck}" +REPORT_ROOT="${OPS_REPORT_ROOT:-}" +NODE_AGENT_SERVICE_NAME="${OPS_NODE_AGENT_SERVICE_NAME:-domaincheck-node-agent}" +NODE_AGENT_ENV_FILE="${OPS_NODE_AGENT_ENV_FILE:-/etc/default/domaincheck-node-agent}" +FORCE_WRITE="${FORCE:-false}" + +if [[ ! -f "${TEMPLATE_PATH}" ]]; then + echo "template not found: ${TEMPLATE_PATH}" >&2 + exit 1 +fi + +mkdir -p "$(dirname "${TARGET_PATH}")" + +if [[ ! -f "${TARGET_PATH}" || "${FORCE_WRITE}" == "true" ]]; then + cp "${TEMPLATE_PATH}" "${TARGET_PATH}" +fi + +upsert_key() { + local key="$1" + local value="$2" + if grep -q "^${key}=" "${TARGET_PATH}"; then + sed -i "s#^${key}=.*#${key}=${value}#" "${TARGET_PATH}" + else + printf '%s=%s\n' "${key}" "${value}" >>"${TARGET_PATH}" + fi +} + +upsert_commented_key() { + local key="$1" + local value="$2" + if [[ -z "${value}" ]]; then + return 0 + fi + upsert_key "${key}" "${value}" +} + +upsert_key "OPS_ACTIVE_PROFILE" "${ACTIVE_PROFILE}" +upsert_key "OPS_PROFILE_NAMES" "${PROFILE_NAMES}" +upsert_key "OPS_MAINLAND_API_BASE_URL" "${MAINLAND_API_BASE_URL}" +upsert_key "OPS_OVERSEAS_API_BASE_URL" "${OVERSEAS_API_BASE_URL}" +upsert_key "OPS_DEFAULT_CONTROL_PLANE_BASE_URL" "${CONTROL_PLANE_BASE_URL}" +upsert_key "OPS_DEFAULT_REQUESTED_BY" "${DEFAULT_REQUESTED_BY}" +upsert_key "OPS_DEFAULT_ISSUED_BY" "${DEFAULT_ISSUED_BY}" +upsert_key "OPS_DEFAULT_EXPIRES_IN_HOURS" "${DEFAULT_EXPIRES_IN_HOURS}" +upsert_key "OPS_DEFAULT_ROOT_DIR" "${DEFAULT_ROOT_DIR}" +upsert_key "OPS_NODE_AGENT_SERVICE_NAME" "${NODE_AGENT_SERVICE_NAME}" +upsert_key "OPS_NODE_AGENT_ENV_FILE" "${NODE_AGENT_ENV_FILE}" +upsert_commented_key "OPS_REPORT_ROOT" "${REPORT_ROOT}" + +python3 - <<'PY' "${TARGET_PATH}" +import json +import sys +from pathlib import Path + +config_path = Path(sys.argv[1]) +keys = [ + "OPS_ACTIVE_PROFILE", + "OPS_PROFILE_NAMES", + "OPS_MAINLAND_API_BASE_URL", + "OPS_OVERSEAS_API_BASE_URL", + "OPS_DEFAULT_CONTROL_PLANE_BASE_URL", + "OPS_DEFAULT_REQUESTED_BY", + "OPS_DEFAULT_ISSUED_BY", + "OPS_DEFAULT_EXPIRES_IN_HOURS", + "OPS_DEFAULT_ROOT_DIR", + "OPS_NODE_AGENT_SERVICE_NAME", + "OPS_NODE_AGENT_ENV_FILE", + "OPS_REPORT_ROOT", +] +values = {} +for line in config_path.read_text(encoding="utf-8").splitlines(): + if not line or line.startswith("#") or "=" not in line: + continue + key, value = line.split("=", 1) + if key in keys: + values[key] = value + +print(json.dumps({ + "ok": True, + "config_path": str(config_path), + "summary": values, + "next_commands": [ + "bash domain-api/deploy/multi-region/drive_ops_center.sh config", + "bash domain-api/deploy/multi-region/drive_ops_center.sh doctor-export", + "bash domain-api/deploy/multi-region/drive_ops_center.sh agent-check", + ], +}, ensure_ascii=False, indent=2)) +PY diff --git a/domain-api/deploy/multi-region/install_mainland_controller_quick.sh b/domain-api/deploy/multi-region/install_mainland_controller_quick.sh index e2ecdf2..797abba 100755 --- a/domain-api/deploy/multi-region/install_mainland_controller_quick.sh +++ b/domain-api/deploy/multi-region/install_mainland_controller_quick.sh @@ -100,7 +100,7 @@ REDIS_DB=0 EOF cd "${DOMAIN_API_DIR}" -bash deploy/multi-region/bootstrap_mainland.sh /opt/domaincheck controller +bash domain-api/deploy/multi-region/bootstrap_mainland.sh /opt/domaincheck controller cat >/etc/default/domaincheck-worker </etc/default/domaincheck-worker </dev/null + cp "${DOMAIN_API_DIR}/deploy/systemd/domain-api.service" /etc/systemd/system/domaincheck-api.service cp "${DOMAIN_API_DIR}/deploy/systemd/domain-worker.service" /etc/systemd/system/domaincheck-worker.service @@ -206,3 +215,5 @@ echo "国外主控机部署完成，建议继续执行：" echo "curl http://127.0.0.1:8100/health" echo "curl http://127.0.0.1:8100/api/v1/runtime/preflight" echo "curl http://127.0.0.1:8100/api/v1/runtime/readiness" +echo "bash ${DOMAIN_API_DIR}/deploy/multi-region/drive_ops_center.sh config" +echo "如后续接入大陆 controller，请记得修改 /etc/default/domaincheck-ops-center 中的 OPS_MAINLAND_API_BASE_URL" diff --git a/domain-api/deploy/multi-region/lib_ops_center.sh b/domain-api/deploy/multi-region/lib_ops_center.sh new file mode 100644 index 0000000..86a6b83 --- /dev/null +++ b/domain-api/deploy/multi-region/lib_ops_center.sh @@ -0,0 +1,106 @@ +#!/usr/bin/env bash + +ops_center_normalize_profile_key() { + local raw_value="${1:-default}" + printf '%s' "${raw_value}" | tr '[:lower:]-./' '[:upper:]____' | tr -cd 'A-Z0-9_' +} + +ops_center_load_config() { + local caller_dir="${1:-}" + if [[ -z "${caller_dir}" ]]; then + caller_dir="$(pwd)" + fi + + local env_active_profile="${OPS_ACTIVE_PROFILE:-}" + local env_profile_names="${OPS_PROFILE_NAMES:-}" + local env_report_root="${OPS_REPORT_ROOT:-}" + local env_mainland_api_base_url="${OPS_MAINLAND_API_BASE_URL:-}" + local env_overseas_api_base_url="${OPS_OVERSEAS_API_BASE_URL:-}" + local env_control_plane_base_url="${OPS_DEFAULT_CONTROL_PLANE_BASE_URL:-}" + local env_requested_by="${OPS_DEFAULT_REQUESTED_BY:-}" + local env_issued_by="${OPS_DEFAULT_ISSUED_BY:-}" + local env_expires_in_hours="${OPS_DEFAULT_EXPIRES_IN_HOURS:-}" + local env_root_dir="${OPS_DEFAULT_ROOT_DIR:-}" + local env_node_agent_service_name="${OPS_NODE_AGENT_SERVICE_NAME:-}" + local env_node_agent_env_file="${OPS_NODE_AGENT_ENV_FILE:-}" + + OPS_CENTER_SCRIPT_DIR="${caller_dir}" + OPS_CENTER_API_DIR="$(cd "${OPS_CENTER_SCRIPT_DIR}/../.." && pwd)" + OPS_CENTER_PROJECT_ROOT="$(cd "${OPS_CENTER_API_DIR}/.." && pwd)" + OPS_CENTER_RUNTIME_DIR="${OPS_CENTER_API_DIR}/runtime" + OPS_CENTER_REPORT_ROOT_DEFAULT="${OPS_CENTER_RUNTIME_DIR}/ops-center-reports" + OPS_CENTER_CONFIG_FILE="${DOMAINCHECK_OPS_CENTER_ENV:-${OPS_CENTER_ENV_FILE:-/etc/default/domaincheck-ops-center}}" + + if [[ -f "${OPS_CENTER_CONFIG_FILE}" ]]; then + # shellcheck disable=SC1090 + source "${OPS_CENTER_CONFIG_FILE}" + fi + + OPS_CENTER_ACTIVE_PROFILE="${OPS_CENTER_PROFILE:-${DOMAINCHECK_OPS_CENTER_PROFILE:-${env_active_profile:-${OPS_ACTIVE_PROFILE:-default}}}}" + OPS_CENTER_ACTIVE_PROFILE_KEY="$(ops_center_normalize_profile_key "${OPS_CENTER_ACTIVE_PROFILE}")" + OPS_CENTER_PROFILE_NAMES="${env_profile_names:-${OPS_PROFILE_NAMES:-${OPS_CENTER_ACTIVE_PROFILE}}}" + OPS_CENTER_REPORT_ROOT="${env_report_root:-${OPS_REPORT_ROOT:-${OPS_CENTER_REPORT_ROOT_DEFAULT}}}" + OPS_CENTER_ENV_MAINLAND_API_BASE_URL="${env_mainland_api_base_url}" + OPS_CENTER_ENV_OVERSEAS_API_BASE_URL="${env_overseas_api_base_url}" + OPS_CENTER_ENV_CONTROL_PLANE_BASE_URL="${env_control_plane_base_url}" + OPS_CENTER_ENV_REQUESTED_BY="${env_requested_by}" + OPS_CENTER_ENV_ISSUED_BY="${env_issued_by}" + OPS_CENTER_ENV_EXPIRES_IN_HOURS="${env_expires_in_hours}" + OPS_CENTER_ENV_ROOT_DIR="${env_root_dir}" + OPS_CENTER_ENV_NODE_AGENT_SERVICE_NAME="${env_node_agent_service_name}" + OPS_CENTER_ENV_NODE_AGENT_ENV_FILE="${env_node_agent_env_file}" +} + +ops_center_resolve_profile_value() { + local suffix="$1" + local fallback_value="${2:-}" + local profile_key="${OPS_CENTER_ACTIVE_PROFILE_KEY:-DEFAULT}" + local variable_name="OPS_PROFILE_${profile_key}_${suffix}" + local profile_value="${!variable_name:-}" + if [[ -n "${profile_value}" ]]; then + printf '%s' "${profile_value}" + else + printf '%s' "${fallback_value}" + fi +} + +ops_center_resolve_mainland_api_base_url() { + ops_center_resolve_profile_value "MAINLAND_API_BASE_URL" "${OPS_CENTER_ENV_MAINLAND_API_BASE_URL:-${OPS_MAINLAND_API_BASE_URL:-http://127.0.0.1:8100}}" +} + +ops_center_resolve_overseas_api_base_url() { + ops_center_resolve_profile_value "OVERSEAS_API_BASE_URL" "${OPS_CENTER_ENV_OVERSEAS_API_BASE_URL:-${OPS_OVERSEAS_API_BASE_URL:-http://127.0.0.1:8100}}" +} + +ops_center_resolve_control_plane_base_url() { + ops_center_resolve_profile_value "DEFAULT_CONTROL_PLANE_BASE_URL" "${OPS_CENTER_ENV_CONTROL_PLANE_BASE_URL:-${OPS_DEFAULT_CONTROL_PLANE_BASE_URL:-}}" +} + +ops_center_resolve_requested_by() { + ops_center_resolve_profile_value "DEFAULT_REQUESTED_BY" "${OPS_CENTER_ENV_REQUESTED_BY:-${OPS_DEFAULT_REQUESTED_BY:-cli/ops-center}}" +} + +ops_center_resolve_issued_by() { + ops_center_resolve_profile_value "DEFAULT_ISSUED_BY" "${OPS_CENTER_ENV_ISSUED_BY:-${OPS_DEFAULT_ISSUED_BY:-ops-center}}" +} + +ops_center_resolve_expires_in_hours() { + ops_center_resolve_profile_value "DEFAULT_EXPIRES_IN_HOURS" "${OPS_CENTER_ENV_EXPIRES_IN_HOURS:-${OPS_DEFAULT_EXPIRES_IN_HOURS:-72}}" +} + +ops_center_resolve_root_dir() { + ops_center_resolve_profile_value "DEFAULT_ROOT_DIR" "${OPS_CENTER_ENV_ROOT_DIR:-${OPS_DEFAULT_ROOT_DIR:-/opt/domaincheck}}" +} + +ops_center_resolve_node_agent_service_name() { + ops_center_resolve_profile_value "NODE_AGENT_SERVICE_NAME" "${OPS_CENTER_ENV_NODE_AGENT_SERVICE_NAME:-${OPS_NODE_AGENT_SERVICE_NAME:-domaincheck-node-agent}}" +} + +ops_center_resolve_node_agent_env_file() { + ops_center_resolve_profile_value "NODE_AGENT_ENV_FILE" "${OPS_CENTER_ENV_NODE_AGENT_ENV_FILE:-${OPS_NODE_AGENT_ENV_FILE:-/etc/default/domaincheck-node-agent}}" +} + +ops_center_ensure_report_dir() { + local report_dir="${1:-${OPS_CENTER_REPORT_ROOT_DEFAULT}}" + mkdir -p "${report_dir}" +} diff --git a/domain-api/deploy/multi-region/review_go_live_bundle.sh b/domain-api/deploy/multi-region/review_go_live_bundle.sh new file mode 100755 index 0000000..7c4b9f8 --- /dev/null +++ b/domain-api/deploy/multi-region/review_go_live_bundle.sh @@ -0,0 +1,317 @@ +#!/usr/bin/env bash +set -euo pipefail + +TARGET_PATH="${1:-}" + +if [[ -z "${TARGET_PATH}" ]]; then + echo "usage: bash domain-api/deploy/multi-region/review_go_live_bundle.sh " >&2 + exit 1 +fi + +if [[ -d "${TARGET_PATH}" ]]; then + MANIFEST_PATH="${TARGET_PATH%/}/manifest.json" +else + MANIFEST_PATH="${TARGET_PATH}" +fi + +if [[ ! -f "${MANIFEST_PATH}" ]]; then + echo "manifest not found: ${MANIFEST_PATH}" >&2 + exit 1 +fi + +python3 - "${MANIFEST_PATH}" <<'PY' +import json +import pathlib +import sys + +manifest_path = pathlib.Path(sys.argv[1]) +payload = json.loads(manifest_path.read_text(encoding="utf-8")) + +artifacts = payload.get("artifacts") or [] +failures = payload.get("failures") or [] +summary = payload.get("summary") or {} + +critical_keys = {"go_live_summary", "stack_diagnosis", "driver_feed", "codex_brief", "release_launchpad"} +artifact_by_key = {str(item.get("key") or ""): item for item in artifacts if isinstance(item, dict)} +critical_failures = [key for key in critical_keys if not bool((artifact_by_key.get(key) or {}).get("ok", False))] +noncritical_failures = [key for key in failures if key not in critical_keys] +env_audit_artifact = artifact_by_key.get("env_audit") or {} + +env_audit_status = "" +env_audit_headline = "" +env_audit_missing_items = [] +env_audit_recommended_actions = [] +env_audit_blocking = False +env_audit_attention = False +env_audit_runtime_may_need_restart = False +go_live_summary_payload = {} +stack_diagnosis_payload = {} +driver_feed_payload = {} +codex_brief_payload = {} + +def load_json_artifact(key: str) -> dict: + artifact = artifact_by_key.get(key) or {} + artifact_path = pathlib.Path(str(artifact.get("path") or "")) + if not artifact_path.is_file(): + return {} + try: + loaded = json.loads(artifact_path.read_text(encoding="utf-8")) + except Exception: + return {} + return loaded if isinstance(loaded, dict) else {} + +env_audit_path = pathlib.Path(str(env_audit_artifact.get("path") or "")) +if env_audit_path.is_file(): + raw_text = env_audit_path.read_text(encoding="utf-8", errors="replace") + marker = "[8/8] condensed env audit summary" + candidate_text = raw_text.split(marker, 1)[1].strip() if marker in raw_text else raw_text.strip() + if candidate_text: + try: + env_payload = json.loads(candidate_text) + except Exception: + env_payload = {} + env_audit_status = str(env_payload.get("status") or "") + env_audit_headline = str(env_payload.get("headline") or "") + env_audit_missing_items = list(env_payload.get("missing_items") or []) + env_audit_recommended_actions = [str(item).strip() for item in list(env_payload.get("recommended_actions") or []) if str(item).strip()] + env_audit_runtime_may_need_restart = bool(((env_payload.get("runtime") or {}).get("runtime_may_need_restart", False))) + env_audit_blocking = env_audit_status == "blocked" + env_audit_attention = env_audit_status == "attention" + +go_live_summary_payload = load_json_artifact("go_live_summary") +stack_diagnosis_payload = load_json_artifact("stack_diagnosis") +driver_feed_payload = load_json_artifact("driver_feed") +codex_brief_payload = load_json_artifact("codex_brief") + +go_live_launchpad_target_node_code = str(go_live_summary_payload.get("launchpad_recommended_target_node_code") or "").strip() +go_live_launchpad_recovery_label = str(go_live_summary_payload.get("launchpad_recommended_recovery_label") or "").strip() +go_live_launchpad_recovery_summary = str(go_live_summary_payload.get("launchpad_recommended_recovery_summary") or "").strip() +go_live_launchpad_bootstrap_pending_nodes = int(go_live_summary_payload.get("launchpad_onboarding_bootstrap_pending_nodes", 0) or 0) +go_live_launchpad_acceptance_ready_nodes = int(go_live_summary_payload.get("launchpad_onboarding_acceptance_ready_nodes", 0) or 0) +stack_diagnosis = stack_diagnosis_payload.get("diagnosis") or {} +stack_launchpad_target_node_code = str(stack_diagnosis.get("launchpad_recommended_target_node_code") or "").strip() +stack_launchpad_recovery_label = str(stack_diagnosis.get("launchpad_recommended_recovery_label") or "").strip() +stack_launchpad_recovery_summary = str(stack_diagnosis.get("launchpad_recommended_recovery_summary") or "").strip() +stack_launchpad_bootstrap_pending_nodes = int(stack_diagnosis.get("launchpad_onboarding_bootstrap_pending_nodes", 0) or 0) +stack_launchpad_acceptance_ready_nodes = int(stack_diagnosis.get("launchpad_onboarding_acceptance_ready_nodes", 0) or 0) +driver_feed_summary = driver_feed_payload.get("summary") or {} +codex_brief_summary = codex_brief_payload.get("summary") or {} + + +def nonempty_distinct(values: dict[str, str]) -> list[str]: + return sorted({str(value or "").strip() for value in values.values() if str(value or "").strip()}) + + +def distinct_ints(values: dict[str, int], available_sources: set[str]) -> list[int]: + return sorted({int(values[source]) for source in values if source in available_sources}) + + +launchpad_target_node_codes = { + "go_live_summary": go_live_launchpad_target_node_code, + "stack_diagnosis": stack_launchpad_target_node_code, + "driver_feed": str(driver_feed_summary.get("launchpad_recommended_target_node_code") or "").strip(), + "codex_brief": str(codex_brief_summary.get("launchpad_recommended_target_node_code") or "").strip(), +} +launchpad_recovery_labels = { + "go_live_summary": go_live_launchpad_recovery_label, + "stack_diagnosis": stack_launchpad_recovery_label, + "driver_feed": str(driver_feed_summary.get("launchpad_recommended_recovery_label") or "").strip(), + "codex_brief": str(codex_brief_summary.get("launchpad_recommended_recovery_label") or "").strip(), +} +launchpad_recovery_summaries = { + "go_live_summary": go_live_launchpad_recovery_summary, + "stack_diagnosis": stack_launchpad_recovery_summary, + "driver_feed": str(driver_feed_summary.get("launchpad_recommended_recovery_summary") or "").strip(), + "codex_brief": str(codex_brief_summary.get("launchpad_recommended_recovery_summary") or "").strip(), +} +launchpad_bootstrap_pending_nodes = { + "go_live_summary": go_live_launchpad_bootstrap_pending_nodes, + "stack_diagnosis": stack_launchpad_bootstrap_pending_nodes, + "driver_feed": int(driver_feed_summary.get("launchpad_onboarding_bootstrap_pending_nodes", 0) or 0), + "codex_brief": int(codex_brief_summary.get("launchpad_onboarding_bootstrap_pending_nodes", 0) or 0), +} +launchpad_acceptance_ready_nodes = { + "go_live_summary": go_live_launchpad_acceptance_ready_nodes, + "stack_diagnosis": stack_launchpad_acceptance_ready_nodes, + "driver_feed": int(driver_feed_summary.get("launchpad_onboarding_acceptance_ready_nodes", 0) or 0), + "codex_brief": int(codex_brief_summary.get("launchpad_onboarding_acceptance_ready_nodes", 0) or 0), +} + +available_alignment_sources = { + source + for source, payload_value in { + "go_live_summary": go_live_summary_payload, + "stack_diagnosis": stack_diagnosis_payload, + "driver_feed": driver_feed_payload, + "codex_brief": codex_brief_payload, + }.items() + if isinstance(payload_value, dict) and payload_value +} + +launchpad_target_consistent = len(nonempty_distinct({ + source: value for source, value in launchpad_target_node_codes.items() if source in available_alignment_sources +})) <= 1 +launchpad_recovery_label_consistent = len(nonempty_distinct({ + source: value for source, value in launchpad_recovery_labels.items() if source in available_alignment_sources +})) <= 1 +launchpad_recovery_summary_consistent = len(nonempty_distinct({ + source: value for source, value in launchpad_recovery_summaries.items() if source in available_alignment_sources +})) <= 1 +launchpad_bootstrap_pending_consistent = len(distinct_ints(launchpad_bootstrap_pending_nodes, available_alignment_sources)) <= 1 +launchpad_acceptance_ready_consistent = len(distinct_ints(launchpad_acceptance_ready_nodes, available_alignment_sources)) <= 1 +launchpad_alignment_consistent = all([ + launchpad_target_consistent, + launchpad_recovery_label_consistent, + launchpad_recovery_summary_consistent, + launchpad_bootstrap_pending_consistent, + launchpad_acceptance_ready_consistent, +]) + +node_agent_env_missing = any( + str(item).strip() == "missing_env:/etc/default/domaincheck-node-agent" + for item in env_audit_missing_items +) + +if critical_failures or env_audit_blocking: + status = "blocked" + if critical_failures and env_audit_blocking: + headline = "关键报告存在缺失，且环境审计已给出阻断结论，当前不适合上线。" + elif critical_failures: + headline = "上线证据包仍缺关键报告，暂不建议直接宣称收口完成。" + else: + headline = "环境审计已给出阻断结论，需先修复现场环境后再进入发布门禁。" +elif noncritical_failures or env_audit_attention or not launchpad_alignment_consistent: + status = "attention" + if not launchpad_alignment_consistent: + headline = "核心报告已齐，但 launchpad 摘要在不同 surface 之间存在漂移，建议先复核再发版。" + elif noncritical_failures and env_audit_attention: + headline = "核心报告已齐，但补充报告和环境审计都提示仍有缺口，建议先复核再发版。" + elif noncritical_failures: + headline = "核心报告已齐，但仍有补充报告超时或失败，建议先复核再发版。" + else: + headline = "核心报告已齐，但环境审计仍提示基础缺口，建议补齐后再发版。" +else: + status = "ready" + headline = "核心上线证据已经齐备，可进入最终人工复核或正式发布门禁。" + +result = { + "report_dir": str(payload.get("report_dir") or manifest_path.parent), + "status": status, + "headline": headline, + "artifact_total": int(summary.get("artifact_total", len(artifacts)) or len(artifacts)), + "failure_total": int(summary.get("failure_total", len(failures)) or len(failures)), + "critical_failures": critical_failures, + "noncritical_failures": noncritical_failures, + "env_audit": { + "ok": bool(env_audit_artifact.get("ok", False)), + "status": env_audit_status, + "headline": env_audit_headline, + "missing_items": env_audit_missing_items, + "runtime_may_need_restart": env_audit_runtime_may_need_restart, + "recommended_actions": env_audit_recommended_actions, + }, + "go_live_summary": { + "go_live_status": str(go_live_summary_payload.get("go_live_status") or "").strip(), + "publish_status": str(go_live_summary_payload.get("publish_status") or "").strip(), + "operator_title": str(go_live_summary_payload.get("operator_title") or "").strip(), + "next_step_action_code": str(go_live_summary_payload.get("next_step_action_code") or "").strip(), + "launchpad_recommended_target_node_code": go_live_launchpad_target_node_code, + "launchpad_recommended_recovery_label": go_live_launchpad_recovery_label, + "launchpad_recommended_recovery_summary": go_live_launchpad_recovery_summary, + "launchpad_onboarding_bootstrap_pending_nodes": go_live_launchpad_bootstrap_pending_nodes, + "launchpad_onboarding_acceptance_ready_nodes": go_live_launchpad_acceptance_ready_nodes, + }, + "stack_diagnosis": { + "stack_status": str((stack_diagnosis_payload.get("diagnosis") or {}).get("stack_status") or "").strip(), + "operator_title": str(((stack_diagnosis_payload.get("diagnosis") or {}).get("operator_decision") or {}).get("title") or "").strip(), + "next_step_action_code": str(((stack_diagnosis_payload.get("diagnosis") or {}).get("next_step") or {}).get("action_code") or "").strip(), + "launchpad_recommended_target_node_code": stack_launchpad_target_node_code, + "launchpad_recommended_recovery_label": stack_launchpad_recovery_label, + "launchpad_recommended_recovery_summary": stack_launchpad_recovery_summary, + "launchpad_onboarding_bootstrap_pending_nodes": stack_launchpad_bootstrap_pending_nodes, + "launchpad_onboarding_acceptance_ready_nodes": stack_launchpad_acceptance_ready_nodes, + }, + "driver_feed": { + "headline": str(driver_feed_payload.get("headline") or "").strip(), + "launch_status": str(((driver_feed_payload.get("automation_coverage") or {}).get("launch_status") or "")).strip(), + "launch_ready": bool(((driver_feed_payload.get("automation_coverage") or {}).get("launch_ready", False))), + "launchpad_recommended_target_node_code": str( + ((driver_feed_payload.get("summary") or {}).get("launchpad_recommended_target_node_code") or "") + ).strip(), + "launchpad_recommended_recovery_label": str( + ((driver_feed_payload.get("summary") or {}).get("launchpad_recommended_recovery_label") or "") + ).strip(), + }, + "codex_brief": { + "headline": str(codex_brief_payload.get("headline") or "").strip(), + "focus_entry_key": str(((codex_brief_payload.get("focus") or {}).get("entry_key") or "")).strip(), + "launch_status": str(((codex_brief_payload.get("automation_coverage") or {}).get("launch_status") or "")).strip(), + "launch_ready": bool(((codex_brief_payload.get("automation_coverage") or {}).get("launch_ready", False))), + "launchpad_recommended_target_node_code": str( + ((codex_brief_payload.get("summary") or {}).get("launchpad_recommended_target_node_code") or "") + ).strip(), + "launchpad_recommended_recovery_label": str( + ((codex_brief_payload.get("summary") or {}).get("launchpad_recommended_recovery_label") or "") + ).strip(), + }, + "launchpad_alignment": { + "consistent": launchpad_alignment_consistent, + "target_node_code_consistent": launchpad_target_consistent, + "recovery_label_consistent": launchpad_recovery_label_consistent, + "recovery_summary_consistent": launchpad_recovery_summary_consistent, + "bootstrap_pending_consistent": launchpad_bootstrap_pending_consistent, + "acceptance_ready_consistent": launchpad_acceptance_ready_consistent, + "available_sources": sorted(available_alignment_sources), + "target_node_codes": launchpad_target_node_codes, + "recovery_labels": launchpad_recovery_labels, + "recovery_summaries": launchpad_recovery_summaries, + "bootstrap_pending_nodes": launchpad_bootstrap_pending_nodes, + "acceptance_ready_nodes": launchpad_acceptance_ready_nodes, + }, + "recommended_reading_order": list(summary.get("recommended_reading_order") or []), + "recommended_next_steps": [ + "先看 00_env_audit.txt，确认当前机器环境是否存在阻断或注意项", + "先看 02_go_live_summary.json 和 03_stack_diagnosis.json", + "再看 04_driver_feed.json 和 05_codex_brief.json", + "若涉及发布门禁，再看 06_release_launchpad.json", + "若 manifest 里仍有失败项，再看对应 artifact 原文件", + ] + + ( + [ + "当前缺口指向 Node Agent 未接管：先执行 agent-gap-export，拿到首个缺口节点的接管包", + "随后执行 node-bootstrap-plan，为缺口节点生成可直接落地的接管脚本", + "如果仓库代码已更新但 bootstrap-plan 输出仍偏旧，先重启控制面 API 再重新导出接管计划", + ] + if node_agent_env_missing + else [] + ) + + ( + [ + "env-audit 已识别运行中 API 可能仍是旧代码：先执行 drive_ops_center.sh runtime-refresh-recover，按固定链路完成 API 重启与复检", + "若仍需人工逐条确认，再执行 recommended_actions 里的 API 重启与复检命令" + ] + if env_audit_runtime_may_need_restart + else [] + ) + + ( + [ + f"当前 launchpad 缺口已经收敛到节点 {go_live_launchpad_target_node_code}，建议优先执行：{go_live_launchpad_recovery_label or '节点恢复'}", + "先对照 02_go_live_summary.json、04_driver_feed.json、05_codex_brief.json 中的 launchpad 字段，确认三处摘要是否一致", + (go_live_launchpad_recovery_summary or "如果 recovery_summary 仍为空，再回到 06_release_launchpad.json 查看 gap rows 详情"), + ] + if go_live_launchpad_target_node_code + else [] + ) + + ( + [ + "当前 bundle 内的 launchpad 摘要在 go_live_summary / stack_diagnosis / driver_feed / codex_brief 之间存在不一致，先不要直接交付。", + "优先复核 02_go_live_summary.json 与 03_stack_diagnosis.json，再检查 04_driver_feed.json 与 05_codex_brief.json 是否使用了同一版控制面数据。", + "若只有局部摘要偏旧，先执行 drive_ops_center.sh runtime-refresh-recover，并重新导出 go-live bundle。", + ] + if not launchpad_alignment_consistent + else [] + ), +} + +print(json.dumps(result, ensure_ascii=False, indent=2)) +PY diff --git a/domain-api/deploy/multi-region/templates/domaincheck-node-agent.env.example b/domain-api/deploy/multi-region/templates/domaincheck-node-agent.env.example new file mode 100644 index 0000000..f867a84 --- /dev/null +++ b/domain-api/deploy/multi-region/templates/domaincheck-node-agent.env.example @@ -0,0 +1,15 @@ +OPS_CONTROL_PLANE_BASE_URL=http://127.0.0.1:8100 +OPS_AGENT_TOKEN= +OPS_AGENT_POLL_INTERVAL_SECONDS=5 + +NODE_CODE=mainland-worker-01 +NODE_REGION=mainland +NODE_ROLE=worker + +WORKER_SERVICE_NAME=domaincheck-worker +API_SERVICE_NAME=domaincheck-api +SYNC_AGENT_SERVICE_NAME=domaincheck-sync-agent +NODE_AGENT_SERVICE_NAME=domaincheck-node-agent + +OPS_AGENT_CAPABILITIES=["service.start","service.stop","service.restart","service.status","runtime.start_worker","runtime.stop_worker","runtime.restart_api","runtime.start_sync_agent","runtime.stop_sync_agent","health.snapshot","logs.collect","diagnostics.collect","deploy.release"] +OPS_AGENT_LABELS={} diff --git a/domain-api/deploy/multi-region/templates/domaincheck-ops-center.env.example b/domain-api/deploy/multi-region/templates/domaincheck-ops-center.env.example new file mode 100644 index 0000000..b329373 --- /dev/null +++ b/domain-api/deploy/multi-region/templates/domaincheck-ops-center.env.example @@ -0,0 +1,49 @@ +# Overseas control-plane unified entrypoint config +# +# Recommended target path: +# /etc/default/domaincheck-ops-center +# +# Usage: +# export DOMAINCHECK_OPS_CENTER_ENV=/etc/default/domaincheck-ops-center +# bash domain-api/deploy/multi-region/drive_ops_center.sh doctor + +# Active profile name +OPS_ACTIVE_PROFILE=default + +# Optional profile list shown by `drive_ops_center.sh config` +# OPS_PROFILE_NAMES=default,prod,staging + +# Mainland controller API base URL +OPS_MAINLAND_API_BASE_URL=http://127.0.0.1:8100 + +# Overseas control-plane API base URL +OPS_OVERSEAS_API_BASE_URL=http://127.0.0.1:8100 + +# Public control-plane base URL used in generated node-agent bootstrap plans +OPS_DEFAULT_CONTROL_PLANE_BASE_URL=https://api.example.com + +# Default request identity used by CLI wrappers +OPS_DEFAULT_REQUESTED_BY=cli/ops-center + +# Default metadata used when generating node-agent bootstrap plans +OPS_DEFAULT_ISSUED_BY=ops-center +OPS_DEFAULT_EXPIRES_IN_HOURS=72 +OPS_DEFAULT_ROOT_DIR=/opt/domaincheck + +# Optional report export root +# OPS_REPORT_ROOT=/opt/domaincheck/domain-api/runtime/ops-center-reports + +# Local node-agent status check defaults +OPS_NODE_AGENT_SERVICE_NAME=domaincheck-node-agent +OPS_NODE_AGENT_ENV_FILE=/etc/default/domaincheck-node-agent + +# Optional per-profile overrides: +# OPS_PROFILE_PROD_MAINLAND_API_BASE_URL=http://10.0.0.10:8100 +# OPS_PROFILE_PROD_OVERSEAS_API_BASE_URL=https://api.example.com +# OPS_PROFILE_PROD_DEFAULT_CONTROL_PLANE_BASE_URL=https://api.example.com +# OPS_PROFILE_PROD_DEFAULT_REQUESTED_BY=cli/ops-center-prod +# OPS_PROFILE_PROD_DEFAULT_ISSUED_BY=ops-center-prod +# OPS_PROFILE_PROD_DEFAULT_EXPIRES_IN_HOURS=72 +# OPS_PROFILE_PROD_DEFAULT_ROOT_DIR=/opt/domaincheck +# OPS_PROFILE_PROD_NODE_AGENT_SERVICE_NAME=domaincheck-node-agent +# OPS_PROFILE_PROD_NODE_AGENT_ENV_FILE=/etc/default/domaincheck-node-agent diff --git a/domain-api/deploy/systemd/domain-node-agent.service b/domain-api/deploy/systemd/domain-node-agent.service new file mode 100644 index 0000000..9e2a520 --- /dev/null +++ b/domain-api/deploy/systemd/domain-node-agent.service @@ -0,0 +1,17 @@ +[Unit] +Description=domainCheck Node Agent +After=network.target + +[Service] +Type=simple +WorkingDirectory=/opt/domaincheck/domain-api +EnvironmentFile=-/etc/default/domaincheck-node-agent +Environment="PATH=/home/www/.nvm/versions/node/v20.20.2/bin:/home/www/.local/bin:/home/www/bin:/sbin:/bin:/usr/sbin:/usr/bin:/usr/local/sbin:/usr/local/bin" +ExecStart=/opt/domaincheck/domainCheck/.venv/bin/python -m app.node_agent +Restart=always +RestartSec=5 +User=www +Group=www + +[Install] +WantedBy=multi-user.target diff --git a/domain-api/tests/test_build_info_service.py b/domain-api/tests/test_build_info_service.py new file mode 100644 index 0000000..78f8fdd --- /dev/null +++ b/domain-api/tests/test_build_info_service.py @@ -0,0 +1,98 @@ +import json +import tempfile +import unittest +from pathlib import Path +from unittest.mock import patch + +from app.core.config import settings +from app.services.build_info_service import ( + _expected_route_paths, + get_runtime_build_info, + remember_registered_route_paths, +) + + +class BuildInfoServiceTests(unittest.TestCase): + def tearDown(self) -> None: + remember_registered_route_paths([]) + + def test_runtime_build_info_prefers_configured_manifest_and_reports_registered_surface(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + manifest_path = Path(temp_dir) / "release_manifest.json" + manifest_path.write_text( + json.dumps( + { + "package_name": "domaincheck_release_demo", + "generated_at": "2026-04-18T12:00:00", + "commit_sha": "bd6bcb2", + "commit_ref": "main", + "sha256": "demo-sha256", + }, + ensure_ascii=False, + ), + encoding="utf-8", + ) + + route_paths = list(_expected_route_paths().values()) + remember_registered_route_paths(route_paths) + + with patch.multiple( + settings, + build_manifest_path=str(manifest_path), + build_commit_sha="", + build_commit_ref="", + build_generated_at="", + build_package_name="", + build_checksum="", + build_source_label="", + ): + payload = get_runtime_build_info() + + self.assertEqual("configured_manifest", payload["source"]) + self.assertEqual("domaincheck_release_demo", payload["package_name"]) + self.assertEqual("bd6bcb2", payload["commit_sha"]) + self.assertEqual("main", payload["commit_ref"]) + self.assertEqual("demo-sha256", payload["checksum"]) + self.assertEqual(str(manifest_path), payload["manifest_path"]) + self.assertTrue(payload["route_surface"]["surface_complete"]) + self.assertEqual([], payload["route_surface"]["missing_keys"]) + self.assertEqual("registered", payload["route_surface"]["mode"]) + self.assertIn("repository_capabilities", payload) + self.assertIn("supports_install_command_block", payload["repository_capabilities"]) + self.assertIn("supports_multi_layout_bootstrap", payload["repository_capabilities"]) + + def test_runtime_build_info_env_override_wins_and_missing_routes_are_visible(self) -> None: + with patch.multiple( + settings, + build_manifest_path="", + build_commit_sha="246838ae4c07", + build_commit_ref="main", + build_generated_at="2026-04-18T12:10:00", + build_package_name="domaincheck_release_env", + build_checksum="env-sha256", + build_source_label="env-override", + ): + payload = get_runtime_build_info(route_paths=[_expected_route_paths()["ops_contracts"]]) + + self.assertEqual("env-override", payload["source"]) + self.assertEqual("domaincheck_release_env", payload["package_name"]) + self.assertEqual("246838ae4c07", payload["commit_sha"]) + self.assertEqual("main", payload["commit_ref"]) + self.assertEqual("env-sha256", payload["checksum"]) + self.assertFalse(payload["route_surface"]["surface_complete"]) + self.assertIn("ops_stack_diagnosis", payload["route_surface"]["missing_keys"]) + self.assertIn("ops_node_onboarding_bootstrap_preview", payload["route_surface"]["missing_keys"]) + self.assertIn("ops_node_onboarding_bootstrap_execute", payload["route_surface"]["missing_keys"]) + self.assertIn("ops_node_onboarding_acceptance_preview", payload["route_surface"]["missing_keys"]) + self.assertIn("ops_node_onboarding_acceptance_execute", payload["route_surface"]["missing_keys"]) + self.assertIn("ops_node_onboarding_recovery_preview", payload["route_surface"]["missing_keys"]) + self.assertIn("ops_node_onboarding_recovery_execute", payload["route_surface"]["missing_keys"]) + self.assertIn("ops_node_scene_log", payload["route_surface"]["missing_keys"]) + self.assertIn("runtime_build_info", payload["route_surface"]["missing_keys"]) + self.assertEqual("registered", payload["route_surface"]["mode"]) + self.assertIn("repository_capabilities", payload) + self.assertIn("supports_install_command_block", payload["repository_capabilities"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/domain-api/tests/test_drive_ops_center_docs.py b/domain-api/tests/test_drive_ops_center_docs.py new file mode 100644 index 0000000..d0070b6 --- /dev/null +++ b/domain-api/tests/test_drive_ops_center_docs.py @@ -0,0 +1,111 @@ +from __future__ import annotations + +import os +import re +import subprocess +import unittest +from pathlib import Path + + +class DriveOpsCenterDocsTests(unittest.TestCase): + def test_authoritative_docs_only_reference_supported_commands(self) -> None: + repo_root = Path(__file__).resolve().parents[2] + script_path = repo_root / "domain-api" / "deploy" / "multi-region" / "drive_ops_center.sh" + doc_paths = [ + repo_root / "domain-api" / "deploy" / "multi-region" / "README.md", + repo_root / "docs" / "25_domainCheck_海外单脑控制面上线收口总表.md", + repo_root / "docs" / "26_domainCheck_发布前运行验证与交付模板.md", + ] + + help_output = subprocess.run( + ["bash", str(script_path), "help"], + cwd=repo_root, + check=True, + capture_output=True, + text=True, + ).stdout + + supported_commands = self._extract_supported_commands(help_output) + documented_commands = self._extract_documented_commands(doc_paths) + + missing_commands = sorted(documented_commands - supported_commands) + self.assertEqual( + [], + missing_commands, + msg=( + "These drive_ops_center.sh commands are referenced by authoritative docs " + "but missing from the CLI help output: " + + ", ".join(missing_commands) + ), + ) + self.assertIn("stack-diagnosis", documented_commands) + self.assertIn("stack-diagnosis", supported_commands) + + def test_stack_diagnosis_summary_argument_normalization_does_not_treat_summary_as_host(self) -> None: + repo_root = Path(__file__).resolve().parents[2] + script_path = repo_root / "domain-api" / "deploy" / "multi-region" / "drive_ops_center.sh" + env = dict(os.environ) + env["DOMAINCHECK_OPS_CENTER_ENV"] = "/tmp/domaincheck-ops-center-does-not-exist" + env["OPS_MAINLAND_API_BASE_URL"] = "http://127.0.0.1:9" + + result = subprocess.run( + ["bash", str(script_path), "stack-diagnosis", "summary"], + cwd=repo_root, + check=True, + capture_output=True, + text=True, + env=env, + ) + + combined_output = f"{result.stdout}\n{result.stderr}" + self.assertNotIn("Could not resolve host: summary", combined_output) + self.assertNotIn("NameError", combined_output) + self.assertIn("[9/9] condensed stack summary", combined_output) + + def test_env_drift_recommends_stack_diagnosis_instead_of_stack_check(self) -> None: + repo_root = Path(__file__).resolve().parents[2] + script_path = repo_root / "domain-api" / "deploy" / "multi-region" / "check_env_drift.sh" + script_text = script_path.read_text(encoding="utf-8") + + self.assertIn( + 'recommended_actions.append("bash domain-api/deploy/multi-region/drive_ops_center.sh stack-diagnosis http://127.0.0.1:8100 summary")', + script_text, + ) + self.assertNotIn( + 'recommended_actions.append("bash domain-api/deploy/multi-region/drive_ops_center.sh stack-check http://127.0.0.1:8100 summary")', + script_text, + ) + + @staticmethod + def _extract_supported_commands(help_output: str) -> set[str]: + commands: set[str] = set() + in_commands_section = False + for raw_line in help_output.splitlines(): + line = raw_line.rstrip() + if line.strip() == "commands:": + in_commands_section = True + continue + if line.strip() == "examples:": + break + if not in_commands_section: + continue + stripped = line.strip() + if not stripped: + continue + command = stripped.split()[0] + if re.fullmatch(r"[a-z0-9-]+", command): + commands.add(command) + return commands + + @staticmethod + def _extract_documented_commands(doc_paths: list[Path]) -> set[str]: + pattern = re.compile(r"drive_ops_center\.sh\s+([a-z0-9-]+)") + commands: set[str] = set() + for doc_path in doc_paths: + text = doc_path.read_text(encoding="utf-8") + commands.update(match.group(1) for match in pattern.finditer(text)) + return commands + + +if __name__ == "__main__": + unittest.main() diff --git a/domain-api/tests/test_node_agent_delivery_queue.py b/domain-api/tests/test_node_agent_delivery_queue.py new file mode 100644 index 0000000..960843a --- /dev/null +++ b/domain-api/tests/test_node_agent_delivery_queue.py @@ -0,0 +1,284 @@ +import json +import os +import tempfile +import unittest +import urllib.error +from unittest.mock import patch + +from app import node_agent + + +class NodeAgentDeliveryQueueTests(unittest.TestCase): + def test_base_payload_includes_delivery_queue_snapshot(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + with patch.object(node_agent, "AGENT_QUEUE_DIR", temp_dir): + node_agent._ensure_queue_dirs() + payload = node_agent._base_payload() + + delivery_queue = payload["metadata"]["delivery_queue"] + self.assertEqual("healthy", delivery_queue["state"]) + self.assertEqual(0, delivery_queue["pending_count"]) + self.assertEqual(0, delivery_queue["dead_letter_count"]) + + def test_job_event_network_failure_is_queued_for_retry(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + with patch.object(node_agent, "AGENT_QUEUE_DIR", temp_dir), patch.object( + node_agent, + "_post", + side_effect=urllib.error.URLError("offline"), + ): + result = node_agent._job_event( + 7, + event_type="executor_received", + message="accepted", + client_event_id="evt-001", + summary_text="已接单", + focus_ref={"kind": "ops_job", "job_id": 7}, + ) + + self.assertEqual("queued", result["state"]) + pending_dir = os.path.join(temp_dir, "pending") + pending_files = sorted(os.listdir(pending_dir)) + self.assertEqual(1, len(pending_files)) + with open(os.path.join(pending_dir, pending_files[0]), "r", encoding="utf-8") as handle: + record = json.load(handle) + self.assertEqual("evt-001", record["payload"]["client_event_id"]) + self.assertEqual("已接单", record["payload"]["summary_text"]) + self.assertEqual("ops_job", record["payload"]["focus_ref"]["kind"]) + self.assertEqual("job_event", record["kind"]) + + def test_job_complete_transport_failure_preserves_client_request_id(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + with patch.object(node_agent, "AGENT_QUEUE_DIR", temp_dir), patch.object( + node_agent, + "_post", + side_effect=urllib.error.URLError("offline"), + ): + result = node_agent._job_complete( + 9, + status="success", + stdout="done", + stderr="", + result={"ok": True}, + client_request_id="complete-001", + duration_ms=321, + summary_text="执行成功", + focus_ref={"kind": "ops_job", "job_id": 9}, + ) + + self.assertEqual("queued", result["state"]) + pending_dir = os.path.join(temp_dir, "pending") + pending_files = sorted(os.listdir(pending_dir)) + self.assertEqual(1, len(pending_files)) + with open(os.path.join(pending_dir, pending_files[0]), "r", encoding="utf-8") as handle: + record = json.load(handle) + self.assertEqual("complete-001", record["payload"]["client_request_id"]) + self.assertEqual(321, record["payload"]["duration_ms"]) + self.assertEqual("执行成功", record["payload"]["summary_text"]) + self.assertEqual("ops_job", record["payload"]["focus_ref"]["kind"]) + self.assertEqual("job_complete", record["kind"]) + + def test_flush_delivery_queue_delivers_pending_records(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + with patch.object(node_agent, "AGENT_QUEUE_DIR", temp_dir): + record = node_agent._build_delivery_record( + "job_event", + "/api/v1/ops/agent/jobs/7/events", + { + "node_code": "mainland-worker-01", + "event_type": "executor_received", + "message": "accepted", + "client_event_id": "evt-002", + }, + "evt-002", + ) + node_agent._store_pending_delivery(record) + with patch.object(node_agent, "_post", return_value={"code": 0, "message": "ok"}): + summary = node_agent._flush_delivery_queue(limit=10) + + self.assertEqual(1, summary["delivered"]) + self.assertEqual([], os.listdir(os.path.join(temp_dir, "pending"))) + + def test_flush_delivery_queue_moves_semantic_failure_to_dead_letter(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + with patch.object(node_agent, "AGENT_QUEUE_DIR", temp_dir): + record = node_agent._build_delivery_record( + "job_complete", + "/api/v1/ops/agent/jobs/9/complete", + { + "node_code": "mainland-worker-01", + "status": "success", + "client_request_id": "complete-002", + }, + "complete-002", + ) + node_agent._store_pending_delivery(record) + with patch.object( + node_agent, + "_post", + return_value={ + "code": 1, + "message": "任务不存在或不属于当前节点", + "detail_code": "ops_job_not_owned_by_agent", + }, + ): + summary = node_agent._flush_delivery_queue(limit=10) + + self.assertEqual(1, summary["dead_letter"]) + self.assertEqual([], os.listdir(os.path.join(temp_dir, "pending"))) + self.assertEqual(1, len(os.listdir(os.path.join(temp_dir, "dead-letter")))) + + def test_replay_dead_letter_records_moves_record_back_to_pending_and_flushes(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + with patch.object(node_agent, "AGENT_QUEUE_DIR", temp_dir): + record = node_agent._build_delivery_record( + "job_event", + "/api/v1/ops/agent/jobs/7/events", + { + "node_code": "mainland-worker-01", + "event_type": "executor_received", + "message": "accepted", + "client_event_id": "evt-003", + }, + "evt-003", + ) + node_agent._store_dead_letter_delivery( + record, + reason="temporary failure", + detail_code="transient_error", + ) + with patch.object(node_agent, "_post", return_value={"code": 0, "message": "ok"}): + ok, message, data = node_agent._execute_action( + "delivery.queue.replay", + { + "selector": {"record_id": "evt-003"}, + "limit": 10, + "flush_after_replay": True, + "reason": "恢复后重放", + }, + ) + + self.assertTrue(ok) + self.assertEqual("已重放 1 条死信记录", message) + self.assertEqual(1, data["replayed_total"]) + self.assertEqual(1, data["flush_summary"]["delivered"]) + self.assertEqual([], os.listdir(os.path.join(temp_dir, "dead-letter"))) + self.assertEqual([], os.listdir(os.path.join(temp_dir, "pending"))) + + def test_discard_dead_letter_records_archives_record(self) -> None: + with tempfile.TemporaryDirectory() as temp_dir: + with patch.object(node_agent, "AGENT_QUEUE_DIR", temp_dir): + record = node_agent._build_delivery_record( + "job_complete", + "/api/v1/ops/agent/jobs/8/complete", + { + "node_code": "mainland-worker-01", + "status": "failed", + "client_request_id": "complete-003", + }, + "complete-003", + ) + node_agent._store_dead_letter_delivery( + record, + reason="permanent failure", + detail_code="ops_job_invalid_status", + ) + ok, message, data = node_agent._execute_action( + "delivery.queue.discard", + { + "selector": {"record_id": "complete-003"}, + "limit": 10, + "discarded_by": "web-ui", + "reason": "确认不需要再补发", + }, + ) + + self.assertTrue(ok) + self.assertEqual("已丢弃 1 条死信记录", message) + self.assertEqual(1, data["discarded_total"]) + self.assertEqual([], os.listdir(os.path.join(temp_dir, "dead-letter"))) + discarded_files = os.listdir(os.path.join(temp_dir, "discarded")) + self.assertEqual(1, len(discarded_files)) + with open(os.path.join(temp_dir, "discarded", discarded_files[0]), "r", encoding="utf-8") as handle: + archived = json.load(handle) + self.assertEqual("web-ui", archived["discarded_by"]) + self.assertEqual("确认不需要再补发", archived["discard_reason"]) + + def test_process_job_passes_envelope_context_to_completion(self) -> None: + job = { + "id": 12, + "job_code": "ops-20260418-000012", + "job_type": "release_deploy", + "action": "logs.collect", + "payload": {"service_name": "domaincheck-worker", "lines": 120}, + "step_key": "worker_logs", + "step_title": "收集 Worker 日志", + "focus_ref": {"kind": "ops_job", "job_id": 12, "job_code": "ops-20260418-000012"}, + "release_context": {"release_version": "v2026.04.18", "rollout_id": 3}, + "step_ref": {"step_id": 22, "step_key": "worker_logs", "step_title": "收集 Worker 日志"}, + } + with patch.object(node_agent, "_job_start") as mock_job_start, patch.object( + node_agent, + "_job_event", + ) as mock_job_event, patch.object( + node_agent, + "_execute_action", + return_value=(True, "logs collected", {"summary": "收集完成", "stdout": "ok", "stderr": ""}), + ) as mock_execute_action, patch.object( + node_agent, + "_job_complete", + return_value={"state": "sent"}, + ) as mock_job_complete: + node_agent._process_job(job) + + self.assertEqual("worker_logs", mock_job_start.call_args.kwargs["job"]["step_key"]) + self.assertEqual("ops_job", mock_job_event.call_args.kwargs["focus_ref"]["kind"]) + self.assertEqual("已接单 logs.collect", mock_job_event.call_args.kwargs["summary_text"]) + self.assertEqual("logs.collect", mock_execute_action.call_args.args[0]) + self.assertEqual("v2026.04.18", mock_job_complete.call_args.kwargs["release_context"]["release_version"]) + self.assertEqual("ops_job", mock_job_complete.call_args.kwargs["focus_ref"]["kind"]) + self.assertEqual("收集完成", mock_job_complete.call_args.kwargs["summary_text"]) + self.assertGreaterEqual(mock_job_complete.call_args.kwargs["duration_ms"], 0) + + def test_process_job_continues_when_job_start_delivery_fails(self) -> None: + job = { + "id": 18, + "job_code": "ops-20260418-000018", + "job_type": "ops_action", + "action": "logs.collect", + "payload": {"service_name": "domaincheck-worker", "lines": 50}, + "step_key": "worker_logs", + "step_title": "收集 Worker 日志", + "focus_ref": {"kind": "ops_job", "job_id": 18, "job_code": "ops-20260418-000018"}, + "release_context": {}, + "step_ref": {"step_id": 28, "step_key": "worker_logs", "step_title": "收集 Worker 日志"}, + } + with patch.object( + node_agent, + "_job_start", + side_effect=urllib.error.URLError("temporary offline"), + ) as mock_job_start, patch.object( + node_agent, + "_job_event", + return_value={"state": "queued"}, + ) as mock_job_event, patch.object( + node_agent, + "_execute_action", + return_value=(True, "logs collected", {"summary": "收集完成", "stdout": "ok", "stderr": ""}), + ) as mock_execute_action, patch.object( + node_agent, + "_job_complete", + return_value={"state": "sent"}, + ) as mock_job_complete: + node_agent._process_job(job) + + mock_job_start.assert_called_once() + mock_execute_action.assert_called_once() + mock_job_complete.assert_called_once() + event_payload = mock_job_event.call_args.kwargs["payload"] + self.assertEqual("failed_local", event_payload["start_delivery_state"]) + self.assertIn("temporary offline", event_payload["start_delivery_error"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/domain-api/tests/test_node_agent_release_normalization.py b/domain-api/tests/test_node_agent_release_normalization.py new file mode 100644 index 0000000..963816d --- /dev/null +++ b/domain-api/tests/test_node_agent_release_normalization.py @@ -0,0 +1,41 @@ +import unittest + +from app import node_agent + + +class NodeAgentReleaseNormalizationTests(unittest.TestCase): + def test_normalize_text_list_supports_strings_and_lists(self) -> None: + self.assertEqual( + ["domaincheck-api", "domaincheck-worker"], + node_agent._normalize_text_list("domaincheck-api\ndomaincheck-worker"), + ) + self.assertEqual( + ["domaincheck-api", "domaincheck-worker"], + node_agent._normalize_text_list(["domaincheck-api", " domaincheck-worker "]), + ) + self.assertEqual([], node_agent._normalize_text_list("")) + + def test_health_check_services_fall_back_to_restart_services_when_missing(self) -> None: + result = node_agent._normalize_release_health_check_services( + {}, + ["domaincheck-worker"], + ) + self.assertEqual(["domaincheck-worker"], result) + + def test_health_check_services_can_be_explicitly_empty(self) -> None: + result = node_agent._normalize_release_health_check_services( + {"health_check_services": ""}, + ["domaincheck-worker"], + ) + self.assertEqual([], result) + + def test_health_check_services_accept_multiline_string(self) -> None: + result = node_agent._normalize_release_health_check_services( + {"health_check_services": "domaincheck-api\ndomaincheck-sync-agent"}, + ["domaincheck-worker"], + ) + self.assertEqual(["domaincheck-api", "domaincheck-sync-agent"], result) + + +if __name__ == "__main__": + unittest.main() diff --git a/domain-api/tests/test_ops_action_executor_core.py b/domain-api/tests/test_ops_action_executor_core.py new file mode 100644 index 0000000..55d94e2 --- /dev/null +++ b/domain-api/tests/test_ops_action_executor_core.py @@ -0,0 +1,115 @@ +import tempfile +import unittest +from pathlib import Path +from unittest.mock import patch + +from app.services.ops_action_executor_core import build_service_name_map, execute_structured_action + + +class _FakeRunner: + def __init__(self, responses: dict[tuple[str, ...], tuple[int, str, str]]) -> None: + self.responses = responses + self.calls: list[tuple[str, ...]] = [] + + def __call__(self, command: list[str], *, timeout: int = 60) -> tuple[int, str, str]: + key = tuple(command) + self.calls.append(key) + return self.responses.get(key, (0, "", "")) + + +class OpsActionExecutorCoreTests(unittest.TestCase): + def test_health_snapshot_includes_host_context_and_ok_flags(self) -> None: + runner = _FakeRunner( + { + ("systemctl", "is-active", "domaincheck-api"): (0, "active", ""), + ("systemctl", "is-active", "domaincheck-worker"): (3, "inactive", ""), + ("systemctl", "is-active", "domaincheck-sync-agent"): (0, "active", ""), + ("systemctl", "is-active", "domaincheck-node-agent"): (0, "active", ""), + } + ) + + ok, message, data = execute_structured_action( + "health.snapshot", + {}, + service_names=build_service_name_map( + api_service_name="domaincheck-api", + worker_service_name="domaincheck-worker", + sync_agent_service_name="domaincheck-sync-agent", + ), + runner=runner, + host_context={"hostname": "mainland-controller-01", "ip": "121.204.244.188"}, + ) + + self.assertTrue(ok) + self.assertEqual("health snapshot collected", message) + self.assertEqual("mainland-controller-01", data["hostname"]) + self.assertEqual("121.204.244.188", data["ip"]) + self.assertTrue(data["checks"]["api"]["ok"]) + self.assertFalse(data["checks"]["worker"]["ok"]) + self.assertEqual(4, len(runner.calls)) + + def test_logs_collect_defaults_to_worker_service_and_clamps_lines(self) -> None: + runner = _FakeRunner( + { + ("journalctl", "-u", "domaincheck-worker", "-n", "500", "--no-pager"): ( + 0, + "line-1\nline-2", + "", + ) + } + ) + + ok, message, data = execute_structured_action( + "logs.collect", + {"lines": 9999}, + service_names=build_service_name_map( + api_service_name="domaincheck-api", + worker_service_name="domaincheck-worker", + sync_agent_service_name="domaincheck-sync-agent", + ), + runner=runner, + ) + + self.assertTrue(ok) + self.assertEqual("line-1\nline-2", message) + self.assertEqual("domaincheck-worker", data["service_name"]) + self.assertEqual(500, data["lines"]) + self.assertEqual( + ("journalctl", "-u", "domaincheck-worker", "-n", "500", "--no-pager"), + runner.calls[0], + ) + + def test_logs_collect_falls_back_to_worker_log_file_when_journal_is_denied(self) -> None: + runner = _FakeRunner( + { + ("journalctl", "-u", "domaincheck-worker", "-n", "120", "--no-pager"): ( + 1, + "", + "No journal files were opened due to insufficient permissions.", + ) + } + ) + with tempfile.TemporaryDirectory() as temp_dir: + log_path = Path(temp_dir) / "detect_worker.log" + log_path.write_text("worker-a\nworker-b\n", encoding="utf-8") + with patch("app.services.ops_action_executor_core.settings.domain_root", temp_dir): + ok, message, data = execute_structured_action( + "logs.collect", + {"lines": 120}, + service_names=build_service_name_map( + api_service_name="domaincheck-api", + worker_service_name="domaincheck-worker", + sync_agent_service_name="domaincheck-sync-agent", + ), + runner=runner, + ) + + self.assertTrue(ok) + self.assertEqual("worker-a\nworker-b", message) + self.assertEqual("file_fallback", data["collection_source"]) + self.assertTrue(data["fallback_used"]) + self.assertIn("detect_worker.log", data["log_paths"][0]) + + +if __name__ == "__main__": + unittest.main() diff --git a/domain-api/tests/test_ops_agent_service.py b/domain-api/tests/test_ops_agent_service.py new file mode 100644 index 0000000..1d4d52d --- /dev/null +++ b/domain-api/tests/test_ops_agent_service.py @@ -0,0 +1,1181 @@ +import unittest +from datetime import datetime +from unittest.mock import patch + +from app.api.routes.ops_agent import _build_agent_response +from app.services.ops_agent_service import ( + agent_append_job_event, + agent_complete_job, + agent_pull_jobs, + agent_register, + build_node_agent_bootstrap_plan, + build_managed_node_handover_bootstrap_plan, + execute_managed_node_onboarding_acceptance, + execute_managed_node_onboarding_bootstrap, + execute_managed_node_onboarding_recovery, + get_managed_node_delivery_queue, + get_managed_node_handover, + get_managed_node_onboarding, + list_ops_job_events, + list_managed_node_delivery_queue_records, + list_managed_nodes_with_agent_state, + preview_managed_node_onboarding_acceptance, + preview_managed_node_onboarding_bootstrap, + preview_managed_node_onboarding_recovery, + request_managed_node_delivery_queue_action, +) + + +class _EmptyCursor: + def __enter__(self): + return self + + def __exit__(self, exc_type, exc, tb): + return False + + def execute(self, query, params=None) -> None: + self.query = query + self.params = params + + def fetchall(self): + return [] + + +class _EmptyConnection: + def __enter__(self): + return self + + def __exit__(self, exc_type, exc, tb): + return False + + def cursor(self): + return _EmptyCursor() + + +class _SequenceCursor: + def __init__(self, *, fetchone_values=None, fetchall_values=None): + self.fetchone_values = list(fetchone_values or []) + self.fetchall_values = list(fetchall_values or []) + self.executed = [] + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc, tb): + return False + + def execute(self, query, params=None) -> None: + self.executed.append((query, params)) + + def fetchone(self): + if self.fetchone_values: + return self.fetchone_values.pop(0) + return None + + def fetchall(self): + if self.fetchall_values: + return self.fetchall_values.pop(0) + return [] + + +class _SequenceConnection: + def __init__(self, cursor): + self.cursor_obj = cursor + self.autocommit = True + self.committed = False + self.rolled_back = False + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc, tb): + return False + + def cursor(self): + return self.cursor_obj + + def commit(self) -> None: + self.committed = True + + def rollback(self) -> None: + self.rolled_back = True + + +class OpsAgentServiceTests(unittest.TestCase): + def test_build_agent_response_extracts_detail_code(self) -> None: + response = _build_agent_response(False, "bad request", {"detail_code": "agent_token_invalid", "foo": "bar"}) + + self.assertEqual(1, response.code) + self.assertEqual("agent_token_invalid", response.detail_code) + self.assertEqual("bar", response.data["foo"]) + + @patch("app.services.ops_agent_service.issue_node_agent_token") + def test_build_node_agent_bootstrap_plan_includes_script_artifacts(self, mock_issue_node_agent_token) -> None: + mock_issue_node_agent_token.return_value = ( + True, + "ok", + { + "token": "agent-token-example", + "token_preview": "agent-...mple", + "record_id": 12, + "node_code": "mainland-worker-02", + "expires_at": "2026-04-20 10:00:00", + "created_at": "2026-04-17 10:00:00", + }, + ) + + ok, message, data = build_node_agent_bootstrap_plan( + node_code="mainland-worker-02", + node_region="mainland", + node_role="worker", + issued_by="test", + control_plane_base_url="http://152.53.37.118:8100/api/v1", + root_dir="/opt/domaincheck", + ) + + self.assertTrue(ok) + self.assertEqual("节点 Agent 接入方案已生成", message) + plan = data["bootstrap_plan"] + self.assertEqual("http://152.53.37.118:8100", data["control_plane_base_url"]) + self.assertEqual("bootstrap-node-agent-mainland-worker-02.sh", plan["bootstrap_script_name"]) + self.assertEqual("/tmp/bootstrap-node-agent-mainland-worker-02.sh", plan["bootstrap_script_path"]) + self.assertIn("OPS_AGENT_TOKEN=agent-token-example", plan["env_content"]) + self.assertIn("bash \"${INSTALL_SCRIPT}\" \"${ROOT_DIR}\"", plan["bootstrap_script_content"]) + self.assertIn("systemctl restart \"${SERVICE_NAME}\"", plan["bootstrap_script_content"]) + self.assertIn("cat >/tmp/bootstrap-node-agent-mainland-worker-02.sh <<'EOF'", plan["bootstrap_run_script_block"]) + self.assertIn("bash /tmp/bootstrap-node-agent-mainland-worker-02.sh", plan["bootstrap_run_script_block"]) + self.assertIn("curl -s http://152.53.37.118:8100/api/v1/ops/overview", plan["health_check_block"]) + + @patch("app.services.ops_agent_service.list_managed_nodes_with_agent_state") + def test_get_managed_node_handover_summarizes_pending_bootstrap_node(self, mock_list_managed_nodes_with_agent_state) -> None: + mock_list_managed_nodes_with_agent_state.return_value = { + "nodes": [ + { + "node_code": "mainland-worker-01", + "region": "mainland", + "role": "worker", + "is_managed": True, + "is_enabled": True, + "has_ssh_access": True, + "ssh_host": "121.204.244.248", + "ssh_port": 22, + "ssh_user": "root", + "auth_mode": "key", + "agent_state": "pending_bootstrap", + "agent_state_label": "待接入", + "agent_state_reason": "已签发有效 Agent Token，但节点尚未完成 register/heartbeat。", + "remote_access_state": "ssh_ready", + "remote_access_label": "SSH 可执行", + "remote_access_reason": "SSH 已备好，可先生成接入方案。", + "delivery_queue_state": "healthy", + "delivery_queue_label": "正常", + "delivery_queue_reason": "当前没有待重试回执。", + "delivery_queue_pending_count": 0, + "delivery_queue_dead_letter_count": 0, + "cluster_status": "online", + "cluster_current_load": 0, + "cluster_last_heartbeat_at": "2026-04-18 12:00:00", + "latest_token": { + "token_status": "active", + "expires_at": "2026-04-21 12:00:00", + }, + "latest_job": { + "job_code": "ops-job-demo", + "status": "success", + }, + "metadata": {}, + } + ] + } + + payload = get_managed_node_handover( + "mainland-worker-01", + control_plane_base_url="http://152.53.37.118:8100/api/v1", + ) + + self.assertEqual("mainland-worker-01", payload["node_code"]) + self.assertEqual("pending_bootstrap", payload["stage"]["code"]) + self.assertEqual("execute_bootstrap_plan", payload["stage"]["next_step"]["code"]) + self.assertEqual("http://152.53.37.118:8100", payload["control_plane_base_url"]) + self.assertEqual("/api/v1/ops/nodes/mainland-worker-01/handover/bootstrap-plan", payload["endpoints"]["bootstrap_plan"]) + self.assertTrue(payload["readiness"]["has_ssh_access"]) + self.assertIn("已签发有效 Token", "".join(payload["blocking_items"])) + + @patch("app.services.ops_agent_service.get_managed_node_handover") + def test_get_managed_node_onboarding_emits_bootstrap_recovery_decision(self, mock_get_managed_node_handover) -> None: + mock_get_managed_node_handover.return_value = { + "node_code": "mainland-worker-01", + "summary": "已签发有效 Token，但节点尚未完成 register/heartbeat。", + "node": { + "node_code": "mainland-worker-01", + "region": "mainland", + "role": "worker", + }, + "stage": { + "code": "pending_bootstrap", + "label": "待接入", + "summary": "已签发有效 Token，但节点尚未完成 register/heartbeat。", + }, + "readiness": { + "has_ssh_access": True, + "is_agent_online": False, + "is_remote_access_ready": True, + }, + "control_plane_base_url": "http://127.0.0.1:8100", + "root_dir": "/opt/domaincheck", + } + + payload = get_managed_node_onboarding("mainland-worker-01") + + self.assertEqual("pending_bootstrap", payload["onboarding_stage"]["code"]) + self.assertEqual("bootstrap_run", payload["recovery_decision"]["action"]) + self.assertEqual("bootstrap", payload["recovery_decision"]["window"]) + self.assertIn("node-bootstrap-run", payload["recovery_decision"]["command_hint"]) + + @patch("app.services.ops_agent_service.get_managed_node_handover") + def test_get_managed_node_onboarding_emits_acceptance_recovery_decision(self, mock_get_managed_node_handover) -> None: + mock_get_managed_node_handover.return_value = { + "node_code": "mainland-worker-01", + "summary": "Node Agent 已在线，可以继续跑接管验收。", + "node": { + "node_code": "mainland-worker-01", + "region": "mainland", + "role": "worker", + }, + "stage": { + "code": "ready", + "label": "已接入", + "summary": "Node Agent 已在线，可以继续跑接管验收。", + }, + "readiness": { + "has_ssh_access": True, + "is_agent_online": True, + "is_remote_access_ready": True, + }, + "control_plane_base_url": "http://127.0.0.1:8100", + "root_dir": "/opt/domaincheck", + } + + payload = get_managed_node_onboarding("mainland-worker-01") + + self.assertEqual("acceptance_ready", payload["onboarding_stage"]["code"]) + self.assertEqual("run_acceptance", payload["recovery_decision"]["action"]) + self.assertEqual("acceptance", payload["recovery_decision"]["window"]) + self.assertIn("node-acceptance-run", payload["recovery_decision"]["command_hint"]) + + @patch("app.services.ops_agent_service.build_node_agent_bootstrap_plan") + @patch("app.services.ops_agent_service.get_managed_node_handover") + def test_build_managed_node_handover_bootstrap_plan_reuses_node_defaults( + self, + mock_get_managed_node_handover, + mock_build_node_agent_bootstrap_plan, + ) -> None: + mock_get_managed_node_handover.side_effect = [ + { + "node": { + "node_code": "mainland-worker-01", + "region": "mainland", + "role": "worker", + }, + "control_plane_base_url": "http://152.53.37.118:8100", + "root_dir": "/srv/domaincheck", + "bootstrap_defaults": { + "node_region": "mainland", + "node_role": "worker", + "control_plane_base_url": "http://152.53.37.118:8100", + "root_dir": "/srv/domaincheck", + "expires_in_hours": 72, + "metadata": { + "issued_from": "ops-managed-node-handover", + }, + }, + }, + { + "node_code": "mainland-worker-01", + "stage": { + "code": "pending_bootstrap", + }, + }, + ] + mock_build_node_agent_bootstrap_plan.return_value = ( + True, + "节点 Agent 接入方案已生成", + { + "token_preview": "agent-...mple", + "bootstrap_plan": { + "bootstrap_script_name": "bootstrap-node-agent-mainland-worker-01.sh", + }, + }, + ) + + ok, message, data = build_managed_node_handover_bootstrap_plan( + node_code="mainland-worker-01", + issued_by="web-ui", + expires_in_hours=48, + metadata={"issued_from": "ops-center-manual"}, + ) + + self.assertTrue(ok) + self.assertEqual("节点 Agent 接入方案已生成", message) + mock_build_node_agent_bootstrap_plan.assert_called_once() + called_kwargs = mock_build_node_agent_bootstrap_plan.call_args.kwargs + self.assertEqual("mainland-worker-01", called_kwargs["node_code"]) + self.assertEqual("mainland", called_kwargs["node_region"]) + self.assertEqual("worker", called_kwargs["node_role"]) + self.assertEqual("http://152.53.37.118:8100", called_kwargs["control_plane_base_url"]) + self.assertEqual("/srv/domaincheck", called_kwargs["root_dir"]) + self.assertEqual("ops-center-manual", called_kwargs["metadata"]["issued_from"]) + self.assertEqual("managed-node-handover", data["requested_via"]) + self.assertEqual("pending_bootstrap", data["handover"]["stage"]["code"]) + + @patch("app.services.ops_playbook_service.preview_ops_playbook") + @patch("app.services.ops_agent_service.get_managed_node_onboarding") + def test_preview_managed_node_onboarding_bootstrap_uses_control_plane_playbook( + self, + mock_get_managed_node_onboarding, + mock_preview_ops_playbook, + ) -> None: + mock_get_managed_node_onboarding.return_value = { + "node_code": "mainland-controller-01", + "bootstrap_plan_request": { + "node_code": "mainland-controller-01", + "node_region": "mainland", + "node_role": "control", + "control_plane_base_url": "http://127.0.0.1:8100", + "root_dir": "/opt/domaincheck", + }, + "acceptance": {"status_code": "blocked"}, + } + mock_preview_ops_playbook.return_value = (True, "ok", {"playbook": {"key": "onboarding.bootstrap"}}) + + ok, message, data = preview_managed_node_onboarding_bootstrap( + node_code="mainland-controller-01", + requested_by="tester", + ) + + self.assertTrue(ok) + self.assertEqual("ok", message) + self.assertEqual("mainland-controller-01", data["node_code"]) + self.assertEqual("control-plane", data["execution_mode"]) + mock_preview_ops_playbook.assert_called_once_with( + { + "playbook_key": "onboarding.bootstrap", + "node_codes": ["mainland-controller-01"], + "execution_mode": "control-plane", + "auto_approve": True, + "requested_by": "tester", + "payload": { + "node_code": "mainland-controller-01", + "node_region": "mainland", + "node_role": "control", + "control_plane_base_url": "http://127.0.0.1:8100", + "root_dir": "/opt/domaincheck", + }, + } + ) + + @patch("app.services.ops_playbook_service.execute_ops_playbook") + @patch("app.services.ops_agent_service.get_managed_node_onboarding") + def test_execute_managed_node_onboarding_bootstrap_uses_control_plane_playbook( + self, + mock_get_managed_node_onboarding, + mock_execute_ops_playbook, + ) -> None: + mock_get_managed_node_onboarding.return_value = { + "node_code": "mainland-controller-01", + "bootstrap_plan_request": { + "node_code": "mainland-controller-01", + "node_region": "mainland", + "node_role": "control", + "control_plane_base_url": "http://127.0.0.1:8100", + "root_dir": "/opt/domaincheck", + }, + "acceptance": {"status_code": "blocked"}, + } + mock_execute_ops_playbook.return_value = ( + True, + "created", + { + "run_code": "pbr-demo", + "playbook_run": { + "run_code": "pbr-demo", + "focus_summary": "接入工单已生成，节点待执行 bootstrap 脚本并回连 Agent。", + "focus_ref": {"kind": "playbook_run", "run_code": "pbr-demo"}, + }, + }, + ) + + ok, message, data = execute_managed_node_onboarding_bootstrap( + node_code="mainland-controller-01", + requested_by="tester", + ) + + self.assertTrue(ok) + self.assertEqual("created", message) + self.assertEqual("control-plane", data["execution_mode"]) + self.assertEqual("acceptance_ready", data["follow_up"]["next_stage_code"]) + self.assertIn("node-recover", data["follow_up"]["recommended_commands"][0]) + self.assertIn("待执行 bootstrap 脚本", data["follow_up"]["summary"]) + mock_execute_ops_playbook.assert_called_once() + called_payload = mock_execute_ops_playbook.call_args.args[0] + self.assertEqual("onboarding.bootstrap", called_payload["playbook_key"]) + self.assertEqual(["mainland-controller-01"], called_payload["node_codes"]) + self.assertEqual("control-plane", called_payload["execution_mode"]) + self.assertTrue(called_payload["auto_approve"]) + self.assertEqual("tester", called_payload["requested_by"]) + + @patch("app.services.ops_playbook_service.preview_ops_playbook") + @patch("app.services.ops_agent_service.get_managed_node_onboarding") + def test_preview_managed_node_onboarding_acceptance_uses_acceptance_execution_mode( + self, + mock_get_managed_node_onboarding, + mock_preview_ops_playbook, + ) -> None: + mock_get_managed_node_onboarding.return_value = { + "node_code": "mainland-worker-01", + "acceptance": { + "status_code": "ssh_only", + "execution_mode": "ssh", + }, + } + mock_preview_ops_playbook.return_value = (True, "ok", {"playbook": {"key": "onboarding.acceptance"}}) + + ok, message, data = preview_managed_node_onboarding_acceptance( + node_code="mainland-worker-01", + requested_by="tester", + ) + + self.assertTrue(ok) + self.assertEqual("ok", message) + self.assertEqual("ssh", data["execution_mode"]) + mock_preview_ops_playbook.assert_called_once_with( + { + "playbook_key": "onboarding.acceptance", + "node_codes": ["mainland-worker-01"], + "execution_mode": "ssh", + "auto_approve": True, + "requested_by": "tester", + } + ) + + @patch("app.services.ops_playbook_service.execute_ops_playbook") + @patch("app.services.ops_agent_service.get_managed_node_onboarding") + def test_execute_managed_node_onboarding_acceptance_blocks_when_not_ready( + self, + mock_get_managed_node_onboarding, + mock_execute_ops_playbook, + ) -> None: + mock_get_managed_node_onboarding.return_value = { + "node_code": "mainland-worker-01", + "acceptance": { + "status_code": "blocked", + "summary": "当前还没进入标准接管就绪状态", + }, + } + + ok, message, data = execute_managed_node_onboarding_acceptance( + node_code="mainland-worker-01", + requested_by="tester", + ) + + self.assertFalse(ok) + self.assertIn("当前还没进入标准接管就绪状态", message) + self.assertIn("onboarding", data) + mock_execute_ops_playbook.assert_not_called() + + @patch("app.services.ops_agent_service.preview_managed_node_onboarding_bootstrap") + @patch("app.services.ops_agent_service.get_managed_node_onboarding") + def test_preview_managed_node_onboarding_recovery_routes_to_bootstrap( + self, + mock_get_managed_node_onboarding, + mock_preview_bootstrap, + ) -> None: + mock_get_managed_node_onboarding.return_value = { + "node_code": "mainland-worker-01", + "recovery_decision": { + "action": "bootstrap_run", + "summary": "建议先签发 onboarding.bootstrap。", + }, + } + mock_preview_bootstrap.return_value = (True, "ok", {"playbook_preview": {"playbook": {"key": "onboarding.bootstrap"}}}) + + ok, message, data = preview_managed_node_onboarding_recovery(node_code="mainland-worker-01", requested_by="tester") + + self.assertTrue(ok) + self.assertEqual("bootstrap_run", data["recovery_action"]) + self.assertEqual("ok", message) + mock_preview_bootstrap.assert_called_once() + + @patch("app.services.ops_agent_service.execute_managed_node_onboarding_acceptance") + @patch("app.services.ops_agent_service.get_managed_node_onboarding") + def test_execute_managed_node_onboarding_recovery_routes_to_acceptance( + self, + mock_get_managed_node_onboarding, + mock_execute_acceptance, + ) -> None: + mock_get_managed_node_onboarding.return_value = { + "node_code": "mainland-worker-01", + "recovery_decision": { + "action": "run_acceptance", + "summary": "建议直接执行 onboarding.acceptance。", + }, + } + mock_execute_acceptance.return_value = ( + True, + "created", + { + "playbook_run": {"run_code": "pbr-acceptance"}, + "follow_up": { + "next_stage_code": "ready", + "summary": "接管验收已发起；接下来应继续确认 health / node-agent / worker 三段验收全部通过。", + "recommended_commands": [ + "bash domain-api/deploy/multi-region/drive_ops_center.sh node-onboarding http://127.0.0.1:8100 mainland-worker-01", + "bash domain-api/deploy/multi-region/drive_ops_center.sh doctor http://127.0.0.1:8100", + ], + }, + }, + ) + + ok, message, data = execute_managed_node_onboarding_recovery(node_code="mainland-worker-01", requested_by="tester") + + self.assertTrue(ok) + self.assertEqual("run_acceptance", data["recovery_action"]) + self.assertEqual("created", message) + self.assertEqual("ready", data["selected_run"]["follow_up"]["next_stage_code"]) + mock_execute_acceptance.assert_called_once() + + @patch("app.services.runtime_status_service.get_runtime_status") + @patch("app.services.cluster_runtime_service.get_cluster_snapshot") + @patch("app.services.ops_job_service.list_managed_nodes") + @patch("app.services.ops_agent_service.get_db") + @patch("app.services.ops_agent_service.ensure_ops_agent_schema") + def test_list_managed_nodes_with_agent_state_merges_detect_participation( + self, + mock_ensure_ops_agent_schema, + mock_get_db, + mock_list_managed_nodes, + mock_get_cluster_snapshot, + mock_get_runtime_status, + ) -> None: + now = datetime.now().strftime("%Y-%m-%d %H:%M:%S") + mock_ensure_ops_agent_schema.return_value = None + mock_get_db.return_value = _EmptyConnection() + mock_list_managed_nodes.return_value = [ + { + "node_code": "mainland-worker-01", + "region": "mainland", + "role": "worker", + "title": "Mainland Worker 01", + "is_enabled": True, + "metadata": {}, + "last_seen_at": now, + }, + { + "node_code": "mainland-controller-01", + "region": "mainland", + "role": "control", + "title": "Mainland Controller 01", + "is_enabled": True, + "metadata": {}, + "last_seen_at": now, + }, + ] + mock_get_cluster_snapshot.return_value = { + "nodes": [ + { + "node_code": "mainland-worker-01", + "region": "mainland", + "role": "worker", + "status": "busy", + "current_load": 4, + "last_heartbeat_at": now, + "is_effective_worker": True, + "detect_participating": True, + }, + { + "node_code": "mainland-controller-01", + "region": "mainland", + "role": "control", + "status": "online", + "current_load": 0, + "last_heartbeat_at": now, + "is_effective_worker": True, + "detect_participating": False, + }, + ] + } + mock_get_runtime_status.return_value = { + "detect": { + "participating_nodes": [ + { + "node_code": "mainland-worker-01", + "participation_state": "running", + "participation_label": "执行中", + "participation_reason": "当前正有 4 个任务线程在跑", + "is_current_participant": True, + "is_dispatch_active": True, + "items_total": 50, + "items_claimed": 12, + "items_running": 4, + "items_completed": 8, + "items_failed": 1, + "processed_recent": 9, + "processed_per_minute": 0.6, + } + ], + "non_participating_nodes": [ + { + "node_code": "mainland-controller-01", + "participation_state": "standby", + "participation_label": "在线待命", + "participation_reason": "节点在线但当前没有领任务", + "is_current_participant": False, + "is_dispatch_active": False, + "items_total": 0, + "items_claimed": 0, + "items_running": 0, + "items_completed": 0, + "items_failed": 0, + "processed_recent": 0, + "processed_per_minute": 0, + } + ], + } + } + + payload = list_managed_nodes_with_agent_state() + + self.assertEqual(2, len(payload["nodes"])) + node_map = {item["node_code"]: item for item in payload["nodes"]} + self.assertTrue(node_map["mainland-worker-01"]["is_current_participant"]) + self.assertTrue(node_map["mainland-worker-01"]["cluster_detect_participating"]) + self.assertEqual("running", node_map["mainland-worker-01"]["participation_state"]) + self.assertEqual("执行中", node_map["mainland-worker-01"]["participation_label"]) + self.assertTrue(node_map["mainland-worker-01"]["is_dispatch_active"]) + self.assertEqual(50, node_map["mainland-worker-01"]["items_total"]) + self.assertEqual(4, node_map["mainland-worker-01"]["items_running"]) + self.assertEqual("standby", node_map["mainland-controller-01"]["participation_state"]) + self.assertEqual("在线待命", node_map["mainland-controller-01"]["participation_label"]) + self.assertFalse(node_map["mainland-controller-01"]["is_current_participant"]) + self.assertEqual(1, payload["summary"]["participating"]) + self.assertEqual(1, payload["summary"]["dispatch_active"]) + self.assertEqual(1, payload["summary"]["standby"]) + self.assertEqual(1, payload["summary"]["non_participating"]) + + @patch("app.services.runtime_status_service.get_runtime_status") + @patch("app.services.cluster_runtime_service.get_cluster_snapshot") + @patch("app.services.ops_job_service.list_managed_nodes") + @patch("app.services.ops_agent_service.get_db") + @patch("app.services.ops_agent_service.ensure_ops_agent_schema") + def test_list_managed_nodes_with_agent_state_marks_ssh_ready_without_agent( + self, + mock_ensure_ops_agent_schema, + mock_get_db, + mock_list_managed_nodes, + mock_get_cluster_snapshot, + mock_get_runtime_status, + ) -> None: + mock_ensure_ops_agent_schema.return_value = None + mock_get_db.return_value = _EmptyConnection() + mock_list_managed_nodes.return_value = [ + { + "node_code": "mainland-worker-02", + "region": "mainland", + "role": "worker", + "title": "Mainland Worker 02", + "ssh_host": "121.204.244.249", + "ssh_port": 22, + "ssh_user": "root", + "auth_mode": "key", + "deploy_channel": "stable", + "is_enabled": True, + "metadata": {}, + "last_seen_at": "", + } + ] + mock_get_cluster_snapshot.return_value = { + "nodes": [ + { + "node_code": "mainland-worker-02", + "region": "mainland", + "role": "worker", + "status": "online", + "current_load": 0, + "hostname": "worker-02", + "ip": "121.204.244.249", + "last_heartbeat_at": "2026-04-18 10:00:00", + "is_effective_worker": True, + "detect_participating": False, + } + ] + } + mock_get_runtime_status.return_value = { + "detect": { + "participating_nodes": [], + "non_participating_nodes": [], + } + } + + payload = list_managed_nodes_with_agent_state() + + self.assertEqual(1, len(payload["nodes"])) + row = payload["nodes"][0] + self.assertFalse(row["is_agent_online"]) + self.assertTrue(row["has_ssh_access"]) + self.assertEqual("SSH 已备好", row["ssh_access_label"]) + self.assertEqual("ssh_ready", row["remote_access_state"]) + self.assertEqual("SSH 可执行", row["remote_access_label"]) + self.assertTrue(row["is_remote_access_ready"]) + self.assertEqual(1, payload["summary"]["ssh_ready"]) + self.assertEqual(0, payload["summary"]["agent_ready"]) + self.assertEqual(1, payload["summary"]["remote_access_ready"]) + + def test_agent_register_requires_node_code_with_detail_code(self) -> None: + ok, message, data = agent_register({}, token="ignored") + + self.assertFalse(ok) + self.assertEqual("node_code 不能为空", message) + self.assertEqual("agent_node_code_required", data["detail_code"]) + + @patch("app.services.ops_agent_service._authenticate_agent_token") + @patch("app.services.ops_agent_service.get_ops_job_event") + @patch("app.services.ops_agent_service.append_ops_job_event") + def test_agent_append_job_event_preserves_client_event_id( + self, + mock_append_ops_job_event, + mock_get_ops_job_event, + mock_authenticate_agent_token, + ) -> None: + mock_authenticate_agent_token.return_value = (True, "ok", {"node_code": "mainland-worker-01"}) + mock_append_ops_job_event.return_value = 88 + mock_get_ops_job_event.return_value = { + "id": 88, + "event_key": "job-event:88", + "focus_ref": {"kind": "ops_job_event", "job_id": 12, "focus_event_id": 88}, + } + + ok, message, data = agent_append_job_event( + 12, + { + "node_code": "mainland-worker-01", + "event_type": "executor_received", + "message": "accepted", + "client_event_id": "evt-001", + "summary_text": "已接单", + "focus_ref": {"kind": "ops_job", "job_id": 12}, + }, + token="token", + ) + + self.assertTrue(ok) + self.assertEqual("事件已写入", message) + self.assertEqual("evt-001", data["client_event_id"]) + self.assertEqual(88, data["event"]["id"]) + self.assertEqual("ops_job_event", data["event"]["focus_ref"]["kind"]) + self.assertEqual("evt-001", mock_append_ops_job_event.call_args.kwargs["client_event_id"]) + self.assertEqual("已接单", mock_append_ops_job_event.call_args.kwargs["payload"]["summary_text"]) + self.assertEqual("ops_job", mock_append_ops_job_event.call_args.kwargs["payload"]["focus_ref"]["kind"]) + + @patch("app.services.ops_agent_service.get_db") + @patch("app.services.ops_agent_service.ensure_ops_agent_schema") + def test_list_ops_job_events_adds_contract_aliases( + self, + mock_ensure_ops_agent_schema, + mock_get_db, + ) -> None: + mock_ensure_ops_agent_schema.return_value = None + cursor = _SequenceCursor( + fetchall_values=[ + [ + ( + 301, + 101, + 201, + "mainland-worker-01", + "evt-001", + "job_dispatch_requested", + "warning", + "任务等待 node-agent 拉取: 101", + '{"dispatch": true, "summary_text": "等待 Agent 拉取", "occurred_at": "2026-04-18 06:45:01", "focus_ref": {"kind": "ops_job", "job_id": 101}}', + datetime(2026, 4, 18, 6, 45, 3), + ) + ] + ] + ) + mock_get_db.return_value = _SequenceConnection(cursor) + + events = list_ops_job_events(101, limit=20) + + self.assertEqual(1, len(events)) + row = events[0] + self.assertEqual("job-event:301", row["event_key"]) + self.assertEqual("warning", row["level"]) + self.assertEqual("警告", row["level_label"]) + self.assertEqual("任务等待 node-agent 拉取: 101", row["summary"]) + self.assertEqual("等待 Agent 拉取", row["summary_text"]) + self.assertTrue(row["has_payload"]) + self.assertEqual("2026-04-18 06:45:01", row["occurred_at"]) + self.assertEqual("ops_job", row["focus_ref"]["kind"]) + self.assertEqual("job_detail", row["ui_intent"]["kind"]) + self.assertEqual(101, row["ui_intent"]["job_id"]) + self.assertEqual(301, row["ui_intent"]["focus_event_id"]) + + @patch("app.services.ops_agent_service.get_ops_job") + @patch("app.services.ops_agent_service._authenticate_agent_token") + @patch("app.services.ops_agent_service.get_db") + def test_agent_complete_job_is_idempotent_for_same_client_request_id( + self, + mock_get_db, + mock_authenticate_agent_token, + mock_get_ops_job, + ) -> None: + mock_authenticate_agent_token.return_value = (True, "ok", {"node_code": "mainland-worker-01"}) + mock_get_ops_job.return_value = {"id": 12, "status": "success"} + cursor = _SequenceCursor(fetchone_values=[(12, "complete-001")]) + connection = _SequenceConnection(cursor) + mock_get_db.return_value = connection + + ok, message, data = agent_complete_job( + 12, + { + "node_code": "mainland-worker-01", + "status": "success", + "result": {"ok": True}, + "client_request_id": "complete-001", + }, + token="token", + ) + + self.assertTrue(ok) + self.assertEqual("任务结果已幂等回写", message) + self.assertTrue(data["deduplicated"]) + self.assertEqual("complete-001", data["client_request_id"]) + self.assertEqual(12, data["completion_summary"]["job_id"]) + self.assertTrue(data["completion_summary"]["deduplicated"]) + self.assertTrue(connection.rolled_back) + + @patch("app.services.ops_agent_service.append_ops_job_event") + @patch("app.services.ops_agent_service.get_ops_job") + @patch("app.services.ops_agent_service._authenticate_agent_token") + @patch("app.services.ops_agent_service.get_db") + def test_agent_pull_jobs_returns_agent_job_envelope( + self, + mock_get_db, + mock_authenticate_agent_token, + mock_get_ops_job, + mock_append_ops_job_event, + ) -> None: + mock_authenticate_agent_token.return_value = (True, "ok", {"node_code": "mainland-worker-01"}) + mock_append_ops_job_event.return_value = 501 + mock_get_ops_job.return_value = { + "id": 101, + "job_code": "ops-20260418-000101", + "action": "logs.collect", + "target_node_code": "mainland-worker-01", + "status": "dispatching", + "execution_mode": "remote-agent", + "execution_mode_label": "Node Agent", + "requested_by": "web-ui", + "payload": {"service_name": "domaincheck-worker", "lines": 120}, + "metadata": {"source": "ops-center"}, + "policy": {"timeout_seconds": 120, "auto_approve": True}, + "summary": "收集 Worker 日志", + "rollout_id": 0, + "focus_ref": {"kind": "ops_job", "job_id": 101, "job_code": "ops-20260418-000101"}, + "steps": [ + { + "id": 201, + "step_key": "worker_logs", + "title": "收集 Worker 日志", + "node_code": "mainland-worker-01", + "status": "dispatching", + } + ], + } + cursor = _SequenceCursor(fetchall_values=[[(101,)]]) + connection = _SequenceConnection(cursor) + mock_get_db.return_value = connection + + ok, message, data = agent_pull_jobs( + {"node_code": "mainland-worker-01"}, + token="token", + limit=2, + ) + + self.assertTrue(ok) + self.assertEqual("ok", message) + self.assertEqual("ops-agent/v1", data["protocol_version"]) + self.assertEqual("agent_job", data["envelope_type"]) + self.assertEqual(1, data["count"]) + envelope = data["jobs"][0] + self.assertEqual(101, envelope["job_id"]) + self.assertEqual("inspection", envelope["job_type"]) + self.assertEqual("worker_logs", envelope["step_key"]) + self.assertEqual("收集 Worker 日志", envelope["step_title"]) + self.assertEqual(120, envelope["policy"]["timeout_seconds"]) + self.assertEqual("ops_job", envelope["focus_ref"]["kind"]) + self.assertEqual("logs.collect", envelope["job_ref"]["action"]) + self.assertEqual(201, envelope["step_ref"]["step_id"]) + self.assertEqual("", envelope["release_context"]["release_version"]) + self.assertTrue(connection.committed) + + @patch("app.services.runtime_status_service.get_runtime_status") + @patch("app.services.cluster_runtime_service.get_cluster_snapshot") + @patch("app.services.ops_job_service.list_managed_nodes") + @patch("app.services.ops_agent_service.get_db") + @patch("app.services.ops_agent_service.ensure_ops_agent_schema") + def test_list_managed_nodes_with_agent_state_includes_delivery_queue_snapshot( + self, + mock_ensure_ops_agent_schema, + mock_get_db, + mock_list_managed_nodes, + mock_get_cluster_snapshot, + mock_get_runtime_status, + ) -> None: + now = datetime.now().strftime("%Y-%m-%d %H:%M:%S") + mock_ensure_ops_agent_schema.return_value = None + mock_get_db.return_value = _EmptyConnection() + mock_list_managed_nodes.return_value = [ + { + "node_code": "mainland-worker-01", + "region": "mainland", + "role": "worker", + "title": "Mainland Worker 01", + "is_enabled": True, + "metadata": { + "delivery_queue": { + "state": "dead_letter", + "label": "死信 2", + "reason": "当前存在 2 条死信记录", + "pending_count": 3, + "dead_letter_count": 2, + "oldest_pending_at": "2026-04-18 05:00:00", + "oldest_dead_letter_at": "2026-04-18 05:01:00", + "last_flush_at": "2026-04-18 05:02:00", + } + }, + "last_seen_at": now, + } + ] + mock_get_cluster_snapshot.return_value = { + "nodes": [ + { + "node_code": "mainland-worker-01", + "region": "mainland", + "role": "worker", + "status": "online", + "current_load": 0, + "last_heartbeat_at": now, + "is_effective_worker": True, + "detect_participating": False, + } + ] + } + mock_get_runtime_status.return_value = { + "detect": { + "participating_nodes": [], + "non_participating_nodes": [], + } + } + + payload = list_managed_nodes_with_agent_state() + + self.assertEqual(1, len(payload["nodes"])) + row = payload["nodes"][0] + self.assertEqual("dead_letter", row["delivery_queue_state"]) + self.assertEqual("死信 2", row["delivery_queue_label"]) + self.assertEqual(3, row["delivery_queue_pending_count"]) + self.assertEqual(2, row["delivery_queue_dead_letter_count"]) + self.assertEqual("2026-04-18 05:01:00", row["delivery_queue_oldest_dead_letter_at"]) + self.assertEqual(1, payload["summary"]["queue_dead_letter_nodes"]) + self.assertEqual(3, payload["summary"]["queue_pending_records"]) + self.assertEqual(2, payload["summary"]["queue_dead_letter_records"]) + + @patch("app.services.ops_agent_service.list_managed_nodes_with_agent_state") + def test_get_managed_node_delivery_queue_returns_summary_and_head_records( + self, + mock_list_managed_nodes_with_agent_state, + ) -> None: + mock_list_managed_nodes_with_agent_state.return_value = { + "nodes": [ + { + "node_code": "mainland-worker-01", + "title": "Mainland Worker 01", + "region": "mainland", + "role": "worker", + "is_managed": True, + "is_enabled": True, + "is_agent_online": True, + "agent_state": "online", + "agent_state_label": "已接管", + "remote_access_state": "hybrid_ready", + "remote_access_label": "Agent + SSH", + "delivery_queue_state": "dead_letter", + "delivery_queue_label": "死信 2", + "delivery_queue_reason": "当前存在 2 条死信记录，建议优先查看节点日志或诊断编排。", + "delivery_queue_pending_count": 3, + "delivery_queue_dead_letter_count": 2, + "delivery_queue_last_flush_at": "2026-04-18 06:02:00", + "delivery_queue_oldest_pending_at": "2026-04-18 06:00:00", + "delivery_queue_oldest_pending_request_id": "complete-ops-123-1", + "delivery_queue_oldest_pending_kind": "job_complete", + "delivery_queue_oldest_dead_letter_at": "2026-04-18 06:01:00", + "delivery_queue_oldest_dead_letter_request_id": "event-ops-123-2", + "delivery_queue_oldest_dead_letter_kind": "job_event", + } + ] + } + + payload = get_managed_node_delivery_queue("mainland-worker-01") + + self.assertEqual("mainland-worker-01", payload["node_code"]) + self.assertEqual("dead_letter", payload["summary"]["state"]) + self.assertEqual(3, payload["summary"]["pending_count"]) + self.assertEqual(2, payload["summary"]["dead_letter_count"]) + self.assertEqual("head_only", payload["record_visibility"]) + self.assertTrue(payload["capabilities"]["can_flush"]) + self.assertTrue(payload["capabilities"]["can_replay"]) + self.assertTrue(payload["capabilities"]["can_discard"]) + self.assertEqual("head_only", payload["records_capability"]["record_visibility"]) + self.assertFalse(payload["records_capability"]["full_records_available"]) + self.assertEqual("job_complete", payload["head_records"]["pending"]["request_kind"]) + self.assertEqual("job_event", payload["head_records"]["dead_letter"]["request_kind"]) + + @patch("app.services.ops_agent_service.list_managed_nodes_with_agent_state") + def test_list_managed_node_delivery_queue_records_filters_head_records_by_state( + self, + mock_list_managed_nodes_with_agent_state, + ) -> None: + mock_list_managed_nodes_with_agent_state.return_value = { + "nodes": [ + { + "node_code": "mainland-worker-01", + "title": "Mainland Worker 01", + "region": "mainland", + "role": "worker", + "is_managed": True, + "is_enabled": True, + "is_agent_online": True, + "agent_state": "online", + "agent_state_label": "已接管", + "remote_access_state": "hybrid_ready", + "remote_access_label": "Agent + SSH", + "delivery_queue_state": "dead_letter", + "delivery_queue_label": "死信 1", + "delivery_queue_reason": "当前存在 1 条死信记录。", + "delivery_queue_pending_count": 1, + "delivery_queue_dead_letter_count": 1, + "delivery_queue_last_flush_at": "2026-04-18 06:02:00", + "delivery_queue_oldest_pending_at": "2026-04-18 06:00:00", + "delivery_queue_oldest_pending_request_id": "complete-ops-123-1", + "delivery_queue_oldest_pending_kind": "job_complete", + "delivery_queue_oldest_dead_letter_at": "2026-04-18 06:01:00", + "delivery_queue_oldest_dead_letter_request_id": "event-ops-123-2", + "delivery_queue_oldest_dead_letter_kind": "job_event", + } + ] + } + + payload = list_managed_node_delivery_queue_records("mainland-worker-01", state="dead_letter", limit=20) + + self.assertEqual("mainland-worker-01", payload["node_code"]) + self.assertEqual(1, payload["summary"]["total"]) + self.assertEqual(1, payload["summary"]["returned_records"]) + self.assertEqual("head_only", payload["summary"]["record_visibility"]) + self.assertEqual("head_only", payload["record_visibility"]) + self.assertTrue(payload["capabilities"]["can_replay"]) + self.assertEqual("dead_letter", payload["records"][0]["state"]) + self.assertEqual("event-ops-123-2", payload["records"][0]["client_request_id"]) + + @patch("app.services.ops_agent_service.create_ops_job") + @patch("app.services.ops_agent_service.get_ops_action_template") + @patch("app.services.ops_agent_service.build_ops_template_payload") + @patch("app.services.ops_agent_service.get_managed_node_delivery_queue") + def test_request_managed_node_delivery_queue_action_creates_remote_agent_job_for_flush( + self, + mock_get_managed_node_delivery_queue, + mock_build_ops_template_payload, + mock_get_ops_action_template, + mock_create_ops_job, + ) -> None: + mock_get_managed_node_delivery_queue.return_value = { + "node_code": "mainland-worker-01", + "summary": {"pending_count": 3, "dead_letter_count": 1}, + "records_capability": {"record_visibility": "head_only"}, + "head_records": {"pending": {}, "dead_letter": {}}, + } + mock_get_ops_action_template.return_value = { + "key": "delivery.queue.flush", + "title": "立即冲刷回执队列", + "default_auto_approve": True, + } + mock_build_ops_template_payload.return_value = (True, "ok", {"limit": 20}) + mock_create_ops_job.return_value = ( + True, + "ok", + {"job": {"id": 91, "action": "delivery.queue.flush", "status": "queued"}}, + ) + + ok, message, data = request_managed_node_delivery_queue_action( + "mainland-worker-01", + "delivery.queue.flush", + payload={"requested_by": "web-ui", "limit": 20}, + ) + + self.assertTrue(ok) + self.assertEqual("立即冲刷回执队列任务已创建", message) + called_payload = mock_create_ops_job.call_args.args[0] + self.assertEqual("delivery.queue.flush", called_payload["action"]) + self.assertEqual("mainland-worker-01", called_payload["target_node_code"]) + self.assertEqual("remote-agent", called_payload["execution_mode"]) + self.assertEqual({"limit": 20}, called_payload["payload"]) + self.assertEqual("delivery.queue.flush", data["action_key"]) + self.assertEqual(91, data["job"]["id"]) + + @patch("app.services.ops_agent_service.get_ops_action_template") + @patch("app.services.ops_agent_service.build_ops_template_payload") + @patch("app.services.ops_agent_service.get_managed_node_delivery_queue") + def test_request_managed_node_delivery_queue_action_rejects_invisible_record( + self, + mock_get_managed_node_delivery_queue, + mock_build_ops_template_payload, + mock_get_ops_action_template, + ) -> None: + mock_get_managed_node_delivery_queue.return_value = { + "node_code": "mainland-worker-01", + "summary": {"pending_count": 0, "dead_letter_count": 1}, + "records_capability": {"record_visibility": "head_only"}, + "head_records": { + "pending": {}, + "dead_letter": { + "record_id": "evt-visible", + "state": "dead_letter", + "client_request_id": "evt-visible", + }, + }, + } + mock_get_ops_action_template.return_value = { + "key": "delivery.queue.discard", + "title": "丢弃死信回执", + "default_auto_approve": False, + } + mock_build_ops_template_payload.return_value = ( + True, + "ok", + {"record_id": "evt-hidden", "limit": 1, "reason": "ignore"}, + ) + + ok, message, data = request_managed_node_delivery_queue_action( + "mainland-worker-01", + "delivery.queue.discard", + payload={"requested_by": "web-ui", "reason": "ignore"}, + record_id="evt-hidden", + ) + + self.assertFalse(ok) + self.assertEqual("当前控制面仅支持对可见头部记录执行单条动作", message) + self.assertEqual("evt-hidden", data["record_id"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/domain-api/tests/test_ops_api_routes.py b/domain-api/tests/test_ops_api_routes.py new file mode 100644 index 0000000..5255cbd --- /dev/null +++ b/domain-api/tests/test_ops_api_routes.py @@ -0,0 +1,96 @@ +from __future__ import annotations + +import unittest +from unittest.mock import patch + +from app.api.routes.ops import ( + ops_doctor_decision, + ops_go_live_bundle, + ops_go_live_review, + ops_go_live_signoff, + ops_go_live_summary, +) + + +class OpsApiRoutesTestCase(unittest.TestCase): + @patch("app.api.routes.ops.get_ops_go_live_signoff") + def test_go_live_signoff_route_uses_service_payload(self, mock_get_ops_go_live_signoff) -> None: + mock_get_ops_go_live_signoff.return_value = { + "signoff_status": "ready", + "headline": "ok", + } + + response = ops_go_live_signoff(base_url="http://127.0.0.1:8100", report_dir="/tmp/signoff") + + self.assertEqual(0, response.code) + self.assertEqual("ready", response.data["signoff_status"]) + self.assertEqual("ok", response.data["headline"]) + mock_get_ops_go_live_signoff.assert_called_once_with( + base_url="http://127.0.0.1:8100", + report_dir="/tmp/signoff", + ) + + @patch("app.api.routes.ops.get_ops_go_live_summary") + def test_go_live_summary_route_passes_base_url(self, mock_get_ops_go_live_summary) -> None: + mock_get_ops_go_live_summary.return_value = { + "go_live_status": "attention", + } + + response = ops_go_live_summary(base_url="http://example.test") + + self.assertEqual(0, response.code) + self.assertEqual("attention", response.data["go_live_status"]) + mock_get_ops_go_live_summary.assert_called_once_with(base_url="http://example.test") + + @patch("app.api.routes.ops.get_ops_go_live_bundle") + def test_go_live_bundle_route_passes_base_url_and_report_dir(self, mock_get_ops_go_live_bundle) -> None: + mock_get_ops_go_live_bundle.return_value = { + "status": "missing", + } + + response = ops_go_live_bundle(base_url="http://example.test", report_dir="/tmp/go-live") + + self.assertEqual(0, response.code) + self.assertEqual("missing", response.data["status"]) + mock_get_ops_go_live_bundle.assert_called_once_with( + base_url="http://example.test", + report_dir="/tmp/go-live", + ) + + @patch("app.api.routes.ops.get_ops_go_live_review") + def test_go_live_review_route_passes_base_url_and_report_dir(self, mock_get_ops_go_live_review) -> None: + mock_get_ops_go_live_review.return_value = { + "status": "attention", + "headline": "review", + } + + response = ops_go_live_review(base_url="http://example.test", report_dir="/tmp/review") + + self.assertEqual(0, response.code) + self.assertEqual("attention", response.data["status"]) + self.assertEqual("review", response.data["headline"]) + mock_get_ops_go_live_review.assert_called_once_with( + base_url="http://example.test", + report_dir="/tmp/review", + ) + + @patch("app.api.routes.ops.get_ops_doctor_decision") + def test_doctor_decision_route_passes_base_url_and_report_dir(self, mock_get_ops_doctor_decision) -> None: + mock_get_ops_doctor_decision.return_value = { + "status": "attention", + "headline": "doctor", + } + + response = ops_doctor_decision(base_url="http://example.test", report_dir="/tmp/doctor") + + self.assertEqual(0, response.code) + self.assertEqual("attention", response.data["status"]) + self.assertEqual("doctor", response.data["headline"]) + mock_get_ops_doctor_decision.assert_called_once_with( + base_url="http://example.test", + report_dir="/tmp/doctor", + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/domain-api/tests/test_ops_command_service.py b/domain-api/tests/test_ops_command_service.py new file mode 100644 index 0000000..b648728 --- /dev/null +++ b/domain-api/tests/test_ops_command_service.py @@ -0,0 +1,44 @@ +import unittest + +from app.services.ops_command_service import build_bash_command, ops_script_path + + +class OpsCommandServiceTests(unittest.TestCase): + def test_ops_script_path_prefixes_multi_region_directory(self) -> None: + self.assertEqual( + "domain-api/deploy/multi-region/drive_ops_center.sh", + ops_script_path("drive_ops_center.sh"), + ) + + def test_ops_script_path_normalizes_leading_slash(self) -> None: + self.assertEqual( + "domain-api/deploy/multi-region/check_ops_center_stack.sh", + ops_script_path("/check_ops_center_stack.sh"), + ) + + def test_build_bash_command_joins_non_empty_args(self) -> None: + self.assertEqual( + "bash domain-api/deploy/multi-region/drive_ops_center.sh driver-resolve http://127.0.0.1:8100 handover_first_gap", + build_bash_command( + "drive_ops_center.sh", + "driver-resolve", + "http://127.0.0.1:8100", + "handover_first_gap", + ), + ) + + def test_build_bash_command_skips_blank_args(self) -> None: + self.assertEqual( + "bash domain-api/deploy/multi-region/drive_release_hub.sh http://127.0.0.1:8100 launchpad", + build_bash_command( + "drive_release_hub.sh", + "http://127.0.0.1:8100", + "", + "launchpad", + " ", + ), + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/domain-api/tests/test_ops_job_service.py b/domain-api/tests/test_ops_job_service.py new file mode 100644 index 0000000..4813d5f --- /dev/null +++ b/domain-api/tests/test_ops_job_service.py @@ -0,0 +1,391 @@ +import json +import unittest +from datetime import datetime +from unittest.mock import MagicMock, patch + +from app.core.config import settings +from app.services.ops_job_service import ( + _execute_control_plane_job, + _serialize_job_row, + create_ops_job, + dispatch_ops_job, + upsert_managed_node, +) + + +class OpsJobServiceTests(unittest.TestCase): + def test_serialize_job_row_adds_execution_mode_label(self) -> None: + row = ( + 9, + "ops-2026041801-abcd12", + "logs.collect", + "node", + "mainland-worker-01", + "queued", + "remote-agent", + "api", + json.dumps({"service_name": "domaincheck-worker", "lines": 120}, ensure_ascii=False), + json.dumps({}, ensure_ascii=False), + json.dumps({}, ensure_ascii=False), + "", + datetime(2026, 4, 18, 10, 0, 0), + None, + None, + datetime(2026, 4, 18, 10, 0, 0), + "low", + False, + "approved", + "", + None, + "", + "", + None, + json.dumps({}, ensure_ascii=False), + json.dumps({}, ensure_ascii=False), + None, + ) + + job = _serialize_job_row(row) + + self.assertEqual("remote-agent", job["execution_mode"]) + self.assertEqual("远端 Agent", job["execution_mode_label"]) + self.assertEqual("排队中", job["status_label"]) + self.assertEqual("已审批", job["approval_status_label"]) + self.assertEqual(["mainland-worker-01"], job["target_node_codes"]) + self.assertEqual(0, job["steps_total"]) + self.assertFalse(job["steps_loaded"]) + self.assertFalse(job["is_compact"]) + self.assertEqual("ops_job", job["focus_ref"]["kind"]) + self.assertEqual(9, job["focus_ref"]["job_id"]) + self.assertIn("目标节点 mainland-worker-01", job["summary_text"]) + + def test_serialize_job_row_counts_loaded_steps(self) -> None: + row = ( + 10, + "ops-2026041801-efgh34", + "diagnostics.collect", + "node", + "mainland-worker-02", + "running", + "ssh", + "web-ui", + json.dumps({}, ensure_ascii=False), + json.dumps({}, ensure_ascii=False), + json.dumps({}, ensure_ascii=False), + "", + datetime(2026, 4, 18, 10, 5, 0), + datetime(2026, 4, 18, 10, 5, 2), + None, + datetime(2026, 4, 18, 10, 5, 3), + "medium", + True, + "approved", + "admin", + datetime(2026, 4, 18, 10, 5, 1), + "", + "", + datetime(2026, 4, 18, 10, 5, 2), + json.dumps({}, ensure_ascii=False), + json.dumps({}, ensure_ascii=False), + None, + ) + + job = _serialize_job_row( + row, + steps=[ + {"id": 1, "status": "success"}, + {"id": 2, "status": "running"}, + {"id": 3, "status": "failed"}, + ], + ) + + self.assertEqual("执行中", job["status_label"]) + self.assertEqual("已审批", job["approval_status_label"]) + self.assertEqual(3, job["steps_total"]) + self.assertEqual(1, job["steps_running"]) + self.assertEqual(1, job["steps_success"]) + self.assertEqual(1, job["steps_failed"]) + self.assertEqual(2, job["steps_terminal"]) + self.assertTrue(job["steps_loaded"]) + self.assertIn("mainland-worker-02", job["summary_text"]) + + @patch("app.services.ops_job_service.list_managed_nodes") + @patch("app.services.ops_agent_service.build_node_agent_bootstrap_plan") + def test_execute_control_plane_bootstrap_uses_managed_node_defaults( + self, + mock_build_node_agent_bootstrap_plan, + mock_list_managed_nodes, + ) -> None: + mock_list_managed_nodes.return_value = [ + { + "node_code": "mainland-worker-02", + "region": "mainland", + "role": "worker", + } + ] + mock_build_node_agent_bootstrap_plan.return_value = ( + True, + "节点 Agent 接入方案已生成", + { + "token_preview": "agent-...1234", + "bootstrap_plan": { + "bootstrap_script_name": "bootstrap-node-agent-mainland-worker-02.sh", + }, + }, + ) + + ok, message, data = _execute_control_plane_job( + "node.bootstrap", + target_node_code="mainland-worker-02", + payload={ + "root_dir": "/opt/domaincheck", + "expires_in_hours": 12, + }, + requested_by="unit-test", + metadata={"source": "test"}, + ) + + self.assertTrue(ok) + self.assertEqual("节点 Agent 接入方案已生成", message) + self.assertEqual("control-plane", data["execution_mode"]) + self.assertEqual("mainland", data["node_region"]) + self.assertEqual("worker", data["node_role"]) + mock_build_node_agent_bootstrap_plan.assert_called_once_with( + node_code="mainland-worker-02", + node_region="mainland", + node_role="worker", + issued_by="unit-test", + expires_in_hours=12, + control_plane_base_url="", + root_dir="/opt/domaincheck", + metadata={ + "source": "test", + "issued_from": "ops-job/control-plane", + "target_node_code": "mainland-worker-02", + "node_region": "mainland", + "node_role": "worker", + }, + ) + + @patch("app.services.ops_job_service.get_db") + def test_upsert_managed_node_preserves_remote_fields_when_sync_payload_omits_them(self, mock_get_db) -> None: + existing_metadata = {"operator_note": "keep-me"} + merged_metadata = {"operator_note": "keep-me", "cluster_status": "online"} + existing_row = ( + "mainland-worker-01", + "mainland", + "worker", + "Mainland Worker 01", + "121.204.244.248", + 22, + "root", + "key", + "stable", + False, + json.dumps(existing_metadata, ensure_ascii=False), + datetime(2026, 4, 18, 1, 0, 0), + None, + datetime(2026, 4, 18, 1, 5, 0), + ) + returned_row = ( + "mainland-worker-01", + "mainland", + "worker", + "S244-248", + "121.204.244.248", + 22, + "root", + "key", + "stable", + False, + json.dumps(merged_metadata, ensure_ascii=False), + datetime(2026, 4, 18, 1, 0, 0), + None, + datetime(2026, 4, 18, 1, 6, 0), + ) + + conn = MagicMock() + cursor = MagicMock() + db_ctx = MagicMock() + cursor_ctx = MagicMock() + db_ctx.__enter__.return_value = conn + db_ctx.__exit__.return_value = False + cursor_ctx.__enter__.return_value = cursor + cursor_ctx.__exit__.return_value = False + conn.cursor.return_value = cursor_ctx + mock_get_db.return_value = db_ctx + cursor.fetchone.side_effect = [existing_row, returned_row] + + ok, message, data = upsert_managed_node( + { + "node_code": "mainland-worker-01", + "region": "mainland", + "role": "worker", + "title": "S244-248", + "metadata": { + "cluster_status": "online", + }, + } + ) + + self.assertTrue(ok) + self.assertEqual("托管节点已保存", message) + node = data["node"] + self.assertEqual("121.204.244.248", node["ssh_host"]) + self.assertEqual("root", node["ssh_user"]) + self.assertFalse(node["is_enabled"]) + self.assertEqual("keep-me", node["metadata"]["operator_note"]) + self.assertEqual("online", node["metadata"]["cluster_status"]) + + insert_call = cursor.execute.call_args_list[-1] + insert_params = insert_call.args[1] + self.assertEqual("121.204.244.248", insert_params[4]) + self.assertEqual(22, insert_params[5]) + self.assertEqual("root", insert_params[6]) + self.assertFalse(insert_params[9]) + self.assertEqual(merged_metadata, json.loads(insert_params[10])) + + @patch("app.services.ops_agent_service.append_ops_job_event") + @patch("app.services.ops_job_service._execute_ssh_job_record") + @patch("app.services.ops_job_service.get_db") + def test_dispatch_ops_job_ssh_executes_via_ssh_record( + self, + mock_get_db, + mock_execute_ssh_job_record, + mock_append_ops_job_event, + ) -> None: + conn = MagicMock() + cursor = MagicMock() + db_ctx = MagicMock() + cursor_ctx = MagicMock() + db_ctx.__enter__.return_value = conn + db_ctx.__exit__.return_value = False + cursor_ctx.__enter__.return_value = cursor + cursor_ctx.__exit__.return_value = False + conn.cursor.return_value = cursor_ctx + mock_get_db.return_value = db_ctx + cursor.fetchone.return_value = ( + 9, + "ops-2026041801-abcd12", + "logs.collect", + "node", + "mainland-worker-01", + "queued", + "ssh", + "api", + json.dumps({"service_name": "domaincheck-worker", "lines": 120}, ensure_ascii=False), + json.dumps({}, ensure_ascii=False), + json.dumps({}, ensure_ascii=False), + "", + datetime(2026, 4, 18, 10, 0, 0), + None, + None, + datetime(2026, 4, 18, 10, 0, 0), + "low", + False, + "approved", + "", + None, + "", + "", + None, + json.dumps({}, ensure_ascii=False), + json.dumps({}, ensure_ascii=False), + None, + ) + mock_execute_ssh_job_record.return_value = ( + True, + "worker logs collected", + {"job": {"id": 9, "status": "success"}}, + ) + + ok, message, data = dispatch_ops_job(9) + + self.assertTrue(ok) + self.assertEqual("已按 SSH 执行派发：worker logs collected", message) + self.assertEqual("success", data["job"]["status"]) + mock_execute_ssh_job_record.assert_called_once_with(9) + mock_append_ops_job_event.assert_called_once() + event_kwargs = mock_append_ops_job_event.call_args.kwargs + self.assertEqual("job_dispatch_requested", event_kwargs["event_type"]) + self.assertEqual("mainland-worker-01", event_kwargs["node_code"]) + self.assertEqual("ssh", event_kwargs["payload"]["execution_mode"]) + + @patch("app.services.ops_agent_service.append_ops_job_event") + @patch("app.services.ops_job_service.get_ops_job") + @patch("app.services.ops_job_service._execute_local_job") + @patch("app.services.ops_policy_service.preview_ops_job_policy") + @patch("app.services.ops_job_service.get_db") + def test_create_ops_job_auto_uses_local_runtime_for_local_structured_action( + self, + mock_get_db, + mock_preview_ops_job_policy, + mock_execute_local_job, + mock_get_ops_job, + mock_append_ops_job_event, + ) -> None: + conn = MagicMock() + cursor = MagicMock() + db_ctx = MagicMock() + cursor_ctx = MagicMock() + db_ctx.__enter__.return_value = conn + db_ctx.__exit__.return_value = False + cursor_ctx.__enter__.return_value = cursor + cursor_ctx.__exit__.return_value = False + conn.cursor.return_value = cursor_ctx + mock_get_db.return_value = db_ctx + cursor.fetchone.return_value = (42,) + + mock_preview_ops_job_policy.return_value = { + "risk_level": "low", + "approval_required": False, + "blocked": False, + "blocking_reasons": [], + } + mock_execute_local_job.return_value = ( + True, + "worker logs collected", + {"executor": "local-runtime"}, + ) + mock_get_ops_job.return_value = { + "id": 42, + "status": "success", + "execution_mode": "local-runtime", + } + + ok, message, data = create_ops_job( + { + "action": "logs.collect", + "target_node_code": settings.node_code, + "payload": { + "service_name": "domaincheck-worker", + "lines": 120, + }, + "requested_by": "unit-test", + } + ) + + self.assertTrue(ok) + self.assertEqual("worker logs collected", message) + self.assertTrue(data["executed_immediately"]) + self.assertEqual("local-runtime", data["job"]["execution_mode"]) + mock_execute_local_job.assert_called_once_with( + "logs.collect", + { + "service_name": "domaincheck-worker", + "lines": 120, + }, + ) + + insert_call = next( + call + for call in cursor.execute.call_args_list + if "INSERT INTO ops_jobs" in str(call.args[0]) + ) + insert_params = insert_call.args[1] + self.assertEqual("local-runtime", insert_params[5]) + mock_append_ops_job_event.assert_called_once() + + +if __name__ == "__main__": + unittest.main() diff --git a/domain-api/tests/test_ops_playbook_service.py b/domain-api/tests/test_ops_playbook_service.py new file mode 100644 index 0000000..64e73ad --- /dev/null +++ b/domain-api/tests/test_ops_playbook_service.py @@ -0,0 +1,220 @@ +import unittest +from unittest.mock import patch + +from app.services.ops_playbook_service import ( + get_recent_ops_playbook_runs, + list_ops_playbook_run_events, + preview_ops_playbook, +) + + +class OpsPlaybookServiceTests(unittest.TestCase): + def test_onboarding_bootstrap_preview_uses_control_plane(self) -> None: + ok, message, data = preview_ops_playbook( + { + "playbook_key": "onboarding.bootstrap", + "node_codes": ["mainland-worker-02"], + } + ) + + self.assertTrue(ok, message) + self.assertEqual("playbook 预览生成成功", message) + self.assertEqual("onboarding.bootstrap", data["playbook"]["key"]) + self.assertEqual("control-plane", data["execution_mode"]) + self.assertEqual("控制面", data["execution_mode_label"]) + self.assertEqual(["control-plane"], data["playbook"]["execution_modes"]) + self.assertEqual("控制面", data["playbook"]["default_execution_mode_label"]) + self.assertEqual("控制面", data["playbook"]["execution_mode_options"][0]["label"]) + self.assertTrue(data["auto_approve"]) + self.assertEqual(1, data["step_count"]) + step = data["steps"][0] + self.assertEqual("bootstrap", step["step_key"]) + self.assertEqual("node.bootstrap", step["template_key"]) + self.assertEqual("node.bootstrap", step["action"]) + self.assertEqual("control-plane", step["execution_mode"]) + self.assertEqual(1, step["expected_jobs"]) + + def test_scene_logs_key_preview_allows_ssh_execution_mode(self) -> None: + ok, message, data = preview_ops_playbook( + { + "playbook_key": "scene.logs.key", + "node_codes": ["mainland-worker-01"], + "execution_mode": "ssh", + } + ) + + self.assertTrue(ok, message) + self.assertEqual("playbook 预览生成成功", message) + self.assertEqual("ssh", data["execution_mode"]) + self.assertEqual("SSH", data["execution_mode_label"]) + self.assertEqual(["remote-agent", "ssh"], data["playbook"]["execution_modes"]) + self.assertEqual("ssh", data["steps"][0]["execution_mode"]) + self.assertEqual("SSH", data["steps"][0]["execution_mode_label"]) + + def test_onboarding_bootstrap_preview_rejects_unsupported_execution_mode(self) -> None: + ok, message, data = preview_ops_playbook( + { + "playbook_key": "onboarding.bootstrap", + "node_codes": ["mainland-worker-02"], + "execution_mode": "ssh", + } + ) + + self.assertFalse(ok) + self.assertEqual("该 playbook 仅支持 control-plane", message) + self.assertEqual(["control-plane"], data["playbook"]["execution_modes"]) + + @patch("app.services.ops_playbook_service.list_managed_nodes_with_agent_state") + @patch("app.services.ops_playbook_service.list_ops_jobs") + def test_recent_onboarding_bootstrap_run_focus_summary_reflects_agent_state( + self, + mock_list_ops_jobs, + mock_list_managed_nodes_with_agent_state, + ) -> None: + mock_list_ops_jobs.return_value = [ + { + "id": 31, + "job_code": "ops-bootstrap-31", + "action": "node.bootstrap", + "status": "success", + "target_node_code": "mainland-worker-02", + "created_at": "2026-04-17 10:00:00", + "updated_at": "2026-04-17 10:01:00", + "metadata": { + "playbook": { + "run_code": "pbr-bootstrap-1", + "key": "onboarding.bootstrap", + "title": "生成节点接入工单", + "group_key": "onboarding", + "group_title": "接管验收", + "requested_by": "tester", + "execution_mode": "control-plane", + "step_key": "bootstrap", + "step_title": "生成 Node Agent 接入工单", + "step_order": 1, + "step_count": 1, + "expected_jobs_total": 1, + "target_nodes_total": 1, + "template_key": "node.bootstrap", + } + }, + "result": { + "bootstrap_plan": { + "bootstrap_script_name": "bootstrap-node-agent-mainland-worker-02.sh", + } + }, + } + ] + mock_list_managed_nodes_with_agent_state.return_value = { + "nodes": [ + { + "node_code": "mainland-worker-02", + "agent_state": "pending_bootstrap", + "is_managed": True, + } + ] + } + + payload = get_recent_ops_playbook_runs(limit=10, scan_limit=20) + + self.assertEqual(1, len(payload["runs"])) + run = payload["runs"][0] + self.assertEqual("success", run["status"]) + self.assertEqual("成功", run["status_label"]) + self.assertEqual("控制面", run["execution_mode_label"]) + self.assertEqual("warning", run["focus_level"]) + self.assertIn("待执行 bootstrap 脚本", run["focus_summary"]) + self.assertEqual(run["focus_summary"], run["summary_text"]) + self.assertEqual(1, run["steps_total"]) + self.assertEqual("pbr-bootstrap-1", run["focus_ref"]["run_code"]) + self.assertEqual("生成 Node Agent 接入工单", run["steps"][0]["title"]) + self.assertEqual("成功", run["steps"][0]["status_label"]) + self.assertTrue(run["steps"][0]["summary_text"]) + + @patch("app.services.ops_playbook_service.list_managed_nodes_with_agent_state") + @patch("app.services.ops_playbook_service.list_ops_job_events_for_jobs") + @patch("app.services.ops_playbook_service.list_ops_jobs") + def test_playbook_run_events_add_contract_aliases( + self, + mock_list_ops_jobs, + mock_list_ops_job_events_for_jobs, + mock_list_managed_nodes_with_agent_state, + ) -> None: + mock_list_managed_nodes_with_agent_state.return_value = {"nodes": []} + mock_list_ops_jobs.return_value = [ + { + "id": 41, + "job_code": "ops-inspection-41", + "action": "logs.collect", + "status": "running", + "target_node_code": "mainland-worker-01", + "created_at": "2026-04-18 06:30:00", + "updated_at": "2026-04-18 06:31:00", + "metadata": { + "playbook": { + "run_code": "pbr-inspection-1", + "key": "inspection.standard", + "title": "标准巡检", + "group_key": "diagnostics", + "group_title": "标准巡检", + "requested_by": "tester", + "execution_mode": "remote-agent", + "step_key": "worker_logs", + "step_title": "收集 Worker 日志", + "step_order": 2, + "step_count": 3, + "expected_jobs_total": 3, + "target_nodes_total": 1, + "template_key": "logs.collect", + } + }, + } + ] + mock_list_ops_job_events_for_jobs.return_value = [ + { + "id": 501, + "event_key": "job-event:501", + "job_id": 41, + "node_code": "mainland-worker-01", + "event_type": "job_running", + "level": "info", + "level_label": "信息", + "message": "开始收集 Worker 日志", + "summary": "开始收集 Worker 日志", + "summary_text": "远端 Agent 已开始收集 Worker 日志", + "focus_ref": { + "kind": "ops_job", + "job_id": 41, + "job_code": "ops-inspection-41", + "action": "logs.collect", + "target_node_code": "mainland-worker-01", + "event_key": "job-event:501", + "step_key": "worker_logs", + }, + "payload": {}, + "created_at": "2026-04-18 06:31:00", + "occurred_at": "2026-04-18 06:31:00", + "ui_intent": {"kind": "job_detail", "job_id": 41, "focus_event_id": 501}, + } + ] + + payload = list_ops_playbook_run_events("pbr-inspection-1", limit=10) + + self.assertEqual("inspection.standard", payload["playbook_run"]["playbook_key"]) + self.assertEqual(1, len(payload["events"])) + event = payload["events"][0] + self.assertEqual("收口中", event["job_status_label"]) + self.assertEqual("远端 Agent 已开始收集 Worker 日志", event["summary_text"]) + self.assertEqual("worker_logs", event["focus_ref"]["focus_step_key"]) + self.assertEqual("job-event:501", event["focus_ref"]["event_key"]) + self.assertEqual("ops_job", event["source_focus_ref"]["kind"]) + self.assertEqual("job-event:501", event["source_focus_ref"]["event_key"]) + summary = payload["summary"] + self.assertEqual(1, summary["returned_total"]) + self.assertEqual({"job_running": 1}, summary["event_type_counts"]) + self.assertEqual({"running": 1}, summary["job_status_counts"]) + self.assertEqual(10, summary["filters"]["limit"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/domain-api/tests/test_ops_policy_service.py b/domain-api/tests/test_ops_policy_service.py new file mode 100644 index 0000000..a1766e5 --- /dev/null +++ b/domain-api/tests/test_ops_policy_service.py @@ -0,0 +1,253 @@ +import unittest +from unittest.mock import patch + +from app.services.ops_policy_service import preview_ops_job_policy + + +class OpsPolicyServiceTests(unittest.TestCase): + @patch("app.services.ops_policy_service.get_cluster_snapshot") + def test_node_bootstrap_preview_requires_control_plane(self, mock_cluster_snapshot) -> None: + mock_cluster_snapshot.return_value = { + "nodes": [ + { + "node_code": "mainland-worker-01", + "region": "mainland", + "role": "worker", + "status": "online", + "current_load": 0, + "is_effective_worker": True, + "detect_participating": False, + } + ], + "summary": { + "online_worker_nodes": 2, + "dedicated_online_worker_nodes": 1, + "online_control_nodes": 1, + "busy_nodes": [], + }, + } + + preview = preview_ops_job_policy( + { + "action": "node.bootstrap", + "target_type": "node", + "target_node_code": "mainland-worker-01", + "execution_mode": "remote-agent", + "payload": {}, + } + ) + + self.assertEqual("critical", preview["risk_level"]) + self.assertTrue(preview["blocked"]) + self.assertTrue(preview["approval_required"]) + self.assertIn("node.bootstrap 仅支持 control-plane", " ".join(preview["blocking_reasons"])) + + @patch("app.services.ops_policy_service.get_cluster_snapshot") + def test_deploy_release_batch_preview_summarizes_node_risks(self, mock_cluster_snapshot) -> None: + mock_cluster_snapshot.return_value = { + "nodes": [ + { + "node_code": "mainland-controller-01", + "region": "mainland", + "role": "control", + "status": "busy", + "current_load": 3, + "is_effective_worker": True, + "detect_participating": False, + }, + { + "node_code": "mainland-worker-01", + "region": "mainland", + "role": "worker", + "status": "online", + "current_load": 0, + "is_effective_worker": True, + "detect_participating": False, + }, + ], + "summary": { + "online_worker_nodes": 2, + "dedicated_online_worker_nodes": 1, + "online_control_nodes": 1, + "busy_nodes": ["mainland-controller-01"], + }, + } + + preview = preview_ops_job_policy( + { + "action": "deploy.release", + "target_type": "batch", + "target_node_codes": ["mainland-controller-01", "mainland-worker-01"], + "execution_mode": "remote-agent", + "payload": { + "release_version": "2026.04.18-rc1", + "artifact_url": "https://example.com/release.tar.gz", + }, + } + ) + + self.assertEqual("batch", preview["target_type"]) + self.assertTrue(preview["blocked"]) + self.assertTrue(preview["approval_required"]) + self.assertEqual(2, preview["target_summary"]["nodes_total"]) + self.assertEqual(1, preview["target_summary"]["blocked_nodes"]) + self.assertEqual(1, preview["target_summary"]["approval_nodes"]) + self.assertEqual(1, preview["target_summary"]["warning_nodes"]) + self.assertEqual(2, len(preview["target_nodes"])) + controller_row = next(item for item in preview["target_nodes"] if item["node_code"] == "mainland-controller-01") + self.assertTrue(controller_row["blocked"]) + self.assertTrue(controller_row["approval_required"]) + + @patch("app.services.ops_policy_service.get_cluster_snapshot") + def test_deploy_release_preview_blocks_missing_artifact_and_wrong_execution_mode(self, mock_cluster_snapshot) -> None: + mock_cluster_snapshot.return_value = { + "nodes": [ + { + "node_code": "mainland-worker-01", + "region": "mainland", + "role": "worker", + "status": "online", + "current_load": 0, + "is_effective_worker": True, + "detect_participating": False, + } + ], + "summary": { + "online_worker_nodes": 2, + "dedicated_online_worker_nodes": 1, + "online_control_nodes": 1, + "busy_nodes": [], + }, + } + + preview = preview_ops_job_policy( + { + "action": "deploy.release", + "target_type": "node", + "target_node_code": "mainland-worker-01", + "execution_mode": "local-runtime", + "payload": { + "release_version": "2026.04.18-rc1", + "artifact_url": "", + }, + } + ) + + self.assertTrue(preview["blocked"]) + self.assertIn("deploy.release 缺少 artifact_url", " ".join(preview["blocking_reasons"])) + self.assertIn("deploy.release 仅支持 remote-agent 或 ssh", " ".join(preview["blocking_reasons"])) + + @patch("app.services.ops_policy_service.get_cluster_snapshot") + def test_deploy_release_preview_allows_ssh_execution_mode(self, mock_cluster_snapshot) -> None: + mock_cluster_snapshot.return_value = { + "nodes": [ + { + "node_code": "mainland-worker-01", + "region": "mainland", + "role": "worker", + "status": "online", + "current_load": 0, + "is_effective_worker": True, + "detect_participating": False, + } + ], + "summary": { + "online_worker_nodes": 2, + "dedicated_online_worker_nodes": 1, + "online_control_nodes": 1, + "busy_nodes": [], + }, + } + + preview = preview_ops_job_policy( + { + "action": "deploy.release", + "target_type": "node", + "target_node_code": "mainland-worker-01", + "execution_mode": "ssh", + "payload": { + "release_version": "2026.04.18-rc1", + "artifact_url": "https://example.com/release.tar.gz", + }, + } + ) + + self.assertFalse(any("仅支持 remote-agent 或 ssh" in item for item in preview["blocking_reasons"])) + self.assertTrue(preview["approval_required"]) + + @patch("app.services.ops_policy_service.get_cluster_snapshot") + def test_service_status_preview_blocks_control_plane_execution_mode(self, mock_cluster_snapshot) -> None: + mock_cluster_snapshot.return_value = { + "nodes": [ + { + "node_code": "mainland-worker-01", + "region": "mainland", + "role": "worker", + "status": "online", + "current_load": 0, + "is_effective_worker": True, + "detect_participating": False, + } + ], + "summary": { + "online_worker_nodes": 2, + "dedicated_online_worker_nodes": 1, + "online_control_nodes": 1, + "busy_nodes": [], + }, + } + + preview = preview_ops_job_policy( + { + "action": "service.status", + "target_type": "node", + "target_node_code": "mainland-worker-01", + "execution_mode": "control-plane", + "payload": {"service_name": "domaincheck-worker"}, + } + ) + + self.assertTrue(preview["blocked"]) + self.assertIn("service.status 当前不支持 control-plane 执行方式", " ".join(preview["blocking_reasons"])) + + @patch("app.services.ops_policy_service.get_cluster_snapshot") + def test_logs_collect_preview_blocks_local_runtime_on_remote_node(self, mock_cluster_snapshot) -> None: + mock_cluster_snapshot.return_value = { + "nodes": [ + { + "node_code": "mainland-worker-01", + "region": "mainland", + "role": "worker", + "status": "online", + "current_load": 0, + "is_effective_worker": True, + "detect_participating": False, + } + ], + "summary": { + "online_worker_nodes": 2, + "dedicated_online_worker_nodes": 1, + "online_control_nodes": 1, + "busy_nodes": [], + }, + } + + preview = preview_ops_job_policy( + { + "action": "logs.collect", + "target_type": "node", + "target_node_code": "mainland-worker-01", + "execution_mode": "local-runtime", + "payload": { + "service_name": "domaincheck-worker", + "lines": 120, + }, + } + ) + + self.assertTrue(preview["blocked"]) + self.assertIn("local-runtime 仅支持当前控制面本机节点", " ".join(preview["blocking_reasons"])) + + +if __name__ == "__main__": + unittest.main() diff --git a/domain-api/tests/test_ops_release_executor_core.py b/domain-api/tests/test_ops_release_executor_core.py new file mode 100644 index 0000000..d5da2eb --- /dev/null +++ b/domain-api/tests/test_ops_release_executor_core.py @@ -0,0 +1,89 @@ +import io +import tarfile +import tempfile +import unittest +from pathlib import Path + +from app.services.ops_release_executor_core import build_remote_release_action_script, execute_release_action + + +class _BytesResponse: + def __init__(self, data: bytes) -> None: + self._data = data + + def __enter__(self): + return self + + def __exit__(self, exc_type, exc, tb): + return False + + def read(self, *args, **kwargs) -> bytes: + return self._data + + +def _build_release_archive() -> bytes: + buffer = io.BytesIO() + with tarfile.open(fileobj=buffer, mode="w:gz") as archive: + content = b"hello-release" + info = tarfile.TarInfo(name="bundle/README.txt") + info.size = len(content) + archive.addfile(info, io.BytesIO(content)) + return buffer.getvalue() + + +class OpsReleaseExecutorCoreTests(unittest.TestCase): + def test_execute_release_action_deploys_archive_and_switches_current(self) -> None: + archive_bytes = _build_release_archive() + commands: list[tuple[str, ...]] = [] + + def run_command(command: list[str], *, timeout: int = 60): + commands.append(tuple(command)) + if command[:2] == ["systemctl", "restart"]: + return 0, "", "" + if command[:2] == ["systemctl", "is-active"]: + return 0, "active", "" + return 0, "", "" + + with tempfile.TemporaryDirectory() as tmpdir: + ok, message, data = execute_release_action( + { + "release_version": "2026.04.18-rc1", + "artifact_url": "https://example.com/domaincheck.tar.gz", + "install_root": tmpdir, + "restart_services": ["domaincheck-worker"], + "health_check_services": ["domaincheck-worker"], + "health_check_urls": [], + "rollback_on_failure": "true", + "switch_current": "true", + }, + run_command=run_command, + default_api_service_name="domaincheck-api", + urlopen_func=lambda request, timeout=120: _BytesResponse(archive_bytes), + user_agent="unit-test/0.1", + ) + + self.assertTrue(ok, message) + self.assertEqual("release deployed", message) + self.assertEqual("2026.04.18-rc1", data["release_version"]) + current_link = Path(tmpdir) / "current" + self.assertTrue(current_link.is_symlink()) + release_dir = Path(data["release_dir"]) + self.assertTrue(release_dir.exists()) + self.assertTrue((release_dir / ".release-meta.json").exists()) + self.assertIn(("systemctl", "restart", "domaincheck-worker"), commands) + self.assertIn(("systemctl", "is-active", "domaincheck-worker"), commands) + + def test_build_remote_release_action_script_is_valid_python(self) -> None: + script = build_remote_release_action_script( + { + "release_version": "2026.04.18-rc1", + "artifact_url": "https://example.com/domaincheck.tar.gz", + } + ) + + compile(script, "", "exec") + self.assertIn("execute_release_action(", script) + + +if __name__ == "__main__": + unittest.main() diff --git a/domain-api/tests/test_ops_release_service_contracts.py b/domain-api/tests/test_ops_release_service_contracts.py new file mode 100644 index 0000000..6befb46 --- /dev/null +++ b/domain-api/tests/test_ops_release_service_contracts.py @@ -0,0 +1,93 @@ +import json +import unittest +from datetime import datetime + +from app.services.ops_release_service import _serialize_release_row, _serialize_rollout_row + + +class OpsReleaseServiceContractTests(unittest.TestCase): + def test_serialize_release_row_adds_contract_aliases(self) -> None: + row = ( + 9, + "2026.04.18+bd6bcb2", + "stable", + "bd6bcb2", + "ready", + "https://example.com/domaincheck.tar.gz", + "sha256:test", + "release notes", + json.dumps( + { + "source": "package", + "manifest": { + "artifact_kind": "tarball", + "package_name": "domaincheck-2026.04.18+bd6bcb2.tgz", + "included_services": ["domaincheck-api", "domaincheck-worker"], + "rollback_hint": "切回上一个 current", + }, + } + ), + "www", + datetime(2026, 4, 18, 9, 0, 0), + datetime(2026, 4, 18, 9, 5, 0), + datetime(2026, 4, 18, 9, 6, 0), + ) + + payload = _serialize_release_row(row) + + self.assertEqual("已就绪", payload["status_label"]) + self.assertIn("已就绪", payload["summary_text"]) + self.assertEqual("release_hub", payload["focus_ref"]["kind"]) + self.assertEqual(9, payload["focus_ref"]["release_id"]) + self.assertEqual("release_detail", payload["focus_ref"]["section"]) + self.assertEqual("tarball", payload["manifest"]["artifact_kind"]) + self.assertEqual("domaincheck-2026.04.18+bd6bcb2.tgz", payload["manifest"]["package_name"]) + self.assertEqual(["domaincheck-api", "domaincheck-worker"], payload["manifest"]["included_services"]) + + def test_serialize_rollout_row_adds_contract_aliases(self) -> None: + row = ( + 11, + 9, + "rollout-20260418090000-a1b2", + json.dumps({"region": "mainland", "role": "worker"}), + json.dumps( + { + "items": [ + {"node_code": "mainland-worker-01", "region": "mainland", "role": "worker"}, + {"node_code": "mainland-worker-02", "region": "mainland", "role": "worker"}, + ] + } + ), + json.dumps({"execution_mode": "ssh", "batch_size": 1}), + "running", + 1, + 2, + 2, + 1, + json.dumps( + { + "status": "running", + "launched_targets": 1, + "remaining_targets": 1, + "active_jobs": 1, + "current_batch_index": 1, + "batches_total": 2, + } + ), + "web-ui", + datetime(2026, 4, 18, 9, 10, 0), + datetime(2026, 4, 18, 9, 11, 0), + ) + + payload = _serialize_rollout_row(row) + + self.assertEqual("执行中", payload["status_label"]) + self.assertEqual(["mainland-worker-01", "mainland-worker-02"], payload["target_node_codes"]) + self.assertIn("当前已覆盖 1/2 台节点", payload["summary_text"]) + self.assertEqual("release_rollout", payload["focus_ref"]["kind"]) + self.assertEqual(11, payload["focus_ref"]["rollout_id"]) + self.assertEqual("rollout_jobs", payload["focus_ref"]["section"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/domain-api/tests/test_ops_release_service_execution_modes.py b/domain-api/tests/test_ops_release_service_execution_modes.py new file mode 100644 index 0000000..ac78276 --- /dev/null +++ b/domain-api/tests/test_ops_release_service_execution_modes.py @@ -0,0 +1,432 @@ +import unittest +from unittest.mock import patch + +from app.services.ops_release_service import ( + build_release_execution_mode_recommendation, + build_release_rollout_gate, + build_rollout_target_operational_readiness, +) + + +class _EmptyCursor: + def __enter__(self): + return self + + def __exit__(self, exc_type, exc, tb): + return False + + def execute(self, query, params=None) -> None: + self.query = query + self.params = params + + def fetchall(self): + return [] + + def fetchone(self): + return None + + +class _EmptyConnection: + def __enter__(self): + return self + + def __exit__(self, exc_type, exc, tb): + return False + + def cursor(self): + return _EmptyCursor() + + +class OpsReleaseServiceExecutionModeTests(unittest.TestCase): + @patch("app.services.ops_release_service.get_db") + @patch("app.services.ops_agent_service.ensure_ops_agent_schema") + @patch("app.services.ops_agent_service.get_managed_node_onboarding") + @patch("app.services.ops_job_service.list_managed_nodes") + @patch("app.services.cluster_runtime_service.get_cluster_snapshot") + def test_build_rollout_target_operational_readiness_accepts_ssh_ready_nodes( + self, + mock_get_cluster_snapshot, + mock_list_managed_nodes, + mock_get_managed_node_onboarding, + mock_ensure_ops_agent_schema, + mock_get_db, + ) -> None: + mock_ensure_ops_agent_schema.return_value = None + mock_get_managed_node_onboarding.return_value = { + "onboarding_stage": {"code": "acceptance_ready", "label": "待接管验收"}, + "summary": "Node Agent 已在线，可以直接跑 onboarding.acceptance。", + "recovery_decision": { + "action": "run_acceptance", + "label": "执行接管验收", + "summary": "Node Agent 已在线，可以直接跑 onboarding.acceptance。", + "command_hint": "bash drive_ops_center.sh node-acceptance-run http://127.0.0.1:8100 mainland-worker-01 cli", + "window": "acceptance", + }, + } + mock_get_db.return_value = _EmptyConnection() + mock_get_cluster_snapshot.return_value = { + "nodes": [ + { + "node_code": "mainland-worker-01", + "region": "mainland", + "role": "worker", + "status": "online", + "current_load": 0, + "is_effective_worker": True, + } + ] + } + mock_list_managed_nodes.return_value = [ + { + "node_code": "mainland-worker-01", + "region": "mainland", + "role": "worker", + "is_enabled": True, + "ssh_host": "121.204.244.248", + "ssh_user": "root", + "metadata": {}, + "last_seen_at": "", + } + ] + + readiness = build_rollout_target_operational_readiness( + [{"node_code": "mainland-worker-01", "region": "mainland", "role": "worker"}], + execution_mode="ssh", + ) + + self.assertEqual("ssh", readiness["execution_mode"]) + self.assertEqual(1, readiness["summary"]["ssh_ready_nodes"]) + self.assertEqual(1, readiness["summary"]["execution_ready_nodes"]) + self.assertEqual([], readiness["blocking_reasons"]) + self.assertTrue(readiness["rows"][0]["ssh_ready"]) + self.assertTrue(readiness["rows"][0]["execution_ready"]) + self.assertTrue(readiness["rows"][0]["agent_managed"]) + self.assertEqual("ssh", readiness["rows"][0]["execution_mode"]) + self.assertEqual("SSH", readiness["rows"][0]["execution_mode_label"]) + self.assertEqual("medium", readiness["rows"][0]["risk_level"]) + self.assertEqual("run_standard_inspection", readiness["rows"][0]["recommended_action_code"]) + self.assertEqual("acceptance_ready", readiness["rows"][0]["onboarding_stage_code"]) + self.assertEqual("run_acceptance", readiness["rows"][0]["recovery_action"]) + self.assertEqual(1, readiness["summary"]["onboarding_acceptance_ready_nodes"]) + + @patch("app.services.ops_release_service.get_db") + @patch("app.services.ops_agent_service.ensure_ops_agent_schema") + @patch("app.services.ops_agent_service.get_managed_node_onboarding") + @patch("app.services.ops_job_service.list_managed_nodes") + @patch("app.services.cluster_runtime_service.get_cluster_snapshot") + def test_build_rollout_target_operational_readiness_adds_release_context_and_focus_ref( + self, + mock_get_cluster_snapshot, + mock_list_managed_nodes, + mock_get_managed_node_onboarding, + mock_ensure_ops_agent_schema, + mock_get_db, + ) -> None: + mock_ensure_ops_agent_schema.return_value = None + mock_get_managed_node_onboarding.return_value = { + "onboarding_stage": {"code": "ready", "label": "已接管"}, + "summary": "当前节点无需额外恢复动作。", + "recovery_decision": { + "action": "noop", + "label": "当前无需额外恢复动作", + "summary": "当前节点暂无需要执行的接管恢复动作。", + "command_hint": "", + "window": "none", + }, + } + mock_get_db.return_value = _EmptyConnection() + mock_get_cluster_snapshot.return_value = { + "nodes": [ + { + "node_code": "mainland-worker-01", + "region": "mainland", + "role": "worker", + "status": "online", + "current_load": 0, + "is_effective_worker": True, + "metadata": {"current_release_version": "2026.04.11+1921318"}, + } + ] + } + mock_list_managed_nodes.return_value = [ + { + "node_code": "mainland-worker-01", + "region": "mainland", + "role": "worker", + "is_enabled": True, + "ssh_host": "121.204.244.248", + "ssh_user": "root", + "metadata": {"current_release_version": "2026.04.11+1921318"}, + "last_seen_at": "2026-04-18 08:00:00", + } + ] + + readiness = build_rollout_target_operational_readiness( + [{"node_code": "mainland-worker-01", "region": "mainland", "role": "worker"}], + execution_mode="remote-agent", + desired_release={"id": 18, "release_version": "2026.04.18+bd6bcb2", "channel": "stable"}, + ) + + row = readiness["rows"][0] + self.assertEqual("2026.04.11+1921318", row["current_release_version"]) + self.assertEqual("2026.04.18+bd6bcb2", row["desired_release_version"]) + self.assertEqual("release_hub", row["focus_ref"]["kind"]) + self.assertEqual("default_rollout_gate", row["focus_ref"]["section"]) + self.assertEqual("mainland-worker-01", row["focus_ref"]["node_code"]) + self.assertEqual("noop", row["recovery_action"]) + + @patch("app.services.ops_release_service.get_db") + @patch("app.services.ops_agent_service.ensure_ops_agent_schema") + @patch("app.services.ops_agent_service.get_managed_node_onboarding") + @patch("app.services.ops_job_service.list_managed_nodes") + @patch("app.services.cluster_runtime_service.get_cluster_snapshot") + def test_build_rollout_target_operational_readiness_prefers_recovery_decision_for_blocked_nodes( + self, + mock_get_cluster_snapshot, + mock_list_managed_nodes, + mock_get_managed_node_onboarding, + mock_ensure_ops_agent_schema, + mock_get_db, + ) -> None: + mock_ensure_ops_agent_schema.return_value = None + mock_get_db.return_value = _EmptyConnection() + mock_get_cluster_snapshot.return_value = { + "nodes": [ + { + "node_code": "mainland-worker-02", + "region": "mainland", + "role": "worker", + "status": "online", + "current_load": 0, + "is_effective_worker": True, + } + ] + } + mock_list_managed_nodes.return_value = [ + { + "node_code": "mainland-worker-02", + "region": "mainland", + "role": "worker", + "is_enabled": True, + "ssh_host": "121.204.244.249", + "ssh_user": "root", + "metadata": {}, + "last_seen_at": "", + } + ] + mock_get_managed_node_onboarding.return_value = { + "onboarding_stage": {"code": "pending_bootstrap", "label": "待接入"}, + "summary": "已签发有效 Agent Token，但节点尚未完成 register/heartbeat。", + "recovery_decision": { + "action": "bootstrap_run", + "label": "签发接入工单", + "summary": "当前节点仍处于接入阶段，建议先签发 onboarding.bootstrap。", + "command_hint": "bash drive_ops_center.sh node-bootstrap-run http://127.0.0.1:8100 mainland-worker-02 cli", + "window": "bootstrap", + }, + } + + readiness = build_rollout_target_operational_readiness( + [{"node_code": "mainland-worker-02", "region": "mainland", "role": "worker"}], + execution_mode="remote-agent", + ) + + row = readiness["rows"][0] + self.assertEqual("bootstrap_run", row["recovery_action"]) + self.assertEqual("bootstrap_run", row["recommended_action_code"]) + self.assertEqual(1, readiness["summary"]["onboarding_bootstrap_pending_nodes"]) + self.assertTrue( + any("onboarding.bootstrap" in item for item in readiness["recommendations"]) + ) + + @patch("app.services.ops_release_service.get_db") + @patch("app.services.ops_agent_service.ensure_ops_agent_schema") + @patch("app.services.ops_agent_service.get_managed_node_onboarding") + @patch("app.services.ops_job_service.list_managed_nodes") + @patch("app.services.cluster_runtime_service.get_cluster_snapshot") + def test_build_rollout_target_operational_readiness_blocks_nodes_with_dead_letter_queue( + self, + mock_get_cluster_snapshot, + mock_list_managed_nodes, + mock_get_managed_node_onboarding, + mock_ensure_ops_agent_schema, + mock_get_db, + ) -> None: + mock_ensure_ops_agent_schema.return_value = None + mock_get_db.return_value = _EmptyConnection() + mock_get_managed_node_onboarding.return_value = { + "onboarding_stage": {"code": "ready", "label": "已接管"}, + "summary": "当前节点无需额外恢复动作。", + "recovery_decision": { + "action": "noop", + "label": "当前无需额外恢复动作", + "summary": "当前节点暂无需要执行的接管恢复动作。", + "command_hint": "", + "window": "none", + }, + } + mock_get_cluster_snapshot.return_value = { + "nodes": [ + { + "node_code": "mainland-worker-03", + "region": "mainland", + "role": "worker", + "status": "online", + "current_load": 0, + "is_effective_worker": True, + } + ] + } + mock_list_managed_nodes.return_value = [ + { + "node_code": "mainland-worker-03", + "region": "mainland", + "role": "worker", + "is_enabled": True, + "ssh_host": "121.204.244.250", + "ssh_user": "root", + "metadata": { + "delivery_queue": { + "state": "dead_letter", + "label": "死信 1", + "reason": "当前存在 1 条死信记录", + "pending_count": 0, + "dead_letter_count": 1, + "oldest_dead_letter_at": "2026-04-18 09:01:00", + } + }, + "last_seen_at": "2026-04-18 09:05:00", + } + ] + + readiness = build_rollout_target_operational_readiness( + [{"node_code": "mainland-worker-03", "region": "mainland", "role": "worker"}], + execution_mode="remote-agent", + ) + + row = readiness["rows"][0] + self.assertEqual("dead_letter", row["delivery_queue_state"]) + self.assertEqual(1, row["delivery_queue_dead_letter_count"]) + self.assertTrue( + any("死信" in item for item in row["blocking_reasons"]) + ) + self.assertEqual(1, readiness["summary"]["queue_dead_letter_nodes"]) + self.assertTrue( + any("死信回执" in item for item in readiness["blocking_reasons"]) + ) + self.assertTrue( + any("死信队列" in item for item in readiness["recommendations"]) + ) + + @patch("app.services.ops_release_service.build_rollout_target_operational_readiness") + def test_build_release_rollout_gate_uses_execution_ready_count_for_ssh(self, mock_build_readiness) -> None: + mock_build_readiness.return_value = { + "execution_mode": "ssh", + "summary": { + "nodes_total": 2, + "execution_ready_nodes": 1, + "inspection_healthy_nodes": 2, + }, + "rows": [], + "blocking_reasons": ["有 1 台目标节点尚未配置完整 SSH 入口，当前不适合直接 SSH 发布。"], + "warning_reasons": [], + "recommendations": [], + } + + gate = build_release_rollout_gate( + { + "id": 11, + "release_version": "2026.04.18-rc1", + "status": "ready", + "artifact_url": "https://example.com/domaincheck.tar.gz", + }, + [{"node_code": "mainland-worker-01"}, {"node_code": "mainland-worker-02"}], + execution_mode="ssh", + ) + + self.assertEqual("ssh", gate["execution_mode"]) + self.assertEqual("SSH", gate["execution_mode_label"]) + self.assertEqual("blocked", gate["status"]) + self.assertIn("1/2 台 SSH 就绪", gate["summary"]) + + @patch("app.services.ops_release_service.build_rollout_target_operational_readiness") + def test_build_release_rollout_gate_uses_human_label_for_remote_agent(self, mock_build_readiness) -> None: + mock_build_readiness.return_value = { + "execution_mode": "remote-agent", + "summary": { + "nodes_total": 2, + "execution_ready_nodes": 1, + "inspection_healthy_nodes": 2, + }, + "rows": [], + "blocking_reasons": ["有 1 台目标节点 Agent 未在线，当前不适合直接 remote-agent 发布。"], + "warning_reasons": [], + "recommendations": [], + } + + gate = build_release_rollout_gate( + { + "id": 12, + "release_version": "2026.04.18-rc1", + "status": "ready", + "artifact_url": "https://example.com/domaincheck.tar.gz", + }, + [{"node_code": "mainland-worker-01"}, {"node_code": "mainland-worker-02"}], + execution_mode="remote-agent", + ) + + self.assertEqual("remote-agent", gate["execution_mode"]) + self.assertEqual("远端 Agent", gate["execution_mode_label"]) + self.assertIn("1/2 台 远端 Agent 就绪", gate["summary"]) + + @patch("app.services.ops_release_service.build_release_rollout_gate") + def test_build_release_execution_mode_recommendation_prefers_ssh_when_gate_is_better(self, mock_build_gate) -> None: + def _fake_gate(release, target_nodes, *, execution_mode="remote-agent"): + if execution_mode == "ssh": + return { + "status": "ready", + "status_label": "可发 Rollout", + "summary": "SSH 路径已收口", + "execution_mode": "ssh", + "execution_mode_label": "SSH", + "blocking_reasons": [], + "warning_reasons": [], + "operational_readiness": { + "summary": {"nodes_total": 2, "execution_ready_nodes": 2}, + "rows": [], + }, + } + return { + "status": "blocked", + "status_label": "存在阻断", + "summary": "remote-agent 尚未收口", + "execution_mode": "remote-agent", + "execution_mode_label": "远端 Agent", + "blocking_reasons": ["Agent 未就绪"], + "warning_reasons": [], + "operational_readiness": { + "summary": {"nodes_total": 2, "execution_ready_nodes": 1}, + "rows": [], + }, + } + + mock_build_gate.side_effect = _fake_gate + + recommendation = build_release_execution_mode_recommendation( + { + "id": 18, + "release_version": "2026.04.18-rc2", + "status": "ready", + "artifact_url": "https://example.com/domaincheck.tar.gz", + }, + [{"node_code": "mainland-worker-01"}, {"node_code": "mainland-worker-02"}], + ) + + self.assertEqual("ssh", recommendation["recommended_mode"]) + self.assertEqual("SSH", recommendation["recommended_mode_label"]) + self.assertEqual("ready", recommendation["recommended_gate"]["status"]) + self.assertIn("SSH", recommendation["reason"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/domain-api/tests/test_ops_release_service_launchpad.py b/domain-api/tests/test_ops_release_service_launchpad.py new file mode 100644 index 0000000..97d1ca6 --- /dev/null +++ b/domain-api/tests/test_ops_release_service_launchpad.py @@ -0,0 +1,212 @@ +import unittest + +from app.services.ops_release_service import ( + _build_release_launchpad_status, + _compact_latest_smart_rollout_preview, +) + + +class OpsReleaseServiceLaunchpadTests(unittest.TestCase): + def test_launchpad_status_blocks_when_package_missing(self) -> None: + status = _build_release_launchpad_status( + package={"available": False}, + latest_package_preview={}, + worker_preview={}, + control_preview={}, + ) + + self.assertEqual("blocked", status["status"]) + self.assertEqual("release_package", status["recommended_action_code"]) + self.assertEqual(status["summary"], status["summary_text"]) + self.assertEqual("release_hub", status["focus_ref"]["kind"]) + self.assertEqual("release_launchpad", status["focus_ref"]["section"]) + + def test_launchpad_status_blocks_when_rollout_preview_has_blockers(self) -> None: + status = _build_release_launchpad_status( + package={"available": True}, + latest_package_preview={}, + worker_preview={"policy_preview": {"blocked": True}}, + control_preview={"policy_preview": {"blocked": False}}, + ) + + self.assertEqual("blocked", status["status"]) + self.assertEqual("fix_rollout_blockers", status["recommended_action_code"]) + self.assertEqual(status["summary"], status["summary_text"]) + + def test_launchpad_status_blocks_when_final_signoff_missing(self) -> None: + status = _build_release_launchpad_status( + package={ + "available": True, + "final_release_ok": False, + "final_release_report_available": True, + "final_release_report_ok": True, + "final_release_report_matches_latest": False, + "final_release_report_stale_reason": "final_release_report does not match latest_release: package_name", + "final_release_gate_decision": "ready", + "final_release_gate_blocked_reasons": [], + }, + latest_package_preview={}, + worker_preview={"smart_rollout": {"available": True}, "policy_preview": {}}, + control_preview={"smart_rollout": {"available": True}, "policy_preview": {}}, + ) + + self.assertEqual("blocked", status["status"]) + self.assertEqual("release_prepare", status["recommended_action_code"]) + self.assertIn("final_release_report does not match latest_release: package_name", status["summary"]) + self.assertIn("release-prepare", status["recommended_command"]) + self.assertEqual(status["summary"], status["summary_text"]) + + def test_launchpad_status_prefers_runtime_refresh_when_runtime_schema_is_stale(self) -> None: + status = _build_release_launchpad_status( + package={"available": True, "final_release_ok": True}, + latest_package_preview={}, + worker_preview={"smart_rollout": {"available": True}, "policy_preview": {}}, + control_preview={"smart_rollout": {"available": True}, "policy_preview": {}}, + runtime_build_info={ + "runtime_schema_stale": True, + "route_surface_complete": True, + "route_surface_missing_keys": ["ops_node_handover_bootstrap_plan"], + }, + ) + + self.assertEqual("attention", status["status"]) + self.assertEqual("api-restart", status["recommended_action_code"]) + self.assertEqual("待刷新运行时", status["status_label"]) + self.assertTrue(status["runtime_refresh_required"]) + self.assertTrue(status["runtime_schema_stale"]) + self.assertIn("runtime-refresh-recover", status["recommended_command"]) + self.assertEqual(status["summary"], status["summary_text"]) + + def test_launchpad_status_marks_attention_when_confirmation_required(self) -> None: + status = _build_release_launchpad_status( + package={"available": True}, + latest_package_preview={}, + worker_preview={"requires_confirmation": True, "policy_preview": {}}, + control_preview={"policy_preview": {}}, + ) + + self.assertEqual("attention", status["status"]) + self.assertEqual("review_smart_rollout_preview", status["recommended_action_code"]) + self.assertEqual("release_launchpad", status["focus_ref"]["section"]) + + def test_launchpad_status_ready_when_worker_rollout_available(self) -> None: + status = _build_release_launchpad_status( + package={"available": True}, + latest_package_preview={}, + worker_preview={"smart_rollout": {"available": True, "execution_mode": "ssh", "execution_mode_label": "SSH"}, "policy_preview": {}}, + control_preview={"smart_rollout": {"available": False}, "policy_preview": {}}, + ) + + self.assertEqual("ready", status["status"]) + self.assertEqual("publish_latest_worker", status["recommended_action_code"]) + self.assertEqual("ssh", status["recommended_execution_mode"]) + self.assertEqual("SSH", status["recommended_execution_mode_label"]) + self.assertEqual(status["summary"], status["summary_text"]) + + def test_launchpad_status_requires_managed_nodes_when_no_targets_available(self) -> None: + status = _build_release_launchpad_status( + package={"available": True}, + latest_package_preview={}, + worker_preview={"smart_rollout": {"available": False}, "policy_preview": {}}, + control_preview={"smart_rollout": {"available": False}, "policy_preview": {}}, + ) + + self.assertEqual("attention", status["status"]) + self.assertEqual("fix_managed_nodes", status["recommended_action_code"]) + self.assertEqual("release_hub", status["focus_ref"]["kind"]) + + def test_launchpad_status_prefers_bootstrap_recovery_when_gap_row_has_recovery_action(self) -> None: + status = _build_release_launchpad_status( + package={"available": True}, + latest_package_preview={}, + worker_preview={ + "smart_rollout": {"available": False}, + "policy_preview": { + "blocked": True, + "release_gate": { + "operational_readiness": { + "rows": [ + { + "node_code": "mainland-worker-02", + "recovery_action": "bootstrap_run", + "recovery_label": "签发接入工单", + "recovery_summary": "当前节点仍处于接入阶段，建议先签发 onboarding.bootstrap。", + } + ] + } + }, + }, + }, + control_preview={"smart_rollout": {"available": False}, "policy_preview": {}}, + ) + + self.assertEqual("blocked", status["status"]) + self.assertEqual("bootstrap_run", status["recommended_action_code"]) + self.assertEqual("mainland-worker-02", status["recommended_target_node_code"]) + self.assertEqual("签发接入工单", status["recommended_recovery_label"]) + self.assertEqual("当前节点仍处于接入阶段，建议先签发 onboarding.bootstrap。", status["recommended_recovery_summary"]) + self.assertEqual(1, status["onboarding_bootstrap_pending_nodes"]) + self.assertEqual(0, status["onboarding_acceptance_ready_nodes"]) + self.assertEqual(1, status["onboarding_gap_nodes_total"]) + self.assertIn("node-bootstrap-run", status["recommended_command"]) + + def test_compact_latest_smart_rollout_preview_adds_rollout_preview_alias(self) -> None: + preview = _compact_latest_smart_rollout_preview( + "智能 Rollout 预检通过", + { + "release": { + "id": 18, + "release_version": "2026.04.18+bd6bcb2", + "channel": "stable", + "status": "ready", + "artifact_url": "https://example.com/domaincheck.tar.gz", + }, + "package": { + "available": True, + "package_name": "domaincheck-2026.04.18+bd6bcb2", + "package_type": "tar.gz", + "sha256": "sha256:test", + "metadata": { + "manifest": { + "included_services": ["domaincheck-api", "domaincheck-worker"], + } + }, + }, + "smart_rollout": { + "available": True, + "execution_mode": "remote-agent", + "execution_mode_label": "远端 Agent", + "summary": "已按 Worker 预选 2 台节点。", + "target_selector": {"node_codes": ["mainland-worker-01", "mainland-worker-02"]}, + "target_nodes": [ + { + "node_code": "mainland-worker-01", + "region": "mainland", + "role": "worker", + "delivery_queue_state": "healthy", + }, + { + "node_code": "mainland-worker-02", + "region": "mainland", + "role": "worker", + "delivery_queue_state": "retrying", + }, + ], + }, + "preview": { + "target_summary": {"nodes_total": 2}, + "batch_plan": {"batches_total": 2, "first_batch_size": 1}, + "release_gate": {"status": "ready", "summary": "当前可推进。"}, + }, + }, + ) + + self.assertEqual("remote-agent", preview["rollout_preview"]["execution_mode"]) + self.assertEqual(2, preview["rollout_preview"]["target_nodes_total"]) + self.assertEqual(2, preview["rollout_preview"]["expected_batches_total"]) + self.assertEqual(2, preview["rollout_preview"]["expected_jobs_total"]) + self.assertEqual("release_hub", preview["rollout_preview"]["focus_ref"]["kind"]) + self.assertEqual("healthy", preview["smart_rollout"]["target_nodes"][0]["delivery_queue_state"]) + self.assertEqual("retrying", preview["smart_rollout"]["target_nodes"][1]["delivery_queue_state"]) + self.assertEqual("rollout_preview", preview["rollout_preview"]["focus_ref"]["section"]) + self.assertEqual(["domaincheck-api", "domaincheck-worker"], preview["package"]["manifest"]["included_services"]) diff --git a/domain-api/tests/test_ops_release_service_package_metadata.py b/domain-api/tests/test_ops_release_service_package_metadata.py new file mode 100644 index 0000000..cc2cfc9 --- /dev/null +++ b/domain-api/tests/test_ops_release_service_package_metadata.py @@ -0,0 +1,915 @@ +import json +import os +import tempfile +import unittest +from pathlib import Path +from unittest.mock import patch + +from app.services.ops_release_service import ( + build_smart_release_rollout_plan, + create_release_and_smart_rollout_from_latest_package, + create_release_from_latest_package, + create_smart_release_rollout, + get_latest_release_package_metadata, + get_release_package_archive_path, + get_release_package_sha256_path, +) + + +class OpsReleaseServicePackageMetadataTests(unittest.TestCase): + def test_get_latest_release_package_metadata_reads_latest_linux_package(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + release_root = Path(tmpdir) + package_name = "domaincheck_release_20260418_120000" + staging_path = release_root / package_name + archive_path = release_root / f"{package_name}.tar.gz" + sha256_path = release_root / f"{package_name}.sha256.txt" + + staging_path.mkdir(parents=True, exist_ok=True) + archive_path.write_bytes(b"linux-package") + sha256_path.write_text("sha256=test-sha256", encoding="utf-8") + (staging_path / "release_manifest.json").write_text( + json.dumps( + { + "package_name": package_name, + "package_type": "tar.gz", + "included": { + "docs": ["08_domainCheck_交付说明.md"], + "scripts": ["package_domain_release.sh", "verify_domain_release.sh"], + }, + }, + ensure_ascii=False, + ), + encoding="utf-8", + ) + (release_root / "latest_release.json").write_text( + json.dumps( + { + "package_name": package_name, + "package_type": "tar.gz", + "generated_at": "2026-04-18T12:00:00", + "staging_path": str(staging_path), + "archive_path": str(archive_path), + "sha256_path": str(sha256_path), + "sha256": "test-sha256", + "smoke_test_ok": True, + }, + ensure_ascii=False, + ), + encoding="utf-8", + ) + (release_root / "final_release_report.json").write_text( + json.dumps( + { + "ok": True, + "latest_release": { + "package_name": package_name, + "archive_path": str(archive_path), + "sha256": "test-sha256", + }, + "release_gate": { + "decision": "ready", + "blocked_reasons": [], + "verify_ok": True, + "smoke_test_ok": True, + }, + }, + ensure_ascii=False, + ), + encoding="utf-8", + ) + + with patch.dict(os.environ, {"DOMAINCHECK_RELEASE_ROOT": str(release_root)}): + data = get_latest_release_package_metadata() + + self.assertTrue(data["available"]) + self.assertEqual(package_name, data["package_name"]) + self.assertEqual("tar.gz", data["package_type"]) + self.assertTrue(data["archive_present"]) + self.assertTrue(data["sha256_present"]) + self.assertTrue(data["staging_present"]) + self.assertEqual("test-sha256", data["sha256"]) + self.assertEqual(f"/api/v1/ops/releases/packages/{package_name}/download", data["download_path"]) + self.assertEqual(package_name, data["release_version_suggestion"]) + self.assertIn("自检通过", data["notes_suggestion"]) + self.assertTrue(data["final_release_ok"]) + self.assertTrue(data["final_release_report_available"]) + self.assertTrue(data["final_release_report_matches_latest"]) + self.assertEqual("ready", data["final_release_gate_decision"]) + self.assertEqual([], data["final_release_gate_blocked_reasons"]) + self.assertEqual(package_name, data["metadata"]["manifest"]["package_name"]) + + def test_get_release_package_paths_support_metadata_fallback_for_zip(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + release_root = Path(tmpdir) + package_name = "domaincheck_release_windows_demo" + package_dir = release_root / "exports" + package_dir.mkdir(parents=True, exist_ok=True) + archive_path = package_dir / f"{package_name}.zip" + sha256_path = package_dir / f"{package_name}.sha256.txt" + + archive_path.write_bytes(b"zip-package") + sha256_path.write_text("sha256=zip-sha256", encoding="utf-8") + (release_root / "latest_release.json").write_text( + json.dumps( + { + "package_name": package_name, + "package_type": "zip", + "generated_at": "2026-04-18T12:10:00", + "zip_path": str(archive_path), + "sha256_path": str(sha256_path), + "sha256": "zip-sha256", + "smoke_test_ok": False, + }, + ensure_ascii=False, + ), + encoding="utf-8", + ) + + with patch.dict(os.environ, {"DOMAINCHECK_RELEASE_ROOT": str(release_root)}): + resolved_archive = get_release_package_archive_path(package_name) + resolved_sha256 = get_release_package_sha256_path(package_name) + metadata = get_latest_release_package_metadata() + + self.assertEqual(archive_path, resolved_archive) + self.assertEqual(sha256_path, resolved_sha256) + self.assertEqual("zip", metadata["package_type"]) + self.assertEqual(archive_path.name, metadata["archive_filename"]) + self.assertFalse(metadata["smoke_test_ok"]) + + def test_get_latest_release_package_metadata_returns_unavailable_when_missing(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + with patch.dict(os.environ, {"DOMAINCHECK_RELEASE_ROOT": tmpdir}): + data = get_latest_release_package_metadata() + + self.assertFalse(data["available"]) + self.assertIn("latest_release.json", data["reason"]) + + def test_get_latest_release_package_metadata_marks_stale_final_release_report(self) -> None: + with tempfile.TemporaryDirectory() as tmpdir: + release_root = Path(tmpdir) + package_name = "domaincheck_release_20260418_130000" + archive_path = release_root / f"{package_name}.tar.gz" + sha256_path = release_root / f"{package_name}.sha256.txt" + + archive_path.write_bytes(b"linux-package") + sha256_path.write_text("sha256=test-sha256", encoding="utf-8") + (release_root / "latest_release.json").write_text( + json.dumps( + { + "package_name": package_name, + "package_type": "tar.gz", + "archive_path": str(archive_path), + "sha256_path": str(sha256_path), + "sha256": "test-sha256", + "smoke_test_ok": True, + }, + ensure_ascii=False, + ), + encoding="utf-8", + ) + (release_root / "final_release_report.json").write_text( + json.dumps( + { + "ok": True, + "latest_release": { + "package_name": "domaincheck_release_20260418_old", + "archive_path": str(release_root / "domaincheck_release_20260418_old.tar.gz"), + "sha256": "old-sha256", + }, + "release_gate": { + "decision": "ready", + "blocked_reasons": [], + "verify_ok": True, + "smoke_test_ok": True, + }, + }, + ensure_ascii=False, + ), + encoding="utf-8", + ) + + with patch.dict(os.environ, {"DOMAINCHECK_RELEASE_ROOT": str(release_root)}): + data = get_latest_release_package_metadata() + + self.assertTrue(data["available"]) + self.assertFalse(data["final_release_ok"]) + self.assertTrue(data["final_release_report_available"]) + self.assertTrue(data["final_release_report_ok"]) + self.assertFalse(data["final_release_report_matches_latest"]) + self.assertIn("package_name", data["final_release_report_stale_reason"]) + + @patch("app.services.ops_release_service.create_release") + @patch("app.services.ops_release_service._get_release_by_version") + @patch("app.services.ops_release_service.get_latest_release_package_metadata") + def test_create_release_from_latest_package_builds_payload( + self, + mock_get_latest_release_package_metadata, + mock_get_release_by_version, + mock_create_release, + ) -> None: + mock_get_latest_release_package_metadata.return_value = { + "available": True, + "package_name": "domaincheck_release_20260418_120000", + "release_version_suggestion": "domaincheck_release_20260418_120000", + "download_path": "/api/v1/ops/releases/packages/domaincheck_release_20260418_120000/download", + "sha256_download_path": "/api/v1/ops/releases/packages/domaincheck_release_20260418_120000/sha256", + "sha256": "test-sha256", + "commit_sha": "bd6bcb2", + "notes_suggestion": "导入自本机发布包 domaincheck_release_20260418_120000.tar.gz；自检通过", + "smoke_test_ok": True, + "final_release_ok": True, + "final_release_report_available": True, + "final_release_report_ok": True, + "final_release_report_matches_latest": True, + "final_release_gate_decision": "ready", + "final_release_gate_blocked_reasons": [], + "metadata": { + "package_name": "domaincheck_release_20260418_120000", + "package_type": "tar.gz", + }, + } + mock_get_release_by_version.return_value = {} + mock_create_release.return_value = ( + True, + "Release 已创建", + { + "release": { + "id": 7, + "release_version": "domaincheck_release_20260418_120000", + } + }, + ) + + ok, message, data = create_release_from_latest_package( + { + "channel": "stable", + "status": "ready", + "created_by": "test-suite", + }, + control_plane_base_url="https://ops.example.com/api/v1", + ) + + self.assertTrue(ok) + self.assertEqual("Release 已创建", message) + mock_create_release.assert_called_once() + create_payload = mock_create_release.call_args.args[0] + self.assertEqual("domaincheck_release_20260418_120000", create_payload["release_version"]) + self.assertEqual("bd6bcb2", create_payload["commit_sha"]) + self.assertEqual( + "https://ops.example.com/api/v1/ops/releases/packages/domaincheck_release_20260418_120000/download", + create_payload["artifact_url"], + ) + self.assertEqual("test-sha256", create_payload["checksum"]) + self.assertEqual("latest_release_package", create_payload["metadata"]["created_from"]) + self.assertEqual( + "https://ops.example.com/api/v1/ops/releases/packages/domaincheck_release_20260418_120000/download", + create_payload["metadata"]["package_source"]["download_url"], + ) + self.assertFalse(data["deduplicated"]) + + @patch("app.services.ops_release_service.create_release") + @patch("app.services.ops_release_service._get_release_by_version") + @patch("app.services.ops_release_service.get_latest_release_package_metadata") + def test_create_release_from_latest_package_is_idempotent( + self, + mock_get_latest_release_package_metadata, + mock_get_release_by_version, + mock_create_release, + ) -> None: + mock_get_latest_release_package_metadata.return_value = { + "available": True, + "package_name": "domaincheck_release_20260418_120000", + "release_version_suggestion": "domaincheck_release_20260418_120000", + "smoke_test_ok": True, + "final_release_ok": True, + "final_release_report_available": True, + "final_release_report_ok": True, + "final_release_report_matches_latest": True, + "final_release_gate_decision": "ready", + "final_release_gate_blocked_reasons": [], + } + mock_get_release_by_version.return_value = { + "id": 9, + "release_version": "domaincheck_release_20260418_120000", + } + + ok, message, data = create_release_from_latest_package({}, control_plane_base_url="https://ops.example.com") + + self.assertTrue(ok) + self.assertEqual("Release 已存在，按幂等返回", message) + self.assertTrue(data["deduplicated"]) + self.assertEqual(9, data["release"]["id"]) + mock_create_release.assert_not_called() + + @patch("app.services.ops_release_service.create_release") + @patch("app.services.ops_release_service._get_release_by_version") + @patch("app.services.ops_release_service.get_latest_release_package_metadata") + def test_create_release_from_latest_package_blocks_when_final_signoff_missing( + self, + mock_get_latest_release_package_metadata, + mock_get_release_by_version, + mock_create_release, + ) -> None: + mock_get_latest_release_package_metadata.return_value = { + "available": True, + "package_name": "domaincheck_release_20260418_130000", + "release_version_suggestion": "domaincheck_release_20260418_130000", + "download_path": "/api/v1/ops/releases/packages/domaincheck_release_20260418_130000/download", + "sha256_download_path": "/api/v1/ops/releases/packages/domaincheck_release_20260418_130000/sha256", + "sha256": "test-sha256", + "commit_sha": "bd6bcb2", + "smoke_test_ok": False, + "final_release_ok": False, + "final_release_report_available": False, + "final_release_report_ok": None, + "final_release_report_matches_latest": None, + "final_release_gate_decision": "", + "final_release_gate_blocked_reasons": [], + "metadata": { + "package_name": "domaincheck_release_20260418_130000", + "package_type": "tar.gz", + }, + } + mock_get_release_by_version.return_value = {} + + ok, message, data = create_release_from_latest_package( + { + "channel": "stable", + "status": "ready", + "created_by": "test-suite", + }, + control_plane_base_url="https://ops.example.com/api/v1", + ) + + self.assertFalse(ok) + self.assertIn("最终签收", message) + self.assertFalse(data["final_release_gate"]["ready"]) + self.assertIn("smoke_test_failed_or_skipped", data["final_release_gate"]["blocked_reasons"]) + self.assertIn("final_release_report_missing", data["final_release_gate"]["blocked_reasons"]) + mock_create_release.assert_not_called() + + @patch("app.services.ops_release_service.create_release") + @patch("app.services.ops_release_service._get_release_by_version") + @patch("app.services.ops_release_service.get_latest_release_package_metadata") + def test_create_release_from_latest_package_blocks_when_final_report_is_stale( + self, + mock_get_latest_release_package_metadata, + mock_get_release_by_version, + mock_create_release, + ) -> None: + mock_get_latest_release_package_metadata.return_value = { + "available": True, + "package_name": "domaincheck_release_20260418_130001", + "release_version_suggestion": "domaincheck_release_20260418_130001", + "download_path": "/api/v1/ops/releases/packages/domaincheck_release_20260418_130001/download", + "sha256_download_path": "/api/v1/ops/releases/packages/domaincheck_release_20260418_130001/sha256", + "sha256": "test-sha256", + "commit_sha": "bd6bcb2", + "smoke_test_ok": True, + "final_release_ok": False, + "final_release_report_available": True, + "final_release_report_ok": True, + "final_release_report_matches_latest": False, + "final_release_report_stale_reason": "final_release_report does not match latest_release: package_name", + "final_release_gate_decision": "ready", + "final_release_gate_blocked_reasons": [], + "metadata": { + "package_name": "domaincheck_release_20260418_130001", + "package_type": "tar.gz", + }, + } + mock_get_release_by_version.return_value = { + "id": 19, + "release_version": "domaincheck_release_20260418_130001", + } + + ok, message, data = create_release_from_latest_package( + {"created_by": "test-suite"}, + control_plane_base_url="https://ops.example.com/api/v1", + ) + + self.assertFalse(ok) + self.assertIn("最终签收", message) + self.assertFalse(data["final_release_gate"]["ready"]) + self.assertEqual( + "final_release_report does not match latest_release: package_name", + data["final_release_gate"]["stale_reason"], + ) + self.assertIn("final_release_report_not_matching_latest", data["final_release_gate"]["blocked_reasons"]) + self.assertEqual(19, data["release"]["id"]) + mock_create_release.assert_not_called() + + @patch("app.services.ops_agent_service.list_managed_nodes_with_agent_state") + def test_build_smart_release_rollout_plan_prefers_remote_ready_workers( + self, + mock_list_managed_nodes_with_agent_state, + ) -> None: + mock_list_managed_nodes_with_agent_state.return_value = { + "nodes": [ + { + "node_code": "mainland-worker-01", + "region": "mainland", + "role": "worker", + "is_enabled": True, + "is_managed": True, + "is_agent_online": True, + "cluster_status": "online", + "cluster_current_load": 0, + "cluster_is_effective_worker": True, + "cluster_detect_participating": False, + }, + { + "node_code": "mainland-controller-01", + "region": "mainland", + "role": "control", + "is_enabled": True, + "is_managed": True, + "is_agent_online": True, + "cluster_status": "online", + "cluster_current_load": 1, + "cluster_is_effective_worker": True, + "cluster_detect_participating": True, + }, + { + "node_code": "mainland-worker-02", + "region": "mainland", + "role": "worker", + "is_enabled": True, + "is_managed": True, + "is_agent_online": False, + "cluster_status": "online", + "cluster_current_load": 0, + "cluster_is_effective_worker": True, + "cluster_detect_participating": False, + }, + ] + } + + worker_plan = build_smart_release_rollout_plan("worker") + control_plan = build_smart_release_rollout_plan("control") + + self.assertTrue(worker_plan["available"]) + self.assertEqual("worker", worker_plan["mode"]) + self.assertEqual(["mainland-worker-01"], worker_plan["target_selector"]["node_codes"]) + self.assertEqual(["domaincheck-worker"], worker_plan["policy"]["deploy_payload"]["restart_services"]) + + self.assertTrue(control_plan["available"]) + self.assertEqual("control", control_plan["mode"]) + self.assertEqual(["mainland-controller-01"], control_plan["target_selector"]["node_codes"]) + self.assertIn("domaincheck-api", control_plan["policy"]["deploy_payload"]["restart_services"]) + self.assertEqual("remote-agent", worker_plan["policy"]["execution_mode"]) + + @patch("app.services.ops_agent_service.list_managed_nodes_with_agent_state") + def test_build_smart_release_rollout_plan_falls_back_to_ssh_ready_workers( + self, + mock_list_managed_nodes_with_agent_state, + ) -> None: + mock_list_managed_nodes_with_agent_state.return_value = { + "nodes": [ + { + "node_code": "mainland-worker-01", + "region": "mainland", + "role": "worker", + "is_enabled": True, + "is_managed": True, + "is_agent_online": False, + "has_ssh_access": True, + "cluster_status": "online", + "cluster_current_load": 0, + "cluster_is_effective_worker": True, + "cluster_detect_participating": False, + }, + { + "node_code": "mainland-worker-02", + "region": "mainland", + "role": "worker", + "is_enabled": True, + "is_managed": True, + "is_agent_online": False, + "has_ssh_access": True, + "cluster_status": "busy", + "cluster_current_load": 1, + "cluster_is_effective_worker": True, + "cluster_detect_participating": True, + }, + ] + } + + worker_plan = build_smart_release_rollout_plan("worker") + + self.assertTrue(worker_plan["available"]) + self.assertEqual("worker", worker_plan["mode"]) + self.assertEqual("ssh", worker_plan["execution_mode"]) + self.assertEqual("SSH", worker_plan["execution_mode_label"]) + self.assertEqual( + ["mainland-worker-01", "mainland-worker-02"], + worker_plan["target_selector"]["node_codes"], + ) + self.assertEqual("ssh", worker_plan["policy"]["execution_mode"]) + + @patch("app.services.ops_agent_service.list_managed_nodes_with_agent_state") + def test_build_smart_release_rollout_plan_excludes_dead_letter_nodes_from_ready_candidates( + self, + mock_list_managed_nodes_with_agent_state, + ) -> None: + mock_list_managed_nodes_with_agent_state.return_value = { + "nodes": [ + { + "node_code": "mainland-worker-01", + "region": "mainland", + "role": "worker", + "is_enabled": True, + "is_managed": True, + "is_agent_online": True, + "has_ssh_access": True, + "delivery_queue_state": "dead_letter", + "cluster_status": "online", + "cluster_current_load": 0, + "cluster_is_effective_worker": True, + "cluster_detect_participating": False, + }, + { + "node_code": "mainland-worker-02", + "region": "mainland", + "role": "worker", + "is_enabled": True, + "is_managed": True, + "is_agent_online": True, + "has_ssh_access": True, + "delivery_queue_state": "healthy", + "cluster_status": "online", + "cluster_current_load": 0, + "cluster_is_effective_worker": True, + "cluster_detect_participating": False, + }, + ] + } + + worker_plan = build_smart_release_rollout_plan("worker") + + self.assertTrue(worker_plan["available"]) + self.assertEqual(["mainland-worker-02"], worker_plan["target_selector"]["node_codes"]) + self.assertEqual("healthy", worker_plan["target_nodes"][0]["delivery_queue_state"]) + + @patch("app.services.ops_agent_service.list_managed_nodes_with_agent_state") + def test_build_smart_release_rollout_plan_returns_unavailable_when_only_dead_letter_nodes_exist( + self, + mock_list_managed_nodes_with_agent_state, + ) -> None: + mock_list_managed_nodes_with_agent_state.return_value = { + "nodes": [ + { + "node_code": "mainland-worker-01", + "region": "mainland", + "role": "worker", + "is_enabled": True, + "is_managed": True, + "is_agent_online": True, + "has_ssh_access": True, + "delivery_queue_state": "dead_letter", + "cluster_status": "online", + "cluster_current_load": 0, + "cluster_is_effective_worker": True, + "cluster_detect_participating": False, + } + ] + } + + worker_plan = build_smart_release_rollout_plan("worker") + + self.assertFalse(worker_plan["available"]) + self.assertIn("死信回执", worker_plan["reason"]) + self.assertEqual("dead_letter", worker_plan["target_nodes"][0]["delivery_queue_state"]) + + @patch("app.services.ops_release_service.create_release_rollout") + @patch("app.services.ops_policy_service.preview_ops_job_policy") + @patch("app.services.ops_agent_service.list_managed_nodes_with_agent_state") + @patch("app.services.ops_release_service.create_release_from_latest_package") + def test_create_release_and_smart_rollout_stops_when_latest_package_gate_blocks( + self, + mock_create_release_from_latest_package, + mock_list_managed_nodes_with_agent_state, + mock_preview_ops_job_policy, + mock_create_release_rollout, + ) -> None: + mock_create_release_from_latest_package.return_value = ( + False, + "最新发布包尚未完成最终签收，暂不允许创建 Release 或发起基于最新包的智能 Rollout。", + { + "package": { + "available": True, + "package_name": "domaincheck_release_20260418_blocked", + "final_release_ok": False, + }, + "final_release_gate": { + "ready": False, + "status": "blocked", + "blocked_reasons": ["final_release_report_missing"], + }, + }, + ) + + ok, message, data = create_release_and_smart_rollout_from_latest_package( + {"mode": "worker"}, + control_plane_base_url="https://ops.example.com", + ) + + self.assertFalse(ok) + self.assertIn("最终签收", message) + self.assertFalse(data["final_release_gate"]["ready"]) + mock_list_managed_nodes_with_agent_state.assert_not_called() + mock_preview_ops_job_policy.assert_not_called() + mock_create_release_rollout.assert_not_called() + + @patch("app.services.ops_release_service.create_release_rollout") + @patch("app.services.ops_policy_service.preview_ops_job_policy") + @patch("app.services.ops_agent_service.list_managed_nodes_with_agent_state") + @patch("app.services.ops_release_service.create_release_from_latest_package") + def test_create_release_and_smart_rollout_requires_confirmation_before_rollout( + self, + mock_create_release_from_latest_package, + mock_list_managed_nodes_with_agent_state, + mock_preview_ops_job_policy, + mock_create_release_rollout, + ) -> None: + mock_create_release_from_latest_package.return_value = ( + True, + "Release 已创建", + { + "release": {"id": 21, "release_version": "domaincheck_release_20260418_120000"}, + "deduplicated": False, + }, + ) + mock_list_managed_nodes_with_agent_state.return_value = { + "nodes": [ + { + "node_code": "mainland-controller-01", + "region": "mainland", + "role": "control", + "is_enabled": True, + "is_managed": True, + "is_agent_online": True, + "cluster_status": "online", + "cluster_current_load": 0, + "cluster_is_effective_worker": True, + "cluster_detect_participating": False, + } + ] + } + mock_preview_ops_job_policy.return_value = { + "blocked": False, + "approval_required": True, + "warnings": ["本次 rollout 覆盖 control 节点，建议审批。"], + "release_gate": {"status_label": "待补巡检", "summary": "建议先确认控制面健康。"}, + } + + ok, message, data = create_release_and_smart_rollout_from_latest_package( + {"mode": "control"}, + control_plane_base_url="https://ops.example.com", + ) + + self.assertTrue(ok) + self.assertEqual("Release 已创建，智能 Rollout 需要确认", message) + self.assertTrue(data["requires_confirmation"]) + self.assertFalse(data["rollout_created"]) + self.assertEqual(["mainland-controller-01"], data["smart_rollout"]["target_selector"]["node_codes"]) + mock_create_release_rollout.assert_not_called() + + @patch("app.services.ops_release_service.create_release_rollout") + @patch("app.services.ops_policy_service.preview_ops_job_policy") + @patch("app.services.ops_agent_service.list_managed_nodes_with_agent_state") + @patch("app.services.ops_release_service.create_release_from_latest_package") + def test_create_release_and_smart_rollout_creates_rollout_after_confirmation( + self, + mock_create_release_from_latest_package, + mock_list_managed_nodes_with_agent_state, + mock_preview_ops_job_policy, + mock_create_release_rollout, + ) -> None: + mock_create_release_from_latest_package.return_value = ( + True, + "Release 已创建", + { + "release": {"id": 22, "release_version": "domaincheck_release_20260418_120000"}, + "deduplicated": True, + }, + ) + mock_list_managed_nodes_with_agent_state.return_value = { + "nodes": [ + { + "node_code": "mainland-worker-01", + "region": "mainland", + "role": "worker", + "is_enabled": True, + "is_managed": True, + "is_agent_online": True, + "cluster_status": "online", + "cluster_current_load": 0, + "cluster_is_effective_worker": True, + "cluster_detect_participating": False, + } + ] + } + mock_preview_ops_job_policy.return_value = { + "blocked": False, + "approval_required": False, + "warnings": [], + "release_gate": {"status_label": "可发 Rollout", "summary": "当前可推进。"}, + } + mock_create_release_rollout.return_value = ( + True, + "Release rollout 已创建", + { + "rollout": {"id": 5, "release_id": 22, "rollout_code": "rollout-20260418-demo"}, + "jobs": [], + }, + ) + + ok, message, data = create_release_and_smart_rollout_from_latest_package( + {"mode": "worker", "confirm_risky": True}, + control_plane_base_url="https://ops.example.com", + ) + + self.assertTrue(ok) + self.assertEqual("Release rollout 已创建", message) + self.assertTrue(data["rollout_created"]) + self.assertEqual(5, data["rollout"]["id"]) + mock_create_release_rollout.assert_called_once() + rollout_payload = mock_create_release_rollout.call_args.args[1] + self.assertEqual(["mainland-worker-01"], rollout_payload["target_selector"]["node_codes"]) + self.assertEqual("remote-agent", rollout_payload["policy"]["execution_mode"]) + + @patch("app.services.ops_release_service.create_release_rollout") + @patch("app.services.ops_policy_service.preview_ops_job_policy") + @patch("app.services.ops_agent_service.list_managed_nodes_with_agent_state") + @patch("app.services.ops_release_service.create_release_from_latest_package") + def test_create_release_and_smart_rollout_uses_ssh_when_only_ssh_path_is_ready( + self, + mock_create_release_from_latest_package, + mock_list_managed_nodes_with_agent_state, + mock_preview_ops_job_policy, + mock_create_release_rollout, + ) -> None: + mock_create_release_from_latest_package.return_value = ( + True, + "Release 已创建", + { + "release": {"id": 23, "release_version": "domaincheck_release_20260418_ssh"}, + "deduplicated": True, + }, + ) + mock_list_managed_nodes_with_agent_state.return_value = { + "nodes": [ + { + "node_code": "mainland-worker-01", + "region": "mainland", + "role": "worker", + "is_enabled": True, + "is_managed": True, + "is_agent_online": False, + "has_ssh_access": True, + "cluster_status": "online", + "cluster_current_load": 0, + "cluster_is_effective_worker": True, + "cluster_detect_participating": False, + } + ] + } + mock_preview_ops_job_policy.return_value = { + "blocked": False, + "approval_required": False, + "warnings": [], + "release_gate": {"status_label": "可发 Rollout", "summary": "当前可推进。"}, + } + mock_create_release_rollout.return_value = ( + True, + "Release rollout 已创建", + { + "rollout": {"id": 6, "release_id": 23, "rollout_code": "rollout-20260418-ssh"}, + "jobs": [], + }, + ) + + ok, message, data = create_release_and_smart_rollout_from_latest_package( + {"mode": "worker", "confirm_risky": True}, + control_plane_base_url="https://ops.example.com", + ) + + self.assertTrue(ok) + self.assertEqual("Release rollout 已创建", message) + self.assertTrue(data["rollout_created"]) + rollout_payload = mock_create_release_rollout.call_args.args[1] + self.assertEqual("ssh", rollout_payload["policy"]["execution_mode"]) + + @patch("app.services.ops_release_service.get_release") + @patch("app.services.ops_release_service.create_release_rollout") + @patch("app.services.ops_policy_service.preview_ops_job_policy") + @patch("app.services.ops_agent_service.list_managed_nodes_with_agent_state") + def test_create_smart_release_rollout_for_existing_release_requires_confirmation( + self, + mock_list_managed_nodes_with_agent_state, + mock_preview_ops_job_policy, + mock_create_release_rollout, + mock_get_release, + ) -> None: + mock_get_release.return_value = { + "id": 41, + "release_version": "domaincheck_release_existing", + "channel": "stable", + } + mock_list_managed_nodes_with_agent_state.return_value = { + "nodes": [ + { + "node_code": "mainland-controller-01", + "region": "mainland", + "role": "control", + "is_enabled": True, + "is_managed": True, + "is_agent_online": True, + "cluster_status": "online", + "cluster_current_load": 0, + "cluster_is_effective_worker": True, + "cluster_detect_participating": False, + } + ] + } + mock_preview_ops_job_policy.return_value = { + "blocked": False, + "approval_required": True, + "warnings": ["控制面节点建议审批。"], + "release_gate": {"status_label": "待审批", "summary": "需要审批。"}, + } + + ok, message, data = create_smart_release_rollout(41, {"mode": "control"}) + + self.assertTrue(ok) + self.assertEqual("智能 Rollout 需要确认", message) + self.assertFalse(data["release_created"]) + self.assertTrue(data["requires_confirmation"]) + self.assertFalse(data["rollout_created"]) + self.assertEqual(["mainland-controller-01"], data["smart_rollout"]["target_selector"]["node_codes"]) + mock_create_release_rollout.assert_not_called() + + @patch("app.services.ops_release_service.get_release") + @patch("app.services.ops_release_service.create_release_rollout") + @patch("app.services.ops_policy_service.preview_ops_job_policy") + @patch("app.services.ops_agent_service.list_managed_nodes_with_agent_state") + def test_create_smart_release_rollout_for_existing_release_creates_rollout( + self, + mock_list_managed_nodes_with_agent_state, + mock_preview_ops_job_policy, + mock_create_release_rollout, + mock_get_release, + ) -> None: + mock_get_release.return_value = { + "id": 42, + "release_version": "domaincheck_release_existing", + "channel": "stable", + } + mock_list_managed_nodes_with_agent_state.return_value = { + "nodes": [ + { + "node_code": "mainland-worker-01", + "region": "mainland", + "role": "worker", + "is_enabled": True, + "is_managed": True, + "is_agent_online": True, + "cluster_status": "online", + "cluster_current_load": 0, + "cluster_is_effective_worker": True, + "cluster_detect_participating": False, + } + ] + } + mock_preview_ops_job_policy.return_value = { + "blocked": False, + "approval_required": False, + "warnings": [], + "release_gate": {"status_label": "可发 Rollout", "summary": "当前可推进。"}, + } + mock_create_release_rollout.return_value = ( + True, + "Release rollout 已创建", + { + "rollout": {"id": 8, "release_id": 42, "rollout_code": "rollout-20260418-existing"}, + "jobs": [], + }, + ) + + ok, message, data = create_smart_release_rollout(42, {"mode": "worker", "confirm_risky": True}) + + self.assertTrue(ok) + self.assertEqual("Release rollout 已创建", message) + self.assertFalse(data["release_created"]) + self.assertTrue(data["rollout_created"]) + self.assertEqual(8, data["rollout"]["id"]) + mock_create_release_rollout.assert_called_once() + rollout_payload = mock_create_release_rollout.call_args.args[1] + self.assertEqual(["mainland-worker-01"], rollout_payload["target_selector"]["node_codes"]) + self.assertEqual("remote-agent", rollout_payload["policy"]["execution_mode"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/domain-api/tests/test_ops_runtime_executor_service.py b/domain-api/tests/test_ops_runtime_executor_service.py new file mode 100644 index 0000000..e4e4f93 --- /dev/null +++ b/domain-api/tests/test_ops_runtime_executor_service.py @@ -0,0 +1,118 @@ +import subprocess +import tempfile +import unittest +from pathlib import Path +from unittest.mock import patch + +from app.services.ops_runtime_executor_service import execute_local_support_action, execute_ssh_action + + +class OpsRuntimeExecutorServiceTests(unittest.TestCase): + @patch("app.services.ops_runtime_executor_service.subprocess.run") + def test_execute_ssh_action_parses_structured_json_result(self, mock_run) -> None: + mock_run.return_value = subprocess.CompletedProcess( + args=["ssh"], + returncode=0, + stdout='login banner\n{"ok": true, "message": "health snapshot collected", "result": {"checks": {"api": {"state": "active"}}}}\n', + stderr="", + ) + + ok, message, data = execute_ssh_action( + { + "node_code": "mainland-worker-01", + "ssh_host": "121.204.244.248", + "ssh_port": 22, + "ssh_user": "root", + }, + "health.snapshot", + {}, + ) + + self.assertTrue(ok) + self.assertEqual("health snapshot collected", message) + self.assertEqual("active", data["checks"]["api"]["state"]) + self.assertEqual("ssh", data["transport"]["executor"]) + self.assertEqual("121.204.244.248", data["transport"]["ssh_host"]) + issued_command = mock_run.call_args.args[0] + self.assertEqual("ssh", issued_command[0]) + self.assertIn("root@121.204.244.248", issued_command) + + @patch("app.services.ops_runtime_executor_service.subprocess.run") + def test_execute_local_support_action_collects_logs_with_payload(self, mock_run) -> None: + mock_run.return_value = subprocess.CompletedProcess( + args=["journalctl"], + returncode=0, + stdout="log line 1\nlog line 2", + stderr="", + ) + + ok, message, data = execute_local_support_action( + "logs.collect", + { + "service_name": "domaincheck-worker", + "lines": 180, + }, + ) + + self.assertTrue(ok) + self.assertEqual("log line 1\nlog line 2", message) + self.assertEqual("domaincheck-worker", data["service_name"]) + self.assertEqual(180, data["lines"]) + issued_command = mock_run.call_args.args[0] + self.assertEqual( + ["journalctl", "-u", "domaincheck-worker", "-n", "180", "--no-pager"], + issued_command, + ) + + @patch("app.services.ops_runtime_executor_service.subprocess.run") + def test_execute_local_support_action_falls_back_to_worker_log_file(self, mock_run) -> None: + mock_run.return_value = subprocess.CompletedProcess( + args=["journalctl"], + returncode=1, + stdout="", + stderr="No journal files were opened due to insufficient permissions.", + ) + + with tempfile.TemporaryDirectory() as temp_dir: + log_path = Path(temp_dir) / "detect_worker.log" + log_path.write_text("fallback-1\nfallback-2\n", encoding="utf-8") + with patch("app.services.ops_action_executor_core.settings.domain_root", temp_dir): + ok, message, data = execute_local_support_action( + "logs.collect", + { + "service_name": "domaincheck-worker", + "lines": 180, + }, + ) + + self.assertTrue(ok) + self.assertEqual("fallback-1\nfallback-2", message) + self.assertEqual("file_fallback", data["collection_source"]) + self.assertTrue(data["fallback_used"]) + + @patch("app.services.ops_runtime_executor_service.subprocess.run") + def test_execute_local_support_action_restarts_service(self, mock_run) -> None: + mock_run.return_value = subprocess.CompletedProcess( + args=["systemctl"], + returncode=0, + stdout="", + stderr="", + ) + + ok, message, data = execute_local_support_action( + "service.restart", + { + "service_name": "domaincheck-worker", + }, + ) + + self.assertTrue(ok) + self.assertEqual("domaincheck-worker restart completed", message) + self.assertEqual("domaincheck-worker", data["service_name"]) + self.assertEqual("restart", data["systemctl_action"]) + issued_command = mock_run.call_args.args[0] + self.assertEqual(["systemctl", "restart", "domaincheck-worker"], issued_command) + + +if __name__ == "__main__": + unittest.main() diff --git a/domain-api/tests/test_ops_schema_init.py b/domain-api/tests/test_ops_schema_init.py new file mode 100644 index 0000000..996cbf2 --- /dev/null +++ b/domain-api/tests/test_ops_schema_init.py @@ -0,0 +1,75 @@ +import unittest +from unittest.mock import MagicMock, patch + +import app.services.ops_agent_service as ops_agent_service +import app.services.ops_job_service as ops_job_service + + +class OpsSchemaInitTests(unittest.TestCase): + @patch("app.services.ops_job_service.get_db") + def test_ensure_ops_schema_uses_advisory_lock_and_skips_after_ready(self, mock_get_db) -> None: + conn = MagicMock() + cursor = MagicMock() + db_ctx = MagicMock() + cursor_ctx = MagicMock() + db_ctx.__enter__.return_value = conn + db_ctx.__exit__.return_value = False + cursor_ctx.__enter__.return_value = cursor + cursor_ctx.__exit__.return_value = False + conn.cursor.return_value = cursor_ctx + mock_get_db.return_value = db_ctx + + previous_ready = ops_job_service._OPS_SCHEMA_READY + ops_job_service._OPS_SCHEMA_READY = False + try: + ops_job_service.ensure_ops_schema() + ops_job_service.ensure_ops_schema() + finally: + ops_job_service._OPS_SCHEMA_READY = previous_ready + + self.assertEqual(1, mock_get_db.call_count) + cursor.execute.assert_any_call( + "SELECT pg_advisory_xact_lock(%s)", + (ops_job_service._OPS_SCHEMA_ADVISORY_LOCK_KEY,), + ) + cursor.execute.assert_any_call(ops_job_service._OPS_SCHEMA_SQL) + conn.commit.assert_called_once() + + @patch("app.services.ops_agent_service.ensure_ops_schema") + @patch("app.services.ops_agent_service.get_db") + def test_ensure_ops_agent_schema_uses_advisory_lock_and_skips_after_ready( + self, + mock_get_db, + mock_ensure_ops_schema, + ) -> None: + conn = MagicMock() + cursor = MagicMock() + db_ctx = MagicMock() + cursor_ctx = MagicMock() + db_ctx.__enter__.return_value = conn + db_ctx.__exit__.return_value = False + cursor_ctx.__enter__.return_value = cursor + cursor_ctx.__exit__.return_value = False + conn.cursor.return_value = cursor_ctx + mock_get_db.return_value = db_ctx + + previous_ready = ops_agent_service._OPS_AGENT_SCHEMA_READY + ops_agent_service._OPS_AGENT_SCHEMA_READY = False + try: + ops_agent_service.ensure_ops_agent_schema() + ops_agent_service.ensure_ops_agent_schema() + finally: + ops_agent_service._OPS_AGENT_SCHEMA_READY = previous_ready + + self.assertEqual(2, mock_ensure_ops_schema.call_count) + self.assertEqual(1, mock_get_db.call_count) + cursor.execute.assert_any_call( + "SELECT pg_advisory_xact_lock(%s)", + (ops_agent_service._OPS_AGENT_SCHEMA_ADVISORY_LOCK_KEY,), + ) + cursor.execute.assert_any_call(ops_agent_service._AGENT_SCHEMA_SQL) + conn.commit.assert_called_once() + + +if __name__ == "__main__": + unittest.main() diff --git a/domain-api/tests/test_ops_service_activity.py b/domain-api/tests/test_ops_service_activity.py new file mode 100644 index 0000000..1cc7a06 --- /dev/null +++ b/domain-api/tests/test_ops_service_activity.py @@ -0,0 +1,607 @@ +import unittest +from typing import Optional +from unittest.mock import patch + +from app.services.ops_service import get_ops_activity_stream + + +class OpsServiceActivityTests(unittest.TestCase): + @staticmethod + def _runtime_status(detect: Optional[dict] = None) -> dict: + return {"detect": dict(detect or {})} + + @patch("app.services.ops_service.get_ops_runbook") + @patch("app.services.ops_service.list_release_rollouts") + @patch("app.services.ops_service.list_ops_jobs") + @patch("app.services.ops_service.get_recent_ops_playbook_runs") + @patch("app.services.ops_service.list_managed_nodes_with_agent_state") + def test_activity_stream_uses_job_detail_for_bootstrap_jobs( + self, + mock_list_managed_nodes_with_agent_state, + mock_get_recent_ops_playbook_runs, + mock_list_ops_jobs, + mock_list_release_rollouts, + mock_get_ops_runbook, + ) -> None: + mock_get_ops_runbook.return_value = {"control_sequences": []} + mock_list_managed_nodes_with_agent_state.return_value = { + "nodes": [ + { + "node_code": "mainland-worker-02", + "agent_state": "pending_bootstrap", + "agent_state_label": "待接入", + } + ] + } + mock_get_recent_ops_playbook_runs.return_value = {"runs": []} + mock_list_ops_jobs.return_value = [ + { + "id": 51, + "job_code": "ops-bootstrap-51", + "action": "node.bootstrap", + "status": "success", + "target_node_code": "mainland-worker-02", + "created_at": "2026-04-17 10:00:00", + "updated_at": "2026-04-17 10:01:00", + "result": { + "bootstrap_plan": { + "bootstrap_script_name": "bootstrap-node-agent-mainland-worker-02.sh", + } + }, + } + ] + mock_list_release_rollouts.return_value = [] + + with patch( + "app.services.ops_service.get_runtime_status", + return_value=self._runtime_status(), + ): + payload = get_ops_activity_stream(limit=10, scan_limit=20) + + self.assertEqual(1, len(payload["items"])) + item = payload["items"][0] + self.assertEqual("ops_job", item["kind"]) + self.assertEqual("节点接入工单", item["title"]) + self.assertEqual("job_detail", item["ui_intent"]["kind"]) + self.assertEqual("job_detail", item["ui_intent_kind"]) + self.assertEqual("控制面", item["execution_mode_label"]) + self.assertEqual("成功", item["status_label"]) + self.assertIn("尚未回连 Agent", item["summary"]) + self.assertEqual(item["summary"], item["summary_text"]) + self.assertIn("方式 控制面", item["meta_text"]) + self.assertIn("bootstrap-node-agent-mainland-worker-02.sh", item["meta_text"]) + self.assertEqual(["mainland-worker-02"], item["target_node_codes"]) + self.assertEqual(51, item["focus_ref"]["job_id"]) + self.assertEqual("node.bootstrap", item["focus_ref"]["action"]) + self.assertEqual("ops_agent_protocol", item["contract_navigation"]["primary_contract_key"]) + self.assertIn("ops_agent_protocol", item["contract_navigation"]["contract_keys"]) + + @patch("app.services.ops_service.get_ops_runbook") + @patch("app.services.ops_service.list_release_rollouts") + @patch("app.services.ops_service.list_ops_jobs") + @patch("app.services.ops_service.get_recent_ops_playbook_runs") + @patch("app.services.ops_service.list_managed_nodes_with_agent_state") + def test_activity_stream_includes_runbook_sequences_after_live_events( + self, + mock_list_managed_nodes_with_agent_state, + mock_get_recent_ops_playbook_runs, + mock_list_ops_jobs, + mock_list_release_rollouts, + mock_get_ops_runbook, + ) -> None: + mock_list_managed_nodes_with_agent_state.return_value = {"nodes": []} + mock_get_recent_ops_playbook_runs.return_value = { + "runs": [ + { + "run_code": "playbook-001", + "playbook_title": "标准巡检", + "status": "running", + "updated_at": "2026-04-18 09:11:00", + } + ] + } + mock_list_ops_jobs.return_value = [] + mock_list_release_rollouts.return_value = [] + mock_get_ops_runbook.return_value = { + "control_sequences": [ + { + "key": "release_progression", + "title": "版本发布与放量", + "status": "warning", + "status_label": "待处理", + "reason": "当前没有足够的执行面接管覆盖。", + "target_scope_label": "Release / Rollout", + "target_node_codes": ["mainland-controller-01"], + "primary_action_code": "fix_managed_nodes", + "primary_resolution": { + "ok": True, + "driver_action_code": "fix_managed_nodes", + "driver_action_label": "补执行面接管", + "driver_node_codes": ["mainland-controller-01"], + "message": "当前建议先补执行面接管。", + "resolved_at": "2026-04-18 09:12:00", + }, + } + ] + } + + with patch( + "app.services.ops_service.get_runtime_status", + return_value=self._runtime_status(), + ): + payload = get_ops_activity_stream(limit=10, scan_limit=20) + + self.assertEqual(2, len(payload["items"])) + self.assertEqual("playbook_run", payload["items"][0]["kind"]) + self.assertEqual("runbook_sequence", payload["items"][1]["kind"]) + runbook_item = payload["items"][1] + self.assertEqual("release_progression", runbook_item["sequence_key"]) + self.assertEqual("focus_runbook_sequence", runbook_item["ui_intent"]["kind"]) + self.assertEqual("fix_managed_nodes", runbook_item["ui_intent"]["driver_action_code"]) + self.assertEqual("attention", runbook_item["status"]) + self.assertIn("当前建议动作：补执行面接管", runbook_item["summary"]) + self.assertIn("release_progression", runbook_item["meta_text"]) + self.assertEqual("ops_playbook_contract", runbook_item["contract_navigation"]["primary_contract_key"]) + self.assertIn("ops_driver_contract", runbook_item["contract_navigation"]["contract_keys"]) + + @patch("app.services.ops_service.get_ops_runbook") + @patch("app.services.ops_service.list_release_rollouts") + @patch("app.services.ops_service.list_ops_jobs") + @patch("app.services.ops_service.get_recent_ops_playbook_runs") + @patch("app.services.ops_service.list_managed_nodes_with_agent_state") + def test_activity_stream_surfaces_execution_mode_labels_for_jobs_and_playbooks( + self, + mock_list_managed_nodes_with_agent_state, + mock_get_recent_ops_playbook_runs, + mock_list_ops_jobs, + mock_list_release_rollouts, + mock_get_ops_runbook, + ) -> None: + mock_list_managed_nodes_with_agent_state.return_value = {"nodes": []} + mock_get_recent_ops_playbook_runs.return_value = { + "runs": [ + { + "run_code": "playbook-ssh-001", + "playbook_title": "关键现场日志", + "group_title": "执行现场", + "status": "running", + "updated_at": "2026-04-18 09:11:00", + "execution_mode": "ssh", + "execution_mode_label": "SSH", + "steps": [{"status": "running"}], + "step_count": 1, + "jobs_total": 2, + "target_node_codes": ["mainland-worker-01"], + } + ] + } + mock_list_ops_jobs.return_value = [ + { + "id": 61, + "job_code": "ops-ssh-61", + "action": "logs.collect", + "status": "running", + "execution_mode": "ssh", + "execution_mode_label": "SSH", + "requested_by": "web-ui", + "target_node_code": "mainland-worker-01", + "target_type": "node", + "risk_level": "low", + "updated_at": "2026-04-18 09:10:00", + "created_at": "2026-04-18 09:09:00", + } + ] + mock_list_release_rollouts.return_value = [] + mock_get_ops_runbook.return_value = {"control_sequences": []} + + with patch( + "app.services.ops_service.get_runtime_status", + return_value=self._runtime_status(), + ): + payload = get_ops_activity_stream(limit=10, scan_limit=20) + + self.assertEqual(2, len(payload["items"])) + self.assertEqual("SSH", payload["items"][0]["execution_mode_label"]) + self.assertIn("执行 SSH", payload["items"][0]["meta_text"]) + self.assertEqual("SSH", payload["items"][1]["execution_mode_label"]) + self.assertIn("方式 SSH", payload["items"][1]["summary"]) + self.assertIn("执行 SSH", payload["items"][1]["meta_text"]) + + @patch("app.services.ops_service.get_ops_runbook") + @patch("app.services.ops_service.list_release_rollouts") + @patch("app.services.ops_service.list_ops_jobs") + @patch("app.services.ops_service.get_recent_ops_playbook_runs") + @patch("app.services.ops_service.list_managed_nodes_with_agent_state") + def test_activity_stream_prefers_job_result_summary_and_focus_ref( + self, + mock_list_managed_nodes_with_agent_state, + mock_get_recent_ops_playbook_runs, + mock_list_ops_jobs, + mock_list_release_rollouts, + mock_get_ops_runbook, + ) -> None: + mock_list_managed_nodes_with_agent_state.return_value = {"nodes": []} + mock_get_recent_ops_playbook_runs.return_value = {"runs": []} + mock_list_ops_jobs.return_value = [ + { + "id": 71, + "job_code": "ops-agent-71", + "action": "logs.collect", + "status": "success", + "execution_mode": "remote-agent", + "execution_mode_label": "远端 Agent", + "requested_by": "codex", + "target_node_code": "mainland-worker-01", + "target_type": "node", + "risk_level": "low", + "updated_at": "2026-04-18 09:20:00", + "created_at": "2026-04-18 09:19:00", + "focus_ref": { + "kind": "ops_job", + "job_id": 71, + "job_code": "ops-agent-71", + "action": "logs.collect", + "target_node_code": "mainland-worker-01", + }, + "result": { + "summary_text": "远端 Agent 已回传 120 行 Worker 日志，可直接继续分析。", + "focus_ref": { + "kind": "ops_job", + "job_id": 71, + "job_code": "ops-agent-71", + "action": "logs.collect", + "target_node_code": "mainland-worker-01", + "event_key": "job-event:888", + "step_key": "worker_logs", + }, + }, + } + ] + mock_list_release_rollouts.return_value = [] + mock_get_ops_runbook.return_value = {"control_sequences": []} + + with patch( + "app.services.ops_service.get_runtime_status", + return_value=self._runtime_status(), + ): + payload = get_ops_activity_stream(limit=10, scan_limit=20) + + self.assertEqual(1, len(payload["items"])) + item = payload["items"][0] + self.assertEqual("ops_job", item["kind"]) + self.assertEqual("远端 Agent 已回传 120 行 Worker 日志，可直接继续分析。", item["summary"]) + self.assertEqual("ops_job", item["focus_ref"]["kind"]) + self.assertEqual("job-event:888", item["focus_ref"]["event_key"]) + self.assertEqual("worker_logs", item["focus_ref"]["step_key"]) + self.assertIn("执行 远端 Agent", item["meta_text"]) + self.assertEqual("ops_observability_contract", item["contract_navigation"]["primary_contract_key"]) + self.assertIn("ops_stack_diagnosis_contract", item["contract_navigation"]["contract_keys"]) + + @patch("app.services.ops_service.get_ops_runbook") + @patch("app.services.ops_service.list_release_rollouts") + @patch("app.services.ops_service.list_ops_jobs") + @patch("app.services.ops_service.get_recent_ops_playbook_runs") + @patch("app.services.ops_service.list_managed_nodes_with_agent_state") + def test_activity_stream_marks_job_attention_when_start_delivery_failed_locally( + self, + mock_list_managed_nodes_with_agent_state, + mock_get_recent_ops_playbook_runs, + mock_list_ops_jobs, + mock_list_release_rollouts, + mock_get_ops_runbook, + ) -> None: + mock_list_managed_nodes_with_agent_state.return_value = {"nodes": []} + mock_get_recent_ops_playbook_runs.return_value = {"runs": []} + mock_list_ops_jobs.return_value = [ + { + "id": 81, + "job_code": "ops-agent-81", + "action": "logs.collect", + "status": "running", + "execution_mode": "remote-agent", + "execution_mode_label": "远端 Agent", + "requested_by": "codex", + "target_node_code": "mainland-worker-01", + "target_type": "node", + "risk_level": "low", + "updated_at": "2026-04-18 10:20:00", + "created_at": "2026-04-18 10:19:00", + "focus_ref": { + "kind": "ops_job", + "job_id": 81, + "job_code": "ops-agent-81", + "action": "logs.collect", + "target_node_code": "mainland-worker-01", + }, + "result": { + "summary_text": "节点端已经收到采集任务。", + "start_delivery_state": "failed_local", + "start_delivery_error": "temporary offline", + "source_focus_ref": { + "kind": "ops_job_event", + "job_id": 81, + "job_code": "ops-agent-81", + "event_key": "job-event:81", + "target_node_code": "mainland-worker-01", + }, + }, + } + ] + mock_list_release_rollouts.return_value = [] + mock_get_ops_runbook.return_value = {"control_sequences": []} + + with patch( + "app.services.ops_service.get_runtime_status", + return_value=self._runtime_status(), + ): + payload = get_ops_activity_stream(limit=10, scan_limit=20) + + self.assertEqual(1, len(payload["items"])) + item = payload["items"][0] + self.assertEqual("ops_job", item["kind"]) + self.assertEqual("attention", item["status"]) + self.assertEqual("running", item["job_status"]) + self.assertEqual("failed_local", item["start_delivery_state"]) + self.assertEqual("ops_job_start_delivery_failed_local", item["problem_code"]) + self.assertIn("开始回执未成功送达控制面", item["summary"]) + self.assertIn("temporary offline", item["summary"]) + self.assertIn("开始回执异常", item["meta_text"]) + self.assertEqual("ops_job_event", item["source_focus_ref"]["kind"]) + self.assertEqual("job-event:81", item["source_focus_ref"]["event_key"]) + self.assertEqual("attention", payload["summary"]["status"]) + self.assertEqual("待处理", payload["summary"]["status_label"]) + + @patch("app.services.ops_service.get_ops_runbook") + @patch("app.services.ops_service.list_release_rollouts") + @patch("app.services.ops_service.list_ops_jobs") + @patch("app.services.ops_service.get_recent_ops_playbook_runs") + @patch("app.services.ops_service.list_managed_nodes_with_agent_state") + def test_activity_stream_surfaces_execution_mode_label_for_rollout( + self, + mock_list_managed_nodes_with_agent_state, + mock_get_recent_ops_playbook_runs, + mock_list_ops_jobs, + mock_list_release_rollouts, + mock_get_ops_runbook, + ) -> None: + mock_list_managed_nodes_with_agent_state.return_value = {"nodes": []} + mock_get_recent_ops_playbook_runs.return_value = {"runs": []} + mock_list_ops_jobs.return_value = [] + mock_list_release_rollouts.return_value = [ + { + "id": 201, + "release_id": 18, + "rollout_code": "rollout-20260418093000-a1b2", + "policy": { + "execution_mode": "ssh", + }, + "status": "running", + "status_label": "执行中", + "target_nodes": [{"node_code": "mainland-worker-01"}], + "jobs_total": 2, + "jobs_created": 1, + "batch_cursor": 1, + "batches_total": 2, + "result_summary": {}, + "summary_text": "当前已覆盖 1/1 台节点，仍有 1 条任务执行中。", + "focus_ref": { + "kind": "release_rollout", + "rollout_id": 201, + "rollout_code": "rollout-20260418093000-a1b2", + "release_id": 18, + "section": "rollout_jobs", + }, + "created_by": "web-ui", + "created_at": "2026-04-18 09:30:00", + "updated_at": "2026-04-18 09:31:00", + } + ] + mock_get_ops_runbook.return_value = {"control_sequences": []} + + with patch( + "app.services.ops_service.get_runtime_status", + return_value=self._runtime_status(), + ): + payload = get_ops_activity_stream(limit=10, scan_limit=20) + + self.assertEqual(1, len(payload["items"])) + item = payload["items"][0] + self.assertEqual("rollout", item["kind"]) + self.assertEqual("SSH", item["execution_mode_label"]) + self.assertIn("执行 SSH", item["meta_text"]) + self.assertEqual("执行中", item["status_label"]) + self.assertIn("当前已覆盖 1/1 台节点", item["summary"]) + self.assertEqual("release_rollout", item["focus_ref"]["kind"]) + self.assertEqual("release_hub_contract", item["contract_navigation"]["primary_contract_key"]) + self.assertIn("release_hub_contract", item["contract_navigation"]["contract_keys"]) + + @patch("app.services.ops_service.get_ops_runbook") + @patch("app.services.ops_service.list_release_rollouts") + @patch("app.services.ops_service.list_ops_jobs") + @patch("app.services.ops_service.get_recent_ops_playbook_runs") + @patch("app.services.ops_service.list_managed_nodes_with_agent_state") + def test_activity_stream_supports_execution_mode_filter( + self, + mock_list_managed_nodes_with_agent_state, + mock_get_recent_ops_playbook_runs, + mock_list_ops_jobs, + mock_list_release_rollouts, + mock_get_ops_runbook, + ) -> None: + mock_list_managed_nodes_with_agent_state.return_value = {"nodes": []} + mock_get_recent_ops_playbook_runs.return_value = { + "runs": [ + { + "run_code": "playbook-ssh-001", + "playbook_title": "关键现场日志", + "status": "running", + "updated_at": "2026-04-18 09:11:00", + "execution_mode": "ssh", + "execution_mode_label": "SSH", + "steps": [{"status": "running"}], + "step_count": 1, + "jobs_total": 1, + "target_node_codes": ["mainland-worker-01"], + } + ] + } + mock_list_ops_jobs.return_value = [ + { + "id": 61, + "job_code": "ops-ssh-61", + "action": "logs.collect", + "status": "running", + "execution_mode": "ssh", + "execution_mode_label": "SSH", + "requested_by": "web-ui", + "target_node_code": "mainland-worker-01", + "target_type": "node", + "risk_level": "low", + "updated_at": "2026-04-18 09:10:00", + "created_at": "2026-04-18 09:09:00", + }, + { + "id": 62, + "job_code": "ops-agent-62", + "action": "health.snapshot", + "status": "success", + "execution_mode": "remote-agent", + "execution_mode_label": "远端 Agent", + "requested_by": "web-ui", + "target_node_code": "mainland-worker-02", + "target_type": "node", + "risk_level": "low", + "updated_at": "2026-04-18 09:08:00", + "created_at": "2026-04-18 09:07:00", + }, + ] + mock_list_release_rollouts.return_value = [] + mock_get_ops_runbook.return_value = { + "control_sequences": [ + { + "key": "release_progression", + "title": "版本发布与放量", + "status": "warning", + "status_label": "待处理", + "reason": "当前没有足够的执行面接管覆盖。", + "target_scope_label": "Release / Rollout", + "target_node_codes": ["mainland-controller-01"], + } + ] + } + + with patch( + "app.services.ops_service.get_runtime_status", + return_value=self._runtime_status(), + ): + payload = get_ops_activity_stream(limit=10, scan_limit=20, execution_mode="ssh") + + self.assertEqual(2, len(payload["items"])) + self.assertTrue(all(item["execution_mode"] == "ssh" for item in payload["items"])) + self.assertEqual("ssh", payload["summary"]["filters"]["execution_mode"]) + self.assertEqual(2, payload["summary"]["available_execution_mode_counts"]["ssh"]) + self.assertEqual(1, payload["summary"]["available_execution_mode_counts"]["remote-agent"]) + + @patch("app.services.ops_service.get_ops_runbook") + @patch("app.services.ops_service.list_release_rollouts") + @patch("app.services.ops_service.list_ops_jobs") + @patch("app.services.ops_service.get_recent_ops_playbook_runs") + @patch("app.services.ops_service.list_managed_nodes_with_agent_state") + def test_activity_stream_includes_execution_scene_and_log_sync_items( + self, + mock_list_managed_nodes_with_agent_state, + mock_get_recent_ops_playbook_runs, + mock_list_ops_jobs, + mock_list_release_rollouts, + mock_get_ops_runbook, + ) -> None: + mock_list_managed_nodes_with_agent_state.return_value = {"nodes": []} + mock_get_recent_ops_playbook_runs.return_value = {"runs": []} + mock_list_ops_jobs.return_value = [] + mock_list_release_rollouts.return_value = [] + mock_get_ops_runbook.return_value = {"control_sequences": []} + + detect = { + "active_job": { + "job_code": "detect-001", + "latest_cycle_event": { + "created_at": "2026-04-18 09:12:30", + }, + }, + "phase_label": "运行中", + "phase_detail": "代理池正常，现场可继续观察", + "recent_event": "域名检测完成: example.com", + "recent_warning": "", + "log_sync": { + "enabled": True, + "mode": "key", + "mode_label": "关键回传", + "description": "当前为关键回传；适合常态联调。", + "line_count": 12, + "source_node_count": 1, + "source_nodes": ["mainland-worker-01"], + "last_at": "2026-04-18 09:12:31", + "preview_lines": ["worker log line"], + }, + "participation_summary": { + "summary": "当前参与检测节点 1 台，其中执行/已领 1 台，近窗刚有吞吐 0 台。", + "effective_online_nodes": 1, + "participating_nodes": 1, + "dispatch_active_nodes": 1, + "recent_only_nodes": 0, + "non_participating_nodes": 0, + "standby_nodes": 0, + "load_syncing_nodes": 0, + }, + "participating_nodes": [ + { + "node_code": "mainland-worker-01", + "region": "mainland", + "role": "worker", + "status": "busy", + "last_heartbeat_at": "2026-04-18 09:12:29", + "participation_state": "dispatch_active", + "is_dispatch_active": True, + } + ], + "non_participating_nodes": [], + } + + with patch( + "app.services.ops_service.get_runtime_status", + return_value=self._runtime_status(detect), + ): + payload = get_ops_activity_stream(limit=10, scan_limit=20) + + items_by_kind = {item["kind"]: item for item in payload["items"]} + self.assertIn("execution_scene", items_by_kind) + self.assertIn("log_sync", items_by_kind) + + execution_scene_item = items_by_kind["execution_scene"] + log_sync_item = items_by_kind["log_sync"] + + self.assertEqual("running", execution_scene_item["status"]) + self.assertEqual("执行中", execution_scene_item["status_label"]) + self.assertEqual("focus_execution_scene", execution_scene_item["ui_intent"]["kind"]) + self.assertEqual("focus_execution_scene", execution_scene_item["ui_intent_kind"]) + self.assertEqual("execution_scene", execution_scene_item["focus_ref"]["kind"]) + self.assertEqual(execution_scene_item["summary"], execution_scene_item["summary_text"]) + self.assertIn("执行/已领 1", execution_scene_item["meta_text"]) + self.assertEqual("ops_observability_contract", execution_scene_item["contract_navigation"]["primary_contract_key"]) + self.assertIn("ops_stack_diagnosis_contract", execution_scene_item["contract_navigation"]["contract_keys"]) + self.assertEqual("running", log_sync_item["status"]) + self.assertEqual("执行中", log_sync_item["status_label"]) + self.assertEqual("control-plane", log_sync_item["execution_mode"]) + self.assertEqual("控制面", log_sync_item["execution_mode_label"]) + self.assertEqual("focus_execution_scene", log_sync_item["ui_intent"]["kind"]) + self.assertTrue(log_sync_item["ui_intent"]["open_log_drawer"]) + self.assertIn("关键日志观察模式", log_sync_item["summary"]) + self.assertEqual("running", payload["summary"]["status"]) + self.assertEqual("执行中", payload["summary"]["status_label"]) + self.assertIn("最近活动里存在正在推进", payload["summary"]["summary_text"]) + self.assertEqual("ops_observability_contract", payload["contract_navigation"]["primary_contract_key"]) + self.assertIn("ops_stack_diagnosis_contract", payload["contract_navigation"]["contract_keys"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/domain-api/tests/test_ops_service_codex_action.py b/domain-api/tests/test_ops_service_codex_action.py new file mode 100644 index 0000000..760eac3 --- /dev/null +++ b/domain-api/tests/test_ops_service_codex_action.py @@ -0,0 +1,925 @@ +from __future__ import annotations + +import unittest +from unittest.mock import patch + +from app.services.ops_service import execute_codex_action, resolve_codex_action + + +def _build_codex_brief_entry( + *, + key: str, + executor_kind: str, + primary_action_code: str, + recommendation: str, + automation_level: str, + target_api: str, + target_method: str = "POST", + sequence_key: str = "", + primary_action_payload: dict | None = None, + secondary_action_code: str = "", + secondary_action_payload: dict | None = None, + focus_ref: dict | None = None, + primary_focus_ref: dict | None = None, + secondary_focus_ref: dict | None = None, +) -> dict: + primary_action_payload = dict(primary_action_payload or {}) + secondary_action_payload = dict(secondary_action_payload or {}) + normalized_focus_ref = dict(focus_ref or {}) + normalized_primary_focus_ref = dict(primary_focus_ref or {}) + normalized_secondary_focus_ref = dict(secondary_focus_ref or {}) + if normalized_primary_focus_ref and not dict(primary_action_payload.get("focus_ref") or {}): + primary_action_payload["focus_ref"] = dict(normalized_primary_focus_ref) + if normalized_secondary_focus_ref and not dict(secondary_action_payload.get("focus_ref") or {}): + secondary_action_payload["focus_ref"] = dict(normalized_secondary_focus_ref) + request_payload_preview = ( + { + "sequence_key": sequence_key, + "secondary": False, + "action_payload": primary_action_payload, + } + if executor_kind == "runbook_sequence" + else { + "action_code": primary_action_code, + "node_codes": ["mainland-controller-01"], + "action_payload": primary_action_payload, + } + ) + return { + "key": key, + "title": "测试动作", + "summary": "测试摘要", + "reason": "测试原因", + "meta_text": "meta", + "executor_kind": executor_kind, + "sequence_key": sequence_key, + "focus_ref": normalized_focus_ref, + "primary_focus_ref": normalized_primary_focus_ref, + "secondary_focus_ref": normalized_secondary_focus_ref, + "primary_label": "主动作", + "secondary_label": "次动作" if secondary_action_code else "", + "primary_action_code": primary_action_code, + "secondary_action_code": secondary_action_code, + "autopilot": { + "status_text": "测试状态", + "automation_level": automation_level, + "recommendation": recommendation, + "executor_mode_hint": "test-mode", + "confirm_required": recommendation == "confirm_then_execute", + "ui_only": recommendation == "open_ui", + "blocked": recommendation == "blocked", + "backend_handled": True, + "risk_level": "low" if recommendation == "auto_execute" else "high", + "reason": "autopilot reason", + "target_method": target_method, + "target_api": target_api, + }, + "primary_execution": { + "label": "主动作", + "node_codes": ["mainland-controller-01"], + "action_payload": primary_action_payload, + "request_payload_preview": request_payload_preview, + }, + "secondary_execution": ( + { + "label": "次动作", + "node_codes": [], + "action_payload": secondary_action_payload, + } + if secondary_action_code + else {} + ), + } + + +def _build_preview_data( + *, + executor_kind: str, + action_code: str, + recommendation: str, + automation_level: str, + target_api: str, + request_payload: dict, + sequence_key: str = "", + focus_ref: dict | None = None, + primary_focus_ref: dict | None = None, + secondary_focus_ref: dict | None = None, +) -> dict: + return { + "executor_kind": executor_kind, + "sequence_key": sequence_key, + "execution_chain": "ops-driver -> preview", + "focus_ref": dict(focus_ref or {}), + "primary_focus_ref": dict(primary_focus_ref or {}), + "secondary_focus_ref": dict(secondary_focus_ref or {}), + "primary": { + "label": "主动作", + "action_code": action_code, + "target_api": target_api, + "target_method": "POST", + "request_payload": dict(request_payload or {}), + "focus_ref": dict(primary_focus_ref or {}), + "execution_profile": { + "action_code": action_code, + "backend_handled": True, + "automation_level": automation_level, + "recommendation": recommendation, + "executor_mode_hint": "test-mode", + "risk_level": "low" if recommendation == "auto_execute" else "high", + "confirm_required": recommendation == "confirm_then_execute", + "ui_only": recommendation == "open_ui", + "blocked": recommendation == "blocked", + "requires_fields": [], + "missing_fields": [], + "reason": "preview reason", + }, + }, + "secondary": {}, + } + + +def _build_codex_activity_focus_entry( + *, + key: str, + activity_key: str, + activity_kind: str, + ui_intent_kind: str, + focus_ref: dict | None = None, + source_focus_ref: dict | None = None, + target_node_codes: list[str] | None = None, +) -> dict: + normalized_focus_ref = dict(focus_ref or {}) + return { + "key": key, + "activity_key": activity_key, + "activity_kind": activity_kind, + "title": "观察焦点", + "subtitle": "日志回传待确认", + "summary": "该焦点属于观察项，应进入对应工作区查看上下文。", + "status": "attention", + "status_label": "待关注", + "meta_text": "activity-meta", + "occurred_at": "2026-04-18T06:00:00", + "target_node_codes": list(target_node_codes or []), + "focus_ref": normalized_focus_ref, + "source_focus_ref": dict(source_focus_ref or {}), + "ui_intent": { + "kind": ui_intent_kind, + "activity_key": activity_key, + }, + "ui_intent_kind": ui_intent_kind, + "focus_action_code": "focus_activity_item", + "focus_action_payload": { + "activity_key": activity_key, + "kind": activity_kind, + "status": "attention", + "ui_intent": { + "kind": ui_intent_kind, + "activity_key": activity_key, + }, + "focus_ref": normalized_focus_ref, + }, + "observation_only": True, + } + + +class OpsServiceCodexActionTests(unittest.TestCase): + @patch("app.services.ops_service.preview_driver_action") + @patch("app.services.ops_service.get_ops_codex_brief") + def test_resolve_codex_action_returns_gate_for_safe_auto(self, mock_get_ops_codex_brief, mock_preview_driver_action) -> None: + mock_get_ops_codex_brief.return_value = { + "focus": {"entry_key": "recommendation:enable-log-sync"}, + "summary": {"total": 1}, + "entries": [ + _build_codex_brief_entry( + key="recommendation:enable-log-sync", + executor_kind="driver_action", + primary_action_code="enable_log_sync_key", + recommendation="auto_execute", + automation_level="safe_auto", + target_api="/api/v1/ops/driver-actions/execute", + focus_ref={ + "kind": "execution_scene", + "scene_key": "scene.logs", + }, + primary_focus_ref={ + "kind": "execution_scene", + "scene_key": "scene.logs", + }, + ) + ], + } + mock_preview_driver_action.return_value = ( + True, + "ok", + _build_preview_data( + executor_kind="driver_action", + action_code="enable_log_sync_key", + recommendation="auto_execute", + automation_level="safe_auto", + target_api="/api/v1/ops/driver-actions/execute", + request_payload={ + "action_code": "enable_log_sync_key", + "node_codes": ["mainland-controller-01"], + "action_payload": { + "focus_ref": { + "kind": "execution_scene", + "scene_key": "scene.logs", + } + }, + "requested_by": "test/codex-action", + }, + focus_ref={ + "kind": "execution_scene", + "scene_key": "scene.logs", + }, + primary_focus_ref={ + "kind": "execution_scene", + "scene_key": "scene.logs", + }, + ), + ) + + ok, message, data = resolve_codex_action({"requested_by": "test/codex-action"}) + + self.assertTrue(ok) + self.assertEqual("Codex 驾驶动作解析完成", message) + self.assertEqual("recommendation:enable-log-sync", data["entry_key"]) + self.assertEqual("execute_now", data["gate"]["decision"]) + self.assertTrue(data["gate"]["will_execute"]) + self.assertEqual("/api/v1/ops/driver-actions/execute", data["gate"]["target_api"]) + self.assertEqual("enable_log_sync_key", data["preview_request"]["primary_action_code"]) + self.assertEqual("execution_scene", data["preview_request"]["focus_ref"]["kind"]) + self.assertEqual("scene.logs", data["gate"]["focus_ref"]["scene_key"]) + self.assertEqual("scene.logs", data["focus_ref"]["scene_key"]) + + @patch("app.services.ops_service.preview_driver_action") + @patch("app.services.ops_service.get_ops_codex_brief") + def test_resolve_codex_action_requires_confirmation_for_guarded_runbook( + self, + mock_get_ops_codex_brief, + mock_preview_driver_action, + ) -> None: + mock_get_ops_codex_brief.return_value = { + "focus": {"entry_key": "runbook:release_progression"}, + "summary": {"total": 1}, + "entries": [ + _build_codex_brief_entry( + key="runbook:release_progression", + executor_kind="runbook_sequence", + sequence_key="release_progression", + primary_action_code="create_release_rollout_worker", + recommendation="confirm_then_execute", + automation_level="guarded_auto", + target_api="/api/v1/ops/runbook/sequences/release_progression/execute", + primary_action_payload={"release_id": 7}, + secondary_action_code="focus_release_hub", + secondary_action_payload={"release_id": 7}, + focus_ref={ + "kind": "release_hub", + "release_id": 7, + "section": "release_launchpad", + }, + primary_focus_ref={ + "kind": "release_hub", + "release_id": 7, + "section": "release_launchpad", + "mode": "worker", + }, + secondary_focus_ref={ + "kind": "release_hub", + "release_id": 7, + "section": "release_detail", + }, + ) + ], + } + mock_preview_driver_action.return_value = ( + True, + "ok", + _build_preview_data( + executor_kind="runbook_sequence", + action_code="create_release_rollout_worker", + recommendation="confirm_then_execute", + automation_level="guarded_auto", + target_api="/api/v1/ops/runbook/sequences/release_progression/execute", + request_payload={ + "secondary": False, + "requested_by": "test/codex-action", + "action_payload": { + "release_id": 7, + "focus_ref": { + "kind": "release_hub", + "release_id": 7, + "section": "release_launchpad", + "mode": "worker", + }, + }, + }, + sequence_key="release_progression", + focus_ref={ + "kind": "release_hub", + "release_id": 7, + "section": "release_launchpad", + }, + primary_focus_ref={ + "kind": "release_hub", + "release_id": 7, + "section": "release_launchpad", + "mode": "worker", + }, + secondary_focus_ref={ + "kind": "release_hub", + "release_id": 7, + "section": "release_detail", + }, + ), + ) + + ok, _, data = resolve_codex_action( + { + "entry_key": "runbook:release_progression", + "requested_by": "test/codex-action", + } + ) + + self.assertTrue(ok) + self.assertEqual("confirmation_required", data["gate"]["decision"]) + self.assertFalse(data["gate"]["will_execute"]) + self.assertTrue(data["gate"]["confirm_required"]) + self.assertEqual("release_progression", data["gate"]["sequence_key"]) + self.assertEqual("worker", data["preview_request"]["primary_focus_ref"]["mode"]) + self.assertEqual("worker", data["gate"]["focus_ref"]["mode"]) + self.assertEqual("worker", data["focus_ref"]["mode"]) + + @patch("app.services.ops_service.preview_driver_action") + @patch("app.services.ops_service.get_ops_codex_brief") + def test_resolve_codex_action_can_select_secondary_section( + self, + mock_get_ops_codex_brief, + mock_preview_driver_action, + ) -> None: + mock_get_ops_codex_brief.return_value = { + "focus": {"entry_key": "runbook:release_progression"}, + "summary": {"total": 1}, + "entries": [ + _build_codex_brief_entry( + key="runbook:release_progression", + executor_kind="runbook_sequence", + sequence_key="release_progression", + primary_action_code="create_release_rollout_worker", + recommendation="confirm_then_execute", + automation_level="guarded_auto", + target_api="/api/v1/ops/runbook/sequences/release_progression/execute", + primary_action_payload={"release_id": 7}, + secondary_action_code="focus_release_hub", + secondary_action_payload={"release_id": 7}, + focus_ref={ + "kind": "release_hub", + "release_id": 7, + "section": "release_launchpad", + }, + primary_focus_ref={ + "kind": "release_hub", + "release_id": 7, + "section": "release_launchpad", + "mode": "worker", + }, + secondary_focus_ref={ + "kind": "release_hub", + "release_id": 7, + "section": "release_detail", + }, + ) + ], + } + preview_data = _build_preview_data( + executor_kind="runbook_sequence", + action_code="create_release_rollout_worker", + recommendation="confirm_then_execute", + automation_level="guarded_auto", + target_api="/api/v1/ops/runbook/sequences/release_progression/execute", + request_payload={ + "secondary": False, + "requested_by": "test/codex-action", + "action_payload": { + "release_id": 7, + "focus_ref": { + "kind": "release_hub", + "release_id": 7, + "section": "release_launchpad", + "mode": "worker", + }, + }, + }, + sequence_key="release_progression", + focus_ref={ + "kind": "release_hub", + "release_id": 7, + "section": "release_launchpad", + }, + primary_focus_ref={ + "kind": "release_hub", + "release_id": 7, + "section": "release_launchpad", + "mode": "worker", + }, + secondary_focus_ref={ + "kind": "release_hub", + "release_id": 7, + "section": "release_detail", + }, + ) + preview_data["secondary"] = { + "label": "查看版本区", + "action_code": "focus_release_hub", + "target_api": "/api/v1/ops/runbook/sequences/release_progression/execute", + "target_method": "POST", + "request_payload": { + "secondary": True, + "requested_by": "test/codex-action", + "action_payload": { + "release_id": 7, + "focus_ref": { + "kind": "release_hub", + "release_id": 7, + "section": "release_detail", + }, + }, + }, + "focus_ref": { + "kind": "release_hub", + "release_id": 7, + "section": "release_detail", + }, + "execution_profile": { + "action_code": "focus_release_hub", + "backend_handled": False, + "automation_level": "safe_auto", + "recommendation": "auto_execute", + "executor_mode_hint": "ui-intent", + "risk_level": "low", + "confirm_required": False, + "ui_only": False, + "blocked": False, + "requires_fields": [], + "missing_fields": [], + "reason": "secondary preview reason", + }, + } + mock_preview_driver_action.return_value = (True, "ok", preview_data) + + ok, _, data = resolve_codex_action( + { + "entry_key": "runbook:release_progression", + "requested_by": "test/codex-action", + "secondary": True, + } + ) + + self.assertTrue(ok) + self.assertEqual("secondary", data["selected_section"]) + self.assertTrue(data["gate"]["secondary"]) + self.assertEqual("focus_release_hub", data["gate"]["action_code"]) + self.assertEqual("auto_execute", data["gate"]["recommendation"]) + self.assertEqual("release_detail", data["gate"]["focus_ref"]["section"]) + self.assertEqual("release_detail", data["focus_ref"]["section"]) + + @patch("app.services.ops_service.preview_driver_action") + @patch("app.services.ops_service.get_ops_codex_brief") + def test_resolve_codex_action_can_fall_back_to_activity_focus_entry( + self, + mock_get_ops_codex_brief, + mock_preview_driver_action, + ) -> None: + mock_get_ops_codex_brief.return_value = { + "focus": {"entry_key": "activity:ops-job-42", "recommendation": "open_ui"}, + "summary": {"total": 0, "activity_focus_total": 1}, + "entries": [], + "activity_focus": [ + _build_codex_activity_focus_entry( + key="activity:ops-job-42", + activity_key="ops-job-42", + activity_kind="ops_job", + ui_intent_kind="job_events", + focus_ref={ + "kind": "ops_job", + "job_key": "job-42", + }, + source_focus_ref={ + "kind": "ops_job_event", + "job_key": "job-42", + "event_key": "event-1", + }, + target_node_codes=["mainland-controller-01"], + ) + ], + } + mock_preview_driver_action.return_value = ( + True, + "ok", + _build_preview_data( + executor_kind="driver_action", + action_code="focus_activity_item", + recommendation="auto_execute", + automation_level="safe_auto", + target_api="/api/v1/ops/driver-actions/execute", + request_payload={ + "action_code": "focus_activity_item", + "node_codes": ["mainland-controller-01"], + "action_payload": { + "activity_key": "ops-job-42", + "kind": "ops_job", + "status": "attention", + "ui_intent": { + "kind": "job_events", + "activity_key": "ops-job-42", + }, + "focus_ref": { + "kind": "ops_job", + "job_key": "job-42", + }, + }, + "requested_by": "test/codex-action", + }, + focus_ref={ + "kind": "ops_job", + "job_key": "job-42", + }, + primary_focus_ref={ + "kind": "ops_job", + "job_key": "job-42", + }, + ), + ) + + ok, _, data = resolve_codex_action({"requested_by": "test/codex-action"}) + + self.assertTrue(ok) + self.assertEqual("activity:ops-job-42", data["entry_key"]) + self.assertTrue(data["selected_entry"]["observation_only"]) + self.assertEqual("ops_job", data["selected_entry"]["activity_kind"]) + self.assertEqual("focus_activity_item", data["preview_request"]["primary_action_code"]) + self.assertEqual("execute_now", data["gate"]["decision"]) + self.assertTrue(data["gate"]["will_execute"]) + self.assertEqual("job-42", data["focus_ref"]["job_key"]) + + @patch("app.services.ops_service.execute_driver_action") + @patch("app.services.ops_service.preview_driver_action") + @patch("app.services.ops_service.get_ops_codex_brief") + def test_execute_codex_action_runs_safe_auto_driver_action( + self, + mock_get_ops_codex_brief, + mock_preview_driver_action, + mock_execute_driver_action, + ) -> None: + mock_get_ops_codex_brief.return_value = { + "focus": {"entry_key": "recommendation:enable-log-sync"}, + "summary": {"total": 1}, + "entries": [ + _build_codex_brief_entry( + key="recommendation:enable-log-sync", + executor_kind="driver_action", + primary_action_code="enable_log_sync_key", + recommendation="auto_execute", + automation_level="safe_auto", + target_api="/api/v1/ops/driver-actions/execute", + ) + ], + } + request_payload = { + "action_code": "enable_log_sync_key", + "node_codes": ["mainland-controller-01"], + "action_payload": {}, + "requested_by": "test/codex-action", + } + mock_preview_driver_action.return_value = ( + True, + "ok", + _build_preview_data( + executor_kind="driver_action", + action_code="enable_log_sync_key", + recommendation="auto_execute", + automation_level="safe_auto", + target_api="/api/v1/ops/driver-actions/execute", + request_payload=request_payload, + ), + ) + mock_execute_driver_action.return_value = ( + True, + "执行成功", + {"handled": True, "action_code": "enable_log_sync_key"}, + ) + + ok, message, data = execute_codex_action({"requested_by": "test/codex-action"}) + + self.assertTrue(ok) + self.assertEqual("执行成功", message) + self.assertTrue(data["executed"]) + mock_execute_driver_action.assert_called_once_with(request_payload) + + @patch("app.services.ops_service.execute_ops_runbook_sequence") + @patch("app.services.ops_service.preview_driver_action") + @patch("app.services.ops_service.get_ops_codex_brief") + def test_execute_codex_action_blocks_guarded_runbook_without_confirm( + self, + mock_get_ops_codex_brief, + mock_preview_driver_action, + mock_execute_ops_runbook_sequence, + ) -> None: + mock_get_ops_codex_brief.return_value = { + "focus": {"entry_key": "runbook:release_progression"}, + "summary": {"total": 1}, + "entries": [ + _build_codex_brief_entry( + key="runbook:release_progression", + executor_kind="runbook_sequence", + sequence_key="release_progression", + primary_action_code="create_release_rollout_worker", + recommendation="confirm_then_execute", + automation_level="guarded_auto", + target_api="/api/v1/ops/runbook/sequences/release_progression/execute", + primary_action_payload={"release_id": 7}, + ) + ], + } + mock_preview_driver_action.return_value = ( + True, + "ok", + _build_preview_data( + executor_kind="runbook_sequence", + action_code="create_release_rollout_worker", + recommendation="confirm_then_execute", + automation_level="guarded_auto", + target_api="/api/v1/ops/runbook/sequences/release_progression/execute", + request_payload={ + "secondary": False, + "requested_by": "test/codex-action", + "action_payload": {"release_id": 7}, + }, + sequence_key="release_progression", + ), + ) + + ok, message, data = execute_codex_action( + { + "entry_key": "runbook:release_progression", + "requested_by": "test/codex-action", + } + ) + + self.assertFalse(ok) + self.assertIn("确认", message) + self.assertFalse(data["executed"]) + mock_execute_ops_runbook_sequence.assert_not_called() + + @patch("app.services.ops_service.execute_driver_action") + @patch("app.services.ops_service.preview_driver_action") + @patch("app.services.ops_service.get_ops_codex_brief") + def test_execute_codex_action_runs_activity_focus_as_safe_auto( + self, + mock_get_ops_codex_brief, + mock_preview_driver_action, + mock_execute_driver_action, + ) -> None: + mock_get_ops_codex_brief.return_value = { + "focus": {"entry_key": "activity:ops-job-42", "recommendation": "open_ui"}, + "summary": {"total": 0, "activity_focus_total": 1}, + "entries": [], + "activity_focus": [ + _build_codex_activity_focus_entry( + key="activity:ops-job-42", + activity_key="ops-job-42", + activity_kind="ops_job", + ui_intent_kind="job_events", + focus_ref={ + "kind": "ops_job", + "job_key": "job-42", + }, + ) + ], + } + mock_preview_driver_action.return_value = ( + True, + "ok", + _build_preview_data( + executor_kind="driver_action", + action_code="focus_activity_item", + recommendation="auto_execute", + automation_level="safe_auto", + target_api="/api/v1/ops/driver-actions/execute", + request_payload={ + "action_code": "focus_activity_item", + "node_codes": [], + "action_payload": { + "activity_key": "ops-job-42", + "kind": "ops_job", + "status": "attention", + "ui_intent": { + "kind": "job_events", + "activity_key": "ops-job-42", + }, + "focus_ref": { + "kind": "ops_job", + "job_key": "job-42", + }, + }, + "requested_by": "test/codex-action", + }, + focus_ref={ + "kind": "ops_job", + "job_key": "job-42", + }, + primary_focus_ref={ + "kind": "ops_job", + "job_key": "job-42", + }, + ), + ) + mock_execute_driver_action.return_value = ( + True, + "执行成功", + {"handled": True, "action_code": "focus_activity_item"}, + ) + + ok, message, data = execute_codex_action({"requested_by": "test/codex-action"}) + + self.assertTrue(ok) + self.assertEqual("执行成功", message) + self.assertTrue(data["executed"]) + mock_execute_driver_action.assert_called_once() + + @patch("app.services.ops_service.preview_driver_action") + @patch("app.services.ops_service.get_ops_codex_brief") + def test_resolve_codex_action_supports_focus_latest_job_events_without_ui_intent( + self, + mock_get_ops_codex_brief, + mock_preview_driver_action, + ) -> None: + mock_get_ops_codex_brief.return_value = { + "focus": {"entry_key": "activity:ops-job-81", "recommendation": "open_ui"}, + "summary": {"total": 0, "activity_focus_total": 1}, + "entries": [], + "activity_focus": [ + { + "key": "activity:ops-job-81", + "activity_key": "ops-job-81", + "activity_kind": "ops_job", + "title": "开始回执异常", + "subtitle": "ops-agent-81", + "summary": "节点已接单执行，但开始回执未成功送达控制面。", + "status": "attention", + "status_label": "待处理", + "meta_text": "ops-agent-81", + "occurred_at": "2026-04-18T06:00:00", + "target_node_codes": ["mainland-worker-01"], + "focus_ref": { + "kind": "ops_job", + "job_id": 81, + "job_code": "ops-agent-81", + "target_node_code": "mainland-worker-01", + }, + "source_focus_ref": { + "kind": "ops_job_event", + "job_id": 81, + "job_code": "ops-agent-81", + "event_key": "job-event:81", + "target_node_code": "mainland-worker-01", + }, + "ui_intent": {}, + "ui_intent_kind": "", + "focus_action_code": "focus_latest_job_events", + "focus_action_payload": { + "focus_ref": { + "kind": "ops_job", + "job_id": 81, + "job_code": "ops-agent-81", + "target_node_code": "mainland-worker-01", + }, + "source_focus_ref": { + "kind": "ops_job_event", + "job_id": 81, + "job_code": "ops-agent-81", + "event_key": "job-event:81", + "target_node_code": "mainland-worker-01", + }, + }, + "observation_only": True, + } + ], + } + mock_preview_driver_action.return_value = ( + True, + "ok", + _build_preview_data( + executor_kind="driver_action", + action_code="focus_latest_job_events", + recommendation="auto_execute", + automation_level="safe_auto", + target_api="/api/v1/ops/driver-actions/execute", + request_payload={ + "action_code": "focus_latest_job_events", + "node_codes": ["mainland-worker-01"], + "action_payload": { + "focus_ref": { + "kind": "ops_job", + "job_id": 81, + "job_code": "ops-agent-81", + "target_node_code": "mainland-worker-01", + }, + "source_focus_ref": { + "kind": "ops_job_event", + "job_id": 81, + "job_code": "ops-agent-81", + "event_key": "job-event:81", + "target_node_code": "mainland-worker-01", + }, + }, + "requested_by": "test/codex-action", + }, + focus_ref={ + "kind": "ops_job", + "job_id": 81, + "job_code": "ops-agent-81", + }, + primary_focus_ref={ + "kind": "ops_job", + "job_id": 81, + "job_code": "ops-agent-81", + }, + ), + ) + + ok, _, data = resolve_codex_action({"requested_by": "test/codex-action"}) + + self.assertTrue(ok) + self.assertEqual("activity:ops-job-81", data["entry_key"]) + self.assertEqual("focus_latest_job_events", data["preview_request"]["primary_action_code"]) + self.assertEqual("execute_now", data["gate"]["decision"]) + self.assertTrue(data["gate"]["will_execute"]) + + @patch("app.services.ops_service.execute_ops_runbook_sequence") + @patch("app.services.ops_service.preview_driver_action") + @patch("app.services.ops_service.get_ops_codex_brief") + def test_execute_codex_action_runs_guarded_runbook_after_confirm( + self, + mock_get_ops_codex_brief, + mock_preview_driver_action, + mock_execute_ops_runbook_sequence, + ) -> None: + mock_get_ops_codex_brief.return_value = { + "focus": {"entry_key": "runbook:release_progression"}, + "summary": {"total": 1}, + "entries": [ + _build_codex_brief_entry( + key="runbook:release_progression", + executor_kind="runbook_sequence", + sequence_key="release_progression", + primary_action_code="create_release_rollout_worker", + recommendation="confirm_then_execute", + automation_level="guarded_auto", + target_api="/api/v1/ops/runbook/sequences/release_progression/execute", + primary_action_payload={"release_id": 7}, + ) + ], + } + request_payload = { + "secondary": False, + "requested_by": "test/codex-action", + "action_payload": {"release_id": 7}, + } + mock_preview_driver_action.return_value = ( + True, + "ok", + _build_preview_data( + executor_kind="runbook_sequence", + action_code="create_release_rollout_worker", + recommendation="confirm_then_execute", + automation_level="guarded_auto", + target_api="/api/v1/ops/runbook/sequences/release_progression/execute", + request_payload=request_payload, + sequence_key="release_progression", + ), + ) + mock_execute_ops_runbook_sequence.return_value = ( + True, + "runbook 执行成功", + {"sequence_key": "release_progression", "driver_action_code": "create_release_rollout_worker"}, + ) + + ok, message, data = execute_codex_action( + { + "entry_key": "runbook:release_progression", + "requested_by": "test/codex-action", + "confirm": True, + } + ) + + self.assertTrue(ok) + self.assertEqual("runbook 执行成功", message) + self.assertTrue(data["executed"]) + mock_execute_ops_runbook_sequence.assert_called_once_with("release_progression", request_payload) + + +if __name__ == "__main__": + unittest.main() diff --git a/domain-api/tests/test_ops_service_codex_brief.py b/domain-api/tests/test_ops_service_codex_brief.py new file mode 100644 index 0000000..adb1c7f --- /dev/null +++ b/domain-api/tests/test_ops_service_codex_brief.py @@ -0,0 +1,497 @@ +import unittest +from unittest.mock import patch + +from app.services.ops_service import get_ops_codex_brief + + +class OpsServiceCodexBriefTests(unittest.TestCase): + @patch("app.services.ops_service.get_ops_driver_feed") + def test_codex_brief_classifies_safe_ui_and_runbook_entries(self, mock_get_ops_driver_feed) -> None: + mock_get_ops_driver_feed.return_value = { + "headline": "当前建议已形成。", + "go_live_summary": { + "go_live_status": "attention", + "publish_ready": False, + "publish_status": "attention", + "publish_status_label": "可发布但建议先复核", + "operator_title": "先补现场日志回传", + "next_step_action_code": "enable_log_sync_key", + "launchpad_recommended_target_node_code": "mainland-controller-01", + "launchpad_recommended_recovery_label": "签发接入工单", + "launchpad_recommended_recovery_summary": "当前节点仍处于接入阶段，建议先签发 onboarding.bootstrap。", + "launchpad_onboarding_bootstrap_pending_nodes": 1, + "launchpad_onboarding_acceptance_ready_nodes": 1, + }, + "activity_focus": [], + "entries": [ + { + "key": "recommendation:enable-scene-log-sync", + "kind": "driver_recommendation", + "lane": "do_now", + "lane_label": "建议动作", + "title": "先补现场日志回传", + "summary": "优先打开关键日志回传。", + "reason": "当前还看不到现场过程日志。", + "status": "attention", + "level_label": "最高优先", + "tag_type": "danger", + "meta_text": "", + "detail_lines": ["参与节点 2 台"], + "node_codes": ["mainland-worker-01"], + "executor_kind": "driver_action", + "focus_ref": { + "kind": "execution_scene", + "scene_key": "scene.logs", + }, + "primary_focus_ref": { + "kind": "execution_scene", + "scene_key": "scene.logs", + }, + "secondary_focus_ref": { + "kind": "execution_scene", + "scene_key": "scene.logs.full", + }, + "primary_label": "开启关键回传", + "secondary_label": "开启全量回传", + "primary_action_code": "enable_log_sync_key", + "secondary_action_code": "enable_log_sync_full", + "primary_node_codes": [], + "secondary_node_codes": [], + "primary_action_payload": { + "focus_ref": { + "kind": "execution_scene", + "scene_key": "scene.logs", + } + }, + "secondary_action_payload": { + "focus_ref": { + "kind": "execution_scene", + "scene_key": "scene.logs.full", + } + }, + }, + { + "key": "recommendation:release-missing", + "kind": "driver_recommendation", + "lane": "do_now", + "lane_label": "建议动作", + "title": "先建立 Release", + "summary": "当前还没有默认 Release。", + "reason": "需要先进入版本区。", + "status": "planned", + "level_label": "待补齐", + "tag_type": "info", + "meta_text": "", + "detail_lines": [], + "node_codes": [], + "executor_kind": "driver_action", + "focus_ref": { + "kind": "release_hub", + "release_id": 7, + "section": "release_launchpad", + }, + "primary_focus_ref": { + "kind": "release_hub", + "release_id": 7, + "section": "release_launchpad", + }, + "primary_label": "创建 Release", + "secondary_label": "", + "primary_action_code": "open_release_dialog", + "secondary_action_code": "", + "primary_node_codes": [], + "secondary_node_codes": [], + "primary_action_payload": {}, + "secondary_action_payload": {}, + }, + { + "key": "runbook:release_progression", + "kind": "runbook_sequence", + "lane": "standard_path", + "lane_label": "标准路径", + "title": "版本发布与放量", + "summary": "当前可以进入正式发布路径。", + "reason": "统一回到 Release / Rollout 闭环。", + "status": "attention", + "level_label": "待处理", + "tag_type": "warning", + "meta_text": "Release / Rollout", + "detail_lines": ["当前解析：Worker 灰度"], + "node_codes": ["mainland-controller-01"], + "executor_kind": "runbook_sequence", + "sequence_key": "release_progression", + "focus_ref": { + "kind": "release_hub", + "release_id": 7, + "section": "release_launchpad", + }, + "primary_focus_ref": { + "kind": "release_hub", + "release_id": 7, + "section": "release_launchpad", + "mode": "worker", + }, + "secondary_focus_ref": { + "kind": "release_hub", + "release_id": 7, + "section": "release_detail", + }, + "primary_label": "Worker 灰度", + "secondary_label": "查看版本区", + "primary_action_code": "create_release_rollout_worker", + "secondary_action_code": "focus_release_hub", + "primary_node_codes": ["mainland-controller-01"], + "secondary_node_codes": [], + "primary_action_payload": { + "release_id": 7, + "focus_ref": { + "kind": "release_hub", + "release_id": 7, + "section": "release_launchpad", + "mode": "worker", + }, + }, + "secondary_action_payload": { + "release_id": 7, + "focus_ref": { + "kind": "release_hub", + "release_id": 7, + "section": "release_detail", + }, + }, + "occurred_at": "2026-04-18T06:00:00", + }, + ], + } + + payload = get_ops_codex_brief() + + self.assertEqual(3, payload["summary"]["total"]) + self.assertEqual("attention", payload["go_live_summary"]["go_live_status"]) + self.assertEqual("attention", payload["summary"]["go_live_status"]) + self.assertEqual("先补现场日志回传", payload["summary"]["operator_title"]) + self.assertEqual("attention", payload["summary"]["publish_status"]) + self.assertFalse(payload["summary"]["publish_ready"]) + self.assertEqual("可发布但建议先复核", payload["summary"]["publish_status_label"]) + self.assertEqual("mainland-controller-01", payload["summary"]["launchpad_recommended_target_node_code"]) + self.assertEqual("签发接入工单", payload["summary"]["launchpad_recommended_recovery_label"]) + self.assertEqual( + "当前节点仍处于接入阶段，建议先签发 onboarding.bootstrap。", + payload["summary"]["launchpad_recommended_recovery_summary"], + ) + self.assertEqual(1, payload["summary"]["launchpad_onboarding_bootstrap_pending_nodes"]) + self.assertEqual(1, payload["summary"]["launchpad_onboarding_acceptance_ready_nodes"]) + self.assertEqual(0, payload["summary"]["activity_focus_total"]) + self.assertEqual("当前建议已形成。", payload["driver_feed"]["headline"]) + self.assertEqual("ops_driver_contract", payload["contract_navigation"]["primary_contract_key"]) + self.assertIn("ops_stack_diagnosis_contract", payload["contract_navigation"]["contract_keys"]) + self.assertEqual(2, payload["summary"]["auto_execute_total"]) + self.assertEqual(1, payload["summary"]["confirm_then_execute_total"]) + self.assertEqual(0, payload["summary"]["open_ui_total"]) + self.assertEqual(1, payload["summary"]["preview_only_total"]) + self.assertEqual(3, payload["summary"]["backend_handled_total"]) + self.assertEqual("attention", payload["summary"]["launch_status"]) + self.assertFalse(payload["summary"]["launch_ready"]) + self.assertEqual("recommendation:enable-scene-log-sync", payload["focus"]["entry_key"]) + self.assertEqual("execution_scene", payload["focus"]["focus_ref"]["kind"]) + self.assertEqual("scene.logs", payload["focus"]["primary_focus_ref"]["scene_key"]) + self.assertEqual("auto_execute", payload["entries"][0]["autopilot"]["recommendation"]) + self.assertIn("ops_observability_contract", payload["entries"][0]["contract_keys"]) + self.assertIn("ops_stack_diagnosis_contract", payload["entries"][0]["contract_keys"]) + self.assertEqual("scene.logs", payload["entries"][0]["primary_focus_ref"]["scene_key"]) + self.assertEqual("safe_auto", payload["entries"][1]["autopilot"]["automation_level"]) + self.assertIn("release_hub_contract", payload["entries"][1]["contract_keys"]) + + runbook_entry = payload["entries"][2] + self.assertEqual("runbook_sequence", runbook_entry["executor_kind"]) + self.assertEqual("confirm_then_execute", runbook_entry["autopilot"]["recommendation"]) + self.assertIn("ops_playbook_contract", runbook_entry["contract_keys"]) + self.assertIn("release_hub_contract", runbook_entry["contract_keys"]) + self.assertEqual("release_launchpad", runbook_entry["focus_ref"]["section"]) + self.assertEqual("worker", runbook_entry["primary_focus_ref"]["mode"]) + self.assertEqual("release_detail", runbook_entry["secondary_focus_ref"]["section"]) + self.assertEqual( + "/api/v1/ops/runbook/sequences/release_progression/execute", + runbook_entry["autopilot"]["target_api"], + ) + self.assertEqual( + { + "sequence_key": "release_progression", + "secondary": False, + "action_payload": { + "release_id": 7, + "focus_ref": { + "kind": "release_hub", + "release_id": 7, + "section": "release_launchpad", + "mode": "worker", + }, + }, + }, + runbook_entry["primary_execution"]["request_payload_preview"], + ) + self.assertEqual( + { + "sequence_key": "release_progression", + "secondary": True, + "action_payload": { + "release_id": 7, + "focus_ref": { + "kind": "release_hub", + "release_id": 7, + "section": "release_detail", + }, + }, + }, + runbook_entry["secondary_execution"]["request_payload_preview"], + ) + self.assertEqual("attention", payload["automation_coverage"]["launch_status"]) + self.assertEqual(2, payload["automation_coverage"]["safe_auto_total"]) + self.assertEqual(1, payload["automation_coverage"]["guarded_auto_total"]) + self.assertEqual(1, payload["automation_coverage"]["preview_only_total"]) + self.assertEqual(3, payload["automation_coverage"]["backend_handled_total"]) + + @patch("app.services.ops_service.get_ops_driver_feed") + def test_codex_brief_marks_mixed_and_blocked_entries(self, mock_get_ops_driver_feed) -> None: + mock_get_ops_driver_feed.return_value = { + "headline": "当前建议需要先复核。", + "go_live_summary": { + "go_live_status": "blocked", + "publish_ready": False, + "publish_status": "blocked", + "publish_status_label": "暂不可发布", + "operator_title": "先补执行面接管", + "next_step_action_code": "fix_managed_nodes", + "launchpad_recommended_target_node_code": "mainland-controller-01", + "launchpad_recommended_recovery_label": "补执行面接管", + "launchpad_recommended_recovery_summary": "先处理 launchpad 缺口。", + "launchpad_onboarding_bootstrap_pending_nodes": 1, + "launchpad_onboarding_acceptance_ready_nodes": 0, + }, + "activity_focus": [], + "entries": [ + { + "key": "recommendation:release-launchpad-fix-managed", + "kind": "driver_recommendation", + "lane": "do_now", + "lane_label": "建议动作", + "title": "先补执行面接管", + "summary": "先处理 launchpad 缺口。", + "reason": "当前动作可能直接 sync，也可能转为补接入。", + "status": "attention", + "level_label": "最高优先", + "tag_type": "danger", + "meta_text": "", + "detail_lines": [], + "node_codes": ["mainland-controller-01"], + "executor_kind": "driver_action", + "focus_ref": { + "kind": "release_hub", + "release_id": 9, + "section": "release_launchpad", + }, + "primary_focus_ref": { + "kind": "release_hub", + "release_id": 9, + "section": "default_rollout_gate", + }, + "primary_label": "补执行面接管", + "secondary_label": "", + "primary_action_code": "fix_managed_nodes", + "secondary_action_code": "", + "primary_node_codes": ["mainland-controller-01"], + "secondary_node_codes": [], + "primary_action_payload": {"channel": "stable"}, + "secondary_action_payload": {}, + }, + { + "key": "recommendation:inspect-participating", + "kind": "driver_recommendation", + "lane": "do_now", + "lane_label": "建议动作", + "title": "先看执行现场", + "summary": "先跑参与节点巡检。", + "reason": "缺少目标节点时不应误判为可自动执行。", + "status": "attention", + "level_label": "推荐", + "tag_type": "warning", + "meta_text": "", + "detail_lines": [], + "node_codes": [], + "executor_kind": "driver_action", + "primary_label": "执行标准巡检", + "secondary_label": "", + "primary_action_code": "run_inspection_participating", + "secondary_action_code": "", + "primary_node_codes": [], + "secondary_node_codes": [], + "primary_action_payload": {}, + "secondary_action_payload": {}, + }, + ], + } + + payload = get_ops_codex_brief() + + self.assertEqual("blocked", payload["summary"]["publish_status"]) + self.assertEqual("暂不可发布", payload["summary"]["publish_status_label"]) + self.assertEqual("mainland-controller-01", payload["summary"]["launchpad_recommended_target_node_code"]) + self.assertEqual("补执行面接管", payload["summary"]["launchpad_recommended_recovery_label"]) + self.assertEqual(1, payload["summary"]["launchpad_onboarding_bootstrap_pending_nodes"]) + self.assertEqual(0, payload["summary"]["launchpad_onboarding_acceptance_ready_nodes"]) + self.assertEqual(1, payload["summary"]["resolve_first_total"]) + self.assertEqual(1, payload["summary"]["blocked_total"]) + self.assertEqual("blocked", payload["summary"]["launch_status"]) + self.assertFalse(payload["summary"]["launch_ready"]) + self.assertEqual("mixed", payload["entries"][0]["autopilot"]["automation_level"]) + self.assertEqual("resolve_first", payload["entries"][0]["autopilot"]["recommendation"]) + self.assertEqual("default_rollout_gate", payload["entries"][0]["primary_focus_ref"]["section"]) + self.assertEqual(1, payload["automation_coverage"]["mixed_total"]) + self.assertEqual(1, payload["automation_coverage"]["blocked_total"]) + self.assertEqual(2, payload["automation_coverage"]["human_dependency_total"]) + + blocked_entry = payload["entries"][1] + self.assertTrue(blocked_entry["autopilot"]["blocked"]) + self.assertEqual(["node_codes"], blocked_entry["primary_execution"]["missing_fields"]) + self.assertEqual("blocked", blocked_entry["autopilot"]["recommendation"]) + + @patch("app.services.ops_service.get_ops_driver_feed") + def test_codex_brief_exposes_activity_focus_and_uses_it_as_fallback_focus(self, mock_get_ops_driver_feed) -> None: + mock_get_ops_driver_feed.return_value = { + "headline": "当前先盯住执行现场。", + "scene_log_observation": {}, + "entries": [], + "activity_focus": [ + { + "key": "activity:execution-scene:live", + "activity_key": "execution-scene:live", + "activity_kind": "execution_scene", + "title": "当前执行现场", + "subtitle": "运行中", + "summary": "当前参与检测节点 2 台，建议先看现场与日志回传。", + "status": "running", + "status_label": "执行中", + "meta_text": "执行/已领 2 / 待命 1", + "occurred_at": "2026-04-18 09:30:00", + "target_node_codes": ["mainland-worker-01", "mainland-controller-01"], + "focus_ref": { + "kind": "execution_scene", + "scene_key": "scene.logs", + }, + "source_focus_ref": {}, + "ui_intent": { + "kind": "focus_execution_scene", + "open_log_drawer": True, + }, + "ui_intent_kind": "focus_execution_scene", + "focus_action_code": "focus_activity_item", + "focus_action_payload": { + "activity_key": "execution-scene:live", + "kind": "execution_scene", + "status": "running", + "ui_intent": { + "kind": "focus_execution_scene", + "open_log_drawer": True, + }, + "focus_ref": { + "kind": "execution_scene", + "scene_key": "scene.logs", + }, + }, + "observation_only": True, + } + ], + } + + payload = get_ops_codex_brief() + + self.assertEqual(0, payload["summary"]["total"]) + self.assertEqual(1, payload["summary"]["activity_focus_total"]) + self.assertEqual("activity:execution-scene:live", payload["focus"]["entry_key"]) + self.assertEqual("open_ui", payload["focus"]["recommendation"]) + self.assertIn("ops_observability_contract", payload["focus"]["contract_keys"]) + self.assertEqual("execution_scene", payload["focus"]["focus_ref"]["kind"]) + self.assertEqual(1, len(payload["activity_focus"])) + self.assertEqual("focus_execution_scene", payload["activity_focus"][0]["ui_intent_kind"]) + self.assertEqual( + "ops_driver_contract", + payload["activity_focus"][0]["contract_navigation"]["primary_contract_key"], + ) + self.assertTrue(payload["activity_focus"][0]["observation_only"]) + + @patch("app.services.ops_service.get_ops_driver_feed") + def test_codex_brief_can_fallback_to_scene_log_observation_focus(self, mock_get_ops_driver_feed) -> None: + mock_get_ops_driver_feed.return_value = { + "headline": "先看现场日志。", + "entries": [], + "activity_focus": [], + "scene_log_observation": { + "status": "waiting_sample", + "status_label": "等待样本", + "summary": "日志回传已开启，但样本还没形成。", + "focus_ref": { + "kind": "node_scene_log", + "node_code": "mainland-worker-01", + "mode": "key", + "limit": 120, + }, + "contract_keys": ["ops_observability_contract", "ops_stack_diagnosis_contract"], + }, + } + + payload = get_ops_codex_brief() + + self.assertEqual("scene-log-observation", payload["focus"]["entry_key"]) + self.assertEqual("open_ui", payload["focus"]["recommendation"]) + self.assertEqual("node_scene_log", payload["focus"]["focus_ref"]["kind"]) + self.assertEqual("mainland-worker-01", payload["focus"]["focus_ref"]["node_code"]) + self.assertEqual("waiting_sample", payload["scene_log_observation"]["status"]) + + @patch("app.services.ops_service.get_ops_driver_feed") + def test_codex_brief_marks_bootstrap_run_as_agent_protocol_flow(self, mock_get_ops_driver_feed) -> None: + mock_get_ops_driver_feed.return_value = { + "headline": "先执行节点接入收口。", + "activity_focus": [], + "entries": [ + { + "key": "priority:handover-gap-bootstrap_run", + "kind": "priority_recommendation", + "lane": "do_now", + "lane_label": "当前优先", + "title": "先补接管缺口", + "summary": "Launchpad 已明确给出首台节点的接入收口动作。", + "reason": "mainland-worker-01 还缺接入收口，先执行 bootstrap。", + "status": "attention", + "level_label": "最高优先", + "tag_type": "warning", + "meta_text": "overview.recommendation", + "detail_lines": ["当前已收敛到单节点接入动作。"], + "node_codes": ["mainland-worker-01"], + "executor_kind": "driver_action", + "focus_ref": {"kind": "managed_node", "node_code": "mainland-worker-01"}, + "primary_focus_ref": {"kind": "managed_node", "node_code": "mainland-worker-01"}, + "secondary_focus_ref": {"kind": "managed_node", "node_code": "mainland-worker-01", "section": "bootstrap_plan"}, + "primary_label": "跑接入收口", + "secondary_label": "查看 Bootstrap 计划", + "primary_action_code": "bootstrap_run", + "secondary_action_code": "create_bootstrap_plan", + "primary_node_codes": ["mainland-worker-01"], + "secondary_node_codes": ["mainland-worker-01"], + "primary_action_payload": {"node_code": "mainland-worker-01"}, + "secondary_action_payload": {"node_code": "mainland-worker-01"}, + } + ], + } + + payload = get_ops_codex_brief() + + self.assertEqual("priority:handover-gap-bootstrap_run", payload["focus"]["entry_key"]) + self.assertEqual("resolve_first", payload["focus"]["recommendation"]) + self.assertIn("ops_agent_protocol", payload["entries"][0]["contract_keys"]) + self.assertEqual("bootstrap_run", payload["entries"][0]["primary_action_code"]) + self.assertEqual("mainland-worker-01", payload["entries"][0]["primary_execution"]["node_codes"][0]) + self.assertEqual("/api/v1/ops/driver-actions/execute", payload["entries"][0]["autopilot"]["target_api"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/domain-api/tests/test_ops_service_contract_registry.py b/domain-api/tests/test_ops_service_contract_registry.py new file mode 100644 index 0000000..340b60d --- /dev/null +++ b/domain-api/tests/test_ops_service_contract_registry.py @@ -0,0 +1,295 @@ +import unittest +from unittest.mock import patch + +from app.services.ops_service import ( + get_ops_blueprint, + get_ops_capabilities, + get_ops_contract_detail, + get_ops_contract_registry, + get_ops_runbook, +) + + +class OpsServiceContractRegistryTests(unittest.TestCase): + def test_contract_registry_lists_all_frozen_ops_contracts(self) -> None: + payload = get_ops_contract_registry() + + self.assertEqual("2026-04-18", payload["registry_version"]) + self.assertEqual("v1", payload["schema_version"]) + self.assertEqual(11, payload["contracts_total"]) + self.assertEqual( + [ + "ops_job_contract", + "ops_agent_protocol", + "release_hub_contract", + "ops_driver_contract", + "ops_playbook_contract", + "ops_observability_contract", + "ops_stack_diagnosis_contract", + "ops_doctor_decision_contract", + "ops_go_live_signoff_contract", + "ops_go_live_bundle_contract", + "ops_go_live_review_contract", + ], + [item["key"] for item in payload["contracts"]], + ) + self.assertIn("/api/v1/ops/contracts/{contract_key}", payload["discovery_entrypoints"]) + self.assertEqual( + "docs/schemas/ops_job_contract.md", + payload["contracts_by_key"]["ops_job_contract"]["schema_doc_path"], + ) + self.assertIn( + "/api/v1/ops/jobs/{job_id}/dispatch", + payload["contracts_by_key"]["ops_job_contract"]["discovery_endpoints"], + ) + self.assertEqual( + "docs/schemas/ops_agent_protocol.md", + payload["contracts_by_key"]["ops_agent_protocol"]["schema_doc_path"], + ) + self.assertIn( + "/api/v1/ops/agent/bootstrap-plan", + payload["contracts_by_key"]["ops_agent_protocol"]["discovery_endpoints"], + ) + self.assertIn( + "/api/v1/ops/nodes/{node_code}/delivery-queue", + payload["contracts_by_key"]["ops_agent_protocol"]["discovery_endpoints"], + ) + self.assertIn( + "/api/v1/ops/nodes/{node_code}/delivery-queue/records", + payload["contracts_by_key"]["ops_agent_protocol"]["discovery_endpoints"], + ) + self.assertIn( + "/api/v1/ops/nodes/{node_code}/delivery-queue/flush", + payload["contracts_by_key"]["ops_agent_protocol"]["discovery_endpoints"], + ) + self.assertIn( + "/api/v1/ops/nodes/{node_code}/delivery-queue/replay", + payload["contracts_by_key"]["ops_agent_protocol"]["discovery_endpoints"], + ) + self.assertIn( + "/api/v1/ops/nodes/{node_code}/delivery-queue/records/{record_id}/replay", + payload["contracts_by_key"]["ops_agent_protocol"]["discovery_endpoints"], + ) + self.assertIn( + "/api/v1/ops/nodes/{node_code}/delivery-queue/records/{record_id}/discard", + payload["contracts_by_key"]["ops_agent_protocol"]["discovery_endpoints"], + ) + self.assertEqual( + "/api/v1/ops/releases/launchpad", + payload["contracts_by_key"]["release_hub_contract"]["primary_endpoint"], + ) + self.assertIn( + "/api/v1/ops/codex-brief", + payload["contracts_by_key"]["ops_driver_contract"]["discovery_endpoints"], + ) + self.assertIn( + "/api/v1/ops/driver-actions/preview", + payload["contracts_by_key"]["ops_driver_contract"]["discovery_endpoints"], + ) + self.assertIn( + "/api/v1/ops/driver-actions/resolve", + payload["contracts_by_key"]["ops_driver_contract"]["discovery_endpoints"], + ) + self.assertIn( + "/api/v1/ops/driver-actions/execute-resolved", + payload["contracts_by_key"]["ops_driver_contract"]["discovery_endpoints"], + ) + self.assertIn( + "/api/v1/ops/codex-actions/resolve", + payload["contracts_by_key"]["ops_driver_contract"]["discovery_endpoints"], + ) + self.assertIn( + "/api/v1/ops/codex-actions/execute", + payload["contracts_by_key"]["ops_driver_contract"]["discovery_endpoints"], + ) + self.assertEqual( + "docs/schemas/ops_playbook_contract.md", + payload["contracts_by_key"]["ops_playbook_contract"]["schema_doc_path"], + ) + self.assertIn( + "/api/v1/ops/playbooks/execute", + payload["contracts_by_key"]["ops_playbook_contract"]["discovery_endpoints"], + ) + self.assertIn( + "/api/v1/ops/playbook-runs/{run_code}/events", + payload["contracts_by_key"]["ops_playbook_contract"]["discovery_endpoints"], + ) + self.assertEqual( + "docs/schemas/ops_observability_contract.md", + payload["contracts_by_key"]["ops_observability_contract"]["schema_doc_path"], + ) + self.assertIn( + "/api/v1/ops/inspection-overview", + payload["contracts_by_key"]["ops_observability_contract"]["discovery_endpoints"], + ) + self.assertIn( + "/api/v1/ops/nodes/{node_code}/scene-log", + payload["contracts_by_key"]["ops_observability_contract"]["discovery_endpoints"], + ) + self.assertEqual( + "docs/schemas/ops_stack_diagnosis_contract.md", + payload["contracts_by_key"]["ops_stack_diagnosis_contract"]["schema_doc_path"], + ) + self.assertIn( + "/api/v1/ops/stack-diagnosis", + payload["contracts_by_key"]["ops_stack_diagnosis_contract"]["discovery_endpoints"], + ) + self.assertIn( + "ops_driver_contract", + payload["contracts_by_key"]["ops_stack_diagnosis_contract"]["related_contract_keys"], + ) + self.assertEqual( + "docs/schemas/ops_doctor_decision_contract.md", + payload["contracts_by_key"]["ops_doctor_decision_contract"]["schema_doc_path"], + ) + self.assertIn( + "/api/v1/ops/doctor-decision", + payload["contracts_by_key"]["ops_doctor_decision_contract"]["discovery_endpoints"], + ) + self.assertIn( + "ops_stack_diagnosis_contract", + payload["contracts_by_key"]["ops_doctor_decision_contract"]["related_contract_keys"], + ) + self.assertEqual( + "docs/schemas/ops_go_live_signoff_contract.md", + payload["contracts_by_key"]["ops_go_live_signoff_contract"]["schema_doc_path"], + ) + self.assertIn( + "/api/v1/ops/go-live-signoff", + payload["contracts_by_key"]["ops_go_live_signoff_contract"]["discovery_endpoints"], + ) + self.assertIn( + "ops_stack_diagnosis_contract", + payload["contracts_by_key"]["ops_go_live_signoff_contract"]["related_contract_keys"], + ) + self.assertEqual( + "docs/schemas/ops_go_live_bundle_contract.md", + payload["contracts_by_key"]["ops_go_live_bundle_contract"]["schema_doc_path"], + ) + self.assertIn( + "/api/v1/ops/go-live-bundle", + payload["contracts_by_key"]["ops_go_live_bundle_contract"]["discovery_endpoints"], + ) + self.assertIn( + "ops_go_live_signoff_contract", + payload["contracts_by_key"]["ops_go_live_bundle_contract"]["related_contract_keys"], + ) + self.assertEqual( + "docs/schemas/ops_go_live_review_contract.md", + payload["contracts_by_key"]["ops_go_live_review_contract"]["schema_doc_path"], + ) + self.assertIn( + "/api/v1/ops/go-live-review", + payload["contracts_by_key"]["ops_go_live_review_contract"]["discovery_endpoints"], + ) + self.assertIn( + "ops_go_live_bundle_contract", + payload["contracts_by_key"]["ops_go_live_review_contract"]["related_contract_keys"], + ) + self.assertIn( + "ops_doctor_decision_contract", + payload["contracts_by_key"]["ops_go_live_review_contract"]["related_contract_keys"], + ) + + def test_contract_detail_exposes_related_contracts(self) -> None: + payload = get_ops_contract_detail("ops_stack_diagnosis_contract") + + self.assertEqual("2026-04-18", payload["registry_version"]) + self.assertEqual("/api/v1/ops/contracts/ops_stack_diagnosis_contract", payload["detail_endpoint"]) + self.assertEqual("ops_stack_diagnosis_contract", payload["contract"]["key"]) + self.assertIn("ops_driver_contract", payload["related_contract_keys"]) + self.assertIn("release_hub_contract", payload["related_contract_keys"]) + self.assertEqual( + "docs/schemas/ops_driver_contract.md", + next( + item["schema_doc_path"] + for item in payload["related_contracts"] + if item["key"] == "ops_driver_contract" + ), + ) + self.assertEqual({}, get_ops_contract_detail("missing-contract")) + + def test_capabilities_blueprint_and_runbook_expose_contract_registry(self) -> None: + capabilities = get_ops_capabilities() + contract_registry = capabilities["contract_registry"] + + self.assertEqual("/api/v1/ops/contracts", contract_registry["endpoint"]) + self.assertEqual("/api/v1/ops/contracts/{contract_key}", contract_registry["detail_endpoint_pattern"]) + self.assertEqual(11, contract_registry["contracts_total"]) + self.assertIn("ops_job_contract", contract_registry["contract_keys"]) + self.assertIn("ops_driver_contract", contract_registry["contract_keys"]) + self.assertIn("ops_playbook_contract", contract_registry["contract_keys"]) + self.assertIn("ops_observability_contract", contract_registry["contract_keys"]) + self.assertIn("ops_stack_diagnosis_contract", contract_registry["contract_keys"]) + self.assertIn("ops_doctor_decision_contract", contract_registry["contract_keys"]) + self.assertIn("ops_go_live_signoff_contract", contract_registry["contract_keys"]) + self.assertIn("ops_go_live_bundle_contract", contract_registry["contract_keys"]) + self.assertIn("ops_go_live_review_contract", contract_registry["contract_keys"]) + remote_log_capability = next( + item for item in capabilities["capabilities"] if item["key"] == "remote_log_sync" + ) + self.assertEqual("ready", remote_log_capability["current_state"]) + self.assertEqual( + "docs/schemas/release_hub_contract.md", + contract_registry["schema_docs"]["release_hub_contract"], + ) + self.assertEqual( + "docs/schemas/ops_playbook_contract.md", + contract_registry["schema_docs"]["ops_playbook_contract"], + ) + self.assertEqual( + "docs/schemas/ops_job_contract.md", + contract_registry["schema_docs"]["ops_job_contract"], + ) + self.assertEqual( + "docs/schemas/ops_observability_contract.md", + contract_registry["schema_docs"]["ops_observability_contract"], + ) + self.assertEqual( + "docs/schemas/ops_stack_diagnosis_contract.md", + contract_registry["schema_docs"]["ops_stack_diagnosis_contract"], + ) + self.assertEqual( + "docs/schemas/ops_doctor_decision_contract.md", + contract_registry["schema_docs"]["ops_doctor_decision_contract"], + ) + self.assertEqual( + "docs/schemas/ops_go_live_signoff_contract.md", + contract_registry["schema_docs"]["ops_go_live_signoff_contract"], + ) + self.assertEqual( + "docs/schemas/ops_go_live_bundle_contract.md", + contract_registry["schema_docs"]["ops_go_live_bundle_contract"], + ) + self.assertEqual( + "docs/schemas/ops_go_live_review_contract.md", + contract_registry["schema_docs"]["ops_go_live_review_contract"], + ) + + blueprint = get_ops_blueprint() + self.assertEqual("2026-04-18", blueprint["contract_registry"]["registry_version"]) + self.assertEqual(11, blueprint["contract_registry"]["contracts_total"]) + + with patch("app.services.ops_service.get_runtime_status") as mock_runtime_status, patch( + "app.services.ops_service.list_managed_nodes_with_agent_state" + ) as mock_list_managed_nodes_with_agent_state, patch( + "app.services.ops_service.get_release_launchpad" + ) as mock_get_release_launchpad, patch( + "app.services.ops_service._attach_ops_runbook_sequence_resolutions" + ) as mock_attach_sequences: + mock_runtime_status.return_value = { + "readiness": {"status": "ready", "summary": "ok"}, + "worker": {"running": True}, + "sync_agent": {"running": True}, + "detect": {}, + } + mock_list_managed_nodes_with_agent_state.return_value = {"nodes": [], "summary": {}} + mock_get_release_launchpad.return_value = {} + mock_attach_sequences.return_value = [] + runbook = get_ops_runbook() + + self.assertIn("GET /api/v1/ops/contracts", runbook["fast_checks"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/domain-api/tests/test_ops_service_doctor_decision.py b/domain-api/tests/test_ops_service_doctor_decision.py new file mode 100644 index 0000000..703a065 --- /dev/null +++ b/domain-api/tests/test_ops_service_doctor_decision.py @@ -0,0 +1,149 @@ +from __future__ import annotations + +import json +import tempfile +import unittest +from pathlib import Path +from unittest.mock import patch + +from app.services.ops_service import get_ops_doctor_decision + + +class OpsServiceDoctorDecisionTests(unittest.TestCase): + @patch("app.services.ops_service.get_ops_contract_registry") + @patch("app.services.ops_service._build_live_doctor_payload") + @patch("app.services.ops_service._find_latest_go_live_bundle_manifest") + def test_doctor_decision_falls_back_to_live_payload_when_bundle_missing( + self, + mock_find_latest_go_live_bundle_manifest, + mock_build_live_doctor_payload, + mock_get_ops_contract_registry, + ) -> None: + mock_find_latest_go_live_bundle_manifest.return_value = (None, Path("/tmp/go-live-bundles")) + mock_build_live_doctor_payload.return_value = ( + { + "status": "attention", + "reason_code": "live_fallback", + "headline": "当前先补 attention 项", + "detail": "需要先顺着 stack next 收口。", + "preferred_surface": "stack-summary", + "next_action_code": "review_stack_next", + "recommended_commands": ["bash primary"], + "evidence": {}, + }, + { + "ok": False, + "required_failures": [], + "optional_unavailable": [], + "scene_log_reports_total": 0, + "scene_log_reports_ok": 0, + "scene_log_status_counts": {}, + "scene_log_reports": [], + "contract_surface_gaps": [], + }, + ) + mock_get_ops_contract_registry.return_value = { + "contracts_by_key": { + "ops_doctor_decision_contract": {"version": "v1"}, + } + } + + payload = get_ops_doctor_decision(base_url="http://127.0.0.1:8100") + + self.assertEqual("attention", payload["status"]) + self.assertEqual("主决策待复核", payload["status_label"]) + self.assertEqual("live_fallback", payload["source"]) + self.assertEqual("review_stack_next", payload["decision"]["next_action_code"]) + self.assertIn("doctor-decision", payload["recommended_commands"]["doctor_decision"]) + self.assertEqual("bash primary", payload["recommended_commands"]["primary_recovery"]) + self.assertEqual( + "bash domain-api/deploy/multi-region/drive_ops_center.sh stack-diagnosis http://127.0.0.1:8100 summary", + payload["recommended_commands"]["stack_check"], + ) + self.assertEqual( + payload["recommended_commands"]["stack_check"], + payload["recommended_commands"]["stack_diagnosis"], + ) + + @patch("app.services.ops_service.get_ops_contract_registry") + def test_doctor_decision_reads_bundle_doctor_artifact_when_available(self, mock_get_ops_contract_registry) -> None: + mock_get_ops_contract_registry.return_value = { + "contracts_by_key": { + "ops_doctor_decision_contract": {"version": "v1"}, + } + } + + with tempfile.TemporaryDirectory() as temp_dir: + report_dir = Path(temp_dir) / "go-live-bundle-001" + report_dir.mkdir(parents=True, exist_ok=True) + doctor_artifact_path = report_dir / "07_doctor_decision.json" + doctor_artifact_path.write_text( + json.dumps( + { + "summary": { + "ok": False, + "required_failures": ["06_contracts_registry.txt"], + "optional_unavailable": ["12_activity_preview.txt"], + "scene_log_reports_total": 2, + "scene_log_reports_ok": 1, + "scene_log_status_counts": {"ok": 1, "missing": 1}, + "scene_log_reports": [ + {"node_code": "mainland-worker-01", "status": "ok", "ok": True}, + {"node_code": "mainland-controller-01", "status": "missing", "ok": False}, + ], + "contract_surface_gaps": ["activity-stream"], + "launchpad_recommended_target_node_code": "mainland-worker-01", + "launchpad_recommended_recovery_label": "先补协议面", + "launchpad_recommended_recovery_summary": "先补 driver / activity 的协议导航。", + }, + "decision": { + "status": "blocked", + "reason_code": "contracts_registry_unavailable", + "headline": "contract registry 当前不可用", + "detail": "先恢复 contracts registry 再继续自动驾驶。", + "preferred_surface": "contracts", + "next_action_code": "restore_contract_registry", + "recommended_commands": [ + "bash domain-api/deploy/multi-region/drive_ops_center.sh contracts http://127.0.0.1:8100", + "bash domain-api/deploy/multi-region/drive_ops_center.sh runtime-refresh-recover http://127.0.0.1:8100", + ], + "evidence": {"surface_http_status": {"contracts_registry": 503}}, + }, + }, + ensure_ascii=False, + indent=2, + ), + encoding="utf-8", + ) + manifest_path = report_dir / "manifest.json" + manifest_path.write_text( + json.dumps( + { + "report_dir": str(report_dir), + "artifacts": [ + {"key": "doctor_decision", "path": str(doctor_artifact_path), "ok": True}, + ], + }, + ensure_ascii=False, + indent=2, + ), + encoding="utf-8", + ) + + with patch( + "app.services.ops_service._find_latest_go_live_bundle_manifest", + return_value=(manifest_path, report_dir), + ): + payload = get_ops_doctor_decision(base_url="http://127.0.0.1:8100") + + self.assertEqual("blocked", payload["status"]) + self.assertEqual("bundle_manifest", payload["source"]) + self.assertEqual("contracts", payload["decision"]["preferred_surface"]) + self.assertEqual(["06_contracts_registry.txt"], payload["summary"]["required_failures"]) + self.assertEqual(["mainland-controller-01"], payload["summary"]["scene_log_problem_node_codes"]) + self.assertIn("contracts", payload["recommended_commands"]["primary_recovery"]) + self.assertIn(str(manifest_path), payload["recommended_commands"]["doctor_decision"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/domain-api/tests/test_ops_service_driver_action_preview.py b/domain-api/tests/test_ops_service_driver_action_preview.py new file mode 100644 index 0000000..5041758 --- /dev/null +++ b/domain-api/tests/test_ops_service_driver_action_preview.py @@ -0,0 +1,274 @@ +import unittest +from unittest.mock import patch + +from app.services.ops_service import preview_driver_action + + +class OpsServiceDriverActionPreviewTests(unittest.TestCase): + def test_preview_driver_action_builds_safe_auto_contract_preview(self) -> None: + ok, message, data = preview_driver_action( + { + "source_label": "overview-recommendation", + "title": "先补现场日志回传", + "summary": "优先打开关键日志回传。", + "reason": "当前还看不到现场过程日志。", + "executor_kind": "driver_action", + "focus_ref": { + "kind": "execution_scene", + "scene_key": "scene.logs", + }, + "primary_focus_ref": { + "kind": "execution_scene", + "scene_key": "scene.logs", + }, + "primary_label": "开启关键回传", + "primary_action_code": "enable_log_sync_key", + "primary_action_payload": {}, + "requested_by": "web-ui/ops-center", + } + ) + + self.assertTrue(ok, message) + self.assertEqual("驾驶动作预览已生成", message) + self.assertEqual("ops_driver_contract", data["contract_key"]) + self.assertEqual("/api/v1/ops/driver-feed", data["contract_primary_endpoint"]) + self.assertEqual("/api/v1/ops/driver-actions/preview", data["preview_endpoint"]) + self.assertEqual("driver_action", data["executor_kind"]) + self.assertEqual( + "ops-driver -> driver-action execute -> playbook / ui-intent / settings", + data["execution_chain"], + ) + + primary = data["primary"] + self.assertEqual("开启关键回传", primary["label"]) + self.assertEqual("enable_log_sync_key", primary["action_code"]) + self.assertEqual("/api/v1/ops/driver-actions/execute", primary["target_api"]) + self.assertEqual("可自动执行", primary["recommendation_label"]) + self.assertEqual("safe_auto", primary["automation_level_label"]) + self.assertEqual("low", primary["risk_level_label"]) + self.assertEqual("execution_scene", data["focus_ref"]["kind"]) + self.assertEqual("execution_scene", data["primary_focus_ref"]["kind"]) + self.assertEqual("execution_scene", primary["focus_ref"]["kind"]) + self.assertEqual( + { + "action_code": "enable_log_sync_key", + "node_codes": [], + "action_payload": { + "focus_ref": { + "kind": "execution_scene", + "scene_key": "scene.logs", + } + }, + "requested_by": "web-ui/ops-center", + }, + primary["request_payload"], + ) + self.assertFalse(data["secondary"]) + + def test_preview_driver_action_marks_missing_required_fields_as_blocked(self) -> None: + ok, message, data = preview_driver_action( + { + "source_label": "driver-feed", + "title": "打开全量现场取证", + "summary": "当前需要进入取证工作区。", + "reason": "需要明确 playbook 上下文。", + "executor_kind": "driver_action", + "primary_label": "打开取证", + "primary_action_code": "open_playbook_dialog", + "primary_action_payload": {}, + "requested_by": "web-ui/ops-center", + } + ) + + self.assertTrue(ok, message) + primary = data["primary"] + self.assertEqual("当前阻断", primary["recommendation_label"]) + self.assertTrue(primary["execution_profile"]["blocked"]) + self.assertEqual(["playbook_key"], primary["execution_profile"]["missing_fields"]) + + def test_preview_driver_action_marks_release_launchpad_review_as_safe_auto(self) -> None: + ok, message, data = preview_driver_action( + { + "source_label": "overview-recommendation", + "title": "查看 Worker 预案", + "summary": "当前建议先复核 Worker launchpad 预案。", + "reason": "需要先判断目标节点、门禁和阻断缺口。", + "executor_kind": "driver_action", + "focus_ref": { + "kind": "release_hub", + "release_id": 88, + "section": "release_launchpad", + "mode": "worker", + }, + "primary_focus_ref": { + "kind": "release_hub", + "release_id": 88, + "section": "release_launchpad", + "mode": "worker", + }, + "primary_label": "查看 Worker 预案", + "primary_action_code": "review_smart_rollout_preview", + "primary_action_payload": { + "channel": "stable", + }, + "requested_by": "web-ui/ops-center", + } + ) + + self.assertTrue(ok, message) + self.assertEqual("驾驶动作预览已生成", message) + primary = data["primary"] + self.assertEqual("review_smart_rollout_preview", primary["action_code"]) + self.assertEqual("可自动执行", primary["recommendation_label"]) + self.assertEqual("safe_auto", primary["automation_level_label"]) + self.assertEqual("low", primary["risk_level_label"]) + self.assertFalse(primary["execution_profile"]["ui_only"]) + self.assertEqual("/api/v1/ops/driver-actions/execute", primary["target_api"]) + + def test_preview_driver_action_marks_handover_gap_as_safe_auto(self) -> None: + ok, message, data = preview_driver_action( + { + "source_label": "overview-recommendation", + "title": "先补接管缺口", + "summary": "当前存在待接管节点。", + "reason": "需要先查看首台缺口节点的接管详情。", + "executor_kind": "driver_action", + "focus_ref": { + "kind": "managed_node", + "node_code": "mainland-worker-01", + }, + "primary_focus_ref": { + "kind": "managed_node", + "node_code": "mainland-worker-01", + }, + "primary_label": "查看接管详情", + "primary_action_code": "handover_first_gap", + "primary_action_payload": { + "node_code": "mainland-worker-01", + }, + "requested_by": "web-ui/ops-center", + } + ) + + self.assertTrue(ok, message) + primary = data["primary"] + self.assertEqual("handover_first_gap", primary["action_code"]) + self.assertEqual("可自动执行", primary["recommendation_label"]) + self.assertEqual("safe_auto", primary["automation_level_label"]) + self.assertEqual("low", primary["risk_level_label"]) + self.assertFalse(primary["execution_profile"]["ui_only"]) + + @patch("app.services.ops_service.get_ops_runbook") + def test_preview_driver_action_supports_runbook_sequence_primary_and_secondary(self, mock_get_ops_runbook) -> None: + mock_get_ops_runbook.return_value = { + "control_sequences": [ + { + "key": "release_progression", + "title": "版本发布与放量", + "status": "ready", + "status_label": "可发 Worker 灰度", + "summary": "当前可以进入正式发布路径。", + "reason": "统一回到 Release / Rollout 闭环。", + "target_scope_label": "Release / Rollout / 2026.04.18", + "step_titles": ["确认发布包", "发灰度"], + "target_node_codes": ["mainland-controller-01"], + "focus_ref": { + "kind": "release_hub", + "release_id": 7, + "section": "release_launchpad", + }, + "primary_label": "Worker 灰度", + "primary_action_code": "create_release_rollout_worker", + "primary_action_payload": {"release_id": 7}, + "primary_focus_ref": { + "kind": "release_hub", + "release_id": 7, + "section": "release_launchpad", + "mode": "worker", + }, + "secondary_label": "查看版本区", + "secondary_action_code": "focus_release_hub", + "secondary_action_payload": {"release_id": 7}, + "secondary_focus_ref": { + "kind": "release_hub", + "release_id": 7, + "section": "release_detail", + }, + } + ] + } + + ok, message, data = preview_driver_action( + { + "source_label": "driver-feed", + "title": "版本发布与放量", + "summary": "当前可以进入正式发布路径。", + "reason": "统一回到 Release / Rollout 闭环。", + "executor_kind": "runbook_sequence", + "sequence_key": "release_progression", + "primary_label": "Worker 灰度", + "requested_by": "web-ui/ops-center", + } + ) + + self.assertTrue(ok, message) + self.assertEqual("驾驶动作预览已生成", message) + self.assertEqual("runbook_sequence", data["executor_kind"]) + self.assertEqual("runbook sequence", data["executor_kind_label"]) + self.assertEqual( + "ops-driver -> runbook-sequence -> resolved driver action / ui-intent", + data["execution_chain"], + ) + self.assertEqual("create_release_rollout_worker", data["resolved_primary"]["driver_action_code"]) + self.assertEqual("release_launchpad", data["focus_ref"]["section"]) + self.assertEqual("worker", data["primary_focus_ref"]["mode"]) + self.assertEqual("release_detail", data["secondary_focus_ref"]["section"]) + + primary = data["primary"] + self.assertEqual("/api/v1/ops/runbook/sequences/release_progression/execute", primary["target_api"]) + self.assertEqual("建议确认后执行", primary["recommendation_label"]) + self.assertEqual("guarded_auto", primary["automation_level_label"]) + self.assertEqual("worker", primary["focus_ref"]["mode"]) + self.assertEqual( + { + "secondary": False, + "requested_by": "web-ui/ops-center", + "node_codes": ["mainland-controller-01"], + "action_payload": { + "release_id": 7, + "focus_ref": { + "kind": "release_hub", + "release_id": 7, + "section": "release_launchpad", + "mode": "worker", + }, + }, + }, + primary["request_payload"], + ) + + secondary = data["secondary"] + self.assertEqual("查看版本区", secondary["label"]) + self.assertEqual("focus_release_hub", secondary["action_code"]) + self.assertEqual("release_detail", secondary["focus_ref"]["section"]) + self.assertEqual( + { + "secondary": True, + "requested_by": "web-ui/ops-center", + "node_codes": ["mainland-controller-01"], + "action_payload": { + "release_id": 7, + "focus_ref": { + "kind": "release_hub", + "release_id": 7, + "section": "release_detail", + }, + }, + }, + secondary["request_payload"], + ) + self.assertEqual("可自动执行", secondary["recommendation_label"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/domain-api/tests/test_ops_service_driver_action_resolve.py b/domain-api/tests/test_ops_service_driver_action_resolve.py new file mode 100644 index 0000000..1eda247 --- /dev/null +++ b/domain-api/tests/test_ops_service_driver_action_resolve.py @@ -0,0 +1,433 @@ +from __future__ import annotations + +import unittest +from unittest.mock import patch + +from app.services.ops_service import execute_resolved_driver_action, resolve_driver_action_request + + +def _build_preview_data( + *, + executor_kind: str, + primary_action_code: str, + recommendation: str, + automation_level: str, + target_api: str, + primary_request_payload: dict, + sequence_key: str = "", + secondary_action_code: str = "", + secondary_request_payload: dict | None = None, + secondary_recommendation: str = "", + secondary_automation_level: str = "", + secondary_target_api: str = "", + focus_ref: dict | None = None, + primary_focus_ref: dict | None = None, + secondary_focus_ref: dict | None = None, +) -> dict: + return { + "executor_kind": executor_kind, + "sequence_key": sequence_key, + "execution_chain": "ops-driver -> preview", + "focus_ref": dict(focus_ref or {}), + "primary_focus_ref": dict(primary_focus_ref or {}), + "secondary_focus_ref": dict(secondary_focus_ref or {}), + "primary": { + "label": "主动作", + "action_code": primary_action_code, + "node_codes": ["mainland-controller-01"], + "target_api": target_api, + "target_method": "POST", + "request_payload": dict(primary_request_payload or {}), + "focus_ref": dict(primary_focus_ref or {}), + "execution_profile": { + "action_code": primary_action_code, + "backend_handled": True, + "automation_level": automation_level, + "recommendation": recommendation, + "executor_mode_hint": "test-mode", + "risk_level": "low" if recommendation == "auto_execute" else "high", + "confirm_required": recommendation == "confirm_then_execute", + "ui_only": recommendation == "open_ui", + "blocked": recommendation == "blocked", + "requires_fields": [], + "missing_fields": [], + "reason": "preview reason", + }, + }, + "secondary": ( + { + "label": "次动作", + "action_code": secondary_action_code, + "node_codes": ["mainland-controller-01"], + "target_api": secondary_target_api or target_api, + "target_method": "POST", + "request_payload": dict(secondary_request_payload or {}), + "focus_ref": dict(secondary_focus_ref or {}), + "execution_profile": { + "action_code": secondary_action_code, + "backend_handled": True, + "automation_level": secondary_automation_level or automation_level, + "recommendation": secondary_recommendation or recommendation, + "executor_mode_hint": "test-mode", + "risk_level": "low" if (secondary_recommendation or recommendation) == "auto_execute" else "high", + "confirm_required": (secondary_recommendation or recommendation) == "confirm_then_execute", + "ui_only": (secondary_recommendation or recommendation) == "open_ui", + "blocked": (secondary_recommendation or recommendation) == "blocked", + "requires_fields": [], + "missing_fields": [], + "reason": "preview reason", + }, + } + if secondary_action_code + else {} + ), + } + + +class OpsServiceDriverActionResolveTests(unittest.TestCase): + @patch("app.services.ops_service.preview_driver_action") + def test_resolve_driver_action_request_returns_gate_for_safe_auto(self, mock_preview_driver_action) -> None: + request_payload = { + "action_code": "enable_log_sync_key", + "node_codes": ["mainland-controller-01"], + "action_payload": {}, + "requested_by": "test/driver-action", + } + mock_preview_driver_action.return_value = ( + True, + "ok", + _build_preview_data( + executor_kind="driver_action", + primary_action_code="enable_log_sync_key", + recommendation="auto_execute", + automation_level="safe_auto", + target_api="/api/v1/ops/driver-actions/execute", + primary_request_payload=request_payload, + focus_ref={ + "kind": "execution_scene", + "scene_key": "scene.logs", + }, + primary_focus_ref={ + "kind": "execution_scene", + "scene_key": "scene.logs", + }, + ), + ) + + ok, message, data = resolve_driver_action_request( + { + "source_label": "driver-feed", + "title": "开启关键日志回传", + "focus_ref": { + "kind": "execution_scene", + "scene_key": "scene.logs", + }, + "primary_focus_ref": { + "kind": "execution_scene", + "scene_key": "scene.logs", + }, + "primary_action_code": "enable_log_sync_key", + "primary_action_payload": {}, + "requested_by": "test/driver-action", + } + ) + + self.assertTrue(ok) + self.assertEqual("驾驶动作解析完成", message) + self.assertEqual("execute_now", data["gate"]["decision"]) + self.assertTrue(data["gate"]["will_execute"]) + self.assertEqual("/api/v1/ops/driver-actions/execute", data["gate"]["target_api"]) + self.assertEqual("enable_log_sync_key", data["preview_request"]["primary_action_code"]) + self.assertEqual("execution_scene", data["preview_request"]["focus_ref"]["kind"]) + self.assertEqual("scene.logs", data["gate"]["focus_ref"]["scene_key"]) + self.assertEqual("scene.logs", data["focus_ref"]["scene_key"]) + + @patch("app.services.ops_service.preview_driver_action") + def test_resolve_driver_action_request_requires_confirmation_for_guarded_runbook(self, mock_preview_driver_action) -> None: + request_payload = { + "secondary": False, + "requested_by": "test/driver-action", + "action_payload": {"release_id": 7}, + } + mock_preview_driver_action.return_value = ( + True, + "ok", + _build_preview_data( + executor_kind="runbook_sequence", + primary_action_code="create_release_rollout_worker", + recommendation="confirm_then_execute", + automation_level="guarded_auto", + target_api="/api/v1/ops/runbook/sequences/release_progression/execute", + primary_request_payload=request_payload, + sequence_key="release_progression", + focus_ref={ + "kind": "release_hub", + "release_id": 7, + "section": "release_launchpad", + }, + primary_focus_ref={ + "kind": "release_hub", + "release_id": 7, + "section": "release_launchpad", + "mode": "worker", + }, + ), + ) + + ok, _, data = resolve_driver_action_request( + { + "source_label": "driver-feed", + "title": "版本发布与放量", + "executor_kind": "runbook_sequence", + "sequence_key": "release_progression", + "focus_ref": { + "kind": "release_hub", + "release_id": 7, + "section": "release_launchpad", + }, + "primary_focus_ref": { + "kind": "release_hub", + "release_id": 7, + "section": "release_launchpad", + "mode": "worker", + }, + "primary_action_code": "create_release_rollout_worker", + "primary_action_payload": {"release_id": 7}, + "requested_by": "test/driver-action", + } + ) + + self.assertTrue(ok) + self.assertEqual("confirmation_required", data["gate"]["decision"]) + self.assertFalse(data["gate"]["will_execute"]) + self.assertTrue(data["gate"]["confirm_required"]) + self.assertEqual("release_progression", data["gate"]["sequence_key"]) + self.assertEqual("worker", data["preview_request"]["primary_focus_ref"]["mode"]) + self.assertEqual("worker", data["gate"]["focus_ref"]["mode"]) + self.assertEqual("worker", data["focus_ref"]["mode"]) + + @patch("app.services.ops_service.preview_driver_action") + def test_resolve_driver_action_request_can_select_secondary_open_ui_section(self, mock_preview_driver_action) -> None: + mock_preview_driver_action.return_value = ( + True, + "ok", + _build_preview_data( + executor_kind="runbook_sequence", + primary_action_code="create_release_rollout_worker", + recommendation="confirm_then_execute", + automation_level="guarded_auto", + target_api="/api/v1/ops/runbook/sequences/release_progression/execute", + primary_request_payload={ + "secondary": False, + "requested_by": "test/driver-action", + "action_payload": {"release_id": 7}, + }, + sequence_key="release_progression", + secondary_action_code="focus_release_hub", + secondary_request_payload={ + "secondary": True, + "requested_by": "test/driver-action", + "action_payload": {"release_id": 7}, + }, + secondary_recommendation="auto_execute", + secondary_automation_level="safe_auto", + secondary_target_api="/api/v1/ops/runbook/sequences/release_progression/execute", + focus_ref={ + "kind": "release_hub", + "release_id": 7, + "section": "release_launchpad", + }, + primary_focus_ref={ + "kind": "release_hub", + "release_id": 7, + "section": "release_launchpad", + "mode": "worker", + }, + secondary_focus_ref={ + "kind": "release_hub", + "release_id": 7, + "section": "release_detail", + }, + ), + ) + + ok, _, data = resolve_driver_action_request( + { + "source_label": "driver-feed", + "title": "版本发布与放量", + "executor_kind": "runbook_sequence", + "sequence_key": "release_progression", + "focus_ref": { + "kind": "release_hub", + "release_id": 7, + "section": "release_launchpad", + }, + "primary_focus_ref": { + "kind": "release_hub", + "release_id": 7, + "section": "release_launchpad", + "mode": "worker", + }, + "secondary_focus_ref": { + "kind": "release_hub", + "release_id": 7, + "section": "release_detail", + }, + "primary_action_code": "create_release_rollout_worker", + "secondary_action_code": "focus_release_hub", + "primary_action_payload": {"release_id": 7}, + "secondary_action_payload": {"release_id": 7}, + "requested_by": "test/driver-action", + "secondary": True, + } + ) + + self.assertTrue(ok) + self.assertEqual("secondary", data["selected_section"]) + self.assertEqual("execute_now", data["gate"]["decision"]) + self.assertTrue(data["gate"]["will_execute"]) + self.assertEqual("focus_release_hub", data["gate"]["action_code"]) + self.assertEqual("release_detail", data["gate"]["focus_ref"]["section"]) + self.assertEqual("release_detail", data["focus_ref"]["section"]) + + @patch("app.services.ops_service.preview_driver_action") + def test_execute_resolved_driver_action_blocks_guarded_action_without_confirm(self, mock_preview_driver_action) -> None: + mock_preview_driver_action.return_value = ( + True, + "ok", + _build_preview_data( + executor_kind="runbook_sequence", + primary_action_code="create_release_rollout_worker", + recommendation="confirm_then_execute", + automation_level="guarded_auto", + target_api="/api/v1/ops/runbook/sequences/release_progression/execute", + primary_request_payload={ + "secondary": False, + "requested_by": "test/driver-action", + "action_payload": {"release_id": 7}, + }, + sequence_key="release_progression", + ), + ) + + ok, message, data = execute_resolved_driver_action( + { + "source_label": "driver-feed", + "title": "版本发布与放量", + "executor_kind": "runbook_sequence", + "sequence_key": "release_progression", + "primary_action_code": "create_release_rollout_worker", + "primary_action_payload": {"release_id": 7}, + "requested_by": "test/driver-action", + } + ) + + self.assertFalse(ok) + self.assertIn("需要显式确认", message) + self.assertFalse(data["executed"]) + self.assertEqual("confirmation_required", data["gate"]["decision"]) + + @patch("app.services.ops_service.execute_driver_action") + @patch("app.services.ops_service.preview_driver_action") + def test_execute_resolved_driver_action_runs_safe_auto_driver_action( + self, + mock_preview_driver_action, + mock_execute_driver_action, + ) -> None: + request_payload = { + "action_code": "enable_log_sync_key", + "node_codes": ["mainland-controller-01"], + "action_payload": {}, + "requested_by": "test/driver-action", + } + mock_preview_driver_action.return_value = ( + True, + "ok", + _build_preview_data( + executor_kind="driver_action", + primary_action_code="enable_log_sync_key", + recommendation="auto_execute", + automation_level="safe_auto", + target_api="/api/v1/ops/driver-actions/execute", + primary_request_payload=request_payload, + ), + ) + mock_execute_driver_action.return_value = ( + True, + "日志回传已开启", + {"handled": True, "mode": "ops-job"}, + ) + + ok, message, data = execute_resolved_driver_action( + { + "source_label": "driver-feed", + "title": "开启关键日志回传", + "primary_action_code": "enable_log_sync_key", + "primary_action_payload": {}, + "requested_by": "test/driver-action", + } + ) + + self.assertTrue(ok) + self.assertEqual("日志回传已开启", message) + self.assertTrue(data["executed"]) + self.assertEqual({"handled": True, "mode": "ops-job"}, data["execution_result"]) + mock_execute_driver_action.assert_called_once_with(request_payload) + + @patch("app.services.ops_service.execute_ops_runbook_sequence") + @patch("app.services.ops_service.preview_driver_action") + def test_execute_resolved_driver_action_runs_guarded_runbook_after_confirm( + self, + mock_preview_driver_action, + mock_execute_ops_runbook_sequence, + ) -> None: + request_payload = { + "secondary": False, + "requested_by": "test/driver-action", + "action_payload": {"release_id": 7}, + } + mock_preview_driver_action.return_value = ( + True, + "ok", + _build_preview_data( + executor_kind="runbook_sequence", + primary_action_code="create_release_rollout_worker", + recommendation="confirm_then_execute", + automation_level="guarded_auto", + target_api="/api/v1/ops/runbook/sequences/release_progression/execute", + primary_request_payload=request_payload, + sequence_key="release_progression", + ), + ) + mock_execute_ops_runbook_sequence.return_value = ( + True, + "Release rollout 已创建", + { + "handled": True, + "mode": "smart-release-rollout", + "driver_action_code": "create_release_rollout_worker", + }, + ) + + ok, message, data = execute_resolved_driver_action( + { + "source_label": "driver-feed", + "title": "版本发布与放量", + "executor_kind": "runbook_sequence", + "sequence_key": "release_progression", + "primary_action_code": "create_release_rollout_worker", + "primary_action_payload": {"release_id": 7}, + "requested_by": "test/driver-action", + "confirm": True, + } + ) + + self.assertTrue(ok) + self.assertEqual("Release rollout 已创建", message) + self.assertTrue(data["executed"]) + self.assertEqual("execute_confirmed", data["gate"]["decision"]) + mock_execute_ops_runbook_sequence.assert_called_once_with("release_progression", request_payload) + + +if __name__ == "__main__": + unittest.main() diff --git a/domain-api/tests/test_ops_service_driver_actions.py b/domain-api/tests/test_ops_service_driver_actions.py new file mode 100644 index 0000000..6b54059 --- /dev/null +++ b/domain-api/tests/test_ops_service_driver_actions.py @@ -0,0 +1,1265 @@ +import unittest +from unittest.mock import patch + +from app.services.ops_service import execute_driver_action + + +class OpsServiceDriverActionTests(unittest.TestCase): + @patch("app.services.ops_service.execute_managed_node_onboarding_recovery") + def test_bootstrap_run_executes_node_onboarding_recovery( + self, + mock_execute_managed_node_onboarding_recovery, + ) -> None: + mock_execute_managed_node_onboarding_recovery.return_value = ( + True, + "自动收口任务已创建", + { + "recovery_action": "bootstrap_run", + "playbook_run": {"run_code": "run-001"}, + }, + ) + + ok, message, data = execute_driver_action( + { + "action_code": "bootstrap_run", + "node_codes": ["mainland-worker-02"], + "action_payload": { + "control_plane_base_url": "http://control.example:8100", + }, + "requested_by": "test", + } + ) + + self.assertTrue(ok, message) + self.assertEqual("自动收口任务已创建", message) + self.assertTrue(data["handled"]) + self.assertEqual("node-onboarding-recovery", data["mode"]) + self.assertEqual("mainland-worker-02", data["node_code"]) + mock_execute_managed_node_onboarding_recovery.assert_called_once_with( + node_code="mainland-worker-02", + requested_by="test/bootstrap_run", + control_plane_base_url="http://control.example:8100", + root_dir="", + ) + + @patch("app.services.ops_service.create_release_and_smart_rollout_from_latest_package") + def test_publish_latest_worker_calls_backend_launchpad_flow( + self, + mock_create_release_and_smart_rollout_from_latest_package, + ) -> None: + mock_create_release_and_smart_rollout_from_latest_package.return_value = ( + True, + "Release rollout 已创建", + { + "release": {"id": 51, "release_version": "domaincheck_release_launchpad"}, + "rollout_created": True, + }, + ) + + ok, message, data = execute_driver_action( + { + "action_code": "publish_latest_worker", + "action_payload": { + "channel": "stable", + "control_plane_base_url": "http://control.example:8100", + }, + "requested_by": "test", + } + ) + + self.assertTrue(ok, message) + self.assertEqual("Release rollout 已创建", message) + self.assertTrue(data["handled"]) + self.assertEqual("smart-release-rollout", data["mode"]) + mock_create_release_and_smart_rollout_from_latest_package.assert_called_once() + called_payload = mock_create_release_and_smart_rollout_from_latest_package.call_args.args[0] + called_kwargs = mock_create_release_and_smart_rollout_from_latest_package.call_args.kwargs + self.assertEqual("worker", called_payload["mode"]) + self.assertEqual("stable", called_payload["channel"]) + self.assertEqual("test/publish_latest_worker", called_payload["created_by"]) + self.assertEqual("test/publish_latest_worker/rollout", called_payload["rollout_created_by"]) + self.assertEqual("http://control.example:8100", called_kwargs["control_plane_base_url"]) + + @patch("app.services.ops_service.prepare_latest_release_package") + def test_release_prepare_executes_final_signoff_flow( + self, + mock_prepare_latest_release_package, + ) -> None: + mock_prepare_latest_release_package.return_value = ( + True, + "最新发布包最终签收已完成", + { + "package": { + "available": True, + "package_name": "domaincheck_release_20260418_220000", + "final_release_ok": True, + }, + "final_release_gate": { + "ready": True, + "status": "ready", + }, + }, + ) + + ok, message, data = execute_driver_action( + { + "action_code": "release_prepare", + "action_payload": {}, + "requested_by": "test", + } + ) + + self.assertTrue(ok, message) + self.assertEqual("最新发布包最终签收已完成", message) + self.assertTrue(data["handled"]) + self.assertEqual("release-prepare", data["mode"]) + self.assertTrue(data["release_prepare_result"]["package"]["final_release_ok"]) + mock_prepare_latest_release_package.assert_called_once_with( + requested_by="test/release_prepare" + ) + + @patch("app.services.ops_service.get_runtime_build_info") + def test_api_restart_returns_runtime_refresh_preview( + self, + mock_get_runtime_build_info, + ) -> None: + mock_get_runtime_build_info.return_value = { + "commit_sha": "246838ae4c07", + "route_surface": { + "surface_complete": False, + "missing_keys": ["ops_node_handover_bootstrap_plan"], + }, + } + + ok, message, data = execute_driver_action( + { + "action_code": "api-restart", + "action_payload": { + "control_plane_base_url": "http://control.example:8100", + }, + "requested_by": "test", + } + ) + + self.assertTrue(ok, message) + self.assertIn("刷新控制面运行时", message) + self.assertTrue(data["handled"]) + self.assertEqual("runtime-refresh-preview", data["mode"]) + self.assertEqual("open_stack_diagnosis", data["ui_intent"]["kind"]) + self.assertIn("runtime-refresh-recover", data["recommended_command"]) + self.assertEqual( + ["ops_node_handover_bootstrap_plan"], + data["runtime_build_info"]["route_surface"]["missing_keys"], + ) + + @patch("app.services.ops_service.get_release_launchpad") + def test_review_smart_rollout_preview_returns_launchpad_preview_payload(self, mock_get_release_launchpad) -> None: + mock_get_release_launchpad.return_value = { + "launchpad_status": { + "status": "attention", + "status_label": "待确认", + }, + "latest_release": { + "id": 88, + "release_version": "domaincheck_release_demo", + }, + "worker_rollout_preview": { + "message": "Worker 智能预案待确认", + "smart_rollout": { + "available": True, + "target_node_codes": ["mainland-worker-01"], + }, + }, + } + + ok, message, data = execute_driver_action( + { + "action_code": "review_smart_rollout_preview", + "action_payload": { + "channel": "stable", + }, + "requested_by": "test", + } + ) + + self.assertTrue(ok, message) + self.assertEqual("", message) + self.assertTrue(data["handled"]) + self.assertEqual("release-launchpad-preview", data["mode"]) + self.assertEqual("worker", data["review_mode"]) + self.assertEqual("worker", data["release_launchpad_review"]["mode"]) + self.assertEqual(88, data["release_launchpad_review"]["latest_release"]["id"]) + self.assertEqual( + ["mainland-worker-01"], + data["release_launchpad_review"]["target_node_codes"], + ) + self.assertEqual("release_launchpad_review", data["ui_intent"]["kind"]) + self.assertEqual("worker", data["ui_intent"]["mode"]) + self.assertEqual(88, data["ui_intent"]["release_id"]) + mock_get_release_launchpad.assert_called_once() + + @patch("app.services.ops_service.get_managed_node_handover") + @patch("app.services.ops_service.get_release_launchpad") + def test_fix_managed_nodes_returns_handover_ui_intent_when_gap_exists( + self, + mock_get_release_launchpad, + mock_get_managed_node_handover, + ) -> None: + mock_get_release_launchpad.return_value = { + "worker_rollout_preview": { + "policy_preview": { + "release_gate": { + "operational_readiness": { + "rows": [ + { + "node_code": "mainland-worker-01", + "is_managed": False, + "is_enabled": False, + "is_agent_online": False, + "remote_agent_ready": False, + "inspection_status": "missing", + } + ] + } + } + } + } + } + mock_get_managed_node_handover.return_value = { + "node_code": "mainland-worker-01", + "stage": {"code": "unmanaged"}, + } + + ok, message, data = execute_driver_action( + { + "action_code": "fix_managed_nodes", + "requested_by": "test", + } + ) + + self.assertTrue(ok, message) + self.assertEqual("", message) + self.assertTrue(data["handled"]) + self.assertEqual("ui-intent", data["mode"]) + self.assertEqual("managed_node_handover", data["ui_intent"]["kind"]) + self.assertEqual("mainland-worker-01", data["ui_intent"]["node_code"]) + self.assertEqual("unmanaged", data["handover"]["stage"]["code"]) + + @patch("app.services.ops_service.sync_managed_nodes_from_cluster") + @patch("app.services.ops_service.get_release_launchpad") + def test_fix_managed_nodes_syncs_cluster_when_gap_not_found( + self, + mock_get_release_launchpad, + mock_sync_managed_nodes_from_cluster, + ) -> None: + mock_get_release_launchpad.return_value = { + "worker_rollout_preview": { + "policy_preview": { + "release_gate": { + "operational_readiness": { + "rows": [] + } + } + } + } + } + mock_sync_managed_nodes_from_cluster.return_value = { + "synced_count": 2, + } + + ok, message, data = execute_driver_action( + { + "action_code": "fix_managed_nodes", + "requested_by": "test", + } + ) + + self.assertTrue(ok, message) + self.assertEqual("已按集群快照同步托管节点，请继续检查接管状态。", message) + self.assertTrue(data["handled"]) + self.assertEqual("cluster-sync", data["mode"]) + self.assertEqual(2, data["sync_result"]["synced_count"]) + mock_sync_managed_nodes_from_cluster.assert_called_once_with(dry_run=False) + + @patch("app.services.ops_service.get_managed_node_handover") + def test_handover_first_gap_includes_handover_payload(self, mock_get_managed_node_handover) -> None: + mock_get_managed_node_handover.return_value = { + "node_code": "mainland-worker-01", + "stage": {"code": "pending_bootstrap"}, + } + + ok, message, data = execute_driver_action( + { + "action_code": "handover_first_gap", + "action_payload": { + "node_code": "mainland-worker-01", + "control_plane_base_url": "http://control.example:8100", + }, + "requested_by": "test", + } + ) + + self.assertTrue(ok, message) + self.assertEqual("", message) + self.assertEqual("handover-preview", data["mode"]) + self.assertEqual("pending_bootstrap", data["handover_preview"]["stage"]["code"]) + self.assertEqual("managed_node_handover", data["ui_intent"]["kind"]) + self.assertEqual("pending_bootstrap", data["handover"]["stage"]["code"]) + mock_get_managed_node_handover.assert_called_once_with( + "mainland-worker-01", + control_plane_base_url="http://control.example:8100", + ) + + @patch("app.services.ops_service.get_managed_node_handover") + def test_view_first_gap_returns_handover_preview_and_edit_ui_intent(self, mock_get_managed_node_handover) -> None: + mock_get_managed_node_handover.return_value = { + "node_code": "mainland-worker-02", + "stage": {"code": "runtime_only"}, + } + + ok, message, data = execute_driver_action( + { + "action_code": "view_first_gap", + "action_payload": { + "node_code": "mainland-worker-02", + }, + "requested_by": "test", + } + ) + + self.assertTrue(ok, message) + self.assertEqual("", message) + self.assertEqual("handover-preview", data["mode"]) + self.assertEqual("runtime_only", data["handover_preview"]["stage"]["code"]) + self.assertEqual("managed_node_edit", data["ui_intent"]["kind"]) + self.assertEqual("runtime_only", data["handover"]["stage"]["code"]) + + @patch("app.services.ops_service.get_latest_release_package_metadata") + def test_release_package_returns_package_preview_and_open_release_dialog_ui_intent( + self, + mock_get_latest_release_package_metadata, + ) -> None: + mock_get_latest_release_package_metadata.return_value = { + "available": True, + "package_name": "domaincheck_release_20260418_120000", + "release_version_suggestion": "domaincheck_release_20260418_120000", + } + + ok, message, data = execute_driver_action( + { + "action_code": "release_package", + "requested_by": "test", + } + ) + + self.assertTrue(ok, message) + self.assertEqual("", message) + self.assertTrue(data["handled"]) + self.assertEqual("release-package-preview", data["mode"]) + self.assertTrue(data["release_package_preview"]["available"]) + self.assertEqual( + "domaincheck_release_20260418_120000", + data["release_package_preview"]["package_name"], + ) + self.assertEqual("open_release_dialog", data["ui_intent"]["kind"]) + mock_get_latest_release_package_metadata.assert_called_once_with() + + @patch("app.services.ops_service.get_release_summary") + @patch("app.services.ops_service.get_latest_release_package_metadata") + def test_open_release_dialog_returns_release_dialog_preview_and_ui_intent( + self, + mock_get_latest_release_package_metadata, + mock_get_release_summary, + ) -> None: + mock_get_latest_release_package_metadata.return_value = { + "available": True, + "package_name": "domaincheck_release_20260418_120000", + } + mock_get_release_summary.return_value = { + "latest_release": { + "id": 7, + "release_version": "2026.04.18", + } + } + + ok, message, data = execute_driver_action( + { + "action_code": "open_release_dialog", + "requested_by": "test", + } + ) + + self.assertTrue(ok, message) + self.assertEqual("", message) + self.assertTrue(data["handled"]) + self.assertEqual("release-dialog-preview", data["mode"]) + self.assertEqual(7, data["release_dialog_preview"]["summary"]["latest_release"]["id"]) + self.assertEqual( + "domaincheck_release_20260418_120000", + data["release_dialog_preview"]["release_package_preview"]["package_name"], + ) + self.assertEqual("open_release_dialog", data["ui_intent"]["kind"]) + mock_get_latest_release_package_metadata.assert_called_once_with() + mock_get_release_summary.assert_called_once_with() + + @patch("app.services.ops_service.get_release_launchpad") + @patch("app.services.ops_service.get_release_summary") + def test_focus_release_hub_returns_release_hub_preview_and_ui_intent( + self, + mock_get_release_summary, + mock_get_release_launchpad, + ) -> None: + mock_get_release_summary.return_value = { + "latest_release": { + "id": 7, + "release_version": "2026.04.18", + } + } + mock_get_release_launchpad.return_value = { + "channel": "stable", + "launchpad_status": { + "status": "attention", + "recommended_action_code": "review_smart_rollout_preview", + }, + } + + ok, message, data = execute_driver_action( + { + "action_code": "focus_release_hub", + "action_payload": { + "release_id": 7, + "channel": "stable", + "control_plane_base_url": "http://control.example:8100", + }, + "requested_by": "test", + } + ) + + self.assertTrue(ok, message) + self.assertEqual("", message) + self.assertTrue(data["handled"]) + self.assertEqual("release-hub-preview", data["mode"]) + self.assertEqual("focus_release_hub", data["ui_intent"]["kind"]) + self.assertEqual(7, data["ui_intent"]["release_id"]) + self.assertEqual( + "review_smart_rollout_preview", + data["release_hub_preview"]["launchpad"]["launchpad_status"]["recommended_action_code"], + ) + self.assertEqual(7, data["release_hub_preview"]["summary"]["latest_release"]["id"]) + mock_get_release_summary.assert_called_once_with() + mock_get_release_launchpad.assert_called_once_with( + control_plane_base_url="http://control.example:8100", + channel="stable", + ) + + def test_open_playbook_dialog_returns_ui_intent(self) -> None: + ok, message, data = execute_driver_action( + { + "action_code": "open_playbook_dialog", + "node_codes": ["mainland-worker-01"], + "action_payload": { + "playbook_key": "scene.logs.key", + "execution_mode": "remote-agent", + "auto_approve": True, + }, + "requested_by": "test", + } + ) + + self.assertTrue(ok, message) + self.assertTrue(data["handled"]) + self.assertEqual("playbook-preview", data["mode"]) + self.assertEqual("scene.logs.key", data["playbook_preview"]["playbook"]["key"]) + self.assertEqual(1, data["playbook_preview"]["target_nodes_total"]) + self.assertEqual(["mainland-worker-01"], data["playbook_preview"]["target_node_codes"]) + self.assertEqual("remote-agent", data["playbook_preview"]["execution_mode"]) + self.assertTrue(data["playbook_preview"]["auto_approve"]) + self.assertEqual("open_playbook_dialog", data["ui_intent"]["kind"]) + self.assertEqual("scene.logs.key", data["ui_intent"]["playbook_key"]) + self.assertEqual(["mainland-worker-01"], data["ui_intent"]["target_node_codes"]) + self.assertEqual("remote-agent", data["ui_intent"]["execution_mode"]) + self.assertTrue(data["ui_intent"]["auto_approve"]) + + @patch("app.services.ops_service.list_ops_playbook_run_events") + @patch("app.services.ops_service.get_ops_playbook_run") + def test_focus_playbook_run_returns_run_focus_preview_and_ui_intent( + self, + mock_get_ops_playbook_run, + mock_list_ops_playbook_run_events, + ) -> None: + mock_get_ops_playbook_run.return_value = { + "run_code": "pbr-demo", + "playbook_key": "inspection.standard", + "status": "running", + } + mock_list_ops_playbook_run_events.return_value = { + "run_code": "pbr-demo", + "events": [{"event_key": "evt-1"}], + } + + ok, message, data = execute_driver_action( + { + "action_code": "focus_playbook_run", + "action_payload": { + "run_code": "pbr-demo", + "focus_step_key": "logs.collect", + "focus_step_title": "收集服务日志", + }, + "requested_by": "test", + } + ) + + self.assertTrue(ok, message) + self.assertEqual("playbook-run-focus", data["mode"]) + self.assertEqual("pbr-demo", data["playbook_run_focus"]["run"]["run_code"]) + self.assertEqual("evt-1", data["playbook_run_focus"]["events"]["events"][0]["event_key"]) + self.assertEqual("playbook_run_detail", data["ui_intent"]["kind"]) + mock_get_ops_playbook_run.assert_called_once_with("pbr-demo") + mock_list_ops_playbook_run_events.assert_called_once_with( + "pbr-demo", + limit=40, + step_key="logs.collect", + node_code="", + ) + + @patch("app.services.ops_service.get_ops_activity_stream") + def test_focus_activity_item_returns_activity_focus_preview_and_ui_intent( + self, + mock_get_ops_activity_stream, + ) -> None: + mock_get_ops_activity_stream.return_value = { + "items": [ + { + "activity_key": "ops-job:42", + "kind": "ops_job", + "status": "attention", + "summary": "任务待关注", + } + ] + } + + ok, message, data = execute_driver_action( + { + "action_code": "focus_activity_item", + "action_payload": { + "activity_key": "ops-job:42", + "kind": "ops_job", + "status": "attention", + "ui_intent": { + "kind": "job_events", + "job_id": 42, + }, + }, + "requested_by": "test", + } + ) + + self.assertTrue(ok, message) + self.assertEqual("activity-focus", data["mode"]) + self.assertEqual("ops-job:42", data["activity_focus_preview"]["activity"]["activity_key"]) + self.assertEqual("job_events", data["ui_intent"]["kind"]) + + @patch("app.services.ops_service.list_ops_job_events_for_jobs") + def test_focus_latest_job_events_returns_job_events_preview_and_ui_intent( + self, + mock_list_ops_job_events_for_jobs, + ) -> None: + mock_list_ops_job_events_for_jobs.return_value = [ + { + "id": 1, + "job_id": 42, + "event_type": "job_started", + "message": "started", + } + ] + + ok, message, data = execute_driver_action( + { + "action_code": "focus_latest_job_events", + "action_payload": { + "focus_ref": { + "kind": "ops_job", + "job_id": 42, + "job_code": "job-42", + } + }, + "requested_by": "test", + } + ) + + self.assertTrue(ok, message) + self.assertEqual("job-events-focus", data["mode"]) + self.assertEqual(42, data["job_events_preview"]["job_id"]) + self.assertEqual("job-42", data["job_events_preview"]["job_code"]) + self.assertEqual(1, data["job_events_preview"]["events"][0]["id"]) + self.assertEqual("job_events", data["ui_intent"]["kind"]) + mock_list_ops_job_events_for_jobs.assert_called_once_with([42], limit=60) + + @patch("app.services.ops_service.list_ops_playbook_run_events") + def test_open_playbook_run_latest_events_returns_events_preview_and_ui_intent( + self, + mock_list_ops_playbook_run_events, + ) -> None: + mock_list_ops_playbook_run_events.return_value = { + "run_code": "pbr-demo", + "events": [{"event_key": "evt-2"}], + } + + ok, message, data = execute_driver_action( + { + "action_code": "open_playbook_run_latest_events", + "action_payload": { + "run_code": "pbr-demo", + "focus_step_key": "logs.collect", + }, + "requested_by": "test", + } + ) + + self.assertTrue(ok, message) + self.assertEqual("playbook-run-events", data["mode"]) + self.assertEqual("evt-2", data["playbook_run_events_preview"]["events"][0]["event_key"]) + self.assertEqual("playbook_run_latest_events", data["ui_intent"]["kind"]) + mock_list_ops_playbook_run_events.assert_called_once_with( + "pbr-demo", + limit=60, + step_key="logs.collect", + node_code="", + ) + + @patch("app.services.ops_service.list_managed_nodes_with_agent_state") + def test_open_playbook_dialog_auto_switches_to_ssh_when_supported_and_target_is_ssh_ready( + self, + mock_list_managed_nodes_with_agent_state, + ) -> None: + mock_list_managed_nodes_with_agent_state.return_value = { + "nodes": [ + { + "node_code": "mainland-worker-01", + "is_managed": True, + "is_enabled": True, + "is_agent_online": False, + "has_ssh_access": True, + } + ] + } + + ok, message, data = execute_driver_action( + { + "action_code": "open_playbook_dialog", + "node_codes": ["mainland-worker-01"], + "action_payload": { + "playbook_key": "scene.logs.key", + }, + "requested_by": "test", + } + ) + + self.assertTrue(ok, message) + self.assertEqual("ssh", data["ui_intent"]["execution_mode"]) + + def test_open_playbook_dialog_keeps_control_plane_when_playbook_only_supports_control_plane(self) -> None: + ok, message, data = execute_driver_action( + { + "action_code": "open_playbook_dialog", + "node_codes": ["mainland-worker-01"], + "action_payload": { + "playbook_key": "onboarding.bootstrap", + }, + "requested_by": "test", + } + ) + + self.assertTrue(ok, message) + self.assertEqual("control-plane", data["ui_intent"]["execution_mode"]) + + def test_open_action_template_dialog_returns_ui_intent(self) -> None: + ok, message, data = execute_driver_action( + { + "action_code": "open_action_template_dialog", + "node_codes": ["mainland-controller-01"], + "action_payload": { + "template_key": "logs.collect", + "execution_mode": "remote-agent", + "auto_approve": False, + "payload": { + "service_name": "domaincheck-worker", + "lines": 120, + }, + }, + "requested_by": "test", + } + ) + + self.assertTrue(ok, message) + self.assertTrue(data["handled"]) + self.assertEqual("action-template-preview", data["mode"]) + self.assertEqual("logs.collect", data["template_preview"]["template"]["key"]) + self.assertEqual("remote-agent", data["template_preview"]["execution_mode"]) + self.assertEqual(["mainland-controller-01"], data["template_preview"]["target_node_codes"]) + self.assertEqual("domaincheck-worker", data["template_preview"]["payload"]["service_name"]) + self.assertEqual("open_action_template_dialog", data["ui_intent"]["kind"]) + self.assertEqual("logs.collect", data["ui_intent"]["template_key"]) + self.assertEqual(["mainland-controller-01"], data["ui_intent"]["target_node_codes"]) + self.assertEqual("remote-agent", data["ui_intent"]["execution_mode"]) + self.assertFalse(data["ui_intent"]["auto_approve"]) + self.assertEqual("domaincheck-worker", data["ui_intent"]["payload"]["service_name"]) + self.assertEqual(120, data["ui_intent"]["payload"]["lines"]) + + @patch("app.services.ops_service.create_ops_job_batch") + def test_replay_delivery_queue_creates_ops_job_batch(self, mock_create_ops_job_batch) -> None: + mock_create_ops_job_batch.return_value = ( + True, + "已创建 2 个运维任务；待审批 0；阻断 0；失败 0", + { + "summary": {"created_count": 2}, + "jobs": [{"id": 1}, {"id": 2}], + }, + ) + + ok, message, data = execute_driver_action( + { + "action_code": "replay_delivery_queue", + "node_codes": ["mainland-worker-01", "mainland-worker-02"], + "action_payload": { + "payload": { + "limit": 2, + "flush_after_replay": True, + }, + }, + "requested_by": "test", + } + ) + + self.assertTrue(ok, message) + self.assertEqual("已创建 2 个运维任务；待审批 0；阻断 0；失败 0", message) + self.assertTrue(data["handled"]) + self.assertEqual("ops-job-batch", data["mode"]) + self.assertEqual("delivery.queue.replay", data["template_key"]) + called_payload = mock_create_ops_job_batch.call_args.args[0] + self.assertEqual("delivery.queue.replay", called_payload["template_key"]) + self.assertEqual(["mainland-worker-01", "mainland-worker-02"], called_payload["target_node_codes"]) + self.assertEqual("test/replay_delivery_queue", called_payload["requested_by"]) + self.assertEqual("remote-agent", called_payload["execution_mode"]) + self.assertFalse(called_payload["auto_approve"]) + self.assertTrue(called_payload["payload"]["flush_after_replay"]) + + @patch("app.services.ops_service.create_ops_job_batch") + def test_flush_delivery_queue_creates_ops_job_batch(self, mock_create_ops_job_batch) -> None: + mock_create_ops_job_batch.return_value = ( + True, + "已创建 1 个运维任务；待审批 0；阻断 0；失败 0", + { + "summary": {"created_count": 1}, + "jobs": [{"id": 9}], + }, + ) + + ok, message, data = execute_driver_action( + { + "action_code": "flush_delivery_queue", + "action_payload": { + "target_node_codes": ["mainland-controller-01"], + "auto_approve": True, + "payload": { + "limit": 5, + }, + }, + "requested_by": "test", + } + ) + + self.assertTrue(ok, message) + self.assertEqual("ops-job-batch", data["mode"]) + self.assertEqual("delivery.queue.flush", data["template_key"]) + called_payload = mock_create_ops_job_batch.call_args.args[0] + self.assertEqual("delivery.queue.flush", called_payload["template_key"]) + self.assertEqual(["mainland-controller-01"], called_payload["target_node_codes"]) + self.assertEqual("test/flush_delivery_queue", called_payload["requested_by"]) + self.assertTrue(called_payload["auto_approve"]) + self.assertEqual(5, called_payload["payload"]["limit"]) + + @patch("app.services.ops_service.list_managed_nodes_with_agent_state") + def test_open_action_template_dialog_auto_switches_to_ssh_when_supported_and_target_is_ssh_ready( + self, + mock_list_managed_nodes_with_agent_state, + ) -> None: + mock_list_managed_nodes_with_agent_state.return_value = { + "nodes": [ + { + "node_code": "mainland-controller-01", + "is_managed": True, + "is_enabled": True, + "is_agent_online": False, + "has_ssh_access": True, + } + ] + } + + ok, message, data = execute_driver_action( + { + "action_code": "open_action_template_dialog", + "node_codes": ["mainland-controller-01"], + "action_payload": { + "template_key": "logs.collect", + "payload": { + "service_name": "domaincheck-api", + "lines": 120, + }, + }, + "requested_by": "test", + } + ) + + self.assertTrue(ok, message) + self.assertEqual("ssh", data["ui_intent"]["execution_mode"]) + + @patch("app.services.ops_service.execute_ops_playbook") + def test_run_scene_logs_full_executes_ops_playbook( + self, + mock_execute_ops_playbook, + ) -> None: + mock_execute_ops_playbook.return_value = ( + True, + "现场取证已创建", + { + "run_code": "playbook-logs-full-001", + }, + ) + + ok, message, data = execute_driver_action( + { + "action_code": "run_scene_logs_full", + "node_codes": ["mainland-worker-01"], + "action_payload": { + "execution_mode": "remote-agent", + "auto_approve": True, + }, + "requested_by": "test", + } + ) + + self.assertTrue(ok, message) + self.assertEqual("现场取证已创建", message) + self.assertTrue(data["handled"]) + self.assertEqual("ops-playbook", data["mode"]) + self.assertEqual("scene.logs.full", data["playbook_key"]) + called_payload = mock_execute_ops_playbook.call_args.args[0] + self.assertEqual("scene.logs.full", called_payload["playbook_key"]) + self.assertEqual(["mainland-worker-01"], called_payload["node_codes"]) + self.assertEqual("remote-agent", called_payload["execution_mode"]) + self.assertTrue(called_payload["auto_approve"]) + self.assertEqual("test/run_scene_logs_full", called_payload["requested_by"]) + + @patch("app.services.ops_service.get_release_launchpad") + @patch("app.services.ops_service.get_release_summary") + def test_open_release_deploy_worker_returns_preview_and_ui_intent( + self, + mock_get_release_summary, + mock_get_release_launchpad, + ) -> None: + mock_get_release_summary.return_value = { + "latest_release": { + "id": 11, + "release_version": "2026.04.18", + } + } + mock_get_release_launchpad.return_value = { + "launchpad_status": { + "status": "attention", + "recommended_action_code": "review_smart_rollout_preview", + } + } + + ok, message, data = execute_driver_action( + { + "action_code": "open_release_deploy_worker", + "node_codes": ["mainland-worker-01"], + "action_payload": { + "release_id": 11, + "execution_mode": "remote-agent", + }, + "requested_by": "test", + } + ) + + self.assertTrue(ok, message) + self.assertTrue(data["handled"]) + self.assertEqual("release-deploy-preview", data["mode"]) + self.assertEqual("deploy.release.worker", data["release_deploy_preview"]["template"]["key"]) + self.assertEqual(11, data["release_deploy_preview"]["release_id"]) + self.assertEqual(["mainland-worker-01"], data["release_deploy_preview"]["target_node_codes"]) + self.assertEqual("remote-agent", data["release_deploy_preview"]["execution_mode"]) + self.assertEqual( + "review_smart_rollout_preview", + data["release_deploy_preview"]["launchpad"]["launchpad_status"]["recommended_action_code"], + ) + self.assertEqual("open_release_deploy_template", data["ui_intent"]["kind"]) + self.assertEqual("deploy.release.worker", data["ui_intent"]["template_key"]) + self.assertEqual(11, data["ui_intent"]["release_id"]) + self.assertEqual(["mainland-worker-01"], data["ui_intent"]["target_node_codes"]) + self.assertEqual("remote-agent", data["ui_intent"]["execution_mode"]) + mock_get_release_summary.assert_called_once_with() + mock_get_release_launchpad.assert_called_once_with() + + @patch("app.services.ops_service.list_managed_nodes_with_agent_state") + def test_open_release_deploy_worker_auto_switches_to_ssh_when_agent_not_ready( + self, + mock_list_managed_nodes_with_agent_state, + ) -> None: + mock_list_managed_nodes_with_agent_state.return_value = { + "nodes": [ + { + "node_code": "mainland-worker-01", + "is_managed": True, + "is_enabled": True, + "is_agent_online": False, + "has_ssh_access": True, + } + ] + } + + ok, message, data = execute_driver_action( + { + "action_code": "open_release_deploy_worker", + "node_codes": ["mainland-worker-01"], + "action_payload": { + "release_id": 11, + }, + "requested_by": "test", + } + ) + + self.assertTrue(ok, message) + self.assertEqual("ssh", data["ui_intent"]["execution_mode"]) + + @patch("app.services.ops_service.get_release_launchpad") + @patch("app.services.ops_service.get_release_summary") + def test_open_rollout_dialog_carries_release_id_and_returns_preview( + self, + mock_get_release_summary, + mock_get_release_launchpad, + ) -> None: + mock_get_release_summary.return_value = { + "latest_release": { + "id": 9, + "release_version": "2026.04.18", + } + } + mock_get_release_launchpad.return_value = { + "launchpad_status": { + "status": "ready", + "recommended_action_code": "publish_latest_worker", + } + } + + ok, message, data = execute_driver_action( + { + "action_code": "open_rollout_dialog", + "action_payload": { + "release_id": 23, + }, + "requested_by": "test", + } + ) + + self.assertTrue(ok, message) + self.assertTrue(data["handled"]) + self.assertEqual("rollout-dialog-preview", data["mode"]) + self.assertEqual( + "publish_latest_worker", + data["rollout_dialog_preview"]["launchpad"]["launchpad_status"]["recommended_action_code"], + ) + self.assertEqual(9, data["rollout_dialog_preview"]["summary"]["latest_release"]["id"]) + self.assertEqual("open_rollout_dialog", data["ui_intent"]["kind"]) + self.assertEqual(23, data["ui_intent"]["release_id"]) + mock_get_release_summary.assert_called_once_with() + mock_get_release_launchpad.assert_called_once_with() + + @patch("app.services.ops_service.create_release_and_smart_rollout_from_latest_package") + def test_create_smart_release_rollout_worker_calls_backend_flow( + self, + mock_create_release_and_smart_rollout_from_latest_package, + ) -> None: + mock_create_release_and_smart_rollout_from_latest_package.return_value = ( + True, + "Release rollout 已创建", + { + "release": {"id": 31, "release_version": "domaincheck_release_demo"}, + "rollout_created": True, + }, + ) + + ok, message, data = execute_driver_action( + { + "action_code": "create_smart_release_rollout_worker", + "action_payload": { + "channel": "stable", + }, + "requested_by": "test", + } + ) + + self.assertTrue(ok, message) + self.assertEqual("Release rollout 已创建", message) + self.assertTrue(data["handled"]) + self.assertEqual("smart-release-rollout", data["mode"]) + self.assertTrue(data["smart_release_result"]["rollout_created"]) + mock_create_release_and_smart_rollout_from_latest_package.assert_called_once() + called_payload = mock_create_release_and_smart_rollout_from_latest_package.call_args.args[0] + self.assertEqual("worker", called_payload["mode"]) + self.assertEqual("stable", called_payload["channel"]) + self.assertEqual("test/create_smart_release_rollout_worker", called_payload["created_by"]) + + @patch("app.services.ops_service.create_release_and_smart_rollout_from_latest_package") + def test_create_smart_release_rollout_control_passes_confirm_risky( + self, + mock_create_release_and_smart_rollout_from_latest_package, + ) -> None: + mock_create_release_and_smart_rollout_from_latest_package.return_value = ( + True, + "Release 已创建，智能 Rollout 需要确认", + { + "release": {"id": 32, "release_version": "domaincheck_release_demo"}, + "requires_confirmation": True, + "rollout_created": False, + }, + ) + + ok, message, data = execute_driver_action( + { + "action_code": "create_smart_release_rollout_control", + "action_payload": { + "confirm_risky": True, + }, + "requested_by": "test", + } + ) + + self.assertTrue(ok, message) + self.assertEqual("Release 已创建，智能 Rollout 需要确认", message) + self.assertTrue(data["handled"]) + self.assertEqual("smart-release-rollout", data["mode"]) + mock_create_release_and_smart_rollout_from_latest_package.assert_called_once() + called_payload = mock_create_release_and_smart_rollout_from_latest_package.call_args.args[0] + self.assertEqual("control", called_payload["mode"]) + self.assertTrue(called_payload["confirm_risky"]) + + @patch("app.services.ops_service.create_smart_release_rollout") + def test_create_release_rollout_worker_calls_existing_release_backend_flow( + self, + mock_create_smart_release_rollout, + ) -> None: + mock_create_smart_release_rollout.return_value = ( + True, + "Release rollout 已创建", + { + "release": {"id": 41, "release_version": "domaincheck_release_existing"}, + "release_created": False, + "rollout_created": True, + }, + ) + + ok, message, data = execute_driver_action( + { + "action_code": "create_release_rollout_worker", + "action_payload": { + "release_id": 41, + }, + "requested_by": "test", + } + ) + + self.assertTrue(ok, message) + self.assertEqual("Release rollout 已创建", message) + self.assertTrue(data["handled"]) + self.assertEqual("smart-release-rollout", data["mode"]) + mock_create_smart_release_rollout.assert_called_once() + called_release_id = mock_create_smart_release_rollout.call_args.args[0] + called_payload = mock_create_smart_release_rollout.call_args.args[1] + self.assertEqual(41, called_release_id) + self.assertEqual("worker", called_payload["mode"]) + self.assertEqual("test/create_release_rollout_worker", called_payload["created_by"]) + + @patch("app.services.ops_service.create_smart_release_rollout") + def test_create_release_rollout_control_passes_confirm_risky( + self, + mock_create_smart_release_rollout, + ) -> None: + mock_create_smart_release_rollout.return_value = ( + True, + "智能 Rollout 需要确认", + { + "release": {"id": 42, "release_version": "domaincheck_release_existing"}, + "release_created": False, + "requires_confirmation": True, + "rollout_created": False, + }, + ) + + ok, message, data = execute_driver_action( + { + "action_code": "create_release_rollout_control", + "action_payload": { + "release_id": 42, + "confirm_risky": True, + }, + "requested_by": "test", + } + ) + + self.assertTrue(ok, message) + self.assertEqual("智能 Rollout 需要确认", message) + self.assertTrue(data["handled"]) + self.assertEqual("smart-release-rollout", data["mode"]) + mock_create_smart_release_rollout.assert_called_once() + called_release_id = mock_create_smart_release_rollout.call_args.args[0] + called_payload = mock_create_smart_release_rollout.call_args.args[1] + self.assertEqual(42, called_release_id) + self.assertEqual("control", called_payload["mode"]) + self.assertTrue(called_payload["confirm_risky"]) + + @patch("app.services.ops_service.update_runtime_settings") + @patch("app.services.ops_service.get_runtime_settings") + def test_enable_log_sync_key_updates_runtime_settings( + self, + mock_get_runtime_settings, + mock_update_runtime_settings, + ) -> None: + mock_get_runtime_settings.return_value = { + "worker_log_sync_enabled": False, + "worker_log_sync_mode": "key", + } + mock_update_runtime_settings.return_value = { + "worker_log_sync_enabled": True, + "worker_log_sync_mode": "key", + } + + ok, message, data = execute_driver_action( + { + "action_code": "enable_log_sync_key", + "requested_by": "test", + } + ) + + self.assertTrue(ok, message) + mock_update_runtime_settings.assert_called_once_with( + { + "worker_log_sync_enabled": True, + "worker_log_sync_mode": "key", + } + ) + self.assertEqual("远端日志回传已切到关键模式", message) + self.assertTrue(data["handled"]) + self.assertEqual("settings", data["mode"]) + self.assertTrue(data["runtime_settings"]["worker_log_sync_enabled"]) + self.assertEqual("key", data["runtime_settings"]["worker_log_sync_mode"]) + + @patch("app.services.ops_service.update_runtime_settings") + @patch("app.services.ops_service.get_runtime_settings") + def test_enable_log_sync_full_updates_runtime_settings( + self, + mock_get_runtime_settings, + mock_update_runtime_settings, + ) -> None: + mock_get_runtime_settings.return_value = { + "worker_log_sync_enabled": False, + "worker_log_sync_mode": "key", + } + mock_update_runtime_settings.return_value = { + "worker_log_sync_enabled": True, + "worker_log_sync_mode": "full", + } + + ok, message, data = execute_driver_action( + { + "action_code": "enable_log_sync_full", + "requested_by": "test", + } + ) + + self.assertTrue(ok, message) + mock_update_runtime_settings.assert_called_once_with( + { + "worker_log_sync_enabled": True, + "worker_log_sync_mode": "full", + } + ) + self.assertEqual("远端日志回传已切到全量模式", message) + self.assertTrue(data["handled"]) + self.assertEqual("settings", data["mode"]) + self.assertTrue(data["runtime_settings"]["worker_log_sync_enabled"]) + self.assertEqual("full", data["runtime_settings"]["worker_log_sync_mode"]) + + @patch("app.services.ops_service.update_runtime_settings") + @patch("app.services.ops_service.get_runtime_settings") + def test_disable_log_sync_updates_runtime_settings( + self, + mock_get_runtime_settings, + mock_update_runtime_settings, + ) -> None: + mock_get_runtime_settings.return_value = { + "worker_log_sync_enabled": True, + "worker_log_sync_mode": "full", + } + mock_update_runtime_settings.return_value = { + "worker_log_sync_enabled": False, + "worker_log_sync_mode": "key", + } + + ok, message, data = execute_driver_action( + { + "action_code": "disable_log_sync", + "requested_by": "test", + } + ) + + self.assertTrue(ok, message) + mock_update_runtime_settings.assert_called_once_with( + { + "worker_log_sync_enabled": False, + "worker_log_sync_mode": "key", + } + ) + self.assertEqual("远端日志回传已关闭", message) + self.assertTrue(data["handled"]) + self.assertEqual("settings", data["mode"]) + self.assertFalse(data["runtime_settings"]["worker_log_sync_enabled"]) + self.assertEqual("key", data["runtime_settings"]["worker_log_sync_mode"]) + + @patch("app.services.ops_service.execute_ops_playbook") + @patch("app.services.ops_service.list_managed_nodes_with_agent_state") + def test_open_worker_logs_auto_switches_to_ssh_when_target_is_ssh_ready( + self, + mock_list_managed_nodes_with_agent_state, + mock_execute_ops_playbook, + ) -> None: + mock_list_managed_nodes_with_agent_state.return_value = { + "nodes": [ + { + "node_code": "mainland-worker-01", + "is_managed": True, + "is_enabled": True, + "is_agent_online": False, + "has_ssh_access": True, + } + ] + } + mock_execute_ops_playbook.return_value = (True, "ok", {"run_code": "playbook-001"}) + + ok, message, data = execute_driver_action( + { + "action_code": "open_worker_logs_participating", + "node_codes": ["mainland-worker-01"], + "requested_by": "test", + } + ) + + self.assertTrue(ok, message) + self.assertEqual("ssh", data["execution_mode"]) + called_payload = mock_execute_ops_playbook.call_args.args[0] + self.assertEqual("ssh", called_payload["execution_mode"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/domain-api/tests/test_ops_service_driver_feed.py b/domain-api/tests/test_ops_service_driver_feed.py new file mode 100644 index 0000000..ffa4ce3 --- /dev/null +++ b/domain-api/tests/test_ops_service_driver_feed.py @@ -0,0 +1,358 @@ +import unittest +from unittest.mock import patch + +from app.services.ops_service import get_ops_driver_feed + + +class OpsServiceDriverFeedTests(unittest.TestCase): + @patch("app.services.ops_service.get_ops_go_live_summary") + @patch("app.services.ops_service.get_ops_runbook") + @patch("app.services.ops_service.get_ops_overview") + def test_driver_feed_merges_priority_recommendation_driver_cards_and_runbook_sequences( + self, + mock_get_ops_overview, + mock_get_ops_runbook, + mock_get_ops_go_live_summary, + ) -> None: + mock_get_ops_go_live_summary.return_value = { + "go_live_status": "blocked", + "publish_ready": False, + "publish_status": "blocked", + "publish_status_label": "暂不可发布", + "publish_summary": "当前仍有硬阻断项，不能直接进入正式发版。", + "operator_title": "先补接管缺口", + "next_step_action_code": "handover_first_gap", + "launchpad_recommended_target_node_code": "mainland-controller-01", + "launchpad_recommended_recovery_label": "签发接入工单", + "launchpad_recommended_recovery_summary": "当前节点仍处于接入阶段，建议先签发 onboarding.bootstrap。", + "launchpad_onboarding_bootstrap_pending_nodes": 1, + "launchpad_onboarding_acceptance_ready_nodes": 1, + "blocking_reasons": ["remote_agent_ready=0"], + "warnings": ["log_sync_waiting_sample=0/2"], + } + mock_get_ops_overview.return_value = { + "execution_scene": { + "participation_summary": { + "participating_nodes": 2, + "dispatch_active_node_codes": ["mainland-worker-01", "mainland-controller-01"], + }, + "participating_nodes": [ + {"node_code": "mainland-worker-01"}, + {"node_code": "mainland-controller-01"}, + ], + "log_sync": { + "enabled": False, + "mode": "key", + "line_count": 0, + "source_node_count": 0, + "participating_node_count": 2, + "covered_participating_node_count": 0, + "missing_participating_nodes": ["mainland-worker-01", "mainland-controller-01"], + }, + }, + "recommendation": { + "key": "fix-handover-gap", + "priority": "先补接管缺口", + "title": "先补接管缺口", + "summary": "当前应该先把接管链路补齐。", + "reason": "否则后续批量动作无法稳定下发。", + "level_label": "最高优先", + "tag_type": "danger", + "primary_label": "纳管首台节点", + "secondary_label": "查看目标节点", + "primary_action_code": "handover_first_gap", + "secondary_action_code": "view_first_gap", + "node_codes": ["mainland-worker-02"], + "primary_node_codes": ["mainland-worker-02"], + "secondary_node_codes": ["mainland-worker-02"], + "primary_action_payload": {"requested_by": "test"}, + "secondary_action_payload": {}, + "reasons": ["当前存在接管缺口。"], + }, + "activity_stream": { + "items": [ + { + "kind": "ops_job", + "activity_key": "ops-job:501", + "title": "logs.collect", + "subtitle": "ops-logs-501", + "summary_text": "远端 Agent 已回传 120 行 Worker 日志，可直接继续分析。", + "summary": "远端 Agent 已回传 120 行 Worker 日志，可直接继续分析。", + "status": "running", + "status_label": "执行中", + "meta_text": "ops-logs-501 / 执行 远端 Agent", + "occurred_at": "2026-04-18 09:18:00", + "target_node_codes": ["mainland-worker-01"], + "focus_ref": { + "kind": "ops_job", + "job_id": 501, + "job_code": "ops-logs-501", + "action": "logs.collect", + "target_node_code": "mainland-worker-01", + }, + "source_focus_ref": { + "kind": "ops_job_event", + "job_id": 501, + "event_key": "job-event:501", + }, + "ui_intent": { + "kind": "job_events", + "job_id": 501, + "job_code": "ops-logs-501", + "action": "logs.collect", + "target_node_code": "mainland-worker-01", + }, + } + ] + }, + "driver_recommendations": [ + { + "key": "fix-handover-gap", + "title": "先补接管缺口", + "summary": "重复建议，不应再次进入 feed。", + "reason": "duplicate", + "level_label": "最高优先", + "tag_type": "danger", + "primary_label": "纳管首台节点", + "primary_action_code": "handover_first_gap", + }, + { + "key": "inspect-scene-log-gap", + "title": "排查现场日志样本缺口", + "summary": "现场日志样本还没有形成。", + "reason": "先看 Worker 日志。", + "meta_text": "参与节点 2 台 / 已回传来源 0 台", + "level_label": "推荐", + "tag_type": "warning", + "primary_label": "看 Worker 日志", + "secondary_label": "执行标准巡检", + "primary_action_code": "open_worker_logs", + "secondary_action_code": "run_standard_inspection", + "node_codes": ["mainland-worker-01", "mainland-controller-01"], + "primary_node_codes": ["mainland-worker-01", "mainland-controller-01"], + "secondary_node_codes": ["mainland-worker-01", "mainland-controller-01"], + "primary_action_payload": {"lines": 120}, + "secondary_action_payload": {}, + }, + ], + } + mock_get_ops_runbook.return_value = { + "control_sequences": [ + { + "key": "release_progression", + "title": "版本发布与放量", + "summary": "当前可进入发布路径。", + "reason": "正式运维时应统一落回 Release / Rollout。", + "status": "warning", + "status_label": "待处理", + "tag_type": "warning", + "target_scope_label": "Release / Rollout", + "target_node_codes": ["mainland-controller-01"], + "focus_ref": { + "kind": "release_hub", + "release_id": 7, + "section": "release_launchpad", + }, + "primary_label": "补执行面接管", + "secondary_label": "查看版本区", + "primary_action_code": "fix_managed_nodes", + "secondary_action_code": "focus_release_hub", + "primary_focus_ref": { + "kind": "release_hub", + "release_id": 7, + "section": "default_rollout_gate", + }, + "secondary_focus_ref": { + "kind": "release_hub", + "release_id": 7, + "section": "release_detail", + }, + "step_titles": ["同步集群节点", "纳管首台缺口", "补齐 Agent 接入"], + "primary_resolution": { + "ok": True, + "driver_action_code": "fix_managed_nodes", + "driver_action_label": "补执行面接管", + "driver_node_codes": ["mainland-controller-01"], + "driver_action_payload": {"channel": "stable"}, + "resolved_at": "2026-04-18T03:00:00", + }, + "secondary_resolution": { + "ok": True, + "driver_action_code": "focus_release_hub", + "driver_action_label": "查看版本区", + "driver_node_codes": [], + "driver_action_payload": {"release_id": 7}, + }, + } + ] + } + + payload = get_ops_driver_feed() + + self.assertEqual(3, payload["summary"]["total"]) + self.assertEqual("blocked", payload["go_live_summary"]["go_live_status"]) + self.assertEqual("blocked", payload["summary"]["go_live_status"]) + self.assertEqual("先补接管缺口", payload["summary"]["operator_title"]) + self.assertEqual("blocked", payload["summary"]["publish_status"]) + self.assertFalse(payload["summary"]["publish_ready"]) + self.assertEqual("暂不可发布", payload["summary"]["publish_status_label"]) + self.assertIn("发布闸门：暂不可发布", payload["headline"]) + self.assertIn("接入缺口：签发接入工单：mainland-controller-01", payload["headline"]) + self.assertEqual("handover_first_gap", payload["go_live_summary"]["next_step_action_code"]) + self.assertEqual("mainland-controller-01", payload["summary"]["launchpad_recommended_target_node_code"]) + self.assertEqual("签发接入工单", payload["summary"]["launchpad_recommended_recovery_label"]) + self.assertEqual( + "当前节点仍处于接入阶段，建议先签发 onboarding.bootstrap。", + payload["summary"]["launchpad_recommended_recovery_summary"], + ) + self.assertEqual(1, payload["summary"]["launchpad_onboarding_bootstrap_pending_nodes"]) + self.assertEqual(1, payload["summary"]["launchpad_onboarding_acceptance_ready_nodes"]) + self.assertEqual(2, payload["summary"]["do_now_total"]) + self.assertEqual(1, payload["summary"]["standard_path_total"]) + self.assertEqual("ops_driver_contract", payload["contract_navigation"]["primary_contract_key"]) + self.assertIn("ops_playbook_contract", payload["contract_navigation"]["contract_keys"]) + self.assertIn("ops_observability_contract", payload["contract_navigation"]["contract_keys"]) + self.assertEqual("priority:fix-handover-gap", payload["top_recommendation"]["key"]) + self.assertEqual("ops_driver_contract", payload["top_recommendation"]["contract_navigation"]["primary_contract_key"]) + self.assertEqual(2, len(payload["driver_recommendations"])) + self.assertEqual(1, len(payload["runbook_sequences"])) + self.assertEqual(1, len(payload["activity_focus"])) + self.assertEqual( + ["priority_recommendation", "driver_recommendation", "runbook_sequence"], + [item["kind"] for item in payload["entries"]], + ) + self.assertEqual(1, payload["summary"]["activity_focus_total"]) + self.assertEqual("disabled", payload["summary"]["scene_log_status"]) + self.assertEqual("blocked", payload["summary"]["launch_status"]) + self.assertFalse(payload["summary"]["launch_ready"]) + self.assertEqual(2, payload["summary"]["preview_only_total"]) + self.assertEqual(3, payload["summary"]["backend_handled_total"]) + self.assertEqual("blocked", payload["automation_coverage"]["launch_status"]) + self.assertEqual(2, payload["automation_coverage"]["safe_auto_total"]) + self.assertEqual(0, payload["automation_coverage"]["guarded_auto_total"]) + self.assertEqual(2, payload["automation_coverage"]["auto_execute_total"]) + self.assertEqual(1, payload["automation_coverage"]["resolve_first_total"]) + self.assertEqual(2, payload["automation_coverage"]["preview_only_total"]) + self.assertEqual(3, payload["automation_coverage"]["backend_handled_total"]) + self.assertEqual(1, payload["automation_coverage"]["human_dependency_total"]) + self.assertFalse(payload["automation_coverage"]["launch_ready"]) + + priority_entry = payload["entries"][0] + self.assertEqual("do_now", priority_entry["lane"]) + self.assertEqual("driver_action", priority_entry["executor_kind"]) + self.assertEqual("handover_first_gap", priority_entry["primary_action_code"]) + self.assertEqual("ops_driver_contract", priority_entry["contract_navigation"]["primary_contract_key"]) + + activity_focus = payload["activity_focus"][0] + self.assertEqual("ops_job", activity_focus["activity_kind"]) + self.assertEqual("远端 Agent 已回传 120 行 Worker 日志，可直接继续分析。", activity_focus["summary"]) + self.assertEqual("focus_latest_job_events", activity_focus["focus_action_code"]) + self.assertEqual("job_events", activity_focus["ui_intent_kind"]) + self.assertEqual("ops_job_event", activity_focus["source_focus_ref"]["kind"]) + self.assertTrue(activity_focus["observation_only"]) + self.assertEqual("ops_observability_contract", activity_focus["contract_navigation"]["primary_contract_key"]) + self.assertIn("ops_stack_diagnosis_contract", activity_focus["contract_navigation"]["contract_keys"]) + + scene_log_observation = payload["scene_log_observation"] + self.assertEqual("disabled", scene_log_observation["status"]) + self.assertEqual("mainland-worker-01", scene_log_observation["target_node_code"]) + self.assertEqual("node_scene_log", scene_log_observation["focus_ref"]["kind"]) + self.assertEqual( + "ops_observability_contract", + scene_log_observation["contract_navigation"]["primary_contract_key"], + ) + + runbook_entry = payload["entries"][2] + self.assertEqual("runbook_sequence", runbook_entry["kind"]) + self.assertEqual("standard_path", runbook_entry["lane"]) + self.assertEqual("runbook_sequence", runbook_entry["executor_kind"]) + self.assertEqual("release_progression", runbook_entry["sequence_key"]) + self.assertEqual("fix_managed_nodes", runbook_entry["primary_action_code"]) + self.assertEqual("focus_release_hub", runbook_entry["secondary_action_code"]) + self.assertIn("当前解析：补执行面接管", runbook_entry["detail_lines"][0]) + self.assertEqual("release_hub", runbook_entry["focus_ref"]["kind"]) + self.assertEqual("release_launchpad", runbook_entry["focus_ref"]["section"]) + self.assertEqual("default_rollout_gate", runbook_entry["primary_focus_ref"]["section"]) + self.assertEqual("release_detail", runbook_entry["secondary_focus_ref"]["section"]) + self.assertEqual("default_rollout_gate", runbook_entry["primary_action_payload"]["focus_ref"]["section"]) + self.assertEqual("release_detail", runbook_entry["secondary_action_payload"]["focus_ref"]["section"]) + self.assertEqual("ops_driver_contract", runbook_entry["contract_navigation"]["primary_contract_key"]) + self.assertIn("ops_playbook_contract", runbook_entry["contract_navigation"]["contract_keys"]) + self.assertIn("release_hub_contract", runbook_entry["contract_navigation"]["contract_keys"]) + + @patch("app.services.ops_service.get_ops_go_live_summary") + @patch("app.services.ops_service.get_ops_runbook") + @patch("app.services.ops_service.get_ops_overview") + def test_driver_feed_preserves_bootstrap_run_as_top_recommendation( + self, + mock_get_ops_overview, + mock_get_ops_runbook, + mock_get_ops_go_live_summary, + ) -> None: + mock_get_ops_go_live_summary.return_value = { + "go_live_status": "attention", + "publish_ready": False, + "publish_status": "attention", + "publish_status_label": "可发布但建议先复核", + "publish_summary": "当前没有硬阻断，但仍有上线前关注项，建议先完成复核再正式发版。", + "operator_title": "先跑接入收口", + "next_step_action_code": "bootstrap_run", + "launchpad_recommended_target_node_code": "mainland-worker-01", + "launchpad_recommended_recovery_label": "跑接入收口", + "launchpad_recommended_recovery_summary": "mainland-worker-01 还缺接入收口，先执行 bootstrap。", + "launchpad_onboarding_bootstrap_pending_nodes": 1, + "launchpad_onboarding_acceptance_ready_nodes": 0, + "blocking_reasons": [], + "warnings": ["remote_agent_ready=1/2"], + } + mock_get_ops_overview.return_value = { + "recommendation": { + "key": "handover-gap-bootstrap_run", + "priority": "先补接管缺口", + "title": "先补接管缺口", + "summary": "Launchpad 已明确收敛到首台节点接入收口。", + "reason": "mainland-worker-01 还缺接入收口，先执行 bootstrap。", + "level_label": "最高优先", + "tag_type": "warning", + "primary_label": "跑接入收口", + "secondary_label": "查看目标节点", + "primary_action_code": "bootstrap_run", + "secondary_action_code": "view_first_gap", + "node_codes": ["mainland-worker-01"], + "primary_node_codes": ["mainland-worker-01"], + "secondary_node_codes": ["mainland-worker-01"], + "primary_action_payload": {"node_code": "mainland-worker-01"}, + "secondary_action_payload": {}, + "focus_ref": {"kind": "managed_node", "node_code": "mainland-worker-01"}, + "primary_focus_ref": {"kind": "managed_node", "node_code": "mainland-worker-01"}, + "secondary_focus_ref": {"kind": "managed_node", "node_code": "mainland-worker-01"}, + }, + "driver_recommendations": [], + "activity_stream": {"items": []}, + "execution_scene": {}, + } + mock_get_ops_runbook.return_value = {"control_sequences": []} + + payload = get_ops_driver_feed() + + self.assertEqual("priority:handover-gap-bootstrap_run", payload["top_recommendation"]["key"]) + self.assertEqual("attention", payload["summary"]["publish_status"]) + self.assertEqual("可发布但建议先复核", payload["summary"]["publish_status_label"]) + self.assertEqual("mainland-worker-01", payload["summary"]["launchpad_recommended_target_node_code"]) + self.assertEqual("跑接入收口", payload["summary"]["launchpad_recommended_recovery_label"]) + self.assertEqual(1, payload["summary"]["launchpad_onboarding_bootstrap_pending_nodes"]) + self.assertEqual(0, payload["summary"]["launchpad_onboarding_acceptance_ready_nodes"]) + self.assertEqual("attention", payload["summary"]["launch_status"]) + self.assertFalse(payload["summary"]["launch_ready"]) + self.assertEqual("bootstrap_run", payload["top_recommendation"]["primary_action_code"]) + self.assertEqual("mainland-worker-01", payload["top_recommendation"]["primary_node_codes"][0]) + self.assertIn("ops_agent_protocol", payload["top_recommendation"]["contract_navigation"]["contract_keys"]) + self.assertEqual("ops_driver_contract", payload["top_recommendation"]["contract_navigation"]["primary_contract_key"]) + self.assertEqual(0, payload["automation_coverage"]["safe_auto_total"]) + self.assertEqual(1, payload["automation_coverage"]["mixed_total"]) + self.assertEqual(0, payload["automation_coverage"]["preview_only_total"]) + self.assertEqual(1, payload["automation_coverage"]["backend_handled_total"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/domain-api/tests/test_ops_service_execution_modes.py b/domain-api/tests/test_ops_service_execution_modes.py new file mode 100644 index 0000000..14d6524 --- /dev/null +++ b/domain-api/tests/test_ops_service_execution_modes.py @@ -0,0 +1,24 @@ +import unittest + +from app.core.config import settings +from app.services.ops_service import _recommend_driver_execution_mode_for_supported_modes + + +class OpsServiceExecutionModeRecommendationTests(unittest.TestCase): + def test_recommend_driver_execution_mode_prefers_local_runtime_for_current_node(self) -> None: + result = _recommend_driver_execution_mode_for_supported_modes( + [settings.node_code], + supported_modes=["remote-agent", "local-runtime", "ssh"], + default_mode="remote-agent", + ) + + self.assertEqual("local-runtime", result["execution_mode"]) + self.assertIn("local-runtime", result["reason"]) + self.assertEqual( + ["remote-agent", "local-runtime", "ssh"], + result["supported_modes"], + ) + + +if __name__ == "__main__": + unittest.main() diff --git a/domain-api/tests/test_ops_service_go_live_bundle.py b/domain-api/tests/test_ops_service_go_live_bundle.py new file mode 100644 index 0000000..81b69e4 --- /dev/null +++ b/domain-api/tests/test_ops_service_go_live_bundle.py @@ -0,0 +1,106 @@ +from __future__ import annotations + +import json +import tempfile +import unittest +from pathlib import Path +from unittest.mock import patch + +from app.services.ops_service import get_ops_go_live_bundle + + +class OpsServiceGoLiveBundleTests(unittest.TestCase): + @patch("app.services.ops_service.get_ops_contract_registry") + def test_go_live_bundle_marks_missing_when_no_manifest_exists(self, mock_get_ops_contract_registry) -> None: + mock_get_ops_contract_registry.return_value = { + "contracts_by_key": { + "ops_go_live_bundle_contract": {"version": "v1"}, + } + } + + with patch("app.services.ops_service._find_latest_go_live_bundle_manifest", return_value=(None, Path("/tmp/go-live-bundles"))): + payload = get_ops_go_live_bundle(base_url="http://127.0.0.1:8100") + + self.assertEqual("missing", payload["status"]) + self.assertFalse(payload["bundle_available"]) + self.assertEqual("待导出", payload["status_label"]) + self.assertIn("go-live-export", payload["recommended_commands"]["go_live_export"]) + + @patch("app.services.ops_service.get_ops_contract_registry") + def test_go_live_bundle_reads_manifest_summary_and_review_state(self, mock_get_ops_contract_registry) -> None: + mock_get_ops_contract_registry.return_value = { + "contracts_by_key": { + "ops_go_live_bundle_contract": {"version": "v1"}, + } + } + + with tempfile.TemporaryDirectory() as temp_dir: + report_dir = Path(temp_dir) / "go-live-bundle-001" + report_dir.mkdir(parents=True, exist_ok=True) + env_audit_path = report_dir / "00_env_audit.txt" + env_audit_path.write_text( + "[8/8] condensed env audit summary\n" + + json.dumps( + { + "status": "attention", + "headline": "仍有轻微环境缺口", + "missing_items": ["missing_tool:jq"], + "runtime": {"runtime_may_need_restart": True}, + "recommended_actions": ["bash domain-api/deploy/multi-region/drive_ops_center.sh runtime-refresh-recover"], + }, + ensure_ascii=False, + ), + encoding="utf-8", + ) + manifest_path = report_dir / "manifest.json" + manifest_path.write_text( + json.dumps( + { + "report_dir": str(report_dir), + "artifacts": [ + {"key": "env_audit", "path": str(env_audit_path), "ok": True}, + {"key": "go_live_summary", "path": str(report_dir / "02_go_live_summary.json"), "ok": True}, + ], + "failures": ["doctor_export"], + "summary": { + "ok": False, + "artifact_total": 9, + "failure_total": 1, + "review_status_hint": "attention", + "review_headline": "核心报告已齐，但建议先复核补充缺口。", + "launchpad_recommended_target_node_code": "mainland-worker-01", + "launchpad_recommended_recovery_label": "先补接管缺口", + "launchpad_recommended_recovery_summary": "mainland-worker-01 还缺接入收口。", + "launchpad_onboarding_bootstrap_pending_nodes": 1, + "launchpad_onboarding_acceptance_ready_nodes": 0, + "launchpad_alignment": {"consistent": False, "target_node_code_consistent": False}, + "recommended_reading_order": ["00_env_audit.txt", "02_go_live_summary.json"], + "contract_surface_gaps": ["ops_go_live_bundle_contract"], + "discovered_contract_keys": ["ops_go_live_signoff_contract"], + }, + }, + ensure_ascii=False, + indent=2, + ), + encoding="utf-8", + ) + + with patch( + "app.services.ops_service._find_latest_go_live_bundle_manifest", + return_value=(manifest_path, report_dir), + ): + payload = get_ops_go_live_bundle(base_url="http://127.0.0.1:8100") + + self.assertEqual("attention", payload["status"]) + self.assertTrue(payload["bundle_available"]) + self.assertEqual(str(manifest_path), payload["manifest_path"]) + self.assertEqual(9, payload["artifact_total"]) + self.assertEqual(["doctor_export"], payload["noncritical_failures"]) + self.assertEqual("attention", payload["env_audit"]["status"]) + self.assertTrue(payload["env_audit"]["runtime_may_need_restart"]) + self.assertFalse(payload["summary"]["launchpad_alignment"]["consistent"]) + self.assertIn(str(manifest_path), payload["recommended_commands"]["go_live_signoff"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/domain-api/tests/test_ops_service_go_live_review.py b/domain-api/tests/test_ops_service_go_live_review.py new file mode 100644 index 0000000..9a79608 --- /dev/null +++ b/domain-api/tests/test_ops_service_go_live_review.py @@ -0,0 +1,169 @@ +from __future__ import annotations + +import json +import tempfile +import unittest +from pathlib import Path +from unittest.mock import patch + +from app.services.ops_service import get_ops_go_live_review + + +class OpsServiceGoLiveReviewTests(unittest.TestCase): + @patch("app.services.ops_service.get_ops_contract_registry") + def test_go_live_review_marks_missing_when_no_manifest_exists(self, mock_get_ops_contract_registry) -> None: + mock_get_ops_contract_registry.return_value = { + "contracts_by_key": { + "ops_go_live_review_contract": {"version": "v1"}, + } + } + + with patch("app.services.ops_service._find_latest_go_live_bundle_manifest", return_value=(None, Path("/tmp/go-live-bundles"))): + payload = get_ops_go_live_review(base_url="http://127.0.0.1:8100") + + self.assertEqual("missing", payload["status"]) + self.assertEqual("待导出", payload["status_label"]) + self.assertIn("go-live-export", payload["recommended_commands"]["go_live_export"]) + + @patch("app.services.ops_service.get_ops_contract_registry") + def test_go_live_review_reads_manifest_and_alignment(self, mock_get_ops_contract_registry) -> None: + mock_get_ops_contract_registry.return_value = { + "contracts_by_key": { + "ops_go_live_review_contract": {"version": "v1"}, + } + } + + with tempfile.TemporaryDirectory() as temp_dir: + report_dir = Path(temp_dir) / "go-live-bundle-001" + report_dir.mkdir(parents=True, exist_ok=True) + env_audit_path = report_dir / "00_env_audit.txt" + env_audit_path.write_text( + "[8/8] condensed env audit summary\n" + + json.dumps( + { + "status": "attention", + "headline": "环境仍有轻微缺口", + "missing_items": ["missing_tool:jq"], + "recommended_actions": ["bash runtime-refresh-recover"], + }, + ensure_ascii=False, + ), + encoding="utf-8", + ) + go_live_summary_path = report_dir / "02_go_live_summary.json" + go_live_summary_path.write_text( + json.dumps( + { + "go_live_status": "attention", + "publish_status": "attention", + "operator_title": "先补接入缺口", + "next_step_action_code": "fix_managed_nodes", + "launchpad_recommended_target_node_code": "mainland-worker-01", + "launchpad_recommended_recovery_label": "先补接入缺口", + "launchpad_recommended_recovery_summary": "mainland-worker-01 还缺接入收口。", + "launchpad_onboarding_bootstrap_pending_nodes": 1, + "launchpad_onboarding_acceptance_ready_nodes": 0, + }, + ensure_ascii=False, + ), + encoding="utf-8", + ) + stack_diagnosis_path = report_dir / "03_stack_diagnosis.json" + stack_diagnosis_path.write_text( + json.dumps( + { + "diagnosis": { + "stack_status": "attention", + "operator_decision": {"title": "先补接入缺口"}, + "next_step": {"action_code": "fix_managed_nodes"}, + "launchpad_recommended_target_node_code": "mainland-worker-02", + "launchpad_recommended_recovery_label": "先看节点接管", + "launchpad_recommended_recovery_summary": "stack 与 summary 存在漂移。", + "launchpad_onboarding_bootstrap_pending_nodes": 1, + "launchpad_onboarding_acceptance_ready_nodes": 0, + } + }, + ensure_ascii=False, + ), + encoding="utf-8", + ) + driver_feed_path = report_dir / "04_driver_feed.json" + driver_feed_path.write_text( + json.dumps( + { + "headline": "driver ok", + "summary": { + "launchpad_recommended_target_node_code": "mainland-worker-01", + "launchpad_recommended_recovery_label": "先补接入缺口", + "launchpad_recommended_recovery_summary": "mainland-worker-01 还缺接入收口。", + "launchpad_onboarding_bootstrap_pending_nodes": 1, + "launchpad_onboarding_acceptance_ready_nodes": 0, + }, + "automation_coverage": {"launch_status": "attention", "launch_ready": False}, + }, + ensure_ascii=False, + ), + encoding="utf-8", + ) + codex_brief_path = report_dir / "05_codex_brief.json" + codex_brief_path.write_text( + json.dumps( + { + "headline": "codex ok", + "summary": { + "launchpad_recommended_target_node_code": "mainland-worker-01", + "launchpad_recommended_recovery_label": "先补接入缺口", + "launchpad_recommended_recovery_summary": "mainland-worker-01 还缺接入收口。", + "launchpad_onboarding_bootstrap_pending_nodes": 1, + "launchpad_onboarding_acceptance_ready_nodes": 0, + }, + "automation_coverage": {"launch_status": "attention", "launch_ready": False}, + "focus": {"entry_key": "fix_managed_nodes"}, + }, + ensure_ascii=False, + ), + encoding="utf-8", + ) + manifest_path = report_dir / "manifest.json" + manifest_path.write_text( + json.dumps( + { + "report_dir": str(report_dir), + "artifacts": [ + {"key": "env_audit", "path": str(env_audit_path), "ok": True}, + {"key": "go_live_summary", "path": str(go_live_summary_path), "ok": True}, + {"key": "stack_diagnosis", "path": str(stack_diagnosis_path), "ok": True}, + {"key": "driver_feed", "path": str(driver_feed_path), "ok": True}, + {"key": "codex_brief", "path": str(codex_brief_path), "ok": True}, + {"key": "release_launchpad", "path": str(report_dir / "06_release_launchpad.json"), "ok": True}, + ], + "failures": ["doctor_export"], + "summary": { + "artifact_total": 9, + "failure_total": 1, + "recommended_reading_order": ["00_env_audit.txt", "02_go_live_summary.json"], + }, + }, + ensure_ascii=False, + indent=2, + ), + encoding="utf-8", + ) + + with patch( + "app.services.ops_service._find_latest_go_live_bundle_manifest", + return_value=(manifest_path, report_dir), + ): + payload = get_ops_go_live_review(base_url="http://127.0.0.1:8100") + + self.assertEqual("attention", payload["status"]) + self.assertEqual(9, payload["artifact_total"]) + self.assertEqual(["doctor_export"], payload["noncritical_failures"]) + self.assertFalse(payload["launchpad_alignment"]["consistent"]) + self.assertIn("go_live_summary", payload["launchpad_alignment"]["available_sources"]) + self.assertIn("mainland-worker-01", payload["recommended_next_steps"][0] + "".join(payload["recommended_next_steps"])) + self.assertIn(str(manifest_path), payload["recommended_commands"]["go_live_review"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/domain-api/tests/test_ops_service_go_live_signoff.py b/domain-api/tests/test_ops_service_go_live_signoff.py new file mode 100644 index 0000000..d162add --- /dev/null +++ b/domain-api/tests/test_ops_service_go_live_signoff.py @@ -0,0 +1,317 @@ +import unittest +from unittest.mock import patch + +from app.services.ops_service import get_ops_go_live_signoff + + +class OpsServiceGoLiveSignoffTests(unittest.TestCase): + @patch("app.services.ops_service.get_ops_contract_registry") + @patch("app.services.ops_service.get_ops_doctor_decision") + @patch("app.services.ops_service.get_ops_go_live_review") + @patch("app.services.ops_service.get_release_launchpad") + @patch("app.services.ops_service.get_ops_codex_brief") + @patch("app.services.ops_service.get_ops_driver_feed") + @patch("app.services.ops_service.get_ops_stack_diagnosis") + @patch("app.services.ops_service.get_ops_go_live_summary") + def test_go_live_signoff_marks_ready_when_all_surfaces_align( + self, + mock_get_ops_go_live_summary, + mock_get_ops_stack_diagnosis, + mock_get_ops_driver_feed, + mock_get_ops_codex_brief, + mock_get_release_launchpad, + mock_get_ops_go_live_review, + mock_get_ops_doctor_decision, + mock_get_ops_contract_registry, + ) -> None: + mock_get_ops_go_live_summary.return_value = { + "go_live_status": "ready", + "publish_status": "ready", + "operator_title": "进入发布", + "next_step_action_code": "publish_latest_worker", + "next_step_reason": "当前门禁已收口。", + "launchpad_recommended_target_node_code": "mainland-worker-01", + "launchpad_recommended_recovery_label": "继续发布", + "launchpad_recommended_recovery_summary": "可以进入正式发布。", + "launchpad_onboarding_bootstrap_pending_nodes": 0, + "launchpad_onboarding_acceptance_ready_nodes": 0, + "recommended_commands": {"next_step": "bash next-step"}, + } + mock_get_ops_stack_diagnosis.return_value = { + "diagnosis": { + "stack_status": "ready", + "operator_decision": {"title": "进入发布"}, + "next_step": {"action_code": "publish_latest_worker"}, + } + } + mock_get_ops_driver_feed.return_value = { + "headline": "主车道清晰", + "summary": { + "launch_status": "ready", + "launchpad_recommended_target_node_code": "mainland-worker-01", + "launchpad_recommended_recovery_label": "继续发布", + "launchpad_recommended_recovery_summary": "可以进入正式发布。", + "launchpad_onboarding_bootstrap_pending_nodes": 0, + "launchpad_onboarding_acceptance_ready_nodes": 0, + }, + } + mock_get_ops_codex_brief.return_value = { + "headline": "可签字", + "summary": { + "launch_status": "ready", + "launchpad_recommended_target_node_code": "mainland-worker-01", + "launchpad_recommended_recovery_label": "继续发布", + "launchpad_recommended_recovery_summary": "可以进入正式发布。", + "launchpad_onboarding_bootstrap_pending_nodes": 0, + "launchpad_onboarding_acceptance_ready_nodes": 0, + }, + "focus": {"entry_key": "publish_latest_worker"}, + } + mock_get_release_launchpad.return_value = { + "launchpad_status": { + "status": "ready", + "status_label": "可发布", + "recommended_target_node_code": "mainland-worker-01", + "recommended_recovery_label": "继续发布", + "recommended_recovery_summary": "可以进入正式发布。", + "onboarding_bootstrap_pending_nodes": 0, + "onboarding_acceptance_ready_nodes": 0, + } + } + mock_get_ops_go_live_review.return_value = { + "status": "ready", + "status_label": "可复核发布", + "headline": "review ok", + "report_dir": "/tmp/review", + "manifest_path": "/tmp/review/manifest.json", + "recommended_commands": {"go_live_review": "bash review"}, + } + mock_get_ops_doctor_decision.return_value = { + "status": "ready", + "headline": "doctor ok", + "report_dir": "/tmp/review", + "manifest_path": "/tmp/review/manifest.json", + "decision": { + "reason_code": "all_green", + "preferred_surface": "go-live-summary", + "next_action_code": "publish_latest_worker", + }, + "recommended_commands": {"doctor_decision": "bash doctor"}, + } + mock_get_ops_contract_registry.return_value = { + "contracts_by_key": { + "ops_go_live_signoff_contract": {"version": "v1"}, + } + } + + payload = get_ops_go_live_signoff(base_url="http://127.0.0.1:8100") + + self.assertEqual("ready", payload["signoff_status"]) + self.assertEqual("可签字上线", payload["status_label"]) + self.assertEqual([], payload["blocked_reasons"]) + self.assertEqual([], payload["attention_reasons"]) + self.assertTrue(payload["launchpad_alignment"]["consistent"]) + self.assertEqual("go-live-summary", payload["decision"]["preferred_surface"]) + self.assertIn("go-live-signoff", payload["recommended_commands"]["go_live_signoff"]) + self.assertEqual("ready", payload["release_gate"]["go_live_review_status"]) + self.assertEqual("ready", payload["release_gate"]["doctor_decision_status"]) + self.assertEqual("/tmp/review/manifest.json", payload["manifest_path"]) + + @patch("app.services.ops_service.get_ops_contract_registry") + @patch("app.services.ops_service.get_ops_doctor_decision") + @patch("app.services.ops_service.get_ops_go_live_review") + @patch("app.services.ops_service.get_release_launchpad") + @patch("app.services.ops_service.get_ops_codex_brief") + @patch("app.services.ops_service.get_ops_driver_feed") + @patch("app.services.ops_service.get_ops_stack_diagnosis") + @patch("app.services.ops_service.get_ops_go_live_summary") + def test_go_live_signoff_marks_attention_when_launchpad_alignment_drifts( + self, + mock_get_ops_go_live_summary, + mock_get_ops_stack_diagnosis, + mock_get_ops_driver_feed, + mock_get_ops_codex_brief, + mock_get_release_launchpad, + mock_get_ops_go_live_review, + mock_get_ops_doctor_decision, + mock_get_ops_contract_registry, + ) -> None: + mock_get_ops_go_live_summary.return_value = { + "go_live_status": "ready", + "publish_status": "ready", + "operator_title": "进入发布", + "next_step_action_code": "publish_latest_worker", + "launchpad_recommended_target_node_code": "mainland-worker-01", + "launchpad_recommended_recovery_label": "继续发布", + "launchpad_recommended_recovery_summary": "可以进入正式发布。", + "launchpad_onboarding_bootstrap_pending_nodes": 0, + "launchpad_onboarding_acceptance_ready_nodes": 0, + "recommended_commands": {}, + } + mock_get_ops_stack_diagnosis.return_value = { + "diagnosis": {"stack_status": "ready", "operator_decision": {}, "next_step": {}} + } + mock_get_ops_driver_feed.return_value = { + "headline": "主车道清晰", + "summary": { + "launch_status": "ready", + "launchpad_recommended_target_node_code": "mainland-worker-02", + "launchpad_recommended_recovery_label": "继续发布", + "launchpad_recommended_recovery_summary": "可以进入正式发布。", + "launchpad_onboarding_bootstrap_pending_nodes": 0, + "launchpad_onboarding_acceptance_ready_nodes": 0, + }, + } + mock_get_ops_codex_brief.return_value = { + "headline": "可签字", + "summary": { + "launch_status": "ready", + "launchpad_recommended_target_node_code": "mainland-worker-01", + "launchpad_recommended_recovery_label": "继续发布", + "launchpad_recommended_recovery_summary": "可以进入正式发布。", + "launchpad_onboarding_bootstrap_pending_nodes": 0, + "launchpad_onboarding_acceptance_ready_nodes": 0, + }, + "focus": {}, + } + mock_get_release_launchpad.return_value = { + "launchpad_status": { + "status": "ready", + "recommended_target_node_code": "mainland-worker-01", + "recommended_recovery_label": "继续发布", + "recommended_recovery_summary": "可以进入正式发布。", + "onboarding_bootstrap_pending_nodes": 0, + "onboarding_acceptance_ready_nodes": 0, + } + } + mock_get_ops_go_live_review.return_value = { + "status": "attention", + "headline": "review drift", + "recommended_commands": {"go_live_review": "bash review"}, + } + mock_get_ops_doctor_decision.return_value = { + "status": "ready", + "headline": "doctor ok", + "decision": {"preferred_surface": "go-live-summary"}, + "recommended_commands": {"doctor_decision": "bash doctor"}, + } + mock_get_ops_contract_registry.return_value = { + "contracts_by_key": { + "ops_go_live_signoff_contract": {"version": "v1"}, + } + } + + payload = get_ops_go_live_signoff() + + self.assertEqual("attention", payload["signoff_status"]) + self.assertIn("launchpad_alignment=inconsistent", payload["attention_reasons"]) + self.assertIn("go_live_review=attention", payload["attention_reasons"]) + self.assertFalse(payload["launchpad_alignment"]["consistent"]) + self.assertFalse(payload["launchpad_alignment"]["target_node_code_consistent"]) + self.assertEqual("go-live-review", payload["decision"]["preferred_surface"]) + + @patch("app.services.ops_service.get_ops_contract_registry") + @patch("app.services.ops_service.get_ops_doctor_decision") + @patch("app.services.ops_service.get_ops_go_live_review") + @patch("app.services.ops_service.get_release_launchpad") + @patch("app.services.ops_service.get_ops_codex_brief") + @patch("app.services.ops_service.get_ops_driver_feed") + @patch("app.services.ops_service.get_ops_stack_diagnosis") + @patch("app.services.ops_service.get_ops_go_live_summary") + def test_go_live_signoff_marks_blocked_when_publish_gate_is_blocked( + self, + mock_get_ops_go_live_summary, + mock_get_ops_stack_diagnosis, + mock_get_ops_driver_feed, + mock_get_ops_codex_brief, + mock_get_release_launchpad, + mock_get_ops_go_live_review, + mock_get_ops_doctor_decision, + mock_get_ops_contract_registry, + ) -> None: + mock_get_ops_go_live_summary.return_value = { + "go_live_status": "attention", + "publish_status": "blocked", + "operator_title": "先处理发布阻断", + "next_step_action_code": "fix_rollout_blockers", + "launchpad_recommended_target_node_code": "mainland-worker-01", + "launchpad_recommended_recovery_label": "处理发布阻断", + "launchpad_recommended_recovery_summary": "当前 rollout gate 未通过。", + "launchpad_onboarding_bootstrap_pending_nodes": 0, + "launchpad_onboarding_acceptance_ready_nodes": 0, + "recommended_commands": {}, + } + mock_get_ops_stack_diagnosis.return_value = { + "diagnosis": { + "stack_status": "attention", + "operator_decision": {"title": "先看发布门禁"}, + "next_step": {"action_code": "fix_rollout_blockers"}, + } + } + mock_get_ops_driver_feed.return_value = { + "headline": "发布门禁阻断", + "summary": { + "launch_status": "attention", + "launchpad_recommended_target_node_code": "mainland-worker-01", + "launchpad_recommended_recovery_label": "处理发布阻断", + "launchpad_recommended_recovery_summary": "当前 rollout gate 未通过。", + "launchpad_onboarding_bootstrap_pending_nodes": 0, + "launchpad_onboarding_acceptance_ready_nodes": 0, + }, + } + mock_get_ops_codex_brief.return_value = { + "headline": "需先排阻断", + "summary": { + "launch_status": "attention", + "launchpad_recommended_target_node_code": "mainland-worker-01", + "launchpad_recommended_recovery_label": "处理发布阻断", + "launchpad_recommended_recovery_summary": "当前 rollout gate 未通过。", + "launchpad_onboarding_bootstrap_pending_nodes": 0, + "launchpad_onboarding_acceptance_ready_nodes": 0, + }, + "focus": {"entry_key": "fix_rollout_blockers"}, + } + mock_get_release_launchpad.return_value = { + "launchpad_status": { + "status": "blocked", + "status_label": "发布阻断", + "recommended_action_code": "fix_rollout_blockers", + "recommended_target_node_code": "mainland-worker-01", + "recommended_recovery_label": "处理发布阻断", + "recommended_recovery_summary": "当前 rollout gate 未通过。", + "onboarding_bootstrap_pending_nodes": 0, + "onboarding_acceptance_ready_nodes": 0, + } + } + mock_get_ops_go_live_review.return_value = { + "status": "ready", + "headline": "review ok", + "recommended_commands": {"go_live_review": "bash review"}, + } + mock_get_ops_doctor_decision.return_value = { + "status": "attention", + "headline": "doctor focus", + "decision": { + "reason_code": "publish_blocked", + "preferred_surface": "release-launchpad", + "next_action_code": "fix_rollout_blockers", + }, + "recommended_commands": {"doctor_decision": "bash doctor"}, + } + mock_get_ops_contract_registry.return_value = { + "contracts_by_key": { + "ops_go_live_signoff_contract": {"version": "v1"}, + } + } + + payload = get_ops_go_live_signoff() + + self.assertEqual("blocked", payload["signoff_status"]) + self.assertIn("publish_status=blocked", payload["blocked_reasons"]) + self.assertIn("release_launchpad=blocked", payload["blocked_reasons"]) + self.assertEqual("release-launchpad", payload["decision"]["preferred_surface"]) + self.assertEqual("publish_blocked", payload["decision"]["doctor_reason_code"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/domain-api/tests/test_ops_service_go_live_summary.py b/domain-api/tests/test_ops_service_go_live_summary.py new file mode 100644 index 0000000..5bc5b15 --- /dev/null +++ b/domain-api/tests/test_ops_service_go_live_summary.py @@ -0,0 +1,664 @@ +import unittest +from unittest.mock import patch + +from app.services.ops_service import get_ops_go_live_summary + + +class OpsServiceGoLiveSummaryTests(unittest.TestCase): + @patch("app.services.ops_service.get_ops_overview") + @patch("app.services.ops_service.list_managed_nodes_with_agent_state") + @patch("app.services.ops_service.get_runtime_build_info") + @patch("app.services.ops_service.get_release_launchpad") + @patch("app.services.ops_service.get_ops_contract_registry") + @patch("app.services.ops_service.get_ops_stack_diagnosis") + def test_go_live_summary_exposes_runtime_schema_stale_and_api_restart_command( + self, + mock_get_ops_stack_diagnosis, + mock_get_ops_contract_registry, + mock_get_release_launchpad, + mock_get_runtime_build_info, + mock_list_managed_nodes_with_agent_state, + mock_get_ops_overview, + ) -> None: + mock_get_ops_stack_diagnosis.return_value = { + "diagnosis": { + "contract_key": "ops_stack_diagnosis_contract", + "stack_status": "attention", + "next_step": { + "action_code": "api-restart", + "reason": "build schema stale", + "focus_ref": {"kind": "runtime_build_info", "section": "schema"}, + }, + "operator_decision": { + "lane": "runtime_recovery", + "title": "优先重启控制面 API 并复检运行时版本", + "primary_command_key": "api-restart", + }, + }, + "release_hub": { + "launchpad_status": "ready", + }, + } + mock_get_ops_contract_registry.return_value = { + "registry_version": "2026-04-18", + "contracts": [{"key": "ops_stack_diagnosis_contract"}], + } + mock_get_release_launchpad.return_value = { + "launchpad_status": { + "status": "ready", + "status_label": "可发布", + "recommended_action_code": "create_smart_release_rollout_worker", + } + } + mock_get_runtime_build_info.return_value = { + "commit_sha": "246838ae4c07", + "repository_capabilities": { + "supports_install_command_block": True, + "supports_multi_layout_bootstrap": True, + }, + "route_surface": { + "surface_complete": True, + "missing_keys": [], + "expected_paths": { + "ops_contracts": "/api/v1/ops/contracts", + }, + }, + } + mock_list_managed_nodes_with_agent_state.return_value = { + "summary": { + "managed_enabled": 1, + "remote_access_ready": 1, + "queue_dead_letter_nodes": 0, + } + } + mock_get_ops_overview.return_value = { + "recommendation": { + "title": "先重启 API", + "reason": "build schema stale", + "primary_action_code": "api-restart", + }, + "execution_scene": { + "log_sync": { + "enabled": False, + "participating_node_count": 0, + "covered_participating_node_count": 0, + "missing_participating_nodes": [], + } + }, + } + + payload = get_ops_go_live_summary(base_url="http://127.0.0.1:8100") + + self.assertEqual("attention", payload["go_live_status"]) + self.assertEqual("attention", payload["publish_status"]) + self.assertTrue(payload["runtime_schema_stale"]) + self.assertFalse(payload["route_surface_declares_bootstrap_plan"]) + self.assertIn("runtime_build_schema_stale", payload["warnings"]) + self.assertIn("runtime_build_schema_stale", payload["publish_warnings"]) + self.assertEqual( + "bash domain-api/deploy/multi-region/drive_ops_center.sh go-live-recover http://127.0.0.1:8100", + payload["recommended_commands"]["go_live_recover"], + ) + self.assertEqual( + "bash domain-api/deploy/multi-region/drive_ops_center.sh runtime-refresh-recover http://127.0.0.1:8100", + payload["recommended_commands"]["api_restart"], + ) + self.assertEqual( + "bash domain-api/deploy/multi-region/drive_ops_center.sh runtime-refresh-recover http://127.0.0.1:8100", + payload["recommended_commands"]["runtime_refresh_recover"], + ) + self.assertIn("node-bootstrap-plan", payload["recommended_commands"]["bootstrap_plan_recheck"]) + + @patch("app.services.ops_service.get_ops_overview") + @patch("app.services.ops_service.list_managed_nodes_with_agent_state") + @patch("app.services.ops_service.get_runtime_build_info") + @patch("app.services.ops_service.get_release_launchpad") + @patch("app.services.ops_service.get_ops_contract_registry") + @patch("app.services.ops_service.get_ops_stack_diagnosis") + def test_go_live_summary_highlights_remote_access_and_log_sync_gaps( + self, + mock_get_ops_stack_diagnosis, + mock_get_ops_contract_registry, + mock_get_release_launchpad, + mock_get_runtime_build_info, + mock_list_managed_nodes_with_agent_state, + mock_get_ops_overview, + ) -> None: + mock_get_ops_stack_diagnosis.return_value = { + "diagnosis": { + "contract_key": "ops_stack_diagnosis_contract", + "stack_status": "blocked", + "next_step": { + "action_code": "handover_first_gap", + "reason": "来自 overview.recommendation.primary_action_code", + }, + "operator_decision": {}, + }, + "release_hub": { + "launchpad_status": "attention", + }, + } + mock_get_ops_contract_registry.return_value = { + "registry_version": "2026-04-18", + "contracts": [{"key": "ops_stack_diagnosis_contract"}], + } + mock_get_release_launchpad.return_value = { + "launchpad_status": { + "status": "attention", + "status_label": "待补执行面", + "recommended_action_code": "fix_managed_nodes", + "focus_ref": {"kind": "release_hub"}, + "onboarding_bootstrap_pending_nodes": 1, + "onboarding_acceptance_ready_nodes": 1, + "recommended_target_node_code": "mainland-controller-01", + "recommended_recovery_label": "签发接入工单", + "recommended_recovery_summary": "当前节点仍处于接入阶段，建议先签发 onboarding.bootstrap。", + } + } + mock_get_runtime_build_info.return_value = { + "commit_sha": "246838ae4c07", + "route_surface": { + "surface_complete": True, + "missing_keys": [], + }, + } + mock_list_managed_nodes_with_agent_state.return_value = { + "summary": { + "managed_enabled": 3, + "remote_access_ready": 0, + "queue_dead_letter_nodes": 0, + } + } + mock_get_ops_overview.return_value = { + "recommendation": { + "title": "先补接管缺口", + "reason": "当前仍有节点没有进入标准执行器就绪状态。", + "primary_action_code": "handover_first_gap", + }, + "execution_scene": { + "log_sync": { + "enabled": True, + "state": "waiting_sample", + "mode": "full", + "participating_node_count": 3, + "covered_participating_node_count": 0, + "missing_participating_nodes": [ + "mainland-worker-01", + "mainland-controller-01", + "overseas-control-01", + ], + } + }, + } + + payload = get_ops_go_live_summary(base_url="http://127.0.0.1:8100") + + self.assertEqual("blocked", payload["go_live_status"]) + self.assertFalse(payload["publish_ready"]) + self.assertEqual("blocked", payload["publish_status"]) + self.assertEqual("暂不可发布", payload["publish_status_label"]) + self.assertIn("stack_diagnosis=blocked", payload["blocking_reasons"]) + self.assertIn("stack_diagnosis=blocked", payload["publish_blocking_reasons"]) + self.assertIn("remote_agent_ready=0", payload["blocking_reasons"]) + self.assertIn("remote_agent_ready=0", payload["publish_blocking_reasons"]) + self.assertIn("release_launchpad=attention", payload["warnings"]) + self.assertIn("release_launchpad=attention", payload["publish_warnings"]) + self.assertIn("launchpad_onboarding_bootstrap_pending=1", payload["warnings"]) + self.assertIn("launchpad_onboarding_acceptance_ready=1", payload["warnings"]) + self.assertIn("launchpad_onboarding_bootstrap_pending=1", payload["publish_warnings"]) + self.assertIn("launchpad_onboarding_acceptance_ready=1", payload["publish_warnings"]) + self.assertIn("log_sync_waiting_sample=0/3", payload["warnings"]) + self.assertIn("log_sync_waiting_sample=0/3", payload["publish_warnings"]) + self.assertTrue(payload["route_surface_complete"]) + self.assertEqual("先补接管缺口", payload["operator_title"]) + self.assertEqual("handover_first_gap", payload["next_step_action_code"]) + self.assertEqual("mainland-controller-01", payload["launchpad_recommended_target_node_code"]) + self.assertEqual("签发接入工单", payload["launchpad_recommended_recovery_label"]) + self.assertEqual( + "当前节点仍处于接入阶段，建议先签发 onboarding.bootstrap。", + payload["launchpad_recommended_recovery_summary"], + ) + self.assertEqual(1, payload["launchpad_onboarding_bootstrap_pending_nodes"]) + self.assertEqual(1, payload["launchpad_onboarding_acceptance_ready_nodes"]) + self.assertEqual( + "bash domain-api/deploy/multi-region/drive_ops_center.sh driver-resolve http://127.0.0.1:8100 handover_first_gap", + payload["recommended_commands"]["next_step"], + ) + self.assertEqual( + "bash domain-api/deploy/multi-region/drive_ops_center.sh driver-resolve http://127.0.0.1:8100 open_worker_logs_participating", + payload["recommended_commands"]["log_sync_logs"], + ) + + @patch("app.services.ops_service.get_ops_overview") + @patch("app.services.ops_service.list_managed_nodes_with_agent_state") + @patch("app.services.ops_service.get_runtime_build_info") + @patch("app.services.ops_service.get_release_launchpad") + @patch("app.services.ops_service.get_ops_contract_registry") + @patch("app.services.ops_service.get_ops_stack_diagnosis") + def test_go_live_summary_falls_back_to_stack_handover_hint_when_launchpad_is_generic( + self, + mock_get_ops_stack_diagnosis, + mock_get_ops_contract_registry, + mock_get_release_launchpad, + mock_get_runtime_build_info, + mock_list_managed_nodes_with_agent_state, + mock_get_ops_overview, + ) -> None: + mock_get_ops_stack_diagnosis.return_value = { + "diagnosis": { + "contract_key": "ops_stack_diagnosis_contract", + "stack_status": "blocked", + "next_step": { + "action_code": "bootstrap_run", + "reason": "当前节点仍处于接入阶段，建议先签发 onboarding.bootstrap。", + "focus_ref": {"kind": "managed_node", "node_code": "mainland-worker-01"}, + }, + "operator_decision": { + "lane": "node_handover", + "title": "优先执行节点接管收口", + "primary_command_key": "bootstrap_run", + }, + "launchpad_recommended_target_node_code": "mainland-worker-01", + "launchpad_recommended_recovery_label": "签发接入工单", + "launchpad_recommended_recovery_summary": "当前节点仍处于接入阶段，建议先签发 onboarding.bootstrap。", + }, + "release_hub": { + "launchpad_status": "attention", + }, + } + mock_get_ops_contract_registry.return_value = { + "registry_version": "2026-04-18", + "contracts": [{"key": "ops_stack_diagnosis_contract"}], + } + mock_get_release_launchpad.return_value = { + "launchpad_status": { + "status": "attention", + "status_label": "待补执行面", + "recommended_action_code": "fix_managed_nodes", + } + } + mock_get_runtime_build_info.return_value = { + "commit_sha": "246838ae4c07", + "route_surface": { + "surface_complete": True, + "missing_keys": [], + "expected_paths": {}, + }, + "repository_capabilities": {}, + } + mock_list_managed_nodes_with_agent_state.return_value = { + "summary": { + "managed_enabled": 1, + "remote_access_ready": 0, + "queue_dead_letter_nodes": 0, + } + } + mock_get_ops_overview.return_value = { + "recommendation": { + "title": "先补接管缺口", + "reason": "overview 仍在泛化描述。", + "primary_action_code": "handover_first_gap", + }, + "execution_scene": { + "log_sync": { + "enabled": False, + "participating_node_count": 0, + "covered_participating_node_count": 0, + "missing_participating_nodes": [], + } + }, + } + + payload = get_ops_go_live_summary(base_url="http://127.0.0.1:8100") + + self.assertEqual("bootstrap_run", payload["next_step_action_code"]) + self.assertEqual("mainland-worker-01", payload["launchpad_recommended_target_node_code"]) + self.assertEqual("签发接入工单", payload["launchpad_recommended_recovery_label"]) + self.assertIn("onboarding.bootstrap", payload["launchpad_recommended_recovery_summary"]) + self.assertEqual( + "bash domain-api/deploy/multi-region/drive_ops_center.sh driver-resolve http://127.0.0.1:8100 bootstrap_run", + payload["recommended_commands"]["next_step"], + ) + + @patch("app.services.ops_service.get_ops_overview") + @patch("app.services.ops_service.list_managed_nodes_with_agent_state") + @patch("app.services.ops_service.get_runtime_build_info") + @patch("app.services.ops_service.get_release_launchpad") + @patch("app.services.ops_service.get_ops_contract_registry") + @patch("app.services.ops_service.get_ops_stack_diagnosis") + def test_go_live_summary_marks_ready_when_surfaces_and_execution_are_closed( + self, + mock_get_ops_stack_diagnosis, + mock_get_ops_contract_registry, + mock_get_release_launchpad, + mock_get_runtime_build_info, + mock_list_managed_nodes_with_agent_state, + mock_get_ops_overview, + ) -> None: + mock_get_ops_stack_diagnosis.return_value = { + "diagnosis": { + "contract_key": "ops_stack_diagnosis_contract", + "stack_status": "healthy", + "next_step": { + "action_code": "run_inspection_participating", + "reason": "继续做上线前复核。", + }, + "operator_decision": { + "lane": "go-live", + "title": "可以进入上线复核", + "primary_command_key": "run_inspection_participating", + }, + }, + "release_hub": { + "launchpad_status": "ready", + }, + } + mock_get_ops_contract_registry.return_value = { + "registry_version": "2026-04-18", + "contracts": [{"key": "ops_stack_diagnosis_contract"}], + } + mock_get_release_launchpad.return_value = { + "launchpad_status": { + "status": "ready", + "status_label": "可发布", + "recommended_action_code": "create_smart_release_rollout_worker", + } + } + mock_get_runtime_build_info.return_value = { + "commit_sha": "246838ae4c07", + "route_surface": { + "surface_complete": True, + "missing_keys": [], + }, + } + mock_list_managed_nodes_with_agent_state.return_value = { + "summary": { + "managed_enabled": 2, + "remote_access_ready": 2, + "queue_dead_letter_nodes": 0, + } + } + mock_get_ops_overview.return_value = { + "recommendation": { + "title": "继续复核", + "reason": "当前执行面与观测面已收口。", + "primary_action_code": "run_inspection_participating", + }, + "execution_scene": { + "log_sync": { + "enabled": True, + "state": "healthy", + "mode": "key", + "participating_node_count": 2, + "covered_participating_node_count": 2, + "missing_participating_nodes": [], + } + }, + } + + payload = get_ops_go_live_summary(base_url="http://127.0.0.1:8100") + + self.assertEqual("ready", payload["go_live_status"]) + self.assertTrue(payload["publish_ready"]) + self.assertEqual("ready", payload["publish_status"]) + self.assertEqual("可发布", payload["publish_status_label"]) + self.assertEqual([], payload["blocking_reasons"]) + self.assertEqual([], payload["warnings"]) + self.assertEqual([], payload["publish_blocking_reasons"]) + self.assertEqual([], payload["publish_warnings"]) + self.assertEqual("go-live", payload["operator_lane"]) + self.assertEqual("run_inspection_participating", payload["operator_primary_command_key"]) + self.assertEqual(2, payload["log_sync_covered_nodes"]) + self.assertEqual( + "bash domain-api/deploy/multi-region/drive_ops_center.sh driver-resolve http://127.0.0.1:8100 run_inspection_participating", + payload["recommended_commands"]["next_step"], + ) + + @patch("app.services.ops_service.get_ops_overview") + @patch("app.services.ops_service.list_managed_nodes_with_agent_state") + @patch("app.services.ops_service.get_runtime_build_info") + @patch("app.services.ops_service.get_release_launchpad") + @patch("app.services.ops_service.get_ops_contract_registry") + @patch("app.services.ops_service.get_ops_stack_diagnosis") + def test_go_live_summary_blocks_publish_when_dead_letter_queue_exists( + self, + mock_get_ops_stack_diagnosis, + mock_get_ops_contract_registry, + mock_get_release_launchpad, + mock_get_runtime_build_info, + mock_list_managed_nodes_with_agent_state, + mock_get_ops_overview, + ) -> None: + mock_get_ops_stack_diagnosis.return_value = { + "diagnosis": { + "contract_key": "ops_stack_diagnosis_contract", + "stack_status": "healthy", + "next_step": {"action_code": "delivery_queue_replay", "reason": "先清死信。"}, + "operator_decision": {}, + }, + "release_hub": { + "launchpad_status": "ready", + }, + } + mock_get_ops_contract_registry.return_value = { + "registry_version": "2026-04-18", + "contracts": [{"key": "ops_stack_diagnosis_contract"}], + } + mock_get_release_launchpad.return_value = { + "launchpad_status": { + "status": "ready", + "status_label": "可发布", + "recommended_action_code": "create_smart_release_rollout_worker", + } + } + mock_get_runtime_build_info.return_value = { + "commit_sha": "246838ae4c07", + "route_surface": { + "surface_complete": True, + "missing_keys": [], + }, + } + mock_list_managed_nodes_with_agent_state.return_value = { + "summary": { + "managed_enabled": 2, + "remote_access_ready": 2, + "queue_dead_letter_nodes": 1, + } + } + mock_get_ops_overview.return_value = { + "recommendation": { + "title": "先清死信", + "reason": "当前仍有节点回执死信。", + "primary_action_code": "delivery_queue_replay", + }, + "execution_scene": { + "log_sync": { + "enabled": True, + "state": "healthy", + "mode": "key", + "participating_node_count": 2, + "covered_participating_node_count": 2, + "missing_participating_nodes": [], + } + }, + } + + payload = get_ops_go_live_summary(base_url="http://127.0.0.1:8100") + + self.assertEqual("attention", payload["go_live_status"]) + self.assertFalse(payload["publish_ready"]) + self.assertEqual("blocked", payload["publish_status"]) + + @patch("app.services.ops_service.get_ops_overview") + @patch("app.services.ops_service.list_managed_nodes_with_agent_state") + @patch("app.services.ops_service.get_runtime_build_info") + @patch("app.services.ops_service.get_release_launchpad") + @patch("app.services.ops_service.get_ops_contract_registry") + @patch("app.services.ops_service.get_ops_stack_diagnosis") + def test_go_live_summary_accepts_log_sync_covered_node_list_and_marks_partial( + self, + mock_get_ops_stack_diagnosis, + mock_get_ops_contract_registry, + mock_get_release_launchpad, + mock_get_runtime_build_info, + mock_list_managed_nodes_with_agent_state, + mock_get_ops_overview, + ) -> None: + mock_get_ops_stack_diagnosis.return_value = { + "diagnosis": { + "contract_key": "ops_stack_diagnosis_contract", + "stack_status": "attention", + "next_step": {"action_code": "bootstrap_run", "reason": "继续补大陆接管。"}, + "operator_decision": {}, + }, + "release_hub": { + "launchpad_status": "attention", + }, + } + mock_get_ops_contract_registry.return_value = { + "registry_version": "2026-04-18", + "contracts": [{"key": "ops_stack_diagnosis_contract"}], + } + mock_get_release_launchpad.return_value = { + "launchpad_status": { + "status": "attention", + "status_label": "待补执行面", + "recommended_action_code": "fix_managed_nodes", + } + } + mock_get_runtime_build_info.return_value = { + "commit_sha": "246838ae4c07", + "route_surface": { + "surface_complete": True, + "missing_keys": [], + }, + } + mock_list_managed_nodes_with_agent_state.return_value = { + "summary": { + "managed_enabled": 3, + "remote_access_ready": 1, + "queue_dead_letter_nodes": 0, + } + } + mock_get_ops_overview.return_value = { + "recommendation": { + "title": "先补大陆接管", + "reason": "当前只有海外样本已形成。", + "primary_action_code": "bootstrap_run", + }, + "execution_scene": { + "log_sync": { + "enabled": True, + "state": "partial_coverage", + "mode": "full", + "participating_node_count": 3, + "covered_participating_node_count": 1, + "covered_participating_nodes": ["overseas-control-01"], + "missing_participating_nodes": ["mainland-worker-01", "mainland-controller-01"], + } + }, + } + + payload = get_ops_go_live_summary(base_url="http://127.0.0.1:8100") + + self.assertEqual("attention", payload["go_live_status"]) + self.assertEqual(1, payload["log_sync_covered_nodes"]) + self.assertIn("log_sync_partial=1/3", payload["warnings"]) + self.assertIn("log_sync_partial=1/3", payload["publish_warnings"]) + + @patch("app.services.ops_service.get_ops_overview") + @patch("app.services.ops_service.list_managed_nodes_with_agent_state") + @patch("app.services.ops_service.get_runtime_build_info") + @patch("app.services.ops_service.get_release_launchpad") + @patch("app.services.ops_service.get_ops_contract_registry") + @patch("app.services.ops_service.get_ops_stack_diagnosis") + def test_go_live_summary_marks_attention_for_start_delivery_gap( + self, + mock_get_ops_stack_diagnosis, + mock_get_ops_contract_registry, + mock_get_release_launchpad, + mock_get_runtime_build_info, + mock_list_managed_nodes_with_agent_state, + mock_get_ops_overview, + ) -> None: + mock_get_ops_stack_diagnosis.return_value = { + "diagnosis": { + "contract_key": "ops_stack_diagnosis_contract", + "stack_status": "healthy", + "next_step": { + "action_code": "focus_latest_job_events", + "reason": "先看任务事件。", + }, + "operator_decision": { + "lane": "ops_jobs", + "title": "优先检查任务事件流", + "primary_command_key": "focus_latest_job_events", + }, + "issues": [ + { + "code": "ops_job_start_delivery_failed_local", + "summary": "最近有任务开始回执异常。", + } + ], + }, + "release_hub": { + "launchpad_status": "ready", + }, + } + mock_get_ops_contract_registry.return_value = { + "registry_version": "2026-04-18", + "contracts": [{"key": "ops_stack_diagnosis_contract"}], + } + mock_get_release_launchpad.return_value = { + "launchpad_status": { + "status": "ready", + "status_label": "可发布", + "recommended_action_code": "create_smart_release_rollout_worker", + } + } + mock_get_runtime_build_info.return_value = { + "commit_sha": "246838ae4c07", + "route_surface": { + "surface_complete": True, + "missing_keys": [], + }, + } + mock_list_managed_nodes_with_agent_state.return_value = { + "summary": { + "managed_enabled": 1, + "remote_access_ready": 1, + "queue_dead_letter_nodes": 0, + } + } + mock_get_ops_overview.return_value = { + "recommendation": { + "title": "先看任务事件", + "reason": "最近出现开始回执异常。", + "primary_action_code": "focus_latest_job_events", + }, + "execution_scene": { + "log_sync": { + "enabled": True, + "state": "healthy", + "mode": "key", + "participating_node_count": 1, + "covered_participating_node_count": 1, + "missing_participating_nodes": [], + } + }, + } + + payload = get_ops_go_live_summary(base_url="http://127.0.0.1:8100") + + self.assertEqual("attention", payload["go_live_status"]) + self.assertFalse(payload["publish_ready"]) + self.assertEqual("attention", payload["publish_status"]) + self.assertEqual(1, payload["activity_start_delivery_issue_total"]) + self.assertIn("ops_job_start_delivery_failed_local=1", payload["warnings"]) + self.assertIn("ops_job_start_delivery_failed_local=1", payload["publish_warnings"]) + self.assertEqual("ops_jobs", payload["operator_lane"]) + self.assertEqual("focus_latest_job_events", payload["next_step_action_code"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/domain-api/tests/test_ops_service_link_snapshot.py b/domain-api/tests/test_ops_service_link_snapshot.py new file mode 100644 index 0000000..195cf93 --- /dev/null +++ b/domain-api/tests/test_ops_service_link_snapshot.py @@ -0,0 +1,182 @@ +import unittest +from unittest.mock import patch + +from app.services.ops_service import get_ops_link_snapshot + + +class OpsServiceLinkSnapshotTests(unittest.TestCase): + @patch("app.services.ops_service.get_ops_overview") + def test_link_snapshot_highlights_log_sync_gap_on_active_scene(self, mock_get_ops_overview) -> None: + mock_get_ops_overview.return_value = { + "current_topology": { + "node_code": "overseas-control-01", + "node_region": "overseas", + "node_role": "control", + "worker_mode": "linux-systemd", + "sync_push_enabled": True, + "sync_target_api_base_url": "http://152.53.37.118:8100", + }, + "cluster_summary": { + "online_control_nodes": 2, + "online_worker_nodes": 3, + "dedicated_online_worker_nodes": 1, + "busy_nodes": ["mainland-worker-01", "overseas-control-01"], + }, + "runtime_summary": { + "readiness_status": "ready", + "readiness_summary": "当前多机与跨地域骨架已进入可联调状态。", + "phase_label": "running", + "phase_detail": "当前任务正在推进", + "active_job_code": "detect-20260417164852-75f7cc", + "progress_percent": 42.5, + }, + "execution_scene": { + "participation_summary": { + "effective_online_nodes": 3, + "participating_nodes": 3, + "dispatch_active_nodes": 2, + "recent_only_nodes": 1, + "non_participating_nodes": 0, + "standby_nodes": 0, + "load_syncing_nodes": 0, + "dispatch_active_node_codes": ["mainland-worker-01", "overseas-control-01"], + "non_participating_node_codes": [], + }, + "log_sync": { + "enabled": False, + "mode": "key", + "line_count": 0, + "source_node_count": 0, + "covered_participating_node_count": 0, + "participating_node_count": 3, + "missing_participating_nodes": [ + "mainland-controller-01", + "mainland-worker-01", + "overseas-control-01", + ], + "last_at": "", + "preview_lines": [], + }, + }, + "sync_summary": { + "enabled": True, + "source_region": "mainland", + "target_region": "overseas", + "projected_batches": 0, + "failed_batches": 0, + "records_total": 2513, + }, + "recommendation": { + "key": "enable-scene-log-sync", + "priority": "先补现场日志回传", + "summary": "当前有 3 台节点正在真实参与检测，但海外控制面还看不到过程日志。", + "reason": "先把远端日志回传切到关键模式，才能看到阶段、异常、代理与执行过程。", + "primary_label": "开启关键回传", + "primary_action_code": "enable_log_sync_key", + "primary_node_codes": ["mainland-controller-01", "mainland-worker-01", "overseas-control-01"], + "primary_action_payload": {}, + "secondary_label": "开启全量回传", + "secondary_action_code": "enable_log_sync_full", + "secondary_node_codes": ["mainland-controller-01", "mainland-worker-01", "overseas-control-01"], + "secondary_action_payload": {}, + }, + } + + snapshot = get_ops_link_snapshot() + + self.assertEqual("attention", snapshot["status"]) + self.assertIn("日志回传", snapshot["headline"]) + self.assertIn("活跃任务 detect-20260417164852-75f7cc", snapshot["summary_lines"][2]) + self.assertIn("远端日志：关闭", snapshot["summary_lines"][4]) + self.assertEqual("enable_log_sync_key", snapshot["next_actions"][0]["action_code"]) + self.assertEqual(3, snapshot["execution"]["participating_nodes"]) + self.assertFalse(snapshot["log_sync"]["enabled"]) + + @patch("app.services.ops_service.get_ops_overview") + def test_link_snapshot_reports_ready_when_scene_is_observable(self, mock_get_ops_overview) -> None: + mock_get_ops_overview.return_value = { + "current_topology": { + "node_code": "overseas-control-01", + "node_region": "overseas", + "node_role": "control", + "worker_mode": "linux-systemd", + "sync_push_enabled": True, + "sync_target_api_base_url": "http://152.53.37.118:8100", + }, + "cluster_summary": { + "online_control_nodes": 2, + "online_worker_nodes": 2, + "dedicated_online_worker_nodes": 1, + "busy_nodes": ["mainland-worker-01"], + }, + "runtime_summary": { + "readiness_status": "ready", + "readiness_summary": "ok", + "phase_label": "running", + "phase_detail": "检测任务正常推进", + "active_job_code": "detect-001", + "progress_percent": 66.0, + }, + "execution_scene": { + "participation_summary": { + "effective_online_nodes": 2, + "participating_nodes": 2, + "dispatch_active_nodes": 2, + "recent_only_nodes": 0, + "non_participating_nodes": 0, + "standby_nodes": 0, + "load_syncing_nodes": 0, + "dispatch_active_node_codes": ["mainland-worker-01", "mainland-controller-01"], + "non_participating_node_codes": [], + }, + "log_sync": { + "enabled": True, + "mode": "key", + "line_count": 18, + "source_node_count": 2, + "covered_participating_node_count": 2, + "participating_node_count": 2, + "missing_participating_nodes": [], + "last_at": "2026-04-17 22:20:00", + "preview_lines": [ + "[2026-04-17 22:19:59] [mainland-worker-01] 开始检测域名: alpha.com", + "[2026-04-17 22:20:00] [mainland-controller-01] 域名检测完成: beta.com", + ], + }, + }, + "sync_summary": { + "enabled": True, + "source_region": "mainland", + "target_region": "overseas", + "projected_batches": 0, + "failed_batches": 0, + "records_total": 2600, + }, + "recommendation": { + "key": "inspect-participating", + "priority": "先看执行现场", + "summary": "当前有 2 台节点正在真实参与检测，优先回收它们的标准巡检结果。", + "reason": "真正参与检测的节点最接近现场。", + "primary_label": "执行标准巡检", + "primary_action_code": "run_inspection_participating", + "primary_node_codes": ["mainland-worker-01", "mainland-controller-01"], + "primary_action_payload": {}, + "secondary_label": "打开诊断模板", + "secondary_action_code": "open_diagnostics_participating", + "secondary_node_codes": ["mainland-worker-01", "mainland-controller-01"], + "secondary_action_payload": {}, + }, + } + + snapshot = get_ops_link_snapshot() + + self.assertEqual("ready", snapshot["status"]) + self.assertIn("可观测状态", snapshot["headline"]) + self.assertIn("覆盖 2/2", snapshot["summary_lines"][4]) + self.assertEqual(2, snapshot["log_sync"]["source_node_count"]) + self.assertEqual(2, len(snapshot["next_actions"])) + self.assertEqual("run_inspection_participating", snapshot["next_actions"][0]["action_code"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/domain-api/tests/test_ops_service_node_scene_log.py b/domain-api/tests/test_ops_service_node_scene_log.py new file mode 100644 index 0000000..b0a8a22 --- /dev/null +++ b/domain-api/tests/test_ops_service_node_scene_log.py @@ -0,0 +1,256 @@ +import unittest +from unittest.mock import patch + +from app.services.ops_service import get_ops_node_scene_log + + +class OpsServiceNodeSceneLogTests(unittest.TestCase): + def _runtime_payload_with_worker_logs(self) -> dict: + return { + "detect": { + "log_sync": { + "enabled": True, + "mode": "key", + "line_count": 2, + "source_node_count": 1, + "source_nodes": ["mainland-worker-01"], + "source_node_summaries": [ + { + "node_code": "mainland-worker-01", + "line_count": 2, + "key_line_count": 2, + "full_line_count": 0, + "last_at": "2026-04-18 05:02:00", + "last_line": "[2026-04-18 05:02:00] [mainland-worker-01] 域名检测完成: alpha.com", + } + ], + "preview_lines": [ + "[2026-04-18 05:00:00] [mainland-worker-01] 开始检测域名: alpha.com", + "[2026-04-18 05:02:00] [mainland-worker-01] 域名检测完成: alpha.com", + ], + "last_at": "2026-04-18 05:02:00", + "last_line": "[2026-04-18 05:02:00] [mainland-worker-01] 域名检测完成: alpha.com", + }, + "participation_summary": { + "participating_nodes": 1, + "dispatch_active_nodes": 1, + "recent_only_nodes": 0, + "non_participating_nodes": 1, + "standby_nodes": 1, + "load_syncing_nodes": 0, + "effective_online_nodes": 2, + "dedicated_worker_nodes": 2, + "controller_worker_nodes": 0, + "dispatch_active_node_codes": ["mainland-worker-01"], + "recent_only_node_codes": [], + "non_participating_node_codes": ["mainland-worker-02"], + "standby_node_codes": ["mainland-worker-02"], + "load_syncing_node_codes": [], + "summary": "有效执行节点 2 台；正在执行/领任务 1 台；近窗刚有吞吐 0 台；在线但未参与 1 台", + }, + "participating_nodes": [ + { + "node_code": "mainland-worker-01", + "region": "mainland", + "role": "worker", + "status": "busy", + "current_load": 2, + "last_heartbeat_at": "2026-04-18 05:02:10", + "participation_state": "claimed", + "participation_label": "已领待跑", + "participation_reason": "已领取 2 项任务，等待线程继续执行。", + "is_dispatch_active": True, + } + ], + "non_participating_nodes": [ + { + "node_code": "mainland-worker-02", + "region": "mainland", + "role": "worker", + "status": "online", + "current_load": 0, + "last_heartbeat_at": "2026-04-18 05:02:11", + "participation_state": "standby", + "participation_label": "在线待命", + "participation_reason": "当前节点在线，但尚未参与本轮检测。", + "is_dispatch_active": False, + } + ], + "active_job": { + "job_code": "detect-001", + "status": "running", + "current_cycle_token": "cycle-1", + "current_cycle_events": [ + { + "event_type": "worker_log", + "created_at": "2026-04-18 05:02:00", + "node_code": "mainland-worker-01", + "message": "域名检测完成: alpha.com", + "payload": {"cycle_token": "cycle-1", "log_mode": "key"}, + }, + { + "event_type": "worker_log", + "created_at": "2026-04-18 05:01:00", + "node_code": "mainland-worker-01", + "message": "代理池刷新完成", + "payload": {"cycle_token": "cycle-1", "log_mode": "full"}, + }, + { + "event_type": "worker_log", + "created_at": "2026-04-18 05:00:00", + "node_code": "mainland-worker-01", + "message": "开始检测域名: alpha.com", + "payload": {"cycle_token": "cycle-1", "log_mode": "key"}, + }, + { + "event_type": "worker_log", + "created_at": "2026-04-18 04:59:00", + "node_code": "mainland-worker-01", + "message": "旧周期日志", + "payload": {"cycle_token": "cycle-0", "log_mode": "key"}, + }, + ], + }, + }, + "cluster": { + "nodes": [ + { + "node_code": "mainland-worker-01", + "region": "mainland", + "role": "worker", + "status": "busy", + "current_load": 2, + "last_heartbeat_at": "2026-04-18 05:02:10", + }, + { + "node_code": "mainland-worker-02", + "region": "mainland", + "role": "worker", + "status": "online", + "current_load": 0, + "last_heartbeat_at": "2026-04-18 05:02:11", + }, + ] + }, + } + + @patch("app.services.ops_service.get_runtime_status") + def test_node_scene_log_filters_key_mode_records_for_participating_node(self, mock_get_runtime_status) -> None: + mock_get_runtime_status.return_value = self._runtime_payload_with_worker_logs() + + payload = get_ops_node_scene_log("mainland-worker-01", limit=10) + + self.assertEqual("mainland-worker-01", payload["node_code"]) + self.assertEqual("healthy", payload["status"]) + self.assertEqual("关键覆盖", payload["status_label"]) + self.assertEqual("key", payload["mode"]) + self.assertTrue(payload["log_sync_enabled"]) + self.assertTrue(payload["participation"]["detect_participating"]) + self.assertEqual(2, payload["records_total"]) + self.assertEqual( + [ + "开始检测域名: alpha.com", + "域名检测完成: alpha.com", + ], + [item["message"] for item in payload["records"]], + ) + self.assertEqual(2, payload["source_summary"]["line_count"]) + self.assertEqual(2, payload["source_summary"]["key_line_count"]) + self.assertEqual(0, payload["source_summary"]["full_line_count"]) + self.assertEqual("ops_observability_contract", payload["contract_navigation"]["primary_contract_key"]) + + @patch("app.services.ops_service.get_runtime_status") + def test_node_scene_log_can_request_full_mode_records(self, mock_get_runtime_status) -> None: + mock_get_runtime_status.return_value = self._runtime_payload_with_worker_logs() + + payload = get_ops_node_scene_log("mainland-worker-01", limit=10, mode="full") + + self.assertEqual("full_capture", payload["status"]) + self.assertEqual("full", payload["mode"]) + self.assertEqual(3, payload["records_total"]) + self.assertEqual( + [ + "开始检测域名: alpha.com", + "代理池刷新完成", + "域名检测完成: alpha.com", + ], + [item["message"] for item in payload["records"]], + ) + self.assertEqual(3, payload["source_summary"]["line_count"]) + self.assertEqual(2, payload["source_summary"]["key_line_count"]) + self.assertEqual(1, payload["source_summary"]["full_line_count"]) + + @patch("app.services.ops_service.get_runtime_status") + def test_node_scene_log_marks_waiting_sample_for_missing_participating_node(self, mock_get_runtime_status) -> None: + mock_get_runtime_status.return_value = { + "detect": { + "log_sync": { + "enabled": True, + "mode": "key", + "line_count": 0, + "source_node_count": 0, + "source_nodes": [], + "source_node_summaries": [], + "preview_lines": [], + "last_at": "", + "last_line": "", + }, + "participation_summary": { + "participating_nodes": 1, + "dispatch_active_nodes": 1, + "recent_only_nodes": 0, + "non_participating_nodes": 0, + "standby_nodes": 0, + "load_syncing_nodes": 0, + "effective_online_nodes": 1, + "dedicated_worker_nodes": 1, + "controller_worker_nodes": 0, + "dispatch_active_node_codes": ["mainland-worker-02"], + "recent_only_node_codes": [], + "non_participating_node_codes": [], + "standby_node_codes": [], + "load_syncing_node_codes": [], + "summary": "有效执行节点 1 台；正在执行/领任务 1 台；近窗刚有吞吐 0 台；在线但未参与 0 台", + }, + "participating_nodes": [ + { + "node_code": "mainland-worker-02", + "region": "mainland", + "role": "worker", + "status": "busy", + "current_load": 1, + "last_heartbeat_at": "2026-04-18 05:05:00", + "participation_state": "running", + "participation_label": "正在执行", + "participation_reason": "当前存在执行中工作负载。", + "is_dispatch_active": True, + } + ], + "non_participating_nodes": [], + "active_job": {"job_code": "detect-002", "status": "running", "current_cycle_events": []}, + }, + "cluster": { + "nodes": [ + { + "node_code": "mainland-worker-02", + "region": "mainland", + "role": "worker", + "status": "busy", + "current_load": 1, + "last_heartbeat_at": "2026-04-18 05:05:00", + } + ] + }, + } + + payload = get_ops_node_scene_log("mainland-worker-02") + + self.assertEqual("waiting_sample", payload["status"]) + self.assertEqual("等待样本", payload["status_label"]) + self.assertEqual(0, payload["records_total"]) + self.assertTrue(payload["participation"]["detect_participating"]) + self.assertIn("现场样本尚未形成", payload["summary"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/domain-api/tests/test_ops_service_overview.py b/domain-api/tests/test_ops_service_overview.py new file mode 100644 index 0000000..a098b4e --- /dev/null +++ b/domain-api/tests/test_ops_service_overview.py @@ -0,0 +1,1186 @@ +import unittest +from unittest.mock import patch + +from app.services.ops_service import get_ops_inspection_overview, get_ops_overview + + +class OpsServiceOverviewTests(unittest.TestCase): + @patch("app.services.ops_service.list_managed_nodes_with_agent_state") + @patch("app.services.ops_service.list_ops_jobs") + def test_inspection_overview_fetches_managed_nodes_when_omitted( + self, + mock_list_ops_jobs, + mock_list_managed_nodes_with_agent_state, + ) -> None: + mock_list_managed_nodes_with_agent_state.return_value = { + "nodes": [ + { + "node_code": "mainland-worker-01", + "region": "mainland", + "role": "worker", + "is_managed": True, + "is_enabled": True, + "is_agent_online": True, + "cluster_status": "online", + "cluster_is_effective_worker": True, + "cluster_detect_participating": True, + } + ] + } + mock_list_ops_jobs.return_value = [ + { + "id": 21, + "job_code": "ops-health-21", + "action": "health.snapshot", + "status": "success", + "target_node_code": "mainland-worker-01", + "created_at": "2026-04-17 10:00:00", + "updated_at": "2026-04-17 10:01:00", + } + ] + + overview = get_ops_inspection_overview(fetch_limit=20) + + self.assertEqual(1, overview["eligible_nodes_total"]) + self.assertEqual(1, overview["rows_total"]) + self.assertEqual("attention", overview["status"]) + self.assertEqual("待处理", overview["status_label"]) + self.assertEqual("mainland", overview["rows"][0]["region"]) + self.assertEqual("worker", overview["rows"][0]["role"]) + self.assertEqual("online", overview["rows"][0]["cluster_status"]) + self.assertIn("latest_health_snapshot", overview["rows"][0]) + self.assertIn("latest_worker_logs", overview["rows"][0]) + self.assertIn("latest_diagnostics", overview["rows"][0]) + + @patch("app.services.ops_service.list_ops_jobs") + def test_inspection_overview_prioritizes_attention_and_supports_filters(self, mock_list_ops_jobs) -> None: + mock_list_ops_jobs.return_value = [ + { + "id": 11, + "job_code": "ops-health-1", + "action": "health.snapshot", + "status": "success", + "target_node_code": "mainland-worker-01", + "created_at": "2026-04-17 10:00:00", + "updated_at": "2026-04-17 10:01:00", + }, + { + "id": 12, + "job_code": "ops-logs-1", + "action": "logs.collect", + "status": "failed", + "target_node_code": "mainland-worker-01", + "created_at": "2026-04-17 10:02:00", + "updated_at": "2026-04-17 10:03:00", + "error_message": "journalctl permission denied", + "payload": { + "service_name": "domaincheck-worker", + "lines": 120, + }, + }, + ] + + managed_nodes = [ + { + "node_code": "mainland-worker-01", + "region": "mainland", + "role": "worker", + "is_managed": True, + "is_enabled": True, + "is_agent_online": True, + "cluster_status": "online", + "cluster_is_effective_worker": True, + "cluster_detect_participating": True, + }, + { + "node_code": "mainland-controller-01", + "region": "mainland", + "role": "control", + "is_managed": True, + "is_enabled": True, + "is_agent_online": True, + "cluster_status": "online", + "cluster_is_effective_worker": True, + "cluster_detect_participating": False, + }, + ] + + overview = get_ops_inspection_overview(managed_nodes=managed_nodes, fetch_limit=20) + + self.assertEqual(2, overview["rows_total"]) + self.assertEqual(1, overview["summary"]["attention"]) + self.assertEqual(1, overview["summary"]["missing"]) + self.assertEqual("attention", overview["status"]) + self.assertEqual("待处理", overview["status_label"]) + self.assertEqual(1, overview["counts"]["attention_nodes"]) + self.assertEqual(1, overview["counts"]["missing_nodes"]) + self.assertEqual("mainland-worker-01", overview["priority_queue"][0]["node_code"]) + self.assertEqual("attention", overview["priority_queue"][0]["overall_status"]) + self.assertEqual("terminal_failure", overview["priority_queue"][0]["problem_kind"]) + self.assertEqual("failed", overview["rows"][0]["latest_worker_logs"]["status"]) + self.assertEqual("成功", overview["rows"][0]["latest_health_snapshot"]["status_label"]) + + filtered = get_ops_inspection_overview( + managed_nodes=managed_nodes, + fetch_limit=20, + only_problem=True, + only_participating=True, + ) + self.assertEqual(1, filtered["filtered_rows_total"]) + self.assertEqual("mainland-worker-01", filtered["rows"][0]["node_code"]) + self.assertEqual("attention", filtered["rows"][0]["overall_status"]) + + @patch("app.services.ops_service.list_ops_jobs") + def test_inspection_overview_includes_visible_handover_gap_nodes(self, mock_list_ops_jobs) -> None: + mock_list_ops_jobs.return_value = [] + + managed_nodes = [ + { + "node_code": "mainland-controller-01", + "region": "mainland", + "role": "control", + "is_managed": True, + "is_enabled": True, + "is_agent_online": True, + "cluster_status": "online", + "cluster_is_effective_worker": True, + "cluster_detect_participating": False, + }, + { + "node_code": "mainland-worker-02", + "region": "mainland", + "role": "worker", + "is_managed": False, + "is_enabled": False, + "is_agent_online": False, + "cluster_status": "online", + "cluster_is_effective_worker": True, + "cluster_detect_participating": False, + }, + ] + + overview = get_ops_inspection_overview(managed_nodes=managed_nodes, fetch_limit=20) + row_map = {item["node_code"]: item for item in overview["rows"]} + + self.assertEqual(2, overview["visible_nodes_total"]) + self.assertEqual(1, overview["eligible_nodes_total"]) + self.assertEqual(1, overview["handover_gap_nodes_total"]) + self.assertEqual("attention", row_map["mainland-worker-02"]["overall_status"]) + self.assertEqual("handover_gap", row_map["mainland-worker-02"]["problem_kind"]) + self.assertEqual("handover_first_gap", row_map["mainland-worker-02"]["recommended_action_code"]) + + @patch("app.services.ops_service.list_ops_jobs") + def test_inspection_overview_marks_participating_nodes_without_inspection_as_attention(self, mock_list_ops_jobs) -> None: + mock_list_ops_jobs.return_value = [] + + managed_nodes = [ + { + "node_code": "mainland-worker-01", + "region": "mainland", + "role": "worker", + "is_managed": True, + "is_enabled": True, + "is_agent_online": True, + "cluster_status": "busy", + "cluster_is_effective_worker": True, + "cluster_detect_participating": True, + } + ] + + overview = get_ops_inspection_overview( + managed_nodes=managed_nodes, + fetch_limit=20, + only_participating=True, + only_problem=True, + ) + + self.assertEqual(1, overview["filtered_rows_total"]) + self.assertEqual("attention", overview["rows"][0]["overall_status"]) + self.assertEqual("participating_no_inspection", overview["rows"][0]["problem_kind"]) + self.assertEqual("open_worker_logs_participating", overview["rows"][0]["recommended_action_code"]) + self.assertEqual("attention", overview["status"]) + + @patch("app.services.ops_service.build_release_execution_mode_recommendation") + @patch("app.services.ops_service.get_release_launchpad") + @patch("app.services.ops_service.get_latest_release") + @patch("app.services.ops_service.get_release_summary") + @patch("app.services.ops_service.list_release_rollouts") + @patch("app.services.ops_service.get_recent_ops_playbook_runs") + @patch("app.services.ops_service.list_ops_jobs") + @patch("app.services.ops_service.list_managed_nodes_with_agent_state") + @patch("app.services.ops_service.get_ops_job_summary") + @patch("app.services.ops_service.get_sync_summary") + @patch("app.services.ops_service.get_runtime_status") + def test_ops_overview_uses_top_driver_recommendation_as_summary( + self, + mock_get_runtime_status, + mock_get_sync_summary, + mock_get_ops_job_summary, + mock_list_managed_nodes_with_agent_state, + mock_list_ops_jobs, + mock_get_recent_ops_playbook_runs, + mock_list_release_rollouts, + mock_get_release_summary, + mock_get_latest_release, + mock_get_release_launchpad, + mock_build_release_execution_mode_recommendation, + ) -> None: + mock_get_runtime_status.return_value = { + "readiness": { + "status": "attention", + "summary": "存在接管缺口", + }, + "detect": { + "log_sync": { + "enabled": False, + "mode": "key", + "line_count": 0, + "source_node_count": 0, + "source_nodes": [], + "last_at": "", + "last_line": "", + "preview_lines": [], + }, + "participation_summary": {}, + "participating_nodes": [], + "non_participating_nodes": [], + "active_job": {}, + "phase_label": "", + "phase_detail": "", + "recent_event": "", + "recent_warning": "", + }, + "cluster": { + "summary": { + "online_worker_nodes": 1, + "online_control_nodes": 1, + "dedicated_online_worker_nodes": 1, + "busy_nodes": [], + }, + "nodes": [ + { + "node_code": "mainland-worker-01", + "region": "mainland", + "role": "worker", + "status": "online", + "current_load": 0, + "hostname": "worker-01", + "last_heartbeat_at": "2026-04-17 10:00:00", + "is_effective_worker": True, + "detect_participating": False, + } + ], + }, + } + mock_get_sync_summary.return_value = {} + mock_get_ops_job_summary.return_value = {"total": 0, "queued": 0, "running": 0, "awaiting_approval": 0} + mock_list_managed_nodes_with_agent_state.return_value = { + "nodes": [ + { + "node_code": "mainland-worker-01", + "region": "mainland", + "role": "worker", + "is_managed": False, + "is_enabled": False, + "is_agent_online": False, + "cluster_status": "online", + "cluster_is_effective_worker": True, + "cluster_detect_participating": False, + } + ], + "summary": { + "total": 1, + "managed_total": 0, + "managed_enabled": 0, + "online": 0, + "stale": 0, + "pending_bootstrap": 1, + "runtime_only": 1, + "unmanaged": 1, + "token_issue": 0, + "status_counts": {}, + }, + } + mock_list_ops_jobs.return_value = [] + mock_get_recent_ops_playbook_runs.return_value = {"runs": []} + mock_list_release_rollouts.return_value = [] + mock_get_release_summary.return_value = {} + mock_get_latest_release.return_value = {} + mock_get_release_launchpad.return_value = {"launchpad_status": {"status": "attention"}} + mock_build_release_execution_mode_recommendation.return_value = { + "recommended_mode": "remote-agent", + "recommended_mode_label": "remote-agent", + "reason": "", + "options": [], + "recommended_gate": { + "status": "release_not_ready", + "summary": "", + "target_nodes": [], + "target_node_codes": [], + "blocking_reasons": [], + "warning_reasons": [], + "execution_mode": "remote-agent", + "execution_mode_label": "remote-agent", + "operational_readiness": {"rows": [], "summary": {}}, + }, + "gates": {}, + } + + overview = get_ops_overview() + + mock_list_managed_nodes_with_agent_state.assert_any_call( + participation_payload=mock_get_runtime_status.return_value["detect"] + ) + self.assertEqual("driver_recommendations", overview["recommendation"]["source"]) + self.assertEqual("handover-gap", overview["recommendation"]["key"]) + self.assertEqual("先补接管缺口", overview["recommendation"]["priority"]) + self.assertIn("有效执行节点", overview["recommendation"]["summary"]) + self.assertEqual("纳管首台节点", overview["recommendation"]["primary_label"]) + self.assertEqual("handover_first_gap", overview["recommendation"]["primary_action_code"]) + self.assertEqual("attention", overview["release_hub"]["launchpad"]["launchpad_status"]["status"]) + + @patch("app.services.ops_service.build_release_execution_mode_recommendation") + @patch("app.services.ops_service.get_release_launchpad") + @patch("app.services.ops_service.get_latest_release") + @patch("app.services.ops_service.get_release_summary") + @patch("app.services.ops_service.list_release_rollouts") + @patch("app.services.ops_service.get_recent_ops_playbook_runs") + @patch("app.services.ops_service.list_ops_jobs") + @patch("app.services.ops_service.list_managed_nodes_with_agent_state") + @patch("app.services.ops_service.get_ops_job_summary") + @patch("app.services.ops_service.get_sync_summary") + @patch("app.services.ops_service.get_runtime_status") + def test_ops_overview_prefers_launchpad_bootstrap_recovery_for_handover_gap( + self, + mock_get_runtime_status, + mock_get_sync_summary, + mock_get_ops_job_summary, + mock_list_managed_nodes_with_agent_state, + mock_list_ops_jobs, + mock_get_recent_ops_playbook_runs, + mock_list_release_rollouts, + mock_get_release_summary, + mock_get_latest_release, + mock_get_release_launchpad, + mock_build_release_execution_mode_recommendation, + ) -> None: + mock_get_runtime_status.return_value = { + "readiness": {"status": "attention", "summary": "当前还需补执行面接管"}, + "worker": {"running": True}, + "sync_agent": {"running": True}, + "detect": {"participation_summary": {"summary": ""}}, + } + mock_get_sync_summary.return_value = {} + mock_get_ops_job_summary.return_value = {} + mock_list_managed_nodes_with_agent_state.return_value = { + "nodes": [ + { + "node_code": "mainland-worker-02", + "role": "worker", + "is_managed": False, + "is_enabled": False, + "is_agent_online": False, + "cluster_status": "online", + "cluster_is_effective_worker": True, + "is_current_participant": False, + } + ], + "summary": { + "total": 1, + "managed_total": 0, + "managed_enabled": 0, + "online": 1, + "stale": 0, + "pending_bootstrap": 1, + "runtime_only": 0, + "unmanaged": 1, + "token_issue": 0, + "status_counts": {}, + }, + } + mock_list_ops_jobs.return_value = [] + mock_get_recent_ops_playbook_runs.return_value = {"runs": []} + mock_list_release_rollouts.return_value = [] + mock_get_release_summary.return_value = {} + mock_get_latest_release.return_value = {} + mock_get_release_launchpad.return_value = { + "launchpad_status": { + "status": "attention", + "status_label": "待补执行面", + "summary": "当前节点仍处于接入阶段，建议先签发 onboarding.bootstrap。", + "recommended_action_code": "bootstrap_run", + }, + "worker_rollout_preview": { + "policy_preview": { + "release_gate": { + "operational_readiness": { + "rows": [ + { + "node_code": "mainland-worker-02", + "execution_ready": False, + "recovery_action": "bootstrap_run", + "recovery_summary": "当前节点仍处于接入阶段，建议先签发 onboarding.bootstrap。", + } + ] + } + } + } + }, + } + mock_build_release_execution_mode_recommendation.return_value = { + "recommended_mode": "remote-agent", + "recommended_mode_label": "remote-agent", + "reason": "", + "options": [], + "recommended_gate": { + "status": "release_not_ready", + "summary": "", + "target_nodes": [], + "target_node_codes": [], + "blocking_reasons": [], + "warning_reasons": [], + "execution_mode": "remote-agent", + "execution_mode_label": "remote-agent", + "operational_readiness": {"rows": [], "summary": {}}, + }, + "gates": {}, + } + + overview = get_ops_overview() + + self.assertEqual("handover-gap-bootstrap_run", overview["recommendation"]["key"]) + self.assertEqual("bootstrap_run", overview["recommendation"]["primary_action_code"]) + self.assertEqual("跑接入收口", overview["recommendation"]["primary_label"]) + self.assertEqual(["mainland-worker-02"], overview["recommendation"]["primary_node_codes"]) + + @patch("app.services.ops_service.build_release_execution_mode_recommendation") + @patch("app.services.ops_service.get_release_launchpad") + @patch("app.services.ops_service.get_latest_release") + @patch("app.services.ops_service.get_release_summary") + @patch("app.services.ops_service.list_release_rollouts") + @patch("app.services.ops_service.get_recent_ops_playbook_runs") + @patch("app.services.ops_service.list_ops_jobs") + @patch("app.services.ops_service.list_managed_nodes_with_agent_state") + @patch("app.services.ops_service.get_ops_job_summary") + @patch("app.services.ops_service.get_sync_summary") + @patch("app.services.ops_service.get_runtime_status") + def test_ops_overview_exposes_remote_log_preview_lines( + self, + mock_get_runtime_status, + mock_get_sync_summary, + mock_get_ops_job_summary, + mock_list_managed_nodes_with_agent_state, + mock_list_ops_jobs, + mock_get_recent_ops_playbook_runs, + mock_list_release_rollouts, + mock_get_release_summary, + mock_get_latest_release, + mock_get_release_launchpad, + mock_build_release_execution_mode_recommendation, + ) -> None: + mock_get_runtime_status.return_value = { + "readiness": {"status": "ready", "summary": "ok"}, + "detect": { + "log_sync": { + "enabled": True, + "mode": "key", + "line_count": 2, + "source_node_count": 1, + "source_nodes": ["mainland-worker-01"], + "last_at": "2026-04-17 10:01:00", + "last_line": "[2026-04-17 10:01:00] [mainland-worker-01] 域名检测完成: alpha.com", + "preview_lines": [ + "[2026-04-17 10:00:00] [mainland-worker-01] 开始检测域名: alpha.com", + "[2026-04-17 10:01:00] [mainland-worker-01] 域名检测完成: alpha.com", + ], + }, + "participation_summary": { + "participating_nodes": 1, + "dispatch_active_nodes": 1, + "recent_only_nodes": 0, + "non_participating_nodes": 0, + "standby_nodes": 0, + "load_syncing_nodes": 0, + "effective_online_nodes": 1, + "dedicated_worker_nodes": 1, + "controller_worker_nodes": 0, + "dispatch_active_node_codes": ["mainland-worker-01"], + "recent_only_node_codes": [], + "non_participating_node_codes": [], + "standby_node_codes": [], + "load_syncing_node_codes": [], + "summary": "有效执行节点 1 台；正在执行/领任务 1 台；近窗刚有吞吐 0 台；在线但未参与 0 台", + }, + "participating_nodes": [ + { + "node_code": "mainland-worker-01", + "region": "mainland", + "role": "worker", + "status": "busy", + "current_load": 1, + "participation_state": "running", + "participation_label": "执行中", + "participation_reason": "当前正在执行 1 项检测任务。", + "is_dispatch_active": True, + } + ], + "non_participating_nodes": [], + "active_job": {"job_code": "detect-001", "status": "running"}, + "phase_label": "运行中", + "phase_detail": "测试中", + "recent_event": "", + "recent_warning": "", + }, + "cluster": { + "summary": { + "online_worker_nodes": 1, + "online_control_nodes": 1, + "dedicated_online_worker_nodes": 1, + "busy_nodes": ["mainland-worker-01"], + }, + "nodes": [ + { + "node_code": "mainland-worker-01", + "region": "mainland", + "role": "worker", + "status": "busy", + "current_load": 1, + "hostname": "worker-01", + "last_heartbeat_at": "2026-04-17 10:01:00", + "is_effective_worker": True, + "detect_participating": True, + } + ], + }, + } + mock_get_sync_summary.return_value = {} + mock_get_ops_job_summary.return_value = {"total": 0, "queued": 0, "running": 0, "awaiting_approval": 0} + mock_list_managed_nodes_with_agent_state.return_value = { + "nodes": [ + { + "node_code": "mainland-worker-01", + "region": "mainland", + "role": "worker", + "is_managed": True, + "is_enabled": True, + "is_agent_online": True, + "cluster_status": "busy", + "cluster_is_effective_worker": True, + "cluster_detect_participating": True, + } + ], + "summary": { + "total": 1, + "managed_total": 1, + "managed_enabled": 1, + "online": 1, + "stale": 0, + "pending_bootstrap": 0, + "runtime_only": 0, + "unmanaged": 0, + "token_issue": 0, + "status_counts": {}, + "participating": 1, + "dispatch_active": 1, + "recent_only": 0, + "standby": 0, + "load_syncing": 0, + "non_participating": 0, + }, + } + mock_list_ops_jobs.return_value = [] + mock_get_recent_ops_playbook_runs.return_value = {"runs": []} + mock_list_release_rollouts.return_value = [] + mock_get_release_summary.return_value = {} + mock_get_latest_release.return_value = {} + mock_get_release_launchpad.return_value = {"launchpad_status": {"status": "ready"}} + mock_build_release_execution_mode_recommendation.return_value = { + "recommended_mode": "remote-agent", + "recommended_mode_label": "remote-agent", + "reason": "", + "options": [], + "recommended_gate": { + "status": "release_not_ready", + "summary": "", + "target_nodes": [], + "target_node_codes": [], + "blocking_reasons": [], + "warning_reasons": [], + "execution_mode": "remote-agent", + "execution_mode_label": "remote-agent", + "operational_readiness": {"rows": [], "summary": {}}, + }, + "gates": {}, + } + + overview = get_ops_overview() + execution_scene = overview["execution_scene"] + log_sync = overview["execution_scene"]["log_sync"] + + self.assertEqual("有效执行节点 1 台；正在执行/领任务 1 台；近窗刚有吞吐 0 台；在线但未参与 0 台", execution_scene["summary"]) + self.assertEqual(1, execution_scene["counts"]["dispatch_active"]) + self.assertEqual(0, execution_scene["counts"]["recent_only"]) + self.assertEqual(0, execution_scene["counts"]["standby"]) + self.assertEqual(0, execution_scene["counts"]["load_syncing"]) + self.assertEqual(1, len(execution_scene["dispatch_active_nodes"])) + self.assertEqual("dispatch_active", execution_scene["dispatch_active_nodes"][0]["participation_bucket"]) + self.assertEqual([], execution_scene["recent_only_nodes"]) + self.assertEqual([], execution_scene["load_syncing_nodes"]) + self.assertTrue(log_sync["enabled"]) + self.assertEqual("key", log_sync["mode"]) + self.assertEqual(2, log_sync["preview_line_count"]) + self.assertEqual( + [ + "[2026-04-17 10:00:00] [mainland-worker-01] 开始检测域名: alpha.com", + "[2026-04-17 10:01:00] [mainland-worker-01] 域名检测完成: alpha.com", + ], + log_sync["preview_lines"], + ) + self.assertEqual(1, log_sync["covered_participating_node_count"]) + self.assertEqual([], log_sync["missing_participating_nodes"]) + self.assertEqual(1, len(log_sync["source_node_summaries"])) + self.assertEqual("mainland-worker-01", log_sync["source_node_summaries"][0]["node_code"]) + self.assertEqual(2, log_sync["source_node_summaries"][0]["line_count"]) + self.assertEqual(2, log_sync["source_node_summaries"][0]["key_line_count"]) + self.assertEqual([], log_sync["missing_participating_node_summaries"]) + self.assertEqual("ready", overview["release_hub"]["launchpad"]["launchpad_status"]["status"]) + + @patch("app.services.ops_service.build_release_execution_mode_recommendation") + @patch("app.services.ops_service.get_release_launchpad") + @patch("app.services.ops_service.get_latest_release") + @patch("app.services.ops_service.get_release_summary") + @patch("app.services.ops_service.list_release_rollouts") + @patch("app.services.ops_service.get_recent_ops_playbook_runs") + @patch("app.services.ops_service.list_ops_jobs") + @patch("app.services.ops_service.list_managed_nodes_with_agent_state") + @patch("app.services.ops_service.get_ops_job_summary") + @patch("app.services.ops_service.get_sync_summary") + @patch("app.services.ops_service.get_runtime_status") + def test_ops_overview_marks_missing_remote_log_reason_code( + self, + mock_get_runtime_status, + mock_get_sync_summary, + mock_get_ops_job_summary, + mock_list_managed_nodes_with_agent_state, + mock_list_ops_jobs, + mock_get_recent_ops_playbook_runs, + mock_list_release_rollouts, + mock_get_release_summary, + mock_get_latest_release, + mock_get_release_launchpad, + mock_build_release_execution_mode_recommendation, + ) -> None: + mock_get_runtime_status.return_value = { + "readiness": {"status": "ready", "summary": "ok"}, + "detect": { + "log_sync": { + "enabled": True, + "mode": "key", + "line_count": 0, + "source_node_count": 0, + "source_nodes": [], + "last_at": "", + "last_line": "", + "preview_lines": [], + }, + "participation_summary": { + "participating_nodes": 1, + "dispatch_active_nodes": 1, + "recent_only_nodes": 0, + "non_participating_nodes": 0, + "standby_nodes": 0, + "load_syncing_nodes": 0, + "effective_online_nodes": 1, + "dedicated_worker_nodes": 1, + "controller_worker_nodes": 0, + "dispatch_active_node_codes": ["mainland-worker-01"], + "recent_only_node_codes": [], + "non_participating_node_codes": [], + "standby_node_codes": [], + "load_syncing_node_codes": [], + "summary": "有效执行节点 1 台；正在执行/领任务 1 台；近窗刚有吞吐 0 台；在线但未参与 0 台", + }, + "participating_nodes": [ + { + "node_code": "mainland-worker-01", + "region": "mainland", + "role": "worker", + "status": "busy", + "current_load": 2, + "participation_state": "claimed", + "participation_label": "已领待跑", + "participation_reason": "已领取 2 项任务，等待线程继续执行。", + "is_dispatch_active": True, + } + ], + "non_participating_nodes": [], + "active_job": {"job_code": "detect-001", "status": "running"}, + "phase_label": "运行中", + "phase_detail": "等待样本", + "recent_event": "", + "recent_warning": "", + }, + "cluster": {"summary": {}, "nodes": []}, + } + mock_get_sync_summary.return_value = {} + mock_get_ops_job_summary.return_value = {"total": 0, "queued": 0, "running": 0, "awaiting_approval": 0} + mock_list_managed_nodes_with_agent_state.return_value = {"nodes": [], "summary": {}} + mock_list_ops_jobs.return_value = [] + mock_get_recent_ops_playbook_runs.return_value = {"runs": []} + mock_list_release_rollouts.return_value = [] + mock_get_release_summary.return_value = {} + mock_get_latest_release.return_value = {} + mock_get_release_launchpad.return_value = {"launchpad_status": {"status": "ready"}} + mock_build_release_execution_mode_recommendation.return_value = { + "recommended_mode": "remote-agent", + "recommended_mode_label": "remote-agent", + "reason": "", + "options": [], + "recommended_gate": { + "status": "release_not_ready", + "summary": "", + "target_nodes": [], + "target_node_codes": [], + "blocking_reasons": [], + "warning_reasons": [], + "execution_mode": "remote-agent", + "execution_mode_label": "remote-agent", + "operational_readiness": {"rows": [], "summary": {}}, + }, + "gates": {}, + } + + overview = get_ops_overview() + missing_rows = overview["execution_scene"]["log_sync"]["missing_participating_node_summaries"] + + self.assertEqual(1, len(missing_rows)) + self.assertEqual("waiting_sample", missing_rows[0]["missing_reason_code"]) + self.assertEqual("dispatch_active", overview["execution_scene"]["participating_nodes"][0]["participation_bucket"]) + + @patch("app.services.ops_service.get_ops_activity_stream") + @patch("app.services.ops_service.build_release_execution_mode_recommendation") + @patch("app.services.ops_service.get_release_launchpad") + @patch("app.services.ops_service.get_latest_release") + @patch("app.services.ops_service.get_release_summary") + @patch("app.services.ops_service.list_release_rollouts") + @patch("app.services.ops_service.get_recent_ops_playbook_runs") + @patch("app.services.ops_service.list_ops_jobs") + @patch("app.services.ops_service.list_managed_nodes_with_agent_state") + @patch("app.services.ops_service.get_ops_job_summary") + @patch("app.services.ops_service.get_sync_summary") + @patch("app.services.ops_service.get_runtime_status") + def test_ops_overview_prioritizes_node_agent_dead_letter_queue( + self, + mock_get_runtime_status, + mock_get_sync_summary, + mock_get_ops_job_summary, + mock_list_managed_nodes_with_agent_state, + mock_list_ops_jobs, + mock_get_recent_ops_playbook_runs, + mock_list_release_rollouts, + mock_get_release_summary, + mock_get_latest_release, + mock_get_release_launchpad, + mock_build_release_execution_mode_recommendation, + mock_get_ops_activity_stream, + ) -> None: + mock_get_runtime_status.return_value = { + "readiness": {"status": "ready", "summary": "ok"}, + "detect": { + "log_sync": { + "enabled": False, + "mode": "key", + "line_count": 0, + "source_node_count": 0, + "source_nodes": [], + "last_at": "", + "last_line": "", + "preview_lines": [], + }, + "participation_summary": {}, + "participating_nodes": [], + "non_participating_nodes": [], + "active_job": {}, + "phase_label": "", + "phase_detail": "", + "recent_event": "", + "recent_warning": "", + }, + "cluster": { + "summary": { + "online_worker_nodes": 1, + "online_control_nodes": 1, + "dedicated_online_worker_nodes": 1, + "busy_nodes": [], + }, + "nodes": [ + { + "node_code": "mainland-worker-01", + "region": "mainland", + "role": "worker", + "status": "online", + "current_load": 0, + "hostname": "worker-01", + "last_heartbeat_at": "2026-04-18 05:10:00", + "is_effective_worker": True, + "detect_participating": False, + } + ], + }, + } + mock_get_sync_summary.return_value = {} + mock_get_ops_job_summary.return_value = {"total": 0, "queued": 0, "running": 0, "awaiting_approval": 0} + mock_list_managed_nodes_with_agent_state.return_value = { + "nodes": [ + { + "node_code": "mainland-worker-01", + "region": "mainland", + "role": "worker", + "is_managed": True, + "is_enabled": True, + "is_agent_online": True, + "cluster_status": "online", + "cluster_is_effective_worker": True, + "cluster_detect_participating": False, + "delivery_queue_state": "dead_letter", + "delivery_queue_label": "死信 2", + "delivery_queue_reason": "当前存在 2 条死信记录，建议优先查看节点日志或诊断编排。", + "delivery_queue_pending_count": 1, + "delivery_queue_dead_letter_count": 2, + } + ], + "summary": { + "total": 1, + "managed_total": 1, + "managed_enabled": 1, + "online": 1, + "stale": 0, + "pending_bootstrap": 0, + "runtime_only": 0, + "unmanaged": 0, + "token_issue": 0, + "queue_retrying_nodes": 0, + "queue_dead_letter_nodes": 1, + "queue_pending_records": 1, + "queue_dead_letter_records": 2, + "status_counts": {}, + }, + } + mock_list_ops_jobs.return_value = [] + mock_get_recent_ops_playbook_runs.return_value = {"runs": []} + mock_list_release_rollouts.return_value = [] + mock_get_release_summary.return_value = {} + mock_get_latest_release.return_value = {} + mock_get_release_launchpad.return_value = {"launchpad_status": {}} + mock_build_release_execution_mode_recommendation.return_value = { + "recommended_mode": "remote-agent", + "recommended_mode_label": "remote-agent", + "reason": "", + "options": [], + "recommended_gate": { + "status": "ready", + "summary": "", + "target_nodes": [], + "target_node_codes": [], + "blocking_reasons": [], + "warning_reasons": [], + "execution_mode": "remote-agent", + "execution_mode_label": "remote-agent", + "operational_readiness": {"rows": [], "summary": {}}, + "release": {"id": 0}, + }, + "gates": {}, + } + mock_get_ops_activity_stream.return_value = {"items": []} + + overview = get_ops_overview() + + self.assertEqual("node-agent-dead-letter", overview["recommendation"]["key"]) + self.assertEqual("先处理 Agent 死信", overview["recommendation"]["priority"]) + self.assertEqual("replay_delivery_queue", overview["recommendation"]["primary_action_code"]) + self.assertEqual( + "delivery.queue.replay", + overview["recommendation"]["primary_action_payload"]["template_key"], + ) + self.assertEqual(1, overview["managed_nodes"]["queue_dead_letter_nodes"]) + self.assertEqual(2, overview["managed_nodes"]["queue_dead_letter_records"]) + + @patch("app.services.ops_service.get_ops_activity_stream") + @patch("app.services.ops_service.build_release_execution_mode_recommendation") + @patch("app.services.ops_service.get_release_launchpad") + @patch("app.services.ops_service.get_latest_release") + @patch("app.services.ops_service.get_release_summary") + @patch("app.services.ops_service.list_release_rollouts") + @patch("app.services.ops_service.get_recent_ops_playbook_runs") + @patch("app.services.ops_service.list_ops_jobs") + @patch("app.services.ops_service.list_managed_nodes_with_agent_state") + @patch("app.services.ops_service.get_ops_job_summary") + @patch("app.services.ops_service.get_sync_summary") + @patch("app.services.ops_service.get_runtime_status") + def test_ops_overview_prioritizes_start_delivery_gap_from_activity_stream( + self, + mock_get_runtime_status, + mock_get_sync_summary, + mock_get_ops_job_summary, + mock_list_managed_nodes_with_agent_state, + mock_list_ops_jobs, + mock_get_recent_ops_playbook_runs, + mock_list_release_rollouts, + mock_get_release_summary, + mock_get_latest_release, + mock_get_release_launchpad, + mock_build_release_execution_mode_recommendation, + mock_get_ops_activity_stream, + ) -> None: + mock_get_runtime_status.return_value = { + "readiness": {"status": "ready", "summary": "ok"}, + "detect": { + "log_sync": { + "enabled": True, + "mode": "key", + "line_count": 12, + "source_node_count": 1, + "source_nodes": ["mainland-worker-01"], + "last_at": "2026-04-18 11:00:00", + "last_line": "ok", + "preview_lines": ["ok"], + }, + "participation_summary": {}, + "participating_nodes": [], + "non_participating_nodes": [], + "active_job": {}, + "phase_label": "", + "phase_detail": "", + "recent_event": "", + "recent_warning": "", + }, + "cluster": {"summary": {}, "nodes": []}, + } + mock_get_sync_summary.return_value = {} + mock_get_ops_job_summary.return_value = {"total": 0, "queued": 0, "running": 0, "awaiting_approval": 0} + mock_list_managed_nodes_with_agent_state.return_value = {"nodes": [], "summary": {}} + mock_list_ops_jobs.return_value = [] + mock_get_recent_ops_playbook_runs.return_value = {"runs": []} + mock_list_release_rollouts.return_value = [] + mock_get_release_summary.return_value = {} + mock_get_latest_release.return_value = {} + mock_get_release_launchpad.return_value = {"launchpad_status": {}} + mock_build_release_execution_mode_recommendation.return_value = { + "recommended_mode": "remote-agent", + "recommended_mode_label": "remote-agent", + "reason": "", + "options": [], + "recommended_gate": { + "status": "ready", + "summary": "", + "target_nodes": [], + "target_node_codes": [], + "blocking_reasons": [], + "warning_reasons": [], + "execution_mode": "remote-agent", + "execution_mode_label": "remote-agent", + "operational_readiness": {"rows": [], "summary": {}}, + "release": {"id": 0}, + }, + "gates": {}, + } + mock_get_ops_activity_stream.return_value = { + "items": [ + { + "kind": "ops_job", + "activity_key": "ops-job:81", + "status": "attention", + "job_status": "running", + "summary": "节点已接单执行，但开始回执未成功送达控制面。", + "job_code": "ops-agent-81", + "start_delivery_state": "failed_local", + "source_focus_ref": { + "kind": "ops_job_event", + "job_id": 81, + "job_code": "ops-agent-81", + "event_key": "job-event:81", + "target_node_code": "mainland-worker-01", + }, + "target_node_codes": ["mainland-worker-01"], + "ui_intent": {"kind": "job_events", "job_id": 81}, + } + ] + } + + overview = get_ops_overview() + + self.assertEqual("activity-start-delivery-gap", overview["recommendation"]["key"]) + self.assertEqual("先检查开始回执异常", overview["recommendation"]["priority"]) + self.assertEqual("focus_latest_job_events", overview["recommendation"]["primary_action_code"]) + self.assertEqual("ops_job_event", overview["recommendation"]["primary_action_payload"]["focus_ref"]["kind"]) + + @patch("app.services.ops_service.build_release_execution_mode_recommendation") + @patch("app.services.ops_service.get_release_launchpad") + @patch("app.services.ops_service.get_latest_release") + @patch("app.services.ops_service.get_release_summary") + @patch("app.services.ops_service.list_release_rollouts") + @patch("app.services.ops_service.get_recent_ops_playbook_runs") + @patch("app.services.ops_service.list_ops_jobs") + @patch("app.services.ops_service.list_managed_nodes_with_agent_state") + @patch("app.services.ops_service.get_ops_job_summary") + @patch("app.services.ops_service.get_sync_summary") + @patch("app.services.ops_service.get_runtime_status") + def test_ops_overview_uses_release_launchpad_driver_card_when_execution_plane_is_clean( + self, + mock_get_runtime_status, + mock_get_sync_summary, + mock_get_ops_job_summary, + mock_list_managed_nodes_with_agent_state, + mock_list_ops_jobs, + mock_get_recent_ops_playbook_runs, + mock_list_release_rollouts, + mock_get_release_summary, + mock_get_latest_release, + mock_get_release_launchpad, + mock_build_release_execution_mode_recommendation, + ) -> None: + mock_get_runtime_status.return_value = { + "readiness": {"status": "ready", "summary": "ok"}, + "detect": { + "log_sync": { + "enabled": True, + "mode": "key", + "line_count": 0, + "source_node_count": 0, + "source_nodes": [], + "last_at": "", + "last_line": "", + "preview_lines": [], + }, + "participation_summary": {}, + "participating_nodes": [], + "non_participating_nodes": [], + "active_job": {}, + "phase_label": "", + "phase_detail": "", + "recent_event": "", + "recent_warning": "", + }, + "cluster": { + "summary": { + "online_worker_nodes": 1, + "online_control_nodes": 1, + "dedicated_online_worker_nodes": 1, + "busy_nodes": [], + }, + "nodes": [ + { + "node_code": "mainland-worker-01", + "region": "mainland", + "role": "worker", + "status": "online", + "current_load": 0, + "hostname": "worker-01", + "last_heartbeat_at": "2026-04-18 10:00:00", + "is_effective_worker": True, + "detect_participating": False, + } + ], + }, + } + mock_get_sync_summary.return_value = {} + mock_get_ops_job_summary.return_value = {"total": 0, "queued": 0, "running": 0, "awaiting_approval": 0} + mock_list_managed_nodes_with_agent_state.return_value = { + "nodes": [ + { + "node_code": "mainland-worker-01", + "region": "mainland", + "role": "worker", + "is_managed": True, + "is_enabled": True, + "is_agent_online": True, + "cluster_status": "online", + "cluster_is_effective_worker": True, + "cluster_detect_participating": False, + } + ], + "summary": { + "total": 1, + "managed_total": 1, + "managed_enabled": 1, + "online": 1, + "stale": 0, + "pending_bootstrap": 0, + "runtime_only": 0, + "unmanaged": 0, + "token_issue": 0, + "status_counts": {}, + }, + } + mock_list_ops_jobs.return_value = [] + mock_get_recent_ops_playbook_runs.return_value = {"runs": []} + mock_list_release_rollouts.return_value = [] + mock_get_release_summary.return_value = {} + mock_get_latest_release.return_value = {} + mock_get_release_launchpad.return_value = { + "channel": "stable", + "control_plane_base_url": "http://control.example:8100", + "latest_package": { + "available": True, + "package_name": "domaincheck_release_demo.tar.gz", + }, + "latest_release": { + "id": 17, + "release_version": "domaincheck_release_demo", + "channel": "stable", + }, + "worker_rollout_preview": { + "message": "智能 Rollout 预检通过", + "smart_rollout": { + "available": True, + "summary": "已按 Worker 模式预选 1 台节点。", + "target_node_codes": ["mainland-worker-01"], + }, + }, + "control_rollout_preview": { + "message": "", + "smart_rollout": { + "available": False, + "target_node_codes": [], + }, + }, + "launchpad_status": { + "status": "ready", + "status_label": "可发 Worker 灰度", + "summary": "最新发布包、Release 记录与 Worker 智能 Rollout 预检均已收口。", + "recommended_action_code": "publish_latest_worker", + }, + } + mock_build_release_execution_mode_recommendation.return_value = { + "recommended_mode": "ssh", + "recommended_mode_label": "SSH", + "reason": "当前目标节点通过 SSH 路径的门禁更完整，建议先走 SSH 发布。", + "options": [], + "recommended_gate": { + "status": "ready", + "summary": "", + "target_nodes": [], + "target_node_codes": [], + "blocking_reasons": [], + "warning_reasons": [], + "execution_mode": "ssh", + "execution_mode_label": "SSH", + "operational_readiness": {"rows": [], "summary": {}}, + }, + "gates": {}, + } + + overview = get_ops_overview() + + self.assertEqual("driver_recommendations", overview["recommendation"]["source"]) + self.assertEqual("release-launchpad-ready", overview["recommendation"]["key"]) + self.assertEqual("当前可以发 Worker 灰度", overview["recommendation"]["priority"]) + self.assertEqual("publish_latest_worker", overview["recommendation"]["primary_action_code"]) + self.assertEqual(["mainland-worker-01"], overview["recommendation"]["primary_node_codes"]) + self.assertEqual("ready", overview["release_hub"]["launchpad"]["launchpad_status"]["status"]) + self.assertEqual("ssh", overview["release_hub"]["default_rollout_gate"]["execution_mode"]) + self.assertEqual("SSH", overview["release_hub"]["default_rollout_execution"]["recommended_mode_label"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/domain-api/tests/test_ops_service_runbook.py b/domain-api/tests/test_ops_service_runbook.py new file mode 100644 index 0000000..84c2f80 --- /dev/null +++ b/domain-api/tests/test_ops_service_runbook.py @@ -0,0 +1,283 @@ +import unittest +from unittest.mock import patch + +from app.services.ops_service import get_ops_runbook + + +class OpsServiceRunbookTests(unittest.TestCase): + @patch("app.services.ops_service.get_release_launchpad") + @patch("app.services.ops_service.get_latest_release") + @patch("app.services.ops_service.list_managed_nodes_with_agent_state") + @patch("app.services.ops_service.get_runtime_status") + def test_ops_runbook_includes_control_sequences( + self, + mock_get_runtime_status, + mock_list_managed_nodes_with_agent_state, + mock_get_latest_release, + mock_get_release_launchpad, + ) -> None: + mock_get_runtime_status.return_value = { + "readiness": { + "status": "ready", + "summary": "当前多机现场已进入标准作业状态", + }, + "worker": { + "running": True, + }, + "sync_agent": { + "running": True, + }, + "detect": { + "participation_summary": { + "summary": "worker 正在执行", + } + }, + } + mock_list_managed_nodes_with_agent_state.return_value = { + "nodes": [ + { + "node_code": "mainland-controller-01", + "role": "control", + "is_managed": True, + "is_enabled": True, + "is_agent_online": True, + "cluster_status": "online", + "cluster_is_effective_worker": True, + "is_current_participant": False, + }, + { + "node_code": "mainland-worker-01", + "role": "worker", + "is_managed": True, + "is_enabled": True, + "is_agent_online": True, + "cluster_status": "busy", + "cluster_is_effective_worker": True, + "is_current_participant": True, + }, + ] + } + mock_get_latest_release.side_effect = lambda channel="stable": ( + { + "id": 15, + "release_version": "2026.04.17", + "channel": "stable", + } + if channel == "stable" + else {} + ) + mock_get_release_launchpad.return_value = { + "channel": "stable", + "latest_release": { + "id": 15, + "release_version": "2026.04.17", + "channel": "stable", + }, + "launchpad_status": { + "status": "ready", + "status_label": "可发 Worker 灰度", + "summary": "最新发布包、Release 记录与 Worker 智能 Rollout 预检均已收口。", + "recommended_action_code": "publish_latest_worker", + "focus_ref": { + "kind": "release_hub", + "release_id": 15, + "section": "release_launchpad", + }, + }, + "worker_rollout_preview": { + "message": "智能 Rollout 预检通过", + "smart_rollout": { + "available": True, + "summary": "已按 Worker 模式预选 1 台节点。", + "target_node_codes": ["mainland-worker-01"], + }, + }, + "control_rollout_preview": { + "message": "", + "smart_rollout": { + "available": False, + "target_node_codes": [], + }, + }, + } + + runbook = get_ops_runbook() + + mock_list_managed_nodes_with_agent_state.assert_called_once_with( + participation_payload=mock_get_runtime_status.return_value["detect"] + ) + self.assertIn("GET /api/v1/ops/releases/launchpad", runbook["fast_checks"]) + self.assertEqual("ready", runbook["release_launchpad"]["launchpad_status"]["status"]) + sequences = {item["key"]: item for item in runbook["control_sequences"]} + self.assertIn("node_onboarding", sequences) + self.assertIn("scene_observe", sequences) + self.assertIn("standard_inspection", sequences) + self.assertIn("release_progression", sequences) + self.assertEqual("bootstrap_run", sequences["node_onboarding"]["primary_action_code"]) + self.assertEqual("run_acceptance", sequences["node_onboarding"]["secondary_action_code"]) + self.assertEqual("run_standard_inspection", sequences["standard_inspection"]["primary_action_code"]) + self.assertEqual("open_diagnostics", sequences["standard_inspection"]["secondary_action_code"]) + self.assertEqual(["mainland-worker-01"], sequences["scene_observe"]["target_node_codes"]) + self.assertEqual("publish_latest_worker", sequences["release_progression"]["primary_action_code"]) + self.assertEqual(15, sequences["release_progression"]["primary_action_payload"]["release_id"]) + self.assertEqual(["mainland-worker-01"], sequences["release_progression"]["target_node_codes"]) + self.assertTrue(sequences["release_progression"]["primary_resolution"]["ok"]) + self.assertEqual("publish_latest_worker", sequences["release_progression"]["primary_resolution"]["driver_action_code"]) + self.assertEqual(["mainland-worker-01"], sequences["release_progression"]["primary_resolution"]["driver_node_codes"]) + self.assertEqual("release_hub", sequences["release_progression"]["focus_ref"]["kind"]) + self.assertEqual("release_launchpad", sequences["release_progression"]["focus_ref"]["section"]) + self.assertEqual("worker", sequences["release_progression"]["primary_focus_ref"]["mode"]) + self.assertEqual("worker", sequences["release_progression"]["primary_action_payload"]["focus_ref"]["mode"]) + self.assertEqual("release_detail", sequences["release_progression"]["secondary_focus_ref"]["section"]) + self.assertTrue(sequences["scene_observe"]["secondary_resolution"]["ok"]) + self.assertEqual("run_scene_logs_full", sequences["scene_observe"]["secondary_resolution"]["driver_action_code"]) + + @patch("app.services.ops_service.get_release_launchpad") + @patch("app.services.ops_service.get_latest_release") + @patch("app.services.ops_service.list_managed_nodes_with_agent_state") + @patch("app.services.ops_service.get_runtime_status") + def test_ops_runbook_release_progression_can_fall_back_to_bootstrap_run( + self, + mock_get_runtime_status, + mock_list_managed_nodes_with_agent_state, + mock_get_latest_release, + mock_get_release_launchpad, + ) -> None: + mock_get_runtime_status.return_value = { + "readiness": { + "status": "attention", + "summary": "当前还需补执行面接管", + }, + "worker": {"running": True}, + "sync_agent": {"running": True}, + "detect": {"participation_summary": {"summary": ""}}, + } + mock_list_managed_nodes_with_agent_state.return_value = { + "nodes": [ + { + "node_code": "mainland-worker-02", + "role": "worker", + "is_managed": False, + "is_enabled": False, + "is_agent_online": False, + "cluster_status": "online", + "cluster_is_effective_worker": True, + "is_current_participant": False, + } + ] + } + mock_get_latest_release.return_value = {} + mock_get_release_launchpad.return_value = { + "channel": "stable", + "latest_release": {}, + "launchpad_status": { + "status": "attention", + "status_label": "待补执行面", + "summary": "当前节点仍处于接入阶段，建议先签发 onboarding.bootstrap。", + "recommended_action_code": "bootstrap_run", + "focus_ref": { + "kind": "release_hub", + "release_id": 19, + "section": "release_launchpad", + }, + }, + "worker_rollout_preview": { + "message": "当前没有满足角色条件的在线节点可用于智能 Rollout。", + "smart_rollout": { + "available": False, + "target_node_codes": [], + }, + "policy_preview": { + "release_gate": { + "operational_readiness": { + "rows": [ + { + "node_code": "mainland-worker-02", + "is_managed": False, + "is_enabled": False, + "is_agent_online": False, + "remote_agent_ready": False, + "inspection_status": "missing", + "recovery_action": "bootstrap_run", + "recovery_summary": "当前节点仍处于接入阶段，建议先签发 onboarding.bootstrap。", + "agent_reason": "节点尚未纳管，当前不能稳定下发 remote-agent 动作。", + } + ] + } + } + }, + }, + "control_rollout_preview": { + "message": "", + "smart_rollout": { + "available": False, + "target_node_codes": [], + }, + }, + } + + runbook = get_ops_runbook() + sequences = {item["key"]: item for item in runbook["control_sequences"]} + + self.assertEqual("bootstrap_run", sequences["release_progression"]["primary_action_code"]) + self.assertEqual(["mainland-worker-02"], sequences["release_progression"]["target_node_codes"]) + self.assertEqual("待补执行面", sequences["release_progression"]["status_label"]) + self.assertTrue(sequences["release_progression"]["primary_resolution"]["ok"]) + self.assertEqual("bootstrap_run", sequences["release_progression"]["primary_resolution"]["driver_action_code"]) + self.assertEqual("default_rollout_gate", sequences["release_progression"]["primary_focus_ref"]["section"]) + self.assertEqual("default_rollout_gate", sequences["release_progression"]["primary_action_payload"]["focus_ref"]["section"]) + + @patch("app.services.ops_service.get_release_launchpad") + @patch("app.services.ops_service.get_latest_release") + @patch("app.services.ops_service.list_managed_nodes_with_agent_state") + @patch("app.services.ops_service.get_runtime_status") + def test_ops_runbook_release_progression_falls_back_to_direct_worker_rollout_when_release_exists( + self, + mock_get_runtime_status, + mock_list_managed_nodes_with_agent_state, + mock_get_latest_release, + mock_get_release_launchpad, + ) -> None: + mock_get_runtime_status.return_value = { + "readiness": { + "status": "ready", + "summary": "当前多机现场已进入标准作业状态", + }, + "worker": {"running": True}, + "sync_agent": {"running": True}, + "detect": {"participation_summary": {"summary": ""}}, + } + mock_list_managed_nodes_with_agent_state.return_value = { + "nodes": [ + { + "node_code": "mainland-worker-01", + "role": "worker", + "is_managed": True, + "is_enabled": True, + "is_agent_online": True, + "cluster_status": "online", + "cluster_is_effective_worker": True, + "is_current_participant": False, + } + ] + } + mock_get_latest_release.return_value = { + "id": 18, + "release_version": "2026.04.18", + "channel": "stable", + } + mock_get_release_launchpad.return_value = {} + + runbook = get_ops_runbook() + sequences = {item["key"]: item for item in runbook["control_sequences"]} + + self.assertEqual("create_release_rollout_worker", sequences["release_progression"]["primary_action_code"]) + self.assertEqual("focus_release_hub", sequences["release_progression"]["secondary_action_code"]) + self.assertEqual(18, sequences["release_progression"]["primary_action_payload"]["release_id"]) + self.assertEqual(["mainland-worker-01"], sequences["release_progression"]["target_node_codes"]) + self.assertTrue(sequences["release_progression"]["primary_resolution"]["ok"]) + self.assertEqual("create_release_rollout_worker", sequences["release_progression"]["primary_resolution"]["driver_action_code"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/domain-api/tests/test_ops_service_runbook_sequence.py b/domain-api/tests/test_ops_service_runbook_sequence.py new file mode 100644 index 0000000..73f2fda --- /dev/null +++ b/domain-api/tests/test_ops_service_runbook_sequence.py @@ -0,0 +1,255 @@ +import unittest +from unittest.mock import patch + +from app.services.ops_service import execute_ops_runbook_sequence, resolve_ops_runbook_sequence + + +class OpsServiceRunbookSequenceTests(unittest.TestCase): + @patch("app.services.ops_service.get_ops_runbook") + def test_resolve_ops_runbook_sequence_uses_primary_action(self, mock_get_ops_runbook) -> None: + mock_get_ops_runbook.return_value = { + "control_sequences": [ + { + "key": "release_progression", + "title": "版本发布与放量", + "status": "ready", + "status_label": "可发 Worker 灰度", + "summary": "可以直接发布。", + "reason": "launchpad 已就绪", + "target_scope_label": "Release / Launchpad / 2026.04.18", + "step_titles": ["确认发布包", "发灰度"], + "target_node_codes": ["mainland-worker-01"], + "focus_ref": { + "kind": "release_hub", + "release_id": 12, + "section": "release_launchpad", + }, + "primary_label": "直接发 Worker 灰度", + "primary_action_code": "publish_latest_worker", + "primary_action_payload": { + "channel": "stable", + "release_id": 12, + }, + "primary_focus_ref": { + "kind": "release_hub", + "release_id": 12, + "section": "release_launchpad", + "mode": "worker", + }, + } + ] + } + + ok, message, data = resolve_ops_runbook_sequence( + "release_progression", + { + "requested_by": "test", + }, + ) + + self.assertTrue(ok, message) + self.assertEqual("标准作业路径解析成功", message) + self.assertEqual("publish_latest_worker", data["driver_action_code"]) + self.assertEqual("直接发 Worker 灰度", data["driver_action_label"]) + self.assertEqual(["mainland-worker-01"], data["driver_node_codes"]) + self.assertEqual("test/runbook/release_progression", data["driver_requested_by"]) + self.assertEqual("Release / Launchpad / 2026.04.18", data["sequence"]["target_scope_label"]) + self.assertEqual(["确认发布包", "发灰度"], data["sequence"]["step_titles"]) + self.assertEqual("release_launchpad", data["sequence"]["focus_ref"]["section"]) + self.assertEqual("worker", data["sequence"]["action_focus_ref"]["mode"]) + self.assertEqual("worker", data["driver_action_payload"]["focus_ref"]["mode"]) + + @patch("app.services.ops_service.get_ops_runbook") + def test_resolve_ops_runbook_sequence_uses_secondary_action_and_allows_payload_override( + self, + mock_get_ops_runbook, + ) -> None: + mock_get_ops_runbook.return_value = { + "control_sequences": [ + { + "key": "node_onboarding", + "title": "纳管新节点", + "status": "warning", + "status_label": "有待接管节点", + "summary": "先补接入。", + "reason": "新增节点需要先纳管。", + "target_node_codes": ["mainland-worker-01"], + "secondary_label": "跑接管验收", + "secondary_action_code": "run_acceptance", + "secondary_action_payload": { + "node_code": "mainland-worker-01", + }, + } + ] + } + + ok, message, data = resolve_ops_runbook_sequence( + "node_onboarding", + { + "secondary": True, + "requested_by": "test", + "node_codes": ["mainland-worker-02"], + "action_payload": { + "auto_approve": True, + }, + }, + ) + + self.assertTrue(ok, message) + self.assertEqual("标准作业路径解析成功", message) + self.assertEqual("run_acceptance", data["driver_action_code"]) + self.assertEqual("跑接管验收", data["driver_action_label"]) + self.assertEqual(["mainland-worker-02"], data["driver_node_codes"]) + self.assertTrue(data["sequence"]["secondary"]) + self.assertEqual( + { + "node_code": "mainland-worker-01", + "auto_approve": True, + }, + data["driver_action_payload"], + ) + + @patch("app.services.ops_service.execute_driver_action") + @patch("app.services.ops_service.get_ops_runbook") + def test_execute_ops_runbook_sequence_uses_primary_action( + self, + mock_get_ops_runbook, + mock_execute_driver_action, + ) -> None: + mock_get_ops_runbook.return_value = { + "control_sequences": [ + { + "key": "release_progression", + "title": "版本发布与放量", + "status": "ready", + "status_label": "可发 Worker 灰度", + "summary": "可以直接发布。", + "reason": "launchpad 已就绪", + "target_node_codes": ["mainland-worker-01"], + "primary_action_code": "publish_latest_worker", + "primary_action_payload": { + "channel": "stable", + "release_id": 12, + }, + "secondary_action_code": "focus_release_hub", + "secondary_action_payload": { + "release_id": 12, + }, + } + ] + } + mock_execute_driver_action.return_value = ( + True, + "Release rollout 已创建", + { + "handled": True, + "mode": "smart-release-rollout", + }, + ) + + ok, message, data = execute_ops_runbook_sequence( + "release_progression", + { + "requested_by": "test", + }, + ) + + self.assertTrue(ok, message) + self.assertEqual("Release rollout 已创建", message) + self.assertTrue(data["handled"]) + self.assertEqual("publish_latest_worker", data["driver_action_code"]) + self.assertEqual(["mainland-worker-01"], data["driver_node_codes"]) + self.assertEqual("release_progression", data["sequence_key"]) + self.assertEqual("版本发布与放量", data["sequence"]["title"]) + mock_execute_driver_action.assert_called_once_with( + { + "action_code": "publish_latest_worker", + "node_codes": ["mainland-worker-01"], + "action_payload": { + "channel": "stable", + "release_id": 12, + }, + "requested_by": "test/runbook/release_progression", + } + ) + + @patch("app.services.ops_service.execute_driver_action") + @patch("app.services.ops_service.get_ops_runbook") + def test_execute_ops_runbook_sequence_uses_secondary_action_and_allows_payload_override( + self, + mock_get_ops_runbook, + mock_execute_driver_action, + ) -> None: + mock_get_ops_runbook.return_value = { + "control_sequences": [ + { + "key": "node_onboarding", + "title": "纳管新节点", + "status": "warning", + "status_label": "有待接管节点", + "summary": "先补接入。", + "reason": "新增节点需要先纳管。", + "target_node_codes": ["mainland-worker-01"], + "primary_action_code": "bootstrap_run", + "primary_action_payload": { + "node_code": "mainland-worker-01", + }, + "secondary_action_code": "run_acceptance", + "secondary_action_payload": { + "node_code": "mainland-worker-01", + }, + } + ] + } + mock_execute_driver_action.return_value = ( + True, + "", + { + "handled": True, + "mode": "ui-intent", + }, + ) + + ok, message, data = execute_ops_runbook_sequence( + "node_onboarding", + { + "secondary": True, + "requested_by": "test", + "node_codes": ["mainland-worker-02"], + "action_payload": { + "auto_approve": True, + }, + }, + ) + + self.assertTrue(ok, message) + self.assertEqual("", message) + self.assertTrue(data["handled"]) + self.assertEqual("run_acceptance", data["driver_action_code"]) + self.assertEqual(["mainland-worker-02"], data["driver_node_codes"]) + self.assertTrue(data["sequence"]["secondary"]) + mock_execute_driver_action.assert_called_once_with( + { + "action_code": "run_acceptance", + "node_codes": ["mainland-worker-02"], + "action_payload": { + "node_code": "mainland-worker-01", + "auto_approve": True, + }, + "requested_by": "test/runbook/node_onboarding", + } + ) + + @patch("app.services.ops_service.get_ops_runbook") + def test_execute_ops_runbook_sequence_returns_error_when_sequence_missing(self, mock_get_ops_runbook) -> None: + mock_get_ops_runbook.return_value = {"control_sequences": []} + + ok, message, data = execute_ops_runbook_sequence("missing-sequence", {}) + + self.assertFalse(ok) + self.assertEqual("标准作业路径不存在", message) + self.assertEqual("missing-sequence", data["sequence_key"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/domain-api/tests/test_ops_service_stack_diagnosis.py b/domain-api/tests/test_ops_service_stack_diagnosis.py new file mode 100644 index 0000000..f513860 --- /dev/null +++ b/domain-api/tests/test_ops_service_stack_diagnosis.py @@ -0,0 +1,971 @@ +import unittest +from unittest.mock import patch + +from app.services.ops_service import get_ops_stack_diagnosis + + +class OpsServiceStackDiagnosisTests(unittest.TestCase): + @patch("app.services.ops_service.get_runtime_build_info") + @patch("app.services.ops_service.get_ops_activity_stream") + @patch("app.services.ops_service.get_recent_ops_playbook_runs") + @patch("app.services.ops_service.get_release_launchpad") + @patch("app.services.ops_service.list_managed_nodes_with_agent_state") + @patch("app.services.ops_service.get_ops_overview") + @patch("app.services.ops_service.get_ops_link_snapshot") + @patch("app.services.ops_service.get_ops_contract_registry") + def test_stack_diagnosis_prioritizes_api_restart_when_runtime_build_schema_is_stale( + self, + mock_get_ops_contract_registry, + mock_get_ops_link_snapshot, + mock_get_ops_overview, + mock_list_managed_nodes_with_agent_state, + mock_get_release_launchpad, + mock_get_recent_ops_playbook_runs, + mock_get_ops_activity_stream, + mock_get_runtime_build_info, + ) -> None: + mock_get_ops_contract_registry.return_value = { + "registry_version": "2026-04-18", + "schema_version": "v1", + "contracts": [{"key": "ops_stack_diagnosis_contract"}], + } + mock_get_ops_link_snapshot.return_value = { + "status": "ready", + "headline": "当前总检可用。", + "next_actions": [], + } + mock_get_ops_overview.return_value = { + "recommendation": { + "primary_action_code": "fix_managed_nodes", + "title": "先补节点接管", + "focus_ref": {"kind": "managed_node", "node_code": "mainland-worker-01"}, + }, + "execution_scene": {}, + "log_sync": {}, + "release_hub": {}, + } + mock_list_managed_nodes_with_agent_state.return_value = {"nodes": [], "summary": {}} + mock_get_release_launchpad.return_value = {"launchpad_status": {}, "latest_release": {}} + mock_get_recent_ops_playbook_runs.return_value = {"runs": []} + mock_get_ops_activity_stream.return_value = {"items": []} + mock_get_runtime_build_info.return_value = { + "source": "git", + "commit_sha": "246838ae4c07", + "repository_capabilities": { + "supports_install_command_block": True, + "supports_multi_layout_bootstrap": True, + }, + "route_surface": { + "mode": "registered", + "registered_paths_total": 24, + "surface_complete": True, + "missing_keys": [], + "expected_paths": { + "ops_contracts": "/api/v1/ops/contracts", + "ops_stack_diagnosis": "/api/v1/ops/stack-diagnosis", + }, + }, + } + + payload = get_ops_stack_diagnosis(base_url="http://127.0.0.1:8100") + + issue = next(item for item in payload["diagnosis"]["issues"] if item["code"] == "runtime_build_schema_stale") + self.assertEqual("api-restart", issue["action_code"]) + self.assertEqual("api-restart", payload["diagnosis"]["next_step"]["action_code"]) + self.assertEqual("runtime_recovery", payload["diagnosis"]["operator_decision"]["lane"]) + self.assertEqual("api-restart", payload["diagnosis"]["operator_decision"]["primary_command_key"]) + self.assertEqual( + "bash domain-api/deploy/multi-region/drive_ops_center.sh runtime-refresh-recover http://127.0.0.1:8100", + payload["diagnosis"]["next_actions"]["primary_command"], + ) + self.assertEqual( + "bash domain-api/deploy/multi-region/drive_ops_center.sh runtime-refresh-recover http://127.0.0.1:8100", + payload["diagnosis"]["recommended_commands"]["api-restart"], + ) + self.assertEqual( + "bash domain-api/deploy/multi-region/drive_ops_center.sh runtime-refresh-recover http://127.0.0.1:8100", + payload["diagnosis"]["recommended_commands"]["runtime-refresh-recover"], + ) + self.assertEqual( + "bash domain-api/deploy/multi-region/drive_ops_center.sh stack-diagnosis http://127.0.0.1:8100 summary", + payload["diagnosis"]["recommended_commands"]["stack-diagnosis"], + ) + self.assertEqual( + payload["diagnosis"]["recommended_commands"]["stack-diagnosis"], + payload["diagnosis"]["recommended_commands"]["stack-check"], + ) + self.assertTrue(payload["runtime_build_info"]["runtime_schema_stale"]) + self.assertFalse(payload["runtime_build_info"]["route_surface_declares_bootstrap_plan"]) + self.assertTrue( + any("运行中 API 还是旧 schema" in hint for hint in payload["diagnosis"]["operator_hints"]) + ) + + @patch("app.services.ops_service.get_runtime_build_info") + @patch("app.services.ops_service.get_ops_activity_stream") + @patch("app.services.ops_service.get_recent_ops_playbook_runs") + @patch("app.services.ops_service.get_release_launchpad") + @patch("app.services.ops_service.list_managed_nodes_with_agent_state") + @patch("app.services.ops_service.get_ops_overview") + @patch("app.services.ops_service.get_ops_link_snapshot") + @patch("app.services.ops_service.get_ops_contract_registry") + def test_stack_diagnosis_surfaces_runtime_route_surface_gap( + self, + mock_get_ops_contract_registry, + mock_get_ops_link_snapshot, + mock_get_ops_overview, + mock_list_managed_nodes_with_agent_state, + mock_get_release_launchpad, + mock_get_recent_ops_playbook_runs, + mock_get_ops_activity_stream, + mock_get_runtime_build_info, + ) -> None: + mock_get_ops_contract_registry.return_value = { + "registry_version": "2026-04-18", + "schema_version": "v1", + "contracts": [], + } + mock_get_ops_link_snapshot.return_value = { + "status": "ready", + "headline": "当前总检可用。", + "next_actions": [], + } + mock_get_ops_overview.return_value = { + "recommendation": {}, + "execution_scene": {}, + "log_sync": {}, + "release_hub": {}, + } + mock_list_managed_nodes_with_agent_state.return_value = {"nodes": [], "summary": {}} + mock_get_release_launchpad.return_value = {"launchpad_status": {}, "latest_release": {}} + mock_get_recent_ops_playbook_runs.return_value = {"runs": []} + mock_get_ops_activity_stream.return_value = {"items": []} + mock_get_runtime_build_info.return_value = { + "source": "git", + "commit_sha": "bd6bcb2", + "route_surface": { + "mode": "registered", + "registered_paths_total": 12, + "surface_complete": False, + "missing_keys": [ + "ops_node_onboarding_acceptance_execute", + "ops_node_onboarding_acceptance_preview", + ], + "missing_paths": [ + "/api/v1/ops/nodes/{node_code}/onboarding/acceptance/execute", + "/api/v1/ops/nodes/{node_code}/onboarding/acceptance/preview", + ], + }, + } + + payload = get_ops_stack_diagnosis(base_url="http://127.0.0.1:8100") + + issue = next( + item for item in payload["diagnosis"]["issues"] if item["code"] == "runtime_route_surface_incomplete" + ) + self.assertEqual("api-restart", issue["action_code"]) + self.assertIn("ops_node_onboarding_acceptance_execute", issue["detail"]) + self.assertIn( + "bash domain-api/deploy/multi-region/drive_ops_center.sh runtime-refresh-recover http://127.0.0.1:8100", + issue["commands"], + ) + self.assertFalse(payload["runtime_build_info"]["route_surface"]["surface_complete"]) + self.assertIn( + "ops_node_onboarding_acceptance_preview", + payload["runtime_build_info"]["route_surface_missing_keys"], + ) + self.assertTrue( + any("API 路由面还不完整" in hint for hint in payload["diagnosis"]["operator_hints"]) + ) + + @patch("app.services.ops_service.get_ops_activity_stream") + @patch("app.services.ops_service.get_recent_ops_playbook_runs") + @patch("app.services.ops_service.get_release_launchpad") + @patch("app.services.ops_service.list_managed_nodes_with_agent_state") + @patch("app.services.ops_service.get_ops_overview") + @patch("app.services.ops_service.get_ops_link_snapshot") + @patch("app.services.ops_service.get_ops_contract_registry") + def test_stack_diagnosis_emits_operator_decision_for_scene_log_gap( + self, + mock_get_ops_contract_registry, + mock_get_ops_link_snapshot, + mock_get_ops_overview, + mock_list_managed_nodes_with_agent_state, + mock_get_release_launchpad, + mock_get_recent_ops_playbook_runs, + mock_get_ops_activity_stream, + ) -> None: + mock_get_ops_contract_registry.return_value = { + "registry_version": "2026-04-18", + "schema_version": "v1", + "contracts": [], + } + mock_get_ops_link_snapshot.return_value = { + "status": "attention", + "headline": "参与节点样本尚未形成。", + "next_actions": [], + } + mock_get_ops_overview.return_value = { + "recommendation": {}, + "execution_scene": { + "participating_nodes": [{"node_code": "mainland-worker-01", "status": "busy"}], + "participation_summary": { + "dispatch_active_node_codes": ["mainland-worker-01"], + "recent_only_node_codes": [], + }, + "log_sync": { + "enabled": True, + "mode": "key", + "line_count": 0, + "source_node_count": 0, + "covered_participating_node_count": 0, + "participating_node_count": 1, + "missing_participating_nodes": ["mainland-worker-01"], + }, + }, + "log_sync": {}, + "release_hub": {}, + } + mock_list_managed_nodes_with_agent_state.return_value = {"nodes": [], "summary": {}} + mock_get_release_launchpad.return_value = {"launchpad_status": {}, "latest_release": {}} + mock_get_recent_ops_playbook_runs.return_value = {"runs": []} + mock_get_ops_activity_stream.return_value = {"items": []} + + payload = get_ops_stack_diagnosis(base_url="http://127.0.0.1:8100") + + decision = payload["diagnosis"]["operator_decision"] + self.assertEqual("observability", decision["lane"]) + self.assertEqual("scene-node-log", decision["primary_command_key"]) + self.assertEqual("mainland-worker-01", decision["next_focus"]["node_code"]) + self.assertIn("scene-node-log", payload["diagnosis"]["recommended_commands"]) + self.assertIn( + "scene-node-log http://127.0.0.1:8100 mainland-worker-01 120 key", + payload["diagnosis"]["next_actions"]["primary_command"], + ) + + @patch("app.services.ops_service.get_ops_activity_stream") + @patch("app.services.ops_service.get_recent_ops_playbook_runs") + @patch("app.services.ops_service.get_release_launchpad") + @patch("app.services.ops_service.get_managed_node_handover") + @patch("app.services.ops_service.list_managed_nodes_with_agent_state") + @patch("app.services.ops_service.get_ops_overview") + @patch("app.services.ops_service.get_ops_link_snapshot") + @patch("app.services.ops_service.get_ops_contract_registry") + def test_stack_diagnosis_emits_operator_decision_for_managed_node_gap( + self, + mock_get_ops_contract_registry, + mock_get_ops_link_snapshot, + mock_get_ops_overview, + mock_list_managed_nodes_with_agent_state, + mock_get_managed_node_handover, + mock_get_release_launchpad, + mock_get_recent_ops_playbook_runs, + mock_get_ops_activity_stream, + ) -> None: + mock_get_ops_contract_registry.return_value = { + "registry_version": "2026-04-18", + "schema_version": "v1", + "contracts": [], + } + mock_get_ops_link_snapshot.return_value = { + "status": "attention", + "headline": "存在接管缺口。", + "next_actions": [], + } + mock_get_ops_overview.return_value = { + "recommendation": {}, + "execution_scene": {}, + "log_sync": {}, + "release_hub": {}, + } + mock_list_managed_nodes_with_agent_state.return_value = { + "nodes": [ + { + "node_code": "mainland-worker-01", + "agent_state": "pending_bootstrap", + "remote_access_state": "ssh_ready", + "delivery_queue_state": "healthy", + "participation_state": "standby", + } + ], + "summary": { + "managed_enabled": 1, + "remote_access_ready": 0, + }, + } + mock_get_managed_node_handover.return_value = { + "node_code": "mainland-worker-01", + "summary": "等待执行 bootstrap。", + "stage": {"code": "pending_bootstrap", "label": "待执行接入", "next_step": {"code": "execute_bootstrap_plan"}}, + } + mock_get_release_launchpad.return_value = { + "launchpad_status": { + "recommended_action_code": "bootstrap_run", + "focus_ref": {"kind": "release_launchpad", "section": "release_launchpad"}, + "recommended_target_node_code": "mainland-worker-01", + }, + "worker_rollout_preview": { + "release_gate": { + "operational_readiness": { + "rows": [ + { + "node_code": "mainland-worker-01", + "recovery_action": "bootstrap_run", + "recovery_summary": "mainland-worker-01 还缺接入收口，先执行 bootstrap。", + "focus_ref": {"kind": "managed_node", "node_code": "mainland-worker-01"}, + } + ] + } + } + }, + "latest_release": {}, + } + mock_get_recent_ops_playbook_runs.return_value = {"runs": []} + mock_get_ops_activity_stream.return_value = {"items": []} + + payload = get_ops_stack_diagnosis(base_url="http://127.0.0.1:8100") + + decision = payload["diagnosis"]["operator_decision"] + self.assertEqual("node_handover", decision["lane"]) + self.assertEqual("bootstrap_run", decision["primary_command_key"]) + self.assertEqual("managed-node-bootstrap-plan", decision["secondary_command_key"]) + self.assertIn("bootstrap_run", payload["diagnosis"]["recommended_commands"]) + self.assertIn("managed-node-handover", payload["diagnosis"]["recommended_commands"]) + self.assertIn("managed-node-bootstrap-plan", payload["diagnosis"]["recommended_commands"]) + self.assertIn( + "node-bootstrap-run http://127.0.0.1:8100 mainland-worker-01 cli", + payload["diagnosis"]["next_actions"]["primary_command"], + ) + self.assertTrue( + any("当前第一动作已经收敛为 接入收口：mainland-worker-01" in hint for hint in payload["diagnosis"]["operator_hints"]) + ) + + @patch("app.services.ops_service.get_runtime_build_info") + @patch("app.services.ops_service.get_ops_activity_stream") + @patch("app.services.ops_service.get_recent_ops_playbook_runs") + @patch("app.services.ops_service.get_release_launchpad") + @patch("app.services.ops_service.get_managed_node_handover") + @patch("app.services.ops_service.list_managed_nodes_with_agent_state") + @patch("app.services.ops_service.get_ops_overview") + @patch("app.services.ops_service.get_ops_link_snapshot") + @patch("app.services.ops_service.get_ops_contract_registry") + def test_stack_diagnosis_promotes_first_handover_gap_when_launchpad_is_still_generic( + self, + mock_get_ops_contract_registry, + mock_get_ops_link_snapshot, + mock_get_ops_overview, + mock_list_managed_nodes_with_agent_state, + mock_get_managed_node_handover, + mock_get_release_launchpad, + mock_get_recent_ops_playbook_runs, + mock_get_ops_activity_stream, + mock_get_runtime_build_info, + ) -> None: + mock_get_ops_contract_registry.return_value = { + "registry_version": "2026-04-18", + "schema_version": "v1", + "contracts": [{"key": "ops_stack_diagnosis_contract"}], + } + mock_get_ops_link_snapshot.return_value = { + "status": "ready", + "headline": "当前总检可用。", + "next_actions": [], + } + mock_get_ops_overview.return_value = { + "recommendation": { + "primary_action_code": "handover_first_gap", + "title": "先补接管缺口", + "reason": "overview 只能先定位到接管缺口。", + "focus_ref": {"kind": "managed_node", "node_code": "mainland-worker-01"}, + }, + "execution_scene": {}, + "log_sync": {}, + "release_hub": {}, + } + mock_list_managed_nodes_with_agent_state.return_value = { + "nodes": [ + { + "node_code": "mainland-worker-01", + "agent_state": "pending_bootstrap", + "remote_access_state": "agent_pending", + "delivery_queue_state": "healthy", + "participation_state": "standby", + } + ], + "summary": { + "managed_enabled": 1, + "remote_access_ready": 0, + }, + } + mock_get_managed_node_handover.return_value = { + "node_code": "mainland-worker-01", + "summary": "当前节点仍处于接入阶段，建议先签发 onboarding.bootstrap 并等待 register / heartbeat 建立。", + "stage": { + "code": "pending_bootstrap", + "label": "待执行接入", + "next_step": { + "code": "execute_bootstrap_plan", + "label": "去目标机执行接入片段", + }, + }, + } + mock_get_release_launchpad.return_value = { + "launchpad_status": { + "recommended_action_code": "fix_managed_nodes", + "status_label": "待补执行面", + "focus_ref": {"kind": "release_launchpad", "section": "release_launchpad"}, + }, + "latest_release": {}, + } + mock_get_recent_ops_playbook_runs.return_value = {"runs": []} + mock_get_ops_activity_stream.return_value = {"items": []} + mock_get_runtime_build_info.return_value = { + "route_surface": {"surface_complete": True, "missing_keys": []}, + "repository_capabilities": {}, + } + + payload = get_ops_stack_diagnosis(base_url="http://127.0.0.1:8100") + + self.assertEqual("bootstrap_run", payload["diagnosis"]["next_step"]["action_code"]) + self.assertEqual( + "mainland-worker-01", + payload["diagnosis"]["launchpad_recommended_target_node_code"], + ) + self.assertEqual( + "签发接入工单", + payload["diagnosis"]["launchpad_recommended_recovery_label"], + ) + self.assertIn( + "onboarding.bootstrap", + payload["diagnosis"]["launchpad_recommended_recovery_summary"], + ) + self.assertEqual("bootstrap_run", payload["diagnosis"]["operator_decision"]["primary_command_key"]) + self.assertEqual( + "bash domain-api/deploy/multi-region/drive_ops_center.sh driver-resolve http://127.0.0.1:8100 bootstrap_run", + payload["diagnosis"]["next_actions"]["primary_command"], + ) + + @patch("app.services.ops_service.get_ops_activity_stream") + @patch("app.services.ops_service.get_recent_ops_playbook_runs") + @patch("app.services.ops_service.get_release_launchpad") + @patch("app.services.ops_service.list_managed_nodes_with_agent_state") + @patch("app.services.ops_service.get_ops_overview") + @patch("app.services.ops_service.get_ops_link_snapshot") + @patch("app.services.ops_service.get_ops_contract_registry") + def test_stack_diagnosis_emits_operator_decision_for_release_launchpad_attention( + self, + mock_get_ops_contract_registry, + mock_get_ops_link_snapshot, + mock_get_ops_overview, + mock_list_managed_nodes_with_agent_state, + mock_get_release_launchpad, + mock_get_recent_ops_playbook_runs, + mock_get_ops_activity_stream, + ) -> None: + mock_get_ops_contract_registry.return_value = { + "registry_version": "2026-04-18", + "schema_version": "v1", + "contracts": [], + } + mock_get_ops_link_snapshot.return_value = { + "status": "attention", + "headline": "发布门禁待处理。", + "next_actions": [], + } + mock_get_ops_overview.return_value = { + "recommendation": {}, + "execution_scene": {}, + "log_sync": {}, + "release_hub": { + "default_rollout_gate": { + "status": "blocked", + "summary_text": "还有节点未满足发布门禁。", + "focus_ref": {"kind": "release_hub", "section": "release_launchpad", "mode": "worker"}, + } + }, + } + mock_list_managed_nodes_with_agent_state.return_value = {"nodes": [], "summary": {}} + mock_get_release_launchpad.return_value = { + "launchpad_status": { + "status": "attention", + "status_label": "待处理", + "recommended_action_code": "review_smart_rollout_preview", + "focus_ref": {"kind": "release_hub", "section": "release_launchpad", "mode": "worker"}, + }, + "latest_release": {"release_version": "release-20260418"}, + } + mock_get_recent_ops_playbook_runs.return_value = {"runs": []} + mock_get_ops_activity_stream.return_value = {"items": []} + + payload = get_ops_stack_diagnosis(base_url="http://127.0.0.1:8100") + + decision = payload["diagnosis"]["operator_decision"] + self.assertEqual("release", decision["lane"]) + self.assertEqual("release-launchpad-worker", decision["primary_command_key"]) + self.assertEqual("review_smart_rollout_preview", decision["secondary_command_key"]) + self.assertIn("release-launchpad-worker", payload["diagnosis"]["recommended_commands"]) + self.assertIn( + "release-launchpad http://127.0.0.1:8100 worker", + payload["diagnosis"]["next_actions"]["primary_command"], + ) + + @patch("app.services.ops_service.get_ops_activity_stream") + @patch("app.services.ops_service.get_recent_ops_playbook_runs") + @patch("app.services.ops_service.get_release_launchpad") + @patch("app.services.ops_service.list_managed_nodes_with_agent_state") + @patch("app.services.ops_service.get_ops_overview") + @patch("app.services.ops_service.get_ops_link_snapshot") + @patch("app.services.ops_service.get_ops_contract_registry") + def test_stack_diagnosis_prefers_node_recovery_when_launchpad_points_to_bootstrap_gap( + self, + mock_get_ops_contract_registry, + mock_get_ops_link_snapshot, + mock_get_ops_overview, + mock_list_managed_nodes_with_agent_state, + mock_get_release_launchpad, + mock_get_recent_ops_playbook_runs, + mock_get_ops_activity_stream, + ) -> None: + mock_get_ops_contract_registry.return_value = { + "registry_version": "2026-04-18", + "schema_version": "v1", + "contracts": [], + } + mock_get_ops_link_snapshot.return_value = { + "status": "attention", + "headline": "发布门禁待处理。", + "next_actions": [], + } + mock_get_ops_overview.return_value = { + "recommendation": {}, + "execution_scene": {}, + "log_sync": {}, + "release_hub": { + "default_rollout_gate": { + "status": "attention", + "summary_text": "当前还有节点未完成接入。", + "focus_ref": {"kind": "release_hub", "section": "release_launchpad", "mode": "worker"}, + } + }, + } + mock_list_managed_nodes_with_agent_state.return_value = {"nodes": [], "summary": {}} + mock_get_release_launchpad.return_value = { + "launchpad_status": { + "status": "attention", + "status_label": "待补执行面", + "recommended_action_code": "bootstrap_run", + "recommended_target_node_code": "mainland-worker-02", + "recommended_recovery_label": "待执行接入", + "recommended_recovery_summary": "当前节点仍处于接入阶段，建议先签发 onboarding.bootstrap。", + "onboarding_bootstrap_pending_nodes": 1, + "onboarding_acceptance_ready_nodes": 0, + "focus_ref": {"kind": "release_hub", "section": "release_launchpad", "mode": "worker"}, + }, + "latest_release": {"release_version": "release-20260418"}, + } + mock_get_recent_ops_playbook_runs.return_value = {"runs": []} + mock_get_ops_activity_stream.return_value = {"items": []} + + payload = get_ops_stack_diagnosis(base_url="http://127.0.0.1:8100") + + decision = payload["diagnosis"]["operator_decision"] + self.assertEqual("node_handover", decision["lane"]) + self.assertEqual("bootstrap_run", decision["primary_command_key"]) + self.assertEqual("focus-release-hub", decision["secondary_command_key"]) + self.assertEqual("mainland-worker-02", payload["diagnosis"]["launchpad_recommended_target_node_code"]) + self.assertEqual("待执行接入", payload["diagnosis"]["launchpad_recommended_recovery_label"]) + self.assertEqual(1, payload["diagnosis"]["launchpad_onboarding_bootstrap_pending_nodes"]) + self.assertEqual(0, payload["diagnosis"]["launchpad_onboarding_acceptance_ready_nodes"]) + self.assertEqual( + "当前节点仍处于接入阶段，建议先签发 onboarding.bootstrap。", + decision["summary"], + ) + self.assertEqual("mainland-worker-02", decision["next_focus"]["node_code"]) + self.assertIn("node-bootstrap-run", payload["diagnosis"]["next_actions"]["primary_command"]) + + @patch("app.services.ops_service.get_ops_activity_stream") + @patch("app.services.ops_service.get_recent_ops_playbook_runs") + @patch("app.services.ops_service.get_release_launchpad") + @patch("app.services.ops_service.get_managed_node_handover") + @patch("app.services.ops_service.list_managed_nodes_with_agent_state") + @patch("app.services.ops_service.get_ops_overview") + @patch("app.services.ops_service.get_ops_link_snapshot") + @patch("app.services.ops_service.get_ops_contract_registry") + def test_stack_diagnosis_includes_first_handover_gap_details( + self, + mock_get_ops_contract_registry, + mock_get_ops_link_snapshot, + mock_get_ops_overview, + mock_list_managed_nodes_with_agent_state, + mock_get_managed_node_handover, + mock_get_release_launchpad, + mock_get_recent_ops_playbook_runs, + mock_get_ops_activity_stream, + ) -> None: + mock_get_ops_contract_registry.return_value = { + "registry_version": "2026-04-18", + "schema_version": "v1", + "contracts": [], + } + mock_get_ops_link_snapshot.return_value = { + "status": "attention", + "headline": "存在接管缺口。", + "next_actions": [], + } + mock_get_ops_overview.return_value = { + "recommendation": {}, + "execution_scene": {}, + "log_sync": {}, + "release_hub": {}, + } + mock_list_managed_nodes_with_agent_state.return_value = { + "nodes": [ + { + "node_code": "mainland-worker-01", + "agent_state": "pending_bootstrap", + "remote_access_state": "ssh_ready", + "delivery_queue_state": "healthy", + "participation_state": "standby", + } + ], + "summary": { + "managed_enabled": 1, + "remote_access_ready": 0, + "status_counts": {"pending_bootstrap": 1}, + "remote_access_state_counts": {"ssh_ready": 1}, + "delivery_queue_state_counts": {"healthy": 1}, + }, + } + mock_get_managed_node_handover.return_value = { + "node_code": "mainland-worker-01", + "summary": "已签发有效 Token，但目标节点尚未执行 bootstrap 片段。", + "stage": { + "code": "pending_bootstrap", + "label": "待执行接入", + "next_step": { + "code": "execute_bootstrap_plan", + "label": "去目标机执行接入片段", + }, + }, + } + mock_get_release_launchpad.return_value = { + "launchpad_status": { + "recommended_action_code": "fix_managed_nodes", + "focus_ref": {"kind": "release_launchpad"}, + }, + "latest_release": {}, + } + mock_get_recent_ops_playbook_runs.return_value = {"runs": []} + mock_get_ops_activity_stream.return_value = {"items": []} + + payload = get_ops_stack_diagnosis(base_url="http://127.0.0.1:8100") + + self.assertEqual( + "mainland-worker-01", + payload["managed_nodes"]["first_handover_gap_node_code"], + ) + self.assertEqual( + "pending_bootstrap", + payload["managed_nodes"]["first_handover_gap"]["stage"]["code"], + ) + issue = next( + item for item in payload["diagnosis"]["issues"] if item["code"] == "managed_nodes_agent_pending" + ) + self.assertIn("mainland-worker-01", issue["detail"]) + self.assertTrue( + any("node-handover http://127.0.0.1:8100 mainland-worker-01" in command for command in issue["commands"]) + ) + self.assertIn( + "bash domain-api/deploy/multi-region/drive_ops_center.sh node-recover http://127.0.0.1:8100 mainland-worker-01", + payload["diagnosis"]["recommended_commands"]["managed-node-recover"], + ) + self.assertTrue( + any("node-bootstrap-plan http://127.0.0.1:8100 mainland-worker-01" in command for command in issue["commands"]) + ) + mock_get_managed_node_handover.assert_called_once() + + @patch("app.services.ops_service.get_ops_activity_stream") + @patch("app.services.ops_service.get_recent_ops_playbook_runs") + @patch("app.services.ops_service.get_release_launchpad") + @patch("app.services.ops_service.list_managed_nodes_with_agent_state") + @patch("app.services.ops_service.get_ops_overview") + @patch("app.services.ops_service.get_ops_link_snapshot") + @patch("app.services.ops_service.get_ops_contract_registry") + def test_stack_diagnosis_keeps_focus_refs_for_problem_runs_and_top_items( + self, + mock_get_ops_contract_registry, + mock_get_ops_link_snapshot, + mock_get_ops_overview, + mock_list_managed_nodes_with_agent_state, + mock_get_release_launchpad, + mock_get_recent_ops_playbook_runs, + mock_get_ops_activity_stream, + ) -> None: + mock_get_ops_contract_registry.return_value = { + "registry_version": "2026-04-18", + "schema_version": "v1", + "contracts": [], + } + mock_get_ops_link_snapshot.return_value = { + "status": "ready", + "headline": "当前总检可用。", + "next_actions": [], + } + mock_get_ops_overview.return_value = { + "recommendation": {}, + "execution_scene": {}, + "log_sync": {}, + "release_hub": {}, + } + mock_list_managed_nodes_with_agent_state.return_value = { + "nodes": [], + "summary": {}, + } + mock_get_release_launchpad.return_value = { + "launchpad_status": {}, + "latest_release": {}, + } + mock_get_recent_ops_playbook_runs.return_value = { + "runs": [ + { + "run_code": "pbr-worker-logs-001", + "status": "attention", + "status_label": "待关注", + "focus_step_key": "worker_logs", + "focus_summary": "Worker 日志步骤仍未收口。", + "focus_ref": { + "kind": "playbook_run", + "run_code": "pbr-worker-logs-001", + "focus_step_key": "worker_logs", + }, + } + ] + } + mock_get_ops_activity_stream.return_value = { + "items": [ + { + "kind": "ops_job", + "status": "running", + "summary_text": "远端 Agent 已回传关键日志样本。", + "occurred_at": "2026-04-18 09:22:00", + "focus_ref": { + "kind": "ops_job", + "job_id": 71, + "job_code": "ops-agent-71", + }, + "source_focus_ref": { + "kind": "ops_job_event", + "job_id": 71, + "event_key": "job-event:71", + }, + "ui_intent": { + "kind": "job_events", + "job_id": 71, + }, + } + ] + } + + payload = get_ops_stack_diagnosis(base_url="http://127.0.0.1:8100") + + self.assertEqual( + "ops_stack_diagnosis_contract", + payload["diagnosis"]["contract_navigation"]["primary_contract_key"], + ) + self.assertIn("ops_driver_contract", payload["diagnosis"]["contract_navigation"]["contract_keys"]) + self.assertEqual("playbook_run", payload["playbook_runs"]["problem_runs"][0]["focus_ref"]["kind"]) + self.assertEqual("worker_logs", payload["playbook_runs"]["problem_runs"][0]["focus_ref"]["focus_step_key"]) + top_item = payload["activity_stream"]["top_items"][0] + self.assertEqual("ops_job", top_item["focus_ref"]["kind"]) + self.assertEqual("ops_job_event", top_item["source_focus_ref"]["kind"]) + self.assertEqual("job_events", top_item["ui_intent_kind"]) + self.assertEqual("2026-04-18 09:22:00", top_item["occurred_at"]) + + @patch("app.services.ops_service.get_ops_activity_stream") + @patch("app.services.ops_service.get_recent_ops_playbook_runs") + @patch("app.services.ops_service.get_release_launchpad") + @patch("app.services.ops_service.list_managed_nodes_with_agent_state") + @patch("app.services.ops_service.get_ops_overview") + @patch("app.services.ops_service.get_ops_link_snapshot") + @patch("app.services.ops_service.get_ops_contract_registry") + def test_stack_diagnosis_adds_node_scene_log_commands_for_log_sample_gap( + self, + mock_get_ops_contract_registry, + mock_get_ops_link_snapshot, + mock_get_ops_overview, + mock_list_managed_nodes_with_agent_state, + mock_get_release_launchpad, + mock_get_recent_ops_playbook_runs, + mock_get_ops_activity_stream, + ) -> None: + mock_get_ops_contract_registry.return_value = { + "registry_version": "2026-04-18", + "schema_version": "v1", + "contracts": [], + } + mock_get_ops_link_snapshot.return_value = { + "status": "attention", + "headline": "参与节点样本尚未形成。", + "next_actions": [ + { + "label": "看参与节点日志", + "action_code": "open_worker_logs_participating", + "node_codes": ["mainland-worker-01"], + "payload": {}, + } + ], + } + mock_get_ops_overview.return_value = { + "recommendation": {}, + "execution_scene": { + "participating_nodes": [ + { + "node_code": "mainland-worker-01", + "status": "busy", + } + ], + "participation_summary": { + "dispatch_active_node_codes": ["mainland-worker-01"], + "recent_only_node_codes": [], + }, + "log_sync": { + "enabled": True, + "mode": "key", + "line_count": 0, + "source_node_count": 0, + "covered_participating_node_count": 0, + "participating_node_count": 1, + "missing_participating_nodes": ["mainland-worker-01"], + }, + }, + "log_sync": {}, + "release_hub": {}, + } + mock_list_managed_nodes_with_agent_state.return_value = { + "nodes": [], + "summary": {}, + } + mock_get_release_launchpad.return_value = { + "launchpad_status": {}, + "latest_release": {}, + } + mock_get_recent_ops_playbook_runs.return_value = {"runs": []} + mock_get_ops_activity_stream.return_value = {"items": []} + + payload = get_ops_stack_diagnosis(base_url="http://127.0.0.1:8100") + + issue = next( + item for item in payload["diagnosis"]["issues"] if item["code"] == "remote_log_sync_waiting_sample" + ) + self.assertEqual("open_worker_logs_participating", issue["action_code"]) + self.assertEqual("node_scene_log", issue["focus_ref"]["kind"]) + self.assertEqual("mainland-worker-01", issue["focus_ref"]["node_code"]) + self.assertTrue( + any( + "scene-node-log http://127.0.0.1:8100 mainland-worker-01 120 key" in command + for command in issue["commands"] + ) + ) + self.assertEqual("node_scene_log", payload["diagnosis"]["next_step"]["focus_ref"]["kind"]) + self.assertTrue( + any( + "scene-node-log http://127.0.0.1:8100 mainland-worker-01 120 key" in command + for command in payload["diagnosis"]["quick_commands"] + ) + ) + self.assertTrue( + any( + "driver-resolve http://127.0.0.1:8100 open_worker_logs_participating" in command + for command in issue["commands"] + ) + ) + + @patch("app.services.ops_service.get_ops_activity_stream") + @patch("app.services.ops_service.get_recent_ops_playbook_runs") + @patch("app.services.ops_service.get_release_launchpad") + @patch("app.services.ops_service.list_managed_nodes_with_agent_state") + @patch("app.services.ops_service.get_ops_overview") + @patch("app.services.ops_service.get_ops_link_snapshot") + @patch("app.services.ops_service.get_ops_contract_registry") + def test_stack_diagnosis_surfaces_start_delivery_gap_from_activity_stream( + self, + mock_get_ops_contract_registry, + mock_get_ops_link_snapshot, + mock_get_ops_overview, + mock_list_managed_nodes_with_agent_state, + mock_get_release_launchpad, + mock_get_recent_ops_playbook_runs, + mock_get_ops_activity_stream, + ) -> None: + mock_get_ops_contract_registry.return_value = { + "registry_version": "2026-04-18", + "schema_version": "v1", + "contracts": [], + } + mock_get_ops_link_snapshot.return_value = { + "status": "attention", + "headline": "最近有任务回执异常。", + "next_actions": [], + } + mock_get_ops_overview.return_value = { + "recommendation": {}, + "execution_scene": {}, + "log_sync": {}, + "release_hub": {}, + } + mock_list_managed_nodes_with_agent_state.return_value = { + "nodes": [], + "summary": {}, + } + mock_get_release_launchpad.return_value = { + "launchpad_status": {}, + "latest_release": {}, + } + mock_get_recent_ops_playbook_runs.return_value = {"runs": []} + mock_get_ops_activity_stream.return_value = { + "items": [ + { + "kind": "ops_job", + "status": "attention", + "job_status": "running", + "summary_text": "节点已接单执行，但开始回执未成功送达控制面。", + "occurred_at": "2026-04-18 10:22:00", + "job_code": "ops-agent-81", + "start_delivery_state": "failed_local", + "start_delivery_error": "temporary offline", + "focus_ref": { + "kind": "ops_job", + "job_id": 81, + "job_code": "ops-agent-81", + "target_node_code": "mainland-worker-01", + }, + "source_focus_ref": { + "kind": "ops_job_event", + "job_id": 81, + "job_code": "ops-agent-81", + "event_key": "job-event:81", + "target_node_code": "mainland-worker-01", + }, + "ui_intent": { + "kind": "job_events", + "job_id": 81, + }, + } + ] + } + + payload = get_ops_stack_diagnosis(base_url="http://127.0.0.1:8100") + + issue = next( + item for item in payload["diagnosis"]["issues"] if item["code"] == "ops_job_start_delivery_failed_local" + ) + self.assertEqual("focus_latest_job_events", issue["action_code"]) + self.assertEqual("ops_job_event", issue["focus_ref"]["kind"]) + self.assertIn("temporary offline", issue["detail"]) + decision = payload["diagnosis"]["operator_decision"] + self.assertEqual("ops_jobs", decision["lane"]) + self.assertEqual("focus_latest_job_events", decision["primary_command_key"]) + self.assertIn("driver-resolve http://127.0.0.1:8100 focus_latest_job_events", payload["diagnosis"]["next_actions"]["primary_command"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/domain-api/tests/test_ops_template_service.py b/domain-api/tests/test_ops_template_service.py new file mode 100644 index 0000000..a8dd3df --- /dev/null +++ b/domain-api/tests/test_ops_template_service.py @@ -0,0 +1,177 @@ +import unittest + +from app.services.ops_template_service import build_ops_template_payload, get_ops_action_template + + +class OpsTemplateServiceTests(unittest.TestCase): + def test_node_bootstrap_defaults_and_clamps_number(self) -> None: + ok, message, data = build_ops_template_payload( + "node.bootstrap", + { + "expires_in_hours": 9999, + }, + ) + + self.assertTrue(ok, message) + self.assertEqual("ok", message) + self.assertEqual("", data["control_plane_base_url"]) + self.assertEqual("/opt/domaincheck", data["root_dir"]) + self.assertEqual(720, data["expires_in_hours"]) + self.assertEqual("", data["node_region"]) + self.assertEqual("", data["node_role"]) + + def test_deploy_release_custom_normalizes_lists_and_booleans(self) -> None: + ok, message, data = build_ops_template_payload( + "deploy.release.custom", + { + "release_version": "2026.04.18-rc1", + "artifact_url": "https://example.com/release.tar.gz", + "restart_services": "domaincheck-api\ndomaincheck-worker", + "health_check_urls": "", + "health_check_services": "", + "rollback_on_failure": "false", + "switch_current": "true", + }, + ) + + self.assertTrue(ok, message) + self.assertEqual("ok", message) + self.assertEqual(["domaincheck-api", "domaincheck-worker"], data["restart_services"]) + self.assertEqual([], data["health_check_urls"]) + self.assertEqual([], data["health_check_services"]) + self.assertFalse(data["rollback_on_failure"]) + self.assertTrue(data["switch_current"]) + self.assertEqual("/opt/domaincheck", data["install_root"]) + + def test_deploy_release_control_applies_defaults(self) -> None: + ok, message, data = build_ops_template_payload( + "deploy.release.control", + { + "release_version": "2026.04.18-rc1", + "artifact_url": "https://example.com/release.tar.gz", + }, + ) + + self.assertTrue(ok, message) + self.assertEqual( + ["domaincheck-api", "domaincheck-worker", "domaincheck-sync-agent"], + data["restart_services"], + ) + self.assertEqual(["http://127.0.0.1:8100/health"], data["health_check_urls"]) + self.assertEqual( + ["domaincheck-api", "domaincheck-worker", "domaincheck-sync-agent"], + data["health_check_services"], + ) + self.assertTrue(data["rollback_on_failure"]) + self.assertTrue(data["switch_current"]) + + def test_deploy_release_missing_required_field_fails(self) -> None: + ok, message, data = build_ops_template_payload( + "deploy.release.custom", + { + "release_version": "2026.04.18-rc1", + "artifact_url": "", + }, + ) + + self.assertFalse(ok) + self.assertEqual("artifact_url 不能为空", message) + self.assertEqual({}, data) + + def test_logs_collect_lines_are_clamped(self) -> None: + ok, message, data = build_ops_template_payload( + "logs.collect", + { + "service_name": "domaincheck-worker", + "lines": 9999, + }, + ) + + self.assertTrue(ok, message) + self.assertEqual(500, data["lines"]) + + def test_delivery_queue_templates_normalize_and_require_expected_fields(self) -> None: + replay_ok, replay_message, replay_data = build_ops_template_payload( + "delivery.queue.replay", + { + "record_id": "evt-001", + "limit": 999, + "flush_after_replay": "false", + "reason": "确认状态已修复", + }, + ) + discard_ok, discard_message, discard_data = build_ops_template_payload( + "delivery.queue.discard", + { + "record_id": "evt-002", + "limit": 0, + "reason": "确认这条死信不再需要补发", + }, + ) + flush_ok, flush_message, flush_data = build_ops_template_payload( + "delivery.queue.flush", + { + "limit": 999, + }, + ) + discard_fail_ok, discard_fail_message, discard_fail_data = build_ops_template_payload( + "delivery.queue.discard", + { + "record_id": "evt-003", + "reason": "", + }, + ) + + self.assertTrue(replay_ok, replay_message) + self.assertEqual(200, replay_data["limit"]) + self.assertFalse(replay_data["flush_after_replay"]) + self.assertEqual("evt-001", replay_data["record_id"]) + + self.assertTrue(discard_ok, discard_message) + self.assertEqual(1, discard_data["limit"]) + self.assertEqual("确认这条死信不再需要补发", discard_data["reason"]) + + self.assertTrue(flush_ok, flush_message) + self.assertEqual(200, flush_data["limit"]) + + self.assertFalse(discard_fail_ok) + self.assertEqual("reason 不能为空", discard_fail_message) + self.assertEqual({}, discard_fail_data) + + def test_diagnostics_and_service_control_templates_expose_ssh(self) -> None: + health_template = get_ops_action_template("health.snapshot") + logs_template = get_ops_action_template("logs.collect") + diagnostics_template = get_ops_action_template("diagnostics.collect") + service_status_template = get_ops_action_template("service.status") + restart_template = get_ops_action_template("service.restart") + restart_api_template = get_ops_action_template("runtime.restart_api") + stop_worker_template = get_ops_action_template("runtime.stop_worker") + deploy_control_template = get_ops_action_template("deploy.release.control") + deploy_worker_template = get_ops_action_template("deploy.release.worker") + deploy_custom_template = get_ops_action_template("deploy.release.custom") + queue_flush_template = get_ops_action_template("delivery.queue.flush") + queue_replay_template = get_ops_action_template("delivery.queue.replay") + queue_discard_template = get_ops_action_template("delivery.queue.discard") + + self.assertIn("ssh", health_template["execution_modes"]) + self.assertEqual("远端 Agent", health_template["default_execution_mode_label"]) + self.assertEqual("SSH", next(item["label"] for item in health_template["execution_mode_options"] if item["value"] == "ssh")) + self.assertIn("ssh", logs_template["execution_modes"]) + self.assertIn("ssh", diagnostics_template["execution_modes"]) + self.assertIn("ssh", service_status_template["execution_modes"]) + self.assertIn("ssh", restart_template["execution_modes"]) + self.assertIn("ssh", restart_api_template["execution_modes"]) + self.assertIn("ssh", stop_worker_template["execution_modes"]) + self.assertEqual("控制面", get_ops_action_template("node.bootstrap")["default_execution_mode_label"]) + self.assertIn("ssh", deploy_control_template["execution_modes"]) + self.assertIn("ssh", deploy_worker_template["execution_modes"]) + self.assertIn("ssh", deploy_custom_template["execution_modes"]) + self.assertEqual(["remote-agent"], queue_flush_template["execution_modes"]) + self.assertEqual(["remote-agent"], queue_replay_template["execution_modes"]) + self.assertEqual(["remote-agent"], queue_discard_template["execution_modes"]) + self.assertEqual("远端 Agent", queue_flush_template["default_execution_mode_label"]) + + + +if __name__ == "__main__": + unittest.main() diff --git a/domain-api/tests/test_prepare_final_release.py b/domain-api/tests/test_prepare_final_release.py new file mode 100644 index 0000000..e57b60f --- /dev/null +++ b/domain-api/tests/test_prepare_final_release.py @@ -0,0 +1,117 @@ +from __future__ import annotations + +import json +import os +import subprocess +import tempfile +import unittest +from pathlib import Path + + +class PrepareFinalReleaseTests(unittest.TestCase): + def test_prepare_final_release_blocks_when_smoke_is_false_even_if_verify_passes(self) -> None: + repo_root = Path(__file__).resolve().parents[2] + source_script = repo_root / "prepare_final_release.sh" + + with tempfile.TemporaryDirectory() as temp_dir: + root = Path(temp_dir) + release_root = root / "release" + release_root.mkdir(parents=True, exist_ok=True) + + (root / "prepare_final_release.sh").write_text(source_script.read_text(encoding="utf-8"), encoding="utf-8") + (root / "package_domain_release.sh").write_text( + "#!/usr/bin/env bash\n" + "set -euo pipefail\n" + "mkdir -p \"$PWD/release\"\n" + "cat > \"$PWD/release/latest_release.json\" <<'EOF'\n" + "{\n" + " \"package_name\": \"domaincheck_release_test\",\n" + " \"archive_path\": \"/tmp/domaincheck_release_test.tar.gz\",\n" + " \"sha256_path\": \"/tmp/domaincheck_release_test.sha256.txt\",\n" + " \"sha256\": \"abc\",\n" + " \"smoke_test_ok\": false,\n" + " \"smoke_test_message\": \"skipped by env\"\n" + "}\n" + "EOF\n", + encoding="utf-8", + ) + (root / "verify_domain_release.sh").write_text( + "#!/usr/bin/env bash\n" + "set -euo pipefail\n" + "printf '%s\\n' '{\"ok\": true, \"archive\": \"/tmp/domaincheck_release_test.tar.gz\"}'\n", + encoding="utf-8", + ) + os.chmod(root / "prepare_final_release.sh", 0o755) + os.chmod(root / "package_domain_release.sh", 0o755) + os.chmod(root / "verify_domain_release.sh", 0o755) + + result = subprocess.run( + ["bash", str(root / "prepare_final_release.sh")], + cwd=root, + check=True, + capture_output=True, + text=True, + ) + + payload = json.loads((release_root / "final_release_report.json").read_text(encoding="utf-8")) + self.assertFalse(payload["ok"]) + self.assertEqual("blocked", payload["release_gate"]["decision"]) + self.assertTrue(payload["release_gate"]["verify_ok"]) + self.assertFalse(payload["release_gate"]["smoke_test_ok"]) + self.assertIn("smoke_test_failed_or_skipped", payload["release_gate"]["blocked_reasons"]) + self.assertIn('"ok": false', result.stdout) + + def test_prepare_final_release_ready_when_verify_and_smoke_both_pass(self) -> None: + repo_root = Path(__file__).resolve().parents[2] + source_script = repo_root / "prepare_final_release.sh" + + with tempfile.TemporaryDirectory() as temp_dir: + root = Path(temp_dir) + release_root = root / "release" + release_root.mkdir(parents=True, exist_ok=True) + + (root / "prepare_final_release.sh").write_text(source_script.read_text(encoding="utf-8"), encoding="utf-8") + (root / "package_domain_release.sh").write_text( + "#!/usr/bin/env bash\n" + "set -euo pipefail\n" + "mkdir -p \"$PWD/release\"\n" + "cat > \"$PWD/release/latest_release.json\" <<'EOF'\n" + "{\n" + " \"package_name\": \"domaincheck_release_test\",\n" + " \"archive_path\": \"/tmp/domaincheck_release_test.tar.gz\",\n" + " \"sha256_path\": \"/tmp/domaincheck_release_test.sha256.txt\",\n" + " \"sha256\": \"abc\",\n" + " \"smoke_test_ok\": true,\n" + " \"smoke_test_message\": \"all checks passed (1/1)\"\n" + "}\n" + "EOF\n", + encoding="utf-8", + ) + (root / "verify_domain_release.sh").write_text( + "#!/usr/bin/env bash\n" + "set -euo pipefail\n" + "printf '%s\\n' '{\"ok\": true, \"archive\": \"/tmp/domaincheck_release_test.tar.gz\"}'\n", + encoding="utf-8", + ) + os.chmod(root / "prepare_final_release.sh", 0o755) + os.chmod(root / "package_domain_release.sh", 0o755) + os.chmod(root / "verify_domain_release.sh", 0o755) + + subprocess.run( + ["bash", str(root / "prepare_final_release.sh")], + cwd=root, + check=True, + capture_output=True, + text=True, + ) + + payload = json.loads((release_root / "final_release_report.json").read_text(encoding="utf-8")) + self.assertTrue(payload["ok"]) + self.assertEqual("ready", payload["release_gate"]["decision"]) + self.assertEqual([], payload["release_gate"]["blocked_reasons"]) + self.assertTrue(payload["release_gate"]["verify_ok"]) + self.assertTrue(payload["release_gate"]["smoke_test_ok"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/domain-api/tests/test_release_metadata_parity.py b/domain-api/tests/test_release_metadata_parity.py new file mode 100644 index 0000000..6797e18 --- /dev/null +++ b/domain-api/tests/test_release_metadata_parity.py @@ -0,0 +1,29 @@ +from __future__ import annotations + +import unittest +from pathlib import Path + + +class ReleaseMetadataParityTests(unittest.TestCase): + def test_powershell_release_package_emits_archive_path_alias(self) -> None: + repo_root = Path(__file__).resolve().parents[2] + script_text = (repo_root / "package_domain_release.ps1").read_text(encoding="utf-8") + + self.assertIn("archive_path = $zipPath", script_text) + self.assertIn('"archive_path=$zipPath"', script_text) + + def test_powershell_release_flow_prefers_archive_path_with_zip_fallback(self) -> None: + repo_root = Path(__file__).resolve().parents[2] + prepare_text = (repo_root / "prepare_final_release.ps1").read_text(encoding="utf-8") + show_text = (repo_root / "show_latest_release.ps1").read_text(encoding="utf-8") + + self.assertIn("$archivePath = if ($latest.archive_path) { $latest.archive_path } else { $latest.zip_path }", prepare_text) + self.assertIn('& (Join-Path $root "package_domain_release.ps1")', prepare_text) + self.assertIn('& (Join-Path $root "verify_domain_release.ps1")', prepare_text) + self.assertNotIn('powershell -ExecutionPolicy Bypass -File (Join-Path $root "package_domain_release.ps1")', prepare_text) + self.assertIn("$reportArchivePath = if ($report.latest_release.archive_path) { [string]$report.latest_release.archive_path } else { [string]$report.latest_release.zip_path }", show_text) + self.assertIn("$latestArchivePath = if ($latest.archive_path) { [string]$latest.archive_path } else { [string]$latest.zip_path }", show_text) + + +if __name__ == "__main__": + unittest.main() diff --git a/domain-api/tests/test_release_script_parity.py b/domain-api/tests/test_release_script_parity.py new file mode 100644 index 0000000..6ec7ee8 --- /dev/null +++ b/domain-api/tests/test_release_script_parity.py @@ -0,0 +1,19 @@ +from __future__ import annotations + +import unittest +from pathlib import Path + + +class ReleaseScriptParityTests(unittest.TestCase): + def test_shell_release_verifier_checks_cross_platform_delivery_scripts(self) -> None: + repo_root = Path(__file__).resolve().parents[2] + script_path = repo_root / "verify_domain_release.sh" + script_text = script_path.read_text(encoding="utf-8") + + self.assertIn('"scripts/package_domain_release.ps1"', script_text) + self.assertIn('"scripts/verify_domain_release.ps1"', script_text) + self.assertIn('"scripts/smoke_test_stack.ps1"', script_text) + + +if __name__ == "__main__": + unittest.main() diff --git a/domain-api/tests/test_show_latest_release.py b/domain-api/tests/test_show_latest_release.py new file mode 100644 index 0000000..93cbd49 --- /dev/null +++ b/domain-api/tests/test_show_latest_release.py @@ -0,0 +1,162 @@ +from __future__ import annotations + +import json +import os +import subprocess +import tempfile +import unittest +from pathlib import Path + + +class ShowLatestReleaseTests(unittest.TestCase): + def test_show_latest_release_marks_stale_final_report_as_not_ok(self) -> None: + repo_root = Path(__file__).resolve().parents[2] + script_path = repo_root / "show_latest_release.sh" + + with tempfile.TemporaryDirectory() as temp_dir: + release_root = Path(temp_dir) + latest_payload = { + "package_name": "domaincheck_release_new", + "archive_path": str(release_root / "domaincheck_release_new.tar.gz"), + "sha256": "new-sha256", + } + final_report = { + "ok": True, + "latest_release": { + "package_name": "domaincheck_release_old", + "archive_path": str(release_root / "domaincheck_release_old.tar.gz"), + "sha256": "old-sha256", + }, + } + (release_root / "latest_release.json").write_text( + json.dumps(latest_payload, ensure_ascii=False, indent=2), + encoding="utf-8", + ) + (release_root / "final_release_report.json").write_text( + json.dumps(final_report, ensure_ascii=False, indent=2), + encoding="utf-8", + ) + + env = dict(os.environ) + env["DOMAINCHECK_RELEASE_ROOT"] = str(release_root) + result = subprocess.run( + ["bash", str(script_path)], + cwd=repo_root, + check=True, + capture_output=True, + text=True, + env=env, + ) + + payload = json.loads(result.stdout) + self.assertFalse(payload["final_release_ok"]) + self.assertTrue(payload["final_release_report_ok"]) + self.assertFalse(payload["final_release_report_matches_latest"]) + self.assertEqual("domaincheck_release_old", payload["final_release_report_package_name"]) + self.assertIn("package_name", payload["final_release_report_stale_reason"]) + self.assertEqual("", payload["final_release_gate_decision"]) + self.assertEqual([], payload["final_release_gate_blocked_reasons"]) + + def test_show_latest_release_keeps_final_report_ok_when_it_matches_latest(self) -> None: + repo_root = Path(__file__).resolve().parents[2] + script_path = repo_root / "show_latest_release.sh" + + with tempfile.TemporaryDirectory() as temp_dir: + release_root = Path(temp_dir) + latest_payload = { + "package_name": "domaincheck_release_new", + "archive_path": str(release_root / "domaincheck_release_new.tar.gz"), + "sha256": "new-sha256", + } + final_report = { + "ok": True, + "latest_release": latest_payload, + "release_gate": { + "decision": "ready", + "blocked_reasons": [], + "verify_ok": True, + "smoke_test_ok": True, + }, + } + (release_root / "latest_release.json").write_text( + json.dumps(latest_payload, ensure_ascii=False, indent=2), + encoding="utf-8", + ) + (release_root / "final_release_report.json").write_text( + json.dumps(final_report, ensure_ascii=False, indent=2), + encoding="utf-8", + ) + + env = dict(os.environ) + env["DOMAINCHECK_RELEASE_ROOT"] = str(release_root) + result = subprocess.run( + ["bash", str(script_path)], + cwd=repo_root, + check=True, + capture_output=True, + text=True, + env=env, + ) + + payload = json.loads(result.stdout) + self.assertTrue(payload["final_release_ok"]) + self.assertTrue(payload["final_release_report_ok"]) + self.assertTrue(payload["final_release_report_matches_latest"]) + self.assertEqual("domaincheck_release_new", payload["final_release_report_package_name"]) + self.assertEqual("", payload["final_release_report_stale_reason"]) + self.assertEqual("ready", payload["final_release_gate_decision"]) + self.assertEqual([], payload["final_release_gate_blocked_reasons"]) + self.assertTrue(payload["final_release_gate_verify_ok"]) + self.assertTrue(payload["final_release_gate_smoke_test_ok"]) + + def test_show_latest_release_accepts_zip_path_fallback(self) -> None: + repo_root = Path(__file__).resolve().parents[2] + script_path = repo_root / "show_latest_release.sh" + + with tempfile.TemporaryDirectory() as temp_dir: + release_root = Path(temp_dir) + latest_payload = { + "package_name": "domaincheck_release_win", + "zip_path": str(release_root / "domaincheck_release_win.zip"), + "sha256": "zip-sha256", + } + final_report = { + "ok": True, + "latest_release": latest_payload, + "release_gate": { + "decision": "ready", + "blocked_reasons": [], + "verify_ok": True, + "smoke_test_ok": True, + }, + } + (release_root / "latest_release.json").write_text( + json.dumps(latest_payload, ensure_ascii=False, indent=2), + encoding="utf-8", + ) + (release_root / "final_release_report.json").write_text( + json.dumps(final_report, ensure_ascii=False, indent=2), + encoding="utf-8", + ) + + env = dict(os.environ) + env["DOMAINCHECK_RELEASE_ROOT"] = str(release_root) + result = subprocess.run( + ["bash", str(script_path)], + cwd=repo_root, + check=True, + capture_output=True, + text=True, + env=env, + ) + + payload = json.loads(result.stdout) + self.assertTrue(payload["final_release_ok"]) + self.assertTrue(payload["final_release_report_ok"]) + self.assertTrue(payload["final_release_report_matches_latest"]) + self.assertEqual("domaincheck_release_win", payload["final_release_report_package_name"]) + self.assertEqual("ready", payload["final_release_gate_decision"]) + + +if __name__ == "__main__": + unittest.main() diff --git a/domain-web/src/api/modules.ts b/domain-web/src/api/modules.ts index f6bfb20..880bb38 100644 --- a/domain-web/src/api/modules.ts +++ b/domain-web/src/api/modules.ts @@ -21,6 +21,107 @@ export const runtimeApi = { action: (action: string) => http.post(`/runtime/actions/${action}`) }; +export const opsApi = { + overview: () => http.get("/ops/overview"), + goLiveSummary: (params?: Record) => http.get("/ops/go-live-summary", { params }), + doctorDecision: (params?: Record) => http.get("/ops/doctor-decision", { params }), + goLiveSignoff: (params?: Record) => http.get("/ops/go-live-signoff", { params }), + goLiveBundle: (params?: Record) => http.get("/ops/go-live-bundle", { params }), + goLiveReview: (params?: Record) => http.get("/ops/go-live-review", { params }), + stackDiagnosis: (params?: Record) => http.get("/ops/stack-diagnosis", { params }), + linkSnapshot: () => http.get("/ops/link-snapshot"), + inspectionOverview: (params?: Record) => http.get("/ops/inspection-overview", { params }), + activityStream: (params?: Record) => http.get("/ops/activity-stream", { params }), + driverFeed: () => http.get("/ops/driver-feed"), + codexBrief: () => http.get("/ops/codex-brief"), + capabilities: () => http.get("/ops/capabilities"), + contracts: () => http.get("/ops/contracts"), + contractDetail: (contractKey: string) => http.get(`/ops/contracts/${encodeURIComponent(contractKey)}`), + actionTemplates: () => http.get("/ops/action-templates"), + playbooks: () => http.get("/ops/playbooks"), + playbookRuns: (params?: Record) => http.get("/ops/playbook-runs", { params }), + playbookRunDetail: (runCode: string) => http.get(`/ops/playbook-runs/${encodeURIComponent(runCode)}`), + playbookRunEvents: (runCode: string, params?: Record) => + http.get(`/ops/playbook-runs/${encodeURIComponent(runCode)}/events`, { params }), + rerunPlaybookRun: (runCode: string, payload?: Record) => + http.post(`/ops/playbook-runs/${encodeURIComponent(runCode)}/rerun`, payload || {}), + cancelPlaybookRun: (runCode: string, payload?: Record) => + http.post(`/ops/playbook-runs/${encodeURIComponent(runCode)}/cancel`, payload || {}), + previewPlaybook: (payload: Record) => http.post("/ops/playbooks/preview", payload), + executePlaybook: (payload: Record) => http.post("/ops/playbooks/execute", payload), + blueprint: () => http.get("/ops/blueprint"), + runbook: () => http.get("/ops/runbook"), + resolveRunbookSequence: (sequenceKey: string, payload?: Record) => + http.post(`/ops/runbook/sequences/${encodeURIComponent(sequenceKey)}/resolve`, payload || {}), + executeRunbookSequence: (sequenceKey: string, payload?: Record) => + http.post(`/ops/runbook/sequences/${encodeURIComponent(sequenceKey)}/execute`, payload || {}), + nodes: () => http.get("/ops/nodes"), + nodeHandover: (nodeCode: string, params?: Record) => + http.get(`/ops/nodes/${encodeURIComponent(nodeCode)}/handover`, { params }), + nodeOnboarding: (nodeCode: string, params?: Record) => + http.get(`/ops/nodes/${encodeURIComponent(nodeCode)}/onboarding`, { params }), + previewNodeOnboardingRecovery: (nodeCode: string, payload?: Record) => + http.post(`/ops/nodes/${encodeURIComponent(nodeCode)}/onboarding/recovery/preview`, payload || {}), + executeNodeOnboardingRecovery: (nodeCode: string, payload?: Record) => + http.post(`/ops/nodes/${encodeURIComponent(nodeCode)}/onboarding/recovery/execute`, payload || {}), + nodeSceneLog: (nodeCode: string, params?: Record) => + http.get(`/ops/nodes/${encodeURIComponent(nodeCode)}/scene-log`, { params }), + buildNodeHandoverBootstrapPlan: (nodeCode: string, payload?: Record) => + http.post(`/ops/nodes/${encodeURIComponent(nodeCode)}/handover/bootstrap-plan`, payload || {}), + deliveryQueue: (nodeCode: string) => http.get(`/ops/nodes/${encodeURIComponent(nodeCode)}/delivery-queue`), + deliveryQueueRecords: (nodeCode: string, params?: Record) => + http.get(`/ops/nodes/${encodeURIComponent(nodeCode)}/delivery-queue/records`, { params }), + flushDeliveryQueue: (nodeCode: string, payload?: Record) => + http.post(`/ops/nodes/${encodeURIComponent(nodeCode)}/delivery-queue/flush`, payload || {}), + replayDeliveryQueue: (nodeCode: string, payload?: Record) => + http.post(`/ops/nodes/${encodeURIComponent(nodeCode)}/delivery-queue/replay`, payload || {}), + replayDeliveryQueueRecord: (nodeCode: string, recordId: string, payload?: Record) => + http.post( + `/ops/nodes/${encodeURIComponent(nodeCode)}/delivery-queue/records/${encodeURIComponent(recordId)}/replay`, + payload || {} + ), + discardDeliveryQueueRecord: (nodeCode: string, recordId: string, payload?: Record) => + http.post( + `/ops/nodes/${encodeURIComponent(nodeCode)}/delivery-queue/records/${encodeURIComponent(recordId)}/discard`, + payload || {} + ), + upsertNode: (payload: Record) => http.post("/ops/nodes", payload), + syncNodesFromCluster: (dryRun = false) => http.post(`/ops/nodes/sync-from-cluster?dry_run=${dryRun ? "true" : "false"}`), + previewPolicy: (payload: Record) => http.post("/ops/policy/preview", payload), + jobs: (params?: Record) => http.get("/ops/jobs", { params }), + jobDetail: (jobId: number) => http.get(`/ops/jobs/${jobId}`), + jobEvents: (jobId: number, params?: Record) => http.get(`/ops/jobs/${jobId}/events`, { params }), + previewDriverAction: (payload: Record) => http.post("/ops/driver-actions/preview", payload), + resolveDriverAction: (payload: Record) => http.post("/ops/driver-actions/resolve", payload), + executeDriverAction: (payload: Record) => http.post("/ops/driver-actions/execute", payload), + executeResolvedDriverAction: (payload: Record) => http.post("/ops/driver-actions/execute-resolved", payload), + resolveCodexAction: (payload: Record) => http.post("/ops/codex-actions/resolve", payload), + executeCodexAction: (payload: Record) => http.post("/ops/codex-actions/execute", payload), + createJobBatch: (payload: Record) => http.post("/ops/jobs/batch", payload), + approveJob: (jobId: number, payload?: Record) => http.post(`/ops/jobs/${jobId}/approve`, payload || {}), + cancelJob: (jobId: number, payload?: Record) => http.post(`/ops/jobs/${jobId}/cancel`, payload || {}), + dispatchJob: (jobId: number) => http.post(`/ops/jobs/${jobId}/dispatch`), + issueAgentToken: (payload: Record) => http.post("/ops/agent/tokens", payload), + buildAgentBootstrapPlan: (payload: Record) => http.post("/ops/agent/bootstrap-plan", payload), + releases: (params?: Record) => http.get("/ops/releases", { params }), + latestRelease: (channel = "stable") => http.get("/ops/releases/latest", { params: { channel } }), + latestReleasePackageMetadata: () => http.get("/ops/releases/package-metadata/latest"), + releaseLaunchpad: (channel = "stable") => http.get("/ops/releases/launchpad", { params: { channel } }), + releaseDetail: (releaseId: number) => http.get(`/ops/releases/${releaseId}`), + createRelease: (payload: Record) => http.post("/ops/releases", payload), + createReleaseFromLatestPackage: (payload?: Record) => http.post("/ops/releases/from-package/latest", payload || {}), + createReleaseFromLatestPackageAndSmartRollout: (payload?: Record) => + http.post("/ops/releases/from-package/latest/smart-rollout", payload || {}), + createSmartRolloutForRelease: (releaseId: number, payload?: Record) => + http.post(`/ops/releases/${releaseId}/smart-rollout`, payload || {}), + activateRelease: (releaseId: number) => http.post(`/ops/releases/${releaseId}/activate`), + releaseRollouts: (releaseId: number, params?: Record) => http.get(`/ops/releases/${releaseId}/rollouts`, { params }), + createRollout: (releaseId: number, payload: Record) => http.post(`/ops/releases/${releaseId}/rollouts`, payload), + rolloutDetail: (rolloutId: number) => http.get(`/ops/rollouts/${rolloutId}`), + rolloutJobs: (rolloutId: number, params?: Record) => http.get(`/ops/rollouts/${rolloutId}/jobs`, { params }), + advanceRollout: (rolloutId: number, payload?: Record) => http.post(`/ops/rollouts/${rolloutId}/advance`, payload || {}) +}; + export const settingsApi = { getSettings: () => http.get("/settings"), updateSettings: (payload: Record) => http.put("/settings", payload), diff --git a/domain-web/src/layouts/MainLayout.vue b/domain-web/src/layouts/MainLayout.vue index 4957b7e..38155c1 100644 --- a/domain-web/src/layouts/MainLayout.vue +++ b/domain-web/src/layouts/MainLayout.vue @@ -67,6 +67,7 @@ const menuItems = [ { path: "/dashboard", label: "概览" }, { path: "/runtime", label: "运行中心" }, { path: "/runtime-debug", label: "运行调试" }, + { path: "/ops-center", label: "运维中枢" }, { path: "/settings", label: "系统设置" }, { path: "/imports", label: "域名导入" }, { path: "/sensitive-words", label: "敏感词配置" }, diff --git a/domain-web/src/router/index.ts b/domain-web/src/router/index.ts index e9bf19f..5d44774 100644 --- a/domain-web/src/router/index.ts +++ b/domain-web/src/router/index.ts @@ -16,6 +16,7 @@ const routes: RouteRecordRaw[] = [ { path: "dashboard", name: "dashboard", component: () => import("@/views/dashboard/DashboardView.vue"), meta: { title: "概览" } }, { path: "runtime", name: "runtime", component: () => import("@/views/runtime/RuntimeView.vue"), meta: { title: "运行中心" } }, { path: "runtime-debug", name: "runtime-debug", component: () => import("@/views/runtime/RuntimeDebugView.vue"), meta: { title: "运行调试" } }, + { path: "ops-center", name: "ops-center", component: () => import("@/views/ops/OpsCenterView.vue"), meta: { title: "运维中枢" } }, { path: "settings", name: "settings", component: () => import("@/views/settings/SettingsView.vue"), meta: { title: "系统设置" } }, { path: "imports", name: "imports", component: () => import("@/views/imports/ImportsView.vue"), meta: { title: "域名导入" } }, { path: "sensitive-words", name: "sensitive-words", component: () => import("@/views/sensitive-words/SensitiveWordsView.vue"), meta: { title: "敏感词配置" } }, diff --git a/domain-web/src/views/detect/DetectView.vue b/domain-web/src/views/detect/DetectView.vue index ba6047f..e821ee6 100644 --- a/domain-web/src/views/detect/DetectView.vue +++ b/domain-web/src/views/detect/DetectView.vue @@ -100,6 +100,13 @@ :title="`当前检测状态：${currentPhaseLabel}；本机 Worker ${status.worker_online ? '在线' : '离线'}。`" style="margin-top: 12px" /> + @@ -328,6 +335,11 @@ const status = ref({ dependency_alerts: [] as any[], log_lines: [] as string[], remote_log_lines: [] as string[], + remote_log_line_count: 0, + remote_log_last_at: "", + remote_log_last_line: "", + remote_log_nodes: [] as string[], + remote_log_node_count: 0, runs: [] as any[], worker_log_sync_enabled: false, worker_log_sync_mode: "key", @@ -487,6 +499,29 @@ const proxyDiagnosisText = computed(() => { return status.value.proxy_runtime_label || "当前可用代理 / 配置代理池链接数"; }); +const remoteLogMirrorText = computed(() => { + if (!status.value.worker_log_sync_enabled) { + return "当前已关闭远端日志回传；日志控制台只展示本机 Worker 输出，不会拼接其他节点的执行过程。"; + } + const modeText = status.value.worker_log_sync_mode === "full" ? "全量" : "关键"; + const nodeCount = Number(status.value.remote_log_node_count || 0); + const lineCount = Number(status.value.remote_log_line_count || 0); + const nodes = Array.isArray(status.value.remote_log_nodes) ? status.value.remote_log_nodes : []; + const lastAt = String(status.value.remote_log_last_at || "").trim(); + const parts = [ + `当前已开启${modeText}回传，日志控制台会自动拼接远端节点回传日志。`, + `来源节点 ${nodeCount} 台`, + `样本 ${lineCount} 条`, + ]; + if (nodes.length) { + parts.push(`节点：${nodes.join("、")}`); + } + if (lastAt) { + parts.push(`最近回传：${lastAt}`); + } + return parts.join("；"); +}); + const queueAlertType = computed<"success" | "warning" | "info" | "error">(() => { if (!queueSummary.value.has_active_job) return "info"; if ((queueSummary.value.queue?.overdue_leases || 0) > 0) return "error"; diff --git a/domain-web/src/views/ops/OpsCenterView.vue b/domain-web/src/views/ops/OpsCenterView.vue new file mode 100644 index 0000000..e50a35d --- /dev/null +++ b/domain-web/src/views/ops/OpsCenterView.vue @@ -0,0 +1,18516 @@ + + + + + diff --git a/domain-web/src/views/runtime/RuntimeView.vue b/domain-web/src/views/runtime/RuntimeView.vue index aeb2353..f133f7a 100644 --- a/domain-web/src/views/runtime/RuntimeView.vue +++ b/domain-web/src/views/runtime/RuntimeView.vue @@ -73,10 +73,10 @@
有效执行节点
{{ runtime.cluster?.summary?.online_worker_nodes || 0 }}
- 可承担检测任务的在线节点 + 真正执行/领任务 {{ runtime.detect?.participation_summary?.dispatch_active_nodes || 0 }} + / 在线未参与 {{ runtime.detect?.participation_summary?.non_participating_nodes || 0 }} / 独立 Worker {{ runtime.cluster?.summary?.dedicated_online_worker_nodes || 0 }} - / 控制面 {{ runtime.cluster?.summary?.online_control_nodes || 0 }} - / 忙碌 {{ runtime.cluster?.summary?.status_counts?.busy || 0 }} + / 控制面兼跑 {{ runtime.detect?.participation_summary?.controller_worker_nodes || 0 }}
@@ -126,6 +126,12 @@ 有效执行节点 {{ runtime.cluster?.summary?.online_worker_nodes || 0 }} + + 真正参与 {{ runtime.detect?.participation_summary?.dispatch_active_nodes || 0 }} + + + 在线未参与 {{ runtime.detect?.participation_summary?.non_participating_nodes || 0 }} + 预检 {{ preflight.ok ? "通过" : "异常" }} @@ -203,6 +209,69 @@ 立即同步 前往检测控制
+
+
+

远端日志回传

+ + {{ workerLogSyncEnabled ? (workerLogSyncMode === "full" ? "全量回传" : "关键回传") : "已关闭" }} + +
+
+ + 关闭回传 + + + 关键回传 + + + 全量回传 + +
+ +
+
+ 样本 + {{ runtime.detect?.log_sync?.line_count || 0 }} 条 +
+
+ 来源节点 + {{ runtime.detect?.log_sync?.source_node_count || 0 }} 台 +
+
+ 节点列表 + {{ workerLogSyncNodesText }} +
+
+ 最近回传 + {{ workerLogSyncLastAtText }} +
+
+ 最近一条 + {{ workerLogSyncLastLineText }} +
+
+

当前参与检测节点

- 直接展示哪些节点正在领任务、执行任务或产生近窗吞吐，controller 兼跑检测也会计入。 + 这里只展示真正有任务动作或近窗吞吐的节点，controller 兼跑检测也会明确计入。
+ + + @@ -379,6 +455,58 @@ +
+
+

在线但未参与节点

+ 这些节点当前在线，但没有执行/领任务；若只是上报了负载，会单独标成“负载待确认”。 +
+ + + + + + + + + + + + + + + + + + + + +
+

动作留痕

@@ -450,12 +578,12 @@ @@ -679,6 +807,8 @@ const loading = reactive({ }); const actionLoading = ref(""); +const workerLogSyncLoading = ref(false); +const pendingWorkerLogMode = ref<"off" | "key" | "full">("off"); const autoRefresh = ref(true); const lastUpdatedAt = ref(""); const lastAction = ref({ @@ -869,12 +999,16 @@ const clusterIssueCount = computed(() => { const clusterAlertText = computed(() => { const summary = runtime.cluster?.summary || {}; + const participationSummary = runtime.detect?.participation_summary || {}; const busy = Array.isArray(summary.busy_nodes) ? summary.busy_nodes.join("、") : ""; const staleNodes = Array.isArray(summary.stale_nodes) ? summary.stale_nodes : []; const offlineNodes = Array.isArray(summary.offline_nodes) ? summary.offline_nodes : []; const parts = [ - `当前有效执行节点 ${summary.online_worker_nodes || 0}，其中独立 Worker ${summary.dedicated_online_worker_nodes || 0}` + `当前有效执行节点 ${summary.online_worker_nodes || 0}，其中独立 Worker ${summary.dedicated_online_worker_nodes || 0}，控制面兼跑 ${participationSummary.controller_worker_nodes || 0}` ]; + parts.push( + `真正执行/领任务 ${participationSummary.dispatch_active_nodes || 0} 台，近窗刚有吞吐 ${participationSummary.recent_only_nodes || 0} 台，在线未参与 ${participationSummary.non_participating_nodes || 0} 台` + ); if (busy) { parts.push(`忙碌节点：${busy}`); } @@ -893,8 +1027,9 @@ const participatingNodesAlertText = computed(() => { return "当前没有节点正在领任务、执行任务或产生近窗吞吐。"; } const names = rows.map((item: Record) => String(item.node_code || "-")).join("、"); - const effectiveCount = rows.filter((item: Record) => Boolean(item.is_effective_worker)).length; - return `当前参与检测节点 ${rows.length} 台，其中有效执行节点 ${effectiveCount} 台：${names}`; + const dispatchActiveCount = rows.filter((item: Record) => ["running", "claimed"].includes(String(item.participation_state || "").trim())).length; + const recentOnlyCount = rows.filter((item: Record) => String(item.participation_state || "").trim() === "recent_throughput").length; + return `当前参与检测节点 ${rows.length} 台：正在执行/领任务 ${dispatchActiveCount} 台，近窗刚有吞吐 ${recentOnlyCount} 台。节点：${names}`; }); const syncAlertText = computed(() => { @@ -920,6 +1055,127 @@ const syncAlertText = computed(() => { return parts.join("；"); }); +const workerLogSyncEnabled = computed(() => { + if (runtime.detect?.log_sync) { + return Boolean(runtime.detect.log_sync.enabled); + } + return Boolean(settingsState.runtime_settings?.worker_log_sync_enabled); +}); + +const workerLogSyncMode = computed(() => { + const sourceValue = runtime.detect?.log_sync?.mode ?? settingsState.runtime_settings?.worker_log_sync_mode; + return String(sourceValue || "").trim().toLowerCase() === "full" ? "full" : "key"; +}); + +const workerLogSyncDescription = computed(() => { + if (!workerLogSyncEnabled.value) { + return "当前已关闭远端日志回传；跨机联调时不会把 Worker 过程日志镜像到当前后台。"; + } + const lineCount = Number(runtime.detect?.log_sync?.line_count || 0); + const sourceNodes = Array.isArray(runtime.detect?.log_sync?.source_nodes) ? runtime.detect.log_sync.source_nodes : []; + const lastAt = String(runtime.detect?.log_sync?.last_at || "").trim(); + const observationSuffix = lineCount > 0 + ? ` 当前已收到 ${lineCount} 条回传日志，来源节点 ${sourceNodes.length || 0} 台${lastAt ? `，最近一条 ${lastAt}` : ""}。` + : " 当前还没有收到远端日志样本。"; + if (workerLogSyncMode.value === "full") { + return `当前为全量回传；会尽量保留更多 Worker 执行过程，适合短时深度排障，但日志量会更大。${observationSuffix}`; + } + return `当前为关键回传；只同步阶段、异常、代理与关键执行事件，适合日常联调观察。${observationSuffix}`; +}); + +const workerLogSyncNodesText = computed(() => { + const nodes = Array.isArray(runtime.detect?.log_sync?.source_nodes) ? runtime.detect.log_sync.source_nodes : []; + return nodes.length ? nodes.join("、") : "当前还没有远端节点回传样本"; +}); + +const workerLogSyncLastAtText = computed(() => { + return String(runtime.detect?.log_sync?.last_at || "").trim() || "暂无"; +}); + +const workerLogSyncLastLineText = computed(() => { + return String(runtime.detect?.log_sync?.last_line || "").trim() || "当前还没有远端日志回传"; +}); + +const participatingNodeCodeSet = computed(() => { + const rows = Array.isArray(runtime.detect?.participating_nodes) ? runtime.detect.participating_nodes : []; + return new Set(rows.map((item: Record) => String(item.node_code || "").trim()).filter(Boolean)); +}); + +const nonParticipatingNodes = computed(() => { + if (Array.isArray(runtime.detect?.non_participating_nodes)) { + return runtime.detect.non_participating_nodes; + } + if (Array.isArray(runtime.detect?.standby_nodes)) { + return runtime.detect.standby_nodes; + } + const rows = Array.isArray(runtime.cluster?.nodes) ? runtime.cluster.nodes : []; + return rows.filter((item: Record) => { + const nodeCode = String(item.node_code || "").trim(); + const status = String(item.status || "").trim(); + const currentLoad = Number(item.current_load || 0); + const isEffectiveWorker = Boolean(item.is_effective_worker); + return Boolean(nodeCode) + && isEffectiveWorker + && ["online", "busy"].includes(status) + && !participatingNodeCodeSet.value.has(nodeCode) + && currentLoad <= 0; + }); +}); + +const nonParticipatingNodesAlertText = computed(() => { + if (!nonParticipatingNodes.value.length) { + return "当前所有有效执行节点要么正在参与检测，要么当前没有额外在线未参与节点。"; + } + const names = nonParticipatingNodes.value.map((item: Record) => String(item.node_code || "-")).join("、"); + const loadSyncingCount = nonParticipatingNodes.value.filter( + (item: Record) => String(item.participation_state || "").trim() === "load_syncing" + ).length; + const standbyCount = nonParticipatingNodes.value.length - loadSyncingCount; + return `当前在线但未参与节点 ${nonParticipatingNodes.value.length} 台：在线待命 ${standbyCount} 台，负载待确认 ${loadSyncingCount} 台。节点：${names}`; +}); + +const nonParticipationStateText = (row: Record) => { + if (row?.participation_label) return String(row.participation_label); + if (String(row?.participation_state || "").trim() === "load_syncing") return "负载待确认"; + return "在线待命"; +}; + +const nonParticipationStateTagType = (row: Record) => { + const state = String(row?.participation_state || "").trim(); + if (state === "load_syncing") return "warning"; + return "info"; +}; + +const participationStateText = (row: Record) => { + if (row?.participation_label) return String(row.participation_label); + const running = Number(row.items_running || 0); + const claimed = Number(row.items_claimed || 0); + const processedRecent = Number(row.processed_recent || 0); + if (running > 0) return "执行中"; + if (claimed > 0) return "已领待跑"; + if (processedRecent > 0) return "近窗有吞吐"; + return "参与中"; +}; + +const participationStateTagType = (row: Record) => { + const state = String(row?.participation_state || participationStateText(row)); + if (["running", "执行中", "busy", "忙碌中"].includes(state)) return "warning"; + if (["claimed", "已领待跑"].includes(state)) return "success"; + if (["recent_throughput", "近窗有吞吐"].includes(state)) return "info"; + return "info"; +}; + +const standbyReasonText = (row: Record) => { + if (row?.standby_reason) return String(row.standby_reason); + if (row?.participation_reason) return String(row.participation_reason); + const metadata = row?.metadata || {}; + if (metadata?.detail) return String(metadata.detail); + if (metadata?.phase_detail) return String(metadata.phase_detail); + if (metadata?.phase) return `当前阶段：${metadata.phase}`; + if (metadata?.phase_label) return `当前阶段：${metadata.phase_label}`; + return "节点在线，当前未领任务、未执行任务，也没有近窗吞吐。"; +}; + const hasNodeOverride = (nodeCode?: string) => { const normalized = String(nodeCode || "").trim(); return Boolean(normalized && settingsState.node_thread_counts?.[normalized] != null); @@ -1046,6 +1302,33 @@ const updateNodeThreadSettings = async (updater: (nodeThreadCounts: Record { + workerLogSyncLoading.value = true; + pendingWorkerLogMode.value = enabled ? mode : "off"; + try { + const latest = await settingsApi.getSettings(); + const payload = latest.data || {}; + await settingsApi.updateSettings({ + thread_count: payload.thread_count, + node_thread_counts: payload.node_thread_counts || {}, + detect_options: payload.detect_options || {}, + proxy_config: payload.proxy_config || {}, + runtime_settings: { + ...(payload.runtime_settings || {}), + worker_log_sync_enabled: enabled, + worker_log_sync_mode: mode + } + }); + ElMessage.success(`远端日志回传已${enabled ? `切到${mode === "full" ? "全量" : "关键"}模式` : "关闭"}`); + await loadAll(false); + } catch (error: any) { + ElMessage.error(error?.message || "更新远端日志回传开关失败"); + } finally { + workerLogSyncLoading.value = false; + pendingWorkerLogMode.value = "off"; + } +}; + const saveNodeThreadOverride = async () => { if (!nodeThreadDialog.nodeCode) { ElMessage.warning("节点编码不能为空"); @@ -1269,6 +1552,11 @@ onBeforeUnmount(clearRefreshTimer); color: #c2410c; } +.status-chip.info { + background: #eff6ff; + color: #1d4ed8; +} + .status-chip.danger { background: #fef2f2; color: #b91c1c; @@ -1300,6 +1588,42 @@ onBeforeUnmount(clearRefreshTimer); font-size: 12px; } +.remote-log-observation { + display: grid; + grid-template-columns: repeat(auto-fit, minmax(180px, 1fr)); + gap: 10px; + margin-top: 12px; +} + +.remote-log-observation__item { + display: flex; + flex-direction: column; + gap: 4px; + padding: 10px 12px; + border-radius: 10px; + background: #f8fbff; + border: 1px solid #dbe4ee; + color: #0f172a; + font-size: 13px; +} + +.remote-log-observation__item .label { + color: #64748b; + font-size: 12px; +} + +.remote-log-observation__item--wide, +.remote-log-observation__item--full { + grid-column: span 2; +} + +@media (max-width: 900px) { + .remote-log-observation__item--wide, + .remote-log-observation__item--full { + grid-column: span 1; + } +} + .cluster-summary { display: flex; flex-wrap: wrap; diff --git a/domain-web/vite.config.ts b/domain-web/vite.config.ts index 149ec8f..f259dbb 100644 --- a/domain-web/vite.config.ts +++ b/domain-web/vite.config.ts @@ -4,6 +4,13 @@ import { fileURLToPath, URL } from "node:url"; export default defineConfig({ plugins: [vue()], + css: { + preprocessorOptions: { + scss: { + api: "modern-compiler" + } + } + }, resolve: { alias: { "@": fileURLToPath(new URL("./src", import.meta.url)) diff --git a/domainCheck/node_thread_counts.json b/domainCheck/node_thread_counts.json index 9e26dfe..a4e3c54 100644 --- a/domainCheck/node_thread_counts.json +++ b/domainCheck/node_thread_counts.json @@ -1 +1,4 @@ -{} \ No newline at end of file +{ + "mainland-controller-01": 100, + "mainland-worker-01": 50 +} \ No newline at end of file diff --git a/package_domain_release.ps1 b/package_domain_release.ps1 index 02aac8f..cdd122c 100644 --- a/package_domain_release.ps1 +++ b/package_domain_release.ps1 @@ -3,17 +3,44 @@ $ErrorActionPreference = "Stop" $root = Split-Path -Parent $MyInvocation.MyCommand.Path $releaseRoot = Join-Path $root "release" $timestamp = Get-Date -Format "yyyyMMdd_HHmmss" -$packageName = "domaincheck_release_$timestamp" +$basePackageName = "domaincheck_release_$timestamp" +$packageName = $basePackageName $stagingRoot = Join-Path $releaseRoot $packageName $zipPath = Join-Path $releaseRoot "$packageName.zip" $hashPath = Join-Path $releaseRoot "$packageName.sha256.txt" +$suffix = 1 +while ((Test-Path -LiteralPath $stagingRoot) -or (Test-Path -LiteralPath $zipPath) -or (Test-Path -LiteralPath $hashPath)) { + $packageName = "${basePackageName}_$suffix" + $stagingRoot = Join-Path $releaseRoot $packageName + $zipPath = Join-Path $releaseRoot "$packageName.zip" + $hashPath = Join-Path $releaseRoot "$packageName.sha256.txt" + $suffix += 1 +} + +if (-not $releaseCommitSha) { + $gitCommand = Get-Command git -ErrorAction SilentlyContinue + if ($gitCommand) { + try { + $insideRepo = (& git -C $root rev-parse --is-inside-work-tree 2>$null).Trim() + if ($insideRepo -eq "true") { + $releaseCommitSha = (& git -C $root rev-parse --short=12 HEAD 2>$null).Trim() + $releaseCommitRef = (& git -C $root rev-parse --abbrev-ref HEAD 2>$null).Trim() + } + } + catch { + } + } +} + $latestTxtPath = Join-Path $releaseRoot "latest_release.txt" $latestJsonPath = Join-Path $releaseRoot "latest_release.json" $smokeScript = Join-Path $root "domainCheck\.venv\Scripts\python.exe" $smokeRunner = Join-Path $root "domain-api\deploy\linux\smoke_test.py" $smokeOutput = Join-Path $stagingRoot "smoke_test_report.json" $apiBaseUrl = if ($env:API_BASE_URL) { $env:API_BASE_URL } else { "http://127.0.0.1:8100" } -$webUrl = if ($env:WEB_URL) { $env:WEB_URL } else { "http://127.0.0.1:3200" } +$webUrl = if ($env:WEB_URL) { $env:WEB_URL } else { "" } +$releaseCommitSha = if ($env:DOMAINCHECK_RELEASE_COMMIT_SHA) { $env:DOMAINCHECK_RELEASE_COMMIT_SHA } else { $releaseCommitSha } +$releaseCommitRef = if ($env:DOMAINCHECK_RELEASE_COMMIT_REF) { $env:DOMAINCHECK_RELEASE_COMMIT_REF } else { $releaseCommitRef } function Copy-OptionalItem { param( @@ -56,7 +83,11 @@ $rootScripts = @( "package_domain_release.ps1", "verify_domain_release.ps1", "prepare_final_release.ps1", - "show_latest_release.ps1" + "show_latest_release.ps1", + "package_domain_release.sh", + "verify_domain_release.sh", + "prepare_final_release.sh", + "show_latest_release.sh" ) foreach ($script in $rootScripts) { Copy-OptionalItem -Source (Join-Path $root $script) -Destination $scriptsTarget @@ -99,10 +130,6 @@ $smokeMessage = "" if ((Test-Path -LiteralPath $smokeScript) -and (Test-Path -LiteralPath $smokeRunner)) { try { & $smokeScript $smokeRunner --base-url $apiBaseUrl --web-url $webUrl --output $smokeOutput | Out-Null - if (Test-Path -LiteralPath $smokeOutput) { - $smokeOk = $true - $smokeMessage = "generated" - } } catch { $smokeMessage = $_.Exception.Message @@ -112,10 +139,40 @@ else { $smokeMessage = "smoke test runtime not found" } +if (Test-Path -LiteralPath $smokeOutput) { + try { + $smokeReport = Get-Content -LiteralPath $smokeOutput -Raw | ConvertFrom-Json + $checks = @($smokeReport.checks) + $totalChecks = $checks.Count + $failedChecks = @($checks | Where-Object { -not $_.ok } | ForEach-Object { [string]$_.name }) + $passedChecks = $totalChecks - $failedChecks.Count + $smokeOk = [bool]$smokeReport.ok + if ($smokeOk) { + $smokeMessage = if ($totalChecks -gt 0) { "all checks passed ($passedChecks/$totalChecks)" } else { "all checks passed" } + } + else { + $failedPreview = ($failedChecks | Select-Object -First 5) -join ", " + $smokeMessage = if ($failedPreview) { + "$passedChecks/$totalChecks checks passed; failed: $failedPreview" + } else { + "$passedChecks/$totalChecks checks passed" + } + } + } + catch { + if (-not $smokeMessage) { + $smokeMessage = "smoke report unreadable" + } + } +} + $manifest = [ordered]@{ package_name = $packageName + package_type = "zip" generated_at = (Get-Date).ToString("s") root = $root + commit_sha = $releaseCommitSha + commit_ref = $releaseCommitRef smoke_test = [ordered]@{ ok = $smokeOk message = $smokeMessage @@ -166,22 +223,31 @@ $zipHash = (Get-FileHash -LiteralPath $zipPath -Algorithm SHA256).Hash.ToLowerIn $latestRelease = [ordered]@{ package_name = $packageName + package_type = "zip" generated_at = (Get-Date).ToString("s") staging_path = $stagingRoot + archive_path = $zipPath zip_path = $zipPath sha256_path = $hashPath sha256 = $zipHash smoke_test_ok = $smokeOk + smoke_test_message = $smokeMessage + smoke_test_report = if (Test-Path -LiteralPath $smokeOutput) { "smoke_test_report.json" } else { "" } + commit_sha = $releaseCommitSha + commit_ref = $releaseCommitRef } $latestRelease | ConvertTo-Json -Depth 6 | Set-Content -LiteralPath $latestJsonPath -Encoding UTF8 @( "package_name=$packageName" "staging_path=$stagingRoot" + "archive_path=$zipPath" "zip_path=$zipPath" "sha256_path=$hashPath" "sha256=$zipHash" "smoke_test_ok=$smokeOk" + "commit_sha=$releaseCommitSha" + "commit_ref=$releaseCommitRef" ) | Set-Content -LiteralPath $latestTxtPath -Encoding UTF8 Write-Host "release package created" diff --git a/package_domain_release.sh b/package_domain_release.sh new file mode 100755 index 0000000..f0cf50b --- /dev/null +++ b/package_domain_release.sh @@ -0,0 +1,328 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +RELEASE_ROOT="${DOMAINCHECK_RELEASE_ROOT:-${ROOT_DIR}/release}" +TIMESTAMP="${DOMAINCHECK_RELEASE_TIMESTAMP:-$(date +%Y%m%d_%H%M%S)}" +BASE_PACKAGE_NAME="domaincheck_release_${TIMESTAMP}" +PACKAGE_NAME="${BASE_PACKAGE_NAME}" +STAGING_ROOT="${RELEASE_ROOT}/${PACKAGE_NAME}" +ARCHIVE_PATH="${RELEASE_ROOT}/${PACKAGE_NAME}.tar.gz" +HASH_PATH="${RELEASE_ROOT}/${PACKAGE_NAME}.sha256.txt" +LATEST_TXT_PATH="${RELEASE_ROOT}/latest_release.txt" +LATEST_JSON_PATH="${RELEASE_ROOT}/latest_release.json" +SMOKE_PYTHON="${DOMAINCHECK_SMOKE_PYTHON:-${ROOT_DIR}/domainCheck/.venv/bin/python}" +SMOKE_RUNNER="${ROOT_DIR}/domain-api/deploy/linux/smoke_test.py" +SMOKE_OUTPUT="${STAGING_ROOT}/smoke_test_report.json" +API_BASE_URL="${API_BASE_URL:-http://127.0.0.1:8100}" +WEB_URL="${WEB_URL:-}" +SKIP_SMOKE_TEST="${DOMAINCHECK_SKIP_SMOKE_TEST:-0}" +RELEASE_COMMIT_SHA="${DOMAINCHECK_RELEASE_COMMIT_SHA:-}" +RELEASE_COMMIT_REF="${DOMAINCHECK_RELEASE_COMMIT_REF:-}" + +copy_optional_item() { + local source_path="$1" + local destination_path="$2" + if [[ -e "${source_path}" ]]; then + cp -a "${source_path}" "${destination_path}" + fi +} + +summarize_smoke_report() { + local report_path="$1" + python3 - "${report_path}" <<'PY' +import json +import sys +from pathlib import Path + +report_path = Path(sys.argv[1]) +if not report_path.exists(): + print("false\treport missing") + raise SystemExit(0) + +payload = json.loads(report_path.read_text(encoding="utf-8")) +checks = [item for item in list(payload.get("checks") or []) if isinstance(item, dict)] +total = len(checks) +failed = [str(item.get("name") or "").strip() for item in checks if not bool(item.get("ok"))] +passed = total - len(failed) +ok = bool(payload.get("ok", False)) + +if ok: + message = f"all checks passed ({passed}/{total})" if total else "all checks passed" +else: + failed_preview = ", ".join([name for name in failed if name][:5]) + if failed_preview: + message = f"{passed}/{total} checks passed; failed: {failed_preview}" + else: + message = f"{passed}/{total} checks passed" + +print(f"{str(ok).lower()}\t{message}") +PY +} + +mkdir -p "${RELEASE_ROOT}" +PACKAGE_SUFFIX=1 +while [[ -e "${STAGING_ROOT}" || -e "${ARCHIVE_PATH}" || -e "${HASH_PATH}" ]]; do + PACKAGE_NAME="${BASE_PACKAGE_NAME}_${PACKAGE_SUFFIX}" + STAGING_ROOT="${RELEASE_ROOT}/${PACKAGE_NAME}" + ARCHIVE_PATH="${RELEASE_ROOT}/${PACKAGE_NAME}.tar.gz" + HASH_PATH="${RELEASE_ROOT}/${PACKAGE_NAME}.sha256.txt" + PACKAGE_SUFFIX=$((PACKAGE_SUFFIX + 1)) +done + +if [[ -z "${RELEASE_COMMIT_SHA}" ]] && command -v git >/dev/null 2>&1; then + if git -C "${ROOT_DIR}" rev-parse --is-inside-work-tree >/dev/null 2>&1; then + RELEASE_COMMIT_SHA="$(git -C "${ROOT_DIR}" rev-parse --short=12 HEAD 2>/dev/null || true)" + RELEASE_COMMIT_REF="$(git -C "${ROOT_DIR}" rev-parse --abbrev-ref HEAD 2>/dev/null || true)" + fi +fi + +rm -rf "${STAGING_ROOT}" +rm -f "${ARCHIVE_PATH}" "${HASH_PATH}" +mkdir -p "${STAGING_ROOT}" + +shopt -s nullglob + +DOCS_TARGET="${STAGING_ROOT}/docs" +mkdir -p "${DOCS_TARGET}" +DOC_FILES=("${ROOT_DIR}"/docs/*.md) +if ((${#DOC_FILES[@]})); then + cp -a "${DOC_FILES[@]}" "${DOCS_TARGET}/" +fi + +SCRIPTS_TARGET="${STAGING_ROOT}/scripts" +mkdir -p "${SCRIPTS_TARGET}" +ROOT_SCRIPTS=( + "start_domain_api.ps1" + "stop_domain_api.ps1" + "start_domain_web.ps1" + "stop_domain_web.ps1" + "start_domain_web_background.ps1" + "start_domain_stack.ps1" + "stop_domain_stack.ps1" + "smoke_test_stack.ps1" + "package_domain_release.ps1" + "verify_domain_release.ps1" + "prepare_final_release.ps1" + "show_latest_release.ps1" + "package_domain_release.sh" + "verify_domain_release.sh" + "prepare_final_release.sh" + "show_latest_release.sh" +) +for script_name in "${ROOT_SCRIPTS[@]}"; do + copy_optional_item "${ROOT_DIR}/${script_name}" "${SCRIPTS_TARGET}/" +done + +API_TARGET="${STAGING_ROOT}/domain-api" +mkdir -p "${API_TARGET}" +API_ITEMS=( + "app" + "deploy" + "README.md" + "requirements.txt" + ".env.example" +) +for item in "${API_ITEMS[@]}"; do + copy_optional_item "${ROOT_DIR}/domain-api/${item}" "${API_TARGET}/" +done + +WEB_TARGET="${STAGING_ROOT}/domain-web" +mkdir -p "${WEB_TARGET}" +WEB_ITEMS=( + "src" + "deploy" + "README.md" + "package.json" + "package-lock.json" + "tsconfig.json" + "tsconfig.node.json" + "vite.config.ts" + ".env.example" + ".env.production.example" +) +for item in "${WEB_ITEMS[@]}"; do + copy_optional_item "${ROOT_DIR}/domain-web/${item}" "${WEB_TARGET}/" +done +copy_optional_item "${ROOT_DIR}/domain-web/dist" "${WEB_TARGET}/" + +SMOKE_OK="false" +SMOKE_MESSAGE="" +if [[ "${SKIP_SMOKE_TEST}" == "1" ]]; then + SMOKE_MESSAGE="skipped by env" +elif [[ -x "${SMOKE_PYTHON}" && -f "${SMOKE_RUNNER}" ]]; then + if smoke_log="$("${SMOKE_PYTHON}" "${SMOKE_RUNNER}" --base-url "${API_BASE_URL}" --web-url "${WEB_URL}" --output "${SMOKE_OUTPUT}" 2>&1)"; then + if [[ -f "${SMOKE_OUTPUT}" ]]; then + SMOKE_OK="true" + SMOKE_MESSAGE="generated" + else + SMOKE_MESSAGE="smoke test succeeded but report missing" + fi + else + SMOKE_MESSAGE="${smoke_log}" + fi +else + SMOKE_MESSAGE="smoke test runtime not found" +fi + +if [[ -f "${SMOKE_OUTPUT}" ]]; then + SMOKE_SUMMARY="$(summarize_smoke_report "${SMOKE_OUTPUT}")" + SMOKE_OK="${SMOKE_SUMMARY%%$'\t'*}" + SMOKE_MESSAGE="${SMOKE_SUMMARY#*$'\t'}" +fi + +python3 - "${STAGING_ROOT}" "${PACKAGE_NAME}" "${ROOT_DIR}" "${SMOKE_OK}" "${SMOKE_MESSAGE}" "${API_BASE_URL}" "${WEB_URL}" "${RELEASE_COMMIT_SHA}" "${RELEASE_COMMIT_REF}" <<'PY' +import json +import os +import sys +from datetime import datetime +from pathlib import Path + +staging_root = Path(sys.argv[1]) +package_name = sys.argv[2] +root_dir = sys.argv[3] +smoke_ok = sys.argv[4].lower() == "true" +smoke_message = sys.argv[5] +api_base_url = sys.argv[6] +web_url = sys.argv[7] +commit_sha = sys.argv[8] +commit_ref = sys.argv[9] + +docs_target = staging_root / "docs" +scripts_target = staging_root / "scripts" +api_target = staging_root / "domain-api" +web_target = staging_root / "domain-web" +smoke_output = staging_root / "smoke_test_report.json" + +manifest = { + "package_name": package_name, + "package_type": "tar.gz", + "generated_at": datetime.now().isoformat(timespec="seconds"), + "root": root_dir, + "commit_sha": commit_sha, + "commit_ref": commit_ref, + "smoke_test": { + "ok": smoke_ok, + "message": smoke_message, + "api_base_url": api_base_url, + "web_url": web_url, + "report": "smoke_test_report.json" if smoke_output.exists() else "", + }, + "included": { + "docs": sorted(item.name for item in docs_target.iterdir() if item.is_file()) if docs_target.exists() else [], + "scripts": sorted(item.name for item in scripts_target.iterdir() if item.is_file()) if scripts_target.exists() else [], + "domain_api": sorted(item.name for item in api_target.iterdir()) if api_target.exists() else [], + "domain_web": sorted(item.name for item in web_target.iterdir()) if web_target.exists() else [], + }, +} + +(staging_root / "release_manifest.json").write_text( + json.dumps(manifest, ensure_ascii=False, indent=2), + encoding="utf-8", +) +PY + +cat > "${STAGING_ROOT}/README_RELEASE.txt" < "${HASH_PATH}" <&2 + exit 1 +fi + +VERIFY_OUTPUT="$(bash "${ROOT_DIR}/verify_domain_release.sh")" + +python3 - "${LATEST_JSON_PATH}" "${REPORT_PATH}" "${VERIFY_OUTPUT}" <<'PY' +import json +import sys +from datetime import datetime +from pathlib import Path + +latest_json_path = Path(sys.argv[1]) +report_path = Path(sys.argv[2]) +verify_payload = json.loads(sys.argv[3]) +latest_payload = json.loads(latest_json_path.read_text(encoding="utf-8")) +verify_ok = bool(verify_payload.get("ok", False)) +smoke_ok = bool(latest_payload.get("smoke_test_ok", False)) +blocked_reasons = [] +if not verify_ok: + blocked_reasons.append("verify_failed") +if not smoke_ok: + blocked_reasons.append("smoke_test_failed_or_skipped") + +report = { + "generated_at": datetime.now().isoformat(timespec="seconds"), + "ok": verify_ok and smoke_ok, + "latest_release": latest_payload, + "verify": verify_payload, + "release_gate": { + "verify_ok": verify_ok, + "smoke_test_ok": smoke_ok, + "blocked_reasons": blocked_reasons, + "decision": "ready" if not blocked_reasons else "blocked", + }, +} +report_path.write_text(json.dumps(report, ensure_ascii=False, indent=2), encoding="utf-8") +print(json.dumps({"report": str(report_path), "ok": report["ok"]}, ensure_ascii=False, indent=2)) +PY + +printf 'final release prepared\n' +printf 'report: %s\n' "${REPORT_PATH}" diff --git a/show_latest_release.ps1 b/show_latest_release.ps1 index b82f339..11d8773 100644 --- a/show_latest_release.ps1 +++ b/show_latest_release.ps1 @@ -15,10 +15,68 @@ if (Test-Path -LiteralPath $finalReportPath) { $report = Get-Content -LiteralPath $finalReportPath -Raw | ConvertFrom-Json } +$reportOk = $null +$reportMatchesLatest = $null +$reportPackageName = "" +$reportStaleReason = "" +$finalReleaseOk = $null +$releaseGateDecision = "" +$releaseGateBlockedReasons = @() +$releaseGateVerifyOk = $null +$releaseGateSmokeTestOk = $null + +if ($report) { + $reportOk = [bool]$report.ok + if ($report.release_gate) { + $releaseGateDecision = [string]$report.release_gate.decision + $releaseGateBlockedReasons = @($report.release_gate.blocked_reasons) + if ($null -ne $report.release_gate.verify_ok) { + $releaseGateVerifyOk = [bool]$report.release_gate.verify_ok + } + if ($null -ne $report.release_gate.smoke_test_ok) { + $releaseGateSmokeTestOk = [bool]$report.release_gate.smoke_test_ok + } + } + if ($report.latest_release) { + $reportPackageName = [string]$report.latest_release.package_name + $mismatches = @() + $reportArchivePath = if ($report.latest_release.archive_path) { [string]$report.latest_release.archive_path } else { [string]$report.latest_release.zip_path } + $latestArchivePath = if ($latest.archive_path) { [string]$latest.archive_path } else { [string]$latest.zip_path } + if ([string]$report.latest_release.package_name -ne [string]$latest.package_name) { + $mismatches += "package_name" + } + if ($reportArchivePath -ne $latestArchivePath) { + $mismatches += "archive_path" + } + if ([string]$report.latest_release.sha256 -ne [string]$latest.sha256) { + $mismatches += "sha256" + } + + $reportMatchesLatest = ($mismatches.Count -eq 0) + if (-not $reportMatchesLatest) { + $reportStaleReason = "final_release_report does not match latest_release: $($mismatches -join ', ')" + } + } + else { + $reportMatchesLatest = $false + $reportStaleReason = "final_release_report.latest_release missing" + } + + $finalReleaseOk = ($reportOk -and $reportMatchesLatest) +} + $summary = [ordered]@{ latest_release = $latest - final_release_ok = if ($report) { [bool]$report.ok } else { $null } + final_release_ok = $finalReleaseOk final_release_report = if ($report) { $finalReportPath } else { "" } + final_release_report_ok = $reportOk + final_release_report_matches_latest = $reportMatchesLatest + final_release_report_package_name = $reportPackageName + final_release_report_stale_reason = $reportStaleReason + final_release_gate_decision = $releaseGateDecision + final_release_gate_blocked_reasons = $releaseGateBlockedReasons + final_release_gate_verify_ok = $releaseGateVerifyOk + final_release_gate_smoke_test_ok = $releaseGateSmokeTestOk } $summary | ConvertTo-Json -Depth 8 diff --git a/show_latest_release.sh b/show_latest_release.sh new file mode 100755 index 0000000..aa0fb33 --- /dev/null +++ b/show_latest_release.sh @@ -0,0 +1,91 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +RELEASE_ROOT="${DOMAINCHECK_RELEASE_ROOT:-${ROOT_DIR}/release}" +LATEST_JSON_PATH="${RELEASE_ROOT}/latest_release.json" +FINAL_REPORT_PATH="${RELEASE_ROOT}/final_release_report.json" + +if [[ ! -f "${LATEST_JSON_PATH}" ]]; then + echo "latest_release.json not found" >&2 + exit 1 +fi + +python3 - "${LATEST_JSON_PATH}" "${FINAL_REPORT_PATH}" <<'PY' +import json +import sys +from pathlib import Path + +latest_json_path = Path(sys.argv[1]) +final_report_path = Path(sys.argv[2]) + +latest_payload = json.loads(latest_json_path.read_text(encoding="utf-8")) +final_payload = None +if final_report_path.exists(): + final_payload = json.loads(final_report_path.read_text(encoding="utf-8")) + +report_ok = None +report_matches_latest = None +report_package_name = "" +report_stale_reason = "" +final_release_ok = None +release_gate_decision = "" +release_gate_blocked_reasons = [] +release_gate_verify_ok = None +release_gate_smoke_test_ok = None + +if final_payload is not None: + report_ok = bool(final_payload.get("ok", False)) + report_latest = final_payload.get("latest_release") if isinstance(final_payload, dict) else None + report_release_gate = final_payload.get("release_gate") if isinstance(final_payload, dict) else None + if isinstance(report_release_gate, dict): + release_gate_decision = str(report_release_gate.get("decision") or "").strip() + release_gate_blocked_reasons = [ + str(item).strip() + for item in list(report_release_gate.get("blocked_reasons") or []) + if str(item).strip() + ] + if "verify_ok" in report_release_gate: + release_gate_verify_ok = bool(report_release_gate.get("verify_ok")) + if "smoke_test_ok" in report_release_gate: + release_gate_smoke_test_ok = bool(report_release_gate.get("smoke_test_ok")) + if isinstance(report_latest, dict): + report_package_name = str(report_latest.get("package_name") or "").strip() + latest_package_name = str(latest_payload.get("package_name") or "").strip() + report_archive_path = str(report_latest.get("archive_path") or report_latest.get("zip_path") or "").strip() + latest_archive_path = str(latest_payload.get("archive_path") or latest_payload.get("zip_path") or "").strip() + report_sha256 = str(report_latest.get("sha256") or "").strip().lower() + latest_sha256 = str(latest_payload.get("sha256") or "").strip().lower() + + mismatches = [] + if report_package_name != latest_package_name: + mismatches.append("package_name") + if report_archive_path != latest_archive_path: + mismatches.append("archive_path") + if report_sha256 != latest_sha256: + mismatches.append("sha256") + + report_matches_latest = not mismatches + if mismatches: + report_stale_reason = "final_release_report does not match latest_release: " + ", ".join(mismatches) + else: + report_matches_latest = False + report_stale_reason = "final_release_report.latest_release missing" + + final_release_ok = report_ok and report_matches_latest + +summary = { + "latest_release": latest_payload, + "final_release_ok": final_release_ok, + "final_release_report": "" if final_payload is None else str(final_report_path), + "final_release_report_ok": report_ok, + "final_release_report_matches_latest": report_matches_latest, + "final_release_report_package_name": report_package_name, + "final_release_report_stale_reason": report_stale_reason, + "final_release_gate_decision": release_gate_decision, + "final_release_gate_blocked_reasons": release_gate_blocked_reasons, + "final_release_gate_verify_ok": release_gate_verify_ok, + "final_release_gate_smoke_test_ok": release_gate_smoke_test_ok, +} +print(json.dumps(summary, ensure_ascii=False, indent=2)) +PY diff --git a/verify_domain_release.ps1 b/verify_domain_release.ps1 index 872b021..d32639a 100644 --- a/verify_domain_release.ps1 +++ b/verify_domain_release.ps1 @@ -59,6 +59,7 @@ Add-Type -AssemblyName System.IO.Compression.FileSystem $zip = [System.IO.Compression.ZipFile]::OpenRead($ZipPath) $missingEntries = @() +$expectsSmokeReport = $false try { $requiredEntries = @( "README_RELEASE.txt", @@ -66,18 +67,42 @@ try { "domain-api/README.md", "domain-web/README.md", "scripts/package_domain_release.ps1", + "scripts/verify_domain_release.ps1", + "scripts/package_domain_release.sh", + "scripts/verify_domain_release.sh", "scripts/smoke_test_stack.ps1" ) $entryNames = $zip.Entries | ForEach-Object { $_.FullName.Replace('\', '/') } + $manifestEntry = $zip.Entries | Where-Object { $_.FullName.Replace('\', '/') -eq "release_manifest.json" } | Select-Object -First 1 + if ($manifestEntry) { + $manifestStream = $manifestEntry.Open() + $manifestReader = $null + try { + $manifestReader = New-Object System.IO.StreamReader($manifestStream) + $manifestText = $manifestReader.ReadToEnd() + $manifest = $manifestText | ConvertFrom-Json + $smokeReport = [string]$manifest.smoke_test.report + if (-not [string]::IsNullOrWhiteSpace($smokeReport)) { + $expectsSmokeReport = $true + } + } + finally { + if ($manifestReader) { + $manifestReader.Dispose() + } + $manifestStream.Dispose() + } + } + foreach ($required in $requiredEntries) { if ($entryNames -notcontains $required) { $missingEntries += $required } } - if ($entryNames -notcontains "smoke_test_report.json") { + if ($expectsSmokeReport -and ($entryNames -notcontains "smoke_test_report.json")) { $missingEntries += "smoke_test_report.json" } } diff --git a/verify_domain_release.sh b/verify_domain_release.sh new file mode 100755 index 0000000..af9e2bb --- /dev/null +++ b/verify_domain_release.sh @@ -0,0 +1,151 @@ +#!/usr/bin/env bash +set -euo pipefail + +ROOT_DIR="$(cd "$(dirname "${BASH_SOURCE[0]}")" && pwd)" +RELEASE_ROOT="${DOMAINCHECK_RELEASE_ROOT:-${ROOT_DIR}/release}" +PACKAGE_PATH="${1:-}" +HASH_PATH="${2:-}" + +python3 - "${RELEASE_ROOT}" "${PACKAGE_PATH}" "${HASH_PATH}" <<'PY' +import hashlib +import json +import sys +import tarfile +import zipfile +from pathlib import Path + +release_root = Path(sys.argv[1]) +package_path_arg = sys.argv[2].strip() +hash_path_arg = sys.argv[3].strip() + + +def resolve_latest_package(root: Path) -> Path: + candidates = sorted( + [ + item + for item in root.glob("*") + if item.is_file() and (item.name.endswith(".tar.gz") or item.name.endswith(".zip")) + ], + key=lambda item: item.stat().st_mtime, + reverse=True, + ) + if not candidates: + raise SystemExit("No release archive found") + return candidates[0] + + +def derive_hash_path(root: Path, package_path: Path) -> Path: + if package_path.name.endswith(".tar.gz"): + base_name = package_path.name[:-7] + else: + base_name = package_path.stem + candidate = root / f"{base_name}.sha256.txt" + if candidate.exists(): + return candidate + candidates = sorted(root.glob("*.sha256.txt"), key=lambda item: item.stat().st_mtime, reverse=True) + if not candidates: + raise SystemExit("No hash file found") + return candidates[0] + + +def compute_sha256(path: Path) -> str: + sha = hashlib.sha256() + with path.open("rb") as handle: + for chunk in iter(lambda: handle.read(1024 * 1024), b""): + sha.update(chunk) + return sha.hexdigest().lower() + + +def normalize_entry_names(names: list[str]) -> list[str]: + normalized: list[str] = [] + for name in names: + item = name.replace("\\", "/") + while item.startswith("./"): + item = item[2:] + item = item.strip("/") + if item: + normalized.append(item) + return normalized + + +def read_archive_entries(path: Path) -> tuple[list[str], dict]: + if path.name.endswith(".zip"): + with zipfile.ZipFile(path, "r") as archive: + entry_names = normalize_entry_names(archive.namelist()) + manifest = {} + if "release_manifest.json" in entry_names: + with archive.open("release_manifest.json", "r") as handle: + manifest = json.loads(handle.read().decode("utf-8")) + return entry_names, manifest + + with tarfile.open(path, "r:*") as archive: + entry_names = normalize_entry_names(archive.getnames()) + manifest = {} + if "release_manifest.json" in entry_names: + manifest_member = archive.extractfile("./release_manifest.json") or archive.extractfile("release_manifest.json") + if manifest_member is not None: + manifest = json.loads(manifest_member.read().decode("utf-8")) + return entry_names, manifest + + +package_path = Path(package_path_arg) if package_path_arg else resolve_latest_package(release_root) +if not package_path.exists(): + raise SystemExit(f"archive not found: {package_path}") + +hash_path = Path(hash_path_arg) if hash_path_arg else derive_hash_path(release_root, package_path) +if not hash_path.exists(): + raise SystemExit(f"hash file not found: {hash_path}") + +expected_hash = "" +expected_file = "" +for line in hash_path.read_text(encoding="utf-8").splitlines(): + if line.startswith("sha256="): + expected_hash = line.split("=", 1)[1].strip().lower() + elif line.startswith("file="): + expected_file = line.split("=", 1)[1].strip() + +if not expected_hash: + raise SystemExit(f"sha256 entry missing in {hash_path}") + +actual_hash = compute_sha256(package_path) +entry_names, manifest = read_archive_entries(package_path) +entry_set = set(entry_names) + +required_entries = [ + "README_RELEASE.txt", + "release_manifest.json", + "domain-api/README.md", + "domain-web/README.md", + "scripts/package_domain_release.ps1", + "scripts/verify_domain_release.ps1", + "scripts/package_domain_release.sh", + "scripts/verify_domain_release.sh", + "scripts/smoke_test_stack.ps1", +] + +missing_entries = [item for item in required_entries if item not in entry_set] + +smoke_report = str(((manifest.get("smoke_test") or {}).get("report") or "")).strip() +if smoke_report and smoke_report not in entry_set: + missing_entries.append(smoke_report) + +ok = ( + actual_hash == expected_hash + and not missing_entries + and (not expected_file or expected_file == package_path.name) +) + +report = { + "ok": ok, + "archive": str(package_path), + "hash_file": str(hash_path), + "expected_file": expected_file, + "actual_file": package_path.name, + "expected_sha256": expected_hash, + "actual_sha256": actual_hash, + "missing_entries": missing_entries, + "package_type": "tar.gz" if package_path.name.endswith(".tar.gz") else "zip", +} +print(json.dumps(report, ensure_ascii=False, indent=2)) +raise SystemExit(0 if ok else 1) +PY