admin/getDomain
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add ops center and node onboarding flow

Browse Source
This commit is contained in:
Your Name
2026-04-18 23:52:51 +08:00
parent 246838ae4c
commit b9c29481b5
142 changed files with 89727 additions and 186 deletions
Download Patch File Download Diff File Expand all files Collapse all files

664
domain-api/tests/test_ops_service_go_live_summary.py Normal file
View File

@@ -0,0 +1,664 @@
import unittest
from unittest.mock import patch
from app.services.ops_service import get_ops_go_live_summary
class OpsServiceGoLiveSummaryTests(unittest.TestCase):
@patch("app.services.ops_service.get_ops_overview")
@patch("app.services.ops_service.list_managed_nodes_with_agent_state")
@patch("app.services.ops_service.get_runtime_build_info")
@patch("app.services.ops_service.get_release_launchpad")
@patch("app.services.ops_service.get_ops_contract_registry")
@patch("app.services.ops_service.get_ops_stack_diagnosis")
def test_go_live_summary_exposes_runtime_schema_stale_and_api_restart_command(
self,
mock_get_ops_stack_diagnosis,
mock_get_ops_contract_registry,
mock_get_release_launchpad,
mock_get_runtime_build_info,
mock_list_managed_nodes_with_agent_state,
mock_get_ops_overview,
) -> None:
mock_get_ops_stack_diagnosis.return_value = {
"diagnosis": {
"contract_key": "ops_stack_diagnosis_contract",
"stack_status": "attention",
"next_step": {
"action_code": "api-restart",
"reason": "build schema stale",
"focus_ref": {"kind": "runtime_build_info", "section": "schema"},
},
"operator_decision": {
"lane": "runtime_recovery",
"title": "优先重启控制面 API 并复检运行时版本",
"primary_command_key": "api-restart",
},
},
"release_hub": {
"launchpad_status": "ready",
},
}
mock_get_ops_contract_registry.return_value = {
"registry_version": "2026-04-18",
"contracts": [{"key": "ops_stack_diagnosis_contract"}],
}
mock_get_release_launchpad.return_value = {
"launchpad_status": {
"status": "ready",
"status_label": "可发布",
"recommended_action_code": "create_smart_release_rollout_worker",
}
}
mock_get_runtime_build_info.return_value = {
"commit_sha": "246838ae4c07",
"repository_capabilities": {
"supports_install_command_block": True,
"supports_multi_layout_bootstrap": True,
},
"route_surface": {
"surface_complete": True,
"missing_keys": [],
"expected_paths": {
"ops_contracts": "/api/v1/ops/contracts",
},
},
}
mock_list_managed_nodes_with_agent_state.return_value = {
"summary": {
"managed_enabled": 1,
"remote_access_ready": 1,
"queue_dead_letter_nodes": 0,
}
}
mock_get_ops_overview.return_value = {
"recommendation": {
"title": "先重启 API",
"reason": "build schema stale",
"primary_action_code": "api-restart",
},
"execution_scene": {
"log_sync": {
"enabled": False,
"participating_node_count": 0,
"covered_participating_node_count": 0,
"missing_participating_nodes": [],
}
},
}
payload = get_ops_go_live_summary(base_url="http://127.0.0.1:8100")
self.assertEqual("attention", payload["go_live_status"])
self.assertEqual("attention", payload["publish_status"])
self.assertTrue(payload["runtime_schema_stale"])
self.assertFalse(payload["route_surface_declares_bootstrap_plan"])
self.assertIn("runtime_build_schema_stale", payload["warnings"])
self.assertIn("runtime_build_schema_stale", payload["publish_warnings"])
self.assertEqual(
"bash domain-api/deploy/multi-region/drive_ops_center.sh go-live-recover http://127.0.0.1:8100",
payload["recommended_commands"]["go_live_recover"],
)
self.assertEqual(
"bash domain-api/deploy/multi-region/drive_ops_center.sh runtime-refresh-recover http://127.0.0.1:8100",
payload["recommended_commands"]["api_restart"],
)
self.assertEqual(
"bash domain-api/deploy/multi-region/drive_ops_center.sh runtime-refresh-recover http://127.0.0.1:8100",
payload["recommended_commands"]["runtime_refresh_recover"],
)
self.assertIn("node-bootstrap-plan", payload["recommended_commands"]["bootstrap_plan_recheck"])
@patch("app.services.ops_service.get_ops_overview")
@patch("app.services.ops_service.list_managed_nodes_with_agent_state")
@patch("app.services.ops_service.get_runtime_build_info")
@patch("app.services.ops_service.get_release_launchpad")
@patch("app.services.ops_service.get_ops_contract_registry")
@patch("app.services.ops_service.get_ops_stack_diagnosis")
def test_go_live_summary_highlights_remote_access_and_log_sync_gaps(
self,
mock_get_ops_stack_diagnosis,
mock_get_ops_contract_registry,
mock_get_release_launchpad,
mock_get_runtime_build_info,
mock_list_managed_nodes_with_agent_state,
mock_get_ops_overview,
) -> None:
mock_get_ops_stack_diagnosis.return_value = {
"diagnosis": {
"contract_key": "ops_stack_diagnosis_contract",
"stack_status": "blocked",
"next_step": {
"action_code": "handover_first_gap",
"reason": "来自 overview.recommendation.primary_action_code",
},
"operator_decision": {},
},
"release_hub": {
"launchpad_status": "attention",
},
}
mock_get_ops_contract_registry.return_value = {
"registry_version": "2026-04-18",
"contracts": [{"key": "ops_stack_diagnosis_contract"}],
}
mock_get_release_launchpad.return_value = {
"launchpad_status": {
"status": "attention",
"status_label": "待补执行面",
"recommended_action_code": "fix_managed_nodes",
"focus_ref": {"kind": "release_hub"},
"onboarding_bootstrap_pending_nodes": 1,
"onboarding_acceptance_ready_nodes": 1,
"recommended_target_node_code": "mainland-controller-01",
"recommended_recovery_label": "签发接入工单",
"recommended_recovery_summary": "当前节点仍处于接入阶段，建议先签发 onboarding.bootstrap。",
}
}
mock_get_runtime_build_info.return_value = {
"commit_sha": "246838ae4c07",
"route_surface": {
"surface_complete": True,
"missing_keys": [],
},
}
mock_list_managed_nodes_with_agent_state.return_value = {
"summary": {
"managed_enabled": 3,
"remote_access_ready": 0,
"queue_dead_letter_nodes": 0,
}
}
mock_get_ops_overview.return_value = {
"recommendation": {
"title": "先补接管缺口",
"reason": "当前仍有节点没有进入标准执行器就绪状态。",
"primary_action_code": "handover_first_gap",
},
"execution_scene": {
"log_sync": {
"enabled": True,
"state": "waiting_sample",
"mode": "full",
"participating_node_count": 3,
"covered_participating_node_count": 0,
"missing_participating_nodes": [
"mainland-worker-01",
"mainland-controller-01",
"overseas-control-01",
],
}
},
}
payload = get_ops_go_live_summary(base_url="http://127.0.0.1:8100")
self.assertEqual("blocked", payload["go_live_status"])
self.assertFalse(payload["publish_ready"])
self.assertEqual("blocked", payload["publish_status"])
self.assertEqual("暂不可发布", payload["publish_status_label"])
self.assertIn("stack_diagnosis=blocked", payload["blocking_reasons"])
self.assertIn("stack_diagnosis=blocked", payload["publish_blocking_reasons"])
self.assertIn("remote_agent_ready=0", payload["blocking_reasons"])
self.assertIn("remote_agent_ready=0", payload["publish_blocking_reasons"])
self.assertIn("release_launchpad=attention", payload["warnings"])
self.assertIn("release_launchpad=attention", payload["publish_warnings"])
self.assertIn("launchpad_onboarding_bootstrap_pending=1", payload["warnings"])
self.assertIn("launchpad_onboarding_acceptance_ready=1", payload["warnings"])
self.assertIn("launchpad_onboarding_bootstrap_pending=1", payload["publish_warnings"])
self.assertIn("launchpad_onboarding_acceptance_ready=1", payload["publish_warnings"])
self.assertIn("log_sync_waiting_sample=0/3", payload["warnings"])
self.assertIn("log_sync_waiting_sample=0/3", payload["publish_warnings"])
self.assertTrue(payload["route_surface_complete"])
self.assertEqual("先补接管缺口", payload["operator_title"])
self.assertEqual("handover_first_gap", payload["next_step_action_code"])
self.assertEqual("mainland-controller-01", payload["launchpad_recommended_target_node_code"])
self.assertEqual("签发接入工单", payload["launchpad_recommended_recovery_label"])
self.assertEqual(
"当前节点仍处于接入阶段，建议先签发 onboarding.bootstrap。",
payload["launchpad_recommended_recovery_summary"],
)
self.assertEqual(1, payload["launchpad_onboarding_bootstrap_pending_nodes"])
self.assertEqual(1, payload["launchpad_onboarding_acceptance_ready_nodes"])
self.assertEqual(
"bash domain-api/deploy/multi-region/drive_ops_center.sh driver-resolve http://127.0.0.1:8100 handover_first_gap",
payload["recommended_commands"]["next_step"],
)
self.assertEqual(
"bash domain-api/deploy/multi-region/drive_ops_center.sh driver-resolve http://127.0.0.1:8100 open_worker_logs_participating",
payload["recommended_commands"]["log_sync_logs"],
)
@patch("app.services.ops_service.get_ops_overview")
@patch("app.services.ops_service.list_managed_nodes_with_agent_state")
@patch("app.services.ops_service.get_runtime_build_info")
@patch("app.services.ops_service.get_release_launchpad")
@patch("app.services.ops_service.get_ops_contract_registry")
@patch("app.services.ops_service.get_ops_stack_diagnosis")
def test_go_live_summary_falls_back_to_stack_handover_hint_when_launchpad_is_generic(
self,
mock_get_ops_stack_diagnosis,
mock_get_ops_contract_registry,
mock_get_release_launchpad,
mock_get_runtime_build_info,
mock_list_managed_nodes_with_agent_state,
mock_get_ops_overview,
) -> None:
mock_get_ops_stack_diagnosis.return_value = {
"diagnosis": {
"contract_key": "ops_stack_diagnosis_contract",
"stack_status": "blocked",
"next_step": {
"action_code": "bootstrap_run",
"reason": "当前节点仍处于接入阶段，建议先签发 onboarding.bootstrap。",
"focus_ref": {"kind": "managed_node", "node_code": "mainland-worker-01"},
},
"operator_decision": {
"lane": "node_handover",
"title": "优先执行节点接管收口",
"primary_command_key": "bootstrap_run",
},
"launchpad_recommended_target_node_code": "mainland-worker-01",
"launchpad_recommended_recovery_label": "签发接入工单",
"launchpad_recommended_recovery_summary": "当前节点仍处于接入阶段，建议先签发 onboarding.bootstrap。",
},
"release_hub": {
"launchpad_status": "attention",
},
}
mock_get_ops_contract_registry.return_value = {
"registry_version": "2026-04-18",
"contracts": [{"key": "ops_stack_diagnosis_contract"}],
}
mock_get_release_launchpad.return_value = {
"launchpad_status": {
"status": "attention",
"status_label": "待补执行面",
"recommended_action_code": "fix_managed_nodes",
}
}
mock_get_runtime_build_info.return_value = {
"commit_sha": "246838ae4c07",
"route_surface": {
"surface_complete": True,
"missing_keys": [],
"expected_paths": {},
},
"repository_capabilities": {},
}
mock_list_managed_nodes_with_agent_state.return_value = {
"summary": {
"managed_enabled": 1,
"remote_access_ready": 0,
"queue_dead_letter_nodes": 0,
}
}
mock_get_ops_overview.return_value = {
"recommendation": {
"title": "先补接管缺口",
"reason": "overview 仍在泛化描述。",
"primary_action_code": "handover_first_gap",
},
"execution_scene": {
"log_sync": {
"enabled": False,
"participating_node_count": 0,
"covered_participating_node_count": 0,
"missing_participating_nodes": [],
}
},
}
payload = get_ops_go_live_summary(base_url="http://127.0.0.1:8100")
self.assertEqual("bootstrap_run", payload["next_step_action_code"])
self.assertEqual("mainland-worker-01", payload["launchpad_recommended_target_node_code"])
self.assertEqual("签发接入工单", payload["launchpad_recommended_recovery_label"])
self.assertIn("onboarding.bootstrap", payload["launchpad_recommended_recovery_summary"])
self.assertEqual(
"bash domain-api/deploy/multi-region/drive_ops_center.sh driver-resolve http://127.0.0.1:8100 bootstrap_run",
payload["recommended_commands"]["next_step"],
)
@patch("app.services.ops_service.get_ops_overview")
@patch("app.services.ops_service.list_managed_nodes_with_agent_state")
@patch("app.services.ops_service.get_runtime_build_info")
@patch("app.services.ops_service.get_release_launchpad")
@patch("app.services.ops_service.get_ops_contract_registry")
@patch("app.services.ops_service.get_ops_stack_diagnosis")
def test_go_live_summary_marks_ready_when_surfaces_and_execution_are_closed(
self,
mock_get_ops_stack_diagnosis,
mock_get_ops_contract_registry,
mock_get_release_launchpad,
mock_get_runtime_build_info,
mock_list_managed_nodes_with_agent_state,
mock_get_ops_overview,
) -> None:
mock_get_ops_stack_diagnosis.return_value = {
"diagnosis": {
"contract_key": "ops_stack_diagnosis_contract",
"stack_status": "healthy",
"next_step": {
"action_code": "run_inspection_participating",
"reason": "继续做上线前复核。",
},
"operator_decision": {
"lane": "go-live",
"title": "可以进入上线复核",
"primary_command_key": "run_inspection_participating",
},
},
"release_hub": {
"launchpad_status": "ready",
},
}
mock_get_ops_contract_registry.return_value = {
"registry_version": "2026-04-18",
"contracts": [{"key": "ops_stack_diagnosis_contract"}],
}
mock_get_release_launchpad.return_value = {
"launchpad_status": {
"status": "ready",
"status_label": "可发布",
"recommended_action_code": "create_smart_release_rollout_worker",
}
}
mock_get_runtime_build_info.return_value = {
"commit_sha": "246838ae4c07",
"route_surface": {
"surface_complete": True,
"missing_keys": [],
},
}
mock_list_managed_nodes_with_agent_state.return_value = {
"summary": {
"managed_enabled": 2,
"remote_access_ready": 2,
"queue_dead_letter_nodes": 0,
}
}
mock_get_ops_overview.return_value = {
"recommendation": {
"title": "继续复核",
"reason": "当前执行面与观测面已收口。",
"primary_action_code": "run_inspection_participating",
},
"execution_scene": {
"log_sync": {
"enabled": True,
"state": "healthy",
"mode": "key",
"participating_node_count": 2,
"covered_participating_node_count": 2,
"missing_participating_nodes": [],
}
},
}
payload = get_ops_go_live_summary(base_url="http://127.0.0.1:8100")
self.assertEqual("ready", payload["go_live_status"])
self.assertTrue(payload["publish_ready"])
self.assertEqual("ready", payload["publish_status"])
self.assertEqual("可发布", payload["publish_status_label"])
self.assertEqual([], payload["blocking_reasons"])
self.assertEqual([], payload["warnings"])
self.assertEqual([], payload["publish_blocking_reasons"])
self.assertEqual([], payload["publish_warnings"])
self.assertEqual("go-live", payload["operator_lane"])
self.assertEqual("run_inspection_participating", payload["operator_primary_command_key"])
self.assertEqual(2, payload["log_sync_covered_nodes"])
self.assertEqual(
"bash domain-api/deploy/multi-region/drive_ops_center.sh driver-resolve http://127.0.0.1:8100 run_inspection_participating",
payload["recommended_commands"]["next_step"],
)
@patch("app.services.ops_service.get_ops_overview")
@patch("app.services.ops_service.list_managed_nodes_with_agent_state")
@patch("app.services.ops_service.get_runtime_build_info")
@patch("app.services.ops_service.get_release_launchpad")
@patch("app.services.ops_service.get_ops_contract_registry")
@patch("app.services.ops_service.get_ops_stack_diagnosis")
def test_go_live_summary_blocks_publish_when_dead_letter_queue_exists(
self,
mock_get_ops_stack_diagnosis,
mock_get_ops_contract_registry,
mock_get_release_launchpad,
mock_get_runtime_build_info,
mock_list_managed_nodes_with_agent_state,
mock_get_ops_overview,
) -> None:
mock_get_ops_stack_diagnosis.return_value = {
"diagnosis": {
"contract_key": "ops_stack_diagnosis_contract",
"stack_status": "healthy",
"next_step": {"action_code": "delivery_queue_replay", "reason": "先清死信。"},
"operator_decision": {},
},
"release_hub": {
"launchpad_status": "ready",
},
}
mock_get_ops_contract_registry.return_value = {
"registry_version": "2026-04-18",
"contracts": [{"key": "ops_stack_diagnosis_contract"}],
}
mock_get_release_launchpad.return_value = {
"launchpad_status": {
"status": "ready",
"status_label": "可发布",
"recommended_action_code": "create_smart_release_rollout_worker",
}
}
mock_get_runtime_build_info.return_value = {
"commit_sha": "246838ae4c07",
"route_surface": {
"surface_complete": True,
"missing_keys": [],
},
}
mock_list_managed_nodes_with_agent_state.return_value = {
"summary": {
"managed_enabled": 2,
"remote_access_ready": 2,
"queue_dead_letter_nodes": 1,
}
}
mock_get_ops_overview.return_value = {
"recommendation": {
"title": "先清死信",
"reason": "当前仍有节点回执死信。",
"primary_action_code": "delivery_queue_replay",
},
"execution_scene": {
"log_sync": {
"enabled": True,
"state": "healthy",
"mode": "key",
"participating_node_count": 2,
"covered_participating_node_count": 2,
"missing_participating_nodes": [],
}
},
}
payload = get_ops_go_live_summary(base_url="http://127.0.0.1:8100")
self.assertEqual("attention", payload["go_live_status"])
self.assertFalse(payload["publish_ready"])
self.assertEqual("blocked", payload["publish_status"])
@patch("app.services.ops_service.get_ops_overview")
@patch("app.services.ops_service.list_managed_nodes_with_agent_state")
@patch("app.services.ops_service.get_runtime_build_info")
@patch("app.services.ops_service.get_release_launchpad")
@patch("app.services.ops_service.get_ops_contract_registry")
@patch("app.services.ops_service.get_ops_stack_diagnosis")
def test_go_live_summary_accepts_log_sync_covered_node_list_and_marks_partial(
self,
mock_get_ops_stack_diagnosis,
mock_get_ops_contract_registry,
mock_get_release_launchpad,
mock_get_runtime_build_info,
mock_list_managed_nodes_with_agent_state,
mock_get_ops_overview,
) -> None:
mock_get_ops_stack_diagnosis.return_value = {
"diagnosis": {
"contract_key": "ops_stack_diagnosis_contract",
"stack_status": "attention",
"next_step": {"action_code": "bootstrap_run", "reason": "继续补大陆接管。"},
"operator_decision": {},
},
"release_hub": {
"launchpad_status": "attention",
},
}
mock_get_ops_contract_registry.return_value = {
"registry_version": "2026-04-18",
"contracts": [{"key": "ops_stack_diagnosis_contract"}],
}
mock_get_release_launchpad.return_value = {
"launchpad_status": {
"status": "attention",
"status_label": "待补执行面",
"recommended_action_code": "fix_managed_nodes",
}
}
mock_get_runtime_build_info.return_value = {
"commit_sha": "246838ae4c07",
"route_surface": {
"surface_complete": True,
"missing_keys": [],
},
}
mock_list_managed_nodes_with_agent_state.return_value = {
"summary": {
"managed_enabled": 3,
"remote_access_ready": 1,
"queue_dead_letter_nodes": 0,
}
}
mock_get_ops_overview.return_value = {
"recommendation": {
"title": "先补大陆接管",
"reason": "当前只有海外样本已形成。",
"primary_action_code": "bootstrap_run",
},
"execution_scene": {
"log_sync": {
"enabled": True,
"state": "partial_coverage",
"mode": "full",
"participating_node_count": 3,
"covered_participating_node_count": 1,
"covered_participating_nodes": ["overseas-control-01"],
"missing_participating_nodes": ["mainland-worker-01", "mainland-controller-01"],
}
},
}
payload = get_ops_go_live_summary(base_url="http://127.0.0.1:8100")
self.assertEqual("attention", payload["go_live_status"])
self.assertEqual(1, payload["log_sync_covered_nodes"])
self.assertIn("log_sync_partial=1/3", payload["warnings"])
self.assertIn("log_sync_partial=1/3", payload["publish_warnings"])
@patch("app.services.ops_service.get_ops_overview")
@patch("app.services.ops_service.list_managed_nodes_with_agent_state")
@patch("app.services.ops_service.get_runtime_build_info")
@patch("app.services.ops_service.get_release_launchpad")
@patch("app.services.ops_service.get_ops_contract_registry")
@patch("app.services.ops_service.get_ops_stack_diagnosis")
def test_go_live_summary_marks_attention_for_start_delivery_gap(
self,
mock_get_ops_stack_diagnosis,
mock_get_ops_contract_registry,
mock_get_release_launchpad,
mock_get_runtime_build_info,
mock_list_managed_nodes_with_agent_state,
mock_get_ops_overview,
) -> None:
mock_get_ops_stack_diagnosis.return_value = {
"diagnosis": {
"contract_key": "ops_stack_diagnosis_contract",
"stack_status": "healthy",
"next_step": {
"action_code": "focus_latest_job_events",
"reason": "先看任务事件。",
},
"operator_decision": {
"lane": "ops_jobs",
"title": "优先检查任务事件流",
"primary_command_key": "focus_latest_job_events",
},
"issues": [
{
"code": "ops_job_start_delivery_failed_local",
"summary": "最近有任务开始回执异常。",
}
],
},
"release_hub": {
"launchpad_status": "ready",
},
}
mock_get_ops_contract_registry.return_value = {
"registry_version": "2026-04-18",
"contracts": [{"key": "ops_stack_diagnosis_contract"}],
}
mock_get_release_launchpad.return_value = {
"launchpad_status": {
"status": "ready",
"status_label": "可发布",
"recommended_action_code": "create_smart_release_rollout_worker",
}
}
mock_get_runtime_build_info.return_value = {
"commit_sha": "246838ae4c07",
"route_surface": {
"surface_complete": True,
"missing_keys": [],
},
}
mock_list_managed_nodes_with_agent_state.return_value = {
"summary": {
"managed_enabled": 1,
"remote_access_ready": 1,
"queue_dead_letter_nodes": 0,
}
}
mock_get_ops_overview.return_value = {
"recommendation": {
"title": "先看任务事件",
"reason": "最近出现开始回执异常。",
"primary_action_code": "focus_latest_job_events",
},
"execution_scene": {
"log_sync": {
"enabled": True,
"state": "healthy",
"mode": "key",
"participating_node_count": 1,
"covered_participating_node_count": 1,
"missing_participating_nodes": [],
}
},
}
payload = get_ops_go_live_summary(base_url="http://127.0.0.1:8100")
self.assertEqual("attention", payload["go_live_status"])
self.assertFalse(payload["publish_ready"])
self.assertEqual("attention", payload["publish_status"])
self.assertEqual(1, payload["activity_start_delivery_issue_total"])
self.assertIn("ops_job_start_delivery_failed_local=1", payload["warnings"])
self.assertIn("ops_job_start_delivery_failed_local=1", payload["publish_warnings"])
self.assertEqual("ops_jobs", payload["operator_lane"])
self.assertEqual("focus_latest_job_events", payload["next_step_action_code"])
if __name__ == "__main__":
unittest.main()