import unittest from unittest.mock import patch from app.services.ops_service import get_ops_stack_diagnosis class OpsServiceStackDiagnosisTests(unittest.TestCase): @patch("app.services.ops_service.get_runtime_build_info") @patch("app.services.ops_service.get_ops_activity_stream") @patch("app.services.ops_service.get_recent_ops_playbook_runs") @patch("app.services.ops_service.get_release_launchpad") @patch("app.services.ops_service.list_managed_nodes_with_agent_state") @patch("app.services.ops_service.get_ops_overview") @patch("app.services.ops_service.get_ops_link_snapshot") @patch("app.services.ops_service.get_ops_contract_registry") def test_stack_diagnosis_prioritizes_api_restart_when_runtime_build_schema_is_stale( self, mock_get_ops_contract_registry, mock_get_ops_link_snapshot, mock_get_ops_overview, mock_list_managed_nodes_with_agent_state, mock_get_release_launchpad, mock_get_recent_ops_playbook_runs, mock_get_ops_activity_stream, mock_get_runtime_build_info, ) -> None: mock_get_ops_contract_registry.return_value = { "registry_version": "2026-04-18", "schema_version": "v1", "contracts": [{"key": "ops_stack_diagnosis_contract"}], } mock_get_ops_link_snapshot.return_value = { "status": "ready", "headline": "当前总检可用。", "next_actions": [], } mock_get_ops_overview.return_value = { "recommendation": { "primary_action_code": "fix_managed_nodes", "title": "先补节点接管", "focus_ref": {"kind": "managed_node", "node_code": "mainland-worker-01"}, }, "execution_scene": {}, "log_sync": {}, "release_hub": {}, } mock_list_managed_nodes_with_agent_state.return_value = {"nodes": [], "summary": {}} mock_get_release_launchpad.return_value = {"launchpad_status": {}, "latest_release": {}} mock_get_recent_ops_playbook_runs.return_value = {"runs": []} mock_get_ops_activity_stream.return_value = {"items": []} mock_get_runtime_build_info.return_value = { "source": "git", "commit_sha": "246838ae4c07", "repository_capabilities": { "supports_install_command_block": True, "supports_multi_layout_bootstrap": True, }, "route_surface": { "mode": "registered", "registered_paths_total": 24, "surface_complete": True, "missing_keys": [], "expected_paths": { "ops_contracts": "/api/v1/ops/contracts", "ops_stack_diagnosis": "/api/v1/ops/stack-diagnosis", }, }, } payload = get_ops_stack_diagnosis(base_url="http://127.0.0.1:8100") issue = next(item for item in payload["diagnosis"]["issues"] if item["code"] == "runtime_build_schema_stale") self.assertEqual("api-restart", issue["action_code"]) self.assertEqual("api-restart", payload["diagnosis"]["next_step"]["action_code"]) self.assertEqual("runtime_recovery", payload["diagnosis"]["operator_decision"]["lane"]) self.assertEqual("api-restart", payload["diagnosis"]["operator_decision"]["primary_command_key"]) self.assertEqual( "bash domain-api/deploy/multi-region/drive_ops_center.sh runtime-refresh-recover http://127.0.0.1:8100", payload["diagnosis"]["next_actions"]["primary_command"], ) self.assertEqual( "bash domain-api/deploy/multi-region/drive_ops_center.sh runtime-refresh-recover http://127.0.0.1:8100", payload["diagnosis"]["recommended_commands"]["api-restart"], ) self.assertEqual( "bash domain-api/deploy/multi-region/drive_ops_center.sh runtime-refresh-recover http://127.0.0.1:8100", payload["diagnosis"]["recommended_commands"]["runtime-refresh-recover"], ) self.assertEqual( "bash domain-api/deploy/multi-region/drive_ops_center.sh stack-diagnosis http://127.0.0.1:8100 summary", payload["diagnosis"]["recommended_commands"]["stack-diagnosis"], ) self.assertEqual( payload["diagnosis"]["recommended_commands"]["stack-diagnosis"], payload["diagnosis"]["recommended_commands"]["stack-check"], ) self.assertTrue(payload["runtime_build_info"]["runtime_schema_stale"]) self.assertFalse(payload["runtime_build_info"]["route_surface_declares_bootstrap_plan"]) self.assertTrue( any("运行中 API 还是旧 schema" in hint for hint in payload["diagnosis"]["operator_hints"]) ) @patch("app.services.ops_service.get_runtime_build_info") @patch("app.services.ops_service.get_ops_activity_stream") @patch("app.services.ops_service.get_recent_ops_playbook_runs") @patch("app.services.ops_service.get_release_launchpad") @patch("app.services.ops_service.list_managed_nodes_with_agent_state") @patch("app.services.ops_service.get_ops_overview") @patch("app.services.ops_service.get_ops_link_snapshot") @patch("app.services.ops_service.get_ops_contract_registry") def test_stack_diagnosis_surfaces_runtime_route_surface_gap( self, mock_get_ops_contract_registry, mock_get_ops_link_snapshot, mock_get_ops_overview, mock_list_managed_nodes_with_agent_state, mock_get_release_launchpad, mock_get_recent_ops_playbook_runs, mock_get_ops_activity_stream, mock_get_runtime_build_info, ) -> None: mock_get_ops_contract_registry.return_value = { "registry_version": "2026-04-18", "schema_version": "v1", "contracts": [], } mock_get_ops_link_snapshot.return_value = { "status": "ready", "headline": "当前总检可用。", "next_actions": [], } mock_get_ops_overview.return_value = { "recommendation": {}, "execution_scene": {}, "log_sync": {}, "release_hub": {}, } mock_list_managed_nodes_with_agent_state.return_value = {"nodes": [], "summary": {}} mock_get_release_launchpad.return_value = {"launchpad_status": {}, "latest_release": {}} mock_get_recent_ops_playbook_runs.return_value = {"runs": []} mock_get_ops_activity_stream.return_value = {"items": []} mock_get_runtime_build_info.return_value = { "source": "git", "commit_sha": "bd6bcb2", "route_surface": { "mode": "registered", "registered_paths_total": 12, "surface_complete": False, "missing_keys": [ "ops_node_onboarding_acceptance_execute", "ops_node_onboarding_acceptance_preview", ], "missing_paths": [ "/api/v1/ops/nodes/{node_code}/onboarding/acceptance/execute", "/api/v1/ops/nodes/{node_code}/onboarding/acceptance/preview", ], }, } payload = get_ops_stack_diagnosis(base_url="http://127.0.0.1:8100") issue = next( item for item in payload["diagnosis"]["issues"] if item["code"] == "runtime_route_surface_incomplete" ) self.assertEqual("api-restart", issue["action_code"]) self.assertIn("ops_node_onboarding_acceptance_execute", issue["detail"]) self.assertIn( "bash domain-api/deploy/multi-region/drive_ops_center.sh runtime-refresh-recover http://127.0.0.1:8100", issue["commands"], ) self.assertFalse(payload["runtime_build_info"]["route_surface"]["surface_complete"]) self.assertIn( "ops_node_onboarding_acceptance_preview", payload["runtime_build_info"]["route_surface_missing_keys"], ) self.assertTrue( any("API 路由面还不完整" in hint for hint in payload["diagnosis"]["operator_hints"]) ) @patch("app.services.ops_service.get_ops_activity_stream") @patch("app.services.ops_service.get_recent_ops_playbook_runs") @patch("app.services.ops_service.get_release_launchpad") @patch("app.services.ops_service.list_managed_nodes_with_agent_state") @patch("app.services.ops_service.get_ops_overview") @patch("app.services.ops_service.get_ops_link_snapshot") @patch("app.services.ops_service.get_ops_contract_registry") def test_stack_diagnosis_emits_operator_decision_for_scene_log_gap( self, mock_get_ops_contract_registry, mock_get_ops_link_snapshot, mock_get_ops_overview, mock_list_managed_nodes_with_agent_state, mock_get_release_launchpad, mock_get_recent_ops_playbook_runs, mock_get_ops_activity_stream, ) -> None: mock_get_ops_contract_registry.return_value = { "registry_version": "2026-04-18", "schema_version": "v1", "contracts": [], } mock_get_ops_link_snapshot.return_value = { "status": "attention", "headline": "参与节点样本尚未形成。", "next_actions": [], } mock_get_ops_overview.return_value = { "recommendation": {}, "execution_scene": { "participating_nodes": [{"node_code": "mainland-worker-01", "status": "busy"}], "participation_summary": { "dispatch_active_node_codes": ["mainland-worker-01"], "recent_only_node_codes": [], }, "log_sync": { "enabled": True, "mode": "key", "line_count": 0, "source_node_count": 0, "covered_participating_node_count": 0, "participating_node_count": 1, "missing_participating_nodes": ["mainland-worker-01"], }, }, "log_sync": {}, "release_hub": {}, } mock_list_managed_nodes_with_agent_state.return_value = {"nodes": [], "summary": {}} mock_get_release_launchpad.return_value = {"launchpad_status": {}, "latest_release": {}} mock_get_recent_ops_playbook_runs.return_value = {"runs": []} mock_get_ops_activity_stream.return_value = {"items": []} payload = get_ops_stack_diagnosis(base_url="http://127.0.0.1:8100") decision = payload["diagnosis"]["operator_decision"] self.assertEqual("observability", decision["lane"]) self.assertEqual("scene-node-log", decision["primary_command_key"]) self.assertEqual("mainland-worker-01", decision["next_focus"]["node_code"]) self.assertIn("scene-node-log", payload["diagnosis"]["recommended_commands"]) self.assertIn( "scene-node-log http://127.0.0.1:8100 mainland-worker-01 120 key", payload["diagnosis"]["next_actions"]["primary_command"], ) @patch("app.services.ops_service.get_ops_activity_stream") @patch("app.services.ops_service.get_recent_ops_playbook_runs") @patch("app.services.ops_service.get_release_launchpad") @patch("app.services.ops_service.get_managed_node_handover") @patch("app.services.ops_service.list_managed_nodes_with_agent_state") @patch("app.services.ops_service.get_ops_overview") @patch("app.services.ops_service.get_ops_link_snapshot") @patch("app.services.ops_service.get_ops_contract_registry") def test_stack_diagnosis_emits_operator_decision_for_managed_node_gap( self, mock_get_ops_contract_registry, mock_get_ops_link_snapshot, mock_get_ops_overview, mock_list_managed_nodes_with_agent_state, mock_get_managed_node_handover, mock_get_release_launchpad, mock_get_recent_ops_playbook_runs, mock_get_ops_activity_stream, ) -> None: mock_get_ops_contract_registry.return_value = { "registry_version": "2026-04-18", "schema_version": "v1", "contracts": [], } mock_get_ops_link_snapshot.return_value = { "status": "attention", "headline": "存在接管缺口。", "next_actions": [], } mock_get_ops_overview.return_value = { "recommendation": {}, "execution_scene": {}, "log_sync": {}, "release_hub": {}, } mock_list_managed_nodes_with_agent_state.return_value = { "nodes": [ { "node_code": "mainland-worker-01", "agent_state": "pending_bootstrap", "remote_access_state": "ssh_ready", "delivery_queue_state": "healthy", "participation_state": "standby", } ], "summary": { "managed_enabled": 1, "remote_access_ready": 0, }, } mock_get_managed_node_handover.return_value = { "node_code": "mainland-worker-01", "summary": "等待执行 bootstrap。", "stage": {"code": "pending_bootstrap", "label": "待执行接入", "next_step": {"code": "execute_bootstrap_plan"}}, } mock_get_release_launchpad.return_value = { "launchpad_status": { "recommended_action_code": "bootstrap_run", "focus_ref": {"kind": "release_launchpad", "section": "release_launchpad"}, "recommended_target_node_code": "mainland-worker-01", }, "worker_rollout_preview": { "release_gate": { "operational_readiness": { "rows": [ { "node_code": "mainland-worker-01", "recovery_action": "bootstrap_run", "recovery_summary": "mainland-worker-01 还缺接入收口，先执行 bootstrap。", "focus_ref": {"kind": "managed_node", "node_code": "mainland-worker-01"}, } ] } } }, "latest_release": {}, } mock_get_recent_ops_playbook_runs.return_value = {"runs": []} mock_get_ops_activity_stream.return_value = {"items": []} payload = get_ops_stack_diagnosis(base_url="http://127.0.0.1:8100") decision = payload["diagnosis"]["operator_decision"] self.assertEqual("node_handover", decision["lane"]) self.assertEqual("bootstrap_run", decision["primary_command_key"]) self.assertEqual("managed-node-bootstrap-plan", decision["secondary_command_key"]) self.assertIn("bootstrap_run", payload["diagnosis"]["recommended_commands"]) self.assertIn("managed-node-handover", payload["diagnosis"]["recommended_commands"]) self.assertIn("managed-node-bootstrap-plan", payload["diagnosis"]["recommended_commands"]) self.assertIn( "node-bootstrap-run http://127.0.0.1:8100 mainland-worker-01 cli", payload["diagnosis"]["next_actions"]["primary_command"], ) self.assertTrue( any("当前第一动作已经收敛为 接入收口：mainland-worker-01" in hint for hint in payload["diagnosis"]["operator_hints"]) ) @patch("app.services.ops_service.get_runtime_build_info") @patch("app.services.ops_service.get_ops_activity_stream") @patch("app.services.ops_service.get_recent_ops_playbook_runs") @patch("app.services.ops_service.get_release_launchpad") @patch("app.services.ops_service.get_managed_node_handover") @patch("app.services.ops_service.list_managed_nodes_with_agent_state") @patch("app.services.ops_service.get_ops_overview") @patch("app.services.ops_service.get_ops_link_snapshot") @patch("app.services.ops_service.get_ops_contract_registry") def test_stack_diagnosis_promotes_first_handover_gap_when_launchpad_is_still_generic( self, mock_get_ops_contract_registry, mock_get_ops_link_snapshot, mock_get_ops_overview, mock_list_managed_nodes_with_agent_state, mock_get_managed_node_handover, mock_get_release_launchpad, mock_get_recent_ops_playbook_runs, mock_get_ops_activity_stream, mock_get_runtime_build_info, ) -> None: mock_get_ops_contract_registry.return_value = { "registry_version": "2026-04-18", "schema_version": "v1", "contracts": [{"key": "ops_stack_diagnosis_contract"}], } mock_get_ops_link_snapshot.return_value = { "status": "ready", "headline": "当前总检可用。", "next_actions": [], } mock_get_ops_overview.return_value = { "recommendation": { "primary_action_code": "handover_first_gap", "title": "先补接管缺口", "reason": "overview 只能先定位到接管缺口。", "focus_ref": {"kind": "managed_node", "node_code": "mainland-worker-01"}, }, "execution_scene": {}, "log_sync": {}, "release_hub": {}, } mock_list_managed_nodes_with_agent_state.return_value = { "nodes": [ { "node_code": "mainland-worker-01", "agent_state": "pending_bootstrap", "remote_access_state": "agent_pending", "delivery_queue_state": "healthy", "participation_state": "standby", } ], "summary": { "managed_enabled": 1, "remote_access_ready": 0, }, } mock_get_managed_node_handover.return_value = { "node_code": "mainland-worker-01", "summary": "当前节点仍处于接入阶段，建议先签发 onboarding.bootstrap 并等待 register / heartbeat 建立。", "stage": { "code": "pending_bootstrap", "label": "待执行接入", "next_step": { "code": "execute_bootstrap_plan", "label": "去目标机执行接入片段", }, }, } mock_get_release_launchpad.return_value = { "launchpad_status": { "recommended_action_code": "fix_managed_nodes", "status_label": "待补执行面", "focus_ref": {"kind": "release_launchpad", "section": "release_launchpad"}, }, "latest_release": {}, } mock_get_recent_ops_playbook_runs.return_value = {"runs": []} mock_get_ops_activity_stream.return_value = {"items": []} mock_get_runtime_build_info.return_value = { "route_surface": {"surface_complete": True, "missing_keys": []}, "repository_capabilities": {}, } payload = get_ops_stack_diagnosis(base_url="http://127.0.0.1:8100") self.assertEqual("bootstrap_run", payload["diagnosis"]["next_step"]["action_code"]) self.assertEqual( "mainland-worker-01", payload["diagnosis"]["launchpad_recommended_target_node_code"], ) self.assertEqual( "签发接入工单", payload["diagnosis"]["launchpad_recommended_recovery_label"], ) self.assertIn( "onboarding.bootstrap", payload["diagnosis"]["launchpad_recommended_recovery_summary"], ) self.assertEqual("bootstrap_run", payload["diagnosis"]["operator_decision"]["primary_command_key"]) self.assertEqual( "bash domain-api/deploy/multi-region/drive_ops_center.sh driver-resolve http://127.0.0.1:8100 bootstrap_run", payload["diagnosis"]["next_actions"]["primary_command"], ) @patch("app.services.ops_service.get_ops_activity_stream") @patch("app.services.ops_service.get_recent_ops_playbook_runs") @patch("app.services.ops_service.get_release_launchpad") @patch("app.services.ops_service.list_managed_nodes_with_agent_state") @patch("app.services.ops_service.get_ops_overview") @patch("app.services.ops_service.get_ops_link_snapshot") @patch("app.services.ops_service.get_ops_contract_registry") def test_stack_diagnosis_emits_operator_decision_for_release_launchpad_attention( self, mock_get_ops_contract_registry, mock_get_ops_link_snapshot, mock_get_ops_overview, mock_list_managed_nodes_with_agent_state, mock_get_release_launchpad, mock_get_recent_ops_playbook_runs, mock_get_ops_activity_stream, ) -> None: mock_get_ops_contract_registry.return_value = { "registry_version": "2026-04-18", "schema_version": "v1", "contracts": [], } mock_get_ops_link_snapshot.return_value = { "status": "attention", "headline": "发布门禁待处理。", "next_actions": [], } mock_get_ops_overview.return_value = { "recommendation": {}, "execution_scene": {}, "log_sync": {}, "release_hub": { "default_rollout_gate": { "status": "blocked", "summary_text": "还有节点未满足发布门禁。", "focus_ref": {"kind": "release_hub", "section": "release_launchpad", "mode": "worker"}, } }, } mock_list_managed_nodes_with_agent_state.return_value = {"nodes": [], "summary": {}} mock_get_release_launchpad.return_value = { "launchpad_status": { "status": "attention", "status_label": "待处理", "recommended_action_code": "review_smart_rollout_preview", "focus_ref": {"kind": "release_hub", "section": "release_launchpad", "mode": "worker"}, }, "latest_release": {"release_version": "release-20260418"}, } mock_get_recent_ops_playbook_runs.return_value = {"runs": []} mock_get_ops_activity_stream.return_value = {"items": []} payload = get_ops_stack_diagnosis(base_url="http://127.0.0.1:8100") decision = payload["diagnosis"]["operator_decision"] self.assertEqual("release", decision["lane"]) self.assertEqual("release-launchpad-worker", decision["primary_command_key"]) self.assertEqual("review_smart_rollout_preview", decision["secondary_command_key"]) self.assertIn("release-launchpad-worker", payload["diagnosis"]["recommended_commands"]) self.assertIn( "release-launchpad http://127.0.0.1:8100 worker", payload["diagnosis"]["next_actions"]["primary_command"], ) @patch("app.services.ops_service.get_ops_activity_stream") @patch("app.services.ops_service.get_recent_ops_playbook_runs") @patch("app.services.ops_service.get_release_launchpad") @patch("app.services.ops_service.list_managed_nodes_with_agent_state") @patch("app.services.ops_service.get_ops_overview") @patch("app.services.ops_service.get_ops_link_snapshot") @patch("app.services.ops_service.get_ops_contract_registry") def test_stack_diagnosis_prefers_node_recovery_when_launchpad_points_to_bootstrap_gap( self, mock_get_ops_contract_registry, mock_get_ops_link_snapshot, mock_get_ops_overview, mock_list_managed_nodes_with_agent_state, mock_get_release_launchpad, mock_get_recent_ops_playbook_runs, mock_get_ops_activity_stream, ) -> None: mock_get_ops_contract_registry.return_value = { "registry_version": "2026-04-18", "schema_version": "v1", "contracts": [], } mock_get_ops_link_snapshot.return_value = { "status": "attention", "headline": "发布门禁待处理。", "next_actions": [], } mock_get_ops_overview.return_value = { "recommendation": {}, "execution_scene": {}, "log_sync": {}, "release_hub": { "default_rollout_gate": { "status": "attention", "summary_text": "当前还有节点未完成接入。", "focus_ref": {"kind": "release_hub", "section": "release_launchpad", "mode": "worker"}, } }, } mock_list_managed_nodes_with_agent_state.return_value = {"nodes": [], "summary": {}} mock_get_release_launchpad.return_value = { "launchpad_status": { "status": "attention", "status_label": "待补执行面", "recommended_action_code": "bootstrap_run", "recommended_target_node_code": "mainland-worker-02", "recommended_recovery_label": "待执行接入", "recommended_recovery_summary": "当前节点仍处于接入阶段，建议先签发 onboarding.bootstrap。", "onboarding_bootstrap_pending_nodes": 1, "onboarding_acceptance_ready_nodes": 0, "focus_ref": {"kind": "release_hub", "section": "release_launchpad", "mode": "worker"}, }, "latest_release": {"release_version": "release-20260418"}, } mock_get_recent_ops_playbook_runs.return_value = {"runs": []} mock_get_ops_activity_stream.return_value = {"items": []} payload = get_ops_stack_diagnosis(base_url="http://127.0.0.1:8100") decision = payload["diagnosis"]["operator_decision"] self.assertEqual("node_handover", decision["lane"]) self.assertEqual("bootstrap_run", decision["primary_command_key"]) self.assertEqual("focus-release-hub", decision["secondary_command_key"]) self.assertEqual("mainland-worker-02", payload["diagnosis"]["launchpad_recommended_target_node_code"]) self.assertEqual("待执行接入", payload["diagnosis"]["launchpad_recommended_recovery_label"]) self.assertEqual(1, payload["diagnosis"]["launchpad_onboarding_bootstrap_pending_nodes"]) self.assertEqual(0, payload["diagnosis"]["launchpad_onboarding_acceptance_ready_nodes"]) self.assertEqual( "当前节点仍处于接入阶段，建议先签发 onboarding.bootstrap。", decision["summary"], ) self.assertEqual("mainland-worker-02", decision["next_focus"]["node_code"]) self.assertIn("node-bootstrap-run", payload["diagnosis"]["next_actions"]["primary_command"]) @patch("app.services.ops_service.get_ops_activity_stream") @patch("app.services.ops_service.get_recent_ops_playbook_runs") @patch("app.services.ops_service.get_release_launchpad") @patch("app.services.ops_service.get_managed_node_handover") @patch("app.services.ops_service.list_managed_nodes_with_agent_state") @patch("app.services.ops_service.get_ops_overview") @patch("app.services.ops_service.get_ops_link_snapshot") @patch("app.services.ops_service.get_ops_contract_registry") def test_stack_diagnosis_includes_first_handover_gap_details( self, mock_get_ops_contract_registry, mock_get_ops_link_snapshot, mock_get_ops_overview, mock_list_managed_nodes_with_agent_state, mock_get_managed_node_handover, mock_get_release_launchpad, mock_get_recent_ops_playbook_runs, mock_get_ops_activity_stream, ) -> None: mock_get_ops_contract_registry.return_value = { "registry_version": "2026-04-18", "schema_version": "v1", "contracts": [], } mock_get_ops_link_snapshot.return_value = { "status": "attention", "headline": "存在接管缺口。", "next_actions": [], } mock_get_ops_overview.return_value = { "recommendation": {}, "execution_scene": {}, "log_sync": {}, "release_hub": {}, } mock_list_managed_nodes_with_agent_state.return_value = { "nodes": [ { "node_code": "mainland-worker-01", "agent_state": "pending_bootstrap", "remote_access_state": "ssh_ready", "delivery_queue_state": "healthy", "participation_state": "standby", } ], "summary": { "managed_enabled": 1, "remote_access_ready": 0, "status_counts": {"pending_bootstrap": 1}, "remote_access_state_counts": {"ssh_ready": 1}, "delivery_queue_state_counts": {"healthy": 1}, }, } mock_get_managed_node_handover.return_value = { "node_code": "mainland-worker-01", "summary": "已签发有效 Token，但目标节点尚未执行 bootstrap 片段。", "stage": { "code": "pending_bootstrap", "label": "待执行接入", "next_step": { "code": "execute_bootstrap_plan", "label": "去目标机执行接入片段", }, }, } mock_get_release_launchpad.return_value = { "launchpad_status": { "recommended_action_code": "fix_managed_nodes", "focus_ref": {"kind": "release_launchpad"}, }, "latest_release": {}, } mock_get_recent_ops_playbook_runs.return_value = {"runs": []} mock_get_ops_activity_stream.return_value = {"items": []} payload = get_ops_stack_diagnosis(base_url="http://127.0.0.1:8100") self.assertEqual( "mainland-worker-01", payload["managed_nodes"]["first_handover_gap_node_code"], ) self.assertEqual( "pending_bootstrap", payload["managed_nodes"]["first_handover_gap"]["stage"]["code"], ) issue = next( item for item in payload["diagnosis"]["issues"] if item["code"] == "managed_nodes_agent_pending" ) self.assertIn("mainland-worker-01", issue["detail"]) self.assertTrue( any("node-handover http://127.0.0.1:8100 mainland-worker-01" in command for command in issue["commands"]) ) self.assertIn( "bash domain-api/deploy/multi-region/drive_ops_center.sh node-recover http://127.0.0.1:8100 mainland-worker-01", payload["diagnosis"]["recommended_commands"]["managed-node-recover"], ) self.assertTrue( any("node-bootstrap-plan http://127.0.0.1:8100 mainland-worker-01" in command for command in issue["commands"]) ) mock_get_managed_node_handover.assert_called_once() @patch("app.services.ops_service.get_ops_activity_stream") @patch("app.services.ops_service.get_recent_ops_playbook_runs") @patch("app.services.ops_service.get_release_launchpad") @patch("app.services.ops_service.list_managed_nodes_with_agent_state") @patch("app.services.ops_service.get_ops_overview") @patch("app.services.ops_service.get_ops_link_snapshot") @patch("app.services.ops_service.get_ops_contract_registry") def test_stack_diagnosis_keeps_focus_refs_for_problem_runs_and_top_items( self, mock_get_ops_contract_registry, mock_get_ops_link_snapshot, mock_get_ops_overview, mock_list_managed_nodes_with_agent_state, mock_get_release_launchpad, mock_get_recent_ops_playbook_runs, mock_get_ops_activity_stream, ) -> None: mock_get_ops_contract_registry.return_value = { "registry_version": "2026-04-18", "schema_version": "v1", "contracts": [], } mock_get_ops_link_snapshot.return_value = { "status": "ready", "headline": "当前总检可用。", "next_actions": [], } mock_get_ops_overview.return_value = { "recommendation": {}, "execution_scene": {}, "log_sync": {}, "release_hub": {}, } mock_list_managed_nodes_with_agent_state.return_value = { "nodes": [], "summary": {}, } mock_get_release_launchpad.return_value = { "launchpad_status": {}, "latest_release": {}, } mock_get_recent_ops_playbook_runs.return_value = { "runs": [ { "run_code": "pbr-worker-logs-001", "status": "attention", "status_label": "待关注", "focus_step_key": "worker_logs", "focus_summary": "Worker 日志步骤仍未收口。", "focus_ref": { "kind": "playbook_run", "run_code": "pbr-worker-logs-001", "focus_step_key": "worker_logs", }, } ] } mock_get_ops_activity_stream.return_value = { "items": [ { "kind": "ops_job", "status": "running", "summary_text": "远端 Agent 已回传关键日志样本。", "occurred_at": "2026-04-18 09:22:00", "focus_ref": { "kind": "ops_job", "job_id": 71, "job_code": "ops-agent-71", }, "source_focus_ref": { "kind": "ops_job_event", "job_id": 71, "event_key": "job-event:71", }, "ui_intent": { "kind": "job_events", "job_id": 71, }, } ] } payload = get_ops_stack_diagnosis(base_url="http://127.0.0.1:8100") self.assertEqual( "ops_stack_diagnosis_contract", payload["diagnosis"]["contract_navigation"]["primary_contract_key"], ) self.assertIn("ops_driver_contract", payload["diagnosis"]["contract_navigation"]["contract_keys"]) self.assertEqual("playbook_run", payload["playbook_runs"]["problem_runs"][0]["focus_ref"]["kind"]) self.assertEqual("worker_logs", payload["playbook_runs"]["problem_runs"][0]["focus_ref"]["focus_step_key"]) top_item = payload["activity_stream"]["top_items"][0] self.assertEqual("ops_job", top_item["focus_ref"]["kind"]) self.assertEqual("ops_job_event", top_item["source_focus_ref"]["kind"]) self.assertEqual("job_events", top_item["ui_intent_kind"]) self.assertEqual("2026-04-18 09:22:00", top_item["occurred_at"]) @patch("app.services.ops_service.get_ops_activity_stream") @patch("app.services.ops_service.get_recent_ops_playbook_runs") @patch("app.services.ops_service.get_release_launchpad") @patch("app.services.ops_service.list_managed_nodes_with_agent_state") @patch("app.services.ops_service.get_ops_overview") @patch("app.services.ops_service.get_ops_link_snapshot") @patch("app.services.ops_service.get_ops_contract_registry") def test_stack_diagnosis_adds_node_scene_log_commands_for_log_sample_gap( self, mock_get_ops_contract_registry, mock_get_ops_link_snapshot, mock_get_ops_overview, mock_list_managed_nodes_with_agent_state, mock_get_release_launchpad, mock_get_recent_ops_playbook_runs, mock_get_ops_activity_stream, ) -> None: mock_get_ops_contract_registry.return_value = { "registry_version": "2026-04-18", "schema_version": "v1", "contracts": [], } mock_get_ops_link_snapshot.return_value = { "status": "attention", "headline": "参与节点样本尚未形成。", "next_actions": [ { "label": "看参与节点日志", "action_code": "open_worker_logs_participating", "node_codes": ["mainland-worker-01"], "payload": {}, } ], } mock_get_ops_overview.return_value = { "recommendation": {}, "execution_scene": { "participating_nodes": [ { "node_code": "mainland-worker-01", "status": "busy", } ], "participation_summary": { "dispatch_active_node_codes": ["mainland-worker-01"], "recent_only_node_codes": [], }, "log_sync": { "enabled": True, "mode": "key", "line_count": 0, "source_node_count": 0, "covered_participating_node_count": 0, "participating_node_count": 1, "missing_participating_nodes": ["mainland-worker-01"], }, }, "log_sync": {}, "release_hub": {}, } mock_list_managed_nodes_with_agent_state.return_value = { "nodes": [], "summary": {}, } mock_get_release_launchpad.return_value = { "launchpad_status": {}, "latest_release": {}, } mock_get_recent_ops_playbook_runs.return_value = {"runs": []} mock_get_ops_activity_stream.return_value = {"items": []} payload = get_ops_stack_diagnosis(base_url="http://127.0.0.1:8100") issue = next( item for item in payload["diagnosis"]["issues"] if item["code"] == "remote_log_sync_waiting_sample" ) self.assertEqual("open_worker_logs_participating", issue["action_code"]) self.assertEqual("node_scene_log", issue["focus_ref"]["kind"]) self.assertEqual("mainland-worker-01", issue["focus_ref"]["node_code"]) self.assertTrue( any( "scene-node-log http://127.0.0.1:8100 mainland-worker-01 120 key" in command for command in issue["commands"] ) ) self.assertEqual("node_scene_log", payload["diagnosis"]["next_step"]["focus_ref"]["kind"]) self.assertTrue( any( "scene-node-log http://127.0.0.1:8100 mainland-worker-01 120 key" in command for command in payload["diagnosis"]["quick_commands"] ) ) self.assertTrue( any( "driver-resolve http://127.0.0.1:8100 open_worker_logs_participating" in command for command in issue["commands"] ) ) @patch("app.services.ops_service.get_ops_activity_stream") @patch("app.services.ops_service.get_recent_ops_playbook_runs") @patch("app.services.ops_service.get_release_launchpad") @patch("app.services.ops_service.list_managed_nodes_with_agent_state") @patch("app.services.ops_service.get_ops_overview") @patch("app.services.ops_service.get_ops_link_snapshot") @patch("app.services.ops_service.get_ops_contract_registry") def test_stack_diagnosis_surfaces_start_delivery_gap_from_activity_stream( self, mock_get_ops_contract_registry, mock_get_ops_link_snapshot, mock_get_ops_overview, mock_list_managed_nodes_with_agent_state, mock_get_release_launchpad, mock_get_recent_ops_playbook_runs, mock_get_ops_activity_stream, ) -> None: mock_get_ops_contract_registry.return_value = { "registry_version": "2026-04-18", "schema_version": "v1", "contracts": [], } mock_get_ops_link_snapshot.return_value = { "status": "attention", "headline": "最近有任务回执异常。", "next_actions": [], } mock_get_ops_overview.return_value = { "recommendation": {}, "execution_scene": {}, "log_sync": {}, "release_hub": {}, } mock_list_managed_nodes_with_agent_state.return_value = { "nodes": [], "summary": {}, } mock_get_release_launchpad.return_value = { "launchpad_status": {}, "latest_release": {}, } mock_get_recent_ops_playbook_runs.return_value = {"runs": []} mock_get_ops_activity_stream.return_value = { "items": [ { "kind": "ops_job", "status": "attention", "job_status": "running", "summary_text": "节点已接单执行，但开始回执未成功送达控制面。", "occurred_at": "2026-04-18 10:22:00", "job_code": "ops-agent-81", "start_delivery_state": "failed_local", "start_delivery_error": "temporary offline", "focus_ref": { "kind": "ops_job", "job_id": 81, "job_code": "ops-agent-81", "target_node_code": "mainland-worker-01", }, "source_focus_ref": { "kind": "ops_job_event", "job_id": 81, "job_code": "ops-agent-81", "event_key": "job-event:81", "target_node_code": "mainland-worker-01", }, "ui_intent": { "kind": "job_events", "job_id": 81, }, } ] } payload = get_ops_stack_diagnosis(base_url="http://127.0.0.1:8100") issue = next( item for item in payload["diagnosis"]["issues"] if item["code"] == "ops_job_start_delivery_failed_local" ) self.assertEqual("focus_latest_job_events", issue["action_code"]) self.assertEqual("ops_job_event", issue["focus_ref"]["kind"]) self.assertIn("temporary offline", issue["detail"]) decision = payload["diagnosis"]["operator_decision"] self.assertEqual("ops_jobs", decision["lane"]) self.assertEqual("focus_latest_job_events", decision["primary_command_key"]) self.assertIn("driver-resolve http://127.0.0.1:8100 focus_latest_job_events", payload["diagnosis"]["next_actions"]["primary_command"]) if __name__ == "__main__": unittest.main()