import unittest from unittest.mock import patch from app.services.ops_service import get_ops_inspection_overview, get_ops_overview class OpsServiceOverviewTests(unittest.TestCase): @patch("app.services.ops_service.list_managed_nodes_with_agent_state") @patch("app.services.ops_service.list_ops_jobs") def test_inspection_overview_fetches_managed_nodes_when_omitted( self, mock_list_ops_jobs, mock_list_managed_nodes_with_agent_state, ) -> None: mock_list_managed_nodes_with_agent_state.return_value = { "nodes": [ { "node_code": "mainland-worker-01", "region": "mainland", "role": "worker", "is_managed": True, "is_enabled": True, "is_agent_online": True, "cluster_status": "online", "cluster_is_effective_worker": True, "cluster_detect_participating": True, } ] } mock_list_ops_jobs.return_value = [ { "id": 21, "job_code": "ops-health-21", "action": "health.snapshot", "status": "success", "target_node_code": "mainland-worker-01", "created_at": "2026-04-17 10:00:00", "updated_at": "2026-04-17 10:01:00", } ] overview = get_ops_inspection_overview(fetch_limit=20) self.assertEqual(1, overview["eligible_nodes_total"]) self.assertEqual(1, overview["rows_total"]) self.assertEqual("attention", overview["status"]) self.assertEqual("待处理", overview["status_label"]) self.assertEqual("mainland", overview["rows"][0]["region"]) self.assertEqual("worker", overview["rows"][0]["role"]) self.assertEqual("online", overview["rows"][0]["cluster_status"]) self.assertIn("latest_health_snapshot", overview["rows"][0]) self.assertIn("latest_worker_logs", overview["rows"][0]) self.assertIn("latest_diagnostics", overview["rows"][0]) @patch("app.services.ops_service.list_ops_jobs") def test_inspection_overview_prioritizes_attention_and_supports_filters(self, mock_list_ops_jobs) -> None: mock_list_ops_jobs.return_value = [ { "id": 11, "job_code": "ops-health-1", "action": "health.snapshot", "status": "success", "target_node_code": "mainland-worker-01", "created_at": "2026-04-17 10:00:00", "updated_at": "2026-04-17 10:01:00", }, { "id": 12, "job_code": "ops-logs-1", "action": "logs.collect", "status": "failed", "target_node_code": "mainland-worker-01", "created_at": "2026-04-17 10:02:00", "updated_at": "2026-04-17 10:03:00", "error_message": "journalctl permission denied", "payload": { "service_name": "domaincheck-worker", "lines": 120, }, }, ] managed_nodes = [ { "node_code": "mainland-worker-01", "region": "mainland", "role": "worker", "is_managed": True, "is_enabled": True, "is_agent_online": True, "cluster_status": "online", "cluster_is_effective_worker": True, "cluster_detect_participating": True, }, { "node_code": "mainland-controller-01", "region": "mainland", "role": "control", "is_managed": True, "is_enabled": True, "is_agent_online": True, "cluster_status": "online", "cluster_is_effective_worker": True, "cluster_detect_participating": False, }, ] overview = get_ops_inspection_overview(managed_nodes=managed_nodes, fetch_limit=20) self.assertEqual(2, overview["rows_total"]) self.assertEqual(1, overview["summary"]["attention"]) self.assertEqual(1, overview["summary"]["missing"]) self.assertEqual("attention", overview["status"]) self.assertEqual("待处理", overview["status_label"]) self.assertEqual(1, overview["counts"]["attention_nodes"]) self.assertEqual(1, overview["counts"]["missing_nodes"]) self.assertEqual("mainland-worker-01", overview["priority_queue"][0]["node_code"]) self.assertEqual("attention", overview["priority_queue"][0]["overall_status"]) self.assertEqual("terminal_failure", overview["priority_queue"][0]["problem_kind"]) self.assertEqual("failed", overview["rows"][0]["latest_worker_logs"]["status"]) self.assertEqual("成功", overview["rows"][0]["latest_health_snapshot"]["status_label"]) filtered = get_ops_inspection_overview( managed_nodes=managed_nodes, fetch_limit=20, only_problem=True, only_participating=True, ) self.assertEqual(1, filtered["filtered_rows_total"]) self.assertEqual("mainland-worker-01", filtered["rows"][0]["node_code"]) self.assertEqual("attention", filtered["rows"][0]["overall_status"]) @patch("app.services.ops_service.list_ops_jobs") def test_inspection_overview_includes_visible_handover_gap_nodes(self, mock_list_ops_jobs) -> None: mock_list_ops_jobs.return_value = [] managed_nodes = [ { "node_code": "mainland-controller-01", "region": "mainland", "role": "control", "is_managed": True, "is_enabled": True, "is_agent_online": True, "cluster_status": "online", "cluster_is_effective_worker": True, "cluster_detect_participating": False, }, { "node_code": "mainland-worker-02", "region": "mainland", "role": "worker", "is_managed": False, "is_enabled": False, "is_agent_online": False, "cluster_status": "online", "cluster_is_effective_worker": True, "cluster_detect_participating": False, }, ] overview = get_ops_inspection_overview(managed_nodes=managed_nodes, fetch_limit=20) row_map = {item["node_code"]: item for item in overview["rows"]} self.assertEqual(2, overview["visible_nodes_total"]) self.assertEqual(1, overview["eligible_nodes_total"]) self.assertEqual(1, overview["handover_gap_nodes_total"]) self.assertEqual("attention", row_map["mainland-worker-02"]["overall_status"]) self.assertEqual("handover_gap", row_map["mainland-worker-02"]["problem_kind"]) self.assertEqual("handover_first_gap", row_map["mainland-worker-02"]["recommended_action_code"]) @patch("app.services.ops_service.list_ops_jobs") def test_inspection_overview_marks_participating_nodes_without_inspection_as_attention(self, mock_list_ops_jobs) -> None: mock_list_ops_jobs.return_value = [] managed_nodes = [ { "node_code": "mainland-worker-01", "region": "mainland", "role": "worker", "is_managed": True, "is_enabled": True, "is_agent_online": True, "cluster_status": "busy", "cluster_is_effective_worker": True, "cluster_detect_participating": True, } ] overview = get_ops_inspection_overview( managed_nodes=managed_nodes, fetch_limit=20, only_participating=True, only_problem=True, ) self.assertEqual(1, overview["filtered_rows_total"]) self.assertEqual("attention", overview["rows"][0]["overall_status"]) self.assertEqual("participating_no_inspection", overview["rows"][0]["problem_kind"]) self.assertEqual("open_worker_logs_participating", overview["rows"][0]["recommended_action_code"]) self.assertEqual("attention", overview["status"]) @patch("app.services.ops_service.build_release_execution_mode_recommendation") @patch("app.services.ops_service.get_release_launchpad") @patch("app.services.ops_service.get_latest_release") @patch("app.services.ops_service.get_release_summary") @patch("app.services.ops_service.list_release_rollouts") @patch("app.services.ops_service.get_recent_ops_playbook_runs") @patch("app.services.ops_service.list_ops_jobs") @patch("app.services.ops_service.list_managed_nodes_with_agent_state") @patch("app.services.ops_service.get_ops_job_summary") @patch("app.services.ops_service.get_sync_summary") @patch("app.services.ops_service.get_runtime_status") def test_ops_overview_uses_top_driver_recommendation_as_summary( self, mock_get_runtime_status, mock_get_sync_summary, mock_get_ops_job_summary, mock_list_managed_nodes_with_agent_state, mock_list_ops_jobs, mock_get_recent_ops_playbook_runs, mock_list_release_rollouts, mock_get_release_summary, mock_get_latest_release, mock_get_release_launchpad, mock_build_release_execution_mode_recommendation, ) -> None: mock_get_runtime_status.return_value = { "readiness": { "status": "attention", "summary": "存在接管缺口", }, "detect": { "log_sync": { "enabled": False, "mode": "key", "line_count": 0, "source_node_count": 0, "source_nodes": [], "last_at": "", "last_line": "", "preview_lines": [], }, "participation_summary": {}, "participating_nodes": [], "non_participating_nodes": [], "active_job": {}, "phase_label": "", "phase_detail": "", "recent_event": "", "recent_warning": "", }, "cluster": { "summary": { "online_worker_nodes": 1, "online_control_nodes": 1, "dedicated_online_worker_nodes": 1, "busy_nodes": [], }, "nodes": [ { "node_code": "mainland-worker-01", "region": "mainland", "role": "worker", "status": "online", "current_load": 0, "hostname": "worker-01", "last_heartbeat_at": "2026-04-17 10:00:00", "is_effective_worker": True, "detect_participating": False, } ], }, } mock_get_sync_summary.return_value = {} mock_get_ops_job_summary.return_value = {"total": 0, "queued": 0, "running": 0, "awaiting_approval": 0} mock_list_managed_nodes_with_agent_state.return_value = { "nodes": [ { "node_code": "mainland-worker-01", "region": "mainland", "role": "worker", "is_managed": False, "is_enabled": False, "is_agent_online": False, "cluster_status": "online", "cluster_is_effective_worker": True, "cluster_detect_participating": False, } ], "summary": { "total": 1, "managed_total": 0, "managed_enabled": 0, "online": 0, "stale": 0, "pending_bootstrap": 1, "runtime_only": 1, "unmanaged": 1, "token_issue": 0, "status_counts": {}, }, } mock_list_ops_jobs.return_value = [] mock_get_recent_ops_playbook_runs.return_value = {"runs": []} mock_list_release_rollouts.return_value = [] mock_get_release_summary.return_value = {} mock_get_latest_release.return_value = {} mock_get_release_launchpad.return_value = {"launchpad_status": {"status": "attention"}} mock_build_release_execution_mode_recommendation.return_value = { "recommended_mode": "remote-agent", "recommended_mode_label": "remote-agent", "reason": "", "options": [], "recommended_gate": { "status": "release_not_ready", "summary": "", "target_nodes": [], "target_node_codes": [], "blocking_reasons": [], "warning_reasons": [], "execution_mode": "remote-agent", "execution_mode_label": "remote-agent", "operational_readiness": {"rows": [], "summary": {}}, }, "gates": {}, } overview = get_ops_overview() mock_list_managed_nodes_with_agent_state.assert_any_call( participation_payload=mock_get_runtime_status.return_value["detect"] ) self.assertEqual("driver_recommendations", overview["recommendation"]["source"]) self.assertEqual("handover-gap", overview["recommendation"]["key"]) self.assertEqual("先补接管缺口", overview["recommendation"]["priority"]) self.assertIn("有效执行节点", overview["recommendation"]["summary"]) self.assertEqual("纳管首台节点", overview["recommendation"]["primary_label"]) self.assertEqual("handover_first_gap", overview["recommendation"]["primary_action_code"]) self.assertEqual("attention", overview["release_hub"]["launchpad"]["launchpad_status"]["status"]) @patch("app.services.ops_service.build_release_execution_mode_recommendation") @patch("app.services.ops_service.get_release_launchpad") @patch("app.services.ops_service.get_latest_release") @patch("app.services.ops_service.get_release_summary") @patch("app.services.ops_service.list_release_rollouts") @patch("app.services.ops_service.get_recent_ops_playbook_runs") @patch("app.services.ops_service.list_ops_jobs") @patch("app.services.ops_service.list_managed_nodes_with_agent_state") @patch("app.services.ops_service.get_ops_job_summary") @patch("app.services.ops_service.get_sync_summary") @patch("app.services.ops_service.get_runtime_status") def test_ops_overview_prefers_launchpad_bootstrap_recovery_for_handover_gap( self, mock_get_runtime_status, mock_get_sync_summary, mock_get_ops_job_summary, mock_list_managed_nodes_with_agent_state, mock_list_ops_jobs, mock_get_recent_ops_playbook_runs, mock_list_release_rollouts, mock_get_release_summary, mock_get_latest_release, mock_get_release_launchpad, mock_build_release_execution_mode_recommendation, ) -> None: mock_get_runtime_status.return_value = { "readiness": {"status": "attention", "summary": "当前还需补执行面接管"}, "worker": {"running": True}, "sync_agent": {"running": True}, "detect": {"participation_summary": {"summary": ""}}, } mock_get_sync_summary.return_value = {} mock_get_ops_job_summary.return_value = {} mock_list_managed_nodes_with_agent_state.return_value = { "nodes": [ { "node_code": "mainland-worker-02", "role": "worker", "is_managed": False, "is_enabled": False, "is_agent_online": False, "cluster_status": "online", "cluster_is_effective_worker": True, "is_current_participant": False, } ], "summary": { "total": 1, "managed_total": 0, "managed_enabled": 0, "online": 1, "stale": 0, "pending_bootstrap": 1, "runtime_only": 0, "unmanaged": 1, "token_issue": 0, "status_counts": {}, }, } mock_list_ops_jobs.return_value = [] mock_get_recent_ops_playbook_runs.return_value = {"runs": []} mock_list_release_rollouts.return_value = [] mock_get_release_summary.return_value = {} mock_get_latest_release.return_value = {} mock_get_release_launchpad.return_value = { "launchpad_status": { "status": "attention", "status_label": "待补执行面", "summary": "当前节点仍处于接入阶段，建议先签发 onboarding.bootstrap。", "recommended_action_code": "bootstrap_run", }, "worker_rollout_preview": { "policy_preview": { "release_gate": { "operational_readiness": { "rows": [ { "node_code": "mainland-worker-02", "execution_ready": False, "recovery_action": "bootstrap_run", "recovery_summary": "当前节点仍处于接入阶段，建议先签发 onboarding.bootstrap。", } ] } } } }, } mock_build_release_execution_mode_recommendation.return_value = { "recommended_mode": "remote-agent", "recommended_mode_label": "remote-agent", "reason": "", "options": [], "recommended_gate": { "status": "release_not_ready", "summary": "", "target_nodes": [], "target_node_codes": [], "blocking_reasons": [], "warning_reasons": [], "execution_mode": "remote-agent", "execution_mode_label": "remote-agent", "operational_readiness": {"rows": [], "summary": {}}, }, "gates": {}, } overview = get_ops_overview() self.assertEqual("handover-gap-bootstrap_run", overview["recommendation"]["key"]) self.assertEqual("bootstrap_run", overview["recommendation"]["primary_action_code"]) self.assertEqual("跑接入收口", overview["recommendation"]["primary_label"]) self.assertEqual(["mainland-worker-02"], overview["recommendation"]["primary_node_codes"]) @patch("app.services.ops_service.build_release_execution_mode_recommendation") @patch("app.services.ops_service.get_release_launchpad") @patch("app.services.ops_service.get_latest_release") @patch("app.services.ops_service.get_release_summary") @patch("app.services.ops_service.list_release_rollouts") @patch("app.services.ops_service.get_recent_ops_playbook_runs") @patch("app.services.ops_service.list_ops_jobs") @patch("app.services.ops_service.list_managed_nodes_with_agent_state") @patch("app.services.ops_service.get_ops_job_summary") @patch("app.services.ops_service.get_sync_summary") @patch("app.services.ops_service.get_runtime_status") def test_ops_overview_exposes_remote_log_preview_lines( self, mock_get_runtime_status, mock_get_sync_summary, mock_get_ops_job_summary, mock_list_managed_nodes_with_agent_state, mock_list_ops_jobs, mock_get_recent_ops_playbook_runs, mock_list_release_rollouts, mock_get_release_summary, mock_get_latest_release, mock_get_release_launchpad, mock_build_release_execution_mode_recommendation, ) -> None: mock_get_runtime_status.return_value = { "readiness": {"status": "ready", "summary": "ok"}, "detect": { "log_sync": { "enabled": True, "mode": "key", "line_count": 2, "source_node_count": 1, "source_nodes": ["mainland-worker-01"], "last_at": "2026-04-17 10:01:00", "last_line": "[2026-04-17 10:01:00] [mainland-worker-01] 域名检测完成: alpha.com", "preview_lines": [ "[2026-04-17 10:00:00] [mainland-worker-01] 开始检测域名: alpha.com", "[2026-04-17 10:01:00] [mainland-worker-01] 域名检测完成: alpha.com", ], }, "participation_summary": { "participating_nodes": 1, "dispatch_active_nodes": 1, "recent_only_nodes": 0, "non_participating_nodes": 0, "standby_nodes": 0, "load_syncing_nodes": 0, "effective_online_nodes": 1, "dedicated_worker_nodes": 1, "controller_worker_nodes": 0, "dispatch_active_node_codes": ["mainland-worker-01"], "recent_only_node_codes": [], "non_participating_node_codes": [], "standby_node_codes": [], "load_syncing_node_codes": [], "summary": "有效执行节点 1 台；正在执行/领任务 1 台；近窗刚有吞吐 0 台；在线但未参与 0 台", }, "participating_nodes": [ { "node_code": "mainland-worker-01", "region": "mainland", "role": "worker", "status": "busy", "current_load": 1, "participation_state": "running", "participation_label": "执行中", "participation_reason": "当前正在执行 1 项检测任务。", "is_dispatch_active": True, } ], "non_participating_nodes": [], "active_job": {"job_code": "detect-001", "status": "running"}, "phase_label": "运行中", "phase_detail": "测试中", "recent_event": "", "recent_warning": "", }, "cluster": { "summary": { "online_worker_nodes": 1, "online_control_nodes": 1, "dedicated_online_worker_nodes": 1, "busy_nodes": ["mainland-worker-01"], }, "nodes": [ { "node_code": "mainland-worker-01", "region": "mainland", "role": "worker", "status": "busy", "current_load": 1, "hostname": "worker-01", "last_heartbeat_at": "2026-04-17 10:01:00", "is_effective_worker": True, "detect_participating": True, } ], }, } mock_get_sync_summary.return_value = {} mock_get_ops_job_summary.return_value = {"total": 0, "queued": 0, "running": 0, "awaiting_approval": 0} mock_list_managed_nodes_with_agent_state.return_value = { "nodes": [ { "node_code": "mainland-worker-01", "region": "mainland", "role": "worker", "is_managed": True, "is_enabled": True, "is_agent_online": True, "cluster_status": "busy", "cluster_is_effective_worker": True, "cluster_detect_participating": True, } ], "summary": { "total": 1, "managed_total": 1, "managed_enabled": 1, "online": 1, "stale": 0, "pending_bootstrap": 0, "runtime_only": 0, "unmanaged": 0, "token_issue": 0, "status_counts": {}, "participating": 1, "dispatch_active": 1, "recent_only": 0, "standby": 0, "load_syncing": 0, "non_participating": 0, }, } mock_list_ops_jobs.return_value = [] mock_get_recent_ops_playbook_runs.return_value = {"runs": []} mock_list_release_rollouts.return_value = [] mock_get_release_summary.return_value = {} mock_get_latest_release.return_value = {} mock_get_release_launchpad.return_value = {"launchpad_status": {"status": "ready"}} mock_build_release_execution_mode_recommendation.return_value = { "recommended_mode": "remote-agent", "recommended_mode_label": "remote-agent", "reason": "", "options": [], "recommended_gate": { "status": "release_not_ready", "summary": "", "target_nodes": [], "target_node_codes": [], "blocking_reasons": [], "warning_reasons": [], "execution_mode": "remote-agent", "execution_mode_label": "remote-agent", "operational_readiness": {"rows": [], "summary": {}}, }, "gates": {}, } overview = get_ops_overview() execution_scene = overview["execution_scene"] log_sync = overview["execution_scene"]["log_sync"] self.assertEqual("有效执行节点 1 台；正在执行/领任务 1 台；近窗刚有吞吐 0 台；在线但未参与 0 台", execution_scene["summary"]) self.assertEqual(1, execution_scene["counts"]["dispatch_active"]) self.assertEqual(0, execution_scene["counts"]["recent_only"]) self.assertEqual(0, execution_scene["counts"]["standby"]) self.assertEqual(0, execution_scene["counts"]["load_syncing"]) self.assertEqual(1, len(execution_scene["dispatch_active_nodes"])) self.assertEqual("dispatch_active", execution_scene["dispatch_active_nodes"][0]["participation_bucket"]) self.assertEqual([], execution_scene["recent_only_nodes"]) self.assertEqual([], execution_scene["load_syncing_nodes"]) self.assertTrue(log_sync["enabled"]) self.assertEqual("key", log_sync["mode"]) self.assertEqual(2, log_sync["preview_line_count"]) self.assertEqual( [ "[2026-04-17 10:00:00] [mainland-worker-01] 开始检测域名: alpha.com", "[2026-04-17 10:01:00] [mainland-worker-01] 域名检测完成: alpha.com", ], log_sync["preview_lines"], ) self.assertEqual(1, log_sync["covered_participating_node_count"]) self.assertEqual([], log_sync["missing_participating_nodes"]) self.assertEqual(1, len(log_sync["source_node_summaries"])) self.assertEqual("mainland-worker-01", log_sync["source_node_summaries"][0]["node_code"]) self.assertEqual(2, log_sync["source_node_summaries"][0]["line_count"]) self.assertEqual(2, log_sync["source_node_summaries"][0]["key_line_count"]) self.assertEqual([], log_sync["missing_participating_node_summaries"]) self.assertEqual("ready", overview["release_hub"]["launchpad"]["launchpad_status"]["status"]) @patch("app.services.ops_service.build_release_execution_mode_recommendation") @patch("app.services.ops_service.get_release_launchpad") @patch("app.services.ops_service.get_latest_release") @patch("app.services.ops_service.get_release_summary") @patch("app.services.ops_service.list_release_rollouts") @patch("app.services.ops_service.get_recent_ops_playbook_runs") @patch("app.services.ops_service.list_ops_jobs") @patch("app.services.ops_service.list_managed_nodes_with_agent_state") @patch("app.services.ops_service.get_ops_job_summary") @patch("app.services.ops_service.get_sync_summary") @patch("app.services.ops_service.get_runtime_status") def test_ops_overview_marks_missing_remote_log_reason_code( self, mock_get_runtime_status, mock_get_sync_summary, mock_get_ops_job_summary, mock_list_managed_nodes_with_agent_state, mock_list_ops_jobs, mock_get_recent_ops_playbook_runs, mock_list_release_rollouts, mock_get_release_summary, mock_get_latest_release, mock_get_release_launchpad, mock_build_release_execution_mode_recommendation, ) -> None: mock_get_runtime_status.return_value = { "readiness": {"status": "ready", "summary": "ok"}, "detect": { "log_sync": { "enabled": True, "mode": "key", "line_count": 0, "source_node_count": 0, "source_nodes": [], "last_at": "", "last_line": "", "preview_lines": [], }, "participation_summary": { "participating_nodes": 1, "dispatch_active_nodes": 1, "recent_only_nodes": 0, "non_participating_nodes": 0, "standby_nodes": 0, "load_syncing_nodes": 0, "effective_online_nodes": 1, "dedicated_worker_nodes": 1, "controller_worker_nodes": 0, "dispatch_active_node_codes": ["mainland-worker-01"], "recent_only_node_codes": [], "non_participating_node_codes": [], "standby_node_codes": [], "load_syncing_node_codes": [], "summary": "有效执行节点 1 台；正在执行/领任务 1 台；近窗刚有吞吐 0 台；在线但未参与 0 台", }, "participating_nodes": [ { "node_code": "mainland-worker-01", "region": "mainland", "role": "worker", "status": "busy", "current_load": 2, "participation_state": "claimed", "participation_label": "已领待跑", "participation_reason": "已领取 2 项任务，等待线程继续执行。", "is_dispatch_active": True, } ], "non_participating_nodes": [], "active_job": {"job_code": "detect-001", "status": "running"}, "phase_label": "运行中", "phase_detail": "等待样本", "recent_event": "", "recent_warning": "", }, "cluster": {"summary": {}, "nodes": []}, } mock_get_sync_summary.return_value = {} mock_get_ops_job_summary.return_value = {"total": 0, "queued": 0, "running": 0, "awaiting_approval": 0} mock_list_managed_nodes_with_agent_state.return_value = {"nodes": [], "summary": {}} mock_list_ops_jobs.return_value = [] mock_get_recent_ops_playbook_runs.return_value = {"runs": []} mock_list_release_rollouts.return_value = [] mock_get_release_summary.return_value = {} mock_get_latest_release.return_value = {} mock_get_release_launchpad.return_value = {"launchpad_status": {"status": "ready"}} mock_build_release_execution_mode_recommendation.return_value = { "recommended_mode": "remote-agent", "recommended_mode_label": "remote-agent", "reason": "", "options": [], "recommended_gate": { "status": "release_not_ready", "summary": "", "target_nodes": [], "target_node_codes": [], "blocking_reasons": [], "warning_reasons": [], "execution_mode": "remote-agent", "execution_mode_label": "remote-agent", "operational_readiness": {"rows": [], "summary": {}}, }, "gates": {}, } overview = get_ops_overview() missing_rows = overview["execution_scene"]["log_sync"]["missing_participating_node_summaries"] self.assertEqual(1, len(missing_rows)) self.assertEqual("waiting_sample", missing_rows[0]["missing_reason_code"]) self.assertEqual("dispatch_active", overview["execution_scene"]["participating_nodes"][0]["participation_bucket"]) @patch("app.services.ops_service.get_ops_activity_stream") @patch("app.services.ops_service.build_release_execution_mode_recommendation") @patch("app.services.ops_service.get_release_launchpad") @patch("app.services.ops_service.get_latest_release") @patch("app.services.ops_service.get_release_summary") @patch("app.services.ops_service.list_release_rollouts") @patch("app.services.ops_service.get_recent_ops_playbook_runs") @patch("app.services.ops_service.list_ops_jobs") @patch("app.services.ops_service.list_managed_nodes_with_agent_state") @patch("app.services.ops_service.get_ops_job_summary") @patch("app.services.ops_service.get_sync_summary") @patch("app.services.ops_service.get_runtime_status") def test_ops_overview_prioritizes_node_agent_dead_letter_queue( self, mock_get_runtime_status, mock_get_sync_summary, mock_get_ops_job_summary, mock_list_managed_nodes_with_agent_state, mock_list_ops_jobs, mock_get_recent_ops_playbook_runs, mock_list_release_rollouts, mock_get_release_summary, mock_get_latest_release, mock_get_release_launchpad, mock_build_release_execution_mode_recommendation, mock_get_ops_activity_stream, ) -> None: mock_get_runtime_status.return_value = { "readiness": {"status": "ready", "summary": "ok"}, "detect": { "log_sync": { "enabled": False, "mode": "key", "line_count": 0, "source_node_count": 0, "source_nodes": [], "last_at": "", "last_line": "", "preview_lines": [], }, "participation_summary": {}, "participating_nodes": [], "non_participating_nodes": [], "active_job": {}, "phase_label": "", "phase_detail": "", "recent_event": "", "recent_warning": "", }, "cluster": { "summary": { "online_worker_nodes": 1, "online_control_nodes": 1, "dedicated_online_worker_nodes": 1, "busy_nodes": [], }, "nodes": [ { "node_code": "mainland-worker-01", "region": "mainland", "role": "worker", "status": "online", "current_load": 0, "hostname": "worker-01", "last_heartbeat_at": "2026-04-18 05:10:00", "is_effective_worker": True, "detect_participating": False, } ], }, } mock_get_sync_summary.return_value = {} mock_get_ops_job_summary.return_value = {"total": 0, "queued": 0, "running": 0, "awaiting_approval": 0} mock_list_managed_nodes_with_agent_state.return_value = { "nodes": [ { "node_code": "mainland-worker-01", "region": "mainland", "role": "worker", "is_managed": True, "is_enabled": True, "is_agent_online": True, "cluster_status": "online", "cluster_is_effective_worker": True, "cluster_detect_participating": False, "delivery_queue_state": "dead_letter", "delivery_queue_label": "死信 2", "delivery_queue_reason": "当前存在 2 条死信记录，建议优先查看节点日志或诊断编排。", "delivery_queue_pending_count": 1, "delivery_queue_dead_letter_count": 2, } ], "summary": { "total": 1, "managed_total": 1, "managed_enabled": 1, "online": 1, "stale": 0, "pending_bootstrap": 0, "runtime_only": 0, "unmanaged": 0, "token_issue": 0, "queue_retrying_nodes": 0, "queue_dead_letter_nodes": 1, "queue_pending_records": 1, "queue_dead_letter_records": 2, "status_counts": {}, }, } mock_list_ops_jobs.return_value = [] mock_get_recent_ops_playbook_runs.return_value = {"runs": []} mock_list_release_rollouts.return_value = [] mock_get_release_summary.return_value = {} mock_get_latest_release.return_value = {} mock_get_release_launchpad.return_value = {"launchpad_status": {}} mock_build_release_execution_mode_recommendation.return_value = { "recommended_mode": "remote-agent", "recommended_mode_label": "remote-agent", "reason": "", "options": [], "recommended_gate": { "status": "ready", "summary": "", "target_nodes": [], "target_node_codes": [], "blocking_reasons": [], "warning_reasons": [], "execution_mode": "remote-agent", "execution_mode_label": "remote-agent", "operational_readiness": {"rows": [], "summary": {}}, "release": {"id": 0}, }, "gates": {}, } mock_get_ops_activity_stream.return_value = {"items": []} overview = get_ops_overview() self.assertEqual("node-agent-dead-letter", overview["recommendation"]["key"]) self.assertEqual("先处理 Agent 死信", overview["recommendation"]["priority"]) self.assertEqual("replay_delivery_queue", overview["recommendation"]["primary_action_code"]) self.assertEqual( "delivery.queue.replay", overview["recommendation"]["primary_action_payload"]["template_key"], ) self.assertEqual(1, overview["managed_nodes"]["queue_dead_letter_nodes"]) self.assertEqual(2, overview["managed_nodes"]["queue_dead_letter_records"]) @patch("app.services.ops_service.get_ops_activity_stream") @patch("app.services.ops_service.build_release_execution_mode_recommendation") @patch("app.services.ops_service.get_release_launchpad") @patch("app.services.ops_service.get_latest_release") @patch("app.services.ops_service.get_release_summary") @patch("app.services.ops_service.list_release_rollouts") @patch("app.services.ops_service.get_recent_ops_playbook_runs") @patch("app.services.ops_service.list_ops_jobs") @patch("app.services.ops_service.list_managed_nodes_with_agent_state") @patch("app.services.ops_service.get_ops_job_summary") @patch("app.services.ops_service.get_sync_summary") @patch("app.services.ops_service.get_runtime_status") def test_ops_overview_prioritizes_start_delivery_gap_from_activity_stream( self, mock_get_runtime_status, mock_get_sync_summary, mock_get_ops_job_summary, mock_list_managed_nodes_with_agent_state, mock_list_ops_jobs, mock_get_recent_ops_playbook_runs, mock_list_release_rollouts, mock_get_release_summary, mock_get_latest_release, mock_get_release_launchpad, mock_build_release_execution_mode_recommendation, mock_get_ops_activity_stream, ) -> None: mock_get_runtime_status.return_value = { "readiness": {"status": "ready", "summary": "ok"}, "detect": { "log_sync": { "enabled": True, "mode": "key", "line_count": 12, "source_node_count": 1, "source_nodes": ["mainland-worker-01"], "last_at": "2026-04-18 11:00:00", "last_line": "ok", "preview_lines": ["ok"], }, "participation_summary": {}, "participating_nodes": [], "non_participating_nodes": [], "active_job": {}, "phase_label": "", "phase_detail": "", "recent_event": "", "recent_warning": "", }, "cluster": {"summary": {}, "nodes": []}, } mock_get_sync_summary.return_value = {} mock_get_ops_job_summary.return_value = {"total": 0, "queued": 0, "running": 0, "awaiting_approval": 0} mock_list_managed_nodes_with_agent_state.return_value = {"nodes": [], "summary": {}} mock_list_ops_jobs.return_value = [] mock_get_recent_ops_playbook_runs.return_value = {"runs": []} mock_list_release_rollouts.return_value = [] mock_get_release_summary.return_value = {} mock_get_latest_release.return_value = {} mock_get_release_launchpad.return_value = {"launchpad_status": {}} mock_build_release_execution_mode_recommendation.return_value = { "recommended_mode": "remote-agent", "recommended_mode_label": "remote-agent", "reason": "", "options": [], "recommended_gate": { "status": "ready", "summary": "", "target_nodes": [], "target_node_codes": [], "blocking_reasons": [], "warning_reasons": [], "execution_mode": "remote-agent", "execution_mode_label": "remote-agent", "operational_readiness": {"rows": [], "summary": {}}, "release": {"id": 0}, }, "gates": {}, } mock_get_ops_activity_stream.return_value = { "items": [ { "kind": "ops_job", "activity_key": "ops-job:81", "status": "attention", "job_status": "running", "summary": "节点已接单执行，但开始回执未成功送达控制面。", "job_code": "ops-agent-81", "start_delivery_state": "failed_local", "source_focus_ref": { "kind": "ops_job_event", "job_id": 81, "job_code": "ops-agent-81", "event_key": "job-event:81", "target_node_code": "mainland-worker-01", }, "target_node_codes": ["mainland-worker-01"], "ui_intent": {"kind": "job_events", "job_id": 81}, } ] } overview = get_ops_overview() self.assertEqual("activity-start-delivery-gap", overview["recommendation"]["key"]) self.assertEqual("先检查开始回执异常", overview["recommendation"]["priority"]) self.assertEqual("focus_latest_job_events", overview["recommendation"]["primary_action_code"]) self.assertEqual("ops_job_event", overview["recommendation"]["primary_action_payload"]["focus_ref"]["kind"]) @patch("app.services.ops_service.build_release_execution_mode_recommendation") @patch("app.services.ops_service.get_release_launchpad") @patch("app.services.ops_service.get_latest_release") @patch("app.services.ops_service.get_release_summary") @patch("app.services.ops_service.list_release_rollouts") @patch("app.services.ops_service.get_recent_ops_playbook_runs") @patch("app.services.ops_service.list_ops_jobs") @patch("app.services.ops_service.list_managed_nodes_with_agent_state") @patch("app.services.ops_service.get_ops_job_summary") @patch("app.services.ops_service.get_sync_summary") @patch("app.services.ops_service.get_runtime_status") def test_ops_overview_uses_release_launchpad_driver_card_when_execution_plane_is_clean( self, mock_get_runtime_status, mock_get_sync_summary, mock_get_ops_job_summary, mock_list_managed_nodes_with_agent_state, mock_list_ops_jobs, mock_get_recent_ops_playbook_runs, mock_list_release_rollouts, mock_get_release_summary, mock_get_latest_release, mock_get_release_launchpad, mock_build_release_execution_mode_recommendation, ) -> None: mock_get_runtime_status.return_value = { "readiness": {"status": "ready", "summary": "ok"}, "detect": { "log_sync": { "enabled": True, "mode": "key", "line_count": 0, "source_node_count": 0, "source_nodes": [], "last_at": "", "last_line": "", "preview_lines": [], }, "participation_summary": {}, "participating_nodes": [], "non_participating_nodes": [], "active_job": {}, "phase_label": "", "phase_detail": "", "recent_event": "", "recent_warning": "", }, "cluster": { "summary": { "online_worker_nodes": 1, "online_control_nodes": 1, "dedicated_online_worker_nodes": 1, "busy_nodes": [], }, "nodes": [ { "node_code": "mainland-worker-01", "region": "mainland", "role": "worker", "status": "online", "current_load": 0, "hostname": "worker-01", "last_heartbeat_at": "2026-04-18 10:00:00", "is_effective_worker": True, "detect_participating": False, } ], }, } mock_get_sync_summary.return_value = {} mock_get_ops_job_summary.return_value = {"total": 0, "queued": 0, "running": 0, "awaiting_approval": 0} mock_list_managed_nodes_with_agent_state.return_value = { "nodes": [ { "node_code": "mainland-worker-01", "region": "mainland", "role": "worker", "is_managed": True, "is_enabled": True, "is_agent_online": True, "cluster_status": "online", "cluster_is_effective_worker": True, "cluster_detect_participating": False, } ], "summary": { "total": 1, "managed_total": 1, "managed_enabled": 1, "online": 1, "stale": 0, "pending_bootstrap": 0, "runtime_only": 0, "unmanaged": 0, "token_issue": 0, "status_counts": {}, }, } mock_list_ops_jobs.return_value = [] mock_get_recent_ops_playbook_runs.return_value = {"runs": []} mock_list_release_rollouts.return_value = [] mock_get_release_summary.return_value = {} mock_get_latest_release.return_value = {} mock_get_release_launchpad.return_value = { "channel": "stable", "control_plane_base_url": "http://control.example:8100", "latest_package": { "available": True, "package_name": "domaincheck_release_demo.tar.gz", }, "latest_release": { "id": 17, "release_version": "domaincheck_release_demo", "channel": "stable", }, "worker_rollout_preview": { "message": "智能 Rollout 预检通过", "smart_rollout": { "available": True, "summary": "已按 Worker 模式预选 1 台节点。", "target_node_codes": ["mainland-worker-01"], }, }, "control_rollout_preview": { "message": "", "smart_rollout": { "available": False, "target_node_codes": [], }, }, "launchpad_status": { "status": "ready", "status_label": "可发 Worker 灰度", "summary": "最新发布包、Release 记录与 Worker 智能 Rollout 预检均已收口。", "recommended_action_code": "publish_latest_worker", }, } mock_build_release_execution_mode_recommendation.return_value = { "recommended_mode": "ssh", "recommended_mode_label": "SSH", "reason": "当前目标节点通过 SSH 路径的门禁更完整，建议先走 SSH 发布。", "options": [], "recommended_gate": { "status": "ready", "summary": "", "target_nodes": [], "target_node_codes": [], "blocking_reasons": [], "warning_reasons": [], "execution_mode": "ssh", "execution_mode_label": "SSH", "operational_readiness": {"rows": [], "summary": {}}, }, "gates": {}, } overview = get_ops_overview() self.assertEqual("driver_recommendations", overview["recommendation"]["source"]) self.assertEqual("release-launchpad-ready", overview["recommendation"]["key"]) self.assertEqual("当前可以发 Worker 灰度", overview["recommendation"]["priority"]) self.assertEqual("publish_latest_worker", overview["recommendation"]["primary_action_code"]) self.assertEqual(["mainland-worker-01"], overview["recommendation"]["primary_node_codes"]) self.assertEqual("ready", overview["release_hub"]["launchpad"]["launchpad_status"]["status"]) self.assertEqual("ssh", overview["release_hub"]["default_rollout_gate"]["execution_mode"]) self.assertEqual("SSH", overview["release_hub"]["default_rollout_execution"]["recommended_mode_label"]) if __name__ == "__main__": unittest.main()