Files
getDomain/domain-api/tests/test_ops_service_overview.py
2026-04-18 23:52:51 +08:00

1187 lines
51 KiB
Python

import unittest
from unittest.mock import patch
from app.services.ops_service import get_ops_inspection_overview, get_ops_overview
class OpsServiceOverviewTests(unittest.TestCase):
@patch("app.services.ops_service.list_managed_nodes_with_agent_state")
@patch("app.services.ops_service.list_ops_jobs")
def test_inspection_overview_fetches_managed_nodes_when_omitted(
self,
mock_list_ops_jobs,
mock_list_managed_nodes_with_agent_state,
) -> None:
mock_list_managed_nodes_with_agent_state.return_value = {
"nodes": [
{
"node_code": "mainland-worker-01",
"region": "mainland",
"role": "worker",
"is_managed": True,
"is_enabled": True,
"is_agent_online": True,
"cluster_status": "online",
"cluster_is_effective_worker": True,
"cluster_detect_participating": True,
}
]
}
mock_list_ops_jobs.return_value = [
{
"id": 21,
"job_code": "ops-health-21",
"action": "health.snapshot",
"status": "success",
"target_node_code": "mainland-worker-01",
"created_at": "2026-04-17 10:00:00",
"updated_at": "2026-04-17 10:01:00",
}
]
overview = get_ops_inspection_overview(fetch_limit=20)
self.assertEqual(1, overview["eligible_nodes_total"])
self.assertEqual(1, overview["rows_total"])
self.assertEqual("attention", overview["status"])
self.assertEqual("待处理", overview["status_label"])
self.assertEqual("mainland", overview["rows"][0]["region"])
self.assertEqual("worker", overview["rows"][0]["role"])
self.assertEqual("online", overview["rows"][0]["cluster_status"])
self.assertIn("latest_health_snapshot", overview["rows"][0])
self.assertIn("latest_worker_logs", overview["rows"][0])
self.assertIn("latest_diagnostics", overview["rows"][0])
@patch("app.services.ops_service.list_ops_jobs")
def test_inspection_overview_prioritizes_attention_and_supports_filters(self, mock_list_ops_jobs) -> None:
mock_list_ops_jobs.return_value = [
{
"id": 11,
"job_code": "ops-health-1",
"action": "health.snapshot",
"status": "success",
"target_node_code": "mainland-worker-01",
"created_at": "2026-04-17 10:00:00",
"updated_at": "2026-04-17 10:01:00",
},
{
"id": 12,
"job_code": "ops-logs-1",
"action": "logs.collect",
"status": "failed",
"target_node_code": "mainland-worker-01",
"created_at": "2026-04-17 10:02:00",
"updated_at": "2026-04-17 10:03:00",
"error_message": "journalctl permission denied",
"payload": {
"service_name": "domaincheck-worker",
"lines": 120,
},
},
]
managed_nodes = [
{
"node_code": "mainland-worker-01",
"region": "mainland",
"role": "worker",
"is_managed": True,
"is_enabled": True,
"is_agent_online": True,
"cluster_status": "online",
"cluster_is_effective_worker": True,
"cluster_detect_participating": True,
},
{
"node_code": "mainland-controller-01",
"region": "mainland",
"role": "control",
"is_managed": True,
"is_enabled": True,
"is_agent_online": True,
"cluster_status": "online",
"cluster_is_effective_worker": True,
"cluster_detect_participating": False,
},
]
overview = get_ops_inspection_overview(managed_nodes=managed_nodes, fetch_limit=20)
self.assertEqual(2, overview["rows_total"])
self.assertEqual(1, overview["summary"]["attention"])
self.assertEqual(1, overview["summary"]["missing"])
self.assertEqual("attention", overview["status"])
self.assertEqual("待处理", overview["status_label"])
self.assertEqual(1, overview["counts"]["attention_nodes"])
self.assertEqual(1, overview["counts"]["missing_nodes"])
self.assertEqual("mainland-worker-01", overview["priority_queue"][0]["node_code"])
self.assertEqual("attention", overview["priority_queue"][0]["overall_status"])
self.assertEqual("terminal_failure", overview["priority_queue"][0]["problem_kind"])
self.assertEqual("failed", overview["rows"][0]["latest_worker_logs"]["status"])
self.assertEqual("成功", overview["rows"][0]["latest_health_snapshot"]["status_label"])
filtered = get_ops_inspection_overview(
managed_nodes=managed_nodes,
fetch_limit=20,
only_problem=True,
only_participating=True,
)
self.assertEqual(1, filtered["filtered_rows_total"])
self.assertEqual("mainland-worker-01", filtered["rows"][0]["node_code"])
self.assertEqual("attention", filtered["rows"][0]["overall_status"])
@patch("app.services.ops_service.list_ops_jobs")
def test_inspection_overview_includes_visible_handover_gap_nodes(self, mock_list_ops_jobs) -> None:
mock_list_ops_jobs.return_value = []
managed_nodes = [
{
"node_code": "mainland-controller-01",
"region": "mainland",
"role": "control",
"is_managed": True,
"is_enabled": True,
"is_agent_online": True,
"cluster_status": "online",
"cluster_is_effective_worker": True,
"cluster_detect_participating": False,
},
{
"node_code": "mainland-worker-02",
"region": "mainland",
"role": "worker",
"is_managed": False,
"is_enabled": False,
"is_agent_online": False,
"cluster_status": "online",
"cluster_is_effective_worker": True,
"cluster_detect_participating": False,
},
]
overview = get_ops_inspection_overview(managed_nodes=managed_nodes, fetch_limit=20)
row_map = {item["node_code"]: item for item in overview["rows"]}
self.assertEqual(2, overview["visible_nodes_total"])
self.assertEqual(1, overview["eligible_nodes_total"])
self.assertEqual(1, overview["handover_gap_nodes_total"])
self.assertEqual("attention", row_map["mainland-worker-02"]["overall_status"])
self.assertEqual("handover_gap", row_map["mainland-worker-02"]["problem_kind"])
self.assertEqual("handover_first_gap", row_map["mainland-worker-02"]["recommended_action_code"])
@patch("app.services.ops_service.list_ops_jobs")
def test_inspection_overview_marks_participating_nodes_without_inspection_as_attention(self, mock_list_ops_jobs) -> None:
mock_list_ops_jobs.return_value = []
managed_nodes = [
{
"node_code": "mainland-worker-01",
"region": "mainland",
"role": "worker",
"is_managed": True,
"is_enabled": True,
"is_agent_online": True,
"cluster_status": "busy",
"cluster_is_effective_worker": True,
"cluster_detect_participating": True,
}
]
overview = get_ops_inspection_overview(
managed_nodes=managed_nodes,
fetch_limit=20,
only_participating=True,
only_problem=True,
)
self.assertEqual(1, overview["filtered_rows_total"])
self.assertEqual("attention", overview["rows"][0]["overall_status"])
self.assertEqual("participating_no_inspection", overview["rows"][0]["problem_kind"])
self.assertEqual("open_worker_logs_participating", overview["rows"][0]["recommended_action_code"])
self.assertEqual("attention", overview["status"])
@patch("app.services.ops_service.build_release_execution_mode_recommendation")
@patch("app.services.ops_service.get_release_launchpad")
@patch("app.services.ops_service.get_latest_release")
@patch("app.services.ops_service.get_release_summary")
@patch("app.services.ops_service.list_release_rollouts")
@patch("app.services.ops_service.get_recent_ops_playbook_runs")
@patch("app.services.ops_service.list_ops_jobs")
@patch("app.services.ops_service.list_managed_nodes_with_agent_state")
@patch("app.services.ops_service.get_ops_job_summary")
@patch("app.services.ops_service.get_sync_summary")
@patch("app.services.ops_service.get_runtime_status")
def test_ops_overview_uses_top_driver_recommendation_as_summary(
self,
mock_get_runtime_status,
mock_get_sync_summary,
mock_get_ops_job_summary,
mock_list_managed_nodes_with_agent_state,
mock_list_ops_jobs,
mock_get_recent_ops_playbook_runs,
mock_list_release_rollouts,
mock_get_release_summary,
mock_get_latest_release,
mock_get_release_launchpad,
mock_build_release_execution_mode_recommendation,
) -> None:
mock_get_runtime_status.return_value = {
"readiness": {
"status": "attention",
"summary": "存在接管缺口",
},
"detect": {
"log_sync": {
"enabled": False,
"mode": "key",
"line_count": 0,
"source_node_count": 0,
"source_nodes": [],
"last_at": "",
"last_line": "",
"preview_lines": [],
},
"participation_summary": {},
"participating_nodes": [],
"non_participating_nodes": [],
"active_job": {},
"phase_label": "",
"phase_detail": "",
"recent_event": "",
"recent_warning": "",
},
"cluster": {
"summary": {
"online_worker_nodes": 1,
"online_control_nodes": 1,
"dedicated_online_worker_nodes": 1,
"busy_nodes": [],
},
"nodes": [
{
"node_code": "mainland-worker-01",
"region": "mainland",
"role": "worker",
"status": "online",
"current_load": 0,
"hostname": "worker-01",
"last_heartbeat_at": "2026-04-17 10:00:00",
"is_effective_worker": True,
"detect_participating": False,
}
],
},
}
mock_get_sync_summary.return_value = {}
mock_get_ops_job_summary.return_value = {"total": 0, "queued": 0, "running": 0, "awaiting_approval": 0}
mock_list_managed_nodes_with_agent_state.return_value = {
"nodes": [
{
"node_code": "mainland-worker-01",
"region": "mainland",
"role": "worker",
"is_managed": False,
"is_enabled": False,
"is_agent_online": False,
"cluster_status": "online",
"cluster_is_effective_worker": True,
"cluster_detect_participating": False,
}
],
"summary": {
"total": 1,
"managed_total": 0,
"managed_enabled": 0,
"online": 0,
"stale": 0,
"pending_bootstrap": 1,
"runtime_only": 1,
"unmanaged": 1,
"token_issue": 0,
"status_counts": {},
},
}
mock_list_ops_jobs.return_value = []
mock_get_recent_ops_playbook_runs.return_value = {"runs": []}
mock_list_release_rollouts.return_value = []
mock_get_release_summary.return_value = {}
mock_get_latest_release.return_value = {}
mock_get_release_launchpad.return_value = {"launchpad_status": {"status": "attention"}}
mock_build_release_execution_mode_recommendation.return_value = {
"recommended_mode": "remote-agent",
"recommended_mode_label": "remote-agent",
"reason": "",
"options": [],
"recommended_gate": {
"status": "release_not_ready",
"summary": "",
"target_nodes": [],
"target_node_codes": [],
"blocking_reasons": [],
"warning_reasons": [],
"execution_mode": "remote-agent",
"execution_mode_label": "remote-agent",
"operational_readiness": {"rows": [], "summary": {}},
},
"gates": {},
}
overview = get_ops_overview()
mock_list_managed_nodes_with_agent_state.assert_any_call(
participation_payload=mock_get_runtime_status.return_value["detect"]
)
self.assertEqual("driver_recommendations", overview["recommendation"]["source"])
self.assertEqual("handover-gap", overview["recommendation"]["key"])
self.assertEqual("先补接管缺口", overview["recommendation"]["priority"])
self.assertIn("有效执行节点", overview["recommendation"]["summary"])
self.assertEqual("纳管首台节点", overview["recommendation"]["primary_label"])
self.assertEqual("handover_first_gap", overview["recommendation"]["primary_action_code"])
self.assertEqual("attention", overview["release_hub"]["launchpad"]["launchpad_status"]["status"])
@patch("app.services.ops_service.build_release_execution_mode_recommendation")
@patch("app.services.ops_service.get_release_launchpad")
@patch("app.services.ops_service.get_latest_release")
@patch("app.services.ops_service.get_release_summary")
@patch("app.services.ops_service.list_release_rollouts")
@patch("app.services.ops_service.get_recent_ops_playbook_runs")
@patch("app.services.ops_service.list_ops_jobs")
@patch("app.services.ops_service.list_managed_nodes_with_agent_state")
@patch("app.services.ops_service.get_ops_job_summary")
@patch("app.services.ops_service.get_sync_summary")
@patch("app.services.ops_service.get_runtime_status")
def test_ops_overview_prefers_launchpad_bootstrap_recovery_for_handover_gap(
self,
mock_get_runtime_status,
mock_get_sync_summary,
mock_get_ops_job_summary,
mock_list_managed_nodes_with_agent_state,
mock_list_ops_jobs,
mock_get_recent_ops_playbook_runs,
mock_list_release_rollouts,
mock_get_release_summary,
mock_get_latest_release,
mock_get_release_launchpad,
mock_build_release_execution_mode_recommendation,
) -> None:
mock_get_runtime_status.return_value = {
"readiness": {"status": "attention", "summary": "当前还需补执行面接管"},
"worker": {"running": True},
"sync_agent": {"running": True},
"detect": {"participation_summary": {"summary": ""}},
}
mock_get_sync_summary.return_value = {}
mock_get_ops_job_summary.return_value = {}
mock_list_managed_nodes_with_agent_state.return_value = {
"nodes": [
{
"node_code": "mainland-worker-02",
"role": "worker",
"is_managed": False,
"is_enabled": False,
"is_agent_online": False,
"cluster_status": "online",
"cluster_is_effective_worker": True,
"is_current_participant": False,
}
],
"summary": {
"total": 1,
"managed_total": 0,
"managed_enabled": 0,
"online": 1,
"stale": 0,
"pending_bootstrap": 1,
"runtime_only": 0,
"unmanaged": 1,
"token_issue": 0,
"status_counts": {},
},
}
mock_list_ops_jobs.return_value = []
mock_get_recent_ops_playbook_runs.return_value = {"runs": []}
mock_list_release_rollouts.return_value = []
mock_get_release_summary.return_value = {}
mock_get_latest_release.return_value = {}
mock_get_release_launchpad.return_value = {
"launchpad_status": {
"status": "attention",
"status_label": "待补执行面",
"summary": "当前节点仍处于接入阶段,建议先签发 onboarding.bootstrap。",
"recommended_action_code": "bootstrap_run",
},
"worker_rollout_preview": {
"policy_preview": {
"release_gate": {
"operational_readiness": {
"rows": [
{
"node_code": "mainland-worker-02",
"execution_ready": False,
"recovery_action": "bootstrap_run",
"recovery_summary": "当前节点仍处于接入阶段,建议先签发 onboarding.bootstrap。",
}
]
}
}
}
},
}
mock_build_release_execution_mode_recommendation.return_value = {
"recommended_mode": "remote-agent",
"recommended_mode_label": "remote-agent",
"reason": "",
"options": [],
"recommended_gate": {
"status": "release_not_ready",
"summary": "",
"target_nodes": [],
"target_node_codes": [],
"blocking_reasons": [],
"warning_reasons": [],
"execution_mode": "remote-agent",
"execution_mode_label": "remote-agent",
"operational_readiness": {"rows": [], "summary": {}},
},
"gates": {},
}
overview = get_ops_overview()
self.assertEqual("handover-gap-bootstrap_run", overview["recommendation"]["key"])
self.assertEqual("bootstrap_run", overview["recommendation"]["primary_action_code"])
self.assertEqual("跑接入收口", overview["recommendation"]["primary_label"])
self.assertEqual(["mainland-worker-02"], overview["recommendation"]["primary_node_codes"])
@patch("app.services.ops_service.build_release_execution_mode_recommendation")
@patch("app.services.ops_service.get_release_launchpad")
@patch("app.services.ops_service.get_latest_release")
@patch("app.services.ops_service.get_release_summary")
@patch("app.services.ops_service.list_release_rollouts")
@patch("app.services.ops_service.get_recent_ops_playbook_runs")
@patch("app.services.ops_service.list_ops_jobs")
@patch("app.services.ops_service.list_managed_nodes_with_agent_state")
@patch("app.services.ops_service.get_ops_job_summary")
@patch("app.services.ops_service.get_sync_summary")
@patch("app.services.ops_service.get_runtime_status")
def test_ops_overview_exposes_remote_log_preview_lines(
self,
mock_get_runtime_status,
mock_get_sync_summary,
mock_get_ops_job_summary,
mock_list_managed_nodes_with_agent_state,
mock_list_ops_jobs,
mock_get_recent_ops_playbook_runs,
mock_list_release_rollouts,
mock_get_release_summary,
mock_get_latest_release,
mock_get_release_launchpad,
mock_build_release_execution_mode_recommendation,
) -> None:
mock_get_runtime_status.return_value = {
"readiness": {"status": "ready", "summary": "ok"},
"detect": {
"log_sync": {
"enabled": True,
"mode": "key",
"line_count": 2,
"source_node_count": 1,
"source_nodes": ["mainland-worker-01"],
"last_at": "2026-04-17 10:01:00",
"last_line": "[2026-04-17 10:01:00] [mainland-worker-01] 域名检测完成: alpha.com",
"preview_lines": [
"[2026-04-17 10:00:00] [mainland-worker-01] 开始检测域名: alpha.com",
"[2026-04-17 10:01:00] [mainland-worker-01] 域名检测完成: alpha.com",
],
},
"participation_summary": {
"participating_nodes": 1,
"dispatch_active_nodes": 1,
"recent_only_nodes": 0,
"non_participating_nodes": 0,
"standby_nodes": 0,
"load_syncing_nodes": 0,
"effective_online_nodes": 1,
"dedicated_worker_nodes": 1,
"controller_worker_nodes": 0,
"dispatch_active_node_codes": ["mainland-worker-01"],
"recent_only_node_codes": [],
"non_participating_node_codes": [],
"standby_node_codes": [],
"load_syncing_node_codes": [],
"summary": "有效执行节点 1 台;正在执行/领任务 1 台;近窗刚有吞吐 0 台;在线但未参与 0 台",
},
"participating_nodes": [
{
"node_code": "mainland-worker-01",
"region": "mainland",
"role": "worker",
"status": "busy",
"current_load": 1,
"participation_state": "running",
"participation_label": "执行中",
"participation_reason": "当前正在执行 1 项检测任务。",
"is_dispatch_active": True,
}
],
"non_participating_nodes": [],
"active_job": {"job_code": "detect-001", "status": "running"},
"phase_label": "运行中",
"phase_detail": "测试中",
"recent_event": "",
"recent_warning": "",
},
"cluster": {
"summary": {
"online_worker_nodes": 1,
"online_control_nodes": 1,
"dedicated_online_worker_nodes": 1,
"busy_nodes": ["mainland-worker-01"],
},
"nodes": [
{
"node_code": "mainland-worker-01",
"region": "mainland",
"role": "worker",
"status": "busy",
"current_load": 1,
"hostname": "worker-01",
"last_heartbeat_at": "2026-04-17 10:01:00",
"is_effective_worker": True,
"detect_participating": True,
}
],
},
}
mock_get_sync_summary.return_value = {}
mock_get_ops_job_summary.return_value = {"total": 0, "queued": 0, "running": 0, "awaiting_approval": 0}
mock_list_managed_nodes_with_agent_state.return_value = {
"nodes": [
{
"node_code": "mainland-worker-01",
"region": "mainland",
"role": "worker",
"is_managed": True,
"is_enabled": True,
"is_agent_online": True,
"cluster_status": "busy",
"cluster_is_effective_worker": True,
"cluster_detect_participating": True,
}
],
"summary": {
"total": 1,
"managed_total": 1,
"managed_enabled": 1,
"online": 1,
"stale": 0,
"pending_bootstrap": 0,
"runtime_only": 0,
"unmanaged": 0,
"token_issue": 0,
"status_counts": {},
"participating": 1,
"dispatch_active": 1,
"recent_only": 0,
"standby": 0,
"load_syncing": 0,
"non_participating": 0,
},
}
mock_list_ops_jobs.return_value = []
mock_get_recent_ops_playbook_runs.return_value = {"runs": []}
mock_list_release_rollouts.return_value = []
mock_get_release_summary.return_value = {}
mock_get_latest_release.return_value = {}
mock_get_release_launchpad.return_value = {"launchpad_status": {"status": "ready"}}
mock_build_release_execution_mode_recommendation.return_value = {
"recommended_mode": "remote-agent",
"recommended_mode_label": "remote-agent",
"reason": "",
"options": [],
"recommended_gate": {
"status": "release_not_ready",
"summary": "",
"target_nodes": [],
"target_node_codes": [],
"blocking_reasons": [],
"warning_reasons": [],
"execution_mode": "remote-agent",
"execution_mode_label": "remote-agent",
"operational_readiness": {"rows": [], "summary": {}},
},
"gates": {},
}
overview = get_ops_overview()
execution_scene = overview["execution_scene"]
log_sync = overview["execution_scene"]["log_sync"]
self.assertEqual("有效执行节点 1 台;正在执行/领任务 1 台;近窗刚有吞吐 0 台;在线但未参与 0 台", execution_scene["summary"])
self.assertEqual(1, execution_scene["counts"]["dispatch_active"])
self.assertEqual(0, execution_scene["counts"]["recent_only"])
self.assertEqual(0, execution_scene["counts"]["standby"])
self.assertEqual(0, execution_scene["counts"]["load_syncing"])
self.assertEqual(1, len(execution_scene["dispatch_active_nodes"]))
self.assertEqual("dispatch_active", execution_scene["dispatch_active_nodes"][0]["participation_bucket"])
self.assertEqual([], execution_scene["recent_only_nodes"])
self.assertEqual([], execution_scene["load_syncing_nodes"])
self.assertTrue(log_sync["enabled"])
self.assertEqual("key", log_sync["mode"])
self.assertEqual(2, log_sync["preview_line_count"])
self.assertEqual(
[
"[2026-04-17 10:00:00] [mainland-worker-01] 开始检测域名: alpha.com",
"[2026-04-17 10:01:00] [mainland-worker-01] 域名检测完成: alpha.com",
],
log_sync["preview_lines"],
)
self.assertEqual(1, log_sync["covered_participating_node_count"])
self.assertEqual([], log_sync["missing_participating_nodes"])
self.assertEqual(1, len(log_sync["source_node_summaries"]))
self.assertEqual("mainland-worker-01", log_sync["source_node_summaries"][0]["node_code"])
self.assertEqual(2, log_sync["source_node_summaries"][0]["line_count"])
self.assertEqual(2, log_sync["source_node_summaries"][0]["key_line_count"])
self.assertEqual([], log_sync["missing_participating_node_summaries"])
self.assertEqual("ready", overview["release_hub"]["launchpad"]["launchpad_status"]["status"])
@patch("app.services.ops_service.build_release_execution_mode_recommendation")
@patch("app.services.ops_service.get_release_launchpad")
@patch("app.services.ops_service.get_latest_release")
@patch("app.services.ops_service.get_release_summary")
@patch("app.services.ops_service.list_release_rollouts")
@patch("app.services.ops_service.get_recent_ops_playbook_runs")
@patch("app.services.ops_service.list_ops_jobs")
@patch("app.services.ops_service.list_managed_nodes_with_agent_state")
@patch("app.services.ops_service.get_ops_job_summary")
@patch("app.services.ops_service.get_sync_summary")
@patch("app.services.ops_service.get_runtime_status")
def test_ops_overview_marks_missing_remote_log_reason_code(
self,
mock_get_runtime_status,
mock_get_sync_summary,
mock_get_ops_job_summary,
mock_list_managed_nodes_with_agent_state,
mock_list_ops_jobs,
mock_get_recent_ops_playbook_runs,
mock_list_release_rollouts,
mock_get_release_summary,
mock_get_latest_release,
mock_get_release_launchpad,
mock_build_release_execution_mode_recommendation,
) -> None:
mock_get_runtime_status.return_value = {
"readiness": {"status": "ready", "summary": "ok"},
"detect": {
"log_sync": {
"enabled": True,
"mode": "key",
"line_count": 0,
"source_node_count": 0,
"source_nodes": [],
"last_at": "",
"last_line": "",
"preview_lines": [],
},
"participation_summary": {
"participating_nodes": 1,
"dispatch_active_nodes": 1,
"recent_only_nodes": 0,
"non_participating_nodes": 0,
"standby_nodes": 0,
"load_syncing_nodes": 0,
"effective_online_nodes": 1,
"dedicated_worker_nodes": 1,
"controller_worker_nodes": 0,
"dispatch_active_node_codes": ["mainland-worker-01"],
"recent_only_node_codes": [],
"non_participating_node_codes": [],
"standby_node_codes": [],
"load_syncing_node_codes": [],
"summary": "有效执行节点 1 台;正在执行/领任务 1 台;近窗刚有吞吐 0 台;在线但未参与 0 台",
},
"participating_nodes": [
{
"node_code": "mainland-worker-01",
"region": "mainland",
"role": "worker",
"status": "busy",
"current_load": 2,
"participation_state": "claimed",
"participation_label": "已领待跑",
"participation_reason": "已领取 2 项任务,等待线程继续执行。",
"is_dispatch_active": True,
}
],
"non_participating_nodes": [],
"active_job": {"job_code": "detect-001", "status": "running"},
"phase_label": "运行中",
"phase_detail": "等待样本",
"recent_event": "",
"recent_warning": "",
},
"cluster": {"summary": {}, "nodes": []},
}
mock_get_sync_summary.return_value = {}
mock_get_ops_job_summary.return_value = {"total": 0, "queued": 0, "running": 0, "awaiting_approval": 0}
mock_list_managed_nodes_with_agent_state.return_value = {"nodes": [], "summary": {}}
mock_list_ops_jobs.return_value = []
mock_get_recent_ops_playbook_runs.return_value = {"runs": []}
mock_list_release_rollouts.return_value = []
mock_get_release_summary.return_value = {}
mock_get_latest_release.return_value = {}
mock_get_release_launchpad.return_value = {"launchpad_status": {"status": "ready"}}
mock_build_release_execution_mode_recommendation.return_value = {
"recommended_mode": "remote-agent",
"recommended_mode_label": "remote-agent",
"reason": "",
"options": [],
"recommended_gate": {
"status": "release_not_ready",
"summary": "",
"target_nodes": [],
"target_node_codes": [],
"blocking_reasons": [],
"warning_reasons": [],
"execution_mode": "remote-agent",
"execution_mode_label": "remote-agent",
"operational_readiness": {"rows": [], "summary": {}},
},
"gates": {},
}
overview = get_ops_overview()
missing_rows = overview["execution_scene"]["log_sync"]["missing_participating_node_summaries"]
self.assertEqual(1, len(missing_rows))
self.assertEqual("waiting_sample", missing_rows[0]["missing_reason_code"])
self.assertEqual("dispatch_active", overview["execution_scene"]["participating_nodes"][0]["participation_bucket"])
@patch("app.services.ops_service.get_ops_activity_stream")
@patch("app.services.ops_service.build_release_execution_mode_recommendation")
@patch("app.services.ops_service.get_release_launchpad")
@patch("app.services.ops_service.get_latest_release")
@patch("app.services.ops_service.get_release_summary")
@patch("app.services.ops_service.list_release_rollouts")
@patch("app.services.ops_service.get_recent_ops_playbook_runs")
@patch("app.services.ops_service.list_ops_jobs")
@patch("app.services.ops_service.list_managed_nodes_with_agent_state")
@patch("app.services.ops_service.get_ops_job_summary")
@patch("app.services.ops_service.get_sync_summary")
@patch("app.services.ops_service.get_runtime_status")
def test_ops_overview_prioritizes_node_agent_dead_letter_queue(
self,
mock_get_runtime_status,
mock_get_sync_summary,
mock_get_ops_job_summary,
mock_list_managed_nodes_with_agent_state,
mock_list_ops_jobs,
mock_get_recent_ops_playbook_runs,
mock_list_release_rollouts,
mock_get_release_summary,
mock_get_latest_release,
mock_get_release_launchpad,
mock_build_release_execution_mode_recommendation,
mock_get_ops_activity_stream,
) -> None:
mock_get_runtime_status.return_value = {
"readiness": {"status": "ready", "summary": "ok"},
"detect": {
"log_sync": {
"enabled": False,
"mode": "key",
"line_count": 0,
"source_node_count": 0,
"source_nodes": [],
"last_at": "",
"last_line": "",
"preview_lines": [],
},
"participation_summary": {},
"participating_nodes": [],
"non_participating_nodes": [],
"active_job": {},
"phase_label": "",
"phase_detail": "",
"recent_event": "",
"recent_warning": "",
},
"cluster": {
"summary": {
"online_worker_nodes": 1,
"online_control_nodes": 1,
"dedicated_online_worker_nodes": 1,
"busy_nodes": [],
},
"nodes": [
{
"node_code": "mainland-worker-01",
"region": "mainland",
"role": "worker",
"status": "online",
"current_load": 0,
"hostname": "worker-01",
"last_heartbeat_at": "2026-04-18 05:10:00",
"is_effective_worker": True,
"detect_participating": False,
}
],
},
}
mock_get_sync_summary.return_value = {}
mock_get_ops_job_summary.return_value = {"total": 0, "queued": 0, "running": 0, "awaiting_approval": 0}
mock_list_managed_nodes_with_agent_state.return_value = {
"nodes": [
{
"node_code": "mainland-worker-01",
"region": "mainland",
"role": "worker",
"is_managed": True,
"is_enabled": True,
"is_agent_online": True,
"cluster_status": "online",
"cluster_is_effective_worker": True,
"cluster_detect_participating": False,
"delivery_queue_state": "dead_letter",
"delivery_queue_label": "死信 2",
"delivery_queue_reason": "当前存在 2 条死信记录,建议优先查看节点日志或诊断编排。",
"delivery_queue_pending_count": 1,
"delivery_queue_dead_letter_count": 2,
}
],
"summary": {
"total": 1,
"managed_total": 1,
"managed_enabled": 1,
"online": 1,
"stale": 0,
"pending_bootstrap": 0,
"runtime_only": 0,
"unmanaged": 0,
"token_issue": 0,
"queue_retrying_nodes": 0,
"queue_dead_letter_nodes": 1,
"queue_pending_records": 1,
"queue_dead_letter_records": 2,
"status_counts": {},
},
}
mock_list_ops_jobs.return_value = []
mock_get_recent_ops_playbook_runs.return_value = {"runs": []}
mock_list_release_rollouts.return_value = []
mock_get_release_summary.return_value = {}
mock_get_latest_release.return_value = {}
mock_get_release_launchpad.return_value = {"launchpad_status": {}}
mock_build_release_execution_mode_recommendation.return_value = {
"recommended_mode": "remote-agent",
"recommended_mode_label": "remote-agent",
"reason": "",
"options": [],
"recommended_gate": {
"status": "ready",
"summary": "",
"target_nodes": [],
"target_node_codes": [],
"blocking_reasons": [],
"warning_reasons": [],
"execution_mode": "remote-agent",
"execution_mode_label": "remote-agent",
"operational_readiness": {"rows": [], "summary": {}},
"release": {"id": 0},
},
"gates": {},
}
mock_get_ops_activity_stream.return_value = {"items": []}
overview = get_ops_overview()
self.assertEqual("node-agent-dead-letter", overview["recommendation"]["key"])
self.assertEqual("先处理 Agent 死信", overview["recommendation"]["priority"])
self.assertEqual("replay_delivery_queue", overview["recommendation"]["primary_action_code"])
self.assertEqual(
"delivery.queue.replay",
overview["recommendation"]["primary_action_payload"]["template_key"],
)
self.assertEqual(1, overview["managed_nodes"]["queue_dead_letter_nodes"])
self.assertEqual(2, overview["managed_nodes"]["queue_dead_letter_records"])
@patch("app.services.ops_service.get_ops_activity_stream")
@patch("app.services.ops_service.build_release_execution_mode_recommendation")
@patch("app.services.ops_service.get_release_launchpad")
@patch("app.services.ops_service.get_latest_release")
@patch("app.services.ops_service.get_release_summary")
@patch("app.services.ops_service.list_release_rollouts")
@patch("app.services.ops_service.get_recent_ops_playbook_runs")
@patch("app.services.ops_service.list_ops_jobs")
@patch("app.services.ops_service.list_managed_nodes_with_agent_state")
@patch("app.services.ops_service.get_ops_job_summary")
@patch("app.services.ops_service.get_sync_summary")
@patch("app.services.ops_service.get_runtime_status")
def test_ops_overview_prioritizes_start_delivery_gap_from_activity_stream(
self,
mock_get_runtime_status,
mock_get_sync_summary,
mock_get_ops_job_summary,
mock_list_managed_nodes_with_agent_state,
mock_list_ops_jobs,
mock_get_recent_ops_playbook_runs,
mock_list_release_rollouts,
mock_get_release_summary,
mock_get_latest_release,
mock_get_release_launchpad,
mock_build_release_execution_mode_recommendation,
mock_get_ops_activity_stream,
) -> None:
mock_get_runtime_status.return_value = {
"readiness": {"status": "ready", "summary": "ok"},
"detect": {
"log_sync": {
"enabled": True,
"mode": "key",
"line_count": 12,
"source_node_count": 1,
"source_nodes": ["mainland-worker-01"],
"last_at": "2026-04-18 11:00:00",
"last_line": "ok",
"preview_lines": ["ok"],
},
"participation_summary": {},
"participating_nodes": [],
"non_participating_nodes": [],
"active_job": {},
"phase_label": "",
"phase_detail": "",
"recent_event": "",
"recent_warning": "",
},
"cluster": {"summary": {}, "nodes": []},
}
mock_get_sync_summary.return_value = {}
mock_get_ops_job_summary.return_value = {"total": 0, "queued": 0, "running": 0, "awaiting_approval": 0}
mock_list_managed_nodes_with_agent_state.return_value = {"nodes": [], "summary": {}}
mock_list_ops_jobs.return_value = []
mock_get_recent_ops_playbook_runs.return_value = {"runs": []}
mock_list_release_rollouts.return_value = []
mock_get_release_summary.return_value = {}
mock_get_latest_release.return_value = {}
mock_get_release_launchpad.return_value = {"launchpad_status": {}}
mock_build_release_execution_mode_recommendation.return_value = {
"recommended_mode": "remote-agent",
"recommended_mode_label": "remote-agent",
"reason": "",
"options": [],
"recommended_gate": {
"status": "ready",
"summary": "",
"target_nodes": [],
"target_node_codes": [],
"blocking_reasons": [],
"warning_reasons": [],
"execution_mode": "remote-agent",
"execution_mode_label": "remote-agent",
"operational_readiness": {"rows": [], "summary": {}},
"release": {"id": 0},
},
"gates": {},
}
mock_get_ops_activity_stream.return_value = {
"items": [
{
"kind": "ops_job",
"activity_key": "ops-job:81",
"status": "attention",
"job_status": "running",
"summary": "节点已接单执行,但开始回执未成功送达控制面。",
"job_code": "ops-agent-81",
"start_delivery_state": "failed_local",
"source_focus_ref": {
"kind": "ops_job_event",
"job_id": 81,
"job_code": "ops-agent-81",
"event_key": "job-event:81",
"target_node_code": "mainland-worker-01",
},
"target_node_codes": ["mainland-worker-01"],
"ui_intent": {"kind": "job_events", "job_id": 81},
}
]
}
overview = get_ops_overview()
self.assertEqual("activity-start-delivery-gap", overview["recommendation"]["key"])
self.assertEqual("先检查开始回执异常", overview["recommendation"]["priority"])
self.assertEqual("focus_latest_job_events", overview["recommendation"]["primary_action_code"])
self.assertEqual("ops_job_event", overview["recommendation"]["primary_action_payload"]["focus_ref"]["kind"])
@patch("app.services.ops_service.build_release_execution_mode_recommendation")
@patch("app.services.ops_service.get_release_launchpad")
@patch("app.services.ops_service.get_latest_release")
@patch("app.services.ops_service.get_release_summary")
@patch("app.services.ops_service.list_release_rollouts")
@patch("app.services.ops_service.get_recent_ops_playbook_runs")
@patch("app.services.ops_service.list_ops_jobs")
@patch("app.services.ops_service.list_managed_nodes_with_agent_state")
@patch("app.services.ops_service.get_ops_job_summary")
@patch("app.services.ops_service.get_sync_summary")
@patch("app.services.ops_service.get_runtime_status")
def test_ops_overview_uses_release_launchpad_driver_card_when_execution_plane_is_clean(
self,
mock_get_runtime_status,
mock_get_sync_summary,
mock_get_ops_job_summary,
mock_list_managed_nodes_with_agent_state,
mock_list_ops_jobs,
mock_get_recent_ops_playbook_runs,
mock_list_release_rollouts,
mock_get_release_summary,
mock_get_latest_release,
mock_get_release_launchpad,
mock_build_release_execution_mode_recommendation,
) -> None:
mock_get_runtime_status.return_value = {
"readiness": {"status": "ready", "summary": "ok"},
"detect": {
"log_sync": {
"enabled": True,
"mode": "key",
"line_count": 0,
"source_node_count": 0,
"source_nodes": [],
"last_at": "",
"last_line": "",
"preview_lines": [],
},
"participation_summary": {},
"participating_nodes": [],
"non_participating_nodes": [],
"active_job": {},
"phase_label": "",
"phase_detail": "",
"recent_event": "",
"recent_warning": "",
},
"cluster": {
"summary": {
"online_worker_nodes": 1,
"online_control_nodes": 1,
"dedicated_online_worker_nodes": 1,
"busy_nodes": [],
},
"nodes": [
{
"node_code": "mainland-worker-01",
"region": "mainland",
"role": "worker",
"status": "online",
"current_load": 0,
"hostname": "worker-01",
"last_heartbeat_at": "2026-04-18 10:00:00",
"is_effective_worker": True,
"detect_participating": False,
}
],
},
}
mock_get_sync_summary.return_value = {}
mock_get_ops_job_summary.return_value = {"total": 0, "queued": 0, "running": 0, "awaiting_approval": 0}
mock_list_managed_nodes_with_agent_state.return_value = {
"nodes": [
{
"node_code": "mainland-worker-01",
"region": "mainland",
"role": "worker",
"is_managed": True,
"is_enabled": True,
"is_agent_online": True,
"cluster_status": "online",
"cluster_is_effective_worker": True,
"cluster_detect_participating": False,
}
],
"summary": {
"total": 1,
"managed_total": 1,
"managed_enabled": 1,
"online": 1,
"stale": 0,
"pending_bootstrap": 0,
"runtime_only": 0,
"unmanaged": 0,
"token_issue": 0,
"status_counts": {},
},
}
mock_list_ops_jobs.return_value = []
mock_get_recent_ops_playbook_runs.return_value = {"runs": []}
mock_list_release_rollouts.return_value = []
mock_get_release_summary.return_value = {}
mock_get_latest_release.return_value = {}
mock_get_release_launchpad.return_value = {
"channel": "stable",
"control_plane_base_url": "http://control.example:8100",
"latest_package": {
"available": True,
"package_name": "domaincheck_release_demo.tar.gz",
},
"latest_release": {
"id": 17,
"release_version": "domaincheck_release_demo",
"channel": "stable",
},
"worker_rollout_preview": {
"message": "智能 Rollout 预检通过",
"smart_rollout": {
"available": True,
"summary": "已按 Worker 模式预选 1 台节点。",
"target_node_codes": ["mainland-worker-01"],
},
},
"control_rollout_preview": {
"message": "",
"smart_rollout": {
"available": False,
"target_node_codes": [],
},
},
"launchpad_status": {
"status": "ready",
"status_label": "可发 Worker 灰度",
"summary": "最新发布包、Release 记录与 Worker 智能 Rollout 预检均已收口。",
"recommended_action_code": "publish_latest_worker",
},
}
mock_build_release_execution_mode_recommendation.return_value = {
"recommended_mode": "ssh",
"recommended_mode_label": "SSH",
"reason": "当前目标节点通过 SSH 路径的门禁更完整,建议先走 SSH 发布。",
"options": [],
"recommended_gate": {
"status": "ready",
"summary": "",
"target_nodes": [],
"target_node_codes": [],
"blocking_reasons": [],
"warning_reasons": [],
"execution_mode": "ssh",
"execution_mode_label": "SSH",
"operational_readiness": {"rows": [], "summary": {}},
},
"gates": {},
}
overview = get_ops_overview()
self.assertEqual("driver_recommendations", overview["recommendation"]["source"])
self.assertEqual("release-launchpad-ready", overview["recommendation"]["key"])
self.assertEqual("当前可以发 Worker 灰度", overview["recommendation"]["priority"])
self.assertEqual("publish_latest_worker", overview["recommendation"]["primary_action_code"])
self.assertEqual(["mainland-worker-01"], overview["recommendation"]["primary_node_codes"])
self.assertEqual("ready", overview["release_hub"]["launchpad"]["launchpad_status"]["status"])
self.assertEqual("ssh", overview["release_hub"]["default_rollout_gate"]["execution_mode"])
self.assertEqual("SSH", overview["release_hub"]["default_rollout_execution"]["recommended_mode_label"])
if __name__ == "__main__":
unittest.main()