Files
getDomain/domain-api/tests/test_ops_service_activity.py
2026-04-18 23:52:51 +08:00

608 lines
27 KiB
Python

import unittest
from typing import Optional
from unittest.mock import patch
from app.services.ops_service import get_ops_activity_stream
class OpsServiceActivityTests(unittest.TestCase):
@staticmethod
def _runtime_status(detect: Optional[dict] = None) -> dict:
return {"detect": dict(detect or {})}
@patch("app.services.ops_service.get_ops_runbook")
@patch("app.services.ops_service.list_release_rollouts")
@patch("app.services.ops_service.list_ops_jobs")
@patch("app.services.ops_service.get_recent_ops_playbook_runs")
@patch("app.services.ops_service.list_managed_nodes_with_agent_state")
def test_activity_stream_uses_job_detail_for_bootstrap_jobs(
self,
mock_list_managed_nodes_with_agent_state,
mock_get_recent_ops_playbook_runs,
mock_list_ops_jobs,
mock_list_release_rollouts,
mock_get_ops_runbook,
) -> None:
mock_get_ops_runbook.return_value = {"control_sequences": []}
mock_list_managed_nodes_with_agent_state.return_value = {
"nodes": [
{
"node_code": "mainland-worker-02",
"agent_state": "pending_bootstrap",
"agent_state_label": "待接入",
}
]
}
mock_get_recent_ops_playbook_runs.return_value = {"runs": []}
mock_list_ops_jobs.return_value = [
{
"id": 51,
"job_code": "ops-bootstrap-51",
"action": "node.bootstrap",
"status": "success",
"target_node_code": "mainland-worker-02",
"created_at": "2026-04-17 10:00:00",
"updated_at": "2026-04-17 10:01:00",
"result": {
"bootstrap_plan": {
"bootstrap_script_name": "bootstrap-node-agent-mainland-worker-02.sh",
}
},
}
]
mock_list_release_rollouts.return_value = []
with patch(
"app.services.ops_service.get_runtime_status",
return_value=self._runtime_status(),
):
payload = get_ops_activity_stream(limit=10, scan_limit=20)
self.assertEqual(1, len(payload["items"]))
item = payload["items"][0]
self.assertEqual("ops_job", item["kind"])
self.assertEqual("节点接入工单", item["title"])
self.assertEqual("job_detail", item["ui_intent"]["kind"])
self.assertEqual("job_detail", item["ui_intent_kind"])
self.assertEqual("控制面", item["execution_mode_label"])
self.assertEqual("成功", item["status_label"])
self.assertIn("尚未回连 Agent", item["summary"])
self.assertEqual(item["summary"], item["summary_text"])
self.assertIn("方式 控制面", item["meta_text"])
self.assertIn("bootstrap-node-agent-mainland-worker-02.sh", item["meta_text"])
self.assertEqual(["mainland-worker-02"], item["target_node_codes"])
self.assertEqual(51, item["focus_ref"]["job_id"])
self.assertEqual("node.bootstrap", item["focus_ref"]["action"])
self.assertEqual("ops_agent_protocol", item["contract_navigation"]["primary_contract_key"])
self.assertIn("ops_agent_protocol", item["contract_navigation"]["contract_keys"])
@patch("app.services.ops_service.get_ops_runbook")
@patch("app.services.ops_service.list_release_rollouts")
@patch("app.services.ops_service.list_ops_jobs")
@patch("app.services.ops_service.get_recent_ops_playbook_runs")
@patch("app.services.ops_service.list_managed_nodes_with_agent_state")
def test_activity_stream_includes_runbook_sequences_after_live_events(
self,
mock_list_managed_nodes_with_agent_state,
mock_get_recent_ops_playbook_runs,
mock_list_ops_jobs,
mock_list_release_rollouts,
mock_get_ops_runbook,
) -> None:
mock_list_managed_nodes_with_agent_state.return_value = {"nodes": []}
mock_get_recent_ops_playbook_runs.return_value = {
"runs": [
{
"run_code": "playbook-001",
"playbook_title": "标准巡检",
"status": "running",
"updated_at": "2026-04-18 09:11:00",
}
]
}
mock_list_ops_jobs.return_value = []
mock_list_release_rollouts.return_value = []
mock_get_ops_runbook.return_value = {
"control_sequences": [
{
"key": "release_progression",
"title": "版本发布与放量",
"status": "warning",
"status_label": "待处理",
"reason": "当前没有足够的执行面接管覆盖。",
"target_scope_label": "Release / Rollout",
"target_node_codes": ["mainland-controller-01"],
"primary_action_code": "fix_managed_nodes",
"primary_resolution": {
"ok": True,
"driver_action_code": "fix_managed_nodes",
"driver_action_label": "补执行面接管",
"driver_node_codes": ["mainland-controller-01"],
"message": "当前建议先补执行面接管。",
"resolved_at": "2026-04-18 09:12:00",
},
}
]
}
with patch(
"app.services.ops_service.get_runtime_status",
return_value=self._runtime_status(),
):
payload = get_ops_activity_stream(limit=10, scan_limit=20)
self.assertEqual(2, len(payload["items"]))
self.assertEqual("playbook_run", payload["items"][0]["kind"])
self.assertEqual("runbook_sequence", payload["items"][1]["kind"])
runbook_item = payload["items"][1]
self.assertEqual("release_progression", runbook_item["sequence_key"])
self.assertEqual("focus_runbook_sequence", runbook_item["ui_intent"]["kind"])
self.assertEqual("fix_managed_nodes", runbook_item["ui_intent"]["driver_action_code"])
self.assertEqual("attention", runbook_item["status"])
self.assertIn("当前建议动作:补执行面接管", runbook_item["summary"])
self.assertIn("release_progression", runbook_item["meta_text"])
self.assertEqual("ops_playbook_contract", runbook_item["contract_navigation"]["primary_contract_key"])
self.assertIn("ops_driver_contract", runbook_item["contract_navigation"]["contract_keys"])
@patch("app.services.ops_service.get_ops_runbook")
@patch("app.services.ops_service.list_release_rollouts")
@patch("app.services.ops_service.list_ops_jobs")
@patch("app.services.ops_service.get_recent_ops_playbook_runs")
@patch("app.services.ops_service.list_managed_nodes_with_agent_state")
def test_activity_stream_surfaces_execution_mode_labels_for_jobs_and_playbooks(
self,
mock_list_managed_nodes_with_agent_state,
mock_get_recent_ops_playbook_runs,
mock_list_ops_jobs,
mock_list_release_rollouts,
mock_get_ops_runbook,
) -> None:
mock_list_managed_nodes_with_agent_state.return_value = {"nodes": []}
mock_get_recent_ops_playbook_runs.return_value = {
"runs": [
{
"run_code": "playbook-ssh-001",
"playbook_title": "关键现场日志",
"group_title": "执行现场",
"status": "running",
"updated_at": "2026-04-18 09:11:00",
"execution_mode": "ssh",
"execution_mode_label": "SSH",
"steps": [{"status": "running"}],
"step_count": 1,
"jobs_total": 2,
"target_node_codes": ["mainland-worker-01"],
}
]
}
mock_list_ops_jobs.return_value = [
{
"id": 61,
"job_code": "ops-ssh-61",
"action": "logs.collect",
"status": "running",
"execution_mode": "ssh",
"execution_mode_label": "SSH",
"requested_by": "web-ui",
"target_node_code": "mainland-worker-01",
"target_type": "node",
"risk_level": "low",
"updated_at": "2026-04-18 09:10:00",
"created_at": "2026-04-18 09:09:00",
}
]
mock_list_release_rollouts.return_value = []
mock_get_ops_runbook.return_value = {"control_sequences": []}
with patch(
"app.services.ops_service.get_runtime_status",
return_value=self._runtime_status(),
):
payload = get_ops_activity_stream(limit=10, scan_limit=20)
self.assertEqual(2, len(payload["items"]))
self.assertEqual("SSH", payload["items"][0]["execution_mode_label"])
self.assertIn("执行 SSH", payload["items"][0]["meta_text"])
self.assertEqual("SSH", payload["items"][1]["execution_mode_label"])
self.assertIn("方式 SSH", payload["items"][1]["summary"])
self.assertIn("执行 SSH", payload["items"][1]["meta_text"])
@patch("app.services.ops_service.get_ops_runbook")
@patch("app.services.ops_service.list_release_rollouts")
@patch("app.services.ops_service.list_ops_jobs")
@patch("app.services.ops_service.get_recent_ops_playbook_runs")
@patch("app.services.ops_service.list_managed_nodes_with_agent_state")
def test_activity_stream_prefers_job_result_summary_and_focus_ref(
self,
mock_list_managed_nodes_with_agent_state,
mock_get_recent_ops_playbook_runs,
mock_list_ops_jobs,
mock_list_release_rollouts,
mock_get_ops_runbook,
) -> None:
mock_list_managed_nodes_with_agent_state.return_value = {"nodes": []}
mock_get_recent_ops_playbook_runs.return_value = {"runs": []}
mock_list_ops_jobs.return_value = [
{
"id": 71,
"job_code": "ops-agent-71",
"action": "logs.collect",
"status": "success",
"execution_mode": "remote-agent",
"execution_mode_label": "远端 Agent",
"requested_by": "codex",
"target_node_code": "mainland-worker-01",
"target_type": "node",
"risk_level": "low",
"updated_at": "2026-04-18 09:20:00",
"created_at": "2026-04-18 09:19:00",
"focus_ref": {
"kind": "ops_job",
"job_id": 71,
"job_code": "ops-agent-71",
"action": "logs.collect",
"target_node_code": "mainland-worker-01",
},
"result": {
"summary_text": "远端 Agent 已回传 120 行 Worker 日志,可直接继续分析。",
"focus_ref": {
"kind": "ops_job",
"job_id": 71,
"job_code": "ops-agent-71",
"action": "logs.collect",
"target_node_code": "mainland-worker-01",
"event_key": "job-event:888",
"step_key": "worker_logs",
},
},
}
]
mock_list_release_rollouts.return_value = []
mock_get_ops_runbook.return_value = {"control_sequences": []}
with patch(
"app.services.ops_service.get_runtime_status",
return_value=self._runtime_status(),
):
payload = get_ops_activity_stream(limit=10, scan_limit=20)
self.assertEqual(1, len(payload["items"]))
item = payload["items"][0]
self.assertEqual("ops_job", item["kind"])
self.assertEqual("远端 Agent 已回传 120 行 Worker 日志,可直接继续分析。", item["summary"])
self.assertEqual("ops_job", item["focus_ref"]["kind"])
self.assertEqual("job-event:888", item["focus_ref"]["event_key"])
self.assertEqual("worker_logs", item["focus_ref"]["step_key"])
self.assertIn("执行 远端 Agent", item["meta_text"])
self.assertEqual("ops_observability_contract", item["contract_navigation"]["primary_contract_key"])
self.assertIn("ops_stack_diagnosis_contract", item["contract_navigation"]["contract_keys"])
@patch("app.services.ops_service.get_ops_runbook")
@patch("app.services.ops_service.list_release_rollouts")
@patch("app.services.ops_service.list_ops_jobs")
@patch("app.services.ops_service.get_recent_ops_playbook_runs")
@patch("app.services.ops_service.list_managed_nodes_with_agent_state")
def test_activity_stream_marks_job_attention_when_start_delivery_failed_locally(
self,
mock_list_managed_nodes_with_agent_state,
mock_get_recent_ops_playbook_runs,
mock_list_ops_jobs,
mock_list_release_rollouts,
mock_get_ops_runbook,
) -> None:
mock_list_managed_nodes_with_agent_state.return_value = {"nodes": []}
mock_get_recent_ops_playbook_runs.return_value = {"runs": []}
mock_list_ops_jobs.return_value = [
{
"id": 81,
"job_code": "ops-agent-81",
"action": "logs.collect",
"status": "running",
"execution_mode": "remote-agent",
"execution_mode_label": "远端 Agent",
"requested_by": "codex",
"target_node_code": "mainland-worker-01",
"target_type": "node",
"risk_level": "low",
"updated_at": "2026-04-18 10:20:00",
"created_at": "2026-04-18 10:19:00",
"focus_ref": {
"kind": "ops_job",
"job_id": 81,
"job_code": "ops-agent-81",
"action": "logs.collect",
"target_node_code": "mainland-worker-01",
},
"result": {
"summary_text": "节点端已经收到采集任务。",
"start_delivery_state": "failed_local",
"start_delivery_error": "temporary offline",
"source_focus_ref": {
"kind": "ops_job_event",
"job_id": 81,
"job_code": "ops-agent-81",
"event_key": "job-event:81",
"target_node_code": "mainland-worker-01",
},
},
}
]
mock_list_release_rollouts.return_value = []
mock_get_ops_runbook.return_value = {"control_sequences": []}
with patch(
"app.services.ops_service.get_runtime_status",
return_value=self._runtime_status(),
):
payload = get_ops_activity_stream(limit=10, scan_limit=20)
self.assertEqual(1, len(payload["items"]))
item = payload["items"][0]
self.assertEqual("ops_job", item["kind"])
self.assertEqual("attention", item["status"])
self.assertEqual("running", item["job_status"])
self.assertEqual("failed_local", item["start_delivery_state"])
self.assertEqual("ops_job_start_delivery_failed_local", item["problem_code"])
self.assertIn("开始回执未成功送达控制面", item["summary"])
self.assertIn("temporary offline", item["summary"])
self.assertIn("开始回执异常", item["meta_text"])
self.assertEqual("ops_job_event", item["source_focus_ref"]["kind"])
self.assertEqual("job-event:81", item["source_focus_ref"]["event_key"])
self.assertEqual("attention", payload["summary"]["status"])
self.assertEqual("待处理", payload["summary"]["status_label"])
@patch("app.services.ops_service.get_ops_runbook")
@patch("app.services.ops_service.list_release_rollouts")
@patch("app.services.ops_service.list_ops_jobs")
@patch("app.services.ops_service.get_recent_ops_playbook_runs")
@patch("app.services.ops_service.list_managed_nodes_with_agent_state")
def test_activity_stream_surfaces_execution_mode_label_for_rollout(
self,
mock_list_managed_nodes_with_agent_state,
mock_get_recent_ops_playbook_runs,
mock_list_ops_jobs,
mock_list_release_rollouts,
mock_get_ops_runbook,
) -> None:
mock_list_managed_nodes_with_agent_state.return_value = {"nodes": []}
mock_get_recent_ops_playbook_runs.return_value = {"runs": []}
mock_list_ops_jobs.return_value = []
mock_list_release_rollouts.return_value = [
{
"id": 201,
"release_id": 18,
"rollout_code": "rollout-20260418093000-a1b2",
"policy": {
"execution_mode": "ssh",
},
"status": "running",
"status_label": "执行中",
"target_nodes": [{"node_code": "mainland-worker-01"}],
"jobs_total": 2,
"jobs_created": 1,
"batch_cursor": 1,
"batches_total": 2,
"result_summary": {},
"summary_text": "当前已覆盖 1/1 台节点,仍有 1 条任务执行中。",
"focus_ref": {
"kind": "release_rollout",
"rollout_id": 201,
"rollout_code": "rollout-20260418093000-a1b2",
"release_id": 18,
"section": "rollout_jobs",
},
"created_by": "web-ui",
"created_at": "2026-04-18 09:30:00",
"updated_at": "2026-04-18 09:31:00",
}
]
mock_get_ops_runbook.return_value = {"control_sequences": []}
with patch(
"app.services.ops_service.get_runtime_status",
return_value=self._runtime_status(),
):
payload = get_ops_activity_stream(limit=10, scan_limit=20)
self.assertEqual(1, len(payload["items"]))
item = payload["items"][0]
self.assertEqual("rollout", item["kind"])
self.assertEqual("SSH", item["execution_mode_label"])
self.assertIn("执行 SSH", item["meta_text"])
self.assertEqual("执行中", item["status_label"])
self.assertIn("当前已覆盖 1/1 台节点", item["summary"])
self.assertEqual("release_rollout", item["focus_ref"]["kind"])
self.assertEqual("release_hub_contract", item["contract_navigation"]["primary_contract_key"])
self.assertIn("release_hub_contract", item["contract_navigation"]["contract_keys"])
@patch("app.services.ops_service.get_ops_runbook")
@patch("app.services.ops_service.list_release_rollouts")
@patch("app.services.ops_service.list_ops_jobs")
@patch("app.services.ops_service.get_recent_ops_playbook_runs")
@patch("app.services.ops_service.list_managed_nodes_with_agent_state")
def test_activity_stream_supports_execution_mode_filter(
self,
mock_list_managed_nodes_with_agent_state,
mock_get_recent_ops_playbook_runs,
mock_list_ops_jobs,
mock_list_release_rollouts,
mock_get_ops_runbook,
) -> None:
mock_list_managed_nodes_with_agent_state.return_value = {"nodes": []}
mock_get_recent_ops_playbook_runs.return_value = {
"runs": [
{
"run_code": "playbook-ssh-001",
"playbook_title": "关键现场日志",
"status": "running",
"updated_at": "2026-04-18 09:11:00",
"execution_mode": "ssh",
"execution_mode_label": "SSH",
"steps": [{"status": "running"}],
"step_count": 1,
"jobs_total": 1,
"target_node_codes": ["mainland-worker-01"],
}
]
}
mock_list_ops_jobs.return_value = [
{
"id": 61,
"job_code": "ops-ssh-61",
"action": "logs.collect",
"status": "running",
"execution_mode": "ssh",
"execution_mode_label": "SSH",
"requested_by": "web-ui",
"target_node_code": "mainland-worker-01",
"target_type": "node",
"risk_level": "low",
"updated_at": "2026-04-18 09:10:00",
"created_at": "2026-04-18 09:09:00",
},
{
"id": 62,
"job_code": "ops-agent-62",
"action": "health.snapshot",
"status": "success",
"execution_mode": "remote-agent",
"execution_mode_label": "远端 Agent",
"requested_by": "web-ui",
"target_node_code": "mainland-worker-02",
"target_type": "node",
"risk_level": "low",
"updated_at": "2026-04-18 09:08:00",
"created_at": "2026-04-18 09:07:00",
},
]
mock_list_release_rollouts.return_value = []
mock_get_ops_runbook.return_value = {
"control_sequences": [
{
"key": "release_progression",
"title": "版本发布与放量",
"status": "warning",
"status_label": "待处理",
"reason": "当前没有足够的执行面接管覆盖。",
"target_scope_label": "Release / Rollout",
"target_node_codes": ["mainland-controller-01"],
}
]
}
with patch(
"app.services.ops_service.get_runtime_status",
return_value=self._runtime_status(),
):
payload = get_ops_activity_stream(limit=10, scan_limit=20, execution_mode="ssh")
self.assertEqual(2, len(payload["items"]))
self.assertTrue(all(item["execution_mode"] == "ssh" for item in payload["items"]))
self.assertEqual("ssh", payload["summary"]["filters"]["execution_mode"])
self.assertEqual(2, payload["summary"]["available_execution_mode_counts"]["ssh"])
self.assertEqual(1, payload["summary"]["available_execution_mode_counts"]["remote-agent"])
@patch("app.services.ops_service.get_ops_runbook")
@patch("app.services.ops_service.list_release_rollouts")
@patch("app.services.ops_service.list_ops_jobs")
@patch("app.services.ops_service.get_recent_ops_playbook_runs")
@patch("app.services.ops_service.list_managed_nodes_with_agent_state")
def test_activity_stream_includes_execution_scene_and_log_sync_items(
self,
mock_list_managed_nodes_with_agent_state,
mock_get_recent_ops_playbook_runs,
mock_list_ops_jobs,
mock_list_release_rollouts,
mock_get_ops_runbook,
) -> None:
mock_list_managed_nodes_with_agent_state.return_value = {"nodes": []}
mock_get_recent_ops_playbook_runs.return_value = {"runs": []}
mock_list_ops_jobs.return_value = []
mock_list_release_rollouts.return_value = []
mock_get_ops_runbook.return_value = {"control_sequences": []}
detect = {
"active_job": {
"job_code": "detect-001",
"latest_cycle_event": {
"created_at": "2026-04-18 09:12:30",
},
},
"phase_label": "运行中",
"phase_detail": "代理池正常,现场可继续观察",
"recent_event": "域名检测完成: example.com",
"recent_warning": "",
"log_sync": {
"enabled": True,
"mode": "key",
"mode_label": "关键回传",
"description": "当前为关键回传;适合常态联调。",
"line_count": 12,
"source_node_count": 1,
"source_nodes": ["mainland-worker-01"],
"last_at": "2026-04-18 09:12:31",
"preview_lines": ["worker log line"],
},
"participation_summary": {
"summary": "当前参与检测节点 1 台,其中执行/已领 1 台,近窗刚有吞吐 0 台。",
"effective_online_nodes": 1,
"participating_nodes": 1,
"dispatch_active_nodes": 1,
"recent_only_nodes": 0,
"non_participating_nodes": 0,
"standby_nodes": 0,
"load_syncing_nodes": 0,
},
"participating_nodes": [
{
"node_code": "mainland-worker-01",
"region": "mainland",
"role": "worker",
"status": "busy",
"last_heartbeat_at": "2026-04-18 09:12:29",
"participation_state": "dispatch_active",
"is_dispatch_active": True,
}
],
"non_participating_nodes": [],
}
with patch(
"app.services.ops_service.get_runtime_status",
return_value=self._runtime_status(detect),
):
payload = get_ops_activity_stream(limit=10, scan_limit=20)
items_by_kind = {item["kind"]: item for item in payload["items"]}
self.assertIn("execution_scene", items_by_kind)
self.assertIn("log_sync", items_by_kind)
execution_scene_item = items_by_kind["execution_scene"]
log_sync_item = items_by_kind["log_sync"]
self.assertEqual("running", execution_scene_item["status"])
self.assertEqual("执行中", execution_scene_item["status_label"])
self.assertEqual("focus_execution_scene", execution_scene_item["ui_intent"]["kind"])
self.assertEqual("focus_execution_scene", execution_scene_item["ui_intent_kind"])
self.assertEqual("execution_scene", execution_scene_item["focus_ref"]["kind"])
self.assertEqual(execution_scene_item["summary"], execution_scene_item["summary_text"])
self.assertIn("执行/已领 1", execution_scene_item["meta_text"])
self.assertEqual("ops_observability_contract", execution_scene_item["contract_navigation"]["primary_contract_key"])
self.assertIn("ops_stack_diagnosis_contract", execution_scene_item["contract_navigation"]["contract_keys"])
self.assertEqual("running", log_sync_item["status"])
self.assertEqual("执行中", log_sync_item["status_label"])
self.assertEqual("control-plane", log_sync_item["execution_mode"])
self.assertEqual("控制面", log_sync_item["execution_mode_label"])
self.assertEqual("focus_execution_scene", log_sync_item["ui_intent"]["kind"])
self.assertTrue(log_sync_item["ui_intent"]["open_log_drawer"])
self.assertIn("关键日志观察模式", log_sync_item["summary"])
self.assertEqual("running", payload["summary"]["status"])
self.assertEqual("执行中", payload["summary"]["status_label"])
self.assertIn("最近活动里存在正在推进", payload["summary"]["summary_text"])
self.assertEqual("ops_observability_contract", payload["contract_navigation"]["primary_contract_key"])
self.assertIn("ops_stack_diagnosis_contract", payload["contract_navigation"]["contract_keys"])
if __name__ == "__main__":
unittest.main()