admin/getDomain
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity
Files
ad513211e648a2ffffec88f120f62fb399abd307
getDomain/domain-api/tests/test_ops_service_stack_diagnosis.py
Your Name b9c29481b5 feat: add ops center and node onboarding flow
2026-04-18 23:52:51 +08:00

972 lines
42 KiB
Python
Raw Blame History

This file contains ambiguous Unicode characters
This file contains Unicode characters that might be confused with other characters. If you think that this is intentional, you can safely ignore this warning. Use the Escape button to reveal them.
import unittest
from unittest.mock import patch
from app.services.ops_service import get_ops_stack_diagnosis
class OpsServiceStackDiagnosisTests(unittest.TestCase):
@patch("app.services.ops_service.get_runtime_build_info")
@patch("app.services.ops_service.get_ops_activity_stream")
@patch("app.services.ops_service.get_recent_ops_playbook_runs")
@patch("app.services.ops_service.get_release_launchpad")
@patch("app.services.ops_service.list_managed_nodes_with_agent_state")
@patch("app.services.ops_service.get_ops_overview")
@patch("app.services.ops_service.get_ops_link_snapshot")
@patch("app.services.ops_service.get_ops_contract_registry")
def test_stack_diagnosis_prioritizes_api_restart_when_runtime_build_schema_is_stale(
self,
mock_get_ops_contract_registry,
mock_get_ops_link_snapshot,
mock_get_ops_overview,
mock_list_managed_nodes_with_agent_state,
mock_get_release_launchpad,
mock_get_recent_ops_playbook_runs,
mock_get_ops_activity_stream,
mock_get_runtime_build_info,
) -> None:
mock_get_ops_contract_registry.return_value = {
"registry_version": "2026-04-18",
"schema_version": "v1",
"contracts": [{"key": "ops_stack_diagnosis_contract"}],
}
mock_get_ops_link_snapshot.return_value = {
"status": "ready",
"headline": "当前总检可用。",
"next_actions": [],
}
mock_get_ops_overview.return_value = {
"recommendation": {
"primary_action_code": "fix_managed_nodes",
"title": "先补节点接管",
"focus_ref": {"kind": "managed_node", "node_code": "mainland-worker-01"},
},
"execution_scene": {},
"log_sync": {},
"release_hub": {},
}
mock_list_managed_nodes_with_agent_state.return_value = {"nodes": [], "summary": {}}
mock_get_release_launchpad.return_value = {"launchpad_status": {}, "latest_release": {}}
mock_get_recent_ops_playbook_runs.return_value = {"runs": []}
mock_get_ops_activity_stream.return_value = {"items": []}
mock_get_runtime_build_info.return_value = {
"source": "git",
"commit_sha": "246838ae4c07",
"repository_capabilities": {
"supports_install_command_block": True,
"supports_multi_layout_bootstrap": True,
},
"route_surface": {
"mode": "registered",
"registered_paths_total": 24,
"surface_complete": True,
"missing_keys": [],
"expected_paths": {
"ops_contracts": "/api/v1/ops/contracts",
"ops_stack_diagnosis": "/api/v1/ops/stack-diagnosis",
},
},
}
payload = get_ops_stack_diagnosis(base_url="http://127.0.0.1:8100")
issue = next(item for item in payload["diagnosis"]["issues"] if item["code"] == "runtime_build_schema_stale")
self.assertEqual("api-restart", issue["action_code"])
self.assertEqual("api-restart", payload["diagnosis"]["next_step"]["action_code"])
self.assertEqual("runtime_recovery", payload["diagnosis"]["operator_decision"]["lane"])
self.assertEqual("api-restart", payload["diagnosis"]["operator_decision"]["primary_command_key"])
self.assertEqual(
"bash domain-api/deploy/multi-region/drive_ops_center.sh runtime-refresh-recover http://127.0.0.1:8100",
payload["diagnosis"]["next_actions"]["primary_command"],
)
self.assertEqual(
"bash domain-api/deploy/multi-region/drive_ops_center.sh runtime-refresh-recover http://127.0.0.1:8100",
payload["diagnosis"]["recommended_commands"]["api-restart"],
)
self.assertEqual(
"bash domain-api/deploy/multi-region/drive_ops_center.sh runtime-refresh-recover http://127.0.0.1:8100",
payload["diagnosis"]["recommended_commands"]["runtime-refresh-recover"],
)
self.assertEqual(
"bash domain-api/deploy/multi-region/drive_ops_center.sh stack-diagnosis http://127.0.0.1:8100 summary",
payload["diagnosis"]["recommended_commands"]["stack-diagnosis"],
)
self.assertEqual(
payload["diagnosis"]["recommended_commands"]["stack-diagnosis"],
payload["diagnosis"]["recommended_commands"]["stack-check"],
)
self.assertTrue(payload["runtime_build_info"]["runtime_schema_stale"])
self.assertFalse(payload["runtime_build_info"]["route_surface_declares_bootstrap_plan"])
self.assertTrue(
any("运行中 API 还是旧 schema" in hint for hint in payload["diagnosis"]["operator_hints"])
)
@patch("app.services.ops_service.get_runtime_build_info")
@patch("app.services.ops_service.get_ops_activity_stream")
@patch("app.services.ops_service.get_recent_ops_playbook_runs")
@patch("app.services.ops_service.get_release_launchpad")
@patch("app.services.ops_service.list_managed_nodes_with_agent_state")
@patch("app.services.ops_service.get_ops_overview")
@patch("app.services.ops_service.get_ops_link_snapshot")
@patch("app.services.ops_service.get_ops_contract_registry")
def test_stack_diagnosis_surfaces_runtime_route_surface_gap(
self,
mock_get_ops_contract_registry,
mock_get_ops_link_snapshot,
mock_get_ops_overview,
mock_list_managed_nodes_with_agent_state,
mock_get_release_launchpad,
mock_get_recent_ops_playbook_runs,
mock_get_ops_activity_stream,
mock_get_runtime_build_info,
) -> None:
mock_get_ops_contract_registry.return_value = {
"registry_version": "2026-04-18",
"schema_version": "v1",
"contracts": [],
}
mock_get_ops_link_snapshot.return_value = {
"status": "ready",
"headline": "当前总检可用。",
"next_actions": [],
}
mock_get_ops_overview.return_value = {
"recommendation": {},
"execution_scene": {},
"log_sync": {},
"release_hub": {},
}
mock_list_managed_nodes_with_agent_state.return_value = {"nodes": [], "summary": {}}
mock_get_release_launchpad.return_value = {"launchpad_status": {}, "latest_release": {}}
mock_get_recent_ops_playbook_runs.return_value = {"runs": []}
mock_get_ops_activity_stream.return_value = {"items": []}
mock_get_runtime_build_info.return_value = {
"source": "git",
"commit_sha": "bd6bcb2",
"route_surface": {
"mode": "registered",
"registered_paths_total": 12,
"surface_complete": False,
"missing_keys": [
"ops_node_onboarding_acceptance_execute",
"ops_node_onboarding_acceptance_preview",
],
"missing_paths": [
"/api/v1/ops/nodes/{node_code}/onboarding/acceptance/execute",
"/api/v1/ops/nodes/{node_code}/onboarding/acceptance/preview",
],
},
}
payload = get_ops_stack_diagnosis(base_url="http://127.0.0.1:8100")
issue = next(
item for item in payload["diagnosis"]["issues"] if item["code"] == "runtime_route_surface_incomplete"
)
self.assertEqual("api-restart", issue["action_code"])
self.assertIn("ops_node_onboarding_acceptance_execute", issue["detail"])
self.assertIn(
"bash domain-api/deploy/multi-region/drive_ops_center.sh runtime-refresh-recover http://127.0.0.1:8100",
issue["commands"],
)
self.assertFalse(payload["runtime_build_info"]["route_surface"]["surface_complete"])
self.assertIn(
"ops_node_onboarding_acceptance_preview",
payload["runtime_build_info"]["route_surface_missing_keys"],
)
self.assertTrue(
any("API 路由面还不完整" in hint for hint in payload["diagnosis"]["operator_hints"])
)
@patch("app.services.ops_service.get_ops_activity_stream")
@patch("app.services.ops_service.get_recent_ops_playbook_runs")
@patch("app.services.ops_service.get_release_launchpad")
@patch("app.services.ops_service.list_managed_nodes_with_agent_state")
@patch("app.services.ops_service.get_ops_overview")
@patch("app.services.ops_service.get_ops_link_snapshot")
@patch("app.services.ops_service.get_ops_contract_registry")
def test_stack_diagnosis_emits_operator_decision_for_scene_log_gap(
self,
mock_get_ops_contract_registry,
mock_get_ops_link_snapshot,
mock_get_ops_overview,
mock_list_managed_nodes_with_agent_state,
mock_get_release_launchpad,
mock_get_recent_ops_playbook_runs,
mock_get_ops_activity_stream,
) -> None:
mock_get_ops_contract_registry.return_value = {
"registry_version": "2026-04-18",
"schema_version": "v1",
"contracts": [],
}
mock_get_ops_link_snapshot.return_value = {
"status": "attention",
"headline": "参与节点样本尚未形成。",
"next_actions": [],
}
mock_get_ops_overview.return_value = {
"recommendation": {},
"execution_scene": {
"participating_nodes": [{"node_code": "mainland-worker-01", "status": "busy"}],
"participation_summary": {
"dispatch_active_node_codes": ["mainland-worker-01"],
"recent_only_node_codes": [],
},
"log_sync": {
"enabled": True,
"mode": "key",
"line_count": 0,
"source_node_count": 0,
"covered_participating_node_count": 0,
"participating_node_count": 1,
"missing_participating_nodes": ["mainland-worker-01"],
},
},
"log_sync": {},
"release_hub": {},
}
mock_list_managed_nodes_with_agent_state.return_value = {"nodes": [], "summary": {}}
mock_get_release_launchpad.return_value = {"launchpad_status": {}, "latest_release": {}}
mock_get_recent_ops_playbook_runs.return_value = {"runs": []}
mock_get_ops_activity_stream.return_value = {"items": []}
payload = get_ops_stack_diagnosis(base_url="http://127.0.0.1:8100")
decision = payload["diagnosis"]["operator_decision"]
self.assertEqual("observability", decision["lane"])
self.assertEqual("scene-node-log", decision["primary_command_key"])
self.assertEqual("mainland-worker-01", decision["next_focus"]["node_code"])
self.assertIn("scene-node-log", payload["diagnosis"]["recommended_commands"])
self.assertIn(
"scene-node-log http://127.0.0.1:8100 mainland-worker-01 120 key",
payload["diagnosis"]["next_actions"]["primary_command"],
)
@patch("app.services.ops_service.get_ops_activity_stream")
@patch("app.services.ops_service.get_recent_ops_playbook_runs")
@patch("app.services.ops_service.get_release_launchpad")
@patch("app.services.ops_service.get_managed_node_handover")
@patch("app.services.ops_service.list_managed_nodes_with_agent_state")
@patch("app.services.ops_service.get_ops_overview")
@patch("app.services.ops_service.get_ops_link_snapshot")
@patch("app.services.ops_service.get_ops_contract_registry")
def test_stack_diagnosis_emits_operator_decision_for_managed_node_gap(
self,
mock_get_ops_contract_registry,
mock_get_ops_link_snapshot,
mock_get_ops_overview,
mock_list_managed_nodes_with_agent_state,
mock_get_managed_node_handover,
mock_get_release_launchpad,
mock_get_recent_ops_playbook_runs,
mock_get_ops_activity_stream,
) -> None:
mock_get_ops_contract_registry.return_value = {
"registry_version": "2026-04-18",
"schema_version": "v1",
"contracts": [],
}
mock_get_ops_link_snapshot.return_value = {
"status": "attention",
"headline": "存在接管缺口。",
"next_actions": [],
}
mock_get_ops_overview.return_value = {
"recommendation": {},
"execution_scene": {},
"log_sync": {},
"release_hub": {},
}
mock_list_managed_nodes_with_agent_state.return_value = {
"nodes": [
{
"node_code": "mainland-worker-01",
"agent_state": "pending_bootstrap",
"remote_access_state": "ssh_ready",
"delivery_queue_state": "healthy",
"participation_state": "standby",
}
],
"summary": {
"managed_enabled": 1,
"remote_access_ready": 0,
},
}
mock_get_managed_node_handover.return_value = {
"node_code": "mainland-worker-01",
"summary": "等待执行 bootstrap。",
"stage": {"code": "pending_bootstrap", "label": "待执行接入", "next_step": {"code": "execute_bootstrap_plan"}},
}
mock_get_release_launchpad.return_value = {
"launchpad_status": {
"recommended_action_code": "bootstrap_run",
"focus_ref": {"kind": "release_launchpad", "section": "release_launchpad"},
"recommended_target_node_code": "mainland-worker-01",
},
"worker_rollout_preview": {
"release_gate": {
"operational_readiness": {
"rows": [
{
"node_code": "mainland-worker-01",
"recovery_action": "bootstrap_run",
"recovery_summary": "mainland-worker-01 还缺接入收口，先执行 bootstrap。",
"focus_ref": {"kind": "managed_node", "node_code": "mainland-worker-01"},
}
]
}
}
},
"latest_release": {},
}
mock_get_recent_ops_playbook_runs.return_value = {"runs": []}
mock_get_ops_activity_stream.return_value = {"items": []}
payload = get_ops_stack_diagnosis(base_url="http://127.0.0.1:8100")
decision = payload["diagnosis"]["operator_decision"]
self.assertEqual("node_handover", decision["lane"])
self.assertEqual("bootstrap_run", decision["primary_command_key"])
self.assertEqual("managed-node-bootstrap-plan", decision["secondary_command_key"])
self.assertIn("bootstrap_run", payload["diagnosis"]["recommended_commands"])
self.assertIn("managed-node-handover", payload["diagnosis"]["recommended_commands"])
self.assertIn("managed-node-bootstrap-plan", payload["diagnosis"]["recommended_commands"])
self.assertIn(
"node-bootstrap-run http://127.0.0.1:8100 mainland-worker-01 cli",
payload["diagnosis"]["next_actions"]["primary_command"],
)
self.assertTrue(
any("当前第一动作已经收敛为 接入收口：mainland-worker-01" in hint for hint in payload["diagnosis"]["operator_hints"])
)
@patch("app.services.ops_service.get_runtime_build_info")
@patch("app.services.ops_service.get_ops_activity_stream")
@patch("app.services.ops_service.get_recent_ops_playbook_runs")
@patch("app.services.ops_service.get_release_launchpad")
@patch("app.services.ops_service.get_managed_node_handover")
@patch("app.services.ops_service.list_managed_nodes_with_agent_state")
@patch("app.services.ops_service.get_ops_overview")
@patch("app.services.ops_service.get_ops_link_snapshot")
@patch("app.services.ops_service.get_ops_contract_registry")
def test_stack_diagnosis_promotes_first_handover_gap_when_launchpad_is_still_generic(
self,
mock_get_ops_contract_registry,
mock_get_ops_link_snapshot,
mock_get_ops_overview,
mock_list_managed_nodes_with_agent_state,
mock_get_managed_node_handover,
mock_get_release_launchpad,
mock_get_recent_ops_playbook_runs,
mock_get_ops_activity_stream,
mock_get_runtime_build_info,
) -> None:
mock_get_ops_contract_registry.return_value = {
"registry_version": "2026-04-18",
"schema_version": "v1",
"contracts": [{"key": "ops_stack_diagnosis_contract"}],
}
mock_get_ops_link_snapshot.return_value = {
"status": "ready",
"headline": "当前总检可用。",
"next_actions": [],
}
mock_get_ops_overview.return_value = {
"recommendation": {
"primary_action_code": "handover_first_gap",
"title": "先补接管缺口",
"reason": "overview 只能先定位到接管缺口。",
"focus_ref": {"kind": "managed_node", "node_code": "mainland-worker-01"},
},
"execution_scene": {},
"log_sync": {},
"release_hub": {},
}
mock_list_managed_nodes_with_agent_state.return_value = {
"nodes": [
{
"node_code": "mainland-worker-01",
"agent_state": "pending_bootstrap",
"remote_access_state": "agent_pending",
"delivery_queue_state": "healthy",
"participation_state": "standby",
}
],
"summary": {
"managed_enabled": 1,
"remote_access_ready": 0,
},
}
mock_get_managed_node_handover.return_value = {
"node_code": "mainland-worker-01",
"summary": "当前节点仍处于接入阶段，建议先签发 onboarding.bootstrap 并等待 register / heartbeat 建立。",
"stage": {
"code": "pending_bootstrap",
"label": "待执行接入",
"next_step": {
"code": "execute_bootstrap_plan",
"label": "去目标机执行接入片段",
},
},
}
mock_get_release_launchpad.return_value = {
"launchpad_status": {
"recommended_action_code": "fix_managed_nodes",
"status_label": "待补执行面",
"focus_ref": {"kind": "release_launchpad", "section": "release_launchpad"},
},
"latest_release": {},
}
mock_get_recent_ops_playbook_runs.return_value = {"runs": []}
mock_get_ops_activity_stream.return_value = {"items": []}
mock_get_runtime_build_info.return_value = {
"route_surface": {"surface_complete": True, "missing_keys": []},
"repository_capabilities": {},
}
payload = get_ops_stack_diagnosis(base_url="http://127.0.0.1:8100")
self.assertEqual("bootstrap_run", payload["diagnosis"]["next_step"]["action_code"])
self.assertEqual(
"mainland-worker-01",
payload["diagnosis"]["launchpad_recommended_target_node_code"],
)
self.assertEqual(
"签发接入工单",
payload["diagnosis"]["launchpad_recommended_recovery_label"],
)
self.assertIn(
"onboarding.bootstrap",
payload["diagnosis"]["launchpad_recommended_recovery_summary"],
)
self.assertEqual("bootstrap_run", payload["diagnosis"]["operator_decision"]["primary_command_key"])
self.assertEqual(
"bash domain-api/deploy/multi-region/drive_ops_center.sh driver-resolve http://127.0.0.1:8100 bootstrap_run",
payload["diagnosis"]["next_actions"]["primary_command"],
)
@patch("app.services.ops_service.get_ops_activity_stream")
@patch("app.services.ops_service.get_recent_ops_playbook_runs")
@patch("app.services.ops_service.get_release_launchpad")
@patch("app.services.ops_service.list_managed_nodes_with_agent_state")
@patch("app.services.ops_service.get_ops_overview")
@patch("app.services.ops_service.get_ops_link_snapshot")
@patch("app.services.ops_service.get_ops_contract_registry")
def test_stack_diagnosis_emits_operator_decision_for_release_launchpad_attention(
self,
mock_get_ops_contract_registry,
mock_get_ops_link_snapshot,
mock_get_ops_overview,
mock_list_managed_nodes_with_agent_state,
mock_get_release_launchpad,
mock_get_recent_ops_playbook_runs,
mock_get_ops_activity_stream,
) -> None:
mock_get_ops_contract_registry.return_value = {
"registry_version": "2026-04-18",
"schema_version": "v1",
"contracts": [],
}
mock_get_ops_link_snapshot.return_value = {
"status": "attention",
"headline": "发布门禁待处理。",
"next_actions": [],
}
mock_get_ops_overview.return_value = {
"recommendation": {},
"execution_scene": {},
"log_sync": {},
"release_hub": {
"default_rollout_gate": {
"status": "blocked",
"summary_text": "还有节点未满足发布门禁。",
"focus_ref": {"kind": "release_hub", "section": "release_launchpad", "mode": "worker"},
}
},
}
mock_list_managed_nodes_with_agent_state.return_value = {"nodes": [], "summary": {}}
mock_get_release_launchpad.return_value = {
"launchpad_status": {
"status": "attention",
"status_label": "待处理",
"recommended_action_code": "review_smart_rollout_preview",
"focus_ref": {"kind": "release_hub", "section": "release_launchpad", "mode": "worker"},
},
"latest_release": {"release_version": "release-20260418"},
}
mock_get_recent_ops_playbook_runs.return_value = {"runs": []}
mock_get_ops_activity_stream.return_value = {"items": []}
payload = get_ops_stack_diagnosis(base_url="http://127.0.0.1:8100")
decision = payload["diagnosis"]["operator_decision"]
self.assertEqual("release", decision["lane"])
self.assertEqual("release-launchpad-worker", decision["primary_command_key"])
self.assertEqual("review_smart_rollout_preview", decision["secondary_command_key"])
self.assertIn("release-launchpad-worker", payload["diagnosis"]["recommended_commands"])
self.assertIn(
"release-launchpad http://127.0.0.1:8100 worker",
payload["diagnosis"]["next_actions"]["primary_command"],
)
@patch("app.services.ops_service.get_ops_activity_stream")
@patch("app.services.ops_service.get_recent_ops_playbook_runs")
@patch("app.services.ops_service.get_release_launchpad")
@patch("app.services.ops_service.list_managed_nodes_with_agent_state")
@patch("app.services.ops_service.get_ops_overview")
@patch("app.services.ops_service.get_ops_link_snapshot")
@patch("app.services.ops_service.get_ops_contract_registry")
def test_stack_diagnosis_prefers_node_recovery_when_launchpad_points_to_bootstrap_gap(
self,
mock_get_ops_contract_registry,
mock_get_ops_link_snapshot,
mock_get_ops_overview,
mock_list_managed_nodes_with_agent_state,
mock_get_release_launchpad,
mock_get_recent_ops_playbook_runs,
mock_get_ops_activity_stream,
) -> None:
mock_get_ops_contract_registry.return_value = {
"registry_version": "2026-04-18",
"schema_version": "v1",
"contracts": [],
}
mock_get_ops_link_snapshot.return_value = {
"status": "attention",
"headline": "发布门禁待处理。",
"next_actions": [],
}
mock_get_ops_overview.return_value = {
"recommendation": {},
"execution_scene": {},
"log_sync": {},
"release_hub": {
"default_rollout_gate": {
"status": "attention",
"summary_text": "当前还有节点未完成接入。",
"focus_ref": {"kind": "release_hub", "section": "release_launchpad", "mode": "worker"},
}
},
}
mock_list_managed_nodes_with_agent_state.return_value = {"nodes": [], "summary": {}}
mock_get_release_launchpad.return_value = {
"launchpad_status": {
"status": "attention",
"status_label": "待补执行面",
"recommended_action_code": "bootstrap_run",
"recommended_target_node_code": "mainland-worker-02",
"recommended_recovery_label": "待执行接入",
"recommended_recovery_summary": "当前节点仍处于接入阶段，建议先签发 onboarding.bootstrap。",
"onboarding_bootstrap_pending_nodes": 1,
"onboarding_acceptance_ready_nodes": 0,
"focus_ref": {"kind": "release_hub", "section": "release_launchpad", "mode": "worker"},
},
"latest_release": {"release_version": "release-20260418"},
}
mock_get_recent_ops_playbook_runs.return_value = {"runs": []}
mock_get_ops_activity_stream.return_value = {"items": []}
payload = get_ops_stack_diagnosis(base_url="http://127.0.0.1:8100")
decision = payload["diagnosis"]["operator_decision"]
self.assertEqual("node_handover", decision["lane"])
self.assertEqual("bootstrap_run", decision["primary_command_key"])
self.assertEqual("focus-release-hub", decision["secondary_command_key"])
self.assertEqual("mainland-worker-02", payload["diagnosis"]["launchpad_recommended_target_node_code"])
self.assertEqual("待执行接入", payload["diagnosis"]["launchpad_recommended_recovery_label"])
self.assertEqual(1, payload["diagnosis"]["launchpad_onboarding_bootstrap_pending_nodes"])
self.assertEqual(0, payload["diagnosis"]["launchpad_onboarding_acceptance_ready_nodes"])
self.assertEqual(
"当前节点仍处于接入阶段，建议先签发 onboarding.bootstrap。",
decision["summary"],
)
self.assertEqual("mainland-worker-02", decision["next_focus"]["node_code"])
self.assertIn("node-bootstrap-run", payload["diagnosis"]["next_actions"]["primary_command"])
@patch("app.services.ops_service.get_ops_activity_stream")
@patch("app.services.ops_service.get_recent_ops_playbook_runs")
@patch("app.services.ops_service.get_release_launchpad")
@patch("app.services.ops_service.get_managed_node_handover")
@patch("app.services.ops_service.list_managed_nodes_with_agent_state")
@patch("app.services.ops_service.get_ops_overview")
@patch("app.services.ops_service.get_ops_link_snapshot")
@patch("app.services.ops_service.get_ops_contract_registry")
def test_stack_diagnosis_includes_first_handover_gap_details(
self,
mock_get_ops_contract_registry,
mock_get_ops_link_snapshot,
mock_get_ops_overview,
mock_list_managed_nodes_with_agent_state,
mock_get_managed_node_handover,
mock_get_release_launchpad,
mock_get_recent_ops_playbook_runs,
mock_get_ops_activity_stream,
) -> None:
mock_get_ops_contract_registry.return_value = {
"registry_version": "2026-04-18",
"schema_version": "v1",
"contracts": [],
}
mock_get_ops_link_snapshot.return_value = {
"status": "attention",
"headline": "存在接管缺口。",
"next_actions": [],
}
mock_get_ops_overview.return_value = {
"recommendation": {},
"execution_scene": {},
"log_sync": {},
"release_hub": {},
}
mock_list_managed_nodes_with_agent_state.return_value = {
"nodes": [
{
"node_code": "mainland-worker-01",
"agent_state": "pending_bootstrap",
"remote_access_state": "ssh_ready",
"delivery_queue_state": "healthy",
"participation_state": "standby",
}
],
"summary": {
"managed_enabled": 1,
"remote_access_ready": 0,
"status_counts": {"pending_bootstrap": 1},
"remote_access_state_counts": {"ssh_ready": 1},
"delivery_queue_state_counts": {"healthy": 1},
},
}
mock_get_managed_node_handover.return_value = {
"node_code": "mainland-worker-01",
"summary": "已签发有效 Token，但目标节点尚未执行 bootstrap 片段。",
"stage": {
"code": "pending_bootstrap",
"label": "待执行接入",
"next_step": {
"code": "execute_bootstrap_plan",
"label": "去目标机执行接入片段",
},
},
}
mock_get_release_launchpad.return_value = {
"launchpad_status": {
"recommended_action_code": "fix_managed_nodes",
"focus_ref": {"kind": "release_launchpad"},
},
"latest_release": {},
}
mock_get_recent_ops_playbook_runs.return_value = {"runs": []}
mock_get_ops_activity_stream.return_value = {"items": []}
payload = get_ops_stack_diagnosis(base_url="http://127.0.0.1:8100")
self.assertEqual(
"mainland-worker-01",
payload["managed_nodes"]["first_handover_gap_node_code"],
)
self.assertEqual(
"pending_bootstrap",
payload["managed_nodes"]["first_handover_gap"]["stage"]["code"],
)
issue = next(
item for item in payload["diagnosis"]["issues"] if item["code"] == "managed_nodes_agent_pending"
)
self.assertIn("mainland-worker-01", issue["detail"])
self.assertTrue(
any("node-handover http://127.0.0.1:8100 mainland-worker-01" in command for command in issue["commands"])
)
self.assertIn(
"bash domain-api/deploy/multi-region/drive_ops_center.sh node-recover http://127.0.0.1:8100 mainland-worker-01",
payload["diagnosis"]["recommended_commands"]["managed-node-recover"],
)
self.assertTrue(
any("node-bootstrap-plan http://127.0.0.1:8100 mainland-worker-01" in command for command in issue["commands"])
)
mock_get_managed_node_handover.assert_called_once()
@patch("app.services.ops_service.get_ops_activity_stream")
@patch("app.services.ops_service.get_recent_ops_playbook_runs")
@patch("app.services.ops_service.get_release_launchpad")
@patch("app.services.ops_service.list_managed_nodes_with_agent_state")
@patch("app.services.ops_service.get_ops_overview")
@patch("app.services.ops_service.get_ops_link_snapshot")
@patch("app.services.ops_service.get_ops_contract_registry")
def test_stack_diagnosis_keeps_focus_refs_for_problem_runs_and_top_items(
self,
mock_get_ops_contract_registry,
mock_get_ops_link_snapshot,
mock_get_ops_overview,
mock_list_managed_nodes_with_agent_state,
mock_get_release_launchpad,
mock_get_recent_ops_playbook_runs,
mock_get_ops_activity_stream,
) -> None:
mock_get_ops_contract_registry.return_value = {
"registry_version": "2026-04-18",
"schema_version": "v1",
"contracts": [],
}
mock_get_ops_link_snapshot.return_value = {
"status": "ready",
"headline": "当前总检可用。",
"next_actions": [],
}
mock_get_ops_overview.return_value = {
"recommendation": {},
"execution_scene": {},
"log_sync": {},
"release_hub": {},
}
mock_list_managed_nodes_with_agent_state.return_value = {
"nodes": [],
"summary": {},
}
mock_get_release_launchpad.return_value = {
"launchpad_status": {},
"latest_release": {},
}
mock_get_recent_ops_playbook_runs.return_value = {
"runs": [
{
"run_code": "pbr-worker-logs-001",
"status": "attention",
"status_label": "待关注",
"focus_step_key": "worker_logs",
"focus_summary": "Worker 日志步骤仍未收口。",
"focus_ref": {
"kind": "playbook_run",
"run_code": "pbr-worker-logs-001",
"focus_step_key": "worker_logs",
},
}
]
}
mock_get_ops_activity_stream.return_value = {
"items": [
{
"kind": "ops_job",
"status": "running",
"summary_text": "远端 Agent 已回传关键日志样本。",
"occurred_at": "2026-04-18 09:22:00",
"focus_ref": {
"kind": "ops_job",
"job_id": 71,
"job_code": "ops-agent-71",
},
"source_focus_ref": {
"kind": "ops_job_event",
"job_id": 71,
"event_key": "job-event:71",
},
"ui_intent": {
"kind": "job_events",
"job_id": 71,
},
}
]
}
payload = get_ops_stack_diagnosis(base_url="http://127.0.0.1:8100")
self.assertEqual(
"ops_stack_diagnosis_contract",
payload["diagnosis"]["contract_navigation"]["primary_contract_key"],
)
self.assertIn("ops_driver_contract", payload["diagnosis"]["contract_navigation"]["contract_keys"])
self.assertEqual("playbook_run", payload["playbook_runs"]["problem_runs"][0]["focus_ref"]["kind"])
self.assertEqual("worker_logs", payload["playbook_runs"]["problem_runs"][0]["focus_ref"]["focus_step_key"])
top_item = payload["activity_stream"]["top_items"][0]
self.assertEqual("ops_job", top_item["focus_ref"]["kind"])
self.assertEqual("ops_job_event", top_item["source_focus_ref"]["kind"])
self.assertEqual("job_events", top_item["ui_intent_kind"])
self.assertEqual("2026-04-18 09:22:00", top_item["occurred_at"])
@patch("app.services.ops_service.get_ops_activity_stream")
@patch("app.services.ops_service.get_recent_ops_playbook_runs")
@patch("app.services.ops_service.get_release_launchpad")
@patch("app.services.ops_service.list_managed_nodes_with_agent_state")
@patch("app.services.ops_service.get_ops_overview")
@patch("app.services.ops_service.get_ops_link_snapshot")
@patch("app.services.ops_service.get_ops_contract_registry")
def test_stack_diagnosis_adds_node_scene_log_commands_for_log_sample_gap(
self,
mock_get_ops_contract_registry,
mock_get_ops_link_snapshot,
mock_get_ops_overview,
mock_list_managed_nodes_with_agent_state,
mock_get_release_launchpad,
mock_get_recent_ops_playbook_runs,
mock_get_ops_activity_stream,
) -> None:
mock_get_ops_contract_registry.return_value = {
"registry_version": "2026-04-18",
"schema_version": "v1",
"contracts": [],
}
mock_get_ops_link_snapshot.return_value = {
"status": "attention",
"headline": "参与节点样本尚未形成。",
"next_actions": [
{
"label": "看参与节点日志",
"action_code": "open_worker_logs_participating",
"node_codes": ["mainland-worker-01"],
"payload": {},
}
],
}
mock_get_ops_overview.return_value = {
"recommendation": {},
"execution_scene": {
"participating_nodes": [
{
"node_code": "mainland-worker-01",
"status": "busy",
}
],
"participation_summary": {
"dispatch_active_node_codes": ["mainland-worker-01"],
"recent_only_node_codes": [],
},
"log_sync": {
"enabled": True,
"mode": "key",
"line_count": 0,
"source_node_count": 0,
"covered_participating_node_count": 0,
"participating_node_count": 1,
"missing_participating_nodes": ["mainland-worker-01"],
},
},
"log_sync": {},
"release_hub": {},
}
mock_list_managed_nodes_with_agent_state.return_value = {
"nodes": [],
"summary": {},
}
mock_get_release_launchpad.return_value = {
"launchpad_status": {},
"latest_release": {},
}
mock_get_recent_ops_playbook_runs.return_value = {"runs": []}
mock_get_ops_activity_stream.return_value = {"items": []}
payload = get_ops_stack_diagnosis(base_url="http://127.0.0.1:8100")
issue = next(
item for item in payload["diagnosis"]["issues"] if item["code"] == "remote_log_sync_waiting_sample"
)
self.assertEqual("open_worker_logs_participating", issue["action_code"])
self.assertEqual("node_scene_log", issue["focus_ref"]["kind"])
self.assertEqual("mainland-worker-01", issue["focus_ref"]["node_code"])
self.assertTrue(
any(
"scene-node-log http://127.0.0.1:8100 mainland-worker-01 120 key" in command
for command in issue["commands"]
)
)
self.assertEqual("node_scene_log", payload["diagnosis"]["next_step"]["focus_ref"]["kind"])
self.assertTrue(
any(
"scene-node-log http://127.0.0.1:8100 mainland-worker-01 120 key" in command
for command in payload["diagnosis"]["quick_commands"]
)
)
self.assertTrue(
any(
"driver-resolve http://127.0.0.1:8100 open_worker_logs_participating" in command
for command in issue["commands"]
)
)
@patch("app.services.ops_service.get_ops_activity_stream")
@patch("app.services.ops_service.get_recent_ops_playbook_runs")
@patch("app.services.ops_service.get_release_launchpad")
@patch("app.services.ops_service.list_managed_nodes_with_agent_state")
@patch("app.services.ops_service.get_ops_overview")
@patch("app.services.ops_service.get_ops_link_snapshot")
@patch("app.services.ops_service.get_ops_contract_registry")
def test_stack_diagnosis_surfaces_start_delivery_gap_from_activity_stream(
self,
mock_get_ops_contract_registry,
mock_get_ops_link_snapshot,
mock_get_ops_overview,
mock_list_managed_nodes_with_agent_state,
mock_get_release_launchpad,
mock_get_recent_ops_playbook_runs,
mock_get_ops_activity_stream,
) -> None:
mock_get_ops_contract_registry.return_value = {
"registry_version": "2026-04-18",
"schema_version": "v1",
"contracts": [],
}
mock_get_ops_link_snapshot.return_value = {
"status": "attention",
"headline": "最近有任务回执异常。",
"next_actions": [],
}
mock_get_ops_overview.return_value = {
"recommendation": {},
"execution_scene": {},
"log_sync": {},
"release_hub": {},
}
mock_list_managed_nodes_with_agent_state.return_value = {
"nodes": [],
"summary": {},
}
mock_get_release_launchpad.return_value = {
"launchpad_status": {},
"latest_release": {},
}
mock_get_recent_ops_playbook_runs.return_value = {"runs": []}
mock_get_ops_activity_stream.return_value = {
"items": [
{
"kind": "ops_job",
"status": "attention",
"job_status": "running",
"summary_text": "节点已接单执行，但开始回执未成功送达控制面。",
"occurred_at": "2026-04-18 10:22:00",
"job_code": "ops-agent-81",
"start_delivery_state": "failed_local",
"start_delivery_error": "temporary offline",
"focus_ref": {
"kind": "ops_job",
"job_id": 81,
"job_code": "ops-agent-81",
"target_node_code": "mainland-worker-01",
},
"source_focus_ref": {
"kind": "ops_job_event",
"job_id": 81,
"job_code": "ops-agent-81",
"event_key": "job-event:81",
"target_node_code": "mainland-worker-01",
},
"ui_intent": {
"kind": "job_events",
"job_id": 81,
},
}
]
}
payload = get_ops_stack_diagnosis(base_url="http://127.0.0.1:8100")
issue = next(
item for item in payload["diagnosis"]["issues"] if item["code"] == "ops_job_start_delivery_failed_local"
)
self.assertEqual("focus_latest_job_events", issue["action_code"])
self.assertEqual("ops_job_event", issue["focus_ref"]["kind"])
self.assertIn("temporary offline", issue["detail"])
decision = payload["diagnosis"]["operator_decision"]
self.assertEqual("ops_jobs", decision["lane"])
self.assertEqual("focus_latest_job_events", decision["primary_command_key"])
self.assertIn("driver-resolve http://127.0.0.1:8100 focus_latest_job_events", payload["diagnosis"]["next_actions"]["primary_command"])
if __name__ == "__main__":
unittest.main()
Reference in New Issue View Git Blame Copy Permalink