getDomain/domain-api/tests/test_ops_service_stack_diagnosis.py

import unittest
from unittest.mock import patch

from app.services.ops_service import get_ops_stack_diagnosis


class OpsServiceStackDiagnosisTests(unittest.TestCase):
    @patch("app.services.ops_service.get_runtime_build_info")
    @patch("app.services.ops_service.get_ops_activity_stream")
    @patch("app.services.ops_service.get_recent_ops_playbook_runs")
    @patch("app.services.ops_service.get_release_launchpad")
    @patch("app.services.ops_service.list_managed_nodes_with_agent_state")
    @patch("app.services.ops_service.get_ops_overview")
    @patch("app.services.ops_service.get_ops_link_snapshot")
    @patch("app.services.ops_service.get_ops_contract_registry")
    def test_stack_diagnosis_prioritizes_api_restart_when_runtime_build_schema_is_stale(
        self,
        mock_get_ops_contract_registry,
        mock_get_ops_link_snapshot,
        mock_get_ops_overview,
        mock_list_managed_nodes_with_agent_state,
        mock_get_release_launchpad,
        mock_get_recent_ops_playbook_runs,
        mock_get_ops_activity_stream,
        mock_get_runtime_build_info,
    ) -> None:
        mock_get_ops_contract_registry.return_value = {
            "registry_version": "2026-04-18",
            "schema_version": "v1",
            "contracts": [{"key": "ops_stack_diagnosis_contract"}],
        }
        mock_get_ops_link_snapshot.return_value = {
            "status": "ready",
            "headline": "当前总检可用。",
            "next_actions": [],
        }
        mock_get_ops_overview.return_value = {
            "recommendation": {
                "primary_action_code": "fix_managed_nodes",
                "title": "先补节点接管",
                "focus_ref": {"kind": "managed_node", "node_code": "mainland-worker-01"},
            },
            "execution_scene": {},
            "log_sync": {},
            "release_hub": {},
        }
        mock_list_managed_nodes_with_agent_state.return_value = {"nodes": [], "summary": {}}
        mock_get_release_launchpad.return_value = {"launchpad_status": {}, "latest_release": {}}
        mock_get_recent_ops_playbook_runs.return_value = {"runs": []}
        mock_get_ops_activity_stream.return_value = {"items": []}
        mock_get_runtime_build_info.return_value = {
            "source": "git",
            "commit_sha": "246838ae4c07",
            "repository_capabilities": {
                "supports_install_command_block": True,
                "supports_multi_layout_bootstrap": True,
            },
            "route_surface": {
                "mode": "registered",
                "registered_paths_total": 24,
                "surface_complete": True,
                "missing_keys": [],
                "expected_paths": {
                    "ops_contracts": "/api/v1/ops/contracts",
                    "ops_stack_diagnosis": "/api/v1/ops/stack-diagnosis",
                },
            },
        }

        payload = get_ops_stack_diagnosis(base_url="http://127.0.0.1:8100")

        issue = next(item for item in payload["diagnosis"]["issues"] if item["code"] == "runtime_build_schema_stale")
        self.assertEqual("api-restart", issue["action_code"])
        self.assertEqual("api-restart", payload["diagnosis"]["next_step"]["action_code"])
        self.assertEqual("runtime_recovery", payload["diagnosis"]["operator_decision"]["lane"])
        self.assertEqual("api-restart", payload["diagnosis"]["operator_decision"]["primary_command_key"])
        self.assertEqual(
            "bash domain-api/deploy/multi-region/drive_ops_center.sh runtime-refresh-recover http://127.0.0.1:8100",
            payload["diagnosis"]["next_actions"]["primary_command"],
        )
        self.assertEqual(
            "bash domain-api/deploy/multi-region/drive_ops_center.sh runtime-refresh-recover http://127.0.0.1:8100",
            payload["diagnosis"]["recommended_commands"]["api-restart"],
        )
        self.assertEqual(
            "bash domain-api/deploy/multi-region/drive_ops_center.sh runtime-refresh-recover http://127.0.0.1:8100",
            payload["diagnosis"]["recommended_commands"]["runtime-refresh-recover"],
        )
        self.assertEqual(
            "bash domain-api/deploy/multi-region/drive_ops_center.sh stack-diagnosis http://127.0.0.1:8100 summary",
            payload["diagnosis"]["recommended_commands"]["stack-diagnosis"],
        )
        self.assertEqual(
            payload["diagnosis"]["recommended_commands"]["stack-diagnosis"],
            payload["diagnosis"]["recommended_commands"]["stack-check"],
        )
        self.assertTrue(payload["runtime_build_info"]["runtime_schema_stale"])
        self.assertFalse(payload["runtime_build_info"]["route_surface_declares_bootstrap_plan"])
        self.assertTrue(
            any("运行中 API 还是旧 schema" in hint for hint in payload["diagnosis"]["operator_hints"])
        )

    @patch("app.services.ops_service.get_runtime_build_info")
    @patch("app.services.ops_service.get_ops_activity_stream")
    @patch("app.services.ops_service.get_recent_ops_playbook_runs")
    @patch("app.services.ops_service.get_release_launchpad")
    @patch("app.services.ops_service.list_managed_nodes_with_agent_state")
    @patch("app.services.ops_service.get_ops_overview")
    @patch("app.services.ops_service.get_ops_link_snapshot")
    @patch("app.services.ops_service.get_ops_contract_registry")
    def test_stack_diagnosis_surfaces_runtime_route_surface_gap(
        self,
        mock_get_ops_contract_registry,
        mock_get_ops_link_snapshot,
        mock_get_ops_overview,
        mock_list_managed_nodes_with_agent_state,
        mock_get_release_launchpad,
        mock_get_recent_ops_playbook_runs,
        mock_get_ops_activity_stream,
        mock_get_runtime_build_info,
    ) -> None:
        mock_get_ops_contract_registry.return_value = {
            "registry_version": "2026-04-18",
            "schema_version": "v1",
            "contracts": [],
        }
        mock_get_ops_link_snapshot.return_value = {
            "status": "ready",
            "headline": "当前总检可用。",
            "next_actions": [],
        }
        mock_get_ops_overview.return_value = {
            "recommendation": {},
            "execution_scene": {},
            "log_sync": {},
            "release_hub": {},
        }
        mock_list_managed_nodes_with_agent_state.return_value = {"nodes": [], "summary": {}}
        mock_get_release_launchpad.return_value = {"launchpad_status": {}, "latest_release": {}}
        mock_get_recent_ops_playbook_runs.return_value = {"runs": []}
        mock_get_ops_activity_stream.return_value = {"items": []}
        mock_get_runtime_build_info.return_value = {
            "source": "git",
            "commit_sha": "bd6bcb2",
            "route_surface": {
                "mode": "registered",
                "registered_paths_total": 12,
                "surface_complete": False,
                "missing_keys": [
                    "ops_node_onboarding_acceptance_execute",
                    "ops_node_onboarding_acceptance_preview",
                ],
                "missing_paths": [
                    "/api/v1/ops/nodes/{node_code}/onboarding/acceptance/execute",
                    "/api/v1/ops/nodes/{node_code}/onboarding/acceptance/preview",
                ],
            },
        }

        payload = get_ops_stack_diagnosis(base_url="http://127.0.0.1:8100")

        issue = next(
            item for item in payload["diagnosis"]["issues"] if item["code"] == "runtime_route_surface_incomplete"
        )
        self.assertEqual("api-restart", issue["action_code"])
        self.assertIn("ops_node_onboarding_acceptance_execute", issue["detail"])
        self.assertIn(
            "bash domain-api/deploy/multi-region/drive_ops_center.sh runtime-refresh-recover http://127.0.0.1:8100",
            issue["commands"],
        )
        self.assertFalse(payload["runtime_build_info"]["route_surface"]["surface_complete"])
        self.assertIn(
            "ops_node_onboarding_acceptance_preview",
            payload["runtime_build_info"]["route_surface_missing_keys"],
        )
        self.assertTrue(
            any("API 路由面还不完整" in hint for hint in payload["diagnosis"]["operator_hints"])
        )

    @patch("app.services.ops_service.get_ops_activity_stream")
    @patch("app.services.ops_service.get_recent_ops_playbook_runs")
    @patch("app.services.ops_service.get_release_launchpad")
    @patch("app.services.ops_service.list_managed_nodes_with_agent_state")
    @patch("app.services.ops_service.get_ops_overview")
    @patch("app.services.ops_service.get_ops_link_snapshot")
    @patch("app.services.ops_service.get_ops_contract_registry")
    def test_stack_diagnosis_emits_operator_decision_for_scene_log_gap(
        self,
        mock_get_ops_contract_registry,
        mock_get_ops_link_snapshot,
        mock_get_ops_overview,
        mock_list_managed_nodes_with_agent_state,
        mock_get_release_launchpad,
        mock_get_recent_ops_playbook_runs,
        mock_get_ops_activity_stream,
    ) -> None:
        mock_get_ops_contract_registry.return_value = {
            "registry_version": "2026-04-18",
            "schema_version": "v1",
            "contracts": [],
        }
        mock_get_ops_link_snapshot.return_value = {
            "status": "attention",
            "headline": "参与节点样本尚未形成。",
            "next_actions": [],
        }
        mock_get_ops_overview.return_value = {
            "recommendation": {},
            "execution_scene": {
                "participating_nodes": [{"node_code": "mainland-worker-01", "status": "busy"}],
                "participation_summary": {
                    "dispatch_active_node_codes": ["mainland-worker-01"],
                    "recent_only_node_codes": [],
                },
                "log_sync": {
                    "enabled": True,
                    "mode": "key",
                    "line_count": 0,
                    "source_node_count": 0,
                    "covered_participating_node_count": 0,
                    "participating_node_count": 1,
                    "missing_participating_nodes": ["mainland-worker-01"],
                },
            },
            "log_sync": {},
            "release_hub": {},
        }
        mock_list_managed_nodes_with_agent_state.return_value = {"nodes": [], "summary": {}}
        mock_get_release_launchpad.return_value = {"launchpad_status": {}, "latest_release": {}}
        mock_get_recent_ops_playbook_runs.return_value = {"runs": []}
        mock_get_ops_activity_stream.return_value = {"items": []}

        payload = get_ops_stack_diagnosis(base_url="http://127.0.0.1:8100")

        decision = payload["diagnosis"]["operator_decision"]
        self.assertEqual("observability", decision["lane"])
        self.assertEqual("scene-node-log", decision["primary_command_key"])
        self.assertEqual("mainland-worker-01", decision["next_focus"]["node_code"])
        self.assertIn("scene-node-log", payload["diagnosis"]["recommended_commands"])
        self.assertIn(
            "scene-node-log http://127.0.0.1:8100 mainland-worker-01 120 key",
            payload["diagnosis"]["next_actions"]["primary_command"],
        )

    @patch("app.services.ops_service.get_ops_activity_stream")
    @patch("app.services.ops_service.get_recent_ops_playbook_runs")
    @patch("app.services.ops_service.get_release_launchpad")
    @patch("app.services.ops_service.get_managed_node_handover")
    @patch("app.services.ops_service.list_managed_nodes_with_agent_state")
    @patch("app.services.ops_service.get_ops_overview")
    @patch("app.services.ops_service.get_ops_link_snapshot")
    @patch("app.services.ops_service.get_ops_contract_registry")
    def test_stack_diagnosis_emits_operator_decision_for_managed_node_gap(
        self,
        mock_get_ops_contract_registry,
        mock_get_ops_link_snapshot,
        mock_get_ops_overview,
        mock_list_managed_nodes_with_agent_state,
        mock_get_managed_node_handover,
        mock_get_release_launchpad,
        mock_get_recent_ops_playbook_runs,
        mock_get_ops_activity_stream,
    ) -> None:
        mock_get_ops_contract_registry.return_value = {
            "registry_version": "2026-04-18",
            "schema_version": "v1",
            "contracts": [],
        }
        mock_get_ops_link_snapshot.return_value = {
            "status": "attention",
            "headline": "存在接管缺口。",
            "next_actions": [],
        }
        mock_get_ops_overview.return_value = {
            "recommendation": {},
            "execution_scene": {},
            "log_sync": {},
            "release_hub": {},
        }
        mock_list_managed_nodes_with_agent_state.return_value = {
            "nodes": [
                {
                    "node_code": "mainland-worker-01",
                    "agent_state": "pending_bootstrap",
                    "remote_access_state": "ssh_ready",
                    "delivery_queue_state": "healthy",
                    "participation_state": "standby",
                }
            ],
            "summary": {
                "managed_enabled": 1,
                "remote_access_ready": 0,
            },
        }
        mock_get_managed_node_handover.return_value = {
            "node_code": "mainland-worker-01",
            "summary": "等待执行 bootstrap。",
            "stage": {"code": "pending_bootstrap", "label": "待执行接入", "next_step": {"code": "execute_bootstrap_plan"}},
        }
        mock_get_release_launchpad.return_value = {
            "launchpad_status": {
                "recommended_action_code": "bootstrap_run",
                "focus_ref": {"kind": "release_launchpad", "section": "release_launchpad"},
                "recommended_target_node_code": "mainland-worker-01",
            },
            "worker_rollout_preview": {
                "release_gate": {
                    "operational_readiness": {
                        "rows": [
                            {
                                "node_code": "mainland-worker-01",
                                "recovery_action": "bootstrap_run",
                                "recovery_summary": "mainland-worker-01 还缺接入收口，先执行 bootstrap。",
                                "focus_ref": {"kind": "managed_node", "node_code": "mainland-worker-01"},
                            }
                        ]
                    }
                }
            },
            "latest_release": {},
        }
        mock_get_recent_ops_playbook_runs.return_value = {"runs": []}
        mock_get_ops_activity_stream.return_value = {"items": []}

        payload = get_ops_stack_diagnosis(base_url="http://127.0.0.1:8100")

        decision = payload["diagnosis"]["operator_decision"]
        self.assertEqual("node_handover", decision["lane"])
        self.assertEqual("bootstrap_run", decision["primary_command_key"])
        self.assertEqual("managed-node-bootstrap-plan", decision["secondary_command_key"])
        self.assertIn("bootstrap_run", payload["diagnosis"]["recommended_commands"])
        self.assertIn("managed-node-handover", payload["diagnosis"]["recommended_commands"])
        self.assertIn("managed-node-bootstrap-plan", payload["diagnosis"]["recommended_commands"])
        self.assertIn(
            "node-bootstrap-run http://127.0.0.1:8100 mainland-worker-01 cli",
            payload["diagnosis"]["next_actions"]["primary_command"],
        )
        self.assertTrue(
            any("当前第一动作已经收敛为 接入收口：mainland-worker-01" in hint for hint in payload["diagnosis"]["operator_hints"])
        )

    @patch("app.services.ops_service.get_runtime_build_info")
    @patch("app.services.ops_service.get_ops_activity_stream")
    @patch("app.services.ops_service.get_recent_ops_playbook_runs")
    @patch("app.services.ops_service.get_release_launchpad")
    @patch("app.services.ops_service.get_managed_node_handover")
    @patch("app.services.ops_service.list_managed_nodes_with_agent_state")
    @patch("app.services.ops_service.get_ops_overview")
    @patch("app.services.ops_service.get_ops_link_snapshot")
    @patch("app.services.ops_service.get_ops_contract_registry")
    def test_stack_diagnosis_promotes_first_handover_gap_when_launchpad_is_still_generic(
        self,
        mock_get_ops_contract_registry,
        mock_get_ops_link_snapshot,
        mock_get_ops_overview,
        mock_list_managed_nodes_with_agent_state,
        mock_get_managed_node_handover,
        mock_get_release_launchpad,
        mock_get_recent_ops_playbook_runs,
        mock_get_ops_activity_stream,
        mock_get_runtime_build_info,
    ) -> None:
        mock_get_ops_contract_registry.return_value = {
            "registry_version": "2026-04-18",
            "schema_version": "v1",
            "contracts": [{"key": "ops_stack_diagnosis_contract"}],
        }
        mock_get_ops_link_snapshot.return_value = {
            "status": "ready",
            "headline": "当前总检可用。",
            "next_actions": [],
        }
        mock_get_ops_overview.return_value = {
            "recommendation": {
                "primary_action_code": "handover_first_gap",
                "title": "先补接管缺口",
                "reason": "overview 只能先定位到接管缺口。",
                "focus_ref": {"kind": "managed_node", "node_code": "mainland-worker-01"},
            },
            "execution_scene": {},
            "log_sync": {},
            "release_hub": {},
        }
        mock_list_managed_nodes_with_agent_state.return_value = {
            "nodes": [
                {
                    "node_code": "mainland-worker-01",
                    "agent_state": "pending_bootstrap",
                    "remote_access_state": "agent_pending",
                    "delivery_queue_state": "healthy",
                    "participation_state": "standby",
                }
            ],
            "summary": {
                "managed_enabled": 1,
                "remote_access_ready": 0,
            },
        }
        mock_get_managed_node_handover.return_value = {
            "node_code": "mainland-worker-01",
            "summary": "当前节点仍处于接入阶段，建议先签发 onboarding.bootstrap 并等待 register / heartbeat 建立。",
            "stage": {
                "code": "pending_bootstrap",
                "label": "待执行接入",
                "next_step": {
                    "code": "execute_bootstrap_plan",
                    "label": "去目标机执行接入片段",
                },
            },
        }
        mock_get_release_launchpad.return_value = {
            "launchpad_status": {
                "recommended_action_code": "fix_managed_nodes",
                "status_label": "待补执行面",
                "focus_ref": {"kind": "release_launchpad", "section": "release_launchpad"},
            },
            "latest_release": {},
        }
        mock_get_recent_ops_playbook_runs.return_value = {"runs": []}
        mock_get_ops_activity_stream.return_value = {"items": []}
        mock_get_runtime_build_info.return_value = {
            "route_surface": {"surface_complete": True, "missing_keys": []},
            "repository_capabilities": {},
        }

        payload = get_ops_stack_diagnosis(base_url="http://127.0.0.1:8100")

        self.assertEqual("bootstrap_run", payload["diagnosis"]["next_step"]["action_code"])
        self.assertEqual(
            "mainland-worker-01",
            payload["diagnosis"]["launchpad_recommended_target_node_code"],
        )
        self.assertEqual(
            "签发接入工单",
            payload["diagnosis"]["launchpad_recommended_recovery_label"],
        )
        self.assertIn(
            "onboarding.bootstrap",
            payload["diagnosis"]["launchpad_recommended_recovery_summary"],
        )
        self.assertEqual("bootstrap_run", payload["diagnosis"]["operator_decision"]["primary_command_key"])
        self.assertEqual(
            "bash domain-api/deploy/multi-region/drive_ops_center.sh driver-resolve http://127.0.0.1:8100 bootstrap_run",
            payload["diagnosis"]["next_actions"]["primary_command"],
        )

    @patch("app.services.ops_service.get_ops_activity_stream")
    @patch("app.services.ops_service.get_recent_ops_playbook_runs")
    @patch("app.services.ops_service.get_release_launchpad")
    @patch("app.services.ops_service.list_managed_nodes_with_agent_state")
    @patch("app.services.ops_service.get_ops_overview")
    @patch("app.services.ops_service.get_ops_link_snapshot")
    @patch("app.services.ops_service.get_ops_contract_registry")
    def test_stack_diagnosis_emits_operator_decision_for_release_launchpad_attention(
        self,
        mock_get_ops_contract_registry,
        mock_get_ops_link_snapshot,
        mock_get_ops_overview,
        mock_list_managed_nodes_with_agent_state,
        mock_get_release_launchpad,
        mock_get_recent_ops_playbook_runs,
        mock_get_ops_activity_stream,
    ) -> None:
        mock_get_ops_contract_registry.return_value = {
            "registry_version": "2026-04-18",
            "schema_version": "v1",
            "contracts": [],
        }
        mock_get_ops_link_snapshot.return_value = {
            "status": "attention",
            "headline": "发布门禁待处理。",
            "next_actions": [],
        }
        mock_get_ops_overview.return_value = {
            "recommendation": {},
            "execution_scene": {},
            "log_sync": {},
            "release_hub": {
                "default_rollout_gate": {
                    "status": "blocked",
                    "summary_text": "还有节点未满足发布门禁。",
                    "focus_ref": {"kind": "release_hub", "section": "release_launchpad", "mode": "worker"},
                }
            },
        }
        mock_list_managed_nodes_with_agent_state.return_value = {"nodes": [], "summary": {}}
        mock_get_release_launchpad.return_value = {
            "launchpad_status": {
                "status": "attention",
                "status_label": "待处理",
                "recommended_action_code": "review_smart_rollout_preview",
                "focus_ref": {"kind": "release_hub", "section": "release_launchpad", "mode": "worker"},
            },
            "latest_release": {"release_version": "release-20260418"},
        }
        mock_get_recent_ops_playbook_runs.return_value = {"runs": []}
        mock_get_ops_activity_stream.return_value = {"items": []}

        payload = get_ops_stack_diagnosis(base_url="http://127.0.0.1:8100")

        decision = payload["diagnosis"]["operator_decision"]
        self.assertEqual("release", decision["lane"])
        self.assertEqual("release-launchpad-worker", decision["primary_command_key"])
        self.assertEqual("review_smart_rollout_preview", decision["secondary_command_key"])
        self.assertIn("release-launchpad-worker", payload["diagnosis"]["recommended_commands"])
        self.assertIn(
            "release-launchpad http://127.0.0.1:8100 worker",
            payload["diagnosis"]["next_actions"]["primary_command"],
        )

    @patch("app.services.ops_service.get_ops_activity_stream")
    @patch("app.services.ops_service.get_recent_ops_playbook_runs")
    @patch("app.services.ops_service.get_release_launchpad")
    @patch("app.services.ops_service.list_managed_nodes_with_agent_state")
    @patch("app.services.ops_service.get_ops_overview")
    @patch("app.services.ops_service.get_ops_link_snapshot")
    @patch("app.services.ops_service.get_ops_contract_registry")
    def test_stack_diagnosis_prefers_node_recovery_when_launchpad_points_to_bootstrap_gap(
        self,
        mock_get_ops_contract_registry,
        mock_get_ops_link_snapshot,
        mock_get_ops_overview,
        mock_list_managed_nodes_with_agent_state,
        mock_get_release_launchpad,
        mock_get_recent_ops_playbook_runs,
        mock_get_ops_activity_stream,
    ) -> None:
        mock_get_ops_contract_registry.return_value = {
            "registry_version": "2026-04-18",
            "schema_version": "v1",
            "contracts": [],
        }
        mock_get_ops_link_snapshot.return_value = {
            "status": "attention",
            "headline": "发布门禁待处理。",
            "next_actions": [],
        }
        mock_get_ops_overview.return_value = {
            "recommendation": {},
            "execution_scene": {},
            "log_sync": {},
            "release_hub": {
                "default_rollout_gate": {
                    "status": "attention",
                    "summary_text": "当前还有节点未完成接入。",
                    "focus_ref": {"kind": "release_hub", "section": "release_launchpad", "mode": "worker"},
                }
            },
        }
        mock_list_managed_nodes_with_agent_state.return_value = {"nodes": [], "summary": {}}
        mock_get_release_launchpad.return_value = {
            "launchpad_status": {
                "status": "attention",
                "status_label": "待补执行面",
                "recommended_action_code": "bootstrap_run",
                "recommended_target_node_code": "mainland-worker-02",
                "recommended_recovery_label": "待执行接入",
                "recommended_recovery_summary": "当前节点仍处于接入阶段，建议先签发 onboarding.bootstrap。",
                "onboarding_bootstrap_pending_nodes": 1,
                "onboarding_acceptance_ready_nodes": 0,
                "focus_ref": {"kind": "release_hub", "section": "release_launchpad", "mode": "worker"},
            },
            "latest_release": {"release_version": "release-20260418"},
        }
        mock_get_recent_ops_playbook_runs.return_value = {"runs": []}
        mock_get_ops_activity_stream.return_value = {"items": []}

        payload = get_ops_stack_diagnosis(base_url="http://127.0.0.1:8100")

        decision = payload["diagnosis"]["operator_decision"]
        self.assertEqual("node_handover", decision["lane"])
        self.assertEqual("bootstrap_run", decision["primary_command_key"])
        self.assertEqual("focus-release-hub", decision["secondary_command_key"])
        self.assertEqual("mainland-worker-02", payload["diagnosis"]["launchpad_recommended_target_node_code"])
        self.assertEqual("待执行接入", payload["diagnosis"]["launchpad_recommended_recovery_label"])
        self.assertEqual(1, payload["diagnosis"]["launchpad_onboarding_bootstrap_pending_nodes"])
        self.assertEqual(0, payload["diagnosis"]["launchpad_onboarding_acceptance_ready_nodes"])
        self.assertEqual(
            "当前节点仍处于接入阶段，建议先签发 onboarding.bootstrap。",
            decision["summary"],
        )
        self.assertEqual("mainland-worker-02", decision["next_focus"]["node_code"])
        self.assertIn("node-bootstrap-run", payload["diagnosis"]["next_actions"]["primary_command"])

    @patch("app.services.ops_service.get_ops_activity_stream")
    @patch("app.services.ops_service.get_recent_ops_playbook_runs")
    @patch("app.services.ops_service.get_release_launchpad")
    @patch("app.services.ops_service.get_managed_node_handover")
    @patch("app.services.ops_service.list_managed_nodes_with_agent_state")
    @patch("app.services.ops_service.get_ops_overview")
    @patch("app.services.ops_service.get_ops_link_snapshot")
    @patch("app.services.ops_service.get_ops_contract_registry")
    def test_stack_diagnosis_includes_first_handover_gap_details(
        self,
        mock_get_ops_contract_registry,
        mock_get_ops_link_snapshot,
        mock_get_ops_overview,
        mock_list_managed_nodes_with_agent_state,
        mock_get_managed_node_handover,
        mock_get_release_launchpad,
        mock_get_recent_ops_playbook_runs,
        mock_get_ops_activity_stream,
    ) -> None:
        mock_get_ops_contract_registry.return_value = {
            "registry_version": "2026-04-18",
            "schema_version": "v1",
            "contracts": [],
        }
        mock_get_ops_link_snapshot.return_value = {
            "status": "attention",
            "headline": "存在接管缺口。",
            "next_actions": [],
        }
        mock_get_ops_overview.return_value = {
            "recommendation": {},
            "execution_scene": {},
            "log_sync": {},
            "release_hub": {},
        }
        mock_list_managed_nodes_with_agent_state.return_value = {
            "nodes": [
                {
                    "node_code": "mainland-worker-01",
                    "agent_state": "pending_bootstrap",
                    "remote_access_state": "ssh_ready",
                    "delivery_queue_state": "healthy",
                    "participation_state": "standby",
                }
            ],
            "summary": {
                "managed_enabled": 1,
                "remote_access_ready": 0,
                "status_counts": {"pending_bootstrap": 1},
                "remote_access_state_counts": {"ssh_ready": 1},
                "delivery_queue_state_counts": {"healthy": 1},
            },
        }
        mock_get_managed_node_handover.return_value = {
            "node_code": "mainland-worker-01",
            "summary": "已签发有效 Token，但目标节点尚未执行 bootstrap 片段。",
            "stage": {
                "code": "pending_bootstrap",
                "label": "待执行接入",
                "next_step": {
                    "code": "execute_bootstrap_plan",
                    "label": "去目标机执行接入片段",
                },
            },
        }
        mock_get_release_launchpad.return_value = {
            "launchpad_status": {
                "recommended_action_code": "fix_managed_nodes",
                "focus_ref": {"kind": "release_launchpad"},
            },
            "latest_release": {},
        }
        mock_get_recent_ops_playbook_runs.return_value = {"runs": []}
        mock_get_ops_activity_stream.return_value = {"items": []}

        payload = get_ops_stack_diagnosis(base_url="http://127.0.0.1:8100")

        self.assertEqual(
            "mainland-worker-01",
            payload["managed_nodes"]["first_handover_gap_node_code"],
        )
        self.assertEqual(
            "pending_bootstrap",
            payload["managed_nodes"]["first_handover_gap"]["stage"]["code"],
        )
        issue = next(
            item for item in payload["diagnosis"]["issues"] if item["code"] == "managed_nodes_agent_pending"
        )
        self.assertIn("mainland-worker-01", issue["detail"])
        self.assertTrue(
            any("node-handover http://127.0.0.1:8100 mainland-worker-01" in command for command in issue["commands"])
        )
        self.assertIn(
            "bash domain-api/deploy/multi-region/drive_ops_center.sh node-recover http://127.0.0.1:8100 mainland-worker-01",
            payload["diagnosis"]["recommended_commands"]["managed-node-recover"],
        )
        self.assertTrue(
            any("node-bootstrap-plan http://127.0.0.1:8100 mainland-worker-01" in command for command in issue["commands"])
        )
        mock_get_managed_node_handover.assert_called_once()

    @patch("app.services.ops_service.get_ops_activity_stream")
    @patch("app.services.ops_service.get_recent_ops_playbook_runs")
    @patch("app.services.ops_service.get_release_launchpad")
    @patch("app.services.ops_service.list_managed_nodes_with_agent_state")
    @patch("app.services.ops_service.get_ops_overview")
    @patch("app.services.ops_service.get_ops_link_snapshot")
    @patch("app.services.ops_service.get_ops_contract_registry")
    def test_stack_diagnosis_keeps_focus_refs_for_problem_runs_and_top_items(
        self,
        mock_get_ops_contract_registry,
        mock_get_ops_link_snapshot,
        mock_get_ops_overview,
        mock_list_managed_nodes_with_agent_state,
        mock_get_release_launchpad,
        mock_get_recent_ops_playbook_runs,
        mock_get_ops_activity_stream,
    ) -> None:
        mock_get_ops_contract_registry.return_value = {
            "registry_version": "2026-04-18",
            "schema_version": "v1",
            "contracts": [],
        }
        mock_get_ops_link_snapshot.return_value = {
            "status": "ready",
            "headline": "当前总检可用。",
            "next_actions": [],
        }
        mock_get_ops_overview.return_value = {
            "recommendation": {},
            "execution_scene": {},
            "log_sync": {},
            "release_hub": {},
        }
        mock_list_managed_nodes_with_agent_state.return_value = {
            "nodes": [],
            "summary": {},
        }
        mock_get_release_launchpad.return_value = {
            "launchpad_status": {},
            "latest_release": {},
        }
        mock_get_recent_ops_playbook_runs.return_value = {
            "runs": [
                {
                    "run_code": "pbr-worker-logs-001",
                    "status": "attention",
                    "status_label": "待关注",
                    "focus_step_key": "worker_logs",
                    "focus_summary": "Worker 日志步骤仍未收口。",
                    "focus_ref": {
                        "kind": "playbook_run",
                        "run_code": "pbr-worker-logs-001",
                        "focus_step_key": "worker_logs",
                    },
                }
            ]
        }
        mock_get_ops_activity_stream.return_value = {
            "items": [
                {
                    "kind": "ops_job",
                    "status": "running",
                    "summary_text": "远端 Agent 已回传关键日志样本。",
                    "occurred_at": "2026-04-18 09:22:00",
                    "focus_ref": {
                        "kind": "ops_job",
                        "job_id": 71,
                        "job_code": "ops-agent-71",
                    },
                    "source_focus_ref": {
                        "kind": "ops_job_event",
                        "job_id": 71,
                        "event_key": "job-event:71",
                    },
                    "ui_intent": {
                        "kind": "job_events",
                        "job_id": 71,
                    },
                }
            ]
        }

        payload = get_ops_stack_diagnosis(base_url="http://127.0.0.1:8100")

        self.assertEqual(
            "ops_stack_diagnosis_contract",
            payload["diagnosis"]["contract_navigation"]["primary_contract_key"],
        )
        self.assertIn("ops_driver_contract", payload["diagnosis"]["contract_navigation"]["contract_keys"])
        self.assertEqual("playbook_run", payload["playbook_runs"]["problem_runs"][0]["focus_ref"]["kind"])
        self.assertEqual("worker_logs", payload["playbook_runs"]["problem_runs"][0]["focus_ref"]["focus_step_key"])
        top_item = payload["activity_stream"]["top_items"][0]
        self.assertEqual("ops_job", top_item["focus_ref"]["kind"])
        self.assertEqual("ops_job_event", top_item["source_focus_ref"]["kind"])
        self.assertEqual("job_events", top_item["ui_intent_kind"])
        self.assertEqual("2026-04-18 09:22:00", top_item["occurred_at"])

    @patch("app.services.ops_service.get_ops_activity_stream")
    @patch("app.services.ops_service.get_recent_ops_playbook_runs")
    @patch("app.services.ops_service.get_release_launchpad")
    @patch("app.services.ops_service.list_managed_nodes_with_agent_state")
    @patch("app.services.ops_service.get_ops_overview")
    @patch("app.services.ops_service.get_ops_link_snapshot")
    @patch("app.services.ops_service.get_ops_contract_registry")
    def test_stack_diagnosis_adds_node_scene_log_commands_for_log_sample_gap(
        self,
        mock_get_ops_contract_registry,
        mock_get_ops_link_snapshot,
        mock_get_ops_overview,
        mock_list_managed_nodes_with_agent_state,
        mock_get_release_launchpad,
        mock_get_recent_ops_playbook_runs,
        mock_get_ops_activity_stream,
    ) -> None:
        mock_get_ops_contract_registry.return_value = {
            "registry_version": "2026-04-18",
            "schema_version": "v1",
            "contracts": [],
        }
        mock_get_ops_link_snapshot.return_value = {
            "status": "attention",
            "headline": "参与节点样本尚未形成。",
            "next_actions": [
                {
                    "label": "看参与节点日志",
                    "action_code": "open_worker_logs_participating",
                    "node_codes": ["mainland-worker-01"],
                    "payload": {},
                }
            ],
        }
        mock_get_ops_overview.return_value = {
            "recommendation": {},
            "execution_scene": {
                "participating_nodes": [
                    {
                        "node_code": "mainland-worker-01",
                        "status": "busy",
                    }
                ],
                "participation_summary": {
                    "dispatch_active_node_codes": ["mainland-worker-01"],
                    "recent_only_node_codes": [],
                },
                "log_sync": {
                    "enabled": True,
                    "mode": "key",
                    "line_count": 0,
                    "source_node_count": 0,
                    "covered_participating_node_count": 0,
                    "participating_node_count": 1,
                    "missing_participating_nodes": ["mainland-worker-01"],
                },
            },
            "log_sync": {},
            "release_hub": {},
        }
        mock_list_managed_nodes_with_agent_state.return_value = {
            "nodes": [],
            "summary": {},
        }
        mock_get_release_launchpad.return_value = {
            "launchpad_status": {},
            "latest_release": {},
        }
        mock_get_recent_ops_playbook_runs.return_value = {"runs": []}
        mock_get_ops_activity_stream.return_value = {"items": []}

        payload = get_ops_stack_diagnosis(base_url="http://127.0.0.1:8100")

        issue = next(
            item for item in payload["diagnosis"]["issues"] if item["code"] == "remote_log_sync_waiting_sample"
        )
        self.assertEqual("open_worker_logs_participating", issue["action_code"])
        self.assertEqual("node_scene_log", issue["focus_ref"]["kind"])
        self.assertEqual("mainland-worker-01", issue["focus_ref"]["node_code"])
        self.assertTrue(
            any(
                "scene-node-log http://127.0.0.1:8100 mainland-worker-01 120 key" in command
                for command in issue["commands"]
            )
        )
        self.assertEqual("node_scene_log", payload["diagnosis"]["next_step"]["focus_ref"]["kind"])
        self.assertTrue(
            any(
                "scene-node-log http://127.0.0.1:8100 mainland-worker-01 120 key" in command
                for command in payload["diagnosis"]["quick_commands"]
            )
        )
        self.assertTrue(
            any(
                "driver-resolve http://127.0.0.1:8100 open_worker_logs_participating" in command
                for command in issue["commands"]
            )
        )

    @patch("app.services.ops_service.get_ops_activity_stream")
    @patch("app.services.ops_service.get_recent_ops_playbook_runs")
    @patch("app.services.ops_service.get_release_launchpad")
    @patch("app.services.ops_service.list_managed_nodes_with_agent_state")
    @patch("app.services.ops_service.get_ops_overview")
    @patch("app.services.ops_service.get_ops_link_snapshot")
    @patch("app.services.ops_service.get_ops_contract_registry")
    def test_stack_diagnosis_surfaces_start_delivery_gap_from_activity_stream(
        self,
        mock_get_ops_contract_registry,
        mock_get_ops_link_snapshot,
        mock_get_ops_overview,
        mock_list_managed_nodes_with_agent_state,
        mock_get_release_launchpad,
        mock_get_recent_ops_playbook_runs,
        mock_get_ops_activity_stream,
    ) -> None:
        mock_get_ops_contract_registry.return_value = {
            "registry_version": "2026-04-18",
            "schema_version": "v1",
            "contracts": [],
        }
        mock_get_ops_link_snapshot.return_value = {
            "status": "attention",
            "headline": "最近有任务回执异常。",
            "next_actions": [],
        }
        mock_get_ops_overview.return_value = {
            "recommendation": {},
            "execution_scene": {},
            "log_sync": {},
            "release_hub": {},
        }
        mock_list_managed_nodes_with_agent_state.return_value = {
            "nodes": [],
            "summary": {},
        }
        mock_get_release_launchpad.return_value = {
            "launchpad_status": {},
            "latest_release": {},
        }
        mock_get_recent_ops_playbook_runs.return_value = {"runs": []}
        mock_get_ops_activity_stream.return_value = {
            "items": [
                {
                    "kind": "ops_job",
                    "status": "attention",
                    "job_status": "running",
                    "summary_text": "节点已接单执行，但开始回执未成功送达控制面。",
                    "occurred_at": "2026-04-18 10:22:00",
                    "job_code": "ops-agent-81",
                    "start_delivery_state": "failed_local",
                    "start_delivery_error": "temporary offline",
                    "focus_ref": {
                        "kind": "ops_job",
                        "job_id": 81,
                        "job_code": "ops-agent-81",
                        "target_node_code": "mainland-worker-01",
                    },
                    "source_focus_ref": {
                        "kind": "ops_job_event",
                        "job_id": 81,
                        "job_code": "ops-agent-81",
                        "event_key": "job-event:81",
                        "target_node_code": "mainland-worker-01",
                    },
                    "ui_intent": {
                        "kind": "job_events",
                        "job_id": 81,
                    },
                }
            ]
        }

        payload = get_ops_stack_diagnosis(base_url="http://127.0.0.1:8100")

        issue = next(
            item for item in payload["diagnosis"]["issues"] if item["code"] == "ops_job_start_delivery_failed_local"
        )
        self.assertEqual("focus_latest_job_events", issue["action_code"])
        self.assertEqual("ops_job_event", issue["focus_ref"]["kind"])
        self.assertIn("temporary offline", issue["detail"])
        decision = payload["diagnosis"]["operator_decision"]
        self.assertEqual("ops_jobs", decision["lane"])
        self.assertEqual("focus_latest_job_events", decision["primary_command_key"])
        self.assertIn("driver-resolve http://127.0.0.1:8100 focus_latest_job_events", payload["diagnosis"]["next_actions"]["primary_command"])


if __name__ == "__main__":
    unittest.main()