import unittest from unittest.mock import patch from app.services.ops_release_service import ( build_release_execution_mode_recommendation, build_release_rollout_gate, build_rollout_target_operational_readiness, ) class _EmptyCursor: def __enter__(self): return self def __exit__(self, exc_type, exc, tb): return False def execute(self, query, params=None) -> None: self.query = query self.params = params def fetchall(self): return [] def fetchone(self): return None class _EmptyConnection: def __enter__(self): return self def __exit__(self, exc_type, exc, tb): return False def cursor(self): return _EmptyCursor() class OpsReleaseServiceExecutionModeTests(unittest.TestCase): @patch("app.services.ops_release_service.get_db") @patch("app.services.ops_agent_service.ensure_ops_agent_schema") @patch("app.services.ops_agent_service.get_managed_node_onboarding") @patch("app.services.ops_job_service.list_managed_nodes") @patch("app.services.cluster_runtime_service.get_cluster_snapshot") def test_build_rollout_target_operational_readiness_accepts_ssh_ready_nodes( self, mock_get_cluster_snapshot, mock_list_managed_nodes, mock_get_managed_node_onboarding, mock_ensure_ops_agent_schema, mock_get_db, ) -> None: mock_ensure_ops_agent_schema.return_value = None mock_get_managed_node_onboarding.return_value = { "onboarding_stage": {"code": "acceptance_ready", "label": "待接管验收"}, "summary": "Node Agent 已在线，可以直接跑 onboarding.acceptance。", "recovery_decision": { "action": "run_acceptance", "label": "执行接管验收", "summary": "Node Agent 已在线，可以直接跑 onboarding.acceptance。", "command_hint": "bash drive_ops_center.sh node-acceptance-run http://127.0.0.1:8100 mainland-worker-01 cli", "window": "acceptance", }, } mock_get_db.return_value = _EmptyConnection() mock_get_cluster_snapshot.return_value = { "nodes": [ { "node_code": "mainland-worker-01", "region": "mainland", "role": "worker", "status": "online", "current_load": 0, "is_effective_worker": True, } ] } mock_list_managed_nodes.return_value = [ { "node_code": "mainland-worker-01", "region": "mainland", "role": "worker", "is_enabled": True, "ssh_host": "121.204.244.248", "ssh_user": "root", "metadata": {}, "last_seen_at": "", } ] readiness = build_rollout_target_operational_readiness( [{"node_code": "mainland-worker-01", "region": "mainland", "role": "worker"}], execution_mode="ssh", ) self.assertEqual("ssh", readiness["execution_mode"]) self.assertEqual(1, readiness["summary"]["ssh_ready_nodes"]) self.assertEqual(1, readiness["summary"]["execution_ready_nodes"]) self.assertEqual([], readiness["blocking_reasons"]) self.assertTrue(readiness["rows"][0]["ssh_ready"]) self.assertTrue(readiness["rows"][0]["execution_ready"]) self.assertTrue(readiness["rows"][0]["agent_managed"]) self.assertEqual("ssh", readiness["rows"][0]["execution_mode"]) self.assertEqual("SSH", readiness["rows"][0]["execution_mode_label"]) self.assertEqual("medium", readiness["rows"][0]["risk_level"]) self.assertEqual("run_standard_inspection", readiness["rows"][0]["recommended_action_code"]) self.assertEqual("acceptance_ready", readiness["rows"][0]["onboarding_stage_code"]) self.assertEqual("run_acceptance", readiness["rows"][0]["recovery_action"]) self.assertEqual(1, readiness["summary"]["onboarding_acceptance_ready_nodes"]) @patch("app.services.ops_release_service.get_db") @patch("app.services.ops_agent_service.ensure_ops_agent_schema") @patch("app.services.ops_agent_service.get_managed_node_onboarding") @patch("app.services.ops_job_service.list_managed_nodes") @patch("app.services.cluster_runtime_service.get_cluster_snapshot") def test_build_rollout_target_operational_readiness_adds_release_context_and_focus_ref( self, mock_get_cluster_snapshot, mock_list_managed_nodes, mock_get_managed_node_onboarding, mock_ensure_ops_agent_schema, mock_get_db, ) -> None: mock_ensure_ops_agent_schema.return_value = None mock_get_managed_node_onboarding.return_value = { "onboarding_stage": {"code": "ready", "label": "已接管"}, "summary": "当前节点无需额外恢复动作。", "recovery_decision": { "action": "noop", "label": "当前无需额外恢复动作", "summary": "当前节点暂无需要执行的接管恢复动作。", "command_hint": "", "window": "none", }, } mock_get_db.return_value = _EmptyConnection() mock_get_cluster_snapshot.return_value = { "nodes": [ { "node_code": "mainland-worker-01", "region": "mainland", "role": "worker", "status": "online", "current_load": 0, "is_effective_worker": True, "metadata": {"current_release_version": "2026.04.11+1921318"}, } ] } mock_list_managed_nodes.return_value = [ { "node_code": "mainland-worker-01", "region": "mainland", "role": "worker", "is_enabled": True, "ssh_host": "121.204.244.248", "ssh_user": "root", "metadata": {"current_release_version": "2026.04.11+1921318"}, "last_seen_at": "2026-04-18 08:00:00", } ] readiness = build_rollout_target_operational_readiness( [{"node_code": "mainland-worker-01", "region": "mainland", "role": "worker"}], execution_mode="remote-agent", desired_release={"id": 18, "release_version": "2026.04.18+bd6bcb2", "channel": "stable"}, ) row = readiness["rows"][0] self.assertEqual("2026.04.11+1921318", row["current_release_version"]) self.assertEqual("2026.04.18+bd6bcb2", row["desired_release_version"]) self.assertEqual("release_hub", row["focus_ref"]["kind"]) self.assertEqual("default_rollout_gate", row["focus_ref"]["section"]) self.assertEqual("mainland-worker-01", row["focus_ref"]["node_code"]) self.assertEqual("noop", row["recovery_action"]) @patch("app.services.ops_release_service.get_db") @patch("app.services.ops_agent_service.ensure_ops_agent_schema") @patch("app.services.ops_agent_service.get_managed_node_onboarding") @patch("app.services.ops_job_service.list_managed_nodes") @patch("app.services.cluster_runtime_service.get_cluster_snapshot") def test_build_rollout_target_operational_readiness_prefers_recovery_decision_for_blocked_nodes( self, mock_get_cluster_snapshot, mock_list_managed_nodes, mock_get_managed_node_onboarding, mock_ensure_ops_agent_schema, mock_get_db, ) -> None: mock_ensure_ops_agent_schema.return_value = None mock_get_db.return_value = _EmptyConnection() mock_get_cluster_snapshot.return_value = { "nodes": [ { "node_code": "mainland-worker-02", "region": "mainland", "role": "worker", "status": "online", "current_load": 0, "is_effective_worker": True, } ] } mock_list_managed_nodes.return_value = [ { "node_code": "mainland-worker-02", "region": "mainland", "role": "worker", "is_enabled": True, "ssh_host": "121.204.244.249", "ssh_user": "root", "metadata": {}, "last_seen_at": "", } ] mock_get_managed_node_onboarding.return_value = { "onboarding_stage": {"code": "pending_bootstrap", "label": "待接入"}, "summary": "已签发有效 Agent Token，但节点尚未完成 register/heartbeat。", "recovery_decision": { "action": "bootstrap_run", "label": "签发接入工单", "summary": "当前节点仍处于接入阶段，建议先签发 onboarding.bootstrap。", "command_hint": "bash drive_ops_center.sh node-bootstrap-run http://127.0.0.1:8100 mainland-worker-02 cli", "window": "bootstrap", }, } readiness = build_rollout_target_operational_readiness( [{"node_code": "mainland-worker-02", "region": "mainland", "role": "worker"}], execution_mode="remote-agent", ) row = readiness["rows"][0] self.assertEqual("bootstrap_run", row["recovery_action"]) self.assertEqual("bootstrap_run", row["recommended_action_code"]) self.assertEqual(1, readiness["summary"]["onboarding_bootstrap_pending_nodes"]) self.assertTrue( any("onboarding.bootstrap" in item for item in readiness["recommendations"]) ) @patch("app.services.ops_release_service.get_db") @patch("app.services.ops_agent_service.ensure_ops_agent_schema") @patch("app.services.ops_agent_service.get_managed_node_onboarding") @patch("app.services.ops_job_service.list_managed_nodes") @patch("app.services.cluster_runtime_service.get_cluster_snapshot") def test_build_rollout_target_operational_readiness_blocks_nodes_with_dead_letter_queue( self, mock_get_cluster_snapshot, mock_list_managed_nodes, mock_get_managed_node_onboarding, mock_ensure_ops_agent_schema, mock_get_db, ) -> None: mock_ensure_ops_agent_schema.return_value = None mock_get_db.return_value = _EmptyConnection() mock_get_managed_node_onboarding.return_value = { "onboarding_stage": {"code": "ready", "label": "已接管"}, "summary": "当前节点无需额外恢复动作。", "recovery_decision": { "action": "noop", "label": "当前无需额外恢复动作", "summary": "当前节点暂无需要执行的接管恢复动作。", "command_hint": "", "window": "none", }, } mock_get_cluster_snapshot.return_value = { "nodes": [ { "node_code": "mainland-worker-03", "region": "mainland", "role": "worker", "status": "online", "current_load": 0, "is_effective_worker": True, } ] } mock_list_managed_nodes.return_value = [ { "node_code": "mainland-worker-03", "region": "mainland", "role": "worker", "is_enabled": True, "ssh_host": "121.204.244.250", "ssh_user": "root", "metadata": { "delivery_queue": { "state": "dead_letter", "label": "死信 1", "reason": "当前存在 1 条死信记录", "pending_count": 0, "dead_letter_count": 1, "oldest_dead_letter_at": "2026-04-18 09:01:00", } }, "last_seen_at": "2026-04-18 09:05:00", } ] readiness = build_rollout_target_operational_readiness( [{"node_code": "mainland-worker-03", "region": "mainland", "role": "worker"}], execution_mode="remote-agent", ) row = readiness["rows"][0] self.assertEqual("dead_letter", row["delivery_queue_state"]) self.assertEqual(1, row["delivery_queue_dead_letter_count"]) self.assertTrue( any("死信" in item for item in row["blocking_reasons"]) ) self.assertEqual(1, readiness["summary"]["queue_dead_letter_nodes"]) self.assertTrue( any("死信回执" in item for item in readiness["blocking_reasons"]) ) self.assertTrue( any("死信队列" in item for item in readiness["recommendations"]) ) @patch("app.services.ops_release_service.build_rollout_target_operational_readiness") def test_build_release_rollout_gate_uses_execution_ready_count_for_ssh(self, mock_build_readiness) -> None: mock_build_readiness.return_value = { "execution_mode": "ssh", "summary": { "nodes_total": 2, "execution_ready_nodes": 1, "inspection_healthy_nodes": 2, }, "rows": [], "blocking_reasons": ["有 1 台目标节点尚未配置完整 SSH 入口，当前不适合直接 SSH 发布。"], "warning_reasons": [], "recommendations": [], } gate = build_release_rollout_gate( { "id": 11, "release_version": "2026.04.18-rc1", "status": "ready", "artifact_url": "https://example.com/domaincheck.tar.gz", }, [{"node_code": "mainland-worker-01"}, {"node_code": "mainland-worker-02"}], execution_mode="ssh", ) self.assertEqual("ssh", gate["execution_mode"]) self.assertEqual("SSH", gate["execution_mode_label"]) self.assertEqual("blocked", gate["status"]) self.assertIn("1/2 台 SSH 就绪", gate["summary"]) @patch("app.services.ops_release_service.build_rollout_target_operational_readiness") def test_build_release_rollout_gate_uses_human_label_for_remote_agent(self, mock_build_readiness) -> None: mock_build_readiness.return_value = { "execution_mode": "remote-agent", "summary": { "nodes_total": 2, "execution_ready_nodes": 1, "inspection_healthy_nodes": 2, }, "rows": [], "blocking_reasons": ["有 1 台目标节点 Agent 未在线，当前不适合直接 remote-agent 发布。"], "warning_reasons": [], "recommendations": [], } gate = build_release_rollout_gate( { "id": 12, "release_version": "2026.04.18-rc1", "status": "ready", "artifact_url": "https://example.com/domaincheck.tar.gz", }, [{"node_code": "mainland-worker-01"}, {"node_code": "mainland-worker-02"}], execution_mode="remote-agent", ) self.assertEqual("remote-agent", gate["execution_mode"]) self.assertEqual("远端 Agent", gate["execution_mode_label"]) self.assertIn("1/2 台 远端 Agent 就绪", gate["summary"]) @patch("app.services.ops_release_service.build_release_rollout_gate") def test_build_release_execution_mode_recommendation_prefers_ssh_when_gate_is_better(self, mock_build_gate) -> None: def _fake_gate(release, target_nodes, *, execution_mode="remote-agent"): if execution_mode == "ssh": return { "status": "ready", "status_label": "可发 Rollout", "summary": "SSH 路径已收口", "execution_mode": "ssh", "execution_mode_label": "SSH", "blocking_reasons": [], "warning_reasons": [], "operational_readiness": { "summary": {"nodes_total": 2, "execution_ready_nodes": 2}, "rows": [], }, } return { "status": "blocked", "status_label": "存在阻断", "summary": "remote-agent 尚未收口", "execution_mode": "remote-agent", "execution_mode_label": "远端 Agent", "blocking_reasons": ["Agent 未就绪"], "warning_reasons": [], "operational_readiness": { "summary": {"nodes_total": 2, "execution_ready_nodes": 1}, "rows": [], }, } mock_build_gate.side_effect = _fake_gate recommendation = build_release_execution_mode_recommendation( { "id": 18, "release_version": "2026.04.18-rc2", "status": "ready", "artifact_url": "https://example.com/domaincheck.tar.gz", }, [{"node_code": "mainland-worker-01"}, {"node_code": "mainland-worker-02"}], ) self.assertEqual("ssh", recommendation["recommended_mode"]) self.assertEqual("SSH", recommendation["recommended_mode_label"]) self.assertEqual("ready", recommendation["recommended_gate"]["status"]) self.assertIn("SSH", recommendation["reason"]) if __name__ == "__main__": unittest.main()