524 lines
21 KiB
Python
524 lines
21 KiB
Python
import unittest
|
||
from unittest.mock import patch
|
||
|
||
from app.services.ops_release_service import (
|
||
build_release_execution_mode_recommendation,
|
||
build_release_rollout_gate,
|
||
build_rollout_target_operational_readiness,
|
||
)
|
||
|
||
|
||
class _EmptyCursor:
|
||
def __enter__(self):
|
||
return self
|
||
|
||
def __exit__(self, exc_type, exc, tb):
|
||
return False
|
||
|
||
def execute(self, query, params=None) -> None:
|
||
self.query = query
|
||
self.params = params
|
||
|
||
def fetchall(self):
|
||
return []
|
||
|
||
def fetchone(self):
|
||
return None
|
||
|
||
|
||
class _EmptyConnection:
|
||
def __enter__(self):
|
||
return self
|
||
|
||
def __exit__(self, exc_type, exc, tb):
|
||
return False
|
||
|
||
def cursor(self):
|
||
return _EmptyCursor()
|
||
|
||
|
||
class OpsReleaseServiceExecutionModeTests(unittest.TestCase):
|
||
@patch("app.services.ops_release_service.get_db")
|
||
@patch("app.services.ops_agent_service.ensure_ops_agent_schema")
|
||
@patch("app.services.ops_agent_service.get_managed_node_onboarding")
|
||
@patch("app.services.ops_agent_service.list_managed_nodes_with_agent_state")
|
||
@patch("app.services.ops_job_service.list_managed_nodes")
|
||
@patch("app.services.cluster_runtime_service.get_cluster_snapshot")
|
||
def test_build_rollout_target_operational_readiness_reuses_managed_node_snapshot_for_onboarding(
|
||
self,
|
||
mock_get_cluster_snapshot,
|
||
mock_list_managed_nodes,
|
||
mock_list_managed_nodes_with_agent_state,
|
||
mock_get_managed_node_onboarding,
|
||
mock_ensure_ops_agent_schema,
|
||
mock_get_db,
|
||
) -> None:
|
||
captured_nodes_payloads = []
|
||
|
||
def _fake_onboarding(node_code, **kwargs):
|
||
captured_nodes_payloads.append(kwargs.get("nodes_payload"))
|
||
return {
|
||
"onboarding_stage": {"code": "ready", "label": "已接管"},
|
||
"summary": f"{node_code} ready",
|
||
"recovery_decision": {
|
||
"action": "noop",
|
||
"label": "当前无需额外恢复动作",
|
||
"summary": "当前节点暂无需要执行的接管恢复动作。",
|
||
"command_hint": "",
|
||
"window": "none",
|
||
},
|
||
}
|
||
|
||
mock_ensure_ops_agent_schema.return_value = None
|
||
mock_get_db.return_value = _EmptyConnection()
|
||
mock_get_cluster_snapshot.return_value = {
|
||
"nodes": [
|
||
{
|
||
"node_code": "mainland-worker-01",
|
||
"region": "mainland",
|
||
"role": "worker",
|
||
"status": "online",
|
||
"current_load": 0,
|
||
"is_effective_worker": True,
|
||
},
|
||
{
|
||
"node_code": "mainland-worker-02",
|
||
"region": "mainland",
|
||
"role": "worker",
|
||
"status": "online",
|
||
"current_load": 0,
|
||
"is_effective_worker": True,
|
||
},
|
||
]
|
||
}
|
||
mock_list_managed_nodes.return_value = [
|
||
{
|
||
"node_code": "mainland-worker-01",
|
||
"region": "mainland",
|
||
"role": "worker",
|
||
"is_enabled": True,
|
||
"ssh_host": "121.204.244.248",
|
||
"ssh_user": "root",
|
||
"metadata": {},
|
||
"last_seen_at": "",
|
||
},
|
||
{
|
||
"node_code": "mainland-worker-02",
|
||
"region": "mainland",
|
||
"role": "worker",
|
||
"is_enabled": True,
|
||
"ssh_host": "121.204.244.249",
|
||
"ssh_user": "root",
|
||
"metadata": {},
|
||
"last_seen_at": "",
|
||
},
|
||
]
|
||
mock_list_managed_nodes_with_agent_state.return_value = {"nodes": [{"node_code": "mainland-worker-01"}, {"node_code": "mainland-worker-02"}]}
|
||
mock_get_managed_node_onboarding.side_effect = _fake_onboarding
|
||
|
||
readiness = build_rollout_target_operational_readiness(
|
||
[
|
||
{"node_code": "mainland-worker-01", "region": "mainland", "role": "worker"},
|
||
{"node_code": "mainland-worker-02", "region": "mainland", "role": "worker"},
|
||
],
|
||
execution_mode="remote-agent",
|
||
)
|
||
|
||
self.assertEqual(2, len(readiness["rows"]))
|
||
mock_list_managed_nodes_with_agent_state.assert_called_once()
|
||
self.assertEqual(2, len(captured_nodes_payloads))
|
||
self.assertTrue(all(payload == {"nodes": [{"node_code": "mainland-worker-01"}, {"node_code": "mainland-worker-02"}]} for payload in captured_nodes_payloads))
|
||
|
||
@patch("app.services.ops_release_service.get_db")
|
||
@patch("app.services.ops_agent_service.ensure_ops_agent_schema")
|
||
@patch("app.services.ops_agent_service.get_managed_node_onboarding")
|
||
@patch("app.services.ops_job_service.list_managed_nodes")
|
||
@patch("app.services.cluster_runtime_service.get_cluster_snapshot")
|
||
def test_build_rollout_target_operational_readiness_accepts_ssh_ready_nodes(
|
||
self,
|
||
mock_get_cluster_snapshot,
|
||
mock_list_managed_nodes,
|
||
mock_get_managed_node_onboarding,
|
||
mock_ensure_ops_agent_schema,
|
||
mock_get_db,
|
||
) -> None:
|
||
mock_ensure_ops_agent_schema.return_value = None
|
||
mock_get_managed_node_onboarding.return_value = {
|
||
"onboarding_stage": {"code": "acceptance_ready", "label": "待接管验收"},
|
||
"summary": "Node Agent 已在线，可以直接跑 onboarding.acceptance。",
|
||
"recovery_decision": {
|
||
"action": "run_acceptance",
|
||
"label": "执行接管验收",
|
||
"summary": "Node Agent 已在线，可以直接跑 onboarding.acceptance。",
|
||
"command_hint": "bash drive_ops_center.sh node-acceptance-run http://127.0.0.1:8100 mainland-worker-01 cli",
|
||
"window": "acceptance",
|
||
},
|
||
}
|
||
mock_get_db.return_value = _EmptyConnection()
|
||
mock_get_cluster_snapshot.return_value = {
|
||
"nodes": [
|
||
{
|
||
"node_code": "mainland-worker-01",
|
||
"region": "mainland",
|
||
"role": "worker",
|
||
"status": "online",
|
||
"current_load": 0,
|
||
"is_effective_worker": True,
|
||
}
|
||
]
|
||
}
|
||
mock_list_managed_nodes.return_value = [
|
||
{
|
||
"node_code": "mainland-worker-01",
|
||
"region": "mainland",
|
||
"role": "worker",
|
||
"is_enabled": True,
|
||
"ssh_host": "121.204.244.248",
|
||
"ssh_user": "root",
|
||
"metadata": {},
|
||
"last_seen_at": "",
|
||
}
|
||
]
|
||
|
||
readiness = build_rollout_target_operational_readiness(
|
||
[{"node_code": "mainland-worker-01", "region": "mainland", "role": "worker"}],
|
||
execution_mode="ssh",
|
||
)
|
||
|
||
self.assertEqual("ssh", readiness["execution_mode"])
|
||
self.assertEqual(1, readiness["summary"]["ssh_ready_nodes"])
|
||
self.assertEqual(1, readiness["summary"]["execution_ready_nodes"])
|
||
self.assertEqual([], readiness["blocking_reasons"])
|
||
self.assertTrue(readiness["rows"][0]["ssh_ready"])
|
||
self.assertTrue(readiness["rows"][0]["execution_ready"])
|
||
self.assertTrue(readiness["rows"][0]["agent_managed"])
|
||
self.assertEqual("ssh", readiness["rows"][0]["execution_mode"])
|
||
self.assertEqual("SSH", readiness["rows"][0]["execution_mode_label"])
|
||
self.assertEqual("medium", readiness["rows"][0]["risk_level"])
|
||
self.assertEqual("run_standard_inspection", readiness["rows"][0]["recommended_action_code"])
|
||
self.assertEqual("acceptance_ready", readiness["rows"][0]["onboarding_stage_code"])
|
||
self.assertEqual("run_acceptance", readiness["rows"][0]["recovery_action"])
|
||
self.assertEqual(1, readiness["summary"]["onboarding_acceptance_ready_nodes"])
|
||
|
||
@patch("app.services.ops_release_service.get_db")
|
||
@patch("app.services.ops_agent_service.ensure_ops_agent_schema")
|
||
@patch("app.services.ops_agent_service.get_managed_node_onboarding")
|
||
@patch("app.services.ops_job_service.list_managed_nodes")
|
||
@patch("app.services.cluster_runtime_service.get_cluster_snapshot")
|
||
def test_build_rollout_target_operational_readiness_adds_release_context_and_focus_ref(
|
||
self,
|
||
mock_get_cluster_snapshot,
|
||
mock_list_managed_nodes,
|
||
mock_get_managed_node_onboarding,
|
||
mock_ensure_ops_agent_schema,
|
||
mock_get_db,
|
||
) -> None:
|
||
mock_ensure_ops_agent_schema.return_value = None
|
||
mock_get_managed_node_onboarding.return_value = {
|
||
"onboarding_stage": {"code": "ready", "label": "已接管"},
|
||
"summary": "当前节点无需额外恢复动作。",
|
||
"recovery_decision": {
|
||
"action": "noop",
|
||
"label": "当前无需额外恢复动作",
|
||
"summary": "当前节点暂无需要执行的接管恢复动作。",
|
||
"command_hint": "",
|
||
"window": "none",
|
||
},
|
||
}
|
||
mock_get_db.return_value = _EmptyConnection()
|
||
mock_get_cluster_snapshot.return_value = {
|
||
"nodes": [
|
||
{
|
||
"node_code": "mainland-worker-01",
|
||
"region": "mainland",
|
||
"role": "worker",
|
||
"status": "online",
|
||
"current_load": 0,
|
||
"is_effective_worker": True,
|
||
"metadata": {"current_release_version": "2026.04.11+1921318"},
|
||
}
|
||
]
|
||
}
|
||
mock_list_managed_nodes.return_value = [
|
||
{
|
||
"node_code": "mainland-worker-01",
|
||
"region": "mainland",
|
||
"role": "worker",
|
||
"is_enabled": True,
|
||
"ssh_host": "121.204.244.248",
|
||
"ssh_user": "root",
|
||
"metadata": {"current_release_version": "2026.04.11+1921318"},
|
||
"last_seen_at": "2026-04-18 08:00:00",
|
||
}
|
||
]
|
||
|
||
readiness = build_rollout_target_operational_readiness(
|
||
[{"node_code": "mainland-worker-01", "region": "mainland", "role": "worker"}],
|
||
execution_mode="remote-agent",
|
||
desired_release={"id": 18, "release_version": "2026.04.18+bd6bcb2", "channel": "stable"},
|
||
)
|
||
|
||
row = readiness["rows"][0]
|
||
self.assertEqual("2026.04.11+1921318", row["current_release_version"])
|
||
self.assertEqual("2026.04.18+bd6bcb2", row["desired_release_version"])
|
||
self.assertEqual("release_hub", row["focus_ref"]["kind"])
|
||
self.assertEqual("default_rollout_gate", row["focus_ref"]["section"])
|
||
self.assertEqual("mainland-worker-01", row["focus_ref"]["node_code"])
|
||
self.assertEqual("noop", row["recovery_action"])
|
||
|
||
@patch("app.services.ops_release_service.get_db")
|
||
@patch("app.services.ops_agent_service.ensure_ops_agent_schema")
|
||
@patch("app.services.ops_agent_service.get_managed_node_onboarding")
|
||
@patch("app.services.ops_job_service.list_managed_nodes")
|
||
@patch("app.services.cluster_runtime_service.get_cluster_snapshot")
|
||
def test_build_rollout_target_operational_readiness_prefers_recovery_decision_for_blocked_nodes(
|
||
self,
|
||
mock_get_cluster_snapshot,
|
||
mock_list_managed_nodes,
|
||
mock_get_managed_node_onboarding,
|
||
mock_ensure_ops_agent_schema,
|
||
mock_get_db,
|
||
) -> None:
|
||
mock_ensure_ops_agent_schema.return_value = None
|
||
mock_get_db.return_value = _EmptyConnection()
|
||
mock_get_cluster_snapshot.return_value = {
|
||
"nodes": [
|
||
{
|
||
"node_code": "mainland-worker-02",
|
||
"region": "mainland",
|
||
"role": "worker",
|
||
"status": "online",
|
||
"current_load": 0,
|
||
"is_effective_worker": True,
|
||
}
|
||
]
|
||
}
|
||
mock_list_managed_nodes.return_value = [
|
||
{
|
||
"node_code": "mainland-worker-02",
|
||
"region": "mainland",
|
||
"role": "worker",
|
||
"is_enabled": True,
|
||
"ssh_host": "121.204.244.249",
|
||
"ssh_user": "root",
|
||
"metadata": {},
|
||
"last_seen_at": "",
|
||
}
|
||
]
|
||
mock_get_managed_node_onboarding.return_value = {
|
||
"onboarding_stage": {"code": "pending_bootstrap", "label": "待接入"},
|
||
"summary": "已签发有效 Agent Token，但节点尚未完成 register/heartbeat。",
|
||
"recovery_decision": {
|
||
"action": "bootstrap_run",
|
||
"label": "签发接入工单",
|
||
"summary": "当前节点仍处于接入阶段，建议先签发 onboarding.bootstrap。",
|
||
"command_hint": "bash drive_ops_center.sh node-bootstrap-run http://127.0.0.1:8100 mainland-worker-02 cli",
|
||
"window": "bootstrap",
|
||
},
|
||
}
|
||
|
||
readiness = build_rollout_target_operational_readiness(
|
||
[{"node_code": "mainland-worker-02", "region": "mainland", "role": "worker"}],
|
||
execution_mode="remote-agent",
|
||
)
|
||
|
||
row = readiness["rows"][0]
|
||
self.assertEqual("bootstrap_run", row["recovery_action"])
|
||
self.assertEqual("bootstrap_run", row["recommended_action_code"])
|
||
self.assertEqual(1, readiness["summary"]["onboarding_bootstrap_pending_nodes"])
|
||
self.assertTrue(
|
||
any("onboarding.bootstrap" in item for item in readiness["recommendations"])
|
||
)
|
||
|
||
@patch("app.services.ops_release_service.get_db")
|
||
@patch("app.services.ops_agent_service.ensure_ops_agent_schema")
|
||
@patch("app.services.ops_agent_service.get_managed_node_onboarding")
|
||
@patch("app.services.ops_job_service.list_managed_nodes")
|
||
@patch("app.services.cluster_runtime_service.get_cluster_snapshot")
|
||
def test_build_rollout_target_operational_readiness_blocks_nodes_with_dead_letter_queue(
|
||
self,
|
||
mock_get_cluster_snapshot,
|
||
mock_list_managed_nodes,
|
||
mock_get_managed_node_onboarding,
|
||
mock_ensure_ops_agent_schema,
|
||
mock_get_db,
|
||
) -> None:
|
||
mock_ensure_ops_agent_schema.return_value = None
|
||
mock_get_db.return_value = _EmptyConnection()
|
||
mock_get_managed_node_onboarding.return_value = {
|
||
"onboarding_stage": {"code": "ready", "label": "已接管"},
|
||
"summary": "当前节点无需额外恢复动作。",
|
||
"recovery_decision": {
|
||
"action": "noop",
|
||
"label": "当前无需额外恢复动作",
|
||
"summary": "当前节点暂无需要执行的接管恢复动作。",
|
||
"command_hint": "",
|
||
"window": "none",
|
||
},
|
||
}
|
||
mock_get_cluster_snapshot.return_value = {
|
||
"nodes": [
|
||
{
|
||
"node_code": "mainland-worker-03",
|
||
"region": "mainland",
|
||
"role": "worker",
|
||
"status": "online",
|
||
"current_load": 0,
|
||
"is_effective_worker": True,
|
||
}
|
||
]
|
||
}
|
||
mock_list_managed_nodes.return_value = [
|
||
{
|
||
"node_code": "mainland-worker-03",
|
||
"region": "mainland",
|
||
"role": "worker",
|
||
"is_enabled": True,
|
||
"ssh_host": "121.204.244.250",
|
||
"ssh_user": "root",
|
||
"metadata": {
|
||
"delivery_queue": {
|
||
"state": "dead_letter",
|
||
"label": "死信 1",
|
||
"reason": "当前存在 1 条死信记录",
|
||
"pending_count": 0,
|
||
"dead_letter_count": 1,
|
||
"oldest_dead_letter_at": "2026-04-18 09:01:00",
|
||
}
|
||
},
|
||
"last_seen_at": "2026-04-18 09:05:00",
|
||
}
|
||
]
|
||
|
||
readiness = build_rollout_target_operational_readiness(
|
||
[{"node_code": "mainland-worker-03", "region": "mainland", "role": "worker"}],
|
||
execution_mode="remote-agent",
|
||
)
|
||
|
||
row = readiness["rows"][0]
|
||
self.assertEqual("dead_letter", row["delivery_queue_state"])
|
||
self.assertEqual(1, row["delivery_queue_dead_letter_count"])
|
||
self.assertTrue(
|
||
any("死信" in item for item in row["blocking_reasons"])
|
||
)
|
||
self.assertEqual(1, readiness["summary"]["queue_dead_letter_nodes"])
|
||
self.assertTrue(
|
||
any("死信回执" in item for item in readiness["blocking_reasons"])
|
||
)
|
||
self.assertTrue(
|
||
any("死信队列" in item for item in readiness["recommendations"])
|
||
)
|
||
|
||
@patch("app.services.ops_release_service.build_rollout_target_operational_readiness")
|
||
def test_build_release_rollout_gate_uses_execution_ready_count_for_ssh(self, mock_build_readiness) -> None:
|
||
mock_build_readiness.return_value = {
|
||
"execution_mode": "ssh",
|
||
"summary": {
|
||
"nodes_total": 2,
|
||
"execution_ready_nodes": 1,
|
||
"inspection_healthy_nodes": 2,
|
||
},
|
||
"rows": [],
|
||
"blocking_reasons": ["有 1 台目标节点尚未配置完整 SSH 入口，当前不适合直接 SSH 发布。"],
|
||
"warning_reasons": [],
|
||
"recommendations": [],
|
||
}
|
||
|
||
gate = build_release_rollout_gate(
|
||
{
|
||
"id": 11,
|
||
"release_version": "2026.04.18-rc1",
|
||
"status": "ready",
|
||
"artifact_url": "https://example.com/domaincheck.tar.gz",
|
||
},
|
||
[{"node_code": "mainland-worker-01"}, {"node_code": "mainland-worker-02"}],
|
||
execution_mode="ssh",
|
||
)
|
||
|
||
self.assertEqual("ssh", gate["execution_mode"])
|
||
self.assertEqual("SSH", gate["execution_mode_label"])
|
||
self.assertEqual("blocked", gate["status"])
|
||
self.assertIn("1/2 台 SSH 就绪", gate["summary"])
|
||
|
||
@patch("app.services.ops_release_service.build_rollout_target_operational_readiness")
|
||
def test_build_release_rollout_gate_uses_human_label_for_remote_agent(self, mock_build_readiness) -> None:
|
||
mock_build_readiness.return_value = {
|
||
"execution_mode": "remote-agent",
|
||
"summary": {
|
||
"nodes_total": 2,
|
||
"execution_ready_nodes": 1,
|
||
"inspection_healthy_nodes": 2,
|
||
},
|
||
"rows": [],
|
||
"blocking_reasons": ["有 1 台目标节点 Agent 未在线，当前不适合直接 remote-agent 发布。"],
|
||
"warning_reasons": [],
|
||
"recommendations": [],
|
||
}
|
||
|
||
gate = build_release_rollout_gate(
|
||
{
|
||
"id": 12,
|
||
"release_version": "2026.04.18-rc1",
|
||
"status": "ready",
|
||
"artifact_url": "https://example.com/domaincheck.tar.gz",
|
||
},
|
||
[{"node_code": "mainland-worker-01"}, {"node_code": "mainland-worker-02"}],
|
||
execution_mode="remote-agent",
|
||
)
|
||
|
||
self.assertEqual("remote-agent", gate["execution_mode"])
|
||
self.assertEqual("远端 Agent", gate["execution_mode_label"])
|
||
self.assertIn("1/2 台 远端 Agent 就绪", gate["summary"])
|
||
|
||
@patch("app.services.ops_release_service.build_release_rollout_gate")
|
||
def test_build_release_execution_mode_recommendation_prefers_ssh_when_gate_is_better(self, mock_build_gate) -> None:
|
||
def _fake_gate(release, target_nodes, *, execution_mode="remote-agent"):
|
||
if execution_mode == "ssh":
|
||
return {
|
||
"status": "ready",
|
||
"status_label": "可发 Rollout",
|
||
"summary": "SSH 路径已收口",
|
||
"execution_mode": "ssh",
|
||
"execution_mode_label": "SSH",
|
||
"blocking_reasons": [],
|
||
"warning_reasons": [],
|
||
"operational_readiness": {
|
||
"summary": {"nodes_total": 2, "execution_ready_nodes": 2},
|
||
"rows": [],
|
||
},
|
||
}
|
||
return {
|
||
"status": "blocked",
|
||
"status_label": "存在阻断",
|
||
"summary": "remote-agent 尚未收口",
|
||
"execution_mode": "remote-agent",
|
||
"execution_mode_label": "远端 Agent",
|
||
"blocking_reasons": ["Agent 未就绪"],
|
||
"warning_reasons": [],
|
||
"operational_readiness": {
|
||
"summary": {"nodes_total": 2, "execution_ready_nodes": 1},
|
||
"rows": [],
|
||
},
|
||
}
|
||
|
||
mock_build_gate.side_effect = _fake_gate
|
||
|
||
recommendation = build_release_execution_mode_recommendation(
|
||
{
|
||
"id": 18,
|
||
"release_version": "2026.04.18-rc2",
|
||
"status": "ready",
|
||
"artifact_url": "https://example.com/domaincheck.tar.gz",
|
||
},
|
||
[{"node_code": "mainland-worker-01"}, {"node_code": "mainland-worker-02"}],
|
||
)
|
||
|
||
self.assertEqual("ssh", recommendation["recommended_mode"])
|
||
self.assertEqual("SSH", recommendation["recommended_mode_label"])
|
||
self.assertEqual("ready", recommendation["recommended_gate"]["status"])
|
||
self.assertIn("SSH", recommendation["reason"])
|
||
|
||
|
||
if __name__ == "__main__":
|
||
unittest.main()
|