admin/getDomain
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add ops center and node onboarding flow

Browse Source
This commit is contained in:
Your Name
2026-04-18 23:52:51 +08:00
parent 246838ae4c
commit b9c29481b5
142 changed files with 89727 additions and 186 deletions
Download Patch File Download Diff File Expand all files Collapse all files

432
domain-api/tests/test_ops_release_service_execution_modes.py Normal file
View File

@@ -0,0 +1,432 @@
import unittest
from unittest.mock import patch
from app.services.ops_release_service import (
build_release_execution_mode_recommendation,
build_release_rollout_gate,
build_rollout_target_operational_readiness,
)
class _EmptyCursor:
def __enter__(self):
return self
def __exit__(self, exc_type, exc, tb):
return False
def execute(self, query, params=None) -> None:
self.query = query
self.params = params
def fetchall(self):
return []
def fetchone(self):
return None
class _EmptyConnection:
def __enter__(self):
return self
def __exit__(self, exc_type, exc, tb):
return False
def cursor(self):
return _EmptyCursor()
class OpsReleaseServiceExecutionModeTests(unittest.TestCase):
@patch("app.services.ops_release_service.get_db")
@patch("app.services.ops_agent_service.ensure_ops_agent_schema")
@patch("app.services.ops_agent_service.get_managed_node_onboarding")
@patch("app.services.ops_job_service.list_managed_nodes")
@patch("app.services.cluster_runtime_service.get_cluster_snapshot")
def test_build_rollout_target_operational_readiness_accepts_ssh_ready_nodes(
self,
mock_get_cluster_snapshot,
mock_list_managed_nodes,
mock_get_managed_node_onboarding,
mock_ensure_ops_agent_schema,
mock_get_db,
) -> None:
mock_ensure_ops_agent_schema.return_value = None
mock_get_managed_node_onboarding.return_value = {
"onboarding_stage": {"code": "acceptance_ready", "label": "待接管验收"},
"summary": "Node Agent 已在线，可以直接跑 onboarding.acceptance。",
"recovery_decision": {
"action": "run_acceptance",
"label": "执行接管验收",
"summary": "Node Agent 已在线，可以直接跑 onboarding.acceptance。",
"command_hint": "bash drive_ops_center.sh node-acceptance-run http://127.0.0.1:8100 mainland-worker-01 cli",
"window": "acceptance",
},
}
mock_get_db.return_value = _EmptyConnection()
mock_get_cluster_snapshot.return_value = {
"nodes": [
{
"node_code": "mainland-worker-01",
"region": "mainland",
"role": "worker",
"status": "online",
"current_load": 0,
"is_effective_worker": True,
}
]
}
mock_list_managed_nodes.return_value = [
{
"node_code": "mainland-worker-01",
"region": "mainland",
"role": "worker",
"is_enabled": True,
"ssh_host": "121.204.244.248",
"ssh_user": "root",
"metadata": {},
"last_seen_at": "",
}
]
readiness = build_rollout_target_operational_readiness(
[{"node_code": "mainland-worker-01", "region": "mainland", "role": "worker"}],
execution_mode="ssh",
)
self.assertEqual("ssh", readiness["execution_mode"])
self.assertEqual(1, readiness["summary"]["ssh_ready_nodes"])
self.assertEqual(1, readiness["summary"]["execution_ready_nodes"])
self.assertEqual([], readiness["blocking_reasons"])
self.assertTrue(readiness["rows"][0]["ssh_ready"])
self.assertTrue(readiness["rows"][0]["execution_ready"])
self.assertTrue(readiness["rows"][0]["agent_managed"])
self.assertEqual("ssh", readiness["rows"][0]["execution_mode"])
self.assertEqual("SSH", readiness["rows"][0]["execution_mode_label"])
self.assertEqual("medium", readiness["rows"][0]["risk_level"])
self.assertEqual("run_standard_inspection", readiness["rows"][0]["recommended_action_code"])
self.assertEqual("acceptance_ready", readiness["rows"][0]["onboarding_stage_code"])
self.assertEqual("run_acceptance", readiness["rows"][0]["recovery_action"])
self.assertEqual(1, readiness["summary"]["onboarding_acceptance_ready_nodes"])
@patch("app.services.ops_release_service.get_db")
@patch("app.services.ops_agent_service.ensure_ops_agent_schema")
@patch("app.services.ops_agent_service.get_managed_node_onboarding")
@patch("app.services.ops_job_service.list_managed_nodes")
@patch("app.services.cluster_runtime_service.get_cluster_snapshot")
def test_build_rollout_target_operational_readiness_adds_release_context_and_focus_ref(
self,
mock_get_cluster_snapshot,
mock_list_managed_nodes,
mock_get_managed_node_onboarding,
mock_ensure_ops_agent_schema,
mock_get_db,
) -> None:
mock_ensure_ops_agent_schema.return_value = None
mock_get_managed_node_onboarding.return_value = {
"onboarding_stage": {"code": "ready", "label": "已接管"},
"summary": "当前节点无需额外恢复动作。",
"recovery_decision": {
"action": "noop",
"label": "当前无需额外恢复动作",
"summary": "当前节点暂无需要执行的接管恢复动作。",
"command_hint": "",
"window": "none",
},
}
mock_get_db.return_value = _EmptyConnection()
mock_get_cluster_snapshot.return_value = {
"nodes": [
{
"node_code": "mainland-worker-01",
"region": "mainland",
"role": "worker",
"status": "online",
"current_load": 0,
"is_effective_worker": True,
"metadata": {"current_release_version": "2026.04.11+1921318"},
}
]
}
mock_list_managed_nodes.return_value = [
{
"node_code": "mainland-worker-01",
"region": "mainland",
"role": "worker",
"is_enabled": True,
"ssh_host": "121.204.244.248",
"ssh_user": "root",
"metadata": {"current_release_version": "2026.04.11+1921318"},
"last_seen_at": "2026-04-18 08:00:00",
}
]
readiness = build_rollout_target_operational_readiness(
[{"node_code": "mainland-worker-01", "region": "mainland", "role": "worker"}],
execution_mode="remote-agent",
desired_release={"id": 18, "release_version": "2026.04.18+bd6bcb2", "channel": "stable"},
)
row = readiness["rows"][0]
self.assertEqual("2026.04.11+1921318", row["current_release_version"])
self.assertEqual("2026.04.18+bd6bcb2", row["desired_release_version"])
self.assertEqual("release_hub", row["focus_ref"]["kind"])
self.assertEqual("default_rollout_gate", row["focus_ref"]["section"])
self.assertEqual("mainland-worker-01", row["focus_ref"]["node_code"])
self.assertEqual("noop", row["recovery_action"])
@patch("app.services.ops_release_service.get_db")
@patch("app.services.ops_agent_service.ensure_ops_agent_schema")
@patch("app.services.ops_agent_service.get_managed_node_onboarding")
@patch("app.services.ops_job_service.list_managed_nodes")
@patch("app.services.cluster_runtime_service.get_cluster_snapshot")
def test_build_rollout_target_operational_readiness_prefers_recovery_decision_for_blocked_nodes(
self,
mock_get_cluster_snapshot,
mock_list_managed_nodes,
mock_get_managed_node_onboarding,
mock_ensure_ops_agent_schema,
mock_get_db,
) -> None:
mock_ensure_ops_agent_schema.return_value = None
mock_get_db.return_value = _EmptyConnection()
mock_get_cluster_snapshot.return_value = {
"nodes": [
{
"node_code": "mainland-worker-02",
"region": "mainland",
"role": "worker",
"status": "online",
"current_load": 0,
"is_effective_worker": True,
}
]
}
mock_list_managed_nodes.return_value = [
{
"node_code": "mainland-worker-02",
"region": "mainland",
"role": "worker",
"is_enabled": True,
"ssh_host": "121.204.244.249",
"ssh_user": "root",
"metadata": {},
"last_seen_at": "",
}
]
mock_get_managed_node_onboarding.return_value = {
"onboarding_stage": {"code": "pending_bootstrap", "label": "待接入"},
"summary": "已签发有效 Agent Token，但节点尚未完成 register/heartbeat。",
"recovery_decision": {
"action": "bootstrap_run",
"label": "签发接入工单",
"summary": "当前节点仍处于接入阶段，建议先签发 onboarding.bootstrap。",
"command_hint": "bash drive_ops_center.sh node-bootstrap-run http://127.0.0.1:8100 mainland-worker-02 cli",
"window": "bootstrap",
},
}
readiness = build_rollout_target_operational_readiness(
[{"node_code": "mainland-worker-02", "region": "mainland", "role": "worker"}],
execution_mode="remote-agent",
)
row = readiness["rows"][0]
self.assertEqual("bootstrap_run", row["recovery_action"])
self.assertEqual("bootstrap_run", row["recommended_action_code"])
self.assertEqual(1, readiness["summary"]["onboarding_bootstrap_pending_nodes"])
self.assertTrue(
any("onboarding.bootstrap" in item for item in readiness["recommendations"])
)
@patch("app.services.ops_release_service.get_db")
@patch("app.services.ops_agent_service.ensure_ops_agent_schema")
@patch("app.services.ops_agent_service.get_managed_node_onboarding")
@patch("app.services.ops_job_service.list_managed_nodes")
@patch("app.services.cluster_runtime_service.get_cluster_snapshot")
def test_build_rollout_target_operational_readiness_blocks_nodes_with_dead_letter_queue(
self,
mock_get_cluster_snapshot,
mock_list_managed_nodes,
mock_get_managed_node_onboarding,
mock_ensure_ops_agent_schema,
mock_get_db,
) -> None:
mock_ensure_ops_agent_schema.return_value = None
mock_get_db.return_value = _EmptyConnection()
mock_get_managed_node_onboarding.return_value = {
"onboarding_stage": {"code": "ready", "label": "已接管"},
"summary": "当前节点无需额外恢复动作。",
"recovery_decision": {
"action": "noop",
"label": "当前无需额外恢复动作",
"summary": "当前节点暂无需要执行的接管恢复动作。",
"command_hint": "",
"window": "none",
},
}
mock_get_cluster_snapshot.return_value = {
"nodes": [
{
"node_code": "mainland-worker-03",
"region": "mainland",
"role": "worker",
"status": "online",
"current_load": 0,
"is_effective_worker": True,
}
]
}
mock_list_managed_nodes.return_value = [
{
"node_code": "mainland-worker-03",
"region": "mainland",
"role": "worker",
"is_enabled": True,
"ssh_host": "121.204.244.250",
"ssh_user": "root",
"metadata": {
"delivery_queue": {
"state": "dead_letter",
"label": "死信 1",
"reason": "当前存在 1 条死信记录",
"pending_count": 0,
"dead_letter_count": 1,
"oldest_dead_letter_at": "2026-04-18 09:01:00",
}
},
"last_seen_at": "2026-04-18 09:05:00",
}
]
readiness = build_rollout_target_operational_readiness(
[{"node_code": "mainland-worker-03", "region": "mainland", "role": "worker"}],
execution_mode="remote-agent",
)
row = readiness["rows"][0]
self.assertEqual("dead_letter", row["delivery_queue_state"])
self.assertEqual(1, row["delivery_queue_dead_letter_count"])
self.assertTrue(
any("死信" in item for item in row["blocking_reasons"])
)
self.assertEqual(1, readiness["summary"]["queue_dead_letter_nodes"])
self.assertTrue(
any("死信回执" in item for item in readiness["blocking_reasons"])
)
self.assertTrue(
any("死信队列" in item for item in readiness["recommendations"])
)
@patch("app.services.ops_release_service.build_rollout_target_operational_readiness")
def test_build_release_rollout_gate_uses_execution_ready_count_for_ssh(self, mock_build_readiness) -> None:
mock_build_readiness.return_value = {
"execution_mode": "ssh",
"summary": {
"nodes_total": 2,
"execution_ready_nodes": 1,
"inspection_healthy_nodes": 2,
},
"rows": [],
"blocking_reasons": ["有 1 台目标节点尚未配置完整 SSH 入口，当前不适合直接 SSH 发布。"],
"warning_reasons": [],
"recommendations": [],
}
gate = build_release_rollout_gate(
{
"id": 11,
"release_version": "2026.04.18-rc1",
"status": "ready",
"artifact_url": "https://example.com/domaincheck.tar.gz",
},
[{"node_code": "mainland-worker-01"}, {"node_code": "mainland-worker-02"}],
execution_mode="ssh",
)
self.assertEqual("ssh", gate["execution_mode"])
self.assertEqual("SSH", gate["execution_mode_label"])
self.assertEqual("blocked", gate["status"])
self.assertIn("1/2 台 SSH 就绪", gate["summary"])
@patch("app.services.ops_release_service.build_rollout_target_operational_readiness")
def test_build_release_rollout_gate_uses_human_label_for_remote_agent(self, mock_build_readiness) -> None:
mock_build_readiness.return_value = {
"execution_mode": "remote-agent",
"summary": {
"nodes_total": 2,
"execution_ready_nodes": 1,
"inspection_healthy_nodes": 2,
},
"rows": [],
"blocking_reasons": ["有 1 台目标节点 Agent 未在线，当前不适合直接 remote-agent 发布。"],
"warning_reasons": [],
"recommendations": [],
}
gate = build_release_rollout_gate(
{
"id": 12,
"release_version": "2026.04.18-rc1",
"status": "ready",
"artifact_url": "https://example.com/domaincheck.tar.gz",
},
[{"node_code": "mainland-worker-01"}, {"node_code": "mainland-worker-02"}],
execution_mode="remote-agent",
)
self.assertEqual("remote-agent", gate["execution_mode"])
self.assertEqual("远端 Agent", gate["execution_mode_label"])
self.assertIn("1/2 台 远端 Agent 就绪", gate["summary"])
@patch("app.services.ops_release_service.build_release_rollout_gate")
def test_build_release_execution_mode_recommendation_prefers_ssh_when_gate_is_better(self, mock_build_gate) -> None:
def _fake_gate(release, target_nodes, *, execution_mode="remote-agent"):
if execution_mode == "ssh":
return {
"status": "ready",
"status_label": "可发 Rollout",
"summary": "SSH 路径已收口",
"execution_mode": "ssh",
"execution_mode_label": "SSH",
"blocking_reasons": [],
"warning_reasons": [],
"operational_readiness": {
"summary": {"nodes_total": 2, "execution_ready_nodes": 2},
"rows": [],
},
}
return {
"status": "blocked",
"status_label": "存在阻断",
"summary": "remote-agent 尚未收口",
"execution_mode": "remote-agent",
"execution_mode_label": "远端 Agent",
"blocking_reasons": ["Agent 未就绪"],
"warning_reasons": [],
"operational_readiness": {
"summary": {"nodes_total": 2, "execution_ready_nodes": 1},
"rows": [],
},
}
mock_build_gate.side_effect = _fake_gate
recommendation = build_release_execution_mode_recommendation(
{
"id": 18,
"release_version": "2026.04.18-rc2",
"status": "ready",
"artifact_url": "https://example.com/domaincheck.tar.gz",
},
[{"node_code": "mainland-worker-01"}, {"node_code": "mainland-worker-02"}],
)
self.assertEqual("ssh", recommendation["recommended_mode"])
self.assertEqual("SSH", recommendation["recommended_mode_label"])
self.assertEqual("ready", recommendation["recommended_gate"]["status"])
self.assertIn("SSH", recommendation["reason"])
if __name__ == "__main__":
unittest.main()