feat: add ops center and node onboarding flow
This commit is contained in:
98
domain-api/tests/test_build_info_service.py
Normal file
98
domain-api/tests/test_build_info_service.py
Normal file
@@ -0,0 +1,98 @@
|
||||
import json
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
from app.core.config import settings
|
||||
from app.services.build_info_service import (
|
||||
_expected_route_paths,
|
||||
get_runtime_build_info,
|
||||
remember_registered_route_paths,
|
||||
)
|
||||
|
||||
|
||||
class BuildInfoServiceTests(unittest.TestCase):
|
||||
def tearDown(self) -> None:
|
||||
remember_registered_route_paths([])
|
||||
|
||||
def test_runtime_build_info_prefers_configured_manifest_and_reports_registered_surface(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
manifest_path = Path(temp_dir) / "release_manifest.json"
|
||||
manifest_path.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"package_name": "domaincheck_release_demo",
|
||||
"generated_at": "2026-04-18T12:00:00",
|
||||
"commit_sha": "bd6bcb2",
|
||||
"commit_ref": "main",
|
||||
"sha256": "demo-sha256",
|
||||
},
|
||||
ensure_ascii=False,
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
route_paths = list(_expected_route_paths().values())
|
||||
remember_registered_route_paths(route_paths)
|
||||
|
||||
with patch.multiple(
|
||||
settings,
|
||||
build_manifest_path=str(manifest_path),
|
||||
build_commit_sha="",
|
||||
build_commit_ref="",
|
||||
build_generated_at="",
|
||||
build_package_name="",
|
||||
build_checksum="",
|
||||
build_source_label="",
|
||||
):
|
||||
payload = get_runtime_build_info()
|
||||
|
||||
self.assertEqual("configured_manifest", payload["source"])
|
||||
self.assertEqual("domaincheck_release_demo", payload["package_name"])
|
||||
self.assertEqual("bd6bcb2", payload["commit_sha"])
|
||||
self.assertEqual("main", payload["commit_ref"])
|
||||
self.assertEqual("demo-sha256", payload["checksum"])
|
||||
self.assertEqual(str(manifest_path), payload["manifest_path"])
|
||||
self.assertTrue(payload["route_surface"]["surface_complete"])
|
||||
self.assertEqual([], payload["route_surface"]["missing_keys"])
|
||||
self.assertEqual("registered", payload["route_surface"]["mode"])
|
||||
self.assertIn("repository_capabilities", payload)
|
||||
self.assertIn("supports_install_command_block", payload["repository_capabilities"])
|
||||
self.assertIn("supports_multi_layout_bootstrap", payload["repository_capabilities"])
|
||||
|
||||
def test_runtime_build_info_env_override_wins_and_missing_routes_are_visible(self) -> None:
|
||||
with patch.multiple(
|
||||
settings,
|
||||
build_manifest_path="",
|
||||
build_commit_sha="246838ae4c07",
|
||||
build_commit_ref="main",
|
||||
build_generated_at="2026-04-18T12:10:00",
|
||||
build_package_name="domaincheck_release_env",
|
||||
build_checksum="env-sha256",
|
||||
build_source_label="env-override",
|
||||
):
|
||||
payload = get_runtime_build_info(route_paths=[_expected_route_paths()["ops_contracts"]])
|
||||
|
||||
self.assertEqual("env-override", payload["source"])
|
||||
self.assertEqual("domaincheck_release_env", payload["package_name"])
|
||||
self.assertEqual("246838ae4c07", payload["commit_sha"])
|
||||
self.assertEqual("main", payload["commit_ref"])
|
||||
self.assertEqual("env-sha256", payload["checksum"])
|
||||
self.assertFalse(payload["route_surface"]["surface_complete"])
|
||||
self.assertIn("ops_stack_diagnosis", payload["route_surface"]["missing_keys"])
|
||||
self.assertIn("ops_node_onboarding_bootstrap_preview", payload["route_surface"]["missing_keys"])
|
||||
self.assertIn("ops_node_onboarding_bootstrap_execute", payload["route_surface"]["missing_keys"])
|
||||
self.assertIn("ops_node_onboarding_acceptance_preview", payload["route_surface"]["missing_keys"])
|
||||
self.assertIn("ops_node_onboarding_acceptance_execute", payload["route_surface"]["missing_keys"])
|
||||
self.assertIn("ops_node_onboarding_recovery_preview", payload["route_surface"]["missing_keys"])
|
||||
self.assertIn("ops_node_onboarding_recovery_execute", payload["route_surface"]["missing_keys"])
|
||||
self.assertIn("ops_node_scene_log", payload["route_surface"]["missing_keys"])
|
||||
self.assertIn("runtime_build_info", payload["route_surface"]["missing_keys"])
|
||||
self.assertEqual("registered", payload["route_surface"]["mode"])
|
||||
self.assertIn("repository_capabilities", payload)
|
||||
self.assertIn("supports_install_command_block", payload["repository_capabilities"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
111
domain-api/tests/test_drive_ops_center_docs.py
Normal file
111
domain-api/tests/test_drive_ops_center_docs.py
Normal file
@@ -0,0 +1,111 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import os
|
||||
import re
|
||||
import subprocess
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
class DriveOpsCenterDocsTests(unittest.TestCase):
|
||||
def test_authoritative_docs_only_reference_supported_commands(self) -> None:
|
||||
repo_root = Path(__file__).resolve().parents[2]
|
||||
script_path = repo_root / "domain-api" / "deploy" / "multi-region" / "drive_ops_center.sh"
|
||||
doc_paths = [
|
||||
repo_root / "domain-api" / "deploy" / "multi-region" / "README.md",
|
||||
repo_root / "docs" / "25_domainCheck_海外单脑控制面上线收口总表.md",
|
||||
repo_root / "docs" / "26_domainCheck_发布前运行验证与交付模板.md",
|
||||
]
|
||||
|
||||
help_output = subprocess.run(
|
||||
["bash", str(script_path), "help"],
|
||||
cwd=repo_root,
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
).stdout
|
||||
|
||||
supported_commands = self._extract_supported_commands(help_output)
|
||||
documented_commands = self._extract_documented_commands(doc_paths)
|
||||
|
||||
missing_commands = sorted(documented_commands - supported_commands)
|
||||
self.assertEqual(
|
||||
[],
|
||||
missing_commands,
|
||||
msg=(
|
||||
"These drive_ops_center.sh commands are referenced by authoritative docs "
|
||||
"but missing from the CLI help output: "
|
||||
+ ", ".join(missing_commands)
|
||||
),
|
||||
)
|
||||
self.assertIn("stack-diagnosis", documented_commands)
|
||||
self.assertIn("stack-diagnosis", supported_commands)
|
||||
|
||||
def test_stack_diagnosis_summary_argument_normalization_does_not_treat_summary_as_host(self) -> None:
|
||||
repo_root = Path(__file__).resolve().parents[2]
|
||||
script_path = repo_root / "domain-api" / "deploy" / "multi-region" / "drive_ops_center.sh"
|
||||
env = dict(os.environ)
|
||||
env["DOMAINCHECK_OPS_CENTER_ENV"] = "/tmp/domaincheck-ops-center-does-not-exist"
|
||||
env["OPS_MAINLAND_API_BASE_URL"] = "http://127.0.0.1:9"
|
||||
|
||||
result = subprocess.run(
|
||||
["bash", str(script_path), "stack-diagnosis", "summary"],
|
||||
cwd=repo_root,
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
env=env,
|
||||
)
|
||||
|
||||
combined_output = f"{result.stdout}\n{result.stderr}"
|
||||
self.assertNotIn("Could not resolve host: summary", combined_output)
|
||||
self.assertNotIn("NameError", combined_output)
|
||||
self.assertIn("[9/9] condensed stack summary", combined_output)
|
||||
|
||||
def test_env_drift_recommends_stack_diagnosis_instead_of_stack_check(self) -> None:
|
||||
repo_root = Path(__file__).resolve().parents[2]
|
||||
script_path = repo_root / "domain-api" / "deploy" / "multi-region" / "check_env_drift.sh"
|
||||
script_text = script_path.read_text(encoding="utf-8")
|
||||
|
||||
self.assertIn(
|
||||
'recommended_actions.append("bash domain-api/deploy/multi-region/drive_ops_center.sh stack-diagnosis http://127.0.0.1:8100 summary")',
|
||||
script_text,
|
||||
)
|
||||
self.assertNotIn(
|
||||
'recommended_actions.append("bash domain-api/deploy/multi-region/drive_ops_center.sh stack-check http://127.0.0.1:8100 summary")',
|
||||
script_text,
|
||||
)
|
||||
|
||||
@staticmethod
|
||||
def _extract_supported_commands(help_output: str) -> set[str]:
|
||||
commands: set[str] = set()
|
||||
in_commands_section = False
|
||||
for raw_line in help_output.splitlines():
|
||||
line = raw_line.rstrip()
|
||||
if line.strip() == "commands:":
|
||||
in_commands_section = True
|
||||
continue
|
||||
if line.strip() == "examples:":
|
||||
break
|
||||
if not in_commands_section:
|
||||
continue
|
||||
stripped = line.strip()
|
||||
if not stripped:
|
||||
continue
|
||||
command = stripped.split()[0]
|
||||
if re.fullmatch(r"[a-z0-9-]+", command):
|
||||
commands.add(command)
|
||||
return commands
|
||||
|
||||
@staticmethod
|
||||
def _extract_documented_commands(doc_paths: list[Path]) -> set[str]:
|
||||
pattern = re.compile(r"drive_ops_center\.sh\s+([a-z0-9-]+)")
|
||||
commands: set[str] = set()
|
||||
for doc_path in doc_paths:
|
||||
text = doc_path.read_text(encoding="utf-8")
|
||||
commands.update(match.group(1) for match in pattern.finditer(text))
|
||||
return commands
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
284
domain-api/tests/test_node_agent_delivery_queue.py
Normal file
284
domain-api/tests/test_node_agent_delivery_queue.py
Normal file
@@ -0,0 +1,284 @@
|
||||
import json
|
||||
import os
|
||||
import tempfile
|
||||
import unittest
|
||||
import urllib.error
|
||||
from unittest.mock import patch
|
||||
|
||||
from app import node_agent
|
||||
|
||||
|
||||
class NodeAgentDeliveryQueueTests(unittest.TestCase):
|
||||
def test_base_payload_includes_delivery_queue_snapshot(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
with patch.object(node_agent, "AGENT_QUEUE_DIR", temp_dir):
|
||||
node_agent._ensure_queue_dirs()
|
||||
payload = node_agent._base_payload()
|
||||
|
||||
delivery_queue = payload["metadata"]["delivery_queue"]
|
||||
self.assertEqual("healthy", delivery_queue["state"])
|
||||
self.assertEqual(0, delivery_queue["pending_count"])
|
||||
self.assertEqual(0, delivery_queue["dead_letter_count"])
|
||||
|
||||
def test_job_event_network_failure_is_queued_for_retry(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
with patch.object(node_agent, "AGENT_QUEUE_DIR", temp_dir), patch.object(
|
||||
node_agent,
|
||||
"_post",
|
||||
side_effect=urllib.error.URLError("offline"),
|
||||
):
|
||||
result = node_agent._job_event(
|
||||
7,
|
||||
event_type="executor_received",
|
||||
message="accepted",
|
||||
client_event_id="evt-001",
|
||||
summary_text="已接单",
|
||||
focus_ref={"kind": "ops_job", "job_id": 7},
|
||||
)
|
||||
|
||||
self.assertEqual("queued", result["state"])
|
||||
pending_dir = os.path.join(temp_dir, "pending")
|
||||
pending_files = sorted(os.listdir(pending_dir))
|
||||
self.assertEqual(1, len(pending_files))
|
||||
with open(os.path.join(pending_dir, pending_files[0]), "r", encoding="utf-8") as handle:
|
||||
record = json.load(handle)
|
||||
self.assertEqual("evt-001", record["payload"]["client_event_id"])
|
||||
self.assertEqual("已接单", record["payload"]["summary_text"])
|
||||
self.assertEqual("ops_job", record["payload"]["focus_ref"]["kind"])
|
||||
self.assertEqual("job_event", record["kind"])
|
||||
|
||||
def test_job_complete_transport_failure_preserves_client_request_id(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
with patch.object(node_agent, "AGENT_QUEUE_DIR", temp_dir), patch.object(
|
||||
node_agent,
|
||||
"_post",
|
||||
side_effect=urllib.error.URLError("offline"),
|
||||
):
|
||||
result = node_agent._job_complete(
|
||||
9,
|
||||
status="success",
|
||||
stdout="done",
|
||||
stderr="",
|
||||
result={"ok": True},
|
||||
client_request_id="complete-001",
|
||||
duration_ms=321,
|
||||
summary_text="执行成功",
|
||||
focus_ref={"kind": "ops_job", "job_id": 9},
|
||||
)
|
||||
|
||||
self.assertEqual("queued", result["state"])
|
||||
pending_dir = os.path.join(temp_dir, "pending")
|
||||
pending_files = sorted(os.listdir(pending_dir))
|
||||
self.assertEqual(1, len(pending_files))
|
||||
with open(os.path.join(pending_dir, pending_files[0]), "r", encoding="utf-8") as handle:
|
||||
record = json.load(handle)
|
||||
self.assertEqual("complete-001", record["payload"]["client_request_id"])
|
||||
self.assertEqual(321, record["payload"]["duration_ms"])
|
||||
self.assertEqual("执行成功", record["payload"]["summary_text"])
|
||||
self.assertEqual("ops_job", record["payload"]["focus_ref"]["kind"])
|
||||
self.assertEqual("job_complete", record["kind"])
|
||||
|
||||
def test_flush_delivery_queue_delivers_pending_records(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
with patch.object(node_agent, "AGENT_QUEUE_DIR", temp_dir):
|
||||
record = node_agent._build_delivery_record(
|
||||
"job_event",
|
||||
"/api/v1/ops/agent/jobs/7/events",
|
||||
{
|
||||
"node_code": "mainland-worker-01",
|
||||
"event_type": "executor_received",
|
||||
"message": "accepted",
|
||||
"client_event_id": "evt-002",
|
||||
},
|
||||
"evt-002",
|
||||
)
|
||||
node_agent._store_pending_delivery(record)
|
||||
with patch.object(node_agent, "_post", return_value={"code": 0, "message": "ok"}):
|
||||
summary = node_agent._flush_delivery_queue(limit=10)
|
||||
|
||||
self.assertEqual(1, summary["delivered"])
|
||||
self.assertEqual([], os.listdir(os.path.join(temp_dir, "pending")))
|
||||
|
||||
def test_flush_delivery_queue_moves_semantic_failure_to_dead_letter(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
with patch.object(node_agent, "AGENT_QUEUE_DIR", temp_dir):
|
||||
record = node_agent._build_delivery_record(
|
||||
"job_complete",
|
||||
"/api/v1/ops/agent/jobs/9/complete",
|
||||
{
|
||||
"node_code": "mainland-worker-01",
|
||||
"status": "success",
|
||||
"client_request_id": "complete-002",
|
||||
},
|
||||
"complete-002",
|
||||
)
|
||||
node_agent._store_pending_delivery(record)
|
||||
with patch.object(
|
||||
node_agent,
|
||||
"_post",
|
||||
return_value={
|
||||
"code": 1,
|
||||
"message": "任务不存在或不属于当前节点",
|
||||
"detail_code": "ops_job_not_owned_by_agent",
|
||||
},
|
||||
):
|
||||
summary = node_agent._flush_delivery_queue(limit=10)
|
||||
|
||||
self.assertEqual(1, summary["dead_letter"])
|
||||
self.assertEqual([], os.listdir(os.path.join(temp_dir, "pending")))
|
||||
self.assertEqual(1, len(os.listdir(os.path.join(temp_dir, "dead-letter"))))
|
||||
|
||||
def test_replay_dead_letter_records_moves_record_back_to_pending_and_flushes(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
with patch.object(node_agent, "AGENT_QUEUE_DIR", temp_dir):
|
||||
record = node_agent._build_delivery_record(
|
||||
"job_event",
|
||||
"/api/v1/ops/agent/jobs/7/events",
|
||||
{
|
||||
"node_code": "mainland-worker-01",
|
||||
"event_type": "executor_received",
|
||||
"message": "accepted",
|
||||
"client_event_id": "evt-003",
|
||||
},
|
||||
"evt-003",
|
||||
)
|
||||
node_agent._store_dead_letter_delivery(
|
||||
record,
|
||||
reason="temporary failure",
|
||||
detail_code="transient_error",
|
||||
)
|
||||
with patch.object(node_agent, "_post", return_value={"code": 0, "message": "ok"}):
|
||||
ok, message, data = node_agent._execute_action(
|
||||
"delivery.queue.replay",
|
||||
{
|
||||
"selector": {"record_id": "evt-003"},
|
||||
"limit": 10,
|
||||
"flush_after_replay": True,
|
||||
"reason": "恢复后重放",
|
||||
},
|
||||
)
|
||||
|
||||
self.assertTrue(ok)
|
||||
self.assertEqual("已重放 1 条死信记录", message)
|
||||
self.assertEqual(1, data["replayed_total"])
|
||||
self.assertEqual(1, data["flush_summary"]["delivered"])
|
||||
self.assertEqual([], os.listdir(os.path.join(temp_dir, "dead-letter")))
|
||||
self.assertEqual([], os.listdir(os.path.join(temp_dir, "pending")))
|
||||
|
||||
def test_discard_dead_letter_records_archives_record(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
with patch.object(node_agent, "AGENT_QUEUE_DIR", temp_dir):
|
||||
record = node_agent._build_delivery_record(
|
||||
"job_complete",
|
||||
"/api/v1/ops/agent/jobs/8/complete",
|
||||
{
|
||||
"node_code": "mainland-worker-01",
|
||||
"status": "failed",
|
||||
"client_request_id": "complete-003",
|
||||
},
|
||||
"complete-003",
|
||||
)
|
||||
node_agent._store_dead_letter_delivery(
|
||||
record,
|
||||
reason="permanent failure",
|
||||
detail_code="ops_job_invalid_status",
|
||||
)
|
||||
ok, message, data = node_agent._execute_action(
|
||||
"delivery.queue.discard",
|
||||
{
|
||||
"selector": {"record_id": "complete-003"},
|
||||
"limit": 10,
|
||||
"discarded_by": "web-ui",
|
||||
"reason": "确认不需要再补发",
|
||||
},
|
||||
)
|
||||
|
||||
self.assertTrue(ok)
|
||||
self.assertEqual("已丢弃 1 条死信记录", message)
|
||||
self.assertEqual(1, data["discarded_total"])
|
||||
self.assertEqual([], os.listdir(os.path.join(temp_dir, "dead-letter")))
|
||||
discarded_files = os.listdir(os.path.join(temp_dir, "discarded"))
|
||||
self.assertEqual(1, len(discarded_files))
|
||||
with open(os.path.join(temp_dir, "discarded", discarded_files[0]), "r", encoding="utf-8") as handle:
|
||||
archived = json.load(handle)
|
||||
self.assertEqual("web-ui", archived["discarded_by"])
|
||||
self.assertEqual("确认不需要再补发", archived["discard_reason"])
|
||||
|
||||
def test_process_job_passes_envelope_context_to_completion(self) -> None:
|
||||
job = {
|
||||
"id": 12,
|
||||
"job_code": "ops-20260418-000012",
|
||||
"job_type": "release_deploy",
|
||||
"action": "logs.collect",
|
||||
"payload": {"service_name": "domaincheck-worker", "lines": 120},
|
||||
"step_key": "worker_logs",
|
||||
"step_title": "收集 Worker 日志",
|
||||
"focus_ref": {"kind": "ops_job", "job_id": 12, "job_code": "ops-20260418-000012"},
|
||||
"release_context": {"release_version": "v2026.04.18", "rollout_id": 3},
|
||||
"step_ref": {"step_id": 22, "step_key": "worker_logs", "step_title": "收集 Worker 日志"},
|
||||
}
|
||||
with patch.object(node_agent, "_job_start") as mock_job_start, patch.object(
|
||||
node_agent,
|
||||
"_job_event",
|
||||
) as mock_job_event, patch.object(
|
||||
node_agent,
|
||||
"_execute_action",
|
||||
return_value=(True, "logs collected", {"summary": "收集完成", "stdout": "ok", "stderr": ""}),
|
||||
) as mock_execute_action, patch.object(
|
||||
node_agent,
|
||||
"_job_complete",
|
||||
return_value={"state": "sent"},
|
||||
) as mock_job_complete:
|
||||
node_agent._process_job(job)
|
||||
|
||||
self.assertEqual("worker_logs", mock_job_start.call_args.kwargs["job"]["step_key"])
|
||||
self.assertEqual("ops_job", mock_job_event.call_args.kwargs["focus_ref"]["kind"])
|
||||
self.assertEqual("已接单 logs.collect", mock_job_event.call_args.kwargs["summary_text"])
|
||||
self.assertEqual("logs.collect", mock_execute_action.call_args.args[0])
|
||||
self.assertEqual("v2026.04.18", mock_job_complete.call_args.kwargs["release_context"]["release_version"])
|
||||
self.assertEqual("ops_job", mock_job_complete.call_args.kwargs["focus_ref"]["kind"])
|
||||
self.assertEqual("收集完成", mock_job_complete.call_args.kwargs["summary_text"])
|
||||
self.assertGreaterEqual(mock_job_complete.call_args.kwargs["duration_ms"], 0)
|
||||
|
||||
def test_process_job_continues_when_job_start_delivery_fails(self) -> None:
|
||||
job = {
|
||||
"id": 18,
|
||||
"job_code": "ops-20260418-000018",
|
||||
"job_type": "ops_action",
|
||||
"action": "logs.collect",
|
||||
"payload": {"service_name": "domaincheck-worker", "lines": 50},
|
||||
"step_key": "worker_logs",
|
||||
"step_title": "收集 Worker 日志",
|
||||
"focus_ref": {"kind": "ops_job", "job_id": 18, "job_code": "ops-20260418-000018"},
|
||||
"release_context": {},
|
||||
"step_ref": {"step_id": 28, "step_key": "worker_logs", "step_title": "收集 Worker 日志"},
|
||||
}
|
||||
with patch.object(
|
||||
node_agent,
|
||||
"_job_start",
|
||||
side_effect=urllib.error.URLError("temporary offline"),
|
||||
) as mock_job_start, patch.object(
|
||||
node_agent,
|
||||
"_job_event",
|
||||
return_value={"state": "queued"},
|
||||
) as mock_job_event, patch.object(
|
||||
node_agent,
|
||||
"_execute_action",
|
||||
return_value=(True, "logs collected", {"summary": "收集完成", "stdout": "ok", "stderr": ""}),
|
||||
) as mock_execute_action, patch.object(
|
||||
node_agent,
|
||||
"_job_complete",
|
||||
return_value={"state": "sent"},
|
||||
) as mock_job_complete:
|
||||
node_agent._process_job(job)
|
||||
|
||||
mock_job_start.assert_called_once()
|
||||
mock_execute_action.assert_called_once()
|
||||
mock_job_complete.assert_called_once()
|
||||
event_payload = mock_job_event.call_args.kwargs["payload"]
|
||||
self.assertEqual("failed_local", event_payload["start_delivery_state"])
|
||||
self.assertIn("temporary offline", event_payload["start_delivery_error"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
41
domain-api/tests/test_node_agent_release_normalization.py
Normal file
41
domain-api/tests/test_node_agent_release_normalization.py
Normal file
@@ -0,0 +1,41 @@
|
||||
import unittest
|
||||
|
||||
from app import node_agent
|
||||
|
||||
|
||||
class NodeAgentReleaseNormalizationTests(unittest.TestCase):
|
||||
def test_normalize_text_list_supports_strings_and_lists(self) -> None:
|
||||
self.assertEqual(
|
||||
["domaincheck-api", "domaincheck-worker"],
|
||||
node_agent._normalize_text_list("domaincheck-api\ndomaincheck-worker"),
|
||||
)
|
||||
self.assertEqual(
|
||||
["domaincheck-api", "domaincheck-worker"],
|
||||
node_agent._normalize_text_list(["domaincheck-api", " domaincheck-worker "]),
|
||||
)
|
||||
self.assertEqual([], node_agent._normalize_text_list(""))
|
||||
|
||||
def test_health_check_services_fall_back_to_restart_services_when_missing(self) -> None:
|
||||
result = node_agent._normalize_release_health_check_services(
|
||||
{},
|
||||
["domaincheck-worker"],
|
||||
)
|
||||
self.assertEqual(["domaincheck-worker"], result)
|
||||
|
||||
def test_health_check_services_can_be_explicitly_empty(self) -> None:
|
||||
result = node_agent._normalize_release_health_check_services(
|
||||
{"health_check_services": ""},
|
||||
["domaincheck-worker"],
|
||||
)
|
||||
self.assertEqual([], result)
|
||||
|
||||
def test_health_check_services_accept_multiline_string(self) -> None:
|
||||
result = node_agent._normalize_release_health_check_services(
|
||||
{"health_check_services": "domaincheck-api\ndomaincheck-sync-agent"},
|
||||
["domaincheck-worker"],
|
||||
)
|
||||
self.assertEqual(["domaincheck-api", "domaincheck-sync-agent"], result)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
115
domain-api/tests/test_ops_action_executor_core.py
Normal file
115
domain-api/tests/test_ops_action_executor_core.py
Normal file
@@ -0,0 +1,115 @@
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
from app.services.ops_action_executor_core import build_service_name_map, execute_structured_action
|
||||
|
||||
|
||||
class _FakeRunner:
|
||||
def __init__(self, responses: dict[tuple[str, ...], tuple[int, str, str]]) -> None:
|
||||
self.responses = responses
|
||||
self.calls: list[tuple[str, ...]] = []
|
||||
|
||||
def __call__(self, command: list[str], *, timeout: int = 60) -> tuple[int, str, str]:
|
||||
key = tuple(command)
|
||||
self.calls.append(key)
|
||||
return self.responses.get(key, (0, "", ""))
|
||||
|
||||
|
||||
class OpsActionExecutorCoreTests(unittest.TestCase):
|
||||
def test_health_snapshot_includes_host_context_and_ok_flags(self) -> None:
|
||||
runner = _FakeRunner(
|
||||
{
|
||||
("systemctl", "is-active", "domaincheck-api"): (0, "active", ""),
|
||||
("systemctl", "is-active", "domaincheck-worker"): (3, "inactive", ""),
|
||||
("systemctl", "is-active", "domaincheck-sync-agent"): (0, "active", ""),
|
||||
("systemctl", "is-active", "domaincheck-node-agent"): (0, "active", ""),
|
||||
}
|
||||
)
|
||||
|
||||
ok, message, data = execute_structured_action(
|
||||
"health.snapshot",
|
||||
{},
|
||||
service_names=build_service_name_map(
|
||||
api_service_name="domaincheck-api",
|
||||
worker_service_name="domaincheck-worker",
|
||||
sync_agent_service_name="domaincheck-sync-agent",
|
||||
),
|
||||
runner=runner,
|
||||
host_context={"hostname": "mainland-controller-01", "ip": "121.204.244.188"},
|
||||
)
|
||||
|
||||
self.assertTrue(ok)
|
||||
self.assertEqual("health snapshot collected", message)
|
||||
self.assertEqual("mainland-controller-01", data["hostname"])
|
||||
self.assertEqual("121.204.244.188", data["ip"])
|
||||
self.assertTrue(data["checks"]["api"]["ok"])
|
||||
self.assertFalse(data["checks"]["worker"]["ok"])
|
||||
self.assertEqual(4, len(runner.calls))
|
||||
|
||||
def test_logs_collect_defaults_to_worker_service_and_clamps_lines(self) -> None:
|
||||
runner = _FakeRunner(
|
||||
{
|
||||
("journalctl", "-u", "domaincheck-worker", "-n", "500", "--no-pager"): (
|
||||
0,
|
||||
"line-1\nline-2",
|
||||
"",
|
||||
)
|
||||
}
|
||||
)
|
||||
|
||||
ok, message, data = execute_structured_action(
|
||||
"logs.collect",
|
||||
{"lines": 9999},
|
||||
service_names=build_service_name_map(
|
||||
api_service_name="domaincheck-api",
|
||||
worker_service_name="domaincheck-worker",
|
||||
sync_agent_service_name="domaincheck-sync-agent",
|
||||
),
|
||||
runner=runner,
|
||||
)
|
||||
|
||||
self.assertTrue(ok)
|
||||
self.assertEqual("line-1\nline-2", message)
|
||||
self.assertEqual("domaincheck-worker", data["service_name"])
|
||||
self.assertEqual(500, data["lines"])
|
||||
self.assertEqual(
|
||||
("journalctl", "-u", "domaincheck-worker", "-n", "500", "--no-pager"),
|
||||
runner.calls[0],
|
||||
)
|
||||
|
||||
def test_logs_collect_falls_back_to_worker_log_file_when_journal_is_denied(self) -> None:
|
||||
runner = _FakeRunner(
|
||||
{
|
||||
("journalctl", "-u", "domaincheck-worker", "-n", "120", "--no-pager"): (
|
||||
1,
|
||||
"",
|
||||
"No journal files were opened due to insufficient permissions.",
|
||||
)
|
||||
}
|
||||
)
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
log_path = Path(temp_dir) / "detect_worker.log"
|
||||
log_path.write_text("worker-a\nworker-b\n", encoding="utf-8")
|
||||
with patch("app.services.ops_action_executor_core.settings.domain_root", temp_dir):
|
||||
ok, message, data = execute_structured_action(
|
||||
"logs.collect",
|
||||
{"lines": 120},
|
||||
service_names=build_service_name_map(
|
||||
api_service_name="domaincheck-api",
|
||||
worker_service_name="domaincheck-worker",
|
||||
sync_agent_service_name="domaincheck-sync-agent",
|
||||
),
|
||||
runner=runner,
|
||||
)
|
||||
|
||||
self.assertTrue(ok)
|
||||
self.assertEqual("worker-a\nworker-b", message)
|
||||
self.assertEqual("file_fallback", data["collection_source"])
|
||||
self.assertTrue(data["fallback_used"])
|
||||
self.assertIn("detect_worker.log", data["log_paths"][0])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
1181
domain-api/tests/test_ops_agent_service.py
Normal file
1181
domain-api/tests/test_ops_agent_service.py
Normal file
File diff suppressed because it is too large
Load Diff
96
domain-api/tests/test_ops_api_routes.py
Normal file
96
domain-api/tests/test_ops_api_routes.py
Normal file
@@ -0,0 +1,96 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
from app.api.routes.ops import (
|
||||
ops_doctor_decision,
|
||||
ops_go_live_bundle,
|
||||
ops_go_live_review,
|
||||
ops_go_live_signoff,
|
||||
ops_go_live_summary,
|
||||
)
|
||||
|
||||
|
||||
class OpsApiRoutesTestCase(unittest.TestCase):
|
||||
@patch("app.api.routes.ops.get_ops_go_live_signoff")
|
||||
def test_go_live_signoff_route_uses_service_payload(self, mock_get_ops_go_live_signoff) -> None:
|
||||
mock_get_ops_go_live_signoff.return_value = {
|
||||
"signoff_status": "ready",
|
||||
"headline": "ok",
|
||||
}
|
||||
|
||||
response = ops_go_live_signoff(base_url="http://127.0.0.1:8100", report_dir="/tmp/signoff")
|
||||
|
||||
self.assertEqual(0, response.code)
|
||||
self.assertEqual("ready", response.data["signoff_status"])
|
||||
self.assertEqual("ok", response.data["headline"])
|
||||
mock_get_ops_go_live_signoff.assert_called_once_with(
|
||||
base_url="http://127.0.0.1:8100",
|
||||
report_dir="/tmp/signoff",
|
||||
)
|
||||
|
||||
@patch("app.api.routes.ops.get_ops_go_live_summary")
|
||||
def test_go_live_summary_route_passes_base_url(self, mock_get_ops_go_live_summary) -> None:
|
||||
mock_get_ops_go_live_summary.return_value = {
|
||||
"go_live_status": "attention",
|
||||
}
|
||||
|
||||
response = ops_go_live_summary(base_url="http://example.test")
|
||||
|
||||
self.assertEqual(0, response.code)
|
||||
self.assertEqual("attention", response.data["go_live_status"])
|
||||
mock_get_ops_go_live_summary.assert_called_once_with(base_url="http://example.test")
|
||||
|
||||
@patch("app.api.routes.ops.get_ops_go_live_bundle")
|
||||
def test_go_live_bundle_route_passes_base_url_and_report_dir(self, mock_get_ops_go_live_bundle) -> None:
|
||||
mock_get_ops_go_live_bundle.return_value = {
|
||||
"status": "missing",
|
||||
}
|
||||
|
||||
response = ops_go_live_bundle(base_url="http://example.test", report_dir="/tmp/go-live")
|
||||
|
||||
self.assertEqual(0, response.code)
|
||||
self.assertEqual("missing", response.data["status"])
|
||||
mock_get_ops_go_live_bundle.assert_called_once_with(
|
||||
base_url="http://example.test",
|
||||
report_dir="/tmp/go-live",
|
||||
)
|
||||
|
||||
@patch("app.api.routes.ops.get_ops_go_live_review")
|
||||
def test_go_live_review_route_passes_base_url_and_report_dir(self, mock_get_ops_go_live_review) -> None:
|
||||
mock_get_ops_go_live_review.return_value = {
|
||||
"status": "attention",
|
||||
"headline": "review",
|
||||
}
|
||||
|
||||
response = ops_go_live_review(base_url="http://example.test", report_dir="/tmp/review")
|
||||
|
||||
self.assertEqual(0, response.code)
|
||||
self.assertEqual("attention", response.data["status"])
|
||||
self.assertEqual("review", response.data["headline"])
|
||||
mock_get_ops_go_live_review.assert_called_once_with(
|
||||
base_url="http://example.test",
|
||||
report_dir="/tmp/review",
|
||||
)
|
||||
|
||||
@patch("app.api.routes.ops.get_ops_doctor_decision")
|
||||
def test_doctor_decision_route_passes_base_url_and_report_dir(self, mock_get_ops_doctor_decision) -> None:
|
||||
mock_get_ops_doctor_decision.return_value = {
|
||||
"status": "attention",
|
||||
"headline": "doctor",
|
||||
}
|
||||
|
||||
response = ops_doctor_decision(base_url="http://example.test", report_dir="/tmp/doctor")
|
||||
|
||||
self.assertEqual(0, response.code)
|
||||
self.assertEqual("attention", response.data["status"])
|
||||
self.assertEqual("doctor", response.data["headline"])
|
||||
mock_get_ops_doctor_decision.assert_called_once_with(
|
||||
base_url="http://example.test",
|
||||
report_dir="/tmp/doctor",
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
44
domain-api/tests/test_ops_command_service.py
Normal file
44
domain-api/tests/test_ops_command_service.py
Normal file
@@ -0,0 +1,44 @@
|
||||
import unittest
|
||||
|
||||
from app.services.ops_command_service import build_bash_command, ops_script_path
|
||||
|
||||
|
||||
class OpsCommandServiceTests(unittest.TestCase):
|
||||
def test_ops_script_path_prefixes_multi_region_directory(self) -> None:
|
||||
self.assertEqual(
|
||||
"domain-api/deploy/multi-region/drive_ops_center.sh",
|
||||
ops_script_path("drive_ops_center.sh"),
|
||||
)
|
||||
|
||||
def test_ops_script_path_normalizes_leading_slash(self) -> None:
|
||||
self.assertEqual(
|
||||
"domain-api/deploy/multi-region/check_ops_center_stack.sh",
|
||||
ops_script_path("/check_ops_center_stack.sh"),
|
||||
)
|
||||
|
||||
def test_build_bash_command_joins_non_empty_args(self) -> None:
|
||||
self.assertEqual(
|
||||
"bash domain-api/deploy/multi-region/drive_ops_center.sh driver-resolve http://127.0.0.1:8100 handover_first_gap",
|
||||
build_bash_command(
|
||||
"drive_ops_center.sh",
|
||||
"driver-resolve",
|
||||
"http://127.0.0.1:8100",
|
||||
"handover_first_gap",
|
||||
),
|
||||
)
|
||||
|
||||
def test_build_bash_command_skips_blank_args(self) -> None:
|
||||
self.assertEqual(
|
||||
"bash domain-api/deploy/multi-region/drive_release_hub.sh http://127.0.0.1:8100 launchpad",
|
||||
build_bash_command(
|
||||
"drive_release_hub.sh",
|
||||
"http://127.0.0.1:8100",
|
||||
"",
|
||||
"launchpad",
|
||||
" ",
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
391
domain-api/tests/test_ops_job_service.py
Normal file
391
domain-api/tests/test_ops_job_service.py
Normal file
@@ -0,0 +1,391 @@
|
||||
import json
|
||||
import unittest
|
||||
from datetime import datetime
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
from app.core.config import settings
|
||||
from app.services.ops_job_service import (
|
||||
_execute_control_plane_job,
|
||||
_serialize_job_row,
|
||||
create_ops_job,
|
||||
dispatch_ops_job,
|
||||
upsert_managed_node,
|
||||
)
|
||||
|
||||
|
||||
class OpsJobServiceTests(unittest.TestCase):
|
||||
def test_serialize_job_row_adds_execution_mode_label(self) -> None:
|
||||
row = (
|
||||
9,
|
||||
"ops-2026041801-abcd12",
|
||||
"logs.collect",
|
||||
"node",
|
||||
"mainland-worker-01",
|
||||
"queued",
|
||||
"remote-agent",
|
||||
"api",
|
||||
json.dumps({"service_name": "domaincheck-worker", "lines": 120}, ensure_ascii=False),
|
||||
json.dumps({}, ensure_ascii=False),
|
||||
json.dumps({}, ensure_ascii=False),
|
||||
"",
|
||||
datetime(2026, 4, 18, 10, 0, 0),
|
||||
None,
|
||||
None,
|
||||
datetime(2026, 4, 18, 10, 0, 0),
|
||||
"low",
|
||||
False,
|
||||
"approved",
|
||||
"",
|
||||
None,
|
||||
"",
|
||||
"",
|
||||
None,
|
||||
json.dumps({}, ensure_ascii=False),
|
||||
json.dumps({}, ensure_ascii=False),
|
||||
None,
|
||||
)
|
||||
|
||||
job = _serialize_job_row(row)
|
||||
|
||||
self.assertEqual("remote-agent", job["execution_mode"])
|
||||
self.assertEqual("远端 Agent", job["execution_mode_label"])
|
||||
self.assertEqual("排队中", job["status_label"])
|
||||
self.assertEqual("已审批", job["approval_status_label"])
|
||||
self.assertEqual(["mainland-worker-01"], job["target_node_codes"])
|
||||
self.assertEqual(0, job["steps_total"])
|
||||
self.assertFalse(job["steps_loaded"])
|
||||
self.assertFalse(job["is_compact"])
|
||||
self.assertEqual("ops_job", job["focus_ref"]["kind"])
|
||||
self.assertEqual(9, job["focus_ref"]["job_id"])
|
||||
self.assertIn("目标节点 mainland-worker-01", job["summary_text"])
|
||||
|
||||
def test_serialize_job_row_counts_loaded_steps(self) -> None:
|
||||
row = (
|
||||
10,
|
||||
"ops-2026041801-efgh34",
|
||||
"diagnostics.collect",
|
||||
"node",
|
||||
"mainland-worker-02",
|
||||
"running",
|
||||
"ssh",
|
||||
"web-ui",
|
||||
json.dumps({}, ensure_ascii=False),
|
||||
json.dumps({}, ensure_ascii=False),
|
||||
json.dumps({}, ensure_ascii=False),
|
||||
"",
|
||||
datetime(2026, 4, 18, 10, 5, 0),
|
||||
datetime(2026, 4, 18, 10, 5, 2),
|
||||
None,
|
||||
datetime(2026, 4, 18, 10, 5, 3),
|
||||
"medium",
|
||||
True,
|
||||
"approved",
|
||||
"admin",
|
||||
datetime(2026, 4, 18, 10, 5, 1),
|
||||
"",
|
||||
"",
|
||||
datetime(2026, 4, 18, 10, 5, 2),
|
||||
json.dumps({}, ensure_ascii=False),
|
||||
json.dumps({}, ensure_ascii=False),
|
||||
None,
|
||||
)
|
||||
|
||||
job = _serialize_job_row(
|
||||
row,
|
||||
steps=[
|
||||
{"id": 1, "status": "success"},
|
||||
{"id": 2, "status": "running"},
|
||||
{"id": 3, "status": "failed"},
|
||||
],
|
||||
)
|
||||
|
||||
self.assertEqual("执行中", job["status_label"])
|
||||
self.assertEqual("已审批", job["approval_status_label"])
|
||||
self.assertEqual(3, job["steps_total"])
|
||||
self.assertEqual(1, job["steps_running"])
|
||||
self.assertEqual(1, job["steps_success"])
|
||||
self.assertEqual(1, job["steps_failed"])
|
||||
self.assertEqual(2, job["steps_terminal"])
|
||||
self.assertTrue(job["steps_loaded"])
|
||||
self.assertIn("mainland-worker-02", job["summary_text"])
|
||||
|
||||
@patch("app.services.ops_job_service.list_managed_nodes")
|
||||
@patch("app.services.ops_agent_service.build_node_agent_bootstrap_plan")
|
||||
def test_execute_control_plane_bootstrap_uses_managed_node_defaults(
|
||||
self,
|
||||
mock_build_node_agent_bootstrap_plan,
|
||||
mock_list_managed_nodes,
|
||||
) -> None:
|
||||
mock_list_managed_nodes.return_value = [
|
||||
{
|
||||
"node_code": "mainland-worker-02",
|
||||
"region": "mainland",
|
||||
"role": "worker",
|
||||
}
|
||||
]
|
||||
mock_build_node_agent_bootstrap_plan.return_value = (
|
||||
True,
|
||||
"节点 Agent 接入方案已生成",
|
||||
{
|
||||
"token_preview": "agent-...1234",
|
||||
"bootstrap_plan": {
|
||||
"bootstrap_script_name": "bootstrap-node-agent-mainland-worker-02.sh",
|
||||
},
|
||||
},
|
||||
)
|
||||
|
||||
ok, message, data = _execute_control_plane_job(
|
||||
"node.bootstrap",
|
||||
target_node_code="mainland-worker-02",
|
||||
payload={
|
||||
"root_dir": "/opt/domaincheck",
|
||||
"expires_in_hours": 12,
|
||||
},
|
||||
requested_by="unit-test",
|
||||
metadata={"source": "test"},
|
||||
)
|
||||
|
||||
self.assertTrue(ok)
|
||||
self.assertEqual("节点 Agent 接入方案已生成", message)
|
||||
self.assertEqual("control-plane", data["execution_mode"])
|
||||
self.assertEqual("mainland", data["node_region"])
|
||||
self.assertEqual("worker", data["node_role"])
|
||||
mock_build_node_agent_bootstrap_plan.assert_called_once_with(
|
||||
node_code="mainland-worker-02",
|
||||
node_region="mainland",
|
||||
node_role="worker",
|
||||
issued_by="unit-test",
|
||||
expires_in_hours=12,
|
||||
control_plane_base_url="",
|
||||
root_dir="/opt/domaincheck",
|
||||
metadata={
|
||||
"source": "test",
|
||||
"issued_from": "ops-job/control-plane",
|
||||
"target_node_code": "mainland-worker-02",
|
||||
"node_region": "mainland",
|
||||
"node_role": "worker",
|
||||
},
|
||||
)
|
||||
|
||||
@patch("app.services.ops_job_service.get_db")
|
||||
def test_upsert_managed_node_preserves_remote_fields_when_sync_payload_omits_them(self, mock_get_db) -> None:
|
||||
existing_metadata = {"operator_note": "keep-me"}
|
||||
merged_metadata = {"operator_note": "keep-me", "cluster_status": "online"}
|
||||
existing_row = (
|
||||
"mainland-worker-01",
|
||||
"mainland",
|
||||
"worker",
|
||||
"Mainland Worker 01",
|
||||
"121.204.244.248",
|
||||
22,
|
||||
"root",
|
||||
"key",
|
||||
"stable",
|
||||
False,
|
||||
json.dumps(existing_metadata, ensure_ascii=False),
|
||||
datetime(2026, 4, 18, 1, 0, 0),
|
||||
None,
|
||||
datetime(2026, 4, 18, 1, 5, 0),
|
||||
)
|
||||
returned_row = (
|
||||
"mainland-worker-01",
|
||||
"mainland",
|
||||
"worker",
|
||||
"S244-248",
|
||||
"121.204.244.248",
|
||||
22,
|
||||
"root",
|
||||
"key",
|
||||
"stable",
|
||||
False,
|
||||
json.dumps(merged_metadata, ensure_ascii=False),
|
||||
datetime(2026, 4, 18, 1, 0, 0),
|
||||
None,
|
||||
datetime(2026, 4, 18, 1, 6, 0),
|
||||
)
|
||||
|
||||
conn = MagicMock()
|
||||
cursor = MagicMock()
|
||||
db_ctx = MagicMock()
|
||||
cursor_ctx = MagicMock()
|
||||
db_ctx.__enter__.return_value = conn
|
||||
db_ctx.__exit__.return_value = False
|
||||
cursor_ctx.__enter__.return_value = cursor
|
||||
cursor_ctx.__exit__.return_value = False
|
||||
conn.cursor.return_value = cursor_ctx
|
||||
mock_get_db.return_value = db_ctx
|
||||
cursor.fetchone.side_effect = [existing_row, returned_row]
|
||||
|
||||
ok, message, data = upsert_managed_node(
|
||||
{
|
||||
"node_code": "mainland-worker-01",
|
||||
"region": "mainland",
|
||||
"role": "worker",
|
||||
"title": "S244-248",
|
||||
"metadata": {
|
||||
"cluster_status": "online",
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
self.assertTrue(ok)
|
||||
self.assertEqual("托管节点已保存", message)
|
||||
node = data["node"]
|
||||
self.assertEqual("121.204.244.248", node["ssh_host"])
|
||||
self.assertEqual("root", node["ssh_user"])
|
||||
self.assertFalse(node["is_enabled"])
|
||||
self.assertEqual("keep-me", node["metadata"]["operator_note"])
|
||||
self.assertEqual("online", node["metadata"]["cluster_status"])
|
||||
|
||||
insert_call = cursor.execute.call_args_list[-1]
|
||||
insert_params = insert_call.args[1]
|
||||
self.assertEqual("121.204.244.248", insert_params[4])
|
||||
self.assertEqual(22, insert_params[5])
|
||||
self.assertEqual("root", insert_params[6])
|
||||
self.assertFalse(insert_params[9])
|
||||
self.assertEqual(merged_metadata, json.loads(insert_params[10]))
|
||||
|
||||
@patch("app.services.ops_agent_service.append_ops_job_event")
|
||||
@patch("app.services.ops_job_service._execute_ssh_job_record")
|
||||
@patch("app.services.ops_job_service.get_db")
|
||||
def test_dispatch_ops_job_ssh_executes_via_ssh_record(
|
||||
self,
|
||||
mock_get_db,
|
||||
mock_execute_ssh_job_record,
|
||||
mock_append_ops_job_event,
|
||||
) -> None:
|
||||
conn = MagicMock()
|
||||
cursor = MagicMock()
|
||||
db_ctx = MagicMock()
|
||||
cursor_ctx = MagicMock()
|
||||
db_ctx.__enter__.return_value = conn
|
||||
db_ctx.__exit__.return_value = False
|
||||
cursor_ctx.__enter__.return_value = cursor
|
||||
cursor_ctx.__exit__.return_value = False
|
||||
conn.cursor.return_value = cursor_ctx
|
||||
mock_get_db.return_value = db_ctx
|
||||
cursor.fetchone.return_value = (
|
||||
9,
|
||||
"ops-2026041801-abcd12",
|
||||
"logs.collect",
|
||||
"node",
|
||||
"mainland-worker-01",
|
||||
"queued",
|
||||
"ssh",
|
||||
"api",
|
||||
json.dumps({"service_name": "domaincheck-worker", "lines": 120}, ensure_ascii=False),
|
||||
json.dumps({}, ensure_ascii=False),
|
||||
json.dumps({}, ensure_ascii=False),
|
||||
"",
|
||||
datetime(2026, 4, 18, 10, 0, 0),
|
||||
None,
|
||||
None,
|
||||
datetime(2026, 4, 18, 10, 0, 0),
|
||||
"low",
|
||||
False,
|
||||
"approved",
|
||||
"",
|
||||
None,
|
||||
"",
|
||||
"",
|
||||
None,
|
||||
json.dumps({}, ensure_ascii=False),
|
||||
json.dumps({}, ensure_ascii=False),
|
||||
None,
|
||||
)
|
||||
mock_execute_ssh_job_record.return_value = (
|
||||
True,
|
||||
"worker logs collected",
|
||||
{"job": {"id": 9, "status": "success"}},
|
||||
)
|
||||
|
||||
ok, message, data = dispatch_ops_job(9)
|
||||
|
||||
self.assertTrue(ok)
|
||||
self.assertEqual("已按 SSH 执行派发:worker logs collected", message)
|
||||
self.assertEqual("success", data["job"]["status"])
|
||||
mock_execute_ssh_job_record.assert_called_once_with(9)
|
||||
mock_append_ops_job_event.assert_called_once()
|
||||
event_kwargs = mock_append_ops_job_event.call_args.kwargs
|
||||
self.assertEqual("job_dispatch_requested", event_kwargs["event_type"])
|
||||
self.assertEqual("mainland-worker-01", event_kwargs["node_code"])
|
||||
self.assertEqual("ssh", event_kwargs["payload"]["execution_mode"])
|
||||
|
||||
@patch("app.services.ops_agent_service.append_ops_job_event")
|
||||
@patch("app.services.ops_job_service.get_ops_job")
|
||||
@patch("app.services.ops_job_service._execute_local_job")
|
||||
@patch("app.services.ops_policy_service.preview_ops_job_policy")
|
||||
@patch("app.services.ops_job_service.get_db")
|
||||
def test_create_ops_job_auto_uses_local_runtime_for_local_structured_action(
|
||||
self,
|
||||
mock_get_db,
|
||||
mock_preview_ops_job_policy,
|
||||
mock_execute_local_job,
|
||||
mock_get_ops_job,
|
||||
mock_append_ops_job_event,
|
||||
) -> None:
|
||||
conn = MagicMock()
|
||||
cursor = MagicMock()
|
||||
db_ctx = MagicMock()
|
||||
cursor_ctx = MagicMock()
|
||||
db_ctx.__enter__.return_value = conn
|
||||
db_ctx.__exit__.return_value = False
|
||||
cursor_ctx.__enter__.return_value = cursor
|
||||
cursor_ctx.__exit__.return_value = False
|
||||
conn.cursor.return_value = cursor_ctx
|
||||
mock_get_db.return_value = db_ctx
|
||||
cursor.fetchone.return_value = (42,)
|
||||
|
||||
mock_preview_ops_job_policy.return_value = {
|
||||
"risk_level": "low",
|
||||
"approval_required": False,
|
||||
"blocked": False,
|
||||
"blocking_reasons": [],
|
||||
}
|
||||
mock_execute_local_job.return_value = (
|
||||
True,
|
||||
"worker logs collected",
|
||||
{"executor": "local-runtime"},
|
||||
)
|
||||
mock_get_ops_job.return_value = {
|
||||
"id": 42,
|
||||
"status": "success",
|
||||
"execution_mode": "local-runtime",
|
||||
}
|
||||
|
||||
ok, message, data = create_ops_job(
|
||||
{
|
||||
"action": "logs.collect",
|
||||
"target_node_code": settings.node_code,
|
||||
"payload": {
|
||||
"service_name": "domaincheck-worker",
|
||||
"lines": 120,
|
||||
},
|
||||
"requested_by": "unit-test",
|
||||
}
|
||||
)
|
||||
|
||||
self.assertTrue(ok)
|
||||
self.assertEqual("worker logs collected", message)
|
||||
self.assertTrue(data["executed_immediately"])
|
||||
self.assertEqual("local-runtime", data["job"]["execution_mode"])
|
||||
mock_execute_local_job.assert_called_once_with(
|
||||
"logs.collect",
|
||||
{
|
||||
"service_name": "domaincheck-worker",
|
||||
"lines": 120,
|
||||
},
|
||||
)
|
||||
|
||||
insert_call = next(
|
||||
call
|
||||
for call in cursor.execute.call_args_list
|
||||
if "INSERT INTO ops_jobs" in str(call.args[0])
|
||||
)
|
||||
insert_params = insert_call.args[1]
|
||||
self.assertEqual("local-runtime", insert_params[5])
|
||||
mock_append_ops_job_event.assert_called_once()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
220
domain-api/tests/test_ops_playbook_service.py
Normal file
220
domain-api/tests/test_ops_playbook_service.py
Normal file
@@ -0,0 +1,220 @@
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
from app.services.ops_playbook_service import (
|
||||
get_recent_ops_playbook_runs,
|
||||
list_ops_playbook_run_events,
|
||||
preview_ops_playbook,
|
||||
)
|
||||
|
||||
|
||||
class OpsPlaybookServiceTests(unittest.TestCase):
|
||||
def test_onboarding_bootstrap_preview_uses_control_plane(self) -> None:
|
||||
ok, message, data = preview_ops_playbook(
|
||||
{
|
||||
"playbook_key": "onboarding.bootstrap",
|
||||
"node_codes": ["mainland-worker-02"],
|
||||
}
|
||||
)
|
||||
|
||||
self.assertTrue(ok, message)
|
||||
self.assertEqual("playbook 预览生成成功", message)
|
||||
self.assertEqual("onboarding.bootstrap", data["playbook"]["key"])
|
||||
self.assertEqual("control-plane", data["execution_mode"])
|
||||
self.assertEqual("控制面", data["execution_mode_label"])
|
||||
self.assertEqual(["control-plane"], data["playbook"]["execution_modes"])
|
||||
self.assertEqual("控制面", data["playbook"]["default_execution_mode_label"])
|
||||
self.assertEqual("控制面", data["playbook"]["execution_mode_options"][0]["label"])
|
||||
self.assertTrue(data["auto_approve"])
|
||||
self.assertEqual(1, data["step_count"])
|
||||
step = data["steps"][0]
|
||||
self.assertEqual("bootstrap", step["step_key"])
|
||||
self.assertEqual("node.bootstrap", step["template_key"])
|
||||
self.assertEqual("node.bootstrap", step["action"])
|
||||
self.assertEqual("control-plane", step["execution_mode"])
|
||||
self.assertEqual(1, step["expected_jobs"])
|
||||
|
||||
def test_scene_logs_key_preview_allows_ssh_execution_mode(self) -> None:
|
||||
ok, message, data = preview_ops_playbook(
|
||||
{
|
||||
"playbook_key": "scene.logs.key",
|
||||
"node_codes": ["mainland-worker-01"],
|
||||
"execution_mode": "ssh",
|
||||
}
|
||||
)
|
||||
|
||||
self.assertTrue(ok, message)
|
||||
self.assertEqual("playbook 预览生成成功", message)
|
||||
self.assertEqual("ssh", data["execution_mode"])
|
||||
self.assertEqual("SSH", data["execution_mode_label"])
|
||||
self.assertEqual(["remote-agent", "ssh"], data["playbook"]["execution_modes"])
|
||||
self.assertEqual("ssh", data["steps"][0]["execution_mode"])
|
||||
self.assertEqual("SSH", data["steps"][0]["execution_mode_label"])
|
||||
|
||||
def test_onboarding_bootstrap_preview_rejects_unsupported_execution_mode(self) -> None:
|
||||
ok, message, data = preview_ops_playbook(
|
||||
{
|
||||
"playbook_key": "onboarding.bootstrap",
|
||||
"node_codes": ["mainland-worker-02"],
|
||||
"execution_mode": "ssh",
|
||||
}
|
||||
)
|
||||
|
||||
self.assertFalse(ok)
|
||||
self.assertEqual("该 playbook 仅支持 control-plane", message)
|
||||
self.assertEqual(["control-plane"], data["playbook"]["execution_modes"])
|
||||
|
||||
@patch("app.services.ops_playbook_service.list_managed_nodes_with_agent_state")
|
||||
@patch("app.services.ops_playbook_service.list_ops_jobs")
|
||||
def test_recent_onboarding_bootstrap_run_focus_summary_reflects_agent_state(
|
||||
self,
|
||||
mock_list_ops_jobs,
|
||||
mock_list_managed_nodes_with_agent_state,
|
||||
) -> None:
|
||||
mock_list_ops_jobs.return_value = [
|
||||
{
|
||||
"id": 31,
|
||||
"job_code": "ops-bootstrap-31",
|
||||
"action": "node.bootstrap",
|
||||
"status": "success",
|
||||
"target_node_code": "mainland-worker-02",
|
||||
"created_at": "2026-04-17 10:00:00",
|
||||
"updated_at": "2026-04-17 10:01:00",
|
||||
"metadata": {
|
||||
"playbook": {
|
||||
"run_code": "pbr-bootstrap-1",
|
||||
"key": "onboarding.bootstrap",
|
||||
"title": "生成节点接入工单",
|
||||
"group_key": "onboarding",
|
||||
"group_title": "接管验收",
|
||||
"requested_by": "tester",
|
||||
"execution_mode": "control-plane",
|
||||
"step_key": "bootstrap",
|
||||
"step_title": "生成 Node Agent 接入工单",
|
||||
"step_order": 1,
|
||||
"step_count": 1,
|
||||
"expected_jobs_total": 1,
|
||||
"target_nodes_total": 1,
|
||||
"template_key": "node.bootstrap",
|
||||
}
|
||||
},
|
||||
"result": {
|
||||
"bootstrap_plan": {
|
||||
"bootstrap_script_name": "bootstrap-node-agent-mainland-worker-02.sh",
|
||||
}
|
||||
},
|
||||
}
|
||||
]
|
||||
mock_list_managed_nodes_with_agent_state.return_value = {
|
||||
"nodes": [
|
||||
{
|
||||
"node_code": "mainland-worker-02",
|
||||
"agent_state": "pending_bootstrap",
|
||||
"is_managed": True,
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
payload = get_recent_ops_playbook_runs(limit=10, scan_limit=20)
|
||||
|
||||
self.assertEqual(1, len(payload["runs"]))
|
||||
run = payload["runs"][0]
|
||||
self.assertEqual("success", run["status"])
|
||||
self.assertEqual("成功", run["status_label"])
|
||||
self.assertEqual("控制面", run["execution_mode_label"])
|
||||
self.assertEqual("warning", run["focus_level"])
|
||||
self.assertIn("待执行 bootstrap 脚本", run["focus_summary"])
|
||||
self.assertEqual(run["focus_summary"], run["summary_text"])
|
||||
self.assertEqual(1, run["steps_total"])
|
||||
self.assertEqual("pbr-bootstrap-1", run["focus_ref"]["run_code"])
|
||||
self.assertEqual("生成 Node Agent 接入工单", run["steps"][0]["title"])
|
||||
self.assertEqual("成功", run["steps"][0]["status_label"])
|
||||
self.assertTrue(run["steps"][0]["summary_text"])
|
||||
|
||||
@patch("app.services.ops_playbook_service.list_managed_nodes_with_agent_state")
|
||||
@patch("app.services.ops_playbook_service.list_ops_job_events_for_jobs")
|
||||
@patch("app.services.ops_playbook_service.list_ops_jobs")
|
||||
def test_playbook_run_events_add_contract_aliases(
|
||||
self,
|
||||
mock_list_ops_jobs,
|
||||
mock_list_ops_job_events_for_jobs,
|
||||
mock_list_managed_nodes_with_agent_state,
|
||||
) -> None:
|
||||
mock_list_managed_nodes_with_agent_state.return_value = {"nodes": []}
|
||||
mock_list_ops_jobs.return_value = [
|
||||
{
|
||||
"id": 41,
|
||||
"job_code": "ops-inspection-41",
|
||||
"action": "logs.collect",
|
||||
"status": "running",
|
||||
"target_node_code": "mainland-worker-01",
|
||||
"created_at": "2026-04-18 06:30:00",
|
||||
"updated_at": "2026-04-18 06:31:00",
|
||||
"metadata": {
|
||||
"playbook": {
|
||||
"run_code": "pbr-inspection-1",
|
||||
"key": "inspection.standard",
|
||||
"title": "标准巡检",
|
||||
"group_key": "diagnostics",
|
||||
"group_title": "标准巡检",
|
||||
"requested_by": "tester",
|
||||
"execution_mode": "remote-agent",
|
||||
"step_key": "worker_logs",
|
||||
"step_title": "收集 Worker 日志",
|
||||
"step_order": 2,
|
||||
"step_count": 3,
|
||||
"expected_jobs_total": 3,
|
||||
"target_nodes_total": 1,
|
||||
"template_key": "logs.collect",
|
||||
}
|
||||
},
|
||||
}
|
||||
]
|
||||
mock_list_ops_job_events_for_jobs.return_value = [
|
||||
{
|
||||
"id": 501,
|
||||
"event_key": "job-event:501",
|
||||
"job_id": 41,
|
||||
"node_code": "mainland-worker-01",
|
||||
"event_type": "job_running",
|
||||
"level": "info",
|
||||
"level_label": "信息",
|
||||
"message": "开始收集 Worker 日志",
|
||||
"summary": "开始收集 Worker 日志",
|
||||
"summary_text": "远端 Agent 已开始收集 Worker 日志",
|
||||
"focus_ref": {
|
||||
"kind": "ops_job",
|
||||
"job_id": 41,
|
||||
"job_code": "ops-inspection-41",
|
||||
"action": "logs.collect",
|
||||
"target_node_code": "mainland-worker-01",
|
||||
"event_key": "job-event:501",
|
||||
"step_key": "worker_logs",
|
||||
},
|
||||
"payload": {},
|
||||
"created_at": "2026-04-18 06:31:00",
|
||||
"occurred_at": "2026-04-18 06:31:00",
|
||||
"ui_intent": {"kind": "job_detail", "job_id": 41, "focus_event_id": 501},
|
||||
}
|
||||
]
|
||||
|
||||
payload = list_ops_playbook_run_events("pbr-inspection-1", limit=10)
|
||||
|
||||
self.assertEqual("inspection.standard", payload["playbook_run"]["playbook_key"])
|
||||
self.assertEqual(1, len(payload["events"]))
|
||||
event = payload["events"][0]
|
||||
self.assertEqual("收口中", event["job_status_label"])
|
||||
self.assertEqual("远端 Agent 已开始收集 Worker 日志", event["summary_text"])
|
||||
self.assertEqual("worker_logs", event["focus_ref"]["focus_step_key"])
|
||||
self.assertEqual("job-event:501", event["focus_ref"]["event_key"])
|
||||
self.assertEqual("ops_job", event["source_focus_ref"]["kind"])
|
||||
self.assertEqual("job-event:501", event["source_focus_ref"]["event_key"])
|
||||
summary = payload["summary"]
|
||||
self.assertEqual(1, summary["returned_total"])
|
||||
self.assertEqual({"job_running": 1}, summary["event_type_counts"])
|
||||
self.assertEqual({"running": 1}, summary["job_status_counts"])
|
||||
self.assertEqual(10, summary["filters"]["limit"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
253
domain-api/tests/test_ops_policy_service.py
Normal file
253
domain-api/tests/test_ops_policy_service.py
Normal file
@@ -0,0 +1,253 @@
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
from app.services.ops_policy_service import preview_ops_job_policy
|
||||
|
||||
|
||||
class OpsPolicyServiceTests(unittest.TestCase):
|
||||
@patch("app.services.ops_policy_service.get_cluster_snapshot")
|
||||
def test_node_bootstrap_preview_requires_control_plane(self, mock_cluster_snapshot) -> None:
|
||||
mock_cluster_snapshot.return_value = {
|
||||
"nodes": [
|
||||
{
|
||||
"node_code": "mainland-worker-01",
|
||||
"region": "mainland",
|
||||
"role": "worker",
|
||||
"status": "online",
|
||||
"current_load": 0,
|
||||
"is_effective_worker": True,
|
||||
"detect_participating": False,
|
||||
}
|
||||
],
|
||||
"summary": {
|
||||
"online_worker_nodes": 2,
|
||||
"dedicated_online_worker_nodes": 1,
|
||||
"online_control_nodes": 1,
|
||||
"busy_nodes": [],
|
||||
},
|
||||
}
|
||||
|
||||
preview = preview_ops_job_policy(
|
||||
{
|
||||
"action": "node.bootstrap",
|
||||
"target_type": "node",
|
||||
"target_node_code": "mainland-worker-01",
|
||||
"execution_mode": "remote-agent",
|
||||
"payload": {},
|
||||
}
|
||||
)
|
||||
|
||||
self.assertEqual("critical", preview["risk_level"])
|
||||
self.assertTrue(preview["blocked"])
|
||||
self.assertTrue(preview["approval_required"])
|
||||
self.assertIn("node.bootstrap 仅支持 control-plane", " ".join(preview["blocking_reasons"]))
|
||||
|
||||
@patch("app.services.ops_policy_service.get_cluster_snapshot")
|
||||
def test_deploy_release_batch_preview_summarizes_node_risks(self, mock_cluster_snapshot) -> None:
|
||||
mock_cluster_snapshot.return_value = {
|
||||
"nodes": [
|
||||
{
|
||||
"node_code": "mainland-controller-01",
|
||||
"region": "mainland",
|
||||
"role": "control",
|
||||
"status": "busy",
|
||||
"current_load": 3,
|
||||
"is_effective_worker": True,
|
||||
"detect_participating": False,
|
||||
},
|
||||
{
|
||||
"node_code": "mainland-worker-01",
|
||||
"region": "mainland",
|
||||
"role": "worker",
|
||||
"status": "online",
|
||||
"current_load": 0,
|
||||
"is_effective_worker": True,
|
||||
"detect_participating": False,
|
||||
},
|
||||
],
|
||||
"summary": {
|
||||
"online_worker_nodes": 2,
|
||||
"dedicated_online_worker_nodes": 1,
|
||||
"online_control_nodes": 1,
|
||||
"busy_nodes": ["mainland-controller-01"],
|
||||
},
|
||||
}
|
||||
|
||||
preview = preview_ops_job_policy(
|
||||
{
|
||||
"action": "deploy.release",
|
||||
"target_type": "batch",
|
||||
"target_node_codes": ["mainland-controller-01", "mainland-worker-01"],
|
||||
"execution_mode": "remote-agent",
|
||||
"payload": {
|
||||
"release_version": "2026.04.18-rc1",
|
||||
"artifact_url": "https://example.com/release.tar.gz",
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
self.assertEqual("batch", preview["target_type"])
|
||||
self.assertTrue(preview["blocked"])
|
||||
self.assertTrue(preview["approval_required"])
|
||||
self.assertEqual(2, preview["target_summary"]["nodes_total"])
|
||||
self.assertEqual(1, preview["target_summary"]["blocked_nodes"])
|
||||
self.assertEqual(1, preview["target_summary"]["approval_nodes"])
|
||||
self.assertEqual(1, preview["target_summary"]["warning_nodes"])
|
||||
self.assertEqual(2, len(preview["target_nodes"]))
|
||||
controller_row = next(item for item in preview["target_nodes"] if item["node_code"] == "mainland-controller-01")
|
||||
self.assertTrue(controller_row["blocked"])
|
||||
self.assertTrue(controller_row["approval_required"])
|
||||
|
||||
@patch("app.services.ops_policy_service.get_cluster_snapshot")
|
||||
def test_deploy_release_preview_blocks_missing_artifact_and_wrong_execution_mode(self, mock_cluster_snapshot) -> None:
|
||||
mock_cluster_snapshot.return_value = {
|
||||
"nodes": [
|
||||
{
|
||||
"node_code": "mainland-worker-01",
|
||||
"region": "mainland",
|
||||
"role": "worker",
|
||||
"status": "online",
|
||||
"current_load": 0,
|
||||
"is_effective_worker": True,
|
||||
"detect_participating": False,
|
||||
}
|
||||
],
|
||||
"summary": {
|
||||
"online_worker_nodes": 2,
|
||||
"dedicated_online_worker_nodes": 1,
|
||||
"online_control_nodes": 1,
|
||||
"busy_nodes": [],
|
||||
},
|
||||
}
|
||||
|
||||
preview = preview_ops_job_policy(
|
||||
{
|
||||
"action": "deploy.release",
|
||||
"target_type": "node",
|
||||
"target_node_code": "mainland-worker-01",
|
||||
"execution_mode": "local-runtime",
|
||||
"payload": {
|
||||
"release_version": "2026.04.18-rc1",
|
||||
"artifact_url": "",
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
self.assertTrue(preview["blocked"])
|
||||
self.assertIn("deploy.release 缺少 artifact_url", " ".join(preview["blocking_reasons"]))
|
||||
self.assertIn("deploy.release 仅支持 remote-agent 或 ssh", " ".join(preview["blocking_reasons"]))
|
||||
|
||||
@patch("app.services.ops_policy_service.get_cluster_snapshot")
|
||||
def test_deploy_release_preview_allows_ssh_execution_mode(self, mock_cluster_snapshot) -> None:
|
||||
mock_cluster_snapshot.return_value = {
|
||||
"nodes": [
|
||||
{
|
||||
"node_code": "mainland-worker-01",
|
||||
"region": "mainland",
|
||||
"role": "worker",
|
||||
"status": "online",
|
||||
"current_load": 0,
|
||||
"is_effective_worker": True,
|
||||
"detect_participating": False,
|
||||
}
|
||||
],
|
||||
"summary": {
|
||||
"online_worker_nodes": 2,
|
||||
"dedicated_online_worker_nodes": 1,
|
||||
"online_control_nodes": 1,
|
||||
"busy_nodes": [],
|
||||
},
|
||||
}
|
||||
|
||||
preview = preview_ops_job_policy(
|
||||
{
|
||||
"action": "deploy.release",
|
||||
"target_type": "node",
|
||||
"target_node_code": "mainland-worker-01",
|
||||
"execution_mode": "ssh",
|
||||
"payload": {
|
||||
"release_version": "2026.04.18-rc1",
|
||||
"artifact_url": "https://example.com/release.tar.gz",
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
self.assertFalse(any("仅支持 remote-agent 或 ssh" in item for item in preview["blocking_reasons"]))
|
||||
self.assertTrue(preview["approval_required"])
|
||||
|
||||
@patch("app.services.ops_policy_service.get_cluster_snapshot")
|
||||
def test_service_status_preview_blocks_control_plane_execution_mode(self, mock_cluster_snapshot) -> None:
|
||||
mock_cluster_snapshot.return_value = {
|
||||
"nodes": [
|
||||
{
|
||||
"node_code": "mainland-worker-01",
|
||||
"region": "mainland",
|
||||
"role": "worker",
|
||||
"status": "online",
|
||||
"current_load": 0,
|
||||
"is_effective_worker": True,
|
||||
"detect_participating": False,
|
||||
}
|
||||
],
|
||||
"summary": {
|
||||
"online_worker_nodes": 2,
|
||||
"dedicated_online_worker_nodes": 1,
|
||||
"online_control_nodes": 1,
|
||||
"busy_nodes": [],
|
||||
},
|
||||
}
|
||||
|
||||
preview = preview_ops_job_policy(
|
||||
{
|
||||
"action": "service.status",
|
||||
"target_type": "node",
|
||||
"target_node_code": "mainland-worker-01",
|
||||
"execution_mode": "control-plane",
|
||||
"payload": {"service_name": "domaincheck-worker"},
|
||||
}
|
||||
)
|
||||
|
||||
self.assertTrue(preview["blocked"])
|
||||
self.assertIn("service.status 当前不支持 control-plane 执行方式", " ".join(preview["blocking_reasons"]))
|
||||
|
||||
@patch("app.services.ops_policy_service.get_cluster_snapshot")
|
||||
def test_logs_collect_preview_blocks_local_runtime_on_remote_node(self, mock_cluster_snapshot) -> None:
|
||||
mock_cluster_snapshot.return_value = {
|
||||
"nodes": [
|
||||
{
|
||||
"node_code": "mainland-worker-01",
|
||||
"region": "mainland",
|
||||
"role": "worker",
|
||||
"status": "online",
|
||||
"current_load": 0,
|
||||
"is_effective_worker": True,
|
||||
"detect_participating": False,
|
||||
}
|
||||
],
|
||||
"summary": {
|
||||
"online_worker_nodes": 2,
|
||||
"dedicated_online_worker_nodes": 1,
|
||||
"online_control_nodes": 1,
|
||||
"busy_nodes": [],
|
||||
},
|
||||
}
|
||||
|
||||
preview = preview_ops_job_policy(
|
||||
{
|
||||
"action": "logs.collect",
|
||||
"target_type": "node",
|
||||
"target_node_code": "mainland-worker-01",
|
||||
"execution_mode": "local-runtime",
|
||||
"payload": {
|
||||
"service_name": "domaincheck-worker",
|
||||
"lines": 120,
|
||||
},
|
||||
}
|
||||
)
|
||||
|
||||
self.assertTrue(preview["blocked"])
|
||||
self.assertIn("local-runtime 仅支持当前控制面本机节点", " ".join(preview["blocking_reasons"]))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
89
domain-api/tests/test_ops_release_executor_core.py
Normal file
89
domain-api/tests/test_ops_release_executor_core.py
Normal file
@@ -0,0 +1,89 @@
|
||||
import io
|
||||
import tarfile
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
from app.services.ops_release_executor_core import build_remote_release_action_script, execute_release_action
|
||||
|
||||
|
||||
class _BytesResponse:
|
||||
def __init__(self, data: bytes) -> None:
|
||||
self._data = data
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, exc_type, exc, tb):
|
||||
return False
|
||||
|
||||
def read(self, *args, **kwargs) -> bytes:
|
||||
return self._data
|
||||
|
||||
|
||||
def _build_release_archive() -> bytes:
|
||||
buffer = io.BytesIO()
|
||||
with tarfile.open(fileobj=buffer, mode="w:gz") as archive:
|
||||
content = b"hello-release"
|
||||
info = tarfile.TarInfo(name="bundle/README.txt")
|
||||
info.size = len(content)
|
||||
archive.addfile(info, io.BytesIO(content))
|
||||
return buffer.getvalue()
|
||||
|
||||
|
||||
class OpsReleaseExecutorCoreTests(unittest.TestCase):
|
||||
def test_execute_release_action_deploys_archive_and_switches_current(self) -> None:
|
||||
archive_bytes = _build_release_archive()
|
||||
commands: list[tuple[str, ...]] = []
|
||||
|
||||
def run_command(command: list[str], *, timeout: int = 60):
|
||||
commands.append(tuple(command))
|
||||
if command[:2] == ["systemctl", "restart"]:
|
||||
return 0, "", ""
|
||||
if command[:2] == ["systemctl", "is-active"]:
|
||||
return 0, "active", ""
|
||||
return 0, "", ""
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
ok, message, data = execute_release_action(
|
||||
{
|
||||
"release_version": "2026.04.18-rc1",
|
||||
"artifact_url": "https://example.com/domaincheck.tar.gz",
|
||||
"install_root": tmpdir,
|
||||
"restart_services": ["domaincheck-worker"],
|
||||
"health_check_services": ["domaincheck-worker"],
|
||||
"health_check_urls": [],
|
||||
"rollback_on_failure": "true",
|
||||
"switch_current": "true",
|
||||
},
|
||||
run_command=run_command,
|
||||
default_api_service_name="domaincheck-api",
|
||||
urlopen_func=lambda request, timeout=120: _BytesResponse(archive_bytes),
|
||||
user_agent="unit-test/0.1",
|
||||
)
|
||||
|
||||
self.assertTrue(ok, message)
|
||||
self.assertEqual("release deployed", message)
|
||||
self.assertEqual("2026.04.18-rc1", data["release_version"])
|
||||
current_link = Path(tmpdir) / "current"
|
||||
self.assertTrue(current_link.is_symlink())
|
||||
release_dir = Path(data["release_dir"])
|
||||
self.assertTrue(release_dir.exists())
|
||||
self.assertTrue((release_dir / ".release-meta.json").exists())
|
||||
self.assertIn(("systemctl", "restart", "domaincheck-worker"), commands)
|
||||
self.assertIn(("systemctl", "is-active", "domaincheck-worker"), commands)
|
||||
|
||||
def test_build_remote_release_action_script_is_valid_python(self) -> None:
|
||||
script = build_remote_release_action_script(
|
||||
{
|
||||
"release_version": "2026.04.18-rc1",
|
||||
"artifact_url": "https://example.com/domaincheck.tar.gz",
|
||||
}
|
||||
)
|
||||
|
||||
compile(script, "<remote-release-script>", "exec")
|
||||
self.assertIn("execute_release_action(", script)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
93
domain-api/tests/test_ops_release_service_contracts.py
Normal file
93
domain-api/tests/test_ops_release_service_contracts.py
Normal file
@@ -0,0 +1,93 @@
|
||||
import json
|
||||
import unittest
|
||||
from datetime import datetime
|
||||
|
||||
from app.services.ops_release_service import _serialize_release_row, _serialize_rollout_row
|
||||
|
||||
|
||||
class OpsReleaseServiceContractTests(unittest.TestCase):
|
||||
def test_serialize_release_row_adds_contract_aliases(self) -> None:
|
||||
row = (
|
||||
9,
|
||||
"2026.04.18+bd6bcb2",
|
||||
"stable",
|
||||
"bd6bcb2",
|
||||
"ready",
|
||||
"https://example.com/domaincheck.tar.gz",
|
||||
"sha256:test",
|
||||
"release notes",
|
||||
json.dumps(
|
||||
{
|
||||
"source": "package",
|
||||
"manifest": {
|
||||
"artifact_kind": "tarball",
|
||||
"package_name": "domaincheck-2026.04.18+bd6bcb2.tgz",
|
||||
"included_services": ["domaincheck-api", "domaincheck-worker"],
|
||||
"rollback_hint": "切回上一个 current",
|
||||
},
|
||||
}
|
||||
),
|
||||
"www",
|
||||
datetime(2026, 4, 18, 9, 0, 0),
|
||||
datetime(2026, 4, 18, 9, 5, 0),
|
||||
datetime(2026, 4, 18, 9, 6, 0),
|
||||
)
|
||||
|
||||
payload = _serialize_release_row(row)
|
||||
|
||||
self.assertEqual("已就绪", payload["status_label"])
|
||||
self.assertIn("已就绪", payload["summary_text"])
|
||||
self.assertEqual("release_hub", payload["focus_ref"]["kind"])
|
||||
self.assertEqual(9, payload["focus_ref"]["release_id"])
|
||||
self.assertEqual("release_detail", payload["focus_ref"]["section"])
|
||||
self.assertEqual("tarball", payload["manifest"]["artifact_kind"])
|
||||
self.assertEqual("domaincheck-2026.04.18+bd6bcb2.tgz", payload["manifest"]["package_name"])
|
||||
self.assertEqual(["domaincheck-api", "domaincheck-worker"], payload["manifest"]["included_services"])
|
||||
|
||||
def test_serialize_rollout_row_adds_contract_aliases(self) -> None:
|
||||
row = (
|
||||
11,
|
||||
9,
|
||||
"rollout-20260418090000-a1b2",
|
||||
json.dumps({"region": "mainland", "role": "worker"}),
|
||||
json.dumps(
|
||||
{
|
||||
"items": [
|
||||
{"node_code": "mainland-worker-01", "region": "mainland", "role": "worker"},
|
||||
{"node_code": "mainland-worker-02", "region": "mainland", "role": "worker"},
|
||||
]
|
||||
}
|
||||
),
|
||||
json.dumps({"execution_mode": "ssh", "batch_size": 1}),
|
||||
"running",
|
||||
1,
|
||||
2,
|
||||
2,
|
||||
1,
|
||||
json.dumps(
|
||||
{
|
||||
"status": "running",
|
||||
"launched_targets": 1,
|
||||
"remaining_targets": 1,
|
||||
"active_jobs": 1,
|
||||
"current_batch_index": 1,
|
||||
"batches_total": 2,
|
||||
}
|
||||
),
|
||||
"web-ui",
|
||||
datetime(2026, 4, 18, 9, 10, 0),
|
||||
datetime(2026, 4, 18, 9, 11, 0),
|
||||
)
|
||||
|
||||
payload = _serialize_rollout_row(row)
|
||||
|
||||
self.assertEqual("执行中", payload["status_label"])
|
||||
self.assertEqual(["mainland-worker-01", "mainland-worker-02"], payload["target_node_codes"])
|
||||
self.assertIn("当前已覆盖 1/2 台节点", payload["summary_text"])
|
||||
self.assertEqual("release_rollout", payload["focus_ref"]["kind"])
|
||||
self.assertEqual(11, payload["focus_ref"]["rollout_id"])
|
||||
self.assertEqual("rollout_jobs", payload["focus_ref"]["section"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
432
domain-api/tests/test_ops_release_service_execution_modes.py
Normal file
432
domain-api/tests/test_ops_release_service_execution_modes.py
Normal file
@@ -0,0 +1,432 @@
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
from app.services.ops_release_service import (
|
||||
build_release_execution_mode_recommendation,
|
||||
build_release_rollout_gate,
|
||||
build_rollout_target_operational_readiness,
|
||||
)
|
||||
|
||||
|
||||
class _EmptyCursor:
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, exc_type, exc, tb):
|
||||
return False
|
||||
|
||||
def execute(self, query, params=None) -> None:
|
||||
self.query = query
|
||||
self.params = params
|
||||
|
||||
def fetchall(self):
|
||||
return []
|
||||
|
||||
def fetchone(self):
|
||||
return None
|
||||
|
||||
|
||||
class _EmptyConnection:
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, exc_type, exc, tb):
|
||||
return False
|
||||
|
||||
def cursor(self):
|
||||
return _EmptyCursor()
|
||||
|
||||
|
||||
class OpsReleaseServiceExecutionModeTests(unittest.TestCase):
|
||||
@patch("app.services.ops_release_service.get_db")
|
||||
@patch("app.services.ops_agent_service.ensure_ops_agent_schema")
|
||||
@patch("app.services.ops_agent_service.get_managed_node_onboarding")
|
||||
@patch("app.services.ops_job_service.list_managed_nodes")
|
||||
@patch("app.services.cluster_runtime_service.get_cluster_snapshot")
|
||||
def test_build_rollout_target_operational_readiness_accepts_ssh_ready_nodes(
|
||||
self,
|
||||
mock_get_cluster_snapshot,
|
||||
mock_list_managed_nodes,
|
||||
mock_get_managed_node_onboarding,
|
||||
mock_ensure_ops_agent_schema,
|
||||
mock_get_db,
|
||||
) -> None:
|
||||
mock_ensure_ops_agent_schema.return_value = None
|
||||
mock_get_managed_node_onboarding.return_value = {
|
||||
"onboarding_stage": {"code": "acceptance_ready", "label": "待接管验收"},
|
||||
"summary": "Node Agent 已在线,可以直接跑 onboarding.acceptance。",
|
||||
"recovery_decision": {
|
||||
"action": "run_acceptance",
|
||||
"label": "执行接管验收",
|
||||
"summary": "Node Agent 已在线,可以直接跑 onboarding.acceptance。",
|
||||
"command_hint": "bash drive_ops_center.sh node-acceptance-run http://127.0.0.1:8100 mainland-worker-01 cli",
|
||||
"window": "acceptance",
|
||||
},
|
||||
}
|
||||
mock_get_db.return_value = _EmptyConnection()
|
||||
mock_get_cluster_snapshot.return_value = {
|
||||
"nodes": [
|
||||
{
|
||||
"node_code": "mainland-worker-01",
|
||||
"region": "mainland",
|
||||
"role": "worker",
|
||||
"status": "online",
|
||||
"current_load": 0,
|
||||
"is_effective_worker": True,
|
||||
}
|
||||
]
|
||||
}
|
||||
mock_list_managed_nodes.return_value = [
|
||||
{
|
||||
"node_code": "mainland-worker-01",
|
||||
"region": "mainland",
|
||||
"role": "worker",
|
||||
"is_enabled": True,
|
||||
"ssh_host": "121.204.244.248",
|
||||
"ssh_user": "root",
|
||||
"metadata": {},
|
||||
"last_seen_at": "",
|
||||
}
|
||||
]
|
||||
|
||||
readiness = build_rollout_target_operational_readiness(
|
||||
[{"node_code": "mainland-worker-01", "region": "mainland", "role": "worker"}],
|
||||
execution_mode="ssh",
|
||||
)
|
||||
|
||||
self.assertEqual("ssh", readiness["execution_mode"])
|
||||
self.assertEqual(1, readiness["summary"]["ssh_ready_nodes"])
|
||||
self.assertEqual(1, readiness["summary"]["execution_ready_nodes"])
|
||||
self.assertEqual([], readiness["blocking_reasons"])
|
||||
self.assertTrue(readiness["rows"][0]["ssh_ready"])
|
||||
self.assertTrue(readiness["rows"][0]["execution_ready"])
|
||||
self.assertTrue(readiness["rows"][0]["agent_managed"])
|
||||
self.assertEqual("ssh", readiness["rows"][0]["execution_mode"])
|
||||
self.assertEqual("SSH", readiness["rows"][0]["execution_mode_label"])
|
||||
self.assertEqual("medium", readiness["rows"][0]["risk_level"])
|
||||
self.assertEqual("run_standard_inspection", readiness["rows"][0]["recommended_action_code"])
|
||||
self.assertEqual("acceptance_ready", readiness["rows"][0]["onboarding_stage_code"])
|
||||
self.assertEqual("run_acceptance", readiness["rows"][0]["recovery_action"])
|
||||
self.assertEqual(1, readiness["summary"]["onboarding_acceptance_ready_nodes"])
|
||||
|
||||
@patch("app.services.ops_release_service.get_db")
|
||||
@patch("app.services.ops_agent_service.ensure_ops_agent_schema")
|
||||
@patch("app.services.ops_agent_service.get_managed_node_onboarding")
|
||||
@patch("app.services.ops_job_service.list_managed_nodes")
|
||||
@patch("app.services.cluster_runtime_service.get_cluster_snapshot")
|
||||
def test_build_rollout_target_operational_readiness_adds_release_context_and_focus_ref(
|
||||
self,
|
||||
mock_get_cluster_snapshot,
|
||||
mock_list_managed_nodes,
|
||||
mock_get_managed_node_onboarding,
|
||||
mock_ensure_ops_agent_schema,
|
||||
mock_get_db,
|
||||
) -> None:
|
||||
mock_ensure_ops_agent_schema.return_value = None
|
||||
mock_get_managed_node_onboarding.return_value = {
|
||||
"onboarding_stage": {"code": "ready", "label": "已接管"},
|
||||
"summary": "当前节点无需额外恢复动作。",
|
||||
"recovery_decision": {
|
||||
"action": "noop",
|
||||
"label": "当前无需额外恢复动作",
|
||||
"summary": "当前节点暂无需要执行的接管恢复动作。",
|
||||
"command_hint": "",
|
||||
"window": "none",
|
||||
},
|
||||
}
|
||||
mock_get_db.return_value = _EmptyConnection()
|
||||
mock_get_cluster_snapshot.return_value = {
|
||||
"nodes": [
|
||||
{
|
||||
"node_code": "mainland-worker-01",
|
||||
"region": "mainland",
|
||||
"role": "worker",
|
||||
"status": "online",
|
||||
"current_load": 0,
|
||||
"is_effective_worker": True,
|
||||
"metadata": {"current_release_version": "2026.04.11+1921318"},
|
||||
}
|
||||
]
|
||||
}
|
||||
mock_list_managed_nodes.return_value = [
|
||||
{
|
||||
"node_code": "mainland-worker-01",
|
||||
"region": "mainland",
|
||||
"role": "worker",
|
||||
"is_enabled": True,
|
||||
"ssh_host": "121.204.244.248",
|
||||
"ssh_user": "root",
|
||||
"metadata": {"current_release_version": "2026.04.11+1921318"},
|
||||
"last_seen_at": "2026-04-18 08:00:00",
|
||||
}
|
||||
]
|
||||
|
||||
readiness = build_rollout_target_operational_readiness(
|
||||
[{"node_code": "mainland-worker-01", "region": "mainland", "role": "worker"}],
|
||||
execution_mode="remote-agent",
|
||||
desired_release={"id": 18, "release_version": "2026.04.18+bd6bcb2", "channel": "stable"},
|
||||
)
|
||||
|
||||
row = readiness["rows"][0]
|
||||
self.assertEqual("2026.04.11+1921318", row["current_release_version"])
|
||||
self.assertEqual("2026.04.18+bd6bcb2", row["desired_release_version"])
|
||||
self.assertEqual("release_hub", row["focus_ref"]["kind"])
|
||||
self.assertEqual("default_rollout_gate", row["focus_ref"]["section"])
|
||||
self.assertEqual("mainland-worker-01", row["focus_ref"]["node_code"])
|
||||
self.assertEqual("noop", row["recovery_action"])
|
||||
|
||||
@patch("app.services.ops_release_service.get_db")
|
||||
@patch("app.services.ops_agent_service.ensure_ops_agent_schema")
|
||||
@patch("app.services.ops_agent_service.get_managed_node_onboarding")
|
||||
@patch("app.services.ops_job_service.list_managed_nodes")
|
||||
@patch("app.services.cluster_runtime_service.get_cluster_snapshot")
|
||||
def test_build_rollout_target_operational_readiness_prefers_recovery_decision_for_blocked_nodes(
|
||||
self,
|
||||
mock_get_cluster_snapshot,
|
||||
mock_list_managed_nodes,
|
||||
mock_get_managed_node_onboarding,
|
||||
mock_ensure_ops_agent_schema,
|
||||
mock_get_db,
|
||||
) -> None:
|
||||
mock_ensure_ops_agent_schema.return_value = None
|
||||
mock_get_db.return_value = _EmptyConnection()
|
||||
mock_get_cluster_snapshot.return_value = {
|
||||
"nodes": [
|
||||
{
|
||||
"node_code": "mainland-worker-02",
|
||||
"region": "mainland",
|
||||
"role": "worker",
|
||||
"status": "online",
|
||||
"current_load": 0,
|
||||
"is_effective_worker": True,
|
||||
}
|
||||
]
|
||||
}
|
||||
mock_list_managed_nodes.return_value = [
|
||||
{
|
||||
"node_code": "mainland-worker-02",
|
||||
"region": "mainland",
|
||||
"role": "worker",
|
||||
"is_enabled": True,
|
||||
"ssh_host": "121.204.244.249",
|
||||
"ssh_user": "root",
|
||||
"metadata": {},
|
||||
"last_seen_at": "",
|
||||
}
|
||||
]
|
||||
mock_get_managed_node_onboarding.return_value = {
|
||||
"onboarding_stage": {"code": "pending_bootstrap", "label": "待接入"},
|
||||
"summary": "已签发有效 Agent Token,但节点尚未完成 register/heartbeat。",
|
||||
"recovery_decision": {
|
||||
"action": "bootstrap_run",
|
||||
"label": "签发接入工单",
|
||||
"summary": "当前节点仍处于接入阶段,建议先签发 onboarding.bootstrap。",
|
||||
"command_hint": "bash drive_ops_center.sh node-bootstrap-run http://127.0.0.1:8100 mainland-worker-02 cli",
|
||||
"window": "bootstrap",
|
||||
},
|
||||
}
|
||||
|
||||
readiness = build_rollout_target_operational_readiness(
|
||||
[{"node_code": "mainland-worker-02", "region": "mainland", "role": "worker"}],
|
||||
execution_mode="remote-agent",
|
||||
)
|
||||
|
||||
row = readiness["rows"][0]
|
||||
self.assertEqual("bootstrap_run", row["recovery_action"])
|
||||
self.assertEqual("bootstrap_run", row["recommended_action_code"])
|
||||
self.assertEqual(1, readiness["summary"]["onboarding_bootstrap_pending_nodes"])
|
||||
self.assertTrue(
|
||||
any("onboarding.bootstrap" in item for item in readiness["recommendations"])
|
||||
)
|
||||
|
||||
@patch("app.services.ops_release_service.get_db")
|
||||
@patch("app.services.ops_agent_service.ensure_ops_agent_schema")
|
||||
@patch("app.services.ops_agent_service.get_managed_node_onboarding")
|
||||
@patch("app.services.ops_job_service.list_managed_nodes")
|
||||
@patch("app.services.cluster_runtime_service.get_cluster_snapshot")
|
||||
def test_build_rollout_target_operational_readiness_blocks_nodes_with_dead_letter_queue(
|
||||
self,
|
||||
mock_get_cluster_snapshot,
|
||||
mock_list_managed_nodes,
|
||||
mock_get_managed_node_onboarding,
|
||||
mock_ensure_ops_agent_schema,
|
||||
mock_get_db,
|
||||
) -> None:
|
||||
mock_ensure_ops_agent_schema.return_value = None
|
||||
mock_get_db.return_value = _EmptyConnection()
|
||||
mock_get_managed_node_onboarding.return_value = {
|
||||
"onboarding_stage": {"code": "ready", "label": "已接管"},
|
||||
"summary": "当前节点无需额外恢复动作。",
|
||||
"recovery_decision": {
|
||||
"action": "noop",
|
||||
"label": "当前无需额外恢复动作",
|
||||
"summary": "当前节点暂无需要执行的接管恢复动作。",
|
||||
"command_hint": "",
|
||||
"window": "none",
|
||||
},
|
||||
}
|
||||
mock_get_cluster_snapshot.return_value = {
|
||||
"nodes": [
|
||||
{
|
||||
"node_code": "mainland-worker-03",
|
||||
"region": "mainland",
|
||||
"role": "worker",
|
||||
"status": "online",
|
||||
"current_load": 0,
|
||||
"is_effective_worker": True,
|
||||
}
|
||||
]
|
||||
}
|
||||
mock_list_managed_nodes.return_value = [
|
||||
{
|
||||
"node_code": "mainland-worker-03",
|
||||
"region": "mainland",
|
||||
"role": "worker",
|
||||
"is_enabled": True,
|
||||
"ssh_host": "121.204.244.250",
|
||||
"ssh_user": "root",
|
||||
"metadata": {
|
||||
"delivery_queue": {
|
||||
"state": "dead_letter",
|
||||
"label": "死信 1",
|
||||
"reason": "当前存在 1 条死信记录",
|
||||
"pending_count": 0,
|
||||
"dead_letter_count": 1,
|
||||
"oldest_dead_letter_at": "2026-04-18 09:01:00",
|
||||
}
|
||||
},
|
||||
"last_seen_at": "2026-04-18 09:05:00",
|
||||
}
|
||||
]
|
||||
|
||||
readiness = build_rollout_target_operational_readiness(
|
||||
[{"node_code": "mainland-worker-03", "region": "mainland", "role": "worker"}],
|
||||
execution_mode="remote-agent",
|
||||
)
|
||||
|
||||
row = readiness["rows"][0]
|
||||
self.assertEqual("dead_letter", row["delivery_queue_state"])
|
||||
self.assertEqual(1, row["delivery_queue_dead_letter_count"])
|
||||
self.assertTrue(
|
||||
any("死信" in item for item in row["blocking_reasons"])
|
||||
)
|
||||
self.assertEqual(1, readiness["summary"]["queue_dead_letter_nodes"])
|
||||
self.assertTrue(
|
||||
any("死信回执" in item for item in readiness["blocking_reasons"])
|
||||
)
|
||||
self.assertTrue(
|
||||
any("死信队列" in item for item in readiness["recommendations"])
|
||||
)
|
||||
|
||||
@patch("app.services.ops_release_service.build_rollout_target_operational_readiness")
|
||||
def test_build_release_rollout_gate_uses_execution_ready_count_for_ssh(self, mock_build_readiness) -> None:
|
||||
mock_build_readiness.return_value = {
|
||||
"execution_mode": "ssh",
|
||||
"summary": {
|
||||
"nodes_total": 2,
|
||||
"execution_ready_nodes": 1,
|
||||
"inspection_healthy_nodes": 2,
|
||||
},
|
||||
"rows": [],
|
||||
"blocking_reasons": ["有 1 台目标节点尚未配置完整 SSH 入口,当前不适合直接 SSH 发布。"],
|
||||
"warning_reasons": [],
|
||||
"recommendations": [],
|
||||
}
|
||||
|
||||
gate = build_release_rollout_gate(
|
||||
{
|
||||
"id": 11,
|
||||
"release_version": "2026.04.18-rc1",
|
||||
"status": "ready",
|
||||
"artifact_url": "https://example.com/domaincheck.tar.gz",
|
||||
},
|
||||
[{"node_code": "mainland-worker-01"}, {"node_code": "mainland-worker-02"}],
|
||||
execution_mode="ssh",
|
||||
)
|
||||
|
||||
self.assertEqual("ssh", gate["execution_mode"])
|
||||
self.assertEqual("SSH", gate["execution_mode_label"])
|
||||
self.assertEqual("blocked", gate["status"])
|
||||
self.assertIn("1/2 台 SSH 就绪", gate["summary"])
|
||||
|
||||
@patch("app.services.ops_release_service.build_rollout_target_operational_readiness")
|
||||
def test_build_release_rollout_gate_uses_human_label_for_remote_agent(self, mock_build_readiness) -> None:
|
||||
mock_build_readiness.return_value = {
|
||||
"execution_mode": "remote-agent",
|
||||
"summary": {
|
||||
"nodes_total": 2,
|
||||
"execution_ready_nodes": 1,
|
||||
"inspection_healthy_nodes": 2,
|
||||
},
|
||||
"rows": [],
|
||||
"blocking_reasons": ["有 1 台目标节点 Agent 未在线,当前不适合直接 remote-agent 发布。"],
|
||||
"warning_reasons": [],
|
||||
"recommendations": [],
|
||||
}
|
||||
|
||||
gate = build_release_rollout_gate(
|
||||
{
|
||||
"id": 12,
|
||||
"release_version": "2026.04.18-rc1",
|
||||
"status": "ready",
|
||||
"artifact_url": "https://example.com/domaincheck.tar.gz",
|
||||
},
|
||||
[{"node_code": "mainland-worker-01"}, {"node_code": "mainland-worker-02"}],
|
||||
execution_mode="remote-agent",
|
||||
)
|
||||
|
||||
self.assertEqual("remote-agent", gate["execution_mode"])
|
||||
self.assertEqual("远端 Agent", gate["execution_mode_label"])
|
||||
self.assertIn("1/2 台 远端 Agent 就绪", gate["summary"])
|
||||
|
||||
@patch("app.services.ops_release_service.build_release_rollout_gate")
|
||||
def test_build_release_execution_mode_recommendation_prefers_ssh_when_gate_is_better(self, mock_build_gate) -> None:
|
||||
def _fake_gate(release, target_nodes, *, execution_mode="remote-agent"):
|
||||
if execution_mode == "ssh":
|
||||
return {
|
||||
"status": "ready",
|
||||
"status_label": "可发 Rollout",
|
||||
"summary": "SSH 路径已收口",
|
||||
"execution_mode": "ssh",
|
||||
"execution_mode_label": "SSH",
|
||||
"blocking_reasons": [],
|
||||
"warning_reasons": [],
|
||||
"operational_readiness": {
|
||||
"summary": {"nodes_total": 2, "execution_ready_nodes": 2},
|
||||
"rows": [],
|
||||
},
|
||||
}
|
||||
return {
|
||||
"status": "blocked",
|
||||
"status_label": "存在阻断",
|
||||
"summary": "remote-agent 尚未收口",
|
||||
"execution_mode": "remote-agent",
|
||||
"execution_mode_label": "远端 Agent",
|
||||
"blocking_reasons": ["Agent 未就绪"],
|
||||
"warning_reasons": [],
|
||||
"operational_readiness": {
|
||||
"summary": {"nodes_total": 2, "execution_ready_nodes": 1},
|
||||
"rows": [],
|
||||
},
|
||||
}
|
||||
|
||||
mock_build_gate.side_effect = _fake_gate
|
||||
|
||||
recommendation = build_release_execution_mode_recommendation(
|
||||
{
|
||||
"id": 18,
|
||||
"release_version": "2026.04.18-rc2",
|
||||
"status": "ready",
|
||||
"artifact_url": "https://example.com/domaincheck.tar.gz",
|
||||
},
|
||||
[{"node_code": "mainland-worker-01"}, {"node_code": "mainland-worker-02"}],
|
||||
)
|
||||
|
||||
self.assertEqual("ssh", recommendation["recommended_mode"])
|
||||
self.assertEqual("SSH", recommendation["recommended_mode_label"])
|
||||
self.assertEqual("ready", recommendation["recommended_gate"]["status"])
|
||||
self.assertIn("SSH", recommendation["reason"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
212
domain-api/tests/test_ops_release_service_launchpad.py
Normal file
212
domain-api/tests/test_ops_release_service_launchpad.py
Normal file
@@ -0,0 +1,212 @@
|
||||
import unittest
|
||||
|
||||
from app.services.ops_release_service import (
|
||||
_build_release_launchpad_status,
|
||||
_compact_latest_smart_rollout_preview,
|
||||
)
|
||||
|
||||
|
||||
class OpsReleaseServiceLaunchpadTests(unittest.TestCase):
|
||||
def test_launchpad_status_blocks_when_package_missing(self) -> None:
|
||||
status = _build_release_launchpad_status(
|
||||
package={"available": False},
|
||||
latest_package_preview={},
|
||||
worker_preview={},
|
||||
control_preview={},
|
||||
)
|
||||
|
||||
self.assertEqual("blocked", status["status"])
|
||||
self.assertEqual("release_package", status["recommended_action_code"])
|
||||
self.assertEqual(status["summary"], status["summary_text"])
|
||||
self.assertEqual("release_hub", status["focus_ref"]["kind"])
|
||||
self.assertEqual("release_launchpad", status["focus_ref"]["section"])
|
||||
|
||||
def test_launchpad_status_blocks_when_rollout_preview_has_blockers(self) -> None:
|
||||
status = _build_release_launchpad_status(
|
||||
package={"available": True},
|
||||
latest_package_preview={},
|
||||
worker_preview={"policy_preview": {"blocked": True}},
|
||||
control_preview={"policy_preview": {"blocked": False}},
|
||||
)
|
||||
|
||||
self.assertEqual("blocked", status["status"])
|
||||
self.assertEqual("fix_rollout_blockers", status["recommended_action_code"])
|
||||
self.assertEqual(status["summary"], status["summary_text"])
|
||||
|
||||
def test_launchpad_status_blocks_when_final_signoff_missing(self) -> None:
|
||||
status = _build_release_launchpad_status(
|
||||
package={
|
||||
"available": True,
|
||||
"final_release_ok": False,
|
||||
"final_release_report_available": True,
|
||||
"final_release_report_ok": True,
|
||||
"final_release_report_matches_latest": False,
|
||||
"final_release_report_stale_reason": "final_release_report does not match latest_release: package_name",
|
||||
"final_release_gate_decision": "ready",
|
||||
"final_release_gate_blocked_reasons": [],
|
||||
},
|
||||
latest_package_preview={},
|
||||
worker_preview={"smart_rollout": {"available": True}, "policy_preview": {}},
|
||||
control_preview={"smart_rollout": {"available": True}, "policy_preview": {}},
|
||||
)
|
||||
|
||||
self.assertEqual("blocked", status["status"])
|
||||
self.assertEqual("release_prepare", status["recommended_action_code"])
|
||||
self.assertIn("final_release_report does not match latest_release: package_name", status["summary"])
|
||||
self.assertIn("release-prepare", status["recommended_command"])
|
||||
self.assertEqual(status["summary"], status["summary_text"])
|
||||
|
||||
def test_launchpad_status_prefers_runtime_refresh_when_runtime_schema_is_stale(self) -> None:
|
||||
status = _build_release_launchpad_status(
|
||||
package={"available": True, "final_release_ok": True},
|
||||
latest_package_preview={},
|
||||
worker_preview={"smart_rollout": {"available": True}, "policy_preview": {}},
|
||||
control_preview={"smart_rollout": {"available": True}, "policy_preview": {}},
|
||||
runtime_build_info={
|
||||
"runtime_schema_stale": True,
|
||||
"route_surface_complete": True,
|
||||
"route_surface_missing_keys": ["ops_node_handover_bootstrap_plan"],
|
||||
},
|
||||
)
|
||||
|
||||
self.assertEqual("attention", status["status"])
|
||||
self.assertEqual("api-restart", status["recommended_action_code"])
|
||||
self.assertEqual("待刷新运行时", status["status_label"])
|
||||
self.assertTrue(status["runtime_refresh_required"])
|
||||
self.assertTrue(status["runtime_schema_stale"])
|
||||
self.assertIn("runtime-refresh-recover", status["recommended_command"])
|
||||
self.assertEqual(status["summary"], status["summary_text"])
|
||||
|
||||
def test_launchpad_status_marks_attention_when_confirmation_required(self) -> None:
|
||||
status = _build_release_launchpad_status(
|
||||
package={"available": True},
|
||||
latest_package_preview={},
|
||||
worker_preview={"requires_confirmation": True, "policy_preview": {}},
|
||||
control_preview={"policy_preview": {}},
|
||||
)
|
||||
|
||||
self.assertEqual("attention", status["status"])
|
||||
self.assertEqual("review_smart_rollout_preview", status["recommended_action_code"])
|
||||
self.assertEqual("release_launchpad", status["focus_ref"]["section"])
|
||||
|
||||
def test_launchpad_status_ready_when_worker_rollout_available(self) -> None:
|
||||
status = _build_release_launchpad_status(
|
||||
package={"available": True},
|
||||
latest_package_preview={},
|
||||
worker_preview={"smart_rollout": {"available": True, "execution_mode": "ssh", "execution_mode_label": "SSH"}, "policy_preview": {}},
|
||||
control_preview={"smart_rollout": {"available": False}, "policy_preview": {}},
|
||||
)
|
||||
|
||||
self.assertEqual("ready", status["status"])
|
||||
self.assertEqual("publish_latest_worker", status["recommended_action_code"])
|
||||
self.assertEqual("ssh", status["recommended_execution_mode"])
|
||||
self.assertEqual("SSH", status["recommended_execution_mode_label"])
|
||||
self.assertEqual(status["summary"], status["summary_text"])
|
||||
|
||||
def test_launchpad_status_requires_managed_nodes_when_no_targets_available(self) -> None:
|
||||
status = _build_release_launchpad_status(
|
||||
package={"available": True},
|
||||
latest_package_preview={},
|
||||
worker_preview={"smart_rollout": {"available": False}, "policy_preview": {}},
|
||||
control_preview={"smart_rollout": {"available": False}, "policy_preview": {}},
|
||||
)
|
||||
|
||||
self.assertEqual("attention", status["status"])
|
||||
self.assertEqual("fix_managed_nodes", status["recommended_action_code"])
|
||||
self.assertEqual("release_hub", status["focus_ref"]["kind"])
|
||||
|
||||
def test_launchpad_status_prefers_bootstrap_recovery_when_gap_row_has_recovery_action(self) -> None:
|
||||
status = _build_release_launchpad_status(
|
||||
package={"available": True},
|
||||
latest_package_preview={},
|
||||
worker_preview={
|
||||
"smart_rollout": {"available": False},
|
||||
"policy_preview": {
|
||||
"blocked": True,
|
||||
"release_gate": {
|
||||
"operational_readiness": {
|
||||
"rows": [
|
||||
{
|
||||
"node_code": "mainland-worker-02",
|
||||
"recovery_action": "bootstrap_run",
|
||||
"recovery_label": "签发接入工单",
|
||||
"recovery_summary": "当前节点仍处于接入阶段,建议先签发 onboarding.bootstrap。",
|
||||
}
|
||||
]
|
||||
}
|
||||
},
|
||||
},
|
||||
},
|
||||
control_preview={"smart_rollout": {"available": False}, "policy_preview": {}},
|
||||
)
|
||||
|
||||
self.assertEqual("blocked", status["status"])
|
||||
self.assertEqual("bootstrap_run", status["recommended_action_code"])
|
||||
self.assertEqual("mainland-worker-02", status["recommended_target_node_code"])
|
||||
self.assertEqual("签发接入工单", status["recommended_recovery_label"])
|
||||
self.assertEqual("当前节点仍处于接入阶段,建议先签发 onboarding.bootstrap。", status["recommended_recovery_summary"])
|
||||
self.assertEqual(1, status["onboarding_bootstrap_pending_nodes"])
|
||||
self.assertEqual(0, status["onboarding_acceptance_ready_nodes"])
|
||||
self.assertEqual(1, status["onboarding_gap_nodes_total"])
|
||||
self.assertIn("node-bootstrap-run", status["recommended_command"])
|
||||
|
||||
def test_compact_latest_smart_rollout_preview_adds_rollout_preview_alias(self) -> None:
|
||||
preview = _compact_latest_smart_rollout_preview(
|
||||
"智能 Rollout 预检通过",
|
||||
{
|
||||
"release": {
|
||||
"id": 18,
|
||||
"release_version": "2026.04.18+bd6bcb2",
|
||||
"channel": "stable",
|
||||
"status": "ready",
|
||||
"artifact_url": "https://example.com/domaincheck.tar.gz",
|
||||
},
|
||||
"package": {
|
||||
"available": True,
|
||||
"package_name": "domaincheck-2026.04.18+bd6bcb2",
|
||||
"package_type": "tar.gz",
|
||||
"sha256": "sha256:test",
|
||||
"metadata": {
|
||||
"manifest": {
|
||||
"included_services": ["domaincheck-api", "domaincheck-worker"],
|
||||
}
|
||||
},
|
||||
},
|
||||
"smart_rollout": {
|
||||
"available": True,
|
||||
"execution_mode": "remote-agent",
|
||||
"execution_mode_label": "远端 Agent",
|
||||
"summary": "已按 Worker 预选 2 台节点。",
|
||||
"target_selector": {"node_codes": ["mainland-worker-01", "mainland-worker-02"]},
|
||||
"target_nodes": [
|
||||
{
|
||||
"node_code": "mainland-worker-01",
|
||||
"region": "mainland",
|
||||
"role": "worker",
|
||||
"delivery_queue_state": "healthy",
|
||||
},
|
||||
{
|
||||
"node_code": "mainland-worker-02",
|
||||
"region": "mainland",
|
||||
"role": "worker",
|
||||
"delivery_queue_state": "retrying",
|
||||
},
|
||||
],
|
||||
},
|
||||
"preview": {
|
||||
"target_summary": {"nodes_total": 2},
|
||||
"batch_plan": {"batches_total": 2, "first_batch_size": 1},
|
||||
"release_gate": {"status": "ready", "summary": "当前可推进。"},
|
||||
},
|
||||
},
|
||||
)
|
||||
|
||||
self.assertEqual("remote-agent", preview["rollout_preview"]["execution_mode"])
|
||||
self.assertEqual(2, preview["rollout_preview"]["target_nodes_total"])
|
||||
self.assertEqual(2, preview["rollout_preview"]["expected_batches_total"])
|
||||
self.assertEqual(2, preview["rollout_preview"]["expected_jobs_total"])
|
||||
self.assertEqual("release_hub", preview["rollout_preview"]["focus_ref"]["kind"])
|
||||
self.assertEqual("healthy", preview["smart_rollout"]["target_nodes"][0]["delivery_queue_state"])
|
||||
self.assertEqual("retrying", preview["smart_rollout"]["target_nodes"][1]["delivery_queue_state"])
|
||||
self.assertEqual("rollout_preview", preview["rollout_preview"]["focus_ref"]["section"])
|
||||
self.assertEqual(["domaincheck-api", "domaincheck-worker"], preview["package"]["manifest"]["included_services"])
|
||||
915
domain-api/tests/test_ops_release_service_package_metadata.py
Normal file
915
domain-api/tests/test_ops_release_service_package_metadata.py
Normal file
@@ -0,0 +1,915 @@
|
||||
import json
|
||||
import os
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
from app.services.ops_release_service import (
|
||||
build_smart_release_rollout_plan,
|
||||
create_release_and_smart_rollout_from_latest_package,
|
||||
create_release_from_latest_package,
|
||||
create_smart_release_rollout,
|
||||
get_latest_release_package_metadata,
|
||||
get_release_package_archive_path,
|
||||
get_release_package_sha256_path,
|
||||
)
|
||||
|
||||
|
||||
class OpsReleaseServicePackageMetadataTests(unittest.TestCase):
|
||||
def test_get_latest_release_package_metadata_reads_latest_linux_package(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
release_root = Path(tmpdir)
|
||||
package_name = "domaincheck_release_20260418_120000"
|
||||
staging_path = release_root / package_name
|
||||
archive_path = release_root / f"{package_name}.tar.gz"
|
||||
sha256_path = release_root / f"{package_name}.sha256.txt"
|
||||
|
||||
staging_path.mkdir(parents=True, exist_ok=True)
|
||||
archive_path.write_bytes(b"linux-package")
|
||||
sha256_path.write_text("sha256=test-sha256", encoding="utf-8")
|
||||
(staging_path / "release_manifest.json").write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"package_name": package_name,
|
||||
"package_type": "tar.gz",
|
||||
"included": {
|
||||
"docs": ["08_domainCheck_交付说明.md"],
|
||||
"scripts": ["package_domain_release.sh", "verify_domain_release.sh"],
|
||||
},
|
||||
},
|
||||
ensure_ascii=False,
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
(release_root / "latest_release.json").write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"package_name": package_name,
|
||||
"package_type": "tar.gz",
|
||||
"generated_at": "2026-04-18T12:00:00",
|
||||
"staging_path": str(staging_path),
|
||||
"archive_path": str(archive_path),
|
||||
"sha256_path": str(sha256_path),
|
||||
"sha256": "test-sha256",
|
||||
"smoke_test_ok": True,
|
||||
},
|
||||
ensure_ascii=False,
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
(release_root / "final_release_report.json").write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"ok": True,
|
||||
"latest_release": {
|
||||
"package_name": package_name,
|
||||
"archive_path": str(archive_path),
|
||||
"sha256": "test-sha256",
|
||||
},
|
||||
"release_gate": {
|
||||
"decision": "ready",
|
||||
"blocked_reasons": [],
|
||||
"verify_ok": True,
|
||||
"smoke_test_ok": True,
|
||||
},
|
||||
},
|
||||
ensure_ascii=False,
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
with patch.dict(os.environ, {"DOMAINCHECK_RELEASE_ROOT": str(release_root)}):
|
||||
data = get_latest_release_package_metadata()
|
||||
|
||||
self.assertTrue(data["available"])
|
||||
self.assertEqual(package_name, data["package_name"])
|
||||
self.assertEqual("tar.gz", data["package_type"])
|
||||
self.assertTrue(data["archive_present"])
|
||||
self.assertTrue(data["sha256_present"])
|
||||
self.assertTrue(data["staging_present"])
|
||||
self.assertEqual("test-sha256", data["sha256"])
|
||||
self.assertEqual(f"/api/v1/ops/releases/packages/{package_name}/download", data["download_path"])
|
||||
self.assertEqual(package_name, data["release_version_suggestion"])
|
||||
self.assertIn("自检通过", data["notes_suggestion"])
|
||||
self.assertTrue(data["final_release_ok"])
|
||||
self.assertTrue(data["final_release_report_available"])
|
||||
self.assertTrue(data["final_release_report_matches_latest"])
|
||||
self.assertEqual("ready", data["final_release_gate_decision"])
|
||||
self.assertEqual([], data["final_release_gate_blocked_reasons"])
|
||||
self.assertEqual(package_name, data["metadata"]["manifest"]["package_name"])
|
||||
|
||||
def test_get_release_package_paths_support_metadata_fallback_for_zip(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
release_root = Path(tmpdir)
|
||||
package_name = "domaincheck_release_windows_demo"
|
||||
package_dir = release_root / "exports"
|
||||
package_dir.mkdir(parents=True, exist_ok=True)
|
||||
archive_path = package_dir / f"{package_name}.zip"
|
||||
sha256_path = package_dir / f"{package_name}.sha256.txt"
|
||||
|
||||
archive_path.write_bytes(b"zip-package")
|
||||
sha256_path.write_text("sha256=zip-sha256", encoding="utf-8")
|
||||
(release_root / "latest_release.json").write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"package_name": package_name,
|
||||
"package_type": "zip",
|
||||
"generated_at": "2026-04-18T12:10:00",
|
||||
"zip_path": str(archive_path),
|
||||
"sha256_path": str(sha256_path),
|
||||
"sha256": "zip-sha256",
|
||||
"smoke_test_ok": False,
|
||||
},
|
||||
ensure_ascii=False,
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
with patch.dict(os.environ, {"DOMAINCHECK_RELEASE_ROOT": str(release_root)}):
|
||||
resolved_archive = get_release_package_archive_path(package_name)
|
||||
resolved_sha256 = get_release_package_sha256_path(package_name)
|
||||
metadata = get_latest_release_package_metadata()
|
||||
|
||||
self.assertEqual(archive_path, resolved_archive)
|
||||
self.assertEqual(sha256_path, resolved_sha256)
|
||||
self.assertEqual("zip", metadata["package_type"])
|
||||
self.assertEqual(archive_path.name, metadata["archive_filename"])
|
||||
self.assertFalse(metadata["smoke_test_ok"])
|
||||
|
||||
def test_get_latest_release_package_metadata_returns_unavailable_when_missing(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
with patch.dict(os.environ, {"DOMAINCHECK_RELEASE_ROOT": tmpdir}):
|
||||
data = get_latest_release_package_metadata()
|
||||
|
||||
self.assertFalse(data["available"])
|
||||
self.assertIn("latest_release.json", data["reason"])
|
||||
|
||||
def test_get_latest_release_package_metadata_marks_stale_final_release_report(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
release_root = Path(tmpdir)
|
||||
package_name = "domaincheck_release_20260418_130000"
|
||||
archive_path = release_root / f"{package_name}.tar.gz"
|
||||
sha256_path = release_root / f"{package_name}.sha256.txt"
|
||||
|
||||
archive_path.write_bytes(b"linux-package")
|
||||
sha256_path.write_text("sha256=test-sha256", encoding="utf-8")
|
||||
(release_root / "latest_release.json").write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"package_name": package_name,
|
||||
"package_type": "tar.gz",
|
||||
"archive_path": str(archive_path),
|
||||
"sha256_path": str(sha256_path),
|
||||
"sha256": "test-sha256",
|
||||
"smoke_test_ok": True,
|
||||
},
|
||||
ensure_ascii=False,
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
(release_root / "final_release_report.json").write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"ok": True,
|
||||
"latest_release": {
|
||||
"package_name": "domaincheck_release_20260418_old",
|
||||
"archive_path": str(release_root / "domaincheck_release_20260418_old.tar.gz"),
|
||||
"sha256": "old-sha256",
|
||||
},
|
||||
"release_gate": {
|
||||
"decision": "ready",
|
||||
"blocked_reasons": [],
|
||||
"verify_ok": True,
|
||||
"smoke_test_ok": True,
|
||||
},
|
||||
},
|
||||
ensure_ascii=False,
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
with patch.dict(os.environ, {"DOMAINCHECK_RELEASE_ROOT": str(release_root)}):
|
||||
data = get_latest_release_package_metadata()
|
||||
|
||||
self.assertTrue(data["available"])
|
||||
self.assertFalse(data["final_release_ok"])
|
||||
self.assertTrue(data["final_release_report_available"])
|
||||
self.assertTrue(data["final_release_report_ok"])
|
||||
self.assertFalse(data["final_release_report_matches_latest"])
|
||||
self.assertIn("package_name", data["final_release_report_stale_reason"])
|
||||
|
||||
@patch("app.services.ops_release_service.create_release")
|
||||
@patch("app.services.ops_release_service._get_release_by_version")
|
||||
@patch("app.services.ops_release_service.get_latest_release_package_metadata")
|
||||
def test_create_release_from_latest_package_builds_payload(
|
||||
self,
|
||||
mock_get_latest_release_package_metadata,
|
||||
mock_get_release_by_version,
|
||||
mock_create_release,
|
||||
) -> None:
|
||||
mock_get_latest_release_package_metadata.return_value = {
|
||||
"available": True,
|
||||
"package_name": "domaincheck_release_20260418_120000",
|
||||
"release_version_suggestion": "domaincheck_release_20260418_120000",
|
||||
"download_path": "/api/v1/ops/releases/packages/domaincheck_release_20260418_120000/download",
|
||||
"sha256_download_path": "/api/v1/ops/releases/packages/domaincheck_release_20260418_120000/sha256",
|
||||
"sha256": "test-sha256",
|
||||
"commit_sha": "bd6bcb2",
|
||||
"notes_suggestion": "导入自本机发布包 domaincheck_release_20260418_120000.tar.gz;自检通过",
|
||||
"smoke_test_ok": True,
|
||||
"final_release_ok": True,
|
||||
"final_release_report_available": True,
|
||||
"final_release_report_ok": True,
|
||||
"final_release_report_matches_latest": True,
|
||||
"final_release_gate_decision": "ready",
|
||||
"final_release_gate_blocked_reasons": [],
|
||||
"metadata": {
|
||||
"package_name": "domaincheck_release_20260418_120000",
|
||||
"package_type": "tar.gz",
|
||||
},
|
||||
}
|
||||
mock_get_release_by_version.return_value = {}
|
||||
mock_create_release.return_value = (
|
||||
True,
|
||||
"Release 已创建",
|
||||
{
|
||||
"release": {
|
||||
"id": 7,
|
||||
"release_version": "domaincheck_release_20260418_120000",
|
||||
}
|
||||
},
|
||||
)
|
||||
|
||||
ok, message, data = create_release_from_latest_package(
|
||||
{
|
||||
"channel": "stable",
|
||||
"status": "ready",
|
||||
"created_by": "test-suite",
|
||||
},
|
||||
control_plane_base_url="https://ops.example.com/api/v1",
|
||||
)
|
||||
|
||||
self.assertTrue(ok)
|
||||
self.assertEqual("Release 已创建", message)
|
||||
mock_create_release.assert_called_once()
|
||||
create_payload = mock_create_release.call_args.args[0]
|
||||
self.assertEqual("domaincheck_release_20260418_120000", create_payload["release_version"])
|
||||
self.assertEqual("bd6bcb2", create_payload["commit_sha"])
|
||||
self.assertEqual(
|
||||
"https://ops.example.com/api/v1/ops/releases/packages/domaincheck_release_20260418_120000/download",
|
||||
create_payload["artifact_url"],
|
||||
)
|
||||
self.assertEqual("test-sha256", create_payload["checksum"])
|
||||
self.assertEqual("latest_release_package", create_payload["metadata"]["created_from"])
|
||||
self.assertEqual(
|
||||
"https://ops.example.com/api/v1/ops/releases/packages/domaincheck_release_20260418_120000/download",
|
||||
create_payload["metadata"]["package_source"]["download_url"],
|
||||
)
|
||||
self.assertFalse(data["deduplicated"])
|
||||
|
||||
@patch("app.services.ops_release_service.create_release")
|
||||
@patch("app.services.ops_release_service._get_release_by_version")
|
||||
@patch("app.services.ops_release_service.get_latest_release_package_metadata")
|
||||
def test_create_release_from_latest_package_is_idempotent(
|
||||
self,
|
||||
mock_get_latest_release_package_metadata,
|
||||
mock_get_release_by_version,
|
||||
mock_create_release,
|
||||
) -> None:
|
||||
mock_get_latest_release_package_metadata.return_value = {
|
||||
"available": True,
|
||||
"package_name": "domaincheck_release_20260418_120000",
|
||||
"release_version_suggestion": "domaincheck_release_20260418_120000",
|
||||
"smoke_test_ok": True,
|
||||
"final_release_ok": True,
|
||||
"final_release_report_available": True,
|
||||
"final_release_report_ok": True,
|
||||
"final_release_report_matches_latest": True,
|
||||
"final_release_gate_decision": "ready",
|
||||
"final_release_gate_blocked_reasons": [],
|
||||
}
|
||||
mock_get_release_by_version.return_value = {
|
||||
"id": 9,
|
||||
"release_version": "domaincheck_release_20260418_120000",
|
||||
}
|
||||
|
||||
ok, message, data = create_release_from_latest_package({}, control_plane_base_url="https://ops.example.com")
|
||||
|
||||
self.assertTrue(ok)
|
||||
self.assertEqual("Release 已存在,按幂等返回", message)
|
||||
self.assertTrue(data["deduplicated"])
|
||||
self.assertEqual(9, data["release"]["id"])
|
||||
mock_create_release.assert_not_called()
|
||||
|
||||
@patch("app.services.ops_release_service.create_release")
|
||||
@patch("app.services.ops_release_service._get_release_by_version")
|
||||
@patch("app.services.ops_release_service.get_latest_release_package_metadata")
|
||||
def test_create_release_from_latest_package_blocks_when_final_signoff_missing(
|
||||
self,
|
||||
mock_get_latest_release_package_metadata,
|
||||
mock_get_release_by_version,
|
||||
mock_create_release,
|
||||
) -> None:
|
||||
mock_get_latest_release_package_metadata.return_value = {
|
||||
"available": True,
|
||||
"package_name": "domaincheck_release_20260418_130000",
|
||||
"release_version_suggestion": "domaincheck_release_20260418_130000",
|
||||
"download_path": "/api/v1/ops/releases/packages/domaincheck_release_20260418_130000/download",
|
||||
"sha256_download_path": "/api/v1/ops/releases/packages/domaincheck_release_20260418_130000/sha256",
|
||||
"sha256": "test-sha256",
|
||||
"commit_sha": "bd6bcb2",
|
||||
"smoke_test_ok": False,
|
||||
"final_release_ok": False,
|
||||
"final_release_report_available": False,
|
||||
"final_release_report_ok": None,
|
||||
"final_release_report_matches_latest": None,
|
||||
"final_release_gate_decision": "",
|
||||
"final_release_gate_blocked_reasons": [],
|
||||
"metadata": {
|
||||
"package_name": "domaincheck_release_20260418_130000",
|
||||
"package_type": "tar.gz",
|
||||
},
|
||||
}
|
||||
mock_get_release_by_version.return_value = {}
|
||||
|
||||
ok, message, data = create_release_from_latest_package(
|
||||
{
|
||||
"channel": "stable",
|
||||
"status": "ready",
|
||||
"created_by": "test-suite",
|
||||
},
|
||||
control_plane_base_url="https://ops.example.com/api/v1",
|
||||
)
|
||||
|
||||
self.assertFalse(ok)
|
||||
self.assertIn("最终签收", message)
|
||||
self.assertFalse(data["final_release_gate"]["ready"])
|
||||
self.assertIn("smoke_test_failed_or_skipped", data["final_release_gate"]["blocked_reasons"])
|
||||
self.assertIn("final_release_report_missing", data["final_release_gate"]["blocked_reasons"])
|
||||
mock_create_release.assert_not_called()
|
||||
|
||||
@patch("app.services.ops_release_service.create_release")
|
||||
@patch("app.services.ops_release_service._get_release_by_version")
|
||||
@patch("app.services.ops_release_service.get_latest_release_package_metadata")
|
||||
def test_create_release_from_latest_package_blocks_when_final_report_is_stale(
|
||||
self,
|
||||
mock_get_latest_release_package_metadata,
|
||||
mock_get_release_by_version,
|
||||
mock_create_release,
|
||||
) -> None:
|
||||
mock_get_latest_release_package_metadata.return_value = {
|
||||
"available": True,
|
||||
"package_name": "domaincheck_release_20260418_130001",
|
||||
"release_version_suggestion": "domaincheck_release_20260418_130001",
|
||||
"download_path": "/api/v1/ops/releases/packages/domaincheck_release_20260418_130001/download",
|
||||
"sha256_download_path": "/api/v1/ops/releases/packages/domaincheck_release_20260418_130001/sha256",
|
||||
"sha256": "test-sha256",
|
||||
"commit_sha": "bd6bcb2",
|
||||
"smoke_test_ok": True,
|
||||
"final_release_ok": False,
|
||||
"final_release_report_available": True,
|
||||
"final_release_report_ok": True,
|
||||
"final_release_report_matches_latest": False,
|
||||
"final_release_report_stale_reason": "final_release_report does not match latest_release: package_name",
|
||||
"final_release_gate_decision": "ready",
|
||||
"final_release_gate_blocked_reasons": [],
|
||||
"metadata": {
|
||||
"package_name": "domaincheck_release_20260418_130001",
|
||||
"package_type": "tar.gz",
|
||||
},
|
||||
}
|
||||
mock_get_release_by_version.return_value = {
|
||||
"id": 19,
|
||||
"release_version": "domaincheck_release_20260418_130001",
|
||||
}
|
||||
|
||||
ok, message, data = create_release_from_latest_package(
|
||||
{"created_by": "test-suite"},
|
||||
control_plane_base_url="https://ops.example.com/api/v1",
|
||||
)
|
||||
|
||||
self.assertFalse(ok)
|
||||
self.assertIn("最终签收", message)
|
||||
self.assertFalse(data["final_release_gate"]["ready"])
|
||||
self.assertEqual(
|
||||
"final_release_report does not match latest_release: package_name",
|
||||
data["final_release_gate"]["stale_reason"],
|
||||
)
|
||||
self.assertIn("final_release_report_not_matching_latest", data["final_release_gate"]["blocked_reasons"])
|
||||
self.assertEqual(19, data["release"]["id"])
|
||||
mock_create_release.assert_not_called()
|
||||
|
||||
@patch("app.services.ops_agent_service.list_managed_nodes_with_agent_state")
|
||||
def test_build_smart_release_rollout_plan_prefers_remote_ready_workers(
|
||||
self,
|
||||
mock_list_managed_nodes_with_agent_state,
|
||||
) -> None:
|
||||
mock_list_managed_nodes_with_agent_state.return_value = {
|
||||
"nodes": [
|
||||
{
|
||||
"node_code": "mainland-worker-01",
|
||||
"region": "mainland",
|
||||
"role": "worker",
|
||||
"is_enabled": True,
|
||||
"is_managed": True,
|
||||
"is_agent_online": True,
|
||||
"cluster_status": "online",
|
||||
"cluster_current_load": 0,
|
||||
"cluster_is_effective_worker": True,
|
||||
"cluster_detect_participating": False,
|
||||
},
|
||||
{
|
||||
"node_code": "mainland-controller-01",
|
||||
"region": "mainland",
|
||||
"role": "control",
|
||||
"is_enabled": True,
|
||||
"is_managed": True,
|
||||
"is_agent_online": True,
|
||||
"cluster_status": "online",
|
||||
"cluster_current_load": 1,
|
||||
"cluster_is_effective_worker": True,
|
||||
"cluster_detect_participating": True,
|
||||
},
|
||||
{
|
||||
"node_code": "mainland-worker-02",
|
||||
"region": "mainland",
|
||||
"role": "worker",
|
||||
"is_enabled": True,
|
||||
"is_managed": True,
|
||||
"is_agent_online": False,
|
||||
"cluster_status": "online",
|
||||
"cluster_current_load": 0,
|
||||
"cluster_is_effective_worker": True,
|
||||
"cluster_detect_participating": False,
|
||||
},
|
||||
]
|
||||
}
|
||||
|
||||
worker_plan = build_smart_release_rollout_plan("worker")
|
||||
control_plan = build_smart_release_rollout_plan("control")
|
||||
|
||||
self.assertTrue(worker_plan["available"])
|
||||
self.assertEqual("worker", worker_plan["mode"])
|
||||
self.assertEqual(["mainland-worker-01"], worker_plan["target_selector"]["node_codes"])
|
||||
self.assertEqual(["domaincheck-worker"], worker_plan["policy"]["deploy_payload"]["restart_services"])
|
||||
|
||||
self.assertTrue(control_plan["available"])
|
||||
self.assertEqual("control", control_plan["mode"])
|
||||
self.assertEqual(["mainland-controller-01"], control_plan["target_selector"]["node_codes"])
|
||||
self.assertIn("domaincheck-api", control_plan["policy"]["deploy_payload"]["restart_services"])
|
||||
self.assertEqual("remote-agent", worker_plan["policy"]["execution_mode"])
|
||||
|
||||
@patch("app.services.ops_agent_service.list_managed_nodes_with_agent_state")
|
||||
def test_build_smart_release_rollout_plan_falls_back_to_ssh_ready_workers(
|
||||
self,
|
||||
mock_list_managed_nodes_with_agent_state,
|
||||
) -> None:
|
||||
mock_list_managed_nodes_with_agent_state.return_value = {
|
||||
"nodes": [
|
||||
{
|
||||
"node_code": "mainland-worker-01",
|
||||
"region": "mainland",
|
||||
"role": "worker",
|
||||
"is_enabled": True,
|
||||
"is_managed": True,
|
||||
"is_agent_online": False,
|
||||
"has_ssh_access": True,
|
||||
"cluster_status": "online",
|
||||
"cluster_current_load": 0,
|
||||
"cluster_is_effective_worker": True,
|
||||
"cluster_detect_participating": False,
|
||||
},
|
||||
{
|
||||
"node_code": "mainland-worker-02",
|
||||
"region": "mainland",
|
||||
"role": "worker",
|
||||
"is_enabled": True,
|
||||
"is_managed": True,
|
||||
"is_agent_online": False,
|
||||
"has_ssh_access": True,
|
||||
"cluster_status": "busy",
|
||||
"cluster_current_load": 1,
|
||||
"cluster_is_effective_worker": True,
|
||||
"cluster_detect_participating": True,
|
||||
},
|
||||
]
|
||||
}
|
||||
|
||||
worker_plan = build_smart_release_rollout_plan("worker")
|
||||
|
||||
self.assertTrue(worker_plan["available"])
|
||||
self.assertEqual("worker", worker_plan["mode"])
|
||||
self.assertEqual("ssh", worker_plan["execution_mode"])
|
||||
self.assertEqual("SSH", worker_plan["execution_mode_label"])
|
||||
self.assertEqual(
|
||||
["mainland-worker-01", "mainland-worker-02"],
|
||||
worker_plan["target_selector"]["node_codes"],
|
||||
)
|
||||
self.assertEqual("ssh", worker_plan["policy"]["execution_mode"])
|
||||
|
||||
@patch("app.services.ops_agent_service.list_managed_nodes_with_agent_state")
|
||||
def test_build_smart_release_rollout_plan_excludes_dead_letter_nodes_from_ready_candidates(
|
||||
self,
|
||||
mock_list_managed_nodes_with_agent_state,
|
||||
) -> None:
|
||||
mock_list_managed_nodes_with_agent_state.return_value = {
|
||||
"nodes": [
|
||||
{
|
||||
"node_code": "mainland-worker-01",
|
||||
"region": "mainland",
|
||||
"role": "worker",
|
||||
"is_enabled": True,
|
||||
"is_managed": True,
|
||||
"is_agent_online": True,
|
||||
"has_ssh_access": True,
|
||||
"delivery_queue_state": "dead_letter",
|
||||
"cluster_status": "online",
|
||||
"cluster_current_load": 0,
|
||||
"cluster_is_effective_worker": True,
|
||||
"cluster_detect_participating": False,
|
||||
},
|
||||
{
|
||||
"node_code": "mainland-worker-02",
|
||||
"region": "mainland",
|
||||
"role": "worker",
|
||||
"is_enabled": True,
|
||||
"is_managed": True,
|
||||
"is_agent_online": True,
|
||||
"has_ssh_access": True,
|
||||
"delivery_queue_state": "healthy",
|
||||
"cluster_status": "online",
|
||||
"cluster_current_load": 0,
|
||||
"cluster_is_effective_worker": True,
|
||||
"cluster_detect_participating": False,
|
||||
},
|
||||
]
|
||||
}
|
||||
|
||||
worker_plan = build_smart_release_rollout_plan("worker")
|
||||
|
||||
self.assertTrue(worker_plan["available"])
|
||||
self.assertEqual(["mainland-worker-02"], worker_plan["target_selector"]["node_codes"])
|
||||
self.assertEqual("healthy", worker_plan["target_nodes"][0]["delivery_queue_state"])
|
||||
|
||||
@patch("app.services.ops_agent_service.list_managed_nodes_with_agent_state")
|
||||
def test_build_smart_release_rollout_plan_returns_unavailable_when_only_dead_letter_nodes_exist(
|
||||
self,
|
||||
mock_list_managed_nodes_with_agent_state,
|
||||
) -> None:
|
||||
mock_list_managed_nodes_with_agent_state.return_value = {
|
||||
"nodes": [
|
||||
{
|
||||
"node_code": "mainland-worker-01",
|
||||
"region": "mainland",
|
||||
"role": "worker",
|
||||
"is_enabled": True,
|
||||
"is_managed": True,
|
||||
"is_agent_online": True,
|
||||
"has_ssh_access": True,
|
||||
"delivery_queue_state": "dead_letter",
|
||||
"cluster_status": "online",
|
||||
"cluster_current_load": 0,
|
||||
"cluster_is_effective_worker": True,
|
||||
"cluster_detect_participating": False,
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
worker_plan = build_smart_release_rollout_plan("worker")
|
||||
|
||||
self.assertFalse(worker_plan["available"])
|
||||
self.assertIn("死信回执", worker_plan["reason"])
|
||||
self.assertEqual("dead_letter", worker_plan["target_nodes"][0]["delivery_queue_state"])
|
||||
|
||||
@patch("app.services.ops_release_service.create_release_rollout")
|
||||
@patch("app.services.ops_policy_service.preview_ops_job_policy")
|
||||
@patch("app.services.ops_agent_service.list_managed_nodes_with_agent_state")
|
||||
@patch("app.services.ops_release_service.create_release_from_latest_package")
|
||||
def test_create_release_and_smart_rollout_stops_when_latest_package_gate_blocks(
|
||||
self,
|
||||
mock_create_release_from_latest_package,
|
||||
mock_list_managed_nodes_with_agent_state,
|
||||
mock_preview_ops_job_policy,
|
||||
mock_create_release_rollout,
|
||||
) -> None:
|
||||
mock_create_release_from_latest_package.return_value = (
|
||||
False,
|
||||
"最新发布包尚未完成最终签收,暂不允许创建 Release 或发起基于最新包的智能 Rollout。",
|
||||
{
|
||||
"package": {
|
||||
"available": True,
|
||||
"package_name": "domaincheck_release_20260418_blocked",
|
||||
"final_release_ok": False,
|
||||
},
|
||||
"final_release_gate": {
|
||||
"ready": False,
|
||||
"status": "blocked",
|
||||
"blocked_reasons": ["final_release_report_missing"],
|
||||
},
|
||||
},
|
||||
)
|
||||
|
||||
ok, message, data = create_release_and_smart_rollout_from_latest_package(
|
||||
{"mode": "worker"},
|
||||
control_plane_base_url="https://ops.example.com",
|
||||
)
|
||||
|
||||
self.assertFalse(ok)
|
||||
self.assertIn("最终签收", message)
|
||||
self.assertFalse(data["final_release_gate"]["ready"])
|
||||
mock_list_managed_nodes_with_agent_state.assert_not_called()
|
||||
mock_preview_ops_job_policy.assert_not_called()
|
||||
mock_create_release_rollout.assert_not_called()
|
||||
|
||||
@patch("app.services.ops_release_service.create_release_rollout")
|
||||
@patch("app.services.ops_policy_service.preview_ops_job_policy")
|
||||
@patch("app.services.ops_agent_service.list_managed_nodes_with_agent_state")
|
||||
@patch("app.services.ops_release_service.create_release_from_latest_package")
|
||||
def test_create_release_and_smart_rollout_requires_confirmation_before_rollout(
|
||||
self,
|
||||
mock_create_release_from_latest_package,
|
||||
mock_list_managed_nodes_with_agent_state,
|
||||
mock_preview_ops_job_policy,
|
||||
mock_create_release_rollout,
|
||||
) -> None:
|
||||
mock_create_release_from_latest_package.return_value = (
|
||||
True,
|
||||
"Release 已创建",
|
||||
{
|
||||
"release": {"id": 21, "release_version": "domaincheck_release_20260418_120000"},
|
||||
"deduplicated": False,
|
||||
},
|
||||
)
|
||||
mock_list_managed_nodes_with_agent_state.return_value = {
|
||||
"nodes": [
|
||||
{
|
||||
"node_code": "mainland-controller-01",
|
||||
"region": "mainland",
|
||||
"role": "control",
|
||||
"is_enabled": True,
|
||||
"is_managed": True,
|
||||
"is_agent_online": True,
|
||||
"cluster_status": "online",
|
||||
"cluster_current_load": 0,
|
||||
"cluster_is_effective_worker": True,
|
||||
"cluster_detect_participating": False,
|
||||
}
|
||||
]
|
||||
}
|
||||
mock_preview_ops_job_policy.return_value = {
|
||||
"blocked": False,
|
||||
"approval_required": True,
|
||||
"warnings": ["本次 rollout 覆盖 control 节点,建议审批。"],
|
||||
"release_gate": {"status_label": "待补巡检", "summary": "建议先确认控制面健康。"},
|
||||
}
|
||||
|
||||
ok, message, data = create_release_and_smart_rollout_from_latest_package(
|
||||
{"mode": "control"},
|
||||
control_plane_base_url="https://ops.example.com",
|
||||
)
|
||||
|
||||
self.assertTrue(ok)
|
||||
self.assertEqual("Release 已创建,智能 Rollout 需要确认", message)
|
||||
self.assertTrue(data["requires_confirmation"])
|
||||
self.assertFalse(data["rollout_created"])
|
||||
self.assertEqual(["mainland-controller-01"], data["smart_rollout"]["target_selector"]["node_codes"])
|
||||
mock_create_release_rollout.assert_not_called()
|
||||
|
||||
@patch("app.services.ops_release_service.create_release_rollout")
|
||||
@patch("app.services.ops_policy_service.preview_ops_job_policy")
|
||||
@patch("app.services.ops_agent_service.list_managed_nodes_with_agent_state")
|
||||
@patch("app.services.ops_release_service.create_release_from_latest_package")
|
||||
def test_create_release_and_smart_rollout_creates_rollout_after_confirmation(
|
||||
self,
|
||||
mock_create_release_from_latest_package,
|
||||
mock_list_managed_nodes_with_agent_state,
|
||||
mock_preview_ops_job_policy,
|
||||
mock_create_release_rollout,
|
||||
) -> None:
|
||||
mock_create_release_from_latest_package.return_value = (
|
||||
True,
|
||||
"Release 已创建",
|
||||
{
|
||||
"release": {"id": 22, "release_version": "domaincheck_release_20260418_120000"},
|
||||
"deduplicated": True,
|
||||
},
|
||||
)
|
||||
mock_list_managed_nodes_with_agent_state.return_value = {
|
||||
"nodes": [
|
||||
{
|
||||
"node_code": "mainland-worker-01",
|
||||
"region": "mainland",
|
||||
"role": "worker",
|
||||
"is_enabled": True,
|
||||
"is_managed": True,
|
||||
"is_agent_online": True,
|
||||
"cluster_status": "online",
|
||||
"cluster_current_load": 0,
|
||||
"cluster_is_effective_worker": True,
|
||||
"cluster_detect_participating": False,
|
||||
}
|
||||
]
|
||||
}
|
||||
mock_preview_ops_job_policy.return_value = {
|
||||
"blocked": False,
|
||||
"approval_required": False,
|
||||
"warnings": [],
|
||||
"release_gate": {"status_label": "可发 Rollout", "summary": "当前可推进。"},
|
||||
}
|
||||
mock_create_release_rollout.return_value = (
|
||||
True,
|
||||
"Release rollout 已创建",
|
||||
{
|
||||
"rollout": {"id": 5, "release_id": 22, "rollout_code": "rollout-20260418-demo"},
|
||||
"jobs": [],
|
||||
},
|
||||
)
|
||||
|
||||
ok, message, data = create_release_and_smart_rollout_from_latest_package(
|
||||
{"mode": "worker", "confirm_risky": True},
|
||||
control_plane_base_url="https://ops.example.com",
|
||||
)
|
||||
|
||||
self.assertTrue(ok)
|
||||
self.assertEqual("Release rollout 已创建", message)
|
||||
self.assertTrue(data["rollout_created"])
|
||||
self.assertEqual(5, data["rollout"]["id"])
|
||||
mock_create_release_rollout.assert_called_once()
|
||||
rollout_payload = mock_create_release_rollout.call_args.args[1]
|
||||
self.assertEqual(["mainland-worker-01"], rollout_payload["target_selector"]["node_codes"])
|
||||
self.assertEqual("remote-agent", rollout_payload["policy"]["execution_mode"])
|
||||
|
||||
@patch("app.services.ops_release_service.create_release_rollout")
|
||||
@patch("app.services.ops_policy_service.preview_ops_job_policy")
|
||||
@patch("app.services.ops_agent_service.list_managed_nodes_with_agent_state")
|
||||
@patch("app.services.ops_release_service.create_release_from_latest_package")
|
||||
def test_create_release_and_smart_rollout_uses_ssh_when_only_ssh_path_is_ready(
|
||||
self,
|
||||
mock_create_release_from_latest_package,
|
||||
mock_list_managed_nodes_with_agent_state,
|
||||
mock_preview_ops_job_policy,
|
||||
mock_create_release_rollout,
|
||||
) -> None:
|
||||
mock_create_release_from_latest_package.return_value = (
|
||||
True,
|
||||
"Release 已创建",
|
||||
{
|
||||
"release": {"id": 23, "release_version": "domaincheck_release_20260418_ssh"},
|
||||
"deduplicated": True,
|
||||
},
|
||||
)
|
||||
mock_list_managed_nodes_with_agent_state.return_value = {
|
||||
"nodes": [
|
||||
{
|
||||
"node_code": "mainland-worker-01",
|
||||
"region": "mainland",
|
||||
"role": "worker",
|
||||
"is_enabled": True,
|
||||
"is_managed": True,
|
||||
"is_agent_online": False,
|
||||
"has_ssh_access": True,
|
||||
"cluster_status": "online",
|
||||
"cluster_current_load": 0,
|
||||
"cluster_is_effective_worker": True,
|
||||
"cluster_detect_participating": False,
|
||||
}
|
||||
]
|
||||
}
|
||||
mock_preview_ops_job_policy.return_value = {
|
||||
"blocked": False,
|
||||
"approval_required": False,
|
||||
"warnings": [],
|
||||
"release_gate": {"status_label": "可发 Rollout", "summary": "当前可推进。"},
|
||||
}
|
||||
mock_create_release_rollout.return_value = (
|
||||
True,
|
||||
"Release rollout 已创建",
|
||||
{
|
||||
"rollout": {"id": 6, "release_id": 23, "rollout_code": "rollout-20260418-ssh"},
|
||||
"jobs": [],
|
||||
},
|
||||
)
|
||||
|
||||
ok, message, data = create_release_and_smart_rollout_from_latest_package(
|
||||
{"mode": "worker", "confirm_risky": True},
|
||||
control_plane_base_url="https://ops.example.com",
|
||||
)
|
||||
|
||||
self.assertTrue(ok)
|
||||
self.assertEqual("Release rollout 已创建", message)
|
||||
self.assertTrue(data["rollout_created"])
|
||||
rollout_payload = mock_create_release_rollout.call_args.args[1]
|
||||
self.assertEqual("ssh", rollout_payload["policy"]["execution_mode"])
|
||||
|
||||
@patch("app.services.ops_release_service.get_release")
|
||||
@patch("app.services.ops_release_service.create_release_rollout")
|
||||
@patch("app.services.ops_policy_service.preview_ops_job_policy")
|
||||
@patch("app.services.ops_agent_service.list_managed_nodes_with_agent_state")
|
||||
def test_create_smart_release_rollout_for_existing_release_requires_confirmation(
|
||||
self,
|
||||
mock_list_managed_nodes_with_agent_state,
|
||||
mock_preview_ops_job_policy,
|
||||
mock_create_release_rollout,
|
||||
mock_get_release,
|
||||
) -> None:
|
||||
mock_get_release.return_value = {
|
||||
"id": 41,
|
||||
"release_version": "domaincheck_release_existing",
|
||||
"channel": "stable",
|
||||
}
|
||||
mock_list_managed_nodes_with_agent_state.return_value = {
|
||||
"nodes": [
|
||||
{
|
||||
"node_code": "mainland-controller-01",
|
||||
"region": "mainland",
|
||||
"role": "control",
|
||||
"is_enabled": True,
|
||||
"is_managed": True,
|
||||
"is_agent_online": True,
|
||||
"cluster_status": "online",
|
||||
"cluster_current_load": 0,
|
||||
"cluster_is_effective_worker": True,
|
||||
"cluster_detect_participating": False,
|
||||
}
|
||||
]
|
||||
}
|
||||
mock_preview_ops_job_policy.return_value = {
|
||||
"blocked": False,
|
||||
"approval_required": True,
|
||||
"warnings": ["控制面节点建议审批。"],
|
||||
"release_gate": {"status_label": "待审批", "summary": "需要审批。"},
|
||||
}
|
||||
|
||||
ok, message, data = create_smart_release_rollout(41, {"mode": "control"})
|
||||
|
||||
self.assertTrue(ok)
|
||||
self.assertEqual("智能 Rollout 需要确认", message)
|
||||
self.assertFalse(data["release_created"])
|
||||
self.assertTrue(data["requires_confirmation"])
|
||||
self.assertFalse(data["rollout_created"])
|
||||
self.assertEqual(["mainland-controller-01"], data["smart_rollout"]["target_selector"]["node_codes"])
|
||||
mock_create_release_rollout.assert_not_called()
|
||||
|
||||
@patch("app.services.ops_release_service.get_release")
|
||||
@patch("app.services.ops_release_service.create_release_rollout")
|
||||
@patch("app.services.ops_policy_service.preview_ops_job_policy")
|
||||
@patch("app.services.ops_agent_service.list_managed_nodes_with_agent_state")
|
||||
def test_create_smart_release_rollout_for_existing_release_creates_rollout(
|
||||
self,
|
||||
mock_list_managed_nodes_with_agent_state,
|
||||
mock_preview_ops_job_policy,
|
||||
mock_create_release_rollout,
|
||||
mock_get_release,
|
||||
) -> None:
|
||||
mock_get_release.return_value = {
|
||||
"id": 42,
|
||||
"release_version": "domaincheck_release_existing",
|
||||
"channel": "stable",
|
||||
}
|
||||
mock_list_managed_nodes_with_agent_state.return_value = {
|
||||
"nodes": [
|
||||
{
|
||||
"node_code": "mainland-worker-01",
|
||||
"region": "mainland",
|
||||
"role": "worker",
|
||||
"is_enabled": True,
|
||||
"is_managed": True,
|
||||
"is_agent_online": True,
|
||||
"cluster_status": "online",
|
||||
"cluster_current_load": 0,
|
||||
"cluster_is_effective_worker": True,
|
||||
"cluster_detect_participating": False,
|
||||
}
|
||||
]
|
||||
}
|
||||
mock_preview_ops_job_policy.return_value = {
|
||||
"blocked": False,
|
||||
"approval_required": False,
|
||||
"warnings": [],
|
||||
"release_gate": {"status_label": "可发 Rollout", "summary": "当前可推进。"},
|
||||
}
|
||||
mock_create_release_rollout.return_value = (
|
||||
True,
|
||||
"Release rollout 已创建",
|
||||
{
|
||||
"rollout": {"id": 8, "release_id": 42, "rollout_code": "rollout-20260418-existing"},
|
||||
"jobs": [],
|
||||
},
|
||||
)
|
||||
|
||||
ok, message, data = create_smart_release_rollout(42, {"mode": "worker", "confirm_risky": True})
|
||||
|
||||
self.assertTrue(ok)
|
||||
self.assertEqual("Release rollout 已创建", message)
|
||||
self.assertFalse(data["release_created"])
|
||||
self.assertTrue(data["rollout_created"])
|
||||
self.assertEqual(8, data["rollout"]["id"])
|
||||
mock_create_release_rollout.assert_called_once()
|
||||
rollout_payload = mock_create_release_rollout.call_args.args[1]
|
||||
self.assertEqual(["mainland-worker-01"], rollout_payload["target_selector"]["node_codes"])
|
||||
self.assertEqual("remote-agent", rollout_payload["policy"]["execution_mode"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
118
domain-api/tests/test_ops_runtime_executor_service.py
Normal file
118
domain-api/tests/test_ops_runtime_executor_service.py
Normal file
@@ -0,0 +1,118 @@
|
||||
import subprocess
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
from app.services.ops_runtime_executor_service import execute_local_support_action, execute_ssh_action
|
||||
|
||||
|
||||
class OpsRuntimeExecutorServiceTests(unittest.TestCase):
|
||||
@patch("app.services.ops_runtime_executor_service.subprocess.run")
|
||||
def test_execute_ssh_action_parses_structured_json_result(self, mock_run) -> None:
|
||||
mock_run.return_value = subprocess.CompletedProcess(
|
||||
args=["ssh"],
|
||||
returncode=0,
|
||||
stdout='login banner\n{"ok": true, "message": "health snapshot collected", "result": {"checks": {"api": {"state": "active"}}}}\n',
|
||||
stderr="",
|
||||
)
|
||||
|
||||
ok, message, data = execute_ssh_action(
|
||||
{
|
||||
"node_code": "mainland-worker-01",
|
||||
"ssh_host": "121.204.244.248",
|
||||
"ssh_port": 22,
|
||||
"ssh_user": "root",
|
||||
},
|
||||
"health.snapshot",
|
||||
{},
|
||||
)
|
||||
|
||||
self.assertTrue(ok)
|
||||
self.assertEqual("health snapshot collected", message)
|
||||
self.assertEqual("active", data["checks"]["api"]["state"])
|
||||
self.assertEqual("ssh", data["transport"]["executor"])
|
||||
self.assertEqual("121.204.244.248", data["transport"]["ssh_host"])
|
||||
issued_command = mock_run.call_args.args[0]
|
||||
self.assertEqual("ssh", issued_command[0])
|
||||
self.assertIn("root@121.204.244.248", issued_command)
|
||||
|
||||
@patch("app.services.ops_runtime_executor_service.subprocess.run")
|
||||
def test_execute_local_support_action_collects_logs_with_payload(self, mock_run) -> None:
|
||||
mock_run.return_value = subprocess.CompletedProcess(
|
||||
args=["journalctl"],
|
||||
returncode=0,
|
||||
stdout="log line 1\nlog line 2",
|
||||
stderr="",
|
||||
)
|
||||
|
||||
ok, message, data = execute_local_support_action(
|
||||
"logs.collect",
|
||||
{
|
||||
"service_name": "domaincheck-worker",
|
||||
"lines": 180,
|
||||
},
|
||||
)
|
||||
|
||||
self.assertTrue(ok)
|
||||
self.assertEqual("log line 1\nlog line 2", message)
|
||||
self.assertEqual("domaincheck-worker", data["service_name"])
|
||||
self.assertEqual(180, data["lines"])
|
||||
issued_command = mock_run.call_args.args[0]
|
||||
self.assertEqual(
|
||||
["journalctl", "-u", "domaincheck-worker", "-n", "180", "--no-pager"],
|
||||
issued_command,
|
||||
)
|
||||
|
||||
@patch("app.services.ops_runtime_executor_service.subprocess.run")
|
||||
def test_execute_local_support_action_falls_back_to_worker_log_file(self, mock_run) -> None:
|
||||
mock_run.return_value = subprocess.CompletedProcess(
|
||||
args=["journalctl"],
|
||||
returncode=1,
|
||||
stdout="",
|
||||
stderr="No journal files were opened due to insufficient permissions.",
|
||||
)
|
||||
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
log_path = Path(temp_dir) / "detect_worker.log"
|
||||
log_path.write_text("fallback-1\nfallback-2\n", encoding="utf-8")
|
||||
with patch("app.services.ops_action_executor_core.settings.domain_root", temp_dir):
|
||||
ok, message, data = execute_local_support_action(
|
||||
"logs.collect",
|
||||
{
|
||||
"service_name": "domaincheck-worker",
|
||||
"lines": 180,
|
||||
},
|
||||
)
|
||||
|
||||
self.assertTrue(ok)
|
||||
self.assertEqual("fallback-1\nfallback-2", message)
|
||||
self.assertEqual("file_fallback", data["collection_source"])
|
||||
self.assertTrue(data["fallback_used"])
|
||||
|
||||
@patch("app.services.ops_runtime_executor_service.subprocess.run")
|
||||
def test_execute_local_support_action_restarts_service(self, mock_run) -> None:
|
||||
mock_run.return_value = subprocess.CompletedProcess(
|
||||
args=["systemctl"],
|
||||
returncode=0,
|
||||
stdout="",
|
||||
stderr="",
|
||||
)
|
||||
|
||||
ok, message, data = execute_local_support_action(
|
||||
"service.restart",
|
||||
{
|
||||
"service_name": "domaincheck-worker",
|
||||
},
|
||||
)
|
||||
|
||||
self.assertTrue(ok)
|
||||
self.assertEqual("domaincheck-worker restart completed", message)
|
||||
self.assertEqual("domaincheck-worker", data["service_name"])
|
||||
self.assertEqual("restart", data["systemctl_action"])
|
||||
issued_command = mock_run.call_args.args[0]
|
||||
self.assertEqual(["systemctl", "restart", "domaincheck-worker"], issued_command)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
75
domain-api/tests/test_ops_schema_init.py
Normal file
75
domain-api/tests/test_ops_schema_init.py
Normal file
@@ -0,0 +1,75 @@
|
||||
import unittest
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import app.services.ops_agent_service as ops_agent_service
|
||||
import app.services.ops_job_service as ops_job_service
|
||||
|
||||
|
||||
class OpsSchemaInitTests(unittest.TestCase):
|
||||
@patch("app.services.ops_job_service.get_db")
|
||||
def test_ensure_ops_schema_uses_advisory_lock_and_skips_after_ready(self, mock_get_db) -> None:
|
||||
conn = MagicMock()
|
||||
cursor = MagicMock()
|
||||
db_ctx = MagicMock()
|
||||
cursor_ctx = MagicMock()
|
||||
db_ctx.__enter__.return_value = conn
|
||||
db_ctx.__exit__.return_value = False
|
||||
cursor_ctx.__enter__.return_value = cursor
|
||||
cursor_ctx.__exit__.return_value = False
|
||||
conn.cursor.return_value = cursor_ctx
|
||||
mock_get_db.return_value = db_ctx
|
||||
|
||||
previous_ready = ops_job_service._OPS_SCHEMA_READY
|
||||
ops_job_service._OPS_SCHEMA_READY = False
|
||||
try:
|
||||
ops_job_service.ensure_ops_schema()
|
||||
ops_job_service.ensure_ops_schema()
|
||||
finally:
|
||||
ops_job_service._OPS_SCHEMA_READY = previous_ready
|
||||
|
||||
self.assertEqual(1, mock_get_db.call_count)
|
||||
cursor.execute.assert_any_call(
|
||||
"SELECT pg_advisory_xact_lock(%s)",
|
||||
(ops_job_service._OPS_SCHEMA_ADVISORY_LOCK_KEY,),
|
||||
)
|
||||
cursor.execute.assert_any_call(ops_job_service._OPS_SCHEMA_SQL)
|
||||
conn.commit.assert_called_once()
|
||||
|
||||
@patch("app.services.ops_agent_service.ensure_ops_schema")
|
||||
@patch("app.services.ops_agent_service.get_db")
|
||||
def test_ensure_ops_agent_schema_uses_advisory_lock_and_skips_after_ready(
|
||||
self,
|
||||
mock_get_db,
|
||||
mock_ensure_ops_schema,
|
||||
) -> None:
|
||||
conn = MagicMock()
|
||||
cursor = MagicMock()
|
||||
db_ctx = MagicMock()
|
||||
cursor_ctx = MagicMock()
|
||||
db_ctx.__enter__.return_value = conn
|
||||
db_ctx.__exit__.return_value = False
|
||||
cursor_ctx.__enter__.return_value = cursor
|
||||
cursor_ctx.__exit__.return_value = False
|
||||
conn.cursor.return_value = cursor_ctx
|
||||
mock_get_db.return_value = db_ctx
|
||||
|
||||
previous_ready = ops_agent_service._OPS_AGENT_SCHEMA_READY
|
||||
ops_agent_service._OPS_AGENT_SCHEMA_READY = False
|
||||
try:
|
||||
ops_agent_service.ensure_ops_agent_schema()
|
||||
ops_agent_service.ensure_ops_agent_schema()
|
||||
finally:
|
||||
ops_agent_service._OPS_AGENT_SCHEMA_READY = previous_ready
|
||||
|
||||
self.assertEqual(2, mock_ensure_ops_schema.call_count)
|
||||
self.assertEqual(1, mock_get_db.call_count)
|
||||
cursor.execute.assert_any_call(
|
||||
"SELECT pg_advisory_xact_lock(%s)",
|
||||
(ops_agent_service._OPS_AGENT_SCHEMA_ADVISORY_LOCK_KEY,),
|
||||
)
|
||||
cursor.execute.assert_any_call(ops_agent_service._AGENT_SCHEMA_SQL)
|
||||
conn.commit.assert_called_once()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
607
domain-api/tests/test_ops_service_activity.py
Normal file
607
domain-api/tests/test_ops_service_activity.py
Normal file
@@ -0,0 +1,607 @@
|
||||
import unittest
|
||||
from typing import Optional
|
||||
from unittest.mock import patch
|
||||
|
||||
from app.services.ops_service import get_ops_activity_stream
|
||||
|
||||
|
||||
class OpsServiceActivityTests(unittest.TestCase):
|
||||
@staticmethod
|
||||
def _runtime_status(detect: Optional[dict] = None) -> dict:
|
||||
return {"detect": dict(detect or {})}
|
||||
|
||||
@patch("app.services.ops_service.get_ops_runbook")
|
||||
@patch("app.services.ops_service.list_release_rollouts")
|
||||
@patch("app.services.ops_service.list_ops_jobs")
|
||||
@patch("app.services.ops_service.get_recent_ops_playbook_runs")
|
||||
@patch("app.services.ops_service.list_managed_nodes_with_agent_state")
|
||||
def test_activity_stream_uses_job_detail_for_bootstrap_jobs(
|
||||
self,
|
||||
mock_list_managed_nodes_with_agent_state,
|
||||
mock_get_recent_ops_playbook_runs,
|
||||
mock_list_ops_jobs,
|
||||
mock_list_release_rollouts,
|
||||
mock_get_ops_runbook,
|
||||
) -> None:
|
||||
mock_get_ops_runbook.return_value = {"control_sequences": []}
|
||||
mock_list_managed_nodes_with_agent_state.return_value = {
|
||||
"nodes": [
|
||||
{
|
||||
"node_code": "mainland-worker-02",
|
||||
"agent_state": "pending_bootstrap",
|
||||
"agent_state_label": "待接入",
|
||||
}
|
||||
]
|
||||
}
|
||||
mock_get_recent_ops_playbook_runs.return_value = {"runs": []}
|
||||
mock_list_ops_jobs.return_value = [
|
||||
{
|
||||
"id": 51,
|
||||
"job_code": "ops-bootstrap-51",
|
||||
"action": "node.bootstrap",
|
||||
"status": "success",
|
||||
"target_node_code": "mainland-worker-02",
|
||||
"created_at": "2026-04-17 10:00:00",
|
||||
"updated_at": "2026-04-17 10:01:00",
|
||||
"result": {
|
||||
"bootstrap_plan": {
|
||||
"bootstrap_script_name": "bootstrap-node-agent-mainland-worker-02.sh",
|
||||
}
|
||||
},
|
||||
}
|
||||
]
|
||||
mock_list_release_rollouts.return_value = []
|
||||
|
||||
with patch(
|
||||
"app.services.ops_service.get_runtime_status",
|
||||
return_value=self._runtime_status(),
|
||||
):
|
||||
payload = get_ops_activity_stream(limit=10, scan_limit=20)
|
||||
|
||||
self.assertEqual(1, len(payload["items"]))
|
||||
item = payload["items"][0]
|
||||
self.assertEqual("ops_job", item["kind"])
|
||||
self.assertEqual("节点接入工单", item["title"])
|
||||
self.assertEqual("job_detail", item["ui_intent"]["kind"])
|
||||
self.assertEqual("job_detail", item["ui_intent_kind"])
|
||||
self.assertEqual("控制面", item["execution_mode_label"])
|
||||
self.assertEqual("成功", item["status_label"])
|
||||
self.assertIn("尚未回连 Agent", item["summary"])
|
||||
self.assertEqual(item["summary"], item["summary_text"])
|
||||
self.assertIn("方式 控制面", item["meta_text"])
|
||||
self.assertIn("bootstrap-node-agent-mainland-worker-02.sh", item["meta_text"])
|
||||
self.assertEqual(["mainland-worker-02"], item["target_node_codes"])
|
||||
self.assertEqual(51, item["focus_ref"]["job_id"])
|
||||
self.assertEqual("node.bootstrap", item["focus_ref"]["action"])
|
||||
self.assertEqual("ops_agent_protocol", item["contract_navigation"]["primary_contract_key"])
|
||||
self.assertIn("ops_agent_protocol", item["contract_navigation"]["contract_keys"])
|
||||
|
||||
@patch("app.services.ops_service.get_ops_runbook")
|
||||
@patch("app.services.ops_service.list_release_rollouts")
|
||||
@patch("app.services.ops_service.list_ops_jobs")
|
||||
@patch("app.services.ops_service.get_recent_ops_playbook_runs")
|
||||
@patch("app.services.ops_service.list_managed_nodes_with_agent_state")
|
||||
def test_activity_stream_includes_runbook_sequences_after_live_events(
|
||||
self,
|
||||
mock_list_managed_nodes_with_agent_state,
|
||||
mock_get_recent_ops_playbook_runs,
|
||||
mock_list_ops_jobs,
|
||||
mock_list_release_rollouts,
|
||||
mock_get_ops_runbook,
|
||||
) -> None:
|
||||
mock_list_managed_nodes_with_agent_state.return_value = {"nodes": []}
|
||||
mock_get_recent_ops_playbook_runs.return_value = {
|
||||
"runs": [
|
||||
{
|
||||
"run_code": "playbook-001",
|
||||
"playbook_title": "标准巡检",
|
||||
"status": "running",
|
||||
"updated_at": "2026-04-18 09:11:00",
|
||||
}
|
||||
]
|
||||
}
|
||||
mock_list_ops_jobs.return_value = []
|
||||
mock_list_release_rollouts.return_value = []
|
||||
mock_get_ops_runbook.return_value = {
|
||||
"control_sequences": [
|
||||
{
|
||||
"key": "release_progression",
|
||||
"title": "版本发布与放量",
|
||||
"status": "warning",
|
||||
"status_label": "待处理",
|
||||
"reason": "当前没有足够的执行面接管覆盖。",
|
||||
"target_scope_label": "Release / Rollout",
|
||||
"target_node_codes": ["mainland-controller-01"],
|
||||
"primary_action_code": "fix_managed_nodes",
|
||||
"primary_resolution": {
|
||||
"ok": True,
|
||||
"driver_action_code": "fix_managed_nodes",
|
||||
"driver_action_label": "补执行面接管",
|
||||
"driver_node_codes": ["mainland-controller-01"],
|
||||
"message": "当前建议先补执行面接管。",
|
||||
"resolved_at": "2026-04-18 09:12:00",
|
||||
},
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
with patch(
|
||||
"app.services.ops_service.get_runtime_status",
|
||||
return_value=self._runtime_status(),
|
||||
):
|
||||
payload = get_ops_activity_stream(limit=10, scan_limit=20)
|
||||
|
||||
self.assertEqual(2, len(payload["items"]))
|
||||
self.assertEqual("playbook_run", payload["items"][0]["kind"])
|
||||
self.assertEqual("runbook_sequence", payload["items"][1]["kind"])
|
||||
runbook_item = payload["items"][1]
|
||||
self.assertEqual("release_progression", runbook_item["sequence_key"])
|
||||
self.assertEqual("focus_runbook_sequence", runbook_item["ui_intent"]["kind"])
|
||||
self.assertEqual("fix_managed_nodes", runbook_item["ui_intent"]["driver_action_code"])
|
||||
self.assertEqual("attention", runbook_item["status"])
|
||||
self.assertIn("当前建议动作:补执行面接管", runbook_item["summary"])
|
||||
self.assertIn("release_progression", runbook_item["meta_text"])
|
||||
self.assertEqual("ops_playbook_contract", runbook_item["contract_navigation"]["primary_contract_key"])
|
||||
self.assertIn("ops_driver_contract", runbook_item["contract_navigation"]["contract_keys"])
|
||||
|
||||
@patch("app.services.ops_service.get_ops_runbook")
|
||||
@patch("app.services.ops_service.list_release_rollouts")
|
||||
@patch("app.services.ops_service.list_ops_jobs")
|
||||
@patch("app.services.ops_service.get_recent_ops_playbook_runs")
|
||||
@patch("app.services.ops_service.list_managed_nodes_with_agent_state")
|
||||
def test_activity_stream_surfaces_execution_mode_labels_for_jobs_and_playbooks(
|
||||
self,
|
||||
mock_list_managed_nodes_with_agent_state,
|
||||
mock_get_recent_ops_playbook_runs,
|
||||
mock_list_ops_jobs,
|
||||
mock_list_release_rollouts,
|
||||
mock_get_ops_runbook,
|
||||
) -> None:
|
||||
mock_list_managed_nodes_with_agent_state.return_value = {"nodes": []}
|
||||
mock_get_recent_ops_playbook_runs.return_value = {
|
||||
"runs": [
|
||||
{
|
||||
"run_code": "playbook-ssh-001",
|
||||
"playbook_title": "关键现场日志",
|
||||
"group_title": "执行现场",
|
||||
"status": "running",
|
||||
"updated_at": "2026-04-18 09:11:00",
|
||||
"execution_mode": "ssh",
|
||||
"execution_mode_label": "SSH",
|
||||
"steps": [{"status": "running"}],
|
||||
"step_count": 1,
|
||||
"jobs_total": 2,
|
||||
"target_node_codes": ["mainland-worker-01"],
|
||||
}
|
||||
]
|
||||
}
|
||||
mock_list_ops_jobs.return_value = [
|
||||
{
|
||||
"id": 61,
|
||||
"job_code": "ops-ssh-61",
|
||||
"action": "logs.collect",
|
||||
"status": "running",
|
||||
"execution_mode": "ssh",
|
||||
"execution_mode_label": "SSH",
|
||||
"requested_by": "web-ui",
|
||||
"target_node_code": "mainland-worker-01",
|
||||
"target_type": "node",
|
||||
"risk_level": "low",
|
||||
"updated_at": "2026-04-18 09:10:00",
|
||||
"created_at": "2026-04-18 09:09:00",
|
||||
}
|
||||
]
|
||||
mock_list_release_rollouts.return_value = []
|
||||
mock_get_ops_runbook.return_value = {"control_sequences": []}
|
||||
|
||||
with patch(
|
||||
"app.services.ops_service.get_runtime_status",
|
||||
return_value=self._runtime_status(),
|
||||
):
|
||||
payload = get_ops_activity_stream(limit=10, scan_limit=20)
|
||||
|
||||
self.assertEqual(2, len(payload["items"]))
|
||||
self.assertEqual("SSH", payload["items"][0]["execution_mode_label"])
|
||||
self.assertIn("执行 SSH", payload["items"][0]["meta_text"])
|
||||
self.assertEqual("SSH", payload["items"][1]["execution_mode_label"])
|
||||
self.assertIn("方式 SSH", payload["items"][1]["summary"])
|
||||
self.assertIn("执行 SSH", payload["items"][1]["meta_text"])
|
||||
|
||||
@patch("app.services.ops_service.get_ops_runbook")
|
||||
@patch("app.services.ops_service.list_release_rollouts")
|
||||
@patch("app.services.ops_service.list_ops_jobs")
|
||||
@patch("app.services.ops_service.get_recent_ops_playbook_runs")
|
||||
@patch("app.services.ops_service.list_managed_nodes_with_agent_state")
|
||||
def test_activity_stream_prefers_job_result_summary_and_focus_ref(
|
||||
self,
|
||||
mock_list_managed_nodes_with_agent_state,
|
||||
mock_get_recent_ops_playbook_runs,
|
||||
mock_list_ops_jobs,
|
||||
mock_list_release_rollouts,
|
||||
mock_get_ops_runbook,
|
||||
) -> None:
|
||||
mock_list_managed_nodes_with_agent_state.return_value = {"nodes": []}
|
||||
mock_get_recent_ops_playbook_runs.return_value = {"runs": []}
|
||||
mock_list_ops_jobs.return_value = [
|
||||
{
|
||||
"id": 71,
|
||||
"job_code": "ops-agent-71",
|
||||
"action": "logs.collect",
|
||||
"status": "success",
|
||||
"execution_mode": "remote-agent",
|
||||
"execution_mode_label": "远端 Agent",
|
||||
"requested_by": "codex",
|
||||
"target_node_code": "mainland-worker-01",
|
||||
"target_type": "node",
|
||||
"risk_level": "low",
|
||||
"updated_at": "2026-04-18 09:20:00",
|
||||
"created_at": "2026-04-18 09:19:00",
|
||||
"focus_ref": {
|
||||
"kind": "ops_job",
|
||||
"job_id": 71,
|
||||
"job_code": "ops-agent-71",
|
||||
"action": "logs.collect",
|
||||
"target_node_code": "mainland-worker-01",
|
||||
},
|
||||
"result": {
|
||||
"summary_text": "远端 Agent 已回传 120 行 Worker 日志,可直接继续分析。",
|
||||
"focus_ref": {
|
||||
"kind": "ops_job",
|
||||
"job_id": 71,
|
||||
"job_code": "ops-agent-71",
|
||||
"action": "logs.collect",
|
||||
"target_node_code": "mainland-worker-01",
|
||||
"event_key": "job-event:888",
|
||||
"step_key": "worker_logs",
|
||||
},
|
||||
},
|
||||
}
|
||||
]
|
||||
mock_list_release_rollouts.return_value = []
|
||||
mock_get_ops_runbook.return_value = {"control_sequences": []}
|
||||
|
||||
with patch(
|
||||
"app.services.ops_service.get_runtime_status",
|
||||
return_value=self._runtime_status(),
|
||||
):
|
||||
payload = get_ops_activity_stream(limit=10, scan_limit=20)
|
||||
|
||||
self.assertEqual(1, len(payload["items"]))
|
||||
item = payload["items"][0]
|
||||
self.assertEqual("ops_job", item["kind"])
|
||||
self.assertEqual("远端 Agent 已回传 120 行 Worker 日志,可直接继续分析。", item["summary"])
|
||||
self.assertEqual("ops_job", item["focus_ref"]["kind"])
|
||||
self.assertEqual("job-event:888", item["focus_ref"]["event_key"])
|
||||
self.assertEqual("worker_logs", item["focus_ref"]["step_key"])
|
||||
self.assertIn("执行 远端 Agent", item["meta_text"])
|
||||
self.assertEqual("ops_observability_contract", item["contract_navigation"]["primary_contract_key"])
|
||||
self.assertIn("ops_stack_diagnosis_contract", item["contract_navigation"]["contract_keys"])
|
||||
|
||||
@patch("app.services.ops_service.get_ops_runbook")
|
||||
@patch("app.services.ops_service.list_release_rollouts")
|
||||
@patch("app.services.ops_service.list_ops_jobs")
|
||||
@patch("app.services.ops_service.get_recent_ops_playbook_runs")
|
||||
@patch("app.services.ops_service.list_managed_nodes_with_agent_state")
|
||||
def test_activity_stream_marks_job_attention_when_start_delivery_failed_locally(
|
||||
self,
|
||||
mock_list_managed_nodes_with_agent_state,
|
||||
mock_get_recent_ops_playbook_runs,
|
||||
mock_list_ops_jobs,
|
||||
mock_list_release_rollouts,
|
||||
mock_get_ops_runbook,
|
||||
) -> None:
|
||||
mock_list_managed_nodes_with_agent_state.return_value = {"nodes": []}
|
||||
mock_get_recent_ops_playbook_runs.return_value = {"runs": []}
|
||||
mock_list_ops_jobs.return_value = [
|
||||
{
|
||||
"id": 81,
|
||||
"job_code": "ops-agent-81",
|
||||
"action": "logs.collect",
|
||||
"status": "running",
|
||||
"execution_mode": "remote-agent",
|
||||
"execution_mode_label": "远端 Agent",
|
||||
"requested_by": "codex",
|
||||
"target_node_code": "mainland-worker-01",
|
||||
"target_type": "node",
|
||||
"risk_level": "low",
|
||||
"updated_at": "2026-04-18 10:20:00",
|
||||
"created_at": "2026-04-18 10:19:00",
|
||||
"focus_ref": {
|
||||
"kind": "ops_job",
|
||||
"job_id": 81,
|
||||
"job_code": "ops-agent-81",
|
||||
"action": "logs.collect",
|
||||
"target_node_code": "mainland-worker-01",
|
||||
},
|
||||
"result": {
|
||||
"summary_text": "节点端已经收到采集任务。",
|
||||
"start_delivery_state": "failed_local",
|
||||
"start_delivery_error": "temporary offline",
|
||||
"source_focus_ref": {
|
||||
"kind": "ops_job_event",
|
||||
"job_id": 81,
|
||||
"job_code": "ops-agent-81",
|
||||
"event_key": "job-event:81",
|
||||
"target_node_code": "mainland-worker-01",
|
||||
},
|
||||
},
|
||||
}
|
||||
]
|
||||
mock_list_release_rollouts.return_value = []
|
||||
mock_get_ops_runbook.return_value = {"control_sequences": []}
|
||||
|
||||
with patch(
|
||||
"app.services.ops_service.get_runtime_status",
|
||||
return_value=self._runtime_status(),
|
||||
):
|
||||
payload = get_ops_activity_stream(limit=10, scan_limit=20)
|
||||
|
||||
self.assertEqual(1, len(payload["items"]))
|
||||
item = payload["items"][0]
|
||||
self.assertEqual("ops_job", item["kind"])
|
||||
self.assertEqual("attention", item["status"])
|
||||
self.assertEqual("running", item["job_status"])
|
||||
self.assertEqual("failed_local", item["start_delivery_state"])
|
||||
self.assertEqual("ops_job_start_delivery_failed_local", item["problem_code"])
|
||||
self.assertIn("开始回执未成功送达控制面", item["summary"])
|
||||
self.assertIn("temporary offline", item["summary"])
|
||||
self.assertIn("开始回执异常", item["meta_text"])
|
||||
self.assertEqual("ops_job_event", item["source_focus_ref"]["kind"])
|
||||
self.assertEqual("job-event:81", item["source_focus_ref"]["event_key"])
|
||||
self.assertEqual("attention", payload["summary"]["status"])
|
||||
self.assertEqual("待处理", payload["summary"]["status_label"])
|
||||
|
||||
@patch("app.services.ops_service.get_ops_runbook")
|
||||
@patch("app.services.ops_service.list_release_rollouts")
|
||||
@patch("app.services.ops_service.list_ops_jobs")
|
||||
@patch("app.services.ops_service.get_recent_ops_playbook_runs")
|
||||
@patch("app.services.ops_service.list_managed_nodes_with_agent_state")
|
||||
def test_activity_stream_surfaces_execution_mode_label_for_rollout(
|
||||
self,
|
||||
mock_list_managed_nodes_with_agent_state,
|
||||
mock_get_recent_ops_playbook_runs,
|
||||
mock_list_ops_jobs,
|
||||
mock_list_release_rollouts,
|
||||
mock_get_ops_runbook,
|
||||
) -> None:
|
||||
mock_list_managed_nodes_with_agent_state.return_value = {"nodes": []}
|
||||
mock_get_recent_ops_playbook_runs.return_value = {"runs": []}
|
||||
mock_list_ops_jobs.return_value = []
|
||||
mock_list_release_rollouts.return_value = [
|
||||
{
|
||||
"id": 201,
|
||||
"release_id": 18,
|
||||
"rollout_code": "rollout-20260418093000-a1b2",
|
||||
"policy": {
|
||||
"execution_mode": "ssh",
|
||||
},
|
||||
"status": "running",
|
||||
"status_label": "执行中",
|
||||
"target_nodes": [{"node_code": "mainland-worker-01"}],
|
||||
"jobs_total": 2,
|
||||
"jobs_created": 1,
|
||||
"batch_cursor": 1,
|
||||
"batches_total": 2,
|
||||
"result_summary": {},
|
||||
"summary_text": "当前已覆盖 1/1 台节点,仍有 1 条任务执行中。",
|
||||
"focus_ref": {
|
||||
"kind": "release_rollout",
|
||||
"rollout_id": 201,
|
||||
"rollout_code": "rollout-20260418093000-a1b2",
|
||||
"release_id": 18,
|
||||
"section": "rollout_jobs",
|
||||
},
|
||||
"created_by": "web-ui",
|
||||
"created_at": "2026-04-18 09:30:00",
|
||||
"updated_at": "2026-04-18 09:31:00",
|
||||
}
|
||||
]
|
||||
mock_get_ops_runbook.return_value = {"control_sequences": []}
|
||||
|
||||
with patch(
|
||||
"app.services.ops_service.get_runtime_status",
|
||||
return_value=self._runtime_status(),
|
||||
):
|
||||
payload = get_ops_activity_stream(limit=10, scan_limit=20)
|
||||
|
||||
self.assertEqual(1, len(payload["items"]))
|
||||
item = payload["items"][0]
|
||||
self.assertEqual("rollout", item["kind"])
|
||||
self.assertEqual("SSH", item["execution_mode_label"])
|
||||
self.assertIn("执行 SSH", item["meta_text"])
|
||||
self.assertEqual("执行中", item["status_label"])
|
||||
self.assertIn("当前已覆盖 1/1 台节点", item["summary"])
|
||||
self.assertEqual("release_rollout", item["focus_ref"]["kind"])
|
||||
self.assertEqual("release_hub_contract", item["contract_navigation"]["primary_contract_key"])
|
||||
self.assertIn("release_hub_contract", item["contract_navigation"]["contract_keys"])
|
||||
|
||||
@patch("app.services.ops_service.get_ops_runbook")
|
||||
@patch("app.services.ops_service.list_release_rollouts")
|
||||
@patch("app.services.ops_service.list_ops_jobs")
|
||||
@patch("app.services.ops_service.get_recent_ops_playbook_runs")
|
||||
@patch("app.services.ops_service.list_managed_nodes_with_agent_state")
|
||||
def test_activity_stream_supports_execution_mode_filter(
|
||||
self,
|
||||
mock_list_managed_nodes_with_agent_state,
|
||||
mock_get_recent_ops_playbook_runs,
|
||||
mock_list_ops_jobs,
|
||||
mock_list_release_rollouts,
|
||||
mock_get_ops_runbook,
|
||||
) -> None:
|
||||
mock_list_managed_nodes_with_agent_state.return_value = {"nodes": []}
|
||||
mock_get_recent_ops_playbook_runs.return_value = {
|
||||
"runs": [
|
||||
{
|
||||
"run_code": "playbook-ssh-001",
|
||||
"playbook_title": "关键现场日志",
|
||||
"status": "running",
|
||||
"updated_at": "2026-04-18 09:11:00",
|
||||
"execution_mode": "ssh",
|
||||
"execution_mode_label": "SSH",
|
||||
"steps": [{"status": "running"}],
|
||||
"step_count": 1,
|
||||
"jobs_total": 1,
|
||||
"target_node_codes": ["mainland-worker-01"],
|
||||
}
|
||||
]
|
||||
}
|
||||
mock_list_ops_jobs.return_value = [
|
||||
{
|
||||
"id": 61,
|
||||
"job_code": "ops-ssh-61",
|
||||
"action": "logs.collect",
|
||||
"status": "running",
|
||||
"execution_mode": "ssh",
|
||||
"execution_mode_label": "SSH",
|
||||
"requested_by": "web-ui",
|
||||
"target_node_code": "mainland-worker-01",
|
||||
"target_type": "node",
|
||||
"risk_level": "low",
|
||||
"updated_at": "2026-04-18 09:10:00",
|
||||
"created_at": "2026-04-18 09:09:00",
|
||||
},
|
||||
{
|
||||
"id": 62,
|
||||
"job_code": "ops-agent-62",
|
||||
"action": "health.snapshot",
|
||||
"status": "success",
|
||||
"execution_mode": "remote-agent",
|
||||
"execution_mode_label": "远端 Agent",
|
||||
"requested_by": "web-ui",
|
||||
"target_node_code": "mainland-worker-02",
|
||||
"target_type": "node",
|
||||
"risk_level": "low",
|
||||
"updated_at": "2026-04-18 09:08:00",
|
||||
"created_at": "2026-04-18 09:07:00",
|
||||
},
|
||||
]
|
||||
mock_list_release_rollouts.return_value = []
|
||||
mock_get_ops_runbook.return_value = {
|
||||
"control_sequences": [
|
||||
{
|
||||
"key": "release_progression",
|
||||
"title": "版本发布与放量",
|
||||
"status": "warning",
|
||||
"status_label": "待处理",
|
||||
"reason": "当前没有足够的执行面接管覆盖。",
|
||||
"target_scope_label": "Release / Rollout",
|
||||
"target_node_codes": ["mainland-controller-01"],
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
with patch(
|
||||
"app.services.ops_service.get_runtime_status",
|
||||
return_value=self._runtime_status(),
|
||||
):
|
||||
payload = get_ops_activity_stream(limit=10, scan_limit=20, execution_mode="ssh")
|
||||
|
||||
self.assertEqual(2, len(payload["items"]))
|
||||
self.assertTrue(all(item["execution_mode"] == "ssh" for item in payload["items"]))
|
||||
self.assertEqual("ssh", payload["summary"]["filters"]["execution_mode"])
|
||||
self.assertEqual(2, payload["summary"]["available_execution_mode_counts"]["ssh"])
|
||||
self.assertEqual(1, payload["summary"]["available_execution_mode_counts"]["remote-agent"])
|
||||
|
||||
@patch("app.services.ops_service.get_ops_runbook")
|
||||
@patch("app.services.ops_service.list_release_rollouts")
|
||||
@patch("app.services.ops_service.list_ops_jobs")
|
||||
@patch("app.services.ops_service.get_recent_ops_playbook_runs")
|
||||
@patch("app.services.ops_service.list_managed_nodes_with_agent_state")
|
||||
def test_activity_stream_includes_execution_scene_and_log_sync_items(
|
||||
self,
|
||||
mock_list_managed_nodes_with_agent_state,
|
||||
mock_get_recent_ops_playbook_runs,
|
||||
mock_list_ops_jobs,
|
||||
mock_list_release_rollouts,
|
||||
mock_get_ops_runbook,
|
||||
) -> None:
|
||||
mock_list_managed_nodes_with_agent_state.return_value = {"nodes": []}
|
||||
mock_get_recent_ops_playbook_runs.return_value = {"runs": []}
|
||||
mock_list_ops_jobs.return_value = []
|
||||
mock_list_release_rollouts.return_value = []
|
||||
mock_get_ops_runbook.return_value = {"control_sequences": []}
|
||||
|
||||
detect = {
|
||||
"active_job": {
|
||||
"job_code": "detect-001",
|
||||
"latest_cycle_event": {
|
||||
"created_at": "2026-04-18 09:12:30",
|
||||
},
|
||||
},
|
||||
"phase_label": "运行中",
|
||||
"phase_detail": "代理池正常,现场可继续观察",
|
||||
"recent_event": "域名检测完成: example.com",
|
||||
"recent_warning": "",
|
||||
"log_sync": {
|
||||
"enabled": True,
|
||||
"mode": "key",
|
||||
"mode_label": "关键回传",
|
||||
"description": "当前为关键回传;适合常态联调。",
|
||||
"line_count": 12,
|
||||
"source_node_count": 1,
|
||||
"source_nodes": ["mainland-worker-01"],
|
||||
"last_at": "2026-04-18 09:12:31",
|
||||
"preview_lines": ["worker log line"],
|
||||
},
|
||||
"participation_summary": {
|
||||
"summary": "当前参与检测节点 1 台,其中执行/已领 1 台,近窗刚有吞吐 0 台。",
|
||||
"effective_online_nodes": 1,
|
||||
"participating_nodes": 1,
|
||||
"dispatch_active_nodes": 1,
|
||||
"recent_only_nodes": 0,
|
||||
"non_participating_nodes": 0,
|
||||
"standby_nodes": 0,
|
||||
"load_syncing_nodes": 0,
|
||||
},
|
||||
"participating_nodes": [
|
||||
{
|
||||
"node_code": "mainland-worker-01",
|
||||
"region": "mainland",
|
||||
"role": "worker",
|
||||
"status": "busy",
|
||||
"last_heartbeat_at": "2026-04-18 09:12:29",
|
||||
"participation_state": "dispatch_active",
|
||||
"is_dispatch_active": True,
|
||||
}
|
||||
],
|
||||
"non_participating_nodes": [],
|
||||
}
|
||||
|
||||
with patch(
|
||||
"app.services.ops_service.get_runtime_status",
|
||||
return_value=self._runtime_status(detect),
|
||||
):
|
||||
payload = get_ops_activity_stream(limit=10, scan_limit=20)
|
||||
|
||||
items_by_kind = {item["kind"]: item for item in payload["items"]}
|
||||
self.assertIn("execution_scene", items_by_kind)
|
||||
self.assertIn("log_sync", items_by_kind)
|
||||
|
||||
execution_scene_item = items_by_kind["execution_scene"]
|
||||
log_sync_item = items_by_kind["log_sync"]
|
||||
|
||||
self.assertEqual("running", execution_scene_item["status"])
|
||||
self.assertEqual("执行中", execution_scene_item["status_label"])
|
||||
self.assertEqual("focus_execution_scene", execution_scene_item["ui_intent"]["kind"])
|
||||
self.assertEqual("focus_execution_scene", execution_scene_item["ui_intent_kind"])
|
||||
self.assertEqual("execution_scene", execution_scene_item["focus_ref"]["kind"])
|
||||
self.assertEqual(execution_scene_item["summary"], execution_scene_item["summary_text"])
|
||||
self.assertIn("执行/已领 1", execution_scene_item["meta_text"])
|
||||
self.assertEqual("ops_observability_contract", execution_scene_item["contract_navigation"]["primary_contract_key"])
|
||||
self.assertIn("ops_stack_diagnosis_contract", execution_scene_item["contract_navigation"]["contract_keys"])
|
||||
self.assertEqual("running", log_sync_item["status"])
|
||||
self.assertEqual("执行中", log_sync_item["status_label"])
|
||||
self.assertEqual("control-plane", log_sync_item["execution_mode"])
|
||||
self.assertEqual("控制面", log_sync_item["execution_mode_label"])
|
||||
self.assertEqual("focus_execution_scene", log_sync_item["ui_intent"]["kind"])
|
||||
self.assertTrue(log_sync_item["ui_intent"]["open_log_drawer"])
|
||||
self.assertIn("关键日志观察模式", log_sync_item["summary"])
|
||||
self.assertEqual("running", payload["summary"]["status"])
|
||||
self.assertEqual("执行中", payload["summary"]["status_label"])
|
||||
self.assertIn("最近活动里存在正在推进", payload["summary"]["summary_text"])
|
||||
self.assertEqual("ops_observability_contract", payload["contract_navigation"]["primary_contract_key"])
|
||||
self.assertIn("ops_stack_diagnosis_contract", payload["contract_navigation"]["contract_keys"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
925
domain-api/tests/test_ops_service_codex_action.py
Normal file
925
domain-api/tests/test_ops_service_codex_action.py
Normal file
@@ -0,0 +1,925 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
from app.services.ops_service import execute_codex_action, resolve_codex_action
|
||||
|
||||
|
||||
def _build_codex_brief_entry(
|
||||
*,
|
||||
key: str,
|
||||
executor_kind: str,
|
||||
primary_action_code: str,
|
||||
recommendation: str,
|
||||
automation_level: str,
|
||||
target_api: str,
|
||||
target_method: str = "POST",
|
||||
sequence_key: str = "",
|
||||
primary_action_payload: dict | None = None,
|
||||
secondary_action_code: str = "",
|
||||
secondary_action_payload: dict | None = None,
|
||||
focus_ref: dict | None = None,
|
||||
primary_focus_ref: dict | None = None,
|
||||
secondary_focus_ref: dict | None = None,
|
||||
) -> dict:
|
||||
primary_action_payload = dict(primary_action_payload or {})
|
||||
secondary_action_payload = dict(secondary_action_payload or {})
|
||||
normalized_focus_ref = dict(focus_ref or {})
|
||||
normalized_primary_focus_ref = dict(primary_focus_ref or {})
|
||||
normalized_secondary_focus_ref = dict(secondary_focus_ref or {})
|
||||
if normalized_primary_focus_ref and not dict(primary_action_payload.get("focus_ref") or {}):
|
||||
primary_action_payload["focus_ref"] = dict(normalized_primary_focus_ref)
|
||||
if normalized_secondary_focus_ref and not dict(secondary_action_payload.get("focus_ref") or {}):
|
||||
secondary_action_payload["focus_ref"] = dict(normalized_secondary_focus_ref)
|
||||
request_payload_preview = (
|
||||
{
|
||||
"sequence_key": sequence_key,
|
||||
"secondary": False,
|
||||
"action_payload": primary_action_payload,
|
||||
}
|
||||
if executor_kind == "runbook_sequence"
|
||||
else {
|
||||
"action_code": primary_action_code,
|
||||
"node_codes": ["mainland-controller-01"],
|
||||
"action_payload": primary_action_payload,
|
||||
}
|
||||
)
|
||||
return {
|
||||
"key": key,
|
||||
"title": "测试动作",
|
||||
"summary": "测试摘要",
|
||||
"reason": "测试原因",
|
||||
"meta_text": "meta",
|
||||
"executor_kind": executor_kind,
|
||||
"sequence_key": sequence_key,
|
||||
"focus_ref": normalized_focus_ref,
|
||||
"primary_focus_ref": normalized_primary_focus_ref,
|
||||
"secondary_focus_ref": normalized_secondary_focus_ref,
|
||||
"primary_label": "主动作",
|
||||
"secondary_label": "次动作" if secondary_action_code else "",
|
||||
"primary_action_code": primary_action_code,
|
||||
"secondary_action_code": secondary_action_code,
|
||||
"autopilot": {
|
||||
"status_text": "测试状态",
|
||||
"automation_level": automation_level,
|
||||
"recommendation": recommendation,
|
||||
"executor_mode_hint": "test-mode",
|
||||
"confirm_required": recommendation == "confirm_then_execute",
|
||||
"ui_only": recommendation == "open_ui",
|
||||
"blocked": recommendation == "blocked",
|
||||
"backend_handled": True,
|
||||
"risk_level": "low" if recommendation == "auto_execute" else "high",
|
||||
"reason": "autopilot reason",
|
||||
"target_method": target_method,
|
||||
"target_api": target_api,
|
||||
},
|
||||
"primary_execution": {
|
||||
"label": "主动作",
|
||||
"node_codes": ["mainland-controller-01"],
|
||||
"action_payload": primary_action_payload,
|
||||
"request_payload_preview": request_payload_preview,
|
||||
},
|
||||
"secondary_execution": (
|
||||
{
|
||||
"label": "次动作",
|
||||
"node_codes": [],
|
||||
"action_payload": secondary_action_payload,
|
||||
}
|
||||
if secondary_action_code
|
||||
else {}
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
def _build_preview_data(
|
||||
*,
|
||||
executor_kind: str,
|
||||
action_code: str,
|
||||
recommendation: str,
|
||||
automation_level: str,
|
||||
target_api: str,
|
||||
request_payload: dict,
|
||||
sequence_key: str = "",
|
||||
focus_ref: dict | None = None,
|
||||
primary_focus_ref: dict | None = None,
|
||||
secondary_focus_ref: dict | None = None,
|
||||
) -> dict:
|
||||
return {
|
||||
"executor_kind": executor_kind,
|
||||
"sequence_key": sequence_key,
|
||||
"execution_chain": "ops-driver -> preview",
|
||||
"focus_ref": dict(focus_ref or {}),
|
||||
"primary_focus_ref": dict(primary_focus_ref or {}),
|
||||
"secondary_focus_ref": dict(secondary_focus_ref or {}),
|
||||
"primary": {
|
||||
"label": "主动作",
|
||||
"action_code": action_code,
|
||||
"target_api": target_api,
|
||||
"target_method": "POST",
|
||||
"request_payload": dict(request_payload or {}),
|
||||
"focus_ref": dict(primary_focus_ref or {}),
|
||||
"execution_profile": {
|
||||
"action_code": action_code,
|
||||
"backend_handled": True,
|
||||
"automation_level": automation_level,
|
||||
"recommendation": recommendation,
|
||||
"executor_mode_hint": "test-mode",
|
||||
"risk_level": "low" if recommendation == "auto_execute" else "high",
|
||||
"confirm_required": recommendation == "confirm_then_execute",
|
||||
"ui_only": recommendation == "open_ui",
|
||||
"blocked": recommendation == "blocked",
|
||||
"requires_fields": [],
|
||||
"missing_fields": [],
|
||||
"reason": "preview reason",
|
||||
},
|
||||
},
|
||||
"secondary": {},
|
||||
}
|
||||
|
||||
|
||||
def _build_codex_activity_focus_entry(
|
||||
*,
|
||||
key: str,
|
||||
activity_key: str,
|
||||
activity_kind: str,
|
||||
ui_intent_kind: str,
|
||||
focus_ref: dict | None = None,
|
||||
source_focus_ref: dict | None = None,
|
||||
target_node_codes: list[str] | None = None,
|
||||
) -> dict:
|
||||
normalized_focus_ref = dict(focus_ref or {})
|
||||
return {
|
||||
"key": key,
|
||||
"activity_key": activity_key,
|
||||
"activity_kind": activity_kind,
|
||||
"title": "观察焦点",
|
||||
"subtitle": "日志回传待确认",
|
||||
"summary": "该焦点属于观察项,应进入对应工作区查看上下文。",
|
||||
"status": "attention",
|
||||
"status_label": "待关注",
|
||||
"meta_text": "activity-meta",
|
||||
"occurred_at": "2026-04-18T06:00:00",
|
||||
"target_node_codes": list(target_node_codes or []),
|
||||
"focus_ref": normalized_focus_ref,
|
||||
"source_focus_ref": dict(source_focus_ref or {}),
|
||||
"ui_intent": {
|
||||
"kind": ui_intent_kind,
|
||||
"activity_key": activity_key,
|
||||
},
|
||||
"ui_intent_kind": ui_intent_kind,
|
||||
"focus_action_code": "focus_activity_item",
|
||||
"focus_action_payload": {
|
||||
"activity_key": activity_key,
|
||||
"kind": activity_kind,
|
||||
"status": "attention",
|
||||
"ui_intent": {
|
||||
"kind": ui_intent_kind,
|
||||
"activity_key": activity_key,
|
||||
},
|
||||
"focus_ref": normalized_focus_ref,
|
||||
},
|
||||
"observation_only": True,
|
||||
}
|
||||
|
||||
|
||||
class OpsServiceCodexActionTests(unittest.TestCase):
|
||||
@patch("app.services.ops_service.preview_driver_action")
|
||||
@patch("app.services.ops_service.get_ops_codex_brief")
|
||||
def test_resolve_codex_action_returns_gate_for_safe_auto(self, mock_get_ops_codex_brief, mock_preview_driver_action) -> None:
|
||||
mock_get_ops_codex_brief.return_value = {
|
||||
"focus": {"entry_key": "recommendation:enable-log-sync"},
|
||||
"summary": {"total": 1},
|
||||
"entries": [
|
||||
_build_codex_brief_entry(
|
||||
key="recommendation:enable-log-sync",
|
||||
executor_kind="driver_action",
|
||||
primary_action_code="enable_log_sync_key",
|
||||
recommendation="auto_execute",
|
||||
automation_level="safe_auto",
|
||||
target_api="/api/v1/ops/driver-actions/execute",
|
||||
focus_ref={
|
||||
"kind": "execution_scene",
|
||||
"scene_key": "scene.logs",
|
||||
},
|
||||
primary_focus_ref={
|
||||
"kind": "execution_scene",
|
||||
"scene_key": "scene.logs",
|
||||
},
|
||||
)
|
||||
],
|
||||
}
|
||||
mock_preview_driver_action.return_value = (
|
||||
True,
|
||||
"ok",
|
||||
_build_preview_data(
|
||||
executor_kind="driver_action",
|
||||
action_code="enable_log_sync_key",
|
||||
recommendation="auto_execute",
|
||||
automation_level="safe_auto",
|
||||
target_api="/api/v1/ops/driver-actions/execute",
|
||||
request_payload={
|
||||
"action_code": "enable_log_sync_key",
|
||||
"node_codes": ["mainland-controller-01"],
|
||||
"action_payload": {
|
||||
"focus_ref": {
|
||||
"kind": "execution_scene",
|
||||
"scene_key": "scene.logs",
|
||||
}
|
||||
},
|
||||
"requested_by": "test/codex-action",
|
||||
},
|
||||
focus_ref={
|
||||
"kind": "execution_scene",
|
||||
"scene_key": "scene.logs",
|
||||
},
|
||||
primary_focus_ref={
|
||||
"kind": "execution_scene",
|
||||
"scene_key": "scene.logs",
|
||||
},
|
||||
),
|
||||
)
|
||||
|
||||
ok, message, data = resolve_codex_action({"requested_by": "test/codex-action"})
|
||||
|
||||
self.assertTrue(ok)
|
||||
self.assertEqual("Codex 驾驶动作解析完成", message)
|
||||
self.assertEqual("recommendation:enable-log-sync", data["entry_key"])
|
||||
self.assertEqual("execute_now", data["gate"]["decision"])
|
||||
self.assertTrue(data["gate"]["will_execute"])
|
||||
self.assertEqual("/api/v1/ops/driver-actions/execute", data["gate"]["target_api"])
|
||||
self.assertEqual("enable_log_sync_key", data["preview_request"]["primary_action_code"])
|
||||
self.assertEqual("execution_scene", data["preview_request"]["focus_ref"]["kind"])
|
||||
self.assertEqual("scene.logs", data["gate"]["focus_ref"]["scene_key"])
|
||||
self.assertEqual("scene.logs", data["focus_ref"]["scene_key"])
|
||||
|
||||
@patch("app.services.ops_service.preview_driver_action")
|
||||
@patch("app.services.ops_service.get_ops_codex_brief")
|
||||
def test_resolve_codex_action_requires_confirmation_for_guarded_runbook(
|
||||
self,
|
||||
mock_get_ops_codex_brief,
|
||||
mock_preview_driver_action,
|
||||
) -> None:
|
||||
mock_get_ops_codex_brief.return_value = {
|
||||
"focus": {"entry_key": "runbook:release_progression"},
|
||||
"summary": {"total": 1},
|
||||
"entries": [
|
||||
_build_codex_brief_entry(
|
||||
key="runbook:release_progression",
|
||||
executor_kind="runbook_sequence",
|
||||
sequence_key="release_progression",
|
||||
primary_action_code="create_release_rollout_worker",
|
||||
recommendation="confirm_then_execute",
|
||||
automation_level="guarded_auto",
|
||||
target_api="/api/v1/ops/runbook/sequences/release_progression/execute",
|
||||
primary_action_payload={"release_id": 7},
|
||||
secondary_action_code="focus_release_hub",
|
||||
secondary_action_payload={"release_id": 7},
|
||||
focus_ref={
|
||||
"kind": "release_hub",
|
||||
"release_id": 7,
|
||||
"section": "release_launchpad",
|
||||
},
|
||||
primary_focus_ref={
|
||||
"kind": "release_hub",
|
||||
"release_id": 7,
|
||||
"section": "release_launchpad",
|
||||
"mode": "worker",
|
||||
},
|
||||
secondary_focus_ref={
|
||||
"kind": "release_hub",
|
||||
"release_id": 7,
|
||||
"section": "release_detail",
|
||||
},
|
||||
)
|
||||
],
|
||||
}
|
||||
mock_preview_driver_action.return_value = (
|
||||
True,
|
||||
"ok",
|
||||
_build_preview_data(
|
||||
executor_kind="runbook_sequence",
|
||||
action_code="create_release_rollout_worker",
|
||||
recommendation="confirm_then_execute",
|
||||
automation_level="guarded_auto",
|
||||
target_api="/api/v1/ops/runbook/sequences/release_progression/execute",
|
||||
request_payload={
|
||||
"secondary": False,
|
||||
"requested_by": "test/codex-action",
|
||||
"action_payload": {
|
||||
"release_id": 7,
|
||||
"focus_ref": {
|
||||
"kind": "release_hub",
|
||||
"release_id": 7,
|
||||
"section": "release_launchpad",
|
||||
"mode": "worker",
|
||||
},
|
||||
},
|
||||
},
|
||||
sequence_key="release_progression",
|
||||
focus_ref={
|
||||
"kind": "release_hub",
|
||||
"release_id": 7,
|
||||
"section": "release_launchpad",
|
||||
},
|
||||
primary_focus_ref={
|
||||
"kind": "release_hub",
|
||||
"release_id": 7,
|
||||
"section": "release_launchpad",
|
||||
"mode": "worker",
|
||||
},
|
||||
secondary_focus_ref={
|
||||
"kind": "release_hub",
|
||||
"release_id": 7,
|
||||
"section": "release_detail",
|
||||
},
|
||||
),
|
||||
)
|
||||
|
||||
ok, _, data = resolve_codex_action(
|
||||
{
|
||||
"entry_key": "runbook:release_progression",
|
||||
"requested_by": "test/codex-action",
|
||||
}
|
||||
)
|
||||
|
||||
self.assertTrue(ok)
|
||||
self.assertEqual("confirmation_required", data["gate"]["decision"])
|
||||
self.assertFalse(data["gate"]["will_execute"])
|
||||
self.assertTrue(data["gate"]["confirm_required"])
|
||||
self.assertEqual("release_progression", data["gate"]["sequence_key"])
|
||||
self.assertEqual("worker", data["preview_request"]["primary_focus_ref"]["mode"])
|
||||
self.assertEqual("worker", data["gate"]["focus_ref"]["mode"])
|
||||
self.assertEqual("worker", data["focus_ref"]["mode"])
|
||||
|
||||
@patch("app.services.ops_service.preview_driver_action")
|
||||
@patch("app.services.ops_service.get_ops_codex_brief")
|
||||
def test_resolve_codex_action_can_select_secondary_section(
|
||||
self,
|
||||
mock_get_ops_codex_brief,
|
||||
mock_preview_driver_action,
|
||||
) -> None:
|
||||
mock_get_ops_codex_brief.return_value = {
|
||||
"focus": {"entry_key": "runbook:release_progression"},
|
||||
"summary": {"total": 1},
|
||||
"entries": [
|
||||
_build_codex_brief_entry(
|
||||
key="runbook:release_progression",
|
||||
executor_kind="runbook_sequence",
|
||||
sequence_key="release_progression",
|
||||
primary_action_code="create_release_rollout_worker",
|
||||
recommendation="confirm_then_execute",
|
||||
automation_level="guarded_auto",
|
||||
target_api="/api/v1/ops/runbook/sequences/release_progression/execute",
|
||||
primary_action_payload={"release_id": 7},
|
||||
secondary_action_code="focus_release_hub",
|
||||
secondary_action_payload={"release_id": 7},
|
||||
focus_ref={
|
||||
"kind": "release_hub",
|
||||
"release_id": 7,
|
||||
"section": "release_launchpad",
|
||||
},
|
||||
primary_focus_ref={
|
||||
"kind": "release_hub",
|
||||
"release_id": 7,
|
||||
"section": "release_launchpad",
|
||||
"mode": "worker",
|
||||
},
|
||||
secondary_focus_ref={
|
||||
"kind": "release_hub",
|
||||
"release_id": 7,
|
||||
"section": "release_detail",
|
||||
},
|
||||
)
|
||||
],
|
||||
}
|
||||
preview_data = _build_preview_data(
|
||||
executor_kind="runbook_sequence",
|
||||
action_code="create_release_rollout_worker",
|
||||
recommendation="confirm_then_execute",
|
||||
automation_level="guarded_auto",
|
||||
target_api="/api/v1/ops/runbook/sequences/release_progression/execute",
|
||||
request_payload={
|
||||
"secondary": False,
|
||||
"requested_by": "test/codex-action",
|
||||
"action_payload": {
|
||||
"release_id": 7,
|
||||
"focus_ref": {
|
||||
"kind": "release_hub",
|
||||
"release_id": 7,
|
||||
"section": "release_launchpad",
|
||||
"mode": "worker",
|
||||
},
|
||||
},
|
||||
},
|
||||
sequence_key="release_progression",
|
||||
focus_ref={
|
||||
"kind": "release_hub",
|
||||
"release_id": 7,
|
||||
"section": "release_launchpad",
|
||||
},
|
||||
primary_focus_ref={
|
||||
"kind": "release_hub",
|
||||
"release_id": 7,
|
||||
"section": "release_launchpad",
|
||||
"mode": "worker",
|
||||
},
|
||||
secondary_focus_ref={
|
||||
"kind": "release_hub",
|
||||
"release_id": 7,
|
||||
"section": "release_detail",
|
||||
},
|
||||
)
|
||||
preview_data["secondary"] = {
|
||||
"label": "查看版本区",
|
||||
"action_code": "focus_release_hub",
|
||||
"target_api": "/api/v1/ops/runbook/sequences/release_progression/execute",
|
||||
"target_method": "POST",
|
||||
"request_payload": {
|
||||
"secondary": True,
|
||||
"requested_by": "test/codex-action",
|
||||
"action_payload": {
|
||||
"release_id": 7,
|
||||
"focus_ref": {
|
||||
"kind": "release_hub",
|
||||
"release_id": 7,
|
||||
"section": "release_detail",
|
||||
},
|
||||
},
|
||||
},
|
||||
"focus_ref": {
|
||||
"kind": "release_hub",
|
||||
"release_id": 7,
|
||||
"section": "release_detail",
|
||||
},
|
||||
"execution_profile": {
|
||||
"action_code": "focus_release_hub",
|
||||
"backend_handled": False,
|
||||
"automation_level": "safe_auto",
|
||||
"recommendation": "auto_execute",
|
||||
"executor_mode_hint": "ui-intent",
|
||||
"risk_level": "low",
|
||||
"confirm_required": False,
|
||||
"ui_only": False,
|
||||
"blocked": False,
|
||||
"requires_fields": [],
|
||||
"missing_fields": [],
|
||||
"reason": "secondary preview reason",
|
||||
},
|
||||
}
|
||||
mock_preview_driver_action.return_value = (True, "ok", preview_data)
|
||||
|
||||
ok, _, data = resolve_codex_action(
|
||||
{
|
||||
"entry_key": "runbook:release_progression",
|
||||
"requested_by": "test/codex-action",
|
||||
"secondary": True,
|
||||
}
|
||||
)
|
||||
|
||||
self.assertTrue(ok)
|
||||
self.assertEqual("secondary", data["selected_section"])
|
||||
self.assertTrue(data["gate"]["secondary"])
|
||||
self.assertEqual("focus_release_hub", data["gate"]["action_code"])
|
||||
self.assertEqual("auto_execute", data["gate"]["recommendation"])
|
||||
self.assertEqual("release_detail", data["gate"]["focus_ref"]["section"])
|
||||
self.assertEqual("release_detail", data["focus_ref"]["section"])
|
||||
|
||||
@patch("app.services.ops_service.preview_driver_action")
|
||||
@patch("app.services.ops_service.get_ops_codex_brief")
|
||||
def test_resolve_codex_action_can_fall_back_to_activity_focus_entry(
|
||||
self,
|
||||
mock_get_ops_codex_brief,
|
||||
mock_preview_driver_action,
|
||||
) -> None:
|
||||
mock_get_ops_codex_brief.return_value = {
|
||||
"focus": {"entry_key": "activity:ops-job-42", "recommendation": "open_ui"},
|
||||
"summary": {"total": 0, "activity_focus_total": 1},
|
||||
"entries": [],
|
||||
"activity_focus": [
|
||||
_build_codex_activity_focus_entry(
|
||||
key="activity:ops-job-42",
|
||||
activity_key="ops-job-42",
|
||||
activity_kind="ops_job",
|
||||
ui_intent_kind="job_events",
|
||||
focus_ref={
|
||||
"kind": "ops_job",
|
||||
"job_key": "job-42",
|
||||
},
|
||||
source_focus_ref={
|
||||
"kind": "ops_job_event",
|
||||
"job_key": "job-42",
|
||||
"event_key": "event-1",
|
||||
},
|
||||
target_node_codes=["mainland-controller-01"],
|
||||
)
|
||||
],
|
||||
}
|
||||
mock_preview_driver_action.return_value = (
|
||||
True,
|
||||
"ok",
|
||||
_build_preview_data(
|
||||
executor_kind="driver_action",
|
||||
action_code="focus_activity_item",
|
||||
recommendation="auto_execute",
|
||||
automation_level="safe_auto",
|
||||
target_api="/api/v1/ops/driver-actions/execute",
|
||||
request_payload={
|
||||
"action_code": "focus_activity_item",
|
||||
"node_codes": ["mainland-controller-01"],
|
||||
"action_payload": {
|
||||
"activity_key": "ops-job-42",
|
||||
"kind": "ops_job",
|
||||
"status": "attention",
|
||||
"ui_intent": {
|
||||
"kind": "job_events",
|
||||
"activity_key": "ops-job-42",
|
||||
},
|
||||
"focus_ref": {
|
||||
"kind": "ops_job",
|
||||
"job_key": "job-42",
|
||||
},
|
||||
},
|
||||
"requested_by": "test/codex-action",
|
||||
},
|
||||
focus_ref={
|
||||
"kind": "ops_job",
|
||||
"job_key": "job-42",
|
||||
},
|
||||
primary_focus_ref={
|
||||
"kind": "ops_job",
|
||||
"job_key": "job-42",
|
||||
},
|
||||
),
|
||||
)
|
||||
|
||||
ok, _, data = resolve_codex_action({"requested_by": "test/codex-action"})
|
||||
|
||||
self.assertTrue(ok)
|
||||
self.assertEqual("activity:ops-job-42", data["entry_key"])
|
||||
self.assertTrue(data["selected_entry"]["observation_only"])
|
||||
self.assertEqual("ops_job", data["selected_entry"]["activity_kind"])
|
||||
self.assertEqual("focus_activity_item", data["preview_request"]["primary_action_code"])
|
||||
self.assertEqual("execute_now", data["gate"]["decision"])
|
||||
self.assertTrue(data["gate"]["will_execute"])
|
||||
self.assertEqual("job-42", data["focus_ref"]["job_key"])
|
||||
|
||||
@patch("app.services.ops_service.execute_driver_action")
|
||||
@patch("app.services.ops_service.preview_driver_action")
|
||||
@patch("app.services.ops_service.get_ops_codex_brief")
|
||||
def test_execute_codex_action_runs_safe_auto_driver_action(
|
||||
self,
|
||||
mock_get_ops_codex_brief,
|
||||
mock_preview_driver_action,
|
||||
mock_execute_driver_action,
|
||||
) -> None:
|
||||
mock_get_ops_codex_brief.return_value = {
|
||||
"focus": {"entry_key": "recommendation:enable-log-sync"},
|
||||
"summary": {"total": 1},
|
||||
"entries": [
|
||||
_build_codex_brief_entry(
|
||||
key="recommendation:enable-log-sync",
|
||||
executor_kind="driver_action",
|
||||
primary_action_code="enable_log_sync_key",
|
||||
recommendation="auto_execute",
|
||||
automation_level="safe_auto",
|
||||
target_api="/api/v1/ops/driver-actions/execute",
|
||||
)
|
||||
],
|
||||
}
|
||||
request_payload = {
|
||||
"action_code": "enable_log_sync_key",
|
||||
"node_codes": ["mainland-controller-01"],
|
||||
"action_payload": {},
|
||||
"requested_by": "test/codex-action",
|
||||
}
|
||||
mock_preview_driver_action.return_value = (
|
||||
True,
|
||||
"ok",
|
||||
_build_preview_data(
|
||||
executor_kind="driver_action",
|
||||
action_code="enable_log_sync_key",
|
||||
recommendation="auto_execute",
|
||||
automation_level="safe_auto",
|
||||
target_api="/api/v1/ops/driver-actions/execute",
|
||||
request_payload=request_payload,
|
||||
),
|
||||
)
|
||||
mock_execute_driver_action.return_value = (
|
||||
True,
|
||||
"执行成功",
|
||||
{"handled": True, "action_code": "enable_log_sync_key"},
|
||||
)
|
||||
|
||||
ok, message, data = execute_codex_action({"requested_by": "test/codex-action"})
|
||||
|
||||
self.assertTrue(ok)
|
||||
self.assertEqual("执行成功", message)
|
||||
self.assertTrue(data["executed"])
|
||||
mock_execute_driver_action.assert_called_once_with(request_payload)
|
||||
|
||||
@patch("app.services.ops_service.execute_ops_runbook_sequence")
|
||||
@patch("app.services.ops_service.preview_driver_action")
|
||||
@patch("app.services.ops_service.get_ops_codex_brief")
|
||||
def test_execute_codex_action_blocks_guarded_runbook_without_confirm(
|
||||
self,
|
||||
mock_get_ops_codex_brief,
|
||||
mock_preview_driver_action,
|
||||
mock_execute_ops_runbook_sequence,
|
||||
) -> None:
|
||||
mock_get_ops_codex_brief.return_value = {
|
||||
"focus": {"entry_key": "runbook:release_progression"},
|
||||
"summary": {"total": 1},
|
||||
"entries": [
|
||||
_build_codex_brief_entry(
|
||||
key="runbook:release_progression",
|
||||
executor_kind="runbook_sequence",
|
||||
sequence_key="release_progression",
|
||||
primary_action_code="create_release_rollout_worker",
|
||||
recommendation="confirm_then_execute",
|
||||
automation_level="guarded_auto",
|
||||
target_api="/api/v1/ops/runbook/sequences/release_progression/execute",
|
||||
primary_action_payload={"release_id": 7},
|
||||
)
|
||||
],
|
||||
}
|
||||
mock_preview_driver_action.return_value = (
|
||||
True,
|
||||
"ok",
|
||||
_build_preview_data(
|
||||
executor_kind="runbook_sequence",
|
||||
action_code="create_release_rollout_worker",
|
||||
recommendation="confirm_then_execute",
|
||||
automation_level="guarded_auto",
|
||||
target_api="/api/v1/ops/runbook/sequences/release_progression/execute",
|
||||
request_payload={
|
||||
"secondary": False,
|
||||
"requested_by": "test/codex-action",
|
||||
"action_payload": {"release_id": 7},
|
||||
},
|
||||
sequence_key="release_progression",
|
||||
),
|
||||
)
|
||||
|
||||
ok, message, data = execute_codex_action(
|
||||
{
|
||||
"entry_key": "runbook:release_progression",
|
||||
"requested_by": "test/codex-action",
|
||||
}
|
||||
)
|
||||
|
||||
self.assertFalse(ok)
|
||||
self.assertIn("确认", message)
|
||||
self.assertFalse(data["executed"])
|
||||
mock_execute_ops_runbook_sequence.assert_not_called()
|
||||
|
||||
@patch("app.services.ops_service.execute_driver_action")
|
||||
@patch("app.services.ops_service.preview_driver_action")
|
||||
@patch("app.services.ops_service.get_ops_codex_brief")
|
||||
def test_execute_codex_action_runs_activity_focus_as_safe_auto(
|
||||
self,
|
||||
mock_get_ops_codex_brief,
|
||||
mock_preview_driver_action,
|
||||
mock_execute_driver_action,
|
||||
) -> None:
|
||||
mock_get_ops_codex_brief.return_value = {
|
||||
"focus": {"entry_key": "activity:ops-job-42", "recommendation": "open_ui"},
|
||||
"summary": {"total": 0, "activity_focus_total": 1},
|
||||
"entries": [],
|
||||
"activity_focus": [
|
||||
_build_codex_activity_focus_entry(
|
||||
key="activity:ops-job-42",
|
||||
activity_key="ops-job-42",
|
||||
activity_kind="ops_job",
|
||||
ui_intent_kind="job_events",
|
||||
focus_ref={
|
||||
"kind": "ops_job",
|
||||
"job_key": "job-42",
|
||||
},
|
||||
)
|
||||
],
|
||||
}
|
||||
mock_preview_driver_action.return_value = (
|
||||
True,
|
||||
"ok",
|
||||
_build_preview_data(
|
||||
executor_kind="driver_action",
|
||||
action_code="focus_activity_item",
|
||||
recommendation="auto_execute",
|
||||
automation_level="safe_auto",
|
||||
target_api="/api/v1/ops/driver-actions/execute",
|
||||
request_payload={
|
||||
"action_code": "focus_activity_item",
|
||||
"node_codes": [],
|
||||
"action_payload": {
|
||||
"activity_key": "ops-job-42",
|
||||
"kind": "ops_job",
|
||||
"status": "attention",
|
||||
"ui_intent": {
|
||||
"kind": "job_events",
|
||||
"activity_key": "ops-job-42",
|
||||
},
|
||||
"focus_ref": {
|
||||
"kind": "ops_job",
|
||||
"job_key": "job-42",
|
||||
},
|
||||
},
|
||||
"requested_by": "test/codex-action",
|
||||
},
|
||||
focus_ref={
|
||||
"kind": "ops_job",
|
||||
"job_key": "job-42",
|
||||
},
|
||||
primary_focus_ref={
|
||||
"kind": "ops_job",
|
||||
"job_key": "job-42",
|
||||
},
|
||||
),
|
||||
)
|
||||
mock_execute_driver_action.return_value = (
|
||||
True,
|
||||
"执行成功",
|
||||
{"handled": True, "action_code": "focus_activity_item"},
|
||||
)
|
||||
|
||||
ok, message, data = execute_codex_action({"requested_by": "test/codex-action"})
|
||||
|
||||
self.assertTrue(ok)
|
||||
self.assertEqual("执行成功", message)
|
||||
self.assertTrue(data["executed"])
|
||||
mock_execute_driver_action.assert_called_once()
|
||||
|
||||
@patch("app.services.ops_service.preview_driver_action")
|
||||
@patch("app.services.ops_service.get_ops_codex_brief")
|
||||
def test_resolve_codex_action_supports_focus_latest_job_events_without_ui_intent(
|
||||
self,
|
||||
mock_get_ops_codex_brief,
|
||||
mock_preview_driver_action,
|
||||
) -> None:
|
||||
mock_get_ops_codex_brief.return_value = {
|
||||
"focus": {"entry_key": "activity:ops-job-81", "recommendation": "open_ui"},
|
||||
"summary": {"total": 0, "activity_focus_total": 1},
|
||||
"entries": [],
|
||||
"activity_focus": [
|
||||
{
|
||||
"key": "activity:ops-job-81",
|
||||
"activity_key": "ops-job-81",
|
||||
"activity_kind": "ops_job",
|
||||
"title": "开始回执异常",
|
||||
"subtitle": "ops-agent-81",
|
||||
"summary": "节点已接单执行,但开始回执未成功送达控制面。",
|
||||
"status": "attention",
|
||||
"status_label": "待处理",
|
||||
"meta_text": "ops-agent-81",
|
||||
"occurred_at": "2026-04-18T06:00:00",
|
||||
"target_node_codes": ["mainland-worker-01"],
|
||||
"focus_ref": {
|
||||
"kind": "ops_job",
|
||||
"job_id": 81,
|
||||
"job_code": "ops-agent-81",
|
||||
"target_node_code": "mainland-worker-01",
|
||||
},
|
||||
"source_focus_ref": {
|
||||
"kind": "ops_job_event",
|
||||
"job_id": 81,
|
||||
"job_code": "ops-agent-81",
|
||||
"event_key": "job-event:81",
|
||||
"target_node_code": "mainland-worker-01",
|
||||
},
|
||||
"ui_intent": {},
|
||||
"ui_intent_kind": "",
|
||||
"focus_action_code": "focus_latest_job_events",
|
||||
"focus_action_payload": {
|
||||
"focus_ref": {
|
||||
"kind": "ops_job",
|
||||
"job_id": 81,
|
||||
"job_code": "ops-agent-81",
|
||||
"target_node_code": "mainland-worker-01",
|
||||
},
|
||||
"source_focus_ref": {
|
||||
"kind": "ops_job_event",
|
||||
"job_id": 81,
|
||||
"job_code": "ops-agent-81",
|
||||
"event_key": "job-event:81",
|
||||
"target_node_code": "mainland-worker-01",
|
||||
},
|
||||
},
|
||||
"observation_only": True,
|
||||
}
|
||||
],
|
||||
}
|
||||
mock_preview_driver_action.return_value = (
|
||||
True,
|
||||
"ok",
|
||||
_build_preview_data(
|
||||
executor_kind="driver_action",
|
||||
action_code="focus_latest_job_events",
|
||||
recommendation="auto_execute",
|
||||
automation_level="safe_auto",
|
||||
target_api="/api/v1/ops/driver-actions/execute",
|
||||
request_payload={
|
||||
"action_code": "focus_latest_job_events",
|
||||
"node_codes": ["mainland-worker-01"],
|
||||
"action_payload": {
|
||||
"focus_ref": {
|
||||
"kind": "ops_job",
|
||||
"job_id": 81,
|
||||
"job_code": "ops-agent-81",
|
||||
"target_node_code": "mainland-worker-01",
|
||||
},
|
||||
"source_focus_ref": {
|
||||
"kind": "ops_job_event",
|
||||
"job_id": 81,
|
||||
"job_code": "ops-agent-81",
|
||||
"event_key": "job-event:81",
|
||||
"target_node_code": "mainland-worker-01",
|
||||
},
|
||||
},
|
||||
"requested_by": "test/codex-action",
|
||||
},
|
||||
focus_ref={
|
||||
"kind": "ops_job",
|
||||
"job_id": 81,
|
||||
"job_code": "ops-agent-81",
|
||||
},
|
||||
primary_focus_ref={
|
||||
"kind": "ops_job",
|
||||
"job_id": 81,
|
||||
"job_code": "ops-agent-81",
|
||||
},
|
||||
),
|
||||
)
|
||||
|
||||
ok, _, data = resolve_codex_action({"requested_by": "test/codex-action"})
|
||||
|
||||
self.assertTrue(ok)
|
||||
self.assertEqual("activity:ops-job-81", data["entry_key"])
|
||||
self.assertEqual("focus_latest_job_events", data["preview_request"]["primary_action_code"])
|
||||
self.assertEqual("execute_now", data["gate"]["decision"])
|
||||
self.assertTrue(data["gate"]["will_execute"])
|
||||
|
||||
@patch("app.services.ops_service.execute_ops_runbook_sequence")
|
||||
@patch("app.services.ops_service.preview_driver_action")
|
||||
@patch("app.services.ops_service.get_ops_codex_brief")
|
||||
def test_execute_codex_action_runs_guarded_runbook_after_confirm(
|
||||
self,
|
||||
mock_get_ops_codex_brief,
|
||||
mock_preview_driver_action,
|
||||
mock_execute_ops_runbook_sequence,
|
||||
) -> None:
|
||||
mock_get_ops_codex_brief.return_value = {
|
||||
"focus": {"entry_key": "runbook:release_progression"},
|
||||
"summary": {"total": 1},
|
||||
"entries": [
|
||||
_build_codex_brief_entry(
|
||||
key="runbook:release_progression",
|
||||
executor_kind="runbook_sequence",
|
||||
sequence_key="release_progression",
|
||||
primary_action_code="create_release_rollout_worker",
|
||||
recommendation="confirm_then_execute",
|
||||
automation_level="guarded_auto",
|
||||
target_api="/api/v1/ops/runbook/sequences/release_progression/execute",
|
||||
primary_action_payload={"release_id": 7},
|
||||
)
|
||||
],
|
||||
}
|
||||
request_payload = {
|
||||
"secondary": False,
|
||||
"requested_by": "test/codex-action",
|
||||
"action_payload": {"release_id": 7},
|
||||
}
|
||||
mock_preview_driver_action.return_value = (
|
||||
True,
|
||||
"ok",
|
||||
_build_preview_data(
|
||||
executor_kind="runbook_sequence",
|
||||
action_code="create_release_rollout_worker",
|
||||
recommendation="confirm_then_execute",
|
||||
automation_level="guarded_auto",
|
||||
target_api="/api/v1/ops/runbook/sequences/release_progression/execute",
|
||||
request_payload=request_payload,
|
||||
sequence_key="release_progression",
|
||||
),
|
||||
)
|
||||
mock_execute_ops_runbook_sequence.return_value = (
|
||||
True,
|
||||
"runbook 执行成功",
|
||||
{"sequence_key": "release_progression", "driver_action_code": "create_release_rollout_worker"},
|
||||
)
|
||||
|
||||
ok, message, data = execute_codex_action(
|
||||
{
|
||||
"entry_key": "runbook:release_progression",
|
||||
"requested_by": "test/codex-action",
|
||||
"confirm": True,
|
||||
}
|
||||
)
|
||||
|
||||
self.assertTrue(ok)
|
||||
self.assertEqual("runbook 执行成功", message)
|
||||
self.assertTrue(data["executed"])
|
||||
mock_execute_ops_runbook_sequence.assert_called_once_with("release_progression", request_payload)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
497
domain-api/tests/test_ops_service_codex_brief.py
Normal file
497
domain-api/tests/test_ops_service_codex_brief.py
Normal file
@@ -0,0 +1,497 @@
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
from app.services.ops_service import get_ops_codex_brief
|
||||
|
||||
|
||||
class OpsServiceCodexBriefTests(unittest.TestCase):
|
||||
@patch("app.services.ops_service.get_ops_driver_feed")
|
||||
def test_codex_brief_classifies_safe_ui_and_runbook_entries(self, mock_get_ops_driver_feed) -> None:
|
||||
mock_get_ops_driver_feed.return_value = {
|
||||
"headline": "当前建议已形成。",
|
||||
"go_live_summary": {
|
||||
"go_live_status": "attention",
|
||||
"publish_ready": False,
|
||||
"publish_status": "attention",
|
||||
"publish_status_label": "可发布但建议先复核",
|
||||
"operator_title": "先补现场日志回传",
|
||||
"next_step_action_code": "enable_log_sync_key",
|
||||
"launchpad_recommended_target_node_code": "mainland-controller-01",
|
||||
"launchpad_recommended_recovery_label": "签发接入工单",
|
||||
"launchpad_recommended_recovery_summary": "当前节点仍处于接入阶段,建议先签发 onboarding.bootstrap。",
|
||||
"launchpad_onboarding_bootstrap_pending_nodes": 1,
|
||||
"launchpad_onboarding_acceptance_ready_nodes": 1,
|
||||
},
|
||||
"activity_focus": [],
|
||||
"entries": [
|
||||
{
|
||||
"key": "recommendation:enable-scene-log-sync",
|
||||
"kind": "driver_recommendation",
|
||||
"lane": "do_now",
|
||||
"lane_label": "建议动作",
|
||||
"title": "先补现场日志回传",
|
||||
"summary": "优先打开关键日志回传。",
|
||||
"reason": "当前还看不到现场过程日志。",
|
||||
"status": "attention",
|
||||
"level_label": "最高优先",
|
||||
"tag_type": "danger",
|
||||
"meta_text": "",
|
||||
"detail_lines": ["参与节点 2 台"],
|
||||
"node_codes": ["mainland-worker-01"],
|
||||
"executor_kind": "driver_action",
|
||||
"focus_ref": {
|
||||
"kind": "execution_scene",
|
||||
"scene_key": "scene.logs",
|
||||
},
|
||||
"primary_focus_ref": {
|
||||
"kind": "execution_scene",
|
||||
"scene_key": "scene.logs",
|
||||
},
|
||||
"secondary_focus_ref": {
|
||||
"kind": "execution_scene",
|
||||
"scene_key": "scene.logs.full",
|
||||
},
|
||||
"primary_label": "开启关键回传",
|
||||
"secondary_label": "开启全量回传",
|
||||
"primary_action_code": "enable_log_sync_key",
|
||||
"secondary_action_code": "enable_log_sync_full",
|
||||
"primary_node_codes": [],
|
||||
"secondary_node_codes": [],
|
||||
"primary_action_payload": {
|
||||
"focus_ref": {
|
||||
"kind": "execution_scene",
|
||||
"scene_key": "scene.logs",
|
||||
}
|
||||
},
|
||||
"secondary_action_payload": {
|
||||
"focus_ref": {
|
||||
"kind": "execution_scene",
|
||||
"scene_key": "scene.logs.full",
|
||||
}
|
||||
},
|
||||
},
|
||||
{
|
||||
"key": "recommendation:release-missing",
|
||||
"kind": "driver_recommendation",
|
||||
"lane": "do_now",
|
||||
"lane_label": "建议动作",
|
||||
"title": "先建立 Release",
|
||||
"summary": "当前还没有默认 Release。",
|
||||
"reason": "需要先进入版本区。",
|
||||
"status": "planned",
|
||||
"level_label": "待补齐",
|
||||
"tag_type": "info",
|
||||
"meta_text": "",
|
||||
"detail_lines": [],
|
||||
"node_codes": [],
|
||||
"executor_kind": "driver_action",
|
||||
"focus_ref": {
|
||||
"kind": "release_hub",
|
||||
"release_id": 7,
|
||||
"section": "release_launchpad",
|
||||
},
|
||||
"primary_focus_ref": {
|
||||
"kind": "release_hub",
|
||||
"release_id": 7,
|
||||
"section": "release_launchpad",
|
||||
},
|
||||
"primary_label": "创建 Release",
|
||||
"secondary_label": "",
|
||||
"primary_action_code": "open_release_dialog",
|
||||
"secondary_action_code": "",
|
||||
"primary_node_codes": [],
|
||||
"secondary_node_codes": [],
|
||||
"primary_action_payload": {},
|
||||
"secondary_action_payload": {},
|
||||
},
|
||||
{
|
||||
"key": "runbook:release_progression",
|
||||
"kind": "runbook_sequence",
|
||||
"lane": "standard_path",
|
||||
"lane_label": "标准路径",
|
||||
"title": "版本发布与放量",
|
||||
"summary": "当前可以进入正式发布路径。",
|
||||
"reason": "统一回到 Release / Rollout 闭环。",
|
||||
"status": "attention",
|
||||
"level_label": "待处理",
|
||||
"tag_type": "warning",
|
||||
"meta_text": "Release / Rollout",
|
||||
"detail_lines": ["当前解析:Worker 灰度"],
|
||||
"node_codes": ["mainland-controller-01"],
|
||||
"executor_kind": "runbook_sequence",
|
||||
"sequence_key": "release_progression",
|
||||
"focus_ref": {
|
||||
"kind": "release_hub",
|
||||
"release_id": 7,
|
||||
"section": "release_launchpad",
|
||||
},
|
||||
"primary_focus_ref": {
|
||||
"kind": "release_hub",
|
||||
"release_id": 7,
|
||||
"section": "release_launchpad",
|
||||
"mode": "worker",
|
||||
},
|
||||
"secondary_focus_ref": {
|
||||
"kind": "release_hub",
|
||||
"release_id": 7,
|
||||
"section": "release_detail",
|
||||
},
|
||||
"primary_label": "Worker 灰度",
|
||||
"secondary_label": "查看版本区",
|
||||
"primary_action_code": "create_release_rollout_worker",
|
||||
"secondary_action_code": "focus_release_hub",
|
||||
"primary_node_codes": ["mainland-controller-01"],
|
||||
"secondary_node_codes": [],
|
||||
"primary_action_payload": {
|
||||
"release_id": 7,
|
||||
"focus_ref": {
|
||||
"kind": "release_hub",
|
||||
"release_id": 7,
|
||||
"section": "release_launchpad",
|
||||
"mode": "worker",
|
||||
},
|
||||
},
|
||||
"secondary_action_payload": {
|
||||
"release_id": 7,
|
||||
"focus_ref": {
|
||||
"kind": "release_hub",
|
||||
"release_id": 7,
|
||||
"section": "release_detail",
|
||||
},
|
||||
},
|
||||
"occurred_at": "2026-04-18T06:00:00",
|
||||
},
|
||||
],
|
||||
}
|
||||
|
||||
payload = get_ops_codex_brief()
|
||||
|
||||
self.assertEqual(3, payload["summary"]["total"])
|
||||
self.assertEqual("attention", payload["go_live_summary"]["go_live_status"])
|
||||
self.assertEqual("attention", payload["summary"]["go_live_status"])
|
||||
self.assertEqual("先补现场日志回传", payload["summary"]["operator_title"])
|
||||
self.assertEqual("attention", payload["summary"]["publish_status"])
|
||||
self.assertFalse(payload["summary"]["publish_ready"])
|
||||
self.assertEqual("可发布但建议先复核", payload["summary"]["publish_status_label"])
|
||||
self.assertEqual("mainland-controller-01", payload["summary"]["launchpad_recommended_target_node_code"])
|
||||
self.assertEqual("签发接入工单", payload["summary"]["launchpad_recommended_recovery_label"])
|
||||
self.assertEqual(
|
||||
"当前节点仍处于接入阶段,建议先签发 onboarding.bootstrap。",
|
||||
payload["summary"]["launchpad_recommended_recovery_summary"],
|
||||
)
|
||||
self.assertEqual(1, payload["summary"]["launchpad_onboarding_bootstrap_pending_nodes"])
|
||||
self.assertEqual(1, payload["summary"]["launchpad_onboarding_acceptance_ready_nodes"])
|
||||
self.assertEqual(0, payload["summary"]["activity_focus_total"])
|
||||
self.assertEqual("当前建议已形成。", payload["driver_feed"]["headline"])
|
||||
self.assertEqual("ops_driver_contract", payload["contract_navigation"]["primary_contract_key"])
|
||||
self.assertIn("ops_stack_diagnosis_contract", payload["contract_navigation"]["contract_keys"])
|
||||
self.assertEqual(2, payload["summary"]["auto_execute_total"])
|
||||
self.assertEqual(1, payload["summary"]["confirm_then_execute_total"])
|
||||
self.assertEqual(0, payload["summary"]["open_ui_total"])
|
||||
self.assertEqual(1, payload["summary"]["preview_only_total"])
|
||||
self.assertEqual(3, payload["summary"]["backend_handled_total"])
|
||||
self.assertEqual("attention", payload["summary"]["launch_status"])
|
||||
self.assertFalse(payload["summary"]["launch_ready"])
|
||||
self.assertEqual("recommendation:enable-scene-log-sync", payload["focus"]["entry_key"])
|
||||
self.assertEqual("execution_scene", payload["focus"]["focus_ref"]["kind"])
|
||||
self.assertEqual("scene.logs", payload["focus"]["primary_focus_ref"]["scene_key"])
|
||||
self.assertEqual("auto_execute", payload["entries"][0]["autopilot"]["recommendation"])
|
||||
self.assertIn("ops_observability_contract", payload["entries"][0]["contract_keys"])
|
||||
self.assertIn("ops_stack_diagnosis_contract", payload["entries"][0]["contract_keys"])
|
||||
self.assertEqual("scene.logs", payload["entries"][0]["primary_focus_ref"]["scene_key"])
|
||||
self.assertEqual("safe_auto", payload["entries"][1]["autopilot"]["automation_level"])
|
||||
self.assertIn("release_hub_contract", payload["entries"][1]["contract_keys"])
|
||||
|
||||
runbook_entry = payload["entries"][2]
|
||||
self.assertEqual("runbook_sequence", runbook_entry["executor_kind"])
|
||||
self.assertEqual("confirm_then_execute", runbook_entry["autopilot"]["recommendation"])
|
||||
self.assertIn("ops_playbook_contract", runbook_entry["contract_keys"])
|
||||
self.assertIn("release_hub_contract", runbook_entry["contract_keys"])
|
||||
self.assertEqual("release_launchpad", runbook_entry["focus_ref"]["section"])
|
||||
self.assertEqual("worker", runbook_entry["primary_focus_ref"]["mode"])
|
||||
self.assertEqual("release_detail", runbook_entry["secondary_focus_ref"]["section"])
|
||||
self.assertEqual(
|
||||
"/api/v1/ops/runbook/sequences/release_progression/execute",
|
||||
runbook_entry["autopilot"]["target_api"],
|
||||
)
|
||||
self.assertEqual(
|
||||
{
|
||||
"sequence_key": "release_progression",
|
||||
"secondary": False,
|
||||
"action_payload": {
|
||||
"release_id": 7,
|
||||
"focus_ref": {
|
||||
"kind": "release_hub",
|
||||
"release_id": 7,
|
||||
"section": "release_launchpad",
|
||||
"mode": "worker",
|
||||
},
|
||||
},
|
||||
},
|
||||
runbook_entry["primary_execution"]["request_payload_preview"],
|
||||
)
|
||||
self.assertEqual(
|
||||
{
|
||||
"sequence_key": "release_progression",
|
||||
"secondary": True,
|
||||
"action_payload": {
|
||||
"release_id": 7,
|
||||
"focus_ref": {
|
||||
"kind": "release_hub",
|
||||
"release_id": 7,
|
||||
"section": "release_detail",
|
||||
},
|
||||
},
|
||||
},
|
||||
runbook_entry["secondary_execution"]["request_payload_preview"],
|
||||
)
|
||||
self.assertEqual("attention", payload["automation_coverage"]["launch_status"])
|
||||
self.assertEqual(2, payload["automation_coverage"]["safe_auto_total"])
|
||||
self.assertEqual(1, payload["automation_coverage"]["guarded_auto_total"])
|
||||
self.assertEqual(1, payload["automation_coverage"]["preview_only_total"])
|
||||
self.assertEqual(3, payload["automation_coverage"]["backend_handled_total"])
|
||||
|
||||
@patch("app.services.ops_service.get_ops_driver_feed")
|
||||
def test_codex_brief_marks_mixed_and_blocked_entries(self, mock_get_ops_driver_feed) -> None:
|
||||
mock_get_ops_driver_feed.return_value = {
|
||||
"headline": "当前建议需要先复核。",
|
||||
"go_live_summary": {
|
||||
"go_live_status": "blocked",
|
||||
"publish_ready": False,
|
||||
"publish_status": "blocked",
|
||||
"publish_status_label": "暂不可发布",
|
||||
"operator_title": "先补执行面接管",
|
||||
"next_step_action_code": "fix_managed_nodes",
|
||||
"launchpad_recommended_target_node_code": "mainland-controller-01",
|
||||
"launchpad_recommended_recovery_label": "补执行面接管",
|
||||
"launchpad_recommended_recovery_summary": "先处理 launchpad 缺口。",
|
||||
"launchpad_onboarding_bootstrap_pending_nodes": 1,
|
||||
"launchpad_onboarding_acceptance_ready_nodes": 0,
|
||||
},
|
||||
"activity_focus": [],
|
||||
"entries": [
|
||||
{
|
||||
"key": "recommendation:release-launchpad-fix-managed",
|
||||
"kind": "driver_recommendation",
|
||||
"lane": "do_now",
|
||||
"lane_label": "建议动作",
|
||||
"title": "先补执行面接管",
|
||||
"summary": "先处理 launchpad 缺口。",
|
||||
"reason": "当前动作可能直接 sync,也可能转为补接入。",
|
||||
"status": "attention",
|
||||
"level_label": "最高优先",
|
||||
"tag_type": "danger",
|
||||
"meta_text": "",
|
||||
"detail_lines": [],
|
||||
"node_codes": ["mainland-controller-01"],
|
||||
"executor_kind": "driver_action",
|
||||
"focus_ref": {
|
||||
"kind": "release_hub",
|
||||
"release_id": 9,
|
||||
"section": "release_launchpad",
|
||||
},
|
||||
"primary_focus_ref": {
|
||||
"kind": "release_hub",
|
||||
"release_id": 9,
|
||||
"section": "default_rollout_gate",
|
||||
},
|
||||
"primary_label": "补执行面接管",
|
||||
"secondary_label": "",
|
||||
"primary_action_code": "fix_managed_nodes",
|
||||
"secondary_action_code": "",
|
||||
"primary_node_codes": ["mainland-controller-01"],
|
||||
"secondary_node_codes": [],
|
||||
"primary_action_payload": {"channel": "stable"},
|
||||
"secondary_action_payload": {},
|
||||
},
|
||||
{
|
||||
"key": "recommendation:inspect-participating",
|
||||
"kind": "driver_recommendation",
|
||||
"lane": "do_now",
|
||||
"lane_label": "建议动作",
|
||||
"title": "先看执行现场",
|
||||
"summary": "先跑参与节点巡检。",
|
||||
"reason": "缺少目标节点时不应误判为可自动执行。",
|
||||
"status": "attention",
|
||||
"level_label": "推荐",
|
||||
"tag_type": "warning",
|
||||
"meta_text": "",
|
||||
"detail_lines": [],
|
||||
"node_codes": [],
|
||||
"executor_kind": "driver_action",
|
||||
"primary_label": "执行标准巡检",
|
||||
"secondary_label": "",
|
||||
"primary_action_code": "run_inspection_participating",
|
||||
"secondary_action_code": "",
|
||||
"primary_node_codes": [],
|
||||
"secondary_node_codes": [],
|
||||
"primary_action_payload": {},
|
||||
"secondary_action_payload": {},
|
||||
},
|
||||
],
|
||||
}
|
||||
|
||||
payload = get_ops_codex_brief()
|
||||
|
||||
self.assertEqual("blocked", payload["summary"]["publish_status"])
|
||||
self.assertEqual("暂不可发布", payload["summary"]["publish_status_label"])
|
||||
self.assertEqual("mainland-controller-01", payload["summary"]["launchpad_recommended_target_node_code"])
|
||||
self.assertEqual("补执行面接管", payload["summary"]["launchpad_recommended_recovery_label"])
|
||||
self.assertEqual(1, payload["summary"]["launchpad_onboarding_bootstrap_pending_nodes"])
|
||||
self.assertEqual(0, payload["summary"]["launchpad_onboarding_acceptance_ready_nodes"])
|
||||
self.assertEqual(1, payload["summary"]["resolve_first_total"])
|
||||
self.assertEqual(1, payload["summary"]["blocked_total"])
|
||||
self.assertEqual("blocked", payload["summary"]["launch_status"])
|
||||
self.assertFalse(payload["summary"]["launch_ready"])
|
||||
self.assertEqual("mixed", payload["entries"][0]["autopilot"]["automation_level"])
|
||||
self.assertEqual("resolve_first", payload["entries"][0]["autopilot"]["recommendation"])
|
||||
self.assertEqual("default_rollout_gate", payload["entries"][0]["primary_focus_ref"]["section"])
|
||||
self.assertEqual(1, payload["automation_coverage"]["mixed_total"])
|
||||
self.assertEqual(1, payload["automation_coverage"]["blocked_total"])
|
||||
self.assertEqual(2, payload["automation_coverage"]["human_dependency_total"])
|
||||
|
||||
blocked_entry = payload["entries"][1]
|
||||
self.assertTrue(blocked_entry["autopilot"]["blocked"])
|
||||
self.assertEqual(["node_codes"], blocked_entry["primary_execution"]["missing_fields"])
|
||||
self.assertEqual("blocked", blocked_entry["autopilot"]["recommendation"])
|
||||
|
||||
@patch("app.services.ops_service.get_ops_driver_feed")
|
||||
def test_codex_brief_exposes_activity_focus_and_uses_it_as_fallback_focus(self, mock_get_ops_driver_feed) -> None:
|
||||
mock_get_ops_driver_feed.return_value = {
|
||||
"headline": "当前先盯住执行现场。",
|
||||
"scene_log_observation": {},
|
||||
"entries": [],
|
||||
"activity_focus": [
|
||||
{
|
||||
"key": "activity:execution-scene:live",
|
||||
"activity_key": "execution-scene:live",
|
||||
"activity_kind": "execution_scene",
|
||||
"title": "当前执行现场",
|
||||
"subtitle": "运行中",
|
||||
"summary": "当前参与检测节点 2 台,建议先看现场与日志回传。",
|
||||
"status": "running",
|
||||
"status_label": "执行中",
|
||||
"meta_text": "执行/已领 2 / 待命 1",
|
||||
"occurred_at": "2026-04-18 09:30:00",
|
||||
"target_node_codes": ["mainland-worker-01", "mainland-controller-01"],
|
||||
"focus_ref": {
|
||||
"kind": "execution_scene",
|
||||
"scene_key": "scene.logs",
|
||||
},
|
||||
"source_focus_ref": {},
|
||||
"ui_intent": {
|
||||
"kind": "focus_execution_scene",
|
||||
"open_log_drawer": True,
|
||||
},
|
||||
"ui_intent_kind": "focus_execution_scene",
|
||||
"focus_action_code": "focus_activity_item",
|
||||
"focus_action_payload": {
|
||||
"activity_key": "execution-scene:live",
|
||||
"kind": "execution_scene",
|
||||
"status": "running",
|
||||
"ui_intent": {
|
||||
"kind": "focus_execution_scene",
|
||||
"open_log_drawer": True,
|
||||
},
|
||||
"focus_ref": {
|
||||
"kind": "execution_scene",
|
||||
"scene_key": "scene.logs",
|
||||
},
|
||||
},
|
||||
"observation_only": True,
|
||||
}
|
||||
],
|
||||
}
|
||||
|
||||
payload = get_ops_codex_brief()
|
||||
|
||||
self.assertEqual(0, payload["summary"]["total"])
|
||||
self.assertEqual(1, payload["summary"]["activity_focus_total"])
|
||||
self.assertEqual("activity:execution-scene:live", payload["focus"]["entry_key"])
|
||||
self.assertEqual("open_ui", payload["focus"]["recommendation"])
|
||||
self.assertIn("ops_observability_contract", payload["focus"]["contract_keys"])
|
||||
self.assertEqual("execution_scene", payload["focus"]["focus_ref"]["kind"])
|
||||
self.assertEqual(1, len(payload["activity_focus"]))
|
||||
self.assertEqual("focus_execution_scene", payload["activity_focus"][0]["ui_intent_kind"])
|
||||
self.assertEqual(
|
||||
"ops_driver_contract",
|
||||
payload["activity_focus"][0]["contract_navigation"]["primary_contract_key"],
|
||||
)
|
||||
self.assertTrue(payload["activity_focus"][0]["observation_only"])
|
||||
|
||||
@patch("app.services.ops_service.get_ops_driver_feed")
|
||||
def test_codex_brief_can_fallback_to_scene_log_observation_focus(self, mock_get_ops_driver_feed) -> None:
|
||||
mock_get_ops_driver_feed.return_value = {
|
||||
"headline": "先看现场日志。",
|
||||
"entries": [],
|
||||
"activity_focus": [],
|
||||
"scene_log_observation": {
|
||||
"status": "waiting_sample",
|
||||
"status_label": "等待样本",
|
||||
"summary": "日志回传已开启,但样本还没形成。",
|
||||
"focus_ref": {
|
||||
"kind": "node_scene_log",
|
||||
"node_code": "mainland-worker-01",
|
||||
"mode": "key",
|
||||
"limit": 120,
|
||||
},
|
||||
"contract_keys": ["ops_observability_contract", "ops_stack_diagnosis_contract"],
|
||||
},
|
||||
}
|
||||
|
||||
payload = get_ops_codex_brief()
|
||||
|
||||
self.assertEqual("scene-log-observation", payload["focus"]["entry_key"])
|
||||
self.assertEqual("open_ui", payload["focus"]["recommendation"])
|
||||
self.assertEqual("node_scene_log", payload["focus"]["focus_ref"]["kind"])
|
||||
self.assertEqual("mainland-worker-01", payload["focus"]["focus_ref"]["node_code"])
|
||||
self.assertEqual("waiting_sample", payload["scene_log_observation"]["status"])
|
||||
|
||||
@patch("app.services.ops_service.get_ops_driver_feed")
|
||||
def test_codex_brief_marks_bootstrap_run_as_agent_protocol_flow(self, mock_get_ops_driver_feed) -> None:
|
||||
mock_get_ops_driver_feed.return_value = {
|
||||
"headline": "先执行节点接入收口。",
|
||||
"activity_focus": [],
|
||||
"entries": [
|
||||
{
|
||||
"key": "priority:handover-gap-bootstrap_run",
|
||||
"kind": "priority_recommendation",
|
||||
"lane": "do_now",
|
||||
"lane_label": "当前优先",
|
||||
"title": "先补接管缺口",
|
||||
"summary": "Launchpad 已明确给出首台节点的接入收口动作。",
|
||||
"reason": "mainland-worker-01 还缺接入收口,先执行 bootstrap。",
|
||||
"status": "attention",
|
||||
"level_label": "最高优先",
|
||||
"tag_type": "warning",
|
||||
"meta_text": "overview.recommendation",
|
||||
"detail_lines": ["当前已收敛到单节点接入动作。"],
|
||||
"node_codes": ["mainland-worker-01"],
|
||||
"executor_kind": "driver_action",
|
||||
"focus_ref": {"kind": "managed_node", "node_code": "mainland-worker-01"},
|
||||
"primary_focus_ref": {"kind": "managed_node", "node_code": "mainland-worker-01"},
|
||||
"secondary_focus_ref": {"kind": "managed_node", "node_code": "mainland-worker-01", "section": "bootstrap_plan"},
|
||||
"primary_label": "跑接入收口",
|
||||
"secondary_label": "查看 Bootstrap 计划",
|
||||
"primary_action_code": "bootstrap_run",
|
||||
"secondary_action_code": "create_bootstrap_plan",
|
||||
"primary_node_codes": ["mainland-worker-01"],
|
||||
"secondary_node_codes": ["mainland-worker-01"],
|
||||
"primary_action_payload": {"node_code": "mainland-worker-01"},
|
||||
"secondary_action_payload": {"node_code": "mainland-worker-01"},
|
||||
}
|
||||
],
|
||||
}
|
||||
|
||||
payload = get_ops_codex_brief()
|
||||
|
||||
self.assertEqual("priority:handover-gap-bootstrap_run", payload["focus"]["entry_key"])
|
||||
self.assertEqual("resolve_first", payload["focus"]["recommendation"])
|
||||
self.assertIn("ops_agent_protocol", payload["entries"][0]["contract_keys"])
|
||||
self.assertEqual("bootstrap_run", payload["entries"][0]["primary_action_code"])
|
||||
self.assertEqual("mainland-worker-01", payload["entries"][0]["primary_execution"]["node_codes"][0])
|
||||
self.assertEqual("/api/v1/ops/driver-actions/execute", payload["entries"][0]["autopilot"]["target_api"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
295
domain-api/tests/test_ops_service_contract_registry.py
Normal file
295
domain-api/tests/test_ops_service_contract_registry.py
Normal file
@@ -0,0 +1,295 @@
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
from app.services.ops_service import (
|
||||
get_ops_blueprint,
|
||||
get_ops_capabilities,
|
||||
get_ops_contract_detail,
|
||||
get_ops_contract_registry,
|
||||
get_ops_runbook,
|
||||
)
|
||||
|
||||
|
||||
class OpsServiceContractRegistryTests(unittest.TestCase):
|
||||
def test_contract_registry_lists_all_frozen_ops_contracts(self) -> None:
|
||||
payload = get_ops_contract_registry()
|
||||
|
||||
self.assertEqual("2026-04-18", payload["registry_version"])
|
||||
self.assertEqual("v1", payload["schema_version"])
|
||||
self.assertEqual(11, payload["contracts_total"])
|
||||
self.assertEqual(
|
||||
[
|
||||
"ops_job_contract",
|
||||
"ops_agent_protocol",
|
||||
"release_hub_contract",
|
||||
"ops_driver_contract",
|
||||
"ops_playbook_contract",
|
||||
"ops_observability_contract",
|
||||
"ops_stack_diagnosis_contract",
|
||||
"ops_doctor_decision_contract",
|
||||
"ops_go_live_signoff_contract",
|
||||
"ops_go_live_bundle_contract",
|
||||
"ops_go_live_review_contract",
|
||||
],
|
||||
[item["key"] for item in payload["contracts"]],
|
||||
)
|
||||
self.assertIn("/api/v1/ops/contracts/{contract_key}", payload["discovery_entrypoints"])
|
||||
self.assertEqual(
|
||||
"docs/schemas/ops_job_contract.md",
|
||||
payload["contracts_by_key"]["ops_job_contract"]["schema_doc_path"],
|
||||
)
|
||||
self.assertIn(
|
||||
"/api/v1/ops/jobs/{job_id}/dispatch",
|
||||
payload["contracts_by_key"]["ops_job_contract"]["discovery_endpoints"],
|
||||
)
|
||||
self.assertEqual(
|
||||
"docs/schemas/ops_agent_protocol.md",
|
||||
payload["contracts_by_key"]["ops_agent_protocol"]["schema_doc_path"],
|
||||
)
|
||||
self.assertIn(
|
||||
"/api/v1/ops/agent/bootstrap-plan",
|
||||
payload["contracts_by_key"]["ops_agent_protocol"]["discovery_endpoints"],
|
||||
)
|
||||
self.assertIn(
|
||||
"/api/v1/ops/nodes/{node_code}/delivery-queue",
|
||||
payload["contracts_by_key"]["ops_agent_protocol"]["discovery_endpoints"],
|
||||
)
|
||||
self.assertIn(
|
||||
"/api/v1/ops/nodes/{node_code}/delivery-queue/records",
|
||||
payload["contracts_by_key"]["ops_agent_protocol"]["discovery_endpoints"],
|
||||
)
|
||||
self.assertIn(
|
||||
"/api/v1/ops/nodes/{node_code}/delivery-queue/flush",
|
||||
payload["contracts_by_key"]["ops_agent_protocol"]["discovery_endpoints"],
|
||||
)
|
||||
self.assertIn(
|
||||
"/api/v1/ops/nodes/{node_code}/delivery-queue/replay",
|
||||
payload["contracts_by_key"]["ops_agent_protocol"]["discovery_endpoints"],
|
||||
)
|
||||
self.assertIn(
|
||||
"/api/v1/ops/nodes/{node_code}/delivery-queue/records/{record_id}/replay",
|
||||
payload["contracts_by_key"]["ops_agent_protocol"]["discovery_endpoints"],
|
||||
)
|
||||
self.assertIn(
|
||||
"/api/v1/ops/nodes/{node_code}/delivery-queue/records/{record_id}/discard",
|
||||
payload["contracts_by_key"]["ops_agent_protocol"]["discovery_endpoints"],
|
||||
)
|
||||
self.assertEqual(
|
||||
"/api/v1/ops/releases/launchpad",
|
||||
payload["contracts_by_key"]["release_hub_contract"]["primary_endpoint"],
|
||||
)
|
||||
self.assertIn(
|
||||
"/api/v1/ops/codex-brief",
|
||||
payload["contracts_by_key"]["ops_driver_contract"]["discovery_endpoints"],
|
||||
)
|
||||
self.assertIn(
|
||||
"/api/v1/ops/driver-actions/preview",
|
||||
payload["contracts_by_key"]["ops_driver_contract"]["discovery_endpoints"],
|
||||
)
|
||||
self.assertIn(
|
||||
"/api/v1/ops/driver-actions/resolve",
|
||||
payload["contracts_by_key"]["ops_driver_contract"]["discovery_endpoints"],
|
||||
)
|
||||
self.assertIn(
|
||||
"/api/v1/ops/driver-actions/execute-resolved",
|
||||
payload["contracts_by_key"]["ops_driver_contract"]["discovery_endpoints"],
|
||||
)
|
||||
self.assertIn(
|
||||
"/api/v1/ops/codex-actions/resolve",
|
||||
payload["contracts_by_key"]["ops_driver_contract"]["discovery_endpoints"],
|
||||
)
|
||||
self.assertIn(
|
||||
"/api/v1/ops/codex-actions/execute",
|
||||
payload["contracts_by_key"]["ops_driver_contract"]["discovery_endpoints"],
|
||||
)
|
||||
self.assertEqual(
|
||||
"docs/schemas/ops_playbook_contract.md",
|
||||
payload["contracts_by_key"]["ops_playbook_contract"]["schema_doc_path"],
|
||||
)
|
||||
self.assertIn(
|
||||
"/api/v1/ops/playbooks/execute",
|
||||
payload["contracts_by_key"]["ops_playbook_contract"]["discovery_endpoints"],
|
||||
)
|
||||
self.assertIn(
|
||||
"/api/v1/ops/playbook-runs/{run_code}/events",
|
||||
payload["contracts_by_key"]["ops_playbook_contract"]["discovery_endpoints"],
|
||||
)
|
||||
self.assertEqual(
|
||||
"docs/schemas/ops_observability_contract.md",
|
||||
payload["contracts_by_key"]["ops_observability_contract"]["schema_doc_path"],
|
||||
)
|
||||
self.assertIn(
|
||||
"/api/v1/ops/inspection-overview",
|
||||
payload["contracts_by_key"]["ops_observability_contract"]["discovery_endpoints"],
|
||||
)
|
||||
self.assertIn(
|
||||
"/api/v1/ops/nodes/{node_code}/scene-log",
|
||||
payload["contracts_by_key"]["ops_observability_contract"]["discovery_endpoints"],
|
||||
)
|
||||
self.assertEqual(
|
||||
"docs/schemas/ops_stack_diagnosis_contract.md",
|
||||
payload["contracts_by_key"]["ops_stack_diagnosis_contract"]["schema_doc_path"],
|
||||
)
|
||||
self.assertIn(
|
||||
"/api/v1/ops/stack-diagnosis",
|
||||
payload["contracts_by_key"]["ops_stack_diagnosis_contract"]["discovery_endpoints"],
|
||||
)
|
||||
self.assertIn(
|
||||
"ops_driver_contract",
|
||||
payload["contracts_by_key"]["ops_stack_diagnosis_contract"]["related_contract_keys"],
|
||||
)
|
||||
self.assertEqual(
|
||||
"docs/schemas/ops_doctor_decision_contract.md",
|
||||
payload["contracts_by_key"]["ops_doctor_decision_contract"]["schema_doc_path"],
|
||||
)
|
||||
self.assertIn(
|
||||
"/api/v1/ops/doctor-decision",
|
||||
payload["contracts_by_key"]["ops_doctor_decision_contract"]["discovery_endpoints"],
|
||||
)
|
||||
self.assertIn(
|
||||
"ops_stack_diagnosis_contract",
|
||||
payload["contracts_by_key"]["ops_doctor_decision_contract"]["related_contract_keys"],
|
||||
)
|
||||
self.assertEqual(
|
||||
"docs/schemas/ops_go_live_signoff_contract.md",
|
||||
payload["contracts_by_key"]["ops_go_live_signoff_contract"]["schema_doc_path"],
|
||||
)
|
||||
self.assertIn(
|
||||
"/api/v1/ops/go-live-signoff",
|
||||
payload["contracts_by_key"]["ops_go_live_signoff_contract"]["discovery_endpoints"],
|
||||
)
|
||||
self.assertIn(
|
||||
"ops_stack_diagnosis_contract",
|
||||
payload["contracts_by_key"]["ops_go_live_signoff_contract"]["related_contract_keys"],
|
||||
)
|
||||
self.assertEqual(
|
||||
"docs/schemas/ops_go_live_bundle_contract.md",
|
||||
payload["contracts_by_key"]["ops_go_live_bundle_contract"]["schema_doc_path"],
|
||||
)
|
||||
self.assertIn(
|
||||
"/api/v1/ops/go-live-bundle",
|
||||
payload["contracts_by_key"]["ops_go_live_bundle_contract"]["discovery_endpoints"],
|
||||
)
|
||||
self.assertIn(
|
||||
"ops_go_live_signoff_contract",
|
||||
payload["contracts_by_key"]["ops_go_live_bundle_contract"]["related_contract_keys"],
|
||||
)
|
||||
self.assertEqual(
|
||||
"docs/schemas/ops_go_live_review_contract.md",
|
||||
payload["contracts_by_key"]["ops_go_live_review_contract"]["schema_doc_path"],
|
||||
)
|
||||
self.assertIn(
|
||||
"/api/v1/ops/go-live-review",
|
||||
payload["contracts_by_key"]["ops_go_live_review_contract"]["discovery_endpoints"],
|
||||
)
|
||||
self.assertIn(
|
||||
"ops_go_live_bundle_contract",
|
||||
payload["contracts_by_key"]["ops_go_live_review_contract"]["related_contract_keys"],
|
||||
)
|
||||
self.assertIn(
|
||||
"ops_doctor_decision_contract",
|
||||
payload["contracts_by_key"]["ops_go_live_review_contract"]["related_contract_keys"],
|
||||
)
|
||||
|
||||
def test_contract_detail_exposes_related_contracts(self) -> None:
|
||||
payload = get_ops_contract_detail("ops_stack_diagnosis_contract")
|
||||
|
||||
self.assertEqual("2026-04-18", payload["registry_version"])
|
||||
self.assertEqual("/api/v1/ops/contracts/ops_stack_diagnosis_contract", payload["detail_endpoint"])
|
||||
self.assertEqual("ops_stack_diagnosis_contract", payload["contract"]["key"])
|
||||
self.assertIn("ops_driver_contract", payload["related_contract_keys"])
|
||||
self.assertIn("release_hub_contract", payload["related_contract_keys"])
|
||||
self.assertEqual(
|
||||
"docs/schemas/ops_driver_contract.md",
|
||||
next(
|
||||
item["schema_doc_path"]
|
||||
for item in payload["related_contracts"]
|
||||
if item["key"] == "ops_driver_contract"
|
||||
),
|
||||
)
|
||||
self.assertEqual({}, get_ops_contract_detail("missing-contract"))
|
||||
|
||||
def test_capabilities_blueprint_and_runbook_expose_contract_registry(self) -> None:
|
||||
capabilities = get_ops_capabilities()
|
||||
contract_registry = capabilities["contract_registry"]
|
||||
|
||||
self.assertEqual("/api/v1/ops/contracts", contract_registry["endpoint"])
|
||||
self.assertEqual("/api/v1/ops/contracts/{contract_key}", contract_registry["detail_endpoint_pattern"])
|
||||
self.assertEqual(11, contract_registry["contracts_total"])
|
||||
self.assertIn("ops_job_contract", contract_registry["contract_keys"])
|
||||
self.assertIn("ops_driver_contract", contract_registry["contract_keys"])
|
||||
self.assertIn("ops_playbook_contract", contract_registry["contract_keys"])
|
||||
self.assertIn("ops_observability_contract", contract_registry["contract_keys"])
|
||||
self.assertIn("ops_stack_diagnosis_contract", contract_registry["contract_keys"])
|
||||
self.assertIn("ops_doctor_decision_contract", contract_registry["contract_keys"])
|
||||
self.assertIn("ops_go_live_signoff_contract", contract_registry["contract_keys"])
|
||||
self.assertIn("ops_go_live_bundle_contract", contract_registry["contract_keys"])
|
||||
self.assertIn("ops_go_live_review_contract", contract_registry["contract_keys"])
|
||||
remote_log_capability = next(
|
||||
item for item in capabilities["capabilities"] if item["key"] == "remote_log_sync"
|
||||
)
|
||||
self.assertEqual("ready", remote_log_capability["current_state"])
|
||||
self.assertEqual(
|
||||
"docs/schemas/release_hub_contract.md",
|
||||
contract_registry["schema_docs"]["release_hub_contract"],
|
||||
)
|
||||
self.assertEqual(
|
||||
"docs/schemas/ops_playbook_contract.md",
|
||||
contract_registry["schema_docs"]["ops_playbook_contract"],
|
||||
)
|
||||
self.assertEqual(
|
||||
"docs/schemas/ops_job_contract.md",
|
||||
contract_registry["schema_docs"]["ops_job_contract"],
|
||||
)
|
||||
self.assertEqual(
|
||||
"docs/schemas/ops_observability_contract.md",
|
||||
contract_registry["schema_docs"]["ops_observability_contract"],
|
||||
)
|
||||
self.assertEqual(
|
||||
"docs/schemas/ops_stack_diagnosis_contract.md",
|
||||
contract_registry["schema_docs"]["ops_stack_diagnosis_contract"],
|
||||
)
|
||||
self.assertEqual(
|
||||
"docs/schemas/ops_doctor_decision_contract.md",
|
||||
contract_registry["schema_docs"]["ops_doctor_decision_contract"],
|
||||
)
|
||||
self.assertEqual(
|
||||
"docs/schemas/ops_go_live_signoff_contract.md",
|
||||
contract_registry["schema_docs"]["ops_go_live_signoff_contract"],
|
||||
)
|
||||
self.assertEqual(
|
||||
"docs/schemas/ops_go_live_bundle_contract.md",
|
||||
contract_registry["schema_docs"]["ops_go_live_bundle_contract"],
|
||||
)
|
||||
self.assertEqual(
|
||||
"docs/schemas/ops_go_live_review_contract.md",
|
||||
contract_registry["schema_docs"]["ops_go_live_review_contract"],
|
||||
)
|
||||
|
||||
blueprint = get_ops_blueprint()
|
||||
self.assertEqual("2026-04-18", blueprint["contract_registry"]["registry_version"])
|
||||
self.assertEqual(11, blueprint["contract_registry"]["contracts_total"])
|
||||
|
||||
with patch("app.services.ops_service.get_runtime_status") as mock_runtime_status, patch(
|
||||
"app.services.ops_service.list_managed_nodes_with_agent_state"
|
||||
) as mock_list_managed_nodes_with_agent_state, patch(
|
||||
"app.services.ops_service.get_release_launchpad"
|
||||
) as mock_get_release_launchpad, patch(
|
||||
"app.services.ops_service._attach_ops_runbook_sequence_resolutions"
|
||||
) as mock_attach_sequences:
|
||||
mock_runtime_status.return_value = {
|
||||
"readiness": {"status": "ready", "summary": "ok"},
|
||||
"worker": {"running": True},
|
||||
"sync_agent": {"running": True},
|
||||
"detect": {},
|
||||
}
|
||||
mock_list_managed_nodes_with_agent_state.return_value = {"nodes": [], "summary": {}}
|
||||
mock_get_release_launchpad.return_value = {}
|
||||
mock_attach_sequences.return_value = []
|
||||
runbook = get_ops_runbook()
|
||||
|
||||
self.assertIn("GET /api/v1/ops/contracts", runbook["fast_checks"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
149
domain-api/tests/test_ops_service_doctor_decision.py
Normal file
149
domain-api/tests/test_ops_service_doctor_decision.py
Normal file
@@ -0,0 +1,149 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
from app.services.ops_service import get_ops_doctor_decision
|
||||
|
||||
|
||||
class OpsServiceDoctorDecisionTests(unittest.TestCase):
|
||||
@patch("app.services.ops_service.get_ops_contract_registry")
|
||||
@patch("app.services.ops_service._build_live_doctor_payload")
|
||||
@patch("app.services.ops_service._find_latest_go_live_bundle_manifest")
|
||||
def test_doctor_decision_falls_back_to_live_payload_when_bundle_missing(
|
||||
self,
|
||||
mock_find_latest_go_live_bundle_manifest,
|
||||
mock_build_live_doctor_payload,
|
||||
mock_get_ops_contract_registry,
|
||||
) -> None:
|
||||
mock_find_latest_go_live_bundle_manifest.return_value = (None, Path("/tmp/go-live-bundles"))
|
||||
mock_build_live_doctor_payload.return_value = (
|
||||
{
|
||||
"status": "attention",
|
||||
"reason_code": "live_fallback",
|
||||
"headline": "当前先补 attention 项",
|
||||
"detail": "需要先顺着 stack next 收口。",
|
||||
"preferred_surface": "stack-summary",
|
||||
"next_action_code": "review_stack_next",
|
||||
"recommended_commands": ["bash primary"],
|
||||
"evidence": {},
|
||||
},
|
||||
{
|
||||
"ok": False,
|
||||
"required_failures": [],
|
||||
"optional_unavailable": [],
|
||||
"scene_log_reports_total": 0,
|
||||
"scene_log_reports_ok": 0,
|
||||
"scene_log_status_counts": {},
|
||||
"scene_log_reports": [],
|
||||
"contract_surface_gaps": [],
|
||||
},
|
||||
)
|
||||
mock_get_ops_contract_registry.return_value = {
|
||||
"contracts_by_key": {
|
||||
"ops_doctor_decision_contract": {"version": "v1"},
|
||||
}
|
||||
}
|
||||
|
||||
payload = get_ops_doctor_decision(base_url="http://127.0.0.1:8100")
|
||||
|
||||
self.assertEqual("attention", payload["status"])
|
||||
self.assertEqual("主决策待复核", payload["status_label"])
|
||||
self.assertEqual("live_fallback", payload["source"])
|
||||
self.assertEqual("review_stack_next", payload["decision"]["next_action_code"])
|
||||
self.assertIn("doctor-decision", payload["recommended_commands"]["doctor_decision"])
|
||||
self.assertEqual("bash primary", payload["recommended_commands"]["primary_recovery"])
|
||||
self.assertEqual(
|
||||
"bash domain-api/deploy/multi-region/drive_ops_center.sh stack-diagnosis http://127.0.0.1:8100 summary",
|
||||
payload["recommended_commands"]["stack_check"],
|
||||
)
|
||||
self.assertEqual(
|
||||
payload["recommended_commands"]["stack_check"],
|
||||
payload["recommended_commands"]["stack_diagnosis"],
|
||||
)
|
||||
|
||||
@patch("app.services.ops_service.get_ops_contract_registry")
|
||||
def test_doctor_decision_reads_bundle_doctor_artifact_when_available(self, mock_get_ops_contract_registry) -> None:
|
||||
mock_get_ops_contract_registry.return_value = {
|
||||
"contracts_by_key": {
|
||||
"ops_doctor_decision_contract": {"version": "v1"},
|
||||
}
|
||||
}
|
||||
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
report_dir = Path(temp_dir) / "go-live-bundle-001"
|
||||
report_dir.mkdir(parents=True, exist_ok=True)
|
||||
doctor_artifact_path = report_dir / "07_doctor_decision.json"
|
||||
doctor_artifact_path.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"summary": {
|
||||
"ok": False,
|
||||
"required_failures": ["06_contracts_registry.txt"],
|
||||
"optional_unavailable": ["12_activity_preview.txt"],
|
||||
"scene_log_reports_total": 2,
|
||||
"scene_log_reports_ok": 1,
|
||||
"scene_log_status_counts": {"ok": 1, "missing": 1},
|
||||
"scene_log_reports": [
|
||||
{"node_code": "mainland-worker-01", "status": "ok", "ok": True},
|
||||
{"node_code": "mainland-controller-01", "status": "missing", "ok": False},
|
||||
],
|
||||
"contract_surface_gaps": ["activity-stream"],
|
||||
"launchpad_recommended_target_node_code": "mainland-worker-01",
|
||||
"launchpad_recommended_recovery_label": "先补协议面",
|
||||
"launchpad_recommended_recovery_summary": "先补 driver / activity 的协议导航。",
|
||||
},
|
||||
"decision": {
|
||||
"status": "blocked",
|
||||
"reason_code": "contracts_registry_unavailable",
|
||||
"headline": "contract registry 当前不可用",
|
||||
"detail": "先恢复 contracts registry 再继续自动驾驶。",
|
||||
"preferred_surface": "contracts",
|
||||
"next_action_code": "restore_contract_registry",
|
||||
"recommended_commands": [
|
||||
"bash domain-api/deploy/multi-region/drive_ops_center.sh contracts http://127.0.0.1:8100",
|
||||
"bash domain-api/deploy/multi-region/drive_ops_center.sh runtime-refresh-recover http://127.0.0.1:8100",
|
||||
],
|
||||
"evidence": {"surface_http_status": {"contracts_registry": 503}},
|
||||
},
|
||||
},
|
||||
ensure_ascii=False,
|
||||
indent=2,
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
manifest_path = report_dir / "manifest.json"
|
||||
manifest_path.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"report_dir": str(report_dir),
|
||||
"artifacts": [
|
||||
{"key": "doctor_decision", "path": str(doctor_artifact_path), "ok": True},
|
||||
],
|
||||
},
|
||||
ensure_ascii=False,
|
||||
indent=2,
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
with patch(
|
||||
"app.services.ops_service._find_latest_go_live_bundle_manifest",
|
||||
return_value=(manifest_path, report_dir),
|
||||
):
|
||||
payload = get_ops_doctor_decision(base_url="http://127.0.0.1:8100")
|
||||
|
||||
self.assertEqual("blocked", payload["status"])
|
||||
self.assertEqual("bundle_manifest", payload["source"])
|
||||
self.assertEqual("contracts", payload["decision"]["preferred_surface"])
|
||||
self.assertEqual(["06_contracts_registry.txt"], payload["summary"]["required_failures"])
|
||||
self.assertEqual(["mainland-controller-01"], payload["summary"]["scene_log_problem_node_codes"])
|
||||
self.assertIn("contracts", payload["recommended_commands"]["primary_recovery"])
|
||||
self.assertIn(str(manifest_path), payload["recommended_commands"]["doctor_decision"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
274
domain-api/tests/test_ops_service_driver_action_preview.py
Normal file
274
domain-api/tests/test_ops_service_driver_action_preview.py
Normal file
@@ -0,0 +1,274 @@
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
from app.services.ops_service import preview_driver_action
|
||||
|
||||
|
||||
class OpsServiceDriverActionPreviewTests(unittest.TestCase):
|
||||
def test_preview_driver_action_builds_safe_auto_contract_preview(self) -> None:
|
||||
ok, message, data = preview_driver_action(
|
||||
{
|
||||
"source_label": "overview-recommendation",
|
||||
"title": "先补现场日志回传",
|
||||
"summary": "优先打开关键日志回传。",
|
||||
"reason": "当前还看不到现场过程日志。",
|
||||
"executor_kind": "driver_action",
|
||||
"focus_ref": {
|
||||
"kind": "execution_scene",
|
||||
"scene_key": "scene.logs",
|
||||
},
|
||||
"primary_focus_ref": {
|
||||
"kind": "execution_scene",
|
||||
"scene_key": "scene.logs",
|
||||
},
|
||||
"primary_label": "开启关键回传",
|
||||
"primary_action_code": "enable_log_sync_key",
|
||||
"primary_action_payload": {},
|
||||
"requested_by": "web-ui/ops-center",
|
||||
}
|
||||
)
|
||||
|
||||
self.assertTrue(ok, message)
|
||||
self.assertEqual("驾驶动作预览已生成", message)
|
||||
self.assertEqual("ops_driver_contract", data["contract_key"])
|
||||
self.assertEqual("/api/v1/ops/driver-feed", data["contract_primary_endpoint"])
|
||||
self.assertEqual("/api/v1/ops/driver-actions/preview", data["preview_endpoint"])
|
||||
self.assertEqual("driver_action", data["executor_kind"])
|
||||
self.assertEqual(
|
||||
"ops-driver -> driver-action execute -> playbook / ui-intent / settings",
|
||||
data["execution_chain"],
|
||||
)
|
||||
|
||||
primary = data["primary"]
|
||||
self.assertEqual("开启关键回传", primary["label"])
|
||||
self.assertEqual("enable_log_sync_key", primary["action_code"])
|
||||
self.assertEqual("/api/v1/ops/driver-actions/execute", primary["target_api"])
|
||||
self.assertEqual("可自动执行", primary["recommendation_label"])
|
||||
self.assertEqual("safe_auto", primary["automation_level_label"])
|
||||
self.assertEqual("low", primary["risk_level_label"])
|
||||
self.assertEqual("execution_scene", data["focus_ref"]["kind"])
|
||||
self.assertEqual("execution_scene", data["primary_focus_ref"]["kind"])
|
||||
self.assertEqual("execution_scene", primary["focus_ref"]["kind"])
|
||||
self.assertEqual(
|
||||
{
|
||||
"action_code": "enable_log_sync_key",
|
||||
"node_codes": [],
|
||||
"action_payload": {
|
||||
"focus_ref": {
|
||||
"kind": "execution_scene",
|
||||
"scene_key": "scene.logs",
|
||||
}
|
||||
},
|
||||
"requested_by": "web-ui/ops-center",
|
||||
},
|
||||
primary["request_payload"],
|
||||
)
|
||||
self.assertFalse(data["secondary"])
|
||||
|
||||
def test_preview_driver_action_marks_missing_required_fields_as_blocked(self) -> None:
|
||||
ok, message, data = preview_driver_action(
|
||||
{
|
||||
"source_label": "driver-feed",
|
||||
"title": "打开全量现场取证",
|
||||
"summary": "当前需要进入取证工作区。",
|
||||
"reason": "需要明确 playbook 上下文。",
|
||||
"executor_kind": "driver_action",
|
||||
"primary_label": "打开取证",
|
||||
"primary_action_code": "open_playbook_dialog",
|
||||
"primary_action_payload": {},
|
||||
"requested_by": "web-ui/ops-center",
|
||||
}
|
||||
)
|
||||
|
||||
self.assertTrue(ok, message)
|
||||
primary = data["primary"]
|
||||
self.assertEqual("当前阻断", primary["recommendation_label"])
|
||||
self.assertTrue(primary["execution_profile"]["blocked"])
|
||||
self.assertEqual(["playbook_key"], primary["execution_profile"]["missing_fields"])
|
||||
|
||||
def test_preview_driver_action_marks_release_launchpad_review_as_safe_auto(self) -> None:
|
||||
ok, message, data = preview_driver_action(
|
||||
{
|
||||
"source_label": "overview-recommendation",
|
||||
"title": "查看 Worker 预案",
|
||||
"summary": "当前建议先复核 Worker launchpad 预案。",
|
||||
"reason": "需要先判断目标节点、门禁和阻断缺口。",
|
||||
"executor_kind": "driver_action",
|
||||
"focus_ref": {
|
||||
"kind": "release_hub",
|
||||
"release_id": 88,
|
||||
"section": "release_launchpad",
|
||||
"mode": "worker",
|
||||
},
|
||||
"primary_focus_ref": {
|
||||
"kind": "release_hub",
|
||||
"release_id": 88,
|
||||
"section": "release_launchpad",
|
||||
"mode": "worker",
|
||||
},
|
||||
"primary_label": "查看 Worker 预案",
|
||||
"primary_action_code": "review_smart_rollout_preview",
|
||||
"primary_action_payload": {
|
||||
"channel": "stable",
|
||||
},
|
||||
"requested_by": "web-ui/ops-center",
|
||||
}
|
||||
)
|
||||
|
||||
self.assertTrue(ok, message)
|
||||
self.assertEqual("驾驶动作预览已生成", message)
|
||||
primary = data["primary"]
|
||||
self.assertEqual("review_smart_rollout_preview", primary["action_code"])
|
||||
self.assertEqual("可自动执行", primary["recommendation_label"])
|
||||
self.assertEqual("safe_auto", primary["automation_level_label"])
|
||||
self.assertEqual("low", primary["risk_level_label"])
|
||||
self.assertFalse(primary["execution_profile"]["ui_only"])
|
||||
self.assertEqual("/api/v1/ops/driver-actions/execute", primary["target_api"])
|
||||
|
||||
def test_preview_driver_action_marks_handover_gap_as_safe_auto(self) -> None:
|
||||
ok, message, data = preview_driver_action(
|
||||
{
|
||||
"source_label": "overview-recommendation",
|
||||
"title": "先补接管缺口",
|
||||
"summary": "当前存在待接管节点。",
|
||||
"reason": "需要先查看首台缺口节点的接管详情。",
|
||||
"executor_kind": "driver_action",
|
||||
"focus_ref": {
|
||||
"kind": "managed_node",
|
||||
"node_code": "mainland-worker-01",
|
||||
},
|
||||
"primary_focus_ref": {
|
||||
"kind": "managed_node",
|
||||
"node_code": "mainland-worker-01",
|
||||
},
|
||||
"primary_label": "查看接管详情",
|
||||
"primary_action_code": "handover_first_gap",
|
||||
"primary_action_payload": {
|
||||
"node_code": "mainland-worker-01",
|
||||
},
|
||||
"requested_by": "web-ui/ops-center",
|
||||
}
|
||||
)
|
||||
|
||||
self.assertTrue(ok, message)
|
||||
primary = data["primary"]
|
||||
self.assertEqual("handover_first_gap", primary["action_code"])
|
||||
self.assertEqual("可自动执行", primary["recommendation_label"])
|
||||
self.assertEqual("safe_auto", primary["automation_level_label"])
|
||||
self.assertEqual("low", primary["risk_level_label"])
|
||||
self.assertFalse(primary["execution_profile"]["ui_only"])
|
||||
|
||||
@patch("app.services.ops_service.get_ops_runbook")
|
||||
def test_preview_driver_action_supports_runbook_sequence_primary_and_secondary(self, mock_get_ops_runbook) -> None:
|
||||
mock_get_ops_runbook.return_value = {
|
||||
"control_sequences": [
|
||||
{
|
||||
"key": "release_progression",
|
||||
"title": "版本发布与放量",
|
||||
"status": "ready",
|
||||
"status_label": "可发 Worker 灰度",
|
||||
"summary": "当前可以进入正式发布路径。",
|
||||
"reason": "统一回到 Release / Rollout 闭环。",
|
||||
"target_scope_label": "Release / Rollout / 2026.04.18",
|
||||
"step_titles": ["确认发布包", "发灰度"],
|
||||
"target_node_codes": ["mainland-controller-01"],
|
||||
"focus_ref": {
|
||||
"kind": "release_hub",
|
||||
"release_id": 7,
|
||||
"section": "release_launchpad",
|
||||
},
|
||||
"primary_label": "Worker 灰度",
|
||||
"primary_action_code": "create_release_rollout_worker",
|
||||
"primary_action_payload": {"release_id": 7},
|
||||
"primary_focus_ref": {
|
||||
"kind": "release_hub",
|
||||
"release_id": 7,
|
||||
"section": "release_launchpad",
|
||||
"mode": "worker",
|
||||
},
|
||||
"secondary_label": "查看版本区",
|
||||
"secondary_action_code": "focus_release_hub",
|
||||
"secondary_action_payload": {"release_id": 7},
|
||||
"secondary_focus_ref": {
|
||||
"kind": "release_hub",
|
||||
"release_id": 7,
|
||||
"section": "release_detail",
|
||||
},
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
ok, message, data = preview_driver_action(
|
||||
{
|
||||
"source_label": "driver-feed",
|
||||
"title": "版本发布与放量",
|
||||
"summary": "当前可以进入正式发布路径。",
|
||||
"reason": "统一回到 Release / Rollout 闭环。",
|
||||
"executor_kind": "runbook_sequence",
|
||||
"sequence_key": "release_progression",
|
||||
"primary_label": "Worker 灰度",
|
||||
"requested_by": "web-ui/ops-center",
|
||||
}
|
||||
)
|
||||
|
||||
self.assertTrue(ok, message)
|
||||
self.assertEqual("驾驶动作预览已生成", message)
|
||||
self.assertEqual("runbook_sequence", data["executor_kind"])
|
||||
self.assertEqual("runbook sequence", data["executor_kind_label"])
|
||||
self.assertEqual(
|
||||
"ops-driver -> runbook-sequence -> resolved driver action / ui-intent",
|
||||
data["execution_chain"],
|
||||
)
|
||||
self.assertEqual("create_release_rollout_worker", data["resolved_primary"]["driver_action_code"])
|
||||
self.assertEqual("release_launchpad", data["focus_ref"]["section"])
|
||||
self.assertEqual("worker", data["primary_focus_ref"]["mode"])
|
||||
self.assertEqual("release_detail", data["secondary_focus_ref"]["section"])
|
||||
|
||||
primary = data["primary"]
|
||||
self.assertEqual("/api/v1/ops/runbook/sequences/release_progression/execute", primary["target_api"])
|
||||
self.assertEqual("建议确认后执行", primary["recommendation_label"])
|
||||
self.assertEqual("guarded_auto", primary["automation_level_label"])
|
||||
self.assertEqual("worker", primary["focus_ref"]["mode"])
|
||||
self.assertEqual(
|
||||
{
|
||||
"secondary": False,
|
||||
"requested_by": "web-ui/ops-center",
|
||||
"node_codes": ["mainland-controller-01"],
|
||||
"action_payload": {
|
||||
"release_id": 7,
|
||||
"focus_ref": {
|
||||
"kind": "release_hub",
|
||||
"release_id": 7,
|
||||
"section": "release_launchpad",
|
||||
"mode": "worker",
|
||||
},
|
||||
},
|
||||
},
|
||||
primary["request_payload"],
|
||||
)
|
||||
|
||||
secondary = data["secondary"]
|
||||
self.assertEqual("查看版本区", secondary["label"])
|
||||
self.assertEqual("focus_release_hub", secondary["action_code"])
|
||||
self.assertEqual("release_detail", secondary["focus_ref"]["section"])
|
||||
self.assertEqual(
|
||||
{
|
||||
"secondary": True,
|
||||
"requested_by": "web-ui/ops-center",
|
||||
"node_codes": ["mainland-controller-01"],
|
||||
"action_payload": {
|
||||
"release_id": 7,
|
||||
"focus_ref": {
|
||||
"kind": "release_hub",
|
||||
"release_id": 7,
|
||||
"section": "release_detail",
|
||||
},
|
||||
},
|
||||
},
|
||||
secondary["request_payload"],
|
||||
)
|
||||
self.assertEqual("可自动执行", secondary["recommendation_label"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
433
domain-api/tests/test_ops_service_driver_action_resolve.py
Normal file
433
domain-api/tests/test_ops_service_driver_action_resolve.py
Normal file
@@ -0,0 +1,433 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
from app.services.ops_service import execute_resolved_driver_action, resolve_driver_action_request
|
||||
|
||||
|
||||
def _build_preview_data(
|
||||
*,
|
||||
executor_kind: str,
|
||||
primary_action_code: str,
|
||||
recommendation: str,
|
||||
automation_level: str,
|
||||
target_api: str,
|
||||
primary_request_payload: dict,
|
||||
sequence_key: str = "",
|
||||
secondary_action_code: str = "",
|
||||
secondary_request_payload: dict | None = None,
|
||||
secondary_recommendation: str = "",
|
||||
secondary_automation_level: str = "",
|
||||
secondary_target_api: str = "",
|
||||
focus_ref: dict | None = None,
|
||||
primary_focus_ref: dict | None = None,
|
||||
secondary_focus_ref: dict | None = None,
|
||||
) -> dict:
|
||||
return {
|
||||
"executor_kind": executor_kind,
|
||||
"sequence_key": sequence_key,
|
||||
"execution_chain": "ops-driver -> preview",
|
||||
"focus_ref": dict(focus_ref or {}),
|
||||
"primary_focus_ref": dict(primary_focus_ref or {}),
|
||||
"secondary_focus_ref": dict(secondary_focus_ref or {}),
|
||||
"primary": {
|
||||
"label": "主动作",
|
||||
"action_code": primary_action_code,
|
||||
"node_codes": ["mainland-controller-01"],
|
||||
"target_api": target_api,
|
||||
"target_method": "POST",
|
||||
"request_payload": dict(primary_request_payload or {}),
|
||||
"focus_ref": dict(primary_focus_ref or {}),
|
||||
"execution_profile": {
|
||||
"action_code": primary_action_code,
|
||||
"backend_handled": True,
|
||||
"automation_level": automation_level,
|
||||
"recommendation": recommendation,
|
||||
"executor_mode_hint": "test-mode",
|
||||
"risk_level": "low" if recommendation == "auto_execute" else "high",
|
||||
"confirm_required": recommendation == "confirm_then_execute",
|
||||
"ui_only": recommendation == "open_ui",
|
||||
"blocked": recommendation == "blocked",
|
||||
"requires_fields": [],
|
||||
"missing_fields": [],
|
||||
"reason": "preview reason",
|
||||
},
|
||||
},
|
||||
"secondary": (
|
||||
{
|
||||
"label": "次动作",
|
||||
"action_code": secondary_action_code,
|
||||
"node_codes": ["mainland-controller-01"],
|
||||
"target_api": secondary_target_api or target_api,
|
||||
"target_method": "POST",
|
||||
"request_payload": dict(secondary_request_payload or {}),
|
||||
"focus_ref": dict(secondary_focus_ref or {}),
|
||||
"execution_profile": {
|
||||
"action_code": secondary_action_code,
|
||||
"backend_handled": True,
|
||||
"automation_level": secondary_automation_level or automation_level,
|
||||
"recommendation": secondary_recommendation or recommendation,
|
||||
"executor_mode_hint": "test-mode",
|
||||
"risk_level": "low" if (secondary_recommendation or recommendation) == "auto_execute" else "high",
|
||||
"confirm_required": (secondary_recommendation or recommendation) == "confirm_then_execute",
|
||||
"ui_only": (secondary_recommendation or recommendation) == "open_ui",
|
||||
"blocked": (secondary_recommendation or recommendation) == "blocked",
|
||||
"requires_fields": [],
|
||||
"missing_fields": [],
|
||||
"reason": "preview reason",
|
||||
},
|
||||
}
|
||||
if secondary_action_code
|
||||
else {}
|
||||
),
|
||||
}
|
||||
|
||||
|
||||
class OpsServiceDriverActionResolveTests(unittest.TestCase):
|
||||
@patch("app.services.ops_service.preview_driver_action")
|
||||
def test_resolve_driver_action_request_returns_gate_for_safe_auto(self, mock_preview_driver_action) -> None:
|
||||
request_payload = {
|
||||
"action_code": "enable_log_sync_key",
|
||||
"node_codes": ["mainland-controller-01"],
|
||||
"action_payload": {},
|
||||
"requested_by": "test/driver-action",
|
||||
}
|
||||
mock_preview_driver_action.return_value = (
|
||||
True,
|
||||
"ok",
|
||||
_build_preview_data(
|
||||
executor_kind="driver_action",
|
||||
primary_action_code="enable_log_sync_key",
|
||||
recommendation="auto_execute",
|
||||
automation_level="safe_auto",
|
||||
target_api="/api/v1/ops/driver-actions/execute",
|
||||
primary_request_payload=request_payload,
|
||||
focus_ref={
|
||||
"kind": "execution_scene",
|
||||
"scene_key": "scene.logs",
|
||||
},
|
||||
primary_focus_ref={
|
||||
"kind": "execution_scene",
|
||||
"scene_key": "scene.logs",
|
||||
},
|
||||
),
|
||||
)
|
||||
|
||||
ok, message, data = resolve_driver_action_request(
|
||||
{
|
||||
"source_label": "driver-feed",
|
||||
"title": "开启关键日志回传",
|
||||
"focus_ref": {
|
||||
"kind": "execution_scene",
|
||||
"scene_key": "scene.logs",
|
||||
},
|
||||
"primary_focus_ref": {
|
||||
"kind": "execution_scene",
|
||||
"scene_key": "scene.logs",
|
||||
},
|
||||
"primary_action_code": "enable_log_sync_key",
|
||||
"primary_action_payload": {},
|
||||
"requested_by": "test/driver-action",
|
||||
}
|
||||
)
|
||||
|
||||
self.assertTrue(ok)
|
||||
self.assertEqual("驾驶动作解析完成", message)
|
||||
self.assertEqual("execute_now", data["gate"]["decision"])
|
||||
self.assertTrue(data["gate"]["will_execute"])
|
||||
self.assertEqual("/api/v1/ops/driver-actions/execute", data["gate"]["target_api"])
|
||||
self.assertEqual("enable_log_sync_key", data["preview_request"]["primary_action_code"])
|
||||
self.assertEqual("execution_scene", data["preview_request"]["focus_ref"]["kind"])
|
||||
self.assertEqual("scene.logs", data["gate"]["focus_ref"]["scene_key"])
|
||||
self.assertEqual("scene.logs", data["focus_ref"]["scene_key"])
|
||||
|
||||
@patch("app.services.ops_service.preview_driver_action")
|
||||
def test_resolve_driver_action_request_requires_confirmation_for_guarded_runbook(self, mock_preview_driver_action) -> None:
|
||||
request_payload = {
|
||||
"secondary": False,
|
||||
"requested_by": "test/driver-action",
|
||||
"action_payload": {"release_id": 7},
|
||||
}
|
||||
mock_preview_driver_action.return_value = (
|
||||
True,
|
||||
"ok",
|
||||
_build_preview_data(
|
||||
executor_kind="runbook_sequence",
|
||||
primary_action_code="create_release_rollout_worker",
|
||||
recommendation="confirm_then_execute",
|
||||
automation_level="guarded_auto",
|
||||
target_api="/api/v1/ops/runbook/sequences/release_progression/execute",
|
||||
primary_request_payload=request_payload,
|
||||
sequence_key="release_progression",
|
||||
focus_ref={
|
||||
"kind": "release_hub",
|
||||
"release_id": 7,
|
||||
"section": "release_launchpad",
|
||||
},
|
||||
primary_focus_ref={
|
||||
"kind": "release_hub",
|
||||
"release_id": 7,
|
||||
"section": "release_launchpad",
|
||||
"mode": "worker",
|
||||
},
|
||||
),
|
||||
)
|
||||
|
||||
ok, _, data = resolve_driver_action_request(
|
||||
{
|
||||
"source_label": "driver-feed",
|
||||
"title": "版本发布与放量",
|
||||
"executor_kind": "runbook_sequence",
|
||||
"sequence_key": "release_progression",
|
||||
"focus_ref": {
|
||||
"kind": "release_hub",
|
||||
"release_id": 7,
|
||||
"section": "release_launchpad",
|
||||
},
|
||||
"primary_focus_ref": {
|
||||
"kind": "release_hub",
|
||||
"release_id": 7,
|
||||
"section": "release_launchpad",
|
||||
"mode": "worker",
|
||||
},
|
||||
"primary_action_code": "create_release_rollout_worker",
|
||||
"primary_action_payload": {"release_id": 7},
|
||||
"requested_by": "test/driver-action",
|
||||
}
|
||||
)
|
||||
|
||||
self.assertTrue(ok)
|
||||
self.assertEqual("confirmation_required", data["gate"]["decision"])
|
||||
self.assertFalse(data["gate"]["will_execute"])
|
||||
self.assertTrue(data["gate"]["confirm_required"])
|
||||
self.assertEqual("release_progression", data["gate"]["sequence_key"])
|
||||
self.assertEqual("worker", data["preview_request"]["primary_focus_ref"]["mode"])
|
||||
self.assertEqual("worker", data["gate"]["focus_ref"]["mode"])
|
||||
self.assertEqual("worker", data["focus_ref"]["mode"])
|
||||
|
||||
@patch("app.services.ops_service.preview_driver_action")
|
||||
def test_resolve_driver_action_request_can_select_secondary_open_ui_section(self, mock_preview_driver_action) -> None:
|
||||
mock_preview_driver_action.return_value = (
|
||||
True,
|
||||
"ok",
|
||||
_build_preview_data(
|
||||
executor_kind="runbook_sequence",
|
||||
primary_action_code="create_release_rollout_worker",
|
||||
recommendation="confirm_then_execute",
|
||||
automation_level="guarded_auto",
|
||||
target_api="/api/v1/ops/runbook/sequences/release_progression/execute",
|
||||
primary_request_payload={
|
||||
"secondary": False,
|
||||
"requested_by": "test/driver-action",
|
||||
"action_payload": {"release_id": 7},
|
||||
},
|
||||
sequence_key="release_progression",
|
||||
secondary_action_code="focus_release_hub",
|
||||
secondary_request_payload={
|
||||
"secondary": True,
|
||||
"requested_by": "test/driver-action",
|
||||
"action_payload": {"release_id": 7},
|
||||
},
|
||||
secondary_recommendation="auto_execute",
|
||||
secondary_automation_level="safe_auto",
|
||||
secondary_target_api="/api/v1/ops/runbook/sequences/release_progression/execute",
|
||||
focus_ref={
|
||||
"kind": "release_hub",
|
||||
"release_id": 7,
|
||||
"section": "release_launchpad",
|
||||
},
|
||||
primary_focus_ref={
|
||||
"kind": "release_hub",
|
||||
"release_id": 7,
|
||||
"section": "release_launchpad",
|
||||
"mode": "worker",
|
||||
},
|
||||
secondary_focus_ref={
|
||||
"kind": "release_hub",
|
||||
"release_id": 7,
|
||||
"section": "release_detail",
|
||||
},
|
||||
),
|
||||
)
|
||||
|
||||
ok, _, data = resolve_driver_action_request(
|
||||
{
|
||||
"source_label": "driver-feed",
|
||||
"title": "版本发布与放量",
|
||||
"executor_kind": "runbook_sequence",
|
||||
"sequence_key": "release_progression",
|
||||
"focus_ref": {
|
||||
"kind": "release_hub",
|
||||
"release_id": 7,
|
||||
"section": "release_launchpad",
|
||||
},
|
||||
"primary_focus_ref": {
|
||||
"kind": "release_hub",
|
||||
"release_id": 7,
|
||||
"section": "release_launchpad",
|
||||
"mode": "worker",
|
||||
},
|
||||
"secondary_focus_ref": {
|
||||
"kind": "release_hub",
|
||||
"release_id": 7,
|
||||
"section": "release_detail",
|
||||
},
|
||||
"primary_action_code": "create_release_rollout_worker",
|
||||
"secondary_action_code": "focus_release_hub",
|
||||
"primary_action_payload": {"release_id": 7},
|
||||
"secondary_action_payload": {"release_id": 7},
|
||||
"requested_by": "test/driver-action",
|
||||
"secondary": True,
|
||||
}
|
||||
)
|
||||
|
||||
self.assertTrue(ok)
|
||||
self.assertEqual("secondary", data["selected_section"])
|
||||
self.assertEqual("execute_now", data["gate"]["decision"])
|
||||
self.assertTrue(data["gate"]["will_execute"])
|
||||
self.assertEqual("focus_release_hub", data["gate"]["action_code"])
|
||||
self.assertEqual("release_detail", data["gate"]["focus_ref"]["section"])
|
||||
self.assertEqual("release_detail", data["focus_ref"]["section"])
|
||||
|
||||
@patch("app.services.ops_service.preview_driver_action")
|
||||
def test_execute_resolved_driver_action_blocks_guarded_action_without_confirm(self, mock_preview_driver_action) -> None:
|
||||
mock_preview_driver_action.return_value = (
|
||||
True,
|
||||
"ok",
|
||||
_build_preview_data(
|
||||
executor_kind="runbook_sequence",
|
||||
primary_action_code="create_release_rollout_worker",
|
||||
recommendation="confirm_then_execute",
|
||||
automation_level="guarded_auto",
|
||||
target_api="/api/v1/ops/runbook/sequences/release_progression/execute",
|
||||
primary_request_payload={
|
||||
"secondary": False,
|
||||
"requested_by": "test/driver-action",
|
||||
"action_payload": {"release_id": 7},
|
||||
},
|
||||
sequence_key="release_progression",
|
||||
),
|
||||
)
|
||||
|
||||
ok, message, data = execute_resolved_driver_action(
|
||||
{
|
||||
"source_label": "driver-feed",
|
||||
"title": "版本发布与放量",
|
||||
"executor_kind": "runbook_sequence",
|
||||
"sequence_key": "release_progression",
|
||||
"primary_action_code": "create_release_rollout_worker",
|
||||
"primary_action_payload": {"release_id": 7},
|
||||
"requested_by": "test/driver-action",
|
||||
}
|
||||
)
|
||||
|
||||
self.assertFalse(ok)
|
||||
self.assertIn("需要显式确认", message)
|
||||
self.assertFalse(data["executed"])
|
||||
self.assertEqual("confirmation_required", data["gate"]["decision"])
|
||||
|
||||
@patch("app.services.ops_service.execute_driver_action")
|
||||
@patch("app.services.ops_service.preview_driver_action")
|
||||
def test_execute_resolved_driver_action_runs_safe_auto_driver_action(
|
||||
self,
|
||||
mock_preview_driver_action,
|
||||
mock_execute_driver_action,
|
||||
) -> None:
|
||||
request_payload = {
|
||||
"action_code": "enable_log_sync_key",
|
||||
"node_codes": ["mainland-controller-01"],
|
||||
"action_payload": {},
|
||||
"requested_by": "test/driver-action",
|
||||
}
|
||||
mock_preview_driver_action.return_value = (
|
||||
True,
|
||||
"ok",
|
||||
_build_preview_data(
|
||||
executor_kind="driver_action",
|
||||
primary_action_code="enable_log_sync_key",
|
||||
recommendation="auto_execute",
|
||||
automation_level="safe_auto",
|
||||
target_api="/api/v1/ops/driver-actions/execute",
|
||||
primary_request_payload=request_payload,
|
||||
),
|
||||
)
|
||||
mock_execute_driver_action.return_value = (
|
||||
True,
|
||||
"日志回传已开启",
|
||||
{"handled": True, "mode": "ops-job"},
|
||||
)
|
||||
|
||||
ok, message, data = execute_resolved_driver_action(
|
||||
{
|
||||
"source_label": "driver-feed",
|
||||
"title": "开启关键日志回传",
|
||||
"primary_action_code": "enable_log_sync_key",
|
||||
"primary_action_payload": {},
|
||||
"requested_by": "test/driver-action",
|
||||
}
|
||||
)
|
||||
|
||||
self.assertTrue(ok)
|
||||
self.assertEqual("日志回传已开启", message)
|
||||
self.assertTrue(data["executed"])
|
||||
self.assertEqual({"handled": True, "mode": "ops-job"}, data["execution_result"])
|
||||
mock_execute_driver_action.assert_called_once_with(request_payload)
|
||||
|
||||
@patch("app.services.ops_service.execute_ops_runbook_sequence")
|
||||
@patch("app.services.ops_service.preview_driver_action")
|
||||
def test_execute_resolved_driver_action_runs_guarded_runbook_after_confirm(
|
||||
self,
|
||||
mock_preview_driver_action,
|
||||
mock_execute_ops_runbook_sequence,
|
||||
) -> None:
|
||||
request_payload = {
|
||||
"secondary": False,
|
||||
"requested_by": "test/driver-action",
|
||||
"action_payload": {"release_id": 7},
|
||||
}
|
||||
mock_preview_driver_action.return_value = (
|
||||
True,
|
||||
"ok",
|
||||
_build_preview_data(
|
||||
executor_kind="runbook_sequence",
|
||||
primary_action_code="create_release_rollout_worker",
|
||||
recommendation="confirm_then_execute",
|
||||
automation_level="guarded_auto",
|
||||
target_api="/api/v1/ops/runbook/sequences/release_progression/execute",
|
||||
primary_request_payload=request_payload,
|
||||
sequence_key="release_progression",
|
||||
),
|
||||
)
|
||||
mock_execute_ops_runbook_sequence.return_value = (
|
||||
True,
|
||||
"Release rollout 已创建",
|
||||
{
|
||||
"handled": True,
|
||||
"mode": "smart-release-rollout",
|
||||
"driver_action_code": "create_release_rollout_worker",
|
||||
},
|
||||
)
|
||||
|
||||
ok, message, data = execute_resolved_driver_action(
|
||||
{
|
||||
"source_label": "driver-feed",
|
||||
"title": "版本发布与放量",
|
||||
"executor_kind": "runbook_sequence",
|
||||
"sequence_key": "release_progression",
|
||||
"primary_action_code": "create_release_rollout_worker",
|
||||
"primary_action_payload": {"release_id": 7},
|
||||
"requested_by": "test/driver-action",
|
||||
"confirm": True,
|
||||
}
|
||||
)
|
||||
|
||||
self.assertTrue(ok)
|
||||
self.assertEqual("Release rollout 已创建", message)
|
||||
self.assertTrue(data["executed"])
|
||||
self.assertEqual("execute_confirmed", data["gate"]["decision"])
|
||||
mock_execute_ops_runbook_sequence.assert_called_once_with("release_progression", request_payload)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
1265
domain-api/tests/test_ops_service_driver_actions.py
Normal file
1265
domain-api/tests/test_ops_service_driver_actions.py
Normal file
File diff suppressed because it is too large
Load Diff
358
domain-api/tests/test_ops_service_driver_feed.py
Normal file
358
domain-api/tests/test_ops_service_driver_feed.py
Normal file
@@ -0,0 +1,358 @@
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
from app.services.ops_service import get_ops_driver_feed
|
||||
|
||||
|
||||
class OpsServiceDriverFeedTests(unittest.TestCase):
|
||||
@patch("app.services.ops_service.get_ops_go_live_summary")
|
||||
@patch("app.services.ops_service.get_ops_runbook")
|
||||
@patch("app.services.ops_service.get_ops_overview")
|
||||
def test_driver_feed_merges_priority_recommendation_driver_cards_and_runbook_sequences(
|
||||
self,
|
||||
mock_get_ops_overview,
|
||||
mock_get_ops_runbook,
|
||||
mock_get_ops_go_live_summary,
|
||||
) -> None:
|
||||
mock_get_ops_go_live_summary.return_value = {
|
||||
"go_live_status": "blocked",
|
||||
"publish_ready": False,
|
||||
"publish_status": "blocked",
|
||||
"publish_status_label": "暂不可发布",
|
||||
"publish_summary": "当前仍有硬阻断项,不能直接进入正式发版。",
|
||||
"operator_title": "先补接管缺口",
|
||||
"next_step_action_code": "handover_first_gap",
|
||||
"launchpad_recommended_target_node_code": "mainland-controller-01",
|
||||
"launchpad_recommended_recovery_label": "签发接入工单",
|
||||
"launchpad_recommended_recovery_summary": "当前节点仍处于接入阶段,建议先签发 onboarding.bootstrap。",
|
||||
"launchpad_onboarding_bootstrap_pending_nodes": 1,
|
||||
"launchpad_onboarding_acceptance_ready_nodes": 1,
|
||||
"blocking_reasons": ["remote_agent_ready=0"],
|
||||
"warnings": ["log_sync_waiting_sample=0/2"],
|
||||
}
|
||||
mock_get_ops_overview.return_value = {
|
||||
"execution_scene": {
|
||||
"participation_summary": {
|
||||
"participating_nodes": 2,
|
||||
"dispatch_active_node_codes": ["mainland-worker-01", "mainland-controller-01"],
|
||||
},
|
||||
"participating_nodes": [
|
||||
{"node_code": "mainland-worker-01"},
|
||||
{"node_code": "mainland-controller-01"},
|
||||
],
|
||||
"log_sync": {
|
||||
"enabled": False,
|
||||
"mode": "key",
|
||||
"line_count": 0,
|
||||
"source_node_count": 0,
|
||||
"participating_node_count": 2,
|
||||
"covered_participating_node_count": 0,
|
||||
"missing_participating_nodes": ["mainland-worker-01", "mainland-controller-01"],
|
||||
},
|
||||
},
|
||||
"recommendation": {
|
||||
"key": "fix-handover-gap",
|
||||
"priority": "先补接管缺口",
|
||||
"title": "先补接管缺口",
|
||||
"summary": "当前应该先把接管链路补齐。",
|
||||
"reason": "否则后续批量动作无法稳定下发。",
|
||||
"level_label": "最高优先",
|
||||
"tag_type": "danger",
|
||||
"primary_label": "纳管首台节点",
|
||||
"secondary_label": "查看目标节点",
|
||||
"primary_action_code": "handover_first_gap",
|
||||
"secondary_action_code": "view_first_gap",
|
||||
"node_codes": ["mainland-worker-02"],
|
||||
"primary_node_codes": ["mainland-worker-02"],
|
||||
"secondary_node_codes": ["mainland-worker-02"],
|
||||
"primary_action_payload": {"requested_by": "test"},
|
||||
"secondary_action_payload": {},
|
||||
"reasons": ["当前存在接管缺口。"],
|
||||
},
|
||||
"activity_stream": {
|
||||
"items": [
|
||||
{
|
||||
"kind": "ops_job",
|
||||
"activity_key": "ops-job:501",
|
||||
"title": "logs.collect",
|
||||
"subtitle": "ops-logs-501",
|
||||
"summary_text": "远端 Agent 已回传 120 行 Worker 日志,可直接继续分析。",
|
||||
"summary": "远端 Agent 已回传 120 行 Worker 日志,可直接继续分析。",
|
||||
"status": "running",
|
||||
"status_label": "执行中",
|
||||
"meta_text": "ops-logs-501 / 执行 远端 Agent",
|
||||
"occurred_at": "2026-04-18 09:18:00",
|
||||
"target_node_codes": ["mainland-worker-01"],
|
||||
"focus_ref": {
|
||||
"kind": "ops_job",
|
||||
"job_id": 501,
|
||||
"job_code": "ops-logs-501",
|
||||
"action": "logs.collect",
|
||||
"target_node_code": "mainland-worker-01",
|
||||
},
|
||||
"source_focus_ref": {
|
||||
"kind": "ops_job_event",
|
||||
"job_id": 501,
|
||||
"event_key": "job-event:501",
|
||||
},
|
||||
"ui_intent": {
|
||||
"kind": "job_events",
|
||||
"job_id": 501,
|
||||
"job_code": "ops-logs-501",
|
||||
"action": "logs.collect",
|
||||
"target_node_code": "mainland-worker-01",
|
||||
},
|
||||
}
|
||||
]
|
||||
},
|
||||
"driver_recommendations": [
|
||||
{
|
||||
"key": "fix-handover-gap",
|
||||
"title": "先补接管缺口",
|
||||
"summary": "重复建议,不应再次进入 feed。",
|
||||
"reason": "duplicate",
|
||||
"level_label": "最高优先",
|
||||
"tag_type": "danger",
|
||||
"primary_label": "纳管首台节点",
|
||||
"primary_action_code": "handover_first_gap",
|
||||
},
|
||||
{
|
||||
"key": "inspect-scene-log-gap",
|
||||
"title": "排查现场日志样本缺口",
|
||||
"summary": "现场日志样本还没有形成。",
|
||||
"reason": "先看 Worker 日志。",
|
||||
"meta_text": "参与节点 2 台 / 已回传来源 0 台",
|
||||
"level_label": "推荐",
|
||||
"tag_type": "warning",
|
||||
"primary_label": "看 Worker 日志",
|
||||
"secondary_label": "执行标准巡检",
|
||||
"primary_action_code": "open_worker_logs",
|
||||
"secondary_action_code": "run_standard_inspection",
|
||||
"node_codes": ["mainland-worker-01", "mainland-controller-01"],
|
||||
"primary_node_codes": ["mainland-worker-01", "mainland-controller-01"],
|
||||
"secondary_node_codes": ["mainland-worker-01", "mainland-controller-01"],
|
||||
"primary_action_payload": {"lines": 120},
|
||||
"secondary_action_payload": {},
|
||||
},
|
||||
],
|
||||
}
|
||||
mock_get_ops_runbook.return_value = {
|
||||
"control_sequences": [
|
||||
{
|
||||
"key": "release_progression",
|
||||
"title": "版本发布与放量",
|
||||
"summary": "当前可进入发布路径。",
|
||||
"reason": "正式运维时应统一落回 Release / Rollout。",
|
||||
"status": "warning",
|
||||
"status_label": "待处理",
|
||||
"tag_type": "warning",
|
||||
"target_scope_label": "Release / Rollout",
|
||||
"target_node_codes": ["mainland-controller-01"],
|
||||
"focus_ref": {
|
||||
"kind": "release_hub",
|
||||
"release_id": 7,
|
||||
"section": "release_launchpad",
|
||||
},
|
||||
"primary_label": "补执行面接管",
|
||||
"secondary_label": "查看版本区",
|
||||
"primary_action_code": "fix_managed_nodes",
|
||||
"secondary_action_code": "focus_release_hub",
|
||||
"primary_focus_ref": {
|
||||
"kind": "release_hub",
|
||||
"release_id": 7,
|
||||
"section": "default_rollout_gate",
|
||||
},
|
||||
"secondary_focus_ref": {
|
||||
"kind": "release_hub",
|
||||
"release_id": 7,
|
||||
"section": "release_detail",
|
||||
},
|
||||
"step_titles": ["同步集群节点", "纳管首台缺口", "补齐 Agent 接入"],
|
||||
"primary_resolution": {
|
||||
"ok": True,
|
||||
"driver_action_code": "fix_managed_nodes",
|
||||
"driver_action_label": "补执行面接管",
|
||||
"driver_node_codes": ["mainland-controller-01"],
|
||||
"driver_action_payload": {"channel": "stable"},
|
||||
"resolved_at": "2026-04-18T03:00:00",
|
||||
},
|
||||
"secondary_resolution": {
|
||||
"ok": True,
|
||||
"driver_action_code": "focus_release_hub",
|
||||
"driver_action_label": "查看版本区",
|
||||
"driver_node_codes": [],
|
||||
"driver_action_payload": {"release_id": 7},
|
||||
},
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
payload = get_ops_driver_feed()
|
||||
|
||||
self.assertEqual(3, payload["summary"]["total"])
|
||||
self.assertEqual("blocked", payload["go_live_summary"]["go_live_status"])
|
||||
self.assertEqual("blocked", payload["summary"]["go_live_status"])
|
||||
self.assertEqual("先补接管缺口", payload["summary"]["operator_title"])
|
||||
self.assertEqual("blocked", payload["summary"]["publish_status"])
|
||||
self.assertFalse(payload["summary"]["publish_ready"])
|
||||
self.assertEqual("暂不可发布", payload["summary"]["publish_status_label"])
|
||||
self.assertIn("发布闸门:暂不可发布", payload["headline"])
|
||||
self.assertIn("接入缺口:签发接入工单:mainland-controller-01", payload["headline"])
|
||||
self.assertEqual("handover_first_gap", payload["go_live_summary"]["next_step_action_code"])
|
||||
self.assertEqual("mainland-controller-01", payload["summary"]["launchpad_recommended_target_node_code"])
|
||||
self.assertEqual("签发接入工单", payload["summary"]["launchpad_recommended_recovery_label"])
|
||||
self.assertEqual(
|
||||
"当前节点仍处于接入阶段,建议先签发 onboarding.bootstrap。",
|
||||
payload["summary"]["launchpad_recommended_recovery_summary"],
|
||||
)
|
||||
self.assertEqual(1, payload["summary"]["launchpad_onboarding_bootstrap_pending_nodes"])
|
||||
self.assertEqual(1, payload["summary"]["launchpad_onboarding_acceptance_ready_nodes"])
|
||||
self.assertEqual(2, payload["summary"]["do_now_total"])
|
||||
self.assertEqual(1, payload["summary"]["standard_path_total"])
|
||||
self.assertEqual("ops_driver_contract", payload["contract_navigation"]["primary_contract_key"])
|
||||
self.assertIn("ops_playbook_contract", payload["contract_navigation"]["contract_keys"])
|
||||
self.assertIn("ops_observability_contract", payload["contract_navigation"]["contract_keys"])
|
||||
self.assertEqual("priority:fix-handover-gap", payload["top_recommendation"]["key"])
|
||||
self.assertEqual("ops_driver_contract", payload["top_recommendation"]["contract_navigation"]["primary_contract_key"])
|
||||
self.assertEqual(2, len(payload["driver_recommendations"]))
|
||||
self.assertEqual(1, len(payload["runbook_sequences"]))
|
||||
self.assertEqual(1, len(payload["activity_focus"]))
|
||||
self.assertEqual(
|
||||
["priority_recommendation", "driver_recommendation", "runbook_sequence"],
|
||||
[item["kind"] for item in payload["entries"]],
|
||||
)
|
||||
self.assertEqual(1, payload["summary"]["activity_focus_total"])
|
||||
self.assertEqual("disabled", payload["summary"]["scene_log_status"])
|
||||
self.assertEqual("blocked", payload["summary"]["launch_status"])
|
||||
self.assertFalse(payload["summary"]["launch_ready"])
|
||||
self.assertEqual(2, payload["summary"]["preview_only_total"])
|
||||
self.assertEqual(3, payload["summary"]["backend_handled_total"])
|
||||
self.assertEqual("blocked", payload["automation_coverage"]["launch_status"])
|
||||
self.assertEqual(2, payload["automation_coverage"]["safe_auto_total"])
|
||||
self.assertEqual(0, payload["automation_coverage"]["guarded_auto_total"])
|
||||
self.assertEqual(2, payload["automation_coverage"]["auto_execute_total"])
|
||||
self.assertEqual(1, payload["automation_coverage"]["resolve_first_total"])
|
||||
self.assertEqual(2, payload["automation_coverage"]["preview_only_total"])
|
||||
self.assertEqual(3, payload["automation_coverage"]["backend_handled_total"])
|
||||
self.assertEqual(1, payload["automation_coverage"]["human_dependency_total"])
|
||||
self.assertFalse(payload["automation_coverage"]["launch_ready"])
|
||||
|
||||
priority_entry = payload["entries"][0]
|
||||
self.assertEqual("do_now", priority_entry["lane"])
|
||||
self.assertEqual("driver_action", priority_entry["executor_kind"])
|
||||
self.assertEqual("handover_first_gap", priority_entry["primary_action_code"])
|
||||
self.assertEqual("ops_driver_contract", priority_entry["contract_navigation"]["primary_contract_key"])
|
||||
|
||||
activity_focus = payload["activity_focus"][0]
|
||||
self.assertEqual("ops_job", activity_focus["activity_kind"])
|
||||
self.assertEqual("远端 Agent 已回传 120 行 Worker 日志,可直接继续分析。", activity_focus["summary"])
|
||||
self.assertEqual("focus_latest_job_events", activity_focus["focus_action_code"])
|
||||
self.assertEqual("job_events", activity_focus["ui_intent_kind"])
|
||||
self.assertEqual("ops_job_event", activity_focus["source_focus_ref"]["kind"])
|
||||
self.assertTrue(activity_focus["observation_only"])
|
||||
self.assertEqual("ops_observability_contract", activity_focus["contract_navigation"]["primary_contract_key"])
|
||||
self.assertIn("ops_stack_diagnosis_contract", activity_focus["contract_navigation"]["contract_keys"])
|
||||
|
||||
scene_log_observation = payload["scene_log_observation"]
|
||||
self.assertEqual("disabled", scene_log_observation["status"])
|
||||
self.assertEqual("mainland-worker-01", scene_log_observation["target_node_code"])
|
||||
self.assertEqual("node_scene_log", scene_log_observation["focus_ref"]["kind"])
|
||||
self.assertEqual(
|
||||
"ops_observability_contract",
|
||||
scene_log_observation["contract_navigation"]["primary_contract_key"],
|
||||
)
|
||||
|
||||
runbook_entry = payload["entries"][2]
|
||||
self.assertEqual("runbook_sequence", runbook_entry["kind"])
|
||||
self.assertEqual("standard_path", runbook_entry["lane"])
|
||||
self.assertEqual("runbook_sequence", runbook_entry["executor_kind"])
|
||||
self.assertEqual("release_progression", runbook_entry["sequence_key"])
|
||||
self.assertEqual("fix_managed_nodes", runbook_entry["primary_action_code"])
|
||||
self.assertEqual("focus_release_hub", runbook_entry["secondary_action_code"])
|
||||
self.assertIn("当前解析:补执行面接管", runbook_entry["detail_lines"][0])
|
||||
self.assertEqual("release_hub", runbook_entry["focus_ref"]["kind"])
|
||||
self.assertEqual("release_launchpad", runbook_entry["focus_ref"]["section"])
|
||||
self.assertEqual("default_rollout_gate", runbook_entry["primary_focus_ref"]["section"])
|
||||
self.assertEqual("release_detail", runbook_entry["secondary_focus_ref"]["section"])
|
||||
self.assertEqual("default_rollout_gate", runbook_entry["primary_action_payload"]["focus_ref"]["section"])
|
||||
self.assertEqual("release_detail", runbook_entry["secondary_action_payload"]["focus_ref"]["section"])
|
||||
self.assertEqual("ops_driver_contract", runbook_entry["contract_navigation"]["primary_contract_key"])
|
||||
self.assertIn("ops_playbook_contract", runbook_entry["contract_navigation"]["contract_keys"])
|
||||
self.assertIn("release_hub_contract", runbook_entry["contract_navigation"]["contract_keys"])
|
||||
|
||||
@patch("app.services.ops_service.get_ops_go_live_summary")
|
||||
@patch("app.services.ops_service.get_ops_runbook")
|
||||
@patch("app.services.ops_service.get_ops_overview")
|
||||
def test_driver_feed_preserves_bootstrap_run_as_top_recommendation(
|
||||
self,
|
||||
mock_get_ops_overview,
|
||||
mock_get_ops_runbook,
|
||||
mock_get_ops_go_live_summary,
|
||||
) -> None:
|
||||
mock_get_ops_go_live_summary.return_value = {
|
||||
"go_live_status": "attention",
|
||||
"publish_ready": False,
|
||||
"publish_status": "attention",
|
||||
"publish_status_label": "可发布但建议先复核",
|
||||
"publish_summary": "当前没有硬阻断,但仍有上线前关注项,建议先完成复核再正式发版。",
|
||||
"operator_title": "先跑接入收口",
|
||||
"next_step_action_code": "bootstrap_run",
|
||||
"launchpad_recommended_target_node_code": "mainland-worker-01",
|
||||
"launchpad_recommended_recovery_label": "跑接入收口",
|
||||
"launchpad_recommended_recovery_summary": "mainland-worker-01 还缺接入收口,先执行 bootstrap。",
|
||||
"launchpad_onboarding_bootstrap_pending_nodes": 1,
|
||||
"launchpad_onboarding_acceptance_ready_nodes": 0,
|
||||
"blocking_reasons": [],
|
||||
"warnings": ["remote_agent_ready=1/2"],
|
||||
}
|
||||
mock_get_ops_overview.return_value = {
|
||||
"recommendation": {
|
||||
"key": "handover-gap-bootstrap_run",
|
||||
"priority": "先补接管缺口",
|
||||
"title": "先补接管缺口",
|
||||
"summary": "Launchpad 已明确收敛到首台节点接入收口。",
|
||||
"reason": "mainland-worker-01 还缺接入收口,先执行 bootstrap。",
|
||||
"level_label": "最高优先",
|
||||
"tag_type": "warning",
|
||||
"primary_label": "跑接入收口",
|
||||
"secondary_label": "查看目标节点",
|
||||
"primary_action_code": "bootstrap_run",
|
||||
"secondary_action_code": "view_first_gap",
|
||||
"node_codes": ["mainland-worker-01"],
|
||||
"primary_node_codes": ["mainland-worker-01"],
|
||||
"secondary_node_codes": ["mainland-worker-01"],
|
||||
"primary_action_payload": {"node_code": "mainland-worker-01"},
|
||||
"secondary_action_payload": {},
|
||||
"focus_ref": {"kind": "managed_node", "node_code": "mainland-worker-01"},
|
||||
"primary_focus_ref": {"kind": "managed_node", "node_code": "mainland-worker-01"},
|
||||
"secondary_focus_ref": {"kind": "managed_node", "node_code": "mainland-worker-01"},
|
||||
},
|
||||
"driver_recommendations": [],
|
||||
"activity_stream": {"items": []},
|
||||
"execution_scene": {},
|
||||
}
|
||||
mock_get_ops_runbook.return_value = {"control_sequences": []}
|
||||
|
||||
payload = get_ops_driver_feed()
|
||||
|
||||
self.assertEqual("priority:handover-gap-bootstrap_run", payload["top_recommendation"]["key"])
|
||||
self.assertEqual("attention", payload["summary"]["publish_status"])
|
||||
self.assertEqual("可发布但建议先复核", payload["summary"]["publish_status_label"])
|
||||
self.assertEqual("mainland-worker-01", payload["summary"]["launchpad_recommended_target_node_code"])
|
||||
self.assertEqual("跑接入收口", payload["summary"]["launchpad_recommended_recovery_label"])
|
||||
self.assertEqual(1, payload["summary"]["launchpad_onboarding_bootstrap_pending_nodes"])
|
||||
self.assertEqual(0, payload["summary"]["launchpad_onboarding_acceptance_ready_nodes"])
|
||||
self.assertEqual("attention", payload["summary"]["launch_status"])
|
||||
self.assertFalse(payload["summary"]["launch_ready"])
|
||||
self.assertEqual("bootstrap_run", payload["top_recommendation"]["primary_action_code"])
|
||||
self.assertEqual("mainland-worker-01", payload["top_recommendation"]["primary_node_codes"][0])
|
||||
self.assertIn("ops_agent_protocol", payload["top_recommendation"]["contract_navigation"]["contract_keys"])
|
||||
self.assertEqual("ops_driver_contract", payload["top_recommendation"]["contract_navigation"]["primary_contract_key"])
|
||||
self.assertEqual(0, payload["automation_coverage"]["safe_auto_total"])
|
||||
self.assertEqual(1, payload["automation_coverage"]["mixed_total"])
|
||||
self.assertEqual(0, payload["automation_coverage"]["preview_only_total"])
|
||||
self.assertEqual(1, payload["automation_coverage"]["backend_handled_total"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
24
domain-api/tests/test_ops_service_execution_modes.py
Normal file
24
domain-api/tests/test_ops_service_execution_modes.py
Normal file
@@ -0,0 +1,24 @@
|
||||
import unittest
|
||||
|
||||
from app.core.config import settings
|
||||
from app.services.ops_service import _recommend_driver_execution_mode_for_supported_modes
|
||||
|
||||
|
||||
class OpsServiceExecutionModeRecommendationTests(unittest.TestCase):
|
||||
def test_recommend_driver_execution_mode_prefers_local_runtime_for_current_node(self) -> None:
|
||||
result = _recommend_driver_execution_mode_for_supported_modes(
|
||||
[settings.node_code],
|
||||
supported_modes=["remote-agent", "local-runtime", "ssh"],
|
||||
default_mode="remote-agent",
|
||||
)
|
||||
|
||||
self.assertEqual("local-runtime", result["execution_mode"])
|
||||
self.assertIn("local-runtime", result["reason"])
|
||||
self.assertEqual(
|
||||
["remote-agent", "local-runtime", "ssh"],
|
||||
result["supported_modes"],
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
106
domain-api/tests/test_ops_service_go_live_bundle.py
Normal file
106
domain-api/tests/test_ops_service_go_live_bundle.py
Normal file
@@ -0,0 +1,106 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
from app.services.ops_service import get_ops_go_live_bundle
|
||||
|
||||
|
||||
class OpsServiceGoLiveBundleTests(unittest.TestCase):
|
||||
@patch("app.services.ops_service.get_ops_contract_registry")
|
||||
def test_go_live_bundle_marks_missing_when_no_manifest_exists(self, mock_get_ops_contract_registry) -> None:
|
||||
mock_get_ops_contract_registry.return_value = {
|
||||
"contracts_by_key": {
|
||||
"ops_go_live_bundle_contract": {"version": "v1"},
|
||||
}
|
||||
}
|
||||
|
||||
with patch("app.services.ops_service._find_latest_go_live_bundle_manifest", return_value=(None, Path("/tmp/go-live-bundles"))):
|
||||
payload = get_ops_go_live_bundle(base_url="http://127.0.0.1:8100")
|
||||
|
||||
self.assertEqual("missing", payload["status"])
|
||||
self.assertFalse(payload["bundle_available"])
|
||||
self.assertEqual("待导出", payload["status_label"])
|
||||
self.assertIn("go-live-export", payload["recommended_commands"]["go_live_export"])
|
||||
|
||||
@patch("app.services.ops_service.get_ops_contract_registry")
|
||||
def test_go_live_bundle_reads_manifest_summary_and_review_state(self, mock_get_ops_contract_registry) -> None:
|
||||
mock_get_ops_contract_registry.return_value = {
|
||||
"contracts_by_key": {
|
||||
"ops_go_live_bundle_contract": {"version": "v1"},
|
||||
}
|
||||
}
|
||||
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
report_dir = Path(temp_dir) / "go-live-bundle-001"
|
||||
report_dir.mkdir(parents=True, exist_ok=True)
|
||||
env_audit_path = report_dir / "00_env_audit.txt"
|
||||
env_audit_path.write_text(
|
||||
"[8/8] condensed env audit summary\n"
|
||||
+ json.dumps(
|
||||
{
|
||||
"status": "attention",
|
||||
"headline": "仍有轻微环境缺口",
|
||||
"missing_items": ["missing_tool:jq"],
|
||||
"runtime": {"runtime_may_need_restart": True},
|
||||
"recommended_actions": ["bash domain-api/deploy/multi-region/drive_ops_center.sh runtime-refresh-recover"],
|
||||
},
|
||||
ensure_ascii=False,
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
manifest_path = report_dir / "manifest.json"
|
||||
manifest_path.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"report_dir": str(report_dir),
|
||||
"artifacts": [
|
||||
{"key": "env_audit", "path": str(env_audit_path), "ok": True},
|
||||
{"key": "go_live_summary", "path": str(report_dir / "02_go_live_summary.json"), "ok": True},
|
||||
],
|
||||
"failures": ["doctor_export"],
|
||||
"summary": {
|
||||
"ok": False,
|
||||
"artifact_total": 9,
|
||||
"failure_total": 1,
|
||||
"review_status_hint": "attention",
|
||||
"review_headline": "核心报告已齐,但建议先复核补充缺口。",
|
||||
"launchpad_recommended_target_node_code": "mainland-worker-01",
|
||||
"launchpad_recommended_recovery_label": "先补接管缺口",
|
||||
"launchpad_recommended_recovery_summary": "mainland-worker-01 还缺接入收口。",
|
||||
"launchpad_onboarding_bootstrap_pending_nodes": 1,
|
||||
"launchpad_onboarding_acceptance_ready_nodes": 0,
|
||||
"launchpad_alignment": {"consistent": False, "target_node_code_consistent": False},
|
||||
"recommended_reading_order": ["00_env_audit.txt", "02_go_live_summary.json"],
|
||||
"contract_surface_gaps": ["ops_go_live_bundle_contract"],
|
||||
"discovered_contract_keys": ["ops_go_live_signoff_contract"],
|
||||
},
|
||||
},
|
||||
ensure_ascii=False,
|
||||
indent=2,
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
with patch(
|
||||
"app.services.ops_service._find_latest_go_live_bundle_manifest",
|
||||
return_value=(manifest_path, report_dir),
|
||||
):
|
||||
payload = get_ops_go_live_bundle(base_url="http://127.0.0.1:8100")
|
||||
|
||||
self.assertEqual("attention", payload["status"])
|
||||
self.assertTrue(payload["bundle_available"])
|
||||
self.assertEqual(str(manifest_path), payload["manifest_path"])
|
||||
self.assertEqual(9, payload["artifact_total"])
|
||||
self.assertEqual(["doctor_export"], payload["noncritical_failures"])
|
||||
self.assertEqual("attention", payload["env_audit"]["status"])
|
||||
self.assertTrue(payload["env_audit"]["runtime_may_need_restart"])
|
||||
self.assertFalse(payload["summary"]["launchpad_alignment"]["consistent"])
|
||||
self.assertIn(str(manifest_path), payload["recommended_commands"]["go_live_signoff"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
169
domain-api/tests/test_ops_service_go_live_review.py
Normal file
169
domain-api/tests/test_ops_service_go_live_review.py
Normal file
@@ -0,0 +1,169 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
from app.services.ops_service import get_ops_go_live_review
|
||||
|
||||
|
||||
class OpsServiceGoLiveReviewTests(unittest.TestCase):
|
||||
@patch("app.services.ops_service.get_ops_contract_registry")
|
||||
def test_go_live_review_marks_missing_when_no_manifest_exists(self, mock_get_ops_contract_registry) -> None:
|
||||
mock_get_ops_contract_registry.return_value = {
|
||||
"contracts_by_key": {
|
||||
"ops_go_live_review_contract": {"version": "v1"},
|
||||
}
|
||||
}
|
||||
|
||||
with patch("app.services.ops_service._find_latest_go_live_bundle_manifest", return_value=(None, Path("/tmp/go-live-bundles"))):
|
||||
payload = get_ops_go_live_review(base_url="http://127.0.0.1:8100")
|
||||
|
||||
self.assertEqual("missing", payload["status"])
|
||||
self.assertEqual("待导出", payload["status_label"])
|
||||
self.assertIn("go-live-export", payload["recommended_commands"]["go_live_export"])
|
||||
|
||||
@patch("app.services.ops_service.get_ops_contract_registry")
|
||||
def test_go_live_review_reads_manifest_and_alignment(self, mock_get_ops_contract_registry) -> None:
|
||||
mock_get_ops_contract_registry.return_value = {
|
||||
"contracts_by_key": {
|
||||
"ops_go_live_review_contract": {"version": "v1"},
|
||||
}
|
||||
}
|
||||
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
report_dir = Path(temp_dir) / "go-live-bundle-001"
|
||||
report_dir.mkdir(parents=True, exist_ok=True)
|
||||
env_audit_path = report_dir / "00_env_audit.txt"
|
||||
env_audit_path.write_text(
|
||||
"[8/8] condensed env audit summary\n"
|
||||
+ json.dumps(
|
||||
{
|
||||
"status": "attention",
|
||||
"headline": "环境仍有轻微缺口",
|
||||
"missing_items": ["missing_tool:jq"],
|
||||
"recommended_actions": ["bash runtime-refresh-recover"],
|
||||
},
|
||||
ensure_ascii=False,
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
go_live_summary_path = report_dir / "02_go_live_summary.json"
|
||||
go_live_summary_path.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"go_live_status": "attention",
|
||||
"publish_status": "attention",
|
||||
"operator_title": "先补接入缺口",
|
||||
"next_step_action_code": "fix_managed_nodes",
|
||||
"launchpad_recommended_target_node_code": "mainland-worker-01",
|
||||
"launchpad_recommended_recovery_label": "先补接入缺口",
|
||||
"launchpad_recommended_recovery_summary": "mainland-worker-01 还缺接入收口。",
|
||||
"launchpad_onboarding_bootstrap_pending_nodes": 1,
|
||||
"launchpad_onboarding_acceptance_ready_nodes": 0,
|
||||
},
|
||||
ensure_ascii=False,
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
stack_diagnosis_path = report_dir / "03_stack_diagnosis.json"
|
||||
stack_diagnosis_path.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"diagnosis": {
|
||||
"stack_status": "attention",
|
||||
"operator_decision": {"title": "先补接入缺口"},
|
||||
"next_step": {"action_code": "fix_managed_nodes"},
|
||||
"launchpad_recommended_target_node_code": "mainland-worker-02",
|
||||
"launchpad_recommended_recovery_label": "先看节点接管",
|
||||
"launchpad_recommended_recovery_summary": "stack 与 summary 存在漂移。",
|
||||
"launchpad_onboarding_bootstrap_pending_nodes": 1,
|
||||
"launchpad_onboarding_acceptance_ready_nodes": 0,
|
||||
}
|
||||
},
|
||||
ensure_ascii=False,
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
driver_feed_path = report_dir / "04_driver_feed.json"
|
||||
driver_feed_path.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"headline": "driver ok",
|
||||
"summary": {
|
||||
"launchpad_recommended_target_node_code": "mainland-worker-01",
|
||||
"launchpad_recommended_recovery_label": "先补接入缺口",
|
||||
"launchpad_recommended_recovery_summary": "mainland-worker-01 还缺接入收口。",
|
||||
"launchpad_onboarding_bootstrap_pending_nodes": 1,
|
||||
"launchpad_onboarding_acceptance_ready_nodes": 0,
|
||||
},
|
||||
"automation_coverage": {"launch_status": "attention", "launch_ready": False},
|
||||
},
|
||||
ensure_ascii=False,
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
codex_brief_path = report_dir / "05_codex_brief.json"
|
||||
codex_brief_path.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"headline": "codex ok",
|
||||
"summary": {
|
||||
"launchpad_recommended_target_node_code": "mainland-worker-01",
|
||||
"launchpad_recommended_recovery_label": "先补接入缺口",
|
||||
"launchpad_recommended_recovery_summary": "mainland-worker-01 还缺接入收口。",
|
||||
"launchpad_onboarding_bootstrap_pending_nodes": 1,
|
||||
"launchpad_onboarding_acceptance_ready_nodes": 0,
|
||||
},
|
||||
"automation_coverage": {"launch_status": "attention", "launch_ready": False},
|
||||
"focus": {"entry_key": "fix_managed_nodes"},
|
||||
},
|
||||
ensure_ascii=False,
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
manifest_path = report_dir / "manifest.json"
|
||||
manifest_path.write_text(
|
||||
json.dumps(
|
||||
{
|
||||
"report_dir": str(report_dir),
|
||||
"artifacts": [
|
||||
{"key": "env_audit", "path": str(env_audit_path), "ok": True},
|
||||
{"key": "go_live_summary", "path": str(go_live_summary_path), "ok": True},
|
||||
{"key": "stack_diagnosis", "path": str(stack_diagnosis_path), "ok": True},
|
||||
{"key": "driver_feed", "path": str(driver_feed_path), "ok": True},
|
||||
{"key": "codex_brief", "path": str(codex_brief_path), "ok": True},
|
||||
{"key": "release_launchpad", "path": str(report_dir / "06_release_launchpad.json"), "ok": True},
|
||||
],
|
||||
"failures": ["doctor_export"],
|
||||
"summary": {
|
||||
"artifact_total": 9,
|
||||
"failure_total": 1,
|
||||
"recommended_reading_order": ["00_env_audit.txt", "02_go_live_summary.json"],
|
||||
},
|
||||
},
|
||||
ensure_ascii=False,
|
||||
indent=2,
|
||||
),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
with patch(
|
||||
"app.services.ops_service._find_latest_go_live_bundle_manifest",
|
||||
return_value=(manifest_path, report_dir),
|
||||
):
|
||||
payload = get_ops_go_live_review(base_url="http://127.0.0.1:8100")
|
||||
|
||||
self.assertEqual("attention", payload["status"])
|
||||
self.assertEqual(9, payload["artifact_total"])
|
||||
self.assertEqual(["doctor_export"], payload["noncritical_failures"])
|
||||
self.assertFalse(payload["launchpad_alignment"]["consistent"])
|
||||
self.assertIn("go_live_summary", payload["launchpad_alignment"]["available_sources"])
|
||||
self.assertIn("mainland-worker-01", payload["recommended_next_steps"][0] + "".join(payload["recommended_next_steps"]))
|
||||
self.assertIn(str(manifest_path), payload["recommended_commands"]["go_live_review"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
317
domain-api/tests/test_ops_service_go_live_signoff.py
Normal file
317
domain-api/tests/test_ops_service_go_live_signoff.py
Normal file
@@ -0,0 +1,317 @@
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
from app.services.ops_service import get_ops_go_live_signoff
|
||||
|
||||
|
||||
class OpsServiceGoLiveSignoffTests(unittest.TestCase):
|
||||
@patch("app.services.ops_service.get_ops_contract_registry")
|
||||
@patch("app.services.ops_service.get_ops_doctor_decision")
|
||||
@patch("app.services.ops_service.get_ops_go_live_review")
|
||||
@patch("app.services.ops_service.get_release_launchpad")
|
||||
@patch("app.services.ops_service.get_ops_codex_brief")
|
||||
@patch("app.services.ops_service.get_ops_driver_feed")
|
||||
@patch("app.services.ops_service.get_ops_stack_diagnosis")
|
||||
@patch("app.services.ops_service.get_ops_go_live_summary")
|
||||
def test_go_live_signoff_marks_ready_when_all_surfaces_align(
|
||||
self,
|
||||
mock_get_ops_go_live_summary,
|
||||
mock_get_ops_stack_diagnosis,
|
||||
mock_get_ops_driver_feed,
|
||||
mock_get_ops_codex_brief,
|
||||
mock_get_release_launchpad,
|
||||
mock_get_ops_go_live_review,
|
||||
mock_get_ops_doctor_decision,
|
||||
mock_get_ops_contract_registry,
|
||||
) -> None:
|
||||
mock_get_ops_go_live_summary.return_value = {
|
||||
"go_live_status": "ready",
|
||||
"publish_status": "ready",
|
||||
"operator_title": "进入发布",
|
||||
"next_step_action_code": "publish_latest_worker",
|
||||
"next_step_reason": "当前门禁已收口。",
|
||||
"launchpad_recommended_target_node_code": "mainland-worker-01",
|
||||
"launchpad_recommended_recovery_label": "继续发布",
|
||||
"launchpad_recommended_recovery_summary": "可以进入正式发布。",
|
||||
"launchpad_onboarding_bootstrap_pending_nodes": 0,
|
||||
"launchpad_onboarding_acceptance_ready_nodes": 0,
|
||||
"recommended_commands": {"next_step": "bash next-step"},
|
||||
}
|
||||
mock_get_ops_stack_diagnosis.return_value = {
|
||||
"diagnosis": {
|
||||
"stack_status": "ready",
|
||||
"operator_decision": {"title": "进入发布"},
|
||||
"next_step": {"action_code": "publish_latest_worker"},
|
||||
}
|
||||
}
|
||||
mock_get_ops_driver_feed.return_value = {
|
||||
"headline": "主车道清晰",
|
||||
"summary": {
|
||||
"launch_status": "ready",
|
||||
"launchpad_recommended_target_node_code": "mainland-worker-01",
|
||||
"launchpad_recommended_recovery_label": "继续发布",
|
||||
"launchpad_recommended_recovery_summary": "可以进入正式发布。",
|
||||
"launchpad_onboarding_bootstrap_pending_nodes": 0,
|
||||
"launchpad_onboarding_acceptance_ready_nodes": 0,
|
||||
},
|
||||
}
|
||||
mock_get_ops_codex_brief.return_value = {
|
||||
"headline": "可签字",
|
||||
"summary": {
|
||||
"launch_status": "ready",
|
||||
"launchpad_recommended_target_node_code": "mainland-worker-01",
|
||||
"launchpad_recommended_recovery_label": "继续发布",
|
||||
"launchpad_recommended_recovery_summary": "可以进入正式发布。",
|
||||
"launchpad_onboarding_bootstrap_pending_nodes": 0,
|
||||
"launchpad_onboarding_acceptance_ready_nodes": 0,
|
||||
},
|
||||
"focus": {"entry_key": "publish_latest_worker"},
|
||||
}
|
||||
mock_get_release_launchpad.return_value = {
|
||||
"launchpad_status": {
|
||||
"status": "ready",
|
||||
"status_label": "可发布",
|
||||
"recommended_target_node_code": "mainland-worker-01",
|
||||
"recommended_recovery_label": "继续发布",
|
||||
"recommended_recovery_summary": "可以进入正式发布。",
|
||||
"onboarding_bootstrap_pending_nodes": 0,
|
||||
"onboarding_acceptance_ready_nodes": 0,
|
||||
}
|
||||
}
|
||||
mock_get_ops_go_live_review.return_value = {
|
||||
"status": "ready",
|
||||
"status_label": "可复核发布",
|
||||
"headline": "review ok",
|
||||
"report_dir": "/tmp/review",
|
||||
"manifest_path": "/tmp/review/manifest.json",
|
||||
"recommended_commands": {"go_live_review": "bash review"},
|
||||
}
|
||||
mock_get_ops_doctor_decision.return_value = {
|
||||
"status": "ready",
|
||||
"headline": "doctor ok",
|
||||
"report_dir": "/tmp/review",
|
||||
"manifest_path": "/tmp/review/manifest.json",
|
||||
"decision": {
|
||||
"reason_code": "all_green",
|
||||
"preferred_surface": "go-live-summary",
|
||||
"next_action_code": "publish_latest_worker",
|
||||
},
|
||||
"recommended_commands": {"doctor_decision": "bash doctor"},
|
||||
}
|
||||
mock_get_ops_contract_registry.return_value = {
|
||||
"contracts_by_key": {
|
||||
"ops_go_live_signoff_contract": {"version": "v1"},
|
||||
}
|
||||
}
|
||||
|
||||
payload = get_ops_go_live_signoff(base_url="http://127.0.0.1:8100")
|
||||
|
||||
self.assertEqual("ready", payload["signoff_status"])
|
||||
self.assertEqual("可签字上线", payload["status_label"])
|
||||
self.assertEqual([], payload["blocked_reasons"])
|
||||
self.assertEqual([], payload["attention_reasons"])
|
||||
self.assertTrue(payload["launchpad_alignment"]["consistent"])
|
||||
self.assertEqual("go-live-summary", payload["decision"]["preferred_surface"])
|
||||
self.assertIn("go-live-signoff", payload["recommended_commands"]["go_live_signoff"])
|
||||
self.assertEqual("ready", payload["release_gate"]["go_live_review_status"])
|
||||
self.assertEqual("ready", payload["release_gate"]["doctor_decision_status"])
|
||||
self.assertEqual("/tmp/review/manifest.json", payload["manifest_path"])
|
||||
|
||||
@patch("app.services.ops_service.get_ops_contract_registry")
|
||||
@patch("app.services.ops_service.get_ops_doctor_decision")
|
||||
@patch("app.services.ops_service.get_ops_go_live_review")
|
||||
@patch("app.services.ops_service.get_release_launchpad")
|
||||
@patch("app.services.ops_service.get_ops_codex_brief")
|
||||
@patch("app.services.ops_service.get_ops_driver_feed")
|
||||
@patch("app.services.ops_service.get_ops_stack_diagnosis")
|
||||
@patch("app.services.ops_service.get_ops_go_live_summary")
|
||||
def test_go_live_signoff_marks_attention_when_launchpad_alignment_drifts(
|
||||
self,
|
||||
mock_get_ops_go_live_summary,
|
||||
mock_get_ops_stack_diagnosis,
|
||||
mock_get_ops_driver_feed,
|
||||
mock_get_ops_codex_brief,
|
||||
mock_get_release_launchpad,
|
||||
mock_get_ops_go_live_review,
|
||||
mock_get_ops_doctor_decision,
|
||||
mock_get_ops_contract_registry,
|
||||
) -> None:
|
||||
mock_get_ops_go_live_summary.return_value = {
|
||||
"go_live_status": "ready",
|
||||
"publish_status": "ready",
|
||||
"operator_title": "进入发布",
|
||||
"next_step_action_code": "publish_latest_worker",
|
||||
"launchpad_recommended_target_node_code": "mainland-worker-01",
|
||||
"launchpad_recommended_recovery_label": "继续发布",
|
||||
"launchpad_recommended_recovery_summary": "可以进入正式发布。",
|
||||
"launchpad_onboarding_bootstrap_pending_nodes": 0,
|
||||
"launchpad_onboarding_acceptance_ready_nodes": 0,
|
||||
"recommended_commands": {},
|
||||
}
|
||||
mock_get_ops_stack_diagnosis.return_value = {
|
||||
"diagnosis": {"stack_status": "ready", "operator_decision": {}, "next_step": {}}
|
||||
}
|
||||
mock_get_ops_driver_feed.return_value = {
|
||||
"headline": "主车道清晰",
|
||||
"summary": {
|
||||
"launch_status": "ready",
|
||||
"launchpad_recommended_target_node_code": "mainland-worker-02",
|
||||
"launchpad_recommended_recovery_label": "继续发布",
|
||||
"launchpad_recommended_recovery_summary": "可以进入正式发布。",
|
||||
"launchpad_onboarding_bootstrap_pending_nodes": 0,
|
||||
"launchpad_onboarding_acceptance_ready_nodes": 0,
|
||||
},
|
||||
}
|
||||
mock_get_ops_codex_brief.return_value = {
|
||||
"headline": "可签字",
|
||||
"summary": {
|
||||
"launch_status": "ready",
|
||||
"launchpad_recommended_target_node_code": "mainland-worker-01",
|
||||
"launchpad_recommended_recovery_label": "继续发布",
|
||||
"launchpad_recommended_recovery_summary": "可以进入正式发布。",
|
||||
"launchpad_onboarding_bootstrap_pending_nodes": 0,
|
||||
"launchpad_onboarding_acceptance_ready_nodes": 0,
|
||||
},
|
||||
"focus": {},
|
||||
}
|
||||
mock_get_release_launchpad.return_value = {
|
||||
"launchpad_status": {
|
||||
"status": "ready",
|
||||
"recommended_target_node_code": "mainland-worker-01",
|
||||
"recommended_recovery_label": "继续发布",
|
||||
"recommended_recovery_summary": "可以进入正式发布。",
|
||||
"onboarding_bootstrap_pending_nodes": 0,
|
||||
"onboarding_acceptance_ready_nodes": 0,
|
||||
}
|
||||
}
|
||||
mock_get_ops_go_live_review.return_value = {
|
||||
"status": "attention",
|
||||
"headline": "review drift",
|
||||
"recommended_commands": {"go_live_review": "bash review"},
|
||||
}
|
||||
mock_get_ops_doctor_decision.return_value = {
|
||||
"status": "ready",
|
||||
"headline": "doctor ok",
|
||||
"decision": {"preferred_surface": "go-live-summary"},
|
||||
"recommended_commands": {"doctor_decision": "bash doctor"},
|
||||
}
|
||||
mock_get_ops_contract_registry.return_value = {
|
||||
"contracts_by_key": {
|
||||
"ops_go_live_signoff_contract": {"version": "v1"},
|
||||
}
|
||||
}
|
||||
|
||||
payload = get_ops_go_live_signoff()
|
||||
|
||||
self.assertEqual("attention", payload["signoff_status"])
|
||||
self.assertIn("launchpad_alignment=inconsistent", payload["attention_reasons"])
|
||||
self.assertIn("go_live_review=attention", payload["attention_reasons"])
|
||||
self.assertFalse(payload["launchpad_alignment"]["consistent"])
|
||||
self.assertFalse(payload["launchpad_alignment"]["target_node_code_consistent"])
|
||||
self.assertEqual("go-live-review", payload["decision"]["preferred_surface"])
|
||||
|
||||
@patch("app.services.ops_service.get_ops_contract_registry")
|
||||
@patch("app.services.ops_service.get_ops_doctor_decision")
|
||||
@patch("app.services.ops_service.get_ops_go_live_review")
|
||||
@patch("app.services.ops_service.get_release_launchpad")
|
||||
@patch("app.services.ops_service.get_ops_codex_brief")
|
||||
@patch("app.services.ops_service.get_ops_driver_feed")
|
||||
@patch("app.services.ops_service.get_ops_stack_diagnosis")
|
||||
@patch("app.services.ops_service.get_ops_go_live_summary")
|
||||
def test_go_live_signoff_marks_blocked_when_publish_gate_is_blocked(
|
||||
self,
|
||||
mock_get_ops_go_live_summary,
|
||||
mock_get_ops_stack_diagnosis,
|
||||
mock_get_ops_driver_feed,
|
||||
mock_get_ops_codex_brief,
|
||||
mock_get_release_launchpad,
|
||||
mock_get_ops_go_live_review,
|
||||
mock_get_ops_doctor_decision,
|
||||
mock_get_ops_contract_registry,
|
||||
) -> None:
|
||||
mock_get_ops_go_live_summary.return_value = {
|
||||
"go_live_status": "attention",
|
||||
"publish_status": "blocked",
|
||||
"operator_title": "先处理发布阻断",
|
||||
"next_step_action_code": "fix_rollout_blockers",
|
||||
"launchpad_recommended_target_node_code": "mainland-worker-01",
|
||||
"launchpad_recommended_recovery_label": "处理发布阻断",
|
||||
"launchpad_recommended_recovery_summary": "当前 rollout gate 未通过。",
|
||||
"launchpad_onboarding_bootstrap_pending_nodes": 0,
|
||||
"launchpad_onboarding_acceptance_ready_nodes": 0,
|
||||
"recommended_commands": {},
|
||||
}
|
||||
mock_get_ops_stack_diagnosis.return_value = {
|
||||
"diagnosis": {
|
||||
"stack_status": "attention",
|
||||
"operator_decision": {"title": "先看发布门禁"},
|
||||
"next_step": {"action_code": "fix_rollout_blockers"},
|
||||
}
|
||||
}
|
||||
mock_get_ops_driver_feed.return_value = {
|
||||
"headline": "发布门禁阻断",
|
||||
"summary": {
|
||||
"launch_status": "attention",
|
||||
"launchpad_recommended_target_node_code": "mainland-worker-01",
|
||||
"launchpad_recommended_recovery_label": "处理发布阻断",
|
||||
"launchpad_recommended_recovery_summary": "当前 rollout gate 未通过。",
|
||||
"launchpad_onboarding_bootstrap_pending_nodes": 0,
|
||||
"launchpad_onboarding_acceptance_ready_nodes": 0,
|
||||
},
|
||||
}
|
||||
mock_get_ops_codex_brief.return_value = {
|
||||
"headline": "需先排阻断",
|
||||
"summary": {
|
||||
"launch_status": "attention",
|
||||
"launchpad_recommended_target_node_code": "mainland-worker-01",
|
||||
"launchpad_recommended_recovery_label": "处理发布阻断",
|
||||
"launchpad_recommended_recovery_summary": "当前 rollout gate 未通过。",
|
||||
"launchpad_onboarding_bootstrap_pending_nodes": 0,
|
||||
"launchpad_onboarding_acceptance_ready_nodes": 0,
|
||||
},
|
||||
"focus": {"entry_key": "fix_rollout_blockers"},
|
||||
}
|
||||
mock_get_release_launchpad.return_value = {
|
||||
"launchpad_status": {
|
||||
"status": "blocked",
|
||||
"status_label": "发布阻断",
|
||||
"recommended_action_code": "fix_rollout_blockers",
|
||||
"recommended_target_node_code": "mainland-worker-01",
|
||||
"recommended_recovery_label": "处理发布阻断",
|
||||
"recommended_recovery_summary": "当前 rollout gate 未通过。",
|
||||
"onboarding_bootstrap_pending_nodes": 0,
|
||||
"onboarding_acceptance_ready_nodes": 0,
|
||||
}
|
||||
}
|
||||
mock_get_ops_go_live_review.return_value = {
|
||||
"status": "ready",
|
||||
"headline": "review ok",
|
||||
"recommended_commands": {"go_live_review": "bash review"},
|
||||
}
|
||||
mock_get_ops_doctor_decision.return_value = {
|
||||
"status": "attention",
|
||||
"headline": "doctor focus",
|
||||
"decision": {
|
||||
"reason_code": "publish_blocked",
|
||||
"preferred_surface": "release-launchpad",
|
||||
"next_action_code": "fix_rollout_blockers",
|
||||
},
|
||||
"recommended_commands": {"doctor_decision": "bash doctor"},
|
||||
}
|
||||
mock_get_ops_contract_registry.return_value = {
|
||||
"contracts_by_key": {
|
||||
"ops_go_live_signoff_contract": {"version": "v1"},
|
||||
}
|
||||
}
|
||||
|
||||
payload = get_ops_go_live_signoff()
|
||||
|
||||
self.assertEqual("blocked", payload["signoff_status"])
|
||||
self.assertIn("publish_status=blocked", payload["blocked_reasons"])
|
||||
self.assertIn("release_launchpad=blocked", payload["blocked_reasons"])
|
||||
self.assertEqual("release-launchpad", payload["decision"]["preferred_surface"])
|
||||
self.assertEqual("publish_blocked", payload["decision"]["doctor_reason_code"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
664
domain-api/tests/test_ops_service_go_live_summary.py
Normal file
664
domain-api/tests/test_ops_service_go_live_summary.py
Normal file
@@ -0,0 +1,664 @@
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
from app.services.ops_service import get_ops_go_live_summary
|
||||
|
||||
|
||||
class OpsServiceGoLiveSummaryTests(unittest.TestCase):
|
||||
@patch("app.services.ops_service.get_ops_overview")
|
||||
@patch("app.services.ops_service.list_managed_nodes_with_agent_state")
|
||||
@patch("app.services.ops_service.get_runtime_build_info")
|
||||
@patch("app.services.ops_service.get_release_launchpad")
|
||||
@patch("app.services.ops_service.get_ops_contract_registry")
|
||||
@patch("app.services.ops_service.get_ops_stack_diagnosis")
|
||||
def test_go_live_summary_exposes_runtime_schema_stale_and_api_restart_command(
|
||||
self,
|
||||
mock_get_ops_stack_diagnosis,
|
||||
mock_get_ops_contract_registry,
|
||||
mock_get_release_launchpad,
|
||||
mock_get_runtime_build_info,
|
||||
mock_list_managed_nodes_with_agent_state,
|
||||
mock_get_ops_overview,
|
||||
) -> None:
|
||||
mock_get_ops_stack_diagnosis.return_value = {
|
||||
"diagnosis": {
|
||||
"contract_key": "ops_stack_diagnosis_contract",
|
||||
"stack_status": "attention",
|
||||
"next_step": {
|
||||
"action_code": "api-restart",
|
||||
"reason": "build schema stale",
|
||||
"focus_ref": {"kind": "runtime_build_info", "section": "schema"},
|
||||
},
|
||||
"operator_decision": {
|
||||
"lane": "runtime_recovery",
|
||||
"title": "优先重启控制面 API 并复检运行时版本",
|
||||
"primary_command_key": "api-restart",
|
||||
},
|
||||
},
|
||||
"release_hub": {
|
||||
"launchpad_status": "ready",
|
||||
},
|
||||
}
|
||||
mock_get_ops_contract_registry.return_value = {
|
||||
"registry_version": "2026-04-18",
|
||||
"contracts": [{"key": "ops_stack_diagnosis_contract"}],
|
||||
}
|
||||
mock_get_release_launchpad.return_value = {
|
||||
"launchpad_status": {
|
||||
"status": "ready",
|
||||
"status_label": "可发布",
|
||||
"recommended_action_code": "create_smart_release_rollout_worker",
|
||||
}
|
||||
}
|
||||
mock_get_runtime_build_info.return_value = {
|
||||
"commit_sha": "246838ae4c07",
|
||||
"repository_capabilities": {
|
||||
"supports_install_command_block": True,
|
||||
"supports_multi_layout_bootstrap": True,
|
||||
},
|
||||
"route_surface": {
|
||||
"surface_complete": True,
|
||||
"missing_keys": [],
|
||||
"expected_paths": {
|
||||
"ops_contracts": "/api/v1/ops/contracts",
|
||||
},
|
||||
},
|
||||
}
|
||||
mock_list_managed_nodes_with_agent_state.return_value = {
|
||||
"summary": {
|
||||
"managed_enabled": 1,
|
||||
"remote_access_ready": 1,
|
||||
"queue_dead_letter_nodes": 0,
|
||||
}
|
||||
}
|
||||
mock_get_ops_overview.return_value = {
|
||||
"recommendation": {
|
||||
"title": "先重启 API",
|
||||
"reason": "build schema stale",
|
||||
"primary_action_code": "api-restart",
|
||||
},
|
||||
"execution_scene": {
|
||||
"log_sync": {
|
||||
"enabled": False,
|
||||
"participating_node_count": 0,
|
||||
"covered_participating_node_count": 0,
|
||||
"missing_participating_nodes": [],
|
||||
}
|
||||
},
|
||||
}
|
||||
|
||||
payload = get_ops_go_live_summary(base_url="http://127.0.0.1:8100")
|
||||
|
||||
self.assertEqual("attention", payload["go_live_status"])
|
||||
self.assertEqual("attention", payload["publish_status"])
|
||||
self.assertTrue(payload["runtime_schema_stale"])
|
||||
self.assertFalse(payload["route_surface_declares_bootstrap_plan"])
|
||||
self.assertIn("runtime_build_schema_stale", payload["warnings"])
|
||||
self.assertIn("runtime_build_schema_stale", payload["publish_warnings"])
|
||||
self.assertEqual(
|
||||
"bash domain-api/deploy/multi-region/drive_ops_center.sh go-live-recover http://127.0.0.1:8100",
|
||||
payload["recommended_commands"]["go_live_recover"],
|
||||
)
|
||||
self.assertEqual(
|
||||
"bash domain-api/deploy/multi-region/drive_ops_center.sh runtime-refresh-recover http://127.0.0.1:8100",
|
||||
payload["recommended_commands"]["api_restart"],
|
||||
)
|
||||
self.assertEqual(
|
||||
"bash domain-api/deploy/multi-region/drive_ops_center.sh runtime-refresh-recover http://127.0.0.1:8100",
|
||||
payload["recommended_commands"]["runtime_refresh_recover"],
|
||||
)
|
||||
self.assertIn("node-bootstrap-plan", payload["recommended_commands"]["bootstrap_plan_recheck"])
|
||||
|
||||
@patch("app.services.ops_service.get_ops_overview")
|
||||
@patch("app.services.ops_service.list_managed_nodes_with_agent_state")
|
||||
@patch("app.services.ops_service.get_runtime_build_info")
|
||||
@patch("app.services.ops_service.get_release_launchpad")
|
||||
@patch("app.services.ops_service.get_ops_contract_registry")
|
||||
@patch("app.services.ops_service.get_ops_stack_diagnosis")
|
||||
def test_go_live_summary_highlights_remote_access_and_log_sync_gaps(
|
||||
self,
|
||||
mock_get_ops_stack_diagnosis,
|
||||
mock_get_ops_contract_registry,
|
||||
mock_get_release_launchpad,
|
||||
mock_get_runtime_build_info,
|
||||
mock_list_managed_nodes_with_agent_state,
|
||||
mock_get_ops_overview,
|
||||
) -> None:
|
||||
mock_get_ops_stack_diagnosis.return_value = {
|
||||
"diagnosis": {
|
||||
"contract_key": "ops_stack_diagnosis_contract",
|
||||
"stack_status": "blocked",
|
||||
"next_step": {
|
||||
"action_code": "handover_first_gap",
|
||||
"reason": "来自 overview.recommendation.primary_action_code",
|
||||
},
|
||||
"operator_decision": {},
|
||||
},
|
||||
"release_hub": {
|
||||
"launchpad_status": "attention",
|
||||
},
|
||||
}
|
||||
mock_get_ops_contract_registry.return_value = {
|
||||
"registry_version": "2026-04-18",
|
||||
"contracts": [{"key": "ops_stack_diagnosis_contract"}],
|
||||
}
|
||||
mock_get_release_launchpad.return_value = {
|
||||
"launchpad_status": {
|
||||
"status": "attention",
|
||||
"status_label": "待补执行面",
|
||||
"recommended_action_code": "fix_managed_nodes",
|
||||
"focus_ref": {"kind": "release_hub"},
|
||||
"onboarding_bootstrap_pending_nodes": 1,
|
||||
"onboarding_acceptance_ready_nodes": 1,
|
||||
"recommended_target_node_code": "mainland-controller-01",
|
||||
"recommended_recovery_label": "签发接入工单",
|
||||
"recommended_recovery_summary": "当前节点仍处于接入阶段,建议先签发 onboarding.bootstrap。",
|
||||
}
|
||||
}
|
||||
mock_get_runtime_build_info.return_value = {
|
||||
"commit_sha": "246838ae4c07",
|
||||
"route_surface": {
|
||||
"surface_complete": True,
|
||||
"missing_keys": [],
|
||||
},
|
||||
}
|
||||
mock_list_managed_nodes_with_agent_state.return_value = {
|
||||
"summary": {
|
||||
"managed_enabled": 3,
|
||||
"remote_access_ready": 0,
|
||||
"queue_dead_letter_nodes": 0,
|
||||
}
|
||||
}
|
||||
mock_get_ops_overview.return_value = {
|
||||
"recommendation": {
|
||||
"title": "先补接管缺口",
|
||||
"reason": "当前仍有节点没有进入标准执行器就绪状态。",
|
||||
"primary_action_code": "handover_first_gap",
|
||||
},
|
||||
"execution_scene": {
|
||||
"log_sync": {
|
||||
"enabled": True,
|
||||
"state": "waiting_sample",
|
||||
"mode": "full",
|
||||
"participating_node_count": 3,
|
||||
"covered_participating_node_count": 0,
|
||||
"missing_participating_nodes": [
|
||||
"mainland-worker-01",
|
||||
"mainland-controller-01",
|
||||
"overseas-control-01",
|
||||
],
|
||||
}
|
||||
},
|
||||
}
|
||||
|
||||
payload = get_ops_go_live_summary(base_url="http://127.0.0.1:8100")
|
||||
|
||||
self.assertEqual("blocked", payload["go_live_status"])
|
||||
self.assertFalse(payload["publish_ready"])
|
||||
self.assertEqual("blocked", payload["publish_status"])
|
||||
self.assertEqual("暂不可发布", payload["publish_status_label"])
|
||||
self.assertIn("stack_diagnosis=blocked", payload["blocking_reasons"])
|
||||
self.assertIn("stack_diagnosis=blocked", payload["publish_blocking_reasons"])
|
||||
self.assertIn("remote_agent_ready=0", payload["blocking_reasons"])
|
||||
self.assertIn("remote_agent_ready=0", payload["publish_blocking_reasons"])
|
||||
self.assertIn("release_launchpad=attention", payload["warnings"])
|
||||
self.assertIn("release_launchpad=attention", payload["publish_warnings"])
|
||||
self.assertIn("launchpad_onboarding_bootstrap_pending=1", payload["warnings"])
|
||||
self.assertIn("launchpad_onboarding_acceptance_ready=1", payload["warnings"])
|
||||
self.assertIn("launchpad_onboarding_bootstrap_pending=1", payload["publish_warnings"])
|
||||
self.assertIn("launchpad_onboarding_acceptance_ready=1", payload["publish_warnings"])
|
||||
self.assertIn("log_sync_waiting_sample=0/3", payload["warnings"])
|
||||
self.assertIn("log_sync_waiting_sample=0/3", payload["publish_warnings"])
|
||||
self.assertTrue(payload["route_surface_complete"])
|
||||
self.assertEqual("先补接管缺口", payload["operator_title"])
|
||||
self.assertEqual("handover_first_gap", payload["next_step_action_code"])
|
||||
self.assertEqual("mainland-controller-01", payload["launchpad_recommended_target_node_code"])
|
||||
self.assertEqual("签发接入工单", payload["launchpad_recommended_recovery_label"])
|
||||
self.assertEqual(
|
||||
"当前节点仍处于接入阶段,建议先签发 onboarding.bootstrap。",
|
||||
payload["launchpad_recommended_recovery_summary"],
|
||||
)
|
||||
self.assertEqual(1, payload["launchpad_onboarding_bootstrap_pending_nodes"])
|
||||
self.assertEqual(1, payload["launchpad_onboarding_acceptance_ready_nodes"])
|
||||
self.assertEqual(
|
||||
"bash domain-api/deploy/multi-region/drive_ops_center.sh driver-resolve http://127.0.0.1:8100 handover_first_gap",
|
||||
payload["recommended_commands"]["next_step"],
|
||||
)
|
||||
self.assertEqual(
|
||||
"bash domain-api/deploy/multi-region/drive_ops_center.sh driver-resolve http://127.0.0.1:8100 open_worker_logs_participating",
|
||||
payload["recommended_commands"]["log_sync_logs"],
|
||||
)
|
||||
|
||||
@patch("app.services.ops_service.get_ops_overview")
|
||||
@patch("app.services.ops_service.list_managed_nodes_with_agent_state")
|
||||
@patch("app.services.ops_service.get_runtime_build_info")
|
||||
@patch("app.services.ops_service.get_release_launchpad")
|
||||
@patch("app.services.ops_service.get_ops_contract_registry")
|
||||
@patch("app.services.ops_service.get_ops_stack_diagnosis")
|
||||
def test_go_live_summary_falls_back_to_stack_handover_hint_when_launchpad_is_generic(
|
||||
self,
|
||||
mock_get_ops_stack_diagnosis,
|
||||
mock_get_ops_contract_registry,
|
||||
mock_get_release_launchpad,
|
||||
mock_get_runtime_build_info,
|
||||
mock_list_managed_nodes_with_agent_state,
|
||||
mock_get_ops_overview,
|
||||
) -> None:
|
||||
mock_get_ops_stack_diagnosis.return_value = {
|
||||
"diagnosis": {
|
||||
"contract_key": "ops_stack_diagnosis_contract",
|
||||
"stack_status": "blocked",
|
||||
"next_step": {
|
||||
"action_code": "bootstrap_run",
|
||||
"reason": "当前节点仍处于接入阶段,建议先签发 onboarding.bootstrap。",
|
||||
"focus_ref": {"kind": "managed_node", "node_code": "mainland-worker-01"},
|
||||
},
|
||||
"operator_decision": {
|
||||
"lane": "node_handover",
|
||||
"title": "优先执行节点接管收口",
|
||||
"primary_command_key": "bootstrap_run",
|
||||
},
|
||||
"launchpad_recommended_target_node_code": "mainland-worker-01",
|
||||
"launchpad_recommended_recovery_label": "签发接入工单",
|
||||
"launchpad_recommended_recovery_summary": "当前节点仍处于接入阶段,建议先签发 onboarding.bootstrap。",
|
||||
},
|
||||
"release_hub": {
|
||||
"launchpad_status": "attention",
|
||||
},
|
||||
}
|
||||
mock_get_ops_contract_registry.return_value = {
|
||||
"registry_version": "2026-04-18",
|
||||
"contracts": [{"key": "ops_stack_diagnosis_contract"}],
|
||||
}
|
||||
mock_get_release_launchpad.return_value = {
|
||||
"launchpad_status": {
|
||||
"status": "attention",
|
||||
"status_label": "待补执行面",
|
||||
"recommended_action_code": "fix_managed_nodes",
|
||||
}
|
||||
}
|
||||
mock_get_runtime_build_info.return_value = {
|
||||
"commit_sha": "246838ae4c07",
|
||||
"route_surface": {
|
||||
"surface_complete": True,
|
||||
"missing_keys": [],
|
||||
"expected_paths": {},
|
||||
},
|
||||
"repository_capabilities": {},
|
||||
}
|
||||
mock_list_managed_nodes_with_agent_state.return_value = {
|
||||
"summary": {
|
||||
"managed_enabled": 1,
|
||||
"remote_access_ready": 0,
|
||||
"queue_dead_letter_nodes": 0,
|
||||
}
|
||||
}
|
||||
mock_get_ops_overview.return_value = {
|
||||
"recommendation": {
|
||||
"title": "先补接管缺口",
|
||||
"reason": "overview 仍在泛化描述。",
|
||||
"primary_action_code": "handover_first_gap",
|
||||
},
|
||||
"execution_scene": {
|
||||
"log_sync": {
|
||||
"enabled": False,
|
||||
"participating_node_count": 0,
|
||||
"covered_participating_node_count": 0,
|
||||
"missing_participating_nodes": [],
|
||||
}
|
||||
},
|
||||
}
|
||||
|
||||
payload = get_ops_go_live_summary(base_url="http://127.0.0.1:8100")
|
||||
|
||||
self.assertEqual("bootstrap_run", payload["next_step_action_code"])
|
||||
self.assertEqual("mainland-worker-01", payload["launchpad_recommended_target_node_code"])
|
||||
self.assertEqual("签发接入工单", payload["launchpad_recommended_recovery_label"])
|
||||
self.assertIn("onboarding.bootstrap", payload["launchpad_recommended_recovery_summary"])
|
||||
self.assertEqual(
|
||||
"bash domain-api/deploy/multi-region/drive_ops_center.sh driver-resolve http://127.0.0.1:8100 bootstrap_run",
|
||||
payload["recommended_commands"]["next_step"],
|
||||
)
|
||||
|
||||
@patch("app.services.ops_service.get_ops_overview")
|
||||
@patch("app.services.ops_service.list_managed_nodes_with_agent_state")
|
||||
@patch("app.services.ops_service.get_runtime_build_info")
|
||||
@patch("app.services.ops_service.get_release_launchpad")
|
||||
@patch("app.services.ops_service.get_ops_contract_registry")
|
||||
@patch("app.services.ops_service.get_ops_stack_diagnosis")
|
||||
def test_go_live_summary_marks_ready_when_surfaces_and_execution_are_closed(
|
||||
self,
|
||||
mock_get_ops_stack_diagnosis,
|
||||
mock_get_ops_contract_registry,
|
||||
mock_get_release_launchpad,
|
||||
mock_get_runtime_build_info,
|
||||
mock_list_managed_nodes_with_agent_state,
|
||||
mock_get_ops_overview,
|
||||
) -> None:
|
||||
mock_get_ops_stack_diagnosis.return_value = {
|
||||
"diagnosis": {
|
||||
"contract_key": "ops_stack_diagnosis_contract",
|
||||
"stack_status": "healthy",
|
||||
"next_step": {
|
||||
"action_code": "run_inspection_participating",
|
||||
"reason": "继续做上线前复核。",
|
||||
},
|
||||
"operator_decision": {
|
||||
"lane": "go-live",
|
||||
"title": "可以进入上线复核",
|
||||
"primary_command_key": "run_inspection_participating",
|
||||
},
|
||||
},
|
||||
"release_hub": {
|
||||
"launchpad_status": "ready",
|
||||
},
|
||||
}
|
||||
mock_get_ops_contract_registry.return_value = {
|
||||
"registry_version": "2026-04-18",
|
||||
"contracts": [{"key": "ops_stack_diagnosis_contract"}],
|
||||
}
|
||||
mock_get_release_launchpad.return_value = {
|
||||
"launchpad_status": {
|
||||
"status": "ready",
|
||||
"status_label": "可发布",
|
||||
"recommended_action_code": "create_smart_release_rollout_worker",
|
||||
}
|
||||
}
|
||||
mock_get_runtime_build_info.return_value = {
|
||||
"commit_sha": "246838ae4c07",
|
||||
"route_surface": {
|
||||
"surface_complete": True,
|
||||
"missing_keys": [],
|
||||
},
|
||||
}
|
||||
mock_list_managed_nodes_with_agent_state.return_value = {
|
||||
"summary": {
|
||||
"managed_enabled": 2,
|
||||
"remote_access_ready": 2,
|
||||
"queue_dead_letter_nodes": 0,
|
||||
}
|
||||
}
|
||||
mock_get_ops_overview.return_value = {
|
||||
"recommendation": {
|
||||
"title": "继续复核",
|
||||
"reason": "当前执行面与观测面已收口。",
|
||||
"primary_action_code": "run_inspection_participating",
|
||||
},
|
||||
"execution_scene": {
|
||||
"log_sync": {
|
||||
"enabled": True,
|
||||
"state": "healthy",
|
||||
"mode": "key",
|
||||
"participating_node_count": 2,
|
||||
"covered_participating_node_count": 2,
|
||||
"missing_participating_nodes": [],
|
||||
}
|
||||
},
|
||||
}
|
||||
|
||||
payload = get_ops_go_live_summary(base_url="http://127.0.0.1:8100")
|
||||
|
||||
self.assertEqual("ready", payload["go_live_status"])
|
||||
self.assertTrue(payload["publish_ready"])
|
||||
self.assertEqual("ready", payload["publish_status"])
|
||||
self.assertEqual("可发布", payload["publish_status_label"])
|
||||
self.assertEqual([], payload["blocking_reasons"])
|
||||
self.assertEqual([], payload["warnings"])
|
||||
self.assertEqual([], payload["publish_blocking_reasons"])
|
||||
self.assertEqual([], payload["publish_warnings"])
|
||||
self.assertEqual("go-live", payload["operator_lane"])
|
||||
self.assertEqual("run_inspection_participating", payload["operator_primary_command_key"])
|
||||
self.assertEqual(2, payload["log_sync_covered_nodes"])
|
||||
self.assertEqual(
|
||||
"bash domain-api/deploy/multi-region/drive_ops_center.sh driver-resolve http://127.0.0.1:8100 run_inspection_participating",
|
||||
payload["recommended_commands"]["next_step"],
|
||||
)
|
||||
|
||||
@patch("app.services.ops_service.get_ops_overview")
|
||||
@patch("app.services.ops_service.list_managed_nodes_with_agent_state")
|
||||
@patch("app.services.ops_service.get_runtime_build_info")
|
||||
@patch("app.services.ops_service.get_release_launchpad")
|
||||
@patch("app.services.ops_service.get_ops_contract_registry")
|
||||
@patch("app.services.ops_service.get_ops_stack_diagnosis")
|
||||
def test_go_live_summary_blocks_publish_when_dead_letter_queue_exists(
|
||||
self,
|
||||
mock_get_ops_stack_diagnosis,
|
||||
mock_get_ops_contract_registry,
|
||||
mock_get_release_launchpad,
|
||||
mock_get_runtime_build_info,
|
||||
mock_list_managed_nodes_with_agent_state,
|
||||
mock_get_ops_overview,
|
||||
) -> None:
|
||||
mock_get_ops_stack_diagnosis.return_value = {
|
||||
"diagnosis": {
|
||||
"contract_key": "ops_stack_diagnosis_contract",
|
||||
"stack_status": "healthy",
|
||||
"next_step": {"action_code": "delivery_queue_replay", "reason": "先清死信。"},
|
||||
"operator_decision": {},
|
||||
},
|
||||
"release_hub": {
|
||||
"launchpad_status": "ready",
|
||||
},
|
||||
}
|
||||
mock_get_ops_contract_registry.return_value = {
|
||||
"registry_version": "2026-04-18",
|
||||
"contracts": [{"key": "ops_stack_diagnosis_contract"}],
|
||||
}
|
||||
mock_get_release_launchpad.return_value = {
|
||||
"launchpad_status": {
|
||||
"status": "ready",
|
||||
"status_label": "可发布",
|
||||
"recommended_action_code": "create_smart_release_rollout_worker",
|
||||
}
|
||||
}
|
||||
mock_get_runtime_build_info.return_value = {
|
||||
"commit_sha": "246838ae4c07",
|
||||
"route_surface": {
|
||||
"surface_complete": True,
|
||||
"missing_keys": [],
|
||||
},
|
||||
}
|
||||
mock_list_managed_nodes_with_agent_state.return_value = {
|
||||
"summary": {
|
||||
"managed_enabled": 2,
|
||||
"remote_access_ready": 2,
|
||||
"queue_dead_letter_nodes": 1,
|
||||
}
|
||||
}
|
||||
mock_get_ops_overview.return_value = {
|
||||
"recommendation": {
|
||||
"title": "先清死信",
|
||||
"reason": "当前仍有节点回执死信。",
|
||||
"primary_action_code": "delivery_queue_replay",
|
||||
},
|
||||
"execution_scene": {
|
||||
"log_sync": {
|
||||
"enabled": True,
|
||||
"state": "healthy",
|
||||
"mode": "key",
|
||||
"participating_node_count": 2,
|
||||
"covered_participating_node_count": 2,
|
||||
"missing_participating_nodes": [],
|
||||
}
|
||||
},
|
||||
}
|
||||
|
||||
payload = get_ops_go_live_summary(base_url="http://127.0.0.1:8100")
|
||||
|
||||
self.assertEqual("attention", payload["go_live_status"])
|
||||
self.assertFalse(payload["publish_ready"])
|
||||
self.assertEqual("blocked", payload["publish_status"])
|
||||
|
||||
@patch("app.services.ops_service.get_ops_overview")
|
||||
@patch("app.services.ops_service.list_managed_nodes_with_agent_state")
|
||||
@patch("app.services.ops_service.get_runtime_build_info")
|
||||
@patch("app.services.ops_service.get_release_launchpad")
|
||||
@patch("app.services.ops_service.get_ops_contract_registry")
|
||||
@patch("app.services.ops_service.get_ops_stack_diagnosis")
|
||||
def test_go_live_summary_accepts_log_sync_covered_node_list_and_marks_partial(
|
||||
self,
|
||||
mock_get_ops_stack_diagnosis,
|
||||
mock_get_ops_contract_registry,
|
||||
mock_get_release_launchpad,
|
||||
mock_get_runtime_build_info,
|
||||
mock_list_managed_nodes_with_agent_state,
|
||||
mock_get_ops_overview,
|
||||
) -> None:
|
||||
mock_get_ops_stack_diagnosis.return_value = {
|
||||
"diagnosis": {
|
||||
"contract_key": "ops_stack_diagnosis_contract",
|
||||
"stack_status": "attention",
|
||||
"next_step": {"action_code": "bootstrap_run", "reason": "继续补大陆接管。"},
|
||||
"operator_decision": {},
|
||||
},
|
||||
"release_hub": {
|
||||
"launchpad_status": "attention",
|
||||
},
|
||||
}
|
||||
mock_get_ops_contract_registry.return_value = {
|
||||
"registry_version": "2026-04-18",
|
||||
"contracts": [{"key": "ops_stack_diagnosis_contract"}],
|
||||
}
|
||||
mock_get_release_launchpad.return_value = {
|
||||
"launchpad_status": {
|
||||
"status": "attention",
|
||||
"status_label": "待补执行面",
|
||||
"recommended_action_code": "fix_managed_nodes",
|
||||
}
|
||||
}
|
||||
mock_get_runtime_build_info.return_value = {
|
||||
"commit_sha": "246838ae4c07",
|
||||
"route_surface": {
|
||||
"surface_complete": True,
|
||||
"missing_keys": [],
|
||||
},
|
||||
}
|
||||
mock_list_managed_nodes_with_agent_state.return_value = {
|
||||
"summary": {
|
||||
"managed_enabled": 3,
|
||||
"remote_access_ready": 1,
|
||||
"queue_dead_letter_nodes": 0,
|
||||
}
|
||||
}
|
||||
mock_get_ops_overview.return_value = {
|
||||
"recommendation": {
|
||||
"title": "先补大陆接管",
|
||||
"reason": "当前只有海外样本已形成。",
|
||||
"primary_action_code": "bootstrap_run",
|
||||
},
|
||||
"execution_scene": {
|
||||
"log_sync": {
|
||||
"enabled": True,
|
||||
"state": "partial_coverage",
|
||||
"mode": "full",
|
||||
"participating_node_count": 3,
|
||||
"covered_participating_node_count": 1,
|
||||
"covered_participating_nodes": ["overseas-control-01"],
|
||||
"missing_participating_nodes": ["mainland-worker-01", "mainland-controller-01"],
|
||||
}
|
||||
},
|
||||
}
|
||||
|
||||
payload = get_ops_go_live_summary(base_url="http://127.0.0.1:8100")
|
||||
|
||||
self.assertEqual("attention", payload["go_live_status"])
|
||||
self.assertEqual(1, payload["log_sync_covered_nodes"])
|
||||
self.assertIn("log_sync_partial=1/3", payload["warnings"])
|
||||
self.assertIn("log_sync_partial=1/3", payload["publish_warnings"])
|
||||
|
||||
@patch("app.services.ops_service.get_ops_overview")
|
||||
@patch("app.services.ops_service.list_managed_nodes_with_agent_state")
|
||||
@patch("app.services.ops_service.get_runtime_build_info")
|
||||
@patch("app.services.ops_service.get_release_launchpad")
|
||||
@patch("app.services.ops_service.get_ops_contract_registry")
|
||||
@patch("app.services.ops_service.get_ops_stack_diagnosis")
|
||||
def test_go_live_summary_marks_attention_for_start_delivery_gap(
|
||||
self,
|
||||
mock_get_ops_stack_diagnosis,
|
||||
mock_get_ops_contract_registry,
|
||||
mock_get_release_launchpad,
|
||||
mock_get_runtime_build_info,
|
||||
mock_list_managed_nodes_with_agent_state,
|
||||
mock_get_ops_overview,
|
||||
) -> None:
|
||||
mock_get_ops_stack_diagnosis.return_value = {
|
||||
"diagnosis": {
|
||||
"contract_key": "ops_stack_diagnosis_contract",
|
||||
"stack_status": "healthy",
|
||||
"next_step": {
|
||||
"action_code": "focus_latest_job_events",
|
||||
"reason": "先看任务事件。",
|
||||
},
|
||||
"operator_decision": {
|
||||
"lane": "ops_jobs",
|
||||
"title": "优先检查任务事件流",
|
||||
"primary_command_key": "focus_latest_job_events",
|
||||
},
|
||||
"issues": [
|
||||
{
|
||||
"code": "ops_job_start_delivery_failed_local",
|
||||
"summary": "最近有任务开始回执异常。",
|
||||
}
|
||||
],
|
||||
},
|
||||
"release_hub": {
|
||||
"launchpad_status": "ready",
|
||||
},
|
||||
}
|
||||
mock_get_ops_contract_registry.return_value = {
|
||||
"registry_version": "2026-04-18",
|
||||
"contracts": [{"key": "ops_stack_diagnosis_contract"}],
|
||||
}
|
||||
mock_get_release_launchpad.return_value = {
|
||||
"launchpad_status": {
|
||||
"status": "ready",
|
||||
"status_label": "可发布",
|
||||
"recommended_action_code": "create_smart_release_rollout_worker",
|
||||
}
|
||||
}
|
||||
mock_get_runtime_build_info.return_value = {
|
||||
"commit_sha": "246838ae4c07",
|
||||
"route_surface": {
|
||||
"surface_complete": True,
|
||||
"missing_keys": [],
|
||||
},
|
||||
}
|
||||
mock_list_managed_nodes_with_agent_state.return_value = {
|
||||
"summary": {
|
||||
"managed_enabled": 1,
|
||||
"remote_access_ready": 1,
|
||||
"queue_dead_letter_nodes": 0,
|
||||
}
|
||||
}
|
||||
mock_get_ops_overview.return_value = {
|
||||
"recommendation": {
|
||||
"title": "先看任务事件",
|
||||
"reason": "最近出现开始回执异常。",
|
||||
"primary_action_code": "focus_latest_job_events",
|
||||
},
|
||||
"execution_scene": {
|
||||
"log_sync": {
|
||||
"enabled": True,
|
||||
"state": "healthy",
|
||||
"mode": "key",
|
||||
"participating_node_count": 1,
|
||||
"covered_participating_node_count": 1,
|
||||
"missing_participating_nodes": [],
|
||||
}
|
||||
},
|
||||
}
|
||||
|
||||
payload = get_ops_go_live_summary(base_url="http://127.0.0.1:8100")
|
||||
|
||||
self.assertEqual("attention", payload["go_live_status"])
|
||||
self.assertFalse(payload["publish_ready"])
|
||||
self.assertEqual("attention", payload["publish_status"])
|
||||
self.assertEqual(1, payload["activity_start_delivery_issue_total"])
|
||||
self.assertIn("ops_job_start_delivery_failed_local=1", payload["warnings"])
|
||||
self.assertIn("ops_job_start_delivery_failed_local=1", payload["publish_warnings"])
|
||||
self.assertEqual("ops_jobs", payload["operator_lane"])
|
||||
self.assertEqual("focus_latest_job_events", payload["next_step_action_code"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
182
domain-api/tests/test_ops_service_link_snapshot.py
Normal file
182
domain-api/tests/test_ops_service_link_snapshot.py
Normal file
@@ -0,0 +1,182 @@
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
from app.services.ops_service import get_ops_link_snapshot
|
||||
|
||||
|
||||
class OpsServiceLinkSnapshotTests(unittest.TestCase):
|
||||
@patch("app.services.ops_service.get_ops_overview")
|
||||
def test_link_snapshot_highlights_log_sync_gap_on_active_scene(self, mock_get_ops_overview) -> None:
|
||||
mock_get_ops_overview.return_value = {
|
||||
"current_topology": {
|
||||
"node_code": "overseas-control-01",
|
||||
"node_region": "overseas",
|
||||
"node_role": "control",
|
||||
"worker_mode": "linux-systemd",
|
||||
"sync_push_enabled": True,
|
||||
"sync_target_api_base_url": "http://152.53.37.118:8100",
|
||||
},
|
||||
"cluster_summary": {
|
||||
"online_control_nodes": 2,
|
||||
"online_worker_nodes": 3,
|
||||
"dedicated_online_worker_nodes": 1,
|
||||
"busy_nodes": ["mainland-worker-01", "overseas-control-01"],
|
||||
},
|
||||
"runtime_summary": {
|
||||
"readiness_status": "ready",
|
||||
"readiness_summary": "当前多机与跨地域骨架已进入可联调状态。",
|
||||
"phase_label": "running",
|
||||
"phase_detail": "当前任务正在推进",
|
||||
"active_job_code": "detect-20260417164852-75f7cc",
|
||||
"progress_percent": 42.5,
|
||||
},
|
||||
"execution_scene": {
|
||||
"participation_summary": {
|
||||
"effective_online_nodes": 3,
|
||||
"participating_nodes": 3,
|
||||
"dispatch_active_nodes": 2,
|
||||
"recent_only_nodes": 1,
|
||||
"non_participating_nodes": 0,
|
||||
"standby_nodes": 0,
|
||||
"load_syncing_nodes": 0,
|
||||
"dispatch_active_node_codes": ["mainland-worker-01", "overseas-control-01"],
|
||||
"non_participating_node_codes": [],
|
||||
},
|
||||
"log_sync": {
|
||||
"enabled": False,
|
||||
"mode": "key",
|
||||
"line_count": 0,
|
||||
"source_node_count": 0,
|
||||
"covered_participating_node_count": 0,
|
||||
"participating_node_count": 3,
|
||||
"missing_participating_nodes": [
|
||||
"mainland-controller-01",
|
||||
"mainland-worker-01",
|
||||
"overseas-control-01",
|
||||
],
|
||||
"last_at": "",
|
||||
"preview_lines": [],
|
||||
},
|
||||
},
|
||||
"sync_summary": {
|
||||
"enabled": True,
|
||||
"source_region": "mainland",
|
||||
"target_region": "overseas",
|
||||
"projected_batches": 0,
|
||||
"failed_batches": 0,
|
||||
"records_total": 2513,
|
||||
},
|
||||
"recommendation": {
|
||||
"key": "enable-scene-log-sync",
|
||||
"priority": "先补现场日志回传",
|
||||
"summary": "当前有 3 台节点正在真实参与检测,但海外控制面还看不到过程日志。",
|
||||
"reason": "先把远端日志回传切到关键模式,才能看到阶段、异常、代理与执行过程。",
|
||||
"primary_label": "开启关键回传",
|
||||
"primary_action_code": "enable_log_sync_key",
|
||||
"primary_node_codes": ["mainland-controller-01", "mainland-worker-01", "overseas-control-01"],
|
||||
"primary_action_payload": {},
|
||||
"secondary_label": "开启全量回传",
|
||||
"secondary_action_code": "enable_log_sync_full",
|
||||
"secondary_node_codes": ["mainland-controller-01", "mainland-worker-01", "overseas-control-01"],
|
||||
"secondary_action_payload": {},
|
||||
},
|
||||
}
|
||||
|
||||
snapshot = get_ops_link_snapshot()
|
||||
|
||||
self.assertEqual("attention", snapshot["status"])
|
||||
self.assertIn("日志回传", snapshot["headline"])
|
||||
self.assertIn("活跃任务 detect-20260417164852-75f7cc", snapshot["summary_lines"][2])
|
||||
self.assertIn("远端日志:关闭", snapshot["summary_lines"][4])
|
||||
self.assertEqual("enable_log_sync_key", snapshot["next_actions"][0]["action_code"])
|
||||
self.assertEqual(3, snapshot["execution"]["participating_nodes"])
|
||||
self.assertFalse(snapshot["log_sync"]["enabled"])
|
||||
|
||||
@patch("app.services.ops_service.get_ops_overview")
|
||||
def test_link_snapshot_reports_ready_when_scene_is_observable(self, mock_get_ops_overview) -> None:
|
||||
mock_get_ops_overview.return_value = {
|
||||
"current_topology": {
|
||||
"node_code": "overseas-control-01",
|
||||
"node_region": "overseas",
|
||||
"node_role": "control",
|
||||
"worker_mode": "linux-systemd",
|
||||
"sync_push_enabled": True,
|
||||
"sync_target_api_base_url": "http://152.53.37.118:8100",
|
||||
},
|
||||
"cluster_summary": {
|
||||
"online_control_nodes": 2,
|
||||
"online_worker_nodes": 2,
|
||||
"dedicated_online_worker_nodes": 1,
|
||||
"busy_nodes": ["mainland-worker-01"],
|
||||
},
|
||||
"runtime_summary": {
|
||||
"readiness_status": "ready",
|
||||
"readiness_summary": "ok",
|
||||
"phase_label": "running",
|
||||
"phase_detail": "检测任务正常推进",
|
||||
"active_job_code": "detect-001",
|
||||
"progress_percent": 66.0,
|
||||
},
|
||||
"execution_scene": {
|
||||
"participation_summary": {
|
||||
"effective_online_nodes": 2,
|
||||
"participating_nodes": 2,
|
||||
"dispatch_active_nodes": 2,
|
||||
"recent_only_nodes": 0,
|
||||
"non_participating_nodes": 0,
|
||||
"standby_nodes": 0,
|
||||
"load_syncing_nodes": 0,
|
||||
"dispatch_active_node_codes": ["mainland-worker-01", "mainland-controller-01"],
|
||||
"non_participating_node_codes": [],
|
||||
},
|
||||
"log_sync": {
|
||||
"enabled": True,
|
||||
"mode": "key",
|
||||
"line_count": 18,
|
||||
"source_node_count": 2,
|
||||
"covered_participating_node_count": 2,
|
||||
"participating_node_count": 2,
|
||||
"missing_participating_nodes": [],
|
||||
"last_at": "2026-04-17 22:20:00",
|
||||
"preview_lines": [
|
||||
"[2026-04-17 22:19:59] [mainland-worker-01] 开始检测域名: alpha.com",
|
||||
"[2026-04-17 22:20:00] [mainland-controller-01] 域名检测完成: beta.com",
|
||||
],
|
||||
},
|
||||
},
|
||||
"sync_summary": {
|
||||
"enabled": True,
|
||||
"source_region": "mainland",
|
||||
"target_region": "overseas",
|
||||
"projected_batches": 0,
|
||||
"failed_batches": 0,
|
||||
"records_total": 2600,
|
||||
},
|
||||
"recommendation": {
|
||||
"key": "inspect-participating",
|
||||
"priority": "先看执行现场",
|
||||
"summary": "当前有 2 台节点正在真实参与检测,优先回收它们的标准巡检结果。",
|
||||
"reason": "真正参与检测的节点最接近现场。",
|
||||
"primary_label": "执行标准巡检",
|
||||
"primary_action_code": "run_inspection_participating",
|
||||
"primary_node_codes": ["mainland-worker-01", "mainland-controller-01"],
|
||||
"primary_action_payload": {},
|
||||
"secondary_label": "打开诊断模板",
|
||||
"secondary_action_code": "open_diagnostics_participating",
|
||||
"secondary_node_codes": ["mainland-worker-01", "mainland-controller-01"],
|
||||
"secondary_action_payload": {},
|
||||
},
|
||||
}
|
||||
|
||||
snapshot = get_ops_link_snapshot()
|
||||
|
||||
self.assertEqual("ready", snapshot["status"])
|
||||
self.assertIn("可观测状态", snapshot["headline"])
|
||||
self.assertIn("覆盖 2/2", snapshot["summary_lines"][4])
|
||||
self.assertEqual(2, snapshot["log_sync"]["source_node_count"])
|
||||
self.assertEqual(2, len(snapshot["next_actions"]))
|
||||
self.assertEqual("run_inspection_participating", snapshot["next_actions"][0]["action_code"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
256
domain-api/tests/test_ops_service_node_scene_log.py
Normal file
256
domain-api/tests/test_ops_service_node_scene_log.py
Normal file
@@ -0,0 +1,256 @@
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
from app.services.ops_service import get_ops_node_scene_log
|
||||
|
||||
|
||||
class OpsServiceNodeSceneLogTests(unittest.TestCase):
|
||||
def _runtime_payload_with_worker_logs(self) -> dict:
|
||||
return {
|
||||
"detect": {
|
||||
"log_sync": {
|
||||
"enabled": True,
|
||||
"mode": "key",
|
||||
"line_count": 2,
|
||||
"source_node_count": 1,
|
||||
"source_nodes": ["mainland-worker-01"],
|
||||
"source_node_summaries": [
|
||||
{
|
||||
"node_code": "mainland-worker-01",
|
||||
"line_count": 2,
|
||||
"key_line_count": 2,
|
||||
"full_line_count": 0,
|
||||
"last_at": "2026-04-18 05:02:00",
|
||||
"last_line": "[2026-04-18 05:02:00] [mainland-worker-01] 域名检测完成: alpha.com",
|
||||
}
|
||||
],
|
||||
"preview_lines": [
|
||||
"[2026-04-18 05:00:00] [mainland-worker-01] 开始检测域名: alpha.com",
|
||||
"[2026-04-18 05:02:00] [mainland-worker-01] 域名检测完成: alpha.com",
|
||||
],
|
||||
"last_at": "2026-04-18 05:02:00",
|
||||
"last_line": "[2026-04-18 05:02:00] [mainland-worker-01] 域名检测完成: alpha.com",
|
||||
},
|
||||
"participation_summary": {
|
||||
"participating_nodes": 1,
|
||||
"dispatch_active_nodes": 1,
|
||||
"recent_only_nodes": 0,
|
||||
"non_participating_nodes": 1,
|
||||
"standby_nodes": 1,
|
||||
"load_syncing_nodes": 0,
|
||||
"effective_online_nodes": 2,
|
||||
"dedicated_worker_nodes": 2,
|
||||
"controller_worker_nodes": 0,
|
||||
"dispatch_active_node_codes": ["mainland-worker-01"],
|
||||
"recent_only_node_codes": [],
|
||||
"non_participating_node_codes": ["mainland-worker-02"],
|
||||
"standby_node_codes": ["mainland-worker-02"],
|
||||
"load_syncing_node_codes": [],
|
||||
"summary": "有效执行节点 2 台;正在执行/领任务 1 台;近窗刚有吞吐 0 台;在线但未参与 1 台",
|
||||
},
|
||||
"participating_nodes": [
|
||||
{
|
||||
"node_code": "mainland-worker-01",
|
||||
"region": "mainland",
|
||||
"role": "worker",
|
||||
"status": "busy",
|
||||
"current_load": 2,
|
||||
"last_heartbeat_at": "2026-04-18 05:02:10",
|
||||
"participation_state": "claimed",
|
||||
"participation_label": "已领待跑",
|
||||
"participation_reason": "已领取 2 项任务,等待线程继续执行。",
|
||||
"is_dispatch_active": True,
|
||||
}
|
||||
],
|
||||
"non_participating_nodes": [
|
||||
{
|
||||
"node_code": "mainland-worker-02",
|
||||
"region": "mainland",
|
||||
"role": "worker",
|
||||
"status": "online",
|
||||
"current_load": 0,
|
||||
"last_heartbeat_at": "2026-04-18 05:02:11",
|
||||
"participation_state": "standby",
|
||||
"participation_label": "在线待命",
|
||||
"participation_reason": "当前节点在线,但尚未参与本轮检测。",
|
||||
"is_dispatch_active": False,
|
||||
}
|
||||
],
|
||||
"active_job": {
|
||||
"job_code": "detect-001",
|
||||
"status": "running",
|
||||
"current_cycle_token": "cycle-1",
|
||||
"current_cycle_events": [
|
||||
{
|
||||
"event_type": "worker_log",
|
||||
"created_at": "2026-04-18 05:02:00",
|
||||
"node_code": "mainland-worker-01",
|
||||
"message": "域名检测完成: alpha.com",
|
||||
"payload": {"cycle_token": "cycle-1", "log_mode": "key"},
|
||||
},
|
||||
{
|
||||
"event_type": "worker_log",
|
||||
"created_at": "2026-04-18 05:01:00",
|
||||
"node_code": "mainland-worker-01",
|
||||
"message": "代理池刷新完成",
|
||||
"payload": {"cycle_token": "cycle-1", "log_mode": "full"},
|
||||
},
|
||||
{
|
||||
"event_type": "worker_log",
|
||||
"created_at": "2026-04-18 05:00:00",
|
||||
"node_code": "mainland-worker-01",
|
||||
"message": "开始检测域名: alpha.com",
|
||||
"payload": {"cycle_token": "cycle-1", "log_mode": "key"},
|
||||
},
|
||||
{
|
||||
"event_type": "worker_log",
|
||||
"created_at": "2026-04-18 04:59:00",
|
||||
"node_code": "mainland-worker-01",
|
||||
"message": "旧周期日志",
|
||||
"payload": {"cycle_token": "cycle-0", "log_mode": "key"},
|
||||
},
|
||||
],
|
||||
},
|
||||
},
|
||||
"cluster": {
|
||||
"nodes": [
|
||||
{
|
||||
"node_code": "mainland-worker-01",
|
||||
"region": "mainland",
|
||||
"role": "worker",
|
||||
"status": "busy",
|
||||
"current_load": 2,
|
||||
"last_heartbeat_at": "2026-04-18 05:02:10",
|
||||
},
|
||||
{
|
||||
"node_code": "mainland-worker-02",
|
||||
"region": "mainland",
|
||||
"role": "worker",
|
||||
"status": "online",
|
||||
"current_load": 0,
|
||||
"last_heartbeat_at": "2026-04-18 05:02:11",
|
||||
},
|
||||
]
|
||||
},
|
||||
}
|
||||
|
||||
@patch("app.services.ops_service.get_runtime_status")
|
||||
def test_node_scene_log_filters_key_mode_records_for_participating_node(self, mock_get_runtime_status) -> None:
|
||||
mock_get_runtime_status.return_value = self._runtime_payload_with_worker_logs()
|
||||
|
||||
payload = get_ops_node_scene_log("mainland-worker-01", limit=10)
|
||||
|
||||
self.assertEqual("mainland-worker-01", payload["node_code"])
|
||||
self.assertEqual("healthy", payload["status"])
|
||||
self.assertEqual("关键覆盖", payload["status_label"])
|
||||
self.assertEqual("key", payload["mode"])
|
||||
self.assertTrue(payload["log_sync_enabled"])
|
||||
self.assertTrue(payload["participation"]["detect_participating"])
|
||||
self.assertEqual(2, payload["records_total"])
|
||||
self.assertEqual(
|
||||
[
|
||||
"开始检测域名: alpha.com",
|
||||
"域名检测完成: alpha.com",
|
||||
],
|
||||
[item["message"] for item in payload["records"]],
|
||||
)
|
||||
self.assertEqual(2, payload["source_summary"]["line_count"])
|
||||
self.assertEqual(2, payload["source_summary"]["key_line_count"])
|
||||
self.assertEqual(0, payload["source_summary"]["full_line_count"])
|
||||
self.assertEqual("ops_observability_contract", payload["contract_navigation"]["primary_contract_key"])
|
||||
|
||||
@patch("app.services.ops_service.get_runtime_status")
|
||||
def test_node_scene_log_can_request_full_mode_records(self, mock_get_runtime_status) -> None:
|
||||
mock_get_runtime_status.return_value = self._runtime_payload_with_worker_logs()
|
||||
|
||||
payload = get_ops_node_scene_log("mainland-worker-01", limit=10, mode="full")
|
||||
|
||||
self.assertEqual("full_capture", payload["status"])
|
||||
self.assertEqual("full", payload["mode"])
|
||||
self.assertEqual(3, payload["records_total"])
|
||||
self.assertEqual(
|
||||
[
|
||||
"开始检测域名: alpha.com",
|
||||
"代理池刷新完成",
|
||||
"域名检测完成: alpha.com",
|
||||
],
|
||||
[item["message"] for item in payload["records"]],
|
||||
)
|
||||
self.assertEqual(3, payload["source_summary"]["line_count"])
|
||||
self.assertEqual(2, payload["source_summary"]["key_line_count"])
|
||||
self.assertEqual(1, payload["source_summary"]["full_line_count"])
|
||||
|
||||
@patch("app.services.ops_service.get_runtime_status")
|
||||
def test_node_scene_log_marks_waiting_sample_for_missing_participating_node(self, mock_get_runtime_status) -> None:
|
||||
mock_get_runtime_status.return_value = {
|
||||
"detect": {
|
||||
"log_sync": {
|
||||
"enabled": True,
|
||||
"mode": "key",
|
||||
"line_count": 0,
|
||||
"source_node_count": 0,
|
||||
"source_nodes": [],
|
||||
"source_node_summaries": [],
|
||||
"preview_lines": [],
|
||||
"last_at": "",
|
||||
"last_line": "",
|
||||
},
|
||||
"participation_summary": {
|
||||
"participating_nodes": 1,
|
||||
"dispatch_active_nodes": 1,
|
||||
"recent_only_nodes": 0,
|
||||
"non_participating_nodes": 0,
|
||||
"standby_nodes": 0,
|
||||
"load_syncing_nodes": 0,
|
||||
"effective_online_nodes": 1,
|
||||
"dedicated_worker_nodes": 1,
|
||||
"controller_worker_nodes": 0,
|
||||
"dispatch_active_node_codes": ["mainland-worker-02"],
|
||||
"recent_only_node_codes": [],
|
||||
"non_participating_node_codes": [],
|
||||
"standby_node_codes": [],
|
||||
"load_syncing_node_codes": [],
|
||||
"summary": "有效执行节点 1 台;正在执行/领任务 1 台;近窗刚有吞吐 0 台;在线但未参与 0 台",
|
||||
},
|
||||
"participating_nodes": [
|
||||
{
|
||||
"node_code": "mainland-worker-02",
|
||||
"region": "mainland",
|
||||
"role": "worker",
|
||||
"status": "busy",
|
||||
"current_load": 1,
|
||||
"last_heartbeat_at": "2026-04-18 05:05:00",
|
||||
"participation_state": "running",
|
||||
"participation_label": "正在执行",
|
||||
"participation_reason": "当前存在执行中工作负载。",
|
||||
"is_dispatch_active": True,
|
||||
}
|
||||
],
|
||||
"non_participating_nodes": [],
|
||||
"active_job": {"job_code": "detect-002", "status": "running", "current_cycle_events": []},
|
||||
},
|
||||
"cluster": {
|
||||
"nodes": [
|
||||
{
|
||||
"node_code": "mainland-worker-02",
|
||||
"region": "mainland",
|
||||
"role": "worker",
|
||||
"status": "busy",
|
||||
"current_load": 1,
|
||||
"last_heartbeat_at": "2026-04-18 05:05:00",
|
||||
}
|
||||
]
|
||||
},
|
||||
}
|
||||
|
||||
payload = get_ops_node_scene_log("mainland-worker-02")
|
||||
|
||||
self.assertEqual("waiting_sample", payload["status"])
|
||||
self.assertEqual("等待样本", payload["status_label"])
|
||||
self.assertEqual(0, payload["records_total"])
|
||||
self.assertTrue(payload["participation"]["detect_participating"])
|
||||
self.assertIn("现场样本尚未形成", payload["summary"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
1186
domain-api/tests/test_ops_service_overview.py
Normal file
1186
domain-api/tests/test_ops_service_overview.py
Normal file
File diff suppressed because it is too large
Load Diff
283
domain-api/tests/test_ops_service_runbook.py
Normal file
283
domain-api/tests/test_ops_service_runbook.py
Normal file
@@ -0,0 +1,283 @@
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
from app.services.ops_service import get_ops_runbook
|
||||
|
||||
|
||||
class OpsServiceRunbookTests(unittest.TestCase):
|
||||
@patch("app.services.ops_service.get_release_launchpad")
|
||||
@patch("app.services.ops_service.get_latest_release")
|
||||
@patch("app.services.ops_service.list_managed_nodes_with_agent_state")
|
||||
@patch("app.services.ops_service.get_runtime_status")
|
||||
def test_ops_runbook_includes_control_sequences(
|
||||
self,
|
||||
mock_get_runtime_status,
|
||||
mock_list_managed_nodes_with_agent_state,
|
||||
mock_get_latest_release,
|
||||
mock_get_release_launchpad,
|
||||
) -> None:
|
||||
mock_get_runtime_status.return_value = {
|
||||
"readiness": {
|
||||
"status": "ready",
|
||||
"summary": "当前多机现场已进入标准作业状态",
|
||||
},
|
||||
"worker": {
|
||||
"running": True,
|
||||
},
|
||||
"sync_agent": {
|
||||
"running": True,
|
||||
},
|
||||
"detect": {
|
||||
"participation_summary": {
|
||||
"summary": "worker 正在执行",
|
||||
}
|
||||
},
|
||||
}
|
||||
mock_list_managed_nodes_with_agent_state.return_value = {
|
||||
"nodes": [
|
||||
{
|
||||
"node_code": "mainland-controller-01",
|
||||
"role": "control",
|
||||
"is_managed": True,
|
||||
"is_enabled": True,
|
||||
"is_agent_online": True,
|
||||
"cluster_status": "online",
|
||||
"cluster_is_effective_worker": True,
|
||||
"is_current_participant": False,
|
||||
},
|
||||
{
|
||||
"node_code": "mainland-worker-01",
|
||||
"role": "worker",
|
||||
"is_managed": True,
|
||||
"is_enabled": True,
|
||||
"is_agent_online": True,
|
||||
"cluster_status": "busy",
|
||||
"cluster_is_effective_worker": True,
|
||||
"is_current_participant": True,
|
||||
},
|
||||
]
|
||||
}
|
||||
mock_get_latest_release.side_effect = lambda channel="stable": (
|
||||
{
|
||||
"id": 15,
|
||||
"release_version": "2026.04.17",
|
||||
"channel": "stable",
|
||||
}
|
||||
if channel == "stable"
|
||||
else {}
|
||||
)
|
||||
mock_get_release_launchpad.return_value = {
|
||||
"channel": "stable",
|
||||
"latest_release": {
|
||||
"id": 15,
|
||||
"release_version": "2026.04.17",
|
||||
"channel": "stable",
|
||||
},
|
||||
"launchpad_status": {
|
||||
"status": "ready",
|
||||
"status_label": "可发 Worker 灰度",
|
||||
"summary": "最新发布包、Release 记录与 Worker 智能 Rollout 预检均已收口。",
|
||||
"recommended_action_code": "publish_latest_worker",
|
||||
"focus_ref": {
|
||||
"kind": "release_hub",
|
||||
"release_id": 15,
|
||||
"section": "release_launchpad",
|
||||
},
|
||||
},
|
||||
"worker_rollout_preview": {
|
||||
"message": "智能 Rollout 预检通过",
|
||||
"smart_rollout": {
|
||||
"available": True,
|
||||
"summary": "已按 Worker 模式预选 1 台节点。",
|
||||
"target_node_codes": ["mainland-worker-01"],
|
||||
},
|
||||
},
|
||||
"control_rollout_preview": {
|
||||
"message": "",
|
||||
"smart_rollout": {
|
||||
"available": False,
|
||||
"target_node_codes": [],
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
runbook = get_ops_runbook()
|
||||
|
||||
mock_list_managed_nodes_with_agent_state.assert_called_once_with(
|
||||
participation_payload=mock_get_runtime_status.return_value["detect"]
|
||||
)
|
||||
self.assertIn("GET /api/v1/ops/releases/launchpad", runbook["fast_checks"])
|
||||
self.assertEqual("ready", runbook["release_launchpad"]["launchpad_status"]["status"])
|
||||
sequences = {item["key"]: item for item in runbook["control_sequences"]}
|
||||
self.assertIn("node_onboarding", sequences)
|
||||
self.assertIn("scene_observe", sequences)
|
||||
self.assertIn("standard_inspection", sequences)
|
||||
self.assertIn("release_progression", sequences)
|
||||
self.assertEqual("bootstrap_run", sequences["node_onboarding"]["primary_action_code"])
|
||||
self.assertEqual("run_acceptance", sequences["node_onboarding"]["secondary_action_code"])
|
||||
self.assertEqual("run_standard_inspection", sequences["standard_inspection"]["primary_action_code"])
|
||||
self.assertEqual("open_diagnostics", sequences["standard_inspection"]["secondary_action_code"])
|
||||
self.assertEqual(["mainland-worker-01"], sequences["scene_observe"]["target_node_codes"])
|
||||
self.assertEqual("publish_latest_worker", sequences["release_progression"]["primary_action_code"])
|
||||
self.assertEqual(15, sequences["release_progression"]["primary_action_payload"]["release_id"])
|
||||
self.assertEqual(["mainland-worker-01"], sequences["release_progression"]["target_node_codes"])
|
||||
self.assertTrue(sequences["release_progression"]["primary_resolution"]["ok"])
|
||||
self.assertEqual("publish_latest_worker", sequences["release_progression"]["primary_resolution"]["driver_action_code"])
|
||||
self.assertEqual(["mainland-worker-01"], sequences["release_progression"]["primary_resolution"]["driver_node_codes"])
|
||||
self.assertEqual("release_hub", sequences["release_progression"]["focus_ref"]["kind"])
|
||||
self.assertEqual("release_launchpad", sequences["release_progression"]["focus_ref"]["section"])
|
||||
self.assertEqual("worker", sequences["release_progression"]["primary_focus_ref"]["mode"])
|
||||
self.assertEqual("worker", sequences["release_progression"]["primary_action_payload"]["focus_ref"]["mode"])
|
||||
self.assertEqual("release_detail", sequences["release_progression"]["secondary_focus_ref"]["section"])
|
||||
self.assertTrue(sequences["scene_observe"]["secondary_resolution"]["ok"])
|
||||
self.assertEqual("run_scene_logs_full", sequences["scene_observe"]["secondary_resolution"]["driver_action_code"])
|
||||
|
||||
@patch("app.services.ops_service.get_release_launchpad")
|
||||
@patch("app.services.ops_service.get_latest_release")
|
||||
@patch("app.services.ops_service.list_managed_nodes_with_agent_state")
|
||||
@patch("app.services.ops_service.get_runtime_status")
|
||||
def test_ops_runbook_release_progression_can_fall_back_to_bootstrap_run(
|
||||
self,
|
||||
mock_get_runtime_status,
|
||||
mock_list_managed_nodes_with_agent_state,
|
||||
mock_get_latest_release,
|
||||
mock_get_release_launchpad,
|
||||
) -> None:
|
||||
mock_get_runtime_status.return_value = {
|
||||
"readiness": {
|
||||
"status": "attention",
|
||||
"summary": "当前还需补执行面接管",
|
||||
},
|
||||
"worker": {"running": True},
|
||||
"sync_agent": {"running": True},
|
||||
"detect": {"participation_summary": {"summary": ""}},
|
||||
}
|
||||
mock_list_managed_nodes_with_agent_state.return_value = {
|
||||
"nodes": [
|
||||
{
|
||||
"node_code": "mainland-worker-02",
|
||||
"role": "worker",
|
||||
"is_managed": False,
|
||||
"is_enabled": False,
|
||||
"is_agent_online": False,
|
||||
"cluster_status": "online",
|
||||
"cluster_is_effective_worker": True,
|
||||
"is_current_participant": False,
|
||||
}
|
||||
]
|
||||
}
|
||||
mock_get_latest_release.return_value = {}
|
||||
mock_get_release_launchpad.return_value = {
|
||||
"channel": "stable",
|
||||
"latest_release": {},
|
||||
"launchpad_status": {
|
||||
"status": "attention",
|
||||
"status_label": "待补执行面",
|
||||
"summary": "当前节点仍处于接入阶段,建议先签发 onboarding.bootstrap。",
|
||||
"recommended_action_code": "bootstrap_run",
|
||||
"focus_ref": {
|
||||
"kind": "release_hub",
|
||||
"release_id": 19,
|
||||
"section": "release_launchpad",
|
||||
},
|
||||
},
|
||||
"worker_rollout_preview": {
|
||||
"message": "当前没有满足角色条件的在线节点可用于智能 Rollout。",
|
||||
"smart_rollout": {
|
||||
"available": False,
|
||||
"target_node_codes": [],
|
||||
},
|
||||
"policy_preview": {
|
||||
"release_gate": {
|
||||
"operational_readiness": {
|
||||
"rows": [
|
||||
{
|
||||
"node_code": "mainland-worker-02",
|
||||
"is_managed": False,
|
||||
"is_enabled": False,
|
||||
"is_agent_online": False,
|
||||
"remote_agent_ready": False,
|
||||
"inspection_status": "missing",
|
||||
"recovery_action": "bootstrap_run",
|
||||
"recovery_summary": "当前节点仍处于接入阶段,建议先签发 onboarding.bootstrap。",
|
||||
"agent_reason": "节点尚未纳管,当前不能稳定下发 remote-agent 动作。",
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
},
|
||||
"control_rollout_preview": {
|
||||
"message": "",
|
||||
"smart_rollout": {
|
||||
"available": False,
|
||||
"target_node_codes": [],
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
runbook = get_ops_runbook()
|
||||
sequences = {item["key"]: item for item in runbook["control_sequences"]}
|
||||
|
||||
self.assertEqual("bootstrap_run", sequences["release_progression"]["primary_action_code"])
|
||||
self.assertEqual(["mainland-worker-02"], sequences["release_progression"]["target_node_codes"])
|
||||
self.assertEqual("待补执行面", sequences["release_progression"]["status_label"])
|
||||
self.assertTrue(sequences["release_progression"]["primary_resolution"]["ok"])
|
||||
self.assertEqual("bootstrap_run", sequences["release_progression"]["primary_resolution"]["driver_action_code"])
|
||||
self.assertEqual("default_rollout_gate", sequences["release_progression"]["primary_focus_ref"]["section"])
|
||||
self.assertEqual("default_rollout_gate", sequences["release_progression"]["primary_action_payload"]["focus_ref"]["section"])
|
||||
|
||||
@patch("app.services.ops_service.get_release_launchpad")
|
||||
@patch("app.services.ops_service.get_latest_release")
|
||||
@patch("app.services.ops_service.list_managed_nodes_with_agent_state")
|
||||
@patch("app.services.ops_service.get_runtime_status")
|
||||
def test_ops_runbook_release_progression_falls_back_to_direct_worker_rollout_when_release_exists(
|
||||
self,
|
||||
mock_get_runtime_status,
|
||||
mock_list_managed_nodes_with_agent_state,
|
||||
mock_get_latest_release,
|
||||
mock_get_release_launchpad,
|
||||
) -> None:
|
||||
mock_get_runtime_status.return_value = {
|
||||
"readiness": {
|
||||
"status": "ready",
|
||||
"summary": "当前多机现场已进入标准作业状态",
|
||||
},
|
||||
"worker": {"running": True},
|
||||
"sync_agent": {"running": True},
|
||||
"detect": {"participation_summary": {"summary": ""}},
|
||||
}
|
||||
mock_list_managed_nodes_with_agent_state.return_value = {
|
||||
"nodes": [
|
||||
{
|
||||
"node_code": "mainland-worker-01",
|
||||
"role": "worker",
|
||||
"is_managed": True,
|
||||
"is_enabled": True,
|
||||
"is_agent_online": True,
|
||||
"cluster_status": "online",
|
||||
"cluster_is_effective_worker": True,
|
||||
"is_current_participant": False,
|
||||
}
|
||||
]
|
||||
}
|
||||
mock_get_latest_release.return_value = {
|
||||
"id": 18,
|
||||
"release_version": "2026.04.18",
|
||||
"channel": "stable",
|
||||
}
|
||||
mock_get_release_launchpad.return_value = {}
|
||||
|
||||
runbook = get_ops_runbook()
|
||||
sequences = {item["key"]: item for item in runbook["control_sequences"]}
|
||||
|
||||
self.assertEqual("create_release_rollout_worker", sequences["release_progression"]["primary_action_code"])
|
||||
self.assertEqual("focus_release_hub", sequences["release_progression"]["secondary_action_code"])
|
||||
self.assertEqual(18, sequences["release_progression"]["primary_action_payload"]["release_id"])
|
||||
self.assertEqual(["mainland-worker-01"], sequences["release_progression"]["target_node_codes"])
|
||||
self.assertTrue(sequences["release_progression"]["primary_resolution"]["ok"])
|
||||
self.assertEqual("create_release_rollout_worker", sequences["release_progression"]["primary_resolution"]["driver_action_code"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
255
domain-api/tests/test_ops_service_runbook_sequence.py
Normal file
255
domain-api/tests/test_ops_service_runbook_sequence.py
Normal file
@@ -0,0 +1,255 @@
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
from app.services.ops_service import execute_ops_runbook_sequence, resolve_ops_runbook_sequence
|
||||
|
||||
|
||||
class OpsServiceRunbookSequenceTests(unittest.TestCase):
|
||||
@patch("app.services.ops_service.get_ops_runbook")
|
||||
def test_resolve_ops_runbook_sequence_uses_primary_action(self, mock_get_ops_runbook) -> None:
|
||||
mock_get_ops_runbook.return_value = {
|
||||
"control_sequences": [
|
||||
{
|
||||
"key": "release_progression",
|
||||
"title": "版本发布与放量",
|
||||
"status": "ready",
|
||||
"status_label": "可发 Worker 灰度",
|
||||
"summary": "可以直接发布。",
|
||||
"reason": "launchpad 已就绪",
|
||||
"target_scope_label": "Release / Launchpad / 2026.04.18",
|
||||
"step_titles": ["确认发布包", "发灰度"],
|
||||
"target_node_codes": ["mainland-worker-01"],
|
||||
"focus_ref": {
|
||||
"kind": "release_hub",
|
||||
"release_id": 12,
|
||||
"section": "release_launchpad",
|
||||
},
|
||||
"primary_label": "直接发 Worker 灰度",
|
||||
"primary_action_code": "publish_latest_worker",
|
||||
"primary_action_payload": {
|
||||
"channel": "stable",
|
||||
"release_id": 12,
|
||||
},
|
||||
"primary_focus_ref": {
|
||||
"kind": "release_hub",
|
||||
"release_id": 12,
|
||||
"section": "release_launchpad",
|
||||
"mode": "worker",
|
||||
},
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
ok, message, data = resolve_ops_runbook_sequence(
|
||||
"release_progression",
|
||||
{
|
||||
"requested_by": "test",
|
||||
},
|
||||
)
|
||||
|
||||
self.assertTrue(ok, message)
|
||||
self.assertEqual("标准作业路径解析成功", message)
|
||||
self.assertEqual("publish_latest_worker", data["driver_action_code"])
|
||||
self.assertEqual("直接发 Worker 灰度", data["driver_action_label"])
|
||||
self.assertEqual(["mainland-worker-01"], data["driver_node_codes"])
|
||||
self.assertEqual("test/runbook/release_progression", data["driver_requested_by"])
|
||||
self.assertEqual("Release / Launchpad / 2026.04.18", data["sequence"]["target_scope_label"])
|
||||
self.assertEqual(["确认发布包", "发灰度"], data["sequence"]["step_titles"])
|
||||
self.assertEqual("release_launchpad", data["sequence"]["focus_ref"]["section"])
|
||||
self.assertEqual("worker", data["sequence"]["action_focus_ref"]["mode"])
|
||||
self.assertEqual("worker", data["driver_action_payload"]["focus_ref"]["mode"])
|
||||
|
||||
@patch("app.services.ops_service.get_ops_runbook")
|
||||
def test_resolve_ops_runbook_sequence_uses_secondary_action_and_allows_payload_override(
|
||||
self,
|
||||
mock_get_ops_runbook,
|
||||
) -> None:
|
||||
mock_get_ops_runbook.return_value = {
|
||||
"control_sequences": [
|
||||
{
|
||||
"key": "node_onboarding",
|
||||
"title": "纳管新节点",
|
||||
"status": "warning",
|
||||
"status_label": "有待接管节点",
|
||||
"summary": "先补接入。",
|
||||
"reason": "新增节点需要先纳管。",
|
||||
"target_node_codes": ["mainland-worker-01"],
|
||||
"secondary_label": "跑接管验收",
|
||||
"secondary_action_code": "run_acceptance",
|
||||
"secondary_action_payload": {
|
||||
"node_code": "mainland-worker-01",
|
||||
},
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
ok, message, data = resolve_ops_runbook_sequence(
|
||||
"node_onboarding",
|
||||
{
|
||||
"secondary": True,
|
||||
"requested_by": "test",
|
||||
"node_codes": ["mainland-worker-02"],
|
||||
"action_payload": {
|
||||
"auto_approve": True,
|
||||
},
|
||||
},
|
||||
)
|
||||
|
||||
self.assertTrue(ok, message)
|
||||
self.assertEqual("标准作业路径解析成功", message)
|
||||
self.assertEqual("run_acceptance", data["driver_action_code"])
|
||||
self.assertEqual("跑接管验收", data["driver_action_label"])
|
||||
self.assertEqual(["mainland-worker-02"], data["driver_node_codes"])
|
||||
self.assertTrue(data["sequence"]["secondary"])
|
||||
self.assertEqual(
|
||||
{
|
||||
"node_code": "mainland-worker-01",
|
||||
"auto_approve": True,
|
||||
},
|
||||
data["driver_action_payload"],
|
||||
)
|
||||
|
||||
@patch("app.services.ops_service.execute_driver_action")
|
||||
@patch("app.services.ops_service.get_ops_runbook")
|
||||
def test_execute_ops_runbook_sequence_uses_primary_action(
|
||||
self,
|
||||
mock_get_ops_runbook,
|
||||
mock_execute_driver_action,
|
||||
) -> None:
|
||||
mock_get_ops_runbook.return_value = {
|
||||
"control_sequences": [
|
||||
{
|
||||
"key": "release_progression",
|
||||
"title": "版本发布与放量",
|
||||
"status": "ready",
|
||||
"status_label": "可发 Worker 灰度",
|
||||
"summary": "可以直接发布。",
|
||||
"reason": "launchpad 已就绪",
|
||||
"target_node_codes": ["mainland-worker-01"],
|
||||
"primary_action_code": "publish_latest_worker",
|
||||
"primary_action_payload": {
|
||||
"channel": "stable",
|
||||
"release_id": 12,
|
||||
},
|
||||
"secondary_action_code": "focus_release_hub",
|
||||
"secondary_action_payload": {
|
||||
"release_id": 12,
|
||||
},
|
||||
}
|
||||
]
|
||||
}
|
||||
mock_execute_driver_action.return_value = (
|
||||
True,
|
||||
"Release rollout 已创建",
|
||||
{
|
||||
"handled": True,
|
||||
"mode": "smart-release-rollout",
|
||||
},
|
||||
)
|
||||
|
||||
ok, message, data = execute_ops_runbook_sequence(
|
||||
"release_progression",
|
||||
{
|
||||
"requested_by": "test",
|
||||
},
|
||||
)
|
||||
|
||||
self.assertTrue(ok, message)
|
||||
self.assertEqual("Release rollout 已创建", message)
|
||||
self.assertTrue(data["handled"])
|
||||
self.assertEqual("publish_latest_worker", data["driver_action_code"])
|
||||
self.assertEqual(["mainland-worker-01"], data["driver_node_codes"])
|
||||
self.assertEqual("release_progression", data["sequence_key"])
|
||||
self.assertEqual("版本发布与放量", data["sequence"]["title"])
|
||||
mock_execute_driver_action.assert_called_once_with(
|
||||
{
|
||||
"action_code": "publish_latest_worker",
|
||||
"node_codes": ["mainland-worker-01"],
|
||||
"action_payload": {
|
||||
"channel": "stable",
|
||||
"release_id": 12,
|
||||
},
|
||||
"requested_by": "test/runbook/release_progression",
|
||||
}
|
||||
)
|
||||
|
||||
@patch("app.services.ops_service.execute_driver_action")
|
||||
@patch("app.services.ops_service.get_ops_runbook")
|
||||
def test_execute_ops_runbook_sequence_uses_secondary_action_and_allows_payload_override(
|
||||
self,
|
||||
mock_get_ops_runbook,
|
||||
mock_execute_driver_action,
|
||||
) -> None:
|
||||
mock_get_ops_runbook.return_value = {
|
||||
"control_sequences": [
|
||||
{
|
||||
"key": "node_onboarding",
|
||||
"title": "纳管新节点",
|
||||
"status": "warning",
|
||||
"status_label": "有待接管节点",
|
||||
"summary": "先补接入。",
|
||||
"reason": "新增节点需要先纳管。",
|
||||
"target_node_codes": ["mainland-worker-01"],
|
||||
"primary_action_code": "bootstrap_run",
|
||||
"primary_action_payload": {
|
||||
"node_code": "mainland-worker-01",
|
||||
},
|
||||
"secondary_action_code": "run_acceptance",
|
||||
"secondary_action_payload": {
|
||||
"node_code": "mainland-worker-01",
|
||||
},
|
||||
}
|
||||
]
|
||||
}
|
||||
mock_execute_driver_action.return_value = (
|
||||
True,
|
||||
"",
|
||||
{
|
||||
"handled": True,
|
||||
"mode": "ui-intent",
|
||||
},
|
||||
)
|
||||
|
||||
ok, message, data = execute_ops_runbook_sequence(
|
||||
"node_onboarding",
|
||||
{
|
||||
"secondary": True,
|
||||
"requested_by": "test",
|
||||
"node_codes": ["mainland-worker-02"],
|
||||
"action_payload": {
|
||||
"auto_approve": True,
|
||||
},
|
||||
},
|
||||
)
|
||||
|
||||
self.assertTrue(ok, message)
|
||||
self.assertEqual("", message)
|
||||
self.assertTrue(data["handled"])
|
||||
self.assertEqual("run_acceptance", data["driver_action_code"])
|
||||
self.assertEqual(["mainland-worker-02"], data["driver_node_codes"])
|
||||
self.assertTrue(data["sequence"]["secondary"])
|
||||
mock_execute_driver_action.assert_called_once_with(
|
||||
{
|
||||
"action_code": "run_acceptance",
|
||||
"node_codes": ["mainland-worker-02"],
|
||||
"action_payload": {
|
||||
"node_code": "mainland-worker-01",
|
||||
"auto_approve": True,
|
||||
},
|
||||
"requested_by": "test/runbook/node_onboarding",
|
||||
}
|
||||
)
|
||||
|
||||
@patch("app.services.ops_service.get_ops_runbook")
|
||||
def test_execute_ops_runbook_sequence_returns_error_when_sequence_missing(self, mock_get_ops_runbook) -> None:
|
||||
mock_get_ops_runbook.return_value = {"control_sequences": []}
|
||||
|
||||
ok, message, data = execute_ops_runbook_sequence("missing-sequence", {})
|
||||
|
||||
self.assertFalse(ok)
|
||||
self.assertEqual("标准作业路径不存在", message)
|
||||
self.assertEqual("missing-sequence", data["sequence_key"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
971
domain-api/tests/test_ops_service_stack_diagnosis.py
Normal file
971
domain-api/tests/test_ops_service_stack_diagnosis.py
Normal file
@@ -0,0 +1,971 @@
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
from app.services.ops_service import get_ops_stack_diagnosis
|
||||
|
||||
|
||||
class OpsServiceStackDiagnosisTests(unittest.TestCase):
|
||||
@patch("app.services.ops_service.get_runtime_build_info")
|
||||
@patch("app.services.ops_service.get_ops_activity_stream")
|
||||
@patch("app.services.ops_service.get_recent_ops_playbook_runs")
|
||||
@patch("app.services.ops_service.get_release_launchpad")
|
||||
@patch("app.services.ops_service.list_managed_nodes_with_agent_state")
|
||||
@patch("app.services.ops_service.get_ops_overview")
|
||||
@patch("app.services.ops_service.get_ops_link_snapshot")
|
||||
@patch("app.services.ops_service.get_ops_contract_registry")
|
||||
def test_stack_diagnosis_prioritizes_api_restart_when_runtime_build_schema_is_stale(
|
||||
self,
|
||||
mock_get_ops_contract_registry,
|
||||
mock_get_ops_link_snapshot,
|
||||
mock_get_ops_overview,
|
||||
mock_list_managed_nodes_with_agent_state,
|
||||
mock_get_release_launchpad,
|
||||
mock_get_recent_ops_playbook_runs,
|
||||
mock_get_ops_activity_stream,
|
||||
mock_get_runtime_build_info,
|
||||
) -> None:
|
||||
mock_get_ops_contract_registry.return_value = {
|
||||
"registry_version": "2026-04-18",
|
||||
"schema_version": "v1",
|
||||
"contracts": [{"key": "ops_stack_diagnosis_contract"}],
|
||||
}
|
||||
mock_get_ops_link_snapshot.return_value = {
|
||||
"status": "ready",
|
||||
"headline": "当前总检可用。",
|
||||
"next_actions": [],
|
||||
}
|
||||
mock_get_ops_overview.return_value = {
|
||||
"recommendation": {
|
||||
"primary_action_code": "fix_managed_nodes",
|
||||
"title": "先补节点接管",
|
||||
"focus_ref": {"kind": "managed_node", "node_code": "mainland-worker-01"},
|
||||
},
|
||||
"execution_scene": {},
|
||||
"log_sync": {},
|
||||
"release_hub": {},
|
||||
}
|
||||
mock_list_managed_nodes_with_agent_state.return_value = {"nodes": [], "summary": {}}
|
||||
mock_get_release_launchpad.return_value = {"launchpad_status": {}, "latest_release": {}}
|
||||
mock_get_recent_ops_playbook_runs.return_value = {"runs": []}
|
||||
mock_get_ops_activity_stream.return_value = {"items": []}
|
||||
mock_get_runtime_build_info.return_value = {
|
||||
"source": "git",
|
||||
"commit_sha": "246838ae4c07",
|
||||
"repository_capabilities": {
|
||||
"supports_install_command_block": True,
|
||||
"supports_multi_layout_bootstrap": True,
|
||||
},
|
||||
"route_surface": {
|
||||
"mode": "registered",
|
||||
"registered_paths_total": 24,
|
||||
"surface_complete": True,
|
||||
"missing_keys": [],
|
||||
"expected_paths": {
|
||||
"ops_contracts": "/api/v1/ops/contracts",
|
||||
"ops_stack_diagnosis": "/api/v1/ops/stack-diagnosis",
|
||||
},
|
||||
},
|
||||
}
|
||||
|
||||
payload = get_ops_stack_diagnosis(base_url="http://127.0.0.1:8100")
|
||||
|
||||
issue = next(item for item in payload["diagnosis"]["issues"] if item["code"] == "runtime_build_schema_stale")
|
||||
self.assertEqual("api-restart", issue["action_code"])
|
||||
self.assertEqual("api-restart", payload["diagnosis"]["next_step"]["action_code"])
|
||||
self.assertEqual("runtime_recovery", payload["diagnosis"]["operator_decision"]["lane"])
|
||||
self.assertEqual("api-restart", payload["diagnosis"]["operator_decision"]["primary_command_key"])
|
||||
self.assertEqual(
|
||||
"bash domain-api/deploy/multi-region/drive_ops_center.sh runtime-refresh-recover http://127.0.0.1:8100",
|
||||
payload["diagnosis"]["next_actions"]["primary_command"],
|
||||
)
|
||||
self.assertEqual(
|
||||
"bash domain-api/deploy/multi-region/drive_ops_center.sh runtime-refresh-recover http://127.0.0.1:8100",
|
||||
payload["diagnosis"]["recommended_commands"]["api-restart"],
|
||||
)
|
||||
self.assertEqual(
|
||||
"bash domain-api/deploy/multi-region/drive_ops_center.sh runtime-refresh-recover http://127.0.0.1:8100",
|
||||
payload["diagnosis"]["recommended_commands"]["runtime-refresh-recover"],
|
||||
)
|
||||
self.assertEqual(
|
||||
"bash domain-api/deploy/multi-region/drive_ops_center.sh stack-diagnosis http://127.0.0.1:8100 summary",
|
||||
payload["diagnosis"]["recommended_commands"]["stack-diagnosis"],
|
||||
)
|
||||
self.assertEqual(
|
||||
payload["diagnosis"]["recommended_commands"]["stack-diagnosis"],
|
||||
payload["diagnosis"]["recommended_commands"]["stack-check"],
|
||||
)
|
||||
self.assertTrue(payload["runtime_build_info"]["runtime_schema_stale"])
|
||||
self.assertFalse(payload["runtime_build_info"]["route_surface_declares_bootstrap_plan"])
|
||||
self.assertTrue(
|
||||
any("运行中 API 还是旧 schema" in hint for hint in payload["diagnosis"]["operator_hints"])
|
||||
)
|
||||
|
||||
@patch("app.services.ops_service.get_runtime_build_info")
|
||||
@patch("app.services.ops_service.get_ops_activity_stream")
|
||||
@patch("app.services.ops_service.get_recent_ops_playbook_runs")
|
||||
@patch("app.services.ops_service.get_release_launchpad")
|
||||
@patch("app.services.ops_service.list_managed_nodes_with_agent_state")
|
||||
@patch("app.services.ops_service.get_ops_overview")
|
||||
@patch("app.services.ops_service.get_ops_link_snapshot")
|
||||
@patch("app.services.ops_service.get_ops_contract_registry")
|
||||
def test_stack_diagnosis_surfaces_runtime_route_surface_gap(
|
||||
self,
|
||||
mock_get_ops_contract_registry,
|
||||
mock_get_ops_link_snapshot,
|
||||
mock_get_ops_overview,
|
||||
mock_list_managed_nodes_with_agent_state,
|
||||
mock_get_release_launchpad,
|
||||
mock_get_recent_ops_playbook_runs,
|
||||
mock_get_ops_activity_stream,
|
||||
mock_get_runtime_build_info,
|
||||
) -> None:
|
||||
mock_get_ops_contract_registry.return_value = {
|
||||
"registry_version": "2026-04-18",
|
||||
"schema_version": "v1",
|
||||
"contracts": [],
|
||||
}
|
||||
mock_get_ops_link_snapshot.return_value = {
|
||||
"status": "ready",
|
||||
"headline": "当前总检可用。",
|
||||
"next_actions": [],
|
||||
}
|
||||
mock_get_ops_overview.return_value = {
|
||||
"recommendation": {},
|
||||
"execution_scene": {},
|
||||
"log_sync": {},
|
||||
"release_hub": {},
|
||||
}
|
||||
mock_list_managed_nodes_with_agent_state.return_value = {"nodes": [], "summary": {}}
|
||||
mock_get_release_launchpad.return_value = {"launchpad_status": {}, "latest_release": {}}
|
||||
mock_get_recent_ops_playbook_runs.return_value = {"runs": []}
|
||||
mock_get_ops_activity_stream.return_value = {"items": []}
|
||||
mock_get_runtime_build_info.return_value = {
|
||||
"source": "git",
|
||||
"commit_sha": "bd6bcb2",
|
||||
"route_surface": {
|
||||
"mode": "registered",
|
||||
"registered_paths_total": 12,
|
||||
"surface_complete": False,
|
||||
"missing_keys": [
|
||||
"ops_node_onboarding_acceptance_execute",
|
||||
"ops_node_onboarding_acceptance_preview",
|
||||
],
|
||||
"missing_paths": [
|
||||
"/api/v1/ops/nodes/{node_code}/onboarding/acceptance/execute",
|
||||
"/api/v1/ops/nodes/{node_code}/onboarding/acceptance/preview",
|
||||
],
|
||||
},
|
||||
}
|
||||
|
||||
payload = get_ops_stack_diagnosis(base_url="http://127.0.0.1:8100")
|
||||
|
||||
issue = next(
|
||||
item for item in payload["diagnosis"]["issues"] if item["code"] == "runtime_route_surface_incomplete"
|
||||
)
|
||||
self.assertEqual("api-restart", issue["action_code"])
|
||||
self.assertIn("ops_node_onboarding_acceptance_execute", issue["detail"])
|
||||
self.assertIn(
|
||||
"bash domain-api/deploy/multi-region/drive_ops_center.sh runtime-refresh-recover http://127.0.0.1:8100",
|
||||
issue["commands"],
|
||||
)
|
||||
self.assertFalse(payload["runtime_build_info"]["route_surface"]["surface_complete"])
|
||||
self.assertIn(
|
||||
"ops_node_onboarding_acceptance_preview",
|
||||
payload["runtime_build_info"]["route_surface_missing_keys"],
|
||||
)
|
||||
self.assertTrue(
|
||||
any("API 路由面还不完整" in hint for hint in payload["diagnosis"]["operator_hints"])
|
||||
)
|
||||
|
||||
@patch("app.services.ops_service.get_ops_activity_stream")
|
||||
@patch("app.services.ops_service.get_recent_ops_playbook_runs")
|
||||
@patch("app.services.ops_service.get_release_launchpad")
|
||||
@patch("app.services.ops_service.list_managed_nodes_with_agent_state")
|
||||
@patch("app.services.ops_service.get_ops_overview")
|
||||
@patch("app.services.ops_service.get_ops_link_snapshot")
|
||||
@patch("app.services.ops_service.get_ops_contract_registry")
|
||||
def test_stack_diagnosis_emits_operator_decision_for_scene_log_gap(
|
||||
self,
|
||||
mock_get_ops_contract_registry,
|
||||
mock_get_ops_link_snapshot,
|
||||
mock_get_ops_overview,
|
||||
mock_list_managed_nodes_with_agent_state,
|
||||
mock_get_release_launchpad,
|
||||
mock_get_recent_ops_playbook_runs,
|
||||
mock_get_ops_activity_stream,
|
||||
) -> None:
|
||||
mock_get_ops_contract_registry.return_value = {
|
||||
"registry_version": "2026-04-18",
|
||||
"schema_version": "v1",
|
||||
"contracts": [],
|
||||
}
|
||||
mock_get_ops_link_snapshot.return_value = {
|
||||
"status": "attention",
|
||||
"headline": "参与节点样本尚未形成。",
|
||||
"next_actions": [],
|
||||
}
|
||||
mock_get_ops_overview.return_value = {
|
||||
"recommendation": {},
|
||||
"execution_scene": {
|
||||
"participating_nodes": [{"node_code": "mainland-worker-01", "status": "busy"}],
|
||||
"participation_summary": {
|
||||
"dispatch_active_node_codes": ["mainland-worker-01"],
|
||||
"recent_only_node_codes": [],
|
||||
},
|
||||
"log_sync": {
|
||||
"enabled": True,
|
||||
"mode": "key",
|
||||
"line_count": 0,
|
||||
"source_node_count": 0,
|
||||
"covered_participating_node_count": 0,
|
||||
"participating_node_count": 1,
|
||||
"missing_participating_nodes": ["mainland-worker-01"],
|
||||
},
|
||||
},
|
||||
"log_sync": {},
|
||||
"release_hub": {},
|
||||
}
|
||||
mock_list_managed_nodes_with_agent_state.return_value = {"nodes": [], "summary": {}}
|
||||
mock_get_release_launchpad.return_value = {"launchpad_status": {}, "latest_release": {}}
|
||||
mock_get_recent_ops_playbook_runs.return_value = {"runs": []}
|
||||
mock_get_ops_activity_stream.return_value = {"items": []}
|
||||
|
||||
payload = get_ops_stack_diagnosis(base_url="http://127.0.0.1:8100")
|
||||
|
||||
decision = payload["diagnosis"]["operator_decision"]
|
||||
self.assertEqual("observability", decision["lane"])
|
||||
self.assertEqual("scene-node-log", decision["primary_command_key"])
|
||||
self.assertEqual("mainland-worker-01", decision["next_focus"]["node_code"])
|
||||
self.assertIn("scene-node-log", payload["diagnosis"]["recommended_commands"])
|
||||
self.assertIn(
|
||||
"scene-node-log http://127.0.0.1:8100 mainland-worker-01 120 key",
|
||||
payload["diagnosis"]["next_actions"]["primary_command"],
|
||||
)
|
||||
|
||||
@patch("app.services.ops_service.get_ops_activity_stream")
|
||||
@patch("app.services.ops_service.get_recent_ops_playbook_runs")
|
||||
@patch("app.services.ops_service.get_release_launchpad")
|
||||
@patch("app.services.ops_service.get_managed_node_handover")
|
||||
@patch("app.services.ops_service.list_managed_nodes_with_agent_state")
|
||||
@patch("app.services.ops_service.get_ops_overview")
|
||||
@patch("app.services.ops_service.get_ops_link_snapshot")
|
||||
@patch("app.services.ops_service.get_ops_contract_registry")
|
||||
def test_stack_diagnosis_emits_operator_decision_for_managed_node_gap(
|
||||
self,
|
||||
mock_get_ops_contract_registry,
|
||||
mock_get_ops_link_snapshot,
|
||||
mock_get_ops_overview,
|
||||
mock_list_managed_nodes_with_agent_state,
|
||||
mock_get_managed_node_handover,
|
||||
mock_get_release_launchpad,
|
||||
mock_get_recent_ops_playbook_runs,
|
||||
mock_get_ops_activity_stream,
|
||||
) -> None:
|
||||
mock_get_ops_contract_registry.return_value = {
|
||||
"registry_version": "2026-04-18",
|
||||
"schema_version": "v1",
|
||||
"contracts": [],
|
||||
}
|
||||
mock_get_ops_link_snapshot.return_value = {
|
||||
"status": "attention",
|
||||
"headline": "存在接管缺口。",
|
||||
"next_actions": [],
|
||||
}
|
||||
mock_get_ops_overview.return_value = {
|
||||
"recommendation": {},
|
||||
"execution_scene": {},
|
||||
"log_sync": {},
|
||||
"release_hub": {},
|
||||
}
|
||||
mock_list_managed_nodes_with_agent_state.return_value = {
|
||||
"nodes": [
|
||||
{
|
||||
"node_code": "mainland-worker-01",
|
||||
"agent_state": "pending_bootstrap",
|
||||
"remote_access_state": "ssh_ready",
|
||||
"delivery_queue_state": "healthy",
|
||||
"participation_state": "standby",
|
||||
}
|
||||
],
|
||||
"summary": {
|
||||
"managed_enabled": 1,
|
||||
"remote_access_ready": 0,
|
||||
},
|
||||
}
|
||||
mock_get_managed_node_handover.return_value = {
|
||||
"node_code": "mainland-worker-01",
|
||||
"summary": "等待执行 bootstrap。",
|
||||
"stage": {"code": "pending_bootstrap", "label": "待执行接入", "next_step": {"code": "execute_bootstrap_plan"}},
|
||||
}
|
||||
mock_get_release_launchpad.return_value = {
|
||||
"launchpad_status": {
|
||||
"recommended_action_code": "bootstrap_run",
|
||||
"focus_ref": {"kind": "release_launchpad", "section": "release_launchpad"},
|
||||
"recommended_target_node_code": "mainland-worker-01",
|
||||
},
|
||||
"worker_rollout_preview": {
|
||||
"release_gate": {
|
||||
"operational_readiness": {
|
||||
"rows": [
|
||||
{
|
||||
"node_code": "mainland-worker-01",
|
||||
"recovery_action": "bootstrap_run",
|
||||
"recovery_summary": "mainland-worker-01 还缺接入收口,先执行 bootstrap。",
|
||||
"focus_ref": {"kind": "managed_node", "node_code": "mainland-worker-01"},
|
||||
}
|
||||
]
|
||||
}
|
||||
}
|
||||
},
|
||||
"latest_release": {},
|
||||
}
|
||||
mock_get_recent_ops_playbook_runs.return_value = {"runs": []}
|
||||
mock_get_ops_activity_stream.return_value = {"items": []}
|
||||
|
||||
payload = get_ops_stack_diagnosis(base_url="http://127.0.0.1:8100")
|
||||
|
||||
decision = payload["diagnosis"]["operator_decision"]
|
||||
self.assertEqual("node_handover", decision["lane"])
|
||||
self.assertEqual("bootstrap_run", decision["primary_command_key"])
|
||||
self.assertEqual("managed-node-bootstrap-plan", decision["secondary_command_key"])
|
||||
self.assertIn("bootstrap_run", payload["diagnosis"]["recommended_commands"])
|
||||
self.assertIn("managed-node-handover", payload["diagnosis"]["recommended_commands"])
|
||||
self.assertIn("managed-node-bootstrap-plan", payload["diagnosis"]["recommended_commands"])
|
||||
self.assertIn(
|
||||
"node-bootstrap-run http://127.0.0.1:8100 mainland-worker-01 cli",
|
||||
payload["diagnosis"]["next_actions"]["primary_command"],
|
||||
)
|
||||
self.assertTrue(
|
||||
any("当前第一动作已经收敛为 接入收口:mainland-worker-01" in hint for hint in payload["diagnosis"]["operator_hints"])
|
||||
)
|
||||
|
||||
@patch("app.services.ops_service.get_runtime_build_info")
|
||||
@patch("app.services.ops_service.get_ops_activity_stream")
|
||||
@patch("app.services.ops_service.get_recent_ops_playbook_runs")
|
||||
@patch("app.services.ops_service.get_release_launchpad")
|
||||
@patch("app.services.ops_service.get_managed_node_handover")
|
||||
@patch("app.services.ops_service.list_managed_nodes_with_agent_state")
|
||||
@patch("app.services.ops_service.get_ops_overview")
|
||||
@patch("app.services.ops_service.get_ops_link_snapshot")
|
||||
@patch("app.services.ops_service.get_ops_contract_registry")
|
||||
def test_stack_diagnosis_promotes_first_handover_gap_when_launchpad_is_still_generic(
|
||||
self,
|
||||
mock_get_ops_contract_registry,
|
||||
mock_get_ops_link_snapshot,
|
||||
mock_get_ops_overview,
|
||||
mock_list_managed_nodes_with_agent_state,
|
||||
mock_get_managed_node_handover,
|
||||
mock_get_release_launchpad,
|
||||
mock_get_recent_ops_playbook_runs,
|
||||
mock_get_ops_activity_stream,
|
||||
mock_get_runtime_build_info,
|
||||
) -> None:
|
||||
mock_get_ops_contract_registry.return_value = {
|
||||
"registry_version": "2026-04-18",
|
||||
"schema_version": "v1",
|
||||
"contracts": [{"key": "ops_stack_diagnosis_contract"}],
|
||||
}
|
||||
mock_get_ops_link_snapshot.return_value = {
|
||||
"status": "ready",
|
||||
"headline": "当前总检可用。",
|
||||
"next_actions": [],
|
||||
}
|
||||
mock_get_ops_overview.return_value = {
|
||||
"recommendation": {
|
||||
"primary_action_code": "handover_first_gap",
|
||||
"title": "先补接管缺口",
|
||||
"reason": "overview 只能先定位到接管缺口。",
|
||||
"focus_ref": {"kind": "managed_node", "node_code": "mainland-worker-01"},
|
||||
},
|
||||
"execution_scene": {},
|
||||
"log_sync": {},
|
||||
"release_hub": {},
|
||||
}
|
||||
mock_list_managed_nodes_with_agent_state.return_value = {
|
||||
"nodes": [
|
||||
{
|
||||
"node_code": "mainland-worker-01",
|
||||
"agent_state": "pending_bootstrap",
|
||||
"remote_access_state": "agent_pending",
|
||||
"delivery_queue_state": "healthy",
|
||||
"participation_state": "standby",
|
||||
}
|
||||
],
|
||||
"summary": {
|
||||
"managed_enabled": 1,
|
||||
"remote_access_ready": 0,
|
||||
},
|
||||
}
|
||||
mock_get_managed_node_handover.return_value = {
|
||||
"node_code": "mainland-worker-01",
|
||||
"summary": "当前节点仍处于接入阶段,建议先签发 onboarding.bootstrap 并等待 register / heartbeat 建立。",
|
||||
"stage": {
|
||||
"code": "pending_bootstrap",
|
||||
"label": "待执行接入",
|
||||
"next_step": {
|
||||
"code": "execute_bootstrap_plan",
|
||||
"label": "去目标机执行接入片段",
|
||||
},
|
||||
},
|
||||
}
|
||||
mock_get_release_launchpad.return_value = {
|
||||
"launchpad_status": {
|
||||
"recommended_action_code": "fix_managed_nodes",
|
||||
"status_label": "待补执行面",
|
||||
"focus_ref": {"kind": "release_launchpad", "section": "release_launchpad"},
|
||||
},
|
||||
"latest_release": {},
|
||||
}
|
||||
mock_get_recent_ops_playbook_runs.return_value = {"runs": []}
|
||||
mock_get_ops_activity_stream.return_value = {"items": []}
|
||||
mock_get_runtime_build_info.return_value = {
|
||||
"route_surface": {"surface_complete": True, "missing_keys": []},
|
||||
"repository_capabilities": {},
|
||||
}
|
||||
|
||||
payload = get_ops_stack_diagnosis(base_url="http://127.0.0.1:8100")
|
||||
|
||||
self.assertEqual("bootstrap_run", payload["diagnosis"]["next_step"]["action_code"])
|
||||
self.assertEqual(
|
||||
"mainland-worker-01",
|
||||
payload["diagnosis"]["launchpad_recommended_target_node_code"],
|
||||
)
|
||||
self.assertEqual(
|
||||
"签发接入工单",
|
||||
payload["diagnosis"]["launchpad_recommended_recovery_label"],
|
||||
)
|
||||
self.assertIn(
|
||||
"onboarding.bootstrap",
|
||||
payload["diagnosis"]["launchpad_recommended_recovery_summary"],
|
||||
)
|
||||
self.assertEqual("bootstrap_run", payload["diagnosis"]["operator_decision"]["primary_command_key"])
|
||||
self.assertEqual(
|
||||
"bash domain-api/deploy/multi-region/drive_ops_center.sh driver-resolve http://127.0.0.1:8100 bootstrap_run",
|
||||
payload["diagnosis"]["next_actions"]["primary_command"],
|
||||
)
|
||||
|
||||
@patch("app.services.ops_service.get_ops_activity_stream")
|
||||
@patch("app.services.ops_service.get_recent_ops_playbook_runs")
|
||||
@patch("app.services.ops_service.get_release_launchpad")
|
||||
@patch("app.services.ops_service.list_managed_nodes_with_agent_state")
|
||||
@patch("app.services.ops_service.get_ops_overview")
|
||||
@patch("app.services.ops_service.get_ops_link_snapshot")
|
||||
@patch("app.services.ops_service.get_ops_contract_registry")
|
||||
def test_stack_diagnosis_emits_operator_decision_for_release_launchpad_attention(
|
||||
self,
|
||||
mock_get_ops_contract_registry,
|
||||
mock_get_ops_link_snapshot,
|
||||
mock_get_ops_overview,
|
||||
mock_list_managed_nodes_with_agent_state,
|
||||
mock_get_release_launchpad,
|
||||
mock_get_recent_ops_playbook_runs,
|
||||
mock_get_ops_activity_stream,
|
||||
) -> None:
|
||||
mock_get_ops_contract_registry.return_value = {
|
||||
"registry_version": "2026-04-18",
|
||||
"schema_version": "v1",
|
||||
"contracts": [],
|
||||
}
|
||||
mock_get_ops_link_snapshot.return_value = {
|
||||
"status": "attention",
|
||||
"headline": "发布门禁待处理。",
|
||||
"next_actions": [],
|
||||
}
|
||||
mock_get_ops_overview.return_value = {
|
||||
"recommendation": {},
|
||||
"execution_scene": {},
|
||||
"log_sync": {},
|
||||
"release_hub": {
|
||||
"default_rollout_gate": {
|
||||
"status": "blocked",
|
||||
"summary_text": "还有节点未满足发布门禁。",
|
||||
"focus_ref": {"kind": "release_hub", "section": "release_launchpad", "mode": "worker"},
|
||||
}
|
||||
},
|
||||
}
|
||||
mock_list_managed_nodes_with_agent_state.return_value = {"nodes": [], "summary": {}}
|
||||
mock_get_release_launchpad.return_value = {
|
||||
"launchpad_status": {
|
||||
"status": "attention",
|
||||
"status_label": "待处理",
|
||||
"recommended_action_code": "review_smart_rollout_preview",
|
||||
"focus_ref": {"kind": "release_hub", "section": "release_launchpad", "mode": "worker"},
|
||||
},
|
||||
"latest_release": {"release_version": "release-20260418"},
|
||||
}
|
||||
mock_get_recent_ops_playbook_runs.return_value = {"runs": []}
|
||||
mock_get_ops_activity_stream.return_value = {"items": []}
|
||||
|
||||
payload = get_ops_stack_diagnosis(base_url="http://127.0.0.1:8100")
|
||||
|
||||
decision = payload["diagnosis"]["operator_decision"]
|
||||
self.assertEqual("release", decision["lane"])
|
||||
self.assertEqual("release-launchpad-worker", decision["primary_command_key"])
|
||||
self.assertEqual("review_smart_rollout_preview", decision["secondary_command_key"])
|
||||
self.assertIn("release-launchpad-worker", payload["diagnosis"]["recommended_commands"])
|
||||
self.assertIn(
|
||||
"release-launchpad http://127.0.0.1:8100 worker",
|
||||
payload["diagnosis"]["next_actions"]["primary_command"],
|
||||
)
|
||||
|
||||
@patch("app.services.ops_service.get_ops_activity_stream")
|
||||
@patch("app.services.ops_service.get_recent_ops_playbook_runs")
|
||||
@patch("app.services.ops_service.get_release_launchpad")
|
||||
@patch("app.services.ops_service.list_managed_nodes_with_agent_state")
|
||||
@patch("app.services.ops_service.get_ops_overview")
|
||||
@patch("app.services.ops_service.get_ops_link_snapshot")
|
||||
@patch("app.services.ops_service.get_ops_contract_registry")
|
||||
def test_stack_diagnosis_prefers_node_recovery_when_launchpad_points_to_bootstrap_gap(
|
||||
self,
|
||||
mock_get_ops_contract_registry,
|
||||
mock_get_ops_link_snapshot,
|
||||
mock_get_ops_overview,
|
||||
mock_list_managed_nodes_with_agent_state,
|
||||
mock_get_release_launchpad,
|
||||
mock_get_recent_ops_playbook_runs,
|
||||
mock_get_ops_activity_stream,
|
||||
) -> None:
|
||||
mock_get_ops_contract_registry.return_value = {
|
||||
"registry_version": "2026-04-18",
|
||||
"schema_version": "v1",
|
||||
"contracts": [],
|
||||
}
|
||||
mock_get_ops_link_snapshot.return_value = {
|
||||
"status": "attention",
|
||||
"headline": "发布门禁待处理。",
|
||||
"next_actions": [],
|
||||
}
|
||||
mock_get_ops_overview.return_value = {
|
||||
"recommendation": {},
|
||||
"execution_scene": {},
|
||||
"log_sync": {},
|
||||
"release_hub": {
|
||||
"default_rollout_gate": {
|
||||
"status": "attention",
|
||||
"summary_text": "当前还有节点未完成接入。",
|
||||
"focus_ref": {"kind": "release_hub", "section": "release_launchpad", "mode": "worker"},
|
||||
}
|
||||
},
|
||||
}
|
||||
mock_list_managed_nodes_with_agent_state.return_value = {"nodes": [], "summary": {}}
|
||||
mock_get_release_launchpad.return_value = {
|
||||
"launchpad_status": {
|
||||
"status": "attention",
|
||||
"status_label": "待补执行面",
|
||||
"recommended_action_code": "bootstrap_run",
|
||||
"recommended_target_node_code": "mainland-worker-02",
|
||||
"recommended_recovery_label": "待执行接入",
|
||||
"recommended_recovery_summary": "当前节点仍处于接入阶段,建议先签发 onboarding.bootstrap。",
|
||||
"onboarding_bootstrap_pending_nodes": 1,
|
||||
"onboarding_acceptance_ready_nodes": 0,
|
||||
"focus_ref": {"kind": "release_hub", "section": "release_launchpad", "mode": "worker"},
|
||||
},
|
||||
"latest_release": {"release_version": "release-20260418"},
|
||||
}
|
||||
mock_get_recent_ops_playbook_runs.return_value = {"runs": []}
|
||||
mock_get_ops_activity_stream.return_value = {"items": []}
|
||||
|
||||
payload = get_ops_stack_diagnosis(base_url="http://127.0.0.1:8100")
|
||||
|
||||
decision = payload["diagnosis"]["operator_decision"]
|
||||
self.assertEqual("node_handover", decision["lane"])
|
||||
self.assertEqual("bootstrap_run", decision["primary_command_key"])
|
||||
self.assertEqual("focus-release-hub", decision["secondary_command_key"])
|
||||
self.assertEqual("mainland-worker-02", payload["diagnosis"]["launchpad_recommended_target_node_code"])
|
||||
self.assertEqual("待执行接入", payload["diagnosis"]["launchpad_recommended_recovery_label"])
|
||||
self.assertEqual(1, payload["diagnosis"]["launchpad_onboarding_bootstrap_pending_nodes"])
|
||||
self.assertEqual(0, payload["diagnosis"]["launchpad_onboarding_acceptance_ready_nodes"])
|
||||
self.assertEqual(
|
||||
"当前节点仍处于接入阶段,建议先签发 onboarding.bootstrap。",
|
||||
decision["summary"],
|
||||
)
|
||||
self.assertEqual("mainland-worker-02", decision["next_focus"]["node_code"])
|
||||
self.assertIn("node-bootstrap-run", payload["diagnosis"]["next_actions"]["primary_command"])
|
||||
|
||||
@patch("app.services.ops_service.get_ops_activity_stream")
|
||||
@patch("app.services.ops_service.get_recent_ops_playbook_runs")
|
||||
@patch("app.services.ops_service.get_release_launchpad")
|
||||
@patch("app.services.ops_service.get_managed_node_handover")
|
||||
@patch("app.services.ops_service.list_managed_nodes_with_agent_state")
|
||||
@patch("app.services.ops_service.get_ops_overview")
|
||||
@patch("app.services.ops_service.get_ops_link_snapshot")
|
||||
@patch("app.services.ops_service.get_ops_contract_registry")
|
||||
def test_stack_diagnosis_includes_first_handover_gap_details(
|
||||
self,
|
||||
mock_get_ops_contract_registry,
|
||||
mock_get_ops_link_snapshot,
|
||||
mock_get_ops_overview,
|
||||
mock_list_managed_nodes_with_agent_state,
|
||||
mock_get_managed_node_handover,
|
||||
mock_get_release_launchpad,
|
||||
mock_get_recent_ops_playbook_runs,
|
||||
mock_get_ops_activity_stream,
|
||||
) -> None:
|
||||
mock_get_ops_contract_registry.return_value = {
|
||||
"registry_version": "2026-04-18",
|
||||
"schema_version": "v1",
|
||||
"contracts": [],
|
||||
}
|
||||
mock_get_ops_link_snapshot.return_value = {
|
||||
"status": "attention",
|
||||
"headline": "存在接管缺口。",
|
||||
"next_actions": [],
|
||||
}
|
||||
mock_get_ops_overview.return_value = {
|
||||
"recommendation": {},
|
||||
"execution_scene": {},
|
||||
"log_sync": {},
|
||||
"release_hub": {},
|
||||
}
|
||||
mock_list_managed_nodes_with_agent_state.return_value = {
|
||||
"nodes": [
|
||||
{
|
||||
"node_code": "mainland-worker-01",
|
||||
"agent_state": "pending_bootstrap",
|
||||
"remote_access_state": "ssh_ready",
|
||||
"delivery_queue_state": "healthy",
|
||||
"participation_state": "standby",
|
||||
}
|
||||
],
|
||||
"summary": {
|
||||
"managed_enabled": 1,
|
||||
"remote_access_ready": 0,
|
||||
"status_counts": {"pending_bootstrap": 1},
|
||||
"remote_access_state_counts": {"ssh_ready": 1},
|
||||
"delivery_queue_state_counts": {"healthy": 1},
|
||||
},
|
||||
}
|
||||
mock_get_managed_node_handover.return_value = {
|
||||
"node_code": "mainland-worker-01",
|
||||
"summary": "已签发有效 Token,但目标节点尚未执行 bootstrap 片段。",
|
||||
"stage": {
|
||||
"code": "pending_bootstrap",
|
||||
"label": "待执行接入",
|
||||
"next_step": {
|
||||
"code": "execute_bootstrap_plan",
|
||||
"label": "去目标机执行接入片段",
|
||||
},
|
||||
},
|
||||
}
|
||||
mock_get_release_launchpad.return_value = {
|
||||
"launchpad_status": {
|
||||
"recommended_action_code": "fix_managed_nodes",
|
||||
"focus_ref": {"kind": "release_launchpad"},
|
||||
},
|
||||
"latest_release": {},
|
||||
}
|
||||
mock_get_recent_ops_playbook_runs.return_value = {"runs": []}
|
||||
mock_get_ops_activity_stream.return_value = {"items": []}
|
||||
|
||||
payload = get_ops_stack_diagnosis(base_url="http://127.0.0.1:8100")
|
||||
|
||||
self.assertEqual(
|
||||
"mainland-worker-01",
|
||||
payload["managed_nodes"]["first_handover_gap_node_code"],
|
||||
)
|
||||
self.assertEqual(
|
||||
"pending_bootstrap",
|
||||
payload["managed_nodes"]["first_handover_gap"]["stage"]["code"],
|
||||
)
|
||||
issue = next(
|
||||
item for item in payload["diagnosis"]["issues"] if item["code"] == "managed_nodes_agent_pending"
|
||||
)
|
||||
self.assertIn("mainland-worker-01", issue["detail"])
|
||||
self.assertTrue(
|
||||
any("node-handover http://127.0.0.1:8100 mainland-worker-01" in command for command in issue["commands"])
|
||||
)
|
||||
self.assertIn(
|
||||
"bash domain-api/deploy/multi-region/drive_ops_center.sh node-recover http://127.0.0.1:8100 mainland-worker-01",
|
||||
payload["diagnosis"]["recommended_commands"]["managed-node-recover"],
|
||||
)
|
||||
self.assertTrue(
|
||||
any("node-bootstrap-plan http://127.0.0.1:8100 mainland-worker-01" in command for command in issue["commands"])
|
||||
)
|
||||
mock_get_managed_node_handover.assert_called_once()
|
||||
|
||||
@patch("app.services.ops_service.get_ops_activity_stream")
|
||||
@patch("app.services.ops_service.get_recent_ops_playbook_runs")
|
||||
@patch("app.services.ops_service.get_release_launchpad")
|
||||
@patch("app.services.ops_service.list_managed_nodes_with_agent_state")
|
||||
@patch("app.services.ops_service.get_ops_overview")
|
||||
@patch("app.services.ops_service.get_ops_link_snapshot")
|
||||
@patch("app.services.ops_service.get_ops_contract_registry")
|
||||
def test_stack_diagnosis_keeps_focus_refs_for_problem_runs_and_top_items(
|
||||
self,
|
||||
mock_get_ops_contract_registry,
|
||||
mock_get_ops_link_snapshot,
|
||||
mock_get_ops_overview,
|
||||
mock_list_managed_nodes_with_agent_state,
|
||||
mock_get_release_launchpad,
|
||||
mock_get_recent_ops_playbook_runs,
|
||||
mock_get_ops_activity_stream,
|
||||
) -> None:
|
||||
mock_get_ops_contract_registry.return_value = {
|
||||
"registry_version": "2026-04-18",
|
||||
"schema_version": "v1",
|
||||
"contracts": [],
|
||||
}
|
||||
mock_get_ops_link_snapshot.return_value = {
|
||||
"status": "ready",
|
||||
"headline": "当前总检可用。",
|
||||
"next_actions": [],
|
||||
}
|
||||
mock_get_ops_overview.return_value = {
|
||||
"recommendation": {},
|
||||
"execution_scene": {},
|
||||
"log_sync": {},
|
||||
"release_hub": {},
|
||||
}
|
||||
mock_list_managed_nodes_with_agent_state.return_value = {
|
||||
"nodes": [],
|
||||
"summary": {},
|
||||
}
|
||||
mock_get_release_launchpad.return_value = {
|
||||
"launchpad_status": {},
|
||||
"latest_release": {},
|
||||
}
|
||||
mock_get_recent_ops_playbook_runs.return_value = {
|
||||
"runs": [
|
||||
{
|
||||
"run_code": "pbr-worker-logs-001",
|
||||
"status": "attention",
|
||||
"status_label": "待关注",
|
||||
"focus_step_key": "worker_logs",
|
||||
"focus_summary": "Worker 日志步骤仍未收口。",
|
||||
"focus_ref": {
|
||||
"kind": "playbook_run",
|
||||
"run_code": "pbr-worker-logs-001",
|
||||
"focus_step_key": "worker_logs",
|
||||
},
|
||||
}
|
||||
]
|
||||
}
|
||||
mock_get_ops_activity_stream.return_value = {
|
||||
"items": [
|
||||
{
|
||||
"kind": "ops_job",
|
||||
"status": "running",
|
||||
"summary_text": "远端 Agent 已回传关键日志样本。",
|
||||
"occurred_at": "2026-04-18 09:22:00",
|
||||
"focus_ref": {
|
||||
"kind": "ops_job",
|
||||
"job_id": 71,
|
||||
"job_code": "ops-agent-71",
|
||||
},
|
||||
"source_focus_ref": {
|
||||
"kind": "ops_job_event",
|
||||
"job_id": 71,
|
||||
"event_key": "job-event:71",
|
||||
},
|
||||
"ui_intent": {
|
||||
"kind": "job_events",
|
||||
"job_id": 71,
|
||||
},
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
payload = get_ops_stack_diagnosis(base_url="http://127.0.0.1:8100")
|
||||
|
||||
self.assertEqual(
|
||||
"ops_stack_diagnosis_contract",
|
||||
payload["diagnosis"]["contract_navigation"]["primary_contract_key"],
|
||||
)
|
||||
self.assertIn("ops_driver_contract", payload["diagnosis"]["contract_navigation"]["contract_keys"])
|
||||
self.assertEqual("playbook_run", payload["playbook_runs"]["problem_runs"][0]["focus_ref"]["kind"])
|
||||
self.assertEqual("worker_logs", payload["playbook_runs"]["problem_runs"][0]["focus_ref"]["focus_step_key"])
|
||||
top_item = payload["activity_stream"]["top_items"][0]
|
||||
self.assertEqual("ops_job", top_item["focus_ref"]["kind"])
|
||||
self.assertEqual("ops_job_event", top_item["source_focus_ref"]["kind"])
|
||||
self.assertEqual("job_events", top_item["ui_intent_kind"])
|
||||
self.assertEqual("2026-04-18 09:22:00", top_item["occurred_at"])
|
||||
|
||||
@patch("app.services.ops_service.get_ops_activity_stream")
|
||||
@patch("app.services.ops_service.get_recent_ops_playbook_runs")
|
||||
@patch("app.services.ops_service.get_release_launchpad")
|
||||
@patch("app.services.ops_service.list_managed_nodes_with_agent_state")
|
||||
@patch("app.services.ops_service.get_ops_overview")
|
||||
@patch("app.services.ops_service.get_ops_link_snapshot")
|
||||
@patch("app.services.ops_service.get_ops_contract_registry")
|
||||
def test_stack_diagnosis_adds_node_scene_log_commands_for_log_sample_gap(
|
||||
self,
|
||||
mock_get_ops_contract_registry,
|
||||
mock_get_ops_link_snapshot,
|
||||
mock_get_ops_overview,
|
||||
mock_list_managed_nodes_with_agent_state,
|
||||
mock_get_release_launchpad,
|
||||
mock_get_recent_ops_playbook_runs,
|
||||
mock_get_ops_activity_stream,
|
||||
) -> None:
|
||||
mock_get_ops_contract_registry.return_value = {
|
||||
"registry_version": "2026-04-18",
|
||||
"schema_version": "v1",
|
||||
"contracts": [],
|
||||
}
|
||||
mock_get_ops_link_snapshot.return_value = {
|
||||
"status": "attention",
|
||||
"headline": "参与节点样本尚未形成。",
|
||||
"next_actions": [
|
||||
{
|
||||
"label": "看参与节点日志",
|
||||
"action_code": "open_worker_logs_participating",
|
||||
"node_codes": ["mainland-worker-01"],
|
||||
"payload": {},
|
||||
}
|
||||
],
|
||||
}
|
||||
mock_get_ops_overview.return_value = {
|
||||
"recommendation": {},
|
||||
"execution_scene": {
|
||||
"participating_nodes": [
|
||||
{
|
||||
"node_code": "mainland-worker-01",
|
||||
"status": "busy",
|
||||
}
|
||||
],
|
||||
"participation_summary": {
|
||||
"dispatch_active_node_codes": ["mainland-worker-01"],
|
||||
"recent_only_node_codes": [],
|
||||
},
|
||||
"log_sync": {
|
||||
"enabled": True,
|
||||
"mode": "key",
|
||||
"line_count": 0,
|
||||
"source_node_count": 0,
|
||||
"covered_participating_node_count": 0,
|
||||
"participating_node_count": 1,
|
||||
"missing_participating_nodes": ["mainland-worker-01"],
|
||||
},
|
||||
},
|
||||
"log_sync": {},
|
||||
"release_hub": {},
|
||||
}
|
||||
mock_list_managed_nodes_with_agent_state.return_value = {
|
||||
"nodes": [],
|
||||
"summary": {},
|
||||
}
|
||||
mock_get_release_launchpad.return_value = {
|
||||
"launchpad_status": {},
|
||||
"latest_release": {},
|
||||
}
|
||||
mock_get_recent_ops_playbook_runs.return_value = {"runs": []}
|
||||
mock_get_ops_activity_stream.return_value = {"items": []}
|
||||
|
||||
payload = get_ops_stack_diagnosis(base_url="http://127.0.0.1:8100")
|
||||
|
||||
issue = next(
|
||||
item for item in payload["diagnosis"]["issues"] if item["code"] == "remote_log_sync_waiting_sample"
|
||||
)
|
||||
self.assertEqual("open_worker_logs_participating", issue["action_code"])
|
||||
self.assertEqual("node_scene_log", issue["focus_ref"]["kind"])
|
||||
self.assertEqual("mainland-worker-01", issue["focus_ref"]["node_code"])
|
||||
self.assertTrue(
|
||||
any(
|
||||
"scene-node-log http://127.0.0.1:8100 mainland-worker-01 120 key" in command
|
||||
for command in issue["commands"]
|
||||
)
|
||||
)
|
||||
self.assertEqual("node_scene_log", payload["diagnosis"]["next_step"]["focus_ref"]["kind"])
|
||||
self.assertTrue(
|
||||
any(
|
||||
"scene-node-log http://127.0.0.1:8100 mainland-worker-01 120 key" in command
|
||||
for command in payload["diagnosis"]["quick_commands"]
|
||||
)
|
||||
)
|
||||
self.assertTrue(
|
||||
any(
|
||||
"driver-resolve http://127.0.0.1:8100 open_worker_logs_participating" in command
|
||||
for command in issue["commands"]
|
||||
)
|
||||
)
|
||||
|
||||
@patch("app.services.ops_service.get_ops_activity_stream")
|
||||
@patch("app.services.ops_service.get_recent_ops_playbook_runs")
|
||||
@patch("app.services.ops_service.get_release_launchpad")
|
||||
@patch("app.services.ops_service.list_managed_nodes_with_agent_state")
|
||||
@patch("app.services.ops_service.get_ops_overview")
|
||||
@patch("app.services.ops_service.get_ops_link_snapshot")
|
||||
@patch("app.services.ops_service.get_ops_contract_registry")
|
||||
def test_stack_diagnosis_surfaces_start_delivery_gap_from_activity_stream(
|
||||
self,
|
||||
mock_get_ops_contract_registry,
|
||||
mock_get_ops_link_snapshot,
|
||||
mock_get_ops_overview,
|
||||
mock_list_managed_nodes_with_agent_state,
|
||||
mock_get_release_launchpad,
|
||||
mock_get_recent_ops_playbook_runs,
|
||||
mock_get_ops_activity_stream,
|
||||
) -> None:
|
||||
mock_get_ops_contract_registry.return_value = {
|
||||
"registry_version": "2026-04-18",
|
||||
"schema_version": "v1",
|
||||
"contracts": [],
|
||||
}
|
||||
mock_get_ops_link_snapshot.return_value = {
|
||||
"status": "attention",
|
||||
"headline": "最近有任务回执异常。",
|
||||
"next_actions": [],
|
||||
}
|
||||
mock_get_ops_overview.return_value = {
|
||||
"recommendation": {},
|
||||
"execution_scene": {},
|
||||
"log_sync": {},
|
||||
"release_hub": {},
|
||||
}
|
||||
mock_list_managed_nodes_with_agent_state.return_value = {
|
||||
"nodes": [],
|
||||
"summary": {},
|
||||
}
|
||||
mock_get_release_launchpad.return_value = {
|
||||
"launchpad_status": {},
|
||||
"latest_release": {},
|
||||
}
|
||||
mock_get_recent_ops_playbook_runs.return_value = {"runs": []}
|
||||
mock_get_ops_activity_stream.return_value = {
|
||||
"items": [
|
||||
{
|
||||
"kind": "ops_job",
|
||||
"status": "attention",
|
||||
"job_status": "running",
|
||||
"summary_text": "节点已接单执行,但开始回执未成功送达控制面。",
|
||||
"occurred_at": "2026-04-18 10:22:00",
|
||||
"job_code": "ops-agent-81",
|
||||
"start_delivery_state": "failed_local",
|
||||
"start_delivery_error": "temporary offline",
|
||||
"focus_ref": {
|
||||
"kind": "ops_job",
|
||||
"job_id": 81,
|
||||
"job_code": "ops-agent-81",
|
||||
"target_node_code": "mainland-worker-01",
|
||||
},
|
||||
"source_focus_ref": {
|
||||
"kind": "ops_job_event",
|
||||
"job_id": 81,
|
||||
"job_code": "ops-agent-81",
|
||||
"event_key": "job-event:81",
|
||||
"target_node_code": "mainland-worker-01",
|
||||
},
|
||||
"ui_intent": {
|
||||
"kind": "job_events",
|
||||
"job_id": 81,
|
||||
},
|
||||
}
|
||||
]
|
||||
}
|
||||
|
||||
payload = get_ops_stack_diagnosis(base_url="http://127.0.0.1:8100")
|
||||
|
||||
issue = next(
|
||||
item for item in payload["diagnosis"]["issues"] if item["code"] == "ops_job_start_delivery_failed_local"
|
||||
)
|
||||
self.assertEqual("focus_latest_job_events", issue["action_code"])
|
||||
self.assertEqual("ops_job_event", issue["focus_ref"]["kind"])
|
||||
self.assertIn("temporary offline", issue["detail"])
|
||||
decision = payload["diagnosis"]["operator_decision"]
|
||||
self.assertEqual("ops_jobs", decision["lane"])
|
||||
self.assertEqual("focus_latest_job_events", decision["primary_command_key"])
|
||||
self.assertIn("driver-resolve http://127.0.0.1:8100 focus_latest_job_events", payload["diagnosis"]["next_actions"]["primary_command"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
177
domain-api/tests/test_ops_template_service.py
Normal file
177
domain-api/tests/test_ops_template_service.py
Normal file
@@ -0,0 +1,177 @@
|
||||
import unittest
|
||||
|
||||
from app.services.ops_template_service import build_ops_template_payload, get_ops_action_template
|
||||
|
||||
|
||||
class OpsTemplateServiceTests(unittest.TestCase):
|
||||
def test_node_bootstrap_defaults_and_clamps_number(self) -> None:
|
||||
ok, message, data = build_ops_template_payload(
|
||||
"node.bootstrap",
|
||||
{
|
||||
"expires_in_hours": 9999,
|
||||
},
|
||||
)
|
||||
|
||||
self.assertTrue(ok, message)
|
||||
self.assertEqual("ok", message)
|
||||
self.assertEqual("", data["control_plane_base_url"])
|
||||
self.assertEqual("/opt/domaincheck", data["root_dir"])
|
||||
self.assertEqual(720, data["expires_in_hours"])
|
||||
self.assertEqual("", data["node_region"])
|
||||
self.assertEqual("", data["node_role"])
|
||||
|
||||
def test_deploy_release_custom_normalizes_lists_and_booleans(self) -> None:
|
||||
ok, message, data = build_ops_template_payload(
|
||||
"deploy.release.custom",
|
||||
{
|
||||
"release_version": "2026.04.18-rc1",
|
||||
"artifact_url": "https://example.com/release.tar.gz",
|
||||
"restart_services": "domaincheck-api\ndomaincheck-worker",
|
||||
"health_check_urls": "",
|
||||
"health_check_services": "",
|
||||
"rollback_on_failure": "false",
|
||||
"switch_current": "true",
|
||||
},
|
||||
)
|
||||
|
||||
self.assertTrue(ok, message)
|
||||
self.assertEqual("ok", message)
|
||||
self.assertEqual(["domaincheck-api", "domaincheck-worker"], data["restart_services"])
|
||||
self.assertEqual([], data["health_check_urls"])
|
||||
self.assertEqual([], data["health_check_services"])
|
||||
self.assertFalse(data["rollback_on_failure"])
|
||||
self.assertTrue(data["switch_current"])
|
||||
self.assertEqual("/opt/domaincheck", data["install_root"])
|
||||
|
||||
def test_deploy_release_control_applies_defaults(self) -> None:
|
||||
ok, message, data = build_ops_template_payload(
|
||||
"deploy.release.control",
|
||||
{
|
||||
"release_version": "2026.04.18-rc1",
|
||||
"artifact_url": "https://example.com/release.tar.gz",
|
||||
},
|
||||
)
|
||||
|
||||
self.assertTrue(ok, message)
|
||||
self.assertEqual(
|
||||
["domaincheck-api", "domaincheck-worker", "domaincheck-sync-agent"],
|
||||
data["restart_services"],
|
||||
)
|
||||
self.assertEqual(["http://127.0.0.1:8100/health"], data["health_check_urls"])
|
||||
self.assertEqual(
|
||||
["domaincheck-api", "domaincheck-worker", "domaincheck-sync-agent"],
|
||||
data["health_check_services"],
|
||||
)
|
||||
self.assertTrue(data["rollback_on_failure"])
|
||||
self.assertTrue(data["switch_current"])
|
||||
|
||||
def test_deploy_release_missing_required_field_fails(self) -> None:
|
||||
ok, message, data = build_ops_template_payload(
|
||||
"deploy.release.custom",
|
||||
{
|
||||
"release_version": "2026.04.18-rc1",
|
||||
"artifact_url": "",
|
||||
},
|
||||
)
|
||||
|
||||
self.assertFalse(ok)
|
||||
self.assertEqual("artifact_url 不能为空", message)
|
||||
self.assertEqual({}, data)
|
||||
|
||||
def test_logs_collect_lines_are_clamped(self) -> None:
|
||||
ok, message, data = build_ops_template_payload(
|
||||
"logs.collect",
|
||||
{
|
||||
"service_name": "domaincheck-worker",
|
||||
"lines": 9999,
|
||||
},
|
||||
)
|
||||
|
||||
self.assertTrue(ok, message)
|
||||
self.assertEqual(500, data["lines"])
|
||||
|
||||
def test_delivery_queue_templates_normalize_and_require_expected_fields(self) -> None:
|
||||
replay_ok, replay_message, replay_data = build_ops_template_payload(
|
||||
"delivery.queue.replay",
|
||||
{
|
||||
"record_id": "evt-001",
|
||||
"limit": 999,
|
||||
"flush_after_replay": "false",
|
||||
"reason": "确认状态已修复",
|
||||
},
|
||||
)
|
||||
discard_ok, discard_message, discard_data = build_ops_template_payload(
|
||||
"delivery.queue.discard",
|
||||
{
|
||||
"record_id": "evt-002",
|
||||
"limit": 0,
|
||||
"reason": "确认这条死信不再需要补发",
|
||||
},
|
||||
)
|
||||
flush_ok, flush_message, flush_data = build_ops_template_payload(
|
||||
"delivery.queue.flush",
|
||||
{
|
||||
"limit": 999,
|
||||
},
|
||||
)
|
||||
discard_fail_ok, discard_fail_message, discard_fail_data = build_ops_template_payload(
|
||||
"delivery.queue.discard",
|
||||
{
|
||||
"record_id": "evt-003",
|
||||
"reason": "",
|
||||
},
|
||||
)
|
||||
|
||||
self.assertTrue(replay_ok, replay_message)
|
||||
self.assertEqual(200, replay_data["limit"])
|
||||
self.assertFalse(replay_data["flush_after_replay"])
|
||||
self.assertEqual("evt-001", replay_data["record_id"])
|
||||
|
||||
self.assertTrue(discard_ok, discard_message)
|
||||
self.assertEqual(1, discard_data["limit"])
|
||||
self.assertEqual("确认这条死信不再需要补发", discard_data["reason"])
|
||||
|
||||
self.assertTrue(flush_ok, flush_message)
|
||||
self.assertEqual(200, flush_data["limit"])
|
||||
|
||||
self.assertFalse(discard_fail_ok)
|
||||
self.assertEqual("reason 不能为空", discard_fail_message)
|
||||
self.assertEqual({}, discard_fail_data)
|
||||
|
||||
def test_diagnostics_and_service_control_templates_expose_ssh(self) -> None:
|
||||
health_template = get_ops_action_template("health.snapshot")
|
||||
logs_template = get_ops_action_template("logs.collect")
|
||||
diagnostics_template = get_ops_action_template("diagnostics.collect")
|
||||
service_status_template = get_ops_action_template("service.status")
|
||||
restart_template = get_ops_action_template("service.restart")
|
||||
restart_api_template = get_ops_action_template("runtime.restart_api")
|
||||
stop_worker_template = get_ops_action_template("runtime.stop_worker")
|
||||
deploy_control_template = get_ops_action_template("deploy.release.control")
|
||||
deploy_worker_template = get_ops_action_template("deploy.release.worker")
|
||||
deploy_custom_template = get_ops_action_template("deploy.release.custom")
|
||||
queue_flush_template = get_ops_action_template("delivery.queue.flush")
|
||||
queue_replay_template = get_ops_action_template("delivery.queue.replay")
|
||||
queue_discard_template = get_ops_action_template("delivery.queue.discard")
|
||||
|
||||
self.assertIn("ssh", health_template["execution_modes"])
|
||||
self.assertEqual("远端 Agent", health_template["default_execution_mode_label"])
|
||||
self.assertEqual("SSH", next(item["label"] for item in health_template["execution_mode_options"] if item["value"] == "ssh"))
|
||||
self.assertIn("ssh", logs_template["execution_modes"])
|
||||
self.assertIn("ssh", diagnostics_template["execution_modes"])
|
||||
self.assertIn("ssh", service_status_template["execution_modes"])
|
||||
self.assertIn("ssh", restart_template["execution_modes"])
|
||||
self.assertIn("ssh", restart_api_template["execution_modes"])
|
||||
self.assertIn("ssh", stop_worker_template["execution_modes"])
|
||||
self.assertEqual("控制面", get_ops_action_template("node.bootstrap")["default_execution_mode_label"])
|
||||
self.assertIn("ssh", deploy_control_template["execution_modes"])
|
||||
self.assertIn("ssh", deploy_worker_template["execution_modes"])
|
||||
self.assertIn("ssh", deploy_custom_template["execution_modes"])
|
||||
self.assertEqual(["remote-agent"], queue_flush_template["execution_modes"])
|
||||
self.assertEqual(["remote-agent"], queue_replay_template["execution_modes"])
|
||||
self.assertEqual(["remote-agent"], queue_discard_template["execution_modes"])
|
||||
self.assertEqual("远端 Agent", queue_flush_template["default_execution_mode_label"])
|
||||
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
117
domain-api/tests/test_prepare_final_release.py
Normal file
117
domain-api/tests/test_prepare_final_release.py
Normal file
@@ -0,0 +1,117 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
class PrepareFinalReleaseTests(unittest.TestCase):
|
||||
def test_prepare_final_release_blocks_when_smoke_is_false_even_if_verify_passes(self) -> None:
|
||||
repo_root = Path(__file__).resolve().parents[2]
|
||||
source_script = repo_root / "prepare_final_release.sh"
|
||||
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
root = Path(temp_dir)
|
||||
release_root = root / "release"
|
||||
release_root.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
(root / "prepare_final_release.sh").write_text(source_script.read_text(encoding="utf-8"), encoding="utf-8")
|
||||
(root / "package_domain_release.sh").write_text(
|
||||
"#!/usr/bin/env bash\n"
|
||||
"set -euo pipefail\n"
|
||||
"mkdir -p \"$PWD/release\"\n"
|
||||
"cat > \"$PWD/release/latest_release.json\" <<'EOF'\n"
|
||||
"{\n"
|
||||
" \"package_name\": \"domaincheck_release_test\",\n"
|
||||
" \"archive_path\": \"/tmp/domaincheck_release_test.tar.gz\",\n"
|
||||
" \"sha256_path\": \"/tmp/domaincheck_release_test.sha256.txt\",\n"
|
||||
" \"sha256\": \"abc\",\n"
|
||||
" \"smoke_test_ok\": false,\n"
|
||||
" \"smoke_test_message\": \"skipped by env\"\n"
|
||||
"}\n"
|
||||
"EOF\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
(root / "verify_domain_release.sh").write_text(
|
||||
"#!/usr/bin/env bash\n"
|
||||
"set -euo pipefail\n"
|
||||
"printf '%s\\n' '{\"ok\": true, \"archive\": \"/tmp/domaincheck_release_test.tar.gz\"}'\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
os.chmod(root / "prepare_final_release.sh", 0o755)
|
||||
os.chmod(root / "package_domain_release.sh", 0o755)
|
||||
os.chmod(root / "verify_domain_release.sh", 0o755)
|
||||
|
||||
result = subprocess.run(
|
||||
["bash", str(root / "prepare_final_release.sh")],
|
||||
cwd=root,
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
|
||||
payload = json.loads((release_root / "final_release_report.json").read_text(encoding="utf-8"))
|
||||
self.assertFalse(payload["ok"])
|
||||
self.assertEqual("blocked", payload["release_gate"]["decision"])
|
||||
self.assertTrue(payload["release_gate"]["verify_ok"])
|
||||
self.assertFalse(payload["release_gate"]["smoke_test_ok"])
|
||||
self.assertIn("smoke_test_failed_or_skipped", payload["release_gate"]["blocked_reasons"])
|
||||
self.assertIn('"ok": false', result.stdout)
|
||||
|
||||
def test_prepare_final_release_ready_when_verify_and_smoke_both_pass(self) -> None:
|
||||
repo_root = Path(__file__).resolve().parents[2]
|
||||
source_script = repo_root / "prepare_final_release.sh"
|
||||
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
root = Path(temp_dir)
|
||||
release_root = root / "release"
|
||||
release_root.mkdir(parents=True, exist_ok=True)
|
||||
|
||||
(root / "prepare_final_release.sh").write_text(source_script.read_text(encoding="utf-8"), encoding="utf-8")
|
||||
(root / "package_domain_release.sh").write_text(
|
||||
"#!/usr/bin/env bash\n"
|
||||
"set -euo pipefail\n"
|
||||
"mkdir -p \"$PWD/release\"\n"
|
||||
"cat > \"$PWD/release/latest_release.json\" <<'EOF'\n"
|
||||
"{\n"
|
||||
" \"package_name\": \"domaincheck_release_test\",\n"
|
||||
" \"archive_path\": \"/tmp/domaincheck_release_test.tar.gz\",\n"
|
||||
" \"sha256_path\": \"/tmp/domaincheck_release_test.sha256.txt\",\n"
|
||||
" \"sha256\": \"abc\",\n"
|
||||
" \"smoke_test_ok\": true,\n"
|
||||
" \"smoke_test_message\": \"all checks passed (1/1)\"\n"
|
||||
"}\n"
|
||||
"EOF\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
(root / "verify_domain_release.sh").write_text(
|
||||
"#!/usr/bin/env bash\n"
|
||||
"set -euo pipefail\n"
|
||||
"printf '%s\\n' '{\"ok\": true, \"archive\": \"/tmp/domaincheck_release_test.tar.gz\"}'\n",
|
||||
encoding="utf-8",
|
||||
)
|
||||
os.chmod(root / "prepare_final_release.sh", 0o755)
|
||||
os.chmod(root / "package_domain_release.sh", 0o755)
|
||||
os.chmod(root / "verify_domain_release.sh", 0o755)
|
||||
|
||||
subprocess.run(
|
||||
["bash", str(root / "prepare_final_release.sh")],
|
||||
cwd=root,
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
)
|
||||
|
||||
payload = json.loads((release_root / "final_release_report.json").read_text(encoding="utf-8"))
|
||||
self.assertTrue(payload["ok"])
|
||||
self.assertEqual("ready", payload["release_gate"]["decision"])
|
||||
self.assertEqual([], payload["release_gate"]["blocked_reasons"])
|
||||
self.assertTrue(payload["release_gate"]["verify_ok"])
|
||||
self.assertTrue(payload["release_gate"]["smoke_test_ok"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
29
domain-api/tests/test_release_metadata_parity.py
Normal file
29
domain-api/tests/test_release_metadata_parity.py
Normal file
@@ -0,0 +1,29 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
class ReleaseMetadataParityTests(unittest.TestCase):
|
||||
def test_powershell_release_package_emits_archive_path_alias(self) -> None:
|
||||
repo_root = Path(__file__).resolve().parents[2]
|
||||
script_text = (repo_root / "package_domain_release.ps1").read_text(encoding="utf-8")
|
||||
|
||||
self.assertIn("archive_path = $zipPath", script_text)
|
||||
self.assertIn('"archive_path=$zipPath"', script_text)
|
||||
|
||||
def test_powershell_release_flow_prefers_archive_path_with_zip_fallback(self) -> None:
|
||||
repo_root = Path(__file__).resolve().parents[2]
|
||||
prepare_text = (repo_root / "prepare_final_release.ps1").read_text(encoding="utf-8")
|
||||
show_text = (repo_root / "show_latest_release.ps1").read_text(encoding="utf-8")
|
||||
|
||||
self.assertIn("$archivePath = if ($latest.archive_path) { $latest.archive_path } else { $latest.zip_path }", prepare_text)
|
||||
self.assertIn('& (Join-Path $root "package_domain_release.ps1")', prepare_text)
|
||||
self.assertIn('& (Join-Path $root "verify_domain_release.ps1")', prepare_text)
|
||||
self.assertNotIn('powershell -ExecutionPolicy Bypass -File (Join-Path $root "package_domain_release.ps1")', prepare_text)
|
||||
self.assertIn("$reportArchivePath = if ($report.latest_release.archive_path) { [string]$report.latest_release.archive_path } else { [string]$report.latest_release.zip_path }", show_text)
|
||||
self.assertIn("$latestArchivePath = if ($latest.archive_path) { [string]$latest.archive_path } else { [string]$latest.zip_path }", show_text)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
19
domain-api/tests/test_release_script_parity.py
Normal file
19
domain-api/tests/test_release_script_parity.py
Normal file
@@ -0,0 +1,19 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
class ReleaseScriptParityTests(unittest.TestCase):
|
||||
def test_shell_release_verifier_checks_cross_platform_delivery_scripts(self) -> None:
|
||||
repo_root = Path(__file__).resolve().parents[2]
|
||||
script_path = repo_root / "verify_domain_release.sh"
|
||||
script_text = script_path.read_text(encoding="utf-8")
|
||||
|
||||
self.assertIn('"scripts/package_domain_release.ps1"', script_text)
|
||||
self.assertIn('"scripts/verify_domain_release.ps1"', script_text)
|
||||
self.assertIn('"scripts/smoke_test_stack.ps1"', script_text)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
162
domain-api/tests/test_show_latest_release.py
Normal file
162
domain-api/tests/test_show_latest_release.py
Normal file
@@ -0,0 +1,162 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import os
|
||||
import subprocess
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
|
||||
|
||||
class ShowLatestReleaseTests(unittest.TestCase):
|
||||
def test_show_latest_release_marks_stale_final_report_as_not_ok(self) -> None:
|
||||
repo_root = Path(__file__).resolve().parents[2]
|
||||
script_path = repo_root / "show_latest_release.sh"
|
||||
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
release_root = Path(temp_dir)
|
||||
latest_payload = {
|
||||
"package_name": "domaincheck_release_new",
|
||||
"archive_path": str(release_root / "domaincheck_release_new.tar.gz"),
|
||||
"sha256": "new-sha256",
|
||||
}
|
||||
final_report = {
|
||||
"ok": True,
|
||||
"latest_release": {
|
||||
"package_name": "domaincheck_release_old",
|
||||
"archive_path": str(release_root / "domaincheck_release_old.tar.gz"),
|
||||
"sha256": "old-sha256",
|
||||
},
|
||||
}
|
||||
(release_root / "latest_release.json").write_text(
|
||||
json.dumps(latest_payload, ensure_ascii=False, indent=2),
|
||||
encoding="utf-8",
|
||||
)
|
||||
(release_root / "final_release_report.json").write_text(
|
||||
json.dumps(final_report, ensure_ascii=False, indent=2),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
env = dict(os.environ)
|
||||
env["DOMAINCHECK_RELEASE_ROOT"] = str(release_root)
|
||||
result = subprocess.run(
|
||||
["bash", str(script_path)],
|
||||
cwd=repo_root,
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
env=env,
|
||||
)
|
||||
|
||||
payload = json.loads(result.stdout)
|
||||
self.assertFalse(payload["final_release_ok"])
|
||||
self.assertTrue(payload["final_release_report_ok"])
|
||||
self.assertFalse(payload["final_release_report_matches_latest"])
|
||||
self.assertEqual("domaincheck_release_old", payload["final_release_report_package_name"])
|
||||
self.assertIn("package_name", payload["final_release_report_stale_reason"])
|
||||
self.assertEqual("", payload["final_release_gate_decision"])
|
||||
self.assertEqual([], payload["final_release_gate_blocked_reasons"])
|
||||
|
||||
def test_show_latest_release_keeps_final_report_ok_when_it_matches_latest(self) -> None:
|
||||
repo_root = Path(__file__).resolve().parents[2]
|
||||
script_path = repo_root / "show_latest_release.sh"
|
||||
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
release_root = Path(temp_dir)
|
||||
latest_payload = {
|
||||
"package_name": "domaincheck_release_new",
|
||||
"archive_path": str(release_root / "domaincheck_release_new.tar.gz"),
|
||||
"sha256": "new-sha256",
|
||||
}
|
||||
final_report = {
|
||||
"ok": True,
|
||||
"latest_release": latest_payload,
|
||||
"release_gate": {
|
||||
"decision": "ready",
|
||||
"blocked_reasons": [],
|
||||
"verify_ok": True,
|
||||
"smoke_test_ok": True,
|
||||
},
|
||||
}
|
||||
(release_root / "latest_release.json").write_text(
|
||||
json.dumps(latest_payload, ensure_ascii=False, indent=2),
|
||||
encoding="utf-8",
|
||||
)
|
||||
(release_root / "final_release_report.json").write_text(
|
||||
json.dumps(final_report, ensure_ascii=False, indent=2),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
env = dict(os.environ)
|
||||
env["DOMAINCHECK_RELEASE_ROOT"] = str(release_root)
|
||||
result = subprocess.run(
|
||||
["bash", str(script_path)],
|
||||
cwd=repo_root,
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
env=env,
|
||||
)
|
||||
|
||||
payload = json.loads(result.stdout)
|
||||
self.assertTrue(payload["final_release_ok"])
|
||||
self.assertTrue(payload["final_release_report_ok"])
|
||||
self.assertTrue(payload["final_release_report_matches_latest"])
|
||||
self.assertEqual("domaincheck_release_new", payload["final_release_report_package_name"])
|
||||
self.assertEqual("", payload["final_release_report_stale_reason"])
|
||||
self.assertEqual("ready", payload["final_release_gate_decision"])
|
||||
self.assertEqual([], payload["final_release_gate_blocked_reasons"])
|
||||
self.assertTrue(payload["final_release_gate_verify_ok"])
|
||||
self.assertTrue(payload["final_release_gate_smoke_test_ok"])
|
||||
|
||||
def test_show_latest_release_accepts_zip_path_fallback(self) -> None:
|
||||
repo_root = Path(__file__).resolve().parents[2]
|
||||
script_path = repo_root / "show_latest_release.sh"
|
||||
|
||||
with tempfile.TemporaryDirectory() as temp_dir:
|
||||
release_root = Path(temp_dir)
|
||||
latest_payload = {
|
||||
"package_name": "domaincheck_release_win",
|
||||
"zip_path": str(release_root / "domaincheck_release_win.zip"),
|
||||
"sha256": "zip-sha256",
|
||||
}
|
||||
final_report = {
|
||||
"ok": True,
|
||||
"latest_release": latest_payload,
|
||||
"release_gate": {
|
||||
"decision": "ready",
|
||||
"blocked_reasons": [],
|
||||
"verify_ok": True,
|
||||
"smoke_test_ok": True,
|
||||
},
|
||||
}
|
||||
(release_root / "latest_release.json").write_text(
|
||||
json.dumps(latest_payload, ensure_ascii=False, indent=2),
|
||||
encoding="utf-8",
|
||||
)
|
||||
(release_root / "final_release_report.json").write_text(
|
||||
json.dumps(final_report, ensure_ascii=False, indent=2),
|
||||
encoding="utf-8",
|
||||
)
|
||||
|
||||
env = dict(os.environ)
|
||||
env["DOMAINCHECK_RELEASE_ROOT"] = str(release_root)
|
||||
result = subprocess.run(
|
||||
["bash", str(script_path)],
|
||||
cwd=repo_root,
|
||||
check=True,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
env=env,
|
||||
)
|
||||
|
||||
payload = json.loads(result.stdout)
|
||||
self.assertTrue(payload["final_release_ok"])
|
||||
self.assertTrue(payload["final_release_report_ok"])
|
||||
self.assertTrue(payload["final_release_report_matches_latest"])
|
||||
self.assertEqual("domaincheck_release_win", payload["final_release_report_package_name"])
|
||||
self.assertEqual("ready", payload["final_release_gate_decision"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
Reference in New Issue
Block a user