feat: add ops center and node onboarding flow

This commit is contained in:
Your Name
2026-04-18 23:52:51 +08:00
parent 246838ae4c
commit b9c29481b5
142 changed files with 89727 additions and 186 deletions

View File

@@ -0,0 +1,98 @@
import json
import tempfile
import unittest
from pathlib import Path
from unittest.mock import patch
from app.core.config import settings
from app.services.build_info_service import (
_expected_route_paths,
get_runtime_build_info,
remember_registered_route_paths,
)
class BuildInfoServiceTests(unittest.TestCase):
def tearDown(self) -> None:
remember_registered_route_paths([])
def test_runtime_build_info_prefers_configured_manifest_and_reports_registered_surface(self) -> None:
with tempfile.TemporaryDirectory() as temp_dir:
manifest_path = Path(temp_dir) / "release_manifest.json"
manifest_path.write_text(
json.dumps(
{
"package_name": "domaincheck_release_demo",
"generated_at": "2026-04-18T12:00:00",
"commit_sha": "bd6bcb2",
"commit_ref": "main",
"sha256": "demo-sha256",
},
ensure_ascii=False,
),
encoding="utf-8",
)
route_paths = list(_expected_route_paths().values())
remember_registered_route_paths(route_paths)
with patch.multiple(
settings,
build_manifest_path=str(manifest_path),
build_commit_sha="",
build_commit_ref="",
build_generated_at="",
build_package_name="",
build_checksum="",
build_source_label="",
):
payload = get_runtime_build_info()
self.assertEqual("configured_manifest", payload["source"])
self.assertEqual("domaincheck_release_demo", payload["package_name"])
self.assertEqual("bd6bcb2", payload["commit_sha"])
self.assertEqual("main", payload["commit_ref"])
self.assertEqual("demo-sha256", payload["checksum"])
self.assertEqual(str(manifest_path), payload["manifest_path"])
self.assertTrue(payload["route_surface"]["surface_complete"])
self.assertEqual([], payload["route_surface"]["missing_keys"])
self.assertEqual("registered", payload["route_surface"]["mode"])
self.assertIn("repository_capabilities", payload)
self.assertIn("supports_install_command_block", payload["repository_capabilities"])
self.assertIn("supports_multi_layout_bootstrap", payload["repository_capabilities"])
def test_runtime_build_info_env_override_wins_and_missing_routes_are_visible(self) -> None:
with patch.multiple(
settings,
build_manifest_path="",
build_commit_sha="246838ae4c07",
build_commit_ref="main",
build_generated_at="2026-04-18T12:10:00",
build_package_name="domaincheck_release_env",
build_checksum="env-sha256",
build_source_label="env-override",
):
payload = get_runtime_build_info(route_paths=[_expected_route_paths()["ops_contracts"]])
self.assertEqual("env-override", payload["source"])
self.assertEqual("domaincheck_release_env", payload["package_name"])
self.assertEqual("246838ae4c07", payload["commit_sha"])
self.assertEqual("main", payload["commit_ref"])
self.assertEqual("env-sha256", payload["checksum"])
self.assertFalse(payload["route_surface"]["surface_complete"])
self.assertIn("ops_stack_diagnosis", payload["route_surface"]["missing_keys"])
self.assertIn("ops_node_onboarding_bootstrap_preview", payload["route_surface"]["missing_keys"])
self.assertIn("ops_node_onboarding_bootstrap_execute", payload["route_surface"]["missing_keys"])
self.assertIn("ops_node_onboarding_acceptance_preview", payload["route_surface"]["missing_keys"])
self.assertIn("ops_node_onboarding_acceptance_execute", payload["route_surface"]["missing_keys"])
self.assertIn("ops_node_onboarding_recovery_preview", payload["route_surface"]["missing_keys"])
self.assertIn("ops_node_onboarding_recovery_execute", payload["route_surface"]["missing_keys"])
self.assertIn("ops_node_scene_log", payload["route_surface"]["missing_keys"])
self.assertIn("runtime_build_info", payload["route_surface"]["missing_keys"])
self.assertEqual("registered", payload["route_surface"]["mode"])
self.assertIn("repository_capabilities", payload)
self.assertIn("supports_install_command_block", payload["repository_capabilities"])
if __name__ == "__main__":
unittest.main()

View File

@@ -0,0 +1,111 @@
from __future__ import annotations
import os
import re
import subprocess
import unittest
from pathlib import Path
class DriveOpsCenterDocsTests(unittest.TestCase):
def test_authoritative_docs_only_reference_supported_commands(self) -> None:
repo_root = Path(__file__).resolve().parents[2]
script_path = repo_root / "domain-api" / "deploy" / "multi-region" / "drive_ops_center.sh"
doc_paths = [
repo_root / "domain-api" / "deploy" / "multi-region" / "README.md",
repo_root / "docs" / "25_domainCheck_海外单脑控制面上线收口总表.md",
repo_root / "docs" / "26_domainCheck_发布前运行验证与交付模板.md",
]
help_output = subprocess.run(
["bash", str(script_path), "help"],
cwd=repo_root,
check=True,
capture_output=True,
text=True,
).stdout
supported_commands = self._extract_supported_commands(help_output)
documented_commands = self._extract_documented_commands(doc_paths)
missing_commands = sorted(documented_commands - supported_commands)
self.assertEqual(
[],
missing_commands,
msg=(
"These drive_ops_center.sh commands are referenced by authoritative docs "
"but missing from the CLI help output: "
+ ", ".join(missing_commands)
),
)
self.assertIn("stack-diagnosis", documented_commands)
self.assertIn("stack-diagnosis", supported_commands)
def test_stack_diagnosis_summary_argument_normalization_does_not_treat_summary_as_host(self) -> None:
repo_root = Path(__file__).resolve().parents[2]
script_path = repo_root / "domain-api" / "deploy" / "multi-region" / "drive_ops_center.sh"
env = dict(os.environ)
env["DOMAINCHECK_OPS_CENTER_ENV"] = "/tmp/domaincheck-ops-center-does-not-exist"
env["OPS_MAINLAND_API_BASE_URL"] = "http://127.0.0.1:9"
result = subprocess.run(
["bash", str(script_path), "stack-diagnosis", "summary"],
cwd=repo_root,
check=True,
capture_output=True,
text=True,
env=env,
)
combined_output = f"{result.stdout}\n{result.stderr}"
self.assertNotIn("Could not resolve host: summary", combined_output)
self.assertNotIn("NameError", combined_output)
self.assertIn("[9/9] condensed stack summary", combined_output)
def test_env_drift_recommends_stack_diagnosis_instead_of_stack_check(self) -> None:
repo_root = Path(__file__).resolve().parents[2]
script_path = repo_root / "domain-api" / "deploy" / "multi-region" / "check_env_drift.sh"
script_text = script_path.read_text(encoding="utf-8")
self.assertIn(
'recommended_actions.append("bash domain-api/deploy/multi-region/drive_ops_center.sh stack-diagnosis http://127.0.0.1:8100 summary")',
script_text,
)
self.assertNotIn(
'recommended_actions.append("bash domain-api/deploy/multi-region/drive_ops_center.sh stack-check http://127.0.0.1:8100 summary")',
script_text,
)
@staticmethod
def _extract_supported_commands(help_output: str) -> set[str]:
commands: set[str] = set()
in_commands_section = False
for raw_line in help_output.splitlines():
line = raw_line.rstrip()
if line.strip() == "commands:":
in_commands_section = True
continue
if line.strip() == "examples:":
break
if not in_commands_section:
continue
stripped = line.strip()
if not stripped:
continue
command = stripped.split()[0]
if re.fullmatch(r"[a-z0-9-]+", command):
commands.add(command)
return commands
@staticmethod
def _extract_documented_commands(doc_paths: list[Path]) -> set[str]:
pattern = re.compile(r"drive_ops_center\.sh\s+([a-z0-9-]+)")
commands: set[str] = set()
for doc_path in doc_paths:
text = doc_path.read_text(encoding="utf-8")
commands.update(match.group(1) for match in pattern.finditer(text))
return commands
if __name__ == "__main__":
unittest.main()

View File

@@ -0,0 +1,284 @@
import json
import os
import tempfile
import unittest
import urllib.error
from unittest.mock import patch
from app import node_agent
class NodeAgentDeliveryQueueTests(unittest.TestCase):
def test_base_payload_includes_delivery_queue_snapshot(self) -> None:
with tempfile.TemporaryDirectory() as temp_dir:
with patch.object(node_agent, "AGENT_QUEUE_DIR", temp_dir):
node_agent._ensure_queue_dirs()
payload = node_agent._base_payload()
delivery_queue = payload["metadata"]["delivery_queue"]
self.assertEqual("healthy", delivery_queue["state"])
self.assertEqual(0, delivery_queue["pending_count"])
self.assertEqual(0, delivery_queue["dead_letter_count"])
def test_job_event_network_failure_is_queued_for_retry(self) -> None:
with tempfile.TemporaryDirectory() as temp_dir:
with patch.object(node_agent, "AGENT_QUEUE_DIR", temp_dir), patch.object(
node_agent,
"_post",
side_effect=urllib.error.URLError("offline"),
):
result = node_agent._job_event(
7,
event_type="executor_received",
message="accepted",
client_event_id="evt-001",
summary_text="已接单",
focus_ref={"kind": "ops_job", "job_id": 7},
)
self.assertEqual("queued", result["state"])
pending_dir = os.path.join(temp_dir, "pending")
pending_files = sorted(os.listdir(pending_dir))
self.assertEqual(1, len(pending_files))
with open(os.path.join(pending_dir, pending_files[0]), "r", encoding="utf-8") as handle:
record = json.load(handle)
self.assertEqual("evt-001", record["payload"]["client_event_id"])
self.assertEqual("已接单", record["payload"]["summary_text"])
self.assertEqual("ops_job", record["payload"]["focus_ref"]["kind"])
self.assertEqual("job_event", record["kind"])
def test_job_complete_transport_failure_preserves_client_request_id(self) -> None:
with tempfile.TemporaryDirectory() as temp_dir:
with patch.object(node_agent, "AGENT_QUEUE_DIR", temp_dir), patch.object(
node_agent,
"_post",
side_effect=urllib.error.URLError("offline"),
):
result = node_agent._job_complete(
9,
status="success",
stdout="done",
stderr="",
result={"ok": True},
client_request_id="complete-001",
duration_ms=321,
summary_text="执行成功",
focus_ref={"kind": "ops_job", "job_id": 9},
)
self.assertEqual("queued", result["state"])
pending_dir = os.path.join(temp_dir, "pending")
pending_files = sorted(os.listdir(pending_dir))
self.assertEqual(1, len(pending_files))
with open(os.path.join(pending_dir, pending_files[0]), "r", encoding="utf-8") as handle:
record = json.load(handle)
self.assertEqual("complete-001", record["payload"]["client_request_id"])
self.assertEqual(321, record["payload"]["duration_ms"])
self.assertEqual("执行成功", record["payload"]["summary_text"])
self.assertEqual("ops_job", record["payload"]["focus_ref"]["kind"])
self.assertEqual("job_complete", record["kind"])
def test_flush_delivery_queue_delivers_pending_records(self) -> None:
with tempfile.TemporaryDirectory() as temp_dir:
with patch.object(node_agent, "AGENT_QUEUE_DIR", temp_dir):
record = node_agent._build_delivery_record(
"job_event",
"/api/v1/ops/agent/jobs/7/events",
{
"node_code": "mainland-worker-01",
"event_type": "executor_received",
"message": "accepted",
"client_event_id": "evt-002",
},
"evt-002",
)
node_agent._store_pending_delivery(record)
with patch.object(node_agent, "_post", return_value={"code": 0, "message": "ok"}):
summary = node_agent._flush_delivery_queue(limit=10)
self.assertEqual(1, summary["delivered"])
self.assertEqual([], os.listdir(os.path.join(temp_dir, "pending")))
def test_flush_delivery_queue_moves_semantic_failure_to_dead_letter(self) -> None:
with tempfile.TemporaryDirectory() as temp_dir:
with patch.object(node_agent, "AGENT_QUEUE_DIR", temp_dir):
record = node_agent._build_delivery_record(
"job_complete",
"/api/v1/ops/agent/jobs/9/complete",
{
"node_code": "mainland-worker-01",
"status": "success",
"client_request_id": "complete-002",
},
"complete-002",
)
node_agent._store_pending_delivery(record)
with patch.object(
node_agent,
"_post",
return_value={
"code": 1,
"message": "任务不存在或不属于当前节点",
"detail_code": "ops_job_not_owned_by_agent",
},
):
summary = node_agent._flush_delivery_queue(limit=10)
self.assertEqual(1, summary["dead_letter"])
self.assertEqual([], os.listdir(os.path.join(temp_dir, "pending")))
self.assertEqual(1, len(os.listdir(os.path.join(temp_dir, "dead-letter"))))
def test_replay_dead_letter_records_moves_record_back_to_pending_and_flushes(self) -> None:
with tempfile.TemporaryDirectory() as temp_dir:
with patch.object(node_agent, "AGENT_QUEUE_DIR", temp_dir):
record = node_agent._build_delivery_record(
"job_event",
"/api/v1/ops/agent/jobs/7/events",
{
"node_code": "mainland-worker-01",
"event_type": "executor_received",
"message": "accepted",
"client_event_id": "evt-003",
},
"evt-003",
)
node_agent._store_dead_letter_delivery(
record,
reason="temporary failure",
detail_code="transient_error",
)
with patch.object(node_agent, "_post", return_value={"code": 0, "message": "ok"}):
ok, message, data = node_agent._execute_action(
"delivery.queue.replay",
{
"selector": {"record_id": "evt-003"},
"limit": 10,
"flush_after_replay": True,
"reason": "恢复后重放",
},
)
self.assertTrue(ok)
self.assertEqual("已重放 1 条死信记录", message)
self.assertEqual(1, data["replayed_total"])
self.assertEqual(1, data["flush_summary"]["delivered"])
self.assertEqual([], os.listdir(os.path.join(temp_dir, "dead-letter")))
self.assertEqual([], os.listdir(os.path.join(temp_dir, "pending")))
def test_discard_dead_letter_records_archives_record(self) -> None:
with tempfile.TemporaryDirectory() as temp_dir:
with patch.object(node_agent, "AGENT_QUEUE_DIR", temp_dir):
record = node_agent._build_delivery_record(
"job_complete",
"/api/v1/ops/agent/jobs/8/complete",
{
"node_code": "mainland-worker-01",
"status": "failed",
"client_request_id": "complete-003",
},
"complete-003",
)
node_agent._store_dead_letter_delivery(
record,
reason="permanent failure",
detail_code="ops_job_invalid_status",
)
ok, message, data = node_agent._execute_action(
"delivery.queue.discard",
{
"selector": {"record_id": "complete-003"},
"limit": 10,
"discarded_by": "web-ui",
"reason": "确认不需要再补发",
},
)
self.assertTrue(ok)
self.assertEqual("已丢弃 1 条死信记录", message)
self.assertEqual(1, data["discarded_total"])
self.assertEqual([], os.listdir(os.path.join(temp_dir, "dead-letter")))
discarded_files = os.listdir(os.path.join(temp_dir, "discarded"))
self.assertEqual(1, len(discarded_files))
with open(os.path.join(temp_dir, "discarded", discarded_files[0]), "r", encoding="utf-8") as handle:
archived = json.load(handle)
self.assertEqual("web-ui", archived["discarded_by"])
self.assertEqual("确认不需要再补发", archived["discard_reason"])
def test_process_job_passes_envelope_context_to_completion(self) -> None:
job = {
"id": 12,
"job_code": "ops-20260418-000012",
"job_type": "release_deploy",
"action": "logs.collect",
"payload": {"service_name": "domaincheck-worker", "lines": 120},
"step_key": "worker_logs",
"step_title": "收集 Worker 日志",
"focus_ref": {"kind": "ops_job", "job_id": 12, "job_code": "ops-20260418-000012"},
"release_context": {"release_version": "v2026.04.18", "rollout_id": 3},
"step_ref": {"step_id": 22, "step_key": "worker_logs", "step_title": "收集 Worker 日志"},
}
with patch.object(node_agent, "_job_start") as mock_job_start, patch.object(
node_agent,
"_job_event",
) as mock_job_event, patch.object(
node_agent,
"_execute_action",
return_value=(True, "logs collected", {"summary": "收集完成", "stdout": "ok", "stderr": ""}),
) as mock_execute_action, patch.object(
node_agent,
"_job_complete",
return_value={"state": "sent"},
) as mock_job_complete:
node_agent._process_job(job)
self.assertEqual("worker_logs", mock_job_start.call_args.kwargs["job"]["step_key"])
self.assertEqual("ops_job", mock_job_event.call_args.kwargs["focus_ref"]["kind"])
self.assertEqual("已接单 logs.collect", mock_job_event.call_args.kwargs["summary_text"])
self.assertEqual("logs.collect", mock_execute_action.call_args.args[0])
self.assertEqual("v2026.04.18", mock_job_complete.call_args.kwargs["release_context"]["release_version"])
self.assertEqual("ops_job", mock_job_complete.call_args.kwargs["focus_ref"]["kind"])
self.assertEqual("收集完成", mock_job_complete.call_args.kwargs["summary_text"])
self.assertGreaterEqual(mock_job_complete.call_args.kwargs["duration_ms"], 0)
def test_process_job_continues_when_job_start_delivery_fails(self) -> None:
job = {
"id": 18,
"job_code": "ops-20260418-000018",
"job_type": "ops_action",
"action": "logs.collect",
"payload": {"service_name": "domaincheck-worker", "lines": 50},
"step_key": "worker_logs",
"step_title": "收集 Worker 日志",
"focus_ref": {"kind": "ops_job", "job_id": 18, "job_code": "ops-20260418-000018"},
"release_context": {},
"step_ref": {"step_id": 28, "step_key": "worker_logs", "step_title": "收集 Worker 日志"},
}
with patch.object(
node_agent,
"_job_start",
side_effect=urllib.error.URLError("temporary offline"),
) as mock_job_start, patch.object(
node_agent,
"_job_event",
return_value={"state": "queued"},
) as mock_job_event, patch.object(
node_agent,
"_execute_action",
return_value=(True, "logs collected", {"summary": "收集完成", "stdout": "ok", "stderr": ""}),
) as mock_execute_action, patch.object(
node_agent,
"_job_complete",
return_value={"state": "sent"},
) as mock_job_complete:
node_agent._process_job(job)
mock_job_start.assert_called_once()
mock_execute_action.assert_called_once()
mock_job_complete.assert_called_once()
event_payload = mock_job_event.call_args.kwargs["payload"]
self.assertEqual("failed_local", event_payload["start_delivery_state"])
self.assertIn("temporary offline", event_payload["start_delivery_error"])
if __name__ == "__main__":
unittest.main()

View File

@@ -0,0 +1,41 @@
import unittest
from app import node_agent
class NodeAgentReleaseNormalizationTests(unittest.TestCase):
def test_normalize_text_list_supports_strings_and_lists(self) -> None:
self.assertEqual(
["domaincheck-api", "domaincheck-worker"],
node_agent._normalize_text_list("domaincheck-api\ndomaincheck-worker"),
)
self.assertEqual(
["domaincheck-api", "domaincheck-worker"],
node_agent._normalize_text_list(["domaincheck-api", " domaincheck-worker "]),
)
self.assertEqual([], node_agent._normalize_text_list(""))
def test_health_check_services_fall_back_to_restart_services_when_missing(self) -> None:
result = node_agent._normalize_release_health_check_services(
{},
["domaincheck-worker"],
)
self.assertEqual(["domaincheck-worker"], result)
def test_health_check_services_can_be_explicitly_empty(self) -> None:
result = node_agent._normalize_release_health_check_services(
{"health_check_services": ""},
["domaincheck-worker"],
)
self.assertEqual([], result)
def test_health_check_services_accept_multiline_string(self) -> None:
result = node_agent._normalize_release_health_check_services(
{"health_check_services": "domaincheck-api\ndomaincheck-sync-agent"},
["domaincheck-worker"],
)
self.assertEqual(["domaincheck-api", "domaincheck-sync-agent"], result)
if __name__ == "__main__":
unittest.main()

View File

@@ -0,0 +1,115 @@
import tempfile
import unittest
from pathlib import Path
from unittest.mock import patch
from app.services.ops_action_executor_core import build_service_name_map, execute_structured_action
class _FakeRunner:
def __init__(self, responses: dict[tuple[str, ...], tuple[int, str, str]]) -> None:
self.responses = responses
self.calls: list[tuple[str, ...]] = []
def __call__(self, command: list[str], *, timeout: int = 60) -> tuple[int, str, str]:
key = tuple(command)
self.calls.append(key)
return self.responses.get(key, (0, "", ""))
class OpsActionExecutorCoreTests(unittest.TestCase):
def test_health_snapshot_includes_host_context_and_ok_flags(self) -> None:
runner = _FakeRunner(
{
("systemctl", "is-active", "domaincheck-api"): (0, "active", ""),
("systemctl", "is-active", "domaincheck-worker"): (3, "inactive", ""),
("systemctl", "is-active", "domaincheck-sync-agent"): (0, "active", ""),
("systemctl", "is-active", "domaincheck-node-agent"): (0, "active", ""),
}
)
ok, message, data = execute_structured_action(
"health.snapshot",
{},
service_names=build_service_name_map(
api_service_name="domaincheck-api",
worker_service_name="domaincheck-worker",
sync_agent_service_name="domaincheck-sync-agent",
),
runner=runner,
host_context={"hostname": "mainland-controller-01", "ip": "121.204.244.188"},
)
self.assertTrue(ok)
self.assertEqual("health snapshot collected", message)
self.assertEqual("mainland-controller-01", data["hostname"])
self.assertEqual("121.204.244.188", data["ip"])
self.assertTrue(data["checks"]["api"]["ok"])
self.assertFalse(data["checks"]["worker"]["ok"])
self.assertEqual(4, len(runner.calls))
def test_logs_collect_defaults_to_worker_service_and_clamps_lines(self) -> None:
runner = _FakeRunner(
{
("journalctl", "-u", "domaincheck-worker", "-n", "500", "--no-pager"): (
0,
"line-1\nline-2",
"",
)
}
)
ok, message, data = execute_structured_action(
"logs.collect",
{"lines": 9999},
service_names=build_service_name_map(
api_service_name="domaincheck-api",
worker_service_name="domaincheck-worker",
sync_agent_service_name="domaincheck-sync-agent",
),
runner=runner,
)
self.assertTrue(ok)
self.assertEqual("line-1\nline-2", message)
self.assertEqual("domaincheck-worker", data["service_name"])
self.assertEqual(500, data["lines"])
self.assertEqual(
("journalctl", "-u", "domaincheck-worker", "-n", "500", "--no-pager"),
runner.calls[0],
)
def test_logs_collect_falls_back_to_worker_log_file_when_journal_is_denied(self) -> None:
runner = _FakeRunner(
{
("journalctl", "-u", "domaincheck-worker", "-n", "120", "--no-pager"): (
1,
"",
"No journal files were opened due to insufficient permissions.",
)
}
)
with tempfile.TemporaryDirectory() as temp_dir:
log_path = Path(temp_dir) / "detect_worker.log"
log_path.write_text("worker-a\nworker-b\n", encoding="utf-8")
with patch("app.services.ops_action_executor_core.settings.domain_root", temp_dir):
ok, message, data = execute_structured_action(
"logs.collect",
{"lines": 120},
service_names=build_service_name_map(
api_service_name="domaincheck-api",
worker_service_name="domaincheck-worker",
sync_agent_service_name="domaincheck-sync-agent",
),
runner=runner,
)
self.assertTrue(ok)
self.assertEqual("worker-a\nworker-b", message)
self.assertEqual("file_fallback", data["collection_source"])
self.assertTrue(data["fallback_used"])
self.assertIn("detect_worker.log", data["log_paths"][0])
if __name__ == "__main__":
unittest.main()

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,96 @@
from __future__ import annotations
import unittest
from unittest.mock import patch
from app.api.routes.ops import (
ops_doctor_decision,
ops_go_live_bundle,
ops_go_live_review,
ops_go_live_signoff,
ops_go_live_summary,
)
class OpsApiRoutesTestCase(unittest.TestCase):
@patch("app.api.routes.ops.get_ops_go_live_signoff")
def test_go_live_signoff_route_uses_service_payload(self, mock_get_ops_go_live_signoff) -> None:
mock_get_ops_go_live_signoff.return_value = {
"signoff_status": "ready",
"headline": "ok",
}
response = ops_go_live_signoff(base_url="http://127.0.0.1:8100", report_dir="/tmp/signoff")
self.assertEqual(0, response.code)
self.assertEqual("ready", response.data["signoff_status"])
self.assertEqual("ok", response.data["headline"])
mock_get_ops_go_live_signoff.assert_called_once_with(
base_url="http://127.0.0.1:8100",
report_dir="/tmp/signoff",
)
@patch("app.api.routes.ops.get_ops_go_live_summary")
def test_go_live_summary_route_passes_base_url(self, mock_get_ops_go_live_summary) -> None:
mock_get_ops_go_live_summary.return_value = {
"go_live_status": "attention",
}
response = ops_go_live_summary(base_url="http://example.test")
self.assertEqual(0, response.code)
self.assertEqual("attention", response.data["go_live_status"])
mock_get_ops_go_live_summary.assert_called_once_with(base_url="http://example.test")
@patch("app.api.routes.ops.get_ops_go_live_bundle")
def test_go_live_bundle_route_passes_base_url_and_report_dir(self, mock_get_ops_go_live_bundle) -> None:
mock_get_ops_go_live_bundle.return_value = {
"status": "missing",
}
response = ops_go_live_bundle(base_url="http://example.test", report_dir="/tmp/go-live")
self.assertEqual(0, response.code)
self.assertEqual("missing", response.data["status"])
mock_get_ops_go_live_bundle.assert_called_once_with(
base_url="http://example.test",
report_dir="/tmp/go-live",
)
@patch("app.api.routes.ops.get_ops_go_live_review")
def test_go_live_review_route_passes_base_url_and_report_dir(self, mock_get_ops_go_live_review) -> None:
mock_get_ops_go_live_review.return_value = {
"status": "attention",
"headline": "review",
}
response = ops_go_live_review(base_url="http://example.test", report_dir="/tmp/review")
self.assertEqual(0, response.code)
self.assertEqual("attention", response.data["status"])
self.assertEqual("review", response.data["headline"])
mock_get_ops_go_live_review.assert_called_once_with(
base_url="http://example.test",
report_dir="/tmp/review",
)
@patch("app.api.routes.ops.get_ops_doctor_decision")
def test_doctor_decision_route_passes_base_url_and_report_dir(self, mock_get_ops_doctor_decision) -> None:
mock_get_ops_doctor_decision.return_value = {
"status": "attention",
"headline": "doctor",
}
response = ops_doctor_decision(base_url="http://example.test", report_dir="/tmp/doctor")
self.assertEqual(0, response.code)
self.assertEqual("attention", response.data["status"])
self.assertEqual("doctor", response.data["headline"])
mock_get_ops_doctor_decision.assert_called_once_with(
base_url="http://example.test",
report_dir="/tmp/doctor",
)
if __name__ == "__main__":
unittest.main()

View File

@@ -0,0 +1,44 @@
import unittest
from app.services.ops_command_service import build_bash_command, ops_script_path
class OpsCommandServiceTests(unittest.TestCase):
def test_ops_script_path_prefixes_multi_region_directory(self) -> None:
self.assertEqual(
"domain-api/deploy/multi-region/drive_ops_center.sh",
ops_script_path("drive_ops_center.sh"),
)
def test_ops_script_path_normalizes_leading_slash(self) -> None:
self.assertEqual(
"domain-api/deploy/multi-region/check_ops_center_stack.sh",
ops_script_path("/check_ops_center_stack.sh"),
)
def test_build_bash_command_joins_non_empty_args(self) -> None:
self.assertEqual(
"bash domain-api/deploy/multi-region/drive_ops_center.sh driver-resolve http://127.0.0.1:8100 handover_first_gap",
build_bash_command(
"drive_ops_center.sh",
"driver-resolve",
"http://127.0.0.1:8100",
"handover_first_gap",
),
)
def test_build_bash_command_skips_blank_args(self) -> None:
self.assertEqual(
"bash domain-api/deploy/multi-region/drive_release_hub.sh http://127.0.0.1:8100 launchpad",
build_bash_command(
"drive_release_hub.sh",
"http://127.0.0.1:8100",
"",
"launchpad",
" ",
),
)
if __name__ == "__main__":
unittest.main()

View File

@@ -0,0 +1,391 @@
import json
import unittest
from datetime import datetime
from unittest.mock import MagicMock, patch
from app.core.config import settings
from app.services.ops_job_service import (
_execute_control_plane_job,
_serialize_job_row,
create_ops_job,
dispatch_ops_job,
upsert_managed_node,
)
class OpsJobServiceTests(unittest.TestCase):
def test_serialize_job_row_adds_execution_mode_label(self) -> None:
row = (
9,
"ops-2026041801-abcd12",
"logs.collect",
"node",
"mainland-worker-01",
"queued",
"remote-agent",
"api",
json.dumps({"service_name": "domaincheck-worker", "lines": 120}, ensure_ascii=False),
json.dumps({}, ensure_ascii=False),
json.dumps({}, ensure_ascii=False),
"",
datetime(2026, 4, 18, 10, 0, 0),
None,
None,
datetime(2026, 4, 18, 10, 0, 0),
"low",
False,
"approved",
"",
None,
"",
"",
None,
json.dumps({}, ensure_ascii=False),
json.dumps({}, ensure_ascii=False),
None,
)
job = _serialize_job_row(row)
self.assertEqual("remote-agent", job["execution_mode"])
self.assertEqual("远端 Agent", job["execution_mode_label"])
self.assertEqual("排队中", job["status_label"])
self.assertEqual("已审批", job["approval_status_label"])
self.assertEqual(["mainland-worker-01"], job["target_node_codes"])
self.assertEqual(0, job["steps_total"])
self.assertFalse(job["steps_loaded"])
self.assertFalse(job["is_compact"])
self.assertEqual("ops_job", job["focus_ref"]["kind"])
self.assertEqual(9, job["focus_ref"]["job_id"])
self.assertIn("目标节点 mainland-worker-01", job["summary_text"])
def test_serialize_job_row_counts_loaded_steps(self) -> None:
row = (
10,
"ops-2026041801-efgh34",
"diagnostics.collect",
"node",
"mainland-worker-02",
"running",
"ssh",
"web-ui",
json.dumps({}, ensure_ascii=False),
json.dumps({}, ensure_ascii=False),
json.dumps({}, ensure_ascii=False),
"",
datetime(2026, 4, 18, 10, 5, 0),
datetime(2026, 4, 18, 10, 5, 2),
None,
datetime(2026, 4, 18, 10, 5, 3),
"medium",
True,
"approved",
"admin",
datetime(2026, 4, 18, 10, 5, 1),
"",
"",
datetime(2026, 4, 18, 10, 5, 2),
json.dumps({}, ensure_ascii=False),
json.dumps({}, ensure_ascii=False),
None,
)
job = _serialize_job_row(
row,
steps=[
{"id": 1, "status": "success"},
{"id": 2, "status": "running"},
{"id": 3, "status": "failed"},
],
)
self.assertEqual("执行中", job["status_label"])
self.assertEqual("已审批", job["approval_status_label"])
self.assertEqual(3, job["steps_total"])
self.assertEqual(1, job["steps_running"])
self.assertEqual(1, job["steps_success"])
self.assertEqual(1, job["steps_failed"])
self.assertEqual(2, job["steps_terminal"])
self.assertTrue(job["steps_loaded"])
self.assertIn("mainland-worker-02", job["summary_text"])
@patch("app.services.ops_job_service.list_managed_nodes")
@patch("app.services.ops_agent_service.build_node_agent_bootstrap_plan")
def test_execute_control_plane_bootstrap_uses_managed_node_defaults(
self,
mock_build_node_agent_bootstrap_plan,
mock_list_managed_nodes,
) -> None:
mock_list_managed_nodes.return_value = [
{
"node_code": "mainland-worker-02",
"region": "mainland",
"role": "worker",
}
]
mock_build_node_agent_bootstrap_plan.return_value = (
True,
"节点 Agent 接入方案已生成",
{
"token_preview": "agent-...1234",
"bootstrap_plan": {
"bootstrap_script_name": "bootstrap-node-agent-mainland-worker-02.sh",
},
},
)
ok, message, data = _execute_control_plane_job(
"node.bootstrap",
target_node_code="mainland-worker-02",
payload={
"root_dir": "/opt/domaincheck",
"expires_in_hours": 12,
},
requested_by="unit-test",
metadata={"source": "test"},
)
self.assertTrue(ok)
self.assertEqual("节点 Agent 接入方案已生成", message)
self.assertEqual("control-plane", data["execution_mode"])
self.assertEqual("mainland", data["node_region"])
self.assertEqual("worker", data["node_role"])
mock_build_node_agent_bootstrap_plan.assert_called_once_with(
node_code="mainland-worker-02",
node_region="mainland",
node_role="worker",
issued_by="unit-test",
expires_in_hours=12,
control_plane_base_url="",
root_dir="/opt/domaincheck",
metadata={
"source": "test",
"issued_from": "ops-job/control-plane",
"target_node_code": "mainland-worker-02",
"node_region": "mainland",
"node_role": "worker",
},
)
@patch("app.services.ops_job_service.get_db")
def test_upsert_managed_node_preserves_remote_fields_when_sync_payload_omits_them(self, mock_get_db) -> None:
existing_metadata = {"operator_note": "keep-me"}
merged_metadata = {"operator_note": "keep-me", "cluster_status": "online"}
existing_row = (
"mainland-worker-01",
"mainland",
"worker",
"Mainland Worker 01",
"121.204.244.248",
22,
"root",
"key",
"stable",
False,
json.dumps(existing_metadata, ensure_ascii=False),
datetime(2026, 4, 18, 1, 0, 0),
None,
datetime(2026, 4, 18, 1, 5, 0),
)
returned_row = (
"mainland-worker-01",
"mainland",
"worker",
"S244-248",
"121.204.244.248",
22,
"root",
"key",
"stable",
False,
json.dumps(merged_metadata, ensure_ascii=False),
datetime(2026, 4, 18, 1, 0, 0),
None,
datetime(2026, 4, 18, 1, 6, 0),
)
conn = MagicMock()
cursor = MagicMock()
db_ctx = MagicMock()
cursor_ctx = MagicMock()
db_ctx.__enter__.return_value = conn
db_ctx.__exit__.return_value = False
cursor_ctx.__enter__.return_value = cursor
cursor_ctx.__exit__.return_value = False
conn.cursor.return_value = cursor_ctx
mock_get_db.return_value = db_ctx
cursor.fetchone.side_effect = [existing_row, returned_row]
ok, message, data = upsert_managed_node(
{
"node_code": "mainland-worker-01",
"region": "mainland",
"role": "worker",
"title": "S244-248",
"metadata": {
"cluster_status": "online",
},
}
)
self.assertTrue(ok)
self.assertEqual("托管节点已保存", message)
node = data["node"]
self.assertEqual("121.204.244.248", node["ssh_host"])
self.assertEqual("root", node["ssh_user"])
self.assertFalse(node["is_enabled"])
self.assertEqual("keep-me", node["metadata"]["operator_note"])
self.assertEqual("online", node["metadata"]["cluster_status"])
insert_call = cursor.execute.call_args_list[-1]
insert_params = insert_call.args[1]
self.assertEqual("121.204.244.248", insert_params[4])
self.assertEqual(22, insert_params[5])
self.assertEqual("root", insert_params[6])
self.assertFalse(insert_params[9])
self.assertEqual(merged_metadata, json.loads(insert_params[10]))
@patch("app.services.ops_agent_service.append_ops_job_event")
@patch("app.services.ops_job_service._execute_ssh_job_record")
@patch("app.services.ops_job_service.get_db")
def test_dispatch_ops_job_ssh_executes_via_ssh_record(
self,
mock_get_db,
mock_execute_ssh_job_record,
mock_append_ops_job_event,
) -> None:
conn = MagicMock()
cursor = MagicMock()
db_ctx = MagicMock()
cursor_ctx = MagicMock()
db_ctx.__enter__.return_value = conn
db_ctx.__exit__.return_value = False
cursor_ctx.__enter__.return_value = cursor
cursor_ctx.__exit__.return_value = False
conn.cursor.return_value = cursor_ctx
mock_get_db.return_value = db_ctx
cursor.fetchone.return_value = (
9,
"ops-2026041801-abcd12",
"logs.collect",
"node",
"mainland-worker-01",
"queued",
"ssh",
"api",
json.dumps({"service_name": "domaincheck-worker", "lines": 120}, ensure_ascii=False),
json.dumps({}, ensure_ascii=False),
json.dumps({}, ensure_ascii=False),
"",
datetime(2026, 4, 18, 10, 0, 0),
None,
None,
datetime(2026, 4, 18, 10, 0, 0),
"low",
False,
"approved",
"",
None,
"",
"",
None,
json.dumps({}, ensure_ascii=False),
json.dumps({}, ensure_ascii=False),
None,
)
mock_execute_ssh_job_record.return_value = (
True,
"worker logs collected",
{"job": {"id": 9, "status": "success"}},
)
ok, message, data = dispatch_ops_job(9)
self.assertTrue(ok)
self.assertEqual("已按 SSH 执行派发worker logs collected", message)
self.assertEqual("success", data["job"]["status"])
mock_execute_ssh_job_record.assert_called_once_with(9)
mock_append_ops_job_event.assert_called_once()
event_kwargs = mock_append_ops_job_event.call_args.kwargs
self.assertEqual("job_dispatch_requested", event_kwargs["event_type"])
self.assertEqual("mainland-worker-01", event_kwargs["node_code"])
self.assertEqual("ssh", event_kwargs["payload"]["execution_mode"])
@patch("app.services.ops_agent_service.append_ops_job_event")
@patch("app.services.ops_job_service.get_ops_job")
@patch("app.services.ops_job_service._execute_local_job")
@patch("app.services.ops_policy_service.preview_ops_job_policy")
@patch("app.services.ops_job_service.get_db")
def test_create_ops_job_auto_uses_local_runtime_for_local_structured_action(
self,
mock_get_db,
mock_preview_ops_job_policy,
mock_execute_local_job,
mock_get_ops_job,
mock_append_ops_job_event,
) -> None:
conn = MagicMock()
cursor = MagicMock()
db_ctx = MagicMock()
cursor_ctx = MagicMock()
db_ctx.__enter__.return_value = conn
db_ctx.__exit__.return_value = False
cursor_ctx.__enter__.return_value = cursor
cursor_ctx.__exit__.return_value = False
conn.cursor.return_value = cursor_ctx
mock_get_db.return_value = db_ctx
cursor.fetchone.return_value = (42,)
mock_preview_ops_job_policy.return_value = {
"risk_level": "low",
"approval_required": False,
"blocked": False,
"blocking_reasons": [],
}
mock_execute_local_job.return_value = (
True,
"worker logs collected",
{"executor": "local-runtime"},
)
mock_get_ops_job.return_value = {
"id": 42,
"status": "success",
"execution_mode": "local-runtime",
}
ok, message, data = create_ops_job(
{
"action": "logs.collect",
"target_node_code": settings.node_code,
"payload": {
"service_name": "domaincheck-worker",
"lines": 120,
},
"requested_by": "unit-test",
}
)
self.assertTrue(ok)
self.assertEqual("worker logs collected", message)
self.assertTrue(data["executed_immediately"])
self.assertEqual("local-runtime", data["job"]["execution_mode"])
mock_execute_local_job.assert_called_once_with(
"logs.collect",
{
"service_name": "domaincheck-worker",
"lines": 120,
},
)
insert_call = next(
call
for call in cursor.execute.call_args_list
if "INSERT INTO ops_jobs" in str(call.args[0])
)
insert_params = insert_call.args[1]
self.assertEqual("local-runtime", insert_params[5])
mock_append_ops_job_event.assert_called_once()
if __name__ == "__main__":
unittest.main()

View File

@@ -0,0 +1,220 @@
import unittest
from unittest.mock import patch
from app.services.ops_playbook_service import (
get_recent_ops_playbook_runs,
list_ops_playbook_run_events,
preview_ops_playbook,
)
class OpsPlaybookServiceTests(unittest.TestCase):
def test_onboarding_bootstrap_preview_uses_control_plane(self) -> None:
ok, message, data = preview_ops_playbook(
{
"playbook_key": "onboarding.bootstrap",
"node_codes": ["mainland-worker-02"],
}
)
self.assertTrue(ok, message)
self.assertEqual("playbook 预览生成成功", message)
self.assertEqual("onboarding.bootstrap", data["playbook"]["key"])
self.assertEqual("control-plane", data["execution_mode"])
self.assertEqual("控制面", data["execution_mode_label"])
self.assertEqual(["control-plane"], data["playbook"]["execution_modes"])
self.assertEqual("控制面", data["playbook"]["default_execution_mode_label"])
self.assertEqual("控制面", data["playbook"]["execution_mode_options"][0]["label"])
self.assertTrue(data["auto_approve"])
self.assertEqual(1, data["step_count"])
step = data["steps"][0]
self.assertEqual("bootstrap", step["step_key"])
self.assertEqual("node.bootstrap", step["template_key"])
self.assertEqual("node.bootstrap", step["action"])
self.assertEqual("control-plane", step["execution_mode"])
self.assertEqual(1, step["expected_jobs"])
def test_scene_logs_key_preview_allows_ssh_execution_mode(self) -> None:
ok, message, data = preview_ops_playbook(
{
"playbook_key": "scene.logs.key",
"node_codes": ["mainland-worker-01"],
"execution_mode": "ssh",
}
)
self.assertTrue(ok, message)
self.assertEqual("playbook 预览生成成功", message)
self.assertEqual("ssh", data["execution_mode"])
self.assertEqual("SSH", data["execution_mode_label"])
self.assertEqual(["remote-agent", "ssh"], data["playbook"]["execution_modes"])
self.assertEqual("ssh", data["steps"][0]["execution_mode"])
self.assertEqual("SSH", data["steps"][0]["execution_mode_label"])
def test_onboarding_bootstrap_preview_rejects_unsupported_execution_mode(self) -> None:
ok, message, data = preview_ops_playbook(
{
"playbook_key": "onboarding.bootstrap",
"node_codes": ["mainland-worker-02"],
"execution_mode": "ssh",
}
)
self.assertFalse(ok)
self.assertEqual("该 playbook 仅支持 control-plane", message)
self.assertEqual(["control-plane"], data["playbook"]["execution_modes"])
@patch("app.services.ops_playbook_service.list_managed_nodes_with_agent_state")
@patch("app.services.ops_playbook_service.list_ops_jobs")
def test_recent_onboarding_bootstrap_run_focus_summary_reflects_agent_state(
self,
mock_list_ops_jobs,
mock_list_managed_nodes_with_agent_state,
) -> None:
mock_list_ops_jobs.return_value = [
{
"id": 31,
"job_code": "ops-bootstrap-31",
"action": "node.bootstrap",
"status": "success",
"target_node_code": "mainland-worker-02",
"created_at": "2026-04-17 10:00:00",
"updated_at": "2026-04-17 10:01:00",
"metadata": {
"playbook": {
"run_code": "pbr-bootstrap-1",
"key": "onboarding.bootstrap",
"title": "生成节点接入工单",
"group_key": "onboarding",
"group_title": "接管验收",
"requested_by": "tester",
"execution_mode": "control-plane",
"step_key": "bootstrap",
"step_title": "生成 Node Agent 接入工单",
"step_order": 1,
"step_count": 1,
"expected_jobs_total": 1,
"target_nodes_total": 1,
"template_key": "node.bootstrap",
}
},
"result": {
"bootstrap_plan": {
"bootstrap_script_name": "bootstrap-node-agent-mainland-worker-02.sh",
}
},
}
]
mock_list_managed_nodes_with_agent_state.return_value = {
"nodes": [
{
"node_code": "mainland-worker-02",
"agent_state": "pending_bootstrap",
"is_managed": True,
}
]
}
payload = get_recent_ops_playbook_runs(limit=10, scan_limit=20)
self.assertEqual(1, len(payload["runs"]))
run = payload["runs"][0]
self.assertEqual("success", run["status"])
self.assertEqual("成功", run["status_label"])
self.assertEqual("控制面", run["execution_mode_label"])
self.assertEqual("warning", run["focus_level"])
self.assertIn("待执行 bootstrap 脚本", run["focus_summary"])
self.assertEqual(run["focus_summary"], run["summary_text"])
self.assertEqual(1, run["steps_total"])
self.assertEqual("pbr-bootstrap-1", run["focus_ref"]["run_code"])
self.assertEqual("生成 Node Agent 接入工单", run["steps"][0]["title"])
self.assertEqual("成功", run["steps"][0]["status_label"])
self.assertTrue(run["steps"][0]["summary_text"])
@patch("app.services.ops_playbook_service.list_managed_nodes_with_agent_state")
@patch("app.services.ops_playbook_service.list_ops_job_events_for_jobs")
@patch("app.services.ops_playbook_service.list_ops_jobs")
def test_playbook_run_events_add_contract_aliases(
self,
mock_list_ops_jobs,
mock_list_ops_job_events_for_jobs,
mock_list_managed_nodes_with_agent_state,
) -> None:
mock_list_managed_nodes_with_agent_state.return_value = {"nodes": []}
mock_list_ops_jobs.return_value = [
{
"id": 41,
"job_code": "ops-inspection-41",
"action": "logs.collect",
"status": "running",
"target_node_code": "mainland-worker-01",
"created_at": "2026-04-18 06:30:00",
"updated_at": "2026-04-18 06:31:00",
"metadata": {
"playbook": {
"run_code": "pbr-inspection-1",
"key": "inspection.standard",
"title": "标准巡检",
"group_key": "diagnostics",
"group_title": "标准巡检",
"requested_by": "tester",
"execution_mode": "remote-agent",
"step_key": "worker_logs",
"step_title": "收集 Worker 日志",
"step_order": 2,
"step_count": 3,
"expected_jobs_total": 3,
"target_nodes_total": 1,
"template_key": "logs.collect",
}
},
}
]
mock_list_ops_job_events_for_jobs.return_value = [
{
"id": 501,
"event_key": "job-event:501",
"job_id": 41,
"node_code": "mainland-worker-01",
"event_type": "job_running",
"level": "info",
"level_label": "信息",
"message": "开始收集 Worker 日志",
"summary": "开始收集 Worker 日志",
"summary_text": "远端 Agent 已开始收集 Worker 日志",
"focus_ref": {
"kind": "ops_job",
"job_id": 41,
"job_code": "ops-inspection-41",
"action": "logs.collect",
"target_node_code": "mainland-worker-01",
"event_key": "job-event:501",
"step_key": "worker_logs",
},
"payload": {},
"created_at": "2026-04-18 06:31:00",
"occurred_at": "2026-04-18 06:31:00",
"ui_intent": {"kind": "job_detail", "job_id": 41, "focus_event_id": 501},
}
]
payload = list_ops_playbook_run_events("pbr-inspection-1", limit=10)
self.assertEqual("inspection.standard", payload["playbook_run"]["playbook_key"])
self.assertEqual(1, len(payload["events"]))
event = payload["events"][0]
self.assertEqual("收口中", event["job_status_label"])
self.assertEqual("远端 Agent 已开始收集 Worker 日志", event["summary_text"])
self.assertEqual("worker_logs", event["focus_ref"]["focus_step_key"])
self.assertEqual("job-event:501", event["focus_ref"]["event_key"])
self.assertEqual("ops_job", event["source_focus_ref"]["kind"])
self.assertEqual("job-event:501", event["source_focus_ref"]["event_key"])
summary = payload["summary"]
self.assertEqual(1, summary["returned_total"])
self.assertEqual({"job_running": 1}, summary["event_type_counts"])
self.assertEqual({"running": 1}, summary["job_status_counts"])
self.assertEqual(10, summary["filters"]["limit"])
if __name__ == "__main__":
unittest.main()

View File

@@ -0,0 +1,253 @@
import unittest
from unittest.mock import patch
from app.services.ops_policy_service import preview_ops_job_policy
class OpsPolicyServiceTests(unittest.TestCase):
@patch("app.services.ops_policy_service.get_cluster_snapshot")
def test_node_bootstrap_preview_requires_control_plane(self, mock_cluster_snapshot) -> None:
mock_cluster_snapshot.return_value = {
"nodes": [
{
"node_code": "mainland-worker-01",
"region": "mainland",
"role": "worker",
"status": "online",
"current_load": 0,
"is_effective_worker": True,
"detect_participating": False,
}
],
"summary": {
"online_worker_nodes": 2,
"dedicated_online_worker_nodes": 1,
"online_control_nodes": 1,
"busy_nodes": [],
},
}
preview = preview_ops_job_policy(
{
"action": "node.bootstrap",
"target_type": "node",
"target_node_code": "mainland-worker-01",
"execution_mode": "remote-agent",
"payload": {},
}
)
self.assertEqual("critical", preview["risk_level"])
self.assertTrue(preview["blocked"])
self.assertTrue(preview["approval_required"])
self.assertIn("node.bootstrap 仅支持 control-plane", " ".join(preview["blocking_reasons"]))
@patch("app.services.ops_policy_service.get_cluster_snapshot")
def test_deploy_release_batch_preview_summarizes_node_risks(self, mock_cluster_snapshot) -> None:
mock_cluster_snapshot.return_value = {
"nodes": [
{
"node_code": "mainland-controller-01",
"region": "mainland",
"role": "control",
"status": "busy",
"current_load": 3,
"is_effective_worker": True,
"detect_participating": False,
},
{
"node_code": "mainland-worker-01",
"region": "mainland",
"role": "worker",
"status": "online",
"current_load": 0,
"is_effective_worker": True,
"detect_participating": False,
},
],
"summary": {
"online_worker_nodes": 2,
"dedicated_online_worker_nodes": 1,
"online_control_nodes": 1,
"busy_nodes": ["mainland-controller-01"],
},
}
preview = preview_ops_job_policy(
{
"action": "deploy.release",
"target_type": "batch",
"target_node_codes": ["mainland-controller-01", "mainland-worker-01"],
"execution_mode": "remote-agent",
"payload": {
"release_version": "2026.04.18-rc1",
"artifact_url": "https://example.com/release.tar.gz",
},
}
)
self.assertEqual("batch", preview["target_type"])
self.assertTrue(preview["blocked"])
self.assertTrue(preview["approval_required"])
self.assertEqual(2, preview["target_summary"]["nodes_total"])
self.assertEqual(1, preview["target_summary"]["blocked_nodes"])
self.assertEqual(1, preview["target_summary"]["approval_nodes"])
self.assertEqual(1, preview["target_summary"]["warning_nodes"])
self.assertEqual(2, len(preview["target_nodes"]))
controller_row = next(item for item in preview["target_nodes"] if item["node_code"] == "mainland-controller-01")
self.assertTrue(controller_row["blocked"])
self.assertTrue(controller_row["approval_required"])
@patch("app.services.ops_policy_service.get_cluster_snapshot")
def test_deploy_release_preview_blocks_missing_artifact_and_wrong_execution_mode(self, mock_cluster_snapshot) -> None:
mock_cluster_snapshot.return_value = {
"nodes": [
{
"node_code": "mainland-worker-01",
"region": "mainland",
"role": "worker",
"status": "online",
"current_load": 0,
"is_effective_worker": True,
"detect_participating": False,
}
],
"summary": {
"online_worker_nodes": 2,
"dedicated_online_worker_nodes": 1,
"online_control_nodes": 1,
"busy_nodes": [],
},
}
preview = preview_ops_job_policy(
{
"action": "deploy.release",
"target_type": "node",
"target_node_code": "mainland-worker-01",
"execution_mode": "local-runtime",
"payload": {
"release_version": "2026.04.18-rc1",
"artifact_url": "",
},
}
)
self.assertTrue(preview["blocked"])
self.assertIn("deploy.release 缺少 artifact_url", " ".join(preview["blocking_reasons"]))
self.assertIn("deploy.release 仅支持 remote-agent 或 ssh", " ".join(preview["blocking_reasons"]))
@patch("app.services.ops_policy_service.get_cluster_snapshot")
def test_deploy_release_preview_allows_ssh_execution_mode(self, mock_cluster_snapshot) -> None:
mock_cluster_snapshot.return_value = {
"nodes": [
{
"node_code": "mainland-worker-01",
"region": "mainland",
"role": "worker",
"status": "online",
"current_load": 0,
"is_effective_worker": True,
"detect_participating": False,
}
],
"summary": {
"online_worker_nodes": 2,
"dedicated_online_worker_nodes": 1,
"online_control_nodes": 1,
"busy_nodes": [],
},
}
preview = preview_ops_job_policy(
{
"action": "deploy.release",
"target_type": "node",
"target_node_code": "mainland-worker-01",
"execution_mode": "ssh",
"payload": {
"release_version": "2026.04.18-rc1",
"artifact_url": "https://example.com/release.tar.gz",
},
}
)
self.assertFalse(any("仅支持 remote-agent 或 ssh" in item for item in preview["blocking_reasons"]))
self.assertTrue(preview["approval_required"])
@patch("app.services.ops_policy_service.get_cluster_snapshot")
def test_service_status_preview_blocks_control_plane_execution_mode(self, mock_cluster_snapshot) -> None:
mock_cluster_snapshot.return_value = {
"nodes": [
{
"node_code": "mainland-worker-01",
"region": "mainland",
"role": "worker",
"status": "online",
"current_load": 0,
"is_effective_worker": True,
"detect_participating": False,
}
],
"summary": {
"online_worker_nodes": 2,
"dedicated_online_worker_nodes": 1,
"online_control_nodes": 1,
"busy_nodes": [],
},
}
preview = preview_ops_job_policy(
{
"action": "service.status",
"target_type": "node",
"target_node_code": "mainland-worker-01",
"execution_mode": "control-plane",
"payload": {"service_name": "domaincheck-worker"},
}
)
self.assertTrue(preview["blocked"])
self.assertIn("service.status 当前不支持 control-plane 执行方式", " ".join(preview["blocking_reasons"]))
@patch("app.services.ops_policy_service.get_cluster_snapshot")
def test_logs_collect_preview_blocks_local_runtime_on_remote_node(self, mock_cluster_snapshot) -> None:
mock_cluster_snapshot.return_value = {
"nodes": [
{
"node_code": "mainland-worker-01",
"region": "mainland",
"role": "worker",
"status": "online",
"current_load": 0,
"is_effective_worker": True,
"detect_participating": False,
}
],
"summary": {
"online_worker_nodes": 2,
"dedicated_online_worker_nodes": 1,
"online_control_nodes": 1,
"busy_nodes": [],
},
}
preview = preview_ops_job_policy(
{
"action": "logs.collect",
"target_type": "node",
"target_node_code": "mainland-worker-01",
"execution_mode": "local-runtime",
"payload": {
"service_name": "domaincheck-worker",
"lines": 120,
},
}
)
self.assertTrue(preview["blocked"])
self.assertIn("local-runtime 仅支持当前控制面本机节点", " ".join(preview["blocking_reasons"]))
if __name__ == "__main__":
unittest.main()

View File

@@ -0,0 +1,89 @@
import io
import tarfile
import tempfile
import unittest
from pathlib import Path
from app.services.ops_release_executor_core import build_remote_release_action_script, execute_release_action
class _BytesResponse:
def __init__(self, data: bytes) -> None:
self._data = data
def __enter__(self):
return self
def __exit__(self, exc_type, exc, tb):
return False
def read(self, *args, **kwargs) -> bytes:
return self._data
def _build_release_archive() -> bytes:
buffer = io.BytesIO()
with tarfile.open(fileobj=buffer, mode="w:gz") as archive:
content = b"hello-release"
info = tarfile.TarInfo(name="bundle/README.txt")
info.size = len(content)
archive.addfile(info, io.BytesIO(content))
return buffer.getvalue()
class OpsReleaseExecutorCoreTests(unittest.TestCase):
def test_execute_release_action_deploys_archive_and_switches_current(self) -> None:
archive_bytes = _build_release_archive()
commands: list[tuple[str, ...]] = []
def run_command(command: list[str], *, timeout: int = 60):
commands.append(tuple(command))
if command[:2] == ["systemctl", "restart"]:
return 0, "", ""
if command[:2] == ["systemctl", "is-active"]:
return 0, "active", ""
return 0, "", ""
with tempfile.TemporaryDirectory() as tmpdir:
ok, message, data = execute_release_action(
{
"release_version": "2026.04.18-rc1",
"artifact_url": "https://example.com/domaincheck.tar.gz",
"install_root": tmpdir,
"restart_services": ["domaincheck-worker"],
"health_check_services": ["domaincheck-worker"],
"health_check_urls": [],
"rollback_on_failure": "true",
"switch_current": "true",
},
run_command=run_command,
default_api_service_name="domaincheck-api",
urlopen_func=lambda request, timeout=120: _BytesResponse(archive_bytes),
user_agent="unit-test/0.1",
)
self.assertTrue(ok, message)
self.assertEqual("release deployed", message)
self.assertEqual("2026.04.18-rc1", data["release_version"])
current_link = Path(tmpdir) / "current"
self.assertTrue(current_link.is_symlink())
release_dir = Path(data["release_dir"])
self.assertTrue(release_dir.exists())
self.assertTrue((release_dir / ".release-meta.json").exists())
self.assertIn(("systemctl", "restart", "domaincheck-worker"), commands)
self.assertIn(("systemctl", "is-active", "domaincheck-worker"), commands)
def test_build_remote_release_action_script_is_valid_python(self) -> None:
script = build_remote_release_action_script(
{
"release_version": "2026.04.18-rc1",
"artifact_url": "https://example.com/domaincheck.tar.gz",
}
)
compile(script, "<remote-release-script>", "exec")
self.assertIn("execute_release_action(", script)
if __name__ == "__main__":
unittest.main()

View File

@@ -0,0 +1,93 @@
import json
import unittest
from datetime import datetime
from app.services.ops_release_service import _serialize_release_row, _serialize_rollout_row
class OpsReleaseServiceContractTests(unittest.TestCase):
def test_serialize_release_row_adds_contract_aliases(self) -> None:
row = (
9,
"2026.04.18+bd6bcb2",
"stable",
"bd6bcb2",
"ready",
"https://example.com/domaincheck.tar.gz",
"sha256:test",
"release notes",
json.dumps(
{
"source": "package",
"manifest": {
"artifact_kind": "tarball",
"package_name": "domaincheck-2026.04.18+bd6bcb2.tgz",
"included_services": ["domaincheck-api", "domaincheck-worker"],
"rollback_hint": "切回上一个 current",
},
}
),
"www",
datetime(2026, 4, 18, 9, 0, 0),
datetime(2026, 4, 18, 9, 5, 0),
datetime(2026, 4, 18, 9, 6, 0),
)
payload = _serialize_release_row(row)
self.assertEqual("已就绪", payload["status_label"])
self.assertIn("已就绪", payload["summary_text"])
self.assertEqual("release_hub", payload["focus_ref"]["kind"])
self.assertEqual(9, payload["focus_ref"]["release_id"])
self.assertEqual("release_detail", payload["focus_ref"]["section"])
self.assertEqual("tarball", payload["manifest"]["artifact_kind"])
self.assertEqual("domaincheck-2026.04.18+bd6bcb2.tgz", payload["manifest"]["package_name"])
self.assertEqual(["domaincheck-api", "domaincheck-worker"], payload["manifest"]["included_services"])
def test_serialize_rollout_row_adds_contract_aliases(self) -> None:
row = (
11,
9,
"rollout-20260418090000-a1b2",
json.dumps({"region": "mainland", "role": "worker"}),
json.dumps(
{
"items": [
{"node_code": "mainland-worker-01", "region": "mainland", "role": "worker"},
{"node_code": "mainland-worker-02", "region": "mainland", "role": "worker"},
]
}
),
json.dumps({"execution_mode": "ssh", "batch_size": 1}),
"running",
1,
2,
2,
1,
json.dumps(
{
"status": "running",
"launched_targets": 1,
"remaining_targets": 1,
"active_jobs": 1,
"current_batch_index": 1,
"batches_total": 2,
}
),
"web-ui",
datetime(2026, 4, 18, 9, 10, 0),
datetime(2026, 4, 18, 9, 11, 0),
)
payload = _serialize_rollout_row(row)
self.assertEqual("执行中", payload["status_label"])
self.assertEqual(["mainland-worker-01", "mainland-worker-02"], payload["target_node_codes"])
self.assertIn("当前已覆盖 1/2 台节点", payload["summary_text"])
self.assertEqual("release_rollout", payload["focus_ref"]["kind"])
self.assertEqual(11, payload["focus_ref"]["rollout_id"])
self.assertEqual("rollout_jobs", payload["focus_ref"]["section"])
if __name__ == "__main__":
unittest.main()

View File

@@ -0,0 +1,432 @@
import unittest
from unittest.mock import patch
from app.services.ops_release_service import (
build_release_execution_mode_recommendation,
build_release_rollout_gate,
build_rollout_target_operational_readiness,
)
class _EmptyCursor:
def __enter__(self):
return self
def __exit__(self, exc_type, exc, tb):
return False
def execute(self, query, params=None) -> None:
self.query = query
self.params = params
def fetchall(self):
return []
def fetchone(self):
return None
class _EmptyConnection:
def __enter__(self):
return self
def __exit__(self, exc_type, exc, tb):
return False
def cursor(self):
return _EmptyCursor()
class OpsReleaseServiceExecutionModeTests(unittest.TestCase):
@patch("app.services.ops_release_service.get_db")
@patch("app.services.ops_agent_service.ensure_ops_agent_schema")
@patch("app.services.ops_agent_service.get_managed_node_onboarding")
@patch("app.services.ops_job_service.list_managed_nodes")
@patch("app.services.cluster_runtime_service.get_cluster_snapshot")
def test_build_rollout_target_operational_readiness_accepts_ssh_ready_nodes(
self,
mock_get_cluster_snapshot,
mock_list_managed_nodes,
mock_get_managed_node_onboarding,
mock_ensure_ops_agent_schema,
mock_get_db,
) -> None:
mock_ensure_ops_agent_schema.return_value = None
mock_get_managed_node_onboarding.return_value = {
"onboarding_stage": {"code": "acceptance_ready", "label": "待接管验收"},
"summary": "Node Agent 已在线,可以直接跑 onboarding.acceptance。",
"recovery_decision": {
"action": "run_acceptance",
"label": "执行接管验收",
"summary": "Node Agent 已在线,可以直接跑 onboarding.acceptance。",
"command_hint": "bash drive_ops_center.sh node-acceptance-run http://127.0.0.1:8100 mainland-worker-01 cli",
"window": "acceptance",
},
}
mock_get_db.return_value = _EmptyConnection()
mock_get_cluster_snapshot.return_value = {
"nodes": [
{
"node_code": "mainland-worker-01",
"region": "mainland",
"role": "worker",
"status": "online",
"current_load": 0,
"is_effective_worker": True,
}
]
}
mock_list_managed_nodes.return_value = [
{
"node_code": "mainland-worker-01",
"region": "mainland",
"role": "worker",
"is_enabled": True,
"ssh_host": "121.204.244.248",
"ssh_user": "root",
"metadata": {},
"last_seen_at": "",
}
]
readiness = build_rollout_target_operational_readiness(
[{"node_code": "mainland-worker-01", "region": "mainland", "role": "worker"}],
execution_mode="ssh",
)
self.assertEqual("ssh", readiness["execution_mode"])
self.assertEqual(1, readiness["summary"]["ssh_ready_nodes"])
self.assertEqual(1, readiness["summary"]["execution_ready_nodes"])
self.assertEqual([], readiness["blocking_reasons"])
self.assertTrue(readiness["rows"][0]["ssh_ready"])
self.assertTrue(readiness["rows"][0]["execution_ready"])
self.assertTrue(readiness["rows"][0]["agent_managed"])
self.assertEqual("ssh", readiness["rows"][0]["execution_mode"])
self.assertEqual("SSH", readiness["rows"][0]["execution_mode_label"])
self.assertEqual("medium", readiness["rows"][0]["risk_level"])
self.assertEqual("run_standard_inspection", readiness["rows"][0]["recommended_action_code"])
self.assertEqual("acceptance_ready", readiness["rows"][0]["onboarding_stage_code"])
self.assertEqual("run_acceptance", readiness["rows"][0]["recovery_action"])
self.assertEqual(1, readiness["summary"]["onboarding_acceptance_ready_nodes"])
@patch("app.services.ops_release_service.get_db")
@patch("app.services.ops_agent_service.ensure_ops_agent_schema")
@patch("app.services.ops_agent_service.get_managed_node_onboarding")
@patch("app.services.ops_job_service.list_managed_nodes")
@patch("app.services.cluster_runtime_service.get_cluster_snapshot")
def test_build_rollout_target_operational_readiness_adds_release_context_and_focus_ref(
self,
mock_get_cluster_snapshot,
mock_list_managed_nodes,
mock_get_managed_node_onboarding,
mock_ensure_ops_agent_schema,
mock_get_db,
) -> None:
mock_ensure_ops_agent_schema.return_value = None
mock_get_managed_node_onboarding.return_value = {
"onboarding_stage": {"code": "ready", "label": "已接管"},
"summary": "当前节点无需额外恢复动作。",
"recovery_decision": {
"action": "noop",
"label": "当前无需额外恢复动作",
"summary": "当前节点暂无需要执行的接管恢复动作。",
"command_hint": "",
"window": "none",
},
}
mock_get_db.return_value = _EmptyConnection()
mock_get_cluster_snapshot.return_value = {
"nodes": [
{
"node_code": "mainland-worker-01",
"region": "mainland",
"role": "worker",
"status": "online",
"current_load": 0,
"is_effective_worker": True,
"metadata": {"current_release_version": "2026.04.11+1921318"},
}
]
}
mock_list_managed_nodes.return_value = [
{
"node_code": "mainland-worker-01",
"region": "mainland",
"role": "worker",
"is_enabled": True,
"ssh_host": "121.204.244.248",
"ssh_user": "root",
"metadata": {"current_release_version": "2026.04.11+1921318"},
"last_seen_at": "2026-04-18 08:00:00",
}
]
readiness = build_rollout_target_operational_readiness(
[{"node_code": "mainland-worker-01", "region": "mainland", "role": "worker"}],
execution_mode="remote-agent",
desired_release={"id": 18, "release_version": "2026.04.18+bd6bcb2", "channel": "stable"},
)
row = readiness["rows"][0]
self.assertEqual("2026.04.11+1921318", row["current_release_version"])
self.assertEqual("2026.04.18+bd6bcb2", row["desired_release_version"])
self.assertEqual("release_hub", row["focus_ref"]["kind"])
self.assertEqual("default_rollout_gate", row["focus_ref"]["section"])
self.assertEqual("mainland-worker-01", row["focus_ref"]["node_code"])
self.assertEqual("noop", row["recovery_action"])
@patch("app.services.ops_release_service.get_db")
@patch("app.services.ops_agent_service.ensure_ops_agent_schema")
@patch("app.services.ops_agent_service.get_managed_node_onboarding")
@patch("app.services.ops_job_service.list_managed_nodes")
@patch("app.services.cluster_runtime_service.get_cluster_snapshot")
def test_build_rollout_target_operational_readiness_prefers_recovery_decision_for_blocked_nodes(
self,
mock_get_cluster_snapshot,
mock_list_managed_nodes,
mock_get_managed_node_onboarding,
mock_ensure_ops_agent_schema,
mock_get_db,
) -> None:
mock_ensure_ops_agent_schema.return_value = None
mock_get_db.return_value = _EmptyConnection()
mock_get_cluster_snapshot.return_value = {
"nodes": [
{
"node_code": "mainland-worker-02",
"region": "mainland",
"role": "worker",
"status": "online",
"current_load": 0,
"is_effective_worker": True,
}
]
}
mock_list_managed_nodes.return_value = [
{
"node_code": "mainland-worker-02",
"region": "mainland",
"role": "worker",
"is_enabled": True,
"ssh_host": "121.204.244.249",
"ssh_user": "root",
"metadata": {},
"last_seen_at": "",
}
]
mock_get_managed_node_onboarding.return_value = {
"onboarding_stage": {"code": "pending_bootstrap", "label": "待接入"},
"summary": "已签发有效 Agent Token但节点尚未完成 register/heartbeat。",
"recovery_decision": {
"action": "bootstrap_run",
"label": "签发接入工单",
"summary": "当前节点仍处于接入阶段,建议先签发 onboarding.bootstrap。",
"command_hint": "bash drive_ops_center.sh node-bootstrap-run http://127.0.0.1:8100 mainland-worker-02 cli",
"window": "bootstrap",
},
}
readiness = build_rollout_target_operational_readiness(
[{"node_code": "mainland-worker-02", "region": "mainland", "role": "worker"}],
execution_mode="remote-agent",
)
row = readiness["rows"][0]
self.assertEqual("bootstrap_run", row["recovery_action"])
self.assertEqual("bootstrap_run", row["recommended_action_code"])
self.assertEqual(1, readiness["summary"]["onboarding_bootstrap_pending_nodes"])
self.assertTrue(
any("onboarding.bootstrap" in item for item in readiness["recommendations"])
)
@patch("app.services.ops_release_service.get_db")
@patch("app.services.ops_agent_service.ensure_ops_agent_schema")
@patch("app.services.ops_agent_service.get_managed_node_onboarding")
@patch("app.services.ops_job_service.list_managed_nodes")
@patch("app.services.cluster_runtime_service.get_cluster_snapshot")
def test_build_rollout_target_operational_readiness_blocks_nodes_with_dead_letter_queue(
self,
mock_get_cluster_snapshot,
mock_list_managed_nodes,
mock_get_managed_node_onboarding,
mock_ensure_ops_agent_schema,
mock_get_db,
) -> None:
mock_ensure_ops_agent_schema.return_value = None
mock_get_db.return_value = _EmptyConnection()
mock_get_managed_node_onboarding.return_value = {
"onboarding_stage": {"code": "ready", "label": "已接管"},
"summary": "当前节点无需额外恢复动作。",
"recovery_decision": {
"action": "noop",
"label": "当前无需额外恢复动作",
"summary": "当前节点暂无需要执行的接管恢复动作。",
"command_hint": "",
"window": "none",
},
}
mock_get_cluster_snapshot.return_value = {
"nodes": [
{
"node_code": "mainland-worker-03",
"region": "mainland",
"role": "worker",
"status": "online",
"current_load": 0,
"is_effective_worker": True,
}
]
}
mock_list_managed_nodes.return_value = [
{
"node_code": "mainland-worker-03",
"region": "mainland",
"role": "worker",
"is_enabled": True,
"ssh_host": "121.204.244.250",
"ssh_user": "root",
"metadata": {
"delivery_queue": {
"state": "dead_letter",
"label": "死信 1",
"reason": "当前存在 1 条死信记录",
"pending_count": 0,
"dead_letter_count": 1,
"oldest_dead_letter_at": "2026-04-18 09:01:00",
}
},
"last_seen_at": "2026-04-18 09:05:00",
}
]
readiness = build_rollout_target_operational_readiness(
[{"node_code": "mainland-worker-03", "region": "mainland", "role": "worker"}],
execution_mode="remote-agent",
)
row = readiness["rows"][0]
self.assertEqual("dead_letter", row["delivery_queue_state"])
self.assertEqual(1, row["delivery_queue_dead_letter_count"])
self.assertTrue(
any("死信" in item for item in row["blocking_reasons"])
)
self.assertEqual(1, readiness["summary"]["queue_dead_letter_nodes"])
self.assertTrue(
any("死信回执" in item for item in readiness["blocking_reasons"])
)
self.assertTrue(
any("死信队列" in item for item in readiness["recommendations"])
)
@patch("app.services.ops_release_service.build_rollout_target_operational_readiness")
def test_build_release_rollout_gate_uses_execution_ready_count_for_ssh(self, mock_build_readiness) -> None:
mock_build_readiness.return_value = {
"execution_mode": "ssh",
"summary": {
"nodes_total": 2,
"execution_ready_nodes": 1,
"inspection_healthy_nodes": 2,
},
"rows": [],
"blocking_reasons": ["有 1 台目标节点尚未配置完整 SSH 入口,当前不适合直接 SSH 发布。"],
"warning_reasons": [],
"recommendations": [],
}
gate = build_release_rollout_gate(
{
"id": 11,
"release_version": "2026.04.18-rc1",
"status": "ready",
"artifact_url": "https://example.com/domaincheck.tar.gz",
},
[{"node_code": "mainland-worker-01"}, {"node_code": "mainland-worker-02"}],
execution_mode="ssh",
)
self.assertEqual("ssh", gate["execution_mode"])
self.assertEqual("SSH", gate["execution_mode_label"])
self.assertEqual("blocked", gate["status"])
self.assertIn("1/2 台 SSH 就绪", gate["summary"])
@patch("app.services.ops_release_service.build_rollout_target_operational_readiness")
def test_build_release_rollout_gate_uses_human_label_for_remote_agent(self, mock_build_readiness) -> None:
mock_build_readiness.return_value = {
"execution_mode": "remote-agent",
"summary": {
"nodes_total": 2,
"execution_ready_nodes": 1,
"inspection_healthy_nodes": 2,
},
"rows": [],
"blocking_reasons": ["有 1 台目标节点 Agent 未在线,当前不适合直接 remote-agent 发布。"],
"warning_reasons": [],
"recommendations": [],
}
gate = build_release_rollout_gate(
{
"id": 12,
"release_version": "2026.04.18-rc1",
"status": "ready",
"artifact_url": "https://example.com/domaincheck.tar.gz",
},
[{"node_code": "mainland-worker-01"}, {"node_code": "mainland-worker-02"}],
execution_mode="remote-agent",
)
self.assertEqual("remote-agent", gate["execution_mode"])
self.assertEqual("远端 Agent", gate["execution_mode_label"])
self.assertIn("1/2 台 远端 Agent 就绪", gate["summary"])
@patch("app.services.ops_release_service.build_release_rollout_gate")
def test_build_release_execution_mode_recommendation_prefers_ssh_when_gate_is_better(self, mock_build_gate) -> None:
def _fake_gate(release, target_nodes, *, execution_mode="remote-agent"):
if execution_mode == "ssh":
return {
"status": "ready",
"status_label": "可发 Rollout",
"summary": "SSH 路径已收口",
"execution_mode": "ssh",
"execution_mode_label": "SSH",
"blocking_reasons": [],
"warning_reasons": [],
"operational_readiness": {
"summary": {"nodes_total": 2, "execution_ready_nodes": 2},
"rows": [],
},
}
return {
"status": "blocked",
"status_label": "存在阻断",
"summary": "remote-agent 尚未收口",
"execution_mode": "remote-agent",
"execution_mode_label": "远端 Agent",
"blocking_reasons": ["Agent 未就绪"],
"warning_reasons": [],
"operational_readiness": {
"summary": {"nodes_total": 2, "execution_ready_nodes": 1},
"rows": [],
},
}
mock_build_gate.side_effect = _fake_gate
recommendation = build_release_execution_mode_recommendation(
{
"id": 18,
"release_version": "2026.04.18-rc2",
"status": "ready",
"artifact_url": "https://example.com/domaincheck.tar.gz",
},
[{"node_code": "mainland-worker-01"}, {"node_code": "mainland-worker-02"}],
)
self.assertEqual("ssh", recommendation["recommended_mode"])
self.assertEqual("SSH", recommendation["recommended_mode_label"])
self.assertEqual("ready", recommendation["recommended_gate"]["status"])
self.assertIn("SSH", recommendation["reason"])
if __name__ == "__main__":
unittest.main()

View File

@@ -0,0 +1,212 @@
import unittest
from app.services.ops_release_service import (
_build_release_launchpad_status,
_compact_latest_smart_rollout_preview,
)
class OpsReleaseServiceLaunchpadTests(unittest.TestCase):
def test_launchpad_status_blocks_when_package_missing(self) -> None:
status = _build_release_launchpad_status(
package={"available": False},
latest_package_preview={},
worker_preview={},
control_preview={},
)
self.assertEqual("blocked", status["status"])
self.assertEqual("release_package", status["recommended_action_code"])
self.assertEqual(status["summary"], status["summary_text"])
self.assertEqual("release_hub", status["focus_ref"]["kind"])
self.assertEqual("release_launchpad", status["focus_ref"]["section"])
def test_launchpad_status_blocks_when_rollout_preview_has_blockers(self) -> None:
status = _build_release_launchpad_status(
package={"available": True},
latest_package_preview={},
worker_preview={"policy_preview": {"blocked": True}},
control_preview={"policy_preview": {"blocked": False}},
)
self.assertEqual("blocked", status["status"])
self.assertEqual("fix_rollout_blockers", status["recommended_action_code"])
self.assertEqual(status["summary"], status["summary_text"])
def test_launchpad_status_blocks_when_final_signoff_missing(self) -> None:
status = _build_release_launchpad_status(
package={
"available": True,
"final_release_ok": False,
"final_release_report_available": True,
"final_release_report_ok": True,
"final_release_report_matches_latest": False,
"final_release_report_stale_reason": "final_release_report does not match latest_release: package_name",
"final_release_gate_decision": "ready",
"final_release_gate_blocked_reasons": [],
},
latest_package_preview={},
worker_preview={"smart_rollout": {"available": True}, "policy_preview": {}},
control_preview={"smart_rollout": {"available": True}, "policy_preview": {}},
)
self.assertEqual("blocked", status["status"])
self.assertEqual("release_prepare", status["recommended_action_code"])
self.assertIn("final_release_report does not match latest_release: package_name", status["summary"])
self.assertIn("release-prepare", status["recommended_command"])
self.assertEqual(status["summary"], status["summary_text"])
def test_launchpad_status_prefers_runtime_refresh_when_runtime_schema_is_stale(self) -> None:
status = _build_release_launchpad_status(
package={"available": True, "final_release_ok": True},
latest_package_preview={},
worker_preview={"smart_rollout": {"available": True}, "policy_preview": {}},
control_preview={"smart_rollout": {"available": True}, "policy_preview": {}},
runtime_build_info={
"runtime_schema_stale": True,
"route_surface_complete": True,
"route_surface_missing_keys": ["ops_node_handover_bootstrap_plan"],
},
)
self.assertEqual("attention", status["status"])
self.assertEqual("api-restart", status["recommended_action_code"])
self.assertEqual("待刷新运行时", status["status_label"])
self.assertTrue(status["runtime_refresh_required"])
self.assertTrue(status["runtime_schema_stale"])
self.assertIn("runtime-refresh-recover", status["recommended_command"])
self.assertEqual(status["summary"], status["summary_text"])
def test_launchpad_status_marks_attention_when_confirmation_required(self) -> None:
status = _build_release_launchpad_status(
package={"available": True},
latest_package_preview={},
worker_preview={"requires_confirmation": True, "policy_preview": {}},
control_preview={"policy_preview": {}},
)
self.assertEqual("attention", status["status"])
self.assertEqual("review_smart_rollout_preview", status["recommended_action_code"])
self.assertEqual("release_launchpad", status["focus_ref"]["section"])
def test_launchpad_status_ready_when_worker_rollout_available(self) -> None:
status = _build_release_launchpad_status(
package={"available": True},
latest_package_preview={},
worker_preview={"smart_rollout": {"available": True, "execution_mode": "ssh", "execution_mode_label": "SSH"}, "policy_preview": {}},
control_preview={"smart_rollout": {"available": False}, "policy_preview": {}},
)
self.assertEqual("ready", status["status"])
self.assertEqual("publish_latest_worker", status["recommended_action_code"])
self.assertEqual("ssh", status["recommended_execution_mode"])
self.assertEqual("SSH", status["recommended_execution_mode_label"])
self.assertEqual(status["summary"], status["summary_text"])
def test_launchpad_status_requires_managed_nodes_when_no_targets_available(self) -> None:
status = _build_release_launchpad_status(
package={"available": True},
latest_package_preview={},
worker_preview={"smart_rollout": {"available": False}, "policy_preview": {}},
control_preview={"smart_rollout": {"available": False}, "policy_preview": {}},
)
self.assertEqual("attention", status["status"])
self.assertEqual("fix_managed_nodes", status["recommended_action_code"])
self.assertEqual("release_hub", status["focus_ref"]["kind"])
def test_launchpad_status_prefers_bootstrap_recovery_when_gap_row_has_recovery_action(self) -> None:
status = _build_release_launchpad_status(
package={"available": True},
latest_package_preview={},
worker_preview={
"smart_rollout": {"available": False},
"policy_preview": {
"blocked": True,
"release_gate": {
"operational_readiness": {
"rows": [
{
"node_code": "mainland-worker-02",
"recovery_action": "bootstrap_run",
"recovery_label": "签发接入工单",
"recovery_summary": "当前节点仍处于接入阶段,建议先签发 onboarding.bootstrap。",
}
]
}
},
},
},
control_preview={"smart_rollout": {"available": False}, "policy_preview": {}},
)
self.assertEqual("blocked", status["status"])
self.assertEqual("bootstrap_run", status["recommended_action_code"])
self.assertEqual("mainland-worker-02", status["recommended_target_node_code"])
self.assertEqual("签发接入工单", status["recommended_recovery_label"])
self.assertEqual("当前节点仍处于接入阶段,建议先签发 onboarding.bootstrap。", status["recommended_recovery_summary"])
self.assertEqual(1, status["onboarding_bootstrap_pending_nodes"])
self.assertEqual(0, status["onboarding_acceptance_ready_nodes"])
self.assertEqual(1, status["onboarding_gap_nodes_total"])
self.assertIn("node-bootstrap-run", status["recommended_command"])
def test_compact_latest_smart_rollout_preview_adds_rollout_preview_alias(self) -> None:
preview = _compact_latest_smart_rollout_preview(
"智能 Rollout 预检通过",
{
"release": {
"id": 18,
"release_version": "2026.04.18+bd6bcb2",
"channel": "stable",
"status": "ready",
"artifact_url": "https://example.com/domaincheck.tar.gz",
},
"package": {
"available": True,
"package_name": "domaincheck-2026.04.18+bd6bcb2",
"package_type": "tar.gz",
"sha256": "sha256:test",
"metadata": {
"manifest": {
"included_services": ["domaincheck-api", "domaincheck-worker"],
}
},
},
"smart_rollout": {
"available": True,
"execution_mode": "remote-agent",
"execution_mode_label": "远端 Agent",
"summary": "已按 Worker 预选 2 台节点。",
"target_selector": {"node_codes": ["mainland-worker-01", "mainland-worker-02"]},
"target_nodes": [
{
"node_code": "mainland-worker-01",
"region": "mainland",
"role": "worker",
"delivery_queue_state": "healthy",
},
{
"node_code": "mainland-worker-02",
"region": "mainland",
"role": "worker",
"delivery_queue_state": "retrying",
},
],
},
"preview": {
"target_summary": {"nodes_total": 2},
"batch_plan": {"batches_total": 2, "first_batch_size": 1},
"release_gate": {"status": "ready", "summary": "当前可推进。"},
},
},
)
self.assertEqual("remote-agent", preview["rollout_preview"]["execution_mode"])
self.assertEqual(2, preview["rollout_preview"]["target_nodes_total"])
self.assertEqual(2, preview["rollout_preview"]["expected_batches_total"])
self.assertEqual(2, preview["rollout_preview"]["expected_jobs_total"])
self.assertEqual("release_hub", preview["rollout_preview"]["focus_ref"]["kind"])
self.assertEqual("healthy", preview["smart_rollout"]["target_nodes"][0]["delivery_queue_state"])
self.assertEqual("retrying", preview["smart_rollout"]["target_nodes"][1]["delivery_queue_state"])
self.assertEqual("rollout_preview", preview["rollout_preview"]["focus_ref"]["section"])
self.assertEqual(["domaincheck-api", "domaincheck-worker"], preview["package"]["manifest"]["included_services"])

View File

@@ -0,0 +1,915 @@
import json
import os
import tempfile
import unittest
from pathlib import Path
from unittest.mock import patch
from app.services.ops_release_service import (
build_smart_release_rollout_plan,
create_release_and_smart_rollout_from_latest_package,
create_release_from_latest_package,
create_smart_release_rollout,
get_latest_release_package_metadata,
get_release_package_archive_path,
get_release_package_sha256_path,
)
class OpsReleaseServicePackageMetadataTests(unittest.TestCase):
def test_get_latest_release_package_metadata_reads_latest_linux_package(self) -> None:
with tempfile.TemporaryDirectory() as tmpdir:
release_root = Path(tmpdir)
package_name = "domaincheck_release_20260418_120000"
staging_path = release_root / package_name
archive_path = release_root / f"{package_name}.tar.gz"
sha256_path = release_root / f"{package_name}.sha256.txt"
staging_path.mkdir(parents=True, exist_ok=True)
archive_path.write_bytes(b"linux-package")
sha256_path.write_text("sha256=test-sha256", encoding="utf-8")
(staging_path / "release_manifest.json").write_text(
json.dumps(
{
"package_name": package_name,
"package_type": "tar.gz",
"included": {
"docs": ["08_domainCheck_交付说明.md"],
"scripts": ["package_domain_release.sh", "verify_domain_release.sh"],
},
},
ensure_ascii=False,
),
encoding="utf-8",
)
(release_root / "latest_release.json").write_text(
json.dumps(
{
"package_name": package_name,
"package_type": "tar.gz",
"generated_at": "2026-04-18T12:00:00",
"staging_path": str(staging_path),
"archive_path": str(archive_path),
"sha256_path": str(sha256_path),
"sha256": "test-sha256",
"smoke_test_ok": True,
},
ensure_ascii=False,
),
encoding="utf-8",
)
(release_root / "final_release_report.json").write_text(
json.dumps(
{
"ok": True,
"latest_release": {
"package_name": package_name,
"archive_path": str(archive_path),
"sha256": "test-sha256",
},
"release_gate": {
"decision": "ready",
"blocked_reasons": [],
"verify_ok": True,
"smoke_test_ok": True,
},
},
ensure_ascii=False,
),
encoding="utf-8",
)
with patch.dict(os.environ, {"DOMAINCHECK_RELEASE_ROOT": str(release_root)}):
data = get_latest_release_package_metadata()
self.assertTrue(data["available"])
self.assertEqual(package_name, data["package_name"])
self.assertEqual("tar.gz", data["package_type"])
self.assertTrue(data["archive_present"])
self.assertTrue(data["sha256_present"])
self.assertTrue(data["staging_present"])
self.assertEqual("test-sha256", data["sha256"])
self.assertEqual(f"/api/v1/ops/releases/packages/{package_name}/download", data["download_path"])
self.assertEqual(package_name, data["release_version_suggestion"])
self.assertIn("自检通过", data["notes_suggestion"])
self.assertTrue(data["final_release_ok"])
self.assertTrue(data["final_release_report_available"])
self.assertTrue(data["final_release_report_matches_latest"])
self.assertEqual("ready", data["final_release_gate_decision"])
self.assertEqual([], data["final_release_gate_blocked_reasons"])
self.assertEqual(package_name, data["metadata"]["manifest"]["package_name"])
def test_get_release_package_paths_support_metadata_fallback_for_zip(self) -> None:
with tempfile.TemporaryDirectory() as tmpdir:
release_root = Path(tmpdir)
package_name = "domaincheck_release_windows_demo"
package_dir = release_root / "exports"
package_dir.mkdir(parents=True, exist_ok=True)
archive_path = package_dir / f"{package_name}.zip"
sha256_path = package_dir / f"{package_name}.sha256.txt"
archive_path.write_bytes(b"zip-package")
sha256_path.write_text("sha256=zip-sha256", encoding="utf-8")
(release_root / "latest_release.json").write_text(
json.dumps(
{
"package_name": package_name,
"package_type": "zip",
"generated_at": "2026-04-18T12:10:00",
"zip_path": str(archive_path),
"sha256_path": str(sha256_path),
"sha256": "zip-sha256",
"smoke_test_ok": False,
},
ensure_ascii=False,
),
encoding="utf-8",
)
with patch.dict(os.environ, {"DOMAINCHECK_RELEASE_ROOT": str(release_root)}):
resolved_archive = get_release_package_archive_path(package_name)
resolved_sha256 = get_release_package_sha256_path(package_name)
metadata = get_latest_release_package_metadata()
self.assertEqual(archive_path, resolved_archive)
self.assertEqual(sha256_path, resolved_sha256)
self.assertEqual("zip", metadata["package_type"])
self.assertEqual(archive_path.name, metadata["archive_filename"])
self.assertFalse(metadata["smoke_test_ok"])
def test_get_latest_release_package_metadata_returns_unavailable_when_missing(self) -> None:
with tempfile.TemporaryDirectory() as tmpdir:
with patch.dict(os.environ, {"DOMAINCHECK_RELEASE_ROOT": tmpdir}):
data = get_latest_release_package_metadata()
self.assertFalse(data["available"])
self.assertIn("latest_release.json", data["reason"])
def test_get_latest_release_package_metadata_marks_stale_final_release_report(self) -> None:
with tempfile.TemporaryDirectory() as tmpdir:
release_root = Path(tmpdir)
package_name = "domaincheck_release_20260418_130000"
archive_path = release_root / f"{package_name}.tar.gz"
sha256_path = release_root / f"{package_name}.sha256.txt"
archive_path.write_bytes(b"linux-package")
sha256_path.write_text("sha256=test-sha256", encoding="utf-8")
(release_root / "latest_release.json").write_text(
json.dumps(
{
"package_name": package_name,
"package_type": "tar.gz",
"archive_path": str(archive_path),
"sha256_path": str(sha256_path),
"sha256": "test-sha256",
"smoke_test_ok": True,
},
ensure_ascii=False,
),
encoding="utf-8",
)
(release_root / "final_release_report.json").write_text(
json.dumps(
{
"ok": True,
"latest_release": {
"package_name": "domaincheck_release_20260418_old",
"archive_path": str(release_root / "domaincheck_release_20260418_old.tar.gz"),
"sha256": "old-sha256",
},
"release_gate": {
"decision": "ready",
"blocked_reasons": [],
"verify_ok": True,
"smoke_test_ok": True,
},
},
ensure_ascii=False,
),
encoding="utf-8",
)
with patch.dict(os.environ, {"DOMAINCHECK_RELEASE_ROOT": str(release_root)}):
data = get_latest_release_package_metadata()
self.assertTrue(data["available"])
self.assertFalse(data["final_release_ok"])
self.assertTrue(data["final_release_report_available"])
self.assertTrue(data["final_release_report_ok"])
self.assertFalse(data["final_release_report_matches_latest"])
self.assertIn("package_name", data["final_release_report_stale_reason"])
@patch("app.services.ops_release_service.create_release")
@patch("app.services.ops_release_service._get_release_by_version")
@patch("app.services.ops_release_service.get_latest_release_package_metadata")
def test_create_release_from_latest_package_builds_payload(
self,
mock_get_latest_release_package_metadata,
mock_get_release_by_version,
mock_create_release,
) -> None:
mock_get_latest_release_package_metadata.return_value = {
"available": True,
"package_name": "domaincheck_release_20260418_120000",
"release_version_suggestion": "domaincheck_release_20260418_120000",
"download_path": "/api/v1/ops/releases/packages/domaincheck_release_20260418_120000/download",
"sha256_download_path": "/api/v1/ops/releases/packages/domaincheck_release_20260418_120000/sha256",
"sha256": "test-sha256",
"commit_sha": "bd6bcb2",
"notes_suggestion": "导入自本机发布包 domaincheck_release_20260418_120000.tar.gz自检通过",
"smoke_test_ok": True,
"final_release_ok": True,
"final_release_report_available": True,
"final_release_report_ok": True,
"final_release_report_matches_latest": True,
"final_release_gate_decision": "ready",
"final_release_gate_blocked_reasons": [],
"metadata": {
"package_name": "domaincheck_release_20260418_120000",
"package_type": "tar.gz",
},
}
mock_get_release_by_version.return_value = {}
mock_create_release.return_value = (
True,
"Release 已创建",
{
"release": {
"id": 7,
"release_version": "domaincheck_release_20260418_120000",
}
},
)
ok, message, data = create_release_from_latest_package(
{
"channel": "stable",
"status": "ready",
"created_by": "test-suite",
},
control_plane_base_url="https://ops.example.com/api/v1",
)
self.assertTrue(ok)
self.assertEqual("Release 已创建", message)
mock_create_release.assert_called_once()
create_payload = mock_create_release.call_args.args[0]
self.assertEqual("domaincheck_release_20260418_120000", create_payload["release_version"])
self.assertEqual("bd6bcb2", create_payload["commit_sha"])
self.assertEqual(
"https://ops.example.com/api/v1/ops/releases/packages/domaincheck_release_20260418_120000/download",
create_payload["artifact_url"],
)
self.assertEqual("test-sha256", create_payload["checksum"])
self.assertEqual("latest_release_package", create_payload["metadata"]["created_from"])
self.assertEqual(
"https://ops.example.com/api/v1/ops/releases/packages/domaincheck_release_20260418_120000/download",
create_payload["metadata"]["package_source"]["download_url"],
)
self.assertFalse(data["deduplicated"])
@patch("app.services.ops_release_service.create_release")
@patch("app.services.ops_release_service._get_release_by_version")
@patch("app.services.ops_release_service.get_latest_release_package_metadata")
def test_create_release_from_latest_package_is_idempotent(
self,
mock_get_latest_release_package_metadata,
mock_get_release_by_version,
mock_create_release,
) -> None:
mock_get_latest_release_package_metadata.return_value = {
"available": True,
"package_name": "domaincheck_release_20260418_120000",
"release_version_suggestion": "domaincheck_release_20260418_120000",
"smoke_test_ok": True,
"final_release_ok": True,
"final_release_report_available": True,
"final_release_report_ok": True,
"final_release_report_matches_latest": True,
"final_release_gate_decision": "ready",
"final_release_gate_blocked_reasons": [],
}
mock_get_release_by_version.return_value = {
"id": 9,
"release_version": "domaincheck_release_20260418_120000",
}
ok, message, data = create_release_from_latest_package({}, control_plane_base_url="https://ops.example.com")
self.assertTrue(ok)
self.assertEqual("Release 已存在,按幂等返回", message)
self.assertTrue(data["deduplicated"])
self.assertEqual(9, data["release"]["id"])
mock_create_release.assert_not_called()
@patch("app.services.ops_release_service.create_release")
@patch("app.services.ops_release_service._get_release_by_version")
@patch("app.services.ops_release_service.get_latest_release_package_metadata")
def test_create_release_from_latest_package_blocks_when_final_signoff_missing(
self,
mock_get_latest_release_package_metadata,
mock_get_release_by_version,
mock_create_release,
) -> None:
mock_get_latest_release_package_metadata.return_value = {
"available": True,
"package_name": "domaincheck_release_20260418_130000",
"release_version_suggestion": "domaincheck_release_20260418_130000",
"download_path": "/api/v1/ops/releases/packages/domaincheck_release_20260418_130000/download",
"sha256_download_path": "/api/v1/ops/releases/packages/domaincheck_release_20260418_130000/sha256",
"sha256": "test-sha256",
"commit_sha": "bd6bcb2",
"smoke_test_ok": False,
"final_release_ok": False,
"final_release_report_available": False,
"final_release_report_ok": None,
"final_release_report_matches_latest": None,
"final_release_gate_decision": "",
"final_release_gate_blocked_reasons": [],
"metadata": {
"package_name": "domaincheck_release_20260418_130000",
"package_type": "tar.gz",
},
}
mock_get_release_by_version.return_value = {}
ok, message, data = create_release_from_latest_package(
{
"channel": "stable",
"status": "ready",
"created_by": "test-suite",
},
control_plane_base_url="https://ops.example.com/api/v1",
)
self.assertFalse(ok)
self.assertIn("最终签收", message)
self.assertFalse(data["final_release_gate"]["ready"])
self.assertIn("smoke_test_failed_or_skipped", data["final_release_gate"]["blocked_reasons"])
self.assertIn("final_release_report_missing", data["final_release_gate"]["blocked_reasons"])
mock_create_release.assert_not_called()
@patch("app.services.ops_release_service.create_release")
@patch("app.services.ops_release_service._get_release_by_version")
@patch("app.services.ops_release_service.get_latest_release_package_metadata")
def test_create_release_from_latest_package_blocks_when_final_report_is_stale(
self,
mock_get_latest_release_package_metadata,
mock_get_release_by_version,
mock_create_release,
) -> None:
mock_get_latest_release_package_metadata.return_value = {
"available": True,
"package_name": "domaincheck_release_20260418_130001",
"release_version_suggestion": "domaincheck_release_20260418_130001",
"download_path": "/api/v1/ops/releases/packages/domaincheck_release_20260418_130001/download",
"sha256_download_path": "/api/v1/ops/releases/packages/domaincheck_release_20260418_130001/sha256",
"sha256": "test-sha256",
"commit_sha": "bd6bcb2",
"smoke_test_ok": True,
"final_release_ok": False,
"final_release_report_available": True,
"final_release_report_ok": True,
"final_release_report_matches_latest": False,
"final_release_report_stale_reason": "final_release_report does not match latest_release: package_name",
"final_release_gate_decision": "ready",
"final_release_gate_blocked_reasons": [],
"metadata": {
"package_name": "domaincheck_release_20260418_130001",
"package_type": "tar.gz",
},
}
mock_get_release_by_version.return_value = {
"id": 19,
"release_version": "domaincheck_release_20260418_130001",
}
ok, message, data = create_release_from_latest_package(
{"created_by": "test-suite"},
control_plane_base_url="https://ops.example.com/api/v1",
)
self.assertFalse(ok)
self.assertIn("最终签收", message)
self.assertFalse(data["final_release_gate"]["ready"])
self.assertEqual(
"final_release_report does not match latest_release: package_name",
data["final_release_gate"]["stale_reason"],
)
self.assertIn("final_release_report_not_matching_latest", data["final_release_gate"]["blocked_reasons"])
self.assertEqual(19, data["release"]["id"])
mock_create_release.assert_not_called()
@patch("app.services.ops_agent_service.list_managed_nodes_with_agent_state")
def test_build_smart_release_rollout_plan_prefers_remote_ready_workers(
self,
mock_list_managed_nodes_with_agent_state,
) -> None:
mock_list_managed_nodes_with_agent_state.return_value = {
"nodes": [
{
"node_code": "mainland-worker-01",
"region": "mainland",
"role": "worker",
"is_enabled": True,
"is_managed": True,
"is_agent_online": True,
"cluster_status": "online",
"cluster_current_load": 0,
"cluster_is_effective_worker": True,
"cluster_detect_participating": False,
},
{
"node_code": "mainland-controller-01",
"region": "mainland",
"role": "control",
"is_enabled": True,
"is_managed": True,
"is_agent_online": True,
"cluster_status": "online",
"cluster_current_load": 1,
"cluster_is_effective_worker": True,
"cluster_detect_participating": True,
},
{
"node_code": "mainland-worker-02",
"region": "mainland",
"role": "worker",
"is_enabled": True,
"is_managed": True,
"is_agent_online": False,
"cluster_status": "online",
"cluster_current_load": 0,
"cluster_is_effective_worker": True,
"cluster_detect_participating": False,
},
]
}
worker_plan = build_smart_release_rollout_plan("worker")
control_plan = build_smart_release_rollout_plan("control")
self.assertTrue(worker_plan["available"])
self.assertEqual("worker", worker_plan["mode"])
self.assertEqual(["mainland-worker-01"], worker_plan["target_selector"]["node_codes"])
self.assertEqual(["domaincheck-worker"], worker_plan["policy"]["deploy_payload"]["restart_services"])
self.assertTrue(control_plan["available"])
self.assertEqual("control", control_plan["mode"])
self.assertEqual(["mainland-controller-01"], control_plan["target_selector"]["node_codes"])
self.assertIn("domaincheck-api", control_plan["policy"]["deploy_payload"]["restart_services"])
self.assertEqual("remote-agent", worker_plan["policy"]["execution_mode"])
@patch("app.services.ops_agent_service.list_managed_nodes_with_agent_state")
def test_build_smart_release_rollout_plan_falls_back_to_ssh_ready_workers(
self,
mock_list_managed_nodes_with_agent_state,
) -> None:
mock_list_managed_nodes_with_agent_state.return_value = {
"nodes": [
{
"node_code": "mainland-worker-01",
"region": "mainland",
"role": "worker",
"is_enabled": True,
"is_managed": True,
"is_agent_online": False,
"has_ssh_access": True,
"cluster_status": "online",
"cluster_current_load": 0,
"cluster_is_effective_worker": True,
"cluster_detect_participating": False,
},
{
"node_code": "mainland-worker-02",
"region": "mainland",
"role": "worker",
"is_enabled": True,
"is_managed": True,
"is_agent_online": False,
"has_ssh_access": True,
"cluster_status": "busy",
"cluster_current_load": 1,
"cluster_is_effective_worker": True,
"cluster_detect_participating": True,
},
]
}
worker_plan = build_smart_release_rollout_plan("worker")
self.assertTrue(worker_plan["available"])
self.assertEqual("worker", worker_plan["mode"])
self.assertEqual("ssh", worker_plan["execution_mode"])
self.assertEqual("SSH", worker_plan["execution_mode_label"])
self.assertEqual(
["mainland-worker-01", "mainland-worker-02"],
worker_plan["target_selector"]["node_codes"],
)
self.assertEqual("ssh", worker_plan["policy"]["execution_mode"])
@patch("app.services.ops_agent_service.list_managed_nodes_with_agent_state")
def test_build_smart_release_rollout_plan_excludes_dead_letter_nodes_from_ready_candidates(
self,
mock_list_managed_nodes_with_agent_state,
) -> None:
mock_list_managed_nodes_with_agent_state.return_value = {
"nodes": [
{
"node_code": "mainland-worker-01",
"region": "mainland",
"role": "worker",
"is_enabled": True,
"is_managed": True,
"is_agent_online": True,
"has_ssh_access": True,
"delivery_queue_state": "dead_letter",
"cluster_status": "online",
"cluster_current_load": 0,
"cluster_is_effective_worker": True,
"cluster_detect_participating": False,
},
{
"node_code": "mainland-worker-02",
"region": "mainland",
"role": "worker",
"is_enabled": True,
"is_managed": True,
"is_agent_online": True,
"has_ssh_access": True,
"delivery_queue_state": "healthy",
"cluster_status": "online",
"cluster_current_load": 0,
"cluster_is_effective_worker": True,
"cluster_detect_participating": False,
},
]
}
worker_plan = build_smart_release_rollout_plan("worker")
self.assertTrue(worker_plan["available"])
self.assertEqual(["mainland-worker-02"], worker_plan["target_selector"]["node_codes"])
self.assertEqual("healthy", worker_plan["target_nodes"][0]["delivery_queue_state"])
@patch("app.services.ops_agent_service.list_managed_nodes_with_agent_state")
def test_build_smart_release_rollout_plan_returns_unavailable_when_only_dead_letter_nodes_exist(
self,
mock_list_managed_nodes_with_agent_state,
) -> None:
mock_list_managed_nodes_with_agent_state.return_value = {
"nodes": [
{
"node_code": "mainland-worker-01",
"region": "mainland",
"role": "worker",
"is_enabled": True,
"is_managed": True,
"is_agent_online": True,
"has_ssh_access": True,
"delivery_queue_state": "dead_letter",
"cluster_status": "online",
"cluster_current_load": 0,
"cluster_is_effective_worker": True,
"cluster_detect_participating": False,
}
]
}
worker_plan = build_smart_release_rollout_plan("worker")
self.assertFalse(worker_plan["available"])
self.assertIn("死信回执", worker_plan["reason"])
self.assertEqual("dead_letter", worker_plan["target_nodes"][0]["delivery_queue_state"])
@patch("app.services.ops_release_service.create_release_rollout")
@patch("app.services.ops_policy_service.preview_ops_job_policy")
@patch("app.services.ops_agent_service.list_managed_nodes_with_agent_state")
@patch("app.services.ops_release_service.create_release_from_latest_package")
def test_create_release_and_smart_rollout_stops_when_latest_package_gate_blocks(
self,
mock_create_release_from_latest_package,
mock_list_managed_nodes_with_agent_state,
mock_preview_ops_job_policy,
mock_create_release_rollout,
) -> None:
mock_create_release_from_latest_package.return_value = (
False,
"最新发布包尚未完成最终签收,暂不允许创建 Release 或发起基于最新包的智能 Rollout。",
{
"package": {
"available": True,
"package_name": "domaincheck_release_20260418_blocked",
"final_release_ok": False,
},
"final_release_gate": {
"ready": False,
"status": "blocked",
"blocked_reasons": ["final_release_report_missing"],
},
},
)
ok, message, data = create_release_and_smart_rollout_from_latest_package(
{"mode": "worker"},
control_plane_base_url="https://ops.example.com",
)
self.assertFalse(ok)
self.assertIn("最终签收", message)
self.assertFalse(data["final_release_gate"]["ready"])
mock_list_managed_nodes_with_agent_state.assert_not_called()
mock_preview_ops_job_policy.assert_not_called()
mock_create_release_rollout.assert_not_called()
@patch("app.services.ops_release_service.create_release_rollout")
@patch("app.services.ops_policy_service.preview_ops_job_policy")
@patch("app.services.ops_agent_service.list_managed_nodes_with_agent_state")
@patch("app.services.ops_release_service.create_release_from_latest_package")
def test_create_release_and_smart_rollout_requires_confirmation_before_rollout(
self,
mock_create_release_from_latest_package,
mock_list_managed_nodes_with_agent_state,
mock_preview_ops_job_policy,
mock_create_release_rollout,
) -> None:
mock_create_release_from_latest_package.return_value = (
True,
"Release 已创建",
{
"release": {"id": 21, "release_version": "domaincheck_release_20260418_120000"},
"deduplicated": False,
},
)
mock_list_managed_nodes_with_agent_state.return_value = {
"nodes": [
{
"node_code": "mainland-controller-01",
"region": "mainland",
"role": "control",
"is_enabled": True,
"is_managed": True,
"is_agent_online": True,
"cluster_status": "online",
"cluster_current_load": 0,
"cluster_is_effective_worker": True,
"cluster_detect_participating": False,
}
]
}
mock_preview_ops_job_policy.return_value = {
"blocked": False,
"approval_required": True,
"warnings": ["本次 rollout 覆盖 control 节点,建议审批。"],
"release_gate": {"status_label": "待补巡检", "summary": "建议先确认控制面健康。"},
}
ok, message, data = create_release_and_smart_rollout_from_latest_package(
{"mode": "control"},
control_plane_base_url="https://ops.example.com",
)
self.assertTrue(ok)
self.assertEqual("Release 已创建,智能 Rollout 需要确认", message)
self.assertTrue(data["requires_confirmation"])
self.assertFalse(data["rollout_created"])
self.assertEqual(["mainland-controller-01"], data["smart_rollout"]["target_selector"]["node_codes"])
mock_create_release_rollout.assert_not_called()
@patch("app.services.ops_release_service.create_release_rollout")
@patch("app.services.ops_policy_service.preview_ops_job_policy")
@patch("app.services.ops_agent_service.list_managed_nodes_with_agent_state")
@patch("app.services.ops_release_service.create_release_from_latest_package")
def test_create_release_and_smart_rollout_creates_rollout_after_confirmation(
self,
mock_create_release_from_latest_package,
mock_list_managed_nodes_with_agent_state,
mock_preview_ops_job_policy,
mock_create_release_rollout,
) -> None:
mock_create_release_from_latest_package.return_value = (
True,
"Release 已创建",
{
"release": {"id": 22, "release_version": "domaincheck_release_20260418_120000"},
"deduplicated": True,
},
)
mock_list_managed_nodes_with_agent_state.return_value = {
"nodes": [
{
"node_code": "mainland-worker-01",
"region": "mainland",
"role": "worker",
"is_enabled": True,
"is_managed": True,
"is_agent_online": True,
"cluster_status": "online",
"cluster_current_load": 0,
"cluster_is_effective_worker": True,
"cluster_detect_participating": False,
}
]
}
mock_preview_ops_job_policy.return_value = {
"blocked": False,
"approval_required": False,
"warnings": [],
"release_gate": {"status_label": "可发 Rollout", "summary": "当前可推进。"},
}
mock_create_release_rollout.return_value = (
True,
"Release rollout 已创建",
{
"rollout": {"id": 5, "release_id": 22, "rollout_code": "rollout-20260418-demo"},
"jobs": [],
},
)
ok, message, data = create_release_and_smart_rollout_from_latest_package(
{"mode": "worker", "confirm_risky": True},
control_plane_base_url="https://ops.example.com",
)
self.assertTrue(ok)
self.assertEqual("Release rollout 已创建", message)
self.assertTrue(data["rollout_created"])
self.assertEqual(5, data["rollout"]["id"])
mock_create_release_rollout.assert_called_once()
rollout_payload = mock_create_release_rollout.call_args.args[1]
self.assertEqual(["mainland-worker-01"], rollout_payload["target_selector"]["node_codes"])
self.assertEqual("remote-agent", rollout_payload["policy"]["execution_mode"])
@patch("app.services.ops_release_service.create_release_rollout")
@patch("app.services.ops_policy_service.preview_ops_job_policy")
@patch("app.services.ops_agent_service.list_managed_nodes_with_agent_state")
@patch("app.services.ops_release_service.create_release_from_latest_package")
def test_create_release_and_smart_rollout_uses_ssh_when_only_ssh_path_is_ready(
self,
mock_create_release_from_latest_package,
mock_list_managed_nodes_with_agent_state,
mock_preview_ops_job_policy,
mock_create_release_rollout,
) -> None:
mock_create_release_from_latest_package.return_value = (
True,
"Release 已创建",
{
"release": {"id": 23, "release_version": "domaincheck_release_20260418_ssh"},
"deduplicated": True,
},
)
mock_list_managed_nodes_with_agent_state.return_value = {
"nodes": [
{
"node_code": "mainland-worker-01",
"region": "mainland",
"role": "worker",
"is_enabled": True,
"is_managed": True,
"is_agent_online": False,
"has_ssh_access": True,
"cluster_status": "online",
"cluster_current_load": 0,
"cluster_is_effective_worker": True,
"cluster_detect_participating": False,
}
]
}
mock_preview_ops_job_policy.return_value = {
"blocked": False,
"approval_required": False,
"warnings": [],
"release_gate": {"status_label": "可发 Rollout", "summary": "当前可推进。"},
}
mock_create_release_rollout.return_value = (
True,
"Release rollout 已创建",
{
"rollout": {"id": 6, "release_id": 23, "rollout_code": "rollout-20260418-ssh"},
"jobs": [],
},
)
ok, message, data = create_release_and_smart_rollout_from_latest_package(
{"mode": "worker", "confirm_risky": True},
control_plane_base_url="https://ops.example.com",
)
self.assertTrue(ok)
self.assertEqual("Release rollout 已创建", message)
self.assertTrue(data["rollout_created"])
rollout_payload = mock_create_release_rollout.call_args.args[1]
self.assertEqual("ssh", rollout_payload["policy"]["execution_mode"])
@patch("app.services.ops_release_service.get_release")
@patch("app.services.ops_release_service.create_release_rollout")
@patch("app.services.ops_policy_service.preview_ops_job_policy")
@patch("app.services.ops_agent_service.list_managed_nodes_with_agent_state")
def test_create_smart_release_rollout_for_existing_release_requires_confirmation(
self,
mock_list_managed_nodes_with_agent_state,
mock_preview_ops_job_policy,
mock_create_release_rollout,
mock_get_release,
) -> None:
mock_get_release.return_value = {
"id": 41,
"release_version": "domaincheck_release_existing",
"channel": "stable",
}
mock_list_managed_nodes_with_agent_state.return_value = {
"nodes": [
{
"node_code": "mainland-controller-01",
"region": "mainland",
"role": "control",
"is_enabled": True,
"is_managed": True,
"is_agent_online": True,
"cluster_status": "online",
"cluster_current_load": 0,
"cluster_is_effective_worker": True,
"cluster_detect_participating": False,
}
]
}
mock_preview_ops_job_policy.return_value = {
"blocked": False,
"approval_required": True,
"warnings": ["控制面节点建议审批。"],
"release_gate": {"status_label": "待审批", "summary": "需要审批。"},
}
ok, message, data = create_smart_release_rollout(41, {"mode": "control"})
self.assertTrue(ok)
self.assertEqual("智能 Rollout 需要确认", message)
self.assertFalse(data["release_created"])
self.assertTrue(data["requires_confirmation"])
self.assertFalse(data["rollout_created"])
self.assertEqual(["mainland-controller-01"], data["smart_rollout"]["target_selector"]["node_codes"])
mock_create_release_rollout.assert_not_called()
@patch("app.services.ops_release_service.get_release")
@patch("app.services.ops_release_service.create_release_rollout")
@patch("app.services.ops_policy_service.preview_ops_job_policy")
@patch("app.services.ops_agent_service.list_managed_nodes_with_agent_state")
def test_create_smart_release_rollout_for_existing_release_creates_rollout(
self,
mock_list_managed_nodes_with_agent_state,
mock_preview_ops_job_policy,
mock_create_release_rollout,
mock_get_release,
) -> None:
mock_get_release.return_value = {
"id": 42,
"release_version": "domaincheck_release_existing",
"channel": "stable",
}
mock_list_managed_nodes_with_agent_state.return_value = {
"nodes": [
{
"node_code": "mainland-worker-01",
"region": "mainland",
"role": "worker",
"is_enabled": True,
"is_managed": True,
"is_agent_online": True,
"cluster_status": "online",
"cluster_current_load": 0,
"cluster_is_effective_worker": True,
"cluster_detect_participating": False,
}
]
}
mock_preview_ops_job_policy.return_value = {
"blocked": False,
"approval_required": False,
"warnings": [],
"release_gate": {"status_label": "可发 Rollout", "summary": "当前可推进。"},
}
mock_create_release_rollout.return_value = (
True,
"Release rollout 已创建",
{
"rollout": {"id": 8, "release_id": 42, "rollout_code": "rollout-20260418-existing"},
"jobs": [],
},
)
ok, message, data = create_smart_release_rollout(42, {"mode": "worker", "confirm_risky": True})
self.assertTrue(ok)
self.assertEqual("Release rollout 已创建", message)
self.assertFalse(data["release_created"])
self.assertTrue(data["rollout_created"])
self.assertEqual(8, data["rollout"]["id"])
mock_create_release_rollout.assert_called_once()
rollout_payload = mock_create_release_rollout.call_args.args[1]
self.assertEqual(["mainland-worker-01"], rollout_payload["target_selector"]["node_codes"])
self.assertEqual("remote-agent", rollout_payload["policy"]["execution_mode"])
if __name__ == "__main__":
unittest.main()

View File

@@ -0,0 +1,118 @@
import subprocess
import tempfile
import unittest
from pathlib import Path
from unittest.mock import patch
from app.services.ops_runtime_executor_service import execute_local_support_action, execute_ssh_action
class OpsRuntimeExecutorServiceTests(unittest.TestCase):
@patch("app.services.ops_runtime_executor_service.subprocess.run")
def test_execute_ssh_action_parses_structured_json_result(self, mock_run) -> None:
mock_run.return_value = subprocess.CompletedProcess(
args=["ssh"],
returncode=0,
stdout='login banner\n{"ok": true, "message": "health snapshot collected", "result": {"checks": {"api": {"state": "active"}}}}\n',
stderr="",
)
ok, message, data = execute_ssh_action(
{
"node_code": "mainland-worker-01",
"ssh_host": "121.204.244.248",
"ssh_port": 22,
"ssh_user": "root",
},
"health.snapshot",
{},
)
self.assertTrue(ok)
self.assertEqual("health snapshot collected", message)
self.assertEqual("active", data["checks"]["api"]["state"])
self.assertEqual("ssh", data["transport"]["executor"])
self.assertEqual("121.204.244.248", data["transport"]["ssh_host"])
issued_command = mock_run.call_args.args[0]
self.assertEqual("ssh", issued_command[0])
self.assertIn("root@121.204.244.248", issued_command)
@patch("app.services.ops_runtime_executor_service.subprocess.run")
def test_execute_local_support_action_collects_logs_with_payload(self, mock_run) -> None:
mock_run.return_value = subprocess.CompletedProcess(
args=["journalctl"],
returncode=0,
stdout="log line 1\nlog line 2",
stderr="",
)
ok, message, data = execute_local_support_action(
"logs.collect",
{
"service_name": "domaincheck-worker",
"lines": 180,
},
)
self.assertTrue(ok)
self.assertEqual("log line 1\nlog line 2", message)
self.assertEqual("domaincheck-worker", data["service_name"])
self.assertEqual(180, data["lines"])
issued_command = mock_run.call_args.args[0]
self.assertEqual(
["journalctl", "-u", "domaincheck-worker", "-n", "180", "--no-pager"],
issued_command,
)
@patch("app.services.ops_runtime_executor_service.subprocess.run")
def test_execute_local_support_action_falls_back_to_worker_log_file(self, mock_run) -> None:
mock_run.return_value = subprocess.CompletedProcess(
args=["journalctl"],
returncode=1,
stdout="",
stderr="No journal files were opened due to insufficient permissions.",
)
with tempfile.TemporaryDirectory() as temp_dir:
log_path = Path(temp_dir) / "detect_worker.log"
log_path.write_text("fallback-1\nfallback-2\n", encoding="utf-8")
with patch("app.services.ops_action_executor_core.settings.domain_root", temp_dir):
ok, message, data = execute_local_support_action(
"logs.collect",
{
"service_name": "domaincheck-worker",
"lines": 180,
},
)
self.assertTrue(ok)
self.assertEqual("fallback-1\nfallback-2", message)
self.assertEqual("file_fallback", data["collection_source"])
self.assertTrue(data["fallback_used"])
@patch("app.services.ops_runtime_executor_service.subprocess.run")
def test_execute_local_support_action_restarts_service(self, mock_run) -> None:
mock_run.return_value = subprocess.CompletedProcess(
args=["systemctl"],
returncode=0,
stdout="",
stderr="",
)
ok, message, data = execute_local_support_action(
"service.restart",
{
"service_name": "domaincheck-worker",
},
)
self.assertTrue(ok)
self.assertEqual("domaincheck-worker restart completed", message)
self.assertEqual("domaincheck-worker", data["service_name"])
self.assertEqual("restart", data["systemctl_action"])
issued_command = mock_run.call_args.args[0]
self.assertEqual(["systemctl", "restart", "domaincheck-worker"], issued_command)
if __name__ == "__main__":
unittest.main()

View File

@@ -0,0 +1,75 @@
import unittest
from unittest.mock import MagicMock, patch
import app.services.ops_agent_service as ops_agent_service
import app.services.ops_job_service as ops_job_service
class OpsSchemaInitTests(unittest.TestCase):
@patch("app.services.ops_job_service.get_db")
def test_ensure_ops_schema_uses_advisory_lock_and_skips_after_ready(self, mock_get_db) -> None:
conn = MagicMock()
cursor = MagicMock()
db_ctx = MagicMock()
cursor_ctx = MagicMock()
db_ctx.__enter__.return_value = conn
db_ctx.__exit__.return_value = False
cursor_ctx.__enter__.return_value = cursor
cursor_ctx.__exit__.return_value = False
conn.cursor.return_value = cursor_ctx
mock_get_db.return_value = db_ctx
previous_ready = ops_job_service._OPS_SCHEMA_READY
ops_job_service._OPS_SCHEMA_READY = False
try:
ops_job_service.ensure_ops_schema()
ops_job_service.ensure_ops_schema()
finally:
ops_job_service._OPS_SCHEMA_READY = previous_ready
self.assertEqual(1, mock_get_db.call_count)
cursor.execute.assert_any_call(
"SELECT pg_advisory_xact_lock(%s)",
(ops_job_service._OPS_SCHEMA_ADVISORY_LOCK_KEY,),
)
cursor.execute.assert_any_call(ops_job_service._OPS_SCHEMA_SQL)
conn.commit.assert_called_once()
@patch("app.services.ops_agent_service.ensure_ops_schema")
@patch("app.services.ops_agent_service.get_db")
def test_ensure_ops_agent_schema_uses_advisory_lock_and_skips_after_ready(
self,
mock_get_db,
mock_ensure_ops_schema,
) -> None:
conn = MagicMock()
cursor = MagicMock()
db_ctx = MagicMock()
cursor_ctx = MagicMock()
db_ctx.__enter__.return_value = conn
db_ctx.__exit__.return_value = False
cursor_ctx.__enter__.return_value = cursor
cursor_ctx.__exit__.return_value = False
conn.cursor.return_value = cursor_ctx
mock_get_db.return_value = db_ctx
previous_ready = ops_agent_service._OPS_AGENT_SCHEMA_READY
ops_agent_service._OPS_AGENT_SCHEMA_READY = False
try:
ops_agent_service.ensure_ops_agent_schema()
ops_agent_service.ensure_ops_agent_schema()
finally:
ops_agent_service._OPS_AGENT_SCHEMA_READY = previous_ready
self.assertEqual(2, mock_ensure_ops_schema.call_count)
self.assertEqual(1, mock_get_db.call_count)
cursor.execute.assert_any_call(
"SELECT pg_advisory_xact_lock(%s)",
(ops_agent_service._OPS_AGENT_SCHEMA_ADVISORY_LOCK_KEY,),
)
cursor.execute.assert_any_call(ops_agent_service._AGENT_SCHEMA_SQL)
conn.commit.assert_called_once()
if __name__ == "__main__":
unittest.main()

View File

@@ -0,0 +1,607 @@
import unittest
from typing import Optional
from unittest.mock import patch
from app.services.ops_service import get_ops_activity_stream
class OpsServiceActivityTests(unittest.TestCase):
@staticmethod
def _runtime_status(detect: Optional[dict] = None) -> dict:
return {"detect": dict(detect or {})}
@patch("app.services.ops_service.get_ops_runbook")
@patch("app.services.ops_service.list_release_rollouts")
@patch("app.services.ops_service.list_ops_jobs")
@patch("app.services.ops_service.get_recent_ops_playbook_runs")
@patch("app.services.ops_service.list_managed_nodes_with_agent_state")
def test_activity_stream_uses_job_detail_for_bootstrap_jobs(
self,
mock_list_managed_nodes_with_agent_state,
mock_get_recent_ops_playbook_runs,
mock_list_ops_jobs,
mock_list_release_rollouts,
mock_get_ops_runbook,
) -> None:
mock_get_ops_runbook.return_value = {"control_sequences": []}
mock_list_managed_nodes_with_agent_state.return_value = {
"nodes": [
{
"node_code": "mainland-worker-02",
"agent_state": "pending_bootstrap",
"agent_state_label": "待接入",
}
]
}
mock_get_recent_ops_playbook_runs.return_value = {"runs": []}
mock_list_ops_jobs.return_value = [
{
"id": 51,
"job_code": "ops-bootstrap-51",
"action": "node.bootstrap",
"status": "success",
"target_node_code": "mainland-worker-02",
"created_at": "2026-04-17 10:00:00",
"updated_at": "2026-04-17 10:01:00",
"result": {
"bootstrap_plan": {
"bootstrap_script_name": "bootstrap-node-agent-mainland-worker-02.sh",
}
},
}
]
mock_list_release_rollouts.return_value = []
with patch(
"app.services.ops_service.get_runtime_status",
return_value=self._runtime_status(),
):
payload = get_ops_activity_stream(limit=10, scan_limit=20)
self.assertEqual(1, len(payload["items"]))
item = payload["items"][0]
self.assertEqual("ops_job", item["kind"])
self.assertEqual("节点接入工单", item["title"])
self.assertEqual("job_detail", item["ui_intent"]["kind"])
self.assertEqual("job_detail", item["ui_intent_kind"])
self.assertEqual("控制面", item["execution_mode_label"])
self.assertEqual("成功", item["status_label"])
self.assertIn("尚未回连 Agent", item["summary"])
self.assertEqual(item["summary"], item["summary_text"])
self.assertIn("方式 控制面", item["meta_text"])
self.assertIn("bootstrap-node-agent-mainland-worker-02.sh", item["meta_text"])
self.assertEqual(["mainland-worker-02"], item["target_node_codes"])
self.assertEqual(51, item["focus_ref"]["job_id"])
self.assertEqual("node.bootstrap", item["focus_ref"]["action"])
self.assertEqual("ops_agent_protocol", item["contract_navigation"]["primary_contract_key"])
self.assertIn("ops_agent_protocol", item["contract_navigation"]["contract_keys"])
@patch("app.services.ops_service.get_ops_runbook")
@patch("app.services.ops_service.list_release_rollouts")
@patch("app.services.ops_service.list_ops_jobs")
@patch("app.services.ops_service.get_recent_ops_playbook_runs")
@patch("app.services.ops_service.list_managed_nodes_with_agent_state")
def test_activity_stream_includes_runbook_sequences_after_live_events(
self,
mock_list_managed_nodes_with_agent_state,
mock_get_recent_ops_playbook_runs,
mock_list_ops_jobs,
mock_list_release_rollouts,
mock_get_ops_runbook,
) -> None:
mock_list_managed_nodes_with_agent_state.return_value = {"nodes": []}
mock_get_recent_ops_playbook_runs.return_value = {
"runs": [
{
"run_code": "playbook-001",
"playbook_title": "标准巡检",
"status": "running",
"updated_at": "2026-04-18 09:11:00",
}
]
}
mock_list_ops_jobs.return_value = []
mock_list_release_rollouts.return_value = []
mock_get_ops_runbook.return_value = {
"control_sequences": [
{
"key": "release_progression",
"title": "版本发布与放量",
"status": "warning",
"status_label": "待处理",
"reason": "当前没有足够的执行面接管覆盖。",
"target_scope_label": "Release / Rollout",
"target_node_codes": ["mainland-controller-01"],
"primary_action_code": "fix_managed_nodes",
"primary_resolution": {
"ok": True,
"driver_action_code": "fix_managed_nodes",
"driver_action_label": "补执行面接管",
"driver_node_codes": ["mainland-controller-01"],
"message": "当前建议先补执行面接管。",
"resolved_at": "2026-04-18 09:12:00",
},
}
]
}
with patch(
"app.services.ops_service.get_runtime_status",
return_value=self._runtime_status(),
):
payload = get_ops_activity_stream(limit=10, scan_limit=20)
self.assertEqual(2, len(payload["items"]))
self.assertEqual("playbook_run", payload["items"][0]["kind"])
self.assertEqual("runbook_sequence", payload["items"][1]["kind"])
runbook_item = payload["items"][1]
self.assertEqual("release_progression", runbook_item["sequence_key"])
self.assertEqual("focus_runbook_sequence", runbook_item["ui_intent"]["kind"])
self.assertEqual("fix_managed_nodes", runbook_item["ui_intent"]["driver_action_code"])
self.assertEqual("attention", runbook_item["status"])
self.assertIn("当前建议动作:补执行面接管", runbook_item["summary"])
self.assertIn("release_progression", runbook_item["meta_text"])
self.assertEqual("ops_playbook_contract", runbook_item["contract_navigation"]["primary_contract_key"])
self.assertIn("ops_driver_contract", runbook_item["contract_navigation"]["contract_keys"])
@patch("app.services.ops_service.get_ops_runbook")
@patch("app.services.ops_service.list_release_rollouts")
@patch("app.services.ops_service.list_ops_jobs")
@patch("app.services.ops_service.get_recent_ops_playbook_runs")
@patch("app.services.ops_service.list_managed_nodes_with_agent_state")
def test_activity_stream_surfaces_execution_mode_labels_for_jobs_and_playbooks(
self,
mock_list_managed_nodes_with_agent_state,
mock_get_recent_ops_playbook_runs,
mock_list_ops_jobs,
mock_list_release_rollouts,
mock_get_ops_runbook,
) -> None:
mock_list_managed_nodes_with_agent_state.return_value = {"nodes": []}
mock_get_recent_ops_playbook_runs.return_value = {
"runs": [
{
"run_code": "playbook-ssh-001",
"playbook_title": "关键现场日志",
"group_title": "执行现场",
"status": "running",
"updated_at": "2026-04-18 09:11:00",
"execution_mode": "ssh",
"execution_mode_label": "SSH",
"steps": [{"status": "running"}],
"step_count": 1,
"jobs_total": 2,
"target_node_codes": ["mainland-worker-01"],
}
]
}
mock_list_ops_jobs.return_value = [
{
"id": 61,
"job_code": "ops-ssh-61",
"action": "logs.collect",
"status": "running",
"execution_mode": "ssh",
"execution_mode_label": "SSH",
"requested_by": "web-ui",
"target_node_code": "mainland-worker-01",
"target_type": "node",
"risk_level": "low",
"updated_at": "2026-04-18 09:10:00",
"created_at": "2026-04-18 09:09:00",
}
]
mock_list_release_rollouts.return_value = []
mock_get_ops_runbook.return_value = {"control_sequences": []}
with patch(
"app.services.ops_service.get_runtime_status",
return_value=self._runtime_status(),
):
payload = get_ops_activity_stream(limit=10, scan_limit=20)
self.assertEqual(2, len(payload["items"]))
self.assertEqual("SSH", payload["items"][0]["execution_mode_label"])
self.assertIn("执行 SSH", payload["items"][0]["meta_text"])
self.assertEqual("SSH", payload["items"][1]["execution_mode_label"])
self.assertIn("方式 SSH", payload["items"][1]["summary"])
self.assertIn("执行 SSH", payload["items"][1]["meta_text"])
@patch("app.services.ops_service.get_ops_runbook")
@patch("app.services.ops_service.list_release_rollouts")
@patch("app.services.ops_service.list_ops_jobs")
@patch("app.services.ops_service.get_recent_ops_playbook_runs")
@patch("app.services.ops_service.list_managed_nodes_with_agent_state")
def test_activity_stream_prefers_job_result_summary_and_focus_ref(
self,
mock_list_managed_nodes_with_agent_state,
mock_get_recent_ops_playbook_runs,
mock_list_ops_jobs,
mock_list_release_rollouts,
mock_get_ops_runbook,
) -> None:
mock_list_managed_nodes_with_agent_state.return_value = {"nodes": []}
mock_get_recent_ops_playbook_runs.return_value = {"runs": []}
mock_list_ops_jobs.return_value = [
{
"id": 71,
"job_code": "ops-agent-71",
"action": "logs.collect",
"status": "success",
"execution_mode": "remote-agent",
"execution_mode_label": "远端 Agent",
"requested_by": "codex",
"target_node_code": "mainland-worker-01",
"target_type": "node",
"risk_level": "low",
"updated_at": "2026-04-18 09:20:00",
"created_at": "2026-04-18 09:19:00",
"focus_ref": {
"kind": "ops_job",
"job_id": 71,
"job_code": "ops-agent-71",
"action": "logs.collect",
"target_node_code": "mainland-worker-01",
},
"result": {
"summary_text": "远端 Agent 已回传 120 行 Worker 日志,可直接继续分析。",
"focus_ref": {
"kind": "ops_job",
"job_id": 71,
"job_code": "ops-agent-71",
"action": "logs.collect",
"target_node_code": "mainland-worker-01",
"event_key": "job-event:888",
"step_key": "worker_logs",
},
},
}
]
mock_list_release_rollouts.return_value = []
mock_get_ops_runbook.return_value = {"control_sequences": []}
with patch(
"app.services.ops_service.get_runtime_status",
return_value=self._runtime_status(),
):
payload = get_ops_activity_stream(limit=10, scan_limit=20)
self.assertEqual(1, len(payload["items"]))
item = payload["items"][0]
self.assertEqual("ops_job", item["kind"])
self.assertEqual("远端 Agent 已回传 120 行 Worker 日志,可直接继续分析。", item["summary"])
self.assertEqual("ops_job", item["focus_ref"]["kind"])
self.assertEqual("job-event:888", item["focus_ref"]["event_key"])
self.assertEqual("worker_logs", item["focus_ref"]["step_key"])
self.assertIn("执行 远端 Agent", item["meta_text"])
self.assertEqual("ops_observability_contract", item["contract_navigation"]["primary_contract_key"])
self.assertIn("ops_stack_diagnosis_contract", item["contract_navigation"]["contract_keys"])
@patch("app.services.ops_service.get_ops_runbook")
@patch("app.services.ops_service.list_release_rollouts")
@patch("app.services.ops_service.list_ops_jobs")
@patch("app.services.ops_service.get_recent_ops_playbook_runs")
@patch("app.services.ops_service.list_managed_nodes_with_agent_state")
def test_activity_stream_marks_job_attention_when_start_delivery_failed_locally(
self,
mock_list_managed_nodes_with_agent_state,
mock_get_recent_ops_playbook_runs,
mock_list_ops_jobs,
mock_list_release_rollouts,
mock_get_ops_runbook,
) -> None:
mock_list_managed_nodes_with_agent_state.return_value = {"nodes": []}
mock_get_recent_ops_playbook_runs.return_value = {"runs": []}
mock_list_ops_jobs.return_value = [
{
"id": 81,
"job_code": "ops-agent-81",
"action": "logs.collect",
"status": "running",
"execution_mode": "remote-agent",
"execution_mode_label": "远端 Agent",
"requested_by": "codex",
"target_node_code": "mainland-worker-01",
"target_type": "node",
"risk_level": "low",
"updated_at": "2026-04-18 10:20:00",
"created_at": "2026-04-18 10:19:00",
"focus_ref": {
"kind": "ops_job",
"job_id": 81,
"job_code": "ops-agent-81",
"action": "logs.collect",
"target_node_code": "mainland-worker-01",
},
"result": {
"summary_text": "节点端已经收到采集任务。",
"start_delivery_state": "failed_local",
"start_delivery_error": "temporary offline",
"source_focus_ref": {
"kind": "ops_job_event",
"job_id": 81,
"job_code": "ops-agent-81",
"event_key": "job-event:81",
"target_node_code": "mainland-worker-01",
},
},
}
]
mock_list_release_rollouts.return_value = []
mock_get_ops_runbook.return_value = {"control_sequences": []}
with patch(
"app.services.ops_service.get_runtime_status",
return_value=self._runtime_status(),
):
payload = get_ops_activity_stream(limit=10, scan_limit=20)
self.assertEqual(1, len(payload["items"]))
item = payload["items"][0]
self.assertEqual("ops_job", item["kind"])
self.assertEqual("attention", item["status"])
self.assertEqual("running", item["job_status"])
self.assertEqual("failed_local", item["start_delivery_state"])
self.assertEqual("ops_job_start_delivery_failed_local", item["problem_code"])
self.assertIn("开始回执未成功送达控制面", item["summary"])
self.assertIn("temporary offline", item["summary"])
self.assertIn("开始回执异常", item["meta_text"])
self.assertEqual("ops_job_event", item["source_focus_ref"]["kind"])
self.assertEqual("job-event:81", item["source_focus_ref"]["event_key"])
self.assertEqual("attention", payload["summary"]["status"])
self.assertEqual("待处理", payload["summary"]["status_label"])
@patch("app.services.ops_service.get_ops_runbook")
@patch("app.services.ops_service.list_release_rollouts")
@patch("app.services.ops_service.list_ops_jobs")
@patch("app.services.ops_service.get_recent_ops_playbook_runs")
@patch("app.services.ops_service.list_managed_nodes_with_agent_state")
def test_activity_stream_surfaces_execution_mode_label_for_rollout(
self,
mock_list_managed_nodes_with_agent_state,
mock_get_recent_ops_playbook_runs,
mock_list_ops_jobs,
mock_list_release_rollouts,
mock_get_ops_runbook,
) -> None:
mock_list_managed_nodes_with_agent_state.return_value = {"nodes": []}
mock_get_recent_ops_playbook_runs.return_value = {"runs": []}
mock_list_ops_jobs.return_value = []
mock_list_release_rollouts.return_value = [
{
"id": 201,
"release_id": 18,
"rollout_code": "rollout-20260418093000-a1b2",
"policy": {
"execution_mode": "ssh",
},
"status": "running",
"status_label": "执行中",
"target_nodes": [{"node_code": "mainland-worker-01"}],
"jobs_total": 2,
"jobs_created": 1,
"batch_cursor": 1,
"batches_total": 2,
"result_summary": {},
"summary_text": "当前已覆盖 1/1 台节点,仍有 1 条任务执行中。",
"focus_ref": {
"kind": "release_rollout",
"rollout_id": 201,
"rollout_code": "rollout-20260418093000-a1b2",
"release_id": 18,
"section": "rollout_jobs",
},
"created_by": "web-ui",
"created_at": "2026-04-18 09:30:00",
"updated_at": "2026-04-18 09:31:00",
}
]
mock_get_ops_runbook.return_value = {"control_sequences": []}
with patch(
"app.services.ops_service.get_runtime_status",
return_value=self._runtime_status(),
):
payload = get_ops_activity_stream(limit=10, scan_limit=20)
self.assertEqual(1, len(payload["items"]))
item = payload["items"][0]
self.assertEqual("rollout", item["kind"])
self.assertEqual("SSH", item["execution_mode_label"])
self.assertIn("执行 SSH", item["meta_text"])
self.assertEqual("执行中", item["status_label"])
self.assertIn("当前已覆盖 1/1 台节点", item["summary"])
self.assertEqual("release_rollout", item["focus_ref"]["kind"])
self.assertEqual("release_hub_contract", item["contract_navigation"]["primary_contract_key"])
self.assertIn("release_hub_contract", item["contract_navigation"]["contract_keys"])
@patch("app.services.ops_service.get_ops_runbook")
@patch("app.services.ops_service.list_release_rollouts")
@patch("app.services.ops_service.list_ops_jobs")
@patch("app.services.ops_service.get_recent_ops_playbook_runs")
@patch("app.services.ops_service.list_managed_nodes_with_agent_state")
def test_activity_stream_supports_execution_mode_filter(
self,
mock_list_managed_nodes_with_agent_state,
mock_get_recent_ops_playbook_runs,
mock_list_ops_jobs,
mock_list_release_rollouts,
mock_get_ops_runbook,
) -> None:
mock_list_managed_nodes_with_agent_state.return_value = {"nodes": []}
mock_get_recent_ops_playbook_runs.return_value = {
"runs": [
{
"run_code": "playbook-ssh-001",
"playbook_title": "关键现场日志",
"status": "running",
"updated_at": "2026-04-18 09:11:00",
"execution_mode": "ssh",
"execution_mode_label": "SSH",
"steps": [{"status": "running"}],
"step_count": 1,
"jobs_total": 1,
"target_node_codes": ["mainland-worker-01"],
}
]
}
mock_list_ops_jobs.return_value = [
{
"id": 61,
"job_code": "ops-ssh-61",
"action": "logs.collect",
"status": "running",
"execution_mode": "ssh",
"execution_mode_label": "SSH",
"requested_by": "web-ui",
"target_node_code": "mainland-worker-01",
"target_type": "node",
"risk_level": "low",
"updated_at": "2026-04-18 09:10:00",
"created_at": "2026-04-18 09:09:00",
},
{
"id": 62,
"job_code": "ops-agent-62",
"action": "health.snapshot",
"status": "success",
"execution_mode": "remote-agent",
"execution_mode_label": "远端 Agent",
"requested_by": "web-ui",
"target_node_code": "mainland-worker-02",
"target_type": "node",
"risk_level": "low",
"updated_at": "2026-04-18 09:08:00",
"created_at": "2026-04-18 09:07:00",
},
]
mock_list_release_rollouts.return_value = []
mock_get_ops_runbook.return_value = {
"control_sequences": [
{
"key": "release_progression",
"title": "版本发布与放量",
"status": "warning",
"status_label": "待处理",
"reason": "当前没有足够的执行面接管覆盖。",
"target_scope_label": "Release / Rollout",
"target_node_codes": ["mainland-controller-01"],
}
]
}
with patch(
"app.services.ops_service.get_runtime_status",
return_value=self._runtime_status(),
):
payload = get_ops_activity_stream(limit=10, scan_limit=20, execution_mode="ssh")
self.assertEqual(2, len(payload["items"]))
self.assertTrue(all(item["execution_mode"] == "ssh" for item in payload["items"]))
self.assertEqual("ssh", payload["summary"]["filters"]["execution_mode"])
self.assertEqual(2, payload["summary"]["available_execution_mode_counts"]["ssh"])
self.assertEqual(1, payload["summary"]["available_execution_mode_counts"]["remote-agent"])
@patch("app.services.ops_service.get_ops_runbook")
@patch("app.services.ops_service.list_release_rollouts")
@patch("app.services.ops_service.list_ops_jobs")
@patch("app.services.ops_service.get_recent_ops_playbook_runs")
@patch("app.services.ops_service.list_managed_nodes_with_agent_state")
def test_activity_stream_includes_execution_scene_and_log_sync_items(
self,
mock_list_managed_nodes_with_agent_state,
mock_get_recent_ops_playbook_runs,
mock_list_ops_jobs,
mock_list_release_rollouts,
mock_get_ops_runbook,
) -> None:
mock_list_managed_nodes_with_agent_state.return_value = {"nodes": []}
mock_get_recent_ops_playbook_runs.return_value = {"runs": []}
mock_list_ops_jobs.return_value = []
mock_list_release_rollouts.return_value = []
mock_get_ops_runbook.return_value = {"control_sequences": []}
detect = {
"active_job": {
"job_code": "detect-001",
"latest_cycle_event": {
"created_at": "2026-04-18 09:12:30",
},
},
"phase_label": "运行中",
"phase_detail": "代理池正常,现场可继续观察",
"recent_event": "域名检测完成: example.com",
"recent_warning": "",
"log_sync": {
"enabled": True,
"mode": "key",
"mode_label": "关键回传",
"description": "当前为关键回传;适合常态联调。",
"line_count": 12,
"source_node_count": 1,
"source_nodes": ["mainland-worker-01"],
"last_at": "2026-04-18 09:12:31",
"preview_lines": ["worker log line"],
},
"participation_summary": {
"summary": "当前参与检测节点 1 台,其中执行/已领 1 台,近窗刚有吞吐 0 台。",
"effective_online_nodes": 1,
"participating_nodes": 1,
"dispatch_active_nodes": 1,
"recent_only_nodes": 0,
"non_participating_nodes": 0,
"standby_nodes": 0,
"load_syncing_nodes": 0,
},
"participating_nodes": [
{
"node_code": "mainland-worker-01",
"region": "mainland",
"role": "worker",
"status": "busy",
"last_heartbeat_at": "2026-04-18 09:12:29",
"participation_state": "dispatch_active",
"is_dispatch_active": True,
}
],
"non_participating_nodes": [],
}
with patch(
"app.services.ops_service.get_runtime_status",
return_value=self._runtime_status(detect),
):
payload = get_ops_activity_stream(limit=10, scan_limit=20)
items_by_kind = {item["kind"]: item for item in payload["items"]}
self.assertIn("execution_scene", items_by_kind)
self.assertIn("log_sync", items_by_kind)
execution_scene_item = items_by_kind["execution_scene"]
log_sync_item = items_by_kind["log_sync"]
self.assertEqual("running", execution_scene_item["status"])
self.assertEqual("执行中", execution_scene_item["status_label"])
self.assertEqual("focus_execution_scene", execution_scene_item["ui_intent"]["kind"])
self.assertEqual("focus_execution_scene", execution_scene_item["ui_intent_kind"])
self.assertEqual("execution_scene", execution_scene_item["focus_ref"]["kind"])
self.assertEqual(execution_scene_item["summary"], execution_scene_item["summary_text"])
self.assertIn("执行/已领 1", execution_scene_item["meta_text"])
self.assertEqual("ops_observability_contract", execution_scene_item["contract_navigation"]["primary_contract_key"])
self.assertIn("ops_stack_diagnosis_contract", execution_scene_item["contract_navigation"]["contract_keys"])
self.assertEqual("running", log_sync_item["status"])
self.assertEqual("执行中", log_sync_item["status_label"])
self.assertEqual("control-plane", log_sync_item["execution_mode"])
self.assertEqual("控制面", log_sync_item["execution_mode_label"])
self.assertEqual("focus_execution_scene", log_sync_item["ui_intent"]["kind"])
self.assertTrue(log_sync_item["ui_intent"]["open_log_drawer"])
self.assertIn("关键日志观察模式", log_sync_item["summary"])
self.assertEqual("running", payload["summary"]["status"])
self.assertEqual("执行中", payload["summary"]["status_label"])
self.assertIn("最近活动里存在正在推进", payload["summary"]["summary_text"])
self.assertEqual("ops_observability_contract", payload["contract_navigation"]["primary_contract_key"])
self.assertIn("ops_stack_diagnosis_contract", payload["contract_navigation"]["contract_keys"])
if __name__ == "__main__":
unittest.main()

View File

@@ -0,0 +1,925 @@
from __future__ import annotations
import unittest
from unittest.mock import patch
from app.services.ops_service import execute_codex_action, resolve_codex_action
def _build_codex_brief_entry(
*,
key: str,
executor_kind: str,
primary_action_code: str,
recommendation: str,
automation_level: str,
target_api: str,
target_method: str = "POST",
sequence_key: str = "",
primary_action_payload: dict | None = None,
secondary_action_code: str = "",
secondary_action_payload: dict | None = None,
focus_ref: dict | None = None,
primary_focus_ref: dict | None = None,
secondary_focus_ref: dict | None = None,
) -> dict:
primary_action_payload = dict(primary_action_payload or {})
secondary_action_payload = dict(secondary_action_payload or {})
normalized_focus_ref = dict(focus_ref or {})
normalized_primary_focus_ref = dict(primary_focus_ref or {})
normalized_secondary_focus_ref = dict(secondary_focus_ref or {})
if normalized_primary_focus_ref and not dict(primary_action_payload.get("focus_ref") or {}):
primary_action_payload["focus_ref"] = dict(normalized_primary_focus_ref)
if normalized_secondary_focus_ref and not dict(secondary_action_payload.get("focus_ref") or {}):
secondary_action_payload["focus_ref"] = dict(normalized_secondary_focus_ref)
request_payload_preview = (
{
"sequence_key": sequence_key,
"secondary": False,
"action_payload": primary_action_payload,
}
if executor_kind == "runbook_sequence"
else {
"action_code": primary_action_code,
"node_codes": ["mainland-controller-01"],
"action_payload": primary_action_payload,
}
)
return {
"key": key,
"title": "测试动作",
"summary": "测试摘要",
"reason": "测试原因",
"meta_text": "meta",
"executor_kind": executor_kind,
"sequence_key": sequence_key,
"focus_ref": normalized_focus_ref,
"primary_focus_ref": normalized_primary_focus_ref,
"secondary_focus_ref": normalized_secondary_focus_ref,
"primary_label": "主动作",
"secondary_label": "次动作" if secondary_action_code else "",
"primary_action_code": primary_action_code,
"secondary_action_code": secondary_action_code,
"autopilot": {
"status_text": "测试状态",
"automation_level": automation_level,
"recommendation": recommendation,
"executor_mode_hint": "test-mode",
"confirm_required": recommendation == "confirm_then_execute",
"ui_only": recommendation == "open_ui",
"blocked": recommendation == "blocked",
"backend_handled": True,
"risk_level": "low" if recommendation == "auto_execute" else "high",
"reason": "autopilot reason",
"target_method": target_method,
"target_api": target_api,
},
"primary_execution": {
"label": "主动作",
"node_codes": ["mainland-controller-01"],
"action_payload": primary_action_payload,
"request_payload_preview": request_payload_preview,
},
"secondary_execution": (
{
"label": "次动作",
"node_codes": [],
"action_payload": secondary_action_payload,
}
if secondary_action_code
else {}
),
}
def _build_preview_data(
*,
executor_kind: str,
action_code: str,
recommendation: str,
automation_level: str,
target_api: str,
request_payload: dict,
sequence_key: str = "",
focus_ref: dict | None = None,
primary_focus_ref: dict | None = None,
secondary_focus_ref: dict | None = None,
) -> dict:
return {
"executor_kind": executor_kind,
"sequence_key": sequence_key,
"execution_chain": "ops-driver -> preview",
"focus_ref": dict(focus_ref or {}),
"primary_focus_ref": dict(primary_focus_ref or {}),
"secondary_focus_ref": dict(secondary_focus_ref or {}),
"primary": {
"label": "主动作",
"action_code": action_code,
"target_api": target_api,
"target_method": "POST",
"request_payload": dict(request_payload or {}),
"focus_ref": dict(primary_focus_ref or {}),
"execution_profile": {
"action_code": action_code,
"backend_handled": True,
"automation_level": automation_level,
"recommendation": recommendation,
"executor_mode_hint": "test-mode",
"risk_level": "low" if recommendation == "auto_execute" else "high",
"confirm_required": recommendation == "confirm_then_execute",
"ui_only": recommendation == "open_ui",
"blocked": recommendation == "blocked",
"requires_fields": [],
"missing_fields": [],
"reason": "preview reason",
},
},
"secondary": {},
}
def _build_codex_activity_focus_entry(
*,
key: str,
activity_key: str,
activity_kind: str,
ui_intent_kind: str,
focus_ref: dict | None = None,
source_focus_ref: dict | None = None,
target_node_codes: list[str] | None = None,
) -> dict:
normalized_focus_ref = dict(focus_ref or {})
return {
"key": key,
"activity_key": activity_key,
"activity_kind": activity_kind,
"title": "观察焦点",
"subtitle": "日志回传待确认",
"summary": "该焦点属于观察项,应进入对应工作区查看上下文。",
"status": "attention",
"status_label": "待关注",
"meta_text": "activity-meta",
"occurred_at": "2026-04-18T06:00:00",
"target_node_codes": list(target_node_codes or []),
"focus_ref": normalized_focus_ref,
"source_focus_ref": dict(source_focus_ref or {}),
"ui_intent": {
"kind": ui_intent_kind,
"activity_key": activity_key,
},
"ui_intent_kind": ui_intent_kind,
"focus_action_code": "focus_activity_item",
"focus_action_payload": {
"activity_key": activity_key,
"kind": activity_kind,
"status": "attention",
"ui_intent": {
"kind": ui_intent_kind,
"activity_key": activity_key,
},
"focus_ref": normalized_focus_ref,
},
"observation_only": True,
}
class OpsServiceCodexActionTests(unittest.TestCase):
@patch("app.services.ops_service.preview_driver_action")
@patch("app.services.ops_service.get_ops_codex_brief")
def test_resolve_codex_action_returns_gate_for_safe_auto(self, mock_get_ops_codex_brief, mock_preview_driver_action) -> None:
mock_get_ops_codex_brief.return_value = {
"focus": {"entry_key": "recommendation:enable-log-sync"},
"summary": {"total": 1},
"entries": [
_build_codex_brief_entry(
key="recommendation:enable-log-sync",
executor_kind="driver_action",
primary_action_code="enable_log_sync_key",
recommendation="auto_execute",
automation_level="safe_auto",
target_api="/api/v1/ops/driver-actions/execute",
focus_ref={
"kind": "execution_scene",
"scene_key": "scene.logs",
},
primary_focus_ref={
"kind": "execution_scene",
"scene_key": "scene.logs",
},
)
],
}
mock_preview_driver_action.return_value = (
True,
"ok",
_build_preview_data(
executor_kind="driver_action",
action_code="enable_log_sync_key",
recommendation="auto_execute",
automation_level="safe_auto",
target_api="/api/v1/ops/driver-actions/execute",
request_payload={
"action_code": "enable_log_sync_key",
"node_codes": ["mainland-controller-01"],
"action_payload": {
"focus_ref": {
"kind": "execution_scene",
"scene_key": "scene.logs",
}
},
"requested_by": "test/codex-action",
},
focus_ref={
"kind": "execution_scene",
"scene_key": "scene.logs",
},
primary_focus_ref={
"kind": "execution_scene",
"scene_key": "scene.logs",
},
),
)
ok, message, data = resolve_codex_action({"requested_by": "test/codex-action"})
self.assertTrue(ok)
self.assertEqual("Codex 驾驶动作解析完成", message)
self.assertEqual("recommendation:enable-log-sync", data["entry_key"])
self.assertEqual("execute_now", data["gate"]["decision"])
self.assertTrue(data["gate"]["will_execute"])
self.assertEqual("/api/v1/ops/driver-actions/execute", data["gate"]["target_api"])
self.assertEqual("enable_log_sync_key", data["preview_request"]["primary_action_code"])
self.assertEqual("execution_scene", data["preview_request"]["focus_ref"]["kind"])
self.assertEqual("scene.logs", data["gate"]["focus_ref"]["scene_key"])
self.assertEqual("scene.logs", data["focus_ref"]["scene_key"])
@patch("app.services.ops_service.preview_driver_action")
@patch("app.services.ops_service.get_ops_codex_brief")
def test_resolve_codex_action_requires_confirmation_for_guarded_runbook(
self,
mock_get_ops_codex_brief,
mock_preview_driver_action,
) -> None:
mock_get_ops_codex_brief.return_value = {
"focus": {"entry_key": "runbook:release_progression"},
"summary": {"total": 1},
"entries": [
_build_codex_brief_entry(
key="runbook:release_progression",
executor_kind="runbook_sequence",
sequence_key="release_progression",
primary_action_code="create_release_rollout_worker",
recommendation="confirm_then_execute",
automation_level="guarded_auto",
target_api="/api/v1/ops/runbook/sequences/release_progression/execute",
primary_action_payload={"release_id": 7},
secondary_action_code="focus_release_hub",
secondary_action_payload={"release_id": 7},
focus_ref={
"kind": "release_hub",
"release_id": 7,
"section": "release_launchpad",
},
primary_focus_ref={
"kind": "release_hub",
"release_id": 7,
"section": "release_launchpad",
"mode": "worker",
},
secondary_focus_ref={
"kind": "release_hub",
"release_id": 7,
"section": "release_detail",
},
)
],
}
mock_preview_driver_action.return_value = (
True,
"ok",
_build_preview_data(
executor_kind="runbook_sequence",
action_code="create_release_rollout_worker",
recommendation="confirm_then_execute",
automation_level="guarded_auto",
target_api="/api/v1/ops/runbook/sequences/release_progression/execute",
request_payload={
"secondary": False,
"requested_by": "test/codex-action",
"action_payload": {
"release_id": 7,
"focus_ref": {
"kind": "release_hub",
"release_id": 7,
"section": "release_launchpad",
"mode": "worker",
},
},
},
sequence_key="release_progression",
focus_ref={
"kind": "release_hub",
"release_id": 7,
"section": "release_launchpad",
},
primary_focus_ref={
"kind": "release_hub",
"release_id": 7,
"section": "release_launchpad",
"mode": "worker",
},
secondary_focus_ref={
"kind": "release_hub",
"release_id": 7,
"section": "release_detail",
},
),
)
ok, _, data = resolve_codex_action(
{
"entry_key": "runbook:release_progression",
"requested_by": "test/codex-action",
}
)
self.assertTrue(ok)
self.assertEqual("confirmation_required", data["gate"]["decision"])
self.assertFalse(data["gate"]["will_execute"])
self.assertTrue(data["gate"]["confirm_required"])
self.assertEqual("release_progression", data["gate"]["sequence_key"])
self.assertEqual("worker", data["preview_request"]["primary_focus_ref"]["mode"])
self.assertEqual("worker", data["gate"]["focus_ref"]["mode"])
self.assertEqual("worker", data["focus_ref"]["mode"])
@patch("app.services.ops_service.preview_driver_action")
@patch("app.services.ops_service.get_ops_codex_brief")
def test_resolve_codex_action_can_select_secondary_section(
self,
mock_get_ops_codex_brief,
mock_preview_driver_action,
) -> None:
mock_get_ops_codex_brief.return_value = {
"focus": {"entry_key": "runbook:release_progression"},
"summary": {"total": 1},
"entries": [
_build_codex_brief_entry(
key="runbook:release_progression",
executor_kind="runbook_sequence",
sequence_key="release_progression",
primary_action_code="create_release_rollout_worker",
recommendation="confirm_then_execute",
automation_level="guarded_auto",
target_api="/api/v1/ops/runbook/sequences/release_progression/execute",
primary_action_payload={"release_id": 7},
secondary_action_code="focus_release_hub",
secondary_action_payload={"release_id": 7},
focus_ref={
"kind": "release_hub",
"release_id": 7,
"section": "release_launchpad",
},
primary_focus_ref={
"kind": "release_hub",
"release_id": 7,
"section": "release_launchpad",
"mode": "worker",
},
secondary_focus_ref={
"kind": "release_hub",
"release_id": 7,
"section": "release_detail",
},
)
],
}
preview_data = _build_preview_data(
executor_kind="runbook_sequence",
action_code="create_release_rollout_worker",
recommendation="confirm_then_execute",
automation_level="guarded_auto",
target_api="/api/v1/ops/runbook/sequences/release_progression/execute",
request_payload={
"secondary": False,
"requested_by": "test/codex-action",
"action_payload": {
"release_id": 7,
"focus_ref": {
"kind": "release_hub",
"release_id": 7,
"section": "release_launchpad",
"mode": "worker",
},
},
},
sequence_key="release_progression",
focus_ref={
"kind": "release_hub",
"release_id": 7,
"section": "release_launchpad",
},
primary_focus_ref={
"kind": "release_hub",
"release_id": 7,
"section": "release_launchpad",
"mode": "worker",
},
secondary_focus_ref={
"kind": "release_hub",
"release_id": 7,
"section": "release_detail",
},
)
preview_data["secondary"] = {
"label": "查看版本区",
"action_code": "focus_release_hub",
"target_api": "/api/v1/ops/runbook/sequences/release_progression/execute",
"target_method": "POST",
"request_payload": {
"secondary": True,
"requested_by": "test/codex-action",
"action_payload": {
"release_id": 7,
"focus_ref": {
"kind": "release_hub",
"release_id": 7,
"section": "release_detail",
},
},
},
"focus_ref": {
"kind": "release_hub",
"release_id": 7,
"section": "release_detail",
},
"execution_profile": {
"action_code": "focus_release_hub",
"backend_handled": False,
"automation_level": "safe_auto",
"recommendation": "auto_execute",
"executor_mode_hint": "ui-intent",
"risk_level": "low",
"confirm_required": False,
"ui_only": False,
"blocked": False,
"requires_fields": [],
"missing_fields": [],
"reason": "secondary preview reason",
},
}
mock_preview_driver_action.return_value = (True, "ok", preview_data)
ok, _, data = resolve_codex_action(
{
"entry_key": "runbook:release_progression",
"requested_by": "test/codex-action",
"secondary": True,
}
)
self.assertTrue(ok)
self.assertEqual("secondary", data["selected_section"])
self.assertTrue(data["gate"]["secondary"])
self.assertEqual("focus_release_hub", data["gate"]["action_code"])
self.assertEqual("auto_execute", data["gate"]["recommendation"])
self.assertEqual("release_detail", data["gate"]["focus_ref"]["section"])
self.assertEqual("release_detail", data["focus_ref"]["section"])
@patch("app.services.ops_service.preview_driver_action")
@patch("app.services.ops_service.get_ops_codex_brief")
def test_resolve_codex_action_can_fall_back_to_activity_focus_entry(
self,
mock_get_ops_codex_brief,
mock_preview_driver_action,
) -> None:
mock_get_ops_codex_brief.return_value = {
"focus": {"entry_key": "activity:ops-job-42", "recommendation": "open_ui"},
"summary": {"total": 0, "activity_focus_total": 1},
"entries": [],
"activity_focus": [
_build_codex_activity_focus_entry(
key="activity:ops-job-42",
activity_key="ops-job-42",
activity_kind="ops_job",
ui_intent_kind="job_events",
focus_ref={
"kind": "ops_job",
"job_key": "job-42",
},
source_focus_ref={
"kind": "ops_job_event",
"job_key": "job-42",
"event_key": "event-1",
},
target_node_codes=["mainland-controller-01"],
)
],
}
mock_preview_driver_action.return_value = (
True,
"ok",
_build_preview_data(
executor_kind="driver_action",
action_code="focus_activity_item",
recommendation="auto_execute",
automation_level="safe_auto",
target_api="/api/v1/ops/driver-actions/execute",
request_payload={
"action_code": "focus_activity_item",
"node_codes": ["mainland-controller-01"],
"action_payload": {
"activity_key": "ops-job-42",
"kind": "ops_job",
"status": "attention",
"ui_intent": {
"kind": "job_events",
"activity_key": "ops-job-42",
},
"focus_ref": {
"kind": "ops_job",
"job_key": "job-42",
},
},
"requested_by": "test/codex-action",
},
focus_ref={
"kind": "ops_job",
"job_key": "job-42",
},
primary_focus_ref={
"kind": "ops_job",
"job_key": "job-42",
},
),
)
ok, _, data = resolve_codex_action({"requested_by": "test/codex-action"})
self.assertTrue(ok)
self.assertEqual("activity:ops-job-42", data["entry_key"])
self.assertTrue(data["selected_entry"]["observation_only"])
self.assertEqual("ops_job", data["selected_entry"]["activity_kind"])
self.assertEqual("focus_activity_item", data["preview_request"]["primary_action_code"])
self.assertEqual("execute_now", data["gate"]["decision"])
self.assertTrue(data["gate"]["will_execute"])
self.assertEqual("job-42", data["focus_ref"]["job_key"])
@patch("app.services.ops_service.execute_driver_action")
@patch("app.services.ops_service.preview_driver_action")
@patch("app.services.ops_service.get_ops_codex_brief")
def test_execute_codex_action_runs_safe_auto_driver_action(
self,
mock_get_ops_codex_brief,
mock_preview_driver_action,
mock_execute_driver_action,
) -> None:
mock_get_ops_codex_brief.return_value = {
"focus": {"entry_key": "recommendation:enable-log-sync"},
"summary": {"total": 1},
"entries": [
_build_codex_brief_entry(
key="recommendation:enable-log-sync",
executor_kind="driver_action",
primary_action_code="enable_log_sync_key",
recommendation="auto_execute",
automation_level="safe_auto",
target_api="/api/v1/ops/driver-actions/execute",
)
],
}
request_payload = {
"action_code": "enable_log_sync_key",
"node_codes": ["mainland-controller-01"],
"action_payload": {},
"requested_by": "test/codex-action",
}
mock_preview_driver_action.return_value = (
True,
"ok",
_build_preview_data(
executor_kind="driver_action",
action_code="enable_log_sync_key",
recommendation="auto_execute",
automation_level="safe_auto",
target_api="/api/v1/ops/driver-actions/execute",
request_payload=request_payload,
),
)
mock_execute_driver_action.return_value = (
True,
"执行成功",
{"handled": True, "action_code": "enable_log_sync_key"},
)
ok, message, data = execute_codex_action({"requested_by": "test/codex-action"})
self.assertTrue(ok)
self.assertEqual("执行成功", message)
self.assertTrue(data["executed"])
mock_execute_driver_action.assert_called_once_with(request_payload)
@patch("app.services.ops_service.execute_ops_runbook_sequence")
@patch("app.services.ops_service.preview_driver_action")
@patch("app.services.ops_service.get_ops_codex_brief")
def test_execute_codex_action_blocks_guarded_runbook_without_confirm(
self,
mock_get_ops_codex_brief,
mock_preview_driver_action,
mock_execute_ops_runbook_sequence,
) -> None:
mock_get_ops_codex_brief.return_value = {
"focus": {"entry_key": "runbook:release_progression"},
"summary": {"total": 1},
"entries": [
_build_codex_brief_entry(
key="runbook:release_progression",
executor_kind="runbook_sequence",
sequence_key="release_progression",
primary_action_code="create_release_rollout_worker",
recommendation="confirm_then_execute",
automation_level="guarded_auto",
target_api="/api/v1/ops/runbook/sequences/release_progression/execute",
primary_action_payload={"release_id": 7},
)
],
}
mock_preview_driver_action.return_value = (
True,
"ok",
_build_preview_data(
executor_kind="runbook_sequence",
action_code="create_release_rollout_worker",
recommendation="confirm_then_execute",
automation_level="guarded_auto",
target_api="/api/v1/ops/runbook/sequences/release_progression/execute",
request_payload={
"secondary": False,
"requested_by": "test/codex-action",
"action_payload": {"release_id": 7},
},
sequence_key="release_progression",
),
)
ok, message, data = execute_codex_action(
{
"entry_key": "runbook:release_progression",
"requested_by": "test/codex-action",
}
)
self.assertFalse(ok)
self.assertIn("确认", message)
self.assertFalse(data["executed"])
mock_execute_ops_runbook_sequence.assert_not_called()
@patch("app.services.ops_service.execute_driver_action")
@patch("app.services.ops_service.preview_driver_action")
@patch("app.services.ops_service.get_ops_codex_brief")
def test_execute_codex_action_runs_activity_focus_as_safe_auto(
self,
mock_get_ops_codex_brief,
mock_preview_driver_action,
mock_execute_driver_action,
) -> None:
mock_get_ops_codex_brief.return_value = {
"focus": {"entry_key": "activity:ops-job-42", "recommendation": "open_ui"},
"summary": {"total": 0, "activity_focus_total": 1},
"entries": [],
"activity_focus": [
_build_codex_activity_focus_entry(
key="activity:ops-job-42",
activity_key="ops-job-42",
activity_kind="ops_job",
ui_intent_kind="job_events",
focus_ref={
"kind": "ops_job",
"job_key": "job-42",
},
)
],
}
mock_preview_driver_action.return_value = (
True,
"ok",
_build_preview_data(
executor_kind="driver_action",
action_code="focus_activity_item",
recommendation="auto_execute",
automation_level="safe_auto",
target_api="/api/v1/ops/driver-actions/execute",
request_payload={
"action_code": "focus_activity_item",
"node_codes": [],
"action_payload": {
"activity_key": "ops-job-42",
"kind": "ops_job",
"status": "attention",
"ui_intent": {
"kind": "job_events",
"activity_key": "ops-job-42",
},
"focus_ref": {
"kind": "ops_job",
"job_key": "job-42",
},
},
"requested_by": "test/codex-action",
},
focus_ref={
"kind": "ops_job",
"job_key": "job-42",
},
primary_focus_ref={
"kind": "ops_job",
"job_key": "job-42",
},
),
)
mock_execute_driver_action.return_value = (
True,
"执行成功",
{"handled": True, "action_code": "focus_activity_item"},
)
ok, message, data = execute_codex_action({"requested_by": "test/codex-action"})
self.assertTrue(ok)
self.assertEqual("执行成功", message)
self.assertTrue(data["executed"])
mock_execute_driver_action.assert_called_once()
@patch("app.services.ops_service.preview_driver_action")
@patch("app.services.ops_service.get_ops_codex_brief")
def test_resolve_codex_action_supports_focus_latest_job_events_without_ui_intent(
self,
mock_get_ops_codex_brief,
mock_preview_driver_action,
) -> None:
mock_get_ops_codex_brief.return_value = {
"focus": {"entry_key": "activity:ops-job-81", "recommendation": "open_ui"},
"summary": {"total": 0, "activity_focus_total": 1},
"entries": [],
"activity_focus": [
{
"key": "activity:ops-job-81",
"activity_key": "ops-job-81",
"activity_kind": "ops_job",
"title": "开始回执异常",
"subtitle": "ops-agent-81",
"summary": "节点已接单执行,但开始回执未成功送达控制面。",
"status": "attention",
"status_label": "待处理",
"meta_text": "ops-agent-81",
"occurred_at": "2026-04-18T06:00:00",
"target_node_codes": ["mainland-worker-01"],
"focus_ref": {
"kind": "ops_job",
"job_id": 81,
"job_code": "ops-agent-81",
"target_node_code": "mainland-worker-01",
},
"source_focus_ref": {
"kind": "ops_job_event",
"job_id": 81,
"job_code": "ops-agent-81",
"event_key": "job-event:81",
"target_node_code": "mainland-worker-01",
},
"ui_intent": {},
"ui_intent_kind": "",
"focus_action_code": "focus_latest_job_events",
"focus_action_payload": {
"focus_ref": {
"kind": "ops_job",
"job_id": 81,
"job_code": "ops-agent-81",
"target_node_code": "mainland-worker-01",
},
"source_focus_ref": {
"kind": "ops_job_event",
"job_id": 81,
"job_code": "ops-agent-81",
"event_key": "job-event:81",
"target_node_code": "mainland-worker-01",
},
},
"observation_only": True,
}
],
}
mock_preview_driver_action.return_value = (
True,
"ok",
_build_preview_data(
executor_kind="driver_action",
action_code="focus_latest_job_events",
recommendation="auto_execute",
automation_level="safe_auto",
target_api="/api/v1/ops/driver-actions/execute",
request_payload={
"action_code": "focus_latest_job_events",
"node_codes": ["mainland-worker-01"],
"action_payload": {
"focus_ref": {
"kind": "ops_job",
"job_id": 81,
"job_code": "ops-agent-81",
"target_node_code": "mainland-worker-01",
},
"source_focus_ref": {
"kind": "ops_job_event",
"job_id": 81,
"job_code": "ops-agent-81",
"event_key": "job-event:81",
"target_node_code": "mainland-worker-01",
},
},
"requested_by": "test/codex-action",
},
focus_ref={
"kind": "ops_job",
"job_id": 81,
"job_code": "ops-agent-81",
},
primary_focus_ref={
"kind": "ops_job",
"job_id": 81,
"job_code": "ops-agent-81",
},
),
)
ok, _, data = resolve_codex_action({"requested_by": "test/codex-action"})
self.assertTrue(ok)
self.assertEqual("activity:ops-job-81", data["entry_key"])
self.assertEqual("focus_latest_job_events", data["preview_request"]["primary_action_code"])
self.assertEqual("execute_now", data["gate"]["decision"])
self.assertTrue(data["gate"]["will_execute"])
@patch("app.services.ops_service.execute_ops_runbook_sequence")
@patch("app.services.ops_service.preview_driver_action")
@patch("app.services.ops_service.get_ops_codex_brief")
def test_execute_codex_action_runs_guarded_runbook_after_confirm(
self,
mock_get_ops_codex_brief,
mock_preview_driver_action,
mock_execute_ops_runbook_sequence,
) -> None:
mock_get_ops_codex_brief.return_value = {
"focus": {"entry_key": "runbook:release_progression"},
"summary": {"total": 1},
"entries": [
_build_codex_brief_entry(
key="runbook:release_progression",
executor_kind="runbook_sequence",
sequence_key="release_progression",
primary_action_code="create_release_rollout_worker",
recommendation="confirm_then_execute",
automation_level="guarded_auto",
target_api="/api/v1/ops/runbook/sequences/release_progression/execute",
primary_action_payload={"release_id": 7},
)
],
}
request_payload = {
"secondary": False,
"requested_by": "test/codex-action",
"action_payload": {"release_id": 7},
}
mock_preview_driver_action.return_value = (
True,
"ok",
_build_preview_data(
executor_kind="runbook_sequence",
action_code="create_release_rollout_worker",
recommendation="confirm_then_execute",
automation_level="guarded_auto",
target_api="/api/v1/ops/runbook/sequences/release_progression/execute",
request_payload=request_payload,
sequence_key="release_progression",
),
)
mock_execute_ops_runbook_sequence.return_value = (
True,
"runbook 执行成功",
{"sequence_key": "release_progression", "driver_action_code": "create_release_rollout_worker"},
)
ok, message, data = execute_codex_action(
{
"entry_key": "runbook:release_progression",
"requested_by": "test/codex-action",
"confirm": True,
}
)
self.assertTrue(ok)
self.assertEqual("runbook 执行成功", message)
self.assertTrue(data["executed"])
mock_execute_ops_runbook_sequence.assert_called_once_with("release_progression", request_payload)
if __name__ == "__main__":
unittest.main()

View File

@@ -0,0 +1,497 @@
import unittest
from unittest.mock import patch
from app.services.ops_service import get_ops_codex_brief
class OpsServiceCodexBriefTests(unittest.TestCase):
@patch("app.services.ops_service.get_ops_driver_feed")
def test_codex_brief_classifies_safe_ui_and_runbook_entries(self, mock_get_ops_driver_feed) -> None:
mock_get_ops_driver_feed.return_value = {
"headline": "当前建议已形成。",
"go_live_summary": {
"go_live_status": "attention",
"publish_ready": False,
"publish_status": "attention",
"publish_status_label": "可发布但建议先复核",
"operator_title": "先补现场日志回传",
"next_step_action_code": "enable_log_sync_key",
"launchpad_recommended_target_node_code": "mainland-controller-01",
"launchpad_recommended_recovery_label": "签发接入工单",
"launchpad_recommended_recovery_summary": "当前节点仍处于接入阶段,建议先签发 onboarding.bootstrap。",
"launchpad_onboarding_bootstrap_pending_nodes": 1,
"launchpad_onboarding_acceptance_ready_nodes": 1,
},
"activity_focus": [],
"entries": [
{
"key": "recommendation:enable-scene-log-sync",
"kind": "driver_recommendation",
"lane": "do_now",
"lane_label": "建议动作",
"title": "先补现场日志回传",
"summary": "优先打开关键日志回传。",
"reason": "当前还看不到现场过程日志。",
"status": "attention",
"level_label": "最高优先",
"tag_type": "danger",
"meta_text": "",
"detail_lines": ["参与节点 2 台"],
"node_codes": ["mainland-worker-01"],
"executor_kind": "driver_action",
"focus_ref": {
"kind": "execution_scene",
"scene_key": "scene.logs",
},
"primary_focus_ref": {
"kind": "execution_scene",
"scene_key": "scene.logs",
},
"secondary_focus_ref": {
"kind": "execution_scene",
"scene_key": "scene.logs.full",
},
"primary_label": "开启关键回传",
"secondary_label": "开启全量回传",
"primary_action_code": "enable_log_sync_key",
"secondary_action_code": "enable_log_sync_full",
"primary_node_codes": [],
"secondary_node_codes": [],
"primary_action_payload": {
"focus_ref": {
"kind": "execution_scene",
"scene_key": "scene.logs",
}
},
"secondary_action_payload": {
"focus_ref": {
"kind": "execution_scene",
"scene_key": "scene.logs.full",
}
},
},
{
"key": "recommendation:release-missing",
"kind": "driver_recommendation",
"lane": "do_now",
"lane_label": "建议动作",
"title": "先建立 Release",
"summary": "当前还没有默认 Release。",
"reason": "需要先进入版本区。",
"status": "planned",
"level_label": "待补齐",
"tag_type": "info",
"meta_text": "",
"detail_lines": [],
"node_codes": [],
"executor_kind": "driver_action",
"focus_ref": {
"kind": "release_hub",
"release_id": 7,
"section": "release_launchpad",
},
"primary_focus_ref": {
"kind": "release_hub",
"release_id": 7,
"section": "release_launchpad",
},
"primary_label": "创建 Release",
"secondary_label": "",
"primary_action_code": "open_release_dialog",
"secondary_action_code": "",
"primary_node_codes": [],
"secondary_node_codes": [],
"primary_action_payload": {},
"secondary_action_payload": {},
},
{
"key": "runbook:release_progression",
"kind": "runbook_sequence",
"lane": "standard_path",
"lane_label": "标准路径",
"title": "版本发布与放量",
"summary": "当前可以进入正式发布路径。",
"reason": "统一回到 Release / Rollout 闭环。",
"status": "attention",
"level_label": "待处理",
"tag_type": "warning",
"meta_text": "Release / Rollout",
"detail_lines": ["当前解析Worker 灰度"],
"node_codes": ["mainland-controller-01"],
"executor_kind": "runbook_sequence",
"sequence_key": "release_progression",
"focus_ref": {
"kind": "release_hub",
"release_id": 7,
"section": "release_launchpad",
},
"primary_focus_ref": {
"kind": "release_hub",
"release_id": 7,
"section": "release_launchpad",
"mode": "worker",
},
"secondary_focus_ref": {
"kind": "release_hub",
"release_id": 7,
"section": "release_detail",
},
"primary_label": "Worker 灰度",
"secondary_label": "查看版本区",
"primary_action_code": "create_release_rollout_worker",
"secondary_action_code": "focus_release_hub",
"primary_node_codes": ["mainland-controller-01"],
"secondary_node_codes": [],
"primary_action_payload": {
"release_id": 7,
"focus_ref": {
"kind": "release_hub",
"release_id": 7,
"section": "release_launchpad",
"mode": "worker",
},
},
"secondary_action_payload": {
"release_id": 7,
"focus_ref": {
"kind": "release_hub",
"release_id": 7,
"section": "release_detail",
},
},
"occurred_at": "2026-04-18T06:00:00",
},
],
}
payload = get_ops_codex_brief()
self.assertEqual(3, payload["summary"]["total"])
self.assertEqual("attention", payload["go_live_summary"]["go_live_status"])
self.assertEqual("attention", payload["summary"]["go_live_status"])
self.assertEqual("先补现场日志回传", payload["summary"]["operator_title"])
self.assertEqual("attention", payload["summary"]["publish_status"])
self.assertFalse(payload["summary"]["publish_ready"])
self.assertEqual("可发布但建议先复核", payload["summary"]["publish_status_label"])
self.assertEqual("mainland-controller-01", payload["summary"]["launchpad_recommended_target_node_code"])
self.assertEqual("签发接入工单", payload["summary"]["launchpad_recommended_recovery_label"])
self.assertEqual(
"当前节点仍处于接入阶段,建议先签发 onboarding.bootstrap。",
payload["summary"]["launchpad_recommended_recovery_summary"],
)
self.assertEqual(1, payload["summary"]["launchpad_onboarding_bootstrap_pending_nodes"])
self.assertEqual(1, payload["summary"]["launchpad_onboarding_acceptance_ready_nodes"])
self.assertEqual(0, payload["summary"]["activity_focus_total"])
self.assertEqual("当前建议已形成。", payload["driver_feed"]["headline"])
self.assertEqual("ops_driver_contract", payload["contract_navigation"]["primary_contract_key"])
self.assertIn("ops_stack_diagnosis_contract", payload["contract_navigation"]["contract_keys"])
self.assertEqual(2, payload["summary"]["auto_execute_total"])
self.assertEqual(1, payload["summary"]["confirm_then_execute_total"])
self.assertEqual(0, payload["summary"]["open_ui_total"])
self.assertEqual(1, payload["summary"]["preview_only_total"])
self.assertEqual(3, payload["summary"]["backend_handled_total"])
self.assertEqual("attention", payload["summary"]["launch_status"])
self.assertFalse(payload["summary"]["launch_ready"])
self.assertEqual("recommendation:enable-scene-log-sync", payload["focus"]["entry_key"])
self.assertEqual("execution_scene", payload["focus"]["focus_ref"]["kind"])
self.assertEqual("scene.logs", payload["focus"]["primary_focus_ref"]["scene_key"])
self.assertEqual("auto_execute", payload["entries"][0]["autopilot"]["recommendation"])
self.assertIn("ops_observability_contract", payload["entries"][0]["contract_keys"])
self.assertIn("ops_stack_diagnosis_contract", payload["entries"][0]["contract_keys"])
self.assertEqual("scene.logs", payload["entries"][0]["primary_focus_ref"]["scene_key"])
self.assertEqual("safe_auto", payload["entries"][1]["autopilot"]["automation_level"])
self.assertIn("release_hub_contract", payload["entries"][1]["contract_keys"])
runbook_entry = payload["entries"][2]
self.assertEqual("runbook_sequence", runbook_entry["executor_kind"])
self.assertEqual("confirm_then_execute", runbook_entry["autopilot"]["recommendation"])
self.assertIn("ops_playbook_contract", runbook_entry["contract_keys"])
self.assertIn("release_hub_contract", runbook_entry["contract_keys"])
self.assertEqual("release_launchpad", runbook_entry["focus_ref"]["section"])
self.assertEqual("worker", runbook_entry["primary_focus_ref"]["mode"])
self.assertEqual("release_detail", runbook_entry["secondary_focus_ref"]["section"])
self.assertEqual(
"/api/v1/ops/runbook/sequences/release_progression/execute",
runbook_entry["autopilot"]["target_api"],
)
self.assertEqual(
{
"sequence_key": "release_progression",
"secondary": False,
"action_payload": {
"release_id": 7,
"focus_ref": {
"kind": "release_hub",
"release_id": 7,
"section": "release_launchpad",
"mode": "worker",
},
},
},
runbook_entry["primary_execution"]["request_payload_preview"],
)
self.assertEqual(
{
"sequence_key": "release_progression",
"secondary": True,
"action_payload": {
"release_id": 7,
"focus_ref": {
"kind": "release_hub",
"release_id": 7,
"section": "release_detail",
},
},
},
runbook_entry["secondary_execution"]["request_payload_preview"],
)
self.assertEqual("attention", payload["automation_coverage"]["launch_status"])
self.assertEqual(2, payload["automation_coverage"]["safe_auto_total"])
self.assertEqual(1, payload["automation_coverage"]["guarded_auto_total"])
self.assertEqual(1, payload["automation_coverage"]["preview_only_total"])
self.assertEqual(3, payload["automation_coverage"]["backend_handled_total"])
@patch("app.services.ops_service.get_ops_driver_feed")
def test_codex_brief_marks_mixed_and_blocked_entries(self, mock_get_ops_driver_feed) -> None:
mock_get_ops_driver_feed.return_value = {
"headline": "当前建议需要先复核。",
"go_live_summary": {
"go_live_status": "blocked",
"publish_ready": False,
"publish_status": "blocked",
"publish_status_label": "暂不可发布",
"operator_title": "先补执行面接管",
"next_step_action_code": "fix_managed_nodes",
"launchpad_recommended_target_node_code": "mainland-controller-01",
"launchpad_recommended_recovery_label": "补执行面接管",
"launchpad_recommended_recovery_summary": "先处理 launchpad 缺口。",
"launchpad_onboarding_bootstrap_pending_nodes": 1,
"launchpad_onboarding_acceptance_ready_nodes": 0,
},
"activity_focus": [],
"entries": [
{
"key": "recommendation:release-launchpad-fix-managed",
"kind": "driver_recommendation",
"lane": "do_now",
"lane_label": "建议动作",
"title": "先补执行面接管",
"summary": "先处理 launchpad 缺口。",
"reason": "当前动作可能直接 sync也可能转为补接入。",
"status": "attention",
"level_label": "最高优先",
"tag_type": "danger",
"meta_text": "",
"detail_lines": [],
"node_codes": ["mainland-controller-01"],
"executor_kind": "driver_action",
"focus_ref": {
"kind": "release_hub",
"release_id": 9,
"section": "release_launchpad",
},
"primary_focus_ref": {
"kind": "release_hub",
"release_id": 9,
"section": "default_rollout_gate",
},
"primary_label": "补执行面接管",
"secondary_label": "",
"primary_action_code": "fix_managed_nodes",
"secondary_action_code": "",
"primary_node_codes": ["mainland-controller-01"],
"secondary_node_codes": [],
"primary_action_payload": {"channel": "stable"},
"secondary_action_payload": {},
},
{
"key": "recommendation:inspect-participating",
"kind": "driver_recommendation",
"lane": "do_now",
"lane_label": "建议动作",
"title": "先看执行现场",
"summary": "先跑参与节点巡检。",
"reason": "缺少目标节点时不应误判为可自动执行。",
"status": "attention",
"level_label": "推荐",
"tag_type": "warning",
"meta_text": "",
"detail_lines": [],
"node_codes": [],
"executor_kind": "driver_action",
"primary_label": "执行标准巡检",
"secondary_label": "",
"primary_action_code": "run_inspection_participating",
"secondary_action_code": "",
"primary_node_codes": [],
"secondary_node_codes": [],
"primary_action_payload": {},
"secondary_action_payload": {},
},
],
}
payload = get_ops_codex_brief()
self.assertEqual("blocked", payload["summary"]["publish_status"])
self.assertEqual("暂不可发布", payload["summary"]["publish_status_label"])
self.assertEqual("mainland-controller-01", payload["summary"]["launchpad_recommended_target_node_code"])
self.assertEqual("补执行面接管", payload["summary"]["launchpad_recommended_recovery_label"])
self.assertEqual(1, payload["summary"]["launchpad_onboarding_bootstrap_pending_nodes"])
self.assertEqual(0, payload["summary"]["launchpad_onboarding_acceptance_ready_nodes"])
self.assertEqual(1, payload["summary"]["resolve_first_total"])
self.assertEqual(1, payload["summary"]["blocked_total"])
self.assertEqual("blocked", payload["summary"]["launch_status"])
self.assertFalse(payload["summary"]["launch_ready"])
self.assertEqual("mixed", payload["entries"][0]["autopilot"]["automation_level"])
self.assertEqual("resolve_first", payload["entries"][0]["autopilot"]["recommendation"])
self.assertEqual("default_rollout_gate", payload["entries"][0]["primary_focus_ref"]["section"])
self.assertEqual(1, payload["automation_coverage"]["mixed_total"])
self.assertEqual(1, payload["automation_coverage"]["blocked_total"])
self.assertEqual(2, payload["automation_coverage"]["human_dependency_total"])
blocked_entry = payload["entries"][1]
self.assertTrue(blocked_entry["autopilot"]["blocked"])
self.assertEqual(["node_codes"], blocked_entry["primary_execution"]["missing_fields"])
self.assertEqual("blocked", blocked_entry["autopilot"]["recommendation"])
@patch("app.services.ops_service.get_ops_driver_feed")
def test_codex_brief_exposes_activity_focus_and_uses_it_as_fallback_focus(self, mock_get_ops_driver_feed) -> None:
mock_get_ops_driver_feed.return_value = {
"headline": "当前先盯住执行现场。",
"scene_log_observation": {},
"entries": [],
"activity_focus": [
{
"key": "activity:execution-scene:live",
"activity_key": "execution-scene:live",
"activity_kind": "execution_scene",
"title": "当前执行现场",
"subtitle": "运行中",
"summary": "当前参与检测节点 2 台,建议先看现场与日志回传。",
"status": "running",
"status_label": "执行中",
"meta_text": "执行/已领 2 / 待命 1",
"occurred_at": "2026-04-18 09:30:00",
"target_node_codes": ["mainland-worker-01", "mainland-controller-01"],
"focus_ref": {
"kind": "execution_scene",
"scene_key": "scene.logs",
},
"source_focus_ref": {},
"ui_intent": {
"kind": "focus_execution_scene",
"open_log_drawer": True,
},
"ui_intent_kind": "focus_execution_scene",
"focus_action_code": "focus_activity_item",
"focus_action_payload": {
"activity_key": "execution-scene:live",
"kind": "execution_scene",
"status": "running",
"ui_intent": {
"kind": "focus_execution_scene",
"open_log_drawer": True,
},
"focus_ref": {
"kind": "execution_scene",
"scene_key": "scene.logs",
},
},
"observation_only": True,
}
],
}
payload = get_ops_codex_brief()
self.assertEqual(0, payload["summary"]["total"])
self.assertEqual(1, payload["summary"]["activity_focus_total"])
self.assertEqual("activity:execution-scene:live", payload["focus"]["entry_key"])
self.assertEqual("open_ui", payload["focus"]["recommendation"])
self.assertIn("ops_observability_contract", payload["focus"]["contract_keys"])
self.assertEqual("execution_scene", payload["focus"]["focus_ref"]["kind"])
self.assertEqual(1, len(payload["activity_focus"]))
self.assertEqual("focus_execution_scene", payload["activity_focus"][0]["ui_intent_kind"])
self.assertEqual(
"ops_driver_contract",
payload["activity_focus"][0]["contract_navigation"]["primary_contract_key"],
)
self.assertTrue(payload["activity_focus"][0]["observation_only"])
@patch("app.services.ops_service.get_ops_driver_feed")
def test_codex_brief_can_fallback_to_scene_log_observation_focus(self, mock_get_ops_driver_feed) -> None:
mock_get_ops_driver_feed.return_value = {
"headline": "先看现场日志。",
"entries": [],
"activity_focus": [],
"scene_log_observation": {
"status": "waiting_sample",
"status_label": "等待样本",
"summary": "日志回传已开启,但样本还没形成。",
"focus_ref": {
"kind": "node_scene_log",
"node_code": "mainland-worker-01",
"mode": "key",
"limit": 120,
},
"contract_keys": ["ops_observability_contract", "ops_stack_diagnosis_contract"],
},
}
payload = get_ops_codex_brief()
self.assertEqual("scene-log-observation", payload["focus"]["entry_key"])
self.assertEqual("open_ui", payload["focus"]["recommendation"])
self.assertEqual("node_scene_log", payload["focus"]["focus_ref"]["kind"])
self.assertEqual("mainland-worker-01", payload["focus"]["focus_ref"]["node_code"])
self.assertEqual("waiting_sample", payload["scene_log_observation"]["status"])
@patch("app.services.ops_service.get_ops_driver_feed")
def test_codex_brief_marks_bootstrap_run_as_agent_protocol_flow(self, mock_get_ops_driver_feed) -> None:
mock_get_ops_driver_feed.return_value = {
"headline": "先执行节点接入收口。",
"activity_focus": [],
"entries": [
{
"key": "priority:handover-gap-bootstrap_run",
"kind": "priority_recommendation",
"lane": "do_now",
"lane_label": "当前优先",
"title": "先补接管缺口",
"summary": "Launchpad 已明确给出首台节点的接入收口动作。",
"reason": "mainland-worker-01 还缺接入收口,先执行 bootstrap。",
"status": "attention",
"level_label": "最高优先",
"tag_type": "warning",
"meta_text": "overview.recommendation",
"detail_lines": ["当前已收敛到单节点接入动作。"],
"node_codes": ["mainland-worker-01"],
"executor_kind": "driver_action",
"focus_ref": {"kind": "managed_node", "node_code": "mainland-worker-01"},
"primary_focus_ref": {"kind": "managed_node", "node_code": "mainland-worker-01"},
"secondary_focus_ref": {"kind": "managed_node", "node_code": "mainland-worker-01", "section": "bootstrap_plan"},
"primary_label": "跑接入收口",
"secondary_label": "查看 Bootstrap 计划",
"primary_action_code": "bootstrap_run",
"secondary_action_code": "create_bootstrap_plan",
"primary_node_codes": ["mainland-worker-01"],
"secondary_node_codes": ["mainland-worker-01"],
"primary_action_payload": {"node_code": "mainland-worker-01"},
"secondary_action_payload": {"node_code": "mainland-worker-01"},
}
],
}
payload = get_ops_codex_brief()
self.assertEqual("priority:handover-gap-bootstrap_run", payload["focus"]["entry_key"])
self.assertEqual("resolve_first", payload["focus"]["recommendation"])
self.assertIn("ops_agent_protocol", payload["entries"][0]["contract_keys"])
self.assertEqual("bootstrap_run", payload["entries"][0]["primary_action_code"])
self.assertEqual("mainland-worker-01", payload["entries"][0]["primary_execution"]["node_codes"][0])
self.assertEqual("/api/v1/ops/driver-actions/execute", payload["entries"][0]["autopilot"]["target_api"])
if __name__ == "__main__":
unittest.main()

View File

@@ -0,0 +1,295 @@
import unittest
from unittest.mock import patch
from app.services.ops_service import (
get_ops_blueprint,
get_ops_capabilities,
get_ops_contract_detail,
get_ops_contract_registry,
get_ops_runbook,
)
class OpsServiceContractRegistryTests(unittest.TestCase):
def test_contract_registry_lists_all_frozen_ops_contracts(self) -> None:
payload = get_ops_contract_registry()
self.assertEqual("2026-04-18", payload["registry_version"])
self.assertEqual("v1", payload["schema_version"])
self.assertEqual(11, payload["contracts_total"])
self.assertEqual(
[
"ops_job_contract",
"ops_agent_protocol",
"release_hub_contract",
"ops_driver_contract",
"ops_playbook_contract",
"ops_observability_contract",
"ops_stack_diagnosis_contract",
"ops_doctor_decision_contract",
"ops_go_live_signoff_contract",
"ops_go_live_bundle_contract",
"ops_go_live_review_contract",
],
[item["key"] for item in payload["contracts"]],
)
self.assertIn("/api/v1/ops/contracts/{contract_key}", payload["discovery_entrypoints"])
self.assertEqual(
"docs/schemas/ops_job_contract.md",
payload["contracts_by_key"]["ops_job_contract"]["schema_doc_path"],
)
self.assertIn(
"/api/v1/ops/jobs/{job_id}/dispatch",
payload["contracts_by_key"]["ops_job_contract"]["discovery_endpoints"],
)
self.assertEqual(
"docs/schemas/ops_agent_protocol.md",
payload["contracts_by_key"]["ops_agent_protocol"]["schema_doc_path"],
)
self.assertIn(
"/api/v1/ops/agent/bootstrap-plan",
payload["contracts_by_key"]["ops_agent_protocol"]["discovery_endpoints"],
)
self.assertIn(
"/api/v1/ops/nodes/{node_code}/delivery-queue",
payload["contracts_by_key"]["ops_agent_protocol"]["discovery_endpoints"],
)
self.assertIn(
"/api/v1/ops/nodes/{node_code}/delivery-queue/records",
payload["contracts_by_key"]["ops_agent_protocol"]["discovery_endpoints"],
)
self.assertIn(
"/api/v1/ops/nodes/{node_code}/delivery-queue/flush",
payload["contracts_by_key"]["ops_agent_protocol"]["discovery_endpoints"],
)
self.assertIn(
"/api/v1/ops/nodes/{node_code}/delivery-queue/replay",
payload["contracts_by_key"]["ops_agent_protocol"]["discovery_endpoints"],
)
self.assertIn(
"/api/v1/ops/nodes/{node_code}/delivery-queue/records/{record_id}/replay",
payload["contracts_by_key"]["ops_agent_protocol"]["discovery_endpoints"],
)
self.assertIn(
"/api/v1/ops/nodes/{node_code}/delivery-queue/records/{record_id}/discard",
payload["contracts_by_key"]["ops_agent_protocol"]["discovery_endpoints"],
)
self.assertEqual(
"/api/v1/ops/releases/launchpad",
payload["contracts_by_key"]["release_hub_contract"]["primary_endpoint"],
)
self.assertIn(
"/api/v1/ops/codex-brief",
payload["contracts_by_key"]["ops_driver_contract"]["discovery_endpoints"],
)
self.assertIn(
"/api/v1/ops/driver-actions/preview",
payload["contracts_by_key"]["ops_driver_contract"]["discovery_endpoints"],
)
self.assertIn(
"/api/v1/ops/driver-actions/resolve",
payload["contracts_by_key"]["ops_driver_contract"]["discovery_endpoints"],
)
self.assertIn(
"/api/v1/ops/driver-actions/execute-resolved",
payload["contracts_by_key"]["ops_driver_contract"]["discovery_endpoints"],
)
self.assertIn(
"/api/v1/ops/codex-actions/resolve",
payload["contracts_by_key"]["ops_driver_contract"]["discovery_endpoints"],
)
self.assertIn(
"/api/v1/ops/codex-actions/execute",
payload["contracts_by_key"]["ops_driver_contract"]["discovery_endpoints"],
)
self.assertEqual(
"docs/schemas/ops_playbook_contract.md",
payload["contracts_by_key"]["ops_playbook_contract"]["schema_doc_path"],
)
self.assertIn(
"/api/v1/ops/playbooks/execute",
payload["contracts_by_key"]["ops_playbook_contract"]["discovery_endpoints"],
)
self.assertIn(
"/api/v1/ops/playbook-runs/{run_code}/events",
payload["contracts_by_key"]["ops_playbook_contract"]["discovery_endpoints"],
)
self.assertEqual(
"docs/schemas/ops_observability_contract.md",
payload["contracts_by_key"]["ops_observability_contract"]["schema_doc_path"],
)
self.assertIn(
"/api/v1/ops/inspection-overview",
payload["contracts_by_key"]["ops_observability_contract"]["discovery_endpoints"],
)
self.assertIn(
"/api/v1/ops/nodes/{node_code}/scene-log",
payload["contracts_by_key"]["ops_observability_contract"]["discovery_endpoints"],
)
self.assertEqual(
"docs/schemas/ops_stack_diagnosis_contract.md",
payload["contracts_by_key"]["ops_stack_diagnosis_contract"]["schema_doc_path"],
)
self.assertIn(
"/api/v1/ops/stack-diagnosis",
payload["contracts_by_key"]["ops_stack_diagnosis_contract"]["discovery_endpoints"],
)
self.assertIn(
"ops_driver_contract",
payload["contracts_by_key"]["ops_stack_diagnosis_contract"]["related_contract_keys"],
)
self.assertEqual(
"docs/schemas/ops_doctor_decision_contract.md",
payload["contracts_by_key"]["ops_doctor_decision_contract"]["schema_doc_path"],
)
self.assertIn(
"/api/v1/ops/doctor-decision",
payload["contracts_by_key"]["ops_doctor_decision_contract"]["discovery_endpoints"],
)
self.assertIn(
"ops_stack_diagnosis_contract",
payload["contracts_by_key"]["ops_doctor_decision_contract"]["related_contract_keys"],
)
self.assertEqual(
"docs/schemas/ops_go_live_signoff_contract.md",
payload["contracts_by_key"]["ops_go_live_signoff_contract"]["schema_doc_path"],
)
self.assertIn(
"/api/v1/ops/go-live-signoff",
payload["contracts_by_key"]["ops_go_live_signoff_contract"]["discovery_endpoints"],
)
self.assertIn(
"ops_stack_diagnosis_contract",
payload["contracts_by_key"]["ops_go_live_signoff_contract"]["related_contract_keys"],
)
self.assertEqual(
"docs/schemas/ops_go_live_bundle_contract.md",
payload["contracts_by_key"]["ops_go_live_bundle_contract"]["schema_doc_path"],
)
self.assertIn(
"/api/v1/ops/go-live-bundle",
payload["contracts_by_key"]["ops_go_live_bundle_contract"]["discovery_endpoints"],
)
self.assertIn(
"ops_go_live_signoff_contract",
payload["contracts_by_key"]["ops_go_live_bundle_contract"]["related_contract_keys"],
)
self.assertEqual(
"docs/schemas/ops_go_live_review_contract.md",
payload["contracts_by_key"]["ops_go_live_review_contract"]["schema_doc_path"],
)
self.assertIn(
"/api/v1/ops/go-live-review",
payload["contracts_by_key"]["ops_go_live_review_contract"]["discovery_endpoints"],
)
self.assertIn(
"ops_go_live_bundle_contract",
payload["contracts_by_key"]["ops_go_live_review_contract"]["related_contract_keys"],
)
self.assertIn(
"ops_doctor_decision_contract",
payload["contracts_by_key"]["ops_go_live_review_contract"]["related_contract_keys"],
)
def test_contract_detail_exposes_related_contracts(self) -> None:
payload = get_ops_contract_detail("ops_stack_diagnosis_contract")
self.assertEqual("2026-04-18", payload["registry_version"])
self.assertEqual("/api/v1/ops/contracts/ops_stack_diagnosis_contract", payload["detail_endpoint"])
self.assertEqual("ops_stack_diagnosis_contract", payload["contract"]["key"])
self.assertIn("ops_driver_contract", payload["related_contract_keys"])
self.assertIn("release_hub_contract", payload["related_contract_keys"])
self.assertEqual(
"docs/schemas/ops_driver_contract.md",
next(
item["schema_doc_path"]
for item in payload["related_contracts"]
if item["key"] == "ops_driver_contract"
),
)
self.assertEqual({}, get_ops_contract_detail("missing-contract"))
def test_capabilities_blueprint_and_runbook_expose_contract_registry(self) -> None:
capabilities = get_ops_capabilities()
contract_registry = capabilities["contract_registry"]
self.assertEqual("/api/v1/ops/contracts", contract_registry["endpoint"])
self.assertEqual("/api/v1/ops/contracts/{contract_key}", contract_registry["detail_endpoint_pattern"])
self.assertEqual(11, contract_registry["contracts_total"])
self.assertIn("ops_job_contract", contract_registry["contract_keys"])
self.assertIn("ops_driver_contract", contract_registry["contract_keys"])
self.assertIn("ops_playbook_contract", contract_registry["contract_keys"])
self.assertIn("ops_observability_contract", contract_registry["contract_keys"])
self.assertIn("ops_stack_diagnosis_contract", contract_registry["contract_keys"])
self.assertIn("ops_doctor_decision_contract", contract_registry["contract_keys"])
self.assertIn("ops_go_live_signoff_contract", contract_registry["contract_keys"])
self.assertIn("ops_go_live_bundle_contract", contract_registry["contract_keys"])
self.assertIn("ops_go_live_review_contract", contract_registry["contract_keys"])
remote_log_capability = next(
item for item in capabilities["capabilities"] if item["key"] == "remote_log_sync"
)
self.assertEqual("ready", remote_log_capability["current_state"])
self.assertEqual(
"docs/schemas/release_hub_contract.md",
contract_registry["schema_docs"]["release_hub_contract"],
)
self.assertEqual(
"docs/schemas/ops_playbook_contract.md",
contract_registry["schema_docs"]["ops_playbook_contract"],
)
self.assertEqual(
"docs/schemas/ops_job_contract.md",
contract_registry["schema_docs"]["ops_job_contract"],
)
self.assertEqual(
"docs/schemas/ops_observability_contract.md",
contract_registry["schema_docs"]["ops_observability_contract"],
)
self.assertEqual(
"docs/schemas/ops_stack_diagnosis_contract.md",
contract_registry["schema_docs"]["ops_stack_diagnosis_contract"],
)
self.assertEqual(
"docs/schemas/ops_doctor_decision_contract.md",
contract_registry["schema_docs"]["ops_doctor_decision_contract"],
)
self.assertEqual(
"docs/schemas/ops_go_live_signoff_contract.md",
contract_registry["schema_docs"]["ops_go_live_signoff_contract"],
)
self.assertEqual(
"docs/schemas/ops_go_live_bundle_contract.md",
contract_registry["schema_docs"]["ops_go_live_bundle_contract"],
)
self.assertEqual(
"docs/schemas/ops_go_live_review_contract.md",
contract_registry["schema_docs"]["ops_go_live_review_contract"],
)
blueprint = get_ops_blueprint()
self.assertEqual("2026-04-18", blueprint["contract_registry"]["registry_version"])
self.assertEqual(11, blueprint["contract_registry"]["contracts_total"])
with patch("app.services.ops_service.get_runtime_status") as mock_runtime_status, patch(
"app.services.ops_service.list_managed_nodes_with_agent_state"
) as mock_list_managed_nodes_with_agent_state, patch(
"app.services.ops_service.get_release_launchpad"
) as mock_get_release_launchpad, patch(
"app.services.ops_service._attach_ops_runbook_sequence_resolutions"
) as mock_attach_sequences:
mock_runtime_status.return_value = {
"readiness": {"status": "ready", "summary": "ok"},
"worker": {"running": True},
"sync_agent": {"running": True},
"detect": {},
}
mock_list_managed_nodes_with_agent_state.return_value = {"nodes": [], "summary": {}}
mock_get_release_launchpad.return_value = {}
mock_attach_sequences.return_value = []
runbook = get_ops_runbook()
self.assertIn("GET /api/v1/ops/contracts", runbook["fast_checks"])
if __name__ == "__main__":
unittest.main()

View File

@@ -0,0 +1,149 @@
from __future__ import annotations
import json
import tempfile
import unittest
from pathlib import Path
from unittest.mock import patch
from app.services.ops_service import get_ops_doctor_decision
class OpsServiceDoctorDecisionTests(unittest.TestCase):
@patch("app.services.ops_service.get_ops_contract_registry")
@patch("app.services.ops_service._build_live_doctor_payload")
@patch("app.services.ops_service._find_latest_go_live_bundle_manifest")
def test_doctor_decision_falls_back_to_live_payload_when_bundle_missing(
self,
mock_find_latest_go_live_bundle_manifest,
mock_build_live_doctor_payload,
mock_get_ops_contract_registry,
) -> None:
mock_find_latest_go_live_bundle_manifest.return_value = (None, Path("/tmp/go-live-bundles"))
mock_build_live_doctor_payload.return_value = (
{
"status": "attention",
"reason_code": "live_fallback",
"headline": "当前先补 attention 项",
"detail": "需要先顺着 stack next 收口。",
"preferred_surface": "stack-summary",
"next_action_code": "review_stack_next",
"recommended_commands": ["bash primary"],
"evidence": {},
},
{
"ok": False,
"required_failures": [],
"optional_unavailable": [],
"scene_log_reports_total": 0,
"scene_log_reports_ok": 0,
"scene_log_status_counts": {},
"scene_log_reports": [],
"contract_surface_gaps": [],
},
)
mock_get_ops_contract_registry.return_value = {
"contracts_by_key": {
"ops_doctor_decision_contract": {"version": "v1"},
}
}
payload = get_ops_doctor_decision(base_url="http://127.0.0.1:8100")
self.assertEqual("attention", payload["status"])
self.assertEqual("主决策待复核", payload["status_label"])
self.assertEqual("live_fallback", payload["source"])
self.assertEqual("review_stack_next", payload["decision"]["next_action_code"])
self.assertIn("doctor-decision", payload["recommended_commands"]["doctor_decision"])
self.assertEqual("bash primary", payload["recommended_commands"]["primary_recovery"])
self.assertEqual(
"bash domain-api/deploy/multi-region/drive_ops_center.sh stack-diagnosis http://127.0.0.1:8100 summary",
payload["recommended_commands"]["stack_check"],
)
self.assertEqual(
payload["recommended_commands"]["stack_check"],
payload["recommended_commands"]["stack_diagnosis"],
)
@patch("app.services.ops_service.get_ops_contract_registry")
def test_doctor_decision_reads_bundle_doctor_artifact_when_available(self, mock_get_ops_contract_registry) -> None:
mock_get_ops_contract_registry.return_value = {
"contracts_by_key": {
"ops_doctor_decision_contract": {"version": "v1"},
}
}
with tempfile.TemporaryDirectory() as temp_dir:
report_dir = Path(temp_dir) / "go-live-bundle-001"
report_dir.mkdir(parents=True, exist_ok=True)
doctor_artifact_path = report_dir / "07_doctor_decision.json"
doctor_artifact_path.write_text(
json.dumps(
{
"summary": {
"ok": False,
"required_failures": ["06_contracts_registry.txt"],
"optional_unavailable": ["12_activity_preview.txt"],
"scene_log_reports_total": 2,
"scene_log_reports_ok": 1,
"scene_log_status_counts": {"ok": 1, "missing": 1},
"scene_log_reports": [
{"node_code": "mainland-worker-01", "status": "ok", "ok": True},
{"node_code": "mainland-controller-01", "status": "missing", "ok": False},
],
"contract_surface_gaps": ["activity-stream"],
"launchpad_recommended_target_node_code": "mainland-worker-01",
"launchpad_recommended_recovery_label": "先补协议面",
"launchpad_recommended_recovery_summary": "先补 driver / activity 的协议导航。",
},
"decision": {
"status": "blocked",
"reason_code": "contracts_registry_unavailable",
"headline": "contract registry 当前不可用",
"detail": "先恢复 contracts registry 再继续自动驾驶。",
"preferred_surface": "contracts",
"next_action_code": "restore_contract_registry",
"recommended_commands": [
"bash domain-api/deploy/multi-region/drive_ops_center.sh contracts http://127.0.0.1:8100",
"bash domain-api/deploy/multi-region/drive_ops_center.sh runtime-refresh-recover http://127.0.0.1:8100",
],
"evidence": {"surface_http_status": {"contracts_registry": 503}},
},
},
ensure_ascii=False,
indent=2,
),
encoding="utf-8",
)
manifest_path = report_dir / "manifest.json"
manifest_path.write_text(
json.dumps(
{
"report_dir": str(report_dir),
"artifacts": [
{"key": "doctor_decision", "path": str(doctor_artifact_path), "ok": True},
],
},
ensure_ascii=False,
indent=2,
),
encoding="utf-8",
)
with patch(
"app.services.ops_service._find_latest_go_live_bundle_manifest",
return_value=(manifest_path, report_dir),
):
payload = get_ops_doctor_decision(base_url="http://127.0.0.1:8100")
self.assertEqual("blocked", payload["status"])
self.assertEqual("bundle_manifest", payload["source"])
self.assertEqual("contracts", payload["decision"]["preferred_surface"])
self.assertEqual(["06_contracts_registry.txt"], payload["summary"]["required_failures"])
self.assertEqual(["mainland-controller-01"], payload["summary"]["scene_log_problem_node_codes"])
self.assertIn("contracts", payload["recommended_commands"]["primary_recovery"])
self.assertIn(str(manifest_path), payload["recommended_commands"]["doctor_decision"])
if __name__ == "__main__":
unittest.main()

View File

@@ -0,0 +1,274 @@
import unittest
from unittest.mock import patch
from app.services.ops_service import preview_driver_action
class OpsServiceDriverActionPreviewTests(unittest.TestCase):
def test_preview_driver_action_builds_safe_auto_contract_preview(self) -> None:
ok, message, data = preview_driver_action(
{
"source_label": "overview-recommendation",
"title": "先补现场日志回传",
"summary": "优先打开关键日志回传。",
"reason": "当前还看不到现场过程日志。",
"executor_kind": "driver_action",
"focus_ref": {
"kind": "execution_scene",
"scene_key": "scene.logs",
},
"primary_focus_ref": {
"kind": "execution_scene",
"scene_key": "scene.logs",
},
"primary_label": "开启关键回传",
"primary_action_code": "enable_log_sync_key",
"primary_action_payload": {},
"requested_by": "web-ui/ops-center",
}
)
self.assertTrue(ok, message)
self.assertEqual("驾驶动作预览已生成", message)
self.assertEqual("ops_driver_contract", data["contract_key"])
self.assertEqual("/api/v1/ops/driver-feed", data["contract_primary_endpoint"])
self.assertEqual("/api/v1/ops/driver-actions/preview", data["preview_endpoint"])
self.assertEqual("driver_action", data["executor_kind"])
self.assertEqual(
"ops-driver -> driver-action execute -> playbook / ui-intent / settings",
data["execution_chain"],
)
primary = data["primary"]
self.assertEqual("开启关键回传", primary["label"])
self.assertEqual("enable_log_sync_key", primary["action_code"])
self.assertEqual("/api/v1/ops/driver-actions/execute", primary["target_api"])
self.assertEqual("可自动执行", primary["recommendation_label"])
self.assertEqual("safe_auto", primary["automation_level_label"])
self.assertEqual("low", primary["risk_level_label"])
self.assertEqual("execution_scene", data["focus_ref"]["kind"])
self.assertEqual("execution_scene", data["primary_focus_ref"]["kind"])
self.assertEqual("execution_scene", primary["focus_ref"]["kind"])
self.assertEqual(
{
"action_code": "enable_log_sync_key",
"node_codes": [],
"action_payload": {
"focus_ref": {
"kind": "execution_scene",
"scene_key": "scene.logs",
}
},
"requested_by": "web-ui/ops-center",
},
primary["request_payload"],
)
self.assertFalse(data["secondary"])
def test_preview_driver_action_marks_missing_required_fields_as_blocked(self) -> None:
ok, message, data = preview_driver_action(
{
"source_label": "driver-feed",
"title": "打开全量现场取证",
"summary": "当前需要进入取证工作区。",
"reason": "需要明确 playbook 上下文。",
"executor_kind": "driver_action",
"primary_label": "打开取证",
"primary_action_code": "open_playbook_dialog",
"primary_action_payload": {},
"requested_by": "web-ui/ops-center",
}
)
self.assertTrue(ok, message)
primary = data["primary"]
self.assertEqual("当前阻断", primary["recommendation_label"])
self.assertTrue(primary["execution_profile"]["blocked"])
self.assertEqual(["playbook_key"], primary["execution_profile"]["missing_fields"])
def test_preview_driver_action_marks_release_launchpad_review_as_safe_auto(self) -> None:
ok, message, data = preview_driver_action(
{
"source_label": "overview-recommendation",
"title": "查看 Worker 预案",
"summary": "当前建议先复核 Worker launchpad 预案。",
"reason": "需要先判断目标节点、门禁和阻断缺口。",
"executor_kind": "driver_action",
"focus_ref": {
"kind": "release_hub",
"release_id": 88,
"section": "release_launchpad",
"mode": "worker",
},
"primary_focus_ref": {
"kind": "release_hub",
"release_id": 88,
"section": "release_launchpad",
"mode": "worker",
},
"primary_label": "查看 Worker 预案",
"primary_action_code": "review_smart_rollout_preview",
"primary_action_payload": {
"channel": "stable",
},
"requested_by": "web-ui/ops-center",
}
)
self.assertTrue(ok, message)
self.assertEqual("驾驶动作预览已生成", message)
primary = data["primary"]
self.assertEqual("review_smart_rollout_preview", primary["action_code"])
self.assertEqual("可自动执行", primary["recommendation_label"])
self.assertEqual("safe_auto", primary["automation_level_label"])
self.assertEqual("low", primary["risk_level_label"])
self.assertFalse(primary["execution_profile"]["ui_only"])
self.assertEqual("/api/v1/ops/driver-actions/execute", primary["target_api"])
def test_preview_driver_action_marks_handover_gap_as_safe_auto(self) -> None:
ok, message, data = preview_driver_action(
{
"source_label": "overview-recommendation",
"title": "先补接管缺口",
"summary": "当前存在待接管节点。",
"reason": "需要先查看首台缺口节点的接管详情。",
"executor_kind": "driver_action",
"focus_ref": {
"kind": "managed_node",
"node_code": "mainland-worker-01",
},
"primary_focus_ref": {
"kind": "managed_node",
"node_code": "mainland-worker-01",
},
"primary_label": "查看接管详情",
"primary_action_code": "handover_first_gap",
"primary_action_payload": {
"node_code": "mainland-worker-01",
},
"requested_by": "web-ui/ops-center",
}
)
self.assertTrue(ok, message)
primary = data["primary"]
self.assertEqual("handover_first_gap", primary["action_code"])
self.assertEqual("可自动执行", primary["recommendation_label"])
self.assertEqual("safe_auto", primary["automation_level_label"])
self.assertEqual("low", primary["risk_level_label"])
self.assertFalse(primary["execution_profile"]["ui_only"])
@patch("app.services.ops_service.get_ops_runbook")
def test_preview_driver_action_supports_runbook_sequence_primary_and_secondary(self, mock_get_ops_runbook) -> None:
mock_get_ops_runbook.return_value = {
"control_sequences": [
{
"key": "release_progression",
"title": "版本发布与放量",
"status": "ready",
"status_label": "可发 Worker 灰度",
"summary": "当前可以进入正式发布路径。",
"reason": "统一回到 Release / Rollout 闭环。",
"target_scope_label": "Release / Rollout / 2026.04.18",
"step_titles": ["确认发布包", "发灰度"],
"target_node_codes": ["mainland-controller-01"],
"focus_ref": {
"kind": "release_hub",
"release_id": 7,
"section": "release_launchpad",
},
"primary_label": "Worker 灰度",
"primary_action_code": "create_release_rollout_worker",
"primary_action_payload": {"release_id": 7},
"primary_focus_ref": {
"kind": "release_hub",
"release_id": 7,
"section": "release_launchpad",
"mode": "worker",
},
"secondary_label": "查看版本区",
"secondary_action_code": "focus_release_hub",
"secondary_action_payload": {"release_id": 7},
"secondary_focus_ref": {
"kind": "release_hub",
"release_id": 7,
"section": "release_detail",
},
}
]
}
ok, message, data = preview_driver_action(
{
"source_label": "driver-feed",
"title": "版本发布与放量",
"summary": "当前可以进入正式发布路径。",
"reason": "统一回到 Release / Rollout 闭环。",
"executor_kind": "runbook_sequence",
"sequence_key": "release_progression",
"primary_label": "Worker 灰度",
"requested_by": "web-ui/ops-center",
}
)
self.assertTrue(ok, message)
self.assertEqual("驾驶动作预览已生成", message)
self.assertEqual("runbook_sequence", data["executor_kind"])
self.assertEqual("runbook sequence", data["executor_kind_label"])
self.assertEqual(
"ops-driver -> runbook-sequence -> resolved driver action / ui-intent",
data["execution_chain"],
)
self.assertEqual("create_release_rollout_worker", data["resolved_primary"]["driver_action_code"])
self.assertEqual("release_launchpad", data["focus_ref"]["section"])
self.assertEqual("worker", data["primary_focus_ref"]["mode"])
self.assertEqual("release_detail", data["secondary_focus_ref"]["section"])
primary = data["primary"]
self.assertEqual("/api/v1/ops/runbook/sequences/release_progression/execute", primary["target_api"])
self.assertEqual("建议确认后执行", primary["recommendation_label"])
self.assertEqual("guarded_auto", primary["automation_level_label"])
self.assertEqual("worker", primary["focus_ref"]["mode"])
self.assertEqual(
{
"secondary": False,
"requested_by": "web-ui/ops-center",
"node_codes": ["mainland-controller-01"],
"action_payload": {
"release_id": 7,
"focus_ref": {
"kind": "release_hub",
"release_id": 7,
"section": "release_launchpad",
"mode": "worker",
},
},
},
primary["request_payload"],
)
secondary = data["secondary"]
self.assertEqual("查看版本区", secondary["label"])
self.assertEqual("focus_release_hub", secondary["action_code"])
self.assertEqual("release_detail", secondary["focus_ref"]["section"])
self.assertEqual(
{
"secondary": True,
"requested_by": "web-ui/ops-center",
"node_codes": ["mainland-controller-01"],
"action_payload": {
"release_id": 7,
"focus_ref": {
"kind": "release_hub",
"release_id": 7,
"section": "release_detail",
},
},
},
secondary["request_payload"],
)
self.assertEqual("可自动执行", secondary["recommendation_label"])
if __name__ == "__main__":
unittest.main()

View File

@@ -0,0 +1,433 @@
from __future__ import annotations
import unittest
from unittest.mock import patch
from app.services.ops_service import execute_resolved_driver_action, resolve_driver_action_request
def _build_preview_data(
*,
executor_kind: str,
primary_action_code: str,
recommendation: str,
automation_level: str,
target_api: str,
primary_request_payload: dict,
sequence_key: str = "",
secondary_action_code: str = "",
secondary_request_payload: dict | None = None,
secondary_recommendation: str = "",
secondary_automation_level: str = "",
secondary_target_api: str = "",
focus_ref: dict | None = None,
primary_focus_ref: dict | None = None,
secondary_focus_ref: dict | None = None,
) -> dict:
return {
"executor_kind": executor_kind,
"sequence_key": sequence_key,
"execution_chain": "ops-driver -> preview",
"focus_ref": dict(focus_ref or {}),
"primary_focus_ref": dict(primary_focus_ref or {}),
"secondary_focus_ref": dict(secondary_focus_ref or {}),
"primary": {
"label": "主动作",
"action_code": primary_action_code,
"node_codes": ["mainland-controller-01"],
"target_api": target_api,
"target_method": "POST",
"request_payload": dict(primary_request_payload or {}),
"focus_ref": dict(primary_focus_ref or {}),
"execution_profile": {
"action_code": primary_action_code,
"backend_handled": True,
"automation_level": automation_level,
"recommendation": recommendation,
"executor_mode_hint": "test-mode",
"risk_level": "low" if recommendation == "auto_execute" else "high",
"confirm_required": recommendation == "confirm_then_execute",
"ui_only": recommendation == "open_ui",
"blocked": recommendation == "blocked",
"requires_fields": [],
"missing_fields": [],
"reason": "preview reason",
},
},
"secondary": (
{
"label": "次动作",
"action_code": secondary_action_code,
"node_codes": ["mainland-controller-01"],
"target_api": secondary_target_api or target_api,
"target_method": "POST",
"request_payload": dict(secondary_request_payload or {}),
"focus_ref": dict(secondary_focus_ref or {}),
"execution_profile": {
"action_code": secondary_action_code,
"backend_handled": True,
"automation_level": secondary_automation_level or automation_level,
"recommendation": secondary_recommendation or recommendation,
"executor_mode_hint": "test-mode",
"risk_level": "low" if (secondary_recommendation or recommendation) == "auto_execute" else "high",
"confirm_required": (secondary_recommendation or recommendation) == "confirm_then_execute",
"ui_only": (secondary_recommendation or recommendation) == "open_ui",
"blocked": (secondary_recommendation or recommendation) == "blocked",
"requires_fields": [],
"missing_fields": [],
"reason": "preview reason",
},
}
if secondary_action_code
else {}
),
}
class OpsServiceDriverActionResolveTests(unittest.TestCase):
@patch("app.services.ops_service.preview_driver_action")
def test_resolve_driver_action_request_returns_gate_for_safe_auto(self, mock_preview_driver_action) -> None:
request_payload = {
"action_code": "enable_log_sync_key",
"node_codes": ["mainland-controller-01"],
"action_payload": {},
"requested_by": "test/driver-action",
}
mock_preview_driver_action.return_value = (
True,
"ok",
_build_preview_data(
executor_kind="driver_action",
primary_action_code="enable_log_sync_key",
recommendation="auto_execute",
automation_level="safe_auto",
target_api="/api/v1/ops/driver-actions/execute",
primary_request_payload=request_payload,
focus_ref={
"kind": "execution_scene",
"scene_key": "scene.logs",
},
primary_focus_ref={
"kind": "execution_scene",
"scene_key": "scene.logs",
},
),
)
ok, message, data = resolve_driver_action_request(
{
"source_label": "driver-feed",
"title": "开启关键日志回传",
"focus_ref": {
"kind": "execution_scene",
"scene_key": "scene.logs",
},
"primary_focus_ref": {
"kind": "execution_scene",
"scene_key": "scene.logs",
},
"primary_action_code": "enable_log_sync_key",
"primary_action_payload": {},
"requested_by": "test/driver-action",
}
)
self.assertTrue(ok)
self.assertEqual("驾驶动作解析完成", message)
self.assertEqual("execute_now", data["gate"]["decision"])
self.assertTrue(data["gate"]["will_execute"])
self.assertEqual("/api/v1/ops/driver-actions/execute", data["gate"]["target_api"])
self.assertEqual("enable_log_sync_key", data["preview_request"]["primary_action_code"])
self.assertEqual("execution_scene", data["preview_request"]["focus_ref"]["kind"])
self.assertEqual("scene.logs", data["gate"]["focus_ref"]["scene_key"])
self.assertEqual("scene.logs", data["focus_ref"]["scene_key"])
@patch("app.services.ops_service.preview_driver_action")
def test_resolve_driver_action_request_requires_confirmation_for_guarded_runbook(self, mock_preview_driver_action) -> None:
request_payload = {
"secondary": False,
"requested_by": "test/driver-action",
"action_payload": {"release_id": 7},
}
mock_preview_driver_action.return_value = (
True,
"ok",
_build_preview_data(
executor_kind="runbook_sequence",
primary_action_code="create_release_rollout_worker",
recommendation="confirm_then_execute",
automation_level="guarded_auto",
target_api="/api/v1/ops/runbook/sequences/release_progression/execute",
primary_request_payload=request_payload,
sequence_key="release_progression",
focus_ref={
"kind": "release_hub",
"release_id": 7,
"section": "release_launchpad",
},
primary_focus_ref={
"kind": "release_hub",
"release_id": 7,
"section": "release_launchpad",
"mode": "worker",
},
),
)
ok, _, data = resolve_driver_action_request(
{
"source_label": "driver-feed",
"title": "版本发布与放量",
"executor_kind": "runbook_sequence",
"sequence_key": "release_progression",
"focus_ref": {
"kind": "release_hub",
"release_id": 7,
"section": "release_launchpad",
},
"primary_focus_ref": {
"kind": "release_hub",
"release_id": 7,
"section": "release_launchpad",
"mode": "worker",
},
"primary_action_code": "create_release_rollout_worker",
"primary_action_payload": {"release_id": 7},
"requested_by": "test/driver-action",
}
)
self.assertTrue(ok)
self.assertEqual("confirmation_required", data["gate"]["decision"])
self.assertFalse(data["gate"]["will_execute"])
self.assertTrue(data["gate"]["confirm_required"])
self.assertEqual("release_progression", data["gate"]["sequence_key"])
self.assertEqual("worker", data["preview_request"]["primary_focus_ref"]["mode"])
self.assertEqual("worker", data["gate"]["focus_ref"]["mode"])
self.assertEqual("worker", data["focus_ref"]["mode"])
@patch("app.services.ops_service.preview_driver_action")
def test_resolve_driver_action_request_can_select_secondary_open_ui_section(self, mock_preview_driver_action) -> None:
mock_preview_driver_action.return_value = (
True,
"ok",
_build_preview_data(
executor_kind="runbook_sequence",
primary_action_code="create_release_rollout_worker",
recommendation="confirm_then_execute",
automation_level="guarded_auto",
target_api="/api/v1/ops/runbook/sequences/release_progression/execute",
primary_request_payload={
"secondary": False,
"requested_by": "test/driver-action",
"action_payload": {"release_id": 7},
},
sequence_key="release_progression",
secondary_action_code="focus_release_hub",
secondary_request_payload={
"secondary": True,
"requested_by": "test/driver-action",
"action_payload": {"release_id": 7},
},
secondary_recommendation="auto_execute",
secondary_automation_level="safe_auto",
secondary_target_api="/api/v1/ops/runbook/sequences/release_progression/execute",
focus_ref={
"kind": "release_hub",
"release_id": 7,
"section": "release_launchpad",
},
primary_focus_ref={
"kind": "release_hub",
"release_id": 7,
"section": "release_launchpad",
"mode": "worker",
},
secondary_focus_ref={
"kind": "release_hub",
"release_id": 7,
"section": "release_detail",
},
),
)
ok, _, data = resolve_driver_action_request(
{
"source_label": "driver-feed",
"title": "版本发布与放量",
"executor_kind": "runbook_sequence",
"sequence_key": "release_progression",
"focus_ref": {
"kind": "release_hub",
"release_id": 7,
"section": "release_launchpad",
},
"primary_focus_ref": {
"kind": "release_hub",
"release_id": 7,
"section": "release_launchpad",
"mode": "worker",
},
"secondary_focus_ref": {
"kind": "release_hub",
"release_id": 7,
"section": "release_detail",
},
"primary_action_code": "create_release_rollout_worker",
"secondary_action_code": "focus_release_hub",
"primary_action_payload": {"release_id": 7},
"secondary_action_payload": {"release_id": 7},
"requested_by": "test/driver-action",
"secondary": True,
}
)
self.assertTrue(ok)
self.assertEqual("secondary", data["selected_section"])
self.assertEqual("execute_now", data["gate"]["decision"])
self.assertTrue(data["gate"]["will_execute"])
self.assertEqual("focus_release_hub", data["gate"]["action_code"])
self.assertEqual("release_detail", data["gate"]["focus_ref"]["section"])
self.assertEqual("release_detail", data["focus_ref"]["section"])
@patch("app.services.ops_service.preview_driver_action")
def test_execute_resolved_driver_action_blocks_guarded_action_without_confirm(self, mock_preview_driver_action) -> None:
mock_preview_driver_action.return_value = (
True,
"ok",
_build_preview_data(
executor_kind="runbook_sequence",
primary_action_code="create_release_rollout_worker",
recommendation="confirm_then_execute",
automation_level="guarded_auto",
target_api="/api/v1/ops/runbook/sequences/release_progression/execute",
primary_request_payload={
"secondary": False,
"requested_by": "test/driver-action",
"action_payload": {"release_id": 7},
},
sequence_key="release_progression",
),
)
ok, message, data = execute_resolved_driver_action(
{
"source_label": "driver-feed",
"title": "版本发布与放量",
"executor_kind": "runbook_sequence",
"sequence_key": "release_progression",
"primary_action_code": "create_release_rollout_worker",
"primary_action_payload": {"release_id": 7},
"requested_by": "test/driver-action",
}
)
self.assertFalse(ok)
self.assertIn("需要显式确认", message)
self.assertFalse(data["executed"])
self.assertEqual("confirmation_required", data["gate"]["decision"])
@patch("app.services.ops_service.execute_driver_action")
@patch("app.services.ops_service.preview_driver_action")
def test_execute_resolved_driver_action_runs_safe_auto_driver_action(
self,
mock_preview_driver_action,
mock_execute_driver_action,
) -> None:
request_payload = {
"action_code": "enable_log_sync_key",
"node_codes": ["mainland-controller-01"],
"action_payload": {},
"requested_by": "test/driver-action",
}
mock_preview_driver_action.return_value = (
True,
"ok",
_build_preview_data(
executor_kind="driver_action",
primary_action_code="enable_log_sync_key",
recommendation="auto_execute",
automation_level="safe_auto",
target_api="/api/v1/ops/driver-actions/execute",
primary_request_payload=request_payload,
),
)
mock_execute_driver_action.return_value = (
True,
"日志回传已开启",
{"handled": True, "mode": "ops-job"},
)
ok, message, data = execute_resolved_driver_action(
{
"source_label": "driver-feed",
"title": "开启关键日志回传",
"primary_action_code": "enable_log_sync_key",
"primary_action_payload": {},
"requested_by": "test/driver-action",
}
)
self.assertTrue(ok)
self.assertEqual("日志回传已开启", message)
self.assertTrue(data["executed"])
self.assertEqual({"handled": True, "mode": "ops-job"}, data["execution_result"])
mock_execute_driver_action.assert_called_once_with(request_payload)
@patch("app.services.ops_service.execute_ops_runbook_sequence")
@patch("app.services.ops_service.preview_driver_action")
def test_execute_resolved_driver_action_runs_guarded_runbook_after_confirm(
self,
mock_preview_driver_action,
mock_execute_ops_runbook_sequence,
) -> None:
request_payload = {
"secondary": False,
"requested_by": "test/driver-action",
"action_payload": {"release_id": 7},
}
mock_preview_driver_action.return_value = (
True,
"ok",
_build_preview_data(
executor_kind="runbook_sequence",
primary_action_code="create_release_rollout_worker",
recommendation="confirm_then_execute",
automation_level="guarded_auto",
target_api="/api/v1/ops/runbook/sequences/release_progression/execute",
primary_request_payload=request_payload,
sequence_key="release_progression",
),
)
mock_execute_ops_runbook_sequence.return_value = (
True,
"Release rollout 已创建",
{
"handled": True,
"mode": "smart-release-rollout",
"driver_action_code": "create_release_rollout_worker",
},
)
ok, message, data = execute_resolved_driver_action(
{
"source_label": "driver-feed",
"title": "版本发布与放量",
"executor_kind": "runbook_sequence",
"sequence_key": "release_progression",
"primary_action_code": "create_release_rollout_worker",
"primary_action_payload": {"release_id": 7},
"requested_by": "test/driver-action",
"confirm": True,
}
)
self.assertTrue(ok)
self.assertEqual("Release rollout 已创建", message)
self.assertTrue(data["executed"])
self.assertEqual("execute_confirmed", data["gate"]["decision"])
mock_execute_ops_runbook_sequence.assert_called_once_with("release_progression", request_payload)
if __name__ == "__main__":
unittest.main()

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,358 @@
import unittest
from unittest.mock import patch
from app.services.ops_service import get_ops_driver_feed
class OpsServiceDriverFeedTests(unittest.TestCase):
@patch("app.services.ops_service.get_ops_go_live_summary")
@patch("app.services.ops_service.get_ops_runbook")
@patch("app.services.ops_service.get_ops_overview")
def test_driver_feed_merges_priority_recommendation_driver_cards_and_runbook_sequences(
self,
mock_get_ops_overview,
mock_get_ops_runbook,
mock_get_ops_go_live_summary,
) -> None:
mock_get_ops_go_live_summary.return_value = {
"go_live_status": "blocked",
"publish_ready": False,
"publish_status": "blocked",
"publish_status_label": "暂不可发布",
"publish_summary": "当前仍有硬阻断项,不能直接进入正式发版。",
"operator_title": "先补接管缺口",
"next_step_action_code": "handover_first_gap",
"launchpad_recommended_target_node_code": "mainland-controller-01",
"launchpad_recommended_recovery_label": "签发接入工单",
"launchpad_recommended_recovery_summary": "当前节点仍处于接入阶段,建议先签发 onboarding.bootstrap。",
"launchpad_onboarding_bootstrap_pending_nodes": 1,
"launchpad_onboarding_acceptance_ready_nodes": 1,
"blocking_reasons": ["remote_agent_ready=0"],
"warnings": ["log_sync_waiting_sample=0/2"],
}
mock_get_ops_overview.return_value = {
"execution_scene": {
"participation_summary": {
"participating_nodes": 2,
"dispatch_active_node_codes": ["mainland-worker-01", "mainland-controller-01"],
},
"participating_nodes": [
{"node_code": "mainland-worker-01"},
{"node_code": "mainland-controller-01"},
],
"log_sync": {
"enabled": False,
"mode": "key",
"line_count": 0,
"source_node_count": 0,
"participating_node_count": 2,
"covered_participating_node_count": 0,
"missing_participating_nodes": ["mainland-worker-01", "mainland-controller-01"],
},
},
"recommendation": {
"key": "fix-handover-gap",
"priority": "先补接管缺口",
"title": "先补接管缺口",
"summary": "当前应该先把接管链路补齐。",
"reason": "否则后续批量动作无法稳定下发。",
"level_label": "最高优先",
"tag_type": "danger",
"primary_label": "纳管首台节点",
"secondary_label": "查看目标节点",
"primary_action_code": "handover_first_gap",
"secondary_action_code": "view_first_gap",
"node_codes": ["mainland-worker-02"],
"primary_node_codes": ["mainland-worker-02"],
"secondary_node_codes": ["mainland-worker-02"],
"primary_action_payload": {"requested_by": "test"},
"secondary_action_payload": {},
"reasons": ["当前存在接管缺口。"],
},
"activity_stream": {
"items": [
{
"kind": "ops_job",
"activity_key": "ops-job:501",
"title": "logs.collect",
"subtitle": "ops-logs-501",
"summary_text": "远端 Agent 已回传 120 行 Worker 日志,可直接继续分析。",
"summary": "远端 Agent 已回传 120 行 Worker 日志,可直接继续分析。",
"status": "running",
"status_label": "执行中",
"meta_text": "ops-logs-501 / 执行 远端 Agent",
"occurred_at": "2026-04-18 09:18:00",
"target_node_codes": ["mainland-worker-01"],
"focus_ref": {
"kind": "ops_job",
"job_id": 501,
"job_code": "ops-logs-501",
"action": "logs.collect",
"target_node_code": "mainland-worker-01",
},
"source_focus_ref": {
"kind": "ops_job_event",
"job_id": 501,
"event_key": "job-event:501",
},
"ui_intent": {
"kind": "job_events",
"job_id": 501,
"job_code": "ops-logs-501",
"action": "logs.collect",
"target_node_code": "mainland-worker-01",
},
}
]
},
"driver_recommendations": [
{
"key": "fix-handover-gap",
"title": "先补接管缺口",
"summary": "重复建议,不应再次进入 feed。",
"reason": "duplicate",
"level_label": "最高优先",
"tag_type": "danger",
"primary_label": "纳管首台节点",
"primary_action_code": "handover_first_gap",
},
{
"key": "inspect-scene-log-gap",
"title": "排查现场日志样本缺口",
"summary": "现场日志样本还没有形成。",
"reason": "先看 Worker 日志。",
"meta_text": "参与节点 2 台 / 已回传来源 0 台",
"level_label": "推荐",
"tag_type": "warning",
"primary_label": "看 Worker 日志",
"secondary_label": "执行标准巡检",
"primary_action_code": "open_worker_logs",
"secondary_action_code": "run_standard_inspection",
"node_codes": ["mainland-worker-01", "mainland-controller-01"],
"primary_node_codes": ["mainland-worker-01", "mainland-controller-01"],
"secondary_node_codes": ["mainland-worker-01", "mainland-controller-01"],
"primary_action_payload": {"lines": 120},
"secondary_action_payload": {},
},
],
}
mock_get_ops_runbook.return_value = {
"control_sequences": [
{
"key": "release_progression",
"title": "版本发布与放量",
"summary": "当前可进入发布路径。",
"reason": "正式运维时应统一落回 Release / Rollout。",
"status": "warning",
"status_label": "待处理",
"tag_type": "warning",
"target_scope_label": "Release / Rollout",
"target_node_codes": ["mainland-controller-01"],
"focus_ref": {
"kind": "release_hub",
"release_id": 7,
"section": "release_launchpad",
},
"primary_label": "补执行面接管",
"secondary_label": "查看版本区",
"primary_action_code": "fix_managed_nodes",
"secondary_action_code": "focus_release_hub",
"primary_focus_ref": {
"kind": "release_hub",
"release_id": 7,
"section": "default_rollout_gate",
},
"secondary_focus_ref": {
"kind": "release_hub",
"release_id": 7,
"section": "release_detail",
},
"step_titles": ["同步集群节点", "纳管首台缺口", "补齐 Agent 接入"],
"primary_resolution": {
"ok": True,
"driver_action_code": "fix_managed_nodes",
"driver_action_label": "补执行面接管",
"driver_node_codes": ["mainland-controller-01"],
"driver_action_payload": {"channel": "stable"},
"resolved_at": "2026-04-18T03:00:00",
},
"secondary_resolution": {
"ok": True,
"driver_action_code": "focus_release_hub",
"driver_action_label": "查看版本区",
"driver_node_codes": [],
"driver_action_payload": {"release_id": 7},
},
}
]
}
payload = get_ops_driver_feed()
self.assertEqual(3, payload["summary"]["total"])
self.assertEqual("blocked", payload["go_live_summary"]["go_live_status"])
self.assertEqual("blocked", payload["summary"]["go_live_status"])
self.assertEqual("先补接管缺口", payload["summary"]["operator_title"])
self.assertEqual("blocked", payload["summary"]["publish_status"])
self.assertFalse(payload["summary"]["publish_ready"])
self.assertEqual("暂不可发布", payload["summary"]["publish_status_label"])
self.assertIn("发布闸门:暂不可发布", payload["headline"])
self.assertIn("接入缺口签发接入工单mainland-controller-01", payload["headline"])
self.assertEqual("handover_first_gap", payload["go_live_summary"]["next_step_action_code"])
self.assertEqual("mainland-controller-01", payload["summary"]["launchpad_recommended_target_node_code"])
self.assertEqual("签发接入工单", payload["summary"]["launchpad_recommended_recovery_label"])
self.assertEqual(
"当前节点仍处于接入阶段,建议先签发 onboarding.bootstrap。",
payload["summary"]["launchpad_recommended_recovery_summary"],
)
self.assertEqual(1, payload["summary"]["launchpad_onboarding_bootstrap_pending_nodes"])
self.assertEqual(1, payload["summary"]["launchpad_onboarding_acceptance_ready_nodes"])
self.assertEqual(2, payload["summary"]["do_now_total"])
self.assertEqual(1, payload["summary"]["standard_path_total"])
self.assertEqual("ops_driver_contract", payload["contract_navigation"]["primary_contract_key"])
self.assertIn("ops_playbook_contract", payload["contract_navigation"]["contract_keys"])
self.assertIn("ops_observability_contract", payload["contract_navigation"]["contract_keys"])
self.assertEqual("priority:fix-handover-gap", payload["top_recommendation"]["key"])
self.assertEqual("ops_driver_contract", payload["top_recommendation"]["contract_navigation"]["primary_contract_key"])
self.assertEqual(2, len(payload["driver_recommendations"]))
self.assertEqual(1, len(payload["runbook_sequences"]))
self.assertEqual(1, len(payload["activity_focus"]))
self.assertEqual(
["priority_recommendation", "driver_recommendation", "runbook_sequence"],
[item["kind"] for item in payload["entries"]],
)
self.assertEqual(1, payload["summary"]["activity_focus_total"])
self.assertEqual("disabled", payload["summary"]["scene_log_status"])
self.assertEqual("blocked", payload["summary"]["launch_status"])
self.assertFalse(payload["summary"]["launch_ready"])
self.assertEqual(2, payload["summary"]["preview_only_total"])
self.assertEqual(3, payload["summary"]["backend_handled_total"])
self.assertEqual("blocked", payload["automation_coverage"]["launch_status"])
self.assertEqual(2, payload["automation_coverage"]["safe_auto_total"])
self.assertEqual(0, payload["automation_coverage"]["guarded_auto_total"])
self.assertEqual(2, payload["automation_coverage"]["auto_execute_total"])
self.assertEqual(1, payload["automation_coverage"]["resolve_first_total"])
self.assertEqual(2, payload["automation_coverage"]["preview_only_total"])
self.assertEqual(3, payload["automation_coverage"]["backend_handled_total"])
self.assertEqual(1, payload["automation_coverage"]["human_dependency_total"])
self.assertFalse(payload["automation_coverage"]["launch_ready"])
priority_entry = payload["entries"][0]
self.assertEqual("do_now", priority_entry["lane"])
self.assertEqual("driver_action", priority_entry["executor_kind"])
self.assertEqual("handover_first_gap", priority_entry["primary_action_code"])
self.assertEqual("ops_driver_contract", priority_entry["contract_navigation"]["primary_contract_key"])
activity_focus = payload["activity_focus"][0]
self.assertEqual("ops_job", activity_focus["activity_kind"])
self.assertEqual("远端 Agent 已回传 120 行 Worker 日志,可直接继续分析。", activity_focus["summary"])
self.assertEqual("focus_latest_job_events", activity_focus["focus_action_code"])
self.assertEqual("job_events", activity_focus["ui_intent_kind"])
self.assertEqual("ops_job_event", activity_focus["source_focus_ref"]["kind"])
self.assertTrue(activity_focus["observation_only"])
self.assertEqual("ops_observability_contract", activity_focus["contract_navigation"]["primary_contract_key"])
self.assertIn("ops_stack_diagnosis_contract", activity_focus["contract_navigation"]["contract_keys"])
scene_log_observation = payload["scene_log_observation"]
self.assertEqual("disabled", scene_log_observation["status"])
self.assertEqual("mainland-worker-01", scene_log_observation["target_node_code"])
self.assertEqual("node_scene_log", scene_log_observation["focus_ref"]["kind"])
self.assertEqual(
"ops_observability_contract",
scene_log_observation["contract_navigation"]["primary_contract_key"],
)
runbook_entry = payload["entries"][2]
self.assertEqual("runbook_sequence", runbook_entry["kind"])
self.assertEqual("standard_path", runbook_entry["lane"])
self.assertEqual("runbook_sequence", runbook_entry["executor_kind"])
self.assertEqual("release_progression", runbook_entry["sequence_key"])
self.assertEqual("fix_managed_nodes", runbook_entry["primary_action_code"])
self.assertEqual("focus_release_hub", runbook_entry["secondary_action_code"])
self.assertIn("当前解析:补执行面接管", runbook_entry["detail_lines"][0])
self.assertEqual("release_hub", runbook_entry["focus_ref"]["kind"])
self.assertEqual("release_launchpad", runbook_entry["focus_ref"]["section"])
self.assertEqual("default_rollout_gate", runbook_entry["primary_focus_ref"]["section"])
self.assertEqual("release_detail", runbook_entry["secondary_focus_ref"]["section"])
self.assertEqual("default_rollout_gate", runbook_entry["primary_action_payload"]["focus_ref"]["section"])
self.assertEqual("release_detail", runbook_entry["secondary_action_payload"]["focus_ref"]["section"])
self.assertEqual("ops_driver_contract", runbook_entry["contract_navigation"]["primary_contract_key"])
self.assertIn("ops_playbook_contract", runbook_entry["contract_navigation"]["contract_keys"])
self.assertIn("release_hub_contract", runbook_entry["contract_navigation"]["contract_keys"])
@patch("app.services.ops_service.get_ops_go_live_summary")
@patch("app.services.ops_service.get_ops_runbook")
@patch("app.services.ops_service.get_ops_overview")
def test_driver_feed_preserves_bootstrap_run_as_top_recommendation(
self,
mock_get_ops_overview,
mock_get_ops_runbook,
mock_get_ops_go_live_summary,
) -> None:
mock_get_ops_go_live_summary.return_value = {
"go_live_status": "attention",
"publish_ready": False,
"publish_status": "attention",
"publish_status_label": "可发布但建议先复核",
"publish_summary": "当前没有硬阻断,但仍有上线前关注项,建议先完成复核再正式发版。",
"operator_title": "先跑接入收口",
"next_step_action_code": "bootstrap_run",
"launchpad_recommended_target_node_code": "mainland-worker-01",
"launchpad_recommended_recovery_label": "跑接入收口",
"launchpad_recommended_recovery_summary": "mainland-worker-01 还缺接入收口,先执行 bootstrap。",
"launchpad_onboarding_bootstrap_pending_nodes": 1,
"launchpad_onboarding_acceptance_ready_nodes": 0,
"blocking_reasons": [],
"warnings": ["remote_agent_ready=1/2"],
}
mock_get_ops_overview.return_value = {
"recommendation": {
"key": "handover-gap-bootstrap_run",
"priority": "先补接管缺口",
"title": "先补接管缺口",
"summary": "Launchpad 已明确收敛到首台节点接入收口。",
"reason": "mainland-worker-01 还缺接入收口,先执行 bootstrap。",
"level_label": "最高优先",
"tag_type": "warning",
"primary_label": "跑接入收口",
"secondary_label": "查看目标节点",
"primary_action_code": "bootstrap_run",
"secondary_action_code": "view_first_gap",
"node_codes": ["mainland-worker-01"],
"primary_node_codes": ["mainland-worker-01"],
"secondary_node_codes": ["mainland-worker-01"],
"primary_action_payload": {"node_code": "mainland-worker-01"},
"secondary_action_payload": {},
"focus_ref": {"kind": "managed_node", "node_code": "mainland-worker-01"},
"primary_focus_ref": {"kind": "managed_node", "node_code": "mainland-worker-01"},
"secondary_focus_ref": {"kind": "managed_node", "node_code": "mainland-worker-01"},
},
"driver_recommendations": [],
"activity_stream": {"items": []},
"execution_scene": {},
}
mock_get_ops_runbook.return_value = {"control_sequences": []}
payload = get_ops_driver_feed()
self.assertEqual("priority:handover-gap-bootstrap_run", payload["top_recommendation"]["key"])
self.assertEqual("attention", payload["summary"]["publish_status"])
self.assertEqual("可发布但建议先复核", payload["summary"]["publish_status_label"])
self.assertEqual("mainland-worker-01", payload["summary"]["launchpad_recommended_target_node_code"])
self.assertEqual("跑接入收口", payload["summary"]["launchpad_recommended_recovery_label"])
self.assertEqual(1, payload["summary"]["launchpad_onboarding_bootstrap_pending_nodes"])
self.assertEqual(0, payload["summary"]["launchpad_onboarding_acceptance_ready_nodes"])
self.assertEqual("attention", payload["summary"]["launch_status"])
self.assertFalse(payload["summary"]["launch_ready"])
self.assertEqual("bootstrap_run", payload["top_recommendation"]["primary_action_code"])
self.assertEqual("mainland-worker-01", payload["top_recommendation"]["primary_node_codes"][0])
self.assertIn("ops_agent_protocol", payload["top_recommendation"]["contract_navigation"]["contract_keys"])
self.assertEqual("ops_driver_contract", payload["top_recommendation"]["contract_navigation"]["primary_contract_key"])
self.assertEqual(0, payload["automation_coverage"]["safe_auto_total"])
self.assertEqual(1, payload["automation_coverage"]["mixed_total"])
self.assertEqual(0, payload["automation_coverage"]["preview_only_total"])
self.assertEqual(1, payload["automation_coverage"]["backend_handled_total"])
if __name__ == "__main__":
unittest.main()

View File

@@ -0,0 +1,24 @@
import unittest
from app.core.config import settings
from app.services.ops_service import _recommend_driver_execution_mode_for_supported_modes
class OpsServiceExecutionModeRecommendationTests(unittest.TestCase):
def test_recommend_driver_execution_mode_prefers_local_runtime_for_current_node(self) -> None:
result = _recommend_driver_execution_mode_for_supported_modes(
[settings.node_code],
supported_modes=["remote-agent", "local-runtime", "ssh"],
default_mode="remote-agent",
)
self.assertEqual("local-runtime", result["execution_mode"])
self.assertIn("local-runtime", result["reason"])
self.assertEqual(
["remote-agent", "local-runtime", "ssh"],
result["supported_modes"],
)
if __name__ == "__main__":
unittest.main()

View File

@@ -0,0 +1,106 @@
from __future__ import annotations
import json
import tempfile
import unittest
from pathlib import Path
from unittest.mock import patch
from app.services.ops_service import get_ops_go_live_bundle
class OpsServiceGoLiveBundleTests(unittest.TestCase):
@patch("app.services.ops_service.get_ops_contract_registry")
def test_go_live_bundle_marks_missing_when_no_manifest_exists(self, mock_get_ops_contract_registry) -> None:
mock_get_ops_contract_registry.return_value = {
"contracts_by_key": {
"ops_go_live_bundle_contract": {"version": "v1"},
}
}
with patch("app.services.ops_service._find_latest_go_live_bundle_manifest", return_value=(None, Path("/tmp/go-live-bundles"))):
payload = get_ops_go_live_bundle(base_url="http://127.0.0.1:8100")
self.assertEqual("missing", payload["status"])
self.assertFalse(payload["bundle_available"])
self.assertEqual("待导出", payload["status_label"])
self.assertIn("go-live-export", payload["recommended_commands"]["go_live_export"])
@patch("app.services.ops_service.get_ops_contract_registry")
def test_go_live_bundle_reads_manifest_summary_and_review_state(self, mock_get_ops_contract_registry) -> None:
mock_get_ops_contract_registry.return_value = {
"contracts_by_key": {
"ops_go_live_bundle_contract": {"version": "v1"},
}
}
with tempfile.TemporaryDirectory() as temp_dir:
report_dir = Path(temp_dir) / "go-live-bundle-001"
report_dir.mkdir(parents=True, exist_ok=True)
env_audit_path = report_dir / "00_env_audit.txt"
env_audit_path.write_text(
"[8/8] condensed env audit summary\n"
+ json.dumps(
{
"status": "attention",
"headline": "仍有轻微环境缺口",
"missing_items": ["missing_tool:jq"],
"runtime": {"runtime_may_need_restart": True},
"recommended_actions": ["bash domain-api/deploy/multi-region/drive_ops_center.sh runtime-refresh-recover"],
},
ensure_ascii=False,
),
encoding="utf-8",
)
manifest_path = report_dir / "manifest.json"
manifest_path.write_text(
json.dumps(
{
"report_dir": str(report_dir),
"artifacts": [
{"key": "env_audit", "path": str(env_audit_path), "ok": True},
{"key": "go_live_summary", "path": str(report_dir / "02_go_live_summary.json"), "ok": True},
],
"failures": ["doctor_export"],
"summary": {
"ok": False,
"artifact_total": 9,
"failure_total": 1,
"review_status_hint": "attention",
"review_headline": "核心报告已齐,但建议先复核补充缺口。",
"launchpad_recommended_target_node_code": "mainland-worker-01",
"launchpad_recommended_recovery_label": "先补接管缺口",
"launchpad_recommended_recovery_summary": "mainland-worker-01 还缺接入收口。",
"launchpad_onboarding_bootstrap_pending_nodes": 1,
"launchpad_onboarding_acceptance_ready_nodes": 0,
"launchpad_alignment": {"consistent": False, "target_node_code_consistent": False},
"recommended_reading_order": ["00_env_audit.txt", "02_go_live_summary.json"],
"contract_surface_gaps": ["ops_go_live_bundle_contract"],
"discovered_contract_keys": ["ops_go_live_signoff_contract"],
},
},
ensure_ascii=False,
indent=2,
),
encoding="utf-8",
)
with patch(
"app.services.ops_service._find_latest_go_live_bundle_manifest",
return_value=(manifest_path, report_dir),
):
payload = get_ops_go_live_bundle(base_url="http://127.0.0.1:8100")
self.assertEqual("attention", payload["status"])
self.assertTrue(payload["bundle_available"])
self.assertEqual(str(manifest_path), payload["manifest_path"])
self.assertEqual(9, payload["artifact_total"])
self.assertEqual(["doctor_export"], payload["noncritical_failures"])
self.assertEqual("attention", payload["env_audit"]["status"])
self.assertTrue(payload["env_audit"]["runtime_may_need_restart"])
self.assertFalse(payload["summary"]["launchpad_alignment"]["consistent"])
self.assertIn(str(manifest_path), payload["recommended_commands"]["go_live_signoff"])
if __name__ == "__main__":
unittest.main()

View File

@@ -0,0 +1,169 @@
from __future__ import annotations
import json
import tempfile
import unittest
from pathlib import Path
from unittest.mock import patch
from app.services.ops_service import get_ops_go_live_review
class OpsServiceGoLiveReviewTests(unittest.TestCase):
@patch("app.services.ops_service.get_ops_contract_registry")
def test_go_live_review_marks_missing_when_no_manifest_exists(self, mock_get_ops_contract_registry) -> None:
mock_get_ops_contract_registry.return_value = {
"contracts_by_key": {
"ops_go_live_review_contract": {"version": "v1"},
}
}
with patch("app.services.ops_service._find_latest_go_live_bundle_manifest", return_value=(None, Path("/tmp/go-live-bundles"))):
payload = get_ops_go_live_review(base_url="http://127.0.0.1:8100")
self.assertEqual("missing", payload["status"])
self.assertEqual("待导出", payload["status_label"])
self.assertIn("go-live-export", payload["recommended_commands"]["go_live_export"])
@patch("app.services.ops_service.get_ops_contract_registry")
def test_go_live_review_reads_manifest_and_alignment(self, mock_get_ops_contract_registry) -> None:
mock_get_ops_contract_registry.return_value = {
"contracts_by_key": {
"ops_go_live_review_contract": {"version": "v1"},
}
}
with tempfile.TemporaryDirectory() as temp_dir:
report_dir = Path(temp_dir) / "go-live-bundle-001"
report_dir.mkdir(parents=True, exist_ok=True)
env_audit_path = report_dir / "00_env_audit.txt"
env_audit_path.write_text(
"[8/8] condensed env audit summary\n"
+ json.dumps(
{
"status": "attention",
"headline": "环境仍有轻微缺口",
"missing_items": ["missing_tool:jq"],
"recommended_actions": ["bash runtime-refresh-recover"],
},
ensure_ascii=False,
),
encoding="utf-8",
)
go_live_summary_path = report_dir / "02_go_live_summary.json"
go_live_summary_path.write_text(
json.dumps(
{
"go_live_status": "attention",
"publish_status": "attention",
"operator_title": "先补接入缺口",
"next_step_action_code": "fix_managed_nodes",
"launchpad_recommended_target_node_code": "mainland-worker-01",
"launchpad_recommended_recovery_label": "先补接入缺口",
"launchpad_recommended_recovery_summary": "mainland-worker-01 还缺接入收口。",
"launchpad_onboarding_bootstrap_pending_nodes": 1,
"launchpad_onboarding_acceptance_ready_nodes": 0,
},
ensure_ascii=False,
),
encoding="utf-8",
)
stack_diagnosis_path = report_dir / "03_stack_diagnosis.json"
stack_diagnosis_path.write_text(
json.dumps(
{
"diagnosis": {
"stack_status": "attention",
"operator_decision": {"title": "先补接入缺口"},
"next_step": {"action_code": "fix_managed_nodes"},
"launchpad_recommended_target_node_code": "mainland-worker-02",
"launchpad_recommended_recovery_label": "先看节点接管",
"launchpad_recommended_recovery_summary": "stack 与 summary 存在漂移。",
"launchpad_onboarding_bootstrap_pending_nodes": 1,
"launchpad_onboarding_acceptance_ready_nodes": 0,
}
},
ensure_ascii=False,
),
encoding="utf-8",
)
driver_feed_path = report_dir / "04_driver_feed.json"
driver_feed_path.write_text(
json.dumps(
{
"headline": "driver ok",
"summary": {
"launchpad_recommended_target_node_code": "mainland-worker-01",
"launchpad_recommended_recovery_label": "先补接入缺口",
"launchpad_recommended_recovery_summary": "mainland-worker-01 还缺接入收口。",
"launchpad_onboarding_bootstrap_pending_nodes": 1,
"launchpad_onboarding_acceptance_ready_nodes": 0,
},
"automation_coverage": {"launch_status": "attention", "launch_ready": False},
},
ensure_ascii=False,
),
encoding="utf-8",
)
codex_brief_path = report_dir / "05_codex_brief.json"
codex_brief_path.write_text(
json.dumps(
{
"headline": "codex ok",
"summary": {
"launchpad_recommended_target_node_code": "mainland-worker-01",
"launchpad_recommended_recovery_label": "先补接入缺口",
"launchpad_recommended_recovery_summary": "mainland-worker-01 还缺接入收口。",
"launchpad_onboarding_bootstrap_pending_nodes": 1,
"launchpad_onboarding_acceptance_ready_nodes": 0,
},
"automation_coverage": {"launch_status": "attention", "launch_ready": False},
"focus": {"entry_key": "fix_managed_nodes"},
},
ensure_ascii=False,
),
encoding="utf-8",
)
manifest_path = report_dir / "manifest.json"
manifest_path.write_text(
json.dumps(
{
"report_dir": str(report_dir),
"artifacts": [
{"key": "env_audit", "path": str(env_audit_path), "ok": True},
{"key": "go_live_summary", "path": str(go_live_summary_path), "ok": True},
{"key": "stack_diagnosis", "path": str(stack_diagnosis_path), "ok": True},
{"key": "driver_feed", "path": str(driver_feed_path), "ok": True},
{"key": "codex_brief", "path": str(codex_brief_path), "ok": True},
{"key": "release_launchpad", "path": str(report_dir / "06_release_launchpad.json"), "ok": True},
],
"failures": ["doctor_export"],
"summary": {
"artifact_total": 9,
"failure_total": 1,
"recommended_reading_order": ["00_env_audit.txt", "02_go_live_summary.json"],
},
},
ensure_ascii=False,
indent=2,
),
encoding="utf-8",
)
with patch(
"app.services.ops_service._find_latest_go_live_bundle_manifest",
return_value=(manifest_path, report_dir),
):
payload = get_ops_go_live_review(base_url="http://127.0.0.1:8100")
self.assertEqual("attention", payload["status"])
self.assertEqual(9, payload["artifact_total"])
self.assertEqual(["doctor_export"], payload["noncritical_failures"])
self.assertFalse(payload["launchpad_alignment"]["consistent"])
self.assertIn("go_live_summary", payload["launchpad_alignment"]["available_sources"])
self.assertIn("mainland-worker-01", payload["recommended_next_steps"][0] + "".join(payload["recommended_next_steps"]))
self.assertIn(str(manifest_path), payload["recommended_commands"]["go_live_review"])
if __name__ == "__main__":
unittest.main()

View File

@@ -0,0 +1,317 @@
import unittest
from unittest.mock import patch
from app.services.ops_service import get_ops_go_live_signoff
class OpsServiceGoLiveSignoffTests(unittest.TestCase):
@patch("app.services.ops_service.get_ops_contract_registry")
@patch("app.services.ops_service.get_ops_doctor_decision")
@patch("app.services.ops_service.get_ops_go_live_review")
@patch("app.services.ops_service.get_release_launchpad")
@patch("app.services.ops_service.get_ops_codex_brief")
@patch("app.services.ops_service.get_ops_driver_feed")
@patch("app.services.ops_service.get_ops_stack_diagnosis")
@patch("app.services.ops_service.get_ops_go_live_summary")
def test_go_live_signoff_marks_ready_when_all_surfaces_align(
self,
mock_get_ops_go_live_summary,
mock_get_ops_stack_diagnosis,
mock_get_ops_driver_feed,
mock_get_ops_codex_brief,
mock_get_release_launchpad,
mock_get_ops_go_live_review,
mock_get_ops_doctor_decision,
mock_get_ops_contract_registry,
) -> None:
mock_get_ops_go_live_summary.return_value = {
"go_live_status": "ready",
"publish_status": "ready",
"operator_title": "进入发布",
"next_step_action_code": "publish_latest_worker",
"next_step_reason": "当前门禁已收口。",
"launchpad_recommended_target_node_code": "mainland-worker-01",
"launchpad_recommended_recovery_label": "继续发布",
"launchpad_recommended_recovery_summary": "可以进入正式发布。",
"launchpad_onboarding_bootstrap_pending_nodes": 0,
"launchpad_onboarding_acceptance_ready_nodes": 0,
"recommended_commands": {"next_step": "bash next-step"},
}
mock_get_ops_stack_diagnosis.return_value = {
"diagnosis": {
"stack_status": "ready",
"operator_decision": {"title": "进入发布"},
"next_step": {"action_code": "publish_latest_worker"},
}
}
mock_get_ops_driver_feed.return_value = {
"headline": "主车道清晰",
"summary": {
"launch_status": "ready",
"launchpad_recommended_target_node_code": "mainland-worker-01",
"launchpad_recommended_recovery_label": "继续发布",
"launchpad_recommended_recovery_summary": "可以进入正式发布。",
"launchpad_onboarding_bootstrap_pending_nodes": 0,
"launchpad_onboarding_acceptance_ready_nodes": 0,
},
}
mock_get_ops_codex_brief.return_value = {
"headline": "可签字",
"summary": {
"launch_status": "ready",
"launchpad_recommended_target_node_code": "mainland-worker-01",
"launchpad_recommended_recovery_label": "继续发布",
"launchpad_recommended_recovery_summary": "可以进入正式发布。",
"launchpad_onboarding_bootstrap_pending_nodes": 0,
"launchpad_onboarding_acceptance_ready_nodes": 0,
},
"focus": {"entry_key": "publish_latest_worker"},
}
mock_get_release_launchpad.return_value = {
"launchpad_status": {
"status": "ready",
"status_label": "可发布",
"recommended_target_node_code": "mainland-worker-01",
"recommended_recovery_label": "继续发布",
"recommended_recovery_summary": "可以进入正式发布。",
"onboarding_bootstrap_pending_nodes": 0,
"onboarding_acceptance_ready_nodes": 0,
}
}
mock_get_ops_go_live_review.return_value = {
"status": "ready",
"status_label": "可复核发布",
"headline": "review ok",
"report_dir": "/tmp/review",
"manifest_path": "/tmp/review/manifest.json",
"recommended_commands": {"go_live_review": "bash review"},
}
mock_get_ops_doctor_decision.return_value = {
"status": "ready",
"headline": "doctor ok",
"report_dir": "/tmp/review",
"manifest_path": "/tmp/review/manifest.json",
"decision": {
"reason_code": "all_green",
"preferred_surface": "go-live-summary",
"next_action_code": "publish_latest_worker",
},
"recommended_commands": {"doctor_decision": "bash doctor"},
}
mock_get_ops_contract_registry.return_value = {
"contracts_by_key": {
"ops_go_live_signoff_contract": {"version": "v1"},
}
}
payload = get_ops_go_live_signoff(base_url="http://127.0.0.1:8100")
self.assertEqual("ready", payload["signoff_status"])
self.assertEqual("可签字上线", payload["status_label"])
self.assertEqual([], payload["blocked_reasons"])
self.assertEqual([], payload["attention_reasons"])
self.assertTrue(payload["launchpad_alignment"]["consistent"])
self.assertEqual("go-live-summary", payload["decision"]["preferred_surface"])
self.assertIn("go-live-signoff", payload["recommended_commands"]["go_live_signoff"])
self.assertEqual("ready", payload["release_gate"]["go_live_review_status"])
self.assertEqual("ready", payload["release_gate"]["doctor_decision_status"])
self.assertEqual("/tmp/review/manifest.json", payload["manifest_path"])
@patch("app.services.ops_service.get_ops_contract_registry")
@patch("app.services.ops_service.get_ops_doctor_decision")
@patch("app.services.ops_service.get_ops_go_live_review")
@patch("app.services.ops_service.get_release_launchpad")
@patch("app.services.ops_service.get_ops_codex_brief")
@patch("app.services.ops_service.get_ops_driver_feed")
@patch("app.services.ops_service.get_ops_stack_diagnosis")
@patch("app.services.ops_service.get_ops_go_live_summary")
def test_go_live_signoff_marks_attention_when_launchpad_alignment_drifts(
self,
mock_get_ops_go_live_summary,
mock_get_ops_stack_diagnosis,
mock_get_ops_driver_feed,
mock_get_ops_codex_brief,
mock_get_release_launchpad,
mock_get_ops_go_live_review,
mock_get_ops_doctor_decision,
mock_get_ops_contract_registry,
) -> None:
mock_get_ops_go_live_summary.return_value = {
"go_live_status": "ready",
"publish_status": "ready",
"operator_title": "进入发布",
"next_step_action_code": "publish_latest_worker",
"launchpad_recommended_target_node_code": "mainland-worker-01",
"launchpad_recommended_recovery_label": "继续发布",
"launchpad_recommended_recovery_summary": "可以进入正式发布。",
"launchpad_onboarding_bootstrap_pending_nodes": 0,
"launchpad_onboarding_acceptance_ready_nodes": 0,
"recommended_commands": {},
}
mock_get_ops_stack_diagnosis.return_value = {
"diagnosis": {"stack_status": "ready", "operator_decision": {}, "next_step": {}}
}
mock_get_ops_driver_feed.return_value = {
"headline": "主车道清晰",
"summary": {
"launch_status": "ready",
"launchpad_recommended_target_node_code": "mainland-worker-02",
"launchpad_recommended_recovery_label": "继续发布",
"launchpad_recommended_recovery_summary": "可以进入正式发布。",
"launchpad_onboarding_bootstrap_pending_nodes": 0,
"launchpad_onboarding_acceptance_ready_nodes": 0,
},
}
mock_get_ops_codex_brief.return_value = {
"headline": "可签字",
"summary": {
"launch_status": "ready",
"launchpad_recommended_target_node_code": "mainland-worker-01",
"launchpad_recommended_recovery_label": "继续发布",
"launchpad_recommended_recovery_summary": "可以进入正式发布。",
"launchpad_onboarding_bootstrap_pending_nodes": 0,
"launchpad_onboarding_acceptance_ready_nodes": 0,
},
"focus": {},
}
mock_get_release_launchpad.return_value = {
"launchpad_status": {
"status": "ready",
"recommended_target_node_code": "mainland-worker-01",
"recommended_recovery_label": "继续发布",
"recommended_recovery_summary": "可以进入正式发布。",
"onboarding_bootstrap_pending_nodes": 0,
"onboarding_acceptance_ready_nodes": 0,
}
}
mock_get_ops_go_live_review.return_value = {
"status": "attention",
"headline": "review drift",
"recommended_commands": {"go_live_review": "bash review"},
}
mock_get_ops_doctor_decision.return_value = {
"status": "ready",
"headline": "doctor ok",
"decision": {"preferred_surface": "go-live-summary"},
"recommended_commands": {"doctor_decision": "bash doctor"},
}
mock_get_ops_contract_registry.return_value = {
"contracts_by_key": {
"ops_go_live_signoff_contract": {"version": "v1"},
}
}
payload = get_ops_go_live_signoff()
self.assertEqual("attention", payload["signoff_status"])
self.assertIn("launchpad_alignment=inconsistent", payload["attention_reasons"])
self.assertIn("go_live_review=attention", payload["attention_reasons"])
self.assertFalse(payload["launchpad_alignment"]["consistent"])
self.assertFalse(payload["launchpad_alignment"]["target_node_code_consistent"])
self.assertEqual("go-live-review", payload["decision"]["preferred_surface"])
@patch("app.services.ops_service.get_ops_contract_registry")
@patch("app.services.ops_service.get_ops_doctor_decision")
@patch("app.services.ops_service.get_ops_go_live_review")
@patch("app.services.ops_service.get_release_launchpad")
@patch("app.services.ops_service.get_ops_codex_brief")
@patch("app.services.ops_service.get_ops_driver_feed")
@patch("app.services.ops_service.get_ops_stack_diagnosis")
@patch("app.services.ops_service.get_ops_go_live_summary")
def test_go_live_signoff_marks_blocked_when_publish_gate_is_blocked(
self,
mock_get_ops_go_live_summary,
mock_get_ops_stack_diagnosis,
mock_get_ops_driver_feed,
mock_get_ops_codex_brief,
mock_get_release_launchpad,
mock_get_ops_go_live_review,
mock_get_ops_doctor_decision,
mock_get_ops_contract_registry,
) -> None:
mock_get_ops_go_live_summary.return_value = {
"go_live_status": "attention",
"publish_status": "blocked",
"operator_title": "先处理发布阻断",
"next_step_action_code": "fix_rollout_blockers",
"launchpad_recommended_target_node_code": "mainland-worker-01",
"launchpad_recommended_recovery_label": "处理发布阻断",
"launchpad_recommended_recovery_summary": "当前 rollout gate 未通过。",
"launchpad_onboarding_bootstrap_pending_nodes": 0,
"launchpad_onboarding_acceptance_ready_nodes": 0,
"recommended_commands": {},
}
mock_get_ops_stack_diagnosis.return_value = {
"diagnosis": {
"stack_status": "attention",
"operator_decision": {"title": "先看发布门禁"},
"next_step": {"action_code": "fix_rollout_blockers"},
}
}
mock_get_ops_driver_feed.return_value = {
"headline": "发布门禁阻断",
"summary": {
"launch_status": "attention",
"launchpad_recommended_target_node_code": "mainland-worker-01",
"launchpad_recommended_recovery_label": "处理发布阻断",
"launchpad_recommended_recovery_summary": "当前 rollout gate 未通过。",
"launchpad_onboarding_bootstrap_pending_nodes": 0,
"launchpad_onboarding_acceptance_ready_nodes": 0,
},
}
mock_get_ops_codex_brief.return_value = {
"headline": "需先排阻断",
"summary": {
"launch_status": "attention",
"launchpad_recommended_target_node_code": "mainland-worker-01",
"launchpad_recommended_recovery_label": "处理发布阻断",
"launchpad_recommended_recovery_summary": "当前 rollout gate 未通过。",
"launchpad_onboarding_bootstrap_pending_nodes": 0,
"launchpad_onboarding_acceptance_ready_nodes": 0,
},
"focus": {"entry_key": "fix_rollout_blockers"},
}
mock_get_release_launchpad.return_value = {
"launchpad_status": {
"status": "blocked",
"status_label": "发布阻断",
"recommended_action_code": "fix_rollout_blockers",
"recommended_target_node_code": "mainland-worker-01",
"recommended_recovery_label": "处理发布阻断",
"recommended_recovery_summary": "当前 rollout gate 未通过。",
"onboarding_bootstrap_pending_nodes": 0,
"onboarding_acceptance_ready_nodes": 0,
}
}
mock_get_ops_go_live_review.return_value = {
"status": "ready",
"headline": "review ok",
"recommended_commands": {"go_live_review": "bash review"},
}
mock_get_ops_doctor_decision.return_value = {
"status": "attention",
"headline": "doctor focus",
"decision": {
"reason_code": "publish_blocked",
"preferred_surface": "release-launchpad",
"next_action_code": "fix_rollout_blockers",
},
"recommended_commands": {"doctor_decision": "bash doctor"},
}
mock_get_ops_contract_registry.return_value = {
"contracts_by_key": {
"ops_go_live_signoff_contract": {"version": "v1"},
}
}
payload = get_ops_go_live_signoff()
self.assertEqual("blocked", payload["signoff_status"])
self.assertIn("publish_status=blocked", payload["blocked_reasons"])
self.assertIn("release_launchpad=blocked", payload["blocked_reasons"])
self.assertEqual("release-launchpad", payload["decision"]["preferred_surface"])
self.assertEqual("publish_blocked", payload["decision"]["doctor_reason_code"])
if __name__ == "__main__":
unittest.main()

View File

@@ -0,0 +1,664 @@
import unittest
from unittest.mock import patch
from app.services.ops_service import get_ops_go_live_summary
class OpsServiceGoLiveSummaryTests(unittest.TestCase):
@patch("app.services.ops_service.get_ops_overview")
@patch("app.services.ops_service.list_managed_nodes_with_agent_state")
@patch("app.services.ops_service.get_runtime_build_info")
@patch("app.services.ops_service.get_release_launchpad")
@patch("app.services.ops_service.get_ops_contract_registry")
@patch("app.services.ops_service.get_ops_stack_diagnosis")
def test_go_live_summary_exposes_runtime_schema_stale_and_api_restart_command(
self,
mock_get_ops_stack_diagnosis,
mock_get_ops_contract_registry,
mock_get_release_launchpad,
mock_get_runtime_build_info,
mock_list_managed_nodes_with_agent_state,
mock_get_ops_overview,
) -> None:
mock_get_ops_stack_diagnosis.return_value = {
"diagnosis": {
"contract_key": "ops_stack_diagnosis_contract",
"stack_status": "attention",
"next_step": {
"action_code": "api-restart",
"reason": "build schema stale",
"focus_ref": {"kind": "runtime_build_info", "section": "schema"},
},
"operator_decision": {
"lane": "runtime_recovery",
"title": "优先重启控制面 API 并复检运行时版本",
"primary_command_key": "api-restart",
},
},
"release_hub": {
"launchpad_status": "ready",
},
}
mock_get_ops_contract_registry.return_value = {
"registry_version": "2026-04-18",
"contracts": [{"key": "ops_stack_diagnosis_contract"}],
}
mock_get_release_launchpad.return_value = {
"launchpad_status": {
"status": "ready",
"status_label": "可发布",
"recommended_action_code": "create_smart_release_rollout_worker",
}
}
mock_get_runtime_build_info.return_value = {
"commit_sha": "246838ae4c07",
"repository_capabilities": {
"supports_install_command_block": True,
"supports_multi_layout_bootstrap": True,
},
"route_surface": {
"surface_complete": True,
"missing_keys": [],
"expected_paths": {
"ops_contracts": "/api/v1/ops/contracts",
},
},
}
mock_list_managed_nodes_with_agent_state.return_value = {
"summary": {
"managed_enabled": 1,
"remote_access_ready": 1,
"queue_dead_letter_nodes": 0,
}
}
mock_get_ops_overview.return_value = {
"recommendation": {
"title": "先重启 API",
"reason": "build schema stale",
"primary_action_code": "api-restart",
},
"execution_scene": {
"log_sync": {
"enabled": False,
"participating_node_count": 0,
"covered_participating_node_count": 0,
"missing_participating_nodes": [],
}
},
}
payload = get_ops_go_live_summary(base_url="http://127.0.0.1:8100")
self.assertEqual("attention", payload["go_live_status"])
self.assertEqual("attention", payload["publish_status"])
self.assertTrue(payload["runtime_schema_stale"])
self.assertFalse(payload["route_surface_declares_bootstrap_plan"])
self.assertIn("runtime_build_schema_stale", payload["warnings"])
self.assertIn("runtime_build_schema_stale", payload["publish_warnings"])
self.assertEqual(
"bash domain-api/deploy/multi-region/drive_ops_center.sh go-live-recover http://127.0.0.1:8100",
payload["recommended_commands"]["go_live_recover"],
)
self.assertEqual(
"bash domain-api/deploy/multi-region/drive_ops_center.sh runtime-refresh-recover http://127.0.0.1:8100",
payload["recommended_commands"]["api_restart"],
)
self.assertEqual(
"bash domain-api/deploy/multi-region/drive_ops_center.sh runtime-refresh-recover http://127.0.0.1:8100",
payload["recommended_commands"]["runtime_refresh_recover"],
)
self.assertIn("node-bootstrap-plan", payload["recommended_commands"]["bootstrap_plan_recheck"])
@patch("app.services.ops_service.get_ops_overview")
@patch("app.services.ops_service.list_managed_nodes_with_agent_state")
@patch("app.services.ops_service.get_runtime_build_info")
@patch("app.services.ops_service.get_release_launchpad")
@patch("app.services.ops_service.get_ops_contract_registry")
@patch("app.services.ops_service.get_ops_stack_diagnosis")
def test_go_live_summary_highlights_remote_access_and_log_sync_gaps(
self,
mock_get_ops_stack_diagnosis,
mock_get_ops_contract_registry,
mock_get_release_launchpad,
mock_get_runtime_build_info,
mock_list_managed_nodes_with_agent_state,
mock_get_ops_overview,
) -> None:
mock_get_ops_stack_diagnosis.return_value = {
"diagnosis": {
"contract_key": "ops_stack_diagnosis_contract",
"stack_status": "blocked",
"next_step": {
"action_code": "handover_first_gap",
"reason": "来自 overview.recommendation.primary_action_code",
},
"operator_decision": {},
},
"release_hub": {
"launchpad_status": "attention",
},
}
mock_get_ops_contract_registry.return_value = {
"registry_version": "2026-04-18",
"contracts": [{"key": "ops_stack_diagnosis_contract"}],
}
mock_get_release_launchpad.return_value = {
"launchpad_status": {
"status": "attention",
"status_label": "待补执行面",
"recommended_action_code": "fix_managed_nodes",
"focus_ref": {"kind": "release_hub"},
"onboarding_bootstrap_pending_nodes": 1,
"onboarding_acceptance_ready_nodes": 1,
"recommended_target_node_code": "mainland-controller-01",
"recommended_recovery_label": "签发接入工单",
"recommended_recovery_summary": "当前节点仍处于接入阶段,建议先签发 onboarding.bootstrap。",
}
}
mock_get_runtime_build_info.return_value = {
"commit_sha": "246838ae4c07",
"route_surface": {
"surface_complete": True,
"missing_keys": [],
},
}
mock_list_managed_nodes_with_agent_state.return_value = {
"summary": {
"managed_enabled": 3,
"remote_access_ready": 0,
"queue_dead_letter_nodes": 0,
}
}
mock_get_ops_overview.return_value = {
"recommendation": {
"title": "先补接管缺口",
"reason": "当前仍有节点没有进入标准执行器就绪状态。",
"primary_action_code": "handover_first_gap",
},
"execution_scene": {
"log_sync": {
"enabled": True,
"state": "waiting_sample",
"mode": "full",
"participating_node_count": 3,
"covered_participating_node_count": 0,
"missing_participating_nodes": [
"mainland-worker-01",
"mainland-controller-01",
"overseas-control-01",
],
}
},
}
payload = get_ops_go_live_summary(base_url="http://127.0.0.1:8100")
self.assertEqual("blocked", payload["go_live_status"])
self.assertFalse(payload["publish_ready"])
self.assertEqual("blocked", payload["publish_status"])
self.assertEqual("暂不可发布", payload["publish_status_label"])
self.assertIn("stack_diagnosis=blocked", payload["blocking_reasons"])
self.assertIn("stack_diagnosis=blocked", payload["publish_blocking_reasons"])
self.assertIn("remote_agent_ready=0", payload["blocking_reasons"])
self.assertIn("remote_agent_ready=0", payload["publish_blocking_reasons"])
self.assertIn("release_launchpad=attention", payload["warnings"])
self.assertIn("release_launchpad=attention", payload["publish_warnings"])
self.assertIn("launchpad_onboarding_bootstrap_pending=1", payload["warnings"])
self.assertIn("launchpad_onboarding_acceptance_ready=1", payload["warnings"])
self.assertIn("launchpad_onboarding_bootstrap_pending=1", payload["publish_warnings"])
self.assertIn("launchpad_onboarding_acceptance_ready=1", payload["publish_warnings"])
self.assertIn("log_sync_waiting_sample=0/3", payload["warnings"])
self.assertIn("log_sync_waiting_sample=0/3", payload["publish_warnings"])
self.assertTrue(payload["route_surface_complete"])
self.assertEqual("先补接管缺口", payload["operator_title"])
self.assertEqual("handover_first_gap", payload["next_step_action_code"])
self.assertEqual("mainland-controller-01", payload["launchpad_recommended_target_node_code"])
self.assertEqual("签发接入工单", payload["launchpad_recommended_recovery_label"])
self.assertEqual(
"当前节点仍处于接入阶段,建议先签发 onboarding.bootstrap。",
payload["launchpad_recommended_recovery_summary"],
)
self.assertEqual(1, payload["launchpad_onboarding_bootstrap_pending_nodes"])
self.assertEqual(1, payload["launchpad_onboarding_acceptance_ready_nodes"])
self.assertEqual(
"bash domain-api/deploy/multi-region/drive_ops_center.sh driver-resolve http://127.0.0.1:8100 handover_first_gap",
payload["recommended_commands"]["next_step"],
)
self.assertEqual(
"bash domain-api/deploy/multi-region/drive_ops_center.sh driver-resolve http://127.0.0.1:8100 open_worker_logs_participating",
payload["recommended_commands"]["log_sync_logs"],
)
@patch("app.services.ops_service.get_ops_overview")
@patch("app.services.ops_service.list_managed_nodes_with_agent_state")
@patch("app.services.ops_service.get_runtime_build_info")
@patch("app.services.ops_service.get_release_launchpad")
@patch("app.services.ops_service.get_ops_contract_registry")
@patch("app.services.ops_service.get_ops_stack_diagnosis")
def test_go_live_summary_falls_back_to_stack_handover_hint_when_launchpad_is_generic(
self,
mock_get_ops_stack_diagnosis,
mock_get_ops_contract_registry,
mock_get_release_launchpad,
mock_get_runtime_build_info,
mock_list_managed_nodes_with_agent_state,
mock_get_ops_overview,
) -> None:
mock_get_ops_stack_diagnosis.return_value = {
"diagnosis": {
"contract_key": "ops_stack_diagnosis_contract",
"stack_status": "blocked",
"next_step": {
"action_code": "bootstrap_run",
"reason": "当前节点仍处于接入阶段,建议先签发 onboarding.bootstrap。",
"focus_ref": {"kind": "managed_node", "node_code": "mainland-worker-01"},
},
"operator_decision": {
"lane": "node_handover",
"title": "优先执行节点接管收口",
"primary_command_key": "bootstrap_run",
},
"launchpad_recommended_target_node_code": "mainland-worker-01",
"launchpad_recommended_recovery_label": "签发接入工单",
"launchpad_recommended_recovery_summary": "当前节点仍处于接入阶段,建议先签发 onboarding.bootstrap。",
},
"release_hub": {
"launchpad_status": "attention",
},
}
mock_get_ops_contract_registry.return_value = {
"registry_version": "2026-04-18",
"contracts": [{"key": "ops_stack_diagnosis_contract"}],
}
mock_get_release_launchpad.return_value = {
"launchpad_status": {
"status": "attention",
"status_label": "待补执行面",
"recommended_action_code": "fix_managed_nodes",
}
}
mock_get_runtime_build_info.return_value = {
"commit_sha": "246838ae4c07",
"route_surface": {
"surface_complete": True,
"missing_keys": [],
"expected_paths": {},
},
"repository_capabilities": {},
}
mock_list_managed_nodes_with_agent_state.return_value = {
"summary": {
"managed_enabled": 1,
"remote_access_ready": 0,
"queue_dead_letter_nodes": 0,
}
}
mock_get_ops_overview.return_value = {
"recommendation": {
"title": "先补接管缺口",
"reason": "overview 仍在泛化描述。",
"primary_action_code": "handover_first_gap",
},
"execution_scene": {
"log_sync": {
"enabled": False,
"participating_node_count": 0,
"covered_participating_node_count": 0,
"missing_participating_nodes": [],
}
},
}
payload = get_ops_go_live_summary(base_url="http://127.0.0.1:8100")
self.assertEqual("bootstrap_run", payload["next_step_action_code"])
self.assertEqual("mainland-worker-01", payload["launchpad_recommended_target_node_code"])
self.assertEqual("签发接入工单", payload["launchpad_recommended_recovery_label"])
self.assertIn("onboarding.bootstrap", payload["launchpad_recommended_recovery_summary"])
self.assertEqual(
"bash domain-api/deploy/multi-region/drive_ops_center.sh driver-resolve http://127.0.0.1:8100 bootstrap_run",
payload["recommended_commands"]["next_step"],
)
@patch("app.services.ops_service.get_ops_overview")
@patch("app.services.ops_service.list_managed_nodes_with_agent_state")
@patch("app.services.ops_service.get_runtime_build_info")
@patch("app.services.ops_service.get_release_launchpad")
@patch("app.services.ops_service.get_ops_contract_registry")
@patch("app.services.ops_service.get_ops_stack_diagnosis")
def test_go_live_summary_marks_ready_when_surfaces_and_execution_are_closed(
self,
mock_get_ops_stack_diagnosis,
mock_get_ops_contract_registry,
mock_get_release_launchpad,
mock_get_runtime_build_info,
mock_list_managed_nodes_with_agent_state,
mock_get_ops_overview,
) -> None:
mock_get_ops_stack_diagnosis.return_value = {
"diagnosis": {
"contract_key": "ops_stack_diagnosis_contract",
"stack_status": "healthy",
"next_step": {
"action_code": "run_inspection_participating",
"reason": "继续做上线前复核。",
},
"operator_decision": {
"lane": "go-live",
"title": "可以进入上线复核",
"primary_command_key": "run_inspection_participating",
},
},
"release_hub": {
"launchpad_status": "ready",
},
}
mock_get_ops_contract_registry.return_value = {
"registry_version": "2026-04-18",
"contracts": [{"key": "ops_stack_diagnosis_contract"}],
}
mock_get_release_launchpad.return_value = {
"launchpad_status": {
"status": "ready",
"status_label": "可发布",
"recommended_action_code": "create_smart_release_rollout_worker",
}
}
mock_get_runtime_build_info.return_value = {
"commit_sha": "246838ae4c07",
"route_surface": {
"surface_complete": True,
"missing_keys": [],
},
}
mock_list_managed_nodes_with_agent_state.return_value = {
"summary": {
"managed_enabled": 2,
"remote_access_ready": 2,
"queue_dead_letter_nodes": 0,
}
}
mock_get_ops_overview.return_value = {
"recommendation": {
"title": "继续复核",
"reason": "当前执行面与观测面已收口。",
"primary_action_code": "run_inspection_participating",
},
"execution_scene": {
"log_sync": {
"enabled": True,
"state": "healthy",
"mode": "key",
"participating_node_count": 2,
"covered_participating_node_count": 2,
"missing_participating_nodes": [],
}
},
}
payload = get_ops_go_live_summary(base_url="http://127.0.0.1:8100")
self.assertEqual("ready", payload["go_live_status"])
self.assertTrue(payload["publish_ready"])
self.assertEqual("ready", payload["publish_status"])
self.assertEqual("可发布", payload["publish_status_label"])
self.assertEqual([], payload["blocking_reasons"])
self.assertEqual([], payload["warnings"])
self.assertEqual([], payload["publish_blocking_reasons"])
self.assertEqual([], payload["publish_warnings"])
self.assertEqual("go-live", payload["operator_lane"])
self.assertEqual("run_inspection_participating", payload["operator_primary_command_key"])
self.assertEqual(2, payload["log_sync_covered_nodes"])
self.assertEqual(
"bash domain-api/deploy/multi-region/drive_ops_center.sh driver-resolve http://127.0.0.1:8100 run_inspection_participating",
payload["recommended_commands"]["next_step"],
)
@patch("app.services.ops_service.get_ops_overview")
@patch("app.services.ops_service.list_managed_nodes_with_agent_state")
@patch("app.services.ops_service.get_runtime_build_info")
@patch("app.services.ops_service.get_release_launchpad")
@patch("app.services.ops_service.get_ops_contract_registry")
@patch("app.services.ops_service.get_ops_stack_diagnosis")
def test_go_live_summary_blocks_publish_when_dead_letter_queue_exists(
self,
mock_get_ops_stack_diagnosis,
mock_get_ops_contract_registry,
mock_get_release_launchpad,
mock_get_runtime_build_info,
mock_list_managed_nodes_with_agent_state,
mock_get_ops_overview,
) -> None:
mock_get_ops_stack_diagnosis.return_value = {
"diagnosis": {
"contract_key": "ops_stack_diagnosis_contract",
"stack_status": "healthy",
"next_step": {"action_code": "delivery_queue_replay", "reason": "先清死信。"},
"operator_decision": {},
},
"release_hub": {
"launchpad_status": "ready",
},
}
mock_get_ops_contract_registry.return_value = {
"registry_version": "2026-04-18",
"contracts": [{"key": "ops_stack_diagnosis_contract"}],
}
mock_get_release_launchpad.return_value = {
"launchpad_status": {
"status": "ready",
"status_label": "可发布",
"recommended_action_code": "create_smart_release_rollout_worker",
}
}
mock_get_runtime_build_info.return_value = {
"commit_sha": "246838ae4c07",
"route_surface": {
"surface_complete": True,
"missing_keys": [],
},
}
mock_list_managed_nodes_with_agent_state.return_value = {
"summary": {
"managed_enabled": 2,
"remote_access_ready": 2,
"queue_dead_letter_nodes": 1,
}
}
mock_get_ops_overview.return_value = {
"recommendation": {
"title": "先清死信",
"reason": "当前仍有节点回执死信。",
"primary_action_code": "delivery_queue_replay",
},
"execution_scene": {
"log_sync": {
"enabled": True,
"state": "healthy",
"mode": "key",
"participating_node_count": 2,
"covered_participating_node_count": 2,
"missing_participating_nodes": [],
}
},
}
payload = get_ops_go_live_summary(base_url="http://127.0.0.1:8100")
self.assertEqual("attention", payload["go_live_status"])
self.assertFalse(payload["publish_ready"])
self.assertEqual("blocked", payload["publish_status"])
@patch("app.services.ops_service.get_ops_overview")
@patch("app.services.ops_service.list_managed_nodes_with_agent_state")
@patch("app.services.ops_service.get_runtime_build_info")
@patch("app.services.ops_service.get_release_launchpad")
@patch("app.services.ops_service.get_ops_contract_registry")
@patch("app.services.ops_service.get_ops_stack_diagnosis")
def test_go_live_summary_accepts_log_sync_covered_node_list_and_marks_partial(
self,
mock_get_ops_stack_diagnosis,
mock_get_ops_contract_registry,
mock_get_release_launchpad,
mock_get_runtime_build_info,
mock_list_managed_nodes_with_agent_state,
mock_get_ops_overview,
) -> None:
mock_get_ops_stack_diagnosis.return_value = {
"diagnosis": {
"contract_key": "ops_stack_diagnosis_contract",
"stack_status": "attention",
"next_step": {"action_code": "bootstrap_run", "reason": "继续补大陆接管。"},
"operator_decision": {},
},
"release_hub": {
"launchpad_status": "attention",
},
}
mock_get_ops_contract_registry.return_value = {
"registry_version": "2026-04-18",
"contracts": [{"key": "ops_stack_diagnosis_contract"}],
}
mock_get_release_launchpad.return_value = {
"launchpad_status": {
"status": "attention",
"status_label": "待补执行面",
"recommended_action_code": "fix_managed_nodes",
}
}
mock_get_runtime_build_info.return_value = {
"commit_sha": "246838ae4c07",
"route_surface": {
"surface_complete": True,
"missing_keys": [],
},
}
mock_list_managed_nodes_with_agent_state.return_value = {
"summary": {
"managed_enabled": 3,
"remote_access_ready": 1,
"queue_dead_letter_nodes": 0,
}
}
mock_get_ops_overview.return_value = {
"recommendation": {
"title": "先补大陆接管",
"reason": "当前只有海外样本已形成。",
"primary_action_code": "bootstrap_run",
},
"execution_scene": {
"log_sync": {
"enabled": True,
"state": "partial_coverage",
"mode": "full",
"participating_node_count": 3,
"covered_participating_node_count": 1,
"covered_participating_nodes": ["overseas-control-01"],
"missing_participating_nodes": ["mainland-worker-01", "mainland-controller-01"],
}
},
}
payload = get_ops_go_live_summary(base_url="http://127.0.0.1:8100")
self.assertEqual("attention", payload["go_live_status"])
self.assertEqual(1, payload["log_sync_covered_nodes"])
self.assertIn("log_sync_partial=1/3", payload["warnings"])
self.assertIn("log_sync_partial=1/3", payload["publish_warnings"])
@patch("app.services.ops_service.get_ops_overview")
@patch("app.services.ops_service.list_managed_nodes_with_agent_state")
@patch("app.services.ops_service.get_runtime_build_info")
@patch("app.services.ops_service.get_release_launchpad")
@patch("app.services.ops_service.get_ops_contract_registry")
@patch("app.services.ops_service.get_ops_stack_diagnosis")
def test_go_live_summary_marks_attention_for_start_delivery_gap(
self,
mock_get_ops_stack_diagnosis,
mock_get_ops_contract_registry,
mock_get_release_launchpad,
mock_get_runtime_build_info,
mock_list_managed_nodes_with_agent_state,
mock_get_ops_overview,
) -> None:
mock_get_ops_stack_diagnosis.return_value = {
"diagnosis": {
"contract_key": "ops_stack_diagnosis_contract",
"stack_status": "healthy",
"next_step": {
"action_code": "focus_latest_job_events",
"reason": "先看任务事件。",
},
"operator_decision": {
"lane": "ops_jobs",
"title": "优先检查任务事件流",
"primary_command_key": "focus_latest_job_events",
},
"issues": [
{
"code": "ops_job_start_delivery_failed_local",
"summary": "最近有任务开始回执异常。",
}
],
},
"release_hub": {
"launchpad_status": "ready",
},
}
mock_get_ops_contract_registry.return_value = {
"registry_version": "2026-04-18",
"contracts": [{"key": "ops_stack_diagnosis_contract"}],
}
mock_get_release_launchpad.return_value = {
"launchpad_status": {
"status": "ready",
"status_label": "可发布",
"recommended_action_code": "create_smart_release_rollout_worker",
}
}
mock_get_runtime_build_info.return_value = {
"commit_sha": "246838ae4c07",
"route_surface": {
"surface_complete": True,
"missing_keys": [],
},
}
mock_list_managed_nodes_with_agent_state.return_value = {
"summary": {
"managed_enabled": 1,
"remote_access_ready": 1,
"queue_dead_letter_nodes": 0,
}
}
mock_get_ops_overview.return_value = {
"recommendation": {
"title": "先看任务事件",
"reason": "最近出现开始回执异常。",
"primary_action_code": "focus_latest_job_events",
},
"execution_scene": {
"log_sync": {
"enabled": True,
"state": "healthy",
"mode": "key",
"participating_node_count": 1,
"covered_participating_node_count": 1,
"missing_participating_nodes": [],
}
},
}
payload = get_ops_go_live_summary(base_url="http://127.0.0.1:8100")
self.assertEqual("attention", payload["go_live_status"])
self.assertFalse(payload["publish_ready"])
self.assertEqual("attention", payload["publish_status"])
self.assertEqual(1, payload["activity_start_delivery_issue_total"])
self.assertIn("ops_job_start_delivery_failed_local=1", payload["warnings"])
self.assertIn("ops_job_start_delivery_failed_local=1", payload["publish_warnings"])
self.assertEqual("ops_jobs", payload["operator_lane"])
self.assertEqual("focus_latest_job_events", payload["next_step_action_code"])
if __name__ == "__main__":
unittest.main()

View File

@@ -0,0 +1,182 @@
import unittest
from unittest.mock import patch
from app.services.ops_service import get_ops_link_snapshot
class OpsServiceLinkSnapshotTests(unittest.TestCase):
@patch("app.services.ops_service.get_ops_overview")
def test_link_snapshot_highlights_log_sync_gap_on_active_scene(self, mock_get_ops_overview) -> None:
mock_get_ops_overview.return_value = {
"current_topology": {
"node_code": "overseas-control-01",
"node_region": "overseas",
"node_role": "control",
"worker_mode": "linux-systemd",
"sync_push_enabled": True,
"sync_target_api_base_url": "http://152.53.37.118:8100",
},
"cluster_summary": {
"online_control_nodes": 2,
"online_worker_nodes": 3,
"dedicated_online_worker_nodes": 1,
"busy_nodes": ["mainland-worker-01", "overseas-control-01"],
},
"runtime_summary": {
"readiness_status": "ready",
"readiness_summary": "当前多机与跨地域骨架已进入可联调状态。",
"phase_label": "running",
"phase_detail": "当前任务正在推进",
"active_job_code": "detect-20260417164852-75f7cc",
"progress_percent": 42.5,
},
"execution_scene": {
"participation_summary": {
"effective_online_nodes": 3,
"participating_nodes": 3,
"dispatch_active_nodes": 2,
"recent_only_nodes": 1,
"non_participating_nodes": 0,
"standby_nodes": 0,
"load_syncing_nodes": 0,
"dispatch_active_node_codes": ["mainland-worker-01", "overseas-control-01"],
"non_participating_node_codes": [],
},
"log_sync": {
"enabled": False,
"mode": "key",
"line_count": 0,
"source_node_count": 0,
"covered_participating_node_count": 0,
"participating_node_count": 3,
"missing_participating_nodes": [
"mainland-controller-01",
"mainland-worker-01",
"overseas-control-01",
],
"last_at": "",
"preview_lines": [],
},
},
"sync_summary": {
"enabled": True,
"source_region": "mainland",
"target_region": "overseas",
"projected_batches": 0,
"failed_batches": 0,
"records_total": 2513,
},
"recommendation": {
"key": "enable-scene-log-sync",
"priority": "先补现场日志回传",
"summary": "当前有 3 台节点正在真实参与检测,但海外控制面还看不到过程日志。",
"reason": "先把远端日志回传切到关键模式,才能看到阶段、异常、代理与执行过程。",
"primary_label": "开启关键回传",
"primary_action_code": "enable_log_sync_key",
"primary_node_codes": ["mainland-controller-01", "mainland-worker-01", "overseas-control-01"],
"primary_action_payload": {},
"secondary_label": "开启全量回传",
"secondary_action_code": "enable_log_sync_full",
"secondary_node_codes": ["mainland-controller-01", "mainland-worker-01", "overseas-control-01"],
"secondary_action_payload": {},
},
}
snapshot = get_ops_link_snapshot()
self.assertEqual("attention", snapshot["status"])
self.assertIn("日志回传", snapshot["headline"])
self.assertIn("活跃任务 detect-20260417164852-75f7cc", snapshot["summary_lines"][2])
self.assertIn("远端日志:关闭", snapshot["summary_lines"][4])
self.assertEqual("enable_log_sync_key", snapshot["next_actions"][0]["action_code"])
self.assertEqual(3, snapshot["execution"]["participating_nodes"])
self.assertFalse(snapshot["log_sync"]["enabled"])
@patch("app.services.ops_service.get_ops_overview")
def test_link_snapshot_reports_ready_when_scene_is_observable(self, mock_get_ops_overview) -> None:
mock_get_ops_overview.return_value = {
"current_topology": {
"node_code": "overseas-control-01",
"node_region": "overseas",
"node_role": "control",
"worker_mode": "linux-systemd",
"sync_push_enabled": True,
"sync_target_api_base_url": "http://152.53.37.118:8100",
},
"cluster_summary": {
"online_control_nodes": 2,
"online_worker_nodes": 2,
"dedicated_online_worker_nodes": 1,
"busy_nodes": ["mainland-worker-01"],
},
"runtime_summary": {
"readiness_status": "ready",
"readiness_summary": "ok",
"phase_label": "running",
"phase_detail": "检测任务正常推进",
"active_job_code": "detect-001",
"progress_percent": 66.0,
},
"execution_scene": {
"participation_summary": {
"effective_online_nodes": 2,
"participating_nodes": 2,
"dispatch_active_nodes": 2,
"recent_only_nodes": 0,
"non_participating_nodes": 0,
"standby_nodes": 0,
"load_syncing_nodes": 0,
"dispatch_active_node_codes": ["mainland-worker-01", "mainland-controller-01"],
"non_participating_node_codes": [],
},
"log_sync": {
"enabled": True,
"mode": "key",
"line_count": 18,
"source_node_count": 2,
"covered_participating_node_count": 2,
"participating_node_count": 2,
"missing_participating_nodes": [],
"last_at": "2026-04-17 22:20:00",
"preview_lines": [
"[2026-04-17 22:19:59] [mainland-worker-01] 开始检测域名: alpha.com",
"[2026-04-17 22:20:00] [mainland-controller-01] 域名检测完成: beta.com",
],
},
},
"sync_summary": {
"enabled": True,
"source_region": "mainland",
"target_region": "overseas",
"projected_batches": 0,
"failed_batches": 0,
"records_total": 2600,
},
"recommendation": {
"key": "inspect-participating",
"priority": "先看执行现场",
"summary": "当前有 2 台节点正在真实参与检测,优先回收它们的标准巡检结果。",
"reason": "真正参与检测的节点最接近现场。",
"primary_label": "执行标准巡检",
"primary_action_code": "run_inspection_participating",
"primary_node_codes": ["mainland-worker-01", "mainland-controller-01"],
"primary_action_payload": {},
"secondary_label": "打开诊断模板",
"secondary_action_code": "open_diagnostics_participating",
"secondary_node_codes": ["mainland-worker-01", "mainland-controller-01"],
"secondary_action_payload": {},
},
}
snapshot = get_ops_link_snapshot()
self.assertEqual("ready", snapshot["status"])
self.assertIn("可观测状态", snapshot["headline"])
self.assertIn("覆盖 2/2", snapshot["summary_lines"][4])
self.assertEqual(2, snapshot["log_sync"]["source_node_count"])
self.assertEqual(2, len(snapshot["next_actions"]))
self.assertEqual("run_inspection_participating", snapshot["next_actions"][0]["action_code"])
if __name__ == "__main__":
unittest.main()

View File

@@ -0,0 +1,256 @@
import unittest
from unittest.mock import patch
from app.services.ops_service import get_ops_node_scene_log
class OpsServiceNodeSceneLogTests(unittest.TestCase):
def _runtime_payload_with_worker_logs(self) -> dict:
return {
"detect": {
"log_sync": {
"enabled": True,
"mode": "key",
"line_count": 2,
"source_node_count": 1,
"source_nodes": ["mainland-worker-01"],
"source_node_summaries": [
{
"node_code": "mainland-worker-01",
"line_count": 2,
"key_line_count": 2,
"full_line_count": 0,
"last_at": "2026-04-18 05:02:00",
"last_line": "[2026-04-18 05:02:00] [mainland-worker-01] 域名检测完成: alpha.com",
}
],
"preview_lines": [
"[2026-04-18 05:00:00] [mainland-worker-01] 开始检测域名: alpha.com",
"[2026-04-18 05:02:00] [mainland-worker-01] 域名检测完成: alpha.com",
],
"last_at": "2026-04-18 05:02:00",
"last_line": "[2026-04-18 05:02:00] [mainland-worker-01] 域名检测完成: alpha.com",
},
"participation_summary": {
"participating_nodes": 1,
"dispatch_active_nodes": 1,
"recent_only_nodes": 0,
"non_participating_nodes": 1,
"standby_nodes": 1,
"load_syncing_nodes": 0,
"effective_online_nodes": 2,
"dedicated_worker_nodes": 2,
"controller_worker_nodes": 0,
"dispatch_active_node_codes": ["mainland-worker-01"],
"recent_only_node_codes": [],
"non_participating_node_codes": ["mainland-worker-02"],
"standby_node_codes": ["mainland-worker-02"],
"load_syncing_node_codes": [],
"summary": "有效执行节点 2 台;正在执行/领任务 1 台;近窗刚有吞吐 0 台;在线但未参与 1 台",
},
"participating_nodes": [
{
"node_code": "mainland-worker-01",
"region": "mainland",
"role": "worker",
"status": "busy",
"current_load": 2,
"last_heartbeat_at": "2026-04-18 05:02:10",
"participation_state": "claimed",
"participation_label": "已领待跑",
"participation_reason": "已领取 2 项任务,等待线程继续执行。",
"is_dispatch_active": True,
}
],
"non_participating_nodes": [
{
"node_code": "mainland-worker-02",
"region": "mainland",
"role": "worker",
"status": "online",
"current_load": 0,
"last_heartbeat_at": "2026-04-18 05:02:11",
"participation_state": "standby",
"participation_label": "在线待命",
"participation_reason": "当前节点在线,但尚未参与本轮检测。",
"is_dispatch_active": False,
}
],
"active_job": {
"job_code": "detect-001",
"status": "running",
"current_cycle_token": "cycle-1",
"current_cycle_events": [
{
"event_type": "worker_log",
"created_at": "2026-04-18 05:02:00",
"node_code": "mainland-worker-01",
"message": "域名检测完成: alpha.com",
"payload": {"cycle_token": "cycle-1", "log_mode": "key"},
},
{
"event_type": "worker_log",
"created_at": "2026-04-18 05:01:00",
"node_code": "mainland-worker-01",
"message": "代理池刷新完成",
"payload": {"cycle_token": "cycle-1", "log_mode": "full"},
},
{
"event_type": "worker_log",
"created_at": "2026-04-18 05:00:00",
"node_code": "mainland-worker-01",
"message": "开始检测域名: alpha.com",
"payload": {"cycle_token": "cycle-1", "log_mode": "key"},
},
{
"event_type": "worker_log",
"created_at": "2026-04-18 04:59:00",
"node_code": "mainland-worker-01",
"message": "旧周期日志",
"payload": {"cycle_token": "cycle-0", "log_mode": "key"},
},
],
},
},
"cluster": {
"nodes": [
{
"node_code": "mainland-worker-01",
"region": "mainland",
"role": "worker",
"status": "busy",
"current_load": 2,
"last_heartbeat_at": "2026-04-18 05:02:10",
},
{
"node_code": "mainland-worker-02",
"region": "mainland",
"role": "worker",
"status": "online",
"current_load": 0,
"last_heartbeat_at": "2026-04-18 05:02:11",
},
]
},
}
@patch("app.services.ops_service.get_runtime_status")
def test_node_scene_log_filters_key_mode_records_for_participating_node(self, mock_get_runtime_status) -> None:
mock_get_runtime_status.return_value = self._runtime_payload_with_worker_logs()
payload = get_ops_node_scene_log("mainland-worker-01", limit=10)
self.assertEqual("mainland-worker-01", payload["node_code"])
self.assertEqual("healthy", payload["status"])
self.assertEqual("关键覆盖", payload["status_label"])
self.assertEqual("key", payload["mode"])
self.assertTrue(payload["log_sync_enabled"])
self.assertTrue(payload["participation"]["detect_participating"])
self.assertEqual(2, payload["records_total"])
self.assertEqual(
[
"开始检测域名: alpha.com",
"域名检测完成: alpha.com",
],
[item["message"] for item in payload["records"]],
)
self.assertEqual(2, payload["source_summary"]["line_count"])
self.assertEqual(2, payload["source_summary"]["key_line_count"])
self.assertEqual(0, payload["source_summary"]["full_line_count"])
self.assertEqual("ops_observability_contract", payload["contract_navigation"]["primary_contract_key"])
@patch("app.services.ops_service.get_runtime_status")
def test_node_scene_log_can_request_full_mode_records(self, mock_get_runtime_status) -> None:
mock_get_runtime_status.return_value = self._runtime_payload_with_worker_logs()
payload = get_ops_node_scene_log("mainland-worker-01", limit=10, mode="full")
self.assertEqual("full_capture", payload["status"])
self.assertEqual("full", payload["mode"])
self.assertEqual(3, payload["records_total"])
self.assertEqual(
[
"开始检测域名: alpha.com",
"代理池刷新完成",
"域名检测完成: alpha.com",
],
[item["message"] for item in payload["records"]],
)
self.assertEqual(3, payload["source_summary"]["line_count"])
self.assertEqual(2, payload["source_summary"]["key_line_count"])
self.assertEqual(1, payload["source_summary"]["full_line_count"])
@patch("app.services.ops_service.get_runtime_status")
def test_node_scene_log_marks_waiting_sample_for_missing_participating_node(self, mock_get_runtime_status) -> None:
mock_get_runtime_status.return_value = {
"detect": {
"log_sync": {
"enabled": True,
"mode": "key",
"line_count": 0,
"source_node_count": 0,
"source_nodes": [],
"source_node_summaries": [],
"preview_lines": [],
"last_at": "",
"last_line": "",
},
"participation_summary": {
"participating_nodes": 1,
"dispatch_active_nodes": 1,
"recent_only_nodes": 0,
"non_participating_nodes": 0,
"standby_nodes": 0,
"load_syncing_nodes": 0,
"effective_online_nodes": 1,
"dedicated_worker_nodes": 1,
"controller_worker_nodes": 0,
"dispatch_active_node_codes": ["mainland-worker-02"],
"recent_only_node_codes": [],
"non_participating_node_codes": [],
"standby_node_codes": [],
"load_syncing_node_codes": [],
"summary": "有效执行节点 1 台;正在执行/领任务 1 台;近窗刚有吞吐 0 台;在线但未参与 0 台",
},
"participating_nodes": [
{
"node_code": "mainland-worker-02",
"region": "mainland",
"role": "worker",
"status": "busy",
"current_load": 1,
"last_heartbeat_at": "2026-04-18 05:05:00",
"participation_state": "running",
"participation_label": "正在执行",
"participation_reason": "当前存在执行中工作负载。",
"is_dispatch_active": True,
}
],
"non_participating_nodes": [],
"active_job": {"job_code": "detect-002", "status": "running", "current_cycle_events": []},
},
"cluster": {
"nodes": [
{
"node_code": "mainland-worker-02",
"region": "mainland",
"role": "worker",
"status": "busy",
"current_load": 1,
"last_heartbeat_at": "2026-04-18 05:05:00",
}
]
},
}
payload = get_ops_node_scene_log("mainland-worker-02")
self.assertEqual("waiting_sample", payload["status"])
self.assertEqual("等待样本", payload["status_label"])
self.assertEqual(0, payload["records_total"])
self.assertTrue(payload["participation"]["detect_participating"])
self.assertIn("现场样本尚未形成", payload["summary"])
if __name__ == "__main__":
unittest.main()

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,283 @@
import unittest
from unittest.mock import patch
from app.services.ops_service import get_ops_runbook
class OpsServiceRunbookTests(unittest.TestCase):
@patch("app.services.ops_service.get_release_launchpad")
@patch("app.services.ops_service.get_latest_release")
@patch("app.services.ops_service.list_managed_nodes_with_agent_state")
@patch("app.services.ops_service.get_runtime_status")
def test_ops_runbook_includes_control_sequences(
self,
mock_get_runtime_status,
mock_list_managed_nodes_with_agent_state,
mock_get_latest_release,
mock_get_release_launchpad,
) -> None:
mock_get_runtime_status.return_value = {
"readiness": {
"status": "ready",
"summary": "当前多机现场已进入标准作业状态",
},
"worker": {
"running": True,
},
"sync_agent": {
"running": True,
},
"detect": {
"participation_summary": {
"summary": "worker 正在执行",
}
},
}
mock_list_managed_nodes_with_agent_state.return_value = {
"nodes": [
{
"node_code": "mainland-controller-01",
"role": "control",
"is_managed": True,
"is_enabled": True,
"is_agent_online": True,
"cluster_status": "online",
"cluster_is_effective_worker": True,
"is_current_participant": False,
},
{
"node_code": "mainland-worker-01",
"role": "worker",
"is_managed": True,
"is_enabled": True,
"is_agent_online": True,
"cluster_status": "busy",
"cluster_is_effective_worker": True,
"is_current_participant": True,
},
]
}
mock_get_latest_release.side_effect = lambda channel="stable": (
{
"id": 15,
"release_version": "2026.04.17",
"channel": "stable",
}
if channel == "stable"
else {}
)
mock_get_release_launchpad.return_value = {
"channel": "stable",
"latest_release": {
"id": 15,
"release_version": "2026.04.17",
"channel": "stable",
},
"launchpad_status": {
"status": "ready",
"status_label": "可发 Worker 灰度",
"summary": "最新发布包、Release 记录与 Worker 智能 Rollout 预检均已收口。",
"recommended_action_code": "publish_latest_worker",
"focus_ref": {
"kind": "release_hub",
"release_id": 15,
"section": "release_launchpad",
},
},
"worker_rollout_preview": {
"message": "智能 Rollout 预检通过",
"smart_rollout": {
"available": True,
"summary": "已按 Worker 模式预选 1 台节点。",
"target_node_codes": ["mainland-worker-01"],
},
},
"control_rollout_preview": {
"message": "",
"smart_rollout": {
"available": False,
"target_node_codes": [],
},
},
}
runbook = get_ops_runbook()
mock_list_managed_nodes_with_agent_state.assert_called_once_with(
participation_payload=mock_get_runtime_status.return_value["detect"]
)
self.assertIn("GET /api/v1/ops/releases/launchpad", runbook["fast_checks"])
self.assertEqual("ready", runbook["release_launchpad"]["launchpad_status"]["status"])
sequences = {item["key"]: item for item in runbook["control_sequences"]}
self.assertIn("node_onboarding", sequences)
self.assertIn("scene_observe", sequences)
self.assertIn("standard_inspection", sequences)
self.assertIn("release_progression", sequences)
self.assertEqual("bootstrap_run", sequences["node_onboarding"]["primary_action_code"])
self.assertEqual("run_acceptance", sequences["node_onboarding"]["secondary_action_code"])
self.assertEqual("run_standard_inspection", sequences["standard_inspection"]["primary_action_code"])
self.assertEqual("open_diagnostics", sequences["standard_inspection"]["secondary_action_code"])
self.assertEqual(["mainland-worker-01"], sequences["scene_observe"]["target_node_codes"])
self.assertEqual("publish_latest_worker", sequences["release_progression"]["primary_action_code"])
self.assertEqual(15, sequences["release_progression"]["primary_action_payload"]["release_id"])
self.assertEqual(["mainland-worker-01"], sequences["release_progression"]["target_node_codes"])
self.assertTrue(sequences["release_progression"]["primary_resolution"]["ok"])
self.assertEqual("publish_latest_worker", sequences["release_progression"]["primary_resolution"]["driver_action_code"])
self.assertEqual(["mainland-worker-01"], sequences["release_progression"]["primary_resolution"]["driver_node_codes"])
self.assertEqual("release_hub", sequences["release_progression"]["focus_ref"]["kind"])
self.assertEqual("release_launchpad", sequences["release_progression"]["focus_ref"]["section"])
self.assertEqual("worker", sequences["release_progression"]["primary_focus_ref"]["mode"])
self.assertEqual("worker", sequences["release_progression"]["primary_action_payload"]["focus_ref"]["mode"])
self.assertEqual("release_detail", sequences["release_progression"]["secondary_focus_ref"]["section"])
self.assertTrue(sequences["scene_observe"]["secondary_resolution"]["ok"])
self.assertEqual("run_scene_logs_full", sequences["scene_observe"]["secondary_resolution"]["driver_action_code"])
@patch("app.services.ops_service.get_release_launchpad")
@patch("app.services.ops_service.get_latest_release")
@patch("app.services.ops_service.list_managed_nodes_with_agent_state")
@patch("app.services.ops_service.get_runtime_status")
def test_ops_runbook_release_progression_can_fall_back_to_bootstrap_run(
self,
mock_get_runtime_status,
mock_list_managed_nodes_with_agent_state,
mock_get_latest_release,
mock_get_release_launchpad,
) -> None:
mock_get_runtime_status.return_value = {
"readiness": {
"status": "attention",
"summary": "当前还需补执行面接管",
},
"worker": {"running": True},
"sync_agent": {"running": True},
"detect": {"participation_summary": {"summary": ""}},
}
mock_list_managed_nodes_with_agent_state.return_value = {
"nodes": [
{
"node_code": "mainland-worker-02",
"role": "worker",
"is_managed": False,
"is_enabled": False,
"is_agent_online": False,
"cluster_status": "online",
"cluster_is_effective_worker": True,
"is_current_participant": False,
}
]
}
mock_get_latest_release.return_value = {}
mock_get_release_launchpad.return_value = {
"channel": "stable",
"latest_release": {},
"launchpad_status": {
"status": "attention",
"status_label": "待补执行面",
"summary": "当前节点仍处于接入阶段,建议先签发 onboarding.bootstrap。",
"recommended_action_code": "bootstrap_run",
"focus_ref": {
"kind": "release_hub",
"release_id": 19,
"section": "release_launchpad",
},
},
"worker_rollout_preview": {
"message": "当前没有满足角色条件的在线节点可用于智能 Rollout。",
"smart_rollout": {
"available": False,
"target_node_codes": [],
},
"policy_preview": {
"release_gate": {
"operational_readiness": {
"rows": [
{
"node_code": "mainland-worker-02",
"is_managed": False,
"is_enabled": False,
"is_agent_online": False,
"remote_agent_ready": False,
"inspection_status": "missing",
"recovery_action": "bootstrap_run",
"recovery_summary": "当前节点仍处于接入阶段,建议先签发 onboarding.bootstrap。",
"agent_reason": "节点尚未纳管,当前不能稳定下发 remote-agent 动作。",
}
]
}
}
},
},
"control_rollout_preview": {
"message": "",
"smart_rollout": {
"available": False,
"target_node_codes": [],
},
},
}
runbook = get_ops_runbook()
sequences = {item["key"]: item for item in runbook["control_sequences"]}
self.assertEqual("bootstrap_run", sequences["release_progression"]["primary_action_code"])
self.assertEqual(["mainland-worker-02"], sequences["release_progression"]["target_node_codes"])
self.assertEqual("待补执行面", sequences["release_progression"]["status_label"])
self.assertTrue(sequences["release_progression"]["primary_resolution"]["ok"])
self.assertEqual("bootstrap_run", sequences["release_progression"]["primary_resolution"]["driver_action_code"])
self.assertEqual("default_rollout_gate", sequences["release_progression"]["primary_focus_ref"]["section"])
self.assertEqual("default_rollout_gate", sequences["release_progression"]["primary_action_payload"]["focus_ref"]["section"])
@patch("app.services.ops_service.get_release_launchpad")
@patch("app.services.ops_service.get_latest_release")
@patch("app.services.ops_service.list_managed_nodes_with_agent_state")
@patch("app.services.ops_service.get_runtime_status")
def test_ops_runbook_release_progression_falls_back_to_direct_worker_rollout_when_release_exists(
self,
mock_get_runtime_status,
mock_list_managed_nodes_with_agent_state,
mock_get_latest_release,
mock_get_release_launchpad,
) -> None:
mock_get_runtime_status.return_value = {
"readiness": {
"status": "ready",
"summary": "当前多机现场已进入标准作业状态",
},
"worker": {"running": True},
"sync_agent": {"running": True},
"detect": {"participation_summary": {"summary": ""}},
}
mock_list_managed_nodes_with_agent_state.return_value = {
"nodes": [
{
"node_code": "mainland-worker-01",
"role": "worker",
"is_managed": True,
"is_enabled": True,
"is_agent_online": True,
"cluster_status": "online",
"cluster_is_effective_worker": True,
"is_current_participant": False,
}
]
}
mock_get_latest_release.return_value = {
"id": 18,
"release_version": "2026.04.18",
"channel": "stable",
}
mock_get_release_launchpad.return_value = {}
runbook = get_ops_runbook()
sequences = {item["key"]: item for item in runbook["control_sequences"]}
self.assertEqual("create_release_rollout_worker", sequences["release_progression"]["primary_action_code"])
self.assertEqual("focus_release_hub", sequences["release_progression"]["secondary_action_code"])
self.assertEqual(18, sequences["release_progression"]["primary_action_payload"]["release_id"])
self.assertEqual(["mainland-worker-01"], sequences["release_progression"]["target_node_codes"])
self.assertTrue(sequences["release_progression"]["primary_resolution"]["ok"])
self.assertEqual("create_release_rollout_worker", sequences["release_progression"]["primary_resolution"]["driver_action_code"])
if __name__ == "__main__":
unittest.main()

View File

@@ -0,0 +1,255 @@
import unittest
from unittest.mock import patch
from app.services.ops_service import execute_ops_runbook_sequence, resolve_ops_runbook_sequence
class OpsServiceRunbookSequenceTests(unittest.TestCase):
@patch("app.services.ops_service.get_ops_runbook")
def test_resolve_ops_runbook_sequence_uses_primary_action(self, mock_get_ops_runbook) -> None:
mock_get_ops_runbook.return_value = {
"control_sequences": [
{
"key": "release_progression",
"title": "版本发布与放量",
"status": "ready",
"status_label": "可发 Worker 灰度",
"summary": "可以直接发布。",
"reason": "launchpad 已就绪",
"target_scope_label": "Release / Launchpad / 2026.04.18",
"step_titles": ["确认发布包", "发灰度"],
"target_node_codes": ["mainland-worker-01"],
"focus_ref": {
"kind": "release_hub",
"release_id": 12,
"section": "release_launchpad",
},
"primary_label": "直接发 Worker 灰度",
"primary_action_code": "publish_latest_worker",
"primary_action_payload": {
"channel": "stable",
"release_id": 12,
},
"primary_focus_ref": {
"kind": "release_hub",
"release_id": 12,
"section": "release_launchpad",
"mode": "worker",
},
}
]
}
ok, message, data = resolve_ops_runbook_sequence(
"release_progression",
{
"requested_by": "test",
},
)
self.assertTrue(ok, message)
self.assertEqual("标准作业路径解析成功", message)
self.assertEqual("publish_latest_worker", data["driver_action_code"])
self.assertEqual("直接发 Worker 灰度", data["driver_action_label"])
self.assertEqual(["mainland-worker-01"], data["driver_node_codes"])
self.assertEqual("test/runbook/release_progression", data["driver_requested_by"])
self.assertEqual("Release / Launchpad / 2026.04.18", data["sequence"]["target_scope_label"])
self.assertEqual(["确认发布包", "发灰度"], data["sequence"]["step_titles"])
self.assertEqual("release_launchpad", data["sequence"]["focus_ref"]["section"])
self.assertEqual("worker", data["sequence"]["action_focus_ref"]["mode"])
self.assertEqual("worker", data["driver_action_payload"]["focus_ref"]["mode"])
@patch("app.services.ops_service.get_ops_runbook")
def test_resolve_ops_runbook_sequence_uses_secondary_action_and_allows_payload_override(
self,
mock_get_ops_runbook,
) -> None:
mock_get_ops_runbook.return_value = {
"control_sequences": [
{
"key": "node_onboarding",
"title": "纳管新节点",
"status": "warning",
"status_label": "有待接管节点",
"summary": "先补接入。",
"reason": "新增节点需要先纳管。",
"target_node_codes": ["mainland-worker-01"],
"secondary_label": "跑接管验收",
"secondary_action_code": "run_acceptance",
"secondary_action_payload": {
"node_code": "mainland-worker-01",
},
}
]
}
ok, message, data = resolve_ops_runbook_sequence(
"node_onboarding",
{
"secondary": True,
"requested_by": "test",
"node_codes": ["mainland-worker-02"],
"action_payload": {
"auto_approve": True,
},
},
)
self.assertTrue(ok, message)
self.assertEqual("标准作业路径解析成功", message)
self.assertEqual("run_acceptance", data["driver_action_code"])
self.assertEqual("跑接管验收", data["driver_action_label"])
self.assertEqual(["mainland-worker-02"], data["driver_node_codes"])
self.assertTrue(data["sequence"]["secondary"])
self.assertEqual(
{
"node_code": "mainland-worker-01",
"auto_approve": True,
},
data["driver_action_payload"],
)
@patch("app.services.ops_service.execute_driver_action")
@patch("app.services.ops_service.get_ops_runbook")
def test_execute_ops_runbook_sequence_uses_primary_action(
self,
mock_get_ops_runbook,
mock_execute_driver_action,
) -> None:
mock_get_ops_runbook.return_value = {
"control_sequences": [
{
"key": "release_progression",
"title": "版本发布与放量",
"status": "ready",
"status_label": "可发 Worker 灰度",
"summary": "可以直接发布。",
"reason": "launchpad 已就绪",
"target_node_codes": ["mainland-worker-01"],
"primary_action_code": "publish_latest_worker",
"primary_action_payload": {
"channel": "stable",
"release_id": 12,
},
"secondary_action_code": "focus_release_hub",
"secondary_action_payload": {
"release_id": 12,
},
}
]
}
mock_execute_driver_action.return_value = (
True,
"Release rollout 已创建",
{
"handled": True,
"mode": "smart-release-rollout",
},
)
ok, message, data = execute_ops_runbook_sequence(
"release_progression",
{
"requested_by": "test",
},
)
self.assertTrue(ok, message)
self.assertEqual("Release rollout 已创建", message)
self.assertTrue(data["handled"])
self.assertEqual("publish_latest_worker", data["driver_action_code"])
self.assertEqual(["mainland-worker-01"], data["driver_node_codes"])
self.assertEqual("release_progression", data["sequence_key"])
self.assertEqual("版本发布与放量", data["sequence"]["title"])
mock_execute_driver_action.assert_called_once_with(
{
"action_code": "publish_latest_worker",
"node_codes": ["mainland-worker-01"],
"action_payload": {
"channel": "stable",
"release_id": 12,
},
"requested_by": "test/runbook/release_progression",
}
)
@patch("app.services.ops_service.execute_driver_action")
@patch("app.services.ops_service.get_ops_runbook")
def test_execute_ops_runbook_sequence_uses_secondary_action_and_allows_payload_override(
self,
mock_get_ops_runbook,
mock_execute_driver_action,
) -> None:
mock_get_ops_runbook.return_value = {
"control_sequences": [
{
"key": "node_onboarding",
"title": "纳管新节点",
"status": "warning",
"status_label": "有待接管节点",
"summary": "先补接入。",
"reason": "新增节点需要先纳管。",
"target_node_codes": ["mainland-worker-01"],
"primary_action_code": "bootstrap_run",
"primary_action_payload": {
"node_code": "mainland-worker-01",
},
"secondary_action_code": "run_acceptance",
"secondary_action_payload": {
"node_code": "mainland-worker-01",
},
}
]
}
mock_execute_driver_action.return_value = (
True,
"",
{
"handled": True,
"mode": "ui-intent",
},
)
ok, message, data = execute_ops_runbook_sequence(
"node_onboarding",
{
"secondary": True,
"requested_by": "test",
"node_codes": ["mainland-worker-02"],
"action_payload": {
"auto_approve": True,
},
},
)
self.assertTrue(ok, message)
self.assertEqual("", message)
self.assertTrue(data["handled"])
self.assertEqual("run_acceptance", data["driver_action_code"])
self.assertEqual(["mainland-worker-02"], data["driver_node_codes"])
self.assertTrue(data["sequence"]["secondary"])
mock_execute_driver_action.assert_called_once_with(
{
"action_code": "run_acceptance",
"node_codes": ["mainland-worker-02"],
"action_payload": {
"node_code": "mainland-worker-01",
"auto_approve": True,
},
"requested_by": "test/runbook/node_onboarding",
}
)
@patch("app.services.ops_service.get_ops_runbook")
def test_execute_ops_runbook_sequence_returns_error_when_sequence_missing(self, mock_get_ops_runbook) -> None:
mock_get_ops_runbook.return_value = {"control_sequences": []}
ok, message, data = execute_ops_runbook_sequence("missing-sequence", {})
self.assertFalse(ok)
self.assertEqual("标准作业路径不存在", message)
self.assertEqual("missing-sequence", data["sequence_key"])
if __name__ == "__main__":
unittest.main()

View File

@@ -0,0 +1,971 @@
import unittest
from unittest.mock import patch
from app.services.ops_service import get_ops_stack_diagnosis
class OpsServiceStackDiagnosisTests(unittest.TestCase):
@patch("app.services.ops_service.get_runtime_build_info")
@patch("app.services.ops_service.get_ops_activity_stream")
@patch("app.services.ops_service.get_recent_ops_playbook_runs")
@patch("app.services.ops_service.get_release_launchpad")
@patch("app.services.ops_service.list_managed_nodes_with_agent_state")
@patch("app.services.ops_service.get_ops_overview")
@patch("app.services.ops_service.get_ops_link_snapshot")
@patch("app.services.ops_service.get_ops_contract_registry")
def test_stack_diagnosis_prioritizes_api_restart_when_runtime_build_schema_is_stale(
self,
mock_get_ops_contract_registry,
mock_get_ops_link_snapshot,
mock_get_ops_overview,
mock_list_managed_nodes_with_agent_state,
mock_get_release_launchpad,
mock_get_recent_ops_playbook_runs,
mock_get_ops_activity_stream,
mock_get_runtime_build_info,
) -> None:
mock_get_ops_contract_registry.return_value = {
"registry_version": "2026-04-18",
"schema_version": "v1",
"contracts": [{"key": "ops_stack_diagnosis_contract"}],
}
mock_get_ops_link_snapshot.return_value = {
"status": "ready",
"headline": "当前总检可用。",
"next_actions": [],
}
mock_get_ops_overview.return_value = {
"recommendation": {
"primary_action_code": "fix_managed_nodes",
"title": "先补节点接管",
"focus_ref": {"kind": "managed_node", "node_code": "mainland-worker-01"},
},
"execution_scene": {},
"log_sync": {},
"release_hub": {},
}
mock_list_managed_nodes_with_agent_state.return_value = {"nodes": [], "summary": {}}
mock_get_release_launchpad.return_value = {"launchpad_status": {}, "latest_release": {}}
mock_get_recent_ops_playbook_runs.return_value = {"runs": []}
mock_get_ops_activity_stream.return_value = {"items": []}
mock_get_runtime_build_info.return_value = {
"source": "git",
"commit_sha": "246838ae4c07",
"repository_capabilities": {
"supports_install_command_block": True,
"supports_multi_layout_bootstrap": True,
},
"route_surface": {
"mode": "registered",
"registered_paths_total": 24,
"surface_complete": True,
"missing_keys": [],
"expected_paths": {
"ops_contracts": "/api/v1/ops/contracts",
"ops_stack_diagnosis": "/api/v1/ops/stack-diagnosis",
},
},
}
payload = get_ops_stack_diagnosis(base_url="http://127.0.0.1:8100")
issue = next(item for item in payload["diagnosis"]["issues"] if item["code"] == "runtime_build_schema_stale")
self.assertEqual("api-restart", issue["action_code"])
self.assertEqual("api-restart", payload["diagnosis"]["next_step"]["action_code"])
self.assertEqual("runtime_recovery", payload["diagnosis"]["operator_decision"]["lane"])
self.assertEqual("api-restart", payload["diagnosis"]["operator_decision"]["primary_command_key"])
self.assertEqual(
"bash domain-api/deploy/multi-region/drive_ops_center.sh runtime-refresh-recover http://127.0.0.1:8100",
payload["diagnosis"]["next_actions"]["primary_command"],
)
self.assertEqual(
"bash domain-api/deploy/multi-region/drive_ops_center.sh runtime-refresh-recover http://127.0.0.1:8100",
payload["diagnosis"]["recommended_commands"]["api-restart"],
)
self.assertEqual(
"bash domain-api/deploy/multi-region/drive_ops_center.sh runtime-refresh-recover http://127.0.0.1:8100",
payload["diagnosis"]["recommended_commands"]["runtime-refresh-recover"],
)
self.assertEqual(
"bash domain-api/deploy/multi-region/drive_ops_center.sh stack-diagnosis http://127.0.0.1:8100 summary",
payload["diagnosis"]["recommended_commands"]["stack-diagnosis"],
)
self.assertEqual(
payload["diagnosis"]["recommended_commands"]["stack-diagnosis"],
payload["diagnosis"]["recommended_commands"]["stack-check"],
)
self.assertTrue(payload["runtime_build_info"]["runtime_schema_stale"])
self.assertFalse(payload["runtime_build_info"]["route_surface_declares_bootstrap_plan"])
self.assertTrue(
any("运行中 API 还是旧 schema" in hint for hint in payload["diagnosis"]["operator_hints"])
)
@patch("app.services.ops_service.get_runtime_build_info")
@patch("app.services.ops_service.get_ops_activity_stream")
@patch("app.services.ops_service.get_recent_ops_playbook_runs")
@patch("app.services.ops_service.get_release_launchpad")
@patch("app.services.ops_service.list_managed_nodes_with_agent_state")
@patch("app.services.ops_service.get_ops_overview")
@patch("app.services.ops_service.get_ops_link_snapshot")
@patch("app.services.ops_service.get_ops_contract_registry")
def test_stack_diagnosis_surfaces_runtime_route_surface_gap(
self,
mock_get_ops_contract_registry,
mock_get_ops_link_snapshot,
mock_get_ops_overview,
mock_list_managed_nodes_with_agent_state,
mock_get_release_launchpad,
mock_get_recent_ops_playbook_runs,
mock_get_ops_activity_stream,
mock_get_runtime_build_info,
) -> None:
mock_get_ops_contract_registry.return_value = {
"registry_version": "2026-04-18",
"schema_version": "v1",
"contracts": [],
}
mock_get_ops_link_snapshot.return_value = {
"status": "ready",
"headline": "当前总检可用。",
"next_actions": [],
}
mock_get_ops_overview.return_value = {
"recommendation": {},
"execution_scene": {},
"log_sync": {},
"release_hub": {},
}
mock_list_managed_nodes_with_agent_state.return_value = {"nodes": [], "summary": {}}
mock_get_release_launchpad.return_value = {"launchpad_status": {}, "latest_release": {}}
mock_get_recent_ops_playbook_runs.return_value = {"runs": []}
mock_get_ops_activity_stream.return_value = {"items": []}
mock_get_runtime_build_info.return_value = {
"source": "git",
"commit_sha": "bd6bcb2",
"route_surface": {
"mode": "registered",
"registered_paths_total": 12,
"surface_complete": False,
"missing_keys": [
"ops_node_onboarding_acceptance_execute",
"ops_node_onboarding_acceptance_preview",
],
"missing_paths": [
"/api/v1/ops/nodes/{node_code}/onboarding/acceptance/execute",
"/api/v1/ops/nodes/{node_code}/onboarding/acceptance/preview",
],
},
}
payload = get_ops_stack_diagnosis(base_url="http://127.0.0.1:8100")
issue = next(
item for item in payload["diagnosis"]["issues"] if item["code"] == "runtime_route_surface_incomplete"
)
self.assertEqual("api-restart", issue["action_code"])
self.assertIn("ops_node_onboarding_acceptance_execute", issue["detail"])
self.assertIn(
"bash domain-api/deploy/multi-region/drive_ops_center.sh runtime-refresh-recover http://127.0.0.1:8100",
issue["commands"],
)
self.assertFalse(payload["runtime_build_info"]["route_surface"]["surface_complete"])
self.assertIn(
"ops_node_onboarding_acceptance_preview",
payload["runtime_build_info"]["route_surface_missing_keys"],
)
self.assertTrue(
any("API 路由面还不完整" in hint for hint in payload["diagnosis"]["operator_hints"])
)
@patch("app.services.ops_service.get_ops_activity_stream")
@patch("app.services.ops_service.get_recent_ops_playbook_runs")
@patch("app.services.ops_service.get_release_launchpad")
@patch("app.services.ops_service.list_managed_nodes_with_agent_state")
@patch("app.services.ops_service.get_ops_overview")
@patch("app.services.ops_service.get_ops_link_snapshot")
@patch("app.services.ops_service.get_ops_contract_registry")
def test_stack_diagnosis_emits_operator_decision_for_scene_log_gap(
self,
mock_get_ops_contract_registry,
mock_get_ops_link_snapshot,
mock_get_ops_overview,
mock_list_managed_nodes_with_agent_state,
mock_get_release_launchpad,
mock_get_recent_ops_playbook_runs,
mock_get_ops_activity_stream,
) -> None:
mock_get_ops_contract_registry.return_value = {
"registry_version": "2026-04-18",
"schema_version": "v1",
"contracts": [],
}
mock_get_ops_link_snapshot.return_value = {
"status": "attention",
"headline": "参与节点样本尚未形成。",
"next_actions": [],
}
mock_get_ops_overview.return_value = {
"recommendation": {},
"execution_scene": {
"participating_nodes": [{"node_code": "mainland-worker-01", "status": "busy"}],
"participation_summary": {
"dispatch_active_node_codes": ["mainland-worker-01"],
"recent_only_node_codes": [],
},
"log_sync": {
"enabled": True,
"mode": "key",
"line_count": 0,
"source_node_count": 0,
"covered_participating_node_count": 0,
"participating_node_count": 1,
"missing_participating_nodes": ["mainland-worker-01"],
},
},
"log_sync": {},
"release_hub": {},
}
mock_list_managed_nodes_with_agent_state.return_value = {"nodes": [], "summary": {}}
mock_get_release_launchpad.return_value = {"launchpad_status": {}, "latest_release": {}}
mock_get_recent_ops_playbook_runs.return_value = {"runs": []}
mock_get_ops_activity_stream.return_value = {"items": []}
payload = get_ops_stack_diagnosis(base_url="http://127.0.0.1:8100")
decision = payload["diagnosis"]["operator_decision"]
self.assertEqual("observability", decision["lane"])
self.assertEqual("scene-node-log", decision["primary_command_key"])
self.assertEqual("mainland-worker-01", decision["next_focus"]["node_code"])
self.assertIn("scene-node-log", payload["diagnosis"]["recommended_commands"])
self.assertIn(
"scene-node-log http://127.0.0.1:8100 mainland-worker-01 120 key",
payload["diagnosis"]["next_actions"]["primary_command"],
)
@patch("app.services.ops_service.get_ops_activity_stream")
@patch("app.services.ops_service.get_recent_ops_playbook_runs")
@patch("app.services.ops_service.get_release_launchpad")
@patch("app.services.ops_service.get_managed_node_handover")
@patch("app.services.ops_service.list_managed_nodes_with_agent_state")
@patch("app.services.ops_service.get_ops_overview")
@patch("app.services.ops_service.get_ops_link_snapshot")
@patch("app.services.ops_service.get_ops_contract_registry")
def test_stack_diagnosis_emits_operator_decision_for_managed_node_gap(
self,
mock_get_ops_contract_registry,
mock_get_ops_link_snapshot,
mock_get_ops_overview,
mock_list_managed_nodes_with_agent_state,
mock_get_managed_node_handover,
mock_get_release_launchpad,
mock_get_recent_ops_playbook_runs,
mock_get_ops_activity_stream,
) -> None:
mock_get_ops_contract_registry.return_value = {
"registry_version": "2026-04-18",
"schema_version": "v1",
"contracts": [],
}
mock_get_ops_link_snapshot.return_value = {
"status": "attention",
"headline": "存在接管缺口。",
"next_actions": [],
}
mock_get_ops_overview.return_value = {
"recommendation": {},
"execution_scene": {},
"log_sync": {},
"release_hub": {},
}
mock_list_managed_nodes_with_agent_state.return_value = {
"nodes": [
{
"node_code": "mainland-worker-01",
"agent_state": "pending_bootstrap",
"remote_access_state": "ssh_ready",
"delivery_queue_state": "healthy",
"participation_state": "standby",
}
],
"summary": {
"managed_enabled": 1,
"remote_access_ready": 0,
},
}
mock_get_managed_node_handover.return_value = {
"node_code": "mainland-worker-01",
"summary": "等待执行 bootstrap。",
"stage": {"code": "pending_bootstrap", "label": "待执行接入", "next_step": {"code": "execute_bootstrap_plan"}},
}
mock_get_release_launchpad.return_value = {
"launchpad_status": {
"recommended_action_code": "bootstrap_run",
"focus_ref": {"kind": "release_launchpad", "section": "release_launchpad"},
"recommended_target_node_code": "mainland-worker-01",
},
"worker_rollout_preview": {
"release_gate": {
"operational_readiness": {
"rows": [
{
"node_code": "mainland-worker-01",
"recovery_action": "bootstrap_run",
"recovery_summary": "mainland-worker-01 还缺接入收口,先执行 bootstrap。",
"focus_ref": {"kind": "managed_node", "node_code": "mainland-worker-01"},
}
]
}
}
},
"latest_release": {},
}
mock_get_recent_ops_playbook_runs.return_value = {"runs": []}
mock_get_ops_activity_stream.return_value = {"items": []}
payload = get_ops_stack_diagnosis(base_url="http://127.0.0.1:8100")
decision = payload["diagnosis"]["operator_decision"]
self.assertEqual("node_handover", decision["lane"])
self.assertEqual("bootstrap_run", decision["primary_command_key"])
self.assertEqual("managed-node-bootstrap-plan", decision["secondary_command_key"])
self.assertIn("bootstrap_run", payload["diagnosis"]["recommended_commands"])
self.assertIn("managed-node-handover", payload["diagnosis"]["recommended_commands"])
self.assertIn("managed-node-bootstrap-plan", payload["diagnosis"]["recommended_commands"])
self.assertIn(
"node-bootstrap-run http://127.0.0.1:8100 mainland-worker-01 cli",
payload["diagnosis"]["next_actions"]["primary_command"],
)
self.assertTrue(
any("当前第一动作已经收敛为 接入收口mainland-worker-01" in hint for hint in payload["diagnosis"]["operator_hints"])
)
@patch("app.services.ops_service.get_runtime_build_info")
@patch("app.services.ops_service.get_ops_activity_stream")
@patch("app.services.ops_service.get_recent_ops_playbook_runs")
@patch("app.services.ops_service.get_release_launchpad")
@patch("app.services.ops_service.get_managed_node_handover")
@patch("app.services.ops_service.list_managed_nodes_with_agent_state")
@patch("app.services.ops_service.get_ops_overview")
@patch("app.services.ops_service.get_ops_link_snapshot")
@patch("app.services.ops_service.get_ops_contract_registry")
def test_stack_diagnosis_promotes_first_handover_gap_when_launchpad_is_still_generic(
self,
mock_get_ops_contract_registry,
mock_get_ops_link_snapshot,
mock_get_ops_overview,
mock_list_managed_nodes_with_agent_state,
mock_get_managed_node_handover,
mock_get_release_launchpad,
mock_get_recent_ops_playbook_runs,
mock_get_ops_activity_stream,
mock_get_runtime_build_info,
) -> None:
mock_get_ops_contract_registry.return_value = {
"registry_version": "2026-04-18",
"schema_version": "v1",
"contracts": [{"key": "ops_stack_diagnosis_contract"}],
}
mock_get_ops_link_snapshot.return_value = {
"status": "ready",
"headline": "当前总检可用。",
"next_actions": [],
}
mock_get_ops_overview.return_value = {
"recommendation": {
"primary_action_code": "handover_first_gap",
"title": "先补接管缺口",
"reason": "overview 只能先定位到接管缺口。",
"focus_ref": {"kind": "managed_node", "node_code": "mainland-worker-01"},
},
"execution_scene": {},
"log_sync": {},
"release_hub": {},
}
mock_list_managed_nodes_with_agent_state.return_value = {
"nodes": [
{
"node_code": "mainland-worker-01",
"agent_state": "pending_bootstrap",
"remote_access_state": "agent_pending",
"delivery_queue_state": "healthy",
"participation_state": "standby",
}
],
"summary": {
"managed_enabled": 1,
"remote_access_ready": 0,
},
}
mock_get_managed_node_handover.return_value = {
"node_code": "mainland-worker-01",
"summary": "当前节点仍处于接入阶段,建议先签发 onboarding.bootstrap 并等待 register / heartbeat 建立。",
"stage": {
"code": "pending_bootstrap",
"label": "待执行接入",
"next_step": {
"code": "execute_bootstrap_plan",
"label": "去目标机执行接入片段",
},
},
}
mock_get_release_launchpad.return_value = {
"launchpad_status": {
"recommended_action_code": "fix_managed_nodes",
"status_label": "待补执行面",
"focus_ref": {"kind": "release_launchpad", "section": "release_launchpad"},
},
"latest_release": {},
}
mock_get_recent_ops_playbook_runs.return_value = {"runs": []}
mock_get_ops_activity_stream.return_value = {"items": []}
mock_get_runtime_build_info.return_value = {
"route_surface": {"surface_complete": True, "missing_keys": []},
"repository_capabilities": {},
}
payload = get_ops_stack_diagnosis(base_url="http://127.0.0.1:8100")
self.assertEqual("bootstrap_run", payload["diagnosis"]["next_step"]["action_code"])
self.assertEqual(
"mainland-worker-01",
payload["diagnosis"]["launchpad_recommended_target_node_code"],
)
self.assertEqual(
"签发接入工单",
payload["diagnosis"]["launchpad_recommended_recovery_label"],
)
self.assertIn(
"onboarding.bootstrap",
payload["diagnosis"]["launchpad_recommended_recovery_summary"],
)
self.assertEqual("bootstrap_run", payload["diagnosis"]["operator_decision"]["primary_command_key"])
self.assertEqual(
"bash domain-api/deploy/multi-region/drive_ops_center.sh driver-resolve http://127.0.0.1:8100 bootstrap_run",
payload["diagnosis"]["next_actions"]["primary_command"],
)
@patch("app.services.ops_service.get_ops_activity_stream")
@patch("app.services.ops_service.get_recent_ops_playbook_runs")
@patch("app.services.ops_service.get_release_launchpad")
@patch("app.services.ops_service.list_managed_nodes_with_agent_state")
@patch("app.services.ops_service.get_ops_overview")
@patch("app.services.ops_service.get_ops_link_snapshot")
@patch("app.services.ops_service.get_ops_contract_registry")
def test_stack_diagnosis_emits_operator_decision_for_release_launchpad_attention(
self,
mock_get_ops_contract_registry,
mock_get_ops_link_snapshot,
mock_get_ops_overview,
mock_list_managed_nodes_with_agent_state,
mock_get_release_launchpad,
mock_get_recent_ops_playbook_runs,
mock_get_ops_activity_stream,
) -> None:
mock_get_ops_contract_registry.return_value = {
"registry_version": "2026-04-18",
"schema_version": "v1",
"contracts": [],
}
mock_get_ops_link_snapshot.return_value = {
"status": "attention",
"headline": "发布门禁待处理。",
"next_actions": [],
}
mock_get_ops_overview.return_value = {
"recommendation": {},
"execution_scene": {},
"log_sync": {},
"release_hub": {
"default_rollout_gate": {
"status": "blocked",
"summary_text": "还有节点未满足发布门禁。",
"focus_ref": {"kind": "release_hub", "section": "release_launchpad", "mode": "worker"},
}
},
}
mock_list_managed_nodes_with_agent_state.return_value = {"nodes": [], "summary": {}}
mock_get_release_launchpad.return_value = {
"launchpad_status": {
"status": "attention",
"status_label": "待处理",
"recommended_action_code": "review_smart_rollout_preview",
"focus_ref": {"kind": "release_hub", "section": "release_launchpad", "mode": "worker"},
},
"latest_release": {"release_version": "release-20260418"},
}
mock_get_recent_ops_playbook_runs.return_value = {"runs": []}
mock_get_ops_activity_stream.return_value = {"items": []}
payload = get_ops_stack_diagnosis(base_url="http://127.0.0.1:8100")
decision = payload["diagnosis"]["operator_decision"]
self.assertEqual("release", decision["lane"])
self.assertEqual("release-launchpad-worker", decision["primary_command_key"])
self.assertEqual("review_smart_rollout_preview", decision["secondary_command_key"])
self.assertIn("release-launchpad-worker", payload["diagnosis"]["recommended_commands"])
self.assertIn(
"release-launchpad http://127.0.0.1:8100 worker",
payload["diagnosis"]["next_actions"]["primary_command"],
)
@patch("app.services.ops_service.get_ops_activity_stream")
@patch("app.services.ops_service.get_recent_ops_playbook_runs")
@patch("app.services.ops_service.get_release_launchpad")
@patch("app.services.ops_service.list_managed_nodes_with_agent_state")
@patch("app.services.ops_service.get_ops_overview")
@patch("app.services.ops_service.get_ops_link_snapshot")
@patch("app.services.ops_service.get_ops_contract_registry")
def test_stack_diagnosis_prefers_node_recovery_when_launchpad_points_to_bootstrap_gap(
self,
mock_get_ops_contract_registry,
mock_get_ops_link_snapshot,
mock_get_ops_overview,
mock_list_managed_nodes_with_agent_state,
mock_get_release_launchpad,
mock_get_recent_ops_playbook_runs,
mock_get_ops_activity_stream,
) -> None:
mock_get_ops_contract_registry.return_value = {
"registry_version": "2026-04-18",
"schema_version": "v1",
"contracts": [],
}
mock_get_ops_link_snapshot.return_value = {
"status": "attention",
"headline": "发布门禁待处理。",
"next_actions": [],
}
mock_get_ops_overview.return_value = {
"recommendation": {},
"execution_scene": {},
"log_sync": {},
"release_hub": {
"default_rollout_gate": {
"status": "attention",
"summary_text": "当前还有节点未完成接入。",
"focus_ref": {"kind": "release_hub", "section": "release_launchpad", "mode": "worker"},
}
},
}
mock_list_managed_nodes_with_agent_state.return_value = {"nodes": [], "summary": {}}
mock_get_release_launchpad.return_value = {
"launchpad_status": {
"status": "attention",
"status_label": "待补执行面",
"recommended_action_code": "bootstrap_run",
"recommended_target_node_code": "mainland-worker-02",
"recommended_recovery_label": "待执行接入",
"recommended_recovery_summary": "当前节点仍处于接入阶段,建议先签发 onboarding.bootstrap。",
"onboarding_bootstrap_pending_nodes": 1,
"onboarding_acceptance_ready_nodes": 0,
"focus_ref": {"kind": "release_hub", "section": "release_launchpad", "mode": "worker"},
},
"latest_release": {"release_version": "release-20260418"},
}
mock_get_recent_ops_playbook_runs.return_value = {"runs": []}
mock_get_ops_activity_stream.return_value = {"items": []}
payload = get_ops_stack_diagnosis(base_url="http://127.0.0.1:8100")
decision = payload["diagnosis"]["operator_decision"]
self.assertEqual("node_handover", decision["lane"])
self.assertEqual("bootstrap_run", decision["primary_command_key"])
self.assertEqual("focus-release-hub", decision["secondary_command_key"])
self.assertEqual("mainland-worker-02", payload["diagnosis"]["launchpad_recommended_target_node_code"])
self.assertEqual("待执行接入", payload["diagnosis"]["launchpad_recommended_recovery_label"])
self.assertEqual(1, payload["diagnosis"]["launchpad_onboarding_bootstrap_pending_nodes"])
self.assertEqual(0, payload["diagnosis"]["launchpad_onboarding_acceptance_ready_nodes"])
self.assertEqual(
"当前节点仍处于接入阶段,建议先签发 onboarding.bootstrap。",
decision["summary"],
)
self.assertEqual("mainland-worker-02", decision["next_focus"]["node_code"])
self.assertIn("node-bootstrap-run", payload["diagnosis"]["next_actions"]["primary_command"])
@patch("app.services.ops_service.get_ops_activity_stream")
@patch("app.services.ops_service.get_recent_ops_playbook_runs")
@patch("app.services.ops_service.get_release_launchpad")
@patch("app.services.ops_service.get_managed_node_handover")
@patch("app.services.ops_service.list_managed_nodes_with_agent_state")
@patch("app.services.ops_service.get_ops_overview")
@patch("app.services.ops_service.get_ops_link_snapshot")
@patch("app.services.ops_service.get_ops_contract_registry")
def test_stack_diagnosis_includes_first_handover_gap_details(
self,
mock_get_ops_contract_registry,
mock_get_ops_link_snapshot,
mock_get_ops_overview,
mock_list_managed_nodes_with_agent_state,
mock_get_managed_node_handover,
mock_get_release_launchpad,
mock_get_recent_ops_playbook_runs,
mock_get_ops_activity_stream,
) -> None:
mock_get_ops_contract_registry.return_value = {
"registry_version": "2026-04-18",
"schema_version": "v1",
"contracts": [],
}
mock_get_ops_link_snapshot.return_value = {
"status": "attention",
"headline": "存在接管缺口。",
"next_actions": [],
}
mock_get_ops_overview.return_value = {
"recommendation": {},
"execution_scene": {},
"log_sync": {},
"release_hub": {},
}
mock_list_managed_nodes_with_agent_state.return_value = {
"nodes": [
{
"node_code": "mainland-worker-01",
"agent_state": "pending_bootstrap",
"remote_access_state": "ssh_ready",
"delivery_queue_state": "healthy",
"participation_state": "standby",
}
],
"summary": {
"managed_enabled": 1,
"remote_access_ready": 0,
"status_counts": {"pending_bootstrap": 1},
"remote_access_state_counts": {"ssh_ready": 1},
"delivery_queue_state_counts": {"healthy": 1},
},
}
mock_get_managed_node_handover.return_value = {
"node_code": "mainland-worker-01",
"summary": "已签发有效 Token但目标节点尚未执行 bootstrap 片段。",
"stage": {
"code": "pending_bootstrap",
"label": "待执行接入",
"next_step": {
"code": "execute_bootstrap_plan",
"label": "去目标机执行接入片段",
},
},
}
mock_get_release_launchpad.return_value = {
"launchpad_status": {
"recommended_action_code": "fix_managed_nodes",
"focus_ref": {"kind": "release_launchpad"},
},
"latest_release": {},
}
mock_get_recent_ops_playbook_runs.return_value = {"runs": []}
mock_get_ops_activity_stream.return_value = {"items": []}
payload = get_ops_stack_diagnosis(base_url="http://127.0.0.1:8100")
self.assertEqual(
"mainland-worker-01",
payload["managed_nodes"]["first_handover_gap_node_code"],
)
self.assertEqual(
"pending_bootstrap",
payload["managed_nodes"]["first_handover_gap"]["stage"]["code"],
)
issue = next(
item for item in payload["diagnosis"]["issues"] if item["code"] == "managed_nodes_agent_pending"
)
self.assertIn("mainland-worker-01", issue["detail"])
self.assertTrue(
any("node-handover http://127.0.0.1:8100 mainland-worker-01" in command for command in issue["commands"])
)
self.assertIn(
"bash domain-api/deploy/multi-region/drive_ops_center.sh node-recover http://127.0.0.1:8100 mainland-worker-01",
payload["diagnosis"]["recommended_commands"]["managed-node-recover"],
)
self.assertTrue(
any("node-bootstrap-plan http://127.0.0.1:8100 mainland-worker-01" in command for command in issue["commands"])
)
mock_get_managed_node_handover.assert_called_once()
@patch("app.services.ops_service.get_ops_activity_stream")
@patch("app.services.ops_service.get_recent_ops_playbook_runs")
@patch("app.services.ops_service.get_release_launchpad")
@patch("app.services.ops_service.list_managed_nodes_with_agent_state")
@patch("app.services.ops_service.get_ops_overview")
@patch("app.services.ops_service.get_ops_link_snapshot")
@patch("app.services.ops_service.get_ops_contract_registry")
def test_stack_diagnosis_keeps_focus_refs_for_problem_runs_and_top_items(
self,
mock_get_ops_contract_registry,
mock_get_ops_link_snapshot,
mock_get_ops_overview,
mock_list_managed_nodes_with_agent_state,
mock_get_release_launchpad,
mock_get_recent_ops_playbook_runs,
mock_get_ops_activity_stream,
) -> None:
mock_get_ops_contract_registry.return_value = {
"registry_version": "2026-04-18",
"schema_version": "v1",
"contracts": [],
}
mock_get_ops_link_snapshot.return_value = {
"status": "ready",
"headline": "当前总检可用。",
"next_actions": [],
}
mock_get_ops_overview.return_value = {
"recommendation": {},
"execution_scene": {},
"log_sync": {},
"release_hub": {},
}
mock_list_managed_nodes_with_agent_state.return_value = {
"nodes": [],
"summary": {},
}
mock_get_release_launchpad.return_value = {
"launchpad_status": {},
"latest_release": {},
}
mock_get_recent_ops_playbook_runs.return_value = {
"runs": [
{
"run_code": "pbr-worker-logs-001",
"status": "attention",
"status_label": "待关注",
"focus_step_key": "worker_logs",
"focus_summary": "Worker 日志步骤仍未收口。",
"focus_ref": {
"kind": "playbook_run",
"run_code": "pbr-worker-logs-001",
"focus_step_key": "worker_logs",
},
}
]
}
mock_get_ops_activity_stream.return_value = {
"items": [
{
"kind": "ops_job",
"status": "running",
"summary_text": "远端 Agent 已回传关键日志样本。",
"occurred_at": "2026-04-18 09:22:00",
"focus_ref": {
"kind": "ops_job",
"job_id": 71,
"job_code": "ops-agent-71",
},
"source_focus_ref": {
"kind": "ops_job_event",
"job_id": 71,
"event_key": "job-event:71",
},
"ui_intent": {
"kind": "job_events",
"job_id": 71,
},
}
]
}
payload = get_ops_stack_diagnosis(base_url="http://127.0.0.1:8100")
self.assertEqual(
"ops_stack_diagnosis_contract",
payload["diagnosis"]["contract_navigation"]["primary_contract_key"],
)
self.assertIn("ops_driver_contract", payload["diagnosis"]["contract_navigation"]["contract_keys"])
self.assertEqual("playbook_run", payload["playbook_runs"]["problem_runs"][0]["focus_ref"]["kind"])
self.assertEqual("worker_logs", payload["playbook_runs"]["problem_runs"][0]["focus_ref"]["focus_step_key"])
top_item = payload["activity_stream"]["top_items"][0]
self.assertEqual("ops_job", top_item["focus_ref"]["kind"])
self.assertEqual("ops_job_event", top_item["source_focus_ref"]["kind"])
self.assertEqual("job_events", top_item["ui_intent_kind"])
self.assertEqual("2026-04-18 09:22:00", top_item["occurred_at"])
@patch("app.services.ops_service.get_ops_activity_stream")
@patch("app.services.ops_service.get_recent_ops_playbook_runs")
@patch("app.services.ops_service.get_release_launchpad")
@patch("app.services.ops_service.list_managed_nodes_with_agent_state")
@patch("app.services.ops_service.get_ops_overview")
@patch("app.services.ops_service.get_ops_link_snapshot")
@patch("app.services.ops_service.get_ops_contract_registry")
def test_stack_diagnosis_adds_node_scene_log_commands_for_log_sample_gap(
self,
mock_get_ops_contract_registry,
mock_get_ops_link_snapshot,
mock_get_ops_overview,
mock_list_managed_nodes_with_agent_state,
mock_get_release_launchpad,
mock_get_recent_ops_playbook_runs,
mock_get_ops_activity_stream,
) -> None:
mock_get_ops_contract_registry.return_value = {
"registry_version": "2026-04-18",
"schema_version": "v1",
"contracts": [],
}
mock_get_ops_link_snapshot.return_value = {
"status": "attention",
"headline": "参与节点样本尚未形成。",
"next_actions": [
{
"label": "看参与节点日志",
"action_code": "open_worker_logs_participating",
"node_codes": ["mainland-worker-01"],
"payload": {},
}
],
}
mock_get_ops_overview.return_value = {
"recommendation": {},
"execution_scene": {
"participating_nodes": [
{
"node_code": "mainland-worker-01",
"status": "busy",
}
],
"participation_summary": {
"dispatch_active_node_codes": ["mainland-worker-01"],
"recent_only_node_codes": [],
},
"log_sync": {
"enabled": True,
"mode": "key",
"line_count": 0,
"source_node_count": 0,
"covered_participating_node_count": 0,
"participating_node_count": 1,
"missing_participating_nodes": ["mainland-worker-01"],
},
},
"log_sync": {},
"release_hub": {},
}
mock_list_managed_nodes_with_agent_state.return_value = {
"nodes": [],
"summary": {},
}
mock_get_release_launchpad.return_value = {
"launchpad_status": {},
"latest_release": {},
}
mock_get_recent_ops_playbook_runs.return_value = {"runs": []}
mock_get_ops_activity_stream.return_value = {"items": []}
payload = get_ops_stack_diagnosis(base_url="http://127.0.0.1:8100")
issue = next(
item for item in payload["diagnosis"]["issues"] if item["code"] == "remote_log_sync_waiting_sample"
)
self.assertEqual("open_worker_logs_participating", issue["action_code"])
self.assertEqual("node_scene_log", issue["focus_ref"]["kind"])
self.assertEqual("mainland-worker-01", issue["focus_ref"]["node_code"])
self.assertTrue(
any(
"scene-node-log http://127.0.0.1:8100 mainland-worker-01 120 key" in command
for command in issue["commands"]
)
)
self.assertEqual("node_scene_log", payload["diagnosis"]["next_step"]["focus_ref"]["kind"])
self.assertTrue(
any(
"scene-node-log http://127.0.0.1:8100 mainland-worker-01 120 key" in command
for command in payload["diagnosis"]["quick_commands"]
)
)
self.assertTrue(
any(
"driver-resolve http://127.0.0.1:8100 open_worker_logs_participating" in command
for command in issue["commands"]
)
)
@patch("app.services.ops_service.get_ops_activity_stream")
@patch("app.services.ops_service.get_recent_ops_playbook_runs")
@patch("app.services.ops_service.get_release_launchpad")
@patch("app.services.ops_service.list_managed_nodes_with_agent_state")
@patch("app.services.ops_service.get_ops_overview")
@patch("app.services.ops_service.get_ops_link_snapshot")
@patch("app.services.ops_service.get_ops_contract_registry")
def test_stack_diagnosis_surfaces_start_delivery_gap_from_activity_stream(
self,
mock_get_ops_contract_registry,
mock_get_ops_link_snapshot,
mock_get_ops_overview,
mock_list_managed_nodes_with_agent_state,
mock_get_release_launchpad,
mock_get_recent_ops_playbook_runs,
mock_get_ops_activity_stream,
) -> None:
mock_get_ops_contract_registry.return_value = {
"registry_version": "2026-04-18",
"schema_version": "v1",
"contracts": [],
}
mock_get_ops_link_snapshot.return_value = {
"status": "attention",
"headline": "最近有任务回执异常。",
"next_actions": [],
}
mock_get_ops_overview.return_value = {
"recommendation": {},
"execution_scene": {},
"log_sync": {},
"release_hub": {},
}
mock_list_managed_nodes_with_agent_state.return_value = {
"nodes": [],
"summary": {},
}
mock_get_release_launchpad.return_value = {
"launchpad_status": {},
"latest_release": {},
}
mock_get_recent_ops_playbook_runs.return_value = {"runs": []}
mock_get_ops_activity_stream.return_value = {
"items": [
{
"kind": "ops_job",
"status": "attention",
"job_status": "running",
"summary_text": "节点已接单执行,但开始回执未成功送达控制面。",
"occurred_at": "2026-04-18 10:22:00",
"job_code": "ops-agent-81",
"start_delivery_state": "failed_local",
"start_delivery_error": "temporary offline",
"focus_ref": {
"kind": "ops_job",
"job_id": 81,
"job_code": "ops-agent-81",
"target_node_code": "mainland-worker-01",
},
"source_focus_ref": {
"kind": "ops_job_event",
"job_id": 81,
"job_code": "ops-agent-81",
"event_key": "job-event:81",
"target_node_code": "mainland-worker-01",
},
"ui_intent": {
"kind": "job_events",
"job_id": 81,
},
}
]
}
payload = get_ops_stack_diagnosis(base_url="http://127.0.0.1:8100")
issue = next(
item for item in payload["diagnosis"]["issues"] if item["code"] == "ops_job_start_delivery_failed_local"
)
self.assertEqual("focus_latest_job_events", issue["action_code"])
self.assertEqual("ops_job_event", issue["focus_ref"]["kind"])
self.assertIn("temporary offline", issue["detail"])
decision = payload["diagnosis"]["operator_decision"]
self.assertEqual("ops_jobs", decision["lane"])
self.assertEqual("focus_latest_job_events", decision["primary_command_key"])
self.assertIn("driver-resolve http://127.0.0.1:8100 focus_latest_job_events", payload["diagnosis"]["next_actions"]["primary_command"])
if __name__ == "__main__":
unittest.main()

View File

@@ -0,0 +1,177 @@
import unittest
from app.services.ops_template_service import build_ops_template_payload, get_ops_action_template
class OpsTemplateServiceTests(unittest.TestCase):
def test_node_bootstrap_defaults_and_clamps_number(self) -> None:
ok, message, data = build_ops_template_payload(
"node.bootstrap",
{
"expires_in_hours": 9999,
},
)
self.assertTrue(ok, message)
self.assertEqual("ok", message)
self.assertEqual("", data["control_plane_base_url"])
self.assertEqual("/opt/domaincheck", data["root_dir"])
self.assertEqual(720, data["expires_in_hours"])
self.assertEqual("", data["node_region"])
self.assertEqual("", data["node_role"])
def test_deploy_release_custom_normalizes_lists_and_booleans(self) -> None:
ok, message, data = build_ops_template_payload(
"deploy.release.custom",
{
"release_version": "2026.04.18-rc1",
"artifact_url": "https://example.com/release.tar.gz",
"restart_services": "domaincheck-api\ndomaincheck-worker",
"health_check_urls": "",
"health_check_services": "",
"rollback_on_failure": "false",
"switch_current": "true",
},
)
self.assertTrue(ok, message)
self.assertEqual("ok", message)
self.assertEqual(["domaincheck-api", "domaincheck-worker"], data["restart_services"])
self.assertEqual([], data["health_check_urls"])
self.assertEqual([], data["health_check_services"])
self.assertFalse(data["rollback_on_failure"])
self.assertTrue(data["switch_current"])
self.assertEqual("/opt/domaincheck", data["install_root"])
def test_deploy_release_control_applies_defaults(self) -> None:
ok, message, data = build_ops_template_payload(
"deploy.release.control",
{
"release_version": "2026.04.18-rc1",
"artifact_url": "https://example.com/release.tar.gz",
},
)
self.assertTrue(ok, message)
self.assertEqual(
["domaincheck-api", "domaincheck-worker", "domaincheck-sync-agent"],
data["restart_services"],
)
self.assertEqual(["http://127.0.0.1:8100/health"], data["health_check_urls"])
self.assertEqual(
["domaincheck-api", "domaincheck-worker", "domaincheck-sync-agent"],
data["health_check_services"],
)
self.assertTrue(data["rollback_on_failure"])
self.assertTrue(data["switch_current"])
def test_deploy_release_missing_required_field_fails(self) -> None:
ok, message, data = build_ops_template_payload(
"deploy.release.custom",
{
"release_version": "2026.04.18-rc1",
"artifact_url": "",
},
)
self.assertFalse(ok)
self.assertEqual("artifact_url 不能为空", message)
self.assertEqual({}, data)
def test_logs_collect_lines_are_clamped(self) -> None:
ok, message, data = build_ops_template_payload(
"logs.collect",
{
"service_name": "domaincheck-worker",
"lines": 9999,
},
)
self.assertTrue(ok, message)
self.assertEqual(500, data["lines"])
def test_delivery_queue_templates_normalize_and_require_expected_fields(self) -> None:
replay_ok, replay_message, replay_data = build_ops_template_payload(
"delivery.queue.replay",
{
"record_id": "evt-001",
"limit": 999,
"flush_after_replay": "false",
"reason": "确认状态已修复",
},
)
discard_ok, discard_message, discard_data = build_ops_template_payload(
"delivery.queue.discard",
{
"record_id": "evt-002",
"limit": 0,
"reason": "确认这条死信不再需要补发",
},
)
flush_ok, flush_message, flush_data = build_ops_template_payload(
"delivery.queue.flush",
{
"limit": 999,
},
)
discard_fail_ok, discard_fail_message, discard_fail_data = build_ops_template_payload(
"delivery.queue.discard",
{
"record_id": "evt-003",
"reason": "",
},
)
self.assertTrue(replay_ok, replay_message)
self.assertEqual(200, replay_data["limit"])
self.assertFalse(replay_data["flush_after_replay"])
self.assertEqual("evt-001", replay_data["record_id"])
self.assertTrue(discard_ok, discard_message)
self.assertEqual(1, discard_data["limit"])
self.assertEqual("确认这条死信不再需要补发", discard_data["reason"])
self.assertTrue(flush_ok, flush_message)
self.assertEqual(200, flush_data["limit"])
self.assertFalse(discard_fail_ok)
self.assertEqual("reason 不能为空", discard_fail_message)
self.assertEqual({}, discard_fail_data)
def test_diagnostics_and_service_control_templates_expose_ssh(self) -> None:
health_template = get_ops_action_template("health.snapshot")
logs_template = get_ops_action_template("logs.collect")
diagnostics_template = get_ops_action_template("diagnostics.collect")
service_status_template = get_ops_action_template("service.status")
restart_template = get_ops_action_template("service.restart")
restart_api_template = get_ops_action_template("runtime.restart_api")
stop_worker_template = get_ops_action_template("runtime.stop_worker")
deploy_control_template = get_ops_action_template("deploy.release.control")
deploy_worker_template = get_ops_action_template("deploy.release.worker")
deploy_custom_template = get_ops_action_template("deploy.release.custom")
queue_flush_template = get_ops_action_template("delivery.queue.flush")
queue_replay_template = get_ops_action_template("delivery.queue.replay")
queue_discard_template = get_ops_action_template("delivery.queue.discard")
self.assertIn("ssh", health_template["execution_modes"])
self.assertEqual("远端 Agent", health_template["default_execution_mode_label"])
self.assertEqual("SSH", next(item["label"] for item in health_template["execution_mode_options"] if item["value"] == "ssh"))
self.assertIn("ssh", logs_template["execution_modes"])
self.assertIn("ssh", diagnostics_template["execution_modes"])
self.assertIn("ssh", service_status_template["execution_modes"])
self.assertIn("ssh", restart_template["execution_modes"])
self.assertIn("ssh", restart_api_template["execution_modes"])
self.assertIn("ssh", stop_worker_template["execution_modes"])
self.assertEqual("控制面", get_ops_action_template("node.bootstrap")["default_execution_mode_label"])
self.assertIn("ssh", deploy_control_template["execution_modes"])
self.assertIn("ssh", deploy_worker_template["execution_modes"])
self.assertIn("ssh", deploy_custom_template["execution_modes"])
self.assertEqual(["remote-agent"], queue_flush_template["execution_modes"])
self.assertEqual(["remote-agent"], queue_replay_template["execution_modes"])
self.assertEqual(["remote-agent"], queue_discard_template["execution_modes"])
self.assertEqual("远端 Agent", queue_flush_template["default_execution_mode_label"])
if __name__ == "__main__":
unittest.main()

View File

@@ -0,0 +1,117 @@
from __future__ import annotations
import json
import os
import subprocess
import tempfile
import unittest
from pathlib import Path
class PrepareFinalReleaseTests(unittest.TestCase):
def test_prepare_final_release_blocks_when_smoke_is_false_even_if_verify_passes(self) -> None:
repo_root = Path(__file__).resolve().parents[2]
source_script = repo_root / "prepare_final_release.sh"
with tempfile.TemporaryDirectory() as temp_dir:
root = Path(temp_dir)
release_root = root / "release"
release_root.mkdir(parents=True, exist_ok=True)
(root / "prepare_final_release.sh").write_text(source_script.read_text(encoding="utf-8"), encoding="utf-8")
(root / "package_domain_release.sh").write_text(
"#!/usr/bin/env bash\n"
"set -euo pipefail\n"
"mkdir -p \"$PWD/release\"\n"
"cat > \"$PWD/release/latest_release.json\" <<'EOF'\n"
"{\n"
" \"package_name\": \"domaincheck_release_test\",\n"
" \"archive_path\": \"/tmp/domaincheck_release_test.tar.gz\",\n"
" \"sha256_path\": \"/tmp/domaincheck_release_test.sha256.txt\",\n"
" \"sha256\": \"abc\",\n"
" \"smoke_test_ok\": false,\n"
" \"smoke_test_message\": \"skipped by env\"\n"
"}\n"
"EOF\n",
encoding="utf-8",
)
(root / "verify_domain_release.sh").write_text(
"#!/usr/bin/env bash\n"
"set -euo pipefail\n"
"printf '%s\\n' '{\"ok\": true, \"archive\": \"/tmp/domaincheck_release_test.tar.gz\"}'\n",
encoding="utf-8",
)
os.chmod(root / "prepare_final_release.sh", 0o755)
os.chmod(root / "package_domain_release.sh", 0o755)
os.chmod(root / "verify_domain_release.sh", 0o755)
result = subprocess.run(
["bash", str(root / "prepare_final_release.sh")],
cwd=root,
check=True,
capture_output=True,
text=True,
)
payload = json.loads((release_root / "final_release_report.json").read_text(encoding="utf-8"))
self.assertFalse(payload["ok"])
self.assertEqual("blocked", payload["release_gate"]["decision"])
self.assertTrue(payload["release_gate"]["verify_ok"])
self.assertFalse(payload["release_gate"]["smoke_test_ok"])
self.assertIn("smoke_test_failed_or_skipped", payload["release_gate"]["blocked_reasons"])
self.assertIn('"ok": false', result.stdout)
def test_prepare_final_release_ready_when_verify_and_smoke_both_pass(self) -> None:
repo_root = Path(__file__).resolve().parents[2]
source_script = repo_root / "prepare_final_release.sh"
with tempfile.TemporaryDirectory() as temp_dir:
root = Path(temp_dir)
release_root = root / "release"
release_root.mkdir(parents=True, exist_ok=True)
(root / "prepare_final_release.sh").write_text(source_script.read_text(encoding="utf-8"), encoding="utf-8")
(root / "package_domain_release.sh").write_text(
"#!/usr/bin/env bash\n"
"set -euo pipefail\n"
"mkdir -p \"$PWD/release\"\n"
"cat > \"$PWD/release/latest_release.json\" <<'EOF'\n"
"{\n"
" \"package_name\": \"domaincheck_release_test\",\n"
" \"archive_path\": \"/tmp/domaincheck_release_test.tar.gz\",\n"
" \"sha256_path\": \"/tmp/domaincheck_release_test.sha256.txt\",\n"
" \"sha256\": \"abc\",\n"
" \"smoke_test_ok\": true,\n"
" \"smoke_test_message\": \"all checks passed (1/1)\"\n"
"}\n"
"EOF\n",
encoding="utf-8",
)
(root / "verify_domain_release.sh").write_text(
"#!/usr/bin/env bash\n"
"set -euo pipefail\n"
"printf '%s\\n' '{\"ok\": true, \"archive\": \"/tmp/domaincheck_release_test.tar.gz\"}'\n",
encoding="utf-8",
)
os.chmod(root / "prepare_final_release.sh", 0o755)
os.chmod(root / "package_domain_release.sh", 0o755)
os.chmod(root / "verify_domain_release.sh", 0o755)
subprocess.run(
["bash", str(root / "prepare_final_release.sh")],
cwd=root,
check=True,
capture_output=True,
text=True,
)
payload = json.loads((release_root / "final_release_report.json").read_text(encoding="utf-8"))
self.assertTrue(payload["ok"])
self.assertEqual("ready", payload["release_gate"]["decision"])
self.assertEqual([], payload["release_gate"]["blocked_reasons"])
self.assertTrue(payload["release_gate"]["verify_ok"])
self.assertTrue(payload["release_gate"]["smoke_test_ok"])
if __name__ == "__main__":
unittest.main()

View File

@@ -0,0 +1,29 @@
from __future__ import annotations
import unittest
from pathlib import Path
class ReleaseMetadataParityTests(unittest.TestCase):
def test_powershell_release_package_emits_archive_path_alias(self) -> None:
repo_root = Path(__file__).resolve().parents[2]
script_text = (repo_root / "package_domain_release.ps1").read_text(encoding="utf-8")
self.assertIn("archive_path = $zipPath", script_text)
self.assertIn('"archive_path=$zipPath"', script_text)
def test_powershell_release_flow_prefers_archive_path_with_zip_fallback(self) -> None:
repo_root = Path(__file__).resolve().parents[2]
prepare_text = (repo_root / "prepare_final_release.ps1").read_text(encoding="utf-8")
show_text = (repo_root / "show_latest_release.ps1").read_text(encoding="utf-8")
self.assertIn("$archivePath = if ($latest.archive_path) { $latest.archive_path } else { $latest.zip_path }", prepare_text)
self.assertIn('& (Join-Path $root "package_domain_release.ps1")', prepare_text)
self.assertIn('& (Join-Path $root "verify_domain_release.ps1")', prepare_text)
self.assertNotIn('powershell -ExecutionPolicy Bypass -File (Join-Path $root "package_domain_release.ps1")', prepare_text)
self.assertIn("$reportArchivePath = if ($report.latest_release.archive_path) { [string]$report.latest_release.archive_path } else { [string]$report.latest_release.zip_path }", show_text)
self.assertIn("$latestArchivePath = if ($latest.archive_path) { [string]$latest.archive_path } else { [string]$latest.zip_path }", show_text)
if __name__ == "__main__":
unittest.main()

View File

@@ -0,0 +1,19 @@
from __future__ import annotations
import unittest
from pathlib import Path
class ReleaseScriptParityTests(unittest.TestCase):
def test_shell_release_verifier_checks_cross_platform_delivery_scripts(self) -> None:
repo_root = Path(__file__).resolve().parents[2]
script_path = repo_root / "verify_domain_release.sh"
script_text = script_path.read_text(encoding="utf-8")
self.assertIn('"scripts/package_domain_release.ps1"', script_text)
self.assertIn('"scripts/verify_domain_release.ps1"', script_text)
self.assertIn('"scripts/smoke_test_stack.ps1"', script_text)
if __name__ == "__main__":
unittest.main()

View File

@@ -0,0 +1,162 @@
from __future__ import annotations
import json
import os
import subprocess
import tempfile
import unittest
from pathlib import Path
class ShowLatestReleaseTests(unittest.TestCase):
def test_show_latest_release_marks_stale_final_report_as_not_ok(self) -> None:
repo_root = Path(__file__).resolve().parents[2]
script_path = repo_root / "show_latest_release.sh"
with tempfile.TemporaryDirectory() as temp_dir:
release_root = Path(temp_dir)
latest_payload = {
"package_name": "domaincheck_release_new",
"archive_path": str(release_root / "domaincheck_release_new.tar.gz"),
"sha256": "new-sha256",
}
final_report = {
"ok": True,
"latest_release": {
"package_name": "domaincheck_release_old",
"archive_path": str(release_root / "domaincheck_release_old.tar.gz"),
"sha256": "old-sha256",
},
}
(release_root / "latest_release.json").write_text(
json.dumps(latest_payload, ensure_ascii=False, indent=2),
encoding="utf-8",
)
(release_root / "final_release_report.json").write_text(
json.dumps(final_report, ensure_ascii=False, indent=2),
encoding="utf-8",
)
env = dict(os.environ)
env["DOMAINCHECK_RELEASE_ROOT"] = str(release_root)
result = subprocess.run(
["bash", str(script_path)],
cwd=repo_root,
check=True,
capture_output=True,
text=True,
env=env,
)
payload = json.loads(result.stdout)
self.assertFalse(payload["final_release_ok"])
self.assertTrue(payload["final_release_report_ok"])
self.assertFalse(payload["final_release_report_matches_latest"])
self.assertEqual("domaincheck_release_old", payload["final_release_report_package_name"])
self.assertIn("package_name", payload["final_release_report_stale_reason"])
self.assertEqual("", payload["final_release_gate_decision"])
self.assertEqual([], payload["final_release_gate_blocked_reasons"])
def test_show_latest_release_keeps_final_report_ok_when_it_matches_latest(self) -> None:
repo_root = Path(__file__).resolve().parents[2]
script_path = repo_root / "show_latest_release.sh"
with tempfile.TemporaryDirectory() as temp_dir:
release_root = Path(temp_dir)
latest_payload = {
"package_name": "domaincheck_release_new",
"archive_path": str(release_root / "domaincheck_release_new.tar.gz"),
"sha256": "new-sha256",
}
final_report = {
"ok": True,
"latest_release": latest_payload,
"release_gate": {
"decision": "ready",
"blocked_reasons": [],
"verify_ok": True,
"smoke_test_ok": True,
},
}
(release_root / "latest_release.json").write_text(
json.dumps(latest_payload, ensure_ascii=False, indent=2),
encoding="utf-8",
)
(release_root / "final_release_report.json").write_text(
json.dumps(final_report, ensure_ascii=False, indent=2),
encoding="utf-8",
)
env = dict(os.environ)
env["DOMAINCHECK_RELEASE_ROOT"] = str(release_root)
result = subprocess.run(
["bash", str(script_path)],
cwd=repo_root,
check=True,
capture_output=True,
text=True,
env=env,
)
payload = json.loads(result.stdout)
self.assertTrue(payload["final_release_ok"])
self.assertTrue(payload["final_release_report_ok"])
self.assertTrue(payload["final_release_report_matches_latest"])
self.assertEqual("domaincheck_release_new", payload["final_release_report_package_name"])
self.assertEqual("", payload["final_release_report_stale_reason"])
self.assertEqual("ready", payload["final_release_gate_decision"])
self.assertEqual([], payload["final_release_gate_blocked_reasons"])
self.assertTrue(payload["final_release_gate_verify_ok"])
self.assertTrue(payload["final_release_gate_smoke_test_ok"])
def test_show_latest_release_accepts_zip_path_fallback(self) -> None:
repo_root = Path(__file__).resolve().parents[2]
script_path = repo_root / "show_latest_release.sh"
with tempfile.TemporaryDirectory() as temp_dir:
release_root = Path(temp_dir)
latest_payload = {
"package_name": "domaincheck_release_win",
"zip_path": str(release_root / "domaincheck_release_win.zip"),
"sha256": "zip-sha256",
}
final_report = {
"ok": True,
"latest_release": latest_payload,
"release_gate": {
"decision": "ready",
"blocked_reasons": [],
"verify_ok": True,
"smoke_test_ok": True,
},
}
(release_root / "latest_release.json").write_text(
json.dumps(latest_payload, ensure_ascii=False, indent=2),
encoding="utf-8",
)
(release_root / "final_release_report.json").write_text(
json.dumps(final_report, ensure_ascii=False, indent=2),
encoding="utf-8",
)
env = dict(os.environ)
env["DOMAINCHECK_RELEASE_ROOT"] = str(release_root)
result = subprocess.run(
["bash", str(script_path)],
cwd=repo_root,
check=True,
capture_output=True,
text=True,
env=env,
)
payload = json.loads(result.stdout)
self.assertTrue(payload["final_release_ok"])
self.assertTrue(payload["final_release_report_ok"])
self.assertTrue(payload["final_release_report_matches_latest"])
self.assertEqual("domaincheck_release_win", payload["final_release_report_package_name"])
self.assertEqual("ready", payload["final_release_gate_decision"])
if __name__ == "__main__":
unittest.main()