feat: add ops center and node onboarding flow

domain-api/app/services/build_info_service.py

@@ -0,0 +1,241 @@
+from __future__ import annotations
+
+import json
+import subprocess
+from pathlib import Path
+from typing import Iterable
+
+from app.core.config import settings
+
+
+DOMAIN_API_ROOT = Path(__file__).resolve().parents[2]
+WORKSPACE_ROOT = DOMAIN_API_ROOT.parent
+_REGISTERED_ROUTE_PATHS: set[str] = set()
+
+
+def remember_registered_route_paths(route_paths: Iterable[str] | None) -> None:
+    global _REGISTERED_ROUTE_PATHS
+    normalized = {
+        str(item).strip()
+        for item in list(route_paths or [])
+        if str(item).strip()
+    }
+    _REGISTERED_ROUTE_PATHS = normalized
+
+
+def _read_json(path: Path) -> dict:
+    try:
+        return json.loads(path.read_text(encoding="utf-8"))
+    except Exception:
+        return {}
+
+
+def _discover_manifest_candidate() -> tuple[Path | None, str]:
+    configured = str(settings.build_manifest_path or "").strip()
+    if configured:
+        path = Path(configured)
+        if path.exists():
+            return path, "configured_manifest"
+
+    root_manifest = WORKSPACE_ROOT / "release_manifest.json"
+    if root_manifest.exists():
+        return root_manifest, "release_manifest"
+
+    latest_release = WORKSPACE_ROOT / "release" / "latest_release.json"
+    if latest_release.exists():
+        return latest_release, "latest_release"
+
+    return None, ""
+
+
+def _build_info_from_env() -> dict | None:
+    if not any(
+        [
+            str(settings.build_commit_sha or "").strip(),
+            str(settings.build_commit_ref or "").strip(),
+            str(settings.build_generated_at or "").strip(),
+            str(settings.build_package_name or "").strip(),
+            str(settings.build_checksum or "").strip(),
+        ]
+    ):
+        return None
+
+    return {
+        "source": str(settings.build_source_label or "").strip() or "env",
+        "package_name": str(settings.build_package_name or "").strip(),
+        "generated_at": str(settings.build_generated_at or "").strip(),
+        "commit_sha": str(settings.build_commit_sha or "").strip(),
+        "commit_ref": str(settings.build_commit_ref or "").strip(),
+        "checksum": str(settings.build_checksum or "").strip(),
+        "manifest_path": str(settings.build_manifest_path or "").strip(),
+    }
+
+
+def _build_info_from_manifest() -> dict | None:
+    manifest_path, source = _discover_manifest_candidate()
+    if manifest_path is None:
+        return None
+
+    payload = _read_json(manifest_path)
+    if not payload:
+        return {
+            "source": source,
+            "package_name": "",
+            "generated_at": "",
+            "commit_sha": "",
+            "commit_ref": "",
+            "checksum": "",
+            "manifest_path": str(manifest_path),
+        }
+
+    checksum = str(payload.get("checksum") or payload.get("sha256") or "").strip()
+    return {
+        "source": source,
+        "package_name": str(payload.get("package_name") or "").strip(),
+        "generated_at": str(payload.get("generated_at") or "").strip(),
+        "commit_sha": str(payload.get("commit_sha") or "").strip(),
+        "commit_ref": str(payload.get("commit_ref") or "").strip(),
+        "checksum": checksum,
+        "manifest_path": str(manifest_path),
+    }
+
+
+def _build_info_from_git() -> dict | None:
+    try:
+        inside_worktree = subprocess.run(
+            ["git", "-C", str(WORKSPACE_ROOT), "rev-parse", "--is-inside-work-tree"],
+            check=False,
+            capture_output=True,
+            text=True,
+            timeout=2,
+        )
+    except Exception:
+        return None
+
+    if inside_worktree.returncode != 0 or str(inside_worktree.stdout or "").strip() != "true":
+        return None
+
+    def _read_git(*args: str) -> str:
+        try:
+            result = subprocess.run(
+                ["git", "-C", str(WORKSPACE_ROOT), *args],
+                check=False,
+                capture_output=True,
+                text=True,
+                timeout=2,
+            )
+        except Exception:
+            return ""
+        if result.returncode != 0:
+            return ""
+        return str(result.stdout or "").strip()
+
+    return {
+        "source": "git",
+        "package_name": "",
+        "generated_at": "",
+        "commit_sha": _read_git("rev-parse", "--short=12", "HEAD"),
+        "commit_ref": _read_git("rev-parse", "--abbrev-ref", "HEAD"),
+        "checksum": "",
+        "manifest_path": "",
+    }
+
+
+def _resolve_build_identity() -> dict:
+    return (
+        _build_info_from_env()
+        or _build_info_from_manifest()
+        or _build_info_from_git()
+        or {
+            "source": "unknown",
+            "package_name": "",
+            "generated_at": "",
+            "commit_sha": "",
+            "commit_ref": "",
+            "checksum": "",
+            "manifest_path": "",
+        }
+    )
+
+
+def _expected_route_paths() -> dict[str, str]:
+    prefix = str(settings.api_prefix or "/api/v1").rstrip("/")
+    return {
+        "ops_contracts": f"{prefix}/ops/contracts",
+        "ops_contract_detail": f"{prefix}/ops/contracts/{{contract_key}}",
+        "ops_stack_diagnosis": f"{prefix}/ops/stack-diagnosis",
+        "ops_node_handover": f"{prefix}/ops/nodes/{{node_code}}/handover",
+        "ops_node_onboarding": f"{prefix}/ops/nodes/{{node_code}}/onboarding",
+        "ops_node_onboarding_bootstrap_preview": f"{prefix}/ops/nodes/{{node_code}}/onboarding/bootstrap/preview",
+        "ops_node_onboarding_bootstrap_execute": f"{prefix}/ops/nodes/{{node_code}}/onboarding/bootstrap/execute",
+        "ops_node_onboarding_acceptance_preview": f"{prefix}/ops/nodes/{{node_code}}/onboarding/acceptance/preview",
+        "ops_node_onboarding_acceptance_execute": f"{prefix}/ops/nodes/{{node_code}}/onboarding/acceptance/execute",
+        "ops_node_onboarding_recovery_preview": f"{prefix}/ops/nodes/{{node_code}}/onboarding/recovery/preview",
+        "ops_node_onboarding_recovery_execute": f"{prefix}/ops/nodes/{{node_code}}/onboarding/recovery/execute",
+        "ops_node_scene_log": f"{prefix}/ops/nodes/{{node_code}}/scene-log",
+        "ops_node_handover_bootstrap_plan": f"{prefix}/ops/nodes/{{node_code}}/handover/bootstrap-plan",
+        "ops_driver_feed": f"{prefix}/ops/driver-feed",
+        "ops_codex_brief": f"{prefix}/ops/codex-brief",
+        "ops_activity_stream": f"{prefix}/ops/activity-stream",
+        "ops_runbook_resolve": f"{prefix}/ops/runbook/sequences/{{sequence_key}}/resolve",
+        "ops_runbook_execute": f"{prefix}/ops/runbook/sequences/{{sequence_key}}/execute",
+        "runtime_build_info": f"{prefix}/runtime/build-info",
+    }
+
+
+def _read_text(path: Path) -> str:
+    try:
+        return path.read_text(encoding="utf-8", errors="replace")
+    except Exception:
+        return ""
+
+
+def _repository_capabilities() -> dict:
+    ops_agent_service_text = _read_text(DOMAIN_API_ROOT / "app" / "services" / "ops_agent_service.py")
+    bootstrap_node_agent_text = _read_text(DOMAIN_API_ROOT / "deploy" / "multi-region" / "bootstrap_node_agent.sh")
+    return {
+        "supports_install_command_block": (
+            "install_command_block" in ops_agent_service_text
+            and "_candidate_node_agent_install_paths" in ops_agent_service_text
+        ),
+        "supports_multi_layout_bootstrap": "resolve_project_root" in bootstrap_node_agent_text,
+    }
+
+
+def _build_route_surface(route_paths: Iterable[str] | None = None) -> dict:
+    expected = _expected_route_paths()
+    normalized_paths = {
+        str(item).strip()
+        for item in (list(route_paths) if route_paths is not None else list(_REGISTERED_ROUTE_PATHS))
+        if str(item).strip()
+    }
+    mode = "registered" if normalized_paths else "declared_contract"
+    flags = {key: (path in normalized_paths if normalized_paths else True) for key, path in expected.items()}
+    missing_keys = [key for key, available in flags.items() if not available]
+    return {
+        "mode": mode,
+        "registered_paths_total": len(normalized_paths),
+        "surface_flags": flags,
+        "expected_paths": expected,
+        "missing_keys": missing_keys,
+        "missing_paths": [expected[key] for key in missing_keys],
+        "surface_complete": len(missing_keys) == 0,
+    }
+
+
+def get_runtime_build_info(route_paths: Iterable[str] | None = None) -> dict:
+    build = _resolve_build_identity()
+    route_surface = _build_route_surface(route_paths=route_paths)
+    return {
+        "source": str(build.get("source") or "").strip() or "unknown",
+        "package_name": str(build.get("package_name") or "").strip(),
+        "generated_at": str(build.get("generated_at") or "").strip(),
+        "commit_sha": str(build.get("commit_sha") or "").strip(),
+        "commit_ref": str(build.get("commit_ref") or "").strip(),
+        "checksum": str(build.get("checksum") or "").strip(),
+        "manifest_path": str(build.get("manifest_path") or "").strip(),
+        "workspace_root": str(WORKSPACE_ROOT),
+        "domain_api_root": str(DOMAIN_API_ROOT),
+        "repository_capabilities": _repository_capabilities(),
+        "route_surface": route_surface,
+    }

domain-api/app/services/detect_service.py

@@ -147,15 +147,50 @@ def _build_remote_log_lines(
     mode: str,
     limit: int = 240,
 ) -> list[str]:
+    return _build_remote_log_snapshot(active_job, enabled=enabled, mode=mode, limit=limit)["lines"]
+
+
+def _build_remote_log_snapshot(
+    active_job: dict | None,
+    *,
+    enabled: bool,
+    mode: str,
+    limit: int = 240,
+) -> dict:
     if not enabled:
-        return []
+        return {
+            "lines": [],
+            "line_count": 0,
+            "last_at": "",
+            "last_line": "",
+            "source_nodes": [],
+            "source_node_count": 0,
+        }
     if not active_job:
-        return []
+        return {
+            "lines": [],
+            "line_count": 0,
+            "last_at": "",
+            "last_line": "",
+            "source_nodes": [],
+            "source_node_count": 0,
+        }
     events = list(active_job.get("current_cycle_events") or active_job.get("recent_events") or [])
     if not events:
-        return []
+        return {
+            "lines": [],
+            "line_count": 0,
+            "last_at": "",
+            "last_line": "",
+            "source_nodes": [],
+            "source_node_count": 0,
+        }
 
     lines: list[str] = []
+    source_nodes: set[str] = set()
+    source_node_summaries: dict[str, dict] = {}
+    last_at = ""
+    last_line = ""
     normalized_mode = str(mode or "key").strip().lower()
     if normalized_mode not in {"key", "full"}:
         normalized_mode = "key"
@@ -180,8 +215,47 @@ def _build_remote_log_lines(
             continue
         if len(message) > _REMOTE_LOG_MAX_CHARS:
             message = f"{message[:_REMOTE_LOG_MAX_CHARS]}..."
-        lines.append(f"[{created_at}] [{node_code}] {message}")
-    return lines[-max(1, int(limit or 240)) :]
+        formatted_line = f"[{created_at}] [{node_code}] {message}"
+        lines.append(formatted_line)
+        source_nodes.add(node_code)
+        node_summary = source_node_summaries.setdefault(
+            node_code,
+            {
+                "node_code": node_code,
+                "line_count": 0,
+                "key_line_count": 0,
+                "full_line_count": 0,
+                "last_at": "",
+                "last_line": "",
+            },
+        )
+        node_summary["line_count"] += 1
+        if event_mode == "full":
+            node_summary["full_line_count"] += 1
+        else:
+            node_summary["key_line_count"] += 1
+        node_summary["last_at"] = created_at
+        node_summary["last_line"] = formatted_line
+        last_at = created_at
+        last_line = formatted_line
+    sliced_lines = lines[-max(1, int(limit or 240)) :]
+    sorted_source_node_summaries = sorted(
+        source_node_summaries.values(),
+        key=lambda item: (
+            str(item.get("last_at") or ""),
+            str(item.get("node_code") or ""),
+        ),
+        reverse=True,
+    )
+    return {
+        "lines": sliced_lines,
+        "line_count": len(sliced_lines),
+        "last_at": last_at,
+        "last_line": last_line,
+        "source_nodes": sorted(source_nodes),
+        "source_node_count": len(source_nodes),
+        "source_node_summaries": sorted_source_node_summaries,
+    }
 
 
 def _resolve_remote_log_lines(
@@ -195,6 +269,17 @@ def _resolve_remote_log_lines(
     return _build_remote_log_lines(active_job, enabled=enabled, mode=mode, limit=limit)
 
 
+def _resolve_remote_log_snapshot(
+    active_job: dict | None,
+    runs: list[dict],
+    *,
+    enabled: bool,
+    mode: str,
+    limit: int = 240,
+) -> dict:
+    return _build_remote_log_snapshot(active_job, enabled=enabled, mode=mode, limit=limit)
+
+
 def _load_runtime_state() -> dict:
     try:
         redis_client = get_redis()
@@ -404,13 +489,14 @@ def get_detect_status() -> dict:
     runs = sync_detect_runs(runtime_snapshot, progress, settings_summary, active_job=active_job)
     worker_log_sync_enabled = bool(runtime_settings.get("worker_log_sync_enabled", False))
     worker_log_sync_mode = str(runtime_settings.get("worker_log_sync_mode", "key") or "key")
-    remote_log_lines = _resolve_remote_log_lines(
+    remote_log_snapshot = _resolve_remote_log_snapshot(
         active_job,
         runs,
         enabled=worker_log_sync_enabled,
         mode=worker_log_sync_mode,
         limit=240,
     )
+    remote_log_lines = list(remote_log_snapshot.get("lines") or [])
     dependency_alerts = _extract_dependency_alerts(recent_lines)
     append_detect_result_projection_if_changed(
         detect={
@@ -465,6 +551,12 @@ def get_detect_status() -> dict:
         "recent_warning": recent_proxy_warning,
         "log_lines": recent_lines,
         "remote_log_lines": remote_log_lines,
+        "remote_log_line_count": int(remote_log_snapshot.get("line_count", 0) or 0),
+        "remote_log_last_at": str(remote_log_snapshot.get("last_at") or ""),
+        "remote_log_last_line": str(remote_log_snapshot.get("last_line") or ""),
+        "remote_log_nodes": list(remote_log_snapshot.get("source_nodes") or []),
+        "remote_log_node_count": int(remote_log_snapshot.get("source_node_count", 0) or 0),
+        "remote_log_node_summaries": list(remote_log_snapshot.get("source_node_summaries") or []),
         "runs": runs,
         "active_job": active_job,
         "worker_log_sync_enabled": worker_log_sync_enabled,

domain-api/app/services/ops_action_executor_core.py

@@ -0,0 +1,278 @@
+from __future__ import annotations
+
+from pathlib import Path
+from typing import Protocol
+
+from app.core.config import settings
+
+
+STRUCTURED_ACTIONS = {
+    "health.snapshot",
+    "service.status",
+    "service.restart",
+    "service.start",
+    "service.stop",
+    "logs.collect",
+    "diagnostics.collect",
+    "runtime.start_worker",
+    "runtime.stop_worker",
+    "runtime.restart_api",
+    "runtime.start_sync_agent",
+    "runtime.stop_sync_agent",
+}
+
+
+class CommandRunner(Protocol):
+    def __call__(self, command: list[str], *, timeout: int = 60) -> tuple[int, str, str]:
+        ...
+
+
+def supports_structured_action(action: str) -> bool:
+    return str(action or "").strip() in STRUCTURED_ACTIONS
+
+
+def trim_output(text: str, limit: int) -> str:
+    normalized = str(text or "").strip()
+    if len(normalized) <= int(limit):
+        return normalized
+    return normalized[-int(limit):]
+
+
+def _read_tail_lines(path: Path, *, max_lines: int) -> list[str]:
+    if not path.exists() or not path.is_file():
+        return []
+    with path.open("r", encoding="utf-8", errors="replace") as handle:
+        return handle.read().splitlines()[-max_lines:]
+
+
+def _runtime_logs_dir() -> Path:
+    return Path(__file__).resolve().parents[2] / "runtime" / "logs"
+
+
+def _service_log_file_candidates(service_name: str, *, service_names: dict[str, str]) -> list[Path]:
+    normalized_service_name = str(service_name or "").strip()
+    if not normalized_service_name:
+        return []
+
+    domain_root = Path(settings.domain_root)
+    runtime_logs_dir = _runtime_logs_dir()
+    candidates: list[Path] = []
+
+    def append_candidate(path: Path) -> None:
+        if path not in candidates:
+            candidates.append(path)
+
+    if normalized_service_name == (service_names.get("worker") or "domaincheck-worker"):
+        append_candidate(domain_root / "detect_worker.log")
+        append_candidate(domain_root / "logs" / "detect_worker.log")
+    elif normalized_service_name == (service_names.get("api") or "domaincheck-api"):
+        append_candidate(runtime_logs_dir / "domain-api.stderr.log")
+        append_candidate(runtime_logs_dir / "domain-api.stdout.log")
+        append_candidate(domain_root / "logs" / "app.log")
+
+    return candidates
+
+
+def _collect_log_file_fallback(
+    service_name: str,
+    *,
+    lines: int,
+    service_names: dict[str, str],
+) -> tuple[str, list[str]]:
+    aggregated_lines: list[str] = []
+    used_paths: list[str] = []
+    for path in _service_log_file_candidates(service_name, service_names=service_names):
+        current_lines = _read_tail_lines(path, max_lines=lines)
+        if not current_lines:
+            continue
+        aggregated_lines.extend(current_lines)
+        used_paths.append(str(path))
+    if not aggregated_lines:
+        return "", []
+    return "\n".join(aggregated_lines[-lines:]), used_paths
+
+
+def build_service_name_map(
+    *,
+    api_service_name: str,
+    worker_service_name: str,
+    sync_agent_service_name: str,
+    node_agent_service_name: str = "domaincheck-node-agent",
+) -> dict[str, str]:
+    return {
+        "api": str(api_service_name or "").strip() or "domaincheck-api",
+        "worker": str(worker_service_name or "").strip() or "domaincheck-worker",
+        "sync_agent": str(sync_agent_service_name or "").strip() or "domaincheck-sync-agent",
+        "node_agent": str(node_agent_service_name or "").strip() or "domaincheck-node-agent",
+    }
+
+
+def service_name_from_payload(payload: dict, *, default_worker_service_name: str) -> str:
+    service_name = str((payload or {}).get("service_name") or "").strip()
+    return service_name or str(default_worker_service_name or "").strip() or "domaincheck-worker"
+
+
+def service_name_for_action(action: str, payload: dict, service_names: dict[str, str]) -> str:
+    normalized_action = str(action or "").strip()
+    if normalized_action in {"service.status", "service.restart", "service.start", "service.stop"}:
+        return service_name_from_payload(payload, default_worker_service_name=service_names.get("worker") or "")
+    runtime_action_map = {
+        "runtime.start_worker": service_names.get("worker") or "domaincheck-worker",
+        "runtime.stop_worker": service_names.get("worker") or "domaincheck-worker",
+        "runtime.restart_api": service_names.get("api") or "domaincheck-api",
+        "runtime.start_sync_agent": service_names.get("sync_agent") or "domaincheck-sync-agent",
+        "runtime.stop_sync_agent": service_names.get("sync_agent") or "domaincheck-sync-agent",
+    }
+    return str(runtime_action_map.get(normalized_action) or "").strip()
+
+
+def systemctl_action_name(action: str) -> str:
+    normalized_action = str(action or "").strip()
+    if normalized_action in {"service.restart", "runtime.restart_api"}:
+        return "restart"
+    if normalized_action in {"service.start", "runtime.start_worker", "runtime.start_sync_agent"}:
+        return "start"
+    if normalized_action in {"service.stop", "runtime.stop_worker", "runtime.stop_sync_agent"}:
+        return "stop"
+    return ""
+
+
+def execute_structured_action(
+    action: str,
+    payload: dict | None,
+    *,
+    service_names: dict[str, str],
+    runner: CommandRunner,
+    host_context: dict | None = None,
+) -> tuple[bool, str, dict]:
+    normalized_action = str(action or "").strip()
+    normalized_payload = dict(payload or {})
+    normalized_service_names = {
+        str(key or "").strip(): str(value or "").strip()
+        for key, value in dict(service_names or {}).items()
+        if str(key or "").strip() and str(value or "").strip()
+    }
+    host_details = {
+        key: str((host_context or {}).get(key) or "").strip()
+        for key in ("hostname", "ip")
+        if str((host_context or {}).get(key) or "").strip()
+    }
+
+    if normalized_action == "health.snapshot":
+        checks: dict[str, dict] = {}
+        for key, service_name in normalized_service_names.items():
+            code, stdout, stderr = runner(["systemctl", "is-active", service_name], timeout=15)
+            checks[key] = {
+                "service_name": service_name,
+                "returncode": int(code or 0),
+                "state": stdout or stderr,
+                "ok": int(code or 0) == 0 and (stdout or stderr) == "active",
+            }
+        result = {"checks": checks}
+        result.update(host_details)
+        return True, "health snapshot collected", result
+
+    if normalized_action == "service.status":
+        service_name = service_name_from_payload(
+            normalized_payload,
+            default_worker_service_name=normalized_service_names.get("worker") or "",
+        )
+        code, stdout, stderr = runner(["systemctl", "status", service_name, "--no-pager", "-l"], timeout=45)
+        result = {
+            "service_name": service_name,
+            "stdout": trim_output(stdout, 12000),
+            "stderr": trim_output(stderr, 4000),
+        }
+        result.update(host_details)
+        return int(code or 0) == 0, stdout or stderr or f"{service_name} status collected", result
+
+    systemctl_action = systemctl_action_name(normalized_action)
+    if systemctl_action:
+        service_name = service_name_for_action(normalized_action, normalized_payload, normalized_service_names)
+        if not service_name:
+            return False, f"当前动作缺少 service_name: {normalized_action}", {"action": normalized_action}
+        code, stdout, stderr = runner(["systemctl", systemctl_action, service_name], timeout=45)
+        result = {
+            "service_name": service_name,
+            "systemctl_action": systemctl_action,
+            "stdout": trim_output(stdout, 12000),
+            "stderr": trim_output(stderr, 4000),
+        }
+        result.update(host_details)
+        return int(code or 0) == 0, stdout or stderr or f"{service_name} {systemctl_action} completed", result
+
+    if normalized_action == "logs.collect":
+        service_name = service_name_from_payload(
+            normalized_payload,
+            default_worker_service_name=normalized_service_names.get("worker") or "",
+        )
+        lines = max(20, min(int(normalized_payload.get("lines") or 120), 500))
+        code, stdout, stderr = runner(["journalctl", "-u", service_name, "-n", str(lines), "--no-pager"], timeout=45)
+        journal_output = stdout or stderr
+        fallback_output, fallback_paths = _collect_log_file_fallback(
+            service_name,
+            lines=lines,
+            service_names=normalized_service_names,
+        )
+        collection_source = "journal"
+        effective_output = journal_output
+        ok = int(code or 0) == 0
+        if (not ok or not stdout.strip()) and fallback_output:
+            collection_source = "file_fallback"
+            effective_output = fallback_output
+            ok = True
+        result = {
+            "service_name": service_name,
+            "lines": lines,
+            "collection_source": collection_source,
+            "fallback_used": bool(collection_source == "file_fallback"),
+            "fallback_reason": trim_output(journal_output, 4000) if collection_source == "file_fallback" else "",
+            "log_paths": fallback_paths,
+            "journal_returncode": int(code or 0),
+            "stdout": trim_output(effective_output, 20000),
+            "stderr": trim_output(stderr, 4000),
+        }
+        result.update(host_details)
+        return ok, effective_output or f"{service_name} logs collected", result
+
+    if normalized_action == "diagnostics.collect":
+        lines = max(20, min(int(normalized_payload.get("lines") or 200), 800))
+        diagnostics: dict[str, object] = {
+            "services": {},
+            "lines": lines,
+        }
+        diagnostics.update(host_details)
+        for key, service_name in normalized_service_names.items():
+            active_code, active_stdout, active_stderr = runner(["systemctl", "is-active", service_name], timeout=15)
+            status_code, status_stdout, status_stderr = runner(
+                ["systemctl", "status", service_name, "--no-pager", "-l"],
+                timeout=45,
+            )
+            log_code, log_stdout, log_stderr = runner(
+                ["journalctl", "-u", service_name, "-n", str(lines), "--no-pager"],
+                timeout=45,
+            )
+            log_output = log_stdout or log_stderr
+            fallback_output, fallback_paths = _collect_log_file_fallback(
+                service_name,
+                lines=lines,
+                service_names=normalized_service_names,
+            )
+            log_source = "journal"
+            if (int(log_code or 0) != 0 or not log_stdout.strip()) and fallback_output:
+                log_output = fallback_output
+                log_source = "file_fallback"
+            diagnostics["services"][key] = {
+                "service_name": service_name,
+                "active_returncode": int(active_code or 0),
+                "active_state": active_stdout or active_stderr,
+                "status_returncode": int(status_code or 0),
+                "status_output": trim_output(status_stdout or status_stderr, 12000),
+                "log_returncode": int(log_code or 0),
+                "log_source": log_source,
+                "log_paths": fallback_paths,
+                "log_output": trim_output(log_output, 20000),
+            }
+        return True, "diagnostics collected", diagnostics
+
+    return False, f"unsupported action: {normalized_action}", {"action": normalized_action}

domain-api/app/services/ops_command_service.py

@@ -0,0 +1,17 @@
+from __future__ import annotations
+
+
+OPS_MULTI_REGION_DIR = "domain-api/deploy/multi-region"
+
+
+def ops_script_path(script_name: str) -> str:
+    normalized_script_name = str(script_name or "").strip().lstrip("/")
+    if not normalized_script_name:
+        return OPS_MULTI_REGION_DIR
+    return f"{OPS_MULTI_REGION_DIR}/{normalized_script_name}"
+
+
+def build_bash_command(script_name: str, *args: object) -> str:
+    command_parts = ["bash", ops_script_path(script_name)]
+    command_parts.extend(str(arg or "").strip() for arg in args if str(arg or "").strip())
+    return " ".join(command_parts)

domain-api/app/services/ops_execution_capability_service.py

@@ -0,0 +1,79 @@
+from __future__ import annotations
+
+from app.services.ops_action_executor_core import STRUCTURED_ACTIONS
+
+
+_LOCAL_RUNTIME_ONLY_ACTIONS = {
+    "runtime.push_sync",
+    "runtime.pull_tasks",
+}
+
+_REMOTE_AGENT_ONLY_ACTIONS = {
+    "delivery.queue.flush",
+    "delivery.queue.replay",
+    "delivery.queue.discard",
+}
+
+_LOCAL_RUNTIME_ACTIONS = set(STRUCTURED_ACTIONS) | _LOCAL_RUNTIME_ONLY_ACTIONS | {"deploy.release"}
+_SSH_ACTIONS = set(STRUCTURED_ACTIONS) | {"deploy.release"}
+_REMOTE_AGENT_ACTIONS = set(STRUCTURED_ACTIONS) | _REMOTE_AGENT_ONLY_ACTIONS | {"deploy.release"}
+_CONTROL_PLANE_ACTIONS = {
+    "node.bootstrap",
+}
+
+
+def supports_local_runtime_action(action: str) -> bool:
+    return str(action or "").strip() in _LOCAL_RUNTIME_ACTIONS
+
+
+def supports_ssh_action(action: str) -> bool:
+    return str(action or "").strip() in _SSH_ACTIONS
+
+
+def supports_remote_agent_action(action: str) -> bool:
+    return str(action or "").strip() in _REMOTE_AGENT_ACTIONS
+
+
+def supports_control_plane_action(action: str) -> bool:
+    return str(action or "").strip() in _CONTROL_PLANE_ACTIONS
+
+
+def validate_action_execution_mode(
+    action: str,
+    execution_mode: str,
+    *,
+    target_node_code: str = "",
+    current_node_code: str = "",
+) -> tuple[bool, str]:
+    normalized_action = str(action or "").strip()
+    normalized_mode = str(execution_mode or "remote-agent").strip() or "remote-agent"
+    normalized_target_node_code = str(target_node_code or "").strip()
+    normalized_current_node_code = str(current_node_code or "").strip()
+
+    if normalized_mode == "local-runtime":
+        if not supports_local_runtime_action(normalized_action):
+            return False, f"{normalized_action} 当前不支持 local-runtime 执行方式。"
+        if (
+            normalized_target_node_code
+            and normalized_current_node_code
+            and normalized_target_node_code != normalized_current_node_code
+        ):
+            return False, "local-runtime 仅支持当前控制面本机节点。"
+        return True, ""
+
+    if normalized_mode == "control-plane":
+        if not supports_control_plane_action(normalized_action):
+            return False, f"{normalized_action} 当前不支持 control-plane 执行方式。"
+        return True, ""
+
+    if normalized_mode == "ssh":
+        if not supports_ssh_action(normalized_action):
+            return False, f"{normalized_action} 当前不支持 ssh 执行方式。"
+        return True, ""
+
+    if normalized_mode == "remote-agent":
+        if not supports_remote_agent_action(normalized_action):
+            return False, f"{normalized_action} 当前不支持 remote-agent 执行方式。"
+        return True, ""
+
+    return False, f"未知执行方式: {normalized_mode}"

domain-api/app/services/ops_execution_mode_service.py

@@ -0,0 +1,60 @@
+from __future__ import annotations
+
+
+def normalize_execution_modes(raw_modes: object, fallback_mode: str = "remote-agent") -> list[str]:
+    normalized_modes: list[str] = []
+    seen: set[str] = set()
+    fallback = str(fallback_mode or "remote-agent").strip() or "remote-agent"
+    for item in list(raw_modes or []):
+        mode = str(item or "").strip()
+        if not mode or mode in seen:
+            continue
+        seen.add(mode)
+        normalized_modes.append(mode)
+    if fallback and fallback not in seen:
+        normalized_modes.insert(0, fallback)
+    return normalized_modes or [fallback]
+
+
+def execution_mode_label(execution_mode: str) -> str:
+    normalized_mode = str(execution_mode or "remote-agent").strip() or "remote-agent"
+    if normalized_mode == "ssh":
+        return "SSH"
+    if normalized_mode == "control-plane":
+        return "控制面"
+    if normalized_mode == "local-runtime":
+        return "本机运行时"
+    if normalized_mode == "remote-agent":
+        return "远端 Agent"
+    return normalized_mode
+
+
+def build_execution_mode_options(raw_modes: object, fallback_mode: str = "remote-agent") -> list[dict]:
+    return [
+        {
+            "value": mode,
+            "label": execution_mode_label(mode),
+        }
+        for mode in normalize_execution_modes(raw_modes, fallback_mode)
+    ]
+
+
+def decorate_execution_mode_fields(
+    payload: dict | None = None,
+    *,
+    default_key: str = "default_execution_mode",
+    modes_key: str = "execution_modes",
+) -> dict:
+    normalized_payload = dict(payload or {})
+    default_mode = str(normalized_payload.get(default_key) or "remote-agent").strip() or "remote-agent"
+    execution_mode_options = build_execution_mode_options(normalized_payload.get(modes_key) or [], default_mode)
+    normalized_payload[default_key] = default_mode
+    normalized_payload[modes_key] = [str(item.get("value") or "").strip() for item in execution_mode_options if str(item.get("value") or "").strip()]
+    normalized_payload["default_execution_mode_label"] = execution_mode_label(default_mode)
+    normalized_payload["execution_mode_options"] = execution_mode_options
+    normalized_payload["execution_mode_labels"] = [
+        str(item.get("label") or "").strip()
+        for item in execution_mode_options
+        if str(item.get("label") or "").strip()
+    ]
+    return normalized_payload

domain-api/app/services/ops_policy_service.py

@@ -0,0 +1,494 @@
+from __future__ import annotations
+
+from app.core.config import settings
+from app.services.cluster_runtime_service import get_cluster_snapshot
+from app.services.ops_execution_capability_service import validate_action_execution_mode
+from app.services.ops_release_service import (
+    _calculate_total_batches,
+    _normalize_rollout_policy,
+    _resolve_rollout_targets,
+    build_release_rollout_gate,
+    get_release,
+)
+
+
+_LOW_RISK_ACTIONS = {
+    "health.snapshot",
+    "runtime.push_sync",
+    "runtime.pull_tasks",
+    "logs.collect",
+    "diagnostics.collect",
+    "delivery.queue.flush",
+}
+
+_MEDIUM_RISK_ACTIONS = {
+    "runtime.start_worker",
+    "runtime.start_sync_agent",
+    "service.status",
+    "health.check",
+    "delivery.queue.replay",
+}
+
+_HIGH_RISK_ACTIONS = {
+    "runtime.stop_worker",
+    "runtime.stop_sync_agent",
+    "runtime.restart_api",
+    "service.restart",
+    "deploy.release",
+    "delivery.queue.discard",
+}
+
+_CRITICAL_RISK_ACTIONS = {
+    "deploy.rollback",
+    "node.bootstrap",
+    "cluster.reconfigure",
+}
+
+
+def _normalize_node_code_list(raw_value: object) -> list[str]:
+    if isinstance(raw_value, list):
+        return [str(item).strip() for item in raw_value if str(item).strip()]
+    text = str(raw_value or "").replace("\r", "\n").strip()
+    if not text:
+        return []
+    normalized = text.replace(",", "\n")
+    return [item.strip() for item in normalized.split("\n") if item.strip()]
+
+
+def _dedupe_text_items(items: list[str]) -> list[str]:
+    return list(dict.fromkeys(str(item or "").strip() for item in items if str(item or "").strip()))
+
+
+def _risk_level_for_action(action: str) -> str:
+    if action in _LOW_RISK_ACTIONS:
+        return "low"
+    if action in _HIGH_RISK_ACTIONS:
+        return "high"
+    if action in _CRITICAL_RISK_ACTIONS:
+        return "critical"
+    return "medium"
+
+
+def _compact_target_node(item: dict) -> dict:
+    return {
+        "node_code": str(item.get("node_code") or ""),
+        "region": str(item.get("region") or ""),
+        "role": str(item.get("role") or ""),
+        "status": str(item.get("status") or ""),
+        "current_load": int(item.get("current_load", 0) or 0),
+        "is_effective_worker": bool(item.get("is_effective_worker", False)),
+        "detect_participating": bool(item.get("detect_participating", False)),
+        "is_enabled": bool(item.get("is_enabled", True)),
+        "deploy_channel": str(item.get("deploy_channel") or ""),
+    }
+
+
+def _preview_action_payload_guardrails(
+    action: str,
+    payload: dict,
+    *,
+    execution_mode: str,
+    target_nodes_total: int,
+) -> dict:
+    warnings: list[str] = []
+    blocking_reasons: list[str] = []
+    approval_reasons: list[str] = []
+    recommendations: list[str] = []
+
+    if action == "deploy.release":
+        release_version = str(payload.get("release_version") or "").strip()
+        artifact_url = str(payload.get("artifact_url") or "").strip()
+        if not release_version:
+            blocking_reasons.append("deploy.release 缺少 release_version，当前不能创建正式发布任务。")
+        if not artifact_url:
+            blocking_reasons.append("deploy.release 缺少 artifact_url，当前不能创建正式发布任务。")
+        if execution_mode not in {"remote-agent", "ssh"}:
+            blocking_reasons.append("deploy.release 仅支持 remote-agent 或 ssh 执行方式。")
+        if target_nodes_total > 1:
+            recommendations.append("建议先单节点 canary 验证发布任务，再逐步扩到更多节点。")
+
+    if action == "node.bootstrap":
+        if execution_mode != "control-plane":
+            blocking_reasons.append("node.bootstrap 仅支持 control-plane 执行方式。")
+        if target_nodes_total > 1:
+            recommendations.append("接管动作建议按单节点节奏推进，先确认首台节点接入成功后再继续放量。")
+
+    if action in _CRITICAL_RISK_ACTIONS:
+        approval_reasons.append("该动作属于 critical 风险动作，正式环境必须审批。")
+    elif action in _HIGH_RISK_ACTIONS:
+        approval_reasons.append("该动作属于 high 风险动作，建议审批后执行。")
+
+    return {
+        "warnings": _dedupe_text_items(warnings),
+        "blocking_reasons": _dedupe_text_items(blocking_reasons),
+        "approval_reasons": _dedupe_text_items(approval_reasons),
+        "recommendations": _dedupe_text_items(recommendations),
+    }
+
+
+def _preview_single_node_policy(
+    action: str,
+    *,
+    target_node_code: str,
+    cluster: dict,
+    payload: dict | None = None,
+    execution_mode: str = "remote-agent",
+    target_type: str = "node",
+    include_payload_guardrails: bool = True,
+) -> dict:
+    payload = dict(payload or {})
+    nodes = list(cluster.get("nodes") or [])
+    summary = cluster.get("summary") or {}
+    target_node = next((item for item in nodes if str(item.get("node_code") or "").strip() == target_node_code), {})
+
+    risk_level = _risk_level_for_action(action)
+    warnings: list[str] = []
+    blocking_reasons: list[str] = []
+    approval_reasons: list[str] = []
+    recommendations: list[str] = []
+
+    execution_supported, execution_reason = validate_action_execution_mode(
+        action,
+        execution_mode,
+        target_node_code=target_node_code,
+        current_node_code=settings.node_code,
+    )
+    if not execution_supported and execution_reason:
+        blocking_reasons.append(execution_reason)
+
+    if target_type == "node" and target_node_code and not target_node:
+        warnings.append("目标节点当前未出现在集群快照中，可能尚未注册或已离线。")
+
+    node_status = str(target_node.get("status") or "")
+    node_role = str(target_node.get("role") or "")
+    node_current_load = int(target_node.get("current_load", 0) or 0)
+    node_effective_worker = bool(target_node.get("is_effective_worker", False))
+    node_detect_participating = bool(target_node.get("detect_participating", False))
+
+    if target_node:
+        if node_status in {"busy"} and action in {"runtime.stop_worker", "runtime.restart_api", "deploy.release", "deploy.rollback", "service.restart"}:
+            blocking_reasons.append("目标节点当前处于 busy 状态，不适合直接执行中断类动作。")
+
+        if node_detect_participating and action in {"runtime.stop_worker", "runtime.restart_api", "deploy.release", "deploy.rollback", "service.restart"}:
+            blocking_reasons.append("目标节点正在参与检测，需先迁移负载或人工确认后再执行。")
+
+        if node_role == "control" and action in {"runtime.restart_api", "deploy.release", "deploy.rollback", "service.restart"}:
+            approval_reasons.append("目标节点是 control 节点，建议强制走审批或维护窗口。")
+
+        if node_effective_worker and int(summary.get("online_worker_nodes", 0) or 0) <= 1 and action in {"runtime.stop_worker", "deploy.release", "deploy.rollback", "service.restart"}:
+            blocking_reasons.append("当前有效执行节点数不足 2，执行该动作可能导致检测能力中断。")
+
+        if node_current_load > 0:
+            warnings.append(f"目标节点当前负载为 {node_current_load}，建议优先观察或转移任务。")
+
+    if include_payload_guardrails:
+        payload_guardrails = _preview_action_payload_guardrails(
+            action,
+            payload,
+            execution_mode=execution_mode,
+            target_nodes_total=1,
+        )
+        warnings.extend(payload_guardrails.get("warnings") or [])
+        blocking_reasons.extend(payload_guardrails.get("blocking_reasons") or [])
+        approval_reasons.extend(payload_guardrails.get("approval_reasons") or [])
+        recommendations.extend(payload_guardrails.get("recommendations") or [])
+
+    warnings = _dedupe_text_items(warnings)
+    blocking_reasons = _dedupe_text_items(blocking_reasons)
+    approval_reasons = _dedupe_text_items(approval_reasons)
+    recommendations = _dedupe_text_items(recommendations)
+
+    approval_required = bool(approval_reasons)
+    blocked = bool(blocking_reasons)
+
+    return {
+        "action": action,
+        "target_type": target_type,
+        "target_node_code": target_node_code,
+        "execution_mode": execution_mode,
+        "risk_level": risk_level,
+        "approval_required": approval_required,
+        "blocked": blocked,
+        "blocking_reasons": blocking_reasons,
+        "approval_reasons": approval_reasons,
+        "warnings": warnings,
+        "recommendations": recommendations,
+        "target_node": {
+            "node_code": str(target_node.get("node_code") or ""),
+            "region": str(target_node.get("region") or ""),
+            "role": node_role,
+            "status": node_status,
+            "current_load": node_current_load,
+            "is_effective_worker": node_effective_worker,
+            "detect_participating": node_detect_participating,
+        } if target_node else {},
+        "cluster_guardrails": {
+            "online_worker_nodes": int(summary.get("online_worker_nodes", 0) or 0),
+            "dedicated_online_worker_nodes": int(summary.get("dedicated_online_worker_nodes", 0) or 0),
+            "online_control_nodes": int(summary.get("online_control_nodes", 0) or 0),
+            "busy_nodes": list(summary.get("busy_nodes") or []),
+        },
+    }
+
+
+def _preview_node_batch_policy(action: str, payload: dict, cluster: dict) -> dict:
+    target_node_codes = _normalize_node_code_list(payload.get("target_node_codes") or payload.get("node_codes"))
+    execution_mode = str(payload.get("execution_mode") or "remote-agent").strip() or "remote-agent"
+    action_payload = dict(payload.get("payload") or {})
+    summary = cluster.get("summary") or {}
+
+    warnings: list[str] = []
+    blocking_reasons: list[str] = []
+    approval_reasons: list[str] = []
+    recommendations: list[str] = []
+
+    if not target_node_codes:
+        blocking_reasons.append("当前没有可预检的目标节点。")
+
+    shared_guardrails = _preview_action_payload_guardrails(
+        action,
+        action_payload,
+        execution_mode=execution_mode,
+        target_nodes_total=len(target_node_codes),
+    )
+    warnings.extend(shared_guardrails.get("warnings") or [])
+    blocking_reasons.extend(shared_guardrails.get("blocking_reasons") or [])
+    approval_reasons.extend(shared_guardrails.get("approval_reasons") or [])
+    recommendations.extend(shared_guardrails.get("recommendations") or [])
+
+    node_previews = [
+        _preview_single_node_policy(
+            action,
+            target_node_code=node_code,
+            cluster=cluster,
+            payload=action_payload,
+            execution_mode=execution_mode,
+            target_type="batch",
+            include_payload_guardrails=False,
+        )
+        for node_code in target_node_codes
+    ]
+
+    blocked_nodes = [item for item in node_previews if bool(item.get("blocked", False))]
+    approval_nodes = [item for item in node_previews if bool(item.get("approval_required", False))]
+    warning_nodes = [item for item in node_previews if list(item.get("warnings") or [])]
+    missing_nodes = [item for item in node_previews if not (item.get("target_node") or {}).get("node_code")]
+    target_control_nodes = [item for item in node_previews if str((item.get("target_node") or {}).get("role") or "") == "control"]
+
+    if blocked_nodes:
+        blocking_reasons.append(f"共有 {len(blocked_nodes)} 个目标节点存在节点级阻断，需先处理后再批量执行。")
+    if approval_nodes:
+        approval_reasons.append(f"共有 {len(approval_nodes)} 个目标节点建议先审批。")
+    if warning_nodes:
+        warnings.append(f"共有 {len(warning_nodes)} 个目标节点存在需关注事项。")
+    if missing_nodes:
+        warnings.append(f"共有 {len(missing_nodes)} 个目标节点未出现在当前集群快照中。")
+    if target_control_nodes and len(target_node_codes) > 1:
+        recommendations.append("建议把 control 节点放到批次尾部，优先在 worker 或空闲节点验证。")
+
+    warnings = _dedupe_text_items(warnings)
+    blocking_reasons = _dedupe_text_items(blocking_reasons)
+    approval_reasons = _dedupe_text_items(approval_reasons)
+    recommendations = _dedupe_text_items(recommendations)
+
+    return {
+        "action": action,
+        "target_type": "batch",
+        "execution_mode": execution_mode,
+        "risk_level": _risk_level_for_action(action),
+        "approval_required": bool(approval_reasons),
+        "blocked": bool(blocking_reasons),
+        "blocking_reasons": blocking_reasons,
+        "approval_reasons": approval_reasons,
+        "warnings": warnings,
+        "recommendations": recommendations,
+        "target_summary": {
+            "nodes_total": len(target_node_codes),
+            "blocked_nodes": len(blocked_nodes),
+            "approval_nodes": len(approval_nodes),
+            "warning_nodes": len(warning_nodes),
+            "missing_nodes": len(missing_nodes),
+        },
+        "target_nodes": [
+            {
+                **(item.get("target_node") or {}),
+                "node_code": str((item.get("target_node") or {}).get("node_code") or item.get("target_node_code") or ""),
+                "risk_level": str(item.get("risk_level") or ""),
+                "approval_required": bool(item.get("approval_required", False)),
+                "blocked": bool(item.get("blocked", False)),
+                "blocking_reasons": list(item.get("blocking_reasons") or []),
+                "approval_reasons": list(item.get("approval_reasons") or []),
+                "warnings": list(item.get("warnings") or []),
+            }
+            for item in node_previews
+        ],
+        "cluster_guardrails": {
+            "online_worker_nodes": int(summary.get("online_worker_nodes", 0) or 0),
+            "dedicated_online_worker_nodes": int(summary.get("dedicated_online_worker_nodes", 0) or 0),
+            "online_control_nodes": int(summary.get("online_control_nodes", 0) or 0),
+            "busy_nodes": list(summary.get("busy_nodes") or []),
+        },
+    }
+
+
+def _preview_rollout_policy(action: str, payload: dict, cluster: dict) -> dict:
+    selector = payload.get("target_selector") or {}
+    input_policy = payload.get("policy") or {}
+    release_id = int(payload.get("release_id") or 0)
+    input_release = dict(payload.get("release") or {})
+    release = input_release or (get_release(release_id) if release_id > 0 else {})
+    target_nodes = _resolve_rollout_targets(selector)
+    normalized_policy = _normalize_rollout_policy(input_policy, total_targets=len(target_nodes))
+    execution_mode = str(normalized_policy.get("execution_mode") or "remote-agent").strip() or "remote-agent"
+    summary = cluster.get("summary") or {}
+    release_gate = build_release_rollout_gate(
+        release,
+        target_nodes,
+        execution_mode=execution_mode,
+    )
+    operational_readiness = dict(release_gate.get("operational_readiness") or {})
+
+    risk_level = _risk_level_for_action(action)
+    warnings: list[str] = []
+    blocking_reasons: list[str] = []
+    approval_reasons: list[str] = []
+    recommendations: list[str] = []
+
+    target_count = len(target_nodes)
+    first_batch_size = int(normalized_policy.get("first_batch_size") or 0)
+    batch_size = int(normalized_policy.get("batch_size") or 0)
+    batches_total = _calculate_total_batches(target_count, normalized_policy)
+    first_batch_targets = target_nodes[:first_batch_size] if first_batch_size > 0 else []
+
+    online_worker_nodes = int(summary.get("online_worker_nodes", 0) or 0)
+    dedicated_online_worker_nodes = int(summary.get("dedicated_online_worker_nodes", 0) or 0)
+    online_control_nodes = int(summary.get("online_control_nodes", 0) or 0)
+
+    target_control_nodes = [item for item in target_nodes if str(item.get("role") or "") == "control"]
+    target_worker_nodes = [item for item in target_nodes if str(item.get("role") or "") == "worker"]
+    target_effective_workers = [item for item in target_nodes if bool(item.get("is_effective_worker", False))]
+    first_batch_effective_workers = [item for item in first_batch_targets if bool(item.get("is_effective_worker", False))]
+    first_batch_control_nodes = [item for item in first_batch_targets if str(item.get("role") or "") == "control"]
+
+    busy_targets = [item for item in target_nodes if int(item.get("current_load", 0) or 0) > 0]
+    participating_targets = [item for item in target_nodes if bool(item.get("detect_participating", False))]
+    offline_targets = [item for item in target_nodes if str(item.get("status") or "") not in {"online", "busy"}]
+
+    if target_count <= 0:
+        blocking_reasons.append("当前筛选条件未命中任何可 rollout 的目标节点。")
+    if release_id > 0 and not input_release and not release:
+        blocking_reasons.append("指定的 Release 不存在，当前无法预检该 rollout。")
+    gate_status = str(release_gate.get("status") or "").strip()
+    if gate_status in {"release_not_ready", "artifact_missing"}:
+        warnings.append(str(release_gate.get("summary") or "").strip())
+
+    if offline_targets:
+        warnings.append(f"命中了 {len(offline_targets)} 个非在线节点，若保留 only_online=false，发布时可能出现排队或失败。")
+
+    if busy_targets:
+        warnings.append(f"命中了 {len(busy_targets)} 个当前有负载的节点，建议优先确认任务是否可中断。")
+
+    if participating_targets:
+        warnings.append(f"命中了 {len(participating_targets)} 个正在参与检测的节点，建议优先滚动到空闲节点。")
+
+    if target_control_nodes:
+        approval_reasons.append("本次 rollout 覆盖 control 节点，建议强制审批，并把 control 放在最后一批。")
+
+    if first_batch_effective_workers and online_worker_nodes > 0 and len(first_batch_effective_workers) >= online_worker_nodes:
+        blocking_reasons.append("第一批会覆盖全部有效执行节点，检测能力可能瞬时归零。请缩小 first_batch_size。")
+
+    if first_batch_control_nodes and online_control_nodes > 0 and len(first_batch_control_nodes) >= online_control_nodes:
+        blocking_reasons.append("第一批会覆盖全部在线控制面，运维中枢本身可能不可用。请调整节点顺序或缩小 first_batch_size。")
+
+    if first_batch_size > 1 and target_control_nodes:
+        warnings.append("当前第一批大于 1，且目标中包含 control 节点；默认建议先 worker、后 control。")
+
+    if action in _CRITICAL_RISK_ACTIONS:
+        approval_reasons.append("该动作属于 critical 风险动作，正式环境必须审批。")
+    elif action in _HIGH_RISK_ACTIONS:
+        approval_reasons.append("该动作属于 high 风险动作，建议审批后执行。")
+
+    blocking_reasons.extend(list(release_gate.get("blocking_reasons") or []))
+    warnings.extend(list(release_gate.get("warning_reasons") or []))
+    recommendations.extend(list(release_gate.get("recommendations") or []))
+
+    readiness_summary = dict(operational_readiness.get("summary") or {})
+    if target_control_nodes and int(readiness_summary.get("inspection_healthy_nodes", 0) or 0) < target_count:
+        approval_reasons.append("本次目标中存在巡检未完全通过的节点，建议先完成标准巡检并经人工确认后再发布。")
+
+    if target_count > 1 and first_batch_size > 1:
+        recommendations.append("正式环境建议 first_batch_size=1，从单节点 canary 开始。")
+    if target_control_nodes:
+        recommendations.append("建议把 control 节点放在 rollout 末尾，并确保 worker 已先通过健康检查。")
+    if busy_targets or participating_targets:
+        recommendations.append("建议先等参与检测的节点降为空闲，再推进该批次。")
+
+    warnings = list(dict.fromkeys(item for item in warnings if str(item or "").strip()))
+    blocking_reasons = list(dict.fromkeys(item for item in blocking_reasons if str(item or "").strip()))
+    approval_reasons = list(dict.fromkeys(item for item in approval_reasons if str(item or "").strip()))
+    recommendations = list(dict.fromkeys(item for item in recommendations if str(item or "").strip()))
+
+    approval_required = bool(approval_reasons)
+    blocked = bool(blocking_reasons)
+
+    return {
+        "action": action,
+        "target_type": "rollout",
+        "risk_level": risk_level,
+        "approval_required": approval_required,
+        "blocked": blocked,
+        "blocking_reasons": blocking_reasons,
+        "approval_reasons": approval_reasons,
+        "warnings": warnings,
+        "recommendations": recommendations,
+        "target_summary": {
+            "nodes_total": target_count,
+            "worker_nodes": len(target_worker_nodes),
+            "control_nodes": len(target_control_nodes),
+            "effective_worker_nodes": len(target_effective_workers),
+            "busy_nodes": len(busy_targets),
+            "participating_nodes": len(participating_targets),
+            "offline_nodes": len(offline_targets),
+        },
+        "batch_plan": {
+            "first_batch_size": first_batch_size,
+            "batch_size": batch_size,
+            "batches_total": batches_total,
+            "first_batch_nodes": [_compact_target_node(item) for item in first_batch_targets],
+            "remaining_after_first_batch": max(target_count - len(first_batch_targets), 0),
+        },
+        "cluster_guardrails": {
+            "online_worker_nodes": online_worker_nodes,
+            "dedicated_online_worker_nodes": dedicated_online_worker_nodes,
+            "online_control_nodes": online_control_nodes,
+            "busy_nodes": list(summary.get("busy_nodes") or []),
+        },
+        "release": release,
+        "release_gate": release_gate,
+        "target_nodes": [_compact_target_node(item) for item in target_nodes],
+        "operational_readiness": operational_readiness,
+    }
+
+
+def preview_ops_job_policy(payload: dict) -> dict:
+    action = str(payload.get("action") or "").strip()
+    target_type = str(payload.get("target_type") or "node").strip() or "node"
+    target_node_code = str(payload.get("target_node_code") or "").strip()
+    execution_mode = str(payload.get("execution_mode") or "remote-agent").strip() or "remote-agent"
+    action_payload = dict(payload.get("payload") or {})
+
+    cluster = get_cluster_snapshot()
+    if target_type == "rollout":
+        return _preview_rollout_policy(action, payload, cluster)
+    if target_type in {"batch", "nodes"} or list(payload.get("target_node_codes") or []):
+        return _preview_node_batch_policy(action, payload, cluster)
+
+    return _preview_single_node_policy(
+        action,
+        target_node_code=target_node_code,
+        cluster=cluster,
+        payload=action_payload,
+        execution_mode=execution_mode,
+        target_type=target_type,
+        include_payload_guardrails=True,
+    )

domain-api/app/services/ops_release_executor_core.py

@@ -0,0 +1,475 @@
+from __future__ import annotations
+
+import hashlib
+import inspect
+import json
+import shutil
+import tarfile
+import textwrap
+import time
+import urllib.error
+import urllib.request
+from datetime import datetime
+from pathlib import Path
+
+
+def normalize_text_list(raw_value: object) -> list[str]:
+    if isinstance(raw_value, list):
+        return [str(item).strip() for item in raw_value if str(item).strip()]
+    text = str(raw_value or "").replace("\r", "\n").strip()
+    if not text:
+        return []
+    normalized = text.replace(",", "\n")
+    return [item.strip() for item in normalized.split("\n") if item.strip()]
+
+
+def coerce_bool(raw_value: object, default: bool = False) -> bool:
+    if raw_value is None:
+        return bool(default)
+    if isinstance(raw_value, bool):
+        return raw_value
+    if isinstance(raw_value, (int, float)):
+        return bool(raw_value)
+    normalized = str(raw_value or "").strip().lower()
+    if normalized in {"1", "true", "yes", "y", "on", "enable", "enabled"}:
+        return True
+    if normalized in {"0", "false", "no", "n", "off", "disable", "disabled"}:
+        return False
+    return bool(default) if normalized == "" else bool(normalized)
+
+
+def normalize_release_health_check_services(
+    payload: dict,
+    restart_services: list[str],
+    *,
+    default_api_service_name: str,
+) -> list[str]:
+    raw_health_check_services = payload.get("health_check_services")
+    if raw_health_check_services is None:
+        health_check_service_source = restart_services or [default_api_service_name]
+    else:
+        health_check_service_source = raw_health_check_services
+    return normalize_text_list(health_check_service_source)
+
+
+def collect_service_state(run_command, service_name: str) -> dict:
+    code, stdout, stderr = run_command(["systemctl", "is-active", service_name], timeout=15)
+    state = stdout or stderr
+    return {
+        "service_name": service_name,
+        "returncode": int(code or 0),
+        "state": state,
+        "ok": int(code or 0) == 0 and state == "active",
+    }
+
+
+def check_health_url(url: str, timeout: int, *, user_agent: str, urlopen_func) -> dict:
+    request = urllib.request.Request(
+        url=str(url).strip(),
+        headers={"User-Agent": str(user_agent or "domaincheck-ops/0.1").strip()},
+        method="GET",
+    )
+    try:
+        with urlopen_func(request, timeout=timeout) as response:
+            body = response.read(4000).decode("utf-8", errors="ignore")
+            status_code = int(getattr(response, "status", 0) or response.getcode() or 0)
+        return {
+            "url": str(url).strip(),
+            "ok": 200 <= status_code < 400,
+            "http_status": status_code,
+            "body_preview": body[-1000:],
+        }
+    except urllib.error.HTTPError as exc:
+        body = exc.read().decode("utf-8", errors="ignore") if exc.fp else ""
+        return {
+            "url": str(url).strip(),
+            "ok": False,
+            "http_status": int(exc.code or 0),
+            "body_preview": body[-1000:],
+            "error": str(exc),
+        }
+    except Exception as exc:
+        return {
+            "url": str(url).strip(),
+            "ok": False,
+            "http_status": 0,
+            "body_preview": "",
+            "error": str(exc),
+        }
+
+
+def run_release_health_checks(
+    *,
+    urls: list[str],
+    services: list[str],
+    timeout: int,
+    retries: int,
+    interval_seconds: int,
+    run_command,
+    user_agent: str,
+    urlopen_func,
+) -> tuple[bool, dict]:
+    normalized_urls = [str(item).strip() for item in urls if str(item).strip()]
+    normalized_services = [str(item).strip() for item in services if str(item).strip()]
+    attempts: list[dict] = []
+    total_attempts = max(1, retries + 1)
+
+    for attempt_index in range(1, total_attempts + 1):
+        service_results = [collect_service_state(run_command, service_name) for service_name in normalized_services]
+        url_results = [
+            check_health_url(url, timeout=timeout, user_agent=user_agent, urlopen_func=urlopen_func)
+            for url in normalized_urls
+        ]
+        services_ok = all(bool(item.get("ok", False)) for item in service_results) if service_results else True
+        urls_ok = all(bool(item.get("ok", False)) for item in url_results) if url_results else True
+        attempt_payload = {
+            "attempt": attempt_index,
+            "services": service_results,
+            "urls": url_results,
+            "ok": services_ok and urls_ok,
+        }
+        attempts.append(attempt_payload)
+        if attempt_payload["ok"]:
+            return True, {
+                "ok": True,
+                "attempts": attempts,
+                "services_checked": normalized_services,
+                "urls_checked": normalized_urls,
+            }
+        if attempt_index < total_attempts:
+            time.sleep(max(0, interval_seconds))
+
+    return False, {
+        "ok": False,
+        "attempts": attempts,
+        "services_checked": normalized_services,
+        "urls_checked": normalized_urls,
+    }
+
+
+def safe_extract_tar(archive: tarfile.TarFile, target_dir: Path) -> None:
+    target_dir_resolved = target_dir.resolve()
+    members = archive.getmembers()
+    for member in members:
+        member_path = (target_dir / member.name).resolve()
+        if not str(member_path).startswith(str(target_dir_resolved)):
+            raise RuntimeError(f"unsafe archive member: {member.name}")
+    try:
+        archive.extractall(target_dir, members=members, filter="data")
+    except TypeError:
+        archive.extractall(target_dir, members=members)
+
+
+def execute_release_action(
+    payload: dict,
+    *,
+    run_command,
+    default_api_service_name: str,
+    event_callback=None,
+    urlopen_func=None,
+    user_agent: str = "domaincheck-ops/0.1",
+) -> tuple[bool, str, dict]:
+    normalized_payload = dict(payload or {})
+    event_callback = event_callback or (lambda event_type, message, level="info", payload=None: None)
+    urlopen_func = urlopen_func or urllib.request.urlopen
+
+    release_version = str(normalized_payload.get("release_version") or "").strip()
+    artifact_url = str(normalized_payload.get("artifact_url") or "").strip()
+    checksum = str(normalized_payload.get("checksum") or "").strip().lower()
+    install_root = Path(str(normalized_payload.get("install_root") or "/opt/domaincheck")).resolve()
+    switch_current = coerce_bool(normalized_payload.get("switch_current", True), default=True)
+    restart_services = normalize_text_list(normalized_payload.get("restart_services"))
+    health_check_urls = normalize_text_list(normalized_payload.get("health_check_urls"))
+    health_check_services = normalize_release_health_check_services(
+        normalized_payload,
+        restart_services,
+        default_api_service_name=default_api_service_name,
+    )
+    health_check_timeout_seconds = max(2, int(normalized_payload.get("health_check_timeout_seconds") or 10))
+    health_check_retries = max(0, int(normalized_payload.get("health_check_retries") or 2))
+    health_check_interval_seconds = max(0, int(normalized_payload.get("health_check_interval_seconds") or 2))
+    rollback_on_failure = coerce_bool(normalized_payload.get("rollback_on_failure", True), default=True)
+
+    if not release_version:
+        return False, "release_version missing", {}
+    if not artifact_url:
+        return False, "artifact_url missing", {}
+
+    downloads_dir = install_root / "downloads"
+    releases_dir = install_root / "releases"
+    target_dir = releases_dir / release_version
+    temp_dir = releases_dir / f"{release_version}.tmp"
+    artifact_path = downloads_dir / f"{release_version}.tar.gz"
+    current_link = install_root / "current"
+    previous_current_target = ""
+    downloads_dir.mkdir(parents=True, exist_ok=True)
+    releases_dir.mkdir(parents=True, exist_ok=True)
+
+    if current_link.exists():
+        try:
+            previous_current_target = str(current_link.resolve(strict=True))
+        except Exception:
+            try:
+                previous_current_target = str(current_link.resolve())
+            except Exception:
+                previous_current_target = ""
+
+    event_callback(
+        "deploy_download_started",
+        f"开始下载发布包: {release_version}",
+        payload={"artifact_url": artifact_url},
+    )
+    request = urllib.request.Request(
+        url=artifact_url,
+        headers={"User-Agent": str(user_agent or "domaincheck-ops/0.1").strip()},
+        method="GET",
+    )
+    with urlopen_func(request, timeout=120) as response:
+        artifact_bytes = response.read()
+    artifact_path.write_bytes(artifact_bytes)
+    event_callback(
+        "deploy_download_completed",
+        f"发布包下载完成: {release_version}",
+        payload={"artifact_path": str(artifact_path), "size_bytes": len(artifact_bytes)},
+    )
+
+    calculated_checksum = hashlib.sha256(artifact_bytes).hexdigest().lower()
+    if checksum and checksum != calculated_checksum:
+        return False, "release checksum mismatch", {
+            "expected_checksum": checksum,
+            "calculated_checksum": calculated_checksum,
+        }
+    event_callback(
+        "deploy_checksum_verified",
+        f"发布包校验通过: {release_version}",
+        payload={"checksum": calculated_checksum},
+    )
+
+    if temp_dir.exists():
+        shutil.rmtree(temp_dir, ignore_errors=True)
+
+    temp_dir.mkdir(parents=True, exist_ok=True)
+    with tarfile.open(artifact_path, "r:gz") as archive:
+        safe_extract_tar(archive, temp_dir)
+
+    if target_dir.exists():
+        extracted_target = target_dir
+        shutil.rmtree(temp_dir, ignore_errors=True)
+    else:
+        temp_dir.rename(target_dir)
+        extracted_target = target_dir
+
+    meta_path = extracted_target / ".release-meta.json"
+    meta_path.write_text(
+        json.dumps(
+            {
+                "release_version": release_version,
+                "artifact_url": artifact_url,
+                "checksum": calculated_checksum,
+                "deployed_at": datetime.now().isoformat(sep=" ", timespec="seconds"),
+            },
+            ensure_ascii=False,
+            indent=2,
+        ),
+        encoding="utf-8",
+    )
+
+    if switch_current:
+        if current_link.is_symlink() or current_link.is_file():
+            current_link.unlink(missing_ok=True)
+        elif current_link.exists():
+            raise RuntimeError(f"current link path exists and is not a symlink/file: {current_link}")
+        current_link.symlink_to(extracted_target)
+        event_callback(
+            "deploy_current_switched",
+            f"current 已切换到 {release_version}",
+            payload={"current_link": str(current_link), "target": str(extracted_target)},
+        )
+
+    restarted: list[dict] = []
+    for service_name in restart_services:
+        normalized_service_name = str(service_name or "").strip()
+        if not normalized_service_name:
+            continue
+        event_callback(
+            "deploy_service_restart",
+            f"重启服务: {normalized_service_name}",
+            payload={"service_name": normalized_service_name},
+        )
+        code, stdout, stderr = run_command(["systemctl", "restart", normalized_service_name], timeout=30)
+        restarted.append(
+            {
+                "service_name": normalized_service_name,
+                "returncode": int(code or 0),
+                "stdout": stdout,
+                "stderr": stderr,
+            }
+        )
+        if int(code or 0) != 0:
+            return False, f"restart failed: {normalized_service_name}", {
+                "release_version": release_version,
+                "release_dir": str(extracted_target),
+                "artifact_path": str(artifact_path),
+                "checksum": calculated_checksum,
+                "previous_current_target": previous_current_target,
+                "restart_results": restarted,
+            }
+
+    health_ok, health_result = run_release_health_checks(
+        urls=health_check_urls,
+        services=health_check_services,
+        timeout=health_check_timeout_seconds,
+        retries=health_check_retries,
+        interval_seconds=health_check_interval_seconds,
+        run_command=run_command,
+        user_agent=user_agent,
+        urlopen_func=urlopen_func,
+    )
+    if not health_ok:
+        rollback_result = {
+            "attempted": rollback_on_failure,
+            "restored_to": previous_current_target,
+            "restart_results": [],
+            "health_result": health_result,
+        }
+        event_callback(
+            "deploy_health_failed",
+            f"发布健康检查失败: {release_version}",
+            level="error",
+            payload=health_result,
+        )
+        if rollback_on_failure and previous_current_target:
+            if current_link.is_symlink() or current_link.is_file():
+                current_link.unlink(missing_ok=True)
+            elif current_link.exists():
+                raise RuntimeError(f"current link path exists and is not a symlink/file: {current_link}")
+            current_link.symlink_to(Path(previous_current_target))
+            for service_name in restart_services:
+                normalized_service_name = str(service_name or "").strip()
+                if not normalized_service_name:
+                    continue
+                code, stdout, stderr = run_command(["systemctl", "restart", normalized_service_name], timeout=30)
+                rollback_result["restart_results"].append(
+                    {
+                        "service_name": normalized_service_name,
+                        "returncode": int(code or 0),
+                        "stdout": stdout,
+                        "stderr": stderr,
+                    }
+                )
+            rollback_result["post_rollback_health"] = run_release_health_checks(
+                urls=health_check_urls,
+                services=health_check_services,
+                timeout=health_check_timeout_seconds,
+                retries=0,
+                interval_seconds=0,
+                run_command=run_command,
+                user_agent=user_agent,
+                urlopen_func=urlopen_func,
+            )[1]
+            event_callback(
+                "deploy_rollback_completed",
+                f"已回滚到旧版本: {previous_current_target}",
+                level="warning",
+                payload=rollback_result,
+            )
+        return False, "release health check failed", {
+            "release_version": release_version,
+            "release_dir": str(extracted_target),
+            "artifact_path": str(artifact_path),
+            "checksum": calculated_checksum,
+            "current_link": str(current_link),
+            "previous_current_target": previous_current_target,
+            "restart_results": restarted,
+            "health_check": health_result,
+            "rollback": rollback_result,
+        }
+
+    event_callback(
+        "deploy_health_passed",
+        f"发布健康检查通过: {release_version}",
+        payload=health_result,
+    )
+    return True, "release deployed", {
+        "release_version": release_version,
+        "release_dir": str(extracted_target),
+        "artifact_path": str(artifact_path),
+        "checksum": calculated_checksum,
+        "current_link": str(current_link),
+        "previous_current_target": previous_current_target,
+        "restart_results": restarted,
+        "health_check": health_result,
+    }
+
+
+def build_remote_release_action_script(
+    payload: dict,
+    *,
+    default_api_service_name: str = "domaincheck-api",
+    user_agent: str = "domaincheck-ssh/0.1",
+) -> str:
+    helper_functions = [
+        normalize_text_list,
+        coerce_bool,
+        normalize_release_health_check_services,
+        collect_service_state,
+        check_health_url,
+        run_release_health_checks,
+        safe_extract_tar,
+        execute_release_action,
+    ]
+    helper_source = "\n\n".join(
+        textwrap.dedent(inspect.getsource(func)).strip("\n")
+        for func in helper_functions
+    )
+    return f"""from __future__ import annotations
+
+import hashlib
+import json
+import shutil
+import tarfile
+import time
+import urllib.error
+import urllib.request
+from datetime import datetime
+from pathlib import Path
+
+
+{helper_source}
+
+
+PAYLOAD = {json.dumps(dict(payload or {{}}), ensure_ascii=False)}
+DEFAULT_API_SERVICE_NAME = {json.dumps(str(default_api_service_name or 'domaincheck-api'), ensure_ascii=False)}
+USER_AGENT = {json.dumps(str(user_agent or 'domaincheck-ssh/0.1'), ensure_ascii=False)}
+
+
+def _run(command, timeout=60):
+    import subprocess
+
+    completed = subprocess.run(command, capture_output=True, text=True, timeout=timeout)
+    return int(completed.returncode or 0), str(completed.stdout or "").strip(), str(completed.stderr or "").strip()
+
+
+def _event_callback(event_type, message, level="info", payload=None):
+    return None
+
+
+def main():
+    try:
+        ok, message, result = execute_release_action(
+            PAYLOAD,
+            run_command=_run,
+            default_api_service_name=DEFAULT_API_SERVICE_NAME,
+            event_callback=_event_callback,
+            urlopen_func=urllib.request.urlopen,
+            user_agent=USER_AGENT,
+        )
+    except Exception as exc:
+        ok, message, result = False, str(exc), {{"exception": str(exc), "action": "deploy.release"}}
+    print(json.dumps({{"ok": ok, "message": message, "result": result}}, ensure_ascii=False))
+    raise SystemExit(0 if ok else 1)
+
+
+if __name__ == "__main__":
+    main()
+"""

domain-api/app/services/ops_runtime_executor_service.py

@@ -0,0 +1,420 @@
+from __future__ import annotations
+
+import json
+import subprocess
+
+from app.core.config import settings
+from app.services.ops_action_executor_core import (
+    STRUCTURED_ACTIONS,
+    build_service_name_map,
+    execute_structured_action,
+    trim_output,
+)
+from app.services.ops_release_executor_core import build_remote_release_action_script, execute_release_action
+
+
+_SSH_SUPPORTED_ACTIONS = set(STRUCTURED_ACTIONS) | {"deploy.release"}
+_SSH_CONNECT_TIMEOUT_SECONDS = 12
+_SSH_REMOTE_TIMEOUT_SECONDS = {
+    "health.snapshot": 30,
+    "service.status": 45,
+    "service.restart": 45,
+    "service.start": 45,
+    "service.stop": 45,
+    "logs.collect": 45,
+    "diagnostics.collect": 90,
+    "runtime.start_worker": 45,
+    "runtime.stop_worker": 45,
+    "runtime.restart_api": 45,
+    "runtime.start_sync_agent": 45,
+    "runtime.stop_sync_agent": 45,
+    "deploy.release": 180,
+}
+
+
+def supports_ssh_execution(action: str) -> bool:
+    return str(action or "").strip() in _SSH_SUPPORTED_ACTIONS
+
+
+def execute_local_support_action(action: str, payload: dict | None = None) -> tuple[bool, str, dict]:
+    normalized_action = str(action or "").strip()
+    if normalized_action not in _SSH_SUPPORTED_ACTIONS:
+        return False, f"当前未实现本机支持动作: {normalized_action}", {}
+    if normalized_action == "deploy.release":
+        return execute_release_action(
+            dict(payload or {}),
+            run_command=_run_local_command,
+            default_api_service_name=settings.api_service_name,
+            user_agent="domaincheck-local-runtime/0.1",
+        )
+    return execute_structured_action(
+        normalized_action,
+        dict(payload or {}),
+        service_names=_service_name_map(),
+        runner=_run_local_command,
+    )
+
+
+def execute_ssh_action(node: dict, action: str, payload: dict | None = None) -> tuple[bool, str, dict]:
+    normalized_action = str(action or "").strip()
+    if normalized_action not in _SSH_SUPPORTED_ACTIONS:
+        return False, f"当前 SSH 执行器暂不支持动作: {normalized_action}", {}
+
+    ssh_host = str(node.get("ssh_host") or "").strip()
+    ssh_user = str(node.get("ssh_user") or "").strip()
+    ssh_port = max(1, int(node.get("ssh_port") or 22))
+    node_code = str(node.get("node_code") or "").strip()
+    if not ssh_host or not ssh_user:
+        return False, "目标节点缺少 SSH 主机或用户信息", {
+            "node_code": node_code,
+            "ssh_host": ssh_host,
+            "ssh_user": ssh_user,
+            "ssh_port": ssh_port,
+        }
+
+    normalized_payload = dict(payload or {})
+    service_names = _service_name_map()
+    if normalized_action == "deploy.release":
+        remote_script = build_remote_release_action_script(
+            normalized_payload,
+            default_api_service_name=service_names.get("api") or "domaincheck-api",
+            user_agent="domaincheck-ssh/0.1",
+        )
+    else:
+        remote_script = _build_remote_python_script(
+            action=normalized_action,
+            payload=normalized_payload,
+            service_names=service_names,
+        )
+    remote_command = "\n".join(
+        [
+            "set -euo pipefail",
+            "if command -v python3 >/dev/null 2>&1; then",
+            "  PYTHON_BIN=python3",
+            "elif command -v python >/dev/null 2>&1; then",
+            "  PYTHON_BIN=python",
+            "else",
+            '  echo "{\\"ok\\": false, \\"message\\": \\"python interpreter missing on remote node\\", \\"result\\": {\\"executor\\": \\"ssh\\"}}"',
+            "  exit 127",
+            "fi",
+            '"${PYTHON_BIN}" - <<\'PY\'',
+            remote_script.rstrip("\n"),
+            "PY",
+        ]
+    )
+    ssh_command = [
+        "ssh",
+        "-o",
+        "BatchMode=yes",
+        "-o",
+        "PreferredAuthentications=publickey",
+        "-o",
+        "StrictHostKeyChecking=accept-new",
+        "-o",
+        f"ConnectTimeout={_SSH_CONNECT_TIMEOUT_SECONDS}",
+        "-p",
+        str(ssh_port),
+        f"{ssh_user}@{ssh_host}",
+        remote_command,
+    ]
+    timeout_seconds = int(_SSH_REMOTE_TIMEOUT_SECONDS.get(normalized_action, 45) or 45) + _SSH_CONNECT_TIMEOUT_SECONDS
+    completed = subprocess.run(
+        ssh_command,
+        capture_output=True,
+        text=True,
+        timeout=timeout_seconds,
+    )
+
+    transport = {
+        "executor": "ssh",
+        "node_code": node_code,
+        "ssh_host": ssh_host,
+        "ssh_user": ssh_user,
+        "ssh_port": ssh_port,
+        "action": normalized_action,
+        "returncode": int(completed.returncode or 0),
+    }
+    parsed_ok, parsed_payload = _extract_json_from_output(completed.stdout or "")
+    if parsed_ok:
+        ok = bool(parsed_payload.get("ok", False))
+        message = str(parsed_payload.get("message") or "").strip() or ("SSH 动作执行成功" if ok else "SSH 动作执行失败")
+        result = dict(parsed_payload.get("result") or {})
+        result["transport"] = {
+            **transport,
+            "stdout_preview": trim_output(completed.stdout, 4000),
+            "stderr_preview": trim_output(completed.stderr, 2000),
+        }
+        if completed.returncode != 0 and ok:
+            ok = False
+            message = f"SSH 命令返回码异常: {completed.returncode}"
+        return ok, message, result
+
+    stderr_preview = trim_output(completed.stderr, 4000)
+    stdout_preview = trim_output(completed.stdout, 4000)
+    message = stderr_preview or stdout_preview or f"SSH 执行失败，返回码 {completed.returncode}"
+    return False, message, {
+        "executor": "ssh",
+        "action": normalized_action,
+        "transport": {
+            **transport,
+            "stdout_preview": stdout_preview,
+            "stderr_preview": stderr_preview,
+        },
+    }
+
+
+def _service_name_map() -> dict[str, str]:
+    return build_service_name_map(
+        api_service_name=settings.api_service_name,
+        worker_service_name=settings.worker_service_name,
+        sync_agent_service_name=settings.sync_agent_service_name,
+    )
+
+
+def _run_local_command(command: list[str], *, timeout: int = 60) -> tuple[int, str, str]:
+    completed = subprocess.run(command, capture_output=True, text=True, timeout=timeout)
+    return int(completed.returncode or 0), str(completed.stdout or "").strip(), str(completed.stderr or "").strip()
+
+
+def _build_remote_python_script(*, action: str, payload: dict, service_names: dict[str, str]) -> str:
+    return f"""import json
+import subprocess
+import sys
+from pathlib import Path
+
+SERVICE_NAMES = {json.dumps(service_names, ensure_ascii=False)}
+ACTION = {json.dumps(str(action or '').strip(), ensure_ascii=False)}
+PAYLOAD = {json.dumps(dict(payload or {}), ensure_ascii=False)}
+
+
+def trim_output(text, limit):
+    value = str(text or "").strip()
+    if len(value) <= int(limit):
+        return value
+    return value[-int(limit):]
+
+
+def run(cmd, timeout=60):
+    completed = subprocess.run(cmd, capture_output=True, text=True, timeout=timeout)
+    return int(completed.returncode or 0), str(completed.stdout or "").strip(), str(completed.stderr or "").strip()
+
+
+def read_tail_lines(path, max_lines):
+    file_path = Path(path)
+    if not file_path.exists() or not file_path.is_file():
+        return []
+    with file_path.open("r", encoding="utf-8", errors="replace") as handle:
+        return handle.read().splitlines()[-int(max_lines):]
+
+
+def candidate_domain_roots():
+    roots = []
+    for candidate in (
+        Path.cwd() / "domainCheck",
+        Path("/opt/domaincheck/domainCheck"),
+        Path("/www/wwwroot/getDomain/domainCheck"),
+    ):
+        if candidate not in roots:
+            roots.append(candidate)
+    return roots
+
+
+def candidate_runtime_logs_dirs():
+    dirs = []
+    for candidate in (
+        Path.cwd() / "domain-api" / "runtime" / "logs",
+        Path("/opt/domaincheck/domain-api/runtime/logs"),
+        Path("/www/wwwroot/getDomain/domain-api/runtime/logs"),
+    ):
+        if candidate not in dirs:
+            dirs.append(candidate)
+    return dirs
+
+
+def service_log_file_candidates(service_name):
+    normalized_service_name = str(service_name or "").strip()
+    if not normalized_service_name:
+        return []
+    candidates = []
+
+    def append_candidate(path):
+        if path not in candidates:
+            candidates.append(path)
+
+    if normalized_service_name == (SERVICE_NAMES.get("worker") or "domaincheck-worker"):
+        for domain_root in candidate_domain_roots():
+            append_candidate(domain_root / "detect_worker.log")
+            append_candidate(domain_root / "logs" / "detect_worker.log")
+    elif normalized_service_name == (SERVICE_NAMES.get("api") or "domaincheck-api"):
+        for runtime_logs_dir in candidate_runtime_logs_dirs():
+            append_candidate(runtime_logs_dir / "domain-api.stderr.log")
+            append_candidate(runtime_logs_dir / "domain-api.stdout.log")
+        for domain_root in candidate_domain_roots():
+            append_candidate(domain_root / "logs" / "app.log")
+
+    return candidates
+
+
+def collect_log_file_fallback(service_name, lines):
+    aggregated_lines = []
+    used_paths = []
+    for path in service_log_file_candidates(service_name):
+        current_lines = read_tail_lines(path, lines)
+        if not current_lines:
+            continue
+        aggregated_lines.extend(current_lines)
+        used_paths.append(str(path))
+    if not aggregated_lines:
+        return "", []
+    return "\\n".join(aggregated_lines[-int(lines):]), used_paths
+
+
+def service_name_from_payload(payload):
+    value = str((payload or {{}}).get("service_name") or "").strip()
+    return value or SERVICE_NAMES.get("worker") or "domaincheck-worker"
+
+
+def service_name_for_action(action, payload):
+    if action in ("service.status", "service.restart", "service.start", "service.stop"):
+        return service_name_from_payload(payload)
+    runtime_action_map = {{
+        "runtime.start_worker": SERVICE_NAMES.get("worker") or "domaincheck-worker",
+        "runtime.stop_worker": SERVICE_NAMES.get("worker") or "domaincheck-worker",
+        "runtime.restart_api": SERVICE_NAMES.get("api") or "domaincheck-api",
+        "runtime.start_sync_agent": SERVICE_NAMES.get("sync_agent") or "domaincheck-sync-agent",
+        "runtime.stop_sync_agent": SERVICE_NAMES.get("sync_agent") or "domaincheck-sync-agent",
+    }}
+    return str(runtime_action_map.get(action) or "").strip()
+
+
+def systemctl_action_name(action):
+    if action in ("service.restart", "runtime.restart_api"):
+        return "restart"
+    if action in ("service.start", "runtime.start_worker", "runtime.start_sync_agent"):
+        return "start"
+    if action in ("service.stop", "runtime.stop_worker", "runtime.stop_sync_agent"):
+        return "stop"
+    return ""
+
+
+def execute(action, payload):
+    if action == "health.snapshot":
+        checks = {{}}
+        for key, service_name in SERVICE_NAMES.items():
+            code, stdout, stderr = run(["systemctl", "is-active", service_name], timeout=15)
+            checks[key] = {{
+                "service_name": service_name,
+                "returncode": int(code or 0),
+                "state": stdout or stderr,
+                "ok": int(code or 0) == 0 and (stdout or stderr) == "active",
+            }}
+        return True, "health snapshot collected", {{"checks": checks}}
+
+    if action == "service.status":
+        service_name = service_name_from_payload(payload)
+        code, stdout, stderr = run(["systemctl", "status", service_name, "--no-pager", "-l"], timeout=45)
+        result = {{
+            "service_name": service_name,
+            "stdout": trim_output(stdout, 12000),
+            "stderr": trim_output(stderr, 4000),
+        }}
+        return int(code or 0) == 0, stdout or stderr or f"{{service_name}} status collected", result
+
+    systemctl_action = systemctl_action_name(action)
+    if systemctl_action:
+        service_name = service_name_for_action(action, payload)
+        if not service_name:
+            return False, f"missing service name for action: {{action}}", {{"action": action}}
+        code, stdout, stderr = run(["systemctl", systemctl_action, service_name], timeout=45)
+        result = {{
+            "service_name": service_name,
+            "systemctl_action": systemctl_action,
+            "stdout": trim_output(stdout, 12000),
+            "stderr": trim_output(stderr, 4000),
+        }}
+        return int(code or 0) == 0, stdout or stderr or f"{{service_name}} {{systemctl_action}} completed", result
+
+    if action == "logs.collect":
+        service_name = service_name_from_payload(payload)
+        lines = max(20, min(int((payload or {{}}).get("lines") or 120), 500))
+        code, stdout, stderr = run(["journalctl", "-u", service_name, "-n", str(lines), "--no-pager"], timeout=45)
+        journal_output = stdout or stderr
+        fallback_output, fallback_paths = collect_log_file_fallback(service_name, lines)
+        collection_source = "journal"
+        effective_output = journal_output
+        ok = int(code or 0) == 0
+        if (not ok or not stdout.strip()) and fallback_output:
+            collection_source = "file_fallback"
+            effective_output = fallback_output
+            ok = True
+        result = {{
+            "service_name": service_name,
+            "lines": lines,
+            "collection_source": collection_source,
+            "fallback_used": bool(collection_source == "file_fallback"),
+            "fallback_reason": trim_output(journal_output, 4000) if collection_source == "file_fallback" else "",
+            "log_paths": fallback_paths,
+            "journal_returncode": int(code or 0),
+            "stdout": trim_output(effective_output, 20000),
+            "stderr": trim_output(stderr, 4000),
+        }}
+        return ok, effective_output or f"{{service_name}} logs collected", result
+
+    if action == "diagnostics.collect":
+        lines = max(20, min(int((payload or {{}}).get("lines") or 200), 800))
+        diagnostics = {{
+            "services": {{}},
+            "lines": lines,
+        }}
+        for key, service_name in SERVICE_NAMES.items():
+            active_code, active_stdout, active_stderr = run(["systemctl", "is-active", service_name], timeout=15)
+            status_code, status_stdout, status_stderr = run(["systemctl", "status", service_name, "--no-pager", "-l"], timeout=45)
+            log_code, log_stdout, log_stderr = run(["journalctl", "-u", service_name, "-n", str(lines), "--no-pager"], timeout=45)
+            log_output = log_stdout or log_stderr
+            fallback_output, fallback_paths = collect_log_file_fallback(service_name, lines)
+            log_source = "journal"
+            if (int(log_code or 0) != 0 or not log_stdout.strip()) and fallback_output:
+                log_output = fallback_output
+                log_source = "file_fallback"
+            diagnostics["services"][key] = {{
+                "service_name": service_name,
+                "active_returncode": int(active_code or 0),
+                "active_state": active_stdout or active_stderr,
+                "status_returncode": int(status_code or 0),
+                "status_output": trim_output(status_stdout or status_stderr, 12000),
+                "log_returncode": int(log_code or 0),
+                "log_source": log_source,
+                "log_paths": fallback_paths,
+                "log_output": trim_output(log_output, 20000),
+            }}
+        return True, "diagnostics collected", diagnostics
+
+    return False, f"unsupported action: {{action}}", {{"action": action}}
+
+
+def main():
+    try:
+        ok, message, result = execute(ACTION, PAYLOAD)
+    except Exception as exc:
+        ok, message, result = False, str(exc), {{"exception": str(exc), "action": ACTION}}
+    print(json.dumps({{"ok": ok, "message": message, "result": result}}, ensure_ascii=False))
+    raise SystemExit(0 if ok else 1)
+
+
+if __name__ == "__main__":
+    main()
+"""
+
+
+def _extract_json_from_output(text: str) -> tuple[bool, dict]:
+    for raw_line in reversed(str(text or "").splitlines()):
+        line = str(raw_line or "").strip()
+        if not line:
+            continue
+        try:
+            data = json.loads(line)
+        except Exception:
+            continue
+        if isinstance(data, dict) and "ok" in data:
+            return True, data
+    return False, {}

domain-api/app/services/ops_template_service.py

@@ -0,0 +1,920 @@
+from __future__ import annotations
+
+from copy import deepcopy
+
+from app.core.config import settings
+from app.services.ops_execution_mode_service import decorate_execution_mode_fields, execution_mode_label
+
+
+def _split_text_list(raw_value: object) -> list[str]:
+    if isinstance(raw_value, list):
+        return [str(item).strip() for item in raw_value if str(item).strip()]
+    text = str(raw_value or "").replace("\r", "\n")
+    if not text.strip():
+        return []
+    normalized = text.replace(",", "\n")
+    return [item.strip() for item in normalized.split("\n") if item.strip()]
+
+
+def _coerce_bool(raw_value: object) -> bool:
+    if isinstance(raw_value, bool):
+        return raw_value
+    if isinstance(raw_value, (int, float)):
+        return bool(raw_value)
+    normalized = str(raw_value or "").strip().lower()
+    if normalized in {"1", "true", "yes", "y", "on", "enable", "enabled"}:
+        return True
+    if normalized in {"0", "false", "no", "n", "off", "disable", "disabled"}:
+        return False
+    return bool(normalized)
+
+
+def _is_missing_value(field_type: str, value: object) -> bool:
+    if field_type == "text_list":
+        return not list(value or [])
+    if field_type == "number":
+        return value is None
+    if field_type == "boolean":
+        return value is None
+    return str(value or "").strip() == ""
+
+
+def _service_options() -> list[dict]:
+    return [
+        {"label": f"API ({settings.api_service_name})", "value": settings.api_service_name},
+        {"label": f"Worker ({settings.worker_service_name})", "value": settings.worker_service_name},
+        {"label": f"Sync Agent ({settings.sync_agent_service_name})", "value": settings.sync_agent_service_name},
+        {"label": "Node Agent (domaincheck-node-agent)", "value": "domaincheck-node-agent"},
+    ]
+
+
+def _template_catalog() -> list[dict]:
+    service_options = _service_options()
+    return [
+        {
+            "group_key": "runtime",
+            "group_title": "运行时控制",
+            "group_description": "适合日常按钮化控制 Worker / API / Sync Agent。",
+            "items": [
+                {
+                    "key": "runtime.start_worker",
+                    "title": "启动 Worker",
+                    "action": "runtime.start_worker",
+                    "description": "启动目标节点的检测 Worker。",
+                    "risk_level": "medium",
+                    "default_execution_mode": "remote-agent",
+                    "execution_modes": ["remote-agent", "local-runtime", "ssh"],
+                    "target_roles": ["control", "worker"],
+                    "default_auto_approve": False,
+                    "fields": [],
+                },
+                {
+                    "key": "runtime.stop_worker",
+                    "title": "停止 Worker",
+                    "action": "runtime.stop_worker",
+                    "description": "停止目标节点的检测 Worker。",
+                    "risk_level": "high",
+                    "default_execution_mode": "remote-agent",
+                    "execution_modes": ["remote-agent", "local-runtime", "ssh"],
+                    "target_roles": ["control", "worker"],
+                    "default_auto_approve": False,
+                    "fields": [],
+                },
+                {
+                    "key": "runtime.restart_api",
+                    "title": "重启 API",
+                    "action": "runtime.restart_api",
+                    "description": "重启目标节点的 domaincheck-api 服务。",
+                    "risk_level": "high",
+                    "default_execution_mode": "remote-agent",
+                    "execution_modes": ["remote-agent", "local-runtime", "ssh"],
+                    "target_roles": ["control"],
+                    "default_auto_approve": False,
+                    "fields": [],
+                },
+                {
+                    "key": "runtime.start_sync_agent",
+                    "title": "启动 Sync Agent",
+                    "action": "runtime.start_sync_agent",
+                    "description": "启动目标节点的同步代理服务。",
+                    "risk_level": "medium",
+                    "default_execution_mode": "remote-agent",
+                    "execution_modes": ["remote-agent", "local-runtime", "ssh"],
+                    "target_roles": ["control"],
+                    "default_auto_approve": False,
+                    "fields": [],
+                },
+                {
+                    "key": "runtime.stop_sync_agent",
+                    "title": "停止 Sync Agent",
+                    "action": "runtime.stop_sync_agent",
+                    "description": "停止目标节点的同步代理服务。",
+                    "risk_level": "high",
+                    "default_execution_mode": "remote-agent",
+                    "execution_modes": ["remote-agent", "local-runtime", "ssh"],
+                    "target_roles": ["control"],
+                    "default_auto_approve": False,
+                    "fields": [],
+                },
+                {
+                    "key": "service.restart",
+                    "title": "重启任意服务",
+                    "action": "service.restart",
+                    "description": "对指定 systemd 服务执行 restart。",
+                    "risk_level": "high",
+                    "default_execution_mode": "remote-agent",
+                    "execution_modes": ["remote-agent", "local-runtime", "ssh"],
+                    "target_roles": ["control", "worker"],
+                    "default_auto_approve": False,
+                    "fields": [
+                        {
+                            "key": "service_name",
+                            "label": "服务名",
+                            "type": "select",
+                            "required": True,
+                            "default": settings.worker_service_name,
+                            "options": service_options,
+                            "help": "默认提供 API / Worker / Sync Agent / Node Agent。",
+                        }
+                    ],
+                },
+            ],
+        },
+        {
+            "group_key": "diagnostics",
+            "group_title": "巡检与取证",
+            "group_description": "适合做联调、排障、日志回收和节点健康诊断。",
+            "items": [
+                {
+                    "key": "health.snapshot",
+                    "title": "采集健康快照",
+                    "action": "health.snapshot",
+                    "description": "采集目标节点关键 systemd 服务状态。",
+                    "risk_level": "low",
+                    "default_execution_mode": "remote-agent",
+                    "execution_modes": ["remote-agent", "local-runtime", "ssh"],
+                    "target_roles": ["control", "worker"],
+                    "default_auto_approve": True,
+                    "fields": [],
+                },
+                {
+                    "key": "logs.collect",
+                    "title": "收集服务日志",
+                    "action": "logs.collect",
+                    "description": "抓取指定服务最近 journalctl 日志。",
+                    "risk_level": "low",
+                    "default_execution_mode": "remote-agent",
+                    "execution_modes": ["remote-agent", "local-runtime", "ssh"],
+                    "target_roles": ["control", "worker"],
+                    "default_auto_approve": True,
+                    "fields": [
+                        {
+                            "key": "service_name",
+                            "label": "服务名",
+                            "type": "select",
+                            "required": True,
+                            "default": settings.worker_service_name,
+                            "options": service_options,
+                            "help": "默认抓 Worker；排 API 问题时可切到 domaincheck-api。",
+                        },
+                        {
+                            "key": "lines",
+                            "label": "日志行数",
+                            "type": "number",
+                            "required": True,
+                            "default": 120,
+                            "min": 20,
+                            "max": 500,
+                            "help": "journalctl -n 的行数，适合短时排障。",
+                        },
+                    ],
+                },
+                {
+                    "key": "diagnostics.collect",
+                    "title": "收集诊断包",
+                    "action": "diagnostics.collect",
+                    "description": "打包 API / Worker / Sync Agent / Node Agent 的状态与近期日志。",
+                    "risk_level": "low",
+                    "default_execution_mode": "remote-agent",
+                    "execution_modes": ["remote-agent", "local-runtime", "ssh"],
+                    "target_roles": ["control", "worker"],
+                    "default_auto_approve": True,
+                    "fields": [
+                        {
+                            "key": "lines",
+                            "label": "每个服务日志行数",
+                            "type": "number",
+                            "required": True,
+                            "default": 200,
+                            "min": 20,
+                            "max": 800,
+                            "help": "越大越适合深度排障，但任务返回会更重。",
+                        }
+                    ],
+                },
+                {
+                    "key": "service.status",
+                    "title": "采集服务状态",
+                    "action": "service.status",
+                    "description": "执行 systemctl status --no-pager -l。",
+                    "risk_level": "low",
+                    "default_execution_mode": "remote-agent",
+                    "execution_modes": ["remote-agent", "local-runtime", "ssh"],
+                    "target_roles": ["control", "worker"],
+                    "default_auto_approve": True,
+                    "fields": [
+                        {
+                            "key": "service_name",
+                            "label": "服务名",
+                            "type": "select",
+                            "required": True,
+                            "default": settings.worker_service_name,
+                            "options": service_options,
+                            "help": "用于查看特定服务当前状态和最近日志片段。",
+                        }
+                    ],
+                },
+            ],
+        },
+        {
+            "group_key": "delivery_queue",
+            "group_title": "回执队列治理",
+            "group_description": "适合统一处理 Node Agent 的 pending / dead-letter 回执队列，而不是手工登机删文件。",
+            "items": [
+                {
+                    "key": "delivery.queue.flush",
+                    "title": "立即冲刷回执队列",
+                    "action": "delivery.queue.flush",
+                    "description": "立即触发目标节点 Node Agent 冲刷 pending 回执队列。",
+                    "risk_level": "low",
+                    "default_execution_mode": "remote-agent",
+                    "execution_modes": ["remote-agent"],
+                    "target_roles": ["control", "worker"],
+                    "default_auto_approve": True,
+                    "fields": [
+                        {
+                            "key": "limit",
+                            "label": "本次冲刷上限",
+                            "type": "number",
+                            "required": True,
+                            "default": 20,
+                            "min": 1,
+                            "max": 200,
+                            "help": "限制本次最多冲刷的回执数量，避免一次返回过大。",
+                        }
+                    ],
+                },
+                {
+                    "key": "delivery.queue.replay",
+                    "title": "重放死信回执",
+                    "action": "delivery.queue.replay",
+                    "description": "把 dead-letter 记录重新放回 pending，并可立即触发冲刷。",
+                    "risk_level": "medium",
+                    "default_execution_mode": "remote-agent",
+                    "execution_modes": ["remote-agent"],
+                    "target_roles": ["control", "worker"],
+                    "default_auto_approve": False,
+                    "fields": [
+                        {
+                            "key": "record_id",
+                            "label": "单条记录 ID",
+                            "type": "text",
+                            "required": False,
+                            "default": "",
+                            "help": "留空表示按下面的 selector 条件批量匹配。",
+                        },
+                        {
+                            "key": "request_kind",
+                            "label": "记录类型",
+                            "type": "text",
+                            "required": False,
+                            "default": "",
+                            "help": "可选 job_complete / job_event。",
+                        },
+                        {
+                            "key": "detail_code",
+                            "label": "错误码",
+                            "type": "text",
+                            "required": False,
+                            "default": "",
+                            "help": "按 last_detail_code 过滤死信记录。",
+                        },
+                        {
+                            "key": "limit",
+                            "label": "重放上限",
+                            "type": "number",
+                            "required": True,
+                            "default": 20,
+                            "min": 1,
+                            "max": 200,
+                            "help": "批量重放时最多处理多少条死信。",
+                        },
+                        {
+                            "key": "flush_after_replay",
+                            "label": "重放后立即冲刷",
+                            "type": "boolean",
+                            "required": True,
+                            "default": True,
+                            "help": "开启后会把重放回 pending 的记录立即补发一次。",
+                        },
+                        {
+                            "key": "reason",
+                            "label": "重放原因",
+                            "type": "text",
+                            "required": False,
+                            "default": "",
+                            "wide": True,
+                            "help": "可选，便于后续回溯为什么执行此次重放。",
+                        },
+                    ],
+                },
+                {
+                    "key": "delivery.queue.discard",
+                    "title": "丢弃死信回执",
+                    "action": "delivery.queue.discard",
+                    "description": "把死信移出活动队列，保存到 discarded 归档目录。",
+                    "risk_level": "high",
+                    "default_execution_mode": "remote-agent",
+                    "execution_modes": ["remote-agent"],
+                    "target_roles": ["control", "worker"],
+                    "default_auto_approve": False,
+                    "fields": [
+                        {
+                            "key": "record_id",
+                            "label": "单条记录 ID",
+                            "type": "text",
+                            "required": False,
+                            "default": "",
+                            "help": "留空表示按下面的 selector 条件批量匹配。",
+                        },
+                        {
+                            "key": "request_kind",
+                            "label": "记录类型",
+                            "type": "text",
+                            "required": False,
+                            "default": "",
+                            "help": "可选 job_complete / job_event。",
+                        },
+                        {
+                            "key": "detail_code",
+                            "label": "错误码",
+                            "type": "text",
+                            "required": False,
+                            "default": "",
+                            "help": "按 last_detail_code 过滤死信记录。",
+                        },
+                        {
+                            "key": "limit",
+                            "label": "丢弃上限",
+                            "type": "number",
+                            "required": True,
+                            "default": 20,
+                            "min": 1,
+                            "max": 200,
+                            "help": "批量丢弃时最多处理多少条死信。",
+                        },
+                        {
+                            "key": "reason",
+                            "label": "丢弃原因",
+                            "type": "text",
+                            "required": True,
+                            "default": "",
+                            "wide": True,
+                            "help": "必须说明为何确认这些死信可以被放弃。",
+                        },
+                    ],
+                },
+            ],
+        },
+        {
+            "group_key": "onboarding",
+            "group_title": "接管与纳管",
+            "group_description": "适合在控制面生成节点接入工单、统一沉淀首轮接管动作。",
+            "items": [
+                {
+                    "key": "node.bootstrap",
+                    "title": "生成节点接入工单",
+                    "action": "node.bootstrap",
+                    "description": "在控制面签发 Node Agent Token，并生成 bootstrap env / 脚本 / 一键落地命令。",
+                    "risk_level": "critical",
+                    "default_execution_mode": "control-plane",
+                    "execution_modes": ["control-plane"],
+                    "target_roles": ["control", "worker"],
+                    "default_auto_approve": False,
+                    "fields": [
+                        {
+                            "key": "control_plane_base_url",
+                            "label": "控制面地址",
+                            "type": "text",
+                            "required": False,
+                            "default": "",
+                            "wide": True,
+                            "help": "可留空。留空时由控制面自动回退到当前 API 地址。",
+                        },
+                        {
+                            "key": "root_dir",
+                            "label": "目标安装目录",
+                            "type": "text",
+                            "required": True,
+                            "default": "/opt/domaincheck",
+                            "help": "bootstrap_node_agent.sh 安装根目录。",
+                        },
+                        {
+                            "key": "expires_in_hours",
+                            "label": "Token 有效期（小时）",
+                            "type": "number",
+                            "required": True,
+                            "default": 72,
+                            "min": 1,
+                            "max": 720,
+                            "help": "控制新签发 token 的过期时间。",
+                        },
+                        {
+                            "key": "node_region",
+                            "label": "节点地域提示",
+                            "type": "text",
+                            "required": False,
+                            "default": "",
+                            "help": "可留空；留空时优先从托管节点或集群快照自动推断。",
+                        },
+                        {
+                            "key": "node_role",
+                            "label": "节点角色提示",
+                            "type": "text",
+                            "required": False,
+                            "default": "",
+                            "help": "可留空；留空时优先从托管节点或集群快照自动推断。",
+                        },
+                    ],
+                },
+            ],
+        },
+        {
+            "group_key": "release",
+            "group_title": "发布与变更",
+            "group_description": "适合把单节点灰度、点状修复和手工发布收编到标准 deploy.release 任务。",
+            "items": [
+                {
+                    "key": "deploy.release.control",
+                    "title": "发布控制面节点",
+                    "action": "deploy.release",
+                    "description": "面向 controller 节点的标准发布任务，默认会重启 API / Worker / Sync Agent 并执行 API 健康检查。",
+                    "risk_level": "high",
+                    "default_execution_mode": "remote-agent",
+                    "execution_modes": ["remote-agent", "ssh"],
+                    "target_roles": ["control"],
+                    "default_auto_approve": False,
+                    "fields": [
+                        {
+                            "key": "release_version",
+                            "label": "版本号",
+                            "type": "text",
+                            "required": True,
+                            "default": "",
+                            "help": "例如 2026.04.18-rc1，用于 releases/<version> 目录名。",
+                        },
+                        {
+                            "key": "artifact_url",
+                            "label": "发布包地址",
+                            "type": "textarea",
+                            "required": True,
+                            "default": "",
+                            "wide": True,
+                            "rows": 2,
+                            "help": "Node Agent 会在目标节点直接下载 tar.gz 发布包。",
+                        },
+                        {
+                            "key": "checksum",
+                            "label": "SHA256 校验",
+                            "type": "text",
+                            "required": False,
+                            "default": "",
+                            "wide": True,
+                            "help": "可留空；填写后会强校验发布包完整性。",
+                        },
+                        {
+                            "key": "install_root",
+                            "label": "安装根目录",
+                            "type": "text",
+                            "required": True,
+                            "default": "/opt/domaincheck",
+                            "help": "目标节点上的 releases/current 根目录。",
+                        },
+                        {
+                            "key": "restart_services",
+                            "label": "重启服务",
+                            "type": "text_list",
+                            "required": False,
+                            "default": [
+                                settings.api_service_name,
+                                settings.worker_service_name,
+                                settings.sync_agent_service_name,
+                            ],
+                            "wide": True,
+                            "rows": 3,
+                            "help": "每行一个服务。控制面默认重启 API / Worker / Sync Agent。",
+                        },
+                        {
+                            "key": "health_check_urls",
+                            "label": "健康检查 URL",
+                            "type": "text_list",
+                            "required": False,
+                            "default": ["http://127.0.0.1:8100/health"],
+                            "wide": True,
+                            "rows": 2,
+                            "help": "每行一个 URL。控制面默认探活本机 API /health。",
+                        },
+                        {
+                            "key": "health_check_services",
+                            "label": "健康检查服务",
+                            "type": "text_list",
+                            "required": False,
+                            "default": [
+                                settings.api_service_name,
+                                settings.worker_service_name,
+                                settings.sync_agent_service_name,
+                            ],
+                            "wide": True,
+                            "rows": 3,
+                            "help": "每行一个 systemd 服务，发布后会检查其 active 状态。",
+                        },
+                        {
+                            "key": "health_check_timeout_seconds",
+                            "label": "URL 超时",
+                            "type": "number",
+                            "required": True,
+                            "default": 10,
+                            "min": 2,
+                            "max": 120,
+                            "help": "每次 URL 探活超时秒数。",
+                        },
+                        {
+                            "key": "health_check_retries",
+                            "label": "重试次数",
+                            "type": "number",
+                            "required": True,
+                            "default": 2,
+                            "min": 0,
+                            "max": 10,
+                            "help": "健康检查失败后的额外重试次数。",
+                        },
+                        {
+                            "key": "health_check_interval_seconds",
+                            "label": "重试间隔",
+                            "type": "number",
+                            "required": True,
+                            "default": 2,
+                            "min": 0,
+                            "max": 60,
+                            "help": "每次重试前的等待秒数。",
+                        },
+                        {
+                            "key": "rollback_on_failure",
+                            "label": "失败自动回滚",
+                            "type": "boolean",
+                            "required": True,
+                            "default": True,
+                            "help": "健康检查失败时自动切回旧 current 并重启服务。",
+                        },
+                        {
+                            "key": "switch_current",
+                            "label": "切换 current 软链",
+                            "type": "boolean",
+                            "required": True,
+                            "default": True,
+                            "help": "关闭后只解压版本目录，不切 current。",
+                        },
+                    ],
+                },
+                {
+                    "key": "deploy.release.worker",
+                    "title": "发布 Worker 节点",
+                    "action": "deploy.release",
+                    "description": "面向独立 Worker 节点的标准发布任务，默认只重启 Worker 并校验 Worker 服务状态。",
+                    "risk_level": "high",
+                    "default_execution_mode": "remote-agent",
+                    "execution_modes": ["remote-agent", "ssh"],
+                    "target_roles": ["worker"],
+                    "default_auto_approve": False,
+                    "fields": [
+                        {
+                            "key": "release_version",
+                            "label": "版本号",
+                            "type": "text",
+                            "required": True,
+                            "default": "",
+                            "help": "例如 2026.04.18-rc1，用于 releases/<version> 目录名。",
+                        },
+                        {
+                            "key": "artifact_url",
+                            "label": "发布包地址",
+                            "type": "textarea",
+                            "required": True,
+                            "default": "",
+                            "wide": True,
+                            "rows": 2,
+                            "help": "Node Agent 会在目标节点直接下载 tar.gz 发布包。",
+                        },
+                        {
+                            "key": "checksum",
+                            "label": "SHA256 校验",
+                            "type": "text",
+                            "required": False,
+                            "default": "",
+                            "wide": True,
+                            "help": "可留空；填写后会强校验发布包完整性。",
+                        },
+                        {
+                            "key": "install_root",
+                            "label": "安装根目录",
+                            "type": "text",
+                            "required": True,
+                            "default": "/opt/domaincheck",
+                            "help": "目标节点上的 releases/current 根目录。",
+                        },
+                        {
+                            "key": "restart_services",
+                            "label": "重启服务",
+                            "type": "text_list",
+                            "required": False,
+                            "default": [settings.worker_service_name],
+                            "wide": True,
+                            "rows": 2,
+                            "help": "每行一个服务。独立 Worker 默认只重启 Worker。",
+                        },
+                        {
+                            "key": "health_check_urls",
+                            "label": "健康检查 URL",
+                            "type": "text_list",
+                            "required": False,
+                            "default": [],
+                            "wide": True,
+                            "rows": 2,
+                            "help": "Worker 节点通常可留空；为空时不做 URL 探活。",
+                        },
+                        {
+                            "key": "health_check_services",
+                            "label": "健康检查服务",
+                            "type": "text_list",
+                            "required": False,
+                            "default": [settings.worker_service_name],
+                            "wide": True,
+                            "rows": 2,
+                            "help": "每行一个 systemd 服务，默认只校验 Worker active 状态。",
+                        },
+                        {
+                            "key": "health_check_timeout_seconds",
+                            "label": "URL 超时",
+                            "type": "number",
+                            "required": True,
+                            "default": 10,
+                            "min": 2,
+                            "max": 120,
+                            "help": "每次 URL 探活超时秒数。",
+                        },
+                        {
+                            "key": "health_check_retries",
+                            "label": "重试次数",
+                            "type": "number",
+                            "required": True,
+                            "default": 2,
+                            "min": 0,
+                            "max": 10,
+                            "help": "健康检查失败后的额外重试次数。",
+                        },
+                        {
+                            "key": "health_check_interval_seconds",
+                            "label": "重试间隔",
+                            "type": "number",
+                            "required": True,
+                            "default": 2,
+                            "min": 0,
+                            "max": 60,
+                            "help": "每次重试前的等待秒数。",
+                        },
+                        {
+                            "key": "rollback_on_failure",
+                            "label": "失败自动回滚",
+                            "type": "boolean",
+                            "required": True,
+                            "default": True,
+                            "help": "健康检查失败时自动切回旧 current 并重启服务。",
+                        },
+                        {
+                            "key": "switch_current",
+                            "label": "切换 current 软链",
+                            "type": "boolean",
+                            "required": True,
+                            "default": True,
+                            "help": "关闭后只解压版本目录，不切 current。",
+                        },
+                    ],
+                },
+                {
+                    "key": "deploy.release.custom",
+                    "title": "发布自定义节点",
+                    "action": "deploy.release",
+                    "description": "完全自定义 deploy.release 参数，适合灰度验证、特殊节点或后续扩展场景。",
+                    "risk_level": "high",
+                    "default_execution_mode": "remote-agent",
+                    "execution_modes": ["remote-agent", "ssh"],
+                    "target_roles": ["control", "worker"],
+                    "default_auto_approve": False,
+                    "fields": [
+                        {
+                            "key": "release_version",
+                            "label": "版本号",
+                            "type": "text",
+                            "required": True,
+                            "default": "",
+                            "help": "例如 2026.04.18-rc1，用于 releases/<version> 目录名。",
+                        },
+                        {
+                            "key": "artifact_url",
+                            "label": "发布包地址",
+                            "type": "textarea",
+                            "required": True,
+                            "default": "",
+                            "wide": True,
+                            "rows": 2,
+                            "help": "Node Agent 会在目标节点直接下载 tar.gz 发布包。",
+                        },
+                        {
+                            "key": "checksum",
+                            "label": "SHA256 校验",
+                            "type": "text",
+                            "required": False,
+                            "default": "",
+                            "wide": True,
+                            "help": "可留空；填写后会强校验发布包完整性。",
+                        },
+                        {
+                            "key": "install_root",
+                            "label": "安装根目录",
+                            "type": "text",
+                            "required": True,
+                            "default": "/opt/domaincheck",
+                            "help": "目标节点上的 releases/current 根目录。",
+                        },
+                        {
+                            "key": "restart_services",
+                            "label": "重启服务",
+                            "type": "text_list",
+                            "required": False,
+                            "default": [],
+                            "wide": True,
+                            "rows": 3,
+                            "help": "每行一个服务。留空表示只落盘版本，不主动重启任何服务。",
+                        },
+                        {
+                            "key": "health_check_urls",
+                            "label": "健康检查 URL",
+                            "type": "text_list",
+                            "required": False,
+                            "default": [],
+                            "wide": True,
+                            "rows": 2,
+                            "help": "每行一个 URL。留空表示不做 URL 探活。",
+                        },
+                        {
+                            "key": "health_check_services",
+                            "label": "健康检查服务",
+                            "type": "text_list",
+                            "required": False,
+                            "default": [],
+                            "wide": True,
+                            "rows": 3,
+                            "help": "每行一个 systemd 服务。显式留空时不会自动回落到默认 API 检查。",
+                        },
+                        {
+                            "key": "health_check_timeout_seconds",
+                            "label": "URL 超时",
+                            "type": "number",
+                            "required": True,
+                            "default": 10,
+                            "min": 2,
+                            "max": 120,
+                            "help": "每次 URL 探活超时秒数。",
+                        },
+                        {
+                            "key": "health_check_retries",
+                            "label": "重试次数",
+                            "type": "number",
+                            "required": True,
+                            "default": 2,
+                            "min": 0,
+                            "max": 10,
+                            "help": "健康检查失败后的额外重试次数。",
+                        },
+                        {
+                            "key": "health_check_interval_seconds",
+                            "label": "重试间隔",
+                            "type": "number",
+                            "required": True,
+                            "default": 2,
+                            "min": 0,
+                            "max": 60,
+                            "help": "每次重试前的等待秒数。",
+                        },
+                        {
+                            "key": "rollback_on_failure",
+                            "label": "失败自动回滚",
+                            "type": "boolean",
+                            "required": True,
+                            "default": True,
+                            "help": "健康检查失败时自动切回旧 current 并重启服务。",
+                        },
+                        {
+                            "key": "switch_current",
+                            "label": "切换 current 软链",
+                            "type": "boolean",
+                            "required": True,
+                            "default": True,
+                            "help": "关闭后只解压版本目录，不切 current。",
+                        },
+                    ],
+                },
+            ],
+        },
+    ]
+
+
+def get_ops_action_templates() -> dict:
+    groups = deepcopy(_template_catalog())
+    items: list[dict] = []
+    for group in groups:
+        group_key = str(group.get("group_key") or "").strip()
+        group_title = str(group.get("group_title") or "").strip()
+        group_description = str(group.get("group_description") or "").strip()
+        normalized_group_items: list[dict] = []
+        for item in list(group.get("items") or []):
+            normalized_item = decorate_execution_mode_fields(dict(item))
+            normalized_item["group_key"] = group_key
+            normalized_item["group_title"] = group_title
+            normalized_item["group_description"] = group_description
+            normalized_group_items.append(normalized_item)
+            items.append(normalized_item)
+        group["items"] = normalized_group_items
+    return {
+        "groups": groups,
+        "items": items,
+        "defaults": {
+            "requested_by": "web-ui",
+            "execution_mode": "remote-agent",
+            "execution_mode_label": execution_mode_label("remote-agent"),
+        },
+    }
+
+
+def get_ops_action_template(template_key: str) -> dict:
+    normalized_key = str(template_key or "").strip()
+    if not normalized_key:
+        return {}
+    for group in _template_catalog():
+        group_key = str(group.get("group_key") or "").strip()
+        group_title = str(group.get("group_title") or "").strip()
+        group_description = str(group.get("group_description") or "").strip()
+        for item in list(group.get("items") or []):
+            if str(item.get("key") or "").strip() == normalized_key:
+                normalized_item = decorate_execution_mode_fields(deepcopy(item))
+                normalized_item["group_key"] = group_key
+                normalized_item["group_title"] = group_title
+                normalized_item["group_description"] = group_description
+                return normalized_item
+    return {}
+
+
+def build_ops_template_payload(template_key: str, payload: dict | None = None) -> tuple[bool, str, dict]:
+    template = get_ops_action_template(template_key)
+    if not template:
+        return False, "动作模板不存在", {}
+
+    source_payload = dict(payload or {})
+    normalized_payload: dict = {}
+    for field in list(template.get("fields") or []):
+        field_key = str(field.get("key") or "").strip()
+        if not field_key:
+            continue
+        field_type = str(field.get("type") or "text").strip()
+        raw_value = source_payload.get(field_key, field.get("default"))
+        if field_type == "number":
+            try:
+                value = int(raw_value or 0)
+            except Exception:
+                return False, f"{field_key} 必须是数字", {}
+            min_value = field.get("min")
+            max_value = field.get("max")
+            if min_value is not None:
+                value = max(int(min_value), value)
+            if max_value is not None:
+                value = min(int(max_value), value)
+        elif field_type == "boolean":
+            value = _coerce_bool(raw_value)
+        elif field_type == "text_list":
+            value = _split_text_list(raw_value)
+        else:
+            value = str(raw_value or "").strip()
+        if bool(field.get("required", False)) and _is_missing_value(field_type, value):
+            return False, f"{field_key} 不能为空", {}
+        normalized_payload[field_key] = value
+    return True, "ok", normalized_payload

domain-api/app/services/runtime_status_service.py

@@ -7,6 +7,7 @@ from app.core.config import settings
 from app.core.db import get_db
 from app.core.files import read_json
 from app.core.redis_client import get_redis
+from app.services.build_info_service import get_runtime_build_info
 from app.services.cluster_runtime_service import get_cluster_snapshot
 from app.services.detect_service import get_detect_status
 from app.services.detect_job_service import get_detect_capacity_plan, get_detect_queue_health
@@ -187,6 +188,89 @@ def _build_multi_region_readiness(
     }
 
 
+def _detect_participation_snapshot(*, row: dict) -> dict:
+    items_running = int(row.get("items_running", 0) or 0)
+    items_claimed = int(row.get("items_claimed", 0) or 0)
+    processed_recent = int(row.get("processed_recent", 0) or 0)
+    current_load = int(row.get("current_load", 0) or 0)
+    status = str(row.get("status") or "").strip().lower()
+
+    if items_running > 0:
+        return {
+            "participation_state": "running",
+            "participation_label": "执行中",
+            "participation_reason": f"当前正在执行 {items_running} 项检测任务。",
+            "is_current_participant": True,
+            "is_dispatch_active": True,
+        }
+    if items_claimed > 0:
+        return {
+            "participation_state": "claimed",
+            "participation_label": "已领待跑",
+            "participation_reason": f"已领取 {items_claimed} 项任务，等待线程继续执行。",
+            "is_current_participant": True,
+            "is_dispatch_active": True,
+        }
+    if processed_recent > 0:
+        return {
+            "participation_state": "recent_throughput",
+            "participation_label": "近窗有吞吐",
+            "participation_reason": f"近 15 分钟内已处理 {processed_recent} 项任务。",
+            "is_current_participant": True,
+            "is_dispatch_active": False,
+        }
+    if current_load > 0 or status == "busy":
+        load_value = max(current_load, 1)
+        return {
+            "participation_state": "load_syncing",
+            "participation_label": "负载待确认",
+            "participation_reason": f"节点当前负载为 {load_value}，但还未观察到已领、执行中或近窗吞吐数据，先归入在线未参与观察。",
+            "is_current_participant": False,
+            "is_dispatch_active": False,
+        }
+    return {
+        "participation_state": "standby",
+        "participation_label": "在线待命",
+        "participation_reason": "当前未领任务、未执行任务，也没有近窗吞吐。",
+        "is_current_participant": False,
+        "is_dispatch_active": False,
+    }
+
+
+def _build_detect_node_row(*, node_code: str, cluster_node: dict, job_node: dict, queue_node: dict) -> dict:
+    metadata = cluster_node.get("metadata") or {}
+    role = str(cluster_node.get("role") or job_node.get("role") or "worker")
+    region = str(cluster_node.get("region") or settings.node_region)
+    is_effective_worker = bool(cluster_node.get("is_effective_worker", False) or role == "worker")
+    items_total = int(job_node.get("items_total", metadata.get("job_items_total", 0)) or 0)
+    items_claimed = int(job_node.get("items_claimed", metadata.get("job_items_claimed", 0)) or 0)
+    items_running = int(job_node.get("items_running", metadata.get("job_items_running", 0)) or 0)
+    items_completed = int(job_node.get("items_completed", metadata.get("job_items_completed", 0)) or 0)
+    items_failed = int(job_node.get("items_failed", metadata.get("job_items_failed", 0)) or 0)
+    row = {
+        "node_code": node_code,
+        "role": role,
+        "region": region,
+        "status": str(cluster_node.get("status") or "unknown"),
+        "is_effective_worker": is_effective_worker,
+        "detect_participating": False,
+        "current_load": int(cluster_node.get("current_load", 0) or 0),
+        "items_total": items_total,
+        "items_pending": int(job_node.get("items_pending", max(items_total - items_claimed - items_completed - items_failed, 0)) or 0),
+        "items_claimed": items_claimed,
+        "items_running": items_running,
+        "items_completed": items_completed,
+        "items_failed": items_failed,
+        "processed_recent": int(queue_node.get("processed_recent", 0) or 0),
+        "processed_per_minute": float(queue_node.get("processed_per_minute", 0) or 0),
+        "last_heartbeat_at": str(cluster_node.get("last_heartbeat_at") or ""),
+        "metrics_source": "active_job" if bool(job_node) else ("cluster_metadata" if metadata else "derived"),
+    }
+    row.update(_detect_participation_snapshot(row=row))
+    row["detect_participating"] = bool(row.get("is_current_participant", False))
+    return row
+
+
 def _build_participating_detect_nodes(*, cluster_snapshot: dict, detect_snapshot: dict, worker_runtime: dict) -> list[dict]:
     cluster_nodes = list(cluster_snapshot.get("nodes") or [])
     active_job = detect_snapshot.get("active_job") or {}
@@ -204,56 +288,25 @@ def _build_participating_detect_nodes(*, cluster_snapshot: dict, detect_snapshot
     }
 
     merged: list[dict] = []
-    seen: set[str] = set()
-    for item in list(active_job.get("node_stats") or []):
-        node_code = str(item.get("node_code") or "").strip()
-        if not node_code or node_code == "unassigned":
-            continue
-        seen.add(node_code)
+    job_map = {
+        str(item.get("node_code") or "").strip(): item
+        for item in list(active_job.get("node_stats") or [])
+        if str(item.get("node_code") or "").strip() and str(item.get("node_code") or "").strip() != "unassigned"
+    }
+    candidate_codes = sorted(set(job_map) | set(queue_map) | set(cluster_map))
+    for node_code in candidate_codes:
         cluster_node = cluster_map.get(node_code, {})
-        queue_node = queue_map.get(node_code, {})
-        merged.append(
-            {
-                "node_code": node_code,
-                "role": str(cluster_node.get("role") or "worker"),
-                "region": str(cluster_node.get("region") or settings.node_region),
-                "status": str(cluster_node.get("status") or "unknown"),
-                "is_effective_worker": bool(cluster_node.get("is_effective_worker", False) or str(cluster_node.get("role") or "") == "worker"),
-                "detect_participating": True,
-                "current_load": int(cluster_node.get("current_load", 0) or 0),
-                "items_total": int(item.get("items_total", 0) or 0),
-                "items_pending": int(item.get("items_pending", 0) or 0),
-                "items_claimed": int(item.get("items_claimed", 0) or 0),
-                "items_running": int(item.get("items_running", 0) or 0),
-                "items_completed": int(item.get("items_completed", 0) or 0),
-                "items_failed": int(item.get("items_failed", 0) or 0),
-                "processed_recent": int(queue_node.get("processed_recent", 0) or 0),
-                "processed_per_minute": float(queue_node.get("processed_per_minute", 0) or 0),
-            }
+        if cluster_node and not bool(cluster_node.get("is_effective_worker", False)):
+            continue
+        row = _build_detect_node_row(
+            node_code=node_code,
+            cluster_node=cluster_node,
+            job_node=job_map.get(node_code, {}),
+            queue_node=queue_map.get(node_code, {}),
         )
-
-    if worker_runtime.get("running", False) and settings.node_code not in seen:
-        cluster_node = cluster_map.get(settings.node_code, {})
-        if cluster_node:
-            merged.append(
-                {
-                    "node_code": settings.node_code,
-                    "role": str(cluster_node.get("role") or settings.node_role),
-                    "region": str(cluster_node.get("region") or settings.node_region),
-                    "status": str(cluster_node.get("status") or "online"),
-                    "is_effective_worker": True,
-                    "detect_participating": True,
-                    "current_load": int(cluster_node.get("current_load", 0) or 0),
-                    "items_total": 0,
-                    "items_pending": 0,
-                    "items_claimed": 0,
-                    "items_running": 0,
-                    "items_completed": 0,
-                    "items_failed": 0,
-                    "processed_recent": 0,
-                    "processed_per_minute": 0,
-                }
-            )
+        if not bool(row.get("is_current_participant", False)):
+            continue
+        merged.append(row)
 
     return sorted(
         merged,
@@ -261,11 +314,143 @@ def _build_participating_detect_nodes(*, cluster_snapshot: dict, detect_snapshot
             -int(item.get("items_running", 0) or 0),
             -int(item.get("items_claimed", 0) or 0),
             -int(item.get("processed_recent", 0) or 0),
+            -int(item.get("current_load", 0) or 0),
             str(item.get("node_code") or ""),
         ),
     )
 
 
+def _build_standby_detect_nodes(*, cluster_snapshot: dict, participating_nodes: list[dict]) -> list[dict]:
+    cluster_nodes = list(cluster_snapshot.get("nodes") or [])
+    participating_codes = {
+        str(item.get("node_code") or "").strip()
+        for item in list(participating_nodes or [])
+        if str(item.get("node_code") or "").strip()
+    }
+    standby_rows: list[dict] = []
+    for node in cluster_nodes:
+        node_code = str(node.get("node_code") or "").strip()
+        node_status = str(node.get("status") or "").strip()
+        node_current_load = int(node.get("current_load", 0) or 0)
+        metadata = node.get("metadata") or {}
+        if not node_code:
+            continue
+        if not bool(node.get("is_effective_worker", False)):
+            continue
+        if node_status not in {"online", "busy"}:
+            continue
+        if node_code in participating_codes:
+            continue
+        standby_state = "load_syncing" if node_current_load > 0 or node_status == "busy" else "standby"
+        standby_label = "负载待确认" if standby_state == "load_syncing" else "在线待命"
+        standby_reason = (
+            str(metadata.get("detail") or "").strip()
+            or str(metadata.get("phase_detail") or "").strip()
+            or (
+                f"当前阶段：{str(metadata.get('phase') or metadata.get('phase_label') or '').strip()}"
+                if str(metadata.get("phase") or metadata.get("phase_label") or "").strip()
+                else ""
+            )
+            or (
+                f"节点当前负载为 {node_current_load}，但还未观察到已领、执行中或近窗吞吐数据。"
+                if standby_state == "load_syncing"
+                else "节点在线，当前未领任务、未执行任务，也没有近窗吞吐。"
+            )
+        )
+        standby_rows.append(
+            {
+                "node_code": node_code,
+                "role": str(node.get("role") or "worker"),
+                "region": str(node.get("region") or settings.node_region),
+                "status": node_status,
+                "is_effective_worker": True,
+                "detect_participating": False,
+                "participation_state": standby_state,
+                "participation_label": standby_label,
+                "participation_reason": standby_reason,
+                "current_load": node_current_load,
+                "last_heartbeat_at": str(node.get("last_heartbeat_at") or ""),
+                "phase": str(metadata.get("phase") or metadata.get("phase_label") or "").strip(),
+                "detail": str(metadata.get("detail") or metadata.get("phase_detail") or "").strip(),
+                "worker_online": bool(metadata.get("worker_online", False)),
+                "standby_reason": standby_reason,
+            }
+        )
+    return sorted(
+        standby_rows,
+        key=lambda item: (
+            str(item.get("status") or ""),
+            str(item.get("role") or ""),
+            str(item.get("node_code") or ""),
+        ),
+    )
+
+
+def _build_detect_participation_summary(
+    *,
+    participating_nodes: list[dict],
+    standby_nodes: list[dict],
+    cluster_snapshot: dict,
+) -> dict:
+    cluster_nodes = list(cluster_snapshot.get("nodes") or [])
+    effective_online_nodes = [
+        node
+        for node in cluster_nodes
+        if bool(node.get("is_effective_worker", False)) and str(node.get("status") or "").strip() in {"online", "busy"}
+    ]
+    dispatch_active_nodes = [
+        row for row in participating_nodes
+        if bool(row.get("is_dispatch_active", False))
+    ]
+    recent_only_nodes = [
+        row for row in participating_nodes
+        if str(row.get("participation_state") or "").strip() == "recent_throughput"
+    ]
+    load_syncing_nodes = [
+        row for row in standby_nodes
+        if str(row.get("participation_state") or "").strip() == "load_syncing"
+    ]
+    pure_standby_nodes = [
+        row for row in standby_nodes
+        if str(row.get("participation_state") or "").strip() == "standby"
+    ]
+    dedicated_worker_nodes = [
+        node for node in effective_online_nodes
+        if str(node.get("role") or "").strip() == "worker"
+    ]
+    controller_worker_nodes = [
+        node for node in effective_online_nodes
+        if str(node.get("role") or "").strip() == "control"
+    ]
+
+    summary_parts = [
+        f"有效执行节点 {len(effective_online_nodes)} 台",
+        f"正在执行/领任务 {len(dispatch_active_nodes)} 台",
+        f"近窗刚有吞吐 {len(recent_only_nodes)} 台",
+        f"在线但未参与 {len(standby_nodes)} 台",
+    ]
+    if load_syncing_nodes:
+        summary_parts.append(f"其中负载待确认 {len(load_syncing_nodes)} 台")
+
+    return {
+        "effective_online_nodes": len(effective_online_nodes),
+        "participating_nodes": len(participating_nodes),
+        "dispatch_active_nodes": len(dispatch_active_nodes),
+        "recent_only_nodes": len(recent_only_nodes),
+        "non_participating_nodes": len(standby_nodes),
+        "standby_nodes": len(pure_standby_nodes),
+        "load_syncing_nodes": len(load_syncing_nodes),
+        "dedicated_worker_nodes": len(dedicated_worker_nodes),
+        "controller_worker_nodes": len(controller_worker_nodes),
+        "dispatch_active_node_codes": [str(item.get("node_code") or "") for item in dispatch_active_nodes],
+        "recent_only_node_codes": [str(item.get("node_code") or "") for item in recent_only_nodes],
+        "non_participating_node_codes": [str(item.get("node_code") or "") for item in standby_nodes],
+        "standby_node_codes": [str(item.get("node_code") or "") for item in pure_standby_nodes],
+        "load_syncing_node_codes": [str(item.get("node_code") or "") for item in load_syncing_nodes],
+        "summary": "；".join(summary_parts),
+    }
+
+
 def get_runtime_status() -> dict:
     runtime_settings = get_runtime_settings()
     worker_runtime = detect_worker_runtime()
@@ -314,6 +499,17 @@ def get_runtime_status() -> dict:
         "worker_mode": worker_runtime.get("mode", runtime_settings.get("worker_mode", "windows-local")),
         "queue_health": queue_health,
         "capacity_plan": capacity_plan,
+        "log_sync": {
+            "enabled": bool(runtime_settings.get("worker_log_sync_enabled", False)),
+            "mode": str(runtime_settings.get("worker_log_sync_mode", "key") or "key"),
+            "line_count": int(detect_snapshot.get("remote_log_line_count", 0) or 0),
+            "source_node_count": int(detect_snapshot.get("remote_log_node_count", 0) or 0),
+            "source_nodes": list(detect_snapshot.get("remote_log_nodes") or []),
+            "source_node_summaries": list(detect_snapshot.get("remote_log_node_summaries") or []),
+            "last_at": str(detect_snapshot.get("remote_log_last_at") or ""),
+            "last_line": str(detect_snapshot.get("remote_log_last_line") or ""),
+            "preview_lines": list(detect_snapshot.get("remote_log_lines") or [])[-20:],
+        },
     }
     if not worker_expected_on_this_node:
         detect_payload.update(
@@ -349,6 +545,16 @@ def get_runtime_status() -> dict:
         detect_snapshot=detect_payload,
         worker_runtime=worker_runtime,
     )
+    detect_payload["standby_nodes"] = _build_standby_detect_nodes(
+        cluster_snapshot=cluster_snapshot,
+        participating_nodes=detect_payload["participating_nodes"],
+    )
+    detect_payload["non_participating_nodes"] = list(detect_payload["standby_nodes"] or [])
+    detect_payload["participation_summary"] = _build_detect_participation_summary(
+        participating_nodes=detect_payload["participating_nodes"],
+        standby_nodes=detect_payload["non_participating_nodes"],
+        cluster_snapshot=cluster_snapshot,
+    )
     append_runtime_projection_if_changed(detect=detect_payload, cluster=cluster_snapshot)
     sync_summary = get_sync_summary(record_limit=5)
     readiness = _build_multi_region_readiness(
@@ -357,6 +563,7 @@ def get_runtime_status() -> dict:
         worker_runtime=worker_runtime,
         sync_agent_runtime=sync_agent_runtime,
     )
+    build_info = get_runtime_build_info()
 
     return {
         "api": {
@@ -371,6 +578,7 @@ def get_runtime_status() -> dict:
             "health_url": f"http://127.0.0.1:{settings.api_port}/health",
             "stdout_log": _runtime_log_path("domain-api.stdout.log"),
             "stderr_log": _runtime_log_path("domain-api.stderr.log"),
+            "build": build_info,
         },
         "node": {
             "code": settings.node_code,

domain-api/app/services/sync_record_service.py

@@ -450,6 +450,17 @@ def append_runtime_projection_if_changed(
             continue
         local_participating = bool(node.get("detect_participating", False) or node.get("current_load", 0))
         break
+    local_job_bucket = {}
+    for item in list(active_job.get("node_stats") or []):
+        if str(item.get("node_code") or "").strip() != settings.node_code:
+            continue
+        local_job_bucket = item
+        break
+    if not local_participating:
+        local_participating = bool(
+            int(local_job_bucket.get("items_running", 0) or 0) > 0
+            or int(local_job_bucket.get("items_claimed", 0) or 0) > 0
+        )
     projection = {
         "node": {
             "node_code": settings.node_code,