feat: add ops center and node onboarding flow
This commit is contained in:
Your Name
420
domain-api/app/services/ops_runtime_executor_service.py
Normal file
420
domain-api/app/services/ops_runtime_executor_service.py
Normal file
@@ -0,0 +1,420 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import subprocess
|
||||
|
||||
from app.core.config import settings
|
||||
from app.services.ops_action_executor_core import (
|
||||
STRUCTURED_ACTIONS,
|
||||
build_service_name_map,
|
||||
execute_structured_action,
|
||||
trim_output,
|
||||
)
|
||||
from app.services.ops_release_executor_core import build_remote_release_action_script, execute_release_action
|
||||
|
||||
|
||||
_SSH_SUPPORTED_ACTIONS = set(STRUCTURED_ACTIONS) | {"deploy.release"}
|
||||
_SSH_CONNECT_TIMEOUT_SECONDS = 12
|
||||
_SSH_REMOTE_TIMEOUT_SECONDS = {
|
||||
"health.snapshot": 30,
|
||||
"service.status": 45,
|
||||
"service.restart": 45,
|
||||
"service.start": 45,
|
||||
"service.stop": 45,
|
||||
"logs.collect": 45,
|
||||
"diagnostics.collect": 90,
|
||||
"runtime.start_worker": 45,
|
||||
"runtime.stop_worker": 45,
|
||||
"runtime.restart_api": 45,
|
||||
"runtime.start_sync_agent": 45,
|
||||
"runtime.stop_sync_agent": 45,
|
||||
"deploy.release": 180,
|
||||
}
|
||||
|
||||
|
||||
def supports_ssh_execution(action: str) -> bool:
|
||||
return str(action or "").strip() in _SSH_SUPPORTED_ACTIONS
|
||||
|
||||
|
||||
def execute_local_support_action(action: str, payload: dict | None = None) -> tuple[bool, str, dict]:
|
||||
normalized_action = str(action or "").strip()
|
||||
if normalized_action not in _SSH_SUPPORTED_ACTIONS:
|
||||
return False, f"当前未实现本机支持动作: {normalized_action}", {}
|
||||
if normalized_action == "deploy.release":
|
||||
return execute_release_action(
|
||||
dict(payload or {}),
|
||||
run_command=_run_local_command,
|
||||
default_api_service_name=settings.api_service_name,
|
||||
user_agent="domaincheck-local-runtime/0.1",
|
||||
)
|
||||
return execute_structured_action(
|
||||
normalized_action,
|
||||
dict(payload or {}),
|
||||
service_names=_service_name_map(),
|
||||
runner=_run_local_command,
|
||||
)
|
||||
|
||||
|
||||
def execute_ssh_action(node: dict, action: str, payload: dict | None = None) -> tuple[bool, str, dict]:
|
||||
normalized_action = str(action or "").strip()
|
||||
if normalized_action not in _SSH_SUPPORTED_ACTIONS:
|
||||
return False, f"当前 SSH 执行器暂不支持动作: {normalized_action}", {}
|
||||
|
||||
ssh_host = str(node.get("ssh_host") or "").strip()
|
||||
ssh_user = str(node.get("ssh_user") or "").strip()
|
||||
ssh_port = max(1, int(node.get("ssh_port") or 22))
|
||||
node_code = str(node.get("node_code") or "").strip()
|
||||
if not ssh_host or not ssh_user:
|
||||
return False, "目标节点缺少 SSH 主机或用户信息", {
|
||||
"node_code": node_code,
|
||||
"ssh_host": ssh_host,
|
||||
"ssh_user": ssh_user,
|
||||
"ssh_port": ssh_port,
|
||||
}
|
||||
|
||||
normalized_payload = dict(payload or {})
|
||||
service_names = _service_name_map()
|
||||
if normalized_action == "deploy.release":
|
||||
remote_script = build_remote_release_action_script(
|
||||
normalized_payload,
|
||||
default_api_service_name=service_names.get("api") or "domaincheck-api",
|
||||
user_agent="domaincheck-ssh/0.1",
|
||||
)
|
||||
else:
|
||||
remote_script = _build_remote_python_script(
|
||||
action=normalized_action,
|
||||
payload=normalized_payload,
|
||||
service_names=service_names,
|
||||
)
|
||||
remote_command = "\n".join(
|
||||
[
|
||||
"set -euo pipefail",
|
||||
"if command -v python3 >/dev/null 2>&1; then",
|
||||
" PYTHON_BIN=python3",
|
||||
"elif command -v python >/dev/null 2>&1; then",
|
||||
" PYTHON_BIN=python",
|
||||
"else",
|
||||
' echo "{\\"ok\\": false, \\"message\\": \\"python interpreter missing on remote node\\", \\"result\\": {\\"executor\\": \\"ssh\\"}}"',
|
||||
" exit 127",
|
||||
"fi",
|
||||
'"${PYTHON_BIN}" - <<\'PY\'',
|
||||
remote_script.rstrip("\n"),
|
||||
"PY",
|
||||
]
|
||||
)
|
||||
ssh_command = [
|
||||
"ssh",
|
||||
"-o",
|
||||
"BatchMode=yes",
|
||||
"-o",
|
||||
"PreferredAuthentications=publickey",
|
||||
"-o",
|
||||
"StrictHostKeyChecking=accept-new",
|
||||
"-o",
|
||||
f"ConnectTimeout={_SSH_CONNECT_TIMEOUT_SECONDS}",
|
||||
"-p",
|
||||
str(ssh_port),
|
||||
f"{ssh_user}@{ssh_host}",
|
||||
remote_command,
|
||||
]
|
||||
timeout_seconds = int(_SSH_REMOTE_TIMEOUT_SECONDS.get(normalized_action, 45) or 45) + _SSH_CONNECT_TIMEOUT_SECONDS
|
||||
completed = subprocess.run(
|
||||
ssh_command,
|
||||
capture_output=True,
|
||||
text=True,
|
||||
timeout=timeout_seconds,
|
||||
)
|
||||
|
||||
transport = {
|
||||
"executor": "ssh",
|
||||
"node_code": node_code,
|
||||
"ssh_host": ssh_host,
|
||||
"ssh_user": ssh_user,
|
||||
"ssh_port": ssh_port,
|
||||
"action": normalized_action,
|
||||
"returncode": int(completed.returncode or 0),
|
||||
}
|
||||
parsed_ok, parsed_payload = _extract_json_from_output(completed.stdout or "")
|
||||
if parsed_ok:
|
||||
ok = bool(parsed_payload.get("ok", False))
|
||||
message = str(parsed_payload.get("message") or "").strip() or ("SSH 动作执行成功" if ok else "SSH 动作执行失败")
|
||||
result = dict(parsed_payload.get("result") or {})
|
||||
result["transport"] = {
|
||||
**transport,
|
||||
"stdout_preview": trim_output(completed.stdout, 4000),
|
||||
"stderr_preview": trim_output(completed.stderr, 2000),
|
||||
}
|
||||
if completed.returncode != 0 and ok:
|
||||
ok = False
|
||||
message = f"SSH 命令返回码异常: {completed.returncode}"
|
||||
return ok, message, result
|
||||
|
||||
stderr_preview = trim_output(completed.stderr, 4000)
|
||||
stdout_preview = trim_output(completed.stdout, 4000)
|
||||
message = stderr_preview or stdout_preview or f"SSH 执行失败，返回码 {completed.returncode}"
|
||||
return False, message, {
|
||||
"executor": "ssh",
|
||||
"action": normalized_action,
|
||||
"transport": {
|
||||
**transport,
|
||||
"stdout_preview": stdout_preview,
|
||||
"stderr_preview": stderr_preview,
|
||||
},
|
||||
}
|
||||
|
||||
|
||||
def _service_name_map() -> dict[str, str]:
|
||||
return build_service_name_map(
|
||||
api_service_name=settings.api_service_name,
|
||||
worker_service_name=settings.worker_service_name,
|
||||
sync_agent_service_name=settings.sync_agent_service_name,
|
||||
)
|
||||
|
||||
|
||||
def _run_local_command(command: list[str], *, timeout: int = 60) -> tuple[int, str, str]:
|
||||
completed = subprocess.run(command, capture_output=True, text=True, timeout=timeout)
|
||||
return int(completed.returncode or 0), str(completed.stdout or "").strip(), str(completed.stderr or "").strip()
|
||||
|
||||
|
||||
def _build_remote_python_script(*, action: str, payload: dict, service_names: dict[str, str]) -> str:
|
||||
return f"""import json
|
||||
import subprocess
|
||||
import sys
|
||||
from pathlib import Path
|
||||
|
||||
SERVICE_NAMES = {json.dumps(service_names, ensure_ascii=False)}
|
||||
ACTION = {json.dumps(str(action or '').strip(), ensure_ascii=False)}
|
||||
PAYLOAD = {json.dumps(dict(payload or {}), ensure_ascii=False)}
|
||||
|
||||
|
||||
def trim_output(text, limit):
|
||||
value = str(text or "").strip()
|
||||
if len(value) <= int(limit):
|
||||
return value
|
||||
return value[-int(limit):]
|
||||
|
||||
|
||||
def run(cmd, timeout=60):
|
||||
completed = subprocess.run(cmd, capture_output=True, text=True, timeout=timeout)
|
||||
return int(completed.returncode or 0), str(completed.stdout or "").strip(), str(completed.stderr or "").strip()
|
||||
|
||||
|
||||
def read_tail_lines(path, max_lines):
|
||||
file_path = Path(path)
|
||||
if not file_path.exists() or not file_path.is_file():
|
||||
return []
|
||||
with file_path.open("r", encoding="utf-8", errors="replace") as handle:
|
||||
return handle.read().splitlines()[-int(max_lines):]
|
||||
|
||||
|
||||
def candidate_domain_roots():
|
||||
roots = []
|
||||
for candidate in (
|
||||
Path.cwd() / "domainCheck",
|
||||
Path("/opt/domaincheck/domainCheck"),
|
||||
Path("/www/wwwroot/getDomain/domainCheck"),
|
||||
):
|
||||
if candidate not in roots:
|
||||
roots.append(candidate)
|
||||
return roots
|
||||
|
||||
|
||||
def candidate_runtime_logs_dirs():
|
||||
dirs = []
|
||||
for candidate in (
|
||||
Path.cwd() / "domain-api" / "runtime" / "logs",
|
||||
Path("/opt/domaincheck/domain-api/runtime/logs"),
|
||||
Path("/www/wwwroot/getDomain/domain-api/runtime/logs"),
|
||||
):
|
||||
if candidate not in dirs:
|
||||
dirs.append(candidate)
|
||||
return dirs
|
||||
|
||||
|
||||
def service_log_file_candidates(service_name):
|
||||
normalized_service_name = str(service_name or "").strip()
|
||||
if not normalized_service_name:
|
||||
return []
|
||||
candidates = []
|
||||
|
||||
def append_candidate(path):
|
||||
if path not in candidates:
|
||||
candidates.append(path)
|
||||
|
||||
if normalized_service_name == (SERVICE_NAMES.get("worker") or "domaincheck-worker"):
|
||||
for domain_root in candidate_domain_roots():
|
||||
append_candidate(domain_root / "detect_worker.log")
|
||||
append_candidate(domain_root / "logs" / "detect_worker.log")
|
||||
elif normalized_service_name == (SERVICE_NAMES.get("api") or "domaincheck-api"):
|
||||
for runtime_logs_dir in candidate_runtime_logs_dirs():
|
||||
append_candidate(runtime_logs_dir / "domain-api.stderr.log")
|
||||
append_candidate(runtime_logs_dir / "domain-api.stdout.log")
|
||||
for domain_root in candidate_domain_roots():
|
||||
append_candidate(domain_root / "logs" / "app.log")
|
||||
|
||||
return candidates
|
||||
|
||||
|
||||
def collect_log_file_fallback(service_name, lines):
|
||||
aggregated_lines = []
|
||||
used_paths = []
|
||||
for path in service_log_file_candidates(service_name):
|
||||
current_lines = read_tail_lines(path, lines)
|
||||
if not current_lines:
|
||||
continue
|
||||
aggregated_lines.extend(current_lines)
|
||||
used_paths.append(str(path))
|
||||
if not aggregated_lines:
|
||||
return "", []
|
||||
return "\\n".join(aggregated_lines[-int(lines):]), used_paths
|
||||
|
||||
|
||||
def service_name_from_payload(payload):
|
||||
value = str((payload or {{}}).get("service_name") or "").strip()
|
||||
return value or SERVICE_NAMES.get("worker") or "domaincheck-worker"
|
||||
|
||||
|
||||
def service_name_for_action(action, payload):
|
||||
if action in ("service.status", "service.restart", "service.start", "service.stop"):
|
||||
return service_name_from_payload(payload)
|
||||
runtime_action_map = {{
|
||||
"runtime.start_worker": SERVICE_NAMES.get("worker") or "domaincheck-worker",
|
||||
"runtime.stop_worker": SERVICE_NAMES.get("worker") or "domaincheck-worker",
|
||||
"runtime.restart_api": SERVICE_NAMES.get("api") or "domaincheck-api",
|
||||
"runtime.start_sync_agent": SERVICE_NAMES.get("sync_agent") or "domaincheck-sync-agent",
|
||||
"runtime.stop_sync_agent": SERVICE_NAMES.get("sync_agent") or "domaincheck-sync-agent",
|
||||
}}
|
||||
return str(runtime_action_map.get(action) or "").strip()
|
||||
|
||||
|
||||
def systemctl_action_name(action):
|
||||
if action in ("service.restart", "runtime.restart_api"):
|
||||
return "restart"
|
||||
if action in ("service.start", "runtime.start_worker", "runtime.start_sync_agent"):
|
||||
return "start"
|
||||
if action in ("service.stop", "runtime.stop_worker", "runtime.stop_sync_agent"):
|
||||
return "stop"
|
||||
return ""
|
||||
|
||||
|
||||
def execute(action, payload):
|
||||
if action == "health.snapshot":
|
||||
checks = {{}}
|
||||
for key, service_name in SERVICE_NAMES.items():
|
||||
code, stdout, stderr = run(["systemctl", "is-active", service_name], timeout=15)
|
||||
checks[key] = {{
|
||||
"service_name": service_name,
|
||||
"returncode": int(code or 0),
|
||||
"state": stdout or stderr,
|
||||
"ok": int(code or 0) == 0 and (stdout or stderr) == "active",
|
||||
}}
|
||||
return True, "health snapshot collected", {{"checks": checks}}
|
||||
|
||||
if action == "service.status":
|
||||
service_name = service_name_from_payload(payload)
|
||||
code, stdout, stderr = run(["systemctl", "status", service_name, "--no-pager", "-l"], timeout=45)
|
||||
result = {{
|
||||
"service_name": service_name,
|
||||
"stdout": trim_output(stdout, 12000),
|
||||
"stderr": trim_output(stderr, 4000),
|
||||
}}
|
||||
return int(code or 0) == 0, stdout or stderr or f"{{service_name}} status collected", result
|
||||
|
||||
systemctl_action = systemctl_action_name(action)
|
||||
if systemctl_action:
|
||||
service_name = service_name_for_action(action, payload)
|
||||
if not service_name:
|
||||
return False, f"missing service name for action: {{action}}", {{"action": action}}
|
||||
code, stdout, stderr = run(["systemctl", systemctl_action, service_name], timeout=45)
|
||||
result = {{
|
||||
"service_name": service_name,
|
||||
"systemctl_action": systemctl_action,
|
||||
"stdout": trim_output(stdout, 12000),
|
||||
"stderr": trim_output(stderr, 4000),
|
||||
}}
|
||||
return int(code or 0) == 0, stdout or stderr or f"{{service_name}} {{systemctl_action}} completed", result
|
||||
|
||||
if action == "logs.collect":
|
||||
service_name = service_name_from_payload(payload)
|
||||
lines = max(20, min(int((payload or {{}}).get("lines") or 120), 500))
|
||||
code, stdout, stderr = run(["journalctl", "-u", service_name, "-n", str(lines), "--no-pager"], timeout=45)
|
||||
journal_output = stdout or stderr
|
||||
fallback_output, fallback_paths = collect_log_file_fallback(service_name, lines)
|
||||
collection_source = "journal"
|
||||
effective_output = journal_output
|
||||
ok = int(code or 0) == 0
|
||||
if (not ok or not stdout.strip()) and fallback_output:
|
||||
collection_source = "file_fallback"
|
||||
effective_output = fallback_output
|
||||
ok = True
|
||||
result = {{
|
||||
"service_name": service_name,
|
||||
"lines": lines,
|
||||
"collection_source": collection_source,
|
||||
"fallback_used": bool(collection_source == "file_fallback"),
|
||||
"fallback_reason": trim_output(journal_output, 4000) if collection_source == "file_fallback" else "",
|
||||
"log_paths": fallback_paths,
|
||||
"journal_returncode": int(code or 0),
|
||||
"stdout": trim_output(effective_output, 20000),
|
||||
"stderr": trim_output(stderr, 4000),
|
||||
}}
|
||||
return ok, effective_output or f"{{service_name}} logs collected", result
|
||||
|
||||
if action == "diagnostics.collect":
|
||||
lines = max(20, min(int((payload or {{}}).get("lines") or 200), 800))
|
||||
diagnostics = {{
|
||||
"services": {{}},
|
||||
"lines": lines,
|
||||
}}
|
||||
for key, service_name in SERVICE_NAMES.items():
|
||||
active_code, active_stdout, active_stderr = run(["systemctl", "is-active", service_name], timeout=15)
|
||||
status_code, status_stdout, status_stderr = run(["systemctl", "status", service_name, "--no-pager", "-l"], timeout=45)
|
||||
log_code, log_stdout, log_stderr = run(["journalctl", "-u", service_name, "-n", str(lines), "--no-pager"], timeout=45)
|
||||
log_output = log_stdout or log_stderr
|
||||
fallback_output, fallback_paths = collect_log_file_fallback(service_name, lines)
|
||||
log_source = "journal"
|
||||
if (int(log_code or 0) != 0 or not log_stdout.strip()) and fallback_output:
|
||||
log_output = fallback_output
|
||||
log_source = "file_fallback"
|
||||
diagnostics["services"][key] = {{
|
||||
"service_name": service_name,
|
||||
"active_returncode": int(active_code or 0),
|
||||
"active_state": active_stdout or active_stderr,
|
||||
"status_returncode": int(status_code or 0),
|
||||
"status_output": trim_output(status_stdout or status_stderr, 12000),
|
||||
"log_returncode": int(log_code or 0),
|
||||
"log_source": log_source,
|
||||
"log_paths": fallback_paths,
|
||||
"log_output": trim_output(log_output, 20000),
|
||||
}}
|
||||
return True, "diagnostics collected", diagnostics
|
||||
|
||||
return False, f"unsupported action: {{action}}", {{"action": action}}
|
||||
|
||||
|
||||
def main():
|
||||
try:
|
||||
ok, message, result = execute(ACTION, PAYLOAD)
|
||||
except Exception as exc:
|
||||
ok, message, result = False, str(exc), {{"exception": str(exc), "action": ACTION}}
|
||||
print(json.dumps({{"ok": ok, "message": message, "result": result}}, ensure_ascii=False))
|
||||
raise SystemExit(0 if ok else 1)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
"""
|
||||
|
||||
|
||||
def _extract_json_from_output(text: str) -> tuple[bool, dict]:
|
||||
for raw_line in reversed(str(text or "").splitlines()):
|
||||
line = str(raw_line or "").strip()
|
||||
if not line:
|
||||
continue
|
||||
try:
|
||||
data = json.loads(line)
|
||||
except Exception:
|
||||
continue
|
||||
if isinstance(data, dict) and "ok" in data:
|
||||
return True, data
|
||||
return False, {}
|
||||
Reference in New Issue
Block a user