195 lines
9.4 KiB
Python
195 lines
9.4 KiB
Python
from __future__ import annotations
|
||
|
||
import os
|
||
import subprocess
|
||
from pathlib import Path
|
||
|
||
from app.core.config import settings
|
||
from app.services.debug_event_service import push_debug_event
|
||
from app.services.sync_push_service import pull_detect_task_batch_now, push_runtime_projection_now
|
||
from app.services.runtime_settings_service import get_runtime_settings
|
||
from app.services.worker_control_service import _run_systemctl, normalize_systemctl_error, start_worker, stop_worker
|
||
|
||
|
||
def _workspace_root() -> Path:
|
||
return Path(settings.domain_root).parent
|
||
|
||
|
||
def _run_shell(command: list[str], timeout: int = 30) -> subprocess.CompletedProcess[str]:
|
||
return subprocess.run(command, capture_output=True, text=True, timeout=timeout)
|
||
|
||
|
||
def restart_api() -> tuple[bool, str]:
|
||
runtime = get_runtime_settings()
|
||
api_service_name = runtime.get("api_service_name", settings.api_service_name)
|
||
worker_mode = runtime.get("worker_mode", settings.worker_mode)
|
||
|
||
if worker_mode == "linux-systemd":
|
||
# When the API restarts itself under systemd, wait-free restart avoids
|
||
# blocking the HTTP request until uvicorn is torn down.
|
||
result = _run_systemctl(["restart", api_service_name], timeout=5, no_block=True)
|
||
if result.returncode != 0:
|
||
return False, normalize_systemctl_error(result.stderr or result.stdout or "重启 Linux API 失败", service_name=api_service_name)
|
||
return True, f"Linux API 重启命令已发送: {api_service_name}"
|
||
|
||
if os.name != "nt":
|
||
return False, "当前仅实现 Windows 本地 API 重启，Linux 请将 worker_mode 设为 linux-systemd。"
|
||
|
||
workspace = _workspace_root()
|
||
stop_script = workspace / "stop_domain_api.ps1"
|
||
start_script = workspace / "start_domain_api.ps1"
|
||
if not stop_script.exists() or not start_script.exists():
|
||
return False, "未找到 API 启停脚本"
|
||
|
||
command = (
|
||
"Start-Process powershell "
|
||
"-ArgumentList '-NoProfile', '-ExecutionPolicy', 'Bypass', '-Command', "
|
||
f"\"Start-Sleep -Seconds 2; & '{stop_script}'; Start-Sleep -Seconds 1; & '{start_script}'\""
|
||
)
|
||
result = _run_shell(["powershell", "-NoProfile", "-ExecutionPolicy", "Bypass", "-Command", command], timeout=20)
|
||
if result.returncode != 0:
|
||
return False, (result.stderr or result.stdout or "重启 API 失败").strip()
|
||
return True, "API 重启命令已发送"
|
||
|
||
|
||
def _run_systemd_action(service_name: str, action: str, *, no_block: bool = False) -> tuple[bool, str]:
|
||
systemctl_command = []
|
||
if no_block:
|
||
systemctl_command.append("--no-block")
|
||
systemctl_command.extend([action, service_name])
|
||
result = _run_systemctl(systemctl_command, timeout=10 if no_block else 30)
|
||
if result.returncode != 0:
|
||
return False, normalize_systemctl_error(result.stderr or result.stdout or f"{action} {service_name} 失败", service_name=service_name)
|
||
return True, f"{service_name} {action} 命令已发送"
|
||
|
||
|
||
def _emit_runtime_action_event(action: str, *, stage: str, ok: bool, message: str, data: dict | None = None) -> None:
|
||
try:
|
||
push_debug_event(
|
||
service="runtime-control",
|
||
event_type=f"runtime_action_{stage}",
|
||
level="info" if ok else "warning",
|
||
message=message,
|
||
payload={
|
||
"action": action,
|
||
"ok": ok,
|
||
**(data or {}),
|
||
},
|
||
)
|
||
except Exception:
|
||
pass
|
||
|
||
|
||
def _build_runtime_action_result(
|
||
*,
|
||
action: str,
|
||
poll_after_seconds: int,
|
||
refresh_runtime: bool,
|
||
ok: bool,
|
||
message: str,
|
||
data: dict | None = None,
|
||
) -> dict:
|
||
result = {
|
||
"action": action,
|
||
"poll_after_seconds": poll_after_seconds,
|
||
"refresh_runtime": refresh_runtime,
|
||
**(data or {}),
|
||
}
|
||
lowered = str(message or "").strip().lower()
|
||
if "ui_level" not in result:
|
||
if not ok:
|
||
result["ui_level"] = "warning" if any(keyword in lowered for keyword in ("当前没有", "无需", "未启用", "未配置")) else "error"
|
||
elif any(keyword in str(message or "") for keyword in ("命令已发送", "确认失败")):
|
||
result["ui_level"] = "warning"
|
||
else:
|
||
result["ui_level"] = "success"
|
||
|
||
if "poll_schedule_seconds" not in result:
|
||
if action in {"start_worker", "stop_worker", "restart_api", "start_sync_agent", "stop_sync_agent"}:
|
||
result["poll_schedule_seconds"] = [1, max(2, poll_after_seconds)]
|
||
elif action in {"push_sync", "pull_tasks"}:
|
||
result["poll_schedule_seconds"] = [1, max(2, poll_after_seconds)]
|
||
elif poll_after_seconds > 0:
|
||
result["poll_schedule_seconds"] = [poll_after_seconds]
|
||
else:
|
||
result["poll_schedule_seconds"] = []
|
||
return result
|
||
|
||
|
||
def start_sync_agent() -> tuple[bool, str]:
|
||
runtime = get_runtime_settings()
|
||
worker_mode = runtime.get("worker_mode", settings.worker_mode)
|
||
service_name = runtime.get("sync_agent_service_name", settings.sync_agent_service_name)
|
||
if worker_mode != "linux-systemd":
|
||
return False, "sync-agent 仅在 Linux systemd 多机部署中使用。"
|
||
return _run_systemd_action(service_name, "start")
|
||
|
||
|
||
def stop_sync_agent() -> tuple[bool, str]:
|
||
runtime = get_runtime_settings()
|
||
worker_mode = runtime.get("worker_mode", settings.worker_mode)
|
||
service_name = runtime.get("sync_agent_service_name", settings.sync_agent_service_name)
|
||
if worker_mode != "linux-systemd":
|
||
return False, "sync-agent 仅在 Linux systemd 多机部署中使用。"
|
||
return _run_systemd_action(service_name, "stop")
|
||
|
||
|
||
def runtime_action(action: str) -> tuple[bool, str, dict]:
|
||
normalized_action = str(action or "").strip().lower().replace("-", "_")
|
||
if normalized_action != "push_debug_probe":
|
||
_emit_runtime_action_event(
|
||
normalized_action,
|
||
stage="requested",
|
||
ok=True,
|
||
message=f"收到运行时动作请求: {normalized_action}",
|
||
)
|
||
if normalized_action == "start_worker":
|
||
ok, message = start_worker()
|
||
result = _build_runtime_action_result(action=normalized_action, poll_after_seconds=2, refresh_runtime=True, ok=ok, message=message)
|
||
_emit_runtime_action_event(normalized_action, stage="finished", ok=ok, message=message, data=result)
|
||
return ok, message, result
|
||
if normalized_action == "stop_worker":
|
||
ok, message = stop_worker()
|
||
result = _build_runtime_action_result(action=normalized_action, poll_after_seconds=2, refresh_runtime=True, ok=ok, message=message)
|
||
_emit_runtime_action_event(normalized_action, stage="finished", ok=ok, message=message, data=result)
|
||
return ok, message, result
|
||
if normalized_action == "restart_api":
|
||
ok, message = restart_api()
|
||
result = _build_runtime_action_result(action=normalized_action, poll_after_seconds=4, refresh_runtime=True, ok=ok, message=message)
|
||
_emit_runtime_action_event(normalized_action, stage="finished", ok=ok, message=message, data=result)
|
||
return ok, message, result
|
||
if normalized_action == "start_sync_agent":
|
||
ok, message = start_sync_agent()
|
||
result = _build_runtime_action_result(action=normalized_action, poll_after_seconds=2, refresh_runtime=True, ok=ok, message=message)
|
||
_emit_runtime_action_event(normalized_action, stage="finished", ok=ok, message=message, data=result)
|
||
return ok, message, result
|
||
if normalized_action == "stop_sync_agent":
|
||
ok, message = stop_sync_agent()
|
||
result = _build_runtime_action_result(action=normalized_action, poll_after_seconds=2, refresh_runtime=True, ok=ok, message=message)
|
||
_emit_runtime_action_event(normalized_action, stage="finished", ok=ok, message=message, data=result)
|
||
return ok, message, result
|
||
if normalized_action == "push_sync":
|
||
ok, message, data = push_runtime_projection_now()
|
||
result = _build_runtime_action_result(action=normalized_action, poll_after_seconds=2, refresh_runtime=True, ok=ok, message=message, data=data)
|
||
_emit_runtime_action_event(normalized_action, stage="finished", ok=ok, message=message, data=result)
|
||
return ok, message, result
|
||
if normalized_action == "pull_tasks":
|
||
ok, message, data = pull_detect_task_batch_now()
|
||
result = _build_runtime_action_result(action=normalized_action, poll_after_seconds=2, refresh_runtime=True, ok=ok, message=message, data=data)
|
||
_emit_runtime_action_event(normalized_action, stage="finished", ok=ok, message=message, data=result)
|
||
return ok, message, result
|
||
if normalized_action == "push_debug_probe":
|
||
ok, message, data = push_debug_event(
|
||
service="domain-api",
|
||
event_type="debug_probe",
|
||
level="info",
|
||
message="manual debug probe",
|
||
payload={"node_code": settings.node_code, "node_region": settings.node_region},
|
||
)
|
||
result = _build_runtime_action_result(action=normalized_action, poll_after_seconds=1, refresh_runtime=False, ok=ok, message=message, data=data)
|
||
_emit_runtime_action_event(normalized_action, stage="finished", ok=ok, message=message, data=result)
|
||
return ok, message, result
|
||
result = _build_runtime_action_result(action=normalized_action, poll_after_seconds=0, refresh_runtime=False, ok=False, message=f"不支持的运行时动作: {action}")
|
||
_emit_runtime_action_event(normalized_action, stage="finished", ok=False, message=f"不支持的运行时动作: {action}", data=result)
|
||
return False, f"不支持的运行时动作: {action}", result
|