feat: stabilize multi-region runtime sync and worker orchestration

domain-api/app/services/worker_control_service.py

@@ -7,6 +7,8 @@ from datetime import datetime
 from pathlib import Path
 from uuid import uuid4
 
+import redis
+
 from app.core.config import settings
 from app.core.redis_client import get_redis
 from app.services.runtime_settings_service import get_runtime_settings
@@ -16,6 +18,101 @@ WORKER_CONTROL_CHANNEL = "domain_tool:worker_control"
 WORKER_PENDING_COMMAND_KEY = "domain_tool:worker_pending_command"
 
 
+def _normalize_target_node_codes(payload: dict | None) -> list[str]:
+    if not isinstance(payload, dict):
+        return []
+
+    normalized_targets: list[str] = []
+
+    def append_target(raw_value: object) -> None:
+        normalized_value = str(raw_value or "").strip()
+        if normalized_value and normalized_value not in normalized_targets:
+            normalized_targets.append(normalized_value)
+
+    for key in ("target_node_codes", "node_codes"):
+        raw_value = payload.get(key)
+        if isinstance(raw_value, (list, tuple, set)):
+            for item in raw_value:
+                append_target(item)
+        elif isinstance(raw_value, str) and raw_value.strip():
+            for item in raw_value.split(","):
+                append_target(item)
+        if normalized_targets:
+            return normalized_targets
+
+    for key in ("target_node_code", "node_code"):
+        raw_value = payload.get(key)
+        if raw_value not in (None, ""):
+            append_target(raw_value)
+        if normalized_targets:
+            return normalized_targets
+
+    return normalized_targets
+
+
+def _pending_command_keys(command_payload: dict) -> list[str]:
+    target_node_codes = _normalize_target_node_codes(command_payload)
+    if not target_node_codes:
+        return [WORKER_PENDING_COMMAND_KEY]
+    return [f"{WORKER_PENDING_COMMAND_KEY}:{node_code}" for node_code in target_node_codes]
+
+
+def _dedupe_target_node_codes(node_codes: list[str]) -> list[str]:
+    deduped: list[str] = []
+    seen: set[str] = set()
+    for raw_value in list(node_codes or []):
+        normalized_value = str(raw_value or "").strip()
+        if not normalized_value or normalized_value in seen:
+            continue
+        seen.add(normalized_value)
+        deduped.append(normalized_value)
+    return deduped
+
+
+def _expand_local_linux_worker_target_node_codes(service_name: str) -> list[str]:
+    base_node_code = str(settings.node_code or "").strip()
+    normalized_service_name = str(service_name or "").strip()
+    if not base_node_code or not normalized_service_name:
+        return []
+
+    target_node_codes = [base_node_code]
+    for unit in _expand_linux_worker_control_units(normalized_service_name):
+        normalized_unit = str(unit or "").strip()
+        if not normalized_unit:
+            continue
+        if normalized_unit.endswith(".service"):
+            normalized_unit = normalized_unit[:-8]
+        if normalized_unit == normalized_service_name:
+            continue
+        template_prefix = f"{normalized_service_name}@"
+        if not normalized_unit.startswith(template_prefix):
+            continue
+        instance_suffix = str(normalized_unit.split("@", 1)[1] or "").strip()
+        if instance_suffix:
+            target_node_codes.append(f"{base_node_code}-{instance_suffix}")
+    return _dedupe_target_node_codes(target_node_codes)
+
+
+def _publish_worker_command(redis_client, *, serialized: str, pending_keys: list[str]) -> None:
+    for key in pending_keys:
+        redis_client.set(key, serialized, ex=120)
+    redis_client.publish(WORKER_CONTROL_CHANNEL, serialized)
+
+
+def _build_direct_redis_client() -> redis.Redis:
+    return redis.Redis(
+        host=settings.redis_host,
+        port=settings.redis_port,
+        password=settings.redis_password or None,
+        db=settings.redis_db,
+        decode_responses=True,
+        socket_connect_timeout=5,
+        socket_timeout=5,
+        retry_on_timeout=True,
+        client_name=f"domain-api-workerctl:{settings.node_code}:{os.getpid()}",
+    )
+
+
 def _domain_root() -> Path:
     return Path(settings.domain_root)
 
@@ -37,6 +134,125 @@ def _run_shell(command: list[str], timeout: int = 20) -> subprocess.CompletedPro
     return subprocess.run(command, capture_output=True, text=True, timeout=timeout)
 
 
+def _parse_process_count_output(result: subprocess.CompletedProcess[str]) -> int | None:
+    raw_output = (result.stdout or "").strip()
+    if raw_output.isdigit():
+        return max(0, int(raw_output or 0))
+    return None
+
+
+def _probe_linux_worker_process_count() -> int:
+    probe_commands = (
+        (["bash", "-lc", "pgrep -fc '[d]etect_worker.py' || true"], 8),
+        (["bash", "-lc", "ps -eo args= | grep '[d]etect_worker.py' | wc -l"], 12),
+    )
+    for command, timeout in probe_commands:
+        try:
+            result = _run_shell(command, timeout=timeout)
+        except Exception:
+            continue
+        parsed_count = _parse_process_count_output(result)
+        if parsed_count is not None:
+            return parsed_count
+    return 0
+
+
+def _probe_linux_worker_instance_count(service_name: str) -> int:
+    normalized_service_name = str(service_name or "").strip()
+    if not normalized_service_name:
+        return 0
+    template_prefix = normalized_service_name[:-8] if normalized_service_name.endswith(".service") else normalized_service_name
+    try:
+        result = _run_systemctl(
+            [
+                "list-units",
+                f"{template_prefix}@*",
+                "--type=service",
+                "--all",
+                "--no-legend",
+                "--plain",
+            ],
+            timeout=12,
+            require_sudo=False,
+        )
+    except Exception:
+        return 0
+    if result.returncode != 0:
+        return 0
+    count = 0
+    for raw_line in (result.stdout or "").splitlines():
+        line = str(raw_line or "").strip()
+        if not line:
+            continue
+        parts = line.split()
+        if len(parts) < 4:
+            continue
+        if parts[2] != "active" or parts[3] != "running":
+            continue
+        count += 1
+    return count
+
+
+def _dedupe_units(units: list[str]) -> list[str]:
+    deduped: list[str] = []
+    seen: set[str] = set()
+    for item in units:
+        normalized = str(item or "").strip()
+        if not normalized or normalized in seen:
+            continue
+        seen.add(normalized)
+        deduped.append(normalized)
+    return deduped
+
+
+def _list_linux_worker_instance_units(service_name: str) -> list[str]:
+    normalized_service_name = str(service_name or "").strip()
+    if not normalized_service_name:
+        return []
+    template_prefix = normalized_service_name[:-8] if normalized_service_name.endswith(".service") else normalized_service_name
+    units: list[str] = []
+    try:
+        result = _run_systemctl(
+            [
+                "list-units",
+                f"{template_prefix}@*",
+                "--type=service",
+                "--all",
+                "--no-legend",
+                "--plain",
+            ],
+            timeout=12,
+            require_sudo=False,
+        )
+        for raw_line in (result.stdout or "").splitlines():
+            parts = str(raw_line or "").strip().split()
+            if parts:
+                units.append(str(parts[0] or "").strip())
+    except Exception:
+        pass
+
+    managed_prefix = f"{normalized_service_name}-"
+    try:
+        for candidate in Path("/etc/default").iterdir():
+            if not candidate.is_file():
+                continue
+            if not candidate.name.startswith(managed_prefix):
+                continue
+            suffix = str(candidate.name[len(managed_prefix):] or "").strip()
+            if suffix:
+                units.append(f"{normalized_service_name}@{suffix}")
+    except Exception:
+        pass
+    return _dedupe_units(units)
+
+
+def _expand_linux_worker_control_units(service_name: str) -> list[str]:
+    normalized_service_name = str(service_name or "").strip()
+    if not normalized_service_name:
+        return []
+    return _dedupe_units([normalized_service_name, *_list_linux_worker_instance_units(normalized_service_name)])
+
+
 def _run_systemctl(
     command: list[str],
     timeout: int = 20,
@@ -192,7 +408,36 @@ def _windows_runtime() -> dict:
 def _linux_runtime() -> dict:
     runtime = _runtime_config()
     service_name = runtime["worker_service_name"]
-    return probe_systemd_service(service_name, mode="linux-systemd")
+    service_probe = probe_systemd_service(service_name, mode="linux-systemd")
+    instance_count = _probe_linux_worker_instance_count(service_name)
+    process_count = _probe_linux_worker_process_count()
+
+    if process_count <= 0 and instance_count <= 0:
+        return service_probe
+
+    latest_start_time = str(service_probe.get("latest_start_time") or "").strip()
+    if service_probe.get("running", False) or instance_count > 0:
+        message = service_probe.get("message") or f"multi-instance active ({process_count})"
+        if instance_count > 0 and not service_probe.get("running", False):
+            message = f"template instances active ({instance_count})"
+        return {
+            **service_probe,
+            "running": True,
+            "process_count": process_count,
+            "latest_start_time": latest_start_time,
+            "message": message,
+        }
+    message = str(service_probe.get("message") or "").strip()
+    if message:
+        message = f"{message}; detected {process_count} unmanaged worker processes"
+    else:
+        message = f"detected {process_count} unmanaged worker processes"
+    return {
+        **service_probe,
+        "process_count": process_count,
+        "latest_start_time": latest_start_time,
+        "message": message,
+    }
 
 
 def detect_sync_agent_runtime() -> dict:
@@ -232,9 +477,13 @@ def start_worker() -> tuple[bool, str]:
     worker_mode = runtime["worker_mode"]
     service_name = runtime["worker_service_name"]
     if worker_mode == "linux-systemd":
-        result = _run_systemctl(["start", service_name], timeout=30)
+        units = _expand_linux_worker_control_units(service_name)
+        result = _run_systemctl(["start", *units], timeout=max(30, 15 * max(1, len(units))))
         if result.returncode != 0:
             return False, normalize_systemctl_error(result.stderr or result.stdout or "启动 Linux Worker 失败", service_name=service_name)
+        extra_units = max(0, len(units) - 1)
+        if extra_units > 0:
+            return True, f"Linux Worker 启动命令已发送: {service_name}，附带 {extra_units} 个实例"
         return True, f"Linux Worker 启动命令已发送: {service_name}"
 
     if os.name != "nt":
@@ -260,9 +509,13 @@ def stop_worker() -> tuple[bool, str]:
     worker_mode = runtime["worker_mode"]
     service_name = runtime["worker_service_name"]
     if worker_mode == "linux-systemd":
-        result = _run_systemctl(["stop", service_name], timeout=30)
+        units = _expand_linux_worker_control_units(service_name)
+        result = _run_systemctl(["stop", *units], timeout=max(30, 15 * max(1, len(units))))
         if result.returncode != 0:
             return False, normalize_systemctl_error(result.stderr or result.stdout or "停止 Linux Worker 失败", service_name=service_name)
+        extra_units = max(0, len(units) - 1)
+        if extra_units > 0:
+            return True, f"Linux Worker 停止命令已发送: {service_name}，附带 {extra_units} 个实例"
         return True, f"Linux Worker 停止命令已发送: {service_name}"
 
     if os.name != "nt":
@@ -289,15 +542,42 @@ def stop_worker() -> tuple[bool, str]:
 
 
 def send_worker_command(action: str, payload: dict | None = None) -> tuple[bool, str]:
+    command_payload = {"action": action}
+    if payload:
+        command_payload.update(payload)
+    runtime = _runtime_config()
+    explicit_targets = _normalize_target_node_codes(command_payload)
+    if not explicit_targets and str(runtime.get("worker_mode") or "").strip() == "linux-systemd":
+        expanded_targets = _expand_local_linux_worker_target_node_codes(
+            str(runtime.get("worker_service_name") or "").strip()
+        )
+        if expanded_targets:
+            command_payload["target_node_codes"] = expanded_targets
+    command_payload["request_id"] = str(command_payload.get("request_id") or f"workerctl-{uuid4().hex[:12]}")
+    serialized = json.dumps(command_payload, ensure_ascii=False)
+    pending_keys = _pending_command_keys(command_payload)
+    target_node_codes = _normalize_target_node_codes(command_payload)
+
+    direct_client = None
     try:
-        redis_client = get_redis()
-        command_payload = {"action": action}
-        if payload:
-            command_payload.update(payload)
-        command_payload["request_id"] = str(command_payload.get("request_id") or f"workerctl-{uuid4().hex[:12]}")
-        serialized = json.dumps(command_payload, ensure_ascii=False)
-        redis_client.set(WORKER_PENDING_COMMAND_KEY, serialized, ex=120)
-        redis_client.publish(WORKER_CONTROL_CHANNEL, serialized)
-        return True, f"已发送 Worker 控制指令: {action}"
+        try:
+            _publish_worker_command(get_redis(), serialized=serialized, pending_keys=pending_keys)
+        except Exception:
+            direct_client = _build_direct_redis_client()
+            _publish_worker_command(direct_client, serialized=serialized, pending_keys=pending_keys)
+
+        if pending_keys == [WORKER_PENDING_COMMAND_KEY]:
+            return True, f"已发送 Worker 控制指令: {action}"
+        if len(target_node_codes) <= 4:
+            target_summary = ",".join(target_node_codes)
+        else:
+            target_summary = f"{len(target_node_codes)} targets"
+        return True, f"已发送 Worker 控制指令: {action} -> {target_summary}"
     except Exception as exc:
         return False, f"发送 Worker 控制指令失败: {exc}"
+    finally:
+        if direct_client is not None:
+            try:
+                direct_client.close()
+            except Exception:
+                pass