d

domain-api/app/services/ops_release_service.py

@@ -3,11 +3,13 @@ from __future__ import annotations
 import json
 import os
 import re
+import socket
 import subprocess
 from datetime import datetime
 from math import ceil
 from pathlib import Path
 from threading import Lock
+from urllib.parse import urlsplit, urlunsplit
 from uuid import uuid4
 
 from app.core.db import get_db
@@ -490,7 +492,119 @@ def _normalize_public_base_url(raw_value: str) -> str:
     normalized = str(raw_value or "").strip().rstrip("/")
     if normalized.endswith("/api/v1"):
         normalized = normalized[: -len("/api/v1")]
-    return normalized
+    return _rewrite_loopback_control_plane_url(normalized)
+
+
+def _is_loopback_hostname(hostname: str) -> bool:
+    normalized = str(hostname or "").strip().lower().strip("[]")
+    return normalized in {"127.0.0.1", "localhost", "0.0.0.0", "::1"}
+
+
+def _build_url_with_host(raw_url: str, *, host: str, scheme: str = "", port: int | None = None) -> str:
+    normalized_url = str(raw_url or "").strip()
+    if not normalized_url:
+        return ""
+    parsed = urlsplit(normalized_url)
+    if not parsed.scheme or not parsed.netloc:
+        return normalized_url
+    normalized_host = str(host or "").strip().strip("[]")
+    if not normalized_host:
+        return normalized_url
+    final_scheme = str(scheme or parsed.scheme or "http").strip() or "http"
+    final_port = parsed.port if port is None else int(port)
+    netloc = f"{normalized_host}:{final_port}" if final_port else normalized_host
+    return urlunsplit((final_scheme, netloc, parsed.path, parsed.query, parsed.fragment))
+
+
+def _resolve_public_control_plane_origin(loopback_url: str) -> str:
+    normalized_loopback_url = str(loopback_url or "").strip()
+    if not normalized_loopback_url:
+        return ""
+    parsed_loopback = urlsplit(normalized_loopback_url)
+    default_scheme = str(parsed_loopback.scheme or "http").strip() or "http"
+    default_port = parsed_loopback.port
+
+    env_candidates = [
+        os.getenv("OPS_CONTROL_PLANE_PUBLIC_BASE_URL", ""),
+        os.getenv("CONTROL_PLANE_PUBLIC_BASE_URL", ""),
+        os.getenv("OPS_CONTROL_PLANE_BASE_URL", ""),
+    ]
+    for candidate in env_candidates:
+        normalized_candidate = str(candidate or "").strip().rstrip("/")
+        if not normalized_candidate:
+            continue
+        parsed_candidate = urlsplit(
+            normalized_candidate if "://" in normalized_candidate else f"{default_scheme}://{normalized_candidate}"
+        )
+        candidate_host = str(parsed_candidate.hostname or "").strip()
+        if candidate_host and not _is_loopback_hostname(candidate_host):
+            return _build_url_with_host(
+                normalized_loopback_url,
+                host=candidate_host,
+                scheme=str(parsed_candidate.scheme or default_scheme),
+                port=parsed_candidate.port if parsed_candidate.port is not None else default_port,
+            )
+
+    try:
+        from app.services.cluster_runtime_service import get_cluster_snapshot
+
+        snapshot = get_cluster_snapshot()
+        local_hostnames = {
+            str(socket.gethostname() or "").strip().lower(),
+            str(socket.getfqdn() or "").strip().lower(),
+        }
+        fallback_control_hosts: list[str] = []
+        for item in list(snapshot.get("nodes") or []):
+            if str(item.get("role") or "").strip() != "control":
+                continue
+            control_host = str(item.get("hostname") or "").strip().lower()
+            control_ip = str(item.get("ip") or "").strip()
+            if not control_ip or _is_loopback_hostname(control_ip):
+                continue
+            if control_host and control_host in local_hostnames:
+                return _build_url_with_host(
+                    normalized_loopback_url,
+                    host=control_ip,
+                    scheme=default_scheme,
+                    port=default_port,
+                )
+            fallback_control_hosts.append(control_ip)
+        for control_ip in fallback_control_hosts:
+            if control_ip and not _is_loopback_hostname(control_ip):
+                return _build_url_with_host(
+                    normalized_loopback_url,
+                    host=control_ip,
+                    scheme=default_scheme,
+                    port=default_port,
+                )
+    except Exception:
+        pass
+
+    try:
+        resolved_host = str(socket.gethostbyname(socket.gethostname()) or "").strip()
+        if resolved_host and not _is_loopback_hostname(resolved_host):
+            return _build_url_with_host(
+                normalized_loopback_url,
+                host=resolved_host,
+                scheme=default_scheme,
+                port=default_port,
+            )
+    except Exception:
+        pass
+    return ""
+
+
+def _rewrite_loopback_control_plane_url(raw_url: str) -> str:
+    normalized = str(raw_url or "").strip()
+    if not normalized:
+        return ""
+    parsed = urlsplit(normalized)
+    if not parsed.scheme or not parsed.netloc:
+        return normalized
+    if not _is_loopback_hostname(str(parsed.hostname or "").strip()):
+        return normalized
+    resolved = _resolve_public_control_plane_origin(normalized)
+    return resolved or normalized
 
 
 def _build_absolute_release_package_url(base_url: str, raw_path: str) -> str:
@@ -1684,7 +1798,7 @@ def _resolve_rollout_targets(selector: dict) -> list[dict]:
             continue
         if only_effective_workers and not bool(item.get("is_effective_worker", False)):
             continue
-        if only_online and str(item.get("status") or "") != "online":
+        if only_online and str(item.get("status") or "") not in {"online", "busy"}:
             continue
         targets.append(item)
 
@@ -2037,9 +2151,12 @@ def _build_smart_rollout_role_policy(mode: str, *, execution_mode: str = "remote
             "restart_services": ["domaincheck-api", "domaincheck-worker", "domaincheck-sync-agent"],
             "health_check_urls": ["http://127.0.0.1:8100/health"],
             "health_check_services": ["domaincheck-api", "domaincheck-worker", "domaincheck-sync-agent"],
-            "health_check_timeout_seconds": 10,
-            "health_check_retries": 2,
-            "health_check_interval_seconds": 2,
+            # Control 节点启动期间会先经历较长的 import / startup hook，
+            # systemd 已经 active 但 /health 仍可能在 15-20 秒内拒绝连接。
+            # 这里把健康检查窗口放宽到约 40 秒，避免被误回滚。
+            "health_check_timeout_seconds": 20,
+            "health_check_retries": 9,
+            "health_check_interval_seconds": 4,
             "rollback_on_failure": True,
             "switch_current": True,
         }
@@ -3254,18 +3371,56 @@ def refresh_release_rollout_for_job(job_id: int) -> dict:
     return refresh_release_rollout(rollout_id)
 
 
-def _build_release_job_payload(release: dict, rollout: dict) -> dict:
+def _default_release_deploy_payload_for_target(target: dict) -> dict:
+    role = str((target or {}).get("role") or "").strip().lower()
+    if role == "control":
+        return {
+            "restart_services": ["domaincheck-api", "domaincheck-worker", "domaincheck-sync-agent"],
+            "health_check_urls": ["http://127.0.0.1:8100/health"],
+            "health_check_services": ["domaincheck-api", "domaincheck-worker", "domaincheck-sync-agent"],
+            "health_check_timeout_seconds": 20,
+            "health_check_retries": 9,
+            "health_check_interval_seconds": 4,
+        }
+    return {
+        "restart_services": ["domaincheck-worker"],
+        "health_check_urls": [],
+        "health_check_services": ["domaincheck-worker"],
+        "health_check_timeout_seconds": 10,
+        "health_check_retries": 2,
+        "health_check_interval_seconds": 2,
+    }
+
+
+def _build_release_job_payload(release: dict, rollout: dict, *, target: dict | None = None) -> dict:
     policy = dict(rollout.get("policy") or {})
     deploy_payload = dict(policy.get("deploy_payload") or {})
+    target_defaults = _default_release_deploy_payload_for_target(target or {})
+
+    if not [str(item).strip() for item in list(deploy_payload.get("restart_services") or []) if str(item).strip()]:
+        deploy_payload["restart_services"] = list(target_defaults.get("restart_services") or [])
+    if not [str(item).strip() for item in list(deploy_payload.get("health_check_services") or []) if str(item).strip()]:
+        deploy_payload["health_check_services"] = list(target_defaults.get("health_check_services") or [])
+    if not [str(item).strip() for item in list(deploy_payload.get("health_check_urls") or []) if str(item).strip()]:
+        deploy_payload["health_check_urls"] = list(target_defaults.get("health_check_urls") or [])
+    if deploy_payload.get("health_check_timeout_seconds") in (None, "", 0, "0"):
+        deploy_payload["health_check_timeout_seconds"] = int(target_defaults.get("health_check_timeout_seconds") or 10)
+    if deploy_payload.get("health_check_retries") in (None, "", 0, "0"):
+        deploy_payload["health_check_retries"] = int(target_defaults.get("health_check_retries") or 2)
+    if deploy_payload.get("health_check_interval_seconds") in (None, "", 0, "0"):
+        deploy_payload["health_check_interval_seconds"] = int(target_defaults.get("health_check_interval_seconds") or 2)
+
+    artifact_url = _rewrite_loopback_control_plane_url(str(release.get("artifact_url") or "").strip())
     return {
         "release_id": int(release.get("id") or 0),
         "rollout_id": int(rollout.get("id") or 0),
         "release_version": str(release.get("release_version") or ""),
-        "artifact_url": str(release.get("artifact_url") or ""),
+        "artifact_url": artifact_url,
         "checksum": str(release.get("checksum") or ""),
         "channel": str(release.get("channel") or ""),
         "commit_sha": str(release.get("commit_sha") or ""),
         "notes": str(release.get("notes") or ""),
+        "target_node_role": str((target or {}).get("role") or "").strip(),
         **deploy_payload,
     }
 
@@ -3308,12 +3463,11 @@ def _enqueue_rollout_batch(rollout_id: int, *, created_by: str, reason: str = "m
     auto_dispatch = bool(policy.get("auto_dispatch", False))
     auto_approve = bool(policy.get("auto_approve", False))
     execution_mode = str(policy.get("execution_mode") or "remote-agent").strip() or "remote-agent"
-    job_payload = _build_release_job_payload(release, rollout)
-
     for target in batch_targets:
         target_node_code = str(target.get("node_code") or "").strip()
         if not target_node_code:
             continue
+        job_payload = _build_release_job_payload(release, rollout, target=target)
         job_ok, _job_message, job_data = create_ops_job(
             {
                 "action": "deploy.release",