feat: stabilize multi-region runtime sync and worker orchestration

domain-api/app/services/ops_release_service.py

@@ -12,6 +12,8 @@ from threading import Lock
 from urllib.parse import urlsplit, urlunsplit
 from uuid import uuid4
 
+from psycopg2 import errors
+
 from app.core.db import get_db
 from app.services.build_info_service import get_runtime_build_info
 from app.services.ops_command_service import build_bash_command
@@ -74,6 +76,19 @@ _ROLLOUT_INSPECTION_ACTION_KEYS = ("health.snapshot", "logs.collect", "diagnosti
 _SAFE_RELEASE_PACKAGE_NAME_RE = re.compile(r"^[A-Za-z0-9._-]+$")
 _RELEASE_SCHEMA_LOCK = Lock()
 _RELEASE_SCHEMA_READY = False
+_RELEASE_SCHEMA_ADVISORY_LOCK_KEY = 90421803
+_RELEASE_REQUIRED_TABLES = ("ops_releases", "ops_release_rollouts")
+_RELEASE_REQUIRED_COLUMNS = {
+    "ops_release_rollouts": (
+        "rollout_code",
+        "target_nodes_json",
+        "batch_cursor",
+        "batches_total",
+        "jobs_total",
+        "jobs_created",
+        "result_summary_json",
+    ),
+}
 
 
 def ensure_ops_release_schema() -> None:
@@ -86,13 +101,52 @@ def ensure_ops_release_schema() -> None:
         if _RELEASE_SCHEMA_READY:
             return
         with get_db() as conn:
-            conn.autocommit = False
             with conn.cursor() as cur:
-                cur.execute(_RELEASE_SCHEMA_SQL)
-            conn.commit()
+                if _ops_release_schema_basics_present(cur):
+                    _RELEASE_SCHEMA_READY = True
+                    return
+            conn.autocommit = False
+            try:
+                with conn.cursor() as cur:
+                    cur.execute("SELECT pg_advisory_xact_lock(%s)", (_RELEASE_SCHEMA_ADVISORY_LOCK_KEY,))
+                    cur.execute(_RELEASE_SCHEMA_SQL)
+                conn.commit()
+            except Exception as exc:
+                recoverable = isinstance(exc, (errors.DeadlockDetected, errors.LockNotAvailable))
+                try:
+                    conn.rollback()
+                except Exception:
+                    pass
+                if not recoverable:
+                    raise
+                with conn.cursor() as cur:
+                    if not _ops_release_schema_basics_present(cur):
+                        raise
         _RELEASE_SCHEMA_READY = True
 
 
+def _ops_release_schema_basics_present(cur) -> bool:
+    for table_name in _RELEASE_REQUIRED_TABLES:
+        cur.execute("SELECT to_regclass(%s)", (f"public.{table_name}",))
+        row = cur.fetchone()
+        if not row or not row[0]:
+            return False
+
+    for table_name, required_columns in _RELEASE_REQUIRED_COLUMNS.items():
+        cur.execute(
+            """
+            SELECT column_name
+            FROM information_schema.columns
+            WHERE table_schema = 'public' AND table_name = %s
+            """,
+            (table_name,),
+        )
+        existing_columns = {str(row[0] or "").strip() for row in list(cur.fetchall() or [])}
+        if not set(required_columns).issubset(existing_columns):
+            return False
+    return True
+
+
 def _decode_json(value: object) -> dict:
     if isinstance(value, dict):
         return value
@@ -1091,7 +1145,11 @@ def build_rollout_target_operational_readiness(
     desired_release: dict | None = None,
 ) -> dict:
     from app.services.cluster_runtime_service import get_cluster_snapshot
-    from app.services.ops_agent_service import ensure_ops_agent_schema, get_managed_node_onboarding
+    from app.services.ops_agent_service import (
+        ensure_ops_agent_schema,
+        get_managed_node_onboarding,
+        list_managed_nodes_with_agent_state,
+    )
     from app.services.ops_job_service import list_managed_nodes
 
     ensure_ops_agent_schema()
@@ -1132,6 +1190,7 @@ def build_rollout_target_operational_readiness(
         for item in list(cluster_snapshot.get("nodes") or [])
         if str(item.get("node_code") or "").strip()
     }
+    managed_nodes_payload = list_managed_nodes_with_agent_state()
     managed_nodes = list_managed_nodes()
     managed_map = {
         str(item.get("node_code") or "").strip(): item
@@ -1239,7 +1298,11 @@ def build_rollout_target_operational_readiness(
         last_seen_at = str(managed.get("last_seen_at") or "").strip() or str(metadata.get("last_seen_at") or "").strip()
         cluster_status = str(cluster_node.get("status") or target.get("status") or "").strip()
         current_load = int(cluster_node.get("current_load", target.get("current_load", 0)) or 0)
-        onboarding = get_managed_node_onboarding(node_code) if node_code else {}
+        onboarding = (
+            get_managed_node_onboarding(node_code, nodes_payload=managed_nodes_payload)
+            if node_code
+            else {}
+        )
         onboarding_stage = dict(onboarding.get("onboarding_stage") or {})
         recovery_decision = dict(onboarding.get("recovery_decision") or {})
         onboarding_stage_code = str(onboarding_stage.get("code") or "").strip()