feat: stabilize multi-region runtime sync and worker orchestration

domain-api/app/services/ops_job_service.py

@@ -5,6 +5,8 @@ import threading
 from datetime import datetime
 from uuid import uuid4
 
+from psycopg2 import errors
+
 from app.core.config import settings
 from app.core.db import get_db
 from app.services.ops_execution_capability_service import (
@@ -116,6 +118,28 @@ _LOCAL_RUNTIME_ACTIONS = {
 _OPS_SCHEMA_LOCK = threading.Lock()
 _OPS_SCHEMA_READY = False
 _OPS_SCHEMA_ADVISORY_LOCK_KEY = 90421801
+_OPS_REQUIRED_TABLES = (
+    "ops_managed_nodes",
+    "ops_managed_node_secrets",
+    "ops_jobs",
+    "ops_job_steps",
+)
+_OPS_REQUIRED_COLUMNS = {
+    "ops_jobs": (
+        "risk_level",
+        "approval_required",
+        "approval_status",
+        "approved_by",
+        "approved_at",
+        "blocked_reason",
+        "cancellation_reason",
+        "dispatched_at",
+        "target_selector_json",
+        "policy_json",
+        "rollout_id",
+    ),
+    "ops_job_steps": ("stdout_text", "stderr_text", "result_json"),
+}
 
 
 def ensure_ops_schema() -> None:
@@ -126,14 +150,52 @@ def ensure_ops_schema() -> None:
         if _OPS_SCHEMA_READY:
             return
         with get_db() as conn:
-            conn.autocommit = False
             with conn.cursor() as cur:
-                cur.execute("SELECT pg_advisory_xact_lock(%s)", (_OPS_SCHEMA_ADVISORY_LOCK_KEY,))
-                cur.execute(_OPS_SCHEMA_SQL)
-            conn.commit()
+                if _ops_schema_basics_present(cur):
+                    _OPS_SCHEMA_READY = True
+                    return
+            conn.autocommit = False
+            try:
+                with conn.cursor() as cur:
+                    cur.execute("SELECT pg_advisory_xact_lock(%s)", (_OPS_SCHEMA_ADVISORY_LOCK_KEY,))
+                    cur.execute(_OPS_SCHEMA_SQL)
+                conn.commit()
+            except Exception as exc:
+                recoverable = isinstance(exc, (errors.DeadlockDetected, errors.LockNotAvailable))
+                try:
+                    conn.rollback()
+                except Exception:
+                    pass
+                if not recoverable:
+                    raise
+                with conn.cursor() as cur:
+                    if not _ops_schema_basics_present(cur):
+                        raise
         _OPS_SCHEMA_READY = True
 
 
+def _ops_schema_basics_present(cur) -> bool:
+    for table_name in _OPS_REQUIRED_TABLES:
+        cur.execute("SELECT to_regclass(%s)", (f"public.{table_name}",))
+        row = cur.fetchone()
+        if not row or not row[0]:
+            return False
+
+    for table_name, required_columns in _OPS_REQUIRED_COLUMNS.items():
+        cur.execute(
+            """
+            SELECT column_name
+            FROM information_schema.columns
+            WHERE table_schema = 'public' AND table_name = %s
+            """,
+            (table_name,),
+        )
+        existing_columns = {str(row[0] or "").strip() for row in list(cur.fetchall() or [])}
+        if not set(required_columns).issubset(existing_columns):
+            return False
+    return True
+
+
 def _decode_json(value: object) -> dict:
     if isinstance(value, dict):
         return value
@@ -819,6 +881,7 @@ def _resolve_bootstrap_target_defaults(target_node_code: str) -> dict:
 def _execute_control_plane_job(
     action: str,
     *,
+    job_id: int = 0,
     target_node_code: str,
     payload: dict | None = None,
     requested_by: str = "api",
@@ -864,6 +927,17 @@ def _execute_control_plane_job(
             }
         return False, message, data
 
+    if action == "migration.execute":
+        from app.services.ops_migration_service import execute_ops_migration_job
+
+        return execute_ops_migration_job(
+            job_id=int(job_id or 0),
+            target_node_code=str(target_node_code or "").strip(),
+            payload=dict(normalized_payload or {}),
+            requested_by=str(requested_by or "api").strip() or "api",
+            metadata=dict(normalized_metadata or {}),
+        )
+
     return False, f"当前未实现控制面执行动作: {action}", {}
 
 
@@ -1006,6 +1080,7 @@ def _execute_control_plane_job_record(job_id: int) -> tuple[bool, str, dict]:
 
     ok, message, result = _execute_control_plane_job(
         str(job.get("action") or ""),
+        job_id=int(job_id),
         target_node_code=str(job.get("target_node_code") or ""),
         payload=dict(job.get("payload") or {}),
         requested_by=str(job.get("requested_by") or "api"),
@@ -1339,6 +1414,7 @@ def create_ops_job(payload: dict) -> tuple[bool, str, dict]:
                 if execution_mode == "control-plane":
                     ok, message, result = _execute_control_plane_job(
                         action,
+                        job_id=job_id,
                         target_node_code=target_node_code,
                         payload=dict(input_payload or {}),
                         requested_by=requested_by,