admin/getDomain
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: stabilize multi-region runtime sync and worker orchestration

Browse Source
This commit is contained in:
root
2026-04-27 15:48:12 +08:00
parent 7cbde2aa78
commit 215a364891
137 changed files with 31931 additions and 1943 deletions
Download Patch File Download Diff File Expand all files Collapse all files

96
domain-api/app/services/ops_agent_service.py
View File

@@ -7,6 +7,8 @@ import shlex
import threading
from datetime import datetime, timedelta
from psycopg2 import errors
from app.core.config import settings
from app.core.db import get_db
from app.services.ops_command_service import build_bash_command
@@ -80,6 +82,32 @@ ALTER TABLE ops_jobs ADD COLUMN IF NOT EXISTS last_agent_complete_request_id VAR
_OPS_AGENT_SCHEMA_LOCK = threading.Lock()
_OPS_AGENT_SCHEMA_READY = False
_OPS_AGENT_SCHEMA_ADVISORY_LOCK_KEY = 90421802
_OPS_AGENT_REQUIRED_TABLES = ("ops_node_tokens", "ops_job_events")
_OPS_AGENT_REQUIRED_COLUMNS = {
"ops_node_tokens": (
"node_code",
"purpose",
"issued_by",
"is_enabled",
"expires_at",
"last_used_at",
"metadata_json",
"created_at",
"updated_at",
),
"ops_job_events": (
"job_id",
"step_id",
"node_code",
"client_event_id",
"event_type",
"level",
"message",
"payload_json",
"created_at",
),
"ops_jobs": ("last_agent_complete_request_id",),
}
def ensure_ops_agent_schema() -> None:
@@ -91,14 +119,52 @@ def ensure_ops_agent_schema() -> None:
if _OPS_AGENT_SCHEMA_READY:
return
with get_db() as conn:
conn.autocommit = False
with conn.cursor() as cur:
cur.execute("SELECT pg_advisory_xact_lock(%s)", (_OPS_AGENT_SCHEMA_ADVISORY_LOCK_KEY,))
cur.execute(_AGENT_SCHEMA_SQL)
conn.commit()
if _ops_agent_schema_basics_present(cur):
_OPS_AGENT_SCHEMA_READY = True
return
conn.autocommit = False
try:
with conn.cursor() as cur:
cur.execute("SELECT pg_advisory_xact_lock(%s)", (_OPS_AGENT_SCHEMA_ADVISORY_LOCK_KEY,))
cur.execute(_AGENT_SCHEMA_SQL)
conn.commit()
except Exception as exc:
recoverable = isinstance(exc, (errors.DeadlockDetected, errors.LockNotAvailable))
try:
conn.rollback()
except Exception:
pass
if not recoverable:
raise
with conn.cursor() as cur:
if not _ops_agent_schema_basics_present(cur):
raise
_OPS_AGENT_SCHEMA_READY = True
def _ops_agent_schema_basics_present(cur) -> bool:
for table_name in _OPS_AGENT_REQUIRED_TABLES:
cur.execute("SELECT to_regclass(%s)", (f"public.{table_name}",))
row = cur.fetchone()
if not row or not row[0]:
return False
for table_name, required_columns in _OPS_AGENT_REQUIRED_COLUMNS.items():
cur.execute(
"""
SELECT column_name
FROM information_schema.columns
WHERE table_schema = 'public' AND table_name = %s
""",
(table_name,),
)
existing_columns = {str(row[0] or "").strip() for row in list(cur.fetchall() or [])}
if not set(required_columns).issubset(existing_columns):
return False
return True
def _hash_token(token: str) -> str:
return hashlib.sha256(str(token or "").encode("utf-8")).hexdigest()
@@ -152,12 +218,21 @@ def _merge_detect_runtime_snapshot(cluster_metadata: dict, metadata: dict, curre
"phase_label": str(cluster_metadata.get("phase_label", metadata.get("phase_label", "")) or "").strip(),
"phase_detail": str(cluster_metadata.get("phase_detail", metadata.get("phase_detail", "")) or "").strip(),
"recent_warning": str(cluster_metadata.get("recent_warning", metadata.get("recent_warning", "")) or "").strip(),
"available_proxy_count": int(
cluster_metadata.get("available_proxy_count", metadata.get("available_proxy_count", 0)) or 0
),
"proxy_runtime_label": str(
cluster_metadata.get("proxy_runtime_label", metadata.get("proxy_runtime_label", "")) or ""
).strip(),
"proxy_runtime_reason": str(
cluster_metadata.get("proxy_runtime_reason", metadata.get("proxy_runtime_reason", "")) or ""
).strip(),
"proxy_last_refresh_status": str(
cluster_metadata.get("proxy_last_refresh_status", metadata.get("proxy_last_refresh_status", "")) or ""
).strip(),
"proxy_last_refresh_time": str(
cluster_metadata.get("proxy_last_refresh_time", metadata.get("proxy_last_refresh_time", "")) or ""
).strip(),
"updated_at": str(cluster_metadata.get("updated_at", metadata.get("updated_at", "")) or "").strip(),
}
@@ -2714,6 +2789,15 @@ def _upsert_agent_detect_runtime(node_code: str, payload: dict) -> None:
"phase_label": phase_label,
"phase_detail": phase_detail,
"recent_warning": recent_warning,
"available_proxy_count": max(0, int(detect_runtime.get("available_proxy_count") or 0)),
"proxy_runtime_label": str(detect_runtime.get("proxy_runtime_label") or "").strip(),
"proxy_runtime_reason": str(detect_runtime.get("proxy_runtime_reason") or "").strip(),
"proxy_last_refresh_status": str(detect_runtime.get("proxy_last_refresh_status") or "").strip(),
"proxy_last_refresh_time": str(detect_runtime.get("proxy_last_refresh_time") or "").strip(),
"proxy_last_refresh_source_count": max(0, int(detect_runtime.get("proxy_last_refresh_source_count") or 0)),
"proxy_last_refresh_total_items": max(0, int(detect_runtime.get("proxy_last_refresh_total_items") or 0)),
"proxy_last_validated_count": max(0, int(detect_runtime.get("proxy_last_validated_count") or 0)),
"proxy_last_available_count": max(0, int(detect_runtime.get("proxy_last_available_count") or 0)),
"updated_at": str(detect_runtime.get("updated_at") or "").strip(),
"agent_heartbeat_at": datetime.now().isoformat(timespec="seconds"),
}
@@ -2778,8 +2862,10 @@ def _build_agent_runtime_config_bundle(node_code: str) -> dict:
"node_code": str(node_code or "").strip(),
"detect_options": dict(settings_payload.get("detect_options") or {}),
"proxy_config": dict(settings_payload.get("proxy_config") or {}),
"thread_count": int(settings_payload.get("thread_count", 2) or 2),
"thread_count": int(settings_payload.get("thread_count", 1000) or 1000),
"node_thread_counts": dict(settings_payload.get("node_thread_counts") or {}),
"process_count": int(settings_payload.get("process_count", 80) or 80),
"node_process_counts": dict(settings_payload.get("node_process_counts") or {}),
"runtime_settings": dict(runtime_settings or {}),
"sensitive_words": {
"text": str(sensitive_words_payload.get("text") or ""),