admin/getDomain
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: stabilize multi-region runtime sync and worker orchestration

Browse Source
This commit is contained in:
root
2026-04-27 15:48:12 +08:00
parent 7cbde2aa78
commit 215a364891
137 changed files with 31931 additions and 1943 deletions
Download Patch File Download Diff File Expand all files Collapse all files

288
domain-api/app/services/sync_record_service.py
View File

@@ -51,6 +51,80 @@ _TERMINAL_DETECT_RESULT_EVENT_TYPES = {
"domain_blacklisted",
}
_RUNTIME_PROJECTION_HEARTBEAT_INTERVAL = timedelta(seconds=45)
_RUNTIME_PROJECTION_FUTURE_SKEW_GRACE = timedelta(minutes=5)
def _runtime_projection_activity_signature(projection: dict) -> dict:
normalized_projection = dict(projection or {})
active_job = dict(normalized_projection.get("active_job") or {})
normalized_nodes: list[tuple] = []
for raw_item in list(active_job.get("distributed_node_stats") or active_job.get("node_stats") or []):
if not isinstance(raw_item, dict):
continue
node_code = str(raw_item.get("node_code") or "").strip()
if not node_code:
continue
normalized_nodes.append(
(
node_code,
int(raw_item.get("display_running", raw_item.get("current_load", 0)) or 0),
int(raw_item.get("active_threads", 0) or 0),
int(raw_item.get("max_threads", 0) or 0),
int(raw_item.get("items_claimed", 0) or 0),
int(raw_item.get("items_running", 0) or 0),
int(raw_item.get("items_total", 0) or 0),
str(raw_item.get("status") or "").strip(),
)
)
normalized_cluster_nodes: list[tuple] = []
for raw_item in list(normalized_projection.get("cluster_nodes") or []):
if not isinstance(raw_item, dict):
continue
node_code = str(raw_item.get("node_code") or "").strip()
if not node_code:
continue
normalized_cluster_nodes.append(
(
node_code,
str(raw_item.get("role") or "").strip(),
str(raw_item.get("status") or "").strip(),
int(raw_item.get("current_load", 0) or 0),
int(raw_item.get("active_threads", 0) or 0),
int(raw_item.get("max_threads", 0) or 0),
bool(raw_item.get("detect_participating", False)),
)
)
return {
"active_thread_count": int(normalized_projection.get("active_thread_count", 0) or 0),
"max_thread_count": int(normalized_projection.get("max_thread_count", 0) or 0),
"job_display_running": int(active_job.get("display_items_running", 0) or 0),
"job_display_claimed": int(active_job.get("display_items_claimed", 0) or 0),
"job_display_max_threads": int(active_job.get("display_max_threads", 0) or 0),
"job_items_total": int(active_job.get("items_total", 0) or 0),
"job_items_running": int(active_job.get("items_running", 0) or 0),
"job_items_claimed": int(active_job.get("items_claimed", 0) or 0),
"node_stats": normalized_nodes,
"cluster_nodes": normalized_cluster_nodes,
}
def _pick_latest_projection_row(rows: list[tuple], *, created_at_index: int) -> tuple | None:
candidates = list(rows or [])
if not candidates:
return None
fallback = candidates[0]
for row in candidates:
if len(row) <= int(created_at_index):
return row
created_at = row[created_at_index]
if not isinstance(created_at, datetime):
return row
now = datetime.now(created_at.tzinfo) if created_at.tzinfo else datetime.now()
if created_at <= now + _RUNTIME_PROJECTION_FUTURE_SKEW_GRACE:
return row
return fallback
def _collect_recent_domain_events(active_job: dict, limit: int = 30) -> list[dict]:
safe_limit = max(1, int(limit or 30))
@@ -214,10 +288,15 @@ def _should_append_runtime_projection(previous_payload: dict, current_projection
if previous_alerts != current_alerts:
return True
if _runtime_projection_activity_signature(previous_projection) != _runtime_projection_activity_signature(current_projection):
return True
if not previous_created_at:
return True
now = datetime.now(previous_created_at.tzinfo) if previous_created_at.tzinfo else datetime.now()
return now - previous_created_at >= timedelta(seconds=45)
if previous_created_at > now + _RUNTIME_PROJECTION_FUTURE_SKEW_GRACE:
return True
return now - previous_created_at >= _RUNTIME_PROJECTION_HEARTBEAT_INTERVAL
@db_read_retry()
@@ -293,6 +372,25 @@ def get_detect_result_sync_batches(limit: int = 5) -> dict:
safe_limit = max(1, min(int(limit or 5), 20))
source_region = _normalize_region(settings.sync_source_region, settings.node_region)
target_region = _normalize_region(settings.sync_target_region, "overseas")
local_worker_expected = _local_node_expected_to_execute_worker()
if not local_worker_expected:
return {
"applicable": False,
"local_worker_expected": False,
"reason": "当前节点不承载本地检测执行，结果批次推送概览不适用。",
"source_region": source_region,
"target_region": target_region,
"jobs_total": 0,
"state_counts": {
"synced": 0,
"delivered": 0,
"pushing": 0,
"projected": 0,
"failed": 0,
"unsynced": 0,
},
"batches": [],
}
batches: list[dict] = []
with get_db() as conn:
@@ -424,6 +522,9 @@ def get_detect_result_sync_batches(limit: int = 5) -> dict:
state_counts[state] = state_counts.get(state, 0) + 1
return {
"applicable": True,
"local_worker_expected": local_worker_expected,
"reason": "",
"source_region": source_region,
"target_region": target_region,
"jobs_total": len(batches),
@@ -436,6 +537,11 @@ def get_detect_result_sync_batches(limit: int = 5) -> dict:
def get_sync_summary(record_limit: int = 10) -> dict:
source_region = _normalize_region(settings.sync_source_region, settings.node_region)
target_region = _normalize_region(settings.sync_target_region, "overseas")
local_worker_expected = _local_node_expected_to_execute_worker()
push_expected_on_this_node = bool(
str(settings.node_region or "").strip() == "mainland"
and str(settings.node_role or "").strip() == "control"
)
with get_db() as conn:
with conn.cursor() as cur:
cur.execute(
@@ -486,6 +592,8 @@ def get_sync_summary(record_limit: int = 10) -> dict:
return {
"enabled": bool(settings.sync_push_enabled),
"push_expected_on_this_node": push_expected_on_this_node,
"local_worker_expected_on_this_node": local_worker_expected,
"source_region": source_region,
"target_region": target_region,
"target_api_base_url": settings.sync_target_api_base_url,
@@ -540,6 +648,107 @@ def _local_node_expected_to_execute_worker() -> bool:
return node_role == "worker" or (node_region == "mainland" and node_role == "control")
def _projection_node_rows(*, detect: dict, active_job: dict) -> list[dict]:
queue_health = dict(detect.get("queue_health") or {})
queue_nodes = [
dict(item)
for item in list(queue_health.get("nodes") or [])
if isinstance(item, dict) and str(item.get("node_code") or "").strip()
]
if queue_nodes:
return queue_nodes
return [
dict(item)
for item in list(active_job.get("distributed_node_stats") or active_job.get("node_stats") or [])
if isinstance(item, dict) and str(item.get("node_code") or "").strip()
]
def _projection_cluster_node_rows(*, cluster: dict) -> list[dict]:
normalized_rows: list[dict] = []
for raw_item in list(cluster.get("nodes") or []):
if not isinstance(raw_item, dict):
continue
node_code = str(raw_item.get("node_code") or "").strip()
if not _is_local_projection_node(node_code):
continue
metadata = dict(raw_item.get("metadata") or {})
normalized_rows.append(
{
"node_code": node_code,
"role": str(raw_item.get("role") or metadata.get("source_role") or "").strip(),
"status": str(raw_item.get("status") or metadata.get("source_status") or "").strip(),
"current_load": int(raw_item.get("current_load", 0) or 0),
"active_threads": int(metadata.get("active_threads", 0) or 0),
"max_threads": int(metadata.get("max_threads", raw_item.get("max_threads", 0)) or 0),
"detect_participating": bool(
raw_item.get("detect_participating", metadata.get("detect_participating", False))
),
}
)
normalized_rows.sort(key=lambda item: str(item.get("node_code") or ""))
return normalized_rows
def _projection_display_summary(*, detect: dict, active_job: dict, node_rows: list[dict]) -> dict:
queue_payload = dict((detect.get("queue_health") or {}).get("queue") or {})
display_running = 0
display_max_threads = 0
display_claimed = 0
running_items = 0
for raw_item in list(node_rows or []):
item = dict(raw_item or {})
running_items += int(item.get("items_running", 0) or 0)
display_running += max(
int(item.get("display_running", 0) or 0),
int(item.get("current_load", 0) or 0),
int(item.get("active_threads", 0) or 0),
int(item.get("items_running", 0) or 0),
)
display_max_threads += max(0, int(item.get("max_threads", 0) or 0))
display_claimed += max(
int(item.get("items_claimed", 0) or 0),
int(item.get("display_claimed", 0) or 0),
)
display_running = max(
display_running,
int(queue_payload.get("display_running", queue_payload.get("running", 0)) or 0),
int(active_job.get("display_active_threads", active_job.get("display_items_running", active_job.get("items_running", 0))) or 0),
)
running_items = max(
running_items,
int(queue_payload.get("running", 0) or 0),
int(active_job.get("items_running", 0) or 0),
)
display_claimed = max(
display_claimed,
int(queue_payload.get("display_claimed", queue_payload.get("claimed", 0)) or 0),
int(active_job.get("display_items_claimed", active_job.get("items_claimed", 0)) or 0),
)
display_max_threads = max(
display_max_threads,
int(active_job.get("display_max_threads", 0) or 0),
int(detect.get("aggregate_max_thread_count", 0) or 0),
int(detect.get("max_thread_count", 0) or 0),
)
return {
"items_running": running_items,
"display_running": display_running,
"display_claimed": display_claimed,
"display_max_threads": display_max_threads,
}
def _is_local_projection_node(node_code: str) -> bool:
normalized_node_code = str(node_code or "").strip()
local_node_code = str(settings.node_code or "").strip()
if not normalized_node_code or not local_node_code:
return False
return normalized_node_code == local_node_code or normalized_node_code.startswith(f"{local_node_code}-")
def _build_runtime_projection_payload(
*,
detect: dict,
@@ -549,6 +758,19 @@ def _build_runtime_projection_payload(
) -> dict:
active_job = detect.get("active_job") or {}
local_worker_expected = _local_node_expected_to_execute_worker()
projection_node_rows = _projection_node_rows(detect=detect, active_job=active_job) if local_worker_expected else []
projection_cluster_rows = _projection_cluster_node_rows(cluster=cluster) if local_worker_expected else []
display_summary = (
_projection_display_summary(detect=detect, active_job=active_job, node_rows=projection_node_rows)
if local_worker_expected
else {
"items_running": 0,
"display_running": 0,
"display_claimed": 0,
"display_max_threads": 0,
}
)
queue_payload = dict((detect.get("queue_health") or {}).get("queue") or {}) if local_worker_expected else {}
local_participating = False
for node in list(cluster.get("nodes") or []):
if str(node.get("node_code") or "").strip() != settings.node_code:
@@ -558,15 +780,22 @@ def _build_runtime_projection_payload(
break
local_job_bucket = {}
if local_worker_expected:
for item in list(active_job.get("node_stats") or []):
if str(item.get("node_code") or "").strip() != settings.node_code:
for item in projection_node_rows:
if not _is_local_projection_node(str(item.get("node_code") or "").strip()):
continue
local_job_bucket = item
local_participating = local_participating or bool(
int(item.get("display_running", 0) or 0) > 0
or int(item.get("active_threads", 0) or 0) > 0
or int(item.get("items_running", 0) or 0) > 0
or int(item.get("items_claimed", 0) or 0) > 0
)
break
if local_worker_expected and not local_participating:
local_participating = bool(
int(local_job_bucket.get("items_running", 0) or 0) > 0
or int(local_job_bucket.get("items_claimed", 0) or 0) > 0
or int(display_summary.get("display_running", 0) or 0) > 0
)
projection_active_job = (
@@ -575,12 +804,18 @@ def _build_runtime_projection_payload(
"job_code": active_job.get("job_code", ""),
"status": active_job.get("status", ""),
"progress_percent": active_job.get("progress_percent", 0),
"items_total": active_job.get("items_total", 0),
"items_terminal": active_job.get("items_terminal", 0),
"items_pending": active_job.get("items_pending", 0),
"items_running": active_job.get("items_running", 0),
"items_failed": active_job.get("items_failed", 0),
"node_stats": list(active_job.get("node_stats") or []),
"items_total": int(queue_payload.get("items_total", active_job.get("items_total", 0)) or 0),
"items_terminal": int(queue_payload.get("terminal", active_job.get("items_terminal", 0)) or 0),
"items_pending": int(queue_payload.get("pending", active_job.get("items_pending", 0)) or 0),
"items_claimed": int(queue_payload.get("claimed", active_job.get("items_claimed", 0)) or 0),
"items_running": int(display_summary.get("items_running", 0) or 0),
"items_failed": int(queue_payload.get("failed", active_job.get("items_failed", 0)) or 0),
"display_items_claimed": int(display_summary.get("display_claimed", 0) or 0),
"display_items_running": int(display_summary.get("display_running", 0) or 0),
"display_active_threads": int(display_summary.get("display_running", 0) or 0),
"display_max_threads": int(display_summary.get("display_max_threads", 0) or 0),
"node_stats": list(projection_node_rows),
"distributed_node_stats": list(projection_node_rows),
}
if local_worker_expected
else {
@@ -591,18 +826,24 @@ def _build_runtime_projection_payload(
"items_total": 0,
"items_terminal": 0,
"items_pending": 0,
"items_claimed": 0,
"items_running": 0,
"items_failed": 0,
"display_items_claimed": 0,
"display_items_running": 0,
"display_active_threads": 0,
"display_max_threads": 0,
"node_stats": [],
"distributed_node_stats": [],
}
)
progress_payload = (
{
"pending": int((detect.get("progress") or {}).get("pending", 0) or 0),
"running": int((detect.get("progress") or {}).get("running", 0) or 0),
"completed": int((detect.get("progress") or {}).get("completed", 0) or 0),
"blacklisted": int((detect.get("progress") or {}).get("blacklisted", 0) or 0),
"failed": int((detect.get("progress") or {}).get("failed", 0) or 0),
"pending": int(queue_payload.get("pending", (detect.get("progress") or {}).get("pending", 0)) or 0),
"running": int(display_summary.get("display_running", 0) or 0),
"completed": int(queue_payload.get("completed", (detect.get("progress") or {}).get("completed", 0)) or 0),
"blacklisted": int(queue_payload.get("blacklisted", (detect.get("progress") or {}).get("blacklisted", 0)) or 0),
"failed": int(queue_payload.get("failed", (detect.get("progress") or {}).get("failed", 0)) or 0),
}
if local_worker_expected
else {
@@ -624,8 +865,8 @@ def _build_runtime_projection_payload(
"worker_online": bool(detect.get("worker_online", False)) if local_worker_expected else False,
"detect_participating": local_participating if local_worker_expected else False,
"worker_mode": detect.get("worker_mode", ""),
"active_thread_count": int(detect.get("active_thread_count", 0) or 0) if local_worker_expected else 0,
"max_thread_count": int(detect.get("max_thread_count", 0) or 0) if local_worker_expected else 0,
"active_thread_count": int(display_summary.get("display_running", 0) or 0) if local_worker_expected else 0,
"max_thread_count": int(display_summary.get("display_max_threads", 0) or 0) if local_worker_expected else 0,
"phase_label": detect.get("phase_label", ""),
"phase_detail": detect.get("phase_detail", ""),
"proxy_runtime_label": detect.get("proxy_runtime_label", ""),
@@ -633,6 +874,7 @@ def _build_runtime_projection_payload(
"progress": progress_payload,
"backlog": dict(detect.get("backlog") or {}) if local_worker_expected else {},
"active_job": projection_active_job,
"cluster_nodes": list(projection_cluster_rows),
"cluster_summary": {
"nodes_total": int(cluster.get("nodes_total", 0) or 0),
"online_worker_nodes": int((cluster.get("summary") or {}).get("online_worker_nodes", 0) or 0),
@@ -670,6 +912,7 @@ def append_runtime_projection_if_changed(
) -> int | None:
normalized_source_region = _normalize_region(source_region, _normalize_region(settings.sync_source_region, settings.node_region))
normalized_target_region = _normalize_region(target_region, _normalize_region(settings.sync_target_region, "overseas"))
future_cutoff = datetime.now() + _RUNTIME_PROJECTION_FUTURE_SKEW_GRACE
payload = _build_runtime_projection_payload(
detect=detect,
cluster=cluster,
@@ -686,16 +929,17 @@ def append_runtime_projection_if_changed(
WHERE sync_type = 'runtime_projection'
AND source_region = %s
AND target_region = %s
ORDER BY created_at DESC, id DESC
LIMIT 1
ORDER BY
CASE WHEN created_at <= %s THEN 0 ELSE 1 END ASC,
created_at DESC,
id DESC
LIMIT 200
""",
(normalized_source_region, normalized_target_region),
(normalized_source_region, normalized_target_region, future_cutoff),
)
latest = cur.fetchone()
latest = _pick_latest_projection_row(list(cur.fetchall() or []), created_at_index=1)
latest_payload = _decode_json(latest[0]) if latest else {}
latest_created_at = latest[1] if latest else None
if latest_payload.get("projection_hash") == payload["projection_hash"]:
return None
if not _should_append_runtime_projection(latest_payload, payload["projection"], latest_created_at):
return None
cur.execute(