1080 lines
45 KiB
Python
1080 lines
45 KiB
Python
from __future__ import annotations
|
|
|
|
import hashlib
|
|
import json
|
|
import socket
|
|
from datetime import datetime, timedelta
|
|
|
|
from app.core.config import settings
|
|
from app.core.db import db_read_retry, get_db
|
|
|
|
|
|
def _format_time(value: datetime | None) -> str:
|
|
return value.isoformat(sep=" ", timespec="seconds") if value else ""
|
|
|
|
|
|
def _resolve_local_ip() -> str:
|
|
try:
|
|
return socket.gethostbyname(socket.gethostname())
|
|
except Exception:
|
|
return ""
|
|
|
|
|
|
def _decode_json(value: object) -> dict:
|
|
if isinstance(value, dict):
|
|
return value
|
|
if value in (None, ""):
|
|
return {}
|
|
try:
|
|
return json.loads(value)
|
|
except Exception:
|
|
return {}
|
|
|
|
|
|
def _normalize_region(value: str | None, fallback: str) -> str:
|
|
text = str(value or "").strip()
|
|
if not text or text == "unknown":
|
|
return str(fallback or "unknown").strip() or "unknown"
|
|
return text
|
|
|
|
|
|
_DETECT_RESULT_EVENT_TYPES = {
|
|
"domain_started",
|
|
"domain_completed",
|
|
"domain_failed",
|
|
"domain_blacklisted",
|
|
}
|
|
|
|
_TERMINAL_DETECT_RESULT_EVENT_TYPES = {
|
|
"domain_completed",
|
|
"domain_failed",
|
|
"domain_blacklisted",
|
|
}
|
|
|
|
_RUNTIME_PROJECTION_HEARTBEAT_INTERVAL = timedelta(seconds=45)
|
|
_RUNTIME_PROJECTION_FUTURE_SKEW_GRACE = timedelta(minutes=5)
|
|
|
|
|
|
def _runtime_projection_activity_signature(projection: dict) -> dict:
|
|
normalized_projection = dict(projection or {})
|
|
active_job = dict(normalized_projection.get("active_job") or {})
|
|
normalized_nodes: list[tuple] = []
|
|
for raw_item in list(active_job.get("distributed_node_stats") or active_job.get("node_stats") or []):
|
|
if not isinstance(raw_item, dict):
|
|
continue
|
|
node_code = str(raw_item.get("node_code") or "").strip()
|
|
if not node_code:
|
|
continue
|
|
normalized_nodes.append(
|
|
(
|
|
node_code,
|
|
int(raw_item.get("display_running", raw_item.get("current_load", 0)) or 0),
|
|
int(raw_item.get("active_threads", 0) or 0),
|
|
int(raw_item.get("max_threads", 0) or 0),
|
|
int(raw_item.get("items_claimed", 0) or 0),
|
|
int(raw_item.get("items_running", 0) or 0),
|
|
int(raw_item.get("items_total", 0) or 0),
|
|
str(raw_item.get("status") or "").strip(),
|
|
)
|
|
)
|
|
normalized_cluster_nodes: list[tuple] = []
|
|
for raw_item in list(normalized_projection.get("cluster_nodes") or []):
|
|
if not isinstance(raw_item, dict):
|
|
continue
|
|
node_code = str(raw_item.get("node_code") or "").strip()
|
|
if not node_code:
|
|
continue
|
|
normalized_cluster_nodes.append(
|
|
(
|
|
node_code,
|
|
str(raw_item.get("role") or "").strip(),
|
|
str(raw_item.get("status") or "").strip(),
|
|
int(raw_item.get("current_load", 0) or 0),
|
|
int(raw_item.get("active_threads", 0) or 0),
|
|
int(raw_item.get("max_threads", 0) or 0),
|
|
bool(raw_item.get("detect_participating", False)),
|
|
)
|
|
)
|
|
return {
|
|
"active_thread_count": int(normalized_projection.get("active_thread_count", 0) or 0),
|
|
"max_thread_count": int(normalized_projection.get("max_thread_count", 0) or 0),
|
|
"job_display_running": int(active_job.get("display_items_running", 0) or 0),
|
|
"job_display_claimed": int(active_job.get("display_items_claimed", 0) or 0),
|
|
"job_display_max_threads": int(active_job.get("display_max_threads", 0) or 0),
|
|
"job_items_total": int(active_job.get("items_total", 0) or 0),
|
|
"job_items_running": int(active_job.get("items_running", 0) or 0),
|
|
"job_items_claimed": int(active_job.get("items_claimed", 0) or 0),
|
|
"node_stats": normalized_nodes,
|
|
"cluster_nodes": normalized_cluster_nodes,
|
|
}
|
|
|
|
|
|
def _pick_latest_projection_row(rows: list[tuple], *, created_at_index: int) -> tuple | None:
|
|
candidates = list(rows or [])
|
|
if not candidates:
|
|
return None
|
|
fallback = candidates[0]
|
|
for row in candidates:
|
|
if len(row) <= int(created_at_index):
|
|
return row
|
|
created_at = row[created_at_index]
|
|
if not isinstance(created_at, datetime):
|
|
return row
|
|
now = datetime.now(created_at.tzinfo) if created_at.tzinfo else datetime.now()
|
|
if created_at <= now + _RUNTIME_PROJECTION_FUTURE_SKEW_GRACE:
|
|
return row
|
|
return fallback
|
|
|
|
|
|
def _collect_recent_domain_events(active_job: dict, limit: int = 30) -> list[dict]:
|
|
safe_limit = max(1, int(limit or 30))
|
|
seen: set[tuple[str, str, str, str]] = set()
|
|
normalized: list[dict] = []
|
|
|
|
def _append_event(raw_event: dict) -> None:
|
|
event_type = str(raw_event.get("event_type") or "").strip()
|
|
if event_type not in _DETECT_RESULT_EVENT_TYPES:
|
|
return
|
|
normalized_event = {
|
|
"node_code": str(raw_event.get("node_code") or "").strip(),
|
|
"event_type": event_type,
|
|
"level": str(raw_event.get("level") or "info").strip() or "info",
|
|
"message": str(raw_event.get("message") or "").strip(),
|
|
"created_at": str(raw_event.get("created_at") or "").strip(),
|
|
"payload": _decode_json(raw_event.get("payload")),
|
|
}
|
|
event_key = (
|
|
normalized_event["node_code"],
|
|
normalized_event["event_type"],
|
|
normalized_event["message"],
|
|
normalized_event["created_at"],
|
|
)
|
|
if event_key in seen:
|
|
return
|
|
seen.add(event_key)
|
|
normalized.append(normalized_event)
|
|
|
|
# Keep a small slice of the current-cycle `domain_started` events so the
|
|
# remote log / live activity view still reflects the node's latest work.
|
|
for event in reversed(list(active_job.get("current_cycle_events") or active_job.get("recent_events") or [])):
|
|
if str(event.get("event_type") or "").strip() != "domain_started":
|
|
continue
|
|
_append_event(event)
|
|
if len(normalized) >= min(10, max(1, safe_limit // 3)):
|
|
break
|
|
|
|
# Always pull the most recent terminal result events from the full job
|
|
# history. Otherwise a flood of newer `domain_started` events can hide
|
|
# terminal completions, and overseas will never advance completed counts.
|
|
job_id = int(active_job.get("job_id") or 0)
|
|
if job_id > 0:
|
|
with get_db() as conn:
|
|
with conn.cursor() as cur:
|
|
cur.execute(
|
|
"""
|
|
SELECT node_code, event_type, level, message, payload_json, created_at
|
|
FROM detect_run_events
|
|
WHERE job_id = %s
|
|
AND event_type IN ('domain_completed', 'domain_failed', 'domain_blacklisted')
|
|
ORDER BY created_at DESC, id DESC
|
|
LIMIT %s
|
|
""",
|
|
(job_id, max(safe_limit * 4, 60)),
|
|
)
|
|
rows = cur.fetchall()
|
|
for row in reversed(rows):
|
|
_append_event(
|
|
{
|
|
"node_code": row[0] or "",
|
|
"event_type": row[1] or "",
|
|
"level": row[2] or "info",
|
|
"message": row[3] or "",
|
|
"payload": _decode_json(row[4]),
|
|
"created_at": _format_time(row[5]),
|
|
}
|
|
)
|
|
else:
|
|
for event in reversed(list(active_job.get("current_cycle_events") or active_job.get("recent_events") or [])):
|
|
if str(event.get("event_type") or "").strip() in _TERMINAL_DETECT_RESULT_EVENT_TYPES:
|
|
_append_event(event)
|
|
|
|
normalized.sort(
|
|
key=lambda item: (
|
|
str(item.get("created_at") or ""),
|
|
str(item.get("node_code") or ""),
|
|
str(item.get("event_type") or ""),
|
|
str(item.get("message") or ""),
|
|
)
|
|
)
|
|
return normalized[-safe_limit:]
|
|
|
|
|
|
def _build_detect_result_batch_digest(batch: dict | None) -> dict:
|
|
batch = batch or {}
|
|
projection = (batch.get("projection") or {}).get("payload") or {}
|
|
latest_push = batch.get("latest_push") or {}
|
|
latest_ingest = batch.get("latest_ingest") or {}
|
|
job = (projection.get("projection") or {}).get("job") or {}
|
|
latest_event = (projection.get("projection") or {}).get("latest_event") or {}
|
|
return {
|
|
"job_id": int(batch.get("job_id") or 0),
|
|
"job_code": str(batch.get("job_code") or ""),
|
|
"job_status": str(batch.get("job_status") or ""),
|
|
"items_total": int(batch.get("items_total") or 0),
|
|
"items_terminal": int(batch.get("items_terminal") or 0),
|
|
"items_pending": int(batch.get("items_pending") or 0),
|
|
"items_running": int(batch.get("items_running") or 0),
|
|
"items_failed": int(batch.get("items_failed") or 0),
|
|
"progress_percent": float(batch.get("progress_percent") or 0),
|
|
"sync_state": str(batch.get("sync_state") or "unsynced"),
|
|
"sync_message": str(batch.get("sync_message") or ""),
|
|
"projection_record_id": int((batch.get("projection") or {}).get("id") or 0),
|
|
"projection_status": str((batch.get("projection") or {}).get("status") or ""),
|
|
"projection_created_at": str((batch.get("projection") or {}).get("created_at") or ""),
|
|
"latest_event_type": str(latest_event.get("event_type") or ""),
|
|
"latest_event_message": str(latest_event.get("message") or ""),
|
|
"latest_event_created_at": str(latest_event.get("created_at") or ""),
|
|
"latest_push_status": str(latest_push.get("status") or ""),
|
|
"latest_push_error": str(latest_push.get("error_message") or ""),
|
|
"latest_push_created_at": str(latest_push.get("created_at") or ""),
|
|
"latest_ingest_status": str(latest_ingest.get("status") or ""),
|
|
"latest_ingest_created_at": str(latest_ingest.get("created_at") or ""),
|
|
"cycle_token": str(job.get("current_cycle_token") or ""),
|
|
}
|
|
|
|
|
|
def _should_append_runtime_projection(previous_payload: dict, current_projection: dict, previous_created_at: datetime | None) -> bool:
|
|
if not previous_payload:
|
|
return True
|
|
previous_projection = previous_payload.get("projection") or {}
|
|
if not previous_projection:
|
|
return True
|
|
|
|
keys_requiring_immediate_write = (
|
|
"worker_online",
|
|
"worker_mode",
|
|
"phase_label",
|
|
"phase_detail",
|
|
"proxy_runtime_label",
|
|
"proxy_runtime_reason",
|
|
)
|
|
for key in keys_requiring_immediate_write:
|
|
if previous_projection.get(key) != current_projection.get(key):
|
|
return True
|
|
|
|
previous_job = previous_projection.get("active_job") or {}
|
|
current_job = current_projection.get("active_job") or {}
|
|
for key in ("job_id", "job_code", "status"):
|
|
if previous_job.get(key) != current_job.get(key):
|
|
return True
|
|
|
|
previous_cluster = previous_projection.get("cluster_summary") or {}
|
|
current_cluster = current_projection.get("cluster_summary") or {}
|
|
for key in ("online_worker_nodes", "dedicated_online_worker_nodes", "online_control_nodes", "busy_nodes", "stale_nodes", "offline_nodes"):
|
|
if previous_cluster.get(key) != current_cluster.get(key):
|
|
return True
|
|
|
|
previous_progress = previous_projection.get("progress") or {}
|
|
current_progress = current_projection.get("progress") or {}
|
|
failed_delta = abs(int(current_progress.get("failed", 0) or 0) - int(previous_progress.get("failed", 0) or 0))
|
|
blacklisted_delta = abs(int(current_progress.get("blacklisted", 0) or 0) - int(previous_progress.get("blacklisted", 0) or 0))
|
|
completed_delta = abs(int(current_progress.get("completed", 0) or 0) - int(previous_progress.get("completed", 0) or 0))
|
|
running_delta = abs(int(current_progress.get("running", 0) or 0) - int(previous_progress.get("running", 0) or 0))
|
|
if failed_delta > 0 or blacklisted_delta >= 10 or completed_delta >= 20 or running_delta >= 5:
|
|
return True
|
|
|
|
previous_alerts = previous_projection.get("dependency_alerts") or []
|
|
current_alerts = current_projection.get("dependency_alerts") or []
|
|
if previous_alerts != current_alerts:
|
|
return True
|
|
|
|
if _runtime_projection_activity_signature(previous_projection) != _runtime_projection_activity_signature(current_projection):
|
|
return True
|
|
|
|
if not previous_created_at:
|
|
return True
|
|
now = datetime.now(previous_created_at.tzinfo) if previous_created_at.tzinfo else datetime.now()
|
|
if previous_created_at > now + _RUNTIME_PROJECTION_FUTURE_SKEW_GRACE:
|
|
return True
|
|
return now - previous_created_at >= _RUNTIME_PROJECTION_HEARTBEAT_INTERVAL
|
|
|
|
|
|
@db_read_retry()
|
|
def list_sync_records(limit: int = 20) -> list[dict]:
|
|
safe_limit = max(1, min(int(limit or 20), 200))
|
|
with get_db() as conn:
|
|
with conn.cursor() as cur:
|
|
cur.execute(
|
|
"""
|
|
SELECT id, sync_type, source_region, target_region, status, payload_json, error_message, created_at, updated_at
|
|
FROM detect_sync_records
|
|
ORDER BY created_at DESC, id DESC
|
|
LIMIT %s
|
|
""",
|
|
(safe_limit,),
|
|
)
|
|
rows = cur.fetchall()
|
|
return [
|
|
{
|
|
"id": row[0],
|
|
"sync_type": row[1],
|
|
"source_region": row[2],
|
|
"target_region": row[3],
|
|
"status": row[4],
|
|
"payload": _decode_json(row[5]),
|
|
"error_message": row[6] or "",
|
|
"created_at": _format_time(row[7]),
|
|
"updated_at": _format_time(row[8]),
|
|
}
|
|
for row in rows
|
|
]
|
|
|
|
|
|
def _latest_sync_record_by_source(
|
|
cur,
|
|
*,
|
|
sync_type: str,
|
|
source_region: str,
|
|
target_region: str,
|
|
source_record_id: int,
|
|
) -> dict | None:
|
|
cur.execute(
|
|
"""
|
|
SELECT id, sync_type, source_region, target_region, status, payload_json, error_message, created_at, updated_at
|
|
FROM detect_sync_records
|
|
WHERE sync_type = %s
|
|
AND source_region = %s
|
|
AND target_region = %s
|
|
AND (payload_json->>'source_record_id') = %s
|
|
ORDER BY created_at DESC, id DESC
|
|
LIMIT 1
|
|
""",
|
|
(sync_type, source_region, target_region, str(int(source_record_id))),
|
|
)
|
|
row = cur.fetchone()
|
|
if not row:
|
|
return None
|
|
return {
|
|
"id": row[0],
|
|
"sync_type": row[1],
|
|
"source_region": row[2],
|
|
"target_region": row[3],
|
|
"status": row[4],
|
|
"payload": _decode_json(row[5]),
|
|
"error_message": row[6] or "",
|
|
"created_at": _format_time(row[7]),
|
|
"updated_at": _format_time(row[8]),
|
|
}
|
|
|
|
|
|
@db_read_retry()
|
|
def get_detect_result_sync_batches(limit: int = 5) -> dict:
|
|
safe_limit = max(1, min(int(limit or 5), 20))
|
|
source_region = _normalize_region(settings.sync_source_region, settings.node_region)
|
|
target_region = _normalize_region(settings.sync_target_region, "overseas")
|
|
local_worker_expected = _local_node_expected_to_execute_worker()
|
|
if not local_worker_expected:
|
|
return {
|
|
"applicable": False,
|
|
"local_worker_expected": False,
|
|
"reason": "当前节点不承载本地检测执行，结果批次推送概览不适用。",
|
|
"source_region": source_region,
|
|
"target_region": target_region,
|
|
"jobs_total": 0,
|
|
"state_counts": {
|
|
"synced": 0,
|
|
"delivered": 0,
|
|
"pushing": 0,
|
|
"projected": 0,
|
|
"failed": 0,
|
|
"unsynced": 0,
|
|
},
|
|
"batches": [],
|
|
}
|
|
batches: list[dict] = []
|
|
|
|
with get_db() as conn:
|
|
with conn.cursor() as cur:
|
|
cur.execute(
|
|
"""
|
|
SELECT id, job_code, status, created_at, started_at, finished_at
|
|
FROM detect_jobs
|
|
ORDER BY created_at DESC, id DESC
|
|
LIMIT %s
|
|
""",
|
|
(safe_limit,),
|
|
)
|
|
jobs = cur.fetchall()
|
|
|
|
for row in jobs:
|
|
job_id = int(row[0])
|
|
job_code = str(row[1] or "")
|
|
job_status = str(row[2] or "")
|
|
created_at = _format_time(row[3])
|
|
started_at = _format_time(row[4])
|
|
finished_at = _format_time(row[5])
|
|
|
|
cur.execute(
|
|
"""
|
|
SELECT status, count(*)
|
|
FROM detect_job_items
|
|
WHERE job_id = %s
|
|
GROUP BY status
|
|
""",
|
|
(job_id,),
|
|
)
|
|
item_counts = {str(status or ""): int(count) for status, count in cur.fetchall()}
|
|
items_total = sum(item_counts.values())
|
|
items_terminal = int(item_counts.get("completed", 0)) + int(item_counts.get("blacklisted", 0)) + int(item_counts.get("failed", 0))
|
|
|
|
cur.execute(
|
|
"""
|
|
SELECT id, status, payload_json, created_at, updated_at
|
|
FROM detect_sync_records
|
|
WHERE sync_type = 'detect_result_projection'
|
|
AND source_region = %s
|
|
AND target_region = %s
|
|
AND (payload_json->'projection'->'job'->>'job_id') = %s
|
|
ORDER BY created_at DESC, id DESC
|
|
LIMIT 1
|
|
""",
|
|
(source_region, target_region, str(job_id)),
|
|
)
|
|
projection_row = cur.fetchone()
|
|
projection = None
|
|
latest_push = None
|
|
latest_ingest = None
|
|
sync_state = "unsynced"
|
|
sync_message = "该任务还没有生成结果投影"
|
|
|
|
if projection_row:
|
|
projection_payload = _decode_json(projection_row[2])
|
|
projection = {
|
|
"id": int(projection_row[0]),
|
|
"status": projection_row[1],
|
|
"payload": projection_payload,
|
|
"created_at": _format_time(projection_row[3]),
|
|
"updated_at": _format_time(projection_row[4]),
|
|
}
|
|
latest_push = _latest_sync_record_by_source(
|
|
cur,
|
|
sync_type="runtime_push",
|
|
source_region=source_region,
|
|
target_region=target_region,
|
|
source_record_id=projection["id"],
|
|
)
|
|
latest_ingest = _latest_sync_record_by_source(
|
|
cur,
|
|
sync_type="detect_result_ingest",
|
|
source_region=source_region,
|
|
target_region=target_region,
|
|
source_record_id=projection["id"],
|
|
)
|
|
|
|
if latest_ingest:
|
|
sync_state = "synced"
|
|
sync_message = "最近一条结果投影已被目标地域接收"
|
|
elif latest_push and latest_push.get("status") == "success":
|
|
sync_state = "delivered"
|
|
sync_message = "结果投影已推送成功，等待目标侧回看接收记录"
|
|
elif latest_push and latest_push.get("status") == "pending":
|
|
sync_state = "pushing"
|
|
sync_message = "结果投影正在推送中"
|
|
elif latest_push and latest_push.get("status") == "failed":
|
|
sync_state = "failed"
|
|
sync_message = latest_push.get("error_message") or "最近一次结果投影推送失败"
|
|
else:
|
|
sync_state = "projected"
|
|
sync_message = "已生成结果投影，等待同步代理推送"
|
|
|
|
batches.append(
|
|
{
|
|
"job_id": job_id,
|
|
"job_code": job_code,
|
|
"job_status": job_status,
|
|
"created_at": created_at,
|
|
"started_at": started_at,
|
|
"finished_at": finished_at,
|
|
"items_total": items_total,
|
|
"items_terminal": items_terminal,
|
|
"items_pending": int(item_counts.get("pending", 0)),
|
|
"items_running": int(item_counts.get("running", 0)) + int(item_counts.get("claimed", 0)),
|
|
"items_failed": int(item_counts.get("failed", 0)),
|
|
"progress_percent": round((items_terminal / items_total) * 100, 2) if items_total else 0,
|
|
"sync_state": sync_state,
|
|
"sync_message": sync_message,
|
|
"projection": projection,
|
|
"latest_push": latest_push,
|
|
"latest_ingest": latest_ingest,
|
|
}
|
|
)
|
|
|
|
state_counts = {
|
|
"synced": 0,
|
|
"delivered": 0,
|
|
"pushing": 0,
|
|
"projected": 0,
|
|
"failed": 0,
|
|
"unsynced": 0,
|
|
}
|
|
for item in batches:
|
|
state = str(item.get("sync_state") or "unsynced")
|
|
state_counts[state] = state_counts.get(state, 0) + 1
|
|
|
|
return {
|
|
"applicable": True,
|
|
"local_worker_expected": local_worker_expected,
|
|
"reason": "",
|
|
"source_region": source_region,
|
|
"target_region": target_region,
|
|
"jobs_total": len(batches),
|
|
"state_counts": state_counts,
|
|
"batches": batches,
|
|
}
|
|
|
|
|
|
@db_read_retry()
|
|
def get_sync_summary(record_limit: int = 10) -> dict:
|
|
source_region = _normalize_region(settings.sync_source_region, settings.node_region)
|
|
target_region = _normalize_region(settings.sync_target_region, "overseas")
|
|
local_worker_expected = _local_node_expected_to_execute_worker()
|
|
push_expected_on_this_node = bool(
|
|
str(settings.node_region or "").strip() == "mainland"
|
|
and str(settings.node_role or "").strip() == "control"
|
|
)
|
|
with get_db() as conn:
|
|
with conn.cursor() as cur:
|
|
cur.execute(
|
|
"""
|
|
SELECT status, count(*)
|
|
FROM detect_sync_records
|
|
GROUP BY status
|
|
"""
|
|
)
|
|
status_counts = {str(status or "unknown"): int(count) for status, count in cur.fetchall()}
|
|
cur.execute(
|
|
"""
|
|
SELECT sync_type, count(*)
|
|
FROM detect_sync_records
|
|
GROUP BY sync_type
|
|
"""
|
|
)
|
|
type_counts = {str(sync_type or "unknown"): int(count) for sync_type, count in cur.fetchall()}
|
|
cur.execute("SELECT count(*) FROM detect_sync_records")
|
|
total = int(cur.fetchone()[0] or 0)
|
|
cur.execute(
|
|
"""
|
|
SELECT id, sync_type, source_region, target_region, status, payload_json, error_message, created_at, updated_at
|
|
FROM detect_sync_records
|
|
ORDER BY created_at DESC, id DESC
|
|
LIMIT 1
|
|
"""
|
|
)
|
|
latest = cur.fetchone()
|
|
|
|
latest_record = None
|
|
if latest:
|
|
latest_record = {
|
|
"id": latest[0],
|
|
"sync_type": latest[1],
|
|
"source_region": latest[2],
|
|
"target_region": latest[3],
|
|
"status": latest[4],
|
|
"payload": _decode_json(latest[5]),
|
|
"error_message": latest[6] or "",
|
|
"created_at": _format_time(latest[7]),
|
|
"updated_at": _format_time(latest[8]),
|
|
}
|
|
|
|
detect_result_batches = get_detect_result_sync_batches(limit=min(5, record_limit))
|
|
recent_result_batches = list(detect_result_batches.get("batches") or [])
|
|
latest_result_batch = recent_result_batches[0] if recent_result_batches else None
|
|
|
|
return {
|
|
"enabled": bool(settings.sync_push_enabled),
|
|
"push_expected_on_this_node": push_expected_on_this_node,
|
|
"local_worker_expected_on_this_node": local_worker_expected,
|
|
"source_region": source_region,
|
|
"target_region": target_region,
|
|
"target_api_base_url": settings.sync_target_api_base_url,
|
|
"batch_size": max(1, int(settings.sync_batch_size or 200)),
|
|
"poll_interval_seconds": max(5, int(settings.sync_poll_interval_seconds or 30)),
|
|
"records_total": total,
|
|
"status_counts": status_counts,
|
|
"type_counts": type_counts,
|
|
"latest_record": latest_record,
|
|
"detect_result_batches": detect_result_batches,
|
|
"latest_detect_result_batch": _build_detect_result_batch_digest(latest_result_batch),
|
|
"detect_result_batch_digests": [_build_detect_result_batch_digest(item) for item in recent_result_batches],
|
|
"recent_records": list_sync_records(limit=record_limit),
|
|
}
|
|
|
|
|
|
def append_sync_record(
|
|
*,
|
|
sync_type: str,
|
|
source_region: str,
|
|
target_region: str,
|
|
status: str,
|
|
payload: dict | None = None,
|
|
error_message: str = "",
|
|
) -> int:
|
|
with get_db() as conn:
|
|
with conn.cursor() as cur:
|
|
cur.execute(
|
|
"""
|
|
INSERT INTO detect_sync_records (
|
|
sync_type, source_region, target_region, status, payload_json, error_message, created_at, updated_at
|
|
) VALUES (%s, %s, %s, %s, %s::jsonb, %s, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)
|
|
RETURNING id
|
|
""",
|
|
(
|
|
str(sync_type or "").strip() or "unknown",
|
|
_normalize_region(source_region, _normalize_region(settings.sync_source_region, settings.node_region)),
|
|
_normalize_region(target_region, _normalize_region(settings.sync_target_region, "overseas")),
|
|
str(status or "").strip() or "pending",
|
|
json.dumps(payload or {}, ensure_ascii=False),
|
|
str(error_message or "").strip(),
|
|
),
|
|
)
|
|
record_id = int(cur.fetchone()[0])
|
|
conn.commit()
|
|
return record_id
|
|
|
|
|
|
def _local_node_expected_to_execute_worker() -> bool:
|
|
node_role = str(settings.node_role or "").strip()
|
|
node_region = str(settings.node_region or "").strip()
|
|
return node_role == "worker" or (node_region == "mainland" and node_role == "control")
|
|
|
|
|
|
def _projection_node_rows(*, detect: dict, active_job: dict) -> list[dict]:
|
|
queue_health = dict(detect.get("queue_health") or {})
|
|
queue_nodes = [
|
|
dict(item)
|
|
for item in list(queue_health.get("nodes") or [])
|
|
if isinstance(item, dict) and str(item.get("node_code") or "").strip()
|
|
]
|
|
if queue_nodes:
|
|
return queue_nodes
|
|
return [
|
|
dict(item)
|
|
for item in list(active_job.get("distributed_node_stats") or active_job.get("node_stats") or [])
|
|
if isinstance(item, dict) and str(item.get("node_code") or "").strip()
|
|
]
|
|
|
|
|
|
def _projection_cluster_node_rows(*, cluster: dict) -> list[dict]:
|
|
normalized_rows: list[dict] = []
|
|
for raw_item in list(cluster.get("nodes") or []):
|
|
if not isinstance(raw_item, dict):
|
|
continue
|
|
node_code = str(raw_item.get("node_code") or "").strip()
|
|
if not _is_local_projection_node(node_code):
|
|
continue
|
|
metadata = dict(raw_item.get("metadata") or {})
|
|
normalized_rows.append(
|
|
{
|
|
"node_code": node_code,
|
|
"role": str(raw_item.get("role") or metadata.get("source_role") or "").strip(),
|
|
"status": str(raw_item.get("status") or metadata.get("source_status") or "").strip(),
|
|
"current_load": int(raw_item.get("current_load", 0) or 0),
|
|
"active_threads": int(metadata.get("active_threads", 0) or 0),
|
|
"max_threads": int(metadata.get("max_threads", raw_item.get("max_threads", 0)) or 0),
|
|
"detect_participating": bool(
|
|
raw_item.get("detect_participating", metadata.get("detect_participating", False))
|
|
),
|
|
}
|
|
)
|
|
normalized_rows.sort(key=lambda item: str(item.get("node_code") or ""))
|
|
return normalized_rows
|
|
|
|
|
|
def _projection_display_summary(*, detect: dict, active_job: dict, node_rows: list[dict]) -> dict:
|
|
queue_payload = dict((detect.get("queue_health") or {}).get("queue") or {})
|
|
display_running = 0
|
|
display_max_threads = 0
|
|
display_claimed = 0
|
|
running_items = 0
|
|
|
|
for raw_item in list(node_rows or []):
|
|
item = dict(raw_item or {})
|
|
running_items += int(item.get("items_running", 0) or 0)
|
|
display_running += max(
|
|
int(item.get("display_running", 0) or 0),
|
|
int(item.get("current_load", 0) or 0),
|
|
int(item.get("active_threads", 0) or 0),
|
|
int(item.get("items_running", 0) or 0),
|
|
)
|
|
display_max_threads += max(0, int(item.get("max_threads", 0) or 0))
|
|
display_claimed += max(
|
|
int(item.get("items_claimed", 0) or 0),
|
|
int(item.get("display_claimed", 0) or 0),
|
|
)
|
|
|
|
display_running = max(
|
|
display_running,
|
|
int(queue_payload.get("display_running", queue_payload.get("running", 0)) or 0),
|
|
int(active_job.get("display_active_threads", active_job.get("display_items_running", active_job.get("items_running", 0))) or 0),
|
|
)
|
|
running_items = max(
|
|
running_items,
|
|
int(queue_payload.get("running", 0) or 0),
|
|
int(active_job.get("items_running", 0) or 0),
|
|
)
|
|
display_claimed = max(
|
|
display_claimed,
|
|
int(queue_payload.get("display_claimed", queue_payload.get("claimed", 0)) or 0),
|
|
int(active_job.get("display_items_claimed", active_job.get("items_claimed", 0)) or 0),
|
|
)
|
|
display_max_threads = max(
|
|
display_max_threads,
|
|
int(active_job.get("display_max_threads", 0) or 0),
|
|
int(detect.get("aggregate_max_thread_count", 0) or 0),
|
|
int(detect.get("max_thread_count", 0) or 0),
|
|
)
|
|
return {
|
|
"items_running": running_items,
|
|
"display_running": display_running,
|
|
"display_claimed": display_claimed,
|
|
"display_max_threads": display_max_threads,
|
|
}
|
|
|
|
|
|
def _is_local_projection_node(node_code: str) -> bool:
|
|
normalized_node_code = str(node_code or "").strip()
|
|
local_node_code = str(settings.node_code or "").strip()
|
|
if not normalized_node_code or not local_node_code:
|
|
return False
|
|
return normalized_node_code == local_node_code or normalized_node_code.startswith(f"{local_node_code}-")
|
|
|
|
|
|
def _build_runtime_projection_payload(
|
|
*,
|
|
detect: dict,
|
|
cluster: dict,
|
|
source_region: str,
|
|
target_region: str,
|
|
) -> dict:
|
|
active_job = detect.get("active_job") or {}
|
|
local_worker_expected = _local_node_expected_to_execute_worker()
|
|
projection_node_rows = _projection_node_rows(detect=detect, active_job=active_job) if local_worker_expected else []
|
|
projection_cluster_rows = _projection_cluster_node_rows(cluster=cluster) if local_worker_expected else []
|
|
display_summary = (
|
|
_projection_display_summary(detect=detect, active_job=active_job, node_rows=projection_node_rows)
|
|
if local_worker_expected
|
|
else {
|
|
"items_running": 0,
|
|
"display_running": 0,
|
|
"display_claimed": 0,
|
|
"display_max_threads": 0,
|
|
}
|
|
)
|
|
queue_payload = dict((detect.get("queue_health") or {}).get("queue") or {}) if local_worker_expected else {}
|
|
local_participating = False
|
|
for node in list(cluster.get("nodes") or []):
|
|
if str(node.get("node_code") or "").strip() != settings.node_code:
|
|
continue
|
|
if local_worker_expected:
|
|
local_participating = bool(node.get("detect_participating", False) or node.get("current_load", 0))
|
|
break
|
|
local_job_bucket = {}
|
|
if local_worker_expected:
|
|
for item in projection_node_rows:
|
|
if not _is_local_projection_node(str(item.get("node_code") or "").strip()):
|
|
continue
|
|
local_job_bucket = item
|
|
local_participating = local_participating or bool(
|
|
int(item.get("display_running", 0) or 0) > 0
|
|
or int(item.get("active_threads", 0) or 0) > 0
|
|
or int(item.get("items_running", 0) or 0) > 0
|
|
or int(item.get("items_claimed", 0) or 0) > 0
|
|
)
|
|
break
|
|
if local_worker_expected and not local_participating:
|
|
local_participating = bool(
|
|
int(local_job_bucket.get("items_running", 0) or 0) > 0
|
|
or int(local_job_bucket.get("items_claimed", 0) or 0) > 0
|
|
or int(display_summary.get("display_running", 0) or 0) > 0
|
|
)
|
|
|
|
projection_active_job = (
|
|
{
|
|
"job_id": active_job.get("job_id"),
|
|
"job_code": active_job.get("job_code", ""),
|
|
"status": active_job.get("status", ""),
|
|
"progress_percent": active_job.get("progress_percent", 0),
|
|
"items_total": int(queue_payload.get("items_total", active_job.get("items_total", 0)) or 0),
|
|
"items_terminal": int(queue_payload.get("terminal", active_job.get("items_terminal", 0)) or 0),
|
|
"items_pending": int(queue_payload.get("pending", active_job.get("items_pending", 0)) or 0),
|
|
"items_claimed": int(queue_payload.get("claimed", active_job.get("items_claimed", 0)) or 0),
|
|
"items_running": int(display_summary.get("items_running", 0) or 0),
|
|
"items_failed": int(queue_payload.get("failed", active_job.get("items_failed", 0)) or 0),
|
|
"display_items_claimed": int(display_summary.get("display_claimed", 0) or 0),
|
|
"display_items_running": int(display_summary.get("display_running", 0) or 0),
|
|
"display_active_threads": int(display_summary.get("display_running", 0) or 0),
|
|
"display_max_threads": int(display_summary.get("display_max_threads", 0) or 0),
|
|
"node_stats": list(projection_node_rows),
|
|
"distributed_node_stats": list(projection_node_rows),
|
|
}
|
|
if local_worker_expected
|
|
else {
|
|
"job_id": None,
|
|
"job_code": "",
|
|
"status": "",
|
|
"progress_percent": 0,
|
|
"items_total": 0,
|
|
"items_terminal": 0,
|
|
"items_pending": 0,
|
|
"items_claimed": 0,
|
|
"items_running": 0,
|
|
"items_failed": 0,
|
|
"display_items_claimed": 0,
|
|
"display_items_running": 0,
|
|
"display_active_threads": 0,
|
|
"display_max_threads": 0,
|
|
"node_stats": [],
|
|
"distributed_node_stats": [],
|
|
}
|
|
)
|
|
progress_payload = (
|
|
{
|
|
"pending": int(queue_payload.get("pending", (detect.get("progress") or {}).get("pending", 0)) or 0),
|
|
"running": int(display_summary.get("display_running", 0) or 0),
|
|
"completed": int(queue_payload.get("completed", (detect.get("progress") or {}).get("completed", 0)) or 0),
|
|
"blacklisted": int(queue_payload.get("blacklisted", (detect.get("progress") or {}).get("blacklisted", 0)) or 0),
|
|
"failed": int(queue_payload.get("failed", (detect.get("progress") or {}).get("failed", 0)) or 0),
|
|
}
|
|
if local_worker_expected
|
|
else {
|
|
"pending": 0,
|
|
"running": 0,
|
|
"completed": 0,
|
|
"blacklisted": 0,
|
|
"failed": 0,
|
|
}
|
|
)
|
|
projection = {
|
|
"node": {
|
|
"node_code": settings.node_code,
|
|
"region": settings.node_region,
|
|
"role": settings.node_role,
|
|
"hostname": socket.gethostname(),
|
|
"ip": _resolve_local_ip(),
|
|
},
|
|
"worker_online": bool(detect.get("worker_online", False)) if local_worker_expected else False,
|
|
"detect_participating": local_participating if local_worker_expected else False,
|
|
"worker_mode": detect.get("worker_mode", ""),
|
|
"active_thread_count": int(display_summary.get("display_running", 0) or 0) if local_worker_expected else 0,
|
|
"max_thread_count": int(display_summary.get("display_max_threads", 0) or 0) if local_worker_expected else 0,
|
|
"phase_label": detect.get("phase_label", ""),
|
|
"phase_detail": detect.get("phase_detail", ""),
|
|
"proxy_runtime_label": detect.get("proxy_runtime_label", ""),
|
|
"proxy_runtime_reason": detect.get("proxy_runtime_reason", ""),
|
|
"progress": progress_payload,
|
|
"backlog": dict(detect.get("backlog") or {}) if local_worker_expected else {},
|
|
"active_job": projection_active_job,
|
|
"cluster_nodes": list(projection_cluster_rows),
|
|
"cluster_summary": {
|
|
"nodes_total": int(cluster.get("nodes_total", 0) or 0),
|
|
"online_worker_nodes": int((cluster.get("summary") or {}).get("online_worker_nodes", 0) or 0),
|
|
"dedicated_online_worker_nodes": int((cluster.get("summary") or {}).get("dedicated_online_worker_nodes", 0) or 0),
|
|
"online_control_nodes": int((cluster.get("summary") or {}).get("online_control_nodes", 0) or 0),
|
|
"busy_nodes": list((cluster.get("summary") or {}).get("busy_nodes") or []),
|
|
"stale_nodes": list((cluster.get("summary") or {}).get("stale_nodes") or []),
|
|
"offline_nodes": list((cluster.get("summary") or {}).get("offline_nodes") or []),
|
|
},
|
|
"dependency_alerts": [
|
|
{
|
|
"kind": item.get("kind", ""),
|
|
"title": item.get("title", ""),
|
|
"level": item.get("level", ""),
|
|
}
|
|
for item in (detect.get("dependency_alerts") or [])[:3]
|
|
],
|
|
}
|
|
return {
|
|
"projection": projection,
|
|
"projection_hash": hashlib.sha1(
|
|
json.dumps(projection, ensure_ascii=False, sort_keys=True).encode("utf-8")
|
|
).hexdigest(),
|
|
"source_region": source_region,
|
|
"target_region": target_region,
|
|
}
|
|
|
|
|
|
def append_runtime_projection_if_changed(
|
|
*,
|
|
detect: dict,
|
|
cluster: dict,
|
|
source_region: str | None = None,
|
|
target_region: str | None = None,
|
|
) -> int | None:
|
|
normalized_source_region = _normalize_region(source_region, _normalize_region(settings.sync_source_region, settings.node_region))
|
|
normalized_target_region = _normalize_region(target_region, _normalize_region(settings.sync_target_region, "overseas"))
|
|
future_cutoff = datetime.now() + _RUNTIME_PROJECTION_FUTURE_SKEW_GRACE
|
|
payload = _build_runtime_projection_payload(
|
|
detect=detect,
|
|
cluster=cluster,
|
|
source_region=normalized_source_region,
|
|
target_region=normalized_target_region,
|
|
)
|
|
|
|
with get_db() as conn:
|
|
with conn.cursor() as cur:
|
|
cur.execute(
|
|
"""
|
|
SELECT payload_json, created_at
|
|
FROM detect_sync_records
|
|
WHERE sync_type = 'runtime_projection'
|
|
AND source_region = %s
|
|
AND target_region = %s
|
|
ORDER BY
|
|
CASE WHEN created_at <= %s THEN 0 ELSE 1 END ASC,
|
|
created_at DESC,
|
|
id DESC
|
|
LIMIT 200
|
|
""",
|
|
(normalized_source_region, normalized_target_region, future_cutoff),
|
|
)
|
|
latest = _pick_latest_projection_row(list(cur.fetchall() or []), created_at_index=1)
|
|
latest_payload = _decode_json(latest[0]) if latest else {}
|
|
latest_created_at = latest[1] if latest else None
|
|
if not _should_append_runtime_projection(latest_payload, payload["projection"], latest_created_at):
|
|
return None
|
|
cur.execute(
|
|
"""
|
|
INSERT INTO detect_sync_records (
|
|
sync_type, source_region, target_region, status, payload_json, error_message, created_at, updated_at
|
|
) VALUES (%s, %s, %s, %s, %s::jsonb, %s, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)
|
|
RETURNING id
|
|
""",
|
|
(
|
|
"runtime_projection",
|
|
normalized_source_region,
|
|
normalized_target_region,
|
|
"projected",
|
|
json.dumps(payload, ensure_ascii=False),
|
|
"",
|
|
),
|
|
)
|
|
record_id = int(cur.fetchone()[0])
|
|
conn.commit()
|
|
return record_id
|
|
|
|
|
|
def append_detect_result_projection_if_changed(
|
|
*,
|
|
detect: dict,
|
|
source_region: str | None = None,
|
|
target_region: str | None = None,
|
|
) -> int | None:
|
|
normalized_source_region = _normalize_region(source_region, _normalize_region(settings.sync_source_region, settings.node_region))
|
|
normalized_target_region = _normalize_region(target_region, _normalize_region(settings.sync_target_region, "overseas"))
|
|
active_job = detect.get("active_job") or {}
|
|
if not active_job:
|
|
return None
|
|
|
|
latest_cycle_event = active_job.get("latest_cycle_event") or active_job.get("latest_event") or {}
|
|
projection = {
|
|
"job": {
|
|
"job_id": active_job.get("job_id"),
|
|
"job_code": active_job.get("job_code", ""),
|
|
"status": active_job.get("status", ""),
|
|
"progress_percent": active_job.get("progress_percent", 0),
|
|
"items_total": active_job.get("items_total", 0),
|
|
"items_pending": active_job.get("items_pending", 0),
|
|
"items_claimed": active_job.get("items_claimed", 0),
|
|
"items_running": active_job.get("items_running", 0),
|
|
"items_completed": active_job.get("items_completed", 0),
|
|
"items_blacklisted": active_job.get("items_blacklisted", 0),
|
|
"items_failed": active_job.get("items_failed", 0),
|
|
"items_terminal": active_job.get("items_terminal", 0),
|
|
"current_cycle_token": active_job.get("current_cycle_token", ""),
|
|
},
|
|
"latest_event": {
|
|
"node_code": latest_cycle_event.get("node_code", ""),
|
|
"event_type": latest_cycle_event.get("event_type", ""),
|
|
"message": latest_cycle_event.get("message", ""),
|
|
"created_at": latest_cycle_event.get("created_at", ""),
|
|
},
|
|
"queue": {
|
|
"pending": int((detect.get("progress") or {}).get("pending", 0) or 0),
|
|
"running": int((detect.get("progress") or {}).get("running", 0) or 0),
|
|
"completed": int((detect.get("progress") or {}).get("completed", 0) or 0),
|
|
"blacklisted": int((detect.get("progress") or {}).get("blacklisted", 0) or 0),
|
|
"failed": int((detect.get("progress") or {}).get("failed", 0) or 0),
|
|
},
|
|
"phase": {
|
|
"label": detect.get("phase_label", ""),
|
|
"detail": detect.get("phase_detail", ""),
|
|
},
|
|
"recent_domain_events": _collect_recent_domain_events(active_job, limit=30),
|
|
}
|
|
payload = {
|
|
"projection": projection,
|
|
"projection_hash": hashlib.sha1(
|
|
json.dumps(projection, ensure_ascii=False, sort_keys=True).encode("utf-8")
|
|
).hexdigest(),
|
|
}
|
|
|
|
with get_db() as conn:
|
|
with conn.cursor() as cur:
|
|
current_job_id = int((projection.get("job") or {}).get("job_id") or 0)
|
|
cur.execute(
|
|
"""
|
|
SELECT payload_json, created_at
|
|
FROM detect_sync_records
|
|
WHERE sync_type = 'detect_result_projection'
|
|
AND source_region = %s
|
|
AND target_region = %s
|
|
AND (
|
|
%s <= 0
|
|
OR (payload_json->'projection'->'job'->>'job_id') = %s
|
|
)
|
|
ORDER BY created_at DESC, id DESC
|
|
LIMIT 1
|
|
""",
|
|
(
|
|
normalized_source_region,
|
|
normalized_target_region,
|
|
current_job_id,
|
|
str(current_job_id),
|
|
),
|
|
)
|
|
latest = cur.fetchone()
|
|
latest_payload = _decode_json(latest[0]) if latest else {}
|
|
latest_created_at = latest[1] if latest else None
|
|
if latest_payload.get("projection_hash") == payload["projection_hash"]:
|
|
return None
|
|
|
|
latest_projection = latest_payload.get("projection") or {}
|
|
latest_job = latest_projection.get("job") or {}
|
|
current_job = projection.get("job") or {}
|
|
latest_event = latest_projection.get("latest_event") or {}
|
|
current_event = projection.get("latest_event") or {}
|
|
if latest_job.get("status") == current_job.get("status") and latest_event == current_event and latest_created_at:
|
|
now = datetime.now(latest_created_at.tzinfo) if latest_created_at.tzinfo else datetime.now()
|
|
if now - latest_created_at < timedelta(seconds=30):
|
|
return None
|
|
|
|
cur.execute(
|
|
"""
|
|
INSERT INTO detect_sync_records (
|
|
sync_type, source_region, target_region, status, payload_json, error_message, created_at, updated_at
|
|
) VALUES (%s, %s, %s, %s, %s::jsonb, %s, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)
|
|
RETURNING id
|
|
""",
|
|
(
|
|
"detect_result_projection",
|
|
normalized_source_region,
|
|
normalized_target_region,
|
|
"projected",
|
|
json.dumps(payload, ensure_ascii=False),
|
|
"",
|
|
),
|
|
)
|
|
record_id = int(cur.fetchone()[0])
|
|
conn.commit()
|
|
return record_id
|