admin/getDomain
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

d

Browse Source
This commit is contained in:
Your Name
2026-04-22 14:13:21 +08:00
parent e0406b5d0e
commit 7cbde2aa78
145 changed files with 23086 additions and 2243 deletions
Download Patch File Download Diff File Expand all files Collapse all files

250
domain-api/app/services/sync_record_service.py
View File

@@ -6,7 +6,7 @@ import socket
from datetime import datetime, timedelta
from app.core.config import settings
from app.core.db import get_db
from app.core.db import db_read_retry, get_db
def _format_time(value: datetime | None) -> str:
@@ -45,28 +45,94 @@ _DETECT_RESULT_EVENT_TYPES = {
"domain_blacklisted",
}
_TERMINAL_DETECT_RESULT_EVENT_TYPES = {
"domain_completed",
"domain_failed",
"domain_blacklisted",
}
def _collect_recent_domain_events(active_job: dict, limit: int = 30) -> list[dict]:
events = list(active_job.get("current_cycle_events") or active_job.get("recent_events") or [])
safe_limit = max(1, int(limit or 30))
seen: set[tuple[str, str, str, str]] = set()
normalized: list[dict] = []
for event in reversed(events):
event_type = str(event.get("event_type") or "").strip()
def _append_event(raw_event: dict) -> None:
event_type = str(raw_event.get("event_type") or "").strip()
if event_type not in _DETECT_RESULT_EVENT_TYPES:
continue
payload = _decode_json(event.get("payload"))
normalized.append(
{
"node_code": str(event.get("node_code") or "").strip(),
"event_type": event_type,
"level": str(event.get("level") or "info").strip() or "info",
"message": str(event.get("message") or "").strip(),
"created_at": str(event.get("created_at") or "").strip(),
"payload": payload,
}
return
normalized_event = {
"node_code": str(raw_event.get("node_code") or "").strip(),
"event_type": event_type,
"level": str(raw_event.get("level") or "info").strip() or "info",
"message": str(raw_event.get("message") or "").strip(),
"created_at": str(raw_event.get("created_at") or "").strip(),
"payload": _decode_json(raw_event.get("payload")),
}
event_key = (
normalized_event["node_code"],
normalized_event["event_type"],
normalized_event["message"],
normalized_event["created_at"],
)
if limit <= 0:
return normalized
return normalized[-int(limit):]
if event_key in seen:
return
seen.add(event_key)
normalized.append(normalized_event)
# Keep a small slice of the current-cycle `domain_started` events so the
# remote log / live activity view still reflects the node's latest work.
for event in reversed(list(active_job.get("current_cycle_events") or active_job.get("recent_events") or [])):
if str(event.get("event_type") or "").strip() != "domain_started":
continue
_append_event(event)
if len(normalized) >= min(10, max(1, safe_limit // 3)):
break
# Always pull the most recent terminal result events from the full job
# history. Otherwise a flood of newer `domain_started` events can hide
# terminal completions, and overseas will never advance completed counts.
job_id = int(active_job.get("job_id") or 0)
if job_id > 0:
with get_db() as conn:
with conn.cursor() as cur:
cur.execute(
"""
SELECT node_code, event_type, level, message, payload_json, created_at
FROM detect_run_events
WHERE job_id = %s
AND event_type IN ('domain_completed', 'domain_failed', 'domain_blacklisted')
ORDER BY created_at DESC, id DESC
LIMIT %s
""",
(job_id, max(safe_limit * 4, 60)),
)
rows = cur.fetchall()
for row in reversed(rows):
_append_event(
{
"node_code": row[0] or "",
"event_type": row[1] or "",
"level": row[2] or "info",
"message": row[3] or "",
"payload": _decode_json(row[4]),
"created_at": _format_time(row[5]),
}
)
else:
for event in reversed(list(active_job.get("current_cycle_events") or active_job.get("recent_events") or [])):
if str(event.get("event_type") or "").strip() in _TERMINAL_DETECT_RESULT_EVENT_TYPES:
_append_event(event)
normalized.sort(
key=lambda item: (
str(item.get("created_at") or ""),
str(item.get("node_code") or ""),
str(item.get("event_type") or ""),
str(item.get("message") or ""),
)
)
return normalized[-safe_limit:]
def _build_detect_result_batch_digest(batch: dict | None) -> dict:
@@ -154,6 +220,7 @@ def _should_append_runtime_projection(previous_payload: dict, current_projection
return now - previous_created_at >= timedelta(seconds=45)
@db_read_retry()
def list_sync_records(limit: int = 20) -> list[dict]:
safe_limit = max(1, min(int(limit or 20), 200))
with get_db() as conn:
@@ -221,6 +288,7 @@ def _latest_sync_record_by_source(
}
@db_read_retry()
def get_detect_result_sync_batches(limit: int = 5) -> dict:
safe_limit = max(1, min(int(limit or 5), 20))
source_region = _normalize_region(settings.sync_source_region, settings.node_region)
@@ -364,6 +432,7 @@ def get_detect_result_sync_batches(limit: int = 5) -> dict:
}
@db_read_retry()
def get_sync_summary(record_limit: int = 10) -> dict:
source_region = _normalize_region(settings.sync_source_region, settings.node_region)
target_region = _normalize_region(settings.sync_target_region, "overseas")
@@ -465,56 +534,43 @@ def append_sync_record(
return record_id
def append_runtime_projection_if_changed(
def _local_node_expected_to_execute_worker() -> bool:
node_role = str(settings.node_role or "").strip()
node_region = str(settings.node_region or "").strip()
return node_role == "worker" or (node_region == "mainland" and node_role == "control")
def _build_runtime_projection_payload(
*,
detect: dict,
cluster: dict,
source_region: str | None = None,
target_region: str | None = None,
) -> int | None:
normalized_source_region = _normalize_region(source_region, _normalize_region(settings.sync_source_region, settings.node_region))
normalized_target_region = _normalize_region(target_region, _normalize_region(settings.sync_target_region, "overseas"))
source_region: str,
target_region: str,
) -> dict:
active_job = detect.get("active_job") or {}
local_worker_expected = _local_node_expected_to_execute_worker()
local_participating = False
for node in list(cluster.get("nodes") or []):
if str(node.get("node_code") or "").strip() != settings.node_code:
continue
local_participating = bool(node.get("detect_participating", False) or node.get("current_load", 0))
if local_worker_expected:
local_participating = bool(node.get("detect_participating", False) or node.get("current_load", 0))
break
local_job_bucket = {}
for item in list(active_job.get("node_stats") or []):
if str(item.get("node_code") or "").strip() != settings.node_code:
continue
local_job_bucket = item
break
if not local_participating:
if local_worker_expected:
for item in list(active_job.get("node_stats") or []):
if str(item.get("node_code") or "").strip() != settings.node_code:
continue
local_job_bucket = item
break
if local_worker_expected and not local_participating:
local_participating = bool(
int(local_job_bucket.get("items_running", 0) or 0) > 0
or int(local_job_bucket.get("items_claimed", 0) or 0) > 0
)
projection = {
"node": {
"node_code": settings.node_code,
"region": settings.node_region,
"role": settings.node_role,
"hostname": socket.gethostname(),
"ip": _resolve_local_ip(),
},
"worker_online": bool(detect.get("worker_online", False)),
"detect_participating": local_participating,
"worker_mode": detect.get("worker_mode", ""),
"phase_label": detect.get("phase_label", ""),
"phase_detail": detect.get("phase_detail", ""),
"proxy_runtime_label": detect.get("proxy_runtime_label", ""),
"proxy_runtime_reason": detect.get("proxy_runtime_reason", ""),
"progress": {
"pending": int((detect.get("progress") or {}).get("pending", 0) or 0),
"running": int((detect.get("progress") or {}).get("running", 0) or 0),
"completed": int((detect.get("progress") or {}).get("completed", 0) or 0),
"blacklisted": int((detect.get("progress") or {}).get("blacklisted", 0) or 0),
"failed": int((detect.get("progress") or {}).get("failed", 0) or 0),
},
"active_job": {
projection_active_job = (
{
"job_id": active_job.get("job_id"),
"job_code": active_job.get("job_code", ""),
"status": active_job.get("status", ""),
@@ -525,7 +581,58 @@ def append_runtime_projection_if_changed(
"items_running": active_job.get("items_running", 0),
"items_failed": active_job.get("items_failed", 0),
"node_stats": list(active_job.get("node_stats") or []),
}
if local_worker_expected
else {
"job_id": None,
"job_code": "",
"status": "",
"progress_percent": 0,
"items_total": 0,
"items_terminal": 0,
"items_pending": 0,
"items_running": 0,
"items_failed": 0,
"node_stats": [],
}
)
progress_payload = (
{
"pending": int((detect.get("progress") or {}).get("pending", 0) or 0),
"running": int((detect.get("progress") or {}).get("running", 0) or 0),
"completed": int((detect.get("progress") or {}).get("completed", 0) or 0),
"blacklisted": int((detect.get("progress") or {}).get("blacklisted", 0) or 0),
"failed": int((detect.get("progress") or {}).get("failed", 0) or 0),
}
if local_worker_expected
else {
"pending": 0,
"running": 0,
"completed": 0,
"blacklisted": 0,
"failed": 0,
}
)
projection = {
"node": {
"node_code": settings.node_code,
"region": settings.node_region,
"role": settings.node_role,
"hostname": socket.gethostname(),
"ip": _resolve_local_ip(),
},
"worker_online": bool(detect.get("worker_online", False)) if local_worker_expected else False,
"detect_participating": local_participating if local_worker_expected else False,
"worker_mode": detect.get("worker_mode", ""),
"active_thread_count": int(detect.get("active_thread_count", 0) or 0) if local_worker_expected else 0,
"max_thread_count": int(detect.get("max_thread_count", 0) or 0) if local_worker_expected else 0,
"phase_label": detect.get("phase_label", ""),
"phase_detail": detect.get("phase_detail", ""),
"proxy_runtime_label": detect.get("proxy_runtime_label", ""),
"proxy_runtime_reason": detect.get("proxy_runtime_reason", ""),
"progress": progress_payload,
"backlog": dict(detect.get("backlog") or {}) if local_worker_expected else {},
"active_job": projection_active_job,
"cluster_summary": {
"nodes_total": int(cluster.get("nodes_total", 0) or 0),
"online_worker_nodes": int((cluster.get("summary") or {}).get("online_worker_nodes", 0) or 0),
@@ -544,13 +651,32 @@ def append_runtime_projection_if_changed(
for item in (detect.get("dependency_alerts") or [])[:3]
],
}
payload = {
return {
"projection": projection,
"projection_hash": hashlib.sha1(
json.dumps(projection, ensure_ascii=False, sort_keys=True).encode("utf-8")
).hexdigest(),
"source_region": source_region,
"target_region": target_region,
}
def append_runtime_projection_if_changed(
*,
detect: dict,
cluster: dict,
source_region: str | None = None,
target_region: str | None = None,
) -> int | None:
normalized_source_region = _normalize_region(source_region, _normalize_region(settings.sync_source_region, settings.node_region))
normalized_target_region = _normalize_region(target_region, _normalize_region(settings.sync_target_region, "overseas"))
payload = _build_runtime_projection_payload(
detect=detect,
cluster=cluster,
source_region=normalized_source_region,
target_region=normalized_target_region,
)
with get_db() as conn:
with conn.cursor() as cur:
cur.execute(
@@ -570,7 +696,7 @@ def append_runtime_projection_if_changed(
latest_created_at = latest[1] if latest else None
if latest_payload.get("projection_hash") == payload["projection_hash"]:
return None
if not _should_append_runtime_projection(latest_payload, projection, latest_created_at):
if not _should_append_runtime_projection(latest_payload, payload["projection"], latest_created_at):
return None
cur.execute(
"""
@@ -650,6 +776,7 @@ def append_detect_result_projection_if_changed(
with get_db() as conn:
with conn.cursor() as cur:
current_job_id = int((projection.get("job") or {}).get("job_id") or 0)
cur.execute(
"""
SELECT payload_json, created_at
@@ -657,10 +784,19 @@ def append_detect_result_projection_if_changed(
WHERE sync_type = 'detect_result_projection'
AND source_region = %s
AND target_region = %s
AND (
%s <= 0
OR (payload_json->'projection'->'job'->>'job_id') = %s
)
ORDER BY created_at DESC, id DESC
LIMIT 1
""",
(normalized_source_region, normalized_target_region),
(
normalized_source_region,
normalized_target_region,
current_job_id,
str(current_job_id),
),
)
latest = cur.fetchone()
latest_payload = _decode_json(latest[0]) if latest else {}