admin/getDomain
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

fix: surface distributed detect participation

Browse Source
This commit is contained in:
Your Name
2026-04-19 02:35:43 +08:00
parent ad513211e6
commit cfff515c03
5 changed files with 474 additions and 189 deletions
Download Patch File Download Diff File Expand all files Collapse all files

181
domain-api/app/services/detect_job_service.py
View File

@@ -11,6 +11,7 @@ from app.services.debug_event_service import push_debug_event
ACTIVE_JOB_STATUSES = ("pending", "running")
_RUNTIME_NODE_STALE_MINUTES = 10
def _selection_sql() -> str:
@@ -40,6 +41,152 @@ def _decode_payload(value: object) -> dict:
return {}
def _int_value(value: object) -> int:
try:
return int(value or 0)
except Exception:
return 0
def _build_runtime_display_bucket(row: tuple) -> dict | None:
node_code = str(row[0] or "").strip()
if not node_code:
return None
metadata = _decode_payload(row[5])
items_total = _int_value(metadata.get("job_items_total"))
items_claimed = _int_value(metadata.get("job_items_claimed"))
items_running = _int_value(metadata.get("job_items_running"))
items_completed = _int_value(metadata.get("job_items_completed"))
items_failed = _int_value(metadata.get("job_items_failed"))
current_load = _int_value(row[4])
detect_participating = bool(metadata.get("detect_participating", False))
if items_total <= 0 and items_claimed <= 0 and items_running <= 0 and items_completed <= 0 and items_failed <= 0 and current_load <= 0 and not detect_participating:
return None
items_pending = max(items_total - items_claimed - items_running - items_completed - items_failed, 0)
return {
"node_code": node_code,
"items_total": items_total,
"items_pending": items_pending,
"items_claimed": items_claimed,
"items_running": items_running,
"items_completed": items_completed,
"items_blacklisted": 0,
"items_failed": items_failed,
"metrics_source": str(metadata.get("service") or "runtime").strip() or "runtime",
"region": str(row[1] or "").strip(),
"role": str(row[2] or "").strip(),
"status": str(row[3] or "").strip(),
"current_load": current_load,
"last_heartbeat_at": _format_time(row[6]) if row[6] else "",
}
def _merge_display_node_stats(*, local_node_stats: list[dict], runtime_node_rows: list[tuple]) -> list[dict]:
merged: dict[str, dict] = {}
def _ensure_bucket(node_code: str) -> dict:
return merged.setdefault(
node_code,
{
"node_code": node_code,
"items_total": 0,
"items_pending": 0,
"items_claimed": 0,
"items_running": 0,
"items_completed": 0,
"items_blacklisted": 0,
"items_failed": 0,
"metrics_source": "",
"region": "",
"role": "",
"status": "",
"current_load": 0,
"last_heartbeat_at": "",
},
)
for item in list(local_node_stats or []):
node_code = str(item.get("node_code") or "").strip()
if not node_code:
continue
bucket = _ensure_bucket(node_code)
for key in (
"items_total",
"items_pending",
"items_claimed",
"items_running",
"items_completed",
"items_blacklisted",
"items_failed",
):
bucket[key] = max(_int_value(bucket.get(key)), _int_value(item.get(key)))
if node_code == "unassigned":
bucket["metrics_source"] = "central_queue"
for row in list(runtime_node_rows or []):
runtime_bucket = _build_runtime_display_bucket(row)
if not runtime_bucket:
continue
node_code = str(runtime_bucket.get("node_code") or "").strip()
bucket = _ensure_bucket(node_code)
for key in (
"items_total",
"items_pending",
"items_claimed",
"items_running",
"items_completed",
"items_blacklisted",
"items_failed",
"current_load",
):
bucket[key] = max(_int_value(bucket.get(key)), _int_value(runtime_bucket.get(key)))
for key in ("metrics_source", "region", "role", "status", "last_heartbeat_at"):
if str(runtime_bucket.get(key) or "").strip():
bucket[key] = runtime_bucket.get(key)
return sorted(
merged.values(),
key=lambda item: (
str(item.get("node_code") or "") == "unassigned",
-_int_value(item.get("items_running")),
-_int_value(item.get("items_claimed")),
-_int_value(item.get("items_total")),
str(item.get("node_code") or ""),
),
)
def _load_runtime_display_rows(cur) -> list[tuple]:
cur.execute(
f"""
SELECT node_code, region, role, status, current_load, metadata_json, last_heartbeat_at
FROM detect_worker_nodes
WHERE last_heartbeat_at >= CURRENT_TIMESTAMP - interval '{_RUNTIME_NODE_STALE_MINUTES} minutes'
ORDER BY last_heartbeat_at DESC, node_code ASC
"""
)
return list(cur.fetchall())
def _build_display_summary(node_stats: list[dict]) -> dict:
effective_nodes = [
item
for item in list(node_stats or [])
if str(item.get("node_code") or "").strip() and str(item.get("node_code") or "").strip() != "unassigned"
]
return {
"items_claimed": sum(_int_value(item.get("items_claimed")) for item in effective_nodes),
"items_running": sum(_int_value(item.get("items_running")) for item in effective_nodes),
"items_completed": sum(_int_value(item.get("items_completed")) for item in effective_nodes),
"items_failed": sum(_int_value(item.get("items_failed")) for item in effective_nodes),
"active_nodes": [
str(item.get("node_code") or "").strip()
for item in effective_nodes
if _int_value(item.get("items_claimed")) > 0 or _int_value(item.get("items_running")) > 0
],
}
def _extract_current_cycle_events(events: list[dict]) -> tuple[str, list[dict]]:
if not events:
return "", []
@@ -142,6 +289,11 @@ def _fetch_job_summary(cur, job_row, event_limit: int = 20) -> dict:
for item in cur.fetchall()
]
cycle_token, current_cycle_events = _extract_current_cycle_events(events)
distributed_node_stats = _merge_display_node_stats(
local_node_stats=list(node_buckets.values()),
runtime_node_rows=_load_runtime_display_rows(cur),
)
display_summary = _build_display_summary(distributed_node_stats)
total = sum(counts.values())
terminal = int(counts.get("completed", 0)) + int(counts.get("blacklisted", 0)) + int(counts.get("failed", 0))
return {
@@ -163,6 +315,12 @@ def _fetch_job_summary(cur, job_row, event_limit: int = 20) -> dict:
"items_terminal": terminal,
"progress_percent": round((terminal / total) * 100, 2) if total else 0,
"node_stats": list(node_buckets.values()),
"distributed_node_stats": distributed_node_stats,
"display_items_claimed": int(display_summary.get("items_claimed", 0) or 0),
"display_items_running": int(display_summary.get("items_running", 0) or 0),
"display_items_completed": int(display_summary.get("items_completed", 0) or 0),
"display_items_failed": int(display_summary.get("items_failed", 0) or 0),
"display_active_node_codes": list(display_summary.get("active_nodes") or []),
"recent_events": events,
"latest_event": events[0] if events else None,
"current_cycle_token": cycle_token,
@@ -350,6 +508,27 @@ def get_detect_queue_health(window_minutes: int = 15) -> dict:
}
for item in active_job.get("node_stats") or []
}
distributed_nodes = list(active_job.get("distributed_node_stats") or [])
if distributed_nodes:
node_map = {
str(item.get("node_code") or "unknown"): {
"node_code": str(item.get("node_code") or "unknown"),
"items_total": _int_value(item.get("items_total")),
"items_pending": _int_value(item.get("items_pending")),
"items_claimed": _int_value(item.get("items_claimed")),
"items_running": _int_value(item.get("items_running")),
"items_completed": _int_value(item.get("items_completed")),
"items_blacklisted": _int_value(item.get("items_blacklisted")),
"items_failed": _int_value(item.get("items_failed")),
"processed_recent": 0,
"processed_per_minute": 0,
"completed_recent": 0,
"blacklisted_recent": 0,
"failed_recent": 0,
"metrics_source": str(item.get("metrics_source") or "").strip(),
}
for item in distributed_nodes
}
total_processed_recent = 0
total_completed_recent = 0
total_blacklisted_recent = 0
@@ -412,6 +591,8 @@ def get_detect_queue_health(window_minutes: int = 15) -> dict:
"pending": int(active_job.get("items_pending", 0) or 0),
"claimed": int(active_job.get("items_claimed", 0) or 0),
"running": int(active_job.get("items_running", 0) or 0),
"display_claimed": int(active_job.get("display_items_claimed", active_job.get("items_claimed", 0)) or 0),
"display_running": int(active_job.get("display_items_running", active_job.get("items_running", 0)) or 0),
"completed": int(active_job.get("items_completed", 0) or 0),
"blacklisted": int(active_job.get("items_blacklisted", 0) or 0),
"failed": int(active_job.get("items_failed", 0) or 0),