919 lines
36 KiB
Python
919 lines
36 KiB
Python
from __future__ import annotations
|
|
|
|
import json
|
|
from datetime import datetime
|
|
import math
|
|
from uuid import uuid4
|
|
|
|
from app.core.config import settings
|
|
from app.core.db import get_db
|
|
from app.services.debug_event_service import push_debug_event
|
|
|
|
|
|
ACTIVE_JOB_STATUSES = ("pending", "running")
|
|
_RUNTIME_NODE_STALE_MINUTES = 10
|
|
|
|
|
|
def _selection_sql() -> str:
|
|
return """
|
|
SELECT id
|
|
FROM domains
|
|
WHERE
|
|
detect_status IN (0, 4)
|
|
OR (use_status = 0 AND detect_status = 1 AND register_status = 3 AND expire_date < CURRENT_DATE)
|
|
ORDER BY id ASC
|
|
LIMIT %s
|
|
"""
|
|
|
|
|
|
def _format_time(value: datetime | None) -> str:
|
|
return value.isoformat(sep=" ", timespec="seconds") if value else ""
|
|
|
|
|
|
def _decode_payload(value: object) -> dict:
|
|
if isinstance(value, dict):
|
|
return value
|
|
if value in (None, ""):
|
|
return {}
|
|
try:
|
|
return json.loads(value)
|
|
except Exception:
|
|
return {}
|
|
|
|
|
|
def _int_value(value: object) -> int:
|
|
try:
|
|
return int(value or 0)
|
|
except Exception:
|
|
return 0
|
|
|
|
|
|
def _build_runtime_display_bucket(row: tuple) -> dict | None:
|
|
node_code = str(row[0] or "").strip()
|
|
if not node_code:
|
|
return None
|
|
metadata = _decode_payload(row[5])
|
|
items_total = _int_value(metadata.get("job_items_total"))
|
|
items_claimed = _int_value(metadata.get("job_items_claimed"))
|
|
items_running = _int_value(metadata.get("job_items_running"))
|
|
items_completed = _int_value(metadata.get("job_items_completed"))
|
|
items_failed = _int_value(metadata.get("job_items_failed"))
|
|
current_load = _int_value(row[4])
|
|
detect_participating = bool(metadata.get("detect_participating", False))
|
|
if items_total <= 0 and items_claimed <= 0 and items_running <= 0 and items_completed <= 0 and items_failed <= 0 and current_load <= 0 and not detect_participating:
|
|
return None
|
|
items_pending = max(items_total - items_claimed - items_running - items_completed - items_failed, 0)
|
|
return {
|
|
"node_code": node_code,
|
|
"items_total": items_total,
|
|
"items_pending": items_pending,
|
|
"items_claimed": items_claimed,
|
|
"items_running": items_running,
|
|
"items_completed": items_completed,
|
|
"items_blacklisted": 0,
|
|
"items_failed": items_failed,
|
|
"metrics_source": str(metadata.get("service") or "runtime").strip() or "runtime",
|
|
"region": str(row[1] or "").strip(),
|
|
"role": str(row[2] or "").strip(),
|
|
"status": str(row[3] or "").strip(),
|
|
"current_load": current_load,
|
|
"last_heartbeat_at": _format_time(row[6]) if row[6] else "",
|
|
}
|
|
|
|
|
|
def _merge_display_node_stats(*, local_node_stats: list[dict], runtime_node_rows: list[tuple]) -> list[dict]:
|
|
merged: dict[str, dict] = {}
|
|
|
|
def _ensure_bucket(node_code: str) -> dict:
|
|
return merged.setdefault(
|
|
node_code,
|
|
{
|
|
"node_code": node_code,
|
|
"items_total": 0,
|
|
"items_pending": 0,
|
|
"items_claimed": 0,
|
|
"items_running": 0,
|
|
"items_completed": 0,
|
|
"items_blacklisted": 0,
|
|
"items_failed": 0,
|
|
"metrics_source": "",
|
|
"region": "",
|
|
"role": "",
|
|
"status": "",
|
|
"current_load": 0,
|
|
"last_heartbeat_at": "",
|
|
},
|
|
)
|
|
|
|
for item in list(local_node_stats or []):
|
|
node_code = str(item.get("node_code") or "").strip()
|
|
if not node_code:
|
|
continue
|
|
bucket = _ensure_bucket(node_code)
|
|
for key in (
|
|
"items_total",
|
|
"items_pending",
|
|
"items_claimed",
|
|
"items_running",
|
|
"items_completed",
|
|
"items_blacklisted",
|
|
"items_failed",
|
|
):
|
|
bucket[key] = max(_int_value(bucket.get(key)), _int_value(item.get(key)))
|
|
if node_code == "unassigned":
|
|
bucket["metrics_source"] = "central_queue"
|
|
|
|
for row in list(runtime_node_rows or []):
|
|
runtime_bucket = _build_runtime_display_bucket(row)
|
|
if not runtime_bucket:
|
|
continue
|
|
node_code = str(runtime_bucket.get("node_code") or "").strip()
|
|
bucket = _ensure_bucket(node_code)
|
|
for key in (
|
|
"items_total",
|
|
"items_pending",
|
|
"items_claimed",
|
|
"items_running",
|
|
"items_completed",
|
|
"items_blacklisted",
|
|
"items_failed",
|
|
"current_load",
|
|
):
|
|
bucket[key] = max(_int_value(bucket.get(key)), _int_value(runtime_bucket.get(key)))
|
|
for key in ("metrics_source", "region", "role", "status", "last_heartbeat_at"):
|
|
if str(runtime_bucket.get(key) or "").strip():
|
|
bucket[key] = runtime_bucket.get(key)
|
|
|
|
return sorted(
|
|
merged.values(),
|
|
key=lambda item: (
|
|
str(item.get("node_code") or "") == "unassigned",
|
|
-_int_value(item.get("items_running")),
|
|
-_int_value(item.get("items_claimed")),
|
|
-_int_value(item.get("items_total")),
|
|
str(item.get("node_code") or ""),
|
|
),
|
|
)
|
|
|
|
|
|
def _load_runtime_display_rows(cur) -> list[tuple]:
|
|
cur.execute(
|
|
f"""
|
|
SELECT node_code, region, role, status, current_load, metadata_json, last_heartbeat_at
|
|
FROM detect_worker_nodes
|
|
WHERE last_heartbeat_at >= CURRENT_TIMESTAMP - interval '{_RUNTIME_NODE_STALE_MINUTES} minutes'
|
|
ORDER BY last_heartbeat_at DESC, node_code ASC
|
|
"""
|
|
)
|
|
return list(cur.fetchall())
|
|
|
|
|
|
def _build_display_summary(node_stats: list[dict]) -> dict:
|
|
effective_nodes = [
|
|
item
|
|
for item in list(node_stats or [])
|
|
if str(item.get("node_code") or "").strip() and str(item.get("node_code") or "").strip() != "unassigned"
|
|
]
|
|
return {
|
|
"items_claimed": sum(_int_value(item.get("items_claimed")) for item in effective_nodes),
|
|
"items_running": sum(_int_value(item.get("items_running")) for item in effective_nodes),
|
|
"items_completed": sum(_int_value(item.get("items_completed")) for item in effective_nodes),
|
|
"items_failed": sum(_int_value(item.get("items_failed")) for item in effective_nodes),
|
|
"active_nodes": [
|
|
str(item.get("node_code") or "").strip()
|
|
for item in effective_nodes
|
|
if _int_value(item.get("items_claimed")) > 0 or _int_value(item.get("items_running")) > 0
|
|
],
|
|
}
|
|
|
|
|
|
def _normalize_node_bucket(item: dict) -> dict:
|
|
node_code = str(item.get("node_code") or "").strip()
|
|
return {
|
|
"node_code": node_code,
|
|
"items_total": _int_value(item.get("items_total")),
|
|
"items_pending": _int_value(item.get("items_pending")),
|
|
"items_claimed": _int_value(item.get("items_claimed")),
|
|
"items_running": _int_value(item.get("items_running")),
|
|
"items_completed": _int_value(item.get("items_completed")),
|
|
"items_blacklisted": _int_value(item.get("items_blacklisted")),
|
|
"items_failed": _int_value(item.get("items_failed")),
|
|
"metrics_source": str(item.get("metrics_source") or "").strip(),
|
|
"region": str(item.get("region") or "").strip(),
|
|
"role": str(item.get("role") or "").strip(),
|
|
"status": str(item.get("status") or "").strip(),
|
|
"current_load": _int_value(item.get("current_load")),
|
|
"last_heartbeat_at": str(item.get("last_heartbeat_at") or "").strip(),
|
|
}
|
|
|
|
|
|
def _build_effective_node_stats(
|
|
*,
|
|
distributed_node_stats: list[dict],
|
|
raw_items_total: int,
|
|
) -> list[dict]:
|
|
normalized_total = max(0, _int_value(raw_items_total))
|
|
assigned_buckets: list[dict] = []
|
|
assigned_total = 0
|
|
for item in list(distributed_node_stats or []):
|
|
bucket = _normalize_node_bucket(item)
|
|
if not bucket["node_code"] or bucket["node_code"] == "unassigned":
|
|
continue
|
|
assigned_buckets.append(bucket)
|
|
assigned_total += bucket["items_total"]
|
|
|
|
effective_total = max(normalized_total, assigned_total)
|
|
remainder_total = max(effective_total - assigned_total, 0)
|
|
if remainder_total > 0:
|
|
assigned_buckets.append(
|
|
{
|
|
"node_code": "unassigned",
|
|
"items_total": remainder_total,
|
|
"items_pending": remainder_total,
|
|
"items_claimed": 0,
|
|
"items_running": 0,
|
|
"items_completed": 0,
|
|
"items_blacklisted": 0,
|
|
"items_failed": 0,
|
|
"metrics_source": "central_queue",
|
|
"region": "",
|
|
"role": "",
|
|
"status": "",
|
|
"current_load": 0,
|
|
"last_heartbeat_at": "",
|
|
}
|
|
)
|
|
|
|
return sorted(
|
|
assigned_buckets,
|
|
key=lambda item: (
|
|
str(item.get("node_code") or "") == "unassigned",
|
|
-_int_value(item.get("items_running")),
|
|
-_int_value(item.get("items_claimed")),
|
|
-_int_value(item.get("items_total")),
|
|
str(item.get("node_code") or ""),
|
|
),
|
|
)
|
|
|
|
|
|
def _build_effective_summary(*, node_stats: list[dict], raw_items_total: int, raw_items_blacklisted: int) -> dict:
|
|
effective_total = max(0, _int_value(raw_items_total))
|
|
effective_blacklisted = max(0, _int_value(raw_items_blacklisted))
|
|
effective_pending = sum(_int_value(item.get("items_pending")) for item in list(node_stats or []))
|
|
effective_claimed = sum(_int_value(item.get("items_claimed")) for item in list(node_stats or []))
|
|
effective_running = sum(_int_value(item.get("items_running")) for item in list(node_stats or []))
|
|
effective_completed = sum(_int_value(item.get("items_completed")) for item in list(node_stats or []))
|
|
effective_failed = sum(_int_value(item.get("items_failed")) for item in list(node_stats or []))
|
|
effective_terminal = effective_completed + effective_blacklisted + effective_failed
|
|
return {
|
|
"items_total": effective_total,
|
|
"items_pending": max(effective_pending, 0),
|
|
"items_claimed": max(effective_claimed, 0),
|
|
"items_running": max(effective_running, 0),
|
|
"items_completed": max(effective_completed, 0),
|
|
"items_blacklisted": effective_blacklisted,
|
|
"items_failed": max(effective_failed, 0),
|
|
"items_terminal": max(effective_terminal, 0),
|
|
}
|
|
|
|
|
|
def _extract_current_cycle_events(events: list[dict]) -> tuple[str, list[dict]]:
|
|
if not events:
|
|
return "", []
|
|
|
|
latest_payload = events[0].get("payload") or {}
|
|
latest_cycle_token = str(latest_payload.get("cycle_token") or "").strip()
|
|
if latest_cycle_token:
|
|
current_cycle = []
|
|
for event in events:
|
|
payload = event.get("payload") or {}
|
|
event_cycle = str(payload.get("cycle_token") or "").strip()
|
|
if event_cycle == latest_cycle_token:
|
|
current_cycle.append(event)
|
|
return latest_cycle_token, current_cycle
|
|
|
|
cycle_token = ""
|
|
anchor_index = -1
|
|
for index, event in enumerate(events):
|
|
payload = event.get("payload") or {}
|
|
event_type = str(event.get("event_type") or "")
|
|
candidate = str(payload.get("cycle_token") or "").strip()
|
|
if candidate and event_type in {"job_dispatch_sent", "job_dispatch_requested", "job_dispatch_failed", "job_dispatch_rejected"}:
|
|
cycle_token = candidate
|
|
anchor_index = index
|
|
break
|
|
if not cycle_token:
|
|
return "", events
|
|
current_cycle = []
|
|
for index, event in enumerate(events[: anchor_index + 1]):
|
|
payload = event.get("payload") or {}
|
|
event_cycle = str(payload.get("cycle_token") or "").strip()
|
|
if index == anchor_index or event_cycle == cycle_token:
|
|
current_cycle.append(event)
|
|
return cycle_token, current_cycle
|
|
|
|
|
|
def _fetch_job_summary(cur, job_row, event_limit: int = 20) -> dict:
|
|
job_id = job_row[0]
|
|
cur.execute(
|
|
"""
|
|
SELECT status, count(*)
|
|
FROM detect_job_items
|
|
WHERE job_id = %s
|
|
GROUP BY status
|
|
""",
|
|
(job_id,),
|
|
)
|
|
counts = {status: int(count) for status, count in cur.fetchall()}
|
|
cur.execute(
|
|
"""
|
|
SELECT COALESCE(NULLIF(claimed_by, ''), 'unassigned') AS node_code,
|
|
status,
|
|
count(*)
|
|
FROM detect_job_items
|
|
WHERE job_id = %s
|
|
GROUP BY COALESCE(NULLIF(claimed_by, ''), 'unassigned'), status
|
|
ORDER BY node_code ASC, status ASC
|
|
""",
|
|
(job_id,),
|
|
)
|
|
node_buckets: dict[str, dict] = {}
|
|
for node_code, status, count in cur.fetchall():
|
|
bucket = node_buckets.setdefault(
|
|
node_code,
|
|
{
|
|
"node_code": node_code,
|
|
"items_total": 0,
|
|
"items_pending": 0,
|
|
"items_claimed": 0,
|
|
"items_running": 0,
|
|
"items_completed": 0,
|
|
"items_blacklisted": 0,
|
|
"items_failed": 0,
|
|
},
|
|
)
|
|
field_name = f"items_{status}"
|
|
if field_name in bucket:
|
|
bucket[field_name] += int(count)
|
|
bucket["items_total"] += int(count)
|
|
|
|
cur.execute(
|
|
"""
|
|
SELECT node_code, event_type, level, message, payload_json, created_at
|
|
FROM detect_run_events
|
|
WHERE job_id = %s
|
|
ORDER BY created_at DESC, id DESC
|
|
LIMIT %s
|
|
""",
|
|
(job_id, max(1, int(event_limit or 20))),
|
|
)
|
|
events = [
|
|
{
|
|
"node_code": item[0],
|
|
"event_type": item[1],
|
|
"level": item[2],
|
|
"message": item[3],
|
|
"payload": _decode_payload(item[4]),
|
|
"created_at": _format_time(item[5]),
|
|
}
|
|
for item in cur.fetchall()
|
|
]
|
|
cycle_token, current_cycle_events = _extract_current_cycle_events(events)
|
|
distributed_node_stats = _merge_display_node_stats(
|
|
local_node_stats=list(node_buckets.values()),
|
|
runtime_node_rows=_load_runtime_display_rows(cur),
|
|
)
|
|
raw_total = sum(counts.values())
|
|
raw_pending = int(counts.get("pending", 0))
|
|
raw_claimed = int(counts.get("claimed", 0))
|
|
raw_running = int(counts.get("running", 0))
|
|
raw_completed = int(counts.get("completed", 0))
|
|
raw_blacklisted = int(counts.get("blacklisted", 0))
|
|
raw_failed = int(counts.get("failed", 0))
|
|
raw_terminal = raw_completed + raw_blacklisted + raw_failed
|
|
effective_node_stats = _build_effective_node_stats(
|
|
distributed_node_stats=distributed_node_stats,
|
|
raw_items_total=raw_total,
|
|
)
|
|
effective_summary = _build_effective_summary(
|
|
node_stats=effective_node_stats,
|
|
raw_items_total=raw_total,
|
|
raw_items_blacklisted=raw_blacklisted,
|
|
)
|
|
display_summary = _build_display_summary(effective_node_stats)
|
|
return {
|
|
"job_id": job_id,
|
|
"job_code": job_row[1],
|
|
"source": job_row[2],
|
|
"status": job_row[3],
|
|
"created_by": job_row[4],
|
|
"created_at": _format_time(job_row[5]),
|
|
"started_at": _format_time(job_row[6]),
|
|
"finished_at": _format_time(job_row[7]),
|
|
"items_total": int(effective_summary.get("items_total", 0) or 0),
|
|
"items_pending": int(effective_summary.get("items_pending", 0) or 0),
|
|
"items_claimed": int(effective_summary.get("items_claimed", 0) or 0),
|
|
"items_running": int(effective_summary.get("items_running", 0) or 0),
|
|
"items_completed": int(effective_summary.get("items_completed", 0) or 0),
|
|
"items_blacklisted": int(effective_summary.get("items_blacklisted", 0) or 0),
|
|
"items_failed": int(effective_summary.get("items_failed", 0) or 0),
|
|
"items_terminal": int(effective_summary.get("items_terminal", 0) or 0),
|
|
"progress_percent": round(
|
|
(int(effective_summary.get("items_terminal", 0) or 0) / int(effective_summary.get("items_total", 0) or 0)) * 100,
|
|
2,
|
|
) if int(effective_summary.get("items_total", 0) or 0) else 0,
|
|
"raw_items_total": raw_total,
|
|
"raw_items_pending": raw_pending,
|
|
"raw_items_claimed": raw_claimed,
|
|
"raw_items_running": raw_running,
|
|
"raw_items_completed": raw_completed,
|
|
"raw_items_blacklisted": raw_blacklisted,
|
|
"raw_items_failed": raw_failed,
|
|
"raw_items_terminal": raw_terminal,
|
|
"raw_node_stats": list(node_buckets.values()),
|
|
"node_stats": effective_node_stats,
|
|
"distributed_node_stats": effective_node_stats,
|
|
"display_items_claimed": int(display_summary.get("items_claimed", 0) or 0),
|
|
"display_items_running": int(display_summary.get("items_running", 0) or 0),
|
|
"display_items_completed": int(display_summary.get("items_completed", 0) or 0),
|
|
"display_items_failed": int(display_summary.get("items_failed", 0) or 0),
|
|
"display_active_node_codes": list(display_summary.get("active_nodes") or []),
|
|
"recent_events": events,
|
|
"latest_event": events[0] if events else None,
|
|
"current_cycle_token": cycle_token,
|
|
"current_cycle_events": current_cycle_events,
|
|
"latest_cycle_event": current_cycle_events[0] if current_cycle_events else None,
|
|
}
|
|
|
|
|
|
def get_detect_job_summary(job_id: int, event_limit: int = 20) -> dict | None:
|
|
with get_db() as conn:
|
|
with conn.cursor() as cur:
|
|
cur.execute(
|
|
"""
|
|
SELECT id, job_code, source, status, created_by, created_at, started_at, finished_at
|
|
FROM detect_jobs
|
|
WHERE id = %s
|
|
LIMIT 1
|
|
""",
|
|
(int(job_id),),
|
|
)
|
|
row = cur.fetchone()
|
|
if not row:
|
|
return None
|
|
return _fetch_job_summary(cur, row, event_limit=event_limit)
|
|
|
|
|
|
def get_active_detect_job_summary(event_limit: int = 20) -> dict | None:
|
|
with get_db() as conn:
|
|
with conn.cursor() as cur:
|
|
cur.execute(
|
|
"""
|
|
SELECT id, job_code, source, status, created_by, created_at, started_at, finished_at
|
|
FROM detect_jobs
|
|
WHERE status IN ('pending', 'running')
|
|
ORDER BY created_at DESC, id DESC
|
|
LIMIT 1
|
|
"""
|
|
)
|
|
row = cur.fetchone()
|
|
if not row:
|
|
return None
|
|
return _fetch_job_summary(cur, row, event_limit=event_limit)
|
|
|
|
|
|
def list_detect_jobs(limit: int = 20) -> list[dict]:
|
|
with get_db() as conn:
|
|
with conn.cursor() as cur:
|
|
cur.execute(
|
|
"""
|
|
SELECT id, job_code, source, status, created_by, created_at, started_at, finished_at
|
|
FROM detect_jobs
|
|
ORDER BY created_at DESC, id DESC
|
|
LIMIT %s
|
|
""",
|
|
(max(1, min(int(limit or 20), 100)),),
|
|
)
|
|
rows = cur.fetchall()
|
|
return [_fetch_job_summary(cur, row, event_limit=10) for row in rows]
|
|
|
|
|
|
def list_recent_detect_run_events(limit: int = 20) -> list[dict]:
|
|
safe_limit = max(1, min(int(limit or 20), 200))
|
|
with get_db() as conn:
|
|
with conn.cursor() as cur:
|
|
cur.execute(
|
|
"""
|
|
SELECT job_id, node_code, event_type, level, message, payload_json, created_at
|
|
FROM detect_run_events
|
|
ORDER BY created_at DESC, id DESC
|
|
LIMIT %s
|
|
""",
|
|
(safe_limit,),
|
|
)
|
|
rows = cur.fetchall()
|
|
return [
|
|
{
|
|
"job_id": int(row[0]) if row[0] is not None else None,
|
|
"node_code": row[1] or "",
|
|
"event_type": row[2] or "",
|
|
"level": row[3] or "info",
|
|
"message": row[4] or "",
|
|
"payload": _decode_payload(row[5]),
|
|
"created_at": _format_time(row[6]),
|
|
}
|
|
for row in rows
|
|
]
|
|
|
|
|
|
def get_detect_queue_health(window_minutes: int = 15) -> dict:
|
|
window_minutes = max(5, min(int(window_minutes or 15), 120))
|
|
active_job = get_active_detect_job_summary(event_limit=10)
|
|
if not active_job:
|
|
return {
|
|
"window_minutes": window_minutes,
|
|
"has_active_job": False,
|
|
"job": None,
|
|
"queue": {
|
|
"items_total": 0,
|
|
"pending": 0,
|
|
"claimed": 0,
|
|
"running": 0,
|
|
"completed": 0,
|
|
"blacklisted": 0,
|
|
"failed": 0,
|
|
"terminal": 0,
|
|
"terminal_percent": 0,
|
|
"oldest_pending_at": "",
|
|
"oldest_pending_age_minutes": 0,
|
|
"nearest_lease_expiry_at": "",
|
|
"overdue_leases": 0,
|
|
"expiring_soon_leases": 0,
|
|
},
|
|
"throughput": {
|
|
"processed_recent": 0,
|
|
"processed_per_minute": 0,
|
|
"completed_recent": 0,
|
|
"blacklisted_recent": 0,
|
|
"failed_recent": 0,
|
|
},
|
|
"nodes": [],
|
|
}
|
|
|
|
job_id = int(active_job["job_id"])
|
|
with get_db() as conn:
|
|
with conn.cursor() as cur:
|
|
cur.execute(
|
|
"""
|
|
SELECT
|
|
MIN(create_time) FILTER (WHERE status = 'pending') AS oldest_pending_at,
|
|
MIN(lease_expires_at) FILTER (WHERE status IN ('claimed', 'running') AND lease_expires_at IS NOT NULL) AS nearest_lease_expiry_at,
|
|
COUNT(*) FILTER (WHERE status IN ('claimed', 'running') AND lease_expires_at IS NOT NULL AND lease_expires_at < CURRENT_TIMESTAMP) AS overdue_leases,
|
|
COUNT(*) FILTER (
|
|
WHERE status IN ('claimed', 'running')
|
|
AND lease_expires_at IS NOT NULL
|
|
AND lease_expires_at >= CURRENT_TIMESTAMP
|
|
AND lease_expires_at < CURRENT_TIMESTAMP + interval '5 minutes'
|
|
) AS expiring_soon_leases
|
|
FROM detect_job_items
|
|
WHERE job_id = %s
|
|
""",
|
|
(job_id,),
|
|
)
|
|
lease_row = cur.fetchone()
|
|
|
|
cur.execute(
|
|
"""
|
|
SELECT
|
|
COALESCE(NULLIF(node_code, ''), 'unknown') AS node_code,
|
|
COUNT(*) AS processed_recent,
|
|
COUNT(*) FILTER (WHERE event_type = 'domain_completed') AS completed_recent,
|
|
COUNT(*) FILTER (WHERE event_type = 'domain_blacklisted') AS blacklisted_recent,
|
|
COUNT(*) FILTER (WHERE event_type = 'domain_failed') AS failed_recent
|
|
FROM detect_run_events
|
|
WHERE job_id = %s
|
|
AND event_type IN ('domain_completed', 'domain_blacklisted', 'domain_failed')
|
|
AND created_at >= CURRENT_TIMESTAMP - (%s || ' minutes')::interval
|
|
GROUP BY COALESCE(NULLIF(node_code, ''), 'unknown')
|
|
ORDER BY processed_recent DESC, node_code ASC
|
|
""",
|
|
(job_id, window_minutes),
|
|
)
|
|
throughput_rows = cur.fetchall()
|
|
|
|
oldest_pending_at = _format_time(lease_row[0]) if lease_row and lease_row[0] else ""
|
|
nearest_lease_expiry_at = _format_time(lease_row[1]) if lease_row and lease_row[1] else ""
|
|
oldest_pending_age_minutes = 0
|
|
if lease_row and lease_row[0]:
|
|
oldest_pending_age_minutes = max(0, int((datetime.now() - lease_row[0]).total_seconds() // 60))
|
|
|
|
node_map = {
|
|
str(item.get("node_code") or "unknown"): {
|
|
"node_code": str(item.get("node_code") or "unknown"),
|
|
"items_total": int(item.get("items_total", 0) or 0),
|
|
"items_pending": int(item.get("items_pending", 0) or 0),
|
|
"items_claimed": int(item.get("items_claimed", 0) or 0),
|
|
"items_running": int(item.get("items_running", 0) or 0),
|
|
"items_completed": int(item.get("items_completed", 0) or 0),
|
|
"items_blacklisted": int(item.get("items_blacklisted", 0) or 0),
|
|
"items_failed": int(item.get("items_failed", 0) or 0),
|
|
"processed_recent": 0,
|
|
"processed_per_minute": 0,
|
|
"completed_recent": 0,
|
|
"blacklisted_recent": 0,
|
|
"failed_recent": 0,
|
|
}
|
|
for item in active_job.get("node_stats") or []
|
|
}
|
|
distributed_nodes = list(active_job.get("distributed_node_stats") or [])
|
|
if distributed_nodes:
|
|
node_map = {
|
|
str(item.get("node_code") or "unknown"): {
|
|
"node_code": str(item.get("node_code") or "unknown"),
|
|
"items_total": _int_value(item.get("items_total")),
|
|
"items_pending": _int_value(item.get("items_pending")),
|
|
"items_claimed": _int_value(item.get("items_claimed")),
|
|
"items_running": _int_value(item.get("items_running")),
|
|
"items_completed": _int_value(item.get("items_completed")),
|
|
"items_blacklisted": _int_value(item.get("items_blacklisted")),
|
|
"items_failed": _int_value(item.get("items_failed")),
|
|
"processed_recent": 0,
|
|
"processed_per_minute": 0,
|
|
"completed_recent": 0,
|
|
"blacklisted_recent": 0,
|
|
"failed_recent": 0,
|
|
"metrics_source": str(item.get("metrics_source") or "").strip(),
|
|
}
|
|
for item in distributed_nodes
|
|
}
|
|
total_processed_recent = 0
|
|
total_completed_recent = 0
|
|
total_blacklisted_recent = 0
|
|
total_failed_recent = 0
|
|
for row in throughput_rows:
|
|
node_code = str(row[0] or "unknown")
|
|
bucket = node_map.setdefault(
|
|
node_code,
|
|
{
|
|
"node_code": node_code,
|
|
"items_total": 0,
|
|
"items_pending": 0,
|
|
"items_claimed": 0,
|
|
"items_running": 0,
|
|
"items_completed": 0,
|
|
"items_blacklisted": 0,
|
|
"items_failed": 0,
|
|
"processed_recent": 0,
|
|
"processed_per_minute": 0,
|
|
"completed_recent": 0,
|
|
"blacklisted_recent": 0,
|
|
"failed_recent": 0,
|
|
},
|
|
)
|
|
processed_recent = int(row[1] or 0)
|
|
completed_recent = int(row[2] or 0)
|
|
blacklisted_recent = int(row[3] or 0)
|
|
failed_recent = int(row[4] or 0)
|
|
bucket["processed_recent"] = processed_recent
|
|
bucket["processed_per_minute"] = round(processed_recent / window_minutes, 2)
|
|
bucket["completed_recent"] = completed_recent
|
|
bucket["blacklisted_recent"] = blacklisted_recent
|
|
bucket["failed_recent"] = failed_recent
|
|
total_processed_recent += processed_recent
|
|
total_completed_recent += completed_recent
|
|
total_blacklisted_recent += blacklisted_recent
|
|
total_failed_recent += failed_recent
|
|
|
|
nodes = sorted(
|
|
node_map.values(),
|
|
key=lambda item: (
|
|
-int(item.get("processed_recent", 0) or 0),
|
|
-int(item.get("items_running", 0) or 0),
|
|
str(item.get("node_code") or ""),
|
|
),
|
|
)
|
|
items_total = int(active_job.get("items_total", 0) or 0)
|
|
terminal = int(active_job.get("items_terminal", 0) or 0)
|
|
return {
|
|
"window_minutes": window_minutes,
|
|
"has_active_job": True,
|
|
"job": {
|
|
"job_id": job_id,
|
|
"job_code": active_job.get("job_code", ""),
|
|
"status": active_job.get("status", ""),
|
|
"progress_percent": active_job.get("progress_percent", 0),
|
|
},
|
|
"queue": {
|
|
"items_total": items_total,
|
|
"pending": int(active_job.get("items_pending", 0) or 0),
|
|
"claimed": int(active_job.get("items_claimed", 0) or 0),
|
|
"running": int(active_job.get("items_running", 0) or 0),
|
|
"display_claimed": int(active_job.get("display_items_claimed", active_job.get("items_claimed", 0)) or 0),
|
|
"display_running": int(active_job.get("display_items_running", active_job.get("items_running", 0)) or 0),
|
|
"completed": int(active_job.get("items_completed", 0) or 0),
|
|
"blacklisted": int(active_job.get("items_blacklisted", 0) or 0),
|
|
"failed": int(active_job.get("items_failed", 0) or 0),
|
|
"terminal": terminal,
|
|
"terminal_percent": round((terminal / items_total) * 100, 2) if items_total else 0,
|
|
"oldest_pending_at": oldest_pending_at,
|
|
"oldest_pending_age_minutes": oldest_pending_age_minutes,
|
|
"nearest_lease_expiry_at": nearest_lease_expiry_at,
|
|
"overdue_leases": int(lease_row[2] or 0) if lease_row else 0,
|
|
"expiring_soon_leases": int(lease_row[3] or 0) if lease_row else 0,
|
|
},
|
|
"throughput": {
|
|
"processed_recent": total_processed_recent,
|
|
"processed_per_minute": round(total_processed_recent / window_minutes, 2),
|
|
"completed_recent": total_completed_recent,
|
|
"blacklisted_recent": total_blacklisted_recent,
|
|
"failed_recent": total_failed_recent,
|
|
},
|
|
"nodes": nodes,
|
|
}
|
|
|
|
|
|
def get_detect_capacity_plan(*, queue_health: dict | None = None, online_worker_nodes: int = 0, target_finish_hours: int = 6) -> dict:
|
|
queue_health = queue_health or get_detect_queue_health(window_minutes=15)
|
|
target_finish_hours = max(1, min(int(target_finish_hours or 6), 72))
|
|
online_worker_nodes = max(0, int(online_worker_nodes or 0))
|
|
|
|
if not queue_health.get("has_active_job"):
|
|
return {
|
|
"has_active_job": False,
|
|
"online_worker_nodes": online_worker_nodes,
|
|
"target_finish_hours": target_finish_hours,
|
|
"estimated_hours_remaining": 0,
|
|
"recommended_total_workers": max(1, online_worker_nodes),
|
|
"recommended_additional_workers": 0,
|
|
"current_processed_per_hour": 0,
|
|
"pending_items": 0,
|
|
"terminal_items": 0,
|
|
"summary": "当前没有活跃任务，无需扩容建议。",
|
|
}
|
|
|
|
queue = queue_health.get("queue") or {}
|
|
throughput = queue_health.get("throughput") or {}
|
|
pending_items = int(queue.get("pending", 0) or 0)
|
|
claimed_items = int(queue.get("claimed", 0) or 0)
|
|
running_items = int(queue.get("running", 0) or 0)
|
|
remaining_items = pending_items + claimed_items + running_items
|
|
current_processed_per_hour = round(float(throughput.get("processed_per_minute", 0) or 0) * 60, 2)
|
|
estimated_hours_remaining = round((remaining_items / current_processed_per_hour), 2) if current_processed_per_hour > 0 else 0
|
|
|
|
recommended_total_workers = max(1, online_worker_nodes or 1)
|
|
recommended_additional_workers = 0
|
|
if remaining_items > 0 and target_finish_hours > 0:
|
|
required_per_hour = remaining_items / target_finish_hours
|
|
if current_processed_per_hour > 0 and max(1, online_worker_nodes) > 0:
|
|
per_worker_per_hour = current_processed_per_hour / max(1, online_worker_nodes)
|
|
recommended_total_workers = max(1, int(math.ceil(required_per_hour / per_worker_per_hour)))
|
|
recommended_additional_workers = max(0, recommended_total_workers - online_worker_nodes)
|
|
elif remaining_items > 0:
|
|
recommended_total_workers = max(1, online_worker_nodes or 1)
|
|
recommended_additional_workers = 0
|
|
|
|
summary = (
|
|
f"当前在线 Worker {online_worker_nodes} 台，近窗吞吐约 {current_processed_per_hour} 项/小时，"
|
|
f"剩余待处理约 {remaining_items} 项，预计还需 {estimated_hours_remaining} 小时。"
|
|
)
|
|
if recommended_additional_workers > 0:
|
|
summary = (
|
|
f"{summary} 若希望在 {target_finish_hours} 小时内收敛，建议总 Worker 数达到 "
|
|
f"{recommended_total_workers} 台，至少再加 {recommended_additional_workers} 台。"
|
|
)
|
|
else:
|
|
summary = f"{summary} 按当前目标 {target_finish_hours} 小时看，现有 Worker 数量暂时够用。"
|
|
|
|
return {
|
|
"has_active_job": True,
|
|
"online_worker_nodes": online_worker_nodes,
|
|
"target_finish_hours": target_finish_hours,
|
|
"estimated_hours_remaining": estimated_hours_remaining,
|
|
"recommended_total_workers": recommended_total_workers,
|
|
"recommended_additional_workers": recommended_additional_workers,
|
|
"current_processed_per_hour": current_processed_per_hour,
|
|
"pending_items": pending_items,
|
|
"remaining_items": remaining_items,
|
|
"terminal_items": int(queue.get("terminal", 0) or 0),
|
|
"summary": summary,
|
|
}
|
|
|
|
|
|
def append_detect_job_event(
|
|
job_id: int,
|
|
*,
|
|
event_type: str,
|
|
message: str,
|
|
level: str = "info",
|
|
payload: dict | None = None,
|
|
node_code: str | None = None,
|
|
) -> None:
|
|
with get_db() as conn:
|
|
with conn.cursor() as cur:
|
|
cur.execute(
|
|
"""
|
|
INSERT INTO detect_run_events (job_id, node_code, event_type, level, message, payload_json)
|
|
VALUES (%s, %s, %s, %s, %s, %s::jsonb)
|
|
""",
|
|
(
|
|
int(job_id),
|
|
node_code or settings.node_code,
|
|
str(event_type or "").strip() or "info",
|
|
str(level or "info").strip() or "info",
|
|
str(message or "").strip(),
|
|
json.dumps(payload or {}, ensure_ascii=False),
|
|
),
|
|
)
|
|
conn.commit()
|
|
try:
|
|
push_debug_event(
|
|
service="detect-job",
|
|
event_type=str(event_type or "").strip() or "info",
|
|
level=str(level or "info").strip() or "info",
|
|
message=str(message or "").strip(),
|
|
payload={
|
|
"job_id": int(job_id),
|
|
"node_code": node_code or settings.node_code,
|
|
**(payload or {}),
|
|
},
|
|
)
|
|
except Exception:
|
|
pass
|
|
|
|
|
|
def create_detect_job_if_needed(limit: int = 1000, created_by: str = "system") -> dict | None:
|
|
existing = get_active_detect_job_summary()
|
|
if existing:
|
|
return existing
|
|
|
|
with get_db() as conn:
|
|
conn.autocommit = False
|
|
with conn.cursor() as cur:
|
|
cur.execute(_selection_sql(), (max(1, int(limit or 1000)),))
|
|
domain_ids = [row[0] for row in cur.fetchall()]
|
|
if not domain_ids:
|
|
conn.rollback()
|
|
|
|
if not domain_ids:
|
|
try:
|
|
from app.services.sync_push_service import pull_detect_task_batch_now
|
|
|
|
pull_detect_task_batch_now(limit=max(1, int(limit or 1000)))
|
|
except Exception:
|
|
pass
|
|
|
|
with get_db() as conn:
|
|
conn.autocommit = False
|
|
with conn.cursor() as cur:
|
|
cur.execute(_selection_sql(), (max(1, int(limit or 1000)),))
|
|
domain_ids = [row[0] for row in cur.fetchall()]
|
|
if not domain_ids:
|
|
conn.rollback()
|
|
return None
|
|
|
|
with get_db() as conn:
|
|
conn.autocommit = False
|
|
with conn.cursor() as cur:
|
|
job_code = f"detect-{datetime.now().strftime('%Y%m%d%H%M%S')}-{uuid4().hex[:6]}"
|
|
plan_hash = uuid4().hex
|
|
cur.execute(
|
|
"""
|
|
INSERT INTO detect_jobs (job_code, source, plan_hash, status, remark, created_by)
|
|
VALUES (%s, %s, %s, 'pending', %s, %s)
|
|
RETURNING id
|
|
""",
|
|
(
|
|
job_code,
|
|
"api-start",
|
|
plan_hash,
|
|
f"API 创建检测任务，待检测域名 {len(domain_ids)} 个",
|
|
created_by,
|
|
),
|
|
)
|
|
job_id = cur.fetchone()[0]
|
|
item_rows = [(job_id, domain_id) for domain_id in domain_ids]
|
|
cur.executemany(
|
|
"""
|
|
INSERT INTO detect_job_items (job_id, domain_id, status)
|
|
VALUES (%s, %s, 'pending')
|
|
ON CONFLICT (job_id, domain_id) DO NOTHING
|
|
""",
|
|
item_rows,
|
|
)
|
|
cur.execute(
|
|
"""
|
|
INSERT INTO detect_run_events (job_id, node_code, event_type, level, message, payload_json)
|
|
VALUES (%s, %s, %s, %s, %s, %s::jsonb)
|
|
""",
|
|
(
|
|
job_id,
|
|
settings.node_code,
|
|
"job_created",
|
|
"info",
|
|
f"创建检测任务 {job_code}，共 {len(domain_ids)} 个域名",
|
|
'{"count": %s}' % len(domain_ids),
|
|
),
|
|
)
|
|
conn.commit()
|
|
return get_active_detect_job_summary()
|