d
This commit is contained in:
Your Name
@@ -1,10 +1,390 @@
|
||||
from __future__ import annotations
|
||||
|
||||
from app.core.db import get_db
|
||||
from app.services.detect_job_service import (
|
||||
_build_step_bucket,
|
||||
get_active_detect_job_summary,
|
||||
get_detect_capacity_plan,
|
||||
get_detect_queue_health,
|
||||
)
|
||||
from app.services.runtime_status_service import get_runtime_status
|
||||
|
||||
|
||||
def _empty_active_jobs_aggregate(window_minutes: int) -> dict:
|
||||
return {
|
||||
"window_minutes": int(window_minutes or 15),
|
||||
"active_jobs_total": 0,
|
||||
"queue": {
|
||||
"items_total": 0,
|
||||
"pending": 0,
|
||||
"claimed": 0,
|
||||
"running": 0,
|
||||
"completed": 0,
|
||||
"blacklisted": 0,
|
||||
"failed": 0,
|
||||
"terminal": 0,
|
||||
},
|
||||
"throughput": {
|
||||
"processed_recent": 0,
|
||||
"processed_per_minute": 0,
|
||||
"completed_recent": 0,
|
||||
"blacklisted_recent": 0,
|
||||
"failed_recent": 0,
|
||||
},
|
||||
"steps": [],
|
||||
"nodes": [],
|
||||
"retry_total": 0,
|
||||
}
|
||||
|
||||
|
||||
def _merge_step_queues_with_runtime_activity(
|
||||
base_steps: list[dict] | None,
|
||||
*,
|
||||
runtime_activity: dict | None = None,
|
||||
window_minutes: int = 15,
|
||||
limit: int = 8,
|
||||
) -> list[dict]:
|
||||
safe_window_minutes = max(1, int(window_minutes or 15))
|
||||
normalized_limit = max(1, int(limit or 8))
|
||||
step_map: dict[str, dict] = {}
|
||||
|
||||
for item in list(base_steps or []):
|
||||
step_code = str(item.get("step_code") or "").strip()
|
||||
if not step_code:
|
||||
continue
|
||||
bucket = _build_step_bucket(step_code)
|
||||
bucket.update(
|
||||
{
|
||||
"items_total": int(item.get("items_total", 0) or 0),
|
||||
"items_pending": int(item.get("items_pending", 0) or 0),
|
||||
"items_claimed": int(item.get("items_claimed", 0) or 0),
|
||||
"items_running": int(item.get("items_running", 0) or 0),
|
||||
"items_completed": int(item.get("items_completed", 0) or 0),
|
||||
"items_blacklisted": int(item.get("items_blacklisted", 0) or 0),
|
||||
"items_failed": int(item.get("items_failed", 0) or 0),
|
||||
"started_recent": int(item.get("started_recent", 0) or 0),
|
||||
"processed_recent": int(item.get("processed_recent", 0) or 0),
|
||||
"processed_per_minute": float(item.get("processed_per_minute", 0) or 0),
|
||||
"completed_recent": int(item.get("completed_recent", 0) or 0),
|
||||
"blacklisted_recent": int(item.get("blacklisted_recent", 0) or 0),
|
||||
"failed_recent": int(item.get("failed_recent", 0) or 0),
|
||||
}
|
||||
)
|
||||
step_map[step_code] = bucket
|
||||
|
||||
runtime_step_stats = dict((runtime_activity or {}).get("step_stats") or {})
|
||||
for step_code, stats in runtime_step_stats.items():
|
||||
normalized_step_code = str(step_code or "").strip()
|
||||
if not normalized_step_code:
|
||||
continue
|
||||
bucket = step_map.setdefault(normalized_step_code, _build_step_bucket(normalized_step_code))
|
||||
started_recent = int((stats or {}).get("started_recent", 0) or 0)
|
||||
processed_recent = int((stats or {}).get("processed_recent", 0) or 0)
|
||||
completed_recent = int((stats or {}).get("completed_recent", 0) or 0)
|
||||
blacklisted_recent = int((stats or {}).get("blacklisted_recent", 0) or 0)
|
||||
failed_recent = int((stats or {}).get("failed_recent", 0) or 0)
|
||||
bucket["started_recent"] = max(int(bucket.get("started_recent", 0) or 0), started_recent)
|
||||
bucket["processed_recent"] = max(int(bucket.get("processed_recent", 0) or 0), processed_recent)
|
||||
bucket["completed_recent"] = max(int(bucket.get("completed_recent", 0) or 0), completed_recent)
|
||||
bucket["blacklisted_recent"] = max(int(bucket.get("blacklisted_recent", 0) or 0), blacklisted_recent)
|
||||
bucket["failed_recent"] = max(int(bucket.get("failed_recent", 0) or 0), failed_recent)
|
||||
bucket["processed_per_minute"] = max(
|
||||
float(bucket.get("processed_per_minute", 0) or 0),
|
||||
round(processed_recent / safe_window_minutes, 2),
|
||||
)
|
||||
|
||||
return sorted(
|
||||
step_map.values(),
|
||||
key=lambda item: (
|
||||
-int(item.get("items_pending", 0) or 0),
|
||||
-int(item.get("items_running", 0) or 0),
|
||||
-int(item.get("started_recent", 0) or 0),
|
||||
-int(item.get("processed_recent", 0) or 0),
|
||||
str(item.get("step_code") or ""),
|
||||
),
|
||||
)[:normalized_limit]
|
||||
|
||||
|
||||
def _align_active_jobs_aggregate_with_runtime(
|
||||
aggregate: dict,
|
||||
*,
|
||||
runtime: dict,
|
||||
queue_health: dict,
|
||||
) -> dict:
|
||||
normalized = dict(aggregate or {})
|
||||
node_payload = dict((runtime or {}).get("node") or {})
|
||||
if str(node_payload.get("region") or "").strip() != "overseas" or str(node_payload.get("role") or "").strip() != "control":
|
||||
return normalized
|
||||
|
||||
backlog = dict(((runtime or {}).get("detect") or {}).get("backlog") or {})
|
||||
snapshot_backlog = dict(queue_health.get("runtime_snapshot_backlog") or {})
|
||||
|
||||
def _backlog_value(key: str) -> int:
|
||||
return max(int(backlog.get(key, 0) or 0), int(snapshot_backlog.get(key, 0) or 0))
|
||||
|
||||
pending_total = _backlog_value("pending_total")
|
||||
claimed_total = _backlog_value("claimed_total")
|
||||
running_total = _backlog_value("running_total")
|
||||
completed_total = _backlog_value("completed_total")
|
||||
blacklisted_total = _backlog_value("blacklisted_total")
|
||||
failed_total = _backlog_value("failed_total")
|
||||
|
||||
queue = dict(queue_health.get("queue") or {})
|
||||
throughput = dict(queue_health.get("throughput") or {})
|
||||
terminal_total = max(
|
||||
completed_total + blacklisted_total + failed_total,
|
||||
int(queue.get("completed", 0) or 0) + int(queue.get("blacklisted", 0) or 0) + int(queue.get("failed", 0) or 0),
|
||||
)
|
||||
items_total = pending_total + claimed_total + running_total + terminal_total
|
||||
has_runtime_work = items_total > 0 or bool(queue_health.get("has_active_job"))
|
||||
|
||||
normalized["active_jobs_total"] = max(
|
||||
int(normalized.get("active_jobs_total", 0) or 0),
|
||||
1 if has_runtime_work else 0,
|
||||
)
|
||||
normalized["queue"] = {
|
||||
"items_total": items_total,
|
||||
"pending": pending_total,
|
||||
"claimed": claimed_total,
|
||||
"running": running_total,
|
||||
"completed": max(completed_total, int(queue.get("completed", 0) or 0)),
|
||||
"blacklisted": max(blacklisted_total, int(queue.get("blacklisted", 0) or 0)),
|
||||
"failed": max(failed_total, int(queue.get("failed", 0) or 0)),
|
||||
"terminal": terminal_total,
|
||||
}
|
||||
normalized["throughput"] = {
|
||||
"processed_recent": int(throughput.get("processed_recent", 0) or 0),
|
||||
"processed_per_minute": float(throughput.get("processed_per_minute", 0) or 0),
|
||||
"completed_recent": int(throughput.get("completed_recent", 0) or 0),
|
||||
"blacklisted_recent": int(throughput.get("blacklisted_recent", 0) or 0),
|
||||
"failed_recent": int(throughput.get("failed_recent", 0) or 0),
|
||||
}
|
||||
normalized["steps"] = _merge_step_queues_with_runtime_activity(
|
||||
list(queue_health.get("steps") or []),
|
||||
runtime_activity=dict(queue_health.get("runtime_activity") or {}),
|
||||
window_minutes=int(queue_health.get("window_minutes", normalized.get("window_minutes", 15)) or 15),
|
||||
limit=8,
|
||||
)
|
||||
normalized["nodes"] = list(queue_health.get("nodes") or [])
|
||||
return normalized
|
||||
|
||||
|
||||
def _fetch_active_jobs_aggregate(window_minutes: int = 15) -> dict:
|
||||
safe_window_minutes = max(5, min(int(window_minutes or 15), 120))
|
||||
payload = _empty_active_jobs_aggregate(safe_window_minutes)
|
||||
with get_db() as conn:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(
|
||||
"""
|
||||
SELECT id
|
||||
FROM detect_jobs
|
||||
WHERE status IN ('pending', 'running', 'partial_failed')
|
||||
ORDER BY
|
||||
CASE
|
||||
WHEN status = 'running' THEN 0
|
||||
WHEN status = 'pending' THEN 1
|
||||
WHEN status = 'partial_failed' THEN 2
|
||||
ELSE 3
|
||||
END ASC,
|
||||
COALESCE(started_at, created_at) DESC,
|
||||
id DESC
|
||||
"""
|
||||
)
|
||||
job_ids = [int(row[0]) for row in cur.fetchall() if row and row[0] is not None]
|
||||
if not job_ids:
|
||||
return payload
|
||||
|
||||
payload["active_jobs_total"] = len(job_ids)
|
||||
|
||||
cur.execute(
|
||||
"""
|
||||
SELECT
|
||||
COUNT(*) AS items_total,
|
||||
COUNT(*) FILTER (WHERE status = 'pending') AS items_pending,
|
||||
COUNT(*) FILTER (WHERE status = 'claimed') AS items_claimed,
|
||||
COUNT(*) FILTER (WHERE status = 'running') AS items_running,
|
||||
COUNT(*) FILTER (WHERE status = 'completed') AS items_completed,
|
||||
COUNT(*) FILTER (WHERE status = 'blacklisted') AS items_blacklisted,
|
||||
COUNT(*) FILTER (WHERE status = 'failed') AS items_failed
|
||||
FROM detect_job_items
|
||||
WHERE job_id = ANY(%s)
|
||||
""",
|
||||
(job_ids,),
|
||||
)
|
||||
queue_row = cur.fetchone() or (0, 0, 0, 0, 0, 0, 0)
|
||||
payload["queue"] = {
|
||||
"items_total": int(queue_row[0] or 0),
|
||||
"pending": int(queue_row[1] or 0),
|
||||
"claimed": int(queue_row[2] or 0),
|
||||
"running": int(queue_row[3] or 0),
|
||||
"completed": int(queue_row[4] or 0),
|
||||
"blacklisted": int(queue_row[5] or 0),
|
||||
"failed": int(queue_row[6] or 0),
|
||||
"terminal": int(queue_row[4] or 0) + int(queue_row[5] or 0) + int(queue_row[6] or 0),
|
||||
}
|
||||
|
||||
cur.execute(
|
||||
"""
|
||||
SELECT
|
||||
COUNT(*) AS processed_recent,
|
||||
COUNT(*) FILTER (WHERE event_type = 'domain_completed') AS completed_recent,
|
||||
COUNT(*) FILTER (WHERE event_type = 'domain_blacklisted') AS blacklisted_recent,
|
||||
COUNT(*) FILTER (WHERE event_type = 'domain_failed') AS failed_recent
|
||||
FROM detect_run_events
|
||||
WHERE job_id = ANY(%s)
|
||||
AND event_type IN ('domain_completed', 'domain_blacklisted', 'domain_failed')
|
||||
AND created_at >= CURRENT_TIMESTAMP - (%s || ' minutes')::interval
|
||||
""",
|
||||
(job_ids, safe_window_minutes),
|
||||
)
|
||||
throughput_row = cur.fetchone() or (0, 0, 0, 0)
|
||||
processed_recent = int(throughput_row[0] or 0)
|
||||
payload["throughput"] = {
|
||||
"processed_recent": processed_recent,
|
||||
"processed_per_minute": round(processed_recent / safe_window_minutes, 2),
|
||||
"completed_recent": int(throughput_row[1] or 0),
|
||||
"blacklisted_recent": int(throughput_row[2] or 0),
|
||||
"failed_recent": int(throughput_row[3] or 0),
|
||||
}
|
||||
|
||||
cur.execute(
|
||||
"""
|
||||
SELECT COUNT(*)
|
||||
FROM detect_job_items
|
||||
WHERE job_id = ANY(%s)
|
||||
AND attempt_count > 1
|
||||
""",
|
||||
(job_ids,),
|
||||
)
|
||||
payload["retry_total"] = int((cur.fetchone() or [0])[0] or 0)
|
||||
|
||||
cur.execute(
|
||||
"""
|
||||
SELECT
|
||||
COALESCE(NULLIF(step_code, ''), 'domain_pipeline') AS step_code,
|
||||
COUNT(*) AS items_total,
|
||||
COUNT(*) FILTER (WHERE status = 'pending') AS items_pending,
|
||||
COUNT(*) FILTER (WHERE status = 'claimed') AS items_claimed,
|
||||
COUNT(*) FILTER (WHERE status = 'running') AS items_running,
|
||||
COUNT(*) FILTER (WHERE status = 'completed') AS items_completed,
|
||||
COUNT(*) FILTER (WHERE status = 'blacklisted') AS items_blacklisted,
|
||||
COUNT(*) FILTER (WHERE status = 'failed') AS items_failed,
|
||||
COUNT(*) FILTER (
|
||||
WHERE status IN ('completed', 'blacklisted', 'failed')
|
||||
AND finished_at >= CURRENT_TIMESTAMP - (%s || ' minutes')::interval
|
||||
) AS processed_recent
|
||||
FROM detect_job_items
|
||||
WHERE job_id = ANY(%s)
|
||||
GROUP BY COALESCE(NULLIF(step_code, ''), 'domain_pipeline')
|
||||
ORDER BY
|
||||
COUNT(*) FILTER (WHERE status = 'pending') DESC,
|
||||
COUNT(*) FILTER (WHERE status = 'running') DESC,
|
||||
COUNT(*) DESC,
|
||||
COALESCE(NULLIF(step_code, ''), 'domain_pipeline') ASC
|
||||
LIMIT 8
|
||||
""",
|
||||
(safe_window_minutes, job_ids),
|
||||
)
|
||||
steps: list[dict] = []
|
||||
for row in cur.fetchall():
|
||||
step_code = str(row[0] or "domain_pipeline")
|
||||
bucket = _build_step_bucket(step_code)
|
||||
bucket.update(
|
||||
{
|
||||
"items_total": int(row[1] or 0),
|
||||
"items_pending": int(row[2] or 0),
|
||||
"items_claimed": int(row[3] or 0),
|
||||
"items_running": int(row[4] or 0),
|
||||
"items_completed": int(row[5] or 0),
|
||||
"items_blacklisted": int(row[6] or 0),
|
||||
"items_failed": int(row[7] or 0),
|
||||
"processed_recent": int(row[8] or 0),
|
||||
"processed_per_minute": round(int(row[8] or 0) / safe_window_minutes, 2),
|
||||
}
|
||||
)
|
||||
steps.append(bucket)
|
||||
payload["steps"] = steps
|
||||
|
||||
cur.execute(
|
||||
"""
|
||||
SELECT
|
||||
COALESCE(NULLIF(claimed_by, ''), 'unassigned') AS node_code,
|
||||
COUNT(*) FILTER (WHERE status = 'claimed') AS items_claimed,
|
||||
COUNT(*) FILTER (WHERE status = 'running') AS items_running,
|
||||
COUNT(*) FILTER (WHERE status = 'completed') AS items_completed_total
|
||||
FROM detect_job_items
|
||||
WHERE job_id = ANY(%s)
|
||||
GROUP BY COALESCE(NULLIF(claimed_by, ''), 'unassigned')
|
||||
""",
|
||||
(job_ids,),
|
||||
)
|
||||
node_map = {
|
||||
str(row[0] or "unassigned"): {
|
||||
"node_code": str(row[0] or "unassigned"),
|
||||
"items_running": int(row[2] or 0),
|
||||
"items_claimed": int(row[1] or 0),
|
||||
"items_completed": int(row[3] or 0),
|
||||
"processed_recent": 0,
|
||||
"processed_per_minute": 0,
|
||||
"completed_recent": 0,
|
||||
"failed_recent": 0,
|
||||
"blacklisted_recent": 0,
|
||||
}
|
||||
for row in cur.fetchall()
|
||||
}
|
||||
cur.execute(
|
||||
"""
|
||||
SELECT
|
||||
COALESCE(NULLIF(node_code, ''), 'unassigned') AS node_code,
|
||||
COUNT(*) AS processed_recent,
|
||||
COUNT(*) FILTER (WHERE event_type = 'domain_completed') AS completed_recent,
|
||||
COUNT(*) FILTER (WHERE event_type = 'domain_blacklisted') AS blacklisted_recent,
|
||||
COUNT(*) FILTER (WHERE event_type = 'domain_failed') AS failed_recent
|
||||
FROM detect_run_events
|
||||
WHERE job_id = ANY(%s)
|
||||
AND event_type IN ('domain_completed', 'domain_blacklisted', 'domain_failed')
|
||||
AND created_at >= CURRENT_TIMESTAMP - (%s || ' minutes')::interval
|
||||
GROUP BY COALESCE(NULLIF(node_code, ''), 'unassigned')
|
||||
""",
|
||||
(job_ids, safe_window_minutes),
|
||||
)
|
||||
for row in cur.fetchall():
|
||||
node_code = str(row[0] or "unassigned")
|
||||
bucket = node_map.setdefault(
|
||||
node_code,
|
||||
{
|
||||
"node_code": node_code,
|
||||
"items_running": 0,
|
||||
"items_claimed": 0,
|
||||
"items_completed": 0,
|
||||
"processed_recent": 0,
|
||||
"processed_per_minute": 0,
|
||||
"completed_recent": 0,
|
||||
"failed_recent": 0,
|
||||
"blacklisted_recent": 0,
|
||||
},
|
||||
)
|
||||
processed_recent = int(row[1] or 0)
|
||||
bucket["processed_recent"] = processed_recent
|
||||
bucket["processed_per_minute"] = round(processed_recent / safe_window_minutes, 2)
|
||||
bucket["completed_recent"] = int(row[2] or 0)
|
||||
bucket["blacklisted_recent"] = int(row[3] or 0)
|
||||
bucket["failed_recent"] = int(row[4] or 0)
|
||||
|
||||
payload["nodes"] = sorted(
|
||||
node_map.values(),
|
||||
key=lambda item: (
|
||||
-int(item.get("items_running", 0) or 0),
|
||||
-int(item.get("processed_recent", 0) or 0),
|
||||
-int(item.get("items_claimed", 0) or 0),
|
||||
str(item.get("node_code") or ""),
|
||||
),
|
||||
)[:8]
|
||||
return payload
|
||||
|
||||
|
||||
def fetch_overview() -> dict:
|
||||
window_minutes = 15
|
||||
queries = {
|
||||
"domains_total": "select count(*) from domains",
|
||||
"pending_total": "select count(*) from domains where detect_status = 0",
|
||||
@@ -12,6 +392,8 @@ def fetch_overview() -> dict:
|
||||
"running_total": "select count(*) from domains where detect_status = 2",
|
||||
"blacklist_total": "select count(*) from domains where detect_status = 3",
|
||||
"failed_total": "select count(*) from domains where detect_status = 4",
|
||||
"registerable_total": "select count(*) from domains where detect_status = 1 and register_status = 2",
|
||||
"purchasable_total": "select count(*) from domains where detect_status = 1 and register_status = 2 and coalesce(use_status, 0) = 0",
|
||||
"sensitive_words_total": "select count(*) from sensitive_words",
|
||||
}
|
||||
result: dict[str, int | str] = {}
|
||||
@@ -23,6 +405,11 @@ def fetch_overview() -> dict:
|
||||
result[key] = cur.fetchone()[0]
|
||||
except Exception:
|
||||
result[key] = 0
|
||||
|
||||
active_jobs_aggregate = _fetch_active_jobs_aggregate(window_minutes=window_minutes)
|
||||
active_job = get_active_detect_job_summary(event_limit=20) or {}
|
||||
aggregate_queue = active_jobs_aggregate.get("queue") or {}
|
||||
|
||||
runtime = get_runtime_status()
|
||||
cluster_summary = ((runtime.get("cluster") or {}).get("summary") or {})
|
||||
online_worker_nodes = int(cluster_summary.get("online_worker_nodes", 0) or 0)
|
||||
@@ -37,4 +424,274 @@ def fetch_overview() -> dict:
|
||||
result["worker_mode"] = runtime["worker"]["mode"]
|
||||
result["node_region"] = runtime["node"]["region"]
|
||||
result["node_role"] = runtime["node"]["role"]
|
||||
|
||||
queue_health = get_detect_queue_health(window_minutes=window_minutes)
|
||||
active_jobs_aggregate = _align_active_jobs_aggregate_with_runtime(
|
||||
active_jobs_aggregate,
|
||||
runtime=runtime,
|
||||
queue_health=queue_health,
|
||||
)
|
||||
runtime_snapshot_backlog = dict(queue_health.get("runtime_snapshot_backlog") or {})
|
||||
aggregate_queue = active_jobs_aggregate.get("queue") or {}
|
||||
aggregate_queue_health = {
|
||||
"has_active_job": bool(int(active_jobs_aggregate.get("active_jobs_total", 0) or 0) > 0),
|
||||
"queue": aggregate_queue,
|
||||
"throughput": active_jobs_aggregate.get("throughput") or {},
|
||||
}
|
||||
selected_queue_health = aggregate_queue_health if aggregate_queue_health["has_active_job"] else queue_health
|
||||
if float((queue_health.get("throughput") or {}).get("processed_per_minute", 0) or 0) > float(
|
||||
(selected_queue_health.get("throughput") or {}).get("processed_per_minute", 0) or 0
|
||||
):
|
||||
selected_queue_health = queue_health
|
||||
capacity_plan = get_detect_capacity_plan(
|
||||
queue_health=selected_queue_health,
|
||||
online_worker_nodes=online_worker_nodes,
|
||||
)
|
||||
|
||||
retry_total = int(active_jobs_aggregate.get("retry_total", 0) or 0)
|
||||
step_queue: list[dict] = []
|
||||
node_throughput: list[dict] = []
|
||||
bottleneck_step: dict | None = None
|
||||
active_job_summary: dict | None = None
|
||||
|
||||
if queue_health.get("has_active_job"):
|
||||
job_payload = queue_health.get("job") or {}
|
||||
queue_payload = queue_health.get("queue") or {}
|
||||
throughput_payload = queue_health.get("throughput") or {}
|
||||
runtime_job_code = str(job_payload.get("runtime_job_code") or "").strip()
|
||||
display_job_code = runtime_job_code or str(job_payload.get("job_code") or "")
|
||||
active_job_summary = {
|
||||
"job_id": int(job_payload.get("job_id", 0) or 0),
|
||||
"job_code": display_job_code,
|
||||
"db_job_code": str(job_payload.get("job_code") or ""),
|
||||
"runtime_job_code": runtime_job_code,
|
||||
"status": str(job_payload.get("status") or ""),
|
||||
"progress_percent": float(job_payload.get("progress_percent", 0) or 0),
|
||||
"items_total": int(queue_payload.get("items_total", 0) or 0),
|
||||
"items_pending": int(queue_payload.get("pending", 0) or 0),
|
||||
"items_claimed": int(queue_payload.get("display_claimed", queue_payload.get("claimed", 0)) or 0),
|
||||
"items_running": int(queue_payload.get("running", 0) or 0),
|
||||
"items_display_running": int(queue_payload.get("display_running", queue_payload.get("running", 0)) or 0),
|
||||
"items_completed": int(queue_payload.get("completed", 0) or 0),
|
||||
"items_blacklisted": int(queue_payload.get("blacklisted", 0) or 0),
|
||||
"items_failed": int(queue_payload.get("failed", 0) or 0),
|
||||
"processed_per_minute": float(throughput_payload.get("processed_per_minute", 0) or 0),
|
||||
"processed_recent": int(throughput_payload.get("processed_recent", 0) or 0),
|
||||
"completed_recent": int(throughput_payload.get("completed_recent", 0) or 0),
|
||||
"failed_recent": int(throughput_payload.get("failed_recent", 0) or 0),
|
||||
"blacklisted_recent": int(throughput_payload.get("blacklisted_recent", 0) or 0),
|
||||
"active_jobs_total": int(active_jobs_aggregate.get("active_jobs_total", 0) or 0),
|
||||
}
|
||||
|
||||
queue_pending_total = 0
|
||||
queue_claimed_total = 0
|
||||
queue_running_total = 0
|
||||
queue_display_running_total = 0
|
||||
queue_completed_total = 0
|
||||
queue_blacklist_total = 0
|
||||
queue_failed_total = 0
|
||||
backlog_payload = ((runtime.get("detect") or {}).get("backlog") or {})
|
||||
backlog_pending_total = max(
|
||||
int(backlog_payload.get("pending_total", 0) or 0),
|
||||
int(runtime_snapshot_backlog.get("pending_total", 0) or 0),
|
||||
)
|
||||
backlog_claimed_total = max(
|
||||
int(backlog_payload.get("claimed_total", 0) or 0),
|
||||
int(runtime_snapshot_backlog.get("claimed_total", 0) or 0),
|
||||
)
|
||||
backlog_running_total = max(
|
||||
int(backlog_payload.get("running_total", 0) or 0),
|
||||
int(runtime_snapshot_backlog.get("running_total", 0) or 0),
|
||||
)
|
||||
backlog_register_pending_total = max(
|
||||
int(backlog_payload.get("register_pending", 0) or 0),
|
||||
int(runtime_snapshot_backlog.get("register_pending", 0) or 0),
|
||||
)
|
||||
backlog_downstream_pending_total = max(
|
||||
int(backlog_payload.get("downstream_pending", 0) or 0),
|
||||
int(runtime_snapshot_backlog.get("downstream_pending", 0) or 0),
|
||||
)
|
||||
if queue_health.get("has_active_job"):
|
||||
queue_payload = queue_health.get("queue") or {}
|
||||
queue_pending_total = int(queue_payload.get("pending", 0) or 0)
|
||||
queue_claimed_total = int(queue_payload.get("display_claimed", queue_payload.get("claimed", 0)) or 0)
|
||||
queue_running_total = int(queue_payload.get("running", 0) or 0)
|
||||
queue_display_running_total = int(queue_payload.get("display_running", queue_payload.get("running", 0)) or 0)
|
||||
queue_completed_total = int(queue_payload.get("completed", 0) or 0)
|
||||
queue_blacklist_total = int(queue_payload.get("blacklisted", 0) or 0)
|
||||
queue_failed_total = int(queue_payload.get("failed", 0) or 0)
|
||||
elif int(active_jobs_aggregate.get("active_jobs_total", 0) or 0) > 0:
|
||||
queue_pending_total = int(aggregate_queue.get("pending", 0) or 0)
|
||||
queue_claimed_total = int(aggregate_queue.get("claimed", 0) or 0)
|
||||
queue_running_total = int(aggregate_queue.get("running", 0) or 0)
|
||||
queue_display_running_total = queue_running_total
|
||||
queue_completed_total = int(aggregate_queue.get("completed", 0) or 0)
|
||||
queue_blacklist_total = int(aggregate_queue.get("blacklisted", 0) or 0)
|
||||
queue_failed_total = int(aggregate_queue.get("failed", 0) or 0)
|
||||
elif active_job:
|
||||
queue_pending_total = int(active_job.get("items_pending", 0) or 0)
|
||||
queue_claimed_total = int(active_job.get("items_claimed", 0) or 0)
|
||||
queue_running_total = int(active_job.get("items_running", 0) or 0)
|
||||
queue_display_running_total = int(
|
||||
active_job.get("display_items_running", active_job.get("items_running", 0)) or 0
|
||||
)
|
||||
queue_completed_total = int(active_job.get("items_completed", 0) or 0)
|
||||
queue_blacklist_total = int(active_job.get("items_blacklisted", 0) or 0)
|
||||
queue_failed_total = int(active_job.get("items_failed", 0) or 0)
|
||||
|
||||
aggregate_step_queue = [
|
||||
{
|
||||
"step_code": str(item.get("step_code") or ""),
|
||||
"step_name": str(item.get("step_name") or ""),
|
||||
"items_pending": int(item.get("items_pending", 0) or 0),
|
||||
"items_running": int(item.get("items_running", 0) or 0),
|
||||
"items_claimed": int(item.get("items_claimed", 0) or 0),
|
||||
"items_completed": int(item.get("items_completed", 0) or 0),
|
||||
"items_blacklisted": int(item.get("items_blacklisted", 0) or 0),
|
||||
"items_failed": int(item.get("items_failed", 0) or 0),
|
||||
"started_recent": int(item.get("started_recent", 0) or 0),
|
||||
"processed_per_minute": float(item.get("processed_per_minute", 0) or 0),
|
||||
"processed_recent": int(item.get("processed_recent", 0) or 0),
|
||||
"completed_recent": int(item.get("completed_recent", 0) or 0),
|
||||
"blacklisted_recent": int(item.get("blacklisted_recent", 0) or 0),
|
||||
"failed_recent": int(item.get("failed_recent", 0) or 0),
|
||||
}
|
||||
for item in list(active_jobs_aggregate.get("steps") or [])[:8]
|
||||
]
|
||||
aggregate_node_throughput = [
|
||||
{
|
||||
"node_code": str(item.get("node_code") or ""),
|
||||
"items_pending": int(item.get("items_pending", 0) or 0),
|
||||
"items_running": int(item.get("items_running", 0) or 0),
|
||||
"display_running": int(item.get("display_running", item.get("items_running", 0)) or 0),
|
||||
"items_claimed": int(item.get("items_claimed", 0) or 0),
|
||||
"current_load": int(item.get("current_load", item.get("display_running", 0)) or 0),
|
||||
"active_threads": int(item.get("active_threads", 0) or 0),
|
||||
"max_threads": int(item.get("max_threads", 0) or 0),
|
||||
"processed_recent": int(item.get("processed_recent", 0) or 0),
|
||||
"processed_per_minute": float(item.get("processed_per_minute", 0) or 0),
|
||||
"completed_recent": int(item.get("completed_recent", 0) or 0),
|
||||
"failed_recent": int(item.get("failed_recent", 0) or 0),
|
||||
"blacklisted_recent": int(item.get("blacklisted_recent", 0) or 0),
|
||||
}
|
||||
for item in list(active_jobs_aggregate.get("nodes") or [])[:8]
|
||||
]
|
||||
queue_step_queue = [
|
||||
{
|
||||
"step_code": str(item.get("step_code") or ""),
|
||||
"step_name": str(item.get("step_name") or ""),
|
||||
"items_pending": int(item.get("items_pending", 0) or 0),
|
||||
"items_running": int(item.get("items_running", 0) or 0),
|
||||
"items_claimed": int(item.get("items_claimed", 0) or 0),
|
||||
"items_completed": int(item.get("items_completed", 0) or 0),
|
||||
"items_blacklisted": int(item.get("items_blacklisted", 0) or 0),
|
||||
"items_failed": int(item.get("items_failed", 0) or 0),
|
||||
"started_recent": int(item.get("started_recent", 0) or 0),
|
||||
"processed_per_minute": float(item.get("processed_per_minute", 0) or 0),
|
||||
"processed_recent": int(item.get("processed_recent", 0) or 0),
|
||||
"completed_recent": int(item.get("completed_recent", 0) or 0),
|
||||
"blacklisted_recent": int(item.get("blacklisted_recent", 0) or 0),
|
||||
"failed_recent": int(item.get("failed_recent", 0) or 0),
|
||||
}
|
||||
for item in _merge_step_queues_with_runtime_activity(
|
||||
list(queue_health.get("steps") or []),
|
||||
runtime_activity=dict(queue_health.get("runtime_activity") or {}),
|
||||
window_minutes=window_minutes,
|
||||
limit=8,
|
||||
)
|
||||
]
|
||||
queue_node_throughput = [
|
||||
{
|
||||
"node_code": str(item.get("node_code") or ""),
|
||||
"items_pending": int(item.get("items_pending", 0) or 0),
|
||||
"items_running": int(item.get("items_running", 0) or 0),
|
||||
"display_running": int(item.get("display_running", item.get("items_running", 0)) or 0),
|
||||
"items_claimed": int(item.get("items_claimed", 0) or 0),
|
||||
"current_load": int(item.get("current_load", item.get("display_running", 0)) or 0),
|
||||
"active_threads": int(item.get("active_threads", 0) or 0),
|
||||
"max_threads": int(item.get("max_threads", 0) or 0),
|
||||
"processed_recent": int(item.get("processed_recent", 0) or 0),
|
||||
"processed_per_minute": float(item.get("processed_per_minute", 0) or 0),
|
||||
"completed_recent": int(item.get("completed_recent", 0) or 0),
|
||||
"failed_recent": int(item.get("failed_recent", 0) or 0),
|
||||
"blacklisted_recent": int(item.get("blacklisted_recent", 0) or 0),
|
||||
}
|
||||
for item in list(queue_health.get("nodes") or [])[:8]
|
||||
]
|
||||
step_queue = aggregate_step_queue
|
||||
node_throughput = aggregate_node_throughput
|
||||
aggregate_ppm = float((active_jobs_aggregate.get("throughput") or {}).get("processed_per_minute", 0) or 0)
|
||||
queue_ppm = float((queue_health.get("throughput") or {}).get("processed_per_minute", 0) or 0)
|
||||
if queue_health.get("has_active_job") or queue_ppm > aggregate_ppm:
|
||||
step_queue = queue_step_queue
|
||||
node_throughput = queue_node_throughput
|
||||
if step_queue:
|
||||
bottleneck_step = max(
|
||||
step_queue,
|
||||
key=lambda item: (
|
||||
int(item.get("items_pending", 0) or 0),
|
||||
int(item.get("items_running", 0) or 0),
|
||||
-float(item.get("processed_per_minute", 0) or 0),
|
||||
),
|
||||
)
|
||||
|
||||
result["active_job"] = active_job_summary or {}
|
||||
result["queue_health"] = queue_health
|
||||
result["active_jobs_aggregate"] = active_jobs_aggregate
|
||||
result["capacity_plan"] = capacity_plan
|
||||
result["step_queue"] = step_queue
|
||||
result["node_throughput"] = node_throughput
|
||||
result["retry_total"] = retry_total
|
||||
result["bottleneck_step"] = bottleneck_step or {}
|
||||
aggregate_throughput = active_jobs_aggregate.get("throughput") or {}
|
||||
queue_throughput = queue_health.get("throughput") or {}
|
||||
ops_processed_per_minute = float(aggregate_throughput.get("processed_per_minute", 0) or 0)
|
||||
ops_processed_recent = int(aggregate_throughput.get("processed_recent", 0) or 0)
|
||||
ops_completed_recent = int(aggregate_throughput.get("completed_recent", 0) or 0)
|
||||
ops_failed_recent = int(aggregate_throughput.get("failed_recent", 0) or 0)
|
||||
ops_blacklisted_recent = int(aggregate_throughput.get("blacklisted_recent", 0) or 0)
|
||||
if queue_health.get("has_active_job") or float(queue_throughput.get("processed_per_minute", 0) or 0) > ops_processed_per_minute:
|
||||
ops_processed_per_minute = float(queue_throughput.get("processed_per_minute", 0) or 0)
|
||||
ops_processed_recent = int(queue_throughput.get("processed_recent", 0) or 0)
|
||||
ops_completed_recent = int(queue_throughput.get("completed_recent", 0) or 0)
|
||||
ops_failed_recent = int(queue_throughput.get("failed_recent", 0) or 0)
|
||||
ops_blacklisted_recent = int(queue_throughput.get("blacklisted_recent", 0) or 0)
|
||||
result["ops_summary"] = {
|
||||
"active_jobs_total": int(active_jobs_aggregate.get("active_jobs_total", 0) or 0),
|
||||
"processed_per_minute": ops_processed_per_minute,
|
||||
"processed_recent": ops_processed_recent,
|
||||
"completed_recent": ops_completed_recent,
|
||||
"failed_recent": ops_failed_recent,
|
||||
"blacklisted_recent": ops_blacklisted_recent,
|
||||
"estimated_hours_remaining": float(capacity_plan.get("estimated_hours_remaining", 0) or 0),
|
||||
"remaining_items": int(capacity_plan.get("remaining_items", 0) or 0),
|
||||
"recommended_additional_workers": int(capacity_plan.get("recommended_additional_workers", 0) or 0),
|
||||
"online_worker_nodes": online_worker_nodes,
|
||||
"dedicated_online_worker_nodes": dedicated_online_worker_nodes,
|
||||
"active_execution_nodes": sum(
|
||||
1
|
||||
for item in node_throughput
|
||||
if int(item.get("items_running", 0) or 0) > 0
|
||||
or int(item.get("items_claimed", 0) or 0) > 0
|
||||
or int(item.get("processed_recent", 0) or 0) > 0
|
||||
),
|
||||
}
|
||||
result["processed_per_minute"] = ops_processed_per_minute
|
||||
result["processed_recent"] = ops_processed_recent
|
||||
result["completed_recent"] = ops_completed_recent
|
||||
result["failed_recent"] = ops_failed_recent
|
||||
result["blacklisted_recent"] = ops_blacklisted_recent
|
||||
result["active_execution_nodes"] = int(result["ops_summary"]["active_execution_nodes"] or 0)
|
||||
result["queue_pending_total"] = queue_pending_total
|
||||
result["queue_claimed_total"] = queue_claimed_total
|
||||
result["queue_running_total"] = queue_running_total
|
||||
result["queue_display_running_total"] = max(queue_display_running_total, queue_running_total)
|
||||
result["queue_completed_total"] = queue_completed_total
|
||||
result["queue_blacklist_total"] = queue_blacklist_total
|
||||
result["queue_failed_total"] = queue_failed_total
|
||||
result["backlog_pending_total"] = max(backlog_pending_total, queue_pending_total)
|
||||
result["backlog_claimed_total"] = max(backlog_claimed_total, queue_claimed_total)
|
||||
result["backlog_running_total"] = max(backlog_running_total, queue_running_total)
|
||||
result["backlog_register_pending_total"] = backlog_register_pending_total
|
||||
result["backlog_downstream_pending_total"] = backlog_downstream_pending_total
|
||||
return result
|
||||
|
||||
Reference in New Issue
Block a user