fix: unify detect effective job metrics
This commit is contained in:
Your Name
@@ -187,6 +187,96 @@ def _build_display_summary(node_stats: list[dict]) -> dict:
|
||||
}
|
||||
|
||||
|
||||
def _normalize_node_bucket(item: dict) -> dict:
|
||||
node_code = str(item.get("node_code") or "").strip()
|
||||
return {
|
||||
"node_code": node_code,
|
||||
"items_total": _int_value(item.get("items_total")),
|
||||
"items_pending": _int_value(item.get("items_pending")),
|
||||
"items_claimed": _int_value(item.get("items_claimed")),
|
||||
"items_running": _int_value(item.get("items_running")),
|
||||
"items_completed": _int_value(item.get("items_completed")),
|
||||
"items_blacklisted": _int_value(item.get("items_blacklisted")),
|
||||
"items_failed": _int_value(item.get("items_failed")),
|
||||
"metrics_source": str(item.get("metrics_source") or "").strip(),
|
||||
"region": str(item.get("region") or "").strip(),
|
||||
"role": str(item.get("role") or "").strip(),
|
||||
"status": str(item.get("status") or "").strip(),
|
||||
"current_load": _int_value(item.get("current_load")),
|
||||
"last_heartbeat_at": str(item.get("last_heartbeat_at") or "").strip(),
|
||||
}
|
||||
|
||||
|
||||
def _build_effective_node_stats(
|
||||
*,
|
||||
distributed_node_stats: list[dict],
|
||||
raw_items_total: int,
|
||||
) -> list[dict]:
|
||||
normalized_total = max(0, _int_value(raw_items_total))
|
||||
assigned_buckets: list[dict] = []
|
||||
assigned_total = 0
|
||||
for item in list(distributed_node_stats or []):
|
||||
bucket = _normalize_node_bucket(item)
|
||||
if not bucket["node_code"] or bucket["node_code"] == "unassigned":
|
||||
continue
|
||||
assigned_buckets.append(bucket)
|
||||
assigned_total += bucket["items_total"]
|
||||
|
||||
effective_total = max(normalized_total, assigned_total)
|
||||
remainder_total = max(effective_total - assigned_total, 0)
|
||||
if remainder_total > 0:
|
||||
assigned_buckets.append(
|
||||
{
|
||||
"node_code": "unassigned",
|
||||
"items_total": remainder_total,
|
||||
"items_pending": remainder_total,
|
||||
"items_claimed": 0,
|
||||
"items_running": 0,
|
||||
"items_completed": 0,
|
||||
"items_blacklisted": 0,
|
||||
"items_failed": 0,
|
||||
"metrics_source": "central_queue",
|
||||
"region": "",
|
||||
"role": "",
|
||||
"status": "",
|
||||
"current_load": 0,
|
||||
"last_heartbeat_at": "",
|
||||
}
|
||||
)
|
||||
|
||||
return sorted(
|
||||
assigned_buckets,
|
||||
key=lambda item: (
|
||||
str(item.get("node_code") or "") == "unassigned",
|
||||
-_int_value(item.get("items_running")),
|
||||
-_int_value(item.get("items_claimed")),
|
||||
-_int_value(item.get("items_total")),
|
||||
str(item.get("node_code") or ""),
|
||||
),
|
||||
)
|
||||
|
||||
|
||||
def _build_effective_summary(*, node_stats: list[dict], raw_items_total: int, raw_items_blacklisted: int) -> dict:
|
||||
effective_total = max(0, _int_value(raw_items_total))
|
||||
effective_blacklisted = max(0, _int_value(raw_items_blacklisted))
|
||||
effective_pending = sum(_int_value(item.get("items_pending")) for item in list(node_stats or []))
|
||||
effective_claimed = sum(_int_value(item.get("items_claimed")) for item in list(node_stats or []))
|
||||
effective_running = sum(_int_value(item.get("items_running")) for item in list(node_stats or []))
|
||||
effective_completed = sum(_int_value(item.get("items_completed")) for item in list(node_stats or []))
|
||||
effective_failed = sum(_int_value(item.get("items_failed")) for item in list(node_stats or []))
|
||||
effective_terminal = effective_completed + effective_blacklisted + effective_failed
|
||||
return {
|
||||
"items_total": effective_total,
|
||||
"items_pending": max(effective_pending, 0),
|
||||
"items_claimed": max(effective_claimed, 0),
|
||||
"items_running": max(effective_running, 0),
|
||||
"items_completed": max(effective_completed, 0),
|
||||
"items_blacklisted": effective_blacklisted,
|
||||
"items_failed": max(effective_failed, 0),
|
||||
"items_terminal": max(effective_terminal, 0),
|
||||
}
|
||||
|
||||
|
||||
def _extract_current_cycle_events(events: list[dict]) -> tuple[str, list[dict]]:
|
||||
if not events:
|
||||
return "", []
|
||||
@@ -293,9 +383,24 @@ def _fetch_job_summary(cur, job_row, event_limit: int = 20) -> dict:
|
||||
local_node_stats=list(node_buckets.values()),
|
||||
runtime_node_rows=_load_runtime_display_rows(cur),
|
||||
)
|
||||
display_summary = _build_display_summary(distributed_node_stats)
|
||||
total = sum(counts.values())
|
||||
terminal = int(counts.get("completed", 0)) + int(counts.get("blacklisted", 0)) + int(counts.get("failed", 0))
|
||||
raw_total = sum(counts.values())
|
||||
raw_pending = int(counts.get("pending", 0))
|
||||
raw_claimed = int(counts.get("claimed", 0))
|
||||
raw_running = int(counts.get("running", 0))
|
||||
raw_completed = int(counts.get("completed", 0))
|
||||
raw_blacklisted = int(counts.get("blacklisted", 0))
|
||||
raw_failed = int(counts.get("failed", 0))
|
||||
raw_terminal = raw_completed + raw_blacklisted + raw_failed
|
||||
effective_node_stats = _build_effective_node_stats(
|
||||
distributed_node_stats=distributed_node_stats,
|
||||
raw_items_total=raw_total,
|
||||
)
|
||||
effective_summary = _build_effective_summary(
|
||||
node_stats=effective_node_stats,
|
||||
raw_items_total=raw_total,
|
||||
raw_items_blacklisted=raw_blacklisted,
|
||||
)
|
||||
display_summary = _build_display_summary(effective_node_stats)
|
||||
return {
|
||||
"job_id": job_id,
|
||||
"job_code": job_row[1],
|
||||
@@ -305,17 +410,29 @@ def _fetch_job_summary(cur, job_row, event_limit: int = 20) -> dict:
|
||||
"created_at": _format_time(job_row[5]),
|
||||
"started_at": _format_time(job_row[6]),
|
||||
"finished_at": _format_time(job_row[7]),
|
||||
"items_total": total,
|
||||
"items_pending": int(counts.get("pending", 0)),
|
||||
"items_claimed": int(counts.get("claimed", 0)),
|
||||
"items_running": int(counts.get("running", 0)),
|
||||
"items_completed": int(counts.get("completed", 0)),
|
||||
"items_blacklisted": int(counts.get("blacklisted", 0)),
|
||||
"items_failed": int(counts.get("failed", 0)),
|
||||
"items_terminal": terminal,
|
||||
"progress_percent": round((terminal / total) * 100, 2) if total else 0,
|
||||
"node_stats": list(node_buckets.values()),
|
||||
"distributed_node_stats": distributed_node_stats,
|
||||
"items_total": int(effective_summary.get("items_total", 0) or 0),
|
||||
"items_pending": int(effective_summary.get("items_pending", 0) or 0),
|
||||
"items_claimed": int(effective_summary.get("items_claimed", 0) or 0),
|
||||
"items_running": int(effective_summary.get("items_running", 0) or 0),
|
||||
"items_completed": int(effective_summary.get("items_completed", 0) or 0),
|
||||
"items_blacklisted": int(effective_summary.get("items_blacklisted", 0) or 0),
|
||||
"items_failed": int(effective_summary.get("items_failed", 0) or 0),
|
||||
"items_terminal": int(effective_summary.get("items_terminal", 0) or 0),
|
||||
"progress_percent": round(
|
||||
(int(effective_summary.get("items_terminal", 0) or 0) / int(effective_summary.get("items_total", 0) or 0)) * 100,
|
||||
2,
|
||||
) if int(effective_summary.get("items_total", 0) or 0) else 0,
|
||||
"raw_items_total": raw_total,
|
||||
"raw_items_pending": raw_pending,
|
||||
"raw_items_claimed": raw_claimed,
|
||||
"raw_items_running": raw_running,
|
||||
"raw_items_completed": raw_completed,
|
||||
"raw_items_blacklisted": raw_blacklisted,
|
||||
"raw_items_failed": raw_failed,
|
||||
"raw_items_terminal": raw_terminal,
|
||||
"raw_node_stats": list(node_buckets.values()),
|
||||
"node_stats": effective_node_stats,
|
||||
"distributed_node_stats": effective_node_stats,
|
||||
"display_items_claimed": int(display_summary.get("items_claimed", 0) or 0),
|
||||
"display_items_running": int(display_summary.get("items_running", 0) or 0),
|
||||
"display_items_completed": int(display_summary.get("items_completed", 0) or 0),
|
||||
|
||||
Reference in New Issue
Block a user