admin/getDomain
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: stabilize multi-region runtime sync and worker orchestration

Browse Source
This commit is contained in:
root
2026-04-27 15:48:12 +08:00
parent 7cbde2aa78
commit 215a364891
137 changed files with 31931 additions and 1943 deletions
Download Patch File Download Diff File Expand all files Collapse all files

203
domain-api/app/services/dashboard.py
View File

@@ -1,13 +1,18 @@
from __future__ import annotations
from app.core.config import settings
from app.core.db import get_db
from app.services.cluster_runtime_service import get_cluster_snapshot
from app.services.detect_service import get_detect_status
from app.services.detect_job_service import (
_build_step_bucket,
order_step_buckets,
get_active_detect_job_summary,
get_detect_capacity_plan,
get_detect_queue_health,
)
from app.services.runtime_status_service import get_runtime_status
from app.services.runtime_settings_service import get_runtime_settings
from app.services.worker_control_service import detect_worker_runtime
def _empty_active_jobs_aggregate(window_minutes: int) -> dict:
@@ -93,16 +98,7 @@ def _merge_step_queues_with_runtime_activity(
round(processed_recent / safe_window_minutes, 2),
)
return sorted(
step_map.values(),
key=lambda item: (
-int(item.get("items_pending", 0) or 0),
-int(item.get("items_running", 0) or 0),
-int(item.get("started_recent", 0) or 0),
-int(item.get("processed_recent", 0) or 0),
str(item.get("step_code") or ""),
),
)[:normalized_limit]
return order_step_buckets(list(step_map.values()), limit=normalized_limit)
def _align_active_jobs_aggregate_with_runtime(
@@ -169,6 +165,58 @@ def _align_active_jobs_aggregate_with_runtime(
return normalized
def _build_dashboard_runtime_summary(*, queue_health: dict) -> dict:
runtime_settings = get_runtime_settings()
worker_runtime = detect_worker_runtime()
worker_expected_on_this_node = not (
str(settings.node_region or "").strip() == "overseas"
and str(settings.node_role or "").strip() == "control"
)
return {
"node": {
"region": settings.node_region,
"role": settings.node_role,
},
"worker": {
"running": bool(worker_runtime.get("running", False)),
"mode": worker_runtime.get("mode", runtime_settings.get("worker_mode", "windows-local")),
"expected_on_this_node": worker_expected_on_this_node,
},
"cluster": get_cluster_snapshot(),
"detect": {
"backlog": dict((queue_health or {}).get("runtime_snapshot_backlog") or {}),
},
}
def _resolve_server_code(node_code: str | None) -> str:
normalized_node_code = str(node_code or "").strip()
if not normalized_node_code:
return ""
parent_node_code, separator, suffix = normalized_node_code.rpartition("-")
if separator and parent_node_code and suffix.isalpha() and len(suffix) <= 3:
if any(char.isdigit() for char in parent_node_code):
return parent_node_code
return normalized_node_code
def _count_active_execution_servers(rows: list[dict] | None) -> int:
active_servers: set[str] = set()
for item in list(rows or []):
if not isinstance(item, dict):
continue
server_code = _resolve_server_code(item.get("node_code"))
if not server_code:
continue
if (
int(item.get("items_running", 0) or 0) > 0
or int(item.get("items_claimed", 0) or 0) > 0
or int(item.get("processed_recent", 0) or 0) > 0
):
active_servers.add(server_code)
return len(active_servers)
def _fetch_active_jobs_aggregate(window_minutes: int = 15) -> dict:
safe_window_minutes = max(5, min(int(window_minutes or 15), 120))
payload = _empty_active_jobs_aggregate(safe_window_minutes)
@@ -303,7 +351,7 @@ def _fetch_active_jobs_aggregate(window_minutes: int = 15) -> dict:
}
)
steps.append(bucket)
payload["steps"] = steps
payload["steps"] = order_step_buckets(steps, limit=8)
cur.execute(
"""
@@ -409,8 +457,28 @@ def fetch_overview() -> dict:
active_jobs_aggregate = _fetch_active_jobs_aggregate(window_minutes=window_minutes)
active_job = get_active_detect_job_summary(event_limit=20) or {}
aggregate_queue = active_jobs_aggregate.get("queue") or {}
active_job_display_claimed = int(active_job.get("display_items_claimed", active_job.get("items_claimed", 0)) or 0)
active_job_display_running = int(
active_job.get("display_items_running", active_job.get("display_active_threads", active_job.get("items_running", 0)))
or 0
)
active_job_display_active_threads = int(
active_job.get("display_active_threads", active_job.get("display_items_running", active_job.get("items_running", 0)))
or 0
)
active_job_display_max_threads = int(active_job.get("display_max_threads", 0) or 0)
active_job_distributed_node_stats = [
dict(item)
for item in list(active_job.get("distributed_node_stats") or [])
if isinstance(item, dict)
]
runtime = get_runtime_status()
queue_health = get_detect_queue_health(window_minutes=window_minutes)
runtime = _build_dashboard_runtime_summary(queue_health=queue_health)
try:
detect_status = get_detect_status()
except Exception:
detect_status = {}
cluster_summary = ((runtime.get("cluster") or {}).get("summary") or {})
online_worker_nodes = int(cluster_summary.get("online_worker_nodes", 0) or 0)
dedicated_online_worker_nodes = int(cluster_summary.get("dedicated_online_worker_nodes", 0) or 0)
@@ -425,7 +493,6 @@ def fetch_overview() -> dict:
result["node_region"] = runtime["node"]["region"]
result["node_role"] = runtime["node"]["role"]
queue_health = get_detect_queue_health(window_minutes=window_minutes)
active_jobs_aggregate = _align_active_jobs_aggregate_with_runtime(
active_jobs_aggregate,
runtime=runtime,
@@ -460,6 +527,25 @@ def fetch_overview() -> dict:
throughput_payload = queue_health.get("throughput") or {}
runtime_job_code = str(job_payload.get("runtime_job_code") or "").strip()
display_job_code = runtime_job_code or str(job_payload.get("job_code") or "")
active_job_matches_display_job = display_job_code == str(active_job.get("job_code") or "").strip()
queue_nodes = [
dict(item)
for item in list(queue_health.get("nodes") or [])
if isinstance(item, dict)
]
summary_display_running = max(
int(queue_payload.get("display_running", queue_payload.get("running", 0)) or 0),
active_job_display_running if active_job_matches_display_job else 0,
)
summary_display_active_threads = max(
int(queue_payload.get("display_running", queue_payload.get("running", 0)) or 0),
active_job_display_active_threads if active_job_matches_display_job else 0,
)
summary_display_max_threads = max(
sum(int(item.get("max_threads", 0) or 0) for item in queue_nodes),
active_job_display_max_threads if active_job_matches_display_job else 0,
)
summary_distributed_node_stats = active_job_distributed_node_stats if active_job_matches_display_job else queue_nodes
active_job_summary = {
"job_id": int(job_payload.get("job_id", 0) or 0),
"job_code": display_job_code,
@@ -469,9 +555,19 @@ def fetch_overview() -> dict:
"progress_percent": float(job_payload.get("progress_percent", 0) or 0),
"items_total": int(queue_payload.get("items_total", 0) or 0),
"items_pending": int(queue_payload.get("pending", 0) or 0),
"items_claimed": int(queue_payload.get("display_claimed", queue_payload.get("claimed", 0)) or 0),
"items_running": int(queue_payload.get("running", 0) or 0),
"items_display_running": int(queue_payload.get("display_running", queue_payload.get("running", 0)) or 0),
"items_claimed": max(
int(queue_payload.get("display_claimed", queue_payload.get("claimed", 0)) or 0),
active_job_display_claimed if active_job_matches_display_job else 0,
),
"items_running": max(
int(queue_payload.get("running", 0) or 0),
int(active_job.get("items_running", 0) or 0) if active_job_matches_display_job else 0,
),
"items_display_running": summary_display_running,
"display_items_running": summary_display_running,
"display_active_threads": summary_display_active_threads,
"display_max_threads": summary_display_max_threads,
"distributed_node_stats": summary_distributed_node_stats,
"items_completed": int(queue_payload.get("completed", 0) or 0),
"items_blacklisted": int(queue_payload.get("blacklisted", 0) or 0),
"items_failed": int(queue_payload.get("failed", 0) or 0),
@@ -482,6 +578,33 @@ def fetch_overview() -> dict:
"blacklisted_recent": int(throughput_payload.get("blacklisted_recent", 0) or 0),
"active_jobs_total": int(active_jobs_aggregate.get("active_jobs_total", 0) or 0),
}
elif active_job:
active_job_summary = {
"job_id": int(active_job.get("job_id", 0) or 0),
"job_code": str(active_job.get("job_code") or active_job.get("runtime_job_code") or ""),
"db_job_code": str(active_job.get("job_code") or ""),
"runtime_job_code": str(active_job.get("runtime_job_code") or ""),
"status": str(active_job.get("status") or ""),
"progress_percent": float(active_job.get("progress_percent", 0) or 0),
"items_total": int(active_job.get("items_total", 0) or 0),
"items_pending": int(active_job.get("items_pending", 0) or 0),
"items_claimed": active_job_display_claimed,
"items_running": int(active_job.get("items_running", 0) or 0),
"items_display_running": active_job_display_running,
"display_items_running": active_job_display_running,
"display_active_threads": active_job_display_active_threads,
"display_max_threads": active_job_display_max_threads,
"distributed_node_stats": active_job_distributed_node_stats,
"items_completed": int(active_job.get("items_completed", 0) or 0),
"items_blacklisted": int(active_job.get("items_blacklisted", 0) or 0),
"items_failed": int(active_job.get("items_failed", 0) or 0),
"processed_per_minute": float((active_jobs_aggregate.get("throughput") or {}).get("processed_per_minute", 0) or 0),
"processed_recent": int((active_jobs_aggregate.get("throughput") or {}).get("processed_recent", 0) or 0),
"completed_recent": int((active_jobs_aggregate.get("throughput") or {}).get("completed_recent", 0) or 0),
"failed_recent": int((active_jobs_aggregate.get("throughput") or {}).get("failed_recent", 0) or 0),
"blacklisted_recent": int((active_jobs_aggregate.get("throughput") or {}).get("blacklisted_recent", 0) or 0),
"active_jobs_total": int(active_jobs_aggregate.get("active_jobs_total", 0) or 0),
}
queue_pending_total = 0
queue_claimed_total = 0
@@ -514,9 +637,15 @@ def fetch_overview() -> dict:
if queue_health.get("has_active_job"):
queue_payload = queue_health.get("queue") or {}
queue_pending_total = int(queue_payload.get("pending", 0) or 0)
queue_claimed_total = int(queue_payload.get("display_claimed", queue_payload.get("claimed", 0)) or 0)
queue_running_total = int(queue_payload.get("running", 0) or 0)
queue_display_running_total = int(queue_payload.get("display_running", queue_payload.get("running", 0)) or 0)
queue_claimed_total = max(
int(queue_payload.get("display_claimed", queue_payload.get("claimed", 0)) or 0),
active_job_display_claimed,
)
queue_running_total = max(int(queue_payload.get("running", 0) or 0), int(active_job.get("items_running", 0) or 0))
queue_display_running_total = max(
int(queue_payload.get("display_running", queue_payload.get("running", 0)) or 0),
active_job_display_running,
)
queue_completed_total = int(queue_payload.get("completed", 0) or 0)
queue_blacklist_total = int(queue_payload.get("blacklisted", 0) or 0)
queue_failed_total = int(queue_payload.get("failed", 0) or 0)
@@ -558,7 +687,7 @@ def fetch_overview() -> dict:
}
for item in list(active_jobs_aggregate.get("steps") or [])[:8]
]
aggregate_node_throughput = [
aggregate_node_throughput_all = [
{
"node_code": str(item.get("node_code") or ""),
"items_pending": int(item.get("items_pending", 0) or 0),
@@ -574,8 +703,9 @@ def fetch_overview() -> dict:
"failed_recent": int(item.get("failed_recent", 0) or 0),
"blacklisted_recent": int(item.get("blacklisted_recent", 0) or 0),
}
for item in list(active_jobs_aggregate.get("nodes") or [])[:8]
for item in list(active_jobs_aggregate.get("nodes") or [])
]
aggregate_node_throughput = aggregate_node_throughput_all[:8]
queue_step_queue = [
{
"step_code": str(item.get("step_code") or ""),
@@ -600,7 +730,7 @@ def fetch_overview() -> dict:
limit=8,
)
]
queue_node_throughput = [
queue_node_throughput_all = [
{
"node_code": str(item.get("node_code") or ""),
"items_pending": int(item.get("items_pending", 0) or 0),
@@ -616,15 +746,18 @@ def fetch_overview() -> dict:
"failed_recent": int(item.get("failed_recent", 0) or 0),
"blacklisted_recent": int(item.get("blacklisted_recent", 0) or 0),
}
for item in list(queue_health.get("nodes") or [])[:8]
for item in list(queue_health.get("nodes") or [])
]
queue_node_throughput = queue_node_throughput_all[:8]
step_queue = aggregate_step_queue
node_throughput = aggregate_node_throughput
active_execution_node_source = aggregate_node_throughput_all
aggregate_ppm = float((active_jobs_aggregate.get("throughput") or {}).get("processed_per_minute", 0) or 0)
queue_ppm = float((queue_health.get("throughput") or {}).get("processed_per_minute", 0) or 0)
if queue_health.get("has_active_job") or queue_ppm > aggregate_ppm:
step_queue = queue_step_queue
node_throughput = queue_node_throughput
active_execution_node_source = queue_node_throughput_all
if step_queue:
bottleneck_step = max(
step_queue,
@@ -668,13 +801,7 @@ def fetch_overview() -> dict:
"recommended_additional_workers": int(capacity_plan.get("recommended_additional_workers", 0) or 0),
"online_worker_nodes": online_worker_nodes,
"dedicated_online_worker_nodes": dedicated_online_worker_nodes,
"active_execution_nodes": sum(
1
for item in node_throughput
if int(item.get("items_running", 0) or 0) > 0
or int(item.get("items_claimed", 0) or 0) > 0
or int(item.get("processed_recent", 0) or 0) > 0
),
"active_execution_nodes": _count_active_execution_servers(active_execution_node_source),
}
result["processed_per_minute"] = ops_processed_per_minute
result["processed_recent"] = ops_processed_recent
@@ -686,12 +813,26 @@ def fetch_overview() -> dict:
result["queue_claimed_total"] = queue_claimed_total
result["queue_running_total"] = queue_running_total
result["queue_display_running_total"] = max(queue_display_running_total, queue_running_total)
result["queue_display_max_threads"] = max(
int((active_job_summary or {}).get("display_max_threads", 0) or 0),
sum(int(item.get("max_threads", 0) or 0) for item in list(queue_health.get("nodes") or []) if isinstance(item, dict)),
)
result["queue_completed_total"] = queue_completed_total
result["queue_blacklist_total"] = queue_blacklist_total
result["queue_failed_total"] = queue_failed_total
result["current_job_blacklisted"] = queue_blacklist_total
result["recent_blacklisted_total"] = ops_blacklisted_recent
result["cumulative_blacklisted_total"] = int(result.get("blacklist_total", 0) or 0)
result["backlog_pending_total"] = max(backlog_pending_total, queue_pending_total)
result["backlog_claimed_total"] = max(backlog_claimed_total, queue_claimed_total)
result["backlog_running_total"] = max(backlog_running_total, queue_running_total)
result["backlog_register_pending_total"] = backlog_register_pending_total
result["backlog_downstream_pending_total"] = backlog_downstream_pending_total
result["cluster_proxy_available_count"] = int(detect_status.get("available_proxy_count", 0) or 0)
result["cluster_proxy_runtime_label"] = str(detect_status.get("proxy_runtime_label") or "").strip()
result["cluster_proxy_runtime_detail"] = str(detect_status.get("proxy_runtime_detail") or "").strip()
result["cluster_proxy_last_refresh_status"] = str(detect_status.get("proxy_last_refresh_status") or "").strip()
result["aggregate_process_count"] = int(detect_status.get("aggregate_process_count", 0) or 0)
result["aggregate_participating_node_count"] = int(detect_status.get("aggregate_participating_node_count", 0) or 0)
result["aggregate_active_thread_count"] = int(detect_status.get("active_thread_count", 0) or 0)
return result