admin/getDomain
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: add ops center and node onboarding flow

Browse Source
This commit is contained in:
Your Name
2026-04-18 23:52:51 +08:00
parent 246838ae4c
commit b9c29481b5
142 changed files with 89727 additions and 186 deletions
Download Patch File Download Diff File Expand all files Collapse all files

304
domain-api/app/services/runtime_status_service.py
View File

@@ -7,6 +7,7 @@ from app.core.config import settings
from app.core.db import get_db
from app.core.files import read_json
from app.core.redis_client import get_redis
from app.services.build_info_service import get_runtime_build_info
from app.services.cluster_runtime_service import get_cluster_snapshot
from app.services.detect_service import get_detect_status
from app.services.detect_job_service import get_detect_capacity_plan, get_detect_queue_health
@@ -187,6 +188,89 @@ def _build_multi_region_readiness(
}
def _detect_participation_snapshot(*, row: dict) -> dict:
items_running = int(row.get("items_running", 0) or 0)
items_claimed = int(row.get("items_claimed", 0) or 0)
processed_recent = int(row.get("processed_recent", 0) or 0)
current_load = int(row.get("current_load", 0) or 0)
status = str(row.get("status") or "").strip().lower()
if items_running > 0:
return {
"participation_state": "running",
"participation_label": "执行中",
"participation_reason": f"当前正在执行 {items_running} 项检测任务。",
"is_current_participant": True,
"is_dispatch_active": True,
}
if items_claimed > 0:
return {
"participation_state": "claimed",
"participation_label": "已领待跑",
"participation_reason": f"已领取 {items_claimed} 项任务，等待线程继续执行。",
"is_current_participant": True,
"is_dispatch_active": True,
}
if processed_recent > 0:
return {
"participation_state": "recent_throughput",
"participation_label": "近窗有吞吐",
"participation_reason": f"近 15 分钟内已处理 {processed_recent} 项任务。",
"is_current_participant": True,
"is_dispatch_active": False,
}
if current_load > 0 or status == "busy":
load_value = max(current_load, 1)
return {
"participation_state": "load_syncing",
"participation_label": "负载待确认",
"participation_reason": f"节点当前负载为 {load_value}，但还未观察到已领、执行中或近窗吞吐数据，先归入在线未参与观察。",
"is_current_participant": False,
"is_dispatch_active": False,
}
return {
"participation_state": "standby",
"participation_label": "在线待命",
"participation_reason": "当前未领任务、未执行任务，也没有近窗吞吐。",
"is_current_participant": False,
"is_dispatch_active": False,
}
def _build_detect_node_row(*, node_code: str, cluster_node: dict, job_node: dict, queue_node: dict) -> dict:
metadata = cluster_node.get("metadata") or {}
role = str(cluster_node.get("role") or job_node.get("role") or "worker")
region = str(cluster_node.get("region") or settings.node_region)
is_effective_worker = bool(cluster_node.get("is_effective_worker", False) or role == "worker")
items_total = int(job_node.get("items_total", metadata.get("job_items_total", 0)) or 0)
items_claimed = int(job_node.get("items_claimed", metadata.get("job_items_claimed", 0)) or 0)
items_running = int(job_node.get("items_running", metadata.get("job_items_running", 0)) or 0)
items_completed = int(job_node.get("items_completed", metadata.get("job_items_completed", 0)) or 0)
items_failed = int(job_node.get("items_failed", metadata.get("job_items_failed", 0)) or 0)
row = {
"node_code": node_code,
"role": role,
"region": region,
"status": str(cluster_node.get("status") or "unknown"),
"is_effective_worker": is_effective_worker,
"detect_participating": False,
"current_load": int(cluster_node.get("current_load", 0) or 0),
"items_total": items_total,
"items_pending": int(job_node.get("items_pending", max(items_total - items_claimed - items_completed - items_failed, 0)) or 0),
"items_claimed": items_claimed,
"items_running": items_running,
"items_completed": items_completed,
"items_failed": items_failed,
"processed_recent": int(queue_node.get("processed_recent", 0) or 0),
"processed_per_minute": float(queue_node.get("processed_per_minute", 0) or 0),
"last_heartbeat_at": str(cluster_node.get("last_heartbeat_at") or ""),
"metrics_source": "active_job" if bool(job_node) else ("cluster_metadata" if metadata else "derived"),
}
row.update(_detect_participation_snapshot(row=row))
row["detect_participating"] = bool(row.get("is_current_participant", False))
return row
def _build_participating_detect_nodes(*, cluster_snapshot: dict, detect_snapshot: dict, worker_runtime: dict) -> list[dict]:
cluster_nodes = list(cluster_snapshot.get("nodes") or [])
active_job = detect_snapshot.get("active_job") or {}
@@ -204,56 +288,25 @@ def _build_participating_detect_nodes(*, cluster_snapshot: dict, detect_snapshot
}
merged: list[dict] = []
seen: set[str] = set()
for item in list(active_job.get("node_stats") or []):
node_code = str(item.get("node_code") or "").strip()
if not node_code or node_code == "unassigned":
continue
seen.add(node_code)
job_map = {
str(item.get("node_code") or "").strip(): item
for item in list(active_job.get("node_stats") or [])
if str(item.get("node_code") or "").strip() and str(item.get("node_code") or "").strip() != "unassigned"
}
candidate_codes = sorted(set(job_map) | set(queue_map) | set(cluster_map))
for node_code in candidate_codes:
cluster_node = cluster_map.get(node_code, {})
queue_node = queue_map.get(node_code, {})
merged.append(
{
"node_code": node_code,
"role": str(cluster_node.get("role") or "worker"),
"region": str(cluster_node.get("region") or settings.node_region),
"status": str(cluster_node.get("status") or "unknown"),
"is_effective_worker": bool(cluster_node.get("is_effective_worker", False) or str(cluster_node.get("role") or "") == "worker"),
"detect_participating": True,
"current_load": int(cluster_node.get("current_load", 0) or 0),
"items_total": int(item.get("items_total", 0) or 0),
"items_pending": int(item.get("items_pending", 0) or 0),
"items_claimed": int(item.get("items_claimed", 0) or 0),
"items_running": int(item.get("items_running", 0) or 0),
"items_completed": int(item.get("items_completed", 0) or 0),
"items_failed": int(item.get("items_failed", 0) or 0),
"processed_recent": int(queue_node.get("processed_recent", 0) or 0),
"processed_per_minute": float(queue_node.get("processed_per_minute", 0) or 0),
}
if cluster_node and not bool(cluster_node.get("is_effective_worker", False)):
continue
row = _build_detect_node_row(
node_code=node_code,
cluster_node=cluster_node,
job_node=job_map.get(node_code, {}),
queue_node=queue_map.get(node_code, {}),
)
if worker_runtime.get("running", False) and settings.node_code not in seen:
cluster_node = cluster_map.get(settings.node_code, {})
if cluster_node:
merged.append(
{
"node_code": settings.node_code,
"role": str(cluster_node.get("role") or settings.node_role),
"region": str(cluster_node.get("region") or settings.node_region),
"status": str(cluster_node.get("status") or "online"),
"is_effective_worker": True,
"detect_participating": True,
"current_load": int(cluster_node.get("current_load", 0) or 0),
"items_total": 0,
"items_pending": 0,
"items_claimed": 0,
"items_running": 0,
"items_completed": 0,
"items_failed": 0,
"processed_recent": 0,
"processed_per_minute": 0,
}
)
if not bool(row.get("is_current_participant", False)):
continue
merged.append(row)
return sorted(
merged,
@@ -261,11 +314,143 @@ def _build_participating_detect_nodes(*, cluster_snapshot: dict, detect_snapshot
-int(item.get("items_running", 0) or 0),
-int(item.get("items_claimed", 0) or 0),
-int(item.get("processed_recent", 0) or 0),
-int(item.get("current_load", 0) or 0),
str(item.get("node_code") or ""),
),
)
def _build_standby_detect_nodes(*, cluster_snapshot: dict, participating_nodes: list[dict]) -> list[dict]:
cluster_nodes = list(cluster_snapshot.get("nodes") or [])
participating_codes = {
str(item.get("node_code") or "").strip()
for item in list(participating_nodes or [])
if str(item.get("node_code") or "").strip()
}
standby_rows: list[dict] = []
for node in cluster_nodes:
node_code = str(node.get("node_code") or "").strip()
node_status = str(node.get("status") or "").strip()
node_current_load = int(node.get("current_load", 0) or 0)
metadata = node.get("metadata") or {}
if not node_code:
continue
if not bool(node.get("is_effective_worker", False)):
continue
if node_status not in {"online", "busy"}:
continue
if node_code in participating_codes:
continue
standby_state = "load_syncing" if node_current_load > 0 or node_status == "busy" else "standby"
standby_label = "负载待确认" if standby_state == "load_syncing" else "在线待命"
standby_reason = (
str(metadata.get("detail") or "").strip()
or str(metadata.get("phase_detail") or "").strip()
or (
f"当前阶段：{str(metadata.get('phase') or metadata.get('phase_label') or '').strip()}"
if str(metadata.get("phase") or metadata.get("phase_label") or "").strip()
else ""
)
or (
f"节点当前负载为 {node_current_load}，但还未观察到已领、执行中或近窗吞吐数据。"
if standby_state == "load_syncing"
else "节点在线，当前未领任务、未执行任务，也没有近窗吞吐。"
)
)
standby_rows.append(
{
"node_code": node_code,
"role": str(node.get("role") or "worker"),
"region": str(node.get("region") or settings.node_region),
"status": node_status,
"is_effective_worker": True,
"detect_participating": False,
"participation_state": standby_state,
"participation_label": standby_label,
"participation_reason": standby_reason,
"current_load": node_current_load,
"last_heartbeat_at": str(node.get("last_heartbeat_at") or ""),
"phase": str(metadata.get("phase") or metadata.get("phase_label") or "").strip(),
"detail": str(metadata.get("detail") or metadata.get("phase_detail") or "").strip(),
"worker_online": bool(metadata.get("worker_online", False)),
"standby_reason": standby_reason,
}
)
return sorted(
standby_rows,
key=lambda item: (
str(item.get("status") or ""),
str(item.get("role") or ""),
str(item.get("node_code") or ""),
),
)
def _build_detect_participation_summary(
*,
participating_nodes: list[dict],
standby_nodes: list[dict],
cluster_snapshot: dict,
) -> dict:
cluster_nodes = list(cluster_snapshot.get("nodes") or [])
effective_online_nodes = [
node
for node in cluster_nodes
if bool(node.get("is_effective_worker", False)) and str(node.get("status") or "").strip() in {"online", "busy"}
]
dispatch_active_nodes = [
row for row in participating_nodes
if bool(row.get("is_dispatch_active", False))
]
recent_only_nodes = [
row for row in participating_nodes
if str(row.get("participation_state") or "").strip() == "recent_throughput"
]
load_syncing_nodes = [
row for row in standby_nodes
if str(row.get("participation_state") or "").strip() == "load_syncing"
]
pure_standby_nodes = [
row for row in standby_nodes
if str(row.get("participation_state") or "").strip() == "standby"
]
dedicated_worker_nodes = [
node for node in effective_online_nodes
if str(node.get("role") or "").strip() == "worker"
]
controller_worker_nodes = [
node for node in effective_online_nodes
if str(node.get("role") or "").strip() == "control"
]
summary_parts = [
f"有效执行节点 {len(effective_online_nodes)} 台",
f"正在执行/领任务 {len(dispatch_active_nodes)} 台",
f"近窗刚有吞吐 {len(recent_only_nodes)} 台",
f"在线但未参与 {len(standby_nodes)} 台",
]
if load_syncing_nodes:
summary_parts.append(f"其中负载待确认 {len(load_syncing_nodes)} 台")
return {
"effective_online_nodes": len(effective_online_nodes),
"participating_nodes": len(participating_nodes),
"dispatch_active_nodes": len(dispatch_active_nodes),
"recent_only_nodes": len(recent_only_nodes),
"non_participating_nodes": len(standby_nodes),
"standby_nodes": len(pure_standby_nodes),
"load_syncing_nodes": len(load_syncing_nodes),
"dedicated_worker_nodes": len(dedicated_worker_nodes),
"controller_worker_nodes": len(controller_worker_nodes),
"dispatch_active_node_codes": [str(item.get("node_code") or "") for item in dispatch_active_nodes],
"recent_only_node_codes": [str(item.get("node_code") or "") for item in recent_only_nodes],
"non_participating_node_codes": [str(item.get("node_code") or "") for item in standby_nodes],
"standby_node_codes": [str(item.get("node_code") or "") for item in pure_standby_nodes],
"load_syncing_node_codes": [str(item.get("node_code") or "") for item in load_syncing_nodes],
"summary": "；".join(summary_parts),
}
def get_runtime_status() -> dict:
runtime_settings = get_runtime_settings()
worker_runtime = detect_worker_runtime()
@@ -314,6 +499,17 @@ def get_runtime_status() -> dict:
"worker_mode": worker_runtime.get("mode", runtime_settings.get("worker_mode", "windows-local")),
"queue_health": queue_health,
"capacity_plan": capacity_plan,
"log_sync": {
"enabled": bool(runtime_settings.get("worker_log_sync_enabled", False)),
"mode": str(runtime_settings.get("worker_log_sync_mode", "key") or "key"),
"line_count": int(detect_snapshot.get("remote_log_line_count", 0) or 0),
"source_node_count": int(detect_snapshot.get("remote_log_node_count", 0) or 0),
"source_nodes": list(detect_snapshot.get("remote_log_nodes") or []),
"source_node_summaries": list(detect_snapshot.get("remote_log_node_summaries") or []),
"last_at": str(detect_snapshot.get("remote_log_last_at") or ""),
"last_line": str(detect_snapshot.get("remote_log_last_line") or ""),
"preview_lines": list(detect_snapshot.get("remote_log_lines") or [])[-20:],
},
}
if not worker_expected_on_this_node:
detect_payload.update(
@@ -349,6 +545,16 @@ def get_runtime_status() -> dict:
detect_snapshot=detect_payload,
worker_runtime=worker_runtime,
)
detect_payload["standby_nodes"] = _build_standby_detect_nodes(
cluster_snapshot=cluster_snapshot,
participating_nodes=detect_payload["participating_nodes"],
)
detect_payload["non_participating_nodes"] = list(detect_payload["standby_nodes"] or [])
detect_payload["participation_summary"] = _build_detect_participation_summary(
participating_nodes=detect_payload["participating_nodes"],
standby_nodes=detect_payload["non_participating_nodes"],
cluster_snapshot=cluster_snapshot,
)
append_runtime_projection_if_changed(detect=detect_payload, cluster=cluster_snapshot)
sync_summary = get_sync_summary(record_limit=5)
readiness = _build_multi_region_readiness(
@@ -357,6 +563,7 @@ def get_runtime_status() -> dict:
worker_runtime=worker_runtime,
sync_agent_runtime=sync_agent_runtime,
)
build_info = get_runtime_build_info()
return {
"api": {
@@ -371,6 +578,7 @@ def get_runtime_status() -> dict:
"health_url": f"http://127.0.0.1:{settings.api_port}/health",
"stdout_log": _runtime_log_path("domain-api.stdout.log"),
"stderr_log": _runtime_log_path("domain-api.stderr.log"),
"build": build_info,
},
"node": {
"code": settings.node_code,