admin/getDomain
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

d

Browse Source
This commit is contained in:
Your Name
2026-04-22 14:13:21 +08:00
parent e0406b5d0e
commit 7cbde2aa78
145 changed files with 23086 additions and 2243 deletions
Download Patch File Download Diff File Expand all files Collapse all files

274
domain-api/app/services/runtime_status_service.py
View File

@@ -10,12 +10,151 @@ from app.core.redis_client import get_redis
from app.services.build_info_service import get_runtime_build_info
from app.services.cluster_runtime_service import get_cluster_snapshot
from app.services.detect_service import get_detect_status
from app.services.detect_job_service import get_detect_capacity_plan, get_detect_queue_health
from app.services.detect_job_service import (
_load_latest_runtime_active_job_snapshot,
get_detect_capacity_plan,
get_detect_queue_health,
)
from app.services.sync_record_service import append_runtime_projection_if_changed, get_sync_summary
from app.services.runtime_settings_service import get_runtime_settings
from app.services.worker_control_service import detect_sync_agent_runtime, detect_worker_runtime
def _align_queue_health_with_backlog(queue_health: dict | None, backlog_snapshot: dict | None) -> dict:
normalized = dict(queue_health or {})
queue = dict(normalized.get("queue") or {})
backlog = dict(backlog_snapshot or {})
pending_total = max(int(queue.get("pending", 0) or 0), int(backlog.get("pending_total", 0) or 0))
claimed_total = max(int(queue.get("claimed", 0) or 0), int(backlog.get("claimed_total", 0) or 0))
running_total = max(int(queue.get("running", 0) or 0), int(backlog.get("running_total", 0) or 0))
completed_total = max(int(queue.get("completed", 0) or 0), int(backlog.get("completed_total", 0) or 0))
blacklisted_total = max(int(queue.get("blacklisted", 0) or 0), int(backlog.get("blacklisted_total", 0) or 0))
failed_total = max(int(queue.get("failed", 0) or 0), int(backlog.get("failed_total", 0) or 0))
terminal_total = max(
int(queue.get("terminal", 0) or 0),
completed_total + blacklisted_total + failed_total,
)
normalized["has_active_job"] = bool(
normalized.get("has_active_job")
or pending_total > 0
or claimed_total > 0
or running_total > 0
or terminal_total > 0
)
normalized["queue"] = {
**queue,
"items_total": pending_total + claimed_total + running_total + terminal_total,
"pending": pending_total,
"claimed": claimed_total,
"running": running_total,
"completed": completed_total,
"blacklisted": blacklisted_total,
"failed": failed_total,
"terminal": terminal_total,
}
return normalized
def _decode_projection_payload(value: object) -> dict:
if isinstance(value, dict):
return dict(value)
if value in (None, ""):
return {}
try:
import json
return dict(json.loads(value))
except Exception:
return {}
def _load_detect_backlog_snapshot() -> dict:
with get_db() as conn:
with conn.cursor() as cur:
cur.execute(
"""
SELECT
COUNT(*) FILTER (WHERE item.status = 'pending') AS pending_total,
COUNT(*) FILTER (WHERE item.status = 'claimed') AS claimed_total,
COUNT(*) FILTER (WHERE item.status = 'running') AS running_total,
COUNT(*) FILTER (WHERE item.status = 'completed') AS completed_total,
COUNT(*) FILTER (WHERE item.status = 'blacklisted') AS blacklisted_total,
COUNT(*) FILTER (WHERE item.status = 'failed') AS failed_total,
COUNT(*) FILTER (WHERE item.status = 'pending' AND item.step_code = 'detect_register') AS register_pending,
COUNT(*) FILTER (WHERE item.status = 'pending' AND item.step_code <> 'detect_register') AS downstream_pending
FROM detect_job_items item
JOIN detect_jobs job ON job.id = item.job_id
WHERE job.status IN ('pending', 'running')
"""
)
row = cur.fetchone() or (0, 0, 0, 0, 0, 0, 0, 0)
return {
"pending_total": int(row[0] or 0),
"claimed_total": int(row[1] or 0),
"running_total": int(row[2] or 0),
"completed_total": int(row[3] or 0),
"blacklisted_total": int(row[4] or 0),
"failed_total": int(row[5] or 0),
"register_pending": int(row[6] or 0),
"downstream_pending": int(row[7] or 0),
}
def _load_latest_remote_runtime_projection_backlog() -> dict:
if not (settings.node_region == "overseas" and settings.node_role == "control"):
return {}
with get_db() as conn:
with conn.cursor() as cur:
cur.execute(
"""
SELECT payload_json
FROM detect_sync_records
WHERE sync_type = 'runtime_projection'
AND source_region = 'mainland'
AND target_region = 'overseas'
AND status IN ('projected', 'pushing', 'synced')
ORDER BY updated_at DESC, id DESC
LIMIT 1
"""
)
row = cur.fetchone()
if not row:
return {}
payload = _decode_projection_payload(row[0])
projection = payload.get("projection") if isinstance(payload, dict) else {}
backlog = projection.get("backlog") if isinstance(projection, dict) else {}
if not isinstance(backlog, dict):
return {}
return {
"pending_total": int(backlog.get("pending_total", 0) or 0),
"claimed_total": int(backlog.get("claimed_total", 0) or 0),
"running_total": int(backlog.get("running_total", 0) or 0),
"completed_total": int(backlog.get("completed_total", 0) or 0),
"blacklisted_total": int(backlog.get("blacklisted_total", 0) or 0),
"failed_total": int(backlog.get("failed_total", 0) or 0),
"register_pending": int(backlog.get("register_pending", 0) or 0),
"downstream_pending": int(backlog.get("downstream_pending", 0) or 0),
}
def _merge_backlog_snapshots(primary: dict, secondary: dict) -> dict:
merged = dict(primary or {})
for key in (
"pending_total",
"claimed_total",
"running_total",
"completed_total",
"blacklisted_total",
"failed_total",
"register_pending",
"downstream_pending",
):
merged[key] = max(int(merged.get(key, 0) or 0), int((secondary or {}).get(key, 0) or 0))
return merged
def _runtime_log_path(filename: str) -> str:
path = Path(__file__).resolve().parents[2] / "runtime" / "logs" / filename
return str(path)
@@ -191,6 +330,8 @@ def _build_multi_region_readiness(
def _detect_participation_snapshot(*, row: dict) -> dict:
items_running = int(row.get("items_running", 0) or 0)
items_claimed = int(row.get("items_claimed", 0) or 0)
active_threads = int(row.get("active_threads", 0) or 0)
max_threads = int(row.get("max_threads", 0) or 0)
processed_recent = int(row.get("processed_recent", 0) or 0)
current_load = int(row.get("current_load", 0) or 0)
status = str(row.get("status") or "").strip().lower()
@@ -211,6 +352,17 @@ def _detect_participation_snapshot(*, row: dict) -> dict:
"is_current_participant": True,
"is_dispatch_active": True,
}
if active_threads > 0:
detail = f"当前活跃线程 {active_threads}"
if max_threads > 0:
detail = f"{detail}/{max_threads}"
return {
"participation_state": "runtime_active",
"participation_label": "执行中",
"participation_reason": detail,
"is_current_participant": True,
"is_dispatch_active": True,
}
if processed_recent > 0:
return {
"participation_state": "recent_throughput",
@@ -242,11 +394,52 @@ def _build_detect_node_row(*, node_code: str, cluster_node: dict, job_node: dict
role = str(cluster_node.get("role") or job_node.get("role") or "worker")
region = str(cluster_node.get("region") or settings.node_region)
is_effective_worker = bool(cluster_node.get("is_effective_worker", False) or role == "worker")
items_total = int(job_node.get("items_total", metadata.get("job_items_total", 0)) or 0)
items_claimed = int(job_node.get("items_claimed", metadata.get("job_items_claimed", 0)) or 0)
items_running = int(job_node.get("items_running", metadata.get("job_items_running", 0)) or 0)
items_completed = int(job_node.get("items_completed", metadata.get("job_items_completed", 0)) or 0)
items_failed = int(job_node.get("items_failed", metadata.get("job_items_failed", 0)) or 0)
items_total = max(
int(job_node.get("items_total", 0) or 0),
int(queue_node.get("items_total", 0) or 0),
int(metadata.get("job_items_total", 0) or 0),
)
items_claimed = max(
int(job_node.get("items_claimed", 0) or 0),
int(queue_node.get("items_claimed", 0) or 0),
int(metadata.get("job_items_claimed", 0) or 0),
)
items_running = max(
int(job_node.get("items_running", 0) or 0),
int(queue_node.get("items_running", 0) or 0),
int(metadata.get("job_items_running", 0) or 0),
)
items_completed = max(
int(job_node.get("items_completed", 0) or 0),
int(queue_node.get("items_completed", 0) or 0),
int(metadata.get("job_items_completed", 0) or 0),
)
items_failed = max(
int(job_node.get("items_failed", 0) or 0),
int(queue_node.get("items_failed", 0) or 0),
int(metadata.get("job_items_failed", 0) or 0),
)
items_blacklisted = max(
int(job_node.get("items_blacklisted", 0) or 0),
int(queue_node.get("items_blacklisted", 0) or 0),
)
active_threads = max(
int((cluster_node.get("metadata") or {}).get("active_threads", 0) or 0),
int(job_node.get("active_threads", 0) or 0),
int(queue_node.get("active_threads", 0) or 0),
)
max_threads = max(
int((cluster_node.get("metadata") or {}).get("max_threads", 0) or 0),
int(job_node.get("max_threads", 0) or 0),
int(queue_node.get("max_threads", 0) or 0),
)
current_load = max(items_running, active_threads, 0)
derived_pending = max(items_total - items_claimed - items_running - items_completed - items_failed - items_blacklisted, 0)
items_pending = max(
int(job_node.get("items_pending", 0) or 0),
int(queue_node.get("items_pending", 0) or 0),
derived_pending,
)
row = {
"node_code": node_code,
"role": role,
@@ -254,13 +447,16 @@ def _build_detect_node_row(*, node_code: str, cluster_node: dict, job_node: dict
"status": str(cluster_node.get("status") or "unknown"),
"is_effective_worker": is_effective_worker,
"detect_participating": False,
"current_load": int(cluster_node.get("current_load", 0) or 0),
"current_load": current_load,
"items_total": items_total,
"items_pending": int(job_node.get("items_pending", max(items_total - items_claimed - items_completed - items_failed, 0)) or 0),
"items_pending": items_pending,
"items_claimed": items_claimed,
"items_running": items_running,
"items_completed": items_completed,
"items_blacklisted": items_blacklisted,
"items_failed": items_failed,
"active_threads": active_threads,
"max_threads": max_threads,
"processed_recent": int(queue_node.get("processed_recent", 0) or 0),
"processed_per_minute": float(queue_node.get("processed_per_minute", 0) or 0),
"last_heartbeat_at": str(cluster_node.get("last_heartbeat_at") or ""),
@@ -465,8 +661,13 @@ def get_runtime_status() -> dict:
effective_online_worker_nodes = int((cluster_snapshot.get("summary") or {}).get("online_worker_nodes", 0) or 0)
if effective_online_worker_nodes <= 0 and worker_runtime.get("running", False):
effective_online_worker_nodes = max(1, worker_runtime.get("process_count", 1) or 1)
backlog_snapshot = _load_detect_backlog_snapshot()
remote_backlog_snapshot = _load_latest_remote_runtime_projection_backlog()
runtime_snapshot_backlog = dict(_load_latest_runtime_active_job_snapshot(15).get("backlog") or {})
backlog_snapshot = _merge_backlog_snapshots(backlog_snapshot, remote_backlog_snapshot)
backlog_snapshot = _merge_backlog_snapshots(backlog_snapshot, runtime_snapshot_backlog)
capacity_plan = get_detect_capacity_plan(
queue_health=queue_health,
queue_health=_align_queue_health_with_backlog(queue_health, backlog_snapshot),
online_worker_nodes=effective_online_worker_nodes,
target_finish_hours=6,
)
@@ -498,6 +699,7 @@ def get_runtime_status() -> dict:
"worker_online": worker_runtime.get("running", False),
"worker_mode": worker_runtime.get("mode", runtime_settings.get("worker_mode", "windows-local")),
"queue_health": queue_health,
"backlog": backlog_snapshot,
"capacity_plan": capacity_plan,
"log_sync": {
"enabled": bool(runtime_settings.get("worker_log_sync_enabled", False)),
@@ -565,8 +767,7 @@ def get_runtime_status() -> dict:
)
build_info = get_runtime_build_info()
return {
"api": {
api_payload = {
"service": "domain-api",
"version": "0.1.0",
"api_prefix": settings.api_prefix,
@@ -579,13 +780,13 @@ def get_runtime_status() -> dict:
"stdout_log": _runtime_log_path("domain-api.stdout.log"),
"stderr_log": _runtime_log_path("domain-api.stderr.log"),
"build": build_info,
},
"node": {
}
node_payload = {
"code": settings.node_code,
"region": settings.node_region,
"role": settings.node_role,
},
"worker": {
}
worker_payload = {
"mode": worker_runtime.get("mode", runtime_settings.get("worker_mode", "windows-local")),
"service_name": runtime_settings.get("worker_service_name", settings.worker_service_name),
"running": worker_runtime.get("running", False),
@@ -594,8 +795,8 @@ def get_runtime_status() -> dict:
"latest_start_time": worker_runtime.get("latest_start_time", ""),
"message": worker_runtime.get("message", ""),
"log_path": str(Path(settings.domain_root) / "detect_worker.log"),
},
"sync_agent": {
}
sync_agent_payload = {
"mode": sync_agent_runtime.get("mode", runtime_settings.get("worker_mode", "windows-local")),
"service_name": runtime_settings.get("sync_agent_service_name", settings.sync_agent_service_name),
"running": sync_agent_runtime.get("running", False),
@@ -603,7 +804,44 @@ def get_runtime_status() -> dict:
"latest_start_time": sync_agent_runtime.get("latest_start_time", ""),
"message": sync_agent_runtime.get("message", ""),
"expected_on_this_node": settings.node_region == "mainland" and settings.node_role == "control",
},
}
compatibility_payload = {
# Backward-compatible flat fields for older pages / stale built assets.
"api_online": bool(api_payload.get("pid")),
"api_service_name": api_payload.get("service_name", ""),
"worker_online": bool(worker_payload.get("running", False)),
"worker_mode": worker_payload.get("mode", ""),
"worker_service_name": worker_payload.get("service_name", ""),
"worker_process_count": worker_payload.get("process_count", 0),
"worker_latest_start_time": worker_payload.get("latest_start_time", ""),
"worker_runtime_message": worker_payload.get("message", ""),
"thread_count": detect_snapshot.get("thread_count", 0),
"thread_count_default": detect_snapshot.get("thread_count_default", 0),
"thread_count_source": detect_snapshot.get("thread_count_source", ""),
"thread_count_override": detect_snapshot.get("thread_count_override"),
"active_thread_count": detect_payload.get("active_thread_count", 0),
"max_thread_count": detect_payload.get("max_thread_count", 0),
"progress": detect_payload.get("progress", {}),
"backlog": detect_payload.get("backlog", {}),
"progress_percent": detect_payload.get("progress_percent", 0),
"available_proxy_count": detect_payload.get("available_proxy_count", 0),
"proxy_pool_count": detect_payload.get("proxy_pool_count", 0),
"proxy_runtime_label": detect_payload.get("proxy_runtime_label", ""),
"proxy_runtime_detail": detect_payload.get("proxy_runtime_detail", ""),
"proxy_runtime_reason": detect_payload.get("proxy_runtime_reason", ""),
"proxy_last_refresh_time": detect_payload.get("proxy_last_refresh_time", ""),
"recent_event": detect_payload.get("recent_event", ""),
"recent_warning": detect_payload.get("recent_warning", ""),
"runtime_state": worker_runtime.get("runtime_state") or {},
"cluster_summary": cluster_snapshot.get("summary") or {},
}
return {
**compatibility_payload,
"api": api_payload,
"node": node_payload,
"worker": worker_payload,
"sync_agent": sync_agent_payload,
"detect": detect_payload,
"cluster": cluster_snapshot,
"sync": sync_summary,