978 lines
43 KiB
Python
978 lines
43 KiB
Python
from __future__ import annotations
|
||
|
||
import os
|
||
from pathlib import Path
|
||
|
||
from app.core.config import settings
|
||
from app.core.db import get_db
|
||
from app.core.files import read_json
|
||
from app.core.redis_client import get_redis
|
||
from app.services.build_info_service import get_runtime_build_info
|
||
from app.services.cluster_runtime_service import get_cluster_snapshot
|
||
from app.services.detect_service import get_detect_status
|
||
from app.services.detect_job_service import (
|
||
_load_latest_runtime_active_job_snapshot,
|
||
get_detect_capacity_plan,
|
||
get_detect_queue_health,
|
||
)
|
||
from app.services.sync_record_service import append_runtime_projection_if_changed, get_sync_summary
|
||
from app.services.runtime_settings_service import get_runtime_settings
|
||
from app.services.worker_control_service import detect_sync_agent_runtime, detect_worker_runtime
|
||
|
||
|
||
def _align_queue_health_with_backlog(queue_health: dict | None, backlog_snapshot: dict | None) -> dict:
|
||
normalized = dict(queue_health or {})
|
||
queue = dict(normalized.get("queue") or {})
|
||
backlog = dict(backlog_snapshot or {})
|
||
|
||
pending_total = max(int(queue.get("pending", 0) or 0), int(backlog.get("pending_total", 0) or 0))
|
||
claimed_total = max(int(queue.get("claimed", 0) or 0), int(backlog.get("claimed_total", 0) or 0))
|
||
running_total = max(int(queue.get("running", 0) or 0), int(backlog.get("running_total", 0) or 0))
|
||
completed_total = max(int(queue.get("completed", 0) or 0), int(backlog.get("completed_total", 0) or 0))
|
||
blacklisted_total = max(int(queue.get("blacklisted", 0) or 0), int(backlog.get("blacklisted_total", 0) or 0))
|
||
failed_total = max(int(queue.get("failed", 0) or 0), int(backlog.get("failed_total", 0) or 0))
|
||
terminal_total = max(
|
||
int(queue.get("terminal", 0) or 0),
|
||
completed_total + blacklisted_total + failed_total,
|
||
)
|
||
|
||
normalized["has_active_job"] = bool(
|
||
normalized.get("has_active_job")
|
||
or pending_total > 0
|
||
or claimed_total > 0
|
||
or running_total > 0
|
||
or terminal_total > 0
|
||
)
|
||
normalized["queue"] = {
|
||
**queue,
|
||
"items_total": pending_total + claimed_total + running_total + terminal_total,
|
||
"pending": pending_total,
|
||
"claimed": claimed_total,
|
||
"running": running_total,
|
||
"completed": completed_total,
|
||
"blacklisted": blacklisted_total,
|
||
"failed": failed_total,
|
||
"terminal": terminal_total,
|
||
}
|
||
return normalized
|
||
|
||
|
||
def _decode_projection_payload(value: object) -> dict:
|
||
if isinstance(value, dict):
|
||
return dict(value)
|
||
if value in (None, ""):
|
||
return {}
|
||
try:
|
||
import json
|
||
|
||
return dict(json.loads(value))
|
||
except Exception:
|
||
return {}
|
||
|
||
|
||
def _load_detect_backlog_snapshot() -> dict:
|
||
with get_db() as conn:
|
||
with conn.cursor() as cur:
|
||
cur.execute(
|
||
"""
|
||
SELECT
|
||
COUNT(*) FILTER (WHERE item.status = 'pending') AS pending_total,
|
||
COUNT(*) FILTER (WHERE item.status = 'claimed') AS claimed_total,
|
||
COUNT(*) FILTER (WHERE item.status = 'running') AS running_total,
|
||
COUNT(*) FILTER (WHERE item.status = 'completed') AS completed_total,
|
||
COUNT(*) FILTER (WHERE item.status = 'blacklisted') AS blacklisted_total,
|
||
COUNT(*) FILTER (WHERE item.status = 'failed') AS failed_total,
|
||
COUNT(*) FILTER (WHERE item.status = 'pending' AND item.step_code = 'detect_register') AS register_pending,
|
||
COUNT(*) FILTER (WHERE item.status = 'pending' AND item.step_code <> 'detect_register') AS downstream_pending
|
||
FROM detect_job_items item
|
||
JOIN detect_jobs job ON job.id = item.job_id
|
||
WHERE job.status IN ('pending', 'running')
|
||
"""
|
||
)
|
||
row = cur.fetchone() or (0, 0, 0, 0, 0, 0, 0, 0)
|
||
return {
|
||
"pending_total": int(row[0] or 0),
|
||
"claimed_total": int(row[1] or 0),
|
||
"running_total": int(row[2] or 0),
|
||
"completed_total": int(row[3] or 0),
|
||
"blacklisted_total": int(row[4] or 0),
|
||
"failed_total": int(row[5] or 0),
|
||
"register_pending": int(row[6] or 0),
|
||
"downstream_pending": int(row[7] or 0),
|
||
}
|
||
|
||
|
||
def _load_latest_remote_runtime_projection_backlog() -> dict:
|
||
if not (settings.node_region == "overseas" and settings.node_role == "control"):
|
||
return {}
|
||
with get_db() as conn:
|
||
with conn.cursor() as cur:
|
||
cur.execute(
|
||
"""
|
||
SELECT payload_json
|
||
FROM detect_sync_records
|
||
WHERE sync_type = 'runtime_projection'
|
||
AND source_region = 'mainland'
|
||
AND target_region = 'overseas'
|
||
AND status IN ('projected', 'pushing', 'synced')
|
||
ORDER BY updated_at DESC, id DESC
|
||
LIMIT 1
|
||
"""
|
||
)
|
||
row = cur.fetchone()
|
||
if not row:
|
||
return {}
|
||
payload = _decode_projection_payload(row[0])
|
||
projection = payload.get("projection") if isinstance(payload, dict) else {}
|
||
backlog = projection.get("backlog") if isinstance(projection, dict) else {}
|
||
if not isinstance(backlog, dict):
|
||
return {}
|
||
return {
|
||
"pending_total": int(backlog.get("pending_total", 0) or 0),
|
||
"claimed_total": int(backlog.get("claimed_total", 0) or 0),
|
||
"running_total": int(backlog.get("running_total", 0) or 0),
|
||
"completed_total": int(backlog.get("completed_total", 0) or 0),
|
||
"blacklisted_total": int(backlog.get("blacklisted_total", 0) or 0),
|
||
"failed_total": int(backlog.get("failed_total", 0) or 0),
|
||
"register_pending": int(backlog.get("register_pending", 0) or 0),
|
||
"downstream_pending": int(backlog.get("downstream_pending", 0) or 0),
|
||
}
|
||
|
||
|
||
def _merge_backlog_snapshots(primary: dict, secondary: dict) -> dict:
|
||
merged = dict(primary or {})
|
||
for key in (
|
||
"pending_total",
|
||
"claimed_total",
|
||
"running_total",
|
||
"completed_total",
|
||
"blacklisted_total",
|
||
"failed_total",
|
||
"register_pending",
|
||
"downstream_pending",
|
||
):
|
||
merged[key] = max(int(merged.get(key, 0) or 0), int((secondary or {}).get(key, 0) or 0))
|
||
return merged
|
||
|
||
|
||
def _runtime_log_path(filename: str) -> str:
|
||
path = Path(__file__).resolve().parents[2] / "runtime" / "logs" / filename
|
||
return str(path)
|
||
|
||
|
||
def _domain_cookie_status(filename: str) -> tuple[bool, str]:
|
||
path = Path(settings.domain_root) / filename
|
||
return path.exists(), str(path)
|
||
|
||
|
||
def _bloom_filter_status() -> tuple[bool, str]:
|
||
try:
|
||
redis_client = get_redis()
|
||
modules = redis_client.execute_command("MODULE", "LIST")
|
||
for module in modules or []:
|
||
module_parts = module[::2]
|
||
module_values = module[1::2]
|
||
module_info = dict(zip(module_parts, module_values))
|
||
module_name = str(module_info.get("name", "")).lower()
|
||
if module_name in {"bf", "redisbloom"}:
|
||
return True, "RedisBloom 已安装"
|
||
return False, "Redis 未安装 RedisBloom，当前将使用普通缓存"
|
||
except Exception as exc:
|
||
return False, f"RedisBloom 检查失败: {exc}"
|
||
|
||
|
||
def _build_multi_region_readiness(
|
||
*,
|
||
cluster_snapshot: dict,
|
||
sync_summary: dict,
|
||
worker_runtime: dict,
|
||
sync_agent_runtime: dict,
|
||
) -> dict:
|
||
nodes = list(cluster_snapshot.get("nodes") or [])
|
||
summary = cluster_snapshot.get("summary") or {}
|
||
batch_summary = (sync_summary.get("detect_result_batches") or {})
|
||
batch_states = batch_summary.get("state_counts") or {}
|
||
|
||
online_control_nodes = int(summary.get("online_control_nodes", 0) or 0)
|
||
online_worker_nodes = int(summary.get("online_worker_nodes", 0) or 0)
|
||
mainland_control_nodes = [
|
||
node for node in nodes
|
||
if str(node.get("region") or "") == "mainland"
|
||
and str(node.get("role") or "") == "control"
|
||
and str(node.get("status") or "") in {"online", "busy"}
|
||
]
|
||
mainland_worker_nodes = [
|
||
node for node in nodes
|
||
if str(node.get("region") or "") == "mainland"
|
||
and bool(node.get("is_effective_worker", False))
|
||
and str(node.get("status") or "") in {"online", "busy"}
|
||
]
|
||
|
||
blocking_issues: list[str] = []
|
||
warning_issues: list[str] = []
|
||
info_items: list[str] = []
|
||
|
||
online_pairs = {
|
||
(
|
||
str(node.get("region") or "").strip(),
|
||
str(node.get("role") or "").strip(),
|
||
)
|
||
for node in nodes
|
||
if str(node.get("status") or "") in {"online", "busy"}
|
||
}
|
||
critical_stale_nodes: list[str] = []
|
||
redundant_stale_nodes: list[str] = []
|
||
critical_offline_nodes: list[str] = []
|
||
redundant_offline_nodes: list[str] = []
|
||
|
||
if online_control_nodes <= 0:
|
||
blocking_issues.append("当前没有在线控制面节点，无法视为正式可用集群。")
|
||
|
||
if settings.node_region == "mainland" and settings.node_role == "control":
|
||
if not sync_agent_runtime.get("running", False):
|
||
blocking_issues.append("当前节点应承载 sync-agent，但服务未运行。")
|
||
if str(settings.sync_target_api_base_url or "").strip() == "":
|
||
blocking_issues.append("当前节点未配置 SYNC_TARGET_API_BASE_URL，无法向海外控制面推送。")
|
||
if not bool(settings.sync_push_enabled):
|
||
blocking_issues.append("当前节点未启用 SYNC_PUSH_ENABLED，结果同步不会自动执行。")
|
||
|
||
if settings.node_region == "overseas" and online_control_nodes > 0 and not mainland_control_nodes:
|
||
warning_issues.append("当前尚未观察到在线的大陆 controller 节点，后续自动结果同步仍未进入正式双地域态。")
|
||
|
||
if online_worker_nodes <= 0:
|
||
warning_issues.append("当前没有在线 Worker 节点，检测任务无法在多机状态下继续推进。")
|
||
|
||
for node in nodes:
|
||
node_status = str(node.get("status") or "").strip()
|
||
if node_status not in {"stale", "offline"}:
|
||
continue
|
||
node_code = str(node.get("node_code") or "").strip()
|
||
node_region = str(node.get("region") or "").strip()
|
||
node_role = str(node.get("role") or "").strip()
|
||
metadata = node.get("metadata") or {}
|
||
replaced_by_online_peer = (node_region, node_role) in online_pairs
|
||
redundant = replaced_by_online_peer and (
|
||
node_role == "control"
|
||
or not bool(node.get("is_effective_worker", False))
|
||
or str(metadata.get("service") or "") == "runtime-ingest"
|
||
)
|
||
if node_status == "stale":
|
||
(redundant_stale_nodes if redundant else critical_stale_nodes).append(node_code)
|
||
else:
|
||
(redundant_offline_nodes if redundant else critical_offline_nodes).append(node_code)
|
||
|
||
if critical_stale_nodes:
|
||
warning_issues.append(f"存在失活节点: {'、'.join(critical_stale_nodes)}")
|
||
if critical_offline_nodes:
|
||
warning_issues.append(f"存在离线节点: {'、'.join(critical_offline_nodes)}")
|
||
if redundant_stale_nodes:
|
||
info_items.append(f"存在历史失活节点（已被在线同类节点覆盖）: {'、'.join(redundant_stale_nodes)}")
|
||
if redundant_offline_nodes:
|
||
info_items.append(f"存在历史离线节点（已被在线同类节点覆盖）: {'、'.join(redundant_offline_nodes)}")
|
||
|
||
failed_batches = int(batch_states.get("failed", 0) or 0)
|
||
projected_batches = int(batch_states.get("projected", 0) or 0)
|
||
pushing_batches = int(batch_states.get("pushing", 0) or 0)
|
||
synced_batches = int(batch_states.get("synced", 0) or 0)
|
||
if failed_batches > 0:
|
||
warning_issues.append(f"存在 {failed_batches} 个结果批次同步失败，需要检查 sync-agent 或目标接收面。")
|
||
if projected_batches > 0:
|
||
warning_issues.append(f"存在 {projected_batches} 个结果批次仍待推送。")
|
||
if pushing_batches > 0:
|
||
info_items.append(f"当前有 {pushing_batches} 个结果批次正在推送。")
|
||
if synced_batches > 0:
|
||
info_items.append(f"最近已接收 {synced_batches} 个结果批次。")
|
||
|
||
if worker_runtime.get("running", False):
|
||
info_items.append("当前节点本机 Worker 进程在线。")
|
||
if settings.node_region == "mainland" and settings.node_role == "control" and sync_agent_runtime.get("running", False):
|
||
info_items.append("当前节点本机 sync-agent 在线。")
|
||
if mainland_worker_nodes:
|
||
info_items.append(f"在线大陆 Worker {len(mainland_worker_nodes)} 台。")
|
||
if mainland_control_nodes:
|
||
info_items.append(f"在线大陆 controller {len(mainland_control_nodes)} 台。")
|
||
|
||
if blocking_issues:
|
||
status = "blocking"
|
||
summary_text = blocking_issues[0]
|
||
elif warning_issues:
|
||
status = "attention"
|
||
summary_text = warning_issues[0]
|
||
else:
|
||
status = "ready"
|
||
summary_text = "当前多机与跨地域骨架已进入可联调、可持续观察状态。"
|
||
|
||
return {
|
||
"status": status,
|
||
"ready": status == "ready",
|
||
"summary": summary_text,
|
||
"blocking_issues": blocking_issues,
|
||
"warnings": warning_issues,
|
||
"info": info_items,
|
||
"cluster": {
|
||
"online_control_nodes": online_control_nodes,
|
||
"online_worker_nodes": online_worker_nodes,
|
||
"mainland_control_nodes": len(mainland_control_nodes),
|
||
"mainland_worker_nodes": len(mainland_worker_nodes),
|
||
},
|
||
"sync": {
|
||
"enabled": bool(sync_summary.get("enabled", False)),
|
||
"source_region": sync_summary.get("source_region", ""),
|
||
"target_region": sync_summary.get("target_region", ""),
|
||
"projected_batches": projected_batches,
|
||
"pushing_batches": pushing_batches,
|
||
"failed_batches": failed_batches,
|
||
"synced_batches": synced_batches,
|
||
},
|
||
}
|
||
|
||
|
||
def _detect_participation_snapshot(*, row: dict) -> dict:
|
||
items_running = int(row.get("items_running", 0) or 0)
|
||
items_claimed = int(row.get("items_claimed", 0) or 0)
|
||
active_threads = int(row.get("active_threads", 0) or 0)
|
||
max_threads = int(row.get("max_threads", 0) or 0)
|
||
processed_recent = int(row.get("processed_recent", 0) or 0)
|
||
current_load = int(row.get("current_load", 0) or 0)
|
||
status = str(row.get("status") or "").strip().lower()
|
||
|
||
if items_running > 0:
|
||
return {
|
||
"participation_state": "running",
|
||
"participation_label": "执行中",
|
||
"participation_reason": f"当前正在执行 {items_running} 项检测任务。",
|
||
"is_current_participant": True,
|
||
"is_dispatch_active": True,
|
||
}
|
||
if items_claimed > 0:
|
||
return {
|
||
"participation_state": "claimed",
|
||
"participation_label": "已领待跑",
|
||
"participation_reason": f"已领取 {items_claimed} 项任务，等待线程继续执行。",
|
||
"is_current_participant": True,
|
||
"is_dispatch_active": True,
|
||
}
|
||
if active_threads > 0:
|
||
detail = f"当前活跃线程 {active_threads}"
|
||
if max_threads > 0:
|
||
detail = f"{detail}/{max_threads}"
|
||
return {
|
||
"participation_state": "runtime_active",
|
||
"participation_label": "执行中",
|
||
"participation_reason": detail,
|
||
"is_current_participant": True,
|
||
"is_dispatch_active": True,
|
||
}
|
||
if processed_recent > 0:
|
||
return {
|
||
"participation_state": "recent_throughput",
|
||
"participation_label": "近窗有吞吐",
|
||
"participation_reason": f"近 15 分钟内已处理 {processed_recent} 项任务。",
|
||
"is_current_participant": True,
|
||
"is_dispatch_active": False,
|
||
}
|
||
if current_load > 0 or status == "busy":
|
||
load_value = max(current_load, 1)
|
||
return {
|
||
"participation_state": "load_syncing",
|
||
"participation_label": "负载待确认",
|
||
"participation_reason": f"节点当前负载为 {load_value}，但还未观察到已领、执行中或近窗吞吐数据，先归入在线未参与观察。",
|
||
"is_current_participant": False,
|
||
"is_dispatch_active": False,
|
||
}
|
||
return {
|
||
"participation_state": "standby",
|
||
"participation_label": "在线待命",
|
||
"participation_reason": "当前未领任务、未执行任务，也没有近窗吞吐。",
|
||
"is_current_participant": False,
|
||
"is_dispatch_active": False,
|
||
}
|
||
|
||
|
||
def _build_detect_node_row(*, node_code: str, cluster_node: dict, job_node: dict, queue_node: dict) -> dict:
|
||
metadata = cluster_node.get("metadata") or {}
|
||
role = str(cluster_node.get("role") or job_node.get("role") or "worker")
|
||
region = str(cluster_node.get("region") or settings.node_region)
|
||
is_effective_worker = bool(cluster_node.get("is_effective_worker", False) or role == "worker")
|
||
items_total = max(
|
||
int(job_node.get("items_total", 0) or 0),
|
||
int(queue_node.get("items_total", 0) or 0),
|
||
int(metadata.get("job_items_total", 0) or 0),
|
||
)
|
||
items_claimed = max(
|
||
int(job_node.get("items_claimed", 0) or 0),
|
||
int(queue_node.get("items_claimed", 0) or 0),
|
||
int(metadata.get("job_items_claimed", 0) or 0),
|
||
)
|
||
items_running = max(
|
||
int(job_node.get("items_running", 0) or 0),
|
||
int(queue_node.get("items_running", 0) or 0),
|
||
int(metadata.get("job_items_running", 0) or 0),
|
||
)
|
||
items_completed = max(
|
||
int(job_node.get("items_completed", 0) or 0),
|
||
int(queue_node.get("items_completed", 0) or 0),
|
||
int(metadata.get("job_items_completed", 0) or 0),
|
||
)
|
||
items_failed = max(
|
||
int(job_node.get("items_failed", 0) or 0),
|
||
int(queue_node.get("items_failed", 0) or 0),
|
||
int(metadata.get("job_items_failed", 0) or 0),
|
||
)
|
||
items_blacklisted = max(
|
||
int(job_node.get("items_blacklisted", 0) or 0),
|
||
int(queue_node.get("items_blacklisted", 0) or 0),
|
||
)
|
||
active_threads = max(
|
||
int((cluster_node.get("metadata") or {}).get("active_threads", 0) or 0),
|
||
int(job_node.get("active_threads", 0) or 0),
|
||
int(queue_node.get("active_threads", 0) or 0),
|
||
)
|
||
max_threads = max(
|
||
int((cluster_node.get("metadata") or {}).get("max_threads", 0) or 0),
|
||
int(job_node.get("max_threads", 0) or 0),
|
||
int(queue_node.get("max_threads", 0) or 0),
|
||
)
|
||
current_load = max(items_running, active_threads, 0)
|
||
derived_pending = max(items_total - items_claimed - items_running - items_completed - items_failed - items_blacklisted, 0)
|
||
items_pending = max(
|
||
int(job_node.get("items_pending", 0) or 0),
|
||
int(queue_node.get("items_pending", 0) or 0),
|
||
derived_pending,
|
||
)
|
||
row = {
|
||
"node_code": node_code,
|
||
"role": role,
|
||
"region": region,
|
||
"status": str(cluster_node.get("status") or "unknown"),
|
||
"is_effective_worker": is_effective_worker,
|
||
"detect_participating": False,
|
||
"current_load": current_load,
|
||
"items_total": items_total,
|
||
"items_pending": items_pending,
|
||
"items_claimed": items_claimed,
|
||
"items_running": items_running,
|
||
"items_completed": items_completed,
|
||
"items_blacklisted": items_blacklisted,
|
||
"items_failed": items_failed,
|
||
"active_threads": active_threads,
|
||
"max_threads": max_threads,
|
||
"processed_recent": int(queue_node.get("processed_recent", 0) or 0),
|
||
"processed_per_minute": float(queue_node.get("processed_per_minute", 0) or 0),
|
||
"last_heartbeat_at": str(cluster_node.get("last_heartbeat_at") or ""),
|
||
"metrics_source": "active_job" if bool(job_node) else ("cluster_metadata" if metadata else "derived"),
|
||
}
|
||
row.update(_detect_participation_snapshot(row=row))
|
||
row["detect_participating"] = bool(row.get("is_current_participant", False))
|
||
return row
|
||
|
||
|
||
def _build_participating_detect_nodes(*, cluster_snapshot: dict, detect_snapshot: dict, worker_runtime: dict) -> list[dict]:
|
||
cluster_nodes = list(cluster_snapshot.get("nodes") or [])
|
||
active_job = detect_snapshot.get("active_job") or {}
|
||
queue_nodes = list((detect_snapshot.get("queue_health") or {}).get("nodes") or [])
|
||
|
||
cluster_map = {
|
||
str(item.get("node_code") or "").strip(): item
|
||
for item in cluster_nodes
|
||
if str(item.get("node_code") or "").strip()
|
||
}
|
||
queue_map = {
|
||
str(item.get("node_code") or "").strip(): item
|
||
for item in queue_nodes
|
||
if str(item.get("node_code") or "").strip()
|
||
}
|
||
|
||
merged: list[dict] = []
|
||
job_map = {
|
||
str(item.get("node_code") or "").strip(): item
|
||
for item in list(active_job.get("node_stats") or [])
|
||
if str(item.get("node_code") or "").strip() and str(item.get("node_code") or "").strip() != "unassigned"
|
||
}
|
||
candidate_codes = sorted(set(job_map) | set(queue_map) | set(cluster_map))
|
||
for node_code in candidate_codes:
|
||
cluster_node = cluster_map.get(node_code, {})
|
||
if cluster_node and not bool(cluster_node.get("is_effective_worker", False)):
|
||
continue
|
||
row = _build_detect_node_row(
|
||
node_code=node_code,
|
||
cluster_node=cluster_node,
|
||
job_node=job_map.get(node_code, {}),
|
||
queue_node=queue_map.get(node_code, {}),
|
||
)
|
||
if not bool(row.get("is_current_participant", False)):
|
||
continue
|
||
merged.append(row)
|
||
|
||
return sorted(
|
||
merged,
|
||
key=lambda item: (
|
||
-int(item.get("items_running", 0) or 0),
|
||
-int(item.get("items_claimed", 0) or 0),
|
||
-int(item.get("processed_recent", 0) or 0),
|
||
-int(item.get("current_load", 0) or 0),
|
||
str(item.get("node_code") or ""),
|
||
),
|
||
)
|
||
|
||
|
||
def _build_standby_detect_nodes(*, cluster_snapshot: dict, participating_nodes: list[dict]) -> list[dict]:
|
||
cluster_nodes = list(cluster_snapshot.get("nodes") or [])
|
||
participating_codes = {
|
||
str(item.get("node_code") or "").strip()
|
||
for item in list(participating_nodes or [])
|
||
if str(item.get("node_code") or "").strip()
|
||
}
|
||
standby_rows: list[dict] = []
|
||
for node in cluster_nodes:
|
||
node_code = str(node.get("node_code") or "").strip()
|
||
node_status = str(node.get("status") or "").strip()
|
||
node_current_load = int(node.get("current_load", 0) or 0)
|
||
metadata = node.get("metadata") or {}
|
||
if not node_code:
|
||
continue
|
||
if not bool(node.get("is_effective_worker", False)):
|
||
continue
|
||
if node_status not in {"online", "busy"}:
|
||
continue
|
||
if node_code in participating_codes:
|
||
continue
|
||
standby_state = "load_syncing" if node_current_load > 0 or node_status == "busy" else "standby"
|
||
standby_label = "负载待确认" if standby_state == "load_syncing" else "在线待命"
|
||
standby_reason = (
|
||
str(metadata.get("detail") or "").strip()
|
||
or str(metadata.get("phase_detail") or "").strip()
|
||
or (
|
||
f"当前阶段：{str(metadata.get('phase') or metadata.get('phase_label') or '').strip()}"
|
||
if str(metadata.get("phase") or metadata.get("phase_label") or "").strip()
|
||
else ""
|
||
)
|
||
or (
|
||
f"节点当前负载为 {node_current_load}，但还未观察到已领、执行中或近窗吞吐数据。"
|
||
if standby_state == "load_syncing"
|
||
else "节点在线，当前未领任务、未执行任务，也没有近窗吞吐。"
|
||
)
|
||
)
|
||
standby_rows.append(
|
||
{
|
||
"node_code": node_code,
|
||
"role": str(node.get("role") or "worker"),
|
||
"region": str(node.get("region") or settings.node_region),
|
||
"status": node_status,
|
||
"is_effective_worker": True,
|
||
"detect_participating": False,
|
||
"participation_state": standby_state,
|
||
"participation_label": standby_label,
|
||
"participation_reason": standby_reason,
|
||
"current_load": node_current_load,
|
||
"last_heartbeat_at": str(node.get("last_heartbeat_at") or ""),
|
||
"phase": str(metadata.get("phase") or metadata.get("phase_label") or "").strip(),
|
||
"detail": str(metadata.get("detail") or metadata.get("phase_detail") or "").strip(),
|
||
"worker_online": bool(metadata.get("worker_online", False)),
|
||
"standby_reason": standby_reason,
|
||
}
|
||
)
|
||
return sorted(
|
||
standby_rows,
|
||
key=lambda item: (
|
||
str(item.get("status") or ""),
|
||
str(item.get("role") or ""),
|
||
str(item.get("node_code") or ""),
|
||
),
|
||
)
|
||
|
||
|
||
def _build_detect_participation_summary(
|
||
*,
|
||
participating_nodes: list[dict],
|
||
standby_nodes: list[dict],
|
||
cluster_snapshot: dict,
|
||
) -> dict:
|
||
cluster_nodes = list(cluster_snapshot.get("nodes") or [])
|
||
effective_online_nodes = [
|
||
node
|
||
for node in cluster_nodes
|
||
if bool(node.get("is_effective_worker", False)) and str(node.get("status") or "").strip() in {"online", "busy"}
|
||
]
|
||
dispatch_active_nodes = [
|
||
row for row in participating_nodes
|
||
if bool(row.get("is_dispatch_active", False))
|
||
]
|
||
recent_only_nodes = [
|
||
row for row in participating_nodes
|
||
if str(row.get("participation_state") or "").strip() == "recent_throughput"
|
||
]
|
||
load_syncing_nodes = [
|
||
row for row in standby_nodes
|
||
if str(row.get("participation_state") or "").strip() == "load_syncing"
|
||
]
|
||
pure_standby_nodes = [
|
||
row for row in standby_nodes
|
||
if str(row.get("participation_state") or "").strip() == "standby"
|
||
]
|
||
dedicated_worker_nodes = [
|
||
node for node in effective_online_nodes
|
||
if str(node.get("role") or "").strip() == "worker"
|
||
]
|
||
controller_worker_nodes = [
|
||
node for node in effective_online_nodes
|
||
if str(node.get("role") or "").strip() == "control"
|
||
]
|
||
|
||
summary_parts = [
|
||
f"有效执行节点 {len(effective_online_nodes)} 台",
|
||
f"正在执行/领任务 {len(dispatch_active_nodes)} 台",
|
||
f"近窗刚有吞吐 {len(recent_only_nodes)} 台",
|
||
f"在线但未参与 {len(standby_nodes)} 台",
|
||
]
|
||
if load_syncing_nodes:
|
||
summary_parts.append(f"其中负载待确认 {len(load_syncing_nodes)} 台")
|
||
|
||
return {
|
||
"effective_online_nodes": len(effective_online_nodes),
|
||
"participating_nodes": len(participating_nodes),
|
||
"dispatch_active_nodes": len(dispatch_active_nodes),
|
||
"recent_only_nodes": len(recent_only_nodes),
|
||
"non_participating_nodes": len(standby_nodes),
|
||
"standby_nodes": len(pure_standby_nodes),
|
||
"load_syncing_nodes": len(load_syncing_nodes),
|
||
"dedicated_worker_nodes": len(dedicated_worker_nodes),
|
||
"controller_worker_nodes": len(controller_worker_nodes),
|
||
"dispatch_active_node_codes": [str(item.get("node_code") or "") for item in dispatch_active_nodes],
|
||
"recent_only_node_codes": [str(item.get("node_code") or "") for item in recent_only_nodes],
|
||
"non_participating_node_codes": [str(item.get("node_code") or "") for item in standby_nodes],
|
||
"standby_node_codes": [str(item.get("node_code") or "") for item in pure_standby_nodes],
|
||
"load_syncing_node_codes": [str(item.get("node_code") or "") for item in load_syncing_nodes],
|
||
"summary": "；".join(summary_parts),
|
||
}
|
||
|
||
|
||
def get_runtime_status() -> dict:
|
||
runtime_settings = get_runtime_settings()
|
||
worker_runtime = detect_worker_runtime()
|
||
sync_agent_runtime = detect_sync_agent_runtime()
|
||
worker_expected_on_this_node = not (settings.node_region == "overseas" and settings.node_role == "control")
|
||
api_pid = os.getpid()
|
||
detect_snapshot = get_detect_status()
|
||
latest_run = (detect_snapshot.get("runs") or [None])[0] or {}
|
||
|
||
cluster_snapshot = get_cluster_snapshot()
|
||
queue_health = get_detect_queue_health(window_minutes=15)
|
||
effective_online_worker_nodes = int((cluster_snapshot.get("summary") or {}).get("online_worker_nodes", 0) or 0)
|
||
if effective_online_worker_nodes <= 0 and worker_runtime.get("running", False):
|
||
effective_online_worker_nodes = max(1, worker_runtime.get("process_count", 1) or 1)
|
||
backlog_snapshot = _load_detect_backlog_snapshot()
|
||
remote_backlog_snapshot = _load_latest_remote_runtime_projection_backlog()
|
||
runtime_snapshot_backlog = dict(_load_latest_runtime_active_job_snapshot(15).get("backlog") or {})
|
||
backlog_snapshot = _merge_backlog_snapshots(backlog_snapshot, remote_backlog_snapshot)
|
||
backlog_snapshot = _merge_backlog_snapshots(backlog_snapshot, runtime_snapshot_backlog)
|
||
capacity_plan = get_detect_capacity_plan(
|
||
queue_health=_align_queue_health_with_backlog(queue_health, backlog_snapshot),
|
||
online_worker_nodes=effective_online_worker_nodes,
|
||
target_finish_hours=6,
|
||
)
|
||
detect_payload = {
|
||
"phase_label": latest_run.get("phase_label", ""),
|
||
"phase_detail": latest_run.get("phase_detail", ""),
|
||
"recent_event": detect_snapshot.get("recent_event", ""),
|
||
"recent_warning": detect_snapshot.get("recent_warning", ""),
|
||
"progress_percent": detect_snapshot.get("progress_percent", 0),
|
||
"progress": detect_snapshot.get("progress", {}),
|
||
"active_thread_count": detect_snapshot.get("active_thread_count", 0),
|
||
"max_thread_count": detect_snapshot.get("max_thread_count", 0),
|
||
"available_proxy_count": detect_snapshot.get("available_proxy_count", 0),
|
||
"proxy_pool_count": detect_snapshot.get("proxy_pool_count", 0),
|
||
"proxy_runtime_label": detect_snapshot.get("proxy_runtime_label", ""),
|
||
"proxy_runtime_detail": detect_snapshot.get("proxy_runtime_detail", ""),
|
||
"proxy_runtime_reason": detect_snapshot.get("proxy_runtime_reason", ""),
|
||
"proxy_supplier_empty": detect_snapshot.get("proxy_supplier_empty", False),
|
||
"proxy_last_refresh_status": detect_snapshot.get("proxy_last_refresh_status", ""),
|
||
"proxy_last_refresh_time": detect_snapshot.get("proxy_last_refresh_time", ""),
|
||
"proxy_last_refresh_source_count": detect_snapshot.get("proxy_last_refresh_source_count", 0),
|
||
"proxy_last_refresh_total_items": detect_snapshot.get("proxy_last_refresh_total_items", 0),
|
||
"proxy_last_validated_count": detect_snapshot.get("proxy_last_validated_count", 0),
|
||
"proxy_last_available_count": detect_snapshot.get("proxy_last_available_count", 0),
|
||
"proxy_source_stats": detect_snapshot.get("proxy_source_stats", []),
|
||
"dependency_alerts": detect_snapshot.get("dependency_alerts", []),
|
||
"active_job": detect_snapshot.get("active_job"),
|
||
"runs_count": len(detect_snapshot.get("runs") or []),
|
||
"worker_online": worker_runtime.get("running", False),
|
||
"worker_mode": worker_runtime.get("mode", runtime_settings.get("worker_mode", "windows-local")),
|
||
"queue_health": queue_health,
|
||
"backlog": backlog_snapshot,
|
||
"capacity_plan": capacity_plan,
|
||
"log_sync": {
|
||
"enabled": bool(runtime_settings.get("worker_log_sync_enabled", False)),
|
||
"mode": str(runtime_settings.get("worker_log_sync_mode", "key") or "key"),
|
||
"line_count": int(detect_snapshot.get("remote_log_line_count", 0) or 0),
|
||
"source_node_count": int(detect_snapshot.get("remote_log_node_count", 0) or 0),
|
||
"source_nodes": list(detect_snapshot.get("remote_log_nodes") or []),
|
||
"source_node_summaries": list(detect_snapshot.get("remote_log_node_summaries") or []),
|
||
"last_at": str(detect_snapshot.get("remote_log_last_at") or ""),
|
||
"last_line": str(detect_snapshot.get("remote_log_last_line") or ""),
|
||
"preview_lines": list(detect_snapshot.get("remote_log_lines") or [])[-20:],
|
||
},
|
||
}
|
||
if not worker_expected_on_this_node:
|
||
detect_payload.update(
|
||
{
|
||
"phase_label": "当前节点不承载",
|
||
"phase_detail": "当前节点为海外控制面，仅承载 API 控制与同步接收，不执行本机检测任务。",
|
||
"recent_event": "",
|
||
"recent_warning": "",
|
||
"progress_percent": 0,
|
||
"active_thread_count": 0,
|
||
"max_thread_count": 0,
|
||
"available_proxy_count": 0,
|
||
"proxy_pool_count": 0,
|
||
"proxy_runtime_label": "不适用",
|
||
"proxy_runtime_detail": "当前节点不承载本机 Worker，代理、线程与检测阶段信息在此节点上不适用。",
|
||
"proxy_runtime_reason": "not_applicable",
|
||
"proxy_supplier_empty": False,
|
||
"proxy_last_refresh_status": "",
|
||
"proxy_last_refresh_time": "",
|
||
"proxy_last_refresh_source_count": 0,
|
||
"proxy_last_refresh_total_items": 0,
|
||
"proxy_last_validated_count": 0,
|
||
"proxy_last_available_count": 0,
|
||
"proxy_source_stats": [],
|
||
"dependency_alerts": [],
|
||
"active_job": None,
|
||
"runs_count": 0,
|
||
"worker_online": False,
|
||
}
|
||
)
|
||
detect_payload["participating_nodes"] = _build_participating_detect_nodes(
|
||
cluster_snapshot=cluster_snapshot,
|
||
detect_snapshot=detect_payload,
|
||
worker_runtime=worker_runtime,
|
||
)
|
||
detect_payload["standby_nodes"] = _build_standby_detect_nodes(
|
||
cluster_snapshot=cluster_snapshot,
|
||
participating_nodes=detect_payload["participating_nodes"],
|
||
)
|
||
detect_payload["non_participating_nodes"] = list(detect_payload["standby_nodes"] or [])
|
||
detect_payload["participation_summary"] = _build_detect_participation_summary(
|
||
participating_nodes=detect_payload["participating_nodes"],
|
||
standby_nodes=detect_payload["non_participating_nodes"],
|
||
cluster_snapshot=cluster_snapshot,
|
||
)
|
||
append_runtime_projection_if_changed(detect=detect_payload, cluster=cluster_snapshot)
|
||
sync_summary = get_sync_summary(record_limit=5)
|
||
readiness = _build_multi_region_readiness(
|
||
cluster_snapshot=cluster_snapshot,
|
||
sync_summary=sync_summary,
|
||
worker_runtime=worker_runtime,
|
||
sync_agent_runtime=sync_agent_runtime,
|
||
)
|
||
build_info = get_runtime_build_info()
|
||
|
||
api_payload = {
|
||
"service": "domain-api",
|
||
"version": "0.1.0",
|
||
"api_prefix": settings.api_prefix,
|
||
"pid": api_pid,
|
||
"host": settings.api_host,
|
||
"port": settings.api_port,
|
||
"mode": runtime_settings.get("worker_mode", "windows-local"),
|
||
"service_name": runtime_settings.get("api_service_name", settings.api_service_name),
|
||
"health_url": f"http://127.0.0.1:{settings.api_port}/health",
|
||
"stdout_log": _runtime_log_path("domain-api.stdout.log"),
|
||
"stderr_log": _runtime_log_path("domain-api.stderr.log"),
|
||
"build": build_info,
|
||
}
|
||
node_payload = {
|
||
"code": settings.node_code,
|
||
"region": settings.node_region,
|
||
"role": settings.node_role,
|
||
}
|
||
worker_payload = {
|
||
"mode": worker_runtime.get("mode", runtime_settings.get("worker_mode", "windows-local")),
|
||
"service_name": runtime_settings.get("worker_service_name", settings.worker_service_name),
|
||
"running": worker_runtime.get("running", False),
|
||
"expected_on_this_node": worker_expected_on_this_node,
|
||
"process_count": worker_runtime.get("process_count", 0),
|
||
"latest_start_time": worker_runtime.get("latest_start_time", ""),
|
||
"message": worker_runtime.get("message", ""),
|
||
"log_path": str(Path(settings.domain_root) / "detect_worker.log"),
|
||
}
|
||
sync_agent_payload = {
|
||
"mode": sync_agent_runtime.get("mode", runtime_settings.get("worker_mode", "windows-local")),
|
||
"service_name": runtime_settings.get("sync_agent_service_name", settings.sync_agent_service_name),
|
||
"running": sync_agent_runtime.get("running", False),
|
||
"process_count": sync_agent_runtime.get("process_count", 0),
|
||
"latest_start_time": sync_agent_runtime.get("latest_start_time", ""),
|
||
"message": sync_agent_runtime.get("message", ""),
|
||
"expected_on_this_node": settings.node_region == "mainland" and settings.node_role == "control",
|
||
}
|
||
compatibility_payload = {
|
||
# Backward-compatible flat fields for older pages / stale built assets.
|
||
"api_online": bool(api_payload.get("pid")),
|
||
"api_service_name": api_payload.get("service_name", ""),
|
||
"worker_online": bool(worker_payload.get("running", False)),
|
||
"worker_mode": worker_payload.get("mode", ""),
|
||
"worker_service_name": worker_payload.get("service_name", ""),
|
||
"worker_process_count": worker_payload.get("process_count", 0),
|
||
"worker_latest_start_time": worker_payload.get("latest_start_time", ""),
|
||
"worker_runtime_message": worker_payload.get("message", ""),
|
||
"thread_count": detect_snapshot.get("thread_count", 0),
|
||
"thread_count_default": detect_snapshot.get("thread_count_default", 0),
|
||
"thread_count_source": detect_snapshot.get("thread_count_source", ""),
|
||
"thread_count_override": detect_snapshot.get("thread_count_override"),
|
||
"active_thread_count": detect_payload.get("active_thread_count", 0),
|
||
"max_thread_count": detect_payload.get("max_thread_count", 0),
|
||
"progress": detect_payload.get("progress", {}),
|
||
"backlog": detect_payload.get("backlog", {}),
|
||
"progress_percent": detect_payload.get("progress_percent", 0),
|
||
"available_proxy_count": detect_payload.get("available_proxy_count", 0),
|
||
"proxy_pool_count": detect_payload.get("proxy_pool_count", 0),
|
||
"proxy_runtime_label": detect_payload.get("proxy_runtime_label", ""),
|
||
"proxy_runtime_detail": detect_payload.get("proxy_runtime_detail", ""),
|
||
"proxy_runtime_reason": detect_payload.get("proxy_runtime_reason", ""),
|
||
"proxy_last_refresh_time": detect_payload.get("proxy_last_refresh_time", ""),
|
||
"recent_event": detect_payload.get("recent_event", ""),
|
||
"recent_warning": detect_payload.get("recent_warning", ""),
|
||
"runtime_state": worker_runtime.get("runtime_state") or {},
|
||
"cluster_summary": cluster_snapshot.get("summary") or {},
|
||
}
|
||
|
||
return {
|
||
**compatibility_payload,
|
||
"api": api_payload,
|
||
"node": node_payload,
|
||
"worker": worker_payload,
|
||
"sync_agent": sync_agent_payload,
|
||
"detect": detect_payload,
|
||
"cluster": cluster_snapshot,
|
||
"sync": sync_summary,
|
||
"readiness": readiness,
|
||
}
|
||
|
||
|
||
def get_runtime_preflight() -> dict:
|
||
runtime_settings = get_runtime_settings()
|
||
checks: list[dict[str, object]] = []
|
||
detect_options = read_json("detect_options.json", default={})
|
||
|
||
domain_root = Path(settings.domain_root)
|
||
checks.append(
|
||
{
|
||
"key": "domain_root",
|
||
"label": "domainCheck 目录",
|
||
"ok": domain_root.exists(),
|
||
"message": str(domain_root),
|
||
}
|
||
)
|
||
|
||
try:
|
||
with get_db() as conn:
|
||
with conn.cursor() as cur:
|
||
cur.execute("select 1")
|
||
cur.fetchone()
|
||
checks.append({"key": "database", "label": "PostgreSQL", "ok": True, "message": f"{settings.db_host}:{settings.db_port}/{settings.db_database}"})
|
||
except Exception as exc:
|
||
checks.append({"key": "database", "label": "PostgreSQL", "ok": False, "message": str(exc)})
|
||
|
||
try:
|
||
redis_client = get_redis()
|
||
redis_client.ping()
|
||
checks.append({"key": "redis", "label": "Redis", "ok": True, "message": f"{settings.redis_host}:{settings.redis_port}/{settings.redis_db}"})
|
||
except Exception as exc:
|
||
checks.append({"key": "redis", "label": "Redis", "ok": False, "message": str(exc)})
|
||
else:
|
||
bloom_ok, bloom_message = _bloom_filter_status()
|
||
checks.append(
|
||
{
|
||
"key": "redis_bloom",
|
||
"label": "RedisBloom",
|
||
"ok": True,
|
||
"message": bloom_message,
|
||
"level": "info" if bloom_ok else "warn",
|
||
"degraded": not bloom_ok,
|
||
}
|
||
)
|
||
|
||
worker_mode = runtime_settings.get("worker_mode", "windows-local")
|
||
checks.append({"key": "worker_mode", "label": "运行模式", "ok": True, "message": worker_mode})
|
||
|
||
jucha_enabled = bool(detect_options.get("detect_jucha"))
|
||
jucha_cookie_ok, jucha_cookie_path = _domain_cookie_status("jucha_cookies.pkl")
|
||
checks.append(
|
||
{
|
||
"key": "detect_jucha",
|
||
"label": "聚查检测",
|
||
"ok": True if not jucha_enabled else jucha_cookie_ok,
|
||
"message": "已启用" if jucha_enabled else "未启用",
|
||
}
|
||
)
|
||
checks.append(
|
||
{
|
||
"key": "jucha_cookie",
|
||
"label": "聚查 Cookie",
|
||
"ok": True if not jucha_enabled else jucha_cookie_ok,
|
||
"message": jucha_cookie_path if jucha_enabled else "未启用聚查检测，无需本地 Cookie",
|
||
"level": "info" if (not jucha_enabled or jucha_cookie_ok) else "warn",
|
||
}
|
||
)
|
||
|
||
juziseo_enabled = bool(detect_options.get("detect_juziseo"))
|
||
juziseo_cookie_ok, juziseo_cookie_path = _domain_cookie_status("juziseo_cookies.pkl")
|
||
checks.append(
|
||
{
|
||
"key": "detect_juziseo",
|
||
"label": "桔子SEO检测",
|
||
"ok": True if not juziseo_enabled else juziseo_cookie_ok,
|
||
"message": "已启用" if juziseo_enabled else "未启用",
|
||
}
|
||
)
|
||
checks.append(
|
||
{
|
||
"key": "juziseo_cookie",
|
||
"label": "桔子SEO Cookie",
|
||
"ok": True if not juziseo_enabled else juziseo_cookie_ok,
|
||
"message": juziseo_cookie_path if juziseo_enabled else "未启用桔子SEO检测，无需本地 Cookie",
|
||
"level": "info" if (not juziseo_enabled or juziseo_cookie_ok) else "warn",
|
||
}
|
||
)
|
||
|
||
if worker_mode == "linux-systemd":
|
||
checks.append(
|
||
{
|
||
"key": "worker_service_name",
|
||
"label": "Worker service 名",
|
||
"ok": bool(runtime_settings.get("worker_service_name")),
|
||
"message": runtime_settings.get("worker_service_name", ""),
|
||
}
|
||
)
|
||
checks.append(
|
||
{
|
||
"key": "api_service_name",
|
||
"label": "API service 名",
|
||
"ok": bool(runtime_settings.get("api_service_name")),
|
||
"message": runtime_settings.get("api_service_name", ""),
|
||
}
|
||
)
|
||
checks.append(
|
||
{
|
||
"key": "sync_agent_service_name",
|
||
"label": "Sync agent service 名",
|
||
"ok": bool(runtime_settings.get("sync_agent_service_name")),
|
||
"message": runtime_settings.get("sync_agent_service_name", ""),
|
||
}
|
||
)
|
||
else:
|
||
checks.append(
|
||
{
|
||
"key": "windows_scripts",
|
||
"label": "Windows 启停脚本",
|
||
"ok": (Path(settings.domain_root).parent / "start_domain_api.ps1").exists() and (Path(settings.domain_root).parent / "stop_domain_api.ps1").exists(),
|
||
"message": "start_domain_api.ps1 / stop_domain_api.ps1",
|
||
}
|
||
)
|
||
|
||
overall_ok = all(bool(item["ok"]) for item in checks)
|
||
return {
|
||
"ok": overall_ok,
|
||
"checks": checks,
|
||
}
|