from __future__ import annotations import os from pathlib import Path from app.core.config import settings from app.core.db import get_db from app.core.files import read_json from app.core.redis_client import get_redis from app.services.build_info_service import get_runtime_build_info from app.services.cluster_runtime_service import get_cluster_snapshot from app.services.detect_service import get_detect_status from app.services.detect_job_service import ( _load_latest_runtime_active_job_snapshot, get_detect_capacity_plan, get_detect_queue_health, ) from app.services.sync_record_service import append_runtime_projection_if_changed, get_sync_summary from app.services.runtime_settings_service import get_runtime_settings from app.services.worker_control_service import detect_sync_agent_runtime, detect_worker_runtime def _align_queue_health_with_backlog(queue_health: dict | None, backlog_snapshot: dict | None) -> dict: normalized = dict(queue_health or {}) queue = dict(normalized.get("queue") or {}) backlog = dict(backlog_snapshot or {}) pending_total = max(int(queue.get("pending", 0) or 0), int(backlog.get("pending_total", 0) or 0)) claimed_total = max(int(queue.get("claimed", 0) or 0), int(backlog.get("claimed_total", 0) or 0)) running_total = max(int(queue.get("running", 0) or 0), int(backlog.get("running_total", 0) or 0)) completed_total = max(int(queue.get("completed", 0) or 0), int(backlog.get("completed_total", 0) or 0)) blacklisted_total = max(int(queue.get("blacklisted", 0) or 0), int(backlog.get("blacklisted_total", 0) or 0)) failed_total = max(int(queue.get("failed", 0) or 0), int(backlog.get("failed_total", 0) or 0)) terminal_total = max( int(queue.get("terminal", 0) or 0), completed_total + blacklisted_total + failed_total, ) normalized["has_active_job"] = bool( normalized.get("has_active_job") or pending_total > 0 or claimed_total > 0 or running_total > 0 or terminal_total > 0 ) normalized["queue"] = { **queue, "items_total": pending_total + claimed_total + running_total + terminal_total, "pending": pending_total, "claimed": claimed_total, "running": running_total, "completed": completed_total, "blacklisted": blacklisted_total, "failed": failed_total, "terminal": terminal_total, } return normalized def _decode_projection_payload(value: object) -> dict: if isinstance(value, dict): return dict(value) if value in (None, ""): return {} try: import json return dict(json.loads(value)) except Exception: return {} def _load_detect_backlog_snapshot() -> dict: with get_db() as conn: with conn.cursor() as cur: cur.execute( """ SELECT COUNT(*) FILTER (WHERE item.status = 'pending') AS pending_total, COUNT(*) FILTER (WHERE item.status = 'claimed') AS claimed_total, COUNT(*) FILTER (WHERE item.status = 'running') AS running_total, COUNT(*) FILTER (WHERE item.status = 'completed') AS completed_total, COUNT(*) FILTER (WHERE item.status = 'blacklisted') AS blacklisted_total, COUNT(*) FILTER (WHERE item.status = 'failed') AS failed_total, COUNT(*) FILTER (WHERE item.status = 'pending' AND item.step_code = 'detect_register') AS register_pending, COUNT(*) FILTER (WHERE item.status = 'pending' AND item.step_code <> 'detect_register') AS downstream_pending FROM detect_job_items item JOIN detect_jobs job ON job.id = item.job_id WHERE job.status IN ('pending', 'running') """ ) row = cur.fetchone() or (0, 0, 0, 0, 0, 0, 0, 0) return { "pending_total": int(row[0] or 0), "claimed_total": int(row[1] or 0), "running_total": int(row[2] or 0), "completed_total": int(row[3] or 0), "blacklisted_total": int(row[4] or 0), "failed_total": int(row[5] or 0), "register_pending": int(row[6] or 0), "downstream_pending": int(row[7] or 0), } def _load_latest_remote_runtime_projection_backlog() -> dict: if not (settings.node_region == "overseas" and settings.node_role == "control"): return {} with get_db() as conn: with conn.cursor() as cur: cur.execute( """ SELECT payload_json FROM detect_sync_records WHERE sync_type = 'runtime_projection' AND source_region = 'mainland' AND target_region = 'overseas' AND status IN ('projected', 'pushing', 'synced') ORDER BY updated_at DESC, id DESC LIMIT 1 """ ) row = cur.fetchone() if not row: return {} payload = _decode_projection_payload(row[0]) projection = payload.get("projection") if isinstance(payload, dict) else {} backlog = projection.get("backlog") if isinstance(projection, dict) else {} if not isinstance(backlog, dict): return {} return { "pending_total": int(backlog.get("pending_total", 0) or 0), "claimed_total": int(backlog.get("claimed_total", 0) or 0), "running_total": int(backlog.get("running_total", 0) or 0), "completed_total": int(backlog.get("completed_total", 0) or 0), "blacklisted_total": int(backlog.get("blacklisted_total", 0) or 0), "failed_total": int(backlog.get("failed_total", 0) or 0), "register_pending": int(backlog.get("register_pending", 0) or 0), "downstream_pending": int(backlog.get("downstream_pending", 0) or 0), } def _merge_backlog_snapshots(primary: dict, secondary: dict) -> dict: merged = dict(primary or {}) for key in ( "pending_total", "claimed_total", "running_total", "completed_total", "blacklisted_total", "failed_total", "register_pending", "downstream_pending", ): merged[key] = max(int(merged.get(key, 0) or 0), int((secondary or {}).get(key, 0) or 0)) return merged def _runtime_log_path(filename: str) -> str: path = Path(__file__).resolve().parents[2] / "runtime" / "logs" / filename return str(path) def _domain_cookie_status(filename: str) -> tuple[bool, str]: path = Path(settings.domain_root) / filename return path.exists(), str(path) def _bloom_filter_status() -> tuple[bool, str]: try: redis_client = get_redis() modules = redis_client.execute_command("MODULE", "LIST") for module in modules or []: module_parts = module[::2] module_values = module[1::2] module_info = dict(zip(module_parts, module_values)) module_name = str(module_info.get("name", "")).lower() if module_name in {"bf", "redisbloom"}: return True, "RedisBloom 已安装" return False, "Redis 未安装 RedisBloom，当前将使用普通缓存" except Exception as exc: return False, f"RedisBloom 检查失败: {exc}" def _build_multi_region_readiness( *, cluster_snapshot: dict, sync_summary: dict, worker_runtime: dict, sync_agent_runtime: dict, ) -> dict: nodes = list(cluster_snapshot.get("nodes") or []) summary = cluster_snapshot.get("summary") or {} batch_summary = (sync_summary.get("detect_result_batches") or {}) batch_states = batch_summary.get("state_counts") or {} online_control_nodes = int(summary.get("online_control_nodes", 0) or 0) online_worker_nodes = int(summary.get("online_worker_nodes", 0) or 0) mainland_control_nodes = [ node for node in nodes if str(node.get("region") or "") == "mainland" and str(node.get("role") or "") == "control" and str(node.get("status") or "") in {"online", "busy"} ] mainland_worker_nodes = [ node for node in nodes if str(node.get("region") or "") == "mainland" and bool(node.get("is_effective_worker", False)) and str(node.get("status") or "") in {"online", "busy"} ] blocking_issues: list[str] = [] warning_issues: list[str] = [] info_items: list[str] = [] online_pairs = { ( str(node.get("region") or "").strip(), str(node.get("role") or "").strip(), ) for node in nodes if str(node.get("status") or "") in {"online", "busy"} } critical_stale_nodes: list[str] = [] redundant_stale_nodes: list[str] = [] critical_offline_nodes: list[str] = [] redundant_offline_nodes: list[str] = [] if online_control_nodes <= 0: blocking_issues.append("当前没有在线控制面节点，无法视为正式可用集群。") if settings.node_region == "mainland" and settings.node_role == "control": if not sync_agent_runtime.get("running", False): blocking_issues.append("当前节点应承载 sync-agent，但服务未运行。") if str(settings.sync_target_api_base_url or "").strip() == "": blocking_issues.append("当前节点未配置 SYNC_TARGET_API_BASE_URL，无法向海外控制面推送。") if not bool(settings.sync_push_enabled): blocking_issues.append("当前节点未启用 SYNC_PUSH_ENABLED，结果同步不会自动执行。") if settings.node_region == "overseas" and online_control_nodes > 0 and not mainland_control_nodes: warning_issues.append("当前尚未观察到在线的大陆 controller 节点，后续自动结果同步仍未进入正式双地域态。") if online_worker_nodes <= 0: warning_issues.append("当前没有在线 Worker 节点，检测任务无法在多机状态下继续推进。") for node in nodes: node_status = str(node.get("status") or "").strip() if node_status not in {"stale", "offline"}: continue node_code = str(node.get("node_code") or "").strip() node_region = str(node.get("region") or "").strip() node_role = str(node.get("role") or "").strip() metadata = node.get("metadata") or {} replaced_by_online_peer = (node_region, node_role) in online_pairs redundant = replaced_by_online_peer and ( node_role == "control" or not bool(node.get("is_effective_worker", False)) or str(metadata.get("service") or "") == "runtime-ingest" ) if node_status == "stale": (redundant_stale_nodes if redundant else critical_stale_nodes).append(node_code) else: (redundant_offline_nodes if redundant else critical_offline_nodes).append(node_code) if critical_stale_nodes: warning_issues.append(f"存在失活节点: {'、'.join(critical_stale_nodes)}") if critical_offline_nodes: warning_issues.append(f"存在离线节点: {'、'.join(critical_offline_nodes)}") if redundant_stale_nodes: info_items.append(f"存在历史失活节点（已被在线同类节点覆盖）: {'、'.join(redundant_stale_nodes)}") if redundant_offline_nodes: info_items.append(f"存在历史离线节点（已被在线同类节点覆盖）: {'、'.join(redundant_offline_nodes)}") failed_batches = int(batch_states.get("failed", 0) or 0) projected_batches = int(batch_states.get("projected", 0) or 0) pushing_batches = int(batch_states.get("pushing", 0) or 0) synced_batches = int(batch_states.get("synced", 0) or 0) if failed_batches > 0: warning_issues.append(f"存在 {failed_batches} 个结果批次同步失败，需要检查 sync-agent 或目标接收面。") if projected_batches > 0: warning_issues.append(f"存在 {projected_batches} 个结果批次仍待推送。") if pushing_batches > 0: info_items.append(f"当前有 {pushing_batches} 个结果批次正在推送。") if synced_batches > 0: info_items.append(f"最近已接收 {synced_batches} 个结果批次。") if worker_runtime.get("running", False): info_items.append("当前节点本机 Worker 进程在线。") if settings.node_region == "mainland" and settings.node_role == "control" and sync_agent_runtime.get("running", False): info_items.append("当前节点本机 sync-agent 在线。") if mainland_worker_nodes: info_items.append(f"在线大陆 Worker {len(mainland_worker_nodes)} 台。") if mainland_control_nodes: info_items.append(f"在线大陆 controller {len(mainland_control_nodes)} 台。") if blocking_issues: status = "blocking" summary_text = blocking_issues[0] elif warning_issues: status = "attention" summary_text = warning_issues[0] else: status = "ready" summary_text = "当前多机与跨地域骨架已进入可联调、可持续观察状态。" return { "status": status, "ready": status == "ready", "summary": summary_text, "blocking_issues": blocking_issues, "warnings": warning_issues, "info": info_items, "cluster": { "online_control_nodes": online_control_nodes, "online_worker_nodes": online_worker_nodes, "mainland_control_nodes": len(mainland_control_nodes), "mainland_worker_nodes": len(mainland_worker_nodes), }, "sync": { "enabled": bool(sync_summary.get("enabled", False)), "source_region": sync_summary.get("source_region", ""), "target_region": sync_summary.get("target_region", ""), "projected_batches": projected_batches, "pushing_batches": pushing_batches, "failed_batches": failed_batches, "synced_batches": synced_batches, }, } def _detect_participation_snapshot(*, row: dict) -> dict: items_running = int(row.get("items_running", 0) or 0) items_claimed = int(row.get("items_claimed", 0) or 0) active_threads = int(row.get("active_threads", 0) or 0) max_threads = int(row.get("max_threads", 0) or 0) processed_recent = int(row.get("processed_recent", 0) or 0) current_load = int(row.get("current_load", 0) or 0) status = str(row.get("status") or "").strip().lower() if items_running > 0: return { "participation_state": "running", "participation_label": "执行中", "participation_reason": f"当前正在执行 {items_running} 项检测任务。", "is_current_participant": True, "is_dispatch_active": True, } if items_claimed > 0: return { "participation_state": "claimed", "participation_label": "已领待跑", "participation_reason": f"已领取 {items_claimed} 项任务，等待线程继续执行。", "is_current_participant": True, "is_dispatch_active": True, } if active_threads > 0: detail = f"当前活跃线程 {active_threads}" if max_threads > 0: detail = f"{detail}/{max_threads}" return { "participation_state": "runtime_active", "participation_label": "执行中", "participation_reason": detail, "is_current_participant": True, "is_dispatch_active": True, } if processed_recent > 0: return { "participation_state": "recent_throughput", "participation_label": "近窗有吞吐", "participation_reason": f"近 15 分钟内已处理 {processed_recent} 项任务。", "is_current_participant": True, "is_dispatch_active": False, } if current_load > 0 or status == "busy": load_value = max(current_load, 1) return { "participation_state": "load_syncing", "participation_label": "负载待确认", "participation_reason": f"节点当前负载为 {load_value}，但还未观察到已领、执行中或近窗吞吐数据，先归入在线未参与观察。", "is_current_participant": False, "is_dispatch_active": False, } return { "participation_state": "standby", "participation_label": "在线待命", "participation_reason": "当前未领任务、未执行任务，也没有近窗吞吐。", "is_current_participant": False, "is_dispatch_active": False, } def _build_detect_node_row(*, node_code: str, cluster_node: dict, job_node: dict, queue_node: dict) -> dict: metadata = cluster_node.get("metadata") or {} role = str(cluster_node.get("role") or job_node.get("role") or "worker") region = str(cluster_node.get("region") or settings.node_region) is_effective_worker = bool(cluster_node.get("is_effective_worker", False) or role == "worker") items_total = max( int(job_node.get("items_total", 0) or 0), int(queue_node.get("items_total", 0) or 0), int(metadata.get("job_items_total", 0) or 0), ) items_claimed = max( int(job_node.get("items_claimed", 0) or 0), int(queue_node.get("items_claimed", 0) or 0), int(metadata.get("job_items_claimed", 0) or 0), ) items_running = max( int(job_node.get("items_running", 0) or 0), int(queue_node.get("items_running", 0) or 0), int(metadata.get("job_items_running", 0) or 0), ) items_completed = max( int(job_node.get("items_completed", 0) or 0), int(queue_node.get("items_completed", 0) or 0), int(metadata.get("job_items_completed", 0) or 0), ) items_failed = max( int(job_node.get("items_failed", 0) or 0), int(queue_node.get("items_failed", 0) or 0), int(metadata.get("job_items_failed", 0) or 0), ) items_blacklisted = max( int(job_node.get("items_blacklisted", 0) or 0), int(queue_node.get("items_blacklisted", 0) or 0), ) active_threads = max( int((cluster_node.get("metadata") or {}).get("active_threads", 0) or 0), int(job_node.get("active_threads", 0) or 0), int(queue_node.get("active_threads", 0) or 0), ) max_threads = max( int((cluster_node.get("metadata") or {}).get("max_threads", 0) or 0), int(job_node.get("max_threads", 0) or 0), int(queue_node.get("max_threads", 0) or 0), ) current_load = max(items_running, active_threads, 0) derived_pending = max(items_total - items_claimed - items_running - items_completed - items_failed - items_blacklisted, 0) items_pending = max( int(job_node.get("items_pending", 0) or 0), int(queue_node.get("items_pending", 0) or 0), derived_pending, ) row = { "node_code": node_code, "role": role, "region": region, "status": str(cluster_node.get("status") or "unknown"), "is_effective_worker": is_effective_worker, "detect_participating": False, "current_load": current_load, "items_total": items_total, "items_pending": items_pending, "items_claimed": items_claimed, "items_running": items_running, "items_completed": items_completed, "items_blacklisted": items_blacklisted, "items_failed": items_failed, "active_threads": active_threads, "max_threads": max_threads, "processed_recent": int(queue_node.get("processed_recent", 0) or 0), "processed_per_minute": float(queue_node.get("processed_per_minute", 0) or 0), "last_heartbeat_at": str(cluster_node.get("last_heartbeat_at") or ""), "metrics_source": "active_job" if bool(job_node) else ("cluster_metadata" if metadata else "derived"), } row.update(_detect_participation_snapshot(row=row)) row["detect_participating"] = bool(row.get("is_current_participant", False)) return row def _build_participating_detect_nodes(*, cluster_snapshot: dict, detect_snapshot: dict, worker_runtime: dict) -> list[dict]: cluster_nodes = list(cluster_snapshot.get("nodes") or []) active_job = detect_snapshot.get("active_job") or {} queue_nodes = list((detect_snapshot.get("queue_health") or {}).get("nodes") or []) cluster_map = { str(item.get("node_code") or "").strip(): item for item in cluster_nodes if str(item.get("node_code") or "").strip() } queue_map = { str(item.get("node_code") or "").strip(): item for item in queue_nodes if str(item.get("node_code") or "").strip() } merged: list[dict] = [] job_map = { str(item.get("node_code") or "").strip(): item for item in list(active_job.get("node_stats") or []) if str(item.get("node_code") or "").strip() and str(item.get("node_code") or "").strip() != "unassigned" } candidate_codes = sorted(set(job_map) | set(queue_map) | set(cluster_map)) for node_code in candidate_codes: cluster_node = cluster_map.get(node_code, {}) if cluster_node and not bool(cluster_node.get("is_effective_worker", False)): continue row = _build_detect_node_row( node_code=node_code, cluster_node=cluster_node, job_node=job_map.get(node_code, {}), queue_node=queue_map.get(node_code, {}), ) if not bool(row.get("is_current_participant", False)): continue merged.append(row) return sorted( merged, key=lambda item: ( -int(item.get("items_running", 0) or 0), -int(item.get("items_claimed", 0) or 0), -int(item.get("processed_recent", 0) or 0), -int(item.get("current_load", 0) or 0), str(item.get("node_code") or ""), ), ) def _build_standby_detect_nodes(*, cluster_snapshot: dict, participating_nodes: list[dict]) -> list[dict]: cluster_nodes = list(cluster_snapshot.get("nodes") or []) participating_codes = { str(item.get("node_code") or "").strip() for item in list(participating_nodes or []) if str(item.get("node_code") or "").strip() } standby_rows: list[dict] = [] for node in cluster_nodes: node_code = str(node.get("node_code") or "").strip() node_status = str(node.get("status") or "").strip() node_current_load = int(node.get("current_load", 0) or 0) metadata = node.get("metadata") or {} if not node_code: continue if not bool(node.get("is_effective_worker", False)): continue if node_status not in {"online", "busy"}: continue if node_code in participating_codes: continue standby_state = "load_syncing" if node_current_load > 0 or node_status == "busy" else "standby" standby_label = "负载待确认" if standby_state == "load_syncing" else "在线待命" standby_reason = ( str(metadata.get("detail") or "").strip() or str(metadata.get("phase_detail") or "").strip() or ( f"当前阶段：{str(metadata.get('phase') or metadata.get('phase_label') or '').strip()}" if str(metadata.get("phase") or metadata.get("phase_label") or "").strip() else "" ) or ( f"节点当前负载为 {node_current_load}，但还未观察到已领、执行中或近窗吞吐数据。" if standby_state == "load_syncing" else "节点在线，当前未领任务、未执行任务，也没有近窗吞吐。" ) ) standby_rows.append( { "node_code": node_code, "role": str(node.get("role") or "worker"), "region": str(node.get("region") or settings.node_region), "status": node_status, "is_effective_worker": True, "detect_participating": False, "participation_state": standby_state, "participation_label": standby_label, "participation_reason": standby_reason, "current_load": node_current_load, "last_heartbeat_at": str(node.get("last_heartbeat_at") or ""), "phase": str(metadata.get("phase") or metadata.get("phase_label") or "").strip(), "detail": str(metadata.get("detail") or metadata.get("phase_detail") or "").strip(), "worker_online": bool(metadata.get("worker_online", False)), "standby_reason": standby_reason, } ) return sorted( standby_rows, key=lambda item: ( str(item.get("status") or ""), str(item.get("role") or ""), str(item.get("node_code") or ""), ), ) def _build_detect_participation_summary( *, participating_nodes: list[dict], standby_nodes: list[dict], cluster_snapshot: dict, ) -> dict: cluster_nodes = list(cluster_snapshot.get("nodes") or []) effective_online_nodes = [ node for node in cluster_nodes if bool(node.get("is_effective_worker", False)) and str(node.get("status") or "").strip() in {"online", "busy"} ] dispatch_active_nodes = [ row for row in participating_nodes if bool(row.get("is_dispatch_active", False)) ] recent_only_nodes = [ row for row in participating_nodes if str(row.get("participation_state") or "").strip() == "recent_throughput" ] load_syncing_nodes = [ row for row in standby_nodes if str(row.get("participation_state") or "").strip() == "load_syncing" ] pure_standby_nodes = [ row for row in standby_nodes if str(row.get("participation_state") or "").strip() == "standby" ] dedicated_worker_nodes = [ node for node in effective_online_nodes if str(node.get("role") or "").strip() == "worker" ] controller_worker_nodes = [ node for node in effective_online_nodes if str(node.get("role") or "").strip() == "control" ] summary_parts = [ f"有效执行节点 {len(effective_online_nodes)} 台", f"正在执行/领任务 {len(dispatch_active_nodes)} 台", f"近窗刚有吞吐 {len(recent_only_nodes)} 台", f"在线但未参与 {len(standby_nodes)} 台", ] if load_syncing_nodes: summary_parts.append(f"其中负载待确认 {len(load_syncing_nodes)} 台") return { "effective_online_nodes": len(effective_online_nodes), "participating_nodes": len(participating_nodes), "dispatch_active_nodes": len(dispatch_active_nodes), "recent_only_nodes": len(recent_only_nodes), "non_participating_nodes": len(standby_nodes), "standby_nodes": len(pure_standby_nodes), "load_syncing_nodes": len(load_syncing_nodes), "dedicated_worker_nodes": len(dedicated_worker_nodes), "controller_worker_nodes": len(controller_worker_nodes), "dispatch_active_node_codes": [str(item.get("node_code") or "") for item in dispatch_active_nodes], "recent_only_node_codes": [str(item.get("node_code") or "") for item in recent_only_nodes], "non_participating_node_codes": [str(item.get("node_code") or "") for item in standby_nodes], "standby_node_codes": [str(item.get("node_code") or "") for item in pure_standby_nodes], "load_syncing_node_codes": [str(item.get("node_code") or "") for item in load_syncing_nodes], "summary": "；".join(summary_parts), } def get_runtime_status() -> dict: runtime_settings = get_runtime_settings() worker_runtime = detect_worker_runtime() sync_agent_runtime = detect_sync_agent_runtime() worker_expected_on_this_node = not (settings.node_region == "overseas" and settings.node_role == "control") api_pid = os.getpid() detect_snapshot = get_detect_status() latest_run = (detect_snapshot.get("runs") or [None])[0] or {} cluster_snapshot = get_cluster_snapshot() queue_health = get_detect_queue_health(window_minutes=15) effective_online_worker_nodes = int((cluster_snapshot.get("summary") or {}).get("online_worker_nodes", 0) or 0) if effective_online_worker_nodes <= 0 and worker_runtime.get("running", False): effective_online_worker_nodes = max(1, worker_runtime.get("process_count", 1) or 1) backlog_snapshot = _load_detect_backlog_snapshot() remote_backlog_snapshot = _load_latest_remote_runtime_projection_backlog() runtime_snapshot_backlog = dict(_load_latest_runtime_active_job_snapshot(15).get("backlog") or {}) backlog_snapshot = _merge_backlog_snapshots(backlog_snapshot, remote_backlog_snapshot) backlog_snapshot = _merge_backlog_snapshots(backlog_snapshot, runtime_snapshot_backlog) capacity_plan = get_detect_capacity_plan( queue_health=_align_queue_health_with_backlog(queue_health, backlog_snapshot), online_worker_nodes=effective_online_worker_nodes, target_finish_hours=6, ) detect_payload = { "phase_label": latest_run.get("phase_label", ""), "phase_detail": latest_run.get("phase_detail", ""), "recent_event": detect_snapshot.get("recent_event", ""), "recent_warning": detect_snapshot.get("recent_warning", ""), "progress_percent": detect_snapshot.get("progress_percent", 0), "progress": detect_snapshot.get("progress", {}), "active_thread_count": detect_snapshot.get("active_thread_count", 0), "max_thread_count": detect_snapshot.get("max_thread_count", 0), "available_proxy_count": detect_snapshot.get("available_proxy_count", 0), "proxy_pool_count": detect_snapshot.get("proxy_pool_count", 0), "proxy_runtime_label": detect_snapshot.get("proxy_runtime_label", ""), "proxy_runtime_detail": detect_snapshot.get("proxy_runtime_detail", ""), "proxy_runtime_reason": detect_snapshot.get("proxy_runtime_reason", ""), "proxy_supplier_empty": detect_snapshot.get("proxy_supplier_empty", False), "proxy_last_refresh_status": detect_snapshot.get("proxy_last_refresh_status", ""), "proxy_last_refresh_time": detect_snapshot.get("proxy_last_refresh_time", ""), "proxy_last_refresh_source_count": detect_snapshot.get("proxy_last_refresh_source_count", 0), "proxy_last_refresh_total_items": detect_snapshot.get("proxy_last_refresh_total_items", 0), "proxy_last_validated_count": detect_snapshot.get("proxy_last_validated_count", 0), "proxy_last_available_count": detect_snapshot.get("proxy_last_available_count", 0), "proxy_source_stats": detect_snapshot.get("proxy_source_stats", []), "dependency_alerts": detect_snapshot.get("dependency_alerts", []), "active_job": detect_snapshot.get("active_job"), "runs_count": len(detect_snapshot.get("runs") or []), "worker_online": worker_runtime.get("running", False), "worker_mode": worker_runtime.get("mode", runtime_settings.get("worker_mode", "windows-local")), "queue_health": queue_health, "backlog": backlog_snapshot, "capacity_plan": capacity_plan, "log_sync": { "enabled": bool(runtime_settings.get("worker_log_sync_enabled", False)), "mode": str(runtime_settings.get("worker_log_sync_mode", "key") or "key"), "line_count": int(detect_snapshot.get("remote_log_line_count", 0) or 0), "source_node_count": int(detect_snapshot.get("remote_log_node_count", 0) or 0), "source_nodes": list(detect_snapshot.get("remote_log_nodes") or []), "source_node_summaries": list(detect_snapshot.get("remote_log_node_summaries") or []), "last_at": str(detect_snapshot.get("remote_log_last_at") or ""), "last_line": str(detect_snapshot.get("remote_log_last_line") or ""), "preview_lines": list(detect_snapshot.get("remote_log_lines") or [])[-20:], }, } if not worker_expected_on_this_node: detect_payload.update( { "phase_label": "当前节点不承载", "phase_detail": "当前节点为海外控制面，仅承载 API 控制与同步接收，不执行本机检测任务。", "recent_event": "", "recent_warning": "", "progress_percent": 0, "active_thread_count": 0, "max_thread_count": 0, "available_proxy_count": 0, "proxy_pool_count": 0, "proxy_runtime_label": "不适用", "proxy_runtime_detail": "当前节点不承载本机 Worker，代理、线程与检测阶段信息在此节点上不适用。", "proxy_runtime_reason": "not_applicable", "proxy_supplier_empty": False, "proxy_last_refresh_status": "", "proxy_last_refresh_time": "", "proxy_last_refresh_source_count": 0, "proxy_last_refresh_total_items": 0, "proxy_last_validated_count": 0, "proxy_last_available_count": 0, "proxy_source_stats": [], "dependency_alerts": [], "active_job": None, "runs_count": 0, "worker_online": False, } ) detect_payload["participating_nodes"] = _build_participating_detect_nodes( cluster_snapshot=cluster_snapshot, detect_snapshot=detect_payload, worker_runtime=worker_runtime, ) detect_payload["standby_nodes"] = _build_standby_detect_nodes( cluster_snapshot=cluster_snapshot, participating_nodes=detect_payload["participating_nodes"], ) detect_payload["non_participating_nodes"] = list(detect_payload["standby_nodes"] or []) detect_payload["participation_summary"] = _build_detect_participation_summary( participating_nodes=detect_payload["participating_nodes"], standby_nodes=detect_payload["non_participating_nodes"], cluster_snapshot=cluster_snapshot, ) append_runtime_projection_if_changed(detect=detect_payload, cluster=cluster_snapshot) sync_summary = get_sync_summary(record_limit=5) readiness = _build_multi_region_readiness( cluster_snapshot=cluster_snapshot, sync_summary=sync_summary, worker_runtime=worker_runtime, sync_agent_runtime=sync_agent_runtime, ) build_info = get_runtime_build_info() api_payload = { "service": "domain-api", "version": "0.1.0", "api_prefix": settings.api_prefix, "pid": api_pid, "host": settings.api_host, "port": settings.api_port, "mode": runtime_settings.get("worker_mode", "windows-local"), "service_name": runtime_settings.get("api_service_name", settings.api_service_name), "health_url": f"http://127.0.0.1:{settings.api_port}/health", "stdout_log": _runtime_log_path("domain-api.stdout.log"), "stderr_log": _runtime_log_path("domain-api.stderr.log"), "build": build_info, } node_payload = { "code": settings.node_code, "region": settings.node_region, "role": settings.node_role, } worker_payload = { "mode": worker_runtime.get("mode", runtime_settings.get("worker_mode", "windows-local")), "service_name": runtime_settings.get("worker_service_name", settings.worker_service_name), "running": worker_runtime.get("running", False), "expected_on_this_node": worker_expected_on_this_node, "process_count": worker_runtime.get("process_count", 0), "latest_start_time": worker_runtime.get("latest_start_time", ""), "message": worker_runtime.get("message", ""), "log_path": str(Path(settings.domain_root) / "detect_worker.log"), } sync_agent_payload = { "mode": sync_agent_runtime.get("mode", runtime_settings.get("worker_mode", "windows-local")), "service_name": runtime_settings.get("sync_agent_service_name", settings.sync_agent_service_name), "running": sync_agent_runtime.get("running", False), "process_count": sync_agent_runtime.get("process_count", 0), "latest_start_time": sync_agent_runtime.get("latest_start_time", ""), "message": sync_agent_runtime.get("message", ""), "expected_on_this_node": settings.node_region == "mainland" and settings.node_role == "control", } compatibility_payload = { # Backward-compatible flat fields for older pages / stale built assets. "api_online": bool(api_payload.get("pid")), "api_service_name": api_payload.get("service_name", ""), "worker_online": bool(worker_payload.get("running", False)), "worker_mode": worker_payload.get("mode", ""), "worker_service_name": worker_payload.get("service_name", ""), "worker_process_count": worker_payload.get("process_count", 0), "worker_latest_start_time": worker_payload.get("latest_start_time", ""), "worker_runtime_message": worker_payload.get("message", ""), "thread_count": detect_snapshot.get("thread_count", 0), "thread_count_default": detect_snapshot.get("thread_count_default", 0), "thread_count_source": detect_snapshot.get("thread_count_source", ""), "thread_count_override": detect_snapshot.get("thread_count_override"), "active_thread_count": detect_payload.get("active_thread_count", 0), "max_thread_count": detect_payload.get("max_thread_count", 0), "progress": detect_payload.get("progress", {}), "backlog": detect_payload.get("backlog", {}), "progress_percent": detect_payload.get("progress_percent", 0), "available_proxy_count": detect_payload.get("available_proxy_count", 0), "proxy_pool_count": detect_payload.get("proxy_pool_count", 0), "proxy_runtime_label": detect_payload.get("proxy_runtime_label", ""), "proxy_runtime_detail": detect_payload.get("proxy_runtime_detail", ""), "proxy_runtime_reason": detect_payload.get("proxy_runtime_reason", ""), "proxy_last_refresh_time": detect_payload.get("proxy_last_refresh_time", ""), "recent_event": detect_payload.get("recent_event", ""), "recent_warning": detect_payload.get("recent_warning", ""), "runtime_state": worker_runtime.get("runtime_state") or {}, "cluster_summary": cluster_snapshot.get("summary") or {}, } return { **compatibility_payload, "api": api_payload, "node": node_payload, "worker": worker_payload, "sync_agent": sync_agent_payload, "detect": detect_payload, "cluster": cluster_snapshot, "sync": sync_summary, "readiness": readiness, } def get_runtime_preflight() -> dict: runtime_settings = get_runtime_settings() checks: list[dict[str, object]] = [] detect_options = read_json("detect_options.json", default={}) domain_root = Path(settings.domain_root) checks.append( { "key": "domain_root", "label": "domainCheck 目录", "ok": domain_root.exists(), "message": str(domain_root), } ) try: with get_db() as conn: with conn.cursor() as cur: cur.execute("select 1") cur.fetchone() checks.append({"key": "database", "label": "PostgreSQL", "ok": True, "message": f"{settings.db_host}:{settings.db_port}/{settings.db_database}"}) except Exception as exc: checks.append({"key": "database", "label": "PostgreSQL", "ok": False, "message": str(exc)}) try: redis_client = get_redis() redis_client.ping() checks.append({"key": "redis", "label": "Redis", "ok": True, "message": f"{settings.redis_host}:{settings.redis_port}/{settings.redis_db}"}) except Exception as exc: checks.append({"key": "redis", "label": "Redis", "ok": False, "message": str(exc)}) else: bloom_ok, bloom_message = _bloom_filter_status() checks.append( { "key": "redis_bloom", "label": "RedisBloom", "ok": True, "message": bloom_message, "level": "info" if bloom_ok else "warn", "degraded": not bloom_ok, } ) worker_mode = runtime_settings.get("worker_mode", "windows-local") checks.append({"key": "worker_mode", "label": "运行模式", "ok": True, "message": worker_mode}) jucha_enabled = bool(detect_options.get("detect_jucha")) jucha_cookie_ok, jucha_cookie_path = _domain_cookie_status("jucha_cookies.pkl") checks.append( { "key": "detect_jucha", "label": "聚查检测", "ok": True if not jucha_enabled else jucha_cookie_ok, "message": "已启用" if jucha_enabled else "未启用", } ) checks.append( { "key": "jucha_cookie", "label": "聚查 Cookie", "ok": True if not jucha_enabled else jucha_cookie_ok, "message": jucha_cookie_path if jucha_enabled else "未启用聚查检测，无需本地 Cookie", "level": "info" if (not jucha_enabled or jucha_cookie_ok) else "warn", } ) juziseo_enabled = bool(detect_options.get("detect_juziseo")) juziseo_cookie_ok, juziseo_cookie_path = _domain_cookie_status("juziseo_cookies.pkl") checks.append( { "key": "detect_juziseo", "label": "桔子SEO检测", "ok": True if not juziseo_enabled else juziseo_cookie_ok, "message": "已启用" if juziseo_enabled else "未启用", } ) checks.append( { "key": "juziseo_cookie", "label": "桔子SEO Cookie", "ok": True if not juziseo_enabled else juziseo_cookie_ok, "message": juziseo_cookie_path if juziseo_enabled else "未启用桔子SEO检测，无需本地 Cookie", "level": "info" if (not juziseo_enabled or juziseo_cookie_ok) else "warn", } ) if worker_mode == "linux-systemd": checks.append( { "key": "worker_service_name", "label": "Worker service 名", "ok": bool(runtime_settings.get("worker_service_name")), "message": runtime_settings.get("worker_service_name", ""), } ) checks.append( { "key": "api_service_name", "label": "API service 名", "ok": bool(runtime_settings.get("api_service_name")), "message": runtime_settings.get("api_service_name", ""), } ) checks.append( { "key": "sync_agent_service_name", "label": "Sync agent service 名", "ok": bool(runtime_settings.get("sync_agent_service_name")), "message": runtime_settings.get("sync_agent_service_name", ""), } ) else: checks.append( { "key": "windows_scripts", "label": "Windows 启停脚本", "ok": (Path(settings.domain_root).parent / "start_domain_api.ps1").exists() and (Path(settings.domain_root).parent / "stop_domain_api.ps1").exists(), "message": "start_domain_api.ps1 / stop_domain_api.ps1", } ) overall_ok = all(bool(item["ok"]) for item in checks) return { "ok": overall_ok, "checks": checks, }