from __future__ import annotations import json import socket from datetime import datetime, timedelta from app.core.config import settings from app.core.db import get_db _RUNTIME_SCHEMA_SQL = """ CREATE TABLE IF NOT EXISTS detect_worker_nodes ( node_code VARCHAR(64) PRIMARY KEY, region VARCHAR(32) NOT NULL DEFAULT 'unknown', role VARCHAR(32) NOT NULL DEFAULT 'worker', hostname VARCHAR(255) NOT NULL DEFAULT '', ip VARCHAR(64) NOT NULL DEFAULT '', status VARCHAR(32) NOT NULL DEFAULT 'unknown', worker_version VARCHAR(32) NOT NULL DEFAULT '', current_load INTEGER NOT NULL DEFAULT 0, metadata_json JSONB, last_heartbeat_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, create_time TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, update_time TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ); CREATE TABLE IF NOT EXISTS detect_jobs ( id BIGSERIAL PRIMARY KEY, job_code VARCHAR(64) NOT NULL UNIQUE, source VARCHAR(64) NOT NULL DEFAULT 'manual', plan_hash VARCHAR(128) NOT NULL DEFAULT '', status VARCHAR(32) NOT NULL DEFAULT 'pending', remark TEXT NOT NULL DEFAULT '', created_by VARCHAR(64) NOT NULL DEFAULT '', created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, started_at TIMESTAMP, finished_at TIMESTAMP ); CREATE TABLE IF NOT EXISTS detect_job_items ( id BIGSERIAL PRIMARY KEY, job_id BIGINT NOT NULL REFERENCES detect_jobs(id) ON DELETE CASCADE, domain_id BIGINT NOT NULL, status VARCHAR(32) NOT NULL DEFAULT 'pending', claimed_by VARCHAR(64) NOT NULL DEFAULT '', claim_token VARCHAR(64) NOT NULL DEFAULT '', lease_expires_at TIMESTAMP, attempt_count INTEGER NOT NULL DEFAULT 0, last_error TEXT NOT NULL DEFAULT '', result_version VARCHAR(64) NOT NULL DEFAULT '', started_at TIMESTAMP, finished_at TIMESTAMP, updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, create_time TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, CONSTRAINT uq_detect_job_items_job_domain UNIQUE (job_id, domain_id) ); CREATE INDEX IF NOT EXISTS idx_detect_job_items_status_lease ON detect_job_items(status, lease_expires_at); CREATE TABLE IF NOT EXISTS detect_run_events ( id BIGSERIAL PRIMARY KEY, job_id BIGINT REFERENCES detect_jobs(id) ON DELETE SET NULL, job_item_id BIGINT REFERENCES detect_job_items(id) ON DELETE SET NULL, node_code VARCHAR(64) NOT NULL DEFAULT '', event_type VARCHAR(64) NOT NULL DEFAULT '', level VARCHAR(16) NOT NULL DEFAULT 'info', message TEXT NOT NULL DEFAULT '', payload_json JSONB, created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ); CREATE INDEX IF NOT EXISTS idx_detect_run_events_job_created ON detect_run_events(job_id, created_at DESC); CREATE TABLE IF NOT EXISTS detect_sync_records ( id BIGSERIAL PRIMARY KEY, sync_type VARCHAR(32) NOT NULL DEFAULT '', source_region VARCHAR(32) NOT NULL DEFAULT '', target_region VARCHAR(32) NOT NULL DEFAULT '', status VARCHAR(32) NOT NULL DEFAULT 'pending', payload_json JSONB, error_message TEXT NOT NULL DEFAULT '', created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP, updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP ); """ _STALE_AFTER_SECONDS = 90 _OFFLINE_AFTER_MINUTES = 5 _PRUNE_IMPORTED_AFTER_MINUTES = 30 _PRUNE_GENERAL_AFTER_HOURS = 6 def _resolve_local_ip() -> str: try: return socket.gethostbyname(socket.gethostname()) except Exception: return "" def _decode_json(value: object) -> dict: if isinstance(value, dict): return value if value in (None, ""): return {} try: return json.loads(value) except Exception: return {} def ensure_runtime_schema() -> None: with get_db() as conn: conn.autocommit = False with conn.cursor() as cur: cur.execute(_RUNTIME_SCHEMA_SQL) conn.commit() def register_node_heartbeat( *, node_code: str, region: str, role: str, status: str, current_load: int = 0, worker_version: str = "0.1.0", metadata: dict | None = None, hostname_override: str | None = None, ip_override: str | None = None, ) -> None: hostname = str(hostname_override or "").strip() or socket.gethostname() ip = str(ip_override or "").strip() or _resolve_local_ip() with get_db() as conn: with conn.cursor() as cur: cur.execute( """ INSERT INTO detect_worker_nodes ( node_code, region, role, hostname, ip, status, worker_version, current_load, metadata_json, last_heartbeat_at, update_time ) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP) ON CONFLICT (node_code) DO UPDATE SET region = EXCLUDED.region, role = EXCLUDED.role, hostname = EXCLUDED.hostname, ip = EXCLUDED.ip, status = EXCLUDED.status, worker_version = EXCLUDED.worker_version, current_load = EXCLUDED.current_load, metadata_json = EXCLUDED.metadata_json, last_heartbeat_at = CURRENT_TIMESTAMP, update_time = CURRENT_TIMESTAMP """, ( node_code, region, role, hostname, ip, status, worker_version, max(0, int(current_load or 0)), json.dumps(metadata or {}, ensure_ascii=False), ), ) conn.commit() def cleanup_imported_runtime_nodes(*, region: str, role: str, keep_node_code: str) -> None: normalized_region = str(region or "").strip() or "unknown" normalized_role = str(role or "").strip() or "unknown" preserved_node_code = str(keep_node_code or "").strip() if not preserved_node_code: return with get_db() as conn: with conn.cursor() as cur: cur.execute( """ DELETE FROM detect_worker_nodes WHERE region = %s AND role = %s AND node_code <> %s AND ( node_code = %s OR (metadata_json->>'service') = 'runtime-ingest' ) """, ( normalized_region, normalized_role, preserved_node_code, f"{normalized_region}-{normalized_role}-imported", ), ) conn.commit() def prune_expired_runtime_nodes() -> None: imported_cutoff = datetime.now() - timedelta(minutes=_PRUNE_IMPORTED_AFTER_MINUTES) general_cutoff = datetime.now() - timedelta(hours=_PRUNE_GENERAL_AFTER_HOURS) with get_db() as conn: with conn.cursor() as cur: cur.execute( """ DELETE FROM detect_worker_nodes WHERE ( (metadata_json->>'service') = 'runtime-ingest' AND last_heartbeat_at < %s ) OR ( COALESCE(metadata_json->>'service', '') <> 'runtime-ingest' AND last_heartbeat_at < %s ) """, (imported_cutoff, general_cutoff), ) conn.commit() def register_local_control_heartbeat() -> None: from app.services.detect_job_service import get_active_detect_job_summary from app.services.worker_control_service import detect_worker_runtime worker_runtime = detect_worker_runtime() worker_online = bool(worker_runtime.get("running", False)) active_job = get_active_detect_job_summary(event_limit=5) or {} node_stats = list(active_job.get("node_stats") or []) local_bucket = next( (item for item in node_stats if str(item.get("node_code") or "").strip() == settings.node_code), {}, ) items_total = int(local_bucket.get("items_total", 0) or 0) items_claimed = int(local_bucket.get("items_claimed", 0) or 0) items_running = int(local_bucket.get("items_running", 0) or 0) items_completed = int(local_bucket.get("items_completed", 0) or 0) current_load = max(items_running, items_claimed, 0) detect_participating = bool(worker_online and (items_total > 0 or current_load > 0)) node_status = "busy" if current_load > 0 else "online" register_node_heartbeat( node_code=settings.node_code, region=settings.node_region, role=settings.node_role, status=node_status, current_load=current_load, metadata={ "service": "domain-api", "api_host": settings.api_host, "api_port": settings.api_port, "worker_mode": settings.worker_mode, "worker_online": worker_online, "worker_process_count": int(worker_runtime.get("process_count", 0) or 0), "detect_participating": detect_participating, "active_job_code": str(active_job.get("job_code") or ""), "active_job_status": str(active_job.get("status") or ""), "job_items_total": items_total, "job_items_claimed": items_claimed, "job_items_running": items_running, "job_items_completed": items_completed, "updated_at": datetime.now().isoformat(timespec="seconds"), }, ) def _normalize_node_status(raw_status: str, last_heartbeat_at: datetime | None) -> str: status = str(raw_status or "").strip() or "unknown" if not last_heartbeat_at: return status now = datetime.now(last_heartbeat_at.tzinfo) if last_heartbeat_at.tzinfo else datetime.now() age = now - last_heartbeat_at if age > timedelta(minutes=_OFFLINE_AFTER_MINUTES): return "offline" if age > timedelta(seconds=_STALE_AFTER_SECONDS): return "stale" return status def get_cluster_snapshot() -> dict: prune_expired_runtime_nodes() register_local_control_heartbeat() with get_db() as conn: with conn.cursor() as cur: cur.execute( """ SELECT node_code, region, role, hostname, ip, status, worker_version, current_load, metadata_json, last_heartbeat_at FROM detect_worker_nodes ORDER BY CASE WHEN status = 'busy' THEN 0 WHEN status = 'online' THEN 1 ELSE 2 END, region ASC, role ASC, node_code ASC LIMIT 100 """ ) rows = cur.fetchall() cur.execute("SELECT count(*) FROM detect_jobs") jobs_total = cur.fetchone()[0] cur.execute("SELECT count(*) FROM detect_job_items WHERE status IN ('pending', 'claimed', 'running')") active_items = cur.fetchone()[0] nodes = [ { "node_code": row[0], "region": row[1], "role": row[2], "hostname": row[3], "ip": row[4], "status": _normalize_node_status(row[5], row[9]), "worker_version": row[6], "current_load": row[7], "metadata": _decode_json(row[8]), "last_heartbeat_at": row[9].isoformat(sep=" ", timespec="seconds") if row[9] else "", } for row in rows ] status_counts: dict[str, int] = {} role_counts: dict[str, int] = {} region_counts: dict[str, int] = {} busy_nodes: list[str] = [] stale_nodes: list[str] = [] offline_nodes: list[str] = [] online_worker_nodes = 0 dedicated_online_worker_nodes = 0 online_control_nodes = 0 for node in nodes: node_status = str(node.get("status") or "unknown") node_role = str(node.get("role") or "unknown") node_region = str(node.get("region") or "unknown") node_code = str(node.get("node_code") or "") metadata = node.get("metadata") or {} node_current_load = int(node.get("current_load", 0) or 0) effective_worker = False status_counts[node_status] = status_counts.get(node_status, 0) + 1 role_counts[node_role] = role_counts.get(node_role, 0) + 1 region_counts[node_region] = region_counts.get(node_region, 0) + 1 if node_status == "busy": busy_nodes.append(node_code) if node_status == "stale": stale_nodes.append(node_code) if node_status == "offline": offline_nodes.append(node_code) if node_role == "worker" and node_status in {"online", "busy"}: dedicated_online_worker_nodes += 1 effective_worker = True elif node_role == "control" and node_status in {"online", "busy"}: if bool(metadata.get("worker_online", False)) or bool(metadata.get("detect_participating", False)) or node_current_load > 0: effective_worker = True if effective_worker: online_worker_nodes += 1 if node_role == "control" and node_status in {"online", "busy"}: online_control_nodes += 1 node["is_effective_worker"] = effective_worker node["detect_participating"] = bool(metadata.get("detect_participating", False) or node_current_load > 0) return { "nodes": nodes, "nodes_total": len(nodes), "jobs_total": jobs_total, "active_job_items": active_items, "summary": { "status_counts": status_counts, "role_counts": role_counts, "region_counts": region_counts, "busy_nodes": busy_nodes, "stale_nodes": stale_nodes, "offline_nodes": offline_nodes, "online_worker_nodes": online_worker_nodes, "dedicated_online_worker_nodes": dedicated_online_worker_nodes, "online_control_nodes": online_control_nodes, }, }