306 lines
10 KiB
Python
306 lines
10 KiB
Python
from __future__ import annotations
|
|
|
|
import json
|
|
import socket
|
|
from datetime import datetime, timedelta
|
|
|
|
from app.core.config import settings
|
|
from app.core.db import get_db
|
|
|
|
|
|
_RUNTIME_SCHEMA_SQL = """
|
|
CREATE TABLE IF NOT EXISTS detect_worker_nodes (
|
|
node_code VARCHAR(64) PRIMARY KEY,
|
|
region VARCHAR(32) NOT NULL DEFAULT 'unknown',
|
|
role VARCHAR(32) NOT NULL DEFAULT 'worker',
|
|
hostname VARCHAR(255) NOT NULL DEFAULT '',
|
|
ip VARCHAR(64) NOT NULL DEFAULT '',
|
|
status VARCHAR(32) NOT NULL DEFAULT 'unknown',
|
|
worker_version VARCHAR(32) NOT NULL DEFAULT '',
|
|
current_load INTEGER NOT NULL DEFAULT 0,
|
|
metadata_json JSONB,
|
|
last_heartbeat_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
|
create_time TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
|
update_time TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP
|
|
);
|
|
|
|
CREATE TABLE IF NOT EXISTS detect_jobs (
|
|
id BIGSERIAL PRIMARY KEY,
|
|
job_code VARCHAR(64) NOT NULL UNIQUE,
|
|
source VARCHAR(64) NOT NULL DEFAULT 'manual',
|
|
plan_hash VARCHAR(128) NOT NULL DEFAULT '',
|
|
status VARCHAR(32) NOT NULL DEFAULT 'pending',
|
|
remark TEXT NOT NULL DEFAULT '',
|
|
created_by VARCHAR(64) NOT NULL DEFAULT '',
|
|
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
|
started_at TIMESTAMP,
|
|
finished_at TIMESTAMP
|
|
);
|
|
|
|
CREATE TABLE IF NOT EXISTS detect_job_items (
|
|
id BIGSERIAL PRIMARY KEY,
|
|
job_id BIGINT NOT NULL REFERENCES detect_jobs(id) ON DELETE CASCADE,
|
|
domain_id BIGINT NOT NULL,
|
|
status VARCHAR(32) NOT NULL DEFAULT 'pending',
|
|
claimed_by VARCHAR(64) NOT NULL DEFAULT '',
|
|
claim_token VARCHAR(64) NOT NULL DEFAULT '',
|
|
lease_expires_at TIMESTAMP,
|
|
attempt_count INTEGER NOT NULL DEFAULT 0,
|
|
last_error TEXT NOT NULL DEFAULT '',
|
|
result_version VARCHAR(64) NOT NULL DEFAULT '',
|
|
started_at TIMESTAMP,
|
|
finished_at TIMESTAMP,
|
|
updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
|
create_time TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
|
CONSTRAINT uq_detect_job_items_job_domain UNIQUE (job_id, domain_id)
|
|
);
|
|
|
|
CREATE INDEX IF NOT EXISTS idx_detect_job_items_status_lease
|
|
ON detect_job_items(status, lease_expires_at);
|
|
|
|
CREATE TABLE IF NOT EXISTS detect_run_events (
|
|
id BIGSERIAL PRIMARY KEY,
|
|
job_id BIGINT REFERENCES detect_jobs(id) ON DELETE SET NULL,
|
|
job_item_id BIGINT REFERENCES detect_job_items(id) ON DELETE SET NULL,
|
|
node_code VARCHAR(64) NOT NULL DEFAULT '',
|
|
event_type VARCHAR(64) NOT NULL DEFAULT '',
|
|
level VARCHAR(16) NOT NULL DEFAULT 'info',
|
|
message TEXT NOT NULL DEFAULT '',
|
|
payload_json JSONB,
|
|
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP
|
|
);
|
|
|
|
CREATE INDEX IF NOT EXISTS idx_detect_run_events_job_created
|
|
ON detect_run_events(job_id, created_at DESC);
|
|
|
|
CREATE TABLE IF NOT EXISTS detect_sync_records (
|
|
id BIGSERIAL PRIMARY KEY,
|
|
sync_type VARCHAR(32) NOT NULL DEFAULT '',
|
|
source_region VARCHAR(32) NOT NULL DEFAULT '',
|
|
target_region VARCHAR(32) NOT NULL DEFAULT '',
|
|
status VARCHAR(32) NOT NULL DEFAULT 'pending',
|
|
payload_json JSONB,
|
|
error_message TEXT NOT NULL DEFAULT '',
|
|
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
|
updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP
|
|
);
|
|
"""
|
|
|
|
|
|
def _resolve_local_ip() -> str:
|
|
try:
|
|
return socket.gethostbyname(socket.gethostname())
|
|
except Exception:
|
|
return ""
|
|
|
|
|
|
def _decode_json(value: object) -> dict:
|
|
if isinstance(value, dict):
|
|
return value
|
|
if value in (None, ""):
|
|
return {}
|
|
try:
|
|
return json.loads(value)
|
|
except Exception:
|
|
return {}
|
|
|
|
|
|
def ensure_runtime_schema() -> None:
|
|
with get_db() as conn:
|
|
conn.autocommit = False
|
|
with conn.cursor() as cur:
|
|
cur.execute(_RUNTIME_SCHEMA_SQL)
|
|
conn.commit()
|
|
|
|
|
|
def register_node_heartbeat(
|
|
*,
|
|
node_code: str,
|
|
region: str,
|
|
role: str,
|
|
status: str,
|
|
current_load: int = 0,
|
|
worker_version: str = "0.1.0",
|
|
metadata: dict | None = None,
|
|
hostname_override: str | None = None,
|
|
ip_override: str | None = None,
|
|
) -> None:
|
|
hostname = str(hostname_override or "").strip() or socket.gethostname()
|
|
ip = str(ip_override or "").strip() or _resolve_local_ip()
|
|
with get_db() as conn:
|
|
with conn.cursor() as cur:
|
|
cur.execute(
|
|
"""
|
|
INSERT INTO detect_worker_nodes (
|
|
node_code, region, role, hostname, ip, status, worker_version, current_load, metadata_json, last_heartbeat_at, update_time
|
|
) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)
|
|
ON CONFLICT (node_code) DO UPDATE SET
|
|
region = EXCLUDED.region,
|
|
role = EXCLUDED.role,
|
|
hostname = EXCLUDED.hostname,
|
|
ip = EXCLUDED.ip,
|
|
status = EXCLUDED.status,
|
|
worker_version = EXCLUDED.worker_version,
|
|
current_load = EXCLUDED.current_load,
|
|
metadata_json = EXCLUDED.metadata_json,
|
|
last_heartbeat_at = CURRENT_TIMESTAMP,
|
|
update_time = CURRENT_TIMESTAMP
|
|
""",
|
|
(
|
|
node_code,
|
|
region,
|
|
role,
|
|
hostname,
|
|
ip,
|
|
status,
|
|
worker_version,
|
|
max(0, int(current_load or 0)),
|
|
json.dumps(metadata or {}, ensure_ascii=False),
|
|
),
|
|
)
|
|
conn.commit()
|
|
|
|
|
|
def cleanup_imported_runtime_nodes(*, region: str, role: str, keep_node_code: str) -> None:
|
|
normalized_region = str(region or "").strip() or "unknown"
|
|
normalized_role = str(role or "").strip() or "unknown"
|
|
preserved_node_code = str(keep_node_code or "").strip()
|
|
if not preserved_node_code:
|
|
return
|
|
with get_db() as conn:
|
|
with conn.cursor() as cur:
|
|
cur.execute(
|
|
"""
|
|
DELETE FROM detect_worker_nodes
|
|
WHERE region = %s
|
|
AND role = %s
|
|
AND node_code <> %s
|
|
AND (
|
|
node_code = %s
|
|
OR (metadata_json->>'service') = 'runtime-ingest'
|
|
)
|
|
""",
|
|
(
|
|
normalized_region,
|
|
normalized_role,
|
|
preserved_node_code,
|
|
f"{normalized_region}-{normalized_role}-imported",
|
|
),
|
|
)
|
|
conn.commit()
|
|
|
|
|
|
def register_local_control_heartbeat() -> None:
|
|
register_node_heartbeat(
|
|
node_code=settings.node_code,
|
|
region=settings.node_region,
|
|
role=settings.node_role,
|
|
status="online",
|
|
current_load=0,
|
|
metadata={
|
|
"service": "domain-api",
|
|
"api_host": settings.api_host,
|
|
"api_port": settings.api_port,
|
|
"worker_mode": settings.worker_mode,
|
|
"updated_at": datetime.now().isoformat(timespec="seconds"),
|
|
},
|
|
)
|
|
|
|
|
|
def _normalize_node_status(raw_status: str, last_heartbeat_at: datetime | None) -> str:
|
|
status = str(raw_status or "").strip() or "unknown"
|
|
if not last_heartbeat_at:
|
|
return status
|
|
now = datetime.now(last_heartbeat_at.tzinfo) if last_heartbeat_at.tzinfo else datetime.now()
|
|
age = now - last_heartbeat_at
|
|
if age > timedelta(minutes=5):
|
|
return "offline"
|
|
if age > timedelta(seconds=90):
|
|
return "stale"
|
|
return status
|
|
|
|
|
|
def get_cluster_snapshot() -> dict:
|
|
register_local_control_heartbeat()
|
|
with get_db() as conn:
|
|
with conn.cursor() as cur:
|
|
cur.execute(
|
|
"""
|
|
SELECT node_code, region, role, hostname, ip, status, worker_version, current_load, metadata_json, last_heartbeat_at
|
|
FROM detect_worker_nodes
|
|
ORDER BY
|
|
CASE WHEN status = 'busy' THEN 0 WHEN status = 'online' THEN 1 ELSE 2 END,
|
|
region ASC,
|
|
role ASC,
|
|
node_code ASC
|
|
LIMIT 100
|
|
"""
|
|
)
|
|
rows = cur.fetchall()
|
|
cur.execute("SELECT count(*) FROM detect_jobs")
|
|
jobs_total = cur.fetchone()[0]
|
|
cur.execute("SELECT count(*) FROM detect_job_items WHERE status IN ('pending', 'claimed', 'running')")
|
|
active_items = cur.fetchone()[0]
|
|
|
|
nodes = [
|
|
{
|
|
"node_code": row[0],
|
|
"region": row[1],
|
|
"role": row[2],
|
|
"hostname": row[3],
|
|
"ip": row[4],
|
|
"status": _normalize_node_status(row[5], row[9]),
|
|
"worker_version": row[6],
|
|
"current_load": row[7],
|
|
"metadata": _decode_json(row[8]),
|
|
"last_heartbeat_at": row[9].isoformat(sep=" ", timespec="seconds") if row[9] else "",
|
|
}
|
|
for row in rows
|
|
]
|
|
status_counts: dict[str, int] = {}
|
|
role_counts: dict[str, int] = {}
|
|
region_counts: dict[str, int] = {}
|
|
busy_nodes: list[str] = []
|
|
stale_nodes: list[str] = []
|
|
offline_nodes: list[str] = []
|
|
online_worker_nodes = 0
|
|
online_control_nodes = 0
|
|
|
|
for node in nodes:
|
|
node_status = str(node.get("status") or "unknown")
|
|
node_role = str(node.get("role") or "unknown")
|
|
node_region = str(node.get("region") or "unknown")
|
|
node_code = str(node.get("node_code") or "")
|
|
|
|
status_counts[node_status] = status_counts.get(node_status, 0) + 1
|
|
role_counts[node_role] = role_counts.get(node_role, 0) + 1
|
|
region_counts[node_region] = region_counts.get(node_region, 0) + 1
|
|
|
|
if node_status == "busy":
|
|
busy_nodes.append(node_code)
|
|
if node_status == "stale":
|
|
stale_nodes.append(node_code)
|
|
if node_status == "offline":
|
|
offline_nodes.append(node_code)
|
|
if node_role == "worker" and node_status in {"online", "busy"}:
|
|
online_worker_nodes += 1
|
|
if node_role == "control" and node_status in {"online", "busy"}:
|
|
online_control_nodes += 1
|
|
|
|
return {
|
|
"nodes": nodes,
|
|
"nodes_total": len(nodes),
|
|
"jobs_total": jobs_total,
|
|
"active_job_items": active_items,
|
|
"summary": {
|
|
"status_counts": status_counts,
|
|
"role_counts": role_counts,
|
|
"region_counts": region_counts,
|
|
"busy_nodes": busy_nodes,
|
|
"stale_nodes": stale_nodes,
|
|
"offline_nodes": offline_nodes,
|
|
"online_worker_nodes": online_worker_nodes,
|
|
"online_control_nodes": online_control_nodes,
|
|
},
|
|
}
|