first
This commit is contained in:
Your Name
272
domain-api/app/services/cluster_runtime_service.py
Normal file
272
domain-api/app/services/cluster_runtime_service.py
Normal file
@@ -0,0 +1,272 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import socket
|
||||
from datetime import datetime, timedelta
|
||||
|
||||
from app.core.config import settings
|
||||
from app.core.db import get_db
|
||||
|
||||
|
||||
_RUNTIME_SCHEMA_SQL = """
|
||||
CREATE TABLE IF NOT EXISTS detect_worker_nodes (
|
||||
node_code VARCHAR(64) PRIMARY KEY,
|
||||
region VARCHAR(32) NOT NULL DEFAULT 'unknown',
|
||||
role VARCHAR(32) NOT NULL DEFAULT 'worker',
|
||||
hostname VARCHAR(255) NOT NULL DEFAULT '',
|
||||
ip VARCHAR(64) NOT NULL DEFAULT '',
|
||||
status VARCHAR(32) NOT NULL DEFAULT 'unknown',
|
||||
worker_version VARCHAR(32) NOT NULL DEFAULT '',
|
||||
current_load INTEGER NOT NULL DEFAULT 0,
|
||||
metadata_json JSONB,
|
||||
last_heartbeat_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
create_time TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
update_time TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS detect_jobs (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
job_code VARCHAR(64) NOT NULL UNIQUE,
|
||||
source VARCHAR(64) NOT NULL DEFAULT 'manual',
|
||||
plan_hash VARCHAR(128) NOT NULL DEFAULT '',
|
||||
status VARCHAR(32) NOT NULL DEFAULT 'pending',
|
||||
remark TEXT NOT NULL DEFAULT '',
|
||||
created_by VARCHAR(64) NOT NULL DEFAULT '',
|
||||
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
started_at TIMESTAMP,
|
||||
finished_at TIMESTAMP
|
||||
);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS detect_job_items (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
job_id BIGINT NOT NULL REFERENCES detect_jobs(id) ON DELETE CASCADE,
|
||||
domain_id BIGINT NOT NULL,
|
||||
status VARCHAR(32) NOT NULL DEFAULT 'pending',
|
||||
claimed_by VARCHAR(64) NOT NULL DEFAULT '',
|
||||
claim_token VARCHAR(64) NOT NULL DEFAULT '',
|
||||
lease_expires_at TIMESTAMP,
|
||||
attempt_count INTEGER NOT NULL DEFAULT 0,
|
||||
last_error TEXT NOT NULL DEFAULT '',
|
||||
result_version VARCHAR(64) NOT NULL DEFAULT '',
|
||||
started_at TIMESTAMP,
|
||||
finished_at TIMESTAMP,
|
||||
updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
create_time TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
CONSTRAINT uq_detect_job_items_job_domain UNIQUE (job_id, domain_id)
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_detect_job_items_status_lease
|
||||
ON detect_job_items(status, lease_expires_at);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS detect_run_events (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
job_id BIGINT REFERENCES detect_jobs(id) ON DELETE SET NULL,
|
||||
job_item_id BIGINT REFERENCES detect_job_items(id) ON DELETE SET NULL,
|
||||
node_code VARCHAR(64) NOT NULL DEFAULT '',
|
||||
event_type VARCHAR(64) NOT NULL DEFAULT '',
|
||||
level VARCHAR(16) NOT NULL DEFAULT 'info',
|
||||
message TEXT NOT NULL DEFAULT '',
|
||||
payload_json JSONB,
|
||||
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
|
||||
CREATE INDEX IF NOT EXISTS idx_detect_run_events_job_created
|
||||
ON detect_run_events(job_id, created_at DESC);
|
||||
|
||||
CREATE TABLE IF NOT EXISTS detect_sync_records (
|
||||
id BIGSERIAL PRIMARY KEY,
|
||||
sync_type VARCHAR(32) NOT NULL DEFAULT '',
|
||||
source_region VARCHAR(32) NOT NULL DEFAULT '',
|
||||
target_region VARCHAR(32) NOT NULL DEFAULT '',
|
||||
status VARCHAR(32) NOT NULL DEFAULT 'pending',
|
||||
payload_json JSONB,
|
||||
error_message TEXT NOT NULL DEFAULT '',
|
||||
created_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
|
||||
updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP
|
||||
);
|
||||
"""
|
||||
|
||||
|
||||
def _resolve_local_ip() -> str:
|
||||
try:
|
||||
return socket.gethostbyname(socket.gethostname())
|
||||
except Exception:
|
||||
return ""
|
||||
|
||||
|
||||
def _decode_json(value: object) -> dict:
|
||||
if isinstance(value, dict):
|
||||
return value
|
||||
if value in (None, ""):
|
||||
return {}
|
||||
try:
|
||||
return json.loads(value)
|
||||
except Exception:
|
||||
return {}
|
||||
|
||||
|
||||
def ensure_runtime_schema() -> None:
|
||||
with get_db() as conn:
|
||||
conn.autocommit = False
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(_RUNTIME_SCHEMA_SQL)
|
||||
conn.commit()
|
||||
|
||||
|
||||
def register_node_heartbeat(
|
||||
*,
|
||||
node_code: str,
|
||||
region: str,
|
||||
role: str,
|
||||
status: str,
|
||||
current_load: int = 0,
|
||||
worker_version: str = "0.1.0",
|
||||
metadata: dict | None = None,
|
||||
) -> None:
|
||||
with get_db() as conn:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(
|
||||
"""
|
||||
INSERT INTO detect_worker_nodes (
|
||||
node_code, region, role, hostname, ip, status, worker_version, current_load, metadata_json, last_heartbeat_at, update_time
|
||||
) VALUES (%s, %s, %s, %s, %s, %s, %s, %s, %s, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)
|
||||
ON CONFLICT (node_code) DO UPDATE SET
|
||||
region = EXCLUDED.region,
|
||||
role = EXCLUDED.role,
|
||||
hostname = EXCLUDED.hostname,
|
||||
ip = EXCLUDED.ip,
|
||||
status = EXCLUDED.status,
|
||||
worker_version = EXCLUDED.worker_version,
|
||||
current_load = EXCLUDED.current_load,
|
||||
metadata_json = EXCLUDED.metadata_json,
|
||||
last_heartbeat_at = CURRENT_TIMESTAMP,
|
||||
update_time = CURRENT_TIMESTAMP
|
||||
""",
|
||||
(
|
||||
node_code,
|
||||
region,
|
||||
role,
|
||||
socket.gethostname(),
|
||||
_resolve_local_ip(),
|
||||
status,
|
||||
worker_version,
|
||||
max(0, int(current_load or 0)),
|
||||
json.dumps(metadata or {}, ensure_ascii=False),
|
||||
),
|
||||
)
|
||||
conn.commit()
|
||||
|
||||
|
||||
def register_local_control_heartbeat() -> None:
|
||||
register_node_heartbeat(
|
||||
node_code=settings.node_code,
|
||||
region=settings.node_region,
|
||||
role=settings.node_role,
|
||||
status="online",
|
||||
current_load=0,
|
||||
metadata={
|
||||
"service": "domain-api",
|
||||
"api_host": settings.api_host,
|
||||
"api_port": settings.api_port,
|
||||
"worker_mode": settings.worker_mode,
|
||||
"updated_at": datetime.now().isoformat(timespec="seconds"),
|
||||
},
|
||||
)
|
||||
|
||||
|
||||
def _normalize_node_status(raw_status: str, last_heartbeat_at: datetime | None) -> str:
|
||||
status = str(raw_status or "").strip() or "unknown"
|
||||
if not last_heartbeat_at:
|
||||
return status
|
||||
now = datetime.now(last_heartbeat_at.tzinfo) if last_heartbeat_at.tzinfo else datetime.now()
|
||||
age = now - last_heartbeat_at
|
||||
if age > timedelta(minutes=5):
|
||||
return "offline"
|
||||
if age > timedelta(seconds=90):
|
||||
return "stale"
|
||||
return status
|
||||
|
||||
|
||||
def get_cluster_snapshot() -> dict:
|
||||
register_local_control_heartbeat()
|
||||
with get_db() as conn:
|
||||
with conn.cursor() as cur:
|
||||
cur.execute(
|
||||
"""
|
||||
SELECT node_code, region, role, hostname, ip, status, worker_version, current_load, metadata_json, last_heartbeat_at
|
||||
FROM detect_worker_nodes
|
||||
ORDER BY
|
||||
CASE WHEN status = 'busy' THEN 0 WHEN status = 'online' THEN 1 ELSE 2 END,
|
||||
region ASC,
|
||||
role ASC,
|
||||
node_code ASC
|
||||
LIMIT 100
|
||||
"""
|
||||
)
|
||||
rows = cur.fetchall()
|
||||
cur.execute("SELECT count(*) FROM detect_jobs")
|
||||
jobs_total = cur.fetchone()[0]
|
||||
cur.execute("SELECT count(*) FROM detect_job_items WHERE status IN ('pending', 'claimed', 'running')")
|
||||
active_items = cur.fetchone()[0]
|
||||
|
||||
nodes = [
|
||||
{
|
||||
"node_code": row[0],
|
||||
"region": row[1],
|
||||
"role": row[2],
|
||||
"hostname": row[3],
|
||||
"ip": row[4],
|
||||
"status": _normalize_node_status(row[5], row[9]),
|
||||
"worker_version": row[6],
|
||||
"current_load": row[7],
|
||||
"metadata": _decode_json(row[8]),
|
||||
"last_heartbeat_at": row[9].isoformat(sep=" ", timespec="seconds") if row[9] else "",
|
||||
}
|
||||
for row in rows
|
||||
]
|
||||
status_counts: dict[str, int] = {}
|
||||
role_counts: dict[str, int] = {}
|
||||
region_counts: dict[str, int] = {}
|
||||
busy_nodes: list[str] = []
|
||||
stale_nodes: list[str] = []
|
||||
offline_nodes: list[str] = []
|
||||
online_worker_nodes = 0
|
||||
online_control_nodes = 0
|
||||
|
||||
for node in nodes:
|
||||
node_status = str(node.get("status") or "unknown")
|
||||
node_role = str(node.get("role") or "unknown")
|
||||
node_region = str(node.get("region") or "unknown")
|
||||
node_code = str(node.get("node_code") or "")
|
||||
|
||||
status_counts[node_status] = status_counts.get(node_status, 0) + 1
|
||||
role_counts[node_role] = role_counts.get(node_role, 0) + 1
|
||||
region_counts[node_region] = region_counts.get(node_region, 0) + 1
|
||||
|
||||
if node_status == "busy":
|
||||
busy_nodes.append(node_code)
|
||||
if node_status == "stale":
|
||||
stale_nodes.append(node_code)
|
||||
if node_status == "offline":
|
||||
offline_nodes.append(node_code)
|
||||
if node_role == "worker" and node_status in {"online", "busy"}:
|
||||
online_worker_nodes += 1
|
||||
if node_role == "control" and node_status in {"online", "busy"}:
|
||||
online_control_nodes += 1
|
||||
|
||||
return {
|
||||
"nodes": nodes,
|
||||
"nodes_total": len(nodes),
|
||||
"jobs_total": jobs_total,
|
||||
"active_job_items": active_items,
|
||||
"summary": {
|
||||
"status_counts": status_counts,
|
||||
"role_counts": role_counts,
|
||||
"region_counts": region_counts,
|
||||
"busy_nodes": busy_nodes,
|
||||
"stale_nodes": stale_nodes,
|
||||
"offline_nodes": offline_nodes,
|
||||
"online_worker_nodes": online_worker_nodes,
|
||||
"online_control_nodes": online_control_nodes,
|
||||
},
|
||||
}
|
||||
Reference in New Issue
Block a user