admin/getDomain
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

d

Browse Source
This commit is contained in:
Your Name
2026-04-22 14:13:21 +08:00
parent e0406b5d0e
commit 7cbde2aa78
145 changed files with 23086 additions and 2243 deletions
Download Patch File Download Diff File Expand all files Collapse all files

232
domain-api/app/services/cluster_runtime_service.py
View File

@@ -2,10 +2,11 @@ from __future__ import annotations
import json
import socket
import threading
from datetime import datetime, timedelta
from app.core.config import settings
from app.core.db import get_db
from app.core.db import db_read_retry, get_db
_RUNTIME_SCHEMA_SQL = """
@@ -29,6 +30,8 @@ CREATE TABLE IF NOT EXISTS detect_jobs (
job_code VARCHAR(64) NOT NULL UNIQUE,
source VARCHAR(64) NOT NULL DEFAULT 'manual',
plan_hash VARCHAR(128) NOT NULL DEFAULT '',
task_mode VARCHAR(32) NOT NULL DEFAULT 'domain_pipeline',
step_code VARCHAR(64) NOT NULL DEFAULT '',
status VARCHAR(32) NOT NULL DEFAULT 'pending',
remark TEXT NOT NULL DEFAULT '',
created_by VARCHAR(64) NOT NULL DEFAULT '',
@@ -41,6 +44,7 @@ CREATE TABLE IF NOT EXISTS detect_job_items (
id BIGSERIAL PRIMARY KEY,
job_id BIGINT NOT NULL REFERENCES detect_jobs(id) ON DELETE CASCADE,
domain_id BIGINT NOT NULL,
step_code VARCHAR(64) NOT NULL DEFAULT '',
status VARCHAR(32) NOT NULL DEFAULT 'pending',
claimed_by VARCHAR(64) NOT NULL DEFAULT '',
claim_token VARCHAR(64) NOT NULL DEFAULT '',
@@ -48,16 +52,32 @@ CREATE TABLE IF NOT EXISTS detect_job_items (
attempt_count INTEGER NOT NULL DEFAULT 0,
last_error TEXT NOT NULL DEFAULT '',
result_version VARCHAR(64) NOT NULL DEFAULT '',
step_payload_json JSONB,
result_payload_json JSONB,
started_at TIMESTAMP,
finished_at TIMESTAMP,
updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
create_time TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
CONSTRAINT uq_detect_job_items_job_domain UNIQUE (job_id, domain_id)
create_time TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP
);
CREATE INDEX IF NOT EXISTS idx_detect_job_items_status_lease
ON detect_job_items(status, lease_expires_at);
ALTER TABLE detect_jobs
ADD COLUMN IF NOT EXISTS task_mode VARCHAR(32) NOT NULL DEFAULT 'domain_pipeline',
ADD COLUMN IF NOT EXISTS step_code VARCHAR(64) NOT NULL DEFAULT '';
ALTER TABLE detect_job_items
ADD COLUMN IF NOT EXISTS step_code VARCHAR(64) NOT NULL DEFAULT '',
ADD COLUMN IF NOT EXISTS step_payload_json JSONB,
ADD COLUMN IF NOT EXISTS result_payload_json JSONB;
ALTER TABLE detect_job_items
DROP CONSTRAINT IF EXISTS uq_detect_job_items_job_domain;
CREATE UNIQUE INDEX IF NOT EXISTS idx_detect_job_items_job_domain_step
ON detect_job_items(job_id, domain_id, step_code);
CREATE TABLE IF NOT EXISTS detect_run_events (
id BIGSERIAL PRIMARY KEY,
job_id BIGINT REFERENCES detect_jobs(id) ON DELETE SET NULL,
@@ -90,6 +110,9 @@ _STALE_AFTER_SECONDS = 90
_OFFLINE_AFTER_MINUTES = 5
_PRUNE_IMPORTED_AFTER_MINUTES = 30
_PRUNE_GENERAL_AFTER_HOURS = 6
_RUNTIME_SCHEMA_READY = False
_RUNTIME_SCHEMA_LOCK = threading.Lock()
_RUNTIME_SCHEMA_ADVISORY_LOCK_ID = 62021001
def _resolve_local_ip() -> str:
@@ -110,12 +133,85 @@ def _decode_json(value: object) -> dict:
return {}
def _control_node_supports_worker(*, region: object, metadata: dict | None) -> bool:
normalized_region = str(region or "").strip()
runtime_metadata = dict(metadata or {})
active_threads = int(runtime_metadata.get("active_threads", 0) or 0)
max_threads = int(runtime_metadata.get("max_threads", 0) or 0)
if normalized_region != "mainland":
return False
return bool(
runtime_metadata.get("worker_online", False)
or runtime_metadata.get("detect_participating", False)
or active_threads > 0
or max_threads > 0
)
def _metadata_idle_without_runtime_work(metadata: dict | None) -> bool:
runtime_metadata = dict(metadata or {})
phase = str(
runtime_metadata.get("phase")
or runtime_metadata.get("phase_label")
or ""
).strip().lower()
if phase not in {"idle", "completed", "stopped"}:
return False
active_job_code = str(runtime_metadata.get("active_job_code") or "").strip()
counters = (
int(runtime_metadata.get("job_items_total", 0) or 0),
int(runtime_metadata.get("job_items_claimed", 0) or 0),
int(runtime_metadata.get("job_items_running", 0) or 0),
int(runtime_metadata.get("job_items_completed", 0) or 0),
int(runtime_metadata.get("job_items_failed", 0) or 0),
)
if active_job_code:
return False
return not any(value > 0 for value in counters)
def _load_managed_node_overlays() -> dict[str, dict]:
try:
with get_db() as conn:
with conn.cursor() as cur:
cur.execute(
"""
SELECT node_code, metadata_json, last_seen_at
FROM ops_managed_nodes
WHERE is_enabled = TRUE
"""
)
rows = cur.fetchall()
except Exception:
return {}
overlays: dict[str, dict] = {}
for row in rows:
node_code = str(row[0] or "").strip()
if not node_code:
continue
overlays[node_code] = {
"metadata": _decode_json(row[1]),
"last_seen_at": row[2],
}
return overlays
def ensure_runtime_schema() -> None:
with get_db() as conn:
conn.autocommit = False
with conn.cursor() as cur:
cur.execute(_RUNTIME_SCHEMA_SQL)
conn.commit()
global _RUNTIME_SCHEMA_READY
if _RUNTIME_SCHEMA_READY:
return
with _RUNTIME_SCHEMA_LOCK:
if _RUNTIME_SCHEMA_READY:
return
with get_db() as conn:
conn.autocommit = False
with conn.cursor() as cur:
cur.execute("SELECT pg_advisory_xact_lock(%s)", (_RUNTIME_SCHEMA_ADVISORY_LOCK_ID,))
cur.execute(_RUNTIME_SCHEMA_SQL)
conn.commit()
_RUNTIME_SCHEMA_READY = True
def register_node_heartbeat(
@@ -195,6 +291,43 @@ def cleanup_imported_runtime_nodes(*, region: str, role: str, keep_node_code: st
conn.commit()
def cleanup_imported_runtime_nodes_many(*, region: str, role: str, keep_node_codes: list[str] | tuple[str, ...] | set[str]) -> None:
normalized_region = str(region or "").strip() or "unknown"
normalized_role = str(role or "").strip() or "unknown"
preserved_node_codes = sorted(
{
str(node_code or "").strip()
for node_code in (keep_node_codes or [])
if str(node_code or "").strip()
}
)
if not preserved_node_codes:
return
placeholders = ", ".join(["%s"] * len(preserved_node_codes))
with get_db() as conn:
with conn.cursor() as cur:
cur.execute(
f"""
DELETE FROM detect_worker_nodes
WHERE region = %s
AND role = %s
AND (
node_code = %s
OR (metadata_json->>'service') = 'runtime-ingest'
)
AND node_code NOT IN ({placeholders})
""",
(
normalized_region,
normalized_role,
f"{normalized_region}-{normalized_role}-imported",
*preserved_node_codes,
),
)
conn.commit()
def prune_expired_runtime_nodes() -> None:
imported_cutoff = datetime.now() - timedelta(minutes=_PRUNE_IMPORTED_AFTER_MINUTES)
general_cutoff = datetime.now() - timedelta(hours=_PRUNE_GENERAL_AFTER_HOURS)
@@ -219,10 +352,12 @@ def prune_expired_runtime_nodes() -> None:
def register_local_control_heartbeat() -> None:
from app.services.detect_job_service import get_active_detect_job_summary
from app.services.detect_service import get_detect_status
from app.services.worker_control_service import detect_worker_runtime
worker_runtime = detect_worker_runtime()
worker_online = bool(worker_runtime.get("running", False))
detect_status = get_detect_status()
active_job = get_active_detect_job_summary(event_limit=5) or {}
node_stats = list(active_job.get("node_stats") or [])
local_bucket = next(
@@ -233,7 +368,9 @@ def register_local_control_heartbeat() -> None:
items_claimed = int(local_bucket.get("items_claimed", 0) or 0)
items_running = int(local_bucket.get("items_running", 0) or 0)
items_completed = int(local_bucket.get("items_completed", 0) or 0)
current_load = max(items_running, items_claimed, 0)
active_threads = int(detect_status.get("active_thread_count", 0) or 0)
max_threads = int(detect_status.get("max_thread_count", 0) or 0)
current_load = max(items_running, active_threads, 0)
detect_participating = bool(worker_online and (items_total > 0 or current_load > 0))
node_status = "busy" if current_load > 0 else "online"
register_node_heartbeat(
@@ -256,6 +393,10 @@ def register_local_control_heartbeat() -> None:
"job_items_claimed": items_claimed,
"job_items_running": items_running,
"job_items_completed": items_completed,
"active_threads": active_threads,
"max_threads": max_threads,
"phase_label": str(detect_status.get("phase_label") or ""),
"phase_detail": str(detect_status.get("phase_detail") or ""),
"updated_at": datetime.now().isoformat(timespec="seconds"),
},
)
@@ -274,9 +415,11 @@ def _normalize_node_status(raw_status: str, last_heartbeat_at: datetime | None)
return status
@db_read_retry()
def get_cluster_snapshot() -> dict:
prune_expired_runtime_nodes()
register_local_control_heartbeat()
managed_overlays = _load_managed_node_overlays()
with get_db() as conn:
with conn.cursor() as cur:
cur.execute(
@@ -297,21 +440,50 @@ def get_cluster_snapshot() -> dict:
cur.execute("SELECT count(*) FROM detect_job_items WHERE status IN ('pending', 'claimed', 'running')")
active_items = cur.fetchone()[0]
nodes = [
{
"node_code": row[0],
"region": row[1],
"role": row[2],
"hostname": row[3],
"ip": row[4],
"status": _normalize_node_status(row[5], row[9]),
"worker_version": row[6],
"current_load": row[7],
"metadata": _decode_json(row[8]),
"last_heartbeat_at": row[9].isoformat(sep=" ", timespec="seconds") if row[9] else "",
}
for row in rows
]
nodes = []
for row in rows:
node_code = str(row[0] or "").strip()
metadata = _decode_json(row[8])
current_load = int(row[7] or 0)
sanitized_idle_runtime = _metadata_idle_without_runtime_work(metadata)
if sanitized_idle_runtime:
current_load = 0
metadata["active_threads"] = 0
metadata["detect_participating"] = False
metadata["sanitized_runtime_state"] = "idle_phase_zeroed"
runtime_last_heartbeat = row[9]
managed_overlay = managed_overlays.get(node_code) or {}
managed_last_seen = managed_overlay.get("last_seen_at")
overlay_is_newer = bool(
managed_last_seen
and (not runtime_last_heartbeat or managed_last_seen > runtime_last_heartbeat)
)
effective_last_heartbeat = managed_last_seen if overlay_is_newer else runtime_last_heartbeat
normalized_status = _normalize_node_status(row[5], effective_last_heartbeat)
if sanitized_idle_runtime and normalized_status == "busy":
normalized_status = "online"
if overlay_is_newer and normalized_status in {"offline", "stale"}:
normalized_status = "busy" if current_load > 0 else "online"
if managed_last_seen:
metadata["agent_last_seen_at"] = managed_last_seen.isoformat(sep=" ", timespec="seconds")
if overlay_is_newer:
metadata["cluster_status_source"] = "managed-agent-overlay"
nodes.append(
{
"node_code": node_code,
"region": row[1],
"role": row[2],
"hostname": row[3],
"ip": row[4],
"status": normalized_status,
"worker_version": row[6],
"current_load": current_load,
"metadata": metadata,
"last_heartbeat_at": effective_last_heartbeat.isoformat(sep=" ", timespec="seconds")
if effective_last_heartbeat
else "",
}
)
status_counts: dict[str, int] = {}
role_counts: dict[str, int] = {}
region_counts: dict[str, int] = {}
@@ -330,6 +502,12 @@ def get_cluster_snapshot() -> dict:
metadata = node.get("metadata") or {}
node_current_load = int(node.get("current_load", 0) or 0)
effective_worker = False
if node_role == "control" and not _control_node_supports_worker(region=node_region, metadata=metadata):
node_current_load = 0
node["current_load"] = 0
if node_status == "busy":
node_status = "online"
node["status"] = "online"
status_counts[node_status] = status_counts.get(node_status, 0) + 1
role_counts[node_role] = role_counts.get(node_role, 0) + 1
@@ -345,14 +523,16 @@ def get_cluster_snapshot() -> dict:
dedicated_online_worker_nodes += 1
effective_worker = True
elif node_role == "control" and node_status in {"online", "busy"}:
if bool(metadata.get("worker_online", False)) or bool(metadata.get("detect_participating", False)) or node_current_load > 0:
if _control_node_supports_worker(region=node_region, metadata=metadata):
effective_worker = True
if effective_worker:
online_worker_nodes += 1
if node_role == "control" and node_status in {"online", "busy"}:
online_control_nodes += 1
node["is_effective_worker"] = effective_worker
node["detect_participating"] = bool(metadata.get("detect_participating", False) or node_current_load > 0)
node["detect_participating"] = bool(
effective_worker and (metadata.get("detect_participating", False) or node_current_load > 0)
)
return {
"nodes": nodes,