debug

domain-api/app/services/cluster_runtime_service.py

@@ -86,6 +86,11 @@ CREATE TABLE IF NOT EXISTS detect_sync_records (
 );
 """
 
+_STALE_AFTER_SECONDS = 90
+_OFFLINE_AFTER_MINUTES = 5
+_PRUNE_IMPORTED_AFTER_MINUTES = 30
+_PRUNE_GENERAL_AFTER_HOURS = 6
+
 
 def _resolve_local_ip() -> str:
     try:
@@ -190,6 +195,28 @@ def cleanup_imported_runtime_nodes(*, region: str, role: str, keep_node_code: st
         conn.commit()
 
 
+def prune_expired_runtime_nodes() -> None:
+    imported_cutoff = datetime.now() - timedelta(minutes=_PRUNE_IMPORTED_AFTER_MINUTES)
+    general_cutoff = datetime.now() - timedelta(hours=_PRUNE_GENERAL_AFTER_HOURS)
+    with get_db() as conn:
+        with conn.cursor() as cur:
+            cur.execute(
+                """
+                DELETE FROM detect_worker_nodes
+                WHERE (
+                    (metadata_json->>'service') = 'runtime-ingest'
+                    AND last_heartbeat_at < %s
+                )
+                OR (
+                    COALESCE(metadata_json->>'service', '') <> 'runtime-ingest'
+                    AND last_heartbeat_at < %s
+                )
+                """,
+                (imported_cutoff, general_cutoff),
+            )
+        conn.commit()
+
+
 def register_local_control_heartbeat() -> None:
     from app.services.detect_job_service import get_active_detect_job_summary
     from app.services.worker_control_service import detect_worker_runtime
@@ -240,14 +267,15 @@ def _normalize_node_status(raw_status: str, last_heartbeat_at: datetime | None)
         return status
     now = datetime.now(last_heartbeat_at.tzinfo) if last_heartbeat_at.tzinfo else datetime.now()
     age = now - last_heartbeat_at
-    if age > timedelta(minutes=5):
+    if age > timedelta(minutes=_OFFLINE_AFTER_MINUTES):
         return "offline"
-    if age > timedelta(seconds=90):
+    if age > timedelta(seconds=_STALE_AFTER_SECONDS):
         return "stale"
     return status
 
 
 def get_cluster_snapshot() -> dict:
+    prune_expired_runtime_nodes()
     register_local_control_heartbeat()
     with get_db() as conn:
         with conn.cursor() as cur: