from __future__ import annotations import json import time from datetime import datetime import math from uuid import uuid4 from app.core.config import settings from app.core.db import db_read_retry, get_db, is_retryable_db_error from app.services.debug_event_service import push_debug_event from app.services.settings_service import get_settings_payload ACTIVE_JOB_STATUSES = ("pending", "running") _RUNTIME_NODE_STALE_MINUTES = 10 _DEFAULT_TASK_MODE = "domain_pipeline" _PIPELINE_STEP_ORDER = [ "detect_register", "detect_baidu_site", "detect_360_site", "detect_chinaz", "detect_aizhan", "detect_wayback", "detect_jucha", "detect_juziseo", ] _PIPELINE_STEP_FIELDS = { "detect_baidu_site": "baidu_site", "detect_360_site": "qihu360_site", "detect_chinaz": "chinaz_info", "detect_aizhan": "aizhan_info", "detect_wayback": "wayback_info", "detect_jucha": "jucha_info", "detect_juziseo": "juziseo_info", } _PIPELINE_STEP_LABELS = { "detect_register": "注册状态检测", "detect_baidu_site": "百度site检测", "detect_360_site": "360 site检测", "detect_chinaz": "站长之家检测", "detect_aizhan": "爱站检测", "detect_wayback": "时光机检测", "detect_jucha": "聚查检测", "detect_juziseo": "桔子检测", } _PIPELINE_STEP_RETRY_LIMITS = { "default": 3, "detect_wayback": 2, } _SINGLE_STEP_JOB_SPECS = { "detect_baidu_site": { "step_code": "detect_baidu_site", "task_mode": "single_step", "label": "百度site检测", "source": "api-step-start", }, "detect_wayback": { "step_code": "detect_wayback", "task_mode": "single_step", "label": "时光机检测", "source": "api-step-start", }, } def _selection_sql() -> str: return """ SELECT id FROM domains WHERE detect_status IN (0, 4) OR (use_status = 0 AND detect_status = 1 AND register_status = 3 AND expire_date < CURRENT_DATE) ORDER BY id ASC LIMIT %s """ def _format_time(value: datetime | None) -> str: return value.isoformat(sep=" ", timespec="seconds") if value else "" def _normalize_sync_region(value: str | None, fallback: str) -> str: text = str(value or "").strip() if not text or text == "unknown": return str(fallback or "unknown").strip() or "unknown" return text def normalize_detect_step_code(step_code: object) -> str: normalized = str(step_code or "").strip() if not normalized: return "" return normalized if normalized in _SINGLE_STEP_JOB_SPECS else "" def resolve_detect_job_definition(step_code: object = None) -> dict: normalized_step_code = normalize_detect_step_code(step_code) if normalized_step_code: spec = dict(_SINGLE_STEP_JOB_SPECS[normalized_step_code]) spec["is_single_step"] = True return spec return { "step_code": "", "task_mode": _DEFAULT_TASK_MODE, "label": "全流程检测", "source": "api-start", "is_single_step": False, } def _step_label(step_code: str) -> str: return _PIPELINE_STEP_LABELS.get(str(step_code or "").strip(), str(step_code or "").strip() or "unknown") def _json_state(value: object) -> str: if isinstance(value, dict): return str(value.get("state") or "").strip().lower() return "" def _pipeline_retry_limit(step_code: str) -> int: return int(_PIPELINE_STEP_RETRY_LIMITS.get(str(step_code or "").strip(), _PIPELINE_STEP_RETRY_LIMITS["default"]) or 1) def _classify_pipeline_item_outcome( *, item_status: str, result_payload: dict | None, step_code: str, attempt_count: int, ) -> dict: normalized_item_status = str(item_status or "").strip().lower() payload = dict(result_payload or {}) result_state = str(payload.get("state") or "").strip().lower() retry_limit = _pipeline_retry_limit(step_code) retry_recommended = bool(payload.get("retry_recommended")) legacy_retry_candidate = normalized_item_status == "failed" and not result_state if normalized_item_status == "blacklisted" or result_state == "blacklisted": return { "action": "black_hit", "result_state": result_state or "blacklisted", "retry_limit": retry_limit, "should_retry": False, "reason_code": "blacklisted", } should_retry = ( result_state in {"degraded", "error"} or retry_recommended or legacy_retry_candidate ) if should_retry: return { "action": "retry" if int(attempt_count or 0) < retry_limit else "reject", "result_state": result_state or ("degraded" if retry_recommended else "failed"), "retry_limit": retry_limit, "should_retry": int(attempt_count or 0) < retry_limit, "reason_code": "external_retry" if int(attempt_count or 0) < retry_limit else "retry_exhausted", } if normalized_item_status == "failed" or result_state in {"failed", "rejected", "reject"}: return { "action": "reject", "result_state": result_state or "failed", "retry_limit": retry_limit, "should_retry": False, "reason_code": "business_reject", } return { "action": "pass", "result_state": result_state or "passed", "retry_limit": retry_limit, "should_retry": False, "reason_code": "passed", } def _load_pipeline_order(settings_payload: dict | None = None) -> list[str]: payload = settings_payload or get_settings_payload() detect_options = dict(payload.get("detect_options") or {}) configured_order = list(detect_options.get("detect_order") or []) normalized_order = [item for item in configured_order if item in _PIPELINE_STEP_ORDER] for item in _PIPELINE_STEP_ORDER: if item not in normalized_order: normalized_order.append(item) return [item for item in normalized_order if bool(detect_options.get(item, False))] def _load_domain_pipeline_snapshot(cur, domain_id: int) -> dict | None: cur.execute( """ SELECT d.id, d.domain, d.source_type, d.register_status, d.detect_status, d.use_status, d.expire_date, dd.baidu_site, dd.qihu360_site, dd.wayback_info, dd.chinaz_info, dd.aizhan_info, dd.jucha_info, dd.juziseo_info FROM domains AS d LEFT JOIN domain_detections AS dd ON dd.domain_id = d.id WHERE d.id = %s LIMIT 1 """, (int(domain_id),), ) row = cur.fetchone() if not row: return None return { "id": int(row[0]), "domain": row[1] or "", "source_type": int(row[2] or 0), "register_status": int(row[3] or 0), "detect_status": int(row[4] or 0), "use_status": int(row[5] or 0), "expire_date": _format_time(row[6]) if row[6] else "", "baidu_site": _decode_payload(row[7]), "qihu360_site": _decode_payload(row[8]), "wayback_info": _decode_payload(row[9]), "chinaz_info": _decode_payload(row[10]), "aizhan_info": _decode_payload(row[11]), "jucha_info": _decode_payload(row[12]), "juziseo_info": _decode_payload(row[13]), } def _is_pipeline_step_completed(domain_snapshot: dict, step_code: str) -> bool: normalized_step_code = str(step_code or "").strip() if normalized_step_code == "detect_register": return int(domain_snapshot.get("register_status", 0) or 0) not in {0, 10} field_name = _PIPELINE_STEP_FIELDS.get(normalized_step_code) if not field_name: return False state = _json_state(domain_snapshot.get(field_name)) return state in {"passed", "blacklisted"} def _should_skip_pipeline_step(domain_snapshot: dict, step_code: str) -> bool: normalized_step_code = str(step_code or "").strip() if normalized_step_code == "detect_register" and int(domain_snapshot.get("source_type", 0) or 0) == 1: return True return False def resolve_domain_pipeline_step(domain_snapshot: dict, *, settings_payload: dict | None = None, after_step_code: str | None = None) -> str: pipeline_order = _load_pipeline_order(settings_payload=settings_payload) if not pipeline_order: return "" normalized_after = str(after_step_code or "").strip() seen_after = not normalized_after for step_code in pipeline_order: if not seen_after: if step_code == normalized_after: seen_after = True continue if _should_skip_pipeline_step(domain_snapshot, step_code): continue if _is_pipeline_step_completed(domain_snapshot, step_code): continue return step_code return "" def resolve_initial_domain_pipeline_item( domain_snapshot: dict, *, settings_payload: dict | None = None, retry_count: int = 0, ) -> tuple[str, dict | None]: step_code = resolve_domain_pipeline_step(domain_snapshot, settings_payload=settings_payload) if not step_code: return "", None return step_code, _build_step_payload( step_code=step_code, domain_snapshot=domain_snapshot, settings_payload=settings_payload, retry_count=retry_count, ) def _build_step_payload(*, step_code: str, domain_snapshot: dict, settings_payload: dict | None = None, retry_count: int = 0) -> dict: pipeline_order = _load_pipeline_order(settings_payload=settings_payload) payload = { "step_code": str(step_code or "").strip(), "step_name": _step_label(step_code), "pipeline_order": pipeline_order, "retry_count": int(retry_count or 0), "domain": str(domain_snapshot.get("domain") or "").strip(), "source_type": int(domain_snapshot.get("source_type", 0) or 0), } if str(step_code or "").strip() == "detect_wayback": payload.update( { "wayback_strategy": "recent_years", "wayback_recent_years": 5, "wayback_stop_on_first_hit": True, } ) return payload def _decode_payload(value: object) -> dict: if isinstance(value, dict): return value if value in (None, ""): return {} try: return json.loads(value) except Exception: return {} def _upsert_domain_detection_field(cur, domain_id: int, field_name: str, payload: dict) -> bool: normalized_field_name = str(field_name or "").strip() if not normalized_field_name or normalized_field_name not in _PIPELINE_STEP_FIELDS.values(): return False normalized_payload = dict(payload or {}) cur.execute("SELECT id FROM domain_detections WHERE domain_id = %s LIMIT 1", (int(domain_id),)) existing = cur.fetchone() payload_json = json.dumps(normalized_payload, ensure_ascii=False) if existing: cur.execute( f""" UPDATE domain_detections SET {normalized_field_name} = %s::jsonb, update_time = NOW() WHERE domain_id = %s """, (payload_json, int(domain_id)), ) return True cur.execute( f""" INSERT INTO domain_detections (domain_id, {normalized_field_name}, create_time, update_time) VALUES (%s, %s::jsonb, NOW(), NOW()) """, (int(domain_id), payload_json), ) return True def _int_value(value: object) -> int: try: return int(value or 0) except Exception: return 0 def _effective_runtime_load(*, items_running: object, active_threads: object) -> int: return max(_int_value(items_running), _int_value(active_threads), 0) def _control_node_supports_worker(*, region: object, metadata: dict | None) -> bool: normalized_region = str(region or "").strip() runtime_metadata = dict(metadata or {}) active_threads = int(runtime_metadata.get("active_threads", 0) or 0) max_threads = int(runtime_metadata.get("max_threads", 0) or 0) if normalized_region != "mainland": return False return bool( runtime_metadata.get("worker_online", False) or runtime_metadata.get("detect_participating", False) or active_threads > 0 or max_threads > 0 ) def _build_runtime_display_bucket(row: tuple) -> dict | None: node_code = str(row[0] or "").strip() if not node_code: return None region = str(row[1] or "").strip() role = str(row[2] or "").strip() metadata = _decode_payload(row[5]) if role == "control" and not _control_node_supports_worker(region=region, metadata=metadata): return None items_total = _int_value(metadata.get("job_items_total")) items_claimed = _int_value(metadata.get("job_items_claimed")) items_running = _int_value(metadata.get("job_items_running")) items_completed = _int_value(metadata.get("job_items_completed")) items_failed = _int_value(metadata.get("job_items_failed")) raw_current_load = _int_value(row[4]) active_threads = _int_value(metadata.get("active_threads")) max_threads = _int_value(metadata.get("max_threads")) current_load = _effective_runtime_load(items_running=items_running, active_threads=active_threads) detect_participating = bool(metadata.get("detect_participating", False)) if ( settings.node_region == "overseas" and settings.node_role == "control" and node_code == str(settings.node_code or "").strip() and not detect_participating ): # Overseas control nodes may report their own control-plane load into # detect_worker_nodes; that load should never be counted as worker execution. return None if ( items_total <= 0 and items_claimed <= 0 and items_running <= 0 and items_completed <= 0 and items_failed <= 0 and current_load <= 0 and raw_current_load <= 0 and active_threads <= 0 and not detect_participating ): return None items_pending = max(items_total - items_claimed - items_running - items_completed - items_failed, 0) return { "node_code": node_code, "items_total": items_total, "items_pending": items_pending, "items_claimed": items_claimed, "items_running": items_running, "display_running": current_load, "items_completed": items_completed, "items_blacklisted": 0, "items_failed": items_failed, "metrics_source": str(metadata.get("service") or "runtime").strip() or "runtime", "region": region, "role": role, "status": str(row[3] or "").strip(), "current_load": current_load, "active_threads": active_threads, "max_threads": max_threads, "last_heartbeat_at": _format_time(row[6]) if row[6] else "", } def _merge_display_node_stats(*, local_node_stats: list[dict], runtime_node_rows: list[tuple]) -> list[dict]: merged: dict[str, dict] = {} def _ensure_bucket(node_code: str) -> dict: return merged.setdefault( node_code, { "node_code": node_code, "items_total": 0, "items_pending": 0, "items_claimed": 0, "items_running": 0, "display_running": 0, "items_completed": 0, "items_blacklisted": 0, "items_failed": 0, "metrics_source": "", "region": "", "role": "", "status": "", "current_load": 0, "active_threads": 0, "max_threads": 0, "last_heartbeat_at": "", }, ) for item in list(local_node_stats or []): node_code = str(item.get("node_code") or "").strip() if not node_code: continue bucket = _ensure_bucket(node_code) for key in ( "items_total", "items_pending", "items_claimed", "items_running", "display_running", "items_completed", "items_blacklisted", "items_failed", ): bucket[key] = max(_int_value(bucket.get(key)), _int_value(item.get(key))) if node_code == "unassigned": bucket["metrics_source"] = "central_queue" for row in list(runtime_node_rows or []): runtime_bucket = _build_runtime_display_bucket(row) if not runtime_bucket: continue node_code = str(runtime_bucket.get("node_code") or "").strip() bucket = _ensure_bucket(node_code) for key in ( "items_total", "items_pending", "items_claimed", "items_running", "display_running", "items_completed", "items_blacklisted", "items_failed", "current_load", "active_threads", "max_threads", ): bucket[key] = max(_int_value(bucket.get(key)), _int_value(runtime_bucket.get(key))) for key in ("metrics_source", "region", "role", "status", "last_heartbeat_at"): if str(runtime_bucket.get(key) or "").strip(): bucket[key] = runtime_bucket.get(key) return sorted( merged.values(), key=lambda item: ( str(item.get("node_code") or "") == "unassigned", -_int_value(item.get("active_threads")), -_int_value(item.get("items_running")), -_int_value(item.get("items_claimed")), -_int_value(item.get("items_total")), str(item.get("node_code") or ""), ), ) def _load_runtime_display_rows(cur) -> list[tuple]: cur.execute( f""" SELECT node_code, region, role, status, current_load, metadata_json, last_heartbeat_at FROM detect_worker_nodes WHERE last_heartbeat_at >= CURRENT_TIMESTAMP - interval '{_RUNTIME_NODE_STALE_MINUTES} minutes' ORDER BY last_heartbeat_at DESC, node_code ASC """ ) return list(cur.fetchall()) def _build_display_summary(node_stats: list[dict]) -> dict: effective_nodes = [ item for item in list(node_stats or []) if str(item.get("node_code") or "").strip() and str(item.get("node_code") or "").strip() != "unassigned" and not ( settings.node_region == "overseas" and settings.node_role == "control" and str(item.get("node_code") or "").strip() == str(settings.node_code or "").strip() ) ] return { "items_claimed": sum(_int_value(item.get("items_claimed")) for item in effective_nodes), "items_running": sum(_int_value(item.get("items_running")) for item in effective_nodes), "display_running": sum( _effective_runtime_load( items_running=item.get("items_running"), active_threads=item.get("active_threads"), ) for item in effective_nodes ), "current_load": sum( _effective_runtime_load( items_running=item.get("items_running"), active_threads=item.get("active_threads"), ) for item in effective_nodes ), "active_threads": sum(_int_value(item.get("active_threads")) for item in effective_nodes), "max_threads": sum(_int_value(item.get("max_threads")) for item in effective_nodes), "items_completed": sum(_int_value(item.get("items_completed")) for item in effective_nodes), "items_failed": sum(_int_value(item.get("items_failed")) for item in effective_nodes), "active_nodes": [ str(item.get("node_code") or "").strip() for item in effective_nodes if _effective_runtime_load( items_running=item.get("items_running"), active_threads=item.get("active_threads"), ) > 0 ], } def _normalize_node_bucket(item: dict) -> dict: node_code = str(item.get("node_code") or "").strip() items_running = _int_value(item.get("items_running")) active_threads = _int_value(item.get("active_threads")) max_threads = _int_value(item.get("max_threads")) current_load = _effective_runtime_load(items_running=items_running, active_threads=active_threads) return { "node_code": node_code, "items_total": _int_value(item.get("items_total")), "items_pending": _int_value(item.get("items_pending")), "items_claimed": _int_value(item.get("items_claimed")), "items_running": items_running, "display_running": current_load, "items_completed": _int_value(item.get("items_completed")), "items_blacklisted": _int_value(item.get("items_blacklisted")), "items_failed": _int_value(item.get("items_failed")), "metrics_source": str(item.get("metrics_source") or "").strip(), "region": str(item.get("region") or "").strip(), "role": str(item.get("role") or "").strip(), "status": str(item.get("status") or "").strip(), "current_load": current_load, "active_threads": active_threads, "max_threads": max_threads, "last_heartbeat_at": str(item.get("last_heartbeat_at") or "").strip(), } def _build_effective_node_stats( *, distributed_node_stats: list[dict], raw_items_total: int, ) -> list[dict]: normalized_total = max(0, _int_value(raw_items_total)) assigned_buckets: list[dict] = [] assigned_total = 0 unassigned_bucket: dict | None = None for item in list(distributed_node_stats or []): bucket = _normalize_node_bucket(item) if not bucket["node_code"]: continue if bucket["node_code"] == "unassigned": unassigned_bucket = bucket continue if ( settings.node_region == "overseas" and settings.node_role == "control" and bucket["node_code"] == str(settings.node_code or "").strip() ): continue assigned_buckets.append(bucket) assigned_total += bucket["items_total"] if unassigned_bucket and unassigned_bucket["items_total"] > 0: assigned_buckets.append(unassigned_bucket) assigned_total += unassigned_bucket["items_total"] effective_total = max(normalized_total, assigned_total) remainder_total = max(effective_total - assigned_total, 0) if remainder_total > 0: if unassigned_bucket: unassigned_bucket["items_total"] += remainder_total unassigned_bucket["items_pending"] += remainder_total assigned_buckets = [ unassigned_bucket if str(item.get("node_code") or "").strip() == "unassigned" else item for item in assigned_buckets ] else: assigned_buckets.append( { "node_code": "unassigned", "items_total": remainder_total, "items_pending": remainder_total, "items_claimed": 0, "items_running": 0, "display_running": 0, "items_completed": 0, "items_blacklisted": 0, "items_failed": 0, "metrics_source": "central_queue", "region": "", "role": "", "status": "", "current_load": 0, "active_threads": 0, "max_threads": 0, "last_heartbeat_at": "", } ) return sorted( assigned_buckets, key=lambda item: ( str(item.get("node_code") or "") == "unassigned", -_int_value(item.get("active_threads")), -_int_value(item.get("items_running")), -_int_value(item.get("items_claimed")), -_int_value(item.get("items_total")), str(item.get("node_code") or ""), ), ) def _build_effective_summary( *, node_stats: list[dict], raw_items_total: int, raw_items_pending: int, raw_items_claimed: int, raw_items_running: int, raw_items_completed: int, raw_items_blacklisted: int, raw_items_failed: int, ) -> dict: effective_total = max(0, _int_value(raw_items_total)) effective_pending = max(0, _int_value(raw_items_pending)) effective_claimed = max(0, _int_value(raw_items_claimed)) effective_running = max(0, _int_value(raw_items_running)) effective_completed = max(0, _int_value(raw_items_completed)) effective_blacklisted = max(0, _int_value(raw_items_blacklisted)) effective_failed = max(0, _int_value(raw_items_failed)) effective_terminal = effective_completed + effective_blacklisted + effective_failed return { "items_total": effective_total, "items_pending": effective_pending, "items_claimed": effective_claimed, "items_running": effective_running, "items_completed": effective_completed, "items_blacklisted": effective_blacklisted, "items_failed": effective_failed, "items_terminal": max(effective_terminal, 0), } def _build_step_bucket(step_code: str) -> dict: normalized_step_code = str(step_code or "").strip() return { "step_code": normalized_step_code, "step_name": _step_label(normalized_step_code), "items_total": 0, "items_pending": 0, "items_claimed": 0, "items_running": 0, "items_completed": 0, "items_blacklisted": 0, "items_failed": 0, "processed_recent": 0, "processed_per_minute": 0, "started_recent": 0, "completed_recent": 0, "blacklisted_recent": 0, "failed_recent": 0, } def _extract_current_cycle_events(events: list[dict]) -> tuple[str, list[dict]]: if not events: return "", [] latest_payload = events[0].get("payload") or {} latest_cycle_token = str(latest_payload.get("cycle_token") or "").strip() if latest_cycle_token: current_cycle = [] for event in events: payload = event.get("payload") or {} event_cycle = str(payload.get("cycle_token") or "").strip() if event_cycle == latest_cycle_token: current_cycle.append(event) return latest_cycle_token, current_cycle cycle_token = "" anchor_index = -1 for index, event in enumerate(events): payload = event.get("payload") or {} event_type = str(event.get("event_type") or "") candidate = str(payload.get("cycle_token") or "").strip() if candidate and event_type in {"job_dispatch_sent", "job_dispatch_requested", "job_dispatch_failed", "job_dispatch_rejected"}: cycle_token = candidate anchor_index = index break if not cycle_token: return "", events current_cycle = [] for index, event in enumerate(events[: anchor_index + 1]): payload = event.get("payload") or {} event_cycle = str(payload.get("cycle_token") or "").strip() if index == anchor_index or event_cycle == cycle_token: current_cycle.append(event) return cycle_token, current_cycle def _classify_runtime_debug_event(*, event_type: str, message: str, payload: dict) -> dict: normalized_event_type = str(event_type or "").strip() normalized_message = str(message or "").strip() normalized_payload = payload if isinstance(payload, dict) else {} normalized_job_code = str(normalized_payload.get("job_code") or "").strip() normalized_node_code = str(normalized_payload.get("node_code") or "").strip() result = { "job_code": normalized_job_code, "node_code": normalized_node_code, "started": False, "terminal": False, "terminal_status": "", "step_code": "", } if normalized_event_type in {"domain_started", "domain_completed", "domain_failed", "domain_blacklisted"}: result["started"] = normalized_event_type == "domain_started" result["terminal"] = normalized_event_type in {"domain_completed", "domain_failed", "domain_blacklisted"} terminal_map = { "domain_completed": "completed", "domain_failed": "failed", "domain_blacklisted": "blacklisted", } result["terminal_status"] = terminal_map.get(normalized_event_type, "") result["step_code"] = normalize_detect_step_code(normalized_payload.get("detect_key")) or "detect_register" return result if normalized_event_type != "worker_log": return result step_markers = ( ("detect_register", ("| step=注册状态检测 |", "detect_register")), ("detect_baidu_site", ("| step=百度site检测 |", "detect_baidu_site", "百度阶段", "detect.baidu")), ("detect_360_site", ("| step=360收录检测 |", "detect_360_site", "360阶段", "detect.c360")), ("detect_chinaz", ("| step=站长之家检测 |", "detect_chinaz", "站长之家", "detect.chinaz")), ("detect_aizhan", ("| step=爱站检测 |", "detect_aizhan", "爱站", "detect.aizhan")), ("detect_wayback", ("| step=时光机检测 |", "detect_wayback", "时光机检测")), ("detect_jucha", ("| step=聚查检测 |", "detect_jucha", "聚查", "detect.jucha")), ("detect_juziseo", ("| step=桔子检测 |", "detect_juziseo", "桔子", "juziseo")), ) for step_code, markers in step_markers: if any(marker in normalized_message for marker in markers): result["step_code"] = step_code break if "| stage=started |" in normalized_message or "开始检测域名:" in normalized_message: result["started"] = True return result if "| stage=single_step_finalized |" in normalized_message: result["terminal"] = True if "final_status=completed" in normalized_message: result["terminal_status"] = "completed" elif "final_status=failed" in normalized_message: result["terminal_status"] = "failed" elif "final_status=blacklisted" in normalized_message: result["terminal_status"] = "blacklisted" return result return result def _load_runtime_activity_snapshot(window_minutes: int) -> dict: safe_window_minutes = max(5, min(int(window_minutes or 15), 120)) snapshot = { "processed_recent": 0, "completed_recent": 0, "failed_recent": 0, "blacklisted_recent": 0, "focus_job_code": "", "job_codes": [], "step_code": "detect_register", "step_stats": {}, "nodes": {}, } if not (settings.node_region == "overseas" and settings.node_role == "control"): return snapshot if settings.node_region == "overseas" and settings.node_role == "control": source_region = "mainland" else: source_region = _normalize_sync_region(settings.sync_source_region, settings.node_region) with get_db() as conn: with conn.cursor() as cur: cur.execute( """ SELECT COALESCE(NULLIF(node_code, ''), 'unknown') AS node_code, event_type, message, payload_json, created_at FROM detect_debug_events WHERE source_region = %s AND created_at >= CURRENT_TIMESTAMP - (%s || ' minutes')::interval AND event_type IN ('worker_log', 'domain_started', 'domain_completed', 'domain_failed', 'domain_blacklisted') ORDER BY created_at DESC, id DESC LIMIT 2000 """, (source_region, safe_window_minutes), ) rows = cur.fetchall() job_activity: dict[str, dict] = {} node_stats: dict[str, dict] = {} step_activity: dict[str, dict] = {} for row in rows: node_code = str(row[0] or "unknown").strip() or "unknown" event_type = str(row[1] or "").strip() message = str(row[2] or "").strip() payload = _decode_payload(row[3]) created_at = _format_time(row[4]) classified = _classify_runtime_debug_event(event_type=event_type, message=message, payload=payload) job_code = str(classified.get("job_code") or "").strip() step_code = str(classified.get("step_code") or "").strip() or "detect_register" if job_code: bucket = job_activity.setdefault( job_code, { "job_code": job_code, "activity_count": 0, "started_count": 0, "terminal_count": 0, "latest_at": "", }, ) bucket["activity_count"] += 1 if classified.get("started"): bucket["started_count"] += 1 if classified.get("terminal"): bucket["terminal_count"] += 1 if created_at > str(bucket.get("latest_at") or ""): bucket["latest_at"] = created_at if classified.get("terminal"): snapshot["processed_recent"] += 1 step_bucket = step_activity.setdefault( step_code, { "step_code": step_code, "started_recent": 0, "processed_recent": 0, "completed_recent": 0, "failed_recent": 0, "blacklisted_recent": 0, }, ) step_bucket["processed_recent"] += 1 node_bucket = node_stats.setdefault( node_code, { "node_code": node_code, "processed_recent": 0, "completed_recent": 0, "failed_recent": 0, "blacklisted_recent": 0, }, ) node_bucket["processed_recent"] += 1 terminal_status = str(classified.get("terminal_status") or "").strip() if terminal_status == "completed": snapshot["completed_recent"] += 1 node_bucket["completed_recent"] += 1 step_bucket["completed_recent"] += 1 elif terminal_status == "failed": snapshot["failed_recent"] += 1 node_bucket["failed_recent"] += 1 step_bucket["failed_recent"] += 1 elif terminal_status == "blacklisted": snapshot["blacklisted_recent"] += 1 node_bucket["blacklisted_recent"] += 1 step_bucket["blacklisted_recent"] += 1 if classified.get("started"): step_bucket = step_activity.setdefault( step_code, { "step_code": step_code, "started_recent": 0, "processed_recent": 0, "completed_recent": 0, "failed_recent": 0, "blacklisted_recent": 0, }, ) step_bucket["started_recent"] += 1 if step_activity: ordered_steps = sorted( step_activity.values(), key=lambda item: ( -int(item.get("processed_recent", 0) or 0), -int(item.get("started_recent", 0) or 0), str(item.get("step_code") or ""), ), ) snapshot["step_code"] = str((ordered_steps[0] or {}).get("step_code") or "detect_register") snapshot["step_stats"] = { str(item.get("step_code") or ""): { "step_code": str(item.get("step_code") or ""), "started_recent": int(item.get("started_recent", 0) or 0), "processed_recent": int(item.get("processed_recent", 0) or 0), "completed_recent": int(item.get("completed_recent", 0) or 0), "failed_recent": int(item.get("failed_recent", 0) or 0), "blacklisted_recent": int(item.get("blacklisted_recent", 0) or 0), } for item in ordered_steps if str(item.get("step_code") or "").strip() } if job_activity: ordered_jobs = sorted( job_activity.values(), key=lambda item: ( -int(item.get("terminal_count", 0) or 0), -int(item.get("activity_count", 0) or 0), str(item.get("latest_at") or ""), str(item.get("job_code") or ""), ), ) snapshot["focus_job_code"] = str(ordered_jobs[0].get("job_code") or "") snapshot["job_codes"] = [str(item.get("job_code") or "") for item in ordered_jobs[:5] if str(item.get("job_code") or "")] snapshot["nodes"] = node_stats return snapshot @db_read_retry() def _load_latest_runtime_active_job_snapshot(window_minutes: int) -> dict: safe_window_minutes = max(5, min(int(window_minutes or 15), 120)) if not (settings.node_region == "overseas" and settings.node_role == "control"): return {} source_region = "mainland" with get_db() as conn: with conn.cursor() as cur: cur.execute( """ SELECT payload_json, created_at FROM detect_debug_events WHERE source_region = %s AND service = 'detect-runtime' AND event_type = 'active_job_snapshot' AND created_at >= CURRENT_TIMESTAMP - (%s || ' minutes')::interval ORDER BY created_at DESC, id DESC LIMIT 1 """, (source_region, safe_window_minutes), ) row = cur.fetchone() if not row: return {} payload = _decode_payload(row[0]) if not isinstance(payload, dict): return {} payload["_created_at"] = _format_time(row[1]) return payload def _select_runtime_snapshot_events( runtime_snapshot: dict, *, job_code: str = "", job_id: int = 0, event_limit: int = 20, ) -> list[dict]: safe_limit = max(1, min(int(event_limit or 20), 100)) target_job_code = str(job_code or "").strip() target_job_id = int(job_id or 0) selected: list[dict] = [] for raw_event in list(runtime_snapshot.get("recent_events") or []): if not isinstance(raw_event, dict): continue payload = raw_event.get("payload") normalized_payload = payload if isinstance(payload, dict) else _decode_payload(payload) event_job_code = str(normalized_payload.get("job_code") or "").strip() event_job_id = _int_value(raw_event.get("job_id")) if target_job_code: if event_job_code == target_job_code or (target_job_id > 0 and event_job_id == target_job_id): selected.append( { "node_code": str(raw_event.get("node_code") or "").strip(), "event_type": str(raw_event.get("event_type") or "").strip(), "level": str(raw_event.get("level") or "info").strip() or "info", "message": str(raw_event.get("message") or "").strip(), "payload": normalized_payload, "created_at": str(raw_event.get("created_at") or "").strip(), } ) else: selected.append( { "node_code": str(raw_event.get("node_code") or "").strip(), "event_type": str(raw_event.get("event_type") or "").strip(), "level": str(raw_event.get("level") or "info").strip() or "info", "message": str(raw_event.get("message") or "").strip(), "payload": normalized_payload, "created_at": str(raw_event.get("created_at") or "").strip(), } ) if len(selected) >= safe_limit: break return selected def _filter_events_for_job(events: list[dict], *, job_code: str = "", job_id: int = 0, limit: int = 20) -> list[dict]: target_job_code = str(job_code or "").strip() target_job_id = int(job_id or 0) safe_limit = max(1, min(int(limit or 20), 100)) filtered: list[dict] = [] for raw_event in list(events or []): if not isinstance(raw_event, dict): continue payload = raw_event.get("payload") normalized_payload = payload if isinstance(payload, dict) else _decode_payload(payload) event_job_code = str(normalized_payload.get("job_code") or "").strip() event_job_id = _int_value(raw_event.get("job_id")) if target_job_code: if event_job_code != target_job_code and (target_job_id <= 0 or event_job_id != target_job_id): continue filtered.append( { "node_code": str(raw_event.get("node_code") or "").strip(), "event_type": str(raw_event.get("event_type") or "").strip(), "level": str(raw_event.get("level") or "info").strip() or "info", "message": str(raw_event.get("message") or "").strip(), "payload": normalized_payload, "created_at": str(raw_event.get("created_at") or "").strip(), } ) if len(filtered) >= safe_limit: break return filtered def _fetch_job_summary(cur, job_row, event_limit: int = 20) -> dict: job_id = job_row[0] cur.execute( """ SELECT status, count(*) FROM detect_job_items WHERE job_id = %s GROUP BY status """, (job_id,), ) counts = {status: int(count) for status, count in cur.fetchall()} cur.execute( """ SELECT COALESCE(NULLIF(claimed_by, ''), 'unassigned') AS node_code, status, count(*) FROM detect_job_items WHERE job_id = %s GROUP BY COALESCE(NULLIF(claimed_by, ''), 'unassigned'), status ORDER BY node_code ASC, status ASC """, (job_id,), ) node_buckets: dict[str, dict] = {} for node_code, status, count in cur.fetchall(): bucket = node_buckets.setdefault( node_code, { "node_code": node_code, "items_total": 0, "items_pending": 0, "items_claimed": 0, "items_running": 0, "items_completed": 0, "items_blacklisted": 0, "items_failed": 0, }, ) field_name = f"items_{status}" if field_name in bucket: bucket[field_name] += int(count) bucket["items_total"] += int(count) cur.execute( """ SELECT COALESCE(NULLIF(step_code, ''), 'domain_pipeline') AS step_code, status, count(*) FROM detect_job_items WHERE job_id = %s GROUP BY COALESCE(NULLIF(step_code, ''), 'domain_pipeline'), status ORDER BY step_code ASC, status ASC """, (job_id,), ) step_buckets: dict[str, dict] = {} for step_code, status, count in cur.fetchall(): bucket = step_buckets.setdefault(str(step_code or ""), _build_step_bucket(str(step_code or ""))) field_name = f"items_{status}" if field_name in bucket: bucket[field_name] += int(count) bucket["items_total"] += int(count) cur.execute( """ SELECT node_code, event_type, level, message, payload_json, created_at FROM detect_run_events WHERE job_id = %s ORDER BY created_at DESC, id DESC LIMIT %s """, (job_id, max(1, int(event_limit or 20))), ) events = [ { "node_code": item[0], "event_type": item[1], "level": item[2], "message": item[3], "payload": _decode_payload(item[4]), "created_at": _format_time(item[5]), } for item in cur.fetchall() ] cycle_token, current_cycle_events = _extract_current_cycle_events(events) distributed_node_stats = _merge_display_node_stats( local_node_stats=list(node_buckets.values()), runtime_node_rows=_load_runtime_display_rows(cur), ) raw_total = sum(counts.values()) raw_pending = int(counts.get("pending", 0)) raw_claimed = int(counts.get("claimed", 0)) raw_running = int(counts.get("running", 0)) raw_completed = int(counts.get("completed", 0)) raw_blacklisted = int(counts.get("blacklisted", 0)) raw_failed = int(counts.get("failed", 0)) raw_terminal = raw_completed + raw_blacklisted + raw_failed effective_node_stats = _build_effective_node_stats( distributed_node_stats=distributed_node_stats, raw_items_total=raw_total, ) effective_summary = _build_effective_summary( node_stats=effective_node_stats, raw_items_total=raw_total, raw_items_pending=raw_pending, raw_items_claimed=raw_claimed, raw_items_running=raw_running, raw_items_completed=raw_completed, raw_items_blacklisted=raw_blacklisted, raw_items_failed=raw_failed, ) display_summary = _build_display_summary(effective_node_stats) return { "job_id": job_id, "job_code": job_row[1], "source": job_row[2], "task_mode": job_row[3] or _DEFAULT_TASK_MODE, "step_code": job_row[4] or "", "status": job_row[5], "created_by": job_row[6], "created_at": _format_time(job_row[7]), "started_at": _format_time(job_row[8]), "finished_at": _format_time(job_row[9]), "items_total": int(effective_summary.get("items_total", 0) or 0), "items_pending": int(effective_summary.get("items_pending", 0) or 0), "items_claimed": int(effective_summary.get("items_claimed", 0) or 0), "items_running": int(effective_summary.get("items_running", 0) or 0), "items_completed": int(effective_summary.get("items_completed", 0) or 0), "items_blacklisted": int(effective_summary.get("items_blacklisted", 0) or 0), "items_failed": int(effective_summary.get("items_failed", 0) or 0), "items_terminal": int(effective_summary.get("items_terminal", 0) or 0), "progress_percent": round( (int(effective_summary.get("items_terminal", 0) or 0) / int(effective_summary.get("items_total", 0) or 0)) * 100, 2, ) if int(effective_summary.get("items_total", 0) or 0) else 0, "raw_items_total": raw_total, "raw_items_pending": raw_pending, "raw_items_claimed": raw_claimed, "raw_items_running": raw_running, "raw_items_completed": raw_completed, "raw_items_blacklisted": raw_blacklisted, "raw_items_failed": raw_failed, "raw_items_terminal": raw_terminal, "raw_node_stats": list(node_buckets.values()), "raw_step_stats": list(step_buckets.values()), "node_stats": effective_node_stats, "distributed_node_stats": effective_node_stats, "step_stats": sorted( step_buckets.values(), key=lambda item: ( -_int_value(item.get("items_running")), -_int_value(item.get("items_pending")), str(item.get("step_code") or ""), ), ), "display_items_claimed": int(display_summary.get("items_claimed", 0) or 0), "display_items_running": int(display_summary.get("display_running", display_summary.get("items_running", 0)) or 0), "display_current_load": int(display_summary.get("current_load", 0) or 0), "display_active_threads": int(display_summary.get("active_threads", 0) or 0), "display_max_threads": int(display_summary.get("max_threads", 0) or 0), "display_items_completed": int(display_summary.get("items_completed", 0) or 0), "display_items_failed": int(display_summary.get("items_failed", 0) or 0), "display_active_node_codes": list(display_summary.get("active_nodes") or []), "recent_events": events, "latest_event": events[0] if events else None, "current_cycle_token": cycle_token, "current_cycle_events": current_cycle_events, "latest_cycle_event": current_cycle_events[0] if current_cycle_events else None, } def _enrich_active_job_summary_with_runtime( summary: dict | None, *, event_limit: int = 20, window_minutes: int = 15, runtime_activity: dict | None = None, runtime_snapshot: dict | None = None, recent_domain_events: list[dict] | None = None, ) -> dict | None: if not summary: return summary enriched = dict(summary) runtime_activity = dict(runtime_activity or _load_runtime_activity_snapshot(window_minutes)) runtime_snapshot = dict(runtime_snapshot or _load_latest_runtime_active_job_snapshot(window_minutes)) runtime_job_code = str(runtime_activity.get("focus_job_code") or "").strip() runtime_job_codes = [ str(item or "").strip() for item in list(runtime_activity.get("job_codes") or []) if str(item or "").strip() ] if runtime_job_code: enriched["runtime_job_code"] = runtime_job_code if runtime_job_codes: enriched["runtime_job_codes"] = runtime_job_codes enriched["processed_recent"] = max( int(enriched.get("processed_recent", 0) or 0), int(runtime_activity.get("processed_recent", 0) or 0), ) enriched["processed_per_minute"] = max( float(enriched.get("processed_per_minute", 0) or 0), round(int(runtime_activity.get("processed_recent", 0) or 0) / max(1, int(window_minutes or 15)), 2), ) enriched["completed_recent"] = max( int(enriched.get("completed_recent", 0) or 0), int(runtime_activity.get("completed_recent", 0) or 0), ) enriched["failed_recent"] = max( int(enriched.get("failed_recent", 0) or 0), int(runtime_activity.get("failed_recent", 0) or 0), ) enriched["blacklisted_recent"] = max( int(enriched.get("blacklisted_recent", 0) or 0), int(runtime_activity.get("blacklisted_recent", 0) or 0), ) if recent_domain_events is not None: enriched["recent_domain_events"] = list(recent_domain_events) elif not enriched.get("recent_domain_events"): try: from app.services.sync_record_service import _collect_recent_domain_events enriched["recent_domain_events"] = _collect_recent_domain_events( enriched, limit=max(10, min(int(event_limit or 20), 60)), ) except Exception: enriched["recent_domain_events"] = [] if runtime_snapshot: runtime_job = dict(runtime_snapshot.get("job") or {}) runtime_queue = dict((runtime_snapshot.get("queue_health") or {}).get("queue") or {}) runtime_nodes = list((runtime_snapshot.get("queue_health") or {}).get("nodes") or []) runtime_steps = list((runtime_snapshot.get("queue_health") or {}).get("steps") or []) runtime_job_nodes = list(runtime_job.get("node_stats") or []) if runtime_job: runtime_snapshot_job_id = int(runtime_job.get("job_id", 0) or 0) enriched["runtime_snapshot_job_id"] = runtime_snapshot_job_id runtime_snapshot_job_code = str(runtime_job.get("job_code") or "").strip() enriched["runtime_snapshot_job_code"] = runtime_snapshot_job_code if runtime_snapshot_job_code: enriched["job_code"] = runtime_snapshot_job_code enriched["runtime_job_code"] = runtime_snapshot_job_code merged_runtime_codes = [runtime_snapshot_job_code] + [ item for item in list(enriched.get("runtime_job_codes") or []) if str(item or "").strip() != runtime_snapshot_job_code ] enriched["runtime_job_codes"] = merged_runtime_codes[:5] if runtime_snapshot_job_id > 0: enriched["job_id"] = runtime_snapshot_job_id runtime_status = str(runtime_job.get("status") or "").strip() if runtime_status: enriched["status"] = runtime_status if float(runtime_job.get("progress_percent", 0) or 0) > float(enriched.get("progress_percent", 0) or 0): enriched["progress_percent"] = float(runtime_job.get("progress_percent", 0) or 0) if runtime_queue: enriched["runtime_snapshot_queue"] = runtime_queue runtime_items_total = int(runtime_queue.get("items_total", 0) or 0) if runtime_items_total > 0: enriched["items_total"] = runtime_items_total enriched["items_pending"] = int(runtime_queue.get("pending", 0) or 0) enriched["items_claimed"] = int(runtime_queue.get("claimed", 0) or 0) enriched["items_running"] = int(runtime_queue.get("running", 0) or 0) enriched["items_completed"] = int(runtime_queue.get("completed", 0) or 0) enriched["items_blacklisted"] = int(runtime_queue.get("blacklisted", 0) or 0) enriched["items_failed"] = int(runtime_queue.get("failed", 0) or 0) enriched["items_terminal"] = int(runtime_queue.get("terminal", 0) or 0) enriched["display_items_claimed"] = int(runtime_queue.get("display_claimed", enriched.get("items_claimed", 0)) or 0) enriched["display_items_running"] = int(runtime_queue.get("display_running", enriched.get("items_running", 0)) or 0) if runtime_job_nodes: enriched["node_stats"] = list(runtime_job_nodes) enriched["distributed_node_stats"] = list(runtime_job_nodes) runtime_display_summary = _build_display_summary(list(runtime_job_nodes)) enriched["display_items_claimed"] = int(runtime_display_summary.get("items_claimed", enriched.get("display_items_claimed", 0)) or 0) enriched["display_items_running"] = int(runtime_display_summary.get("display_running", enriched.get("display_items_running", 0)) or 0) enriched["display_current_load"] = int(runtime_display_summary.get("current_load", enriched.get("display_current_load", 0)) or 0) enriched["display_active_threads"] = int(runtime_display_summary.get("active_threads", enriched.get("display_active_threads", 0)) or 0) enriched["display_max_threads"] = int(runtime_display_summary.get("max_threads", enriched.get("display_max_threads", 0)) or 0) enriched["display_items_completed"] = int(runtime_display_summary.get("items_completed", enriched.get("display_items_completed", 0)) or 0) enriched["display_items_failed"] = int(runtime_display_summary.get("items_failed", enriched.get("display_items_failed", 0)) or 0) enriched["display_active_node_codes"] = list(runtime_display_summary.get("active_nodes") or []) elif runtime_nodes: enriched["node_stats"] = list(runtime_nodes) enriched["distributed_node_stats"] = list(runtime_nodes) if runtime_steps: enriched["step_stats"] = list(runtime_steps) runtime_recent_events = _select_runtime_snapshot_events( runtime_snapshot, job_code=str(runtime_job.get("job_code") or "").strip(), job_id=int(runtime_job.get("job_id", 0) or 0), event_limit=event_limit, ) if runtime_recent_events: enriched["recent_events"] = runtime_recent_events current_cycle_token, current_cycle_events = _extract_current_cycle_events(runtime_recent_events) enriched["current_cycle_token"] = current_cycle_token enriched["current_cycle_events"] = current_cycle_events enriched["latest_event"] = runtime_recent_events[0] enriched["latest_cycle_event"] = current_cycle_events[0] if current_cycle_events else runtime_recent_events[0] aligned_job_code = str(enriched.get("job_code") or enriched.get("runtime_job_code") or "").strip() aligned_job_id = int(enriched.get("job_id", 0) or 0) aligned_recent_events = _filter_events_for_job( list(enriched.get("recent_events") or []), job_code=aligned_job_code, job_id=aligned_job_id, limit=event_limit, ) if list(enriched.get("recent_events") or []) and not aligned_recent_events: enriched["recent_events"] = [] enriched["current_cycle_token"] = "" enriched["current_cycle_events"] = [] enriched["latest_event"] = None enriched["latest_cycle_event"] = None elif aligned_recent_events: enriched["recent_events"] = aligned_recent_events current_cycle_token, current_cycle_events = _extract_current_cycle_events(aligned_recent_events) enriched["current_cycle_token"] = current_cycle_token enriched["current_cycle_events"] = current_cycle_events enriched["latest_event"] = aligned_recent_events[0] enriched["latest_cycle_event"] = current_cycle_events[0] if current_cycle_events else aligned_recent_events[0] return enriched @db_read_retry() def get_detect_job_summary(job_id: int, event_limit: int = 20) -> dict | None: with get_db() as conn: with conn.cursor() as cur: cur.execute( """ SELECT id, job_code, source, task_mode, step_code, status, created_by, created_at, started_at, finished_at FROM detect_jobs WHERE id = %s LIMIT 1 """, (int(job_id),), ) row = cur.fetchone() if not row: return None return _enrich_active_job_summary_with_runtime(_fetch_job_summary(cur, row, event_limit=event_limit), event_limit=event_limit) @db_read_retry() def get_active_detect_job_summary(event_limit: int = 20) -> dict | None: with get_db() as conn: with conn.cursor() as cur: cur.execute( """ SELECT id, job_code, source, task_mode, step_code, status, created_by, created_at, started_at, finished_at FROM detect_jobs WHERE status IN ('pending', 'running') ORDER BY CASE WHEN status = 'running' THEN 0 ELSE 1 END ASC, COALESCE(started_at, created_at) DESC, id DESC LIMIT 1 """ ) row = cur.fetchone() if not row: return None return _enrich_active_job_summary_with_runtime(_fetch_job_summary(cur, row, event_limit=event_limit), event_limit=event_limit) @db_read_retry() def get_latest_detect_job_summary( *, event_limit: int = 20, statuses: tuple[str, ...] | list[str] | None = None, recent_minutes: int | None = None, ) -> dict | None: normalized_statuses = [ str(item or "").strip() for item in list(statuses or []) if str(item or "").strip() ] if not normalized_statuses: normalized_statuses = ["pending", "running", "completed", "partial_failed", "failed"] where_clauses = ["status = ANY(%s)"] params: list[object] = [normalized_statuses] if recent_minutes is not None: safe_recent_minutes = max(1, min(int(recent_minutes or 1), 24 * 60)) where_clauses.append( """ COALESCE(finished_at, started_at, created_at) >= CURRENT_TIMESTAMP - (%s || ' minutes')::interval """ ) params.append(safe_recent_minutes) with get_db() as conn: with conn.cursor() as cur: cur.execute( f""" SELECT id, job_code, source, task_mode, step_code, status, created_by, created_at, started_at, finished_at FROM detect_jobs WHERE {' AND '.join(where_clauses)} ORDER BY CASE WHEN status = 'running' THEN 0 WHEN status = 'pending' THEN 1 WHEN status = 'completed' THEN 2 WHEN status = 'partial_failed' THEN 3 WHEN status = 'failed' THEN 4 ELSE 5 END ASC, COALESCE(finished_at, started_at, created_at) DESC, id DESC LIMIT 1 """, tuple(params), ) row = cur.fetchone() if not row: return None return _enrich_active_job_summary_with_runtime(_fetch_job_summary(cur, row, event_limit=event_limit), event_limit=event_limit) @db_read_retry() def get_latest_unprojected_detect_job_summary( *, event_limit: int = 20, statuses: tuple[str, ...] | list[str] | None = None, recent_minutes: int | None = None, source_region: str | None = None, target_region: str | None = None, ) -> dict | None: normalized_statuses = [ str(item or "").strip() for item in list(statuses or []) if str(item or "").strip() ] if not normalized_statuses: normalized_statuses = ["completed", "partial_failed", "failed"] normalized_source_region = _normalize_sync_region( source_region, _normalize_sync_region(settings.sync_source_region, settings.node_region), ) normalized_target_region = _normalize_sync_region( target_region, _normalize_sync_region(settings.sync_target_region, "overseas"), ) where_clauses = [ "job.status = ANY(%s)", """ NOT EXISTS ( SELECT 1 FROM detect_sync_records AS sync_record WHERE sync_record.sync_type = 'detect_result_projection' AND sync_record.source_region = %s AND sync_record.target_region = %s AND (sync_record.payload_json->'projection'->'job'->>'job_id') = job.id::text ) """, ] params: list[object] = [normalized_statuses, normalized_source_region, normalized_target_region] if recent_minutes is not None: safe_recent_minutes = max(1, min(int(recent_minutes or 1), 24 * 60)) where_clauses.append( """ COALESCE(job.finished_at, job.started_at, job.created_at) >= CURRENT_TIMESTAMP - (%s || ' minutes')::interval """ ) params.append(safe_recent_minutes) with get_db() as conn: with conn.cursor() as cur: cur.execute( f""" SELECT job.id, job.job_code, job.source, job.task_mode, job.step_code, job.status, job.created_by, job.created_at, job.started_at, job.finished_at FROM detect_jobs AS job WHERE {' AND '.join(where_clauses)} ORDER BY COALESCE(job.finished_at, job.started_at, job.created_at) DESC, job.id DESC LIMIT 1 """, tuple(params), ) row = cur.fetchone() if not row: return None return _enrich_active_job_summary_with_runtime(_fetch_job_summary(cur, row, event_limit=event_limit), event_limit=event_limit) def list_detect_jobs(limit: int = 20) -> list[dict]: with get_db() as conn: with conn.cursor() as cur: cur.execute( """ SELECT id, job_code, source, task_mode, step_code, status, created_by, created_at, started_at, finished_at FROM detect_jobs ORDER BY created_at DESC, id DESC LIMIT %s """, (max(1, min(int(limit or 20), 100)),), ) rows = cur.fetchall() return [_fetch_job_summary(cur, row, event_limit=10) for row in rows] def list_recent_detect_run_events(limit: int = 20) -> list[dict]: safe_limit = max(1, min(int(limit or 20), 200)) with get_db() as conn: with conn.cursor() as cur: cur.execute( """ SELECT job_id, node_code, event_type, level, message, payload_json, created_at FROM detect_run_events ORDER BY created_at DESC, id DESC LIMIT %s """, (safe_limit,), ) rows = cur.fetchall() return [ { "job_id": int(row[0]) if row[0] is not None else None, "node_code": row[1] or "", "event_type": row[2] or "", "level": row[3] or "info", "message": row[4] or "", "payload": _decode_payload(row[5]), "created_at": _format_time(row[6]), } for row in rows ] @db_read_retry() def get_detect_queue_health(window_minutes: int = 15) -> dict: window_minutes = max(5, min(int(window_minutes or 15), 120)) active_job = get_active_detect_job_summary(event_limit=10) runtime_activity = _load_runtime_activity_snapshot(window_minutes) runtime_snapshot = _load_latest_runtime_active_job_snapshot(window_minutes) if not active_job: return { "window_minutes": window_minutes, "has_active_job": False, "job": None, "queue": { "items_total": 0, "pending": 0, "claimed": 0, "running": 0, "completed": 0, "blacklisted": 0, "failed": 0, "terminal": 0, "terminal_percent": 0, "oldest_pending_at": "", "oldest_pending_age_minutes": 0, "nearest_lease_expiry_at": "", "overdue_leases": 0, "expiring_soon_leases": 0, }, "throughput": { "processed_recent": int(runtime_activity.get("processed_recent", 0) or 0), "processed_per_minute": round(int(runtime_activity.get("processed_recent", 0) or 0) / window_minutes, 2), "completed_recent": int(runtime_activity.get("completed_recent", 0) or 0), "blacklisted_recent": int(runtime_activity.get("blacklisted_recent", 0) or 0), "failed_recent": int(runtime_activity.get("failed_recent", 0) or 0), }, "nodes": [], "steps": [], "runtime_activity": runtime_activity, } job_id = int(active_job["job_id"]) with get_db() as conn: with conn.cursor() as cur: cur.execute( """ SELECT MIN(create_time) FILTER (WHERE status = 'pending') AS oldest_pending_at, MIN(lease_expires_at) FILTER (WHERE status IN ('claimed', 'running') AND lease_expires_at IS NOT NULL) AS nearest_lease_expiry_at, COUNT(*) FILTER (WHERE status IN ('claimed', 'running') AND lease_expires_at IS NOT NULL AND lease_expires_at < CURRENT_TIMESTAMP) AS overdue_leases, COUNT(*) FILTER ( WHERE status IN ('claimed', 'running') AND lease_expires_at IS NOT NULL AND lease_expires_at >= CURRENT_TIMESTAMP AND lease_expires_at < CURRENT_TIMESTAMP + interval '5 minutes' ) AS expiring_soon_leases FROM detect_job_items WHERE job_id = %s """, (job_id,), ) lease_row = cur.fetchone() cur.execute( """ SELECT COALESCE(NULLIF(node_code, ''), 'unknown') AS node_code, COUNT(*) AS processed_recent, COUNT(*) FILTER (WHERE event_type = 'domain_completed') AS completed_recent, COUNT(*) FILTER (WHERE event_type = 'domain_blacklisted') AS blacklisted_recent, COUNT(*) FILTER (WHERE event_type = 'domain_failed') AS failed_recent FROM detect_run_events WHERE job_id = %s AND event_type IN ('domain_completed', 'domain_blacklisted', 'domain_failed') AND created_at >= CURRENT_TIMESTAMP - (%s || ' minutes')::interval GROUP BY COALESCE(NULLIF(node_code, ''), 'unknown') ORDER BY processed_recent DESC, node_code ASC """, (job_id, window_minutes), ) throughput_rows = cur.fetchall() cur.execute( """ SELECT COALESCE(NULLIF(step_code, ''), 'domain_pipeline') AS step_code, COUNT(*) FILTER ( WHERE status IN ('completed', 'blacklisted', 'failed') AND finished_at >= CURRENT_TIMESTAMP - (%s || ' minutes')::interval ) AS processed_recent, COUNT(*) FILTER ( WHERE status = 'completed' AND finished_at >= CURRENT_TIMESTAMP - (%s || ' minutes')::interval ) AS completed_recent, COUNT(*) FILTER ( WHERE status = 'blacklisted' AND finished_at >= CURRENT_TIMESTAMP - (%s || ' minutes')::interval ) AS blacklisted_recent, COUNT(*) FILTER ( WHERE status = 'failed' AND finished_at >= CURRENT_TIMESTAMP - (%s || ' minutes')::interval ) AS failed_recent FROM detect_job_items WHERE job_id = %s GROUP BY COALESCE(NULLIF(step_code, ''), 'domain_pipeline') ORDER BY step_code ASC """, (window_minutes, window_minutes, window_minutes, window_minutes, job_id), ) step_throughput_rows = cur.fetchall() runtime_display_rows = _load_runtime_display_rows(cur) oldest_pending_at = _format_time(lease_row[0]) if lease_row and lease_row[0] else "" nearest_lease_expiry_at = _format_time(lease_row[1]) if lease_row and lease_row[1] else "" oldest_pending_age_minutes = 0 if lease_row and lease_row[0]: oldest_pending_age_minutes = max(0, int((datetime.now() - lease_row[0]).total_seconds() // 60)) node_map = { str(item.get("node_code") or "unknown"): { "node_code": str(item.get("node_code") or "unknown"), "items_total": int(item.get("items_total", 0) or 0), "items_pending": int(item.get("items_pending", 0) or 0), "items_claimed": int(item.get("items_claimed", 0) or 0), "items_running": int(item.get("items_running", 0) or 0), "items_completed": int(item.get("items_completed", 0) or 0), "items_blacklisted": int(item.get("items_blacklisted", 0) or 0), "items_failed": int(item.get("items_failed", 0) or 0), "processed_recent": 0, "processed_per_minute": 0, "completed_recent": 0, "blacklisted_recent": 0, "failed_recent": 0, } for item in active_job.get("node_stats") or [] } distributed_nodes = list(active_job.get("distributed_node_stats") or []) if distributed_nodes: node_map = { str(item.get("node_code") or "unknown"): { "node_code": str(item.get("node_code") or "unknown"), "items_total": _int_value(item.get("items_total")), "items_pending": _int_value(item.get("items_pending")), "items_claimed": _int_value(item.get("items_claimed")), "items_running": _int_value(item.get("items_running")), "items_completed": _int_value(item.get("items_completed")), "items_blacklisted": _int_value(item.get("items_blacklisted")), "items_failed": _int_value(item.get("items_failed")), "processed_recent": 0, "processed_per_minute": 0, "completed_recent": 0, "blacklisted_recent": 0, "failed_recent": 0, "metrics_source": str(item.get("metrics_source") or "").strip(), } for item in distributed_nodes } total_processed_recent = 0 total_completed_recent = 0 total_blacklisted_recent = 0 total_failed_recent = 0 step_map = { str(item.get("step_code") or "domain_pipeline"): { **_build_step_bucket(str(item.get("step_code") or "domain_pipeline")), "items_total": _int_value(item.get("items_total")), "items_pending": _int_value(item.get("items_pending")), "items_claimed": _int_value(item.get("items_claimed")), "items_running": _int_value(item.get("items_running")), "items_completed": _int_value(item.get("items_completed")), "items_blacklisted": _int_value(item.get("items_blacklisted")), "items_failed": _int_value(item.get("items_failed")), } for item in active_job.get("step_stats") or [] } for row in throughput_rows: node_code = str(row[0] or "unknown") bucket = node_map.setdefault( node_code, { "node_code": node_code, "items_total": 0, "items_pending": 0, "items_claimed": 0, "items_running": 0, "items_completed": 0, "items_blacklisted": 0, "items_failed": 0, "processed_recent": 0, "processed_per_minute": 0, "completed_recent": 0, "blacklisted_recent": 0, "failed_recent": 0, }, ) processed_recent = int(row[1] or 0) completed_recent = int(row[2] or 0) blacklisted_recent = int(row[3] or 0) failed_recent = int(row[4] or 0) bucket["processed_recent"] = processed_recent bucket["processed_per_minute"] = round(processed_recent / window_minutes, 2) bucket["completed_recent"] = completed_recent bucket["blacklisted_recent"] = blacklisted_recent bucket["failed_recent"] = failed_recent total_processed_recent += processed_recent total_completed_recent += completed_recent total_blacklisted_recent += blacklisted_recent total_failed_recent += failed_recent for row in list(runtime_display_rows or []): runtime_bucket = _build_runtime_display_bucket(row) if not runtime_bucket: continue node_code = str(runtime_bucket.get("node_code") or "unknown") bucket = node_map.setdefault( node_code, { "node_code": node_code, "items_total": 0, "items_pending": 0, "items_claimed": 0, "items_running": 0, "items_completed": 0, "items_blacklisted": 0, "items_failed": 0, "processed_recent": 0, "processed_per_minute": 0, "completed_recent": 0, "blacklisted_recent": 0, "failed_recent": 0, "metrics_source": "runtime", "display_running": 0, "current_load": 0, "active_threads": 0, "max_threads": 0, "region": "", "role": "", "status": "", "last_heartbeat_at": "", }, ) runtime_display_running = _effective_runtime_load( items_running=runtime_bucket.get("items_running"), active_threads=runtime_bucket.get("active_threads"), ) for key in ( "items_total", "items_pending", "items_claimed", "items_completed", "items_blacklisted", "items_failed", "current_load", "active_threads", "max_threads", ): bucket[key] = max(_int_value(bucket.get(key)), _int_value(runtime_bucket.get(key))) bucket["display_running"] = max( _int_value(bucket.get("display_running")), runtime_display_running, ) bucket["items_running"] = max( _int_value(bucket.get("items_running")), _int_value(runtime_bucket.get("items_running")), ) for key in ("metrics_source", "region", "role", "status", "last_heartbeat_at"): if str(runtime_bucket.get(key) or "").strip(): bucket[key] = runtime_bucket.get(key) for node_code, runtime_bucket in dict(runtime_activity.get("nodes") or {}).items(): bucket = node_map.setdefault( node_code, { "node_code": node_code, "items_total": 0, "items_pending": 0, "items_claimed": 0, "items_running": 0, "items_completed": 0, "items_blacklisted": 0, "items_failed": 0, "processed_recent": 0, "processed_per_minute": 0, "completed_recent": 0, "blacklisted_recent": 0, "failed_recent": 0, "metrics_source": "runtime", "display_running": 0, "current_load": 0, "active_threads": 0, "max_threads": 0, "region": "", "role": "", "status": "", "last_heartbeat_at": "", }, ) runtime_processed_recent = int(runtime_bucket.get("processed_recent", 0) or 0) if runtime_processed_recent > int(bucket.get("processed_recent", 0) or 0): bucket["processed_recent"] = runtime_processed_recent bucket["processed_per_minute"] = round(runtime_processed_recent / window_minutes, 2) bucket["completed_recent"] = int(runtime_bucket.get("completed_recent", 0) or 0) bucket["blacklisted_recent"] = int(runtime_bucket.get("blacklisted_recent", 0) or 0) bucket["failed_recent"] = int(runtime_bucket.get("failed_recent", 0) or 0) for row in step_throughput_rows: step_code = str(row[0] or "domain_pipeline") bucket = step_map.setdefault(step_code, _build_step_bucket(step_code)) processed_recent = int(row[1] or 0) completed_recent = int(row[2] or 0) blacklisted_recent = int(row[3] or 0) failed_recent = int(row[4] or 0) bucket["processed_recent"] = processed_recent bucket["processed_per_minute"] = round(processed_recent / window_minutes, 2) bucket["completed_recent"] = completed_recent bucket["blacklisted_recent"] = blacklisted_recent bucket["failed_recent"] = failed_recent runtime_step_stats = dict(runtime_activity.get("step_stats") or {}) for runtime_step_code, runtime_step_bucket in runtime_step_stats.items(): normalized_step_code = str(runtime_step_code or "").strip() if not normalized_step_code: continue bucket = step_map.setdefault(normalized_step_code, _build_step_bucket(normalized_step_code)) runtime_processed_recent = int(runtime_step_bucket.get("processed_recent", 0) or 0) runtime_started_recent = int(runtime_step_bucket.get("started_recent", 0) or 0) bucket["started_recent"] = max(int(bucket.get("started_recent", 0) or 0), runtime_started_recent) if runtime_processed_recent > int(bucket.get("processed_recent", 0) or 0): bucket["processed_recent"] = runtime_processed_recent bucket["processed_per_minute"] = round(runtime_processed_recent / window_minutes, 2) bucket["completed_recent"] = int(runtime_step_bucket.get("completed_recent", 0) or 0) bucket["blacklisted_recent"] = int(runtime_step_bucket.get("blacklisted_recent", 0) or 0) bucket["failed_recent"] = int(runtime_step_bucket.get("failed_recent", 0) or 0) elif int(bucket.get("processed_recent", 0) or 0) <= 0 and runtime_started_recent > 0: bucket["processed_recent"] = runtime_started_recent bucket["processed_per_minute"] = round(runtime_started_recent / window_minutes, 2) runtime_step_code = str(runtime_activity.get("step_code") or "").strip() runtime_processed_recent = int(runtime_activity.get("processed_recent", 0) or 0) if runtime_step_code and runtime_processed_recent > 0 and runtime_step_code not in runtime_step_stats: runtime_bucket = step_map.setdefault(runtime_step_code, _build_step_bucket(runtime_step_code)) if runtime_processed_recent > int(runtime_bucket.get("processed_recent", 0) or 0): runtime_bucket["processed_recent"] = runtime_processed_recent runtime_bucket["processed_per_minute"] = round(runtime_processed_recent / window_minutes, 2) runtime_bucket["completed_recent"] = int(runtime_activity.get("completed_recent", 0) or 0) runtime_bucket["blacklisted_recent"] = int(runtime_activity.get("blacklisted_recent", 0) or 0) runtime_bucket["failed_recent"] = int(runtime_activity.get("failed_recent", 0) or 0) total_processed_recent = max(total_processed_recent, int(runtime_activity.get("processed_recent", 0) or 0)) total_completed_recent = max(total_completed_recent, int(runtime_activity.get("completed_recent", 0) or 0)) total_blacklisted_recent = max(total_blacklisted_recent, int(runtime_activity.get("blacklisted_recent", 0) or 0)) total_failed_recent = max(total_failed_recent, int(runtime_activity.get("failed_recent", 0) or 0)) runtime_snapshot_job = dict(runtime_snapshot.get("job") or {}) runtime_snapshot_queue = dict((runtime_snapshot.get("queue_health") or {}).get("queue") or {}) runtime_snapshot_nodes = list((runtime_snapshot.get("queue_health") or {}).get("nodes") or []) runtime_snapshot_steps = list((runtime_snapshot.get("queue_health") or {}).get("steps") or []) runtime_snapshot_job_code = str(runtime_snapshot_job.get("job_code") or "").strip() items_total = int(active_job.get("items_total", 0) or 0) pending_items = int(active_job.get("items_pending", 0) or 0) claimed_items = int(active_job.get("items_claimed", 0) or 0) running_items = int(active_job.get("items_running", 0) or 0) completed_items = int(active_job.get("items_completed", 0) or 0) blacklisted_items = int(active_job.get("items_blacklisted", 0) or 0) failed_items = int(active_job.get("items_failed", 0) or 0) display_claimed = int(active_job.get("display_items_claimed", active_job.get("items_claimed", 0)) or 0) display_running = int( active_job.get("display_active_threads", active_job.get("display_items_running", active_job.get("items_running", 0))) or 0 ) progress_percent = float(active_job.get("progress_percent", 0) or 0) if runtime_snapshot_queue: runtime_items_total = int(runtime_snapshot_queue.get("items_total", 0) or 0) if runtime_items_total > 0: items_total = runtime_items_total pending_items = int(runtime_snapshot_queue.get("pending", 0) or 0) claimed_items = int(runtime_snapshot_queue.get("claimed", 0) or 0) running_items = int(runtime_snapshot_queue.get("running", 0) or 0) completed_items = int(runtime_snapshot_queue.get("completed", 0) or 0) blacklisted_items = int(runtime_snapshot_queue.get("blacklisted", 0) or 0) failed_items = int(runtime_snapshot_queue.get("failed", 0) or 0) display_claimed = int(runtime_snapshot_queue.get("display_claimed", claimed_items) or 0) display_running = int(runtime_snapshot_queue.get("display_running", running_items) or 0) progress_percent = float(runtime_snapshot_job.get("progress_percent", progress_percent) or 0) if runtime_snapshot_nodes: node_map = { str(item.get("node_code") or "unknown"): { "node_code": str(item.get("node_code") or "unknown"), "items_total": _int_value(item.get("items_total")), "items_pending": _int_value(item.get("items_pending")), "items_claimed": _int_value(item.get("items_claimed")), "items_running": _int_value(item.get("items_running")), "items_completed": _int_value(item.get("items_completed")), "items_blacklisted": _int_value(item.get("items_blacklisted")), "items_failed": _int_value(item.get("items_failed")), "processed_recent": _int_value(item.get("processed_recent")), "processed_per_minute": float(item.get("processed_per_minute", 0) or 0), "completed_recent": _int_value(item.get("completed_recent")), "blacklisted_recent": _int_value(item.get("blacklisted_recent")), "failed_recent": _int_value(item.get("failed_recent")), "metrics_source": str(item.get("metrics_source") or "").strip(), } for item in runtime_snapshot_nodes } if runtime_snapshot_steps: step_map = { str(item.get("step_code") or "domain_pipeline"): { **_build_step_bucket(str(item.get("step_code") or "domain_pipeline")), "items_total": _int_value(item.get("items_total")), "items_pending": _int_value(item.get("items_pending")), "items_claimed": _int_value(item.get("items_claimed")), "items_running": _int_value(item.get("items_running")), "items_completed": _int_value(item.get("items_completed")), "items_blacklisted": _int_value(item.get("items_blacklisted")), "items_failed": _int_value(item.get("items_failed")), "processed_recent": _int_value(item.get("processed_recent")), "processed_per_minute": float(item.get("processed_per_minute", 0) or 0), "completed_recent": _int_value(item.get("completed_recent")), "blacklisted_recent": _int_value(item.get("blacklisted_recent")), "failed_recent": _int_value(item.get("failed_recent")), } for item in runtime_snapshot_steps } for row in list(runtime_display_rows or []): runtime_bucket = _build_runtime_display_bucket(row) if not runtime_bucket: continue node_code = str(runtime_bucket.get("node_code") or "unknown") bucket = node_map.setdefault( node_code, { "node_code": node_code, "items_total": 0, "items_pending": 0, "items_claimed": 0, "items_running": 0, "items_completed": 0, "items_blacklisted": 0, "items_failed": 0, "processed_recent": 0, "processed_per_minute": 0, "completed_recent": 0, "blacklisted_recent": 0, "failed_recent": 0, "metrics_source": "runtime", "display_running": 0, "current_load": 0, "active_threads": 0, "max_threads": 0, "region": "", "role": "", "status": "", "last_heartbeat_at": "", }, ) runtime_display_running = _effective_runtime_load( items_running=runtime_bucket.get("items_running"), active_threads=runtime_bucket.get("active_threads"), ) for key in ( "items_total", "items_pending", "items_claimed", "items_completed", "items_blacklisted", "items_failed", "current_load", "active_threads", "max_threads", ): bucket[key] = max(_int_value(bucket.get(key)), _int_value(runtime_bucket.get(key))) bucket["display_running"] = max( _int_value(bucket.get("display_running")), runtime_display_running, ) bucket["items_running"] = max( _int_value(bucket.get("items_running")), _int_value(runtime_bucket.get("items_running")), ) for key in ("metrics_source", "region", "role", "status", "last_heartbeat_at"): if str(runtime_bucket.get(key) or "").strip(): bucket[key] = runtime_bucket.get(key) for node_code, runtime_bucket in dict(runtime_activity.get("nodes") or {}).items(): bucket = node_map.setdefault( node_code, { "node_code": node_code, "items_total": 0, "items_pending": 0, "items_claimed": 0, "items_running": 0, "items_completed": 0, "items_blacklisted": 0, "items_failed": 0, "processed_recent": 0, "processed_per_minute": 0, "completed_recent": 0, "blacklisted_recent": 0, "failed_recent": 0, "metrics_source": "runtime", }, ) runtime_processed_recent = int(runtime_bucket.get("processed_recent", 0) or 0) if runtime_processed_recent > int(bucket.get("processed_recent", 0) or 0): bucket["processed_recent"] = runtime_processed_recent bucket["processed_per_minute"] = round(runtime_processed_recent / window_minutes, 2) bucket["completed_recent"] = int(runtime_bucket.get("completed_recent", 0) or 0) bucket["blacklisted_recent"] = int(runtime_bucket.get("blacklisted_recent", 0) or 0) bucket["failed_recent"] = int(runtime_bucket.get("failed_recent", 0) or 0) if not str(bucket.get("metrics_source") or "").strip(): bucket["metrics_source"] = "runtime" for bucket in node_map.values(): active_threads = int(bucket.get("active_threads", 0) or 0) current_load = int(bucket.get("current_load", 0) or 0) display_running_value = max( int(bucket.get("display_running", 0) or 0), active_threads, current_load, ) bucket["display_running"] = display_running_value if active_threads > 0: bucket["items_running"] = min( max(int(bucket.get("items_running", 0) or 0), active_threads), active_threads, ) bucket["current_load"] = max(current_load, active_threads) for bucket in step_map.values(): runtime_bucket = dict(runtime_activity.get("step_stats") or {}).get(str(bucket.get("step_code") or "").strip()) or {} if runtime_bucket: bucket["started_recent"] = max( int(bucket.get("started_recent", 0) or 0), int(runtime_bucket.get("started_recent", 0) or 0), ) runtime_processed_recent = int(runtime_bucket.get("processed_recent", 0) or 0) if runtime_processed_recent > int(bucket.get("processed_recent", 0) or 0): bucket["processed_recent"] = runtime_processed_recent bucket["processed_per_minute"] = round(runtime_processed_recent / window_minutes, 2) bucket["completed_recent"] = int(runtime_bucket.get("completed_recent", 0) or 0) bucket["blacklisted_recent"] = int(runtime_bucket.get("blacklisted_recent", 0) or 0) bucket["failed_recent"] = int(runtime_bucket.get("failed_recent", 0) or 0) normalized_running_items = sum( int(item.get("items_running", 0) or 0) for item in node_map.values() if str(item.get("node_code") or "").strip() and str(item.get("node_code") or "").strip() != "unassigned" ) normalized_display_running = sum( int(item.get("display_running", 0) or 0) for item in node_map.values() if str(item.get("node_code") or "").strip() and str(item.get("node_code") or "").strip() != "unassigned" ) # Prefer normalized per-node sums once runtime heartbeats are present. # Stale snapshot queue rows can temporarily over-report thread load from an # older cycle, which is exactly what misleads the ops dashboard. if normalized_running_items > 0: running_items = normalized_running_items if normalized_display_running > 0: display_running = normalized_display_running nodes = sorted( node_map.values(), key=lambda item: ( -int(item.get("processed_recent", 0) or 0), -int(item.get("display_running", item.get("items_running", 0)) or 0), -int(item.get("items_running", 0) or 0), str(item.get("node_code") or ""), ), ) terminal = completed_items + blacklisted_items + failed_items steps = sorted( step_map.values(), key=lambda item: ( -int(item.get("processed_recent", 0) or 0), -int(item.get("items_running", 0) or 0), -int(item.get("items_pending", 0) or 0), str(item.get("step_code") or ""), ), ) return { "window_minutes": window_minutes, "has_active_job": True, "job": { "job_id": job_id, "job_code": active_job.get("job_code", ""), "runtime_job_code": runtime_snapshot_job_code or str(runtime_activity.get("focus_job_code") or "").strip(), "runtime_job_codes": list(runtime_activity.get("job_codes") or []), "status": active_job.get("status", ""), "progress_percent": progress_percent, }, "queue": { "items_total": items_total, "pending": pending_items, "claimed": claimed_items, "running": running_items, "display_claimed": display_claimed, "display_running": display_running, "completed": completed_items, "blacklisted": blacklisted_items, "failed": failed_items, "terminal": terminal, "terminal_percent": round((terminal / items_total) * 100, 2) if items_total else 0, "oldest_pending_at": oldest_pending_at, "oldest_pending_age_minutes": oldest_pending_age_minutes, "nearest_lease_expiry_at": nearest_lease_expiry_at, "overdue_leases": int(lease_row[2] or 0) if lease_row else 0, "expiring_soon_leases": int(lease_row[3] or 0) if lease_row else 0, }, "throughput": { "processed_recent": total_processed_recent, "processed_per_minute": round(total_processed_recent / window_minutes, 2), "completed_recent": total_completed_recent, "blacklisted_recent": total_blacklisted_recent, "failed_recent": total_failed_recent, }, "nodes": nodes, "steps": steps, "runtime_activity": runtime_activity, "runtime_snapshot_backlog": dict(runtime_snapshot.get("backlog") or {}), } def get_detect_capacity_plan(*, queue_health: dict | None = None, online_worker_nodes: int = 0, target_finish_hours: int = 6) -> dict: queue_health = queue_health or get_detect_queue_health(window_minutes=15) target_finish_hours = max(1, min(int(target_finish_hours or 6), 72)) online_worker_nodes = max(0, int(online_worker_nodes or 0)) if not queue_health.get("has_active_job"): return { "has_active_job": False, "online_worker_nodes": online_worker_nodes, "target_finish_hours": target_finish_hours, "estimated_hours_remaining": 0, "recommended_total_workers": max(1, online_worker_nodes), "recommended_additional_workers": 0, "current_processed_per_hour": 0, "pending_items": 0, "terminal_items": 0, "summary": "当前没有活跃任务，无需扩容建议。", } queue = queue_health.get("queue") or {} throughput = queue_health.get("throughput") or {} pending_items = int(queue.get("pending", 0) or 0) claimed_items = int(queue.get("claimed", 0) or 0) running_items = int(queue.get("running", 0) or 0) remaining_items = pending_items + claimed_items + running_items current_processed_per_hour = round(float(throughput.get("processed_per_minute", 0) or 0) * 60, 2) estimated_hours_remaining = round((remaining_items / current_processed_per_hour), 2) if current_processed_per_hour > 0 else 0 recommended_total_workers = max(1, online_worker_nodes or 1) recommended_additional_workers = 0 if remaining_items > 0 and target_finish_hours > 0: required_per_hour = remaining_items / target_finish_hours if current_processed_per_hour > 0 and max(1, online_worker_nodes) > 0: per_worker_per_hour = current_processed_per_hour / max(1, online_worker_nodes) recommended_total_workers = max(1, int(math.ceil(required_per_hour / per_worker_per_hour))) recommended_additional_workers = max(0, recommended_total_workers - online_worker_nodes) elif remaining_items > 0: recommended_total_workers = max(1, online_worker_nodes or 1) recommended_additional_workers = 0 summary = ( f"当前在线 Worker {online_worker_nodes} 台，近窗吞吐约 {current_processed_per_hour} 项/小时，" f"剩余待处理约 {remaining_items} 项，预计还需 {estimated_hours_remaining} 小时。" ) if recommended_additional_workers > 0: summary = ( f"{summary} 若希望在 {target_finish_hours} 小时内收敛，建议总 Worker 数达到 " f"{recommended_total_workers} 台，至少再加 {recommended_additional_workers} 台。" ) else: summary = f"{summary} 按当前目标 {target_finish_hours} 小时看，现有 Worker 数量暂时够用。" return { "has_active_job": True, "online_worker_nodes": online_worker_nodes, "target_finish_hours": target_finish_hours, "estimated_hours_remaining": estimated_hours_remaining, "recommended_total_workers": recommended_total_workers, "recommended_additional_workers": recommended_additional_workers, "current_processed_per_hour": current_processed_per_hour, "pending_items": pending_items, "remaining_items": remaining_items, "terminal_items": int(queue.get("terminal", 0) or 0), "summary": summary, } def process_detect_pipeline(limit: int = 5000, job_id: int | None = None) -> dict: safe_limit = max(1, min(int(limit or 5000), 5000)) normalized_job_id = int(job_id or 0) if job_id not in (None, "", 0, "0") else 0 summary = { "processed_items": 0, "advanced_items": 0, "retried_items": 0, "terminal_items": 0, "skipped_items": 0, "created_next_items": 0, "retry_reset_items": 0, "legacy_items_upgraded": 0, "legacy_items_completed": 0, "job_ids": [], } with get_db() as conn: conn.autocommit = False with conn.cursor() as cur: settings_payload = get_settings_payload() touched_job_ids: set[int] = set() cur.execute( """ SELECT item.id, item.job_id, item.domain_id FROM detect_job_items AS item JOIN detect_jobs AS job ON job.id = item.job_id WHERE job.task_mode = 'domain_pipeline' AND (%s = 0 OR item.job_id = %s) AND job.status IN ('pending', 'running', 'completed', 'failed', 'partial_failed') AND COALESCE(item.step_code, '') = '' AND item.status = 'pending' ORDER BY item.id ASC LIMIT %s FOR UPDATE SKIP LOCKED """, (normalized_job_id, normalized_job_id, safe_limit), ) legacy_rows = cur.fetchall() for row in legacy_rows: item_id = int(row[0]) legacy_job_id = int(row[1]) domain_id = int(row[2]) domain_snapshot = _load_domain_pipeline_snapshot(cur, domain_id) if not domain_snapshot: continue step_code, step_payload = resolve_initial_domain_pipeline_item( domain_snapshot, settings_payload=settings_payload, ) if step_code and step_payload: cur.execute( """ UPDATE detect_job_items SET step_code = %s, step_payload_json = %s::jsonb, updated_at = CURRENT_TIMESTAMP WHERE id = %s """, (step_code, json.dumps(step_payload, ensure_ascii=False), item_id), ) touched_job_ids.add(legacy_job_id) summary["processed_items"] += 1 summary["legacy_items_upgraded"] += 1 continue processed_payload = { "controller_processed": True, "controller_action": "pipeline_already_complete", "controller_processed_at": datetime.now().isoformat(timespec="seconds"), } cur.execute( """ UPDATE detect_job_items SET status = 'completed', finished_at = COALESCE(finished_at, CURRENT_TIMESTAMP), result_payload_json = %s::jsonb, updated_at = CURRENT_TIMESTAMP WHERE id = %s """, (json.dumps(processed_payload, ensure_ascii=False), item_id), ) touched_job_ids.add(legacy_job_id) summary["processed_items"] += 1 summary["legacy_items_completed"] += 1 cur.execute( """ SELECT item.id, item.job_id, item.domain_id, item.step_code, item.status, item.attempt_count, item.result_payload_json, job.task_mode FROM detect_job_items AS item JOIN detect_jobs AS job ON job.id = item.job_id WHERE job.task_mode IN ('domain_pipeline', 'single_step') AND (%s = 0 OR item.job_id = %s) AND job.status IN ('pending', 'running', 'completed', 'failed', 'partial_failed') AND COALESCE(item.step_code, '') <> '' AND item.status IN ('completed', 'blacklisted', 'failed') AND COALESCE(item.result_payload_json->>'controller_processed', 'false') <> 'true' ORDER BY COALESCE(item.finished_at, item.updated_at) ASC, item.id ASC LIMIT %s FOR UPDATE SKIP LOCKED """, (normalized_job_id, normalized_job_id, safe_limit), ) rows = cur.fetchall() for row in rows: item_id = int(row[0]) job_id = int(row[1]) domain_id = int(row[2]) current_step_code = str(row[3] or "").strip() item_status = str(row[4] or "").strip() attempt_count = int(row[5] or 0) result_payload = _decode_payload(row[6]) task_mode = str(row[7] or "").strip() domain_snapshot = _load_domain_pipeline_snapshot(cur, domain_id) if not domain_snapshot: cur.execute( """ UPDATE detect_job_items SET result_payload_json = jsonb_set(COALESCE(result_payload_json, '{}'::jsonb), '{controller_processed}', 'true'::jsonb, true), updated_at = CURRENT_TIMESTAMP WHERE id = %s """, (item_id,), ) summary["processed_items"] += 1 summary["skipped_items"] += 1 continue if task_mode == "single_step": result_field_name = str( result_payload.get("field_name") or _PIPELINE_STEP_FIELDS.get(current_step_code) or "" ).strip() persisted_payload = { key: value for key, value in dict(result_payload or {}).items() if key not in {"controller_processed", "controller_action", "controller_processed_at"} } action = "single_step_ignored" if result_field_name: persisted_payload.setdefault("step_code", current_step_code) persisted_payload.setdefault("step_name", _step_label(current_step_code)) persisted_payload.setdefault("field_name", result_field_name) persisted_payload.setdefault("task_mode", "single_step") persisted_payload.setdefault("domain_id", domain_id) _upsert_domain_detection_field(cur, domain_id, result_field_name, persisted_payload) action = "single_step_persisted" processed_payload = { **dict(result_payload or {}), "controller_processed": True, "controller_action": action, "controller_processed_at": datetime.now().isoformat(timespec="seconds"), } cur.execute( """ UPDATE detect_job_items SET result_payload_json = %s::jsonb, updated_at = CURRENT_TIMESTAMP WHERE id = %s """, (json.dumps(processed_payload, ensure_ascii=False), item_id), ) touched_job_ids.add(job_id) summary["processed_items"] += 1 if action == "single_step_persisted": summary["advanced_items"] += 1 else: summary["skipped_items"] += 1 continue result_message = str(result_payload.get("message") or "").strip() outcome = _classify_pipeline_item_outcome( item_status=item_status, result_payload=result_payload, step_code=current_step_code, attempt_count=attempt_count, ) action = str(outcome.get("action") or "pass").strip() result_state = str(outcome.get("result_state") or "").strip().lower() retry_limit = int(outcome.get("retry_limit", 0) or 0) if action == "retry": retry_payload = { **result_payload, "controller_action": "retry", "controller_processed": False, "controller_retry_count": int(attempt_count), "controller_reason_code": str(outcome.get("reason_code") or "external_retry"), } cur.execute( """ UPDATE detect_job_items SET status = 'pending', claimed_by = '', claim_token = '', lease_expires_at = NULL, finished_at = NULL, last_error = %s, result_payload_json = %s::jsonb, updated_at = CURRENT_TIMESTAMP WHERE id = %s """, ( result_message or f"{_step_label(current_step_code)} 执行失败，等待重试", json.dumps(retry_payload, ensure_ascii=False), item_id, ), ) _append_detect_job_event_with_cursor( cur, job_id, event_type="step_retry_scheduled", message=f"{domain_snapshot['domain']} 的 {_step_label(current_step_code)} 已重投", payload={ "domain_id": domain_id, "domain": domain_snapshot["domain"], "step_code": current_step_code, "attempt_count": attempt_count, "retry_limit": retry_limit, "result_state": result_state, }, ) touched_job_ids.add(job_id) summary["processed_items"] += 1 summary["retried_items"] += 1 summary["retry_reset_items"] += 1 continue if action == "pass": next_step_code = resolve_domain_pipeline_step( domain_snapshot, settings_payload=settings_payload, after_step_code=current_step_code, ) if next_step_code: next_payload = _build_step_payload( step_code=next_step_code, domain_snapshot=domain_snapshot, settings_payload=settings_payload, ) cur.execute( """ INSERT INTO detect_job_items (job_id, domain_id, step_code, status, step_payload_json) VALUES (%s, %s, %s, 'pending', %s::jsonb) ON CONFLICT (job_id, domain_id, step_code) DO NOTHING RETURNING id """, ( job_id, domain_id, next_step_code, json.dumps(next_payload, ensure_ascii=False), ), ) created_next = bool(cur.fetchone()) action = "pass" summary["advanced_items"] += 1 if created_next: summary["created_next_items"] += 1 _append_detect_job_event_with_cursor( cur, job_id, event_type="pipeline_step_advanced", message=f"{domain_snapshot['domain']} 进入下一步骤: {_step_label(next_step_code)}", payload={ "domain_id": domain_id, "domain": domain_snapshot["domain"], "current_step_code": current_step_code, "next_step_code": next_step_code, }, ) else: action = "pipeline_completed" _append_detect_job_event_with_cursor( cur, job_id, event_type="pipeline_completed", message=f"{domain_snapshot['domain']} 本次 pipeline 已完成", payload={ "domain_id": domain_id, "domain": domain_snapshot["domain"], "current_step_code": current_step_code, }, ) elif action == "black_hit": _append_detect_job_event_with_cursor( cur, job_id, event_type="pipeline_black_hit", message=f"{domain_snapshot['domain']} 在 {_step_label(current_step_code)} 命中黑名单并终止后续步骤", level="warning", payload={ "domain_id": domain_id, "domain": domain_snapshot["domain"], "current_step_code": current_step_code, "result_state": result_state or "blacklisted", }, ) elif action == "reject": _append_detect_job_event_with_cursor( cur, job_id, event_type="pipeline_rejected", message=f"{domain_snapshot['domain']} 在 {_step_label(current_step_code)} 被判定终止", level="warning", payload={ "domain_id": domain_id, "domain": domain_snapshot["domain"], "current_step_code": current_step_code, "result_state": result_state or "failed", "attempt_count": attempt_count, "retry_limit": retry_limit, "reason_code": str(outcome.get("reason_code") or ""), }, ) processed_payload = { **result_payload, "controller_processed": True, "controller_action": action, "controller_reason_code": str(outcome.get("reason_code") or ""), "controller_processed_at": datetime.now().isoformat(timespec="seconds"), } if action == "pass": processed_payload["controller_result_state"] = result_state or "passed" cur.execute( """ UPDATE detect_job_items SET result_payload_json = %s::jsonb, updated_at = CURRENT_TIMESTAMP WHERE id = %s """, (json.dumps(processed_payload, ensure_ascii=False), item_id), ) touched_job_ids.add(job_id) summary["processed_items"] += 1 if action in {"black_hit", "reject", "pipeline_completed"}: summary["terminal_items"] += 1 for job_id in touched_job_ids: cur.execute( """ SELECT job.task_mode, count(*) FILTER (WHERE item.status = 'pending') AS pending_count, count(*) FILTER (WHERE item.status IN ('claimed', 'running')) AS dispatch_active_count, count(*) FILTER (WHERE item.status = 'failed') AS failed_count, count(*) FILTER (WHERE item.status IN ('completed', 'blacklisted')) AS done_count, count(*) FILTER ( WHERE item.status IN ('completed', 'blacklisted', 'failed') AND COALESCE(item.step_code, '') <> '' AND COALESCE(item.result_payload_json->>'controller_processed', 'false') <> 'true' ) AS unprocessed_terminal_count FROM detect_job_items AS item JOIN detect_jobs AS job ON job.id = item.job_id WHERE item.job_id = %s GROUP BY job.task_mode """, (job_id,), ) refresh_row = cur.fetchone() task_mode = str((refresh_row or [""])[0] or "").strip() pending_count = int((refresh_row or ["", 0])[1] or 0) dispatch_active_count = int((refresh_row or ["", 0, 0])[2] or 0) failed_count = int((refresh_row or ["", 0, 0, 0])[3] or 0) done_count = int((refresh_row or ["", 0, 0, 0, 0])[4] or 0) unprocessed_terminal_count = int((refresh_row or ["", 0, 0, 0, 0, 0])[5] or 0) if dispatch_active_count > 0 or (task_mode == "domain_pipeline" and unprocessed_terminal_count > 0): cur.execute( """ UPDATE detect_jobs SET status = 'running', started_at = COALESCE(started_at, CURRENT_TIMESTAMP), finished_at = NULL WHERE id = %s """, (job_id,), ) elif pending_count > 0: cur.execute( """ UPDATE detect_jobs SET status = 'pending', finished_at = NULL WHERE id = %s """, (job_id,), ) else: final_status = "completed" if failed_count > 0 and done_count > 0: final_status = "partial_failed" elif failed_count > 0: final_status = "failed" cur.execute( """ UPDATE detect_jobs SET status = %s, finished_at = CURRENT_TIMESTAMP, started_at = COALESCE(started_at, CURRENT_TIMESTAMP) WHERE id = %s """, (final_status, job_id), ) summary["job_ids"].append(job_id) conn.commit() summary["job_ids"] = sorted(set(int(item) for item in summary["job_ids"])) return summary def process_detect_pipeline_now(limit: int = 2000, job_id: int | None = None) -> tuple[bool, str, dict]: attempts = 3 delay_seconds = 0.05 for attempt in range(1, attempts + 1): try: data = process_detect_pipeline(limit=limit, job_id=job_id) processed = int(data.get("processed_items", 0) or 0) advanced = int(data.get("advanced_items", 0) or 0) retried = int(data.get("retried_items", 0) or 0) message = f"pipeline 处理完成：processed={processed}, advanced={advanced}, retried={retried}" if attempt > 1: message = f"{message}（deadlock 自动重试 {attempt - 1} 次后成功）" return True, message, {"action": "process_pipeline", "retry_attempts": attempt - 1, **data} except Exception as exc: if not is_retryable_db_error(exc) or attempt >= attempts: raise time.sleep(delay_seconds) delay_seconds *= 2 data = process_detect_pipeline(limit=limit, job_id=job_id) processed = int(data.get("processed_items", 0) or 0) advanced = int(data.get("advanced_items", 0) or 0) retried = int(data.get("retried_items", 0) or 0) message = f"pipeline 处理完成：processed={processed}, advanced={advanced}, retried={retried}" return True, message, {"action": "process_pipeline", **data} def _append_detect_job_event_with_cursor( cur, job_id: int, *, event_type: str, message: str, level: str = "info", payload: dict | None = None, node_code: str | None = None, ) -> bool: cur.execute( "SELECT 1 FROM detect_jobs WHERE id = %s LIMIT 1", (int(job_id),), ) if cur.fetchone() is None: return False cur.execute( """ INSERT INTO detect_run_events (job_id, node_code, event_type, level, message, payload_json) VALUES (%s, %s, %s, %s, %s, %s::jsonb) """, ( int(job_id), node_code or settings.node_code, str(event_type or "").strip() or "info", str(level or "info").strip() or "info", str(message or "").strip(), json.dumps(payload or {}, ensure_ascii=False), ), ) return True def append_detect_job_event( job_id: int, *, event_type: str, message: str, level: str = "info", payload: dict | None = None, node_code: str | None = None, ) -> None: inserted = False with get_db() as conn: with conn.cursor() as cur: inserted = _append_detect_job_event_with_cursor( cur, job_id, event_type=event_type, message=message, level=level, payload=payload, node_code=node_code, ) if inserted: conn.commit() try: push_debug_event( service="detect-job", event_type=str(event_type or "").strip() or "info", level=str(level or "info").strip() or "info", message=str(message or "").strip(), payload={ "job_id": int(job_id), "node_code": node_code or settings.node_code, **(payload or {}), }, ) except Exception: pass def create_detect_job_if_needed(limit: int = 1000, created_by: str = "system", step_code: str | None = None) -> dict | None: job_definition = resolve_detect_job_definition(step_code) existing = get_active_detect_job_summary() if existing: existing_task_mode = str(existing.get("task_mode") or "").strip() existing_step_code = normalize_detect_step_code(existing.get("step_code")) if ( existing_task_mode == str(job_definition["task_mode"]).strip() and existing_step_code == normalize_detect_step_code(job_definition["step_code"]) ): return existing settings_payload = get_settings_payload() with get_db() as conn: conn.autocommit = False with conn.cursor() as cur: cur.execute(_selection_sql(), (max(1, int(limit or 1000)),)) domain_ids = [row[0] for row in cur.fetchall()] if not domain_ids: conn.rollback() if not domain_ids: try: from app.services.sync_push_service import pull_detect_task_batch_now pull_detect_task_batch_now(limit=max(1, int(limit or 1000))) except Exception: pass with get_db() as conn: conn.autocommit = False with conn.cursor() as cur: cur.execute(_selection_sql(), (max(1, int(limit or 1000)),)) domain_ids = [row[0] for row in cur.fetchall()] if not domain_ids: conn.rollback() return None with get_db() as conn: conn.autocommit = False with conn.cursor() as cur: domain_snapshots: list[dict] = [] for domain_id in domain_ids: snapshot = _load_domain_pipeline_snapshot(cur, int(domain_id)) if snapshot: domain_snapshots.append(snapshot) if not domain_snapshots: conn.rollback() return None job_prefix = "step" if job_definition["is_single_step"] else "detect" job_code = f"{job_prefix}-{datetime.now().strftime('%Y%m%d%H%M%S')}-{uuid4().hex[:6]}" plan_hash = uuid4().hex cur.execute( """ INSERT INTO detect_jobs (job_code, source, plan_hash, task_mode, step_code, status, remark, created_by) VALUES (%s, %s, %s, %s, %s, 'pending', %s, %s) RETURNING id """, ( job_code, job_definition["source"], plan_hash, job_definition["task_mode"], job_definition["step_code"], f"API 创建{job_definition['label']}任务，待检测域名 {len(domain_ids)} 个", created_by, ), ) job_id = cur.fetchone()[0] queued_count = 0 skipped_count = 0 for snapshot in domain_snapshots: item_step_code = ( job_definition["step_code"] if job_definition["is_single_step"] else resolve_initial_domain_pipeline_item(snapshot, settings_payload=settings_payload)[0] ) if not item_step_code: skipped_count += 1 continue _, step_payload = resolve_initial_domain_pipeline_item(snapshot, settings_payload=settings_payload) cur.execute( """ INSERT INTO detect_job_items (job_id, domain_id, step_code, status, step_payload_json) VALUES (%s, %s, %s, 'pending', %s::jsonb) ON CONFLICT (job_id, domain_id, step_code) DO NOTHING RETURNING id """, ( job_id, int(snapshot["id"]), item_step_code, json.dumps(step_payload, ensure_ascii=False), ), ) if cur.fetchone(): queued_count += 1 else: skipped_count += 1 if queued_count <= 0: conn.rollback() return None cur.execute( """ INSERT INTO detect_run_events (job_id, node_code, event_type, level, message, payload_json) VALUES (%s, %s, %s, %s, %s, %s::jsonb) """, ( job_id, settings.node_code, "job_created", "info", f"创建检测任务 {job_code}，已入队 {queued_count} 个任务项", json.dumps( { "count": len(domain_ids), "queued_count": queued_count, "skipped_count": skipped_count, "task_mode": job_definition["task_mode"], "step_code": job_definition["step_code"], }, ensure_ascii=False, ), ), ) conn.commit() return get_detect_job_summary(job_id)