from __future__ import annotations import hashlib import json import socket from datetime import datetime, timedelta from app.core.config import settings from app.core.db import get_db def _format_time(value: datetime | None) -> str: return value.isoformat(sep=" ", timespec="seconds") if value else "" def _resolve_local_ip() -> str: try: return socket.gethostbyname(socket.gethostname()) except Exception: return "" def _decode_json(value: object) -> dict: if isinstance(value, dict): return value if value in (None, ""): return {} try: return json.loads(value) except Exception: return {} def _normalize_region(value: str | None, fallback: str) -> str: text = str(value or "").strip() if not text or text == "unknown": return str(fallback or "unknown").strip() or "unknown" return text def _build_detect_result_batch_digest(batch: dict | None) -> dict: batch = batch or {} projection = (batch.get("projection") or {}).get("payload") or {} latest_push = batch.get("latest_push") or {} latest_ingest = batch.get("latest_ingest") or {} job = (projection.get("projection") or {}).get("job") or {} latest_event = (projection.get("projection") or {}).get("latest_event") or {} return { "job_id": int(batch.get("job_id") or 0), "job_code": str(batch.get("job_code") or ""), "job_status": str(batch.get("job_status") or ""), "items_total": int(batch.get("items_total") or 0), "items_terminal": int(batch.get("items_terminal") or 0), "items_pending": int(batch.get("items_pending") or 0), "items_running": int(batch.get("items_running") or 0), "items_failed": int(batch.get("items_failed") or 0), "progress_percent": float(batch.get("progress_percent") or 0), "sync_state": str(batch.get("sync_state") or "unsynced"), "sync_message": str(batch.get("sync_message") or ""), "projection_record_id": int((batch.get("projection") or {}).get("id") or 0), "projection_status": str((batch.get("projection") or {}).get("status") or ""), "projection_created_at": str((batch.get("projection") or {}).get("created_at") or ""), "latest_event_type": str(latest_event.get("event_type") or ""), "latest_event_message": str(latest_event.get("message") or ""), "latest_event_created_at": str(latest_event.get("created_at") or ""), "latest_push_status": str(latest_push.get("status") or ""), "latest_push_error": str(latest_push.get("error_message") or ""), "latest_push_created_at": str(latest_push.get("created_at") or ""), "latest_ingest_status": str(latest_ingest.get("status") or ""), "latest_ingest_created_at": str(latest_ingest.get("created_at") or ""), "cycle_token": str(job.get("current_cycle_token") or ""), } def _should_append_runtime_projection(previous_payload: dict, current_projection: dict, previous_created_at: datetime | None) -> bool: if not previous_payload: return True previous_projection = previous_payload.get("projection") or {} if not previous_projection: return True keys_requiring_immediate_write = ( "worker_online", "worker_mode", "phase_label", "phase_detail", "proxy_runtime_label", "proxy_runtime_reason", ) for key in keys_requiring_immediate_write: if previous_projection.get(key) != current_projection.get(key): return True previous_job = previous_projection.get("active_job") or {} current_job = current_projection.get("active_job") or {} for key in ("job_id", "job_code", "status"): if previous_job.get(key) != current_job.get(key): return True previous_cluster = previous_projection.get("cluster_summary") or {} current_cluster = current_projection.get("cluster_summary") or {} for key in ("online_worker_nodes", "online_control_nodes", "busy_nodes", "stale_nodes", "offline_nodes"): if previous_cluster.get(key) != current_cluster.get(key): return True previous_progress = previous_projection.get("progress") or {} current_progress = current_projection.get("progress") or {} failed_delta = abs(int(current_progress.get("failed", 0) or 0) - int(previous_progress.get("failed", 0) or 0)) blacklisted_delta = abs(int(current_progress.get("blacklisted", 0) or 0) - int(previous_progress.get("blacklisted", 0) or 0)) completed_delta = abs(int(current_progress.get("completed", 0) or 0) - int(previous_progress.get("completed", 0) or 0)) running_delta = abs(int(current_progress.get("running", 0) or 0) - int(previous_progress.get("running", 0) or 0)) if failed_delta > 0 or blacklisted_delta >= 10 or completed_delta >= 20 or running_delta >= 5: return True previous_alerts = previous_projection.get("dependency_alerts") or [] current_alerts = current_projection.get("dependency_alerts") or [] if previous_alerts != current_alerts: return True if not previous_created_at: return True now = datetime.now(previous_created_at.tzinfo) if previous_created_at.tzinfo else datetime.now() return now - previous_created_at >= timedelta(seconds=45) def list_sync_records(limit: int = 20) -> list[dict]: safe_limit = max(1, min(int(limit or 20), 200)) with get_db() as conn: with conn.cursor() as cur: cur.execute( """ SELECT id, sync_type, source_region, target_region, status, payload_json, error_message, created_at, updated_at FROM detect_sync_records ORDER BY created_at DESC, id DESC LIMIT %s """, (safe_limit,), ) rows = cur.fetchall() return [ { "id": row[0], "sync_type": row[1], "source_region": row[2], "target_region": row[3], "status": row[4], "payload": _decode_json(row[5]), "error_message": row[6] or "", "created_at": _format_time(row[7]), "updated_at": _format_time(row[8]), } for row in rows ] def _latest_sync_record_by_source( cur, *, sync_type: str, source_region: str, target_region: str, source_record_id: int, ) -> dict | None: cur.execute( """ SELECT id, sync_type, source_region, target_region, status, payload_json, error_message, created_at, updated_at FROM detect_sync_records WHERE sync_type = %s AND source_region = %s AND target_region = %s AND (payload_json->>'source_record_id') = %s ORDER BY created_at DESC, id DESC LIMIT 1 """, (sync_type, source_region, target_region, str(int(source_record_id))), ) row = cur.fetchone() if not row: return None return { "id": row[0], "sync_type": row[1], "source_region": row[2], "target_region": row[3], "status": row[4], "payload": _decode_json(row[5]), "error_message": row[6] or "", "created_at": _format_time(row[7]), "updated_at": _format_time(row[8]), } def get_detect_result_sync_batches(limit: int = 5) -> dict: safe_limit = max(1, min(int(limit or 5), 20)) source_region = _normalize_region(settings.sync_source_region, settings.node_region) target_region = _normalize_region(settings.sync_target_region, "overseas") batches: list[dict] = [] with get_db() as conn: with conn.cursor() as cur: cur.execute( """ SELECT id, job_code, status, created_at, started_at, finished_at FROM detect_jobs ORDER BY created_at DESC, id DESC LIMIT %s """, (safe_limit,), ) jobs = cur.fetchall() for row in jobs: job_id = int(row[0]) job_code = str(row[1] or "") job_status = str(row[2] or "") created_at = _format_time(row[3]) started_at = _format_time(row[4]) finished_at = _format_time(row[5]) cur.execute( """ SELECT status, count(*) FROM detect_job_items WHERE job_id = %s GROUP BY status """, (job_id,), ) item_counts = {str(status or ""): int(count) for status, count in cur.fetchall()} items_total = sum(item_counts.values()) items_terminal = int(item_counts.get("completed", 0)) + int(item_counts.get("blacklisted", 0)) + int(item_counts.get("failed", 0)) cur.execute( """ SELECT id, status, payload_json, created_at, updated_at FROM detect_sync_records WHERE sync_type = 'detect_result_projection' AND source_region = %s AND target_region = %s AND (payload_json->'projection'->'job'->>'job_id') = %s ORDER BY created_at DESC, id DESC LIMIT 1 """, (source_region, target_region, str(job_id)), ) projection_row = cur.fetchone() projection = None latest_push = None latest_ingest = None sync_state = "unsynced" sync_message = "该任务还没有生成结果投影" if projection_row: projection_payload = _decode_json(projection_row[2]) projection = { "id": int(projection_row[0]), "status": projection_row[1], "payload": projection_payload, "created_at": _format_time(projection_row[3]), "updated_at": _format_time(projection_row[4]), } latest_push = _latest_sync_record_by_source( cur, sync_type="runtime_push", source_region=source_region, target_region=target_region, source_record_id=projection["id"], ) latest_ingest = _latest_sync_record_by_source( cur, sync_type="detect_result_ingest", source_region=source_region, target_region=target_region, source_record_id=projection["id"], ) if latest_ingest: sync_state = "synced" sync_message = "最近一条结果投影已被目标地域接收" elif latest_push and latest_push.get("status") == "success": sync_state = "delivered" sync_message = "结果投影已推送成功,等待目标侧回看接收记录" elif latest_push and latest_push.get("status") == "pending": sync_state = "pushing" sync_message = "结果投影正在推送中" elif latest_push and latest_push.get("status") == "failed": sync_state = "failed" sync_message = latest_push.get("error_message") or "最近一次结果投影推送失败" else: sync_state = "projected" sync_message = "已生成结果投影,等待同步代理推送" batches.append( { "job_id": job_id, "job_code": job_code, "job_status": job_status, "created_at": created_at, "started_at": started_at, "finished_at": finished_at, "items_total": items_total, "items_terminal": items_terminal, "items_pending": int(item_counts.get("pending", 0)), "items_running": int(item_counts.get("running", 0)) + int(item_counts.get("claimed", 0)), "items_failed": int(item_counts.get("failed", 0)), "progress_percent": round((items_terminal / items_total) * 100, 2) if items_total else 0, "sync_state": sync_state, "sync_message": sync_message, "projection": projection, "latest_push": latest_push, "latest_ingest": latest_ingest, } ) state_counts = { "synced": 0, "delivered": 0, "pushing": 0, "projected": 0, "failed": 0, "unsynced": 0, } for item in batches: state = str(item.get("sync_state") or "unsynced") state_counts[state] = state_counts.get(state, 0) + 1 return { "source_region": source_region, "target_region": target_region, "jobs_total": len(batches), "state_counts": state_counts, "batches": batches, } def get_sync_summary(record_limit: int = 10) -> dict: source_region = _normalize_region(settings.sync_source_region, settings.node_region) target_region = _normalize_region(settings.sync_target_region, "overseas") with get_db() as conn: with conn.cursor() as cur: cur.execute( """ SELECT status, count(*) FROM detect_sync_records GROUP BY status """ ) status_counts = {str(status or "unknown"): int(count) for status, count in cur.fetchall()} cur.execute( """ SELECT sync_type, count(*) FROM detect_sync_records GROUP BY sync_type """ ) type_counts = {str(sync_type or "unknown"): int(count) for sync_type, count in cur.fetchall()} cur.execute("SELECT count(*) FROM detect_sync_records") total = int(cur.fetchone()[0] or 0) cur.execute( """ SELECT id, sync_type, source_region, target_region, status, payload_json, error_message, created_at, updated_at FROM detect_sync_records ORDER BY created_at DESC, id DESC LIMIT 1 """ ) latest = cur.fetchone() latest_record = None if latest: latest_record = { "id": latest[0], "sync_type": latest[1], "source_region": latest[2], "target_region": latest[3], "status": latest[4], "payload": _decode_json(latest[5]), "error_message": latest[6] or "", "created_at": _format_time(latest[7]), "updated_at": _format_time(latest[8]), } detect_result_batches = get_detect_result_sync_batches(limit=min(5, record_limit)) recent_result_batches = list(detect_result_batches.get("batches") or []) latest_result_batch = recent_result_batches[0] if recent_result_batches else None return { "enabled": bool(settings.sync_push_enabled), "source_region": source_region, "target_region": target_region, "target_api_base_url": settings.sync_target_api_base_url, "batch_size": max(1, int(settings.sync_batch_size or 200)), "poll_interval_seconds": max(5, int(settings.sync_poll_interval_seconds or 30)), "records_total": total, "status_counts": status_counts, "type_counts": type_counts, "latest_record": latest_record, "detect_result_batches": detect_result_batches, "latest_detect_result_batch": _build_detect_result_batch_digest(latest_result_batch), "detect_result_batch_digests": [_build_detect_result_batch_digest(item) for item in recent_result_batches], "recent_records": list_sync_records(limit=record_limit), } def append_sync_record( *, sync_type: str, source_region: str, target_region: str, status: str, payload: dict | None = None, error_message: str = "", ) -> int: with get_db() as conn: with conn.cursor() as cur: cur.execute( """ INSERT INTO detect_sync_records ( sync_type, source_region, target_region, status, payload_json, error_message, created_at, updated_at ) VALUES (%s, %s, %s, %s, %s::jsonb, %s, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP) RETURNING id """, ( str(sync_type or "").strip() or "unknown", _normalize_region(source_region, _normalize_region(settings.sync_source_region, settings.node_region)), _normalize_region(target_region, _normalize_region(settings.sync_target_region, "overseas")), str(status or "").strip() or "pending", json.dumps(payload or {}, ensure_ascii=False), str(error_message or "").strip(), ), ) record_id = int(cur.fetchone()[0]) conn.commit() return record_id def append_runtime_projection_if_changed( *, detect: dict, cluster: dict, source_region: str | None = None, target_region: str | None = None, ) -> int | None: normalized_source_region = _normalize_region(source_region, _normalize_region(settings.sync_source_region, settings.node_region)) normalized_target_region = _normalize_region(target_region, _normalize_region(settings.sync_target_region, "overseas")) active_job = detect.get("active_job") or {} projection = { "node": { "node_code": settings.node_code, "region": settings.node_region, "role": settings.node_role, "hostname": socket.gethostname(), "ip": _resolve_local_ip(), }, "worker_online": bool(detect.get("worker_online", False)), "worker_mode": detect.get("worker_mode", ""), "phase_label": detect.get("phase_label", ""), "phase_detail": detect.get("phase_detail", ""), "proxy_runtime_label": detect.get("proxy_runtime_label", ""), "proxy_runtime_reason": detect.get("proxy_runtime_reason", ""), "progress": { "pending": int((detect.get("progress") or {}).get("pending", 0) or 0), "running": int((detect.get("progress") or {}).get("running", 0) or 0), "completed": int((detect.get("progress") or {}).get("completed", 0) or 0), "blacklisted": int((detect.get("progress") or {}).get("blacklisted", 0) or 0), "failed": int((detect.get("progress") or {}).get("failed", 0) or 0), }, "active_job": { "job_id": active_job.get("job_id"), "job_code": active_job.get("job_code", ""), "status": active_job.get("status", ""), "progress_percent": active_job.get("progress_percent", 0), "items_total": active_job.get("items_total", 0), "items_terminal": active_job.get("items_terminal", 0), "items_pending": active_job.get("items_pending", 0), "items_running": active_job.get("items_running", 0), "items_failed": active_job.get("items_failed", 0), "node_stats": list(active_job.get("node_stats") or []), }, "cluster_summary": { "nodes_total": int(cluster.get("nodes_total", 0) or 0), "online_worker_nodes": int((cluster.get("summary") or {}).get("online_worker_nodes", 0) or 0), "online_control_nodes": int((cluster.get("summary") or {}).get("online_control_nodes", 0) or 0), "busy_nodes": list((cluster.get("summary") or {}).get("busy_nodes") or []), "stale_nodes": list((cluster.get("summary") or {}).get("stale_nodes") or []), "offline_nodes": list((cluster.get("summary") or {}).get("offline_nodes") or []), }, "dependency_alerts": [ { "kind": item.get("kind", ""), "title": item.get("title", ""), "level": item.get("level", ""), } for item in (detect.get("dependency_alerts") or [])[:3] ], } payload = { "projection": projection, "projection_hash": hashlib.sha1( json.dumps(projection, ensure_ascii=False, sort_keys=True).encode("utf-8") ).hexdigest(), } with get_db() as conn: with conn.cursor() as cur: cur.execute( """ SELECT payload_json, created_at FROM detect_sync_records WHERE sync_type = 'runtime_projection' AND source_region = %s AND target_region = %s ORDER BY created_at DESC, id DESC LIMIT 1 """, (normalized_source_region, normalized_target_region), ) latest = cur.fetchone() latest_payload = _decode_json(latest[0]) if latest else {} latest_created_at = latest[1] if latest else None if latest_payload.get("projection_hash") == payload["projection_hash"]: return None if not _should_append_runtime_projection(latest_payload, projection, latest_created_at): return None cur.execute( """ INSERT INTO detect_sync_records ( sync_type, source_region, target_region, status, payload_json, error_message, created_at, updated_at ) VALUES (%s, %s, %s, %s, %s::jsonb, %s, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP) RETURNING id """, ( "runtime_projection", normalized_source_region, normalized_target_region, "projected", json.dumps(payload, ensure_ascii=False), "", ), ) record_id = int(cur.fetchone()[0]) conn.commit() return record_id def append_detect_result_projection_if_changed( *, detect: dict, source_region: str | None = None, target_region: str | None = None, ) -> int | None: normalized_source_region = _normalize_region(source_region, _normalize_region(settings.sync_source_region, settings.node_region)) normalized_target_region = _normalize_region(target_region, _normalize_region(settings.sync_target_region, "overseas")) active_job = detect.get("active_job") or {} if not active_job: return None latest_cycle_event = active_job.get("latest_cycle_event") or active_job.get("latest_event") or {} projection = { "job": { "job_id": active_job.get("job_id"), "job_code": active_job.get("job_code", ""), "status": active_job.get("status", ""), "progress_percent": active_job.get("progress_percent", 0), "items_total": active_job.get("items_total", 0), "items_pending": active_job.get("items_pending", 0), "items_claimed": active_job.get("items_claimed", 0), "items_running": active_job.get("items_running", 0), "items_completed": active_job.get("items_completed", 0), "items_blacklisted": active_job.get("items_blacklisted", 0), "items_failed": active_job.get("items_failed", 0), "items_terminal": active_job.get("items_terminal", 0), "current_cycle_token": active_job.get("current_cycle_token", ""), }, "latest_event": { "node_code": latest_cycle_event.get("node_code", ""), "event_type": latest_cycle_event.get("event_type", ""), "message": latest_cycle_event.get("message", ""), "created_at": latest_cycle_event.get("created_at", ""), }, "queue": { "pending": int((detect.get("progress") or {}).get("pending", 0) or 0), "running": int((detect.get("progress") or {}).get("running", 0) or 0), "completed": int((detect.get("progress") or {}).get("completed", 0) or 0), "blacklisted": int((detect.get("progress") or {}).get("blacklisted", 0) or 0), "failed": int((detect.get("progress") or {}).get("failed", 0) or 0), }, "phase": { "label": detect.get("phase_label", ""), "detail": detect.get("phase_detail", ""), }, } payload = { "projection": projection, "projection_hash": hashlib.sha1( json.dumps(projection, ensure_ascii=False, sort_keys=True).encode("utf-8") ).hexdigest(), } with get_db() as conn: with conn.cursor() as cur: cur.execute( """ SELECT payload_json, created_at FROM detect_sync_records WHERE sync_type = 'detect_result_projection' AND source_region = %s AND target_region = %s ORDER BY created_at DESC, id DESC LIMIT 1 """, (normalized_source_region, normalized_target_region), ) latest = cur.fetchone() latest_payload = _decode_json(latest[0]) if latest else {} latest_created_at = latest[1] if latest else None if latest_payload.get("projection_hash") == payload["projection_hash"]: return None latest_projection = latest_payload.get("projection") or {} latest_job = latest_projection.get("job") or {} current_job = projection.get("job") or {} latest_event = latest_projection.get("latest_event") or {} current_event = projection.get("latest_event") or {} if latest_job.get("status") == current_job.get("status") and latest_event == current_event and latest_created_at: now = datetime.now(latest_created_at.tzinfo) if latest_created_at.tzinfo else datetime.now() if now - latest_created_at < timedelta(seconds=30): return None cur.execute( """ INSERT INTO detect_sync_records ( sync_type, source_region, target_region, status, payload_json, error_message, created_at, updated_at ) VALUES (%s, %s, %s, %s, %s::jsonb, %s, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP) RETURNING id """, ( "detect_result_projection", normalized_source_region, normalized_target_region, "projected", json.dumps(payload, ensure_ascii=False), "", ), ) record_id = int(cur.fetchone()[0]) conn.commit() return record_id