from __future__ import annotations import json import socket import urllib.error import urllib.parse import urllib.request from datetime import datetime, timedelta from app.core.config import settings from app.core.db import get_db from app.services.cluster_runtime_service import register_node_heartbeat from app.services.sync_record_service import _decode_json, _normalize_region def _format_time(value: datetime | None) -> str: return value.isoformat(sep=" ", timespec="seconds") if value else "" def _ingest_url(base_url: str) -> str: text = str(base_url or "").strip().rstrip("/") if not text: return "" if text.endswith("/api/v1"): return f"{text}/runtime/sync-ingest" if text.endswith("/api/v1/runtime"): return f"{text}/sync-ingest" return f"{text}/api/v1/runtime/sync-ingest" def _projection_ingest_type(sync_type: str) -> str: if sync_type == "runtime_projection": return "runtime_ingest" if sync_type == "detect_result_projection": return "detect_result_ingest" return "sync_ingest" def _refresh_remote_runtime_node(*, source_region: str, projection: dict, received_at: datetime | None = None) -> None: node_info = projection.get("node") or {} node_code = str(node_info.get("node_code") or "").strip() region = _normalize_region(node_info.get("region"), source_region) role = str(node_info.get("role") or "control").strip() or "control" hostname = str(node_info.get("hostname") or "").strip() or socket.gethostname() ip = str(node_info.get("ip") or "").strip() if not node_code: node_code = f"{region}-{role}-imported" metadata = { "service": "runtime-ingest", "projection_source_region": source_region, "worker_mode": projection.get("worker_mode", ""), "phase_label": projection.get("phase_label", ""), "phase_detail": projection.get("phase_detail", ""), "proxy_runtime_label": projection.get("proxy_runtime_label", ""), "proxy_runtime_reason": projection.get("proxy_runtime_reason", ""), "updated_at": _format_time(received_at or datetime.now()), } register_node_heartbeat( node_code=node_code, region=region, role=role, status="online", current_load=int(((projection.get("progress") or {}).get("running", 0) or 0)), metadata=metadata, hostname_override=hostname, ip_override=ip, ) def _load_latest_projection(sync_type: str) -> dict | None: source_region = _normalize_region(settings.sync_source_region, settings.node_region) target_region = _normalize_region(settings.sync_target_region, "overseas") with get_db() as conn: with conn.cursor() as cur: cur.execute( """ SELECT id, source_region, target_region, status, payload_json, created_at, updated_at FROM detect_sync_records WHERE sync_type = %s AND source_region = %s AND target_region = %s ORDER BY created_at DESC, id DESC LIMIT 1 """, (sync_type, source_region, target_region), ) row = cur.fetchone() if not row: return None return { "id": row[0], "source_region": row[1], "target_region": row[2], "status": row[3], "payload": _decode_json(row[4]), "created_at": row[5], "updated_at": row[6], } def _load_pushable_projections(sync_type: str, limit: int) -> list[dict]: safe_limit = max(1, min(int(limit or 1), max(1, int(settings.sync_batch_size or 200)))) source_region = _normalize_region(settings.sync_source_region, settings.node_region) target_region = _normalize_region(settings.sync_target_region, "overseas") with get_db() as conn: with conn.cursor() as cur: cur.execute( """ SELECT id, source_region, target_region, status, payload_json, created_at, updated_at FROM detect_sync_records WHERE sync_type = %s AND source_region = %s AND target_region = %s ORDER BY created_at ASC, id ASC LIMIT %s """, (sync_type, source_region, target_region, safe_limit * 5), ) rows = cur.fetchall() selected: list[dict] = [] for row in rows: projection = { "id": row[0], "source_region": row[1], "target_region": row[2], "status": row[3], "payload": _decode_json(row[4]), "created_at": row[5], "updated_at": row[6], } latest_attempt = _latest_push_attempt(projection["id"], projection["target_region"], sync_type) if latest_attempt and latest_attempt["status"] == "success": continue if latest_attempt and latest_attempt["status"] == "pending": continue if latest_attempt and latest_attempt["status"] == "failed": last_created_at = latest_attempt.get("created_at") if isinstance(last_created_at, datetime): now = datetime.now(last_created_at.tzinfo) if last_created_at.tzinfo else datetime.now() if now - last_created_at < timedelta(seconds=max(10, int(settings.sync_poll_interval_seconds or 30))): continue selected.append(projection) if len(selected) >= safe_limit: break return selected def _latest_push_attempt(source_record_id: int, target_region: str, sync_type: str) -> dict | None: with get_db() as conn: with conn.cursor() as cur: cur.execute( """ SELECT id, status, payload_json, error_message, created_at, updated_at FROM detect_sync_records WHERE sync_type = 'runtime_push' AND source_region = %s AND target_region = %s AND (payload_json->>'sync_type') = %s AND (payload_json->>'source_record_id') = %s ORDER BY created_at DESC, id DESC LIMIT 1 """, ( _normalize_region(settings.sync_source_region, settings.node_region), target_region, sync_type, str(int(source_record_id)), ), ) row = cur.fetchone() if not row: return None return { "id": row[0], "status": row[1], "payload": _decode_json(row[2]), "error_message": row[3] or "", "created_at": row[4], "updated_at": row[5], } def _create_push_attempt(*, source_record: dict, ingest_url: str, sync_type: str) -> int: payload = { "sync_type": sync_type, "source_record_id": source_record["id"], "projection_hash": (source_record.get("payload") or {}).get("projection_hash", ""), "ingest_url": ingest_url, } with get_db() as conn: with conn.cursor() as cur: cur.execute( """ INSERT INTO detect_sync_records ( sync_type, source_region, target_region, status, payload_json, error_message, created_at, updated_at ) VALUES (%s, %s, %s, %s, %s::jsonb, %s, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP) RETURNING id """, ( "runtime_push", source_record["source_region"], source_record["target_region"], "pending", json.dumps(payload, ensure_ascii=False), "", ), ) record_id = int(cur.fetchone()[0]) conn.commit() return record_id def _update_push_attempt(record_id: int, *, status: str, payload: dict | None = None, error_message: str = "") -> None: with get_db() as conn: with conn.cursor() as cur: cur.execute( """ UPDATE detect_sync_records SET status = %s, payload_json = %s::jsonb, error_message = %s, updated_at = CURRENT_TIMESTAMP WHERE id = %s """, ( str(status or "").strip() or "pending", json.dumps(payload or {}, ensure_ascii=False), str(error_message or "").strip(), int(record_id), ), ) conn.commit() def ingest_runtime_projection(payload: dict, *, shared_token: str | None = None) -> tuple[bool, str, dict]: configured_token = str(settings.sync_shared_token or "").strip() incoming_token = str(shared_token or "").strip() if configured_token and incoming_token != configured_token: return False, "同步 token 校验失败", {} sync_type = str(payload.get("sync_type") or "runtime_projection").strip() or "runtime_projection" ingest_sync_type = _projection_ingest_type(sync_type) source_region = _normalize_region(payload.get("source_region"), "unknown") source_record_id = int(payload.get("source_record_id") or 0) projection_hash = str(payload.get("projection_hash") or "").strip() projection = payload.get("projection") or {} target_region = _normalize_region(settings.node_region, "overseas") received_at = datetime.now() with get_db() as conn: with conn.cursor() as cur: cur.execute( """ SELECT id FROM detect_sync_records WHERE sync_type = %s AND source_region = %s AND target_region = %s AND (payload_json->>'source_record_id') = %s AND (payload_json->>'projection_hash') = %s ORDER BY created_at DESC, id DESC LIMIT 1 """, (ingest_sync_type, source_region, target_region, str(source_record_id), projection_hash), ) existing = cur.fetchone() if existing: if sync_type == "runtime_projection": _refresh_remote_runtime_node(source_region=source_region, projection=projection, received_at=received_at) cur.execute( """ UPDATE detect_sync_records SET updated_at = CURRENT_TIMESTAMP WHERE id = %s """, (int(existing[0]),), ) conn.commit() return True, "同步投影已存在,已按幂等处理", {"record_id": int(existing[0]), "deduplicated": True} stored_payload = { "sync_type": sync_type, "source_record_id": source_record_id, "projection_hash": projection_hash, "projection": projection, "received_at": _format_time(received_at), } cur.execute( """ INSERT INTO detect_sync_records ( sync_type, source_region, target_region, status, payload_json, error_message, created_at, updated_at ) VALUES (%s, %s, %s, %s, %s::jsonb, %s, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP) RETURNING id """, ( ingest_sync_type, source_region, target_region, "received", json.dumps(stored_payload, ensure_ascii=False), "", ), ) record_id = int(cur.fetchone()[0]) conn.commit() if sync_type == "runtime_projection": _refresh_remote_runtime_node(source_region=source_region, projection=projection, received_at=received_at) return True, "同步投影接收成功", {"record_id": record_id, "deduplicated": False} def _push_projection_now(sync_type: str, ingest_url: str) -> tuple[bool, str, dict]: source_record = _load_latest_projection(sync_type) if not source_record: return False, f"当前没有可推送的{sync_type}", {"action": "push_sync", "sync_type": sync_type} return _push_projection_record(source_record, sync_type, ingest_url) def _push_projection_batch(sync_type: str, ingest_url: str) -> tuple[bool, str, dict]: pending_records = _load_pushable_projections(sync_type, limit=max(1, int(settings.sync_batch_size or 200))) if not pending_records: latest = _load_latest_projection(sync_type) if latest: latest_attempt = _latest_push_attempt(latest["id"], latest["target_region"], sync_type) if latest_attempt and latest_attempt["status"] == "success": return True, "当前批次已全部同步完成", { "action": "push_sync", "sync_type": sync_type, "deduplicated": True, "batch_count": 0, } return False, f"当前没有可推送的{sync_type}", {"action": "push_sync", "sync_type": sync_type, "batch_count": 0} results: list[dict] = [] success_count = 0 for record in pending_records: ok, message, data = _push_projection_record(record, sync_type, ingest_url) results.append( { "source_record_id": record["id"], "ok": ok, "message": message, "data": data, } ) if ok: success_count += 1 overall_ok = success_count > 0 message = f"{sync_type} 批量推送完成,成功 {success_count}/{len(results)}" return overall_ok, message, { "action": "push_sync", "sync_type": sync_type, "batch_count": len(results), "success_count": success_count, "results": results, } def _push_projection_record(source_record: dict, sync_type: str, ingest_url: str) -> tuple[bool, str, dict]: latest_attempt = _latest_push_attempt(source_record["id"], source_record["target_region"], sync_type) if latest_attempt and latest_attempt["status"] == "success": last_created_at = latest_attempt.get("created_at") if sync_type != "runtime_projection" or not isinstance(last_created_at, datetime): return True, "该投影已推送,无需重复发送", { "action": "push_sync", "sync_type": sync_type, "source_record_id": source_record["id"], "deduplicated": True, } now = datetime.now(last_created_at.tzinfo) if last_created_at.tzinfo else datetime.now() if now - last_created_at < timedelta(seconds=max(20, int(settings.sync_poll_interval_seconds or 30))): return True, "该投影已推送,无需重复发送", { "action": "push_sync", "sync_type": sync_type, "source_record_id": source_record["id"], "deduplicated": True, } if latest_attempt and latest_attempt["status"] == "pending": return True, "该投影已有同步推送进行中,暂不重复发送", { "action": "push_sync", "sync_type": sync_type, "source_record_id": source_record["id"], "deduplicated": True, } if latest_attempt and latest_attempt["status"] == "failed": last_created_at = latest_attempt.get("created_at") if isinstance(last_created_at, datetime): now = datetime.now(last_created_at.tzinfo) if last_created_at.tzinfo else datetime.now() if now - last_created_at < timedelta(seconds=max(10, int(settings.sync_poll_interval_seconds or 30))): return False, "最近一次同步推送刚失败,等待下个重试窗口", { "action": "push_sync", "sync_type": sync_type, "source_record_id": source_record["id"], "deduplicated": True, } attempt_id = _create_push_attempt(source_record=source_record, ingest_url=ingest_url, sync_type=sync_type) request_payload = { "sync_type": sync_type, "source_region": source_record["source_region"], "source_record_id": source_record["id"], "projection_hash": (source_record.get("payload") or {}).get("projection_hash", ""), "projection": (source_record.get("payload") or {}).get("projection", {}), "created_at": _format_time(source_record.get("created_at")), } request_body = json.dumps(request_payload, ensure_ascii=False).encode("utf-8") request = urllib.request.Request( ingest_url, data=request_body, headers={ "Content-Type": "application/json", **({"X-Domaincheck-Sync-Token": settings.sync_shared_token} if settings.sync_shared_token else {}), }, method="POST", ) try: with urllib.request.urlopen(request, timeout=15) as response: raw = response.read().decode("utf-8") data = json.loads(raw) if raw else {} status_code = getattr(response, "status", 200) except urllib.error.HTTPError as exc: error_body = exc.read().decode("utf-8", errors="ignore") if hasattr(exc, "read") else "" payload = { "sync_type": sync_type, "source_record_id": source_record["id"], "projection_hash": request_payload["projection_hash"], "ingest_url": ingest_url, "http_status": getattr(exc, "code", 500), "response_text": error_body[:1000], } _update_push_attempt(attempt_id, status="failed", payload=payload, error_message=f"HTTP {getattr(exc, 'code', 500)}") return False, f"同步推送失败: HTTP {getattr(exc, 'code', 500)}", {"action": "push_sync", "sync_type": sync_type, "attempt_id": attempt_id} except Exception as exc: payload = { "sync_type": sync_type, "source_record_id": source_record["id"], "projection_hash": request_payload["projection_hash"], "ingest_url": ingest_url, } _update_push_attempt(attempt_id, status="failed", payload=payload, error_message=str(exc)) return False, f"同步推送失败: {exc}", {"action": "push_sync", "sync_type": sync_type, "attempt_id": attempt_id} payload = { "sync_type": sync_type, "source_record_id": source_record["id"], "projection_hash": request_payload["projection_hash"], "ingest_url": ingest_url, "http_status": status_code, "response": data, } _update_push_attempt(attempt_id, status="success", payload=payload, error_message="") return True, "投影推送成功", { "action": "push_sync", "sync_type": sync_type, "attempt_id": attempt_id, "source_record_id": source_record["id"], "response": data, } def push_runtime_projection_now() -> tuple[bool, str, dict]: if not settings.sync_push_enabled: return False, "未启用同步推送", {"action": "push_sync"} ingest_url = _ingest_url(settings.sync_target_api_base_url) if not ingest_url: return False, "未配置同步目标地址", {"action": "push_sync"} results = [] ok, message, data = _push_projection_now("runtime_projection", ingest_url) results.append({"sync_type": "runtime_projection", "ok": ok, "message": message, "data": data}) ok, message, data = _push_projection_batch("detect_result_projection", ingest_url) results.append({"sync_type": "detect_result_projection", "ok": ok, "message": message, "data": data}) success_count = sum(1 for item in results if item["ok"]) if success_count == 0: return False, "当前没有成功推送的同步投影", {"action": "push_sync", "results": results} return True, f"同步推送完成,成功 {success_count}/{len(results)}", {"action": "push_sync", "results": results}