424 lines
17 KiB
Python
424 lines
17 KiB
Python
from __future__ import annotations
|
|
|
|
import json
|
|
import urllib.error
|
|
import urllib.parse
|
|
import urllib.request
|
|
from datetime import datetime, timedelta
|
|
|
|
from app.core.config import settings
|
|
from app.core.db import get_db
|
|
from app.services.sync_record_service import _decode_json, _normalize_region
|
|
|
|
|
|
def _format_time(value: datetime | None) -> str:
|
|
return value.isoformat(sep=" ", timespec="seconds") if value else ""
|
|
|
|
|
|
def _ingest_url(base_url: str) -> str:
|
|
text = str(base_url or "").strip().rstrip("/")
|
|
if not text:
|
|
return ""
|
|
if text.endswith("/api/v1"):
|
|
return f"{text}/runtime/sync-ingest"
|
|
if text.endswith("/api/v1/runtime"):
|
|
return f"{text}/sync-ingest"
|
|
return f"{text}/api/v1/runtime/sync-ingest"
|
|
|
|
|
|
def _projection_ingest_type(sync_type: str) -> str:
|
|
if sync_type == "runtime_projection":
|
|
return "runtime_ingest"
|
|
if sync_type == "detect_result_projection":
|
|
return "detect_result_ingest"
|
|
return "sync_ingest"
|
|
|
|
|
|
def _load_latest_projection(sync_type: str) -> dict | None:
|
|
source_region = _normalize_region(settings.sync_source_region, settings.node_region)
|
|
target_region = _normalize_region(settings.sync_target_region, "overseas")
|
|
with get_db() as conn:
|
|
with conn.cursor() as cur:
|
|
cur.execute(
|
|
"""
|
|
SELECT id, source_region, target_region, status, payload_json, created_at, updated_at
|
|
FROM detect_sync_records
|
|
WHERE sync_type = %s
|
|
AND source_region = %s
|
|
AND target_region = %s
|
|
ORDER BY created_at DESC, id DESC
|
|
LIMIT 1
|
|
""",
|
|
(sync_type, source_region, target_region),
|
|
)
|
|
row = cur.fetchone()
|
|
if not row:
|
|
return None
|
|
return {
|
|
"id": row[0],
|
|
"source_region": row[1],
|
|
"target_region": row[2],
|
|
"status": row[3],
|
|
"payload": _decode_json(row[4]),
|
|
"created_at": row[5],
|
|
"updated_at": row[6],
|
|
}
|
|
|
|
|
|
def _load_pushable_projections(sync_type: str, limit: int) -> list[dict]:
|
|
safe_limit = max(1, min(int(limit or 1), max(1, int(settings.sync_batch_size or 200))))
|
|
source_region = _normalize_region(settings.sync_source_region, settings.node_region)
|
|
target_region = _normalize_region(settings.sync_target_region, "overseas")
|
|
with get_db() as conn:
|
|
with conn.cursor() as cur:
|
|
cur.execute(
|
|
"""
|
|
SELECT id, source_region, target_region, status, payload_json, created_at, updated_at
|
|
FROM detect_sync_records
|
|
WHERE sync_type = %s
|
|
AND source_region = %s
|
|
AND target_region = %s
|
|
ORDER BY created_at ASC, id ASC
|
|
LIMIT %s
|
|
""",
|
|
(sync_type, source_region, target_region, safe_limit * 5),
|
|
)
|
|
rows = cur.fetchall()
|
|
|
|
selected: list[dict] = []
|
|
for row in rows:
|
|
projection = {
|
|
"id": row[0],
|
|
"source_region": row[1],
|
|
"target_region": row[2],
|
|
"status": row[3],
|
|
"payload": _decode_json(row[4]),
|
|
"created_at": row[5],
|
|
"updated_at": row[6],
|
|
}
|
|
latest_attempt = _latest_push_attempt(projection["id"], projection["target_region"], sync_type)
|
|
if latest_attempt and latest_attempt["status"] == "success":
|
|
continue
|
|
if latest_attempt and latest_attempt["status"] == "pending":
|
|
continue
|
|
if latest_attempt and latest_attempt["status"] == "failed":
|
|
last_created_at = latest_attempt.get("created_at")
|
|
if isinstance(last_created_at, datetime):
|
|
now = datetime.now(last_created_at.tzinfo) if last_created_at.tzinfo else datetime.now()
|
|
if now - last_created_at < timedelta(seconds=max(10, int(settings.sync_poll_interval_seconds or 30))):
|
|
continue
|
|
selected.append(projection)
|
|
if len(selected) >= safe_limit:
|
|
break
|
|
return selected
|
|
|
|
|
|
def _latest_push_attempt(source_record_id: int, target_region: str, sync_type: str) -> dict | None:
|
|
with get_db() as conn:
|
|
with conn.cursor() as cur:
|
|
cur.execute(
|
|
"""
|
|
SELECT id, status, payload_json, error_message, created_at, updated_at
|
|
FROM detect_sync_records
|
|
WHERE sync_type = 'runtime_push'
|
|
AND source_region = %s
|
|
AND target_region = %s
|
|
AND (payload_json->>'sync_type') = %s
|
|
AND (payload_json->>'source_record_id') = %s
|
|
ORDER BY created_at DESC, id DESC
|
|
LIMIT 1
|
|
""",
|
|
(
|
|
_normalize_region(settings.sync_source_region, settings.node_region),
|
|
target_region,
|
|
sync_type,
|
|
str(int(source_record_id)),
|
|
),
|
|
)
|
|
row = cur.fetchone()
|
|
if not row:
|
|
return None
|
|
return {
|
|
"id": row[0],
|
|
"status": row[1],
|
|
"payload": _decode_json(row[2]),
|
|
"error_message": row[3] or "",
|
|
"created_at": row[4],
|
|
"updated_at": row[5],
|
|
}
|
|
|
|
|
|
def _create_push_attempt(*, source_record: dict, ingest_url: str, sync_type: str) -> int:
|
|
payload = {
|
|
"sync_type": sync_type,
|
|
"source_record_id": source_record["id"],
|
|
"projection_hash": (source_record.get("payload") or {}).get("projection_hash", ""),
|
|
"ingest_url": ingest_url,
|
|
}
|
|
with get_db() as conn:
|
|
with conn.cursor() as cur:
|
|
cur.execute(
|
|
"""
|
|
INSERT INTO detect_sync_records (
|
|
sync_type, source_region, target_region, status, payload_json, error_message, created_at, updated_at
|
|
) VALUES (%s, %s, %s, %s, %s::jsonb, %s, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)
|
|
RETURNING id
|
|
""",
|
|
(
|
|
"runtime_push",
|
|
source_record["source_region"],
|
|
source_record["target_region"],
|
|
"pending",
|
|
json.dumps(payload, ensure_ascii=False),
|
|
"",
|
|
),
|
|
)
|
|
record_id = int(cur.fetchone()[0])
|
|
conn.commit()
|
|
return record_id
|
|
|
|
|
|
def _update_push_attempt(record_id: int, *, status: str, payload: dict | None = None, error_message: str = "") -> None:
|
|
with get_db() as conn:
|
|
with conn.cursor() as cur:
|
|
cur.execute(
|
|
"""
|
|
UPDATE detect_sync_records
|
|
SET status = %s,
|
|
payload_json = %s::jsonb,
|
|
error_message = %s,
|
|
updated_at = CURRENT_TIMESTAMP
|
|
WHERE id = %s
|
|
""",
|
|
(
|
|
str(status or "").strip() or "pending",
|
|
json.dumps(payload or {}, ensure_ascii=False),
|
|
str(error_message or "").strip(),
|
|
int(record_id),
|
|
),
|
|
)
|
|
conn.commit()
|
|
|
|
|
|
def ingest_runtime_projection(payload: dict, *, shared_token: str | None = None) -> tuple[bool, str, dict]:
|
|
configured_token = str(settings.sync_shared_token or "").strip()
|
|
incoming_token = str(shared_token or "").strip()
|
|
if configured_token and incoming_token != configured_token:
|
|
return False, "同步 token 校验失败", {}
|
|
|
|
sync_type = str(payload.get("sync_type") or "runtime_projection").strip() or "runtime_projection"
|
|
ingest_sync_type = _projection_ingest_type(sync_type)
|
|
source_region = _normalize_region(payload.get("source_region"), "unknown")
|
|
source_record_id = int(payload.get("source_record_id") or 0)
|
|
projection_hash = str(payload.get("projection_hash") or "").strip()
|
|
projection = payload.get("projection") or {}
|
|
target_region = _normalize_region(settings.node_region, "overseas")
|
|
|
|
with get_db() as conn:
|
|
with conn.cursor() as cur:
|
|
cur.execute(
|
|
"""
|
|
SELECT id
|
|
FROM detect_sync_records
|
|
WHERE sync_type = %s
|
|
AND source_region = %s
|
|
AND target_region = %s
|
|
AND (payload_json->>'source_record_id') = %s
|
|
AND (payload_json->>'projection_hash') = %s
|
|
ORDER BY created_at DESC, id DESC
|
|
LIMIT 1
|
|
""",
|
|
(ingest_sync_type, source_region, target_region, str(source_record_id), projection_hash),
|
|
)
|
|
existing = cur.fetchone()
|
|
if existing:
|
|
return True, "同步投影已存在，已按幂等处理", {"record_id": int(existing[0]), "deduplicated": True}
|
|
|
|
stored_payload = {
|
|
"sync_type": sync_type,
|
|
"source_record_id": source_record_id,
|
|
"projection_hash": projection_hash,
|
|
"projection": projection,
|
|
"received_at": _format_time(datetime.now()),
|
|
}
|
|
cur.execute(
|
|
"""
|
|
INSERT INTO detect_sync_records (
|
|
sync_type, source_region, target_region, status, payload_json, error_message, created_at, updated_at
|
|
) VALUES (%s, %s, %s, %s, %s::jsonb, %s, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)
|
|
RETURNING id
|
|
""",
|
|
(
|
|
ingest_sync_type,
|
|
source_region,
|
|
target_region,
|
|
"received",
|
|
json.dumps(stored_payload, ensure_ascii=False),
|
|
"",
|
|
),
|
|
)
|
|
record_id = int(cur.fetchone()[0])
|
|
conn.commit()
|
|
return True, "同步投影接收成功", {"record_id": record_id, "deduplicated": False}
|
|
|
|
|
|
def _push_projection_now(sync_type: str, ingest_url: str) -> tuple[bool, str, dict]:
|
|
source_record = _load_latest_projection(sync_type)
|
|
if not source_record:
|
|
return False, f"当前没有可推送的{sync_type}", {"action": "push_sync", "sync_type": sync_type}
|
|
return _push_projection_record(source_record, sync_type, ingest_url)
|
|
|
|
|
|
def _push_projection_batch(sync_type: str, ingest_url: str) -> tuple[bool, str, dict]:
|
|
pending_records = _load_pushable_projections(sync_type, limit=max(1, int(settings.sync_batch_size or 200)))
|
|
if not pending_records:
|
|
latest = _load_latest_projection(sync_type)
|
|
if latest:
|
|
latest_attempt = _latest_push_attempt(latest["id"], latest["target_region"], sync_type)
|
|
if latest_attempt and latest_attempt["status"] == "success":
|
|
return True, "当前批次已全部同步完成", {
|
|
"action": "push_sync",
|
|
"sync_type": sync_type,
|
|
"deduplicated": True,
|
|
"batch_count": 0,
|
|
}
|
|
return False, f"当前没有可推送的{sync_type}", {"action": "push_sync", "sync_type": sync_type, "batch_count": 0}
|
|
|
|
results: list[dict] = []
|
|
success_count = 0
|
|
for record in pending_records:
|
|
ok, message, data = _push_projection_record(record, sync_type, ingest_url)
|
|
results.append(
|
|
{
|
|
"source_record_id": record["id"],
|
|
"ok": ok,
|
|
"message": message,
|
|
"data": data,
|
|
}
|
|
)
|
|
if ok:
|
|
success_count += 1
|
|
|
|
overall_ok = success_count > 0
|
|
message = f"{sync_type} 批量推送完成，成功 {success_count}/{len(results)}"
|
|
return overall_ok, message, {
|
|
"action": "push_sync",
|
|
"sync_type": sync_type,
|
|
"batch_count": len(results),
|
|
"success_count": success_count,
|
|
"results": results,
|
|
}
|
|
|
|
|
|
def _push_projection_record(source_record: dict, sync_type: str, ingest_url: str) -> tuple[bool, str, dict]:
|
|
latest_attempt = _latest_push_attempt(source_record["id"], source_record["target_region"], sync_type)
|
|
if latest_attempt and latest_attempt["status"] == "success":
|
|
return True, "该投影已推送，无需重复发送", {
|
|
"action": "push_sync",
|
|
"sync_type": sync_type,
|
|
"source_record_id": source_record["id"],
|
|
"deduplicated": True,
|
|
}
|
|
if latest_attempt and latest_attempt["status"] == "pending":
|
|
return True, "该投影已有同步推送进行中，暂不重复发送", {
|
|
"action": "push_sync",
|
|
"sync_type": sync_type,
|
|
"source_record_id": source_record["id"],
|
|
"deduplicated": True,
|
|
}
|
|
if latest_attempt and latest_attempt["status"] == "failed":
|
|
last_created_at = latest_attempt.get("created_at")
|
|
if isinstance(last_created_at, datetime):
|
|
now = datetime.now(last_created_at.tzinfo) if last_created_at.tzinfo else datetime.now()
|
|
if now - last_created_at < timedelta(seconds=max(10, int(settings.sync_poll_interval_seconds or 30))):
|
|
return False, "最近一次同步推送刚失败，等待下个重试窗口", {
|
|
"action": "push_sync",
|
|
"sync_type": sync_type,
|
|
"source_record_id": source_record["id"],
|
|
"deduplicated": True,
|
|
}
|
|
|
|
attempt_id = _create_push_attempt(source_record=source_record, ingest_url=ingest_url, sync_type=sync_type)
|
|
request_payload = {
|
|
"sync_type": sync_type,
|
|
"source_region": source_record["source_region"],
|
|
"source_record_id": source_record["id"],
|
|
"projection_hash": (source_record.get("payload") or {}).get("projection_hash", ""),
|
|
"projection": (source_record.get("payload") or {}).get("projection", {}),
|
|
"created_at": _format_time(source_record.get("created_at")),
|
|
}
|
|
request_body = json.dumps(request_payload, ensure_ascii=False).encode("utf-8")
|
|
request = urllib.request.Request(
|
|
ingest_url,
|
|
data=request_body,
|
|
headers={
|
|
"Content-Type": "application/json",
|
|
**({"X-Domaincheck-Sync-Token": settings.sync_shared_token} if settings.sync_shared_token else {}),
|
|
},
|
|
method="POST",
|
|
)
|
|
try:
|
|
with urllib.request.urlopen(request, timeout=15) as response:
|
|
raw = response.read().decode("utf-8")
|
|
data = json.loads(raw) if raw else {}
|
|
status_code = getattr(response, "status", 200)
|
|
except urllib.error.HTTPError as exc:
|
|
error_body = exc.read().decode("utf-8", errors="ignore") if hasattr(exc, "read") else ""
|
|
payload = {
|
|
"sync_type": sync_type,
|
|
"source_record_id": source_record["id"],
|
|
"projection_hash": request_payload["projection_hash"],
|
|
"ingest_url": ingest_url,
|
|
"http_status": getattr(exc, "code", 500),
|
|
"response_text": error_body[:1000],
|
|
}
|
|
_update_push_attempt(attempt_id, status="failed", payload=payload, error_message=f"HTTP {getattr(exc, 'code', 500)}")
|
|
return False, f"同步推送失败: HTTP {getattr(exc, 'code', 500)}", {"action": "push_sync", "sync_type": sync_type, "attempt_id": attempt_id}
|
|
except Exception as exc:
|
|
payload = {
|
|
"sync_type": sync_type,
|
|
"source_record_id": source_record["id"],
|
|
"projection_hash": request_payload["projection_hash"],
|
|
"ingest_url": ingest_url,
|
|
}
|
|
_update_push_attempt(attempt_id, status="failed", payload=payload, error_message=str(exc))
|
|
return False, f"同步推送失败: {exc}", {"action": "push_sync", "sync_type": sync_type, "attempt_id": attempt_id}
|
|
|
|
payload = {
|
|
"sync_type": sync_type,
|
|
"source_record_id": source_record["id"],
|
|
"projection_hash": request_payload["projection_hash"],
|
|
"ingest_url": ingest_url,
|
|
"http_status": status_code,
|
|
"response": data,
|
|
}
|
|
_update_push_attempt(attempt_id, status="success", payload=payload, error_message="")
|
|
return True, "投影推送成功", {
|
|
"action": "push_sync",
|
|
"sync_type": sync_type,
|
|
"attempt_id": attempt_id,
|
|
"source_record_id": source_record["id"],
|
|
"response": data,
|
|
}
|
|
|
|
|
|
def push_runtime_projection_now() -> tuple[bool, str, dict]:
|
|
if not settings.sync_push_enabled:
|
|
return False, "未启用同步推送", {"action": "push_sync"}
|
|
|
|
ingest_url = _ingest_url(settings.sync_target_api_base_url)
|
|
if not ingest_url:
|
|
return False, "未配置同步目标地址", {"action": "push_sync"}
|
|
|
|
results = []
|
|
ok, message, data = _push_projection_now("runtime_projection", ingest_url)
|
|
results.append({"sync_type": "runtime_projection", "ok": ok, "message": message, "data": data})
|
|
|
|
ok, message, data = _push_projection_batch("detect_result_projection", ingest_url)
|
|
results.append({"sync_type": "detect_result_projection", "ok": ok, "message": message, "data": data})
|
|
|
|
success_count = sum(1 for item in results if item["ok"])
|
|
if success_count == 0:
|
|
return False, "当前没有成功推送的同步投影", {"action": "push_sync", "results": results}
|
|
return True, f"同步推送完成，成功 {success_count}/{len(results)}", {"action": "push_sync", "results": results}
|