1086 lines
46 KiB
Python
1086 lines
46 KiB
Python
from __future__ import annotations
|
|
|
|
import json
|
|
import socket
|
|
import urllib.error
|
|
import urllib.parse
|
|
import urllib.request
|
|
from datetime import datetime, timedelta
|
|
from uuid import uuid4
|
|
|
|
from app.core.config import settings
|
|
from app.core.db import get_db
|
|
from app.services.cluster_runtime_service import cleanup_imported_runtime_nodes, register_node_heartbeat
|
|
from app.services.sync_record_service import _decode_json, _normalize_region
|
|
|
|
|
|
def _format_time(value: datetime | None) -> str:
|
|
return value.isoformat(sep=" ", timespec="seconds") if value else ""
|
|
|
|
|
|
def _ingest_url(base_url: str) -> str:
|
|
text = str(base_url or "").strip().rstrip("/")
|
|
if not text:
|
|
return ""
|
|
if text.endswith("/api/v1"):
|
|
return f"{text}/runtime/sync-ingest"
|
|
if text.endswith("/api/v1/runtime"):
|
|
return f"{text}/sync-ingest"
|
|
return f"{text}/api/v1/runtime/sync-ingest"
|
|
|
|
|
|
def _task_export_url(base_url: str) -> str:
|
|
text = str(base_url or "").strip().rstrip("/")
|
|
if not text:
|
|
return ""
|
|
if text.endswith("/api/v1"):
|
|
return f"{text}/runtime/task-export"
|
|
if text.endswith("/api/v1/runtime"):
|
|
return f"{text}/task-export"
|
|
return f"{text}/api/v1/runtime/task-export"
|
|
|
|
|
|
def _task_ack_url(base_url: str) -> str:
|
|
text = str(base_url or "").strip().rstrip("/")
|
|
if not text:
|
|
return ""
|
|
if text.endswith("/api/v1"):
|
|
return f"{text}/runtime/task-ack"
|
|
if text.endswith("/api/v1/runtime"):
|
|
return f"{text}/task-ack"
|
|
return f"{text}/api/v1/runtime/task-ack"
|
|
|
|
|
|
def _projection_ingest_type(sync_type: str) -> str:
|
|
if sync_type == "runtime_projection":
|
|
return "runtime_ingest"
|
|
if sync_type == "detect_result_projection":
|
|
return "detect_result_ingest"
|
|
if sync_type == "detect_task_projection":
|
|
return "detect_task_ingest"
|
|
return "sync_ingest"
|
|
|
|
|
|
def _refresh_remote_runtime_node(*, source_region: str, projection: dict, received_at: datetime | None = None) -> None:
|
|
node_info = projection.get("node") or {}
|
|
node_code = str(node_info.get("node_code") or "").strip()
|
|
region = _normalize_region(node_info.get("region"), source_region)
|
|
role = str(node_info.get("role") or "control").strip() or "control"
|
|
hostname = str(node_info.get("hostname") or "").strip() or socket.gethostname()
|
|
ip = str(node_info.get("ip") or "").strip()
|
|
if not node_code:
|
|
node_code = f"{region}-{role}-imported"
|
|
|
|
metadata = {
|
|
"service": "runtime-ingest",
|
|
"projection_source_region": source_region,
|
|
"worker_mode": projection.get("worker_mode", ""),
|
|
"phase_label": projection.get("phase_label", ""),
|
|
"phase_detail": projection.get("phase_detail", ""),
|
|
"proxy_runtime_label": projection.get("proxy_runtime_label", ""),
|
|
"proxy_runtime_reason": projection.get("proxy_runtime_reason", ""),
|
|
"updated_at": _format_time(received_at or datetime.now()),
|
|
}
|
|
register_node_heartbeat(
|
|
node_code=node_code,
|
|
region=region,
|
|
role=role,
|
|
status="online",
|
|
current_load=int(((projection.get("progress") or {}).get("running", 0) or 0)),
|
|
metadata=metadata,
|
|
hostname_override=hostname,
|
|
ip_override=ip,
|
|
)
|
|
cleanup_imported_runtime_nodes(region=region, role=role, keep_node_code=node_code)
|
|
|
|
active_job = projection.get("active_job") or {}
|
|
for node_stat in list(active_job.get("node_stats") or []):
|
|
worker_node_code = str(node_stat.get("node_code") or "").strip()
|
|
if not worker_node_code or worker_node_code == "unassigned":
|
|
continue
|
|
items_running = int(node_stat.get("items_running", 0) or 0)
|
|
items_claimed = int(node_stat.get("items_claimed", 0) or 0)
|
|
items_total = int(node_stat.get("items_total", 0) or 0)
|
|
worker_status = "busy" if (items_running > 0 or items_claimed > 0) else "online"
|
|
worker_load = max(items_running, items_claimed, 0)
|
|
worker_metadata = {
|
|
"service": "runtime-ingest",
|
|
"projection_source_region": source_region,
|
|
"worker_mode": projection.get("worker_mode", ""),
|
|
"phase_label": projection.get("phase_label", ""),
|
|
"phase_detail": projection.get("phase_detail", ""),
|
|
"proxy_runtime_label": projection.get("proxy_runtime_label", ""),
|
|
"proxy_runtime_reason": projection.get("proxy_runtime_reason", ""),
|
|
"updated_at": _format_time(received_at or datetime.now()),
|
|
"job_items_total": items_total,
|
|
"job_items_running": items_running,
|
|
"job_items_claimed": items_claimed,
|
|
"derived_from": node_code,
|
|
}
|
|
register_node_heartbeat(
|
|
node_code=worker_node_code,
|
|
region=region,
|
|
role="worker",
|
|
status=worker_status,
|
|
current_load=worker_load,
|
|
metadata=worker_metadata,
|
|
hostname_override=hostname,
|
|
ip_override=ip,
|
|
)
|
|
cleanup_imported_runtime_nodes(region=region, role="worker", keep_node_code=worker_node_code)
|
|
|
|
|
|
def _load_latest_projection(sync_type: str) -> dict | None:
|
|
source_region = _normalize_region(settings.sync_source_region, settings.node_region)
|
|
target_region = _normalize_region(settings.sync_target_region, "overseas")
|
|
with get_db() as conn:
|
|
with conn.cursor() as cur:
|
|
cur.execute(
|
|
"""
|
|
SELECT id, source_region, target_region, status, payload_json, created_at, updated_at
|
|
FROM detect_sync_records
|
|
WHERE sync_type = %s
|
|
AND source_region = %s
|
|
AND target_region = %s
|
|
ORDER BY created_at DESC, id DESC
|
|
LIMIT 1
|
|
""",
|
|
(sync_type, source_region, target_region),
|
|
)
|
|
row = cur.fetchone()
|
|
if not row:
|
|
return None
|
|
return {
|
|
"id": row[0],
|
|
"source_region": row[1],
|
|
"target_region": row[2],
|
|
"status": row[3],
|
|
"payload": _decode_json(row[4]),
|
|
"created_at": row[5],
|
|
"updated_at": row[6],
|
|
}
|
|
|
|
|
|
def _load_pushable_projections(sync_type: str, limit: int) -> list[dict]:
|
|
safe_limit = max(1, min(int(limit or 1), max(1, int(settings.sync_batch_size or 200))))
|
|
source_region = _normalize_region(settings.sync_source_region, settings.node_region)
|
|
target_region = _normalize_region(settings.sync_target_region, "overseas")
|
|
with get_db() as conn:
|
|
with conn.cursor() as cur:
|
|
cur.execute(
|
|
"""
|
|
SELECT id, source_region, target_region, status, payload_json, created_at, updated_at
|
|
FROM detect_sync_records
|
|
WHERE sync_type = %s
|
|
AND source_region = %s
|
|
AND target_region = %s
|
|
ORDER BY created_at ASC, id ASC
|
|
LIMIT %s
|
|
""",
|
|
(sync_type, source_region, target_region, safe_limit * 5),
|
|
)
|
|
rows = cur.fetchall()
|
|
|
|
selected: list[dict] = []
|
|
for row in rows:
|
|
projection = {
|
|
"id": row[0],
|
|
"source_region": row[1],
|
|
"target_region": row[2],
|
|
"status": row[3],
|
|
"payload": _decode_json(row[4]),
|
|
"created_at": row[5],
|
|
"updated_at": row[6],
|
|
}
|
|
latest_attempt = _latest_push_attempt(projection["id"], projection["target_region"], sync_type)
|
|
if latest_attempt and latest_attempt["status"] == "success":
|
|
continue
|
|
if latest_attempt and latest_attempt["status"] == "pending":
|
|
continue
|
|
if latest_attempt and latest_attempt["status"] == "failed":
|
|
last_created_at = latest_attempt.get("created_at")
|
|
if isinstance(last_created_at, datetime):
|
|
now = datetime.now(last_created_at.tzinfo) if last_created_at.tzinfo else datetime.now()
|
|
if now - last_created_at < timedelta(seconds=max(10, int(settings.sync_poll_interval_seconds or 30))):
|
|
continue
|
|
selected.append(projection)
|
|
if len(selected) >= safe_limit:
|
|
break
|
|
return selected
|
|
|
|
|
|
def _latest_push_attempt(source_record_id: int, target_region: str, sync_type: str) -> dict | None:
|
|
with get_db() as conn:
|
|
with conn.cursor() as cur:
|
|
cur.execute(
|
|
"""
|
|
SELECT id, status, payload_json, error_message, created_at, updated_at
|
|
FROM detect_sync_records
|
|
WHERE sync_type = 'runtime_push'
|
|
AND source_region = %s
|
|
AND target_region = %s
|
|
AND (payload_json->>'sync_type') = %s
|
|
AND (payload_json->>'source_record_id') = %s
|
|
ORDER BY created_at DESC, id DESC
|
|
LIMIT 1
|
|
""",
|
|
(
|
|
_normalize_region(settings.sync_source_region, settings.node_region),
|
|
target_region,
|
|
sync_type,
|
|
str(int(source_record_id)),
|
|
),
|
|
)
|
|
row = cur.fetchone()
|
|
if not row:
|
|
return None
|
|
return {
|
|
"id": row[0],
|
|
"status": row[1],
|
|
"payload": _decode_json(row[2]),
|
|
"error_message": row[3] or "",
|
|
"created_at": row[4],
|
|
"updated_at": row[5],
|
|
}
|
|
|
|
|
|
def _latest_ingest_attempt(source_record_id: int, target_region: str, sync_type: str) -> dict | None:
|
|
ingest_sync_type = _projection_ingest_type(sync_type)
|
|
with get_db() as conn:
|
|
with conn.cursor() as cur:
|
|
cur.execute(
|
|
"""
|
|
SELECT id, status, payload_json, error_message, created_at, updated_at
|
|
FROM detect_sync_records
|
|
WHERE sync_type = %s
|
|
AND source_region = %s
|
|
AND target_region = %s
|
|
AND (payload_json->>'source_record_id') = %s
|
|
ORDER BY created_at DESC, id DESC
|
|
LIMIT 1
|
|
""",
|
|
(
|
|
ingest_sync_type,
|
|
_normalize_region(settings.sync_source_region, settings.node_region),
|
|
target_region,
|
|
str(int(source_record_id)),
|
|
),
|
|
)
|
|
row = cur.fetchone()
|
|
if not row:
|
|
return None
|
|
return {
|
|
"id": row[0],
|
|
"status": row[1],
|
|
"payload": _decode_json(row[2]),
|
|
"error_message": row[3] or "",
|
|
"created_at": row[4],
|
|
"updated_at": row[5],
|
|
}
|
|
|
|
|
|
def _task_selection_sql() -> str:
|
|
return """
|
|
SELECT id, domain, tld, source_type, use_status, detect_status, register_status, expire_date, jucha_status, juziseo_status
|
|
FROM domains
|
|
WHERE
|
|
detect_status IN (0, 4)
|
|
OR (use_status = 0 AND detect_status = 1 AND register_status = 3 AND expire_date < CURRENT_DATE)
|
|
ORDER BY id ASC
|
|
LIMIT %s
|
|
"""
|
|
|
|
|
|
def _load_pending_task_projection(limit: int) -> dict | None:
|
|
source_region = _normalize_region(settings.sync_source_region, settings.node_region)
|
|
target_region = _normalize_region(settings.sync_target_region, "overseas")
|
|
safe_limit = max(1, min(int(limit or 1000), max(1, int(settings.sync_batch_size or 200))))
|
|
with get_db() as conn:
|
|
with conn.cursor() as cur:
|
|
cur.execute(
|
|
"""
|
|
SELECT id, source_region, target_region, status, payload_json, created_at, updated_at
|
|
FROM detect_sync_records
|
|
WHERE sync_type = 'detect_task_projection'
|
|
AND source_region = %s
|
|
AND target_region = %s
|
|
ORDER BY created_at ASC, id ASC
|
|
LIMIT %s
|
|
""",
|
|
(source_region, target_region, safe_limit * 3),
|
|
)
|
|
rows = cur.fetchall()
|
|
for row in rows:
|
|
projection = {
|
|
"id": row[0],
|
|
"source_region": row[1],
|
|
"target_region": row[2],
|
|
"status": row[3],
|
|
"payload": _decode_json(row[4]),
|
|
"created_at": row[5],
|
|
"updated_at": row[6],
|
|
}
|
|
latest_ingest = _latest_ingest_attempt(projection["id"], projection["target_region"], "detect_task_projection")
|
|
if latest_ingest and latest_ingest["status"] == "received":
|
|
continue
|
|
return projection
|
|
return None
|
|
|
|
|
|
def export_detect_task_projection(limit: int = 1000, *, shared_token: str | None = None) -> tuple[bool, str, dict]:
|
|
configured_token = str(settings.sync_shared_token or "").strip()
|
|
incoming_token = str(shared_token or "").strip()
|
|
if configured_token and incoming_token != configured_token:
|
|
return False, "同步 token 校验失败", {}
|
|
|
|
existing = _load_pending_task_projection(limit)
|
|
if existing:
|
|
return True, "已返回待确认的任务批次", {
|
|
"source_record_id": existing["id"],
|
|
"projection_hash": (existing.get("payload") or {}).get("projection_hash", ""),
|
|
"projection": (existing.get("payload") or {}).get("projection", {}),
|
|
"deduplicated": True,
|
|
}
|
|
|
|
source_region = _normalize_region(settings.sync_source_region, settings.node_region)
|
|
target_region = _normalize_region(settings.sync_target_region, "overseas")
|
|
safe_limit = max(1, min(int(limit or 1000), max(1, int(settings.sync_batch_size or 200))))
|
|
with get_db() as conn:
|
|
with conn.cursor() as cur:
|
|
cur.execute(_task_selection_sql(), (safe_limit,))
|
|
rows = cur.fetchall()
|
|
if not rows:
|
|
return False, "当前没有可下发的待检测任务", {"batch_size": 0}
|
|
|
|
items = []
|
|
for row in rows:
|
|
items.append(
|
|
{
|
|
"source_domain_id": int(row[0]),
|
|
"domain": str(row[1] or "").strip(),
|
|
"tld": str(row[2] or "").strip(),
|
|
"source_type": int(row[3] or 0),
|
|
"use_status": int(row[4] or 0),
|
|
"detect_status": int(row[5] or 0),
|
|
"register_status": int(row[6] or 0),
|
|
"expire_date": row[7].isoformat(sep=" ", timespec="seconds") if row[7] else "",
|
|
"jucha_status": int(row[8] or 0),
|
|
"juziseo_status": int(row[9] or 0),
|
|
}
|
|
)
|
|
|
|
projection = {
|
|
"batch_code": f"task-{datetime.now().strftime('%Y%m%d%H%M%S')}-{uuid4().hex[:6]}",
|
|
"items_total": len(items),
|
|
"selection_limit": safe_limit,
|
|
"items": items,
|
|
}
|
|
payload = {
|
|
"projection": projection,
|
|
"projection_hash": f"{uuid4().hex}-{len(items)}",
|
|
}
|
|
cur.execute(
|
|
"""
|
|
INSERT INTO detect_sync_records (
|
|
sync_type, source_region, target_region, status, payload_json, error_message, created_at, updated_at
|
|
) VALUES (%s, %s, %s, %s, %s::jsonb, %s, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)
|
|
RETURNING id
|
|
""",
|
|
(
|
|
"detect_task_projection",
|
|
source_region,
|
|
target_region,
|
|
"projected",
|
|
json.dumps(payload, ensure_ascii=False),
|
|
"",
|
|
),
|
|
)
|
|
record_id = int(cur.fetchone()[0])
|
|
conn.commit()
|
|
return True, "待检测任务批次已生成", {
|
|
"source_record_id": record_id,
|
|
"projection_hash": payload["projection_hash"],
|
|
"projection": projection,
|
|
"deduplicated": False,
|
|
}
|
|
|
|
|
|
def ack_detect_task_projection(payload: dict, *, shared_token: str | None = None) -> tuple[bool, str, dict]:
|
|
configured_token = str(settings.sync_shared_token or "").strip()
|
|
incoming_token = str(shared_token or "").strip()
|
|
if configured_token and incoming_token != configured_token:
|
|
return False, "同步 token 校验失败", {}
|
|
|
|
source_record_id = int(payload.get("source_record_id") or 0)
|
|
projection_hash = str(payload.get("projection_hash") or "").strip()
|
|
if source_record_id <= 0 or not projection_hash:
|
|
return False, "任务确认参数不完整", {}
|
|
|
|
source_region = _normalize_region(settings.sync_source_region, settings.node_region)
|
|
target_region = _normalize_region(settings.sync_target_region, "overseas")
|
|
with get_db() as conn:
|
|
with conn.cursor() as cur:
|
|
cur.execute(
|
|
"""
|
|
SELECT id
|
|
FROM detect_sync_records
|
|
WHERE sync_type = 'detect_task_ingest'
|
|
AND source_region = %s
|
|
AND target_region = %s
|
|
AND (payload_json->>'source_record_id') = %s
|
|
AND (payload_json->>'projection_hash') = %s
|
|
ORDER BY created_at DESC, id DESC
|
|
LIMIT 1
|
|
""",
|
|
(source_region, target_region, str(source_record_id), projection_hash),
|
|
)
|
|
existing = cur.fetchone()
|
|
if existing:
|
|
return True, "任务批次确认已存在", {"record_id": int(existing[0]), "deduplicated": True}
|
|
|
|
cur.execute(
|
|
"""
|
|
INSERT INTO detect_sync_records (
|
|
sync_type, source_region, target_region, status, payload_json, error_message, created_at, updated_at
|
|
) VALUES (%s, %s, %s, %s, %s::jsonb, %s, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)
|
|
RETURNING id
|
|
""",
|
|
(
|
|
"detect_task_ingest",
|
|
source_region,
|
|
target_region,
|
|
"received",
|
|
json.dumps(
|
|
{
|
|
"source_record_id": source_record_id,
|
|
"projection_hash": projection_hash,
|
|
"received_at": _format_time(datetime.now()),
|
|
},
|
|
ensure_ascii=False,
|
|
),
|
|
"",
|
|
),
|
|
)
|
|
record_id = int(cur.fetchone()[0])
|
|
conn.commit()
|
|
return True, "任务批次确认成功", {"record_id": record_id, "deduplicated": False}
|
|
|
|
|
|
def ingest_detect_task_projection(payload: dict, *, shared_token: str | None = None) -> tuple[bool, str, dict]:
|
|
configured_token = str(settings.sync_shared_token or "").strip()
|
|
incoming_token = str(shared_token or "").strip()
|
|
if configured_token and incoming_token != configured_token:
|
|
return False, "同步 token 校验失败", {}
|
|
|
|
sync_type = str(payload.get("sync_type") or "detect_task_projection").strip() or "detect_task_projection"
|
|
if sync_type != "detect_task_projection":
|
|
return False, "同步类型不匹配", {"sync_type": sync_type}
|
|
|
|
source_region = _normalize_region(payload.get("source_region"), "unknown")
|
|
source_record_id = int(payload.get("source_record_id") or 0)
|
|
projection_hash = str(payload.get("projection_hash") or "").strip()
|
|
projection = payload.get("projection") or {}
|
|
items = list(projection.get("items") or [])
|
|
target_region = _normalize_region(settings.node_region, "overseas")
|
|
received_at = datetime.now()
|
|
|
|
if source_record_id <= 0 or not projection_hash:
|
|
return False, "任务批次参数不完整", {}
|
|
if not items:
|
|
return False, "任务批次为空", {"source_record_id": source_record_id}
|
|
|
|
with get_db() as conn:
|
|
with conn.cursor() as cur:
|
|
cur.execute(
|
|
"""
|
|
SELECT id
|
|
FROM detect_sync_records
|
|
WHERE sync_type = 'detect_task_ingest'
|
|
AND source_region = %s
|
|
AND target_region = %s
|
|
AND (payload_json->>'source_record_id') = %s
|
|
AND (payload_json->>'projection_hash') = %s
|
|
ORDER BY created_at DESC, id DESC
|
|
LIMIT 1
|
|
""",
|
|
(source_region, target_region, str(source_record_id), projection_hash),
|
|
)
|
|
existing = cur.fetchone()
|
|
if existing:
|
|
return True, "任务批次已接收，已按幂等处理", {"record_id": int(existing[0]), "deduplicated": True}
|
|
|
|
inserted_count = 0
|
|
updated_count = 0
|
|
for item in items:
|
|
domain = str(item.get("domain") or "").strip().lower()
|
|
if not domain:
|
|
continue
|
|
tld = str(item.get("tld") or "").strip()
|
|
if not tld and "." in domain:
|
|
tld = domain.rsplit(".", 1)[-1]
|
|
expire_date_text = str(item.get("expire_date") or "").strip()
|
|
cur.execute(
|
|
"""
|
|
INSERT INTO domains (
|
|
domain, tld, source_type, use_status, detect_status, register_status,
|
|
has_beian, company_type, website_url, beian_year, snapshot_years,
|
|
expire_date, create_time, update_time, review_status, detect_time,
|
|
backlink_count, jucha_status, juziseo_status
|
|
) VALUES (
|
|
%s, %s, %s, %s, %s, %s,
|
|
1, NULL, NULL, NULL, NULL,
|
|
NULLIF(%s, '')::timestamp, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP, 0, NULL,
|
|
0, %s, %s
|
|
)
|
|
ON CONFLICT (domain) DO UPDATE SET
|
|
tld = EXCLUDED.tld,
|
|
source_type = EXCLUDED.source_type,
|
|
use_status = EXCLUDED.use_status,
|
|
register_status = EXCLUDED.register_status,
|
|
expire_date = COALESCE(EXCLUDED.expire_date, domains.expire_date),
|
|
jucha_status = EXCLUDED.jucha_status,
|
|
juziseo_status = EXCLUDED.juziseo_status,
|
|
detect_status = CASE
|
|
WHEN domains.detect_status IN (1, 2, 3, 4) THEN domains.detect_status
|
|
ELSE EXCLUDED.detect_status
|
|
END,
|
|
update_time = CURRENT_TIMESTAMP
|
|
RETURNING (xmax = 0) AS inserted
|
|
""",
|
|
(
|
|
domain,
|
|
tld,
|
|
int(item.get("source_type") or 0),
|
|
int(item.get("use_status") or 0),
|
|
int(item.get("detect_status") or 0),
|
|
int(item.get("register_status") or 0),
|
|
expire_date_text,
|
|
int(item.get("jucha_status") or 0),
|
|
int(item.get("juziseo_status") or 0),
|
|
),
|
|
)
|
|
inserted = bool((cur.fetchone() or [False])[0])
|
|
if inserted:
|
|
inserted_count += 1
|
|
else:
|
|
updated_count += 1
|
|
|
|
cur.execute(
|
|
"""
|
|
INSERT INTO detect_sync_records (
|
|
sync_type, source_region, target_region, status, payload_json, error_message, created_at, updated_at
|
|
) VALUES (%s, %s, %s, %s, %s::jsonb, %s, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)
|
|
RETURNING id
|
|
""",
|
|
(
|
|
"detect_task_ingest",
|
|
source_region,
|
|
target_region,
|
|
"received",
|
|
json.dumps(
|
|
{
|
|
"source_record_id": source_record_id,
|
|
"projection_hash": projection_hash,
|
|
"batch_code": str(projection.get("batch_code") or "").strip(),
|
|
"items_total": len(items),
|
|
"inserted_count": inserted_count,
|
|
"updated_count": updated_count,
|
|
"received_at": _format_time(received_at),
|
|
},
|
|
ensure_ascii=False,
|
|
),
|
|
"",
|
|
),
|
|
)
|
|
record_id = int(cur.fetchone()[0])
|
|
conn.commit()
|
|
return True, "任务批次接收成功", {
|
|
"record_id": record_id,
|
|
"source_record_id": source_record_id,
|
|
"items_total": len(items),
|
|
"inserted_count": inserted_count,
|
|
"updated_count": updated_count,
|
|
"deduplicated": False,
|
|
}
|
|
|
|
|
|
def _create_push_attempt(*, source_record: dict, ingest_url: str, sync_type: str) -> int:
|
|
payload = {
|
|
"sync_type": sync_type,
|
|
"source_record_id": source_record["id"],
|
|
"projection_hash": (source_record.get("payload") or {}).get("projection_hash", ""),
|
|
"ingest_url": ingest_url,
|
|
}
|
|
with get_db() as conn:
|
|
with conn.cursor() as cur:
|
|
cur.execute(
|
|
"""
|
|
INSERT INTO detect_sync_records (
|
|
sync_type, source_region, target_region, status, payload_json, error_message, created_at, updated_at
|
|
) VALUES (%s, %s, %s, %s, %s::jsonb, %s, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)
|
|
RETURNING id
|
|
""",
|
|
(
|
|
"runtime_push",
|
|
source_record["source_region"],
|
|
source_record["target_region"],
|
|
"pending",
|
|
json.dumps(payload, ensure_ascii=False),
|
|
"",
|
|
),
|
|
)
|
|
record_id = int(cur.fetchone()[0])
|
|
conn.commit()
|
|
return record_id
|
|
|
|
|
|
def _update_push_attempt(record_id: int, *, status: str, payload: dict | None = None, error_message: str = "") -> None:
|
|
with get_db() as conn:
|
|
with conn.cursor() as cur:
|
|
cur.execute(
|
|
"""
|
|
UPDATE detect_sync_records
|
|
SET status = %s,
|
|
payload_json = %s::jsonb,
|
|
error_message = %s,
|
|
updated_at = CURRENT_TIMESTAMP
|
|
WHERE id = %s
|
|
""",
|
|
(
|
|
str(status or "").strip() or "pending",
|
|
json.dumps(payload or {}, ensure_ascii=False),
|
|
str(error_message or "").strip(),
|
|
int(record_id),
|
|
),
|
|
)
|
|
conn.commit()
|
|
|
|
|
|
def ingest_runtime_projection(payload: dict, *, shared_token: str | None = None) -> tuple[bool, str, dict]:
|
|
configured_token = str(settings.sync_shared_token or "").strip()
|
|
incoming_token = str(shared_token or "").strip()
|
|
if configured_token and incoming_token != configured_token:
|
|
return False, "同步 token 校验失败", {}
|
|
|
|
sync_type = str(payload.get("sync_type") or "runtime_projection").strip() or "runtime_projection"
|
|
ingest_sync_type = _projection_ingest_type(sync_type)
|
|
source_region = _normalize_region(payload.get("source_region"), "unknown")
|
|
source_record_id = int(payload.get("source_record_id") or 0)
|
|
projection_hash = str(payload.get("projection_hash") or "").strip()
|
|
projection = payload.get("projection") or {}
|
|
target_region = _normalize_region(settings.node_region, "overseas")
|
|
received_at = datetime.now()
|
|
|
|
with get_db() as conn:
|
|
with conn.cursor() as cur:
|
|
cur.execute(
|
|
"""
|
|
SELECT id
|
|
FROM detect_sync_records
|
|
WHERE sync_type = %s
|
|
AND source_region = %s
|
|
AND target_region = %s
|
|
AND (payload_json->>'source_record_id') = %s
|
|
AND (payload_json->>'projection_hash') = %s
|
|
ORDER BY created_at DESC, id DESC
|
|
LIMIT 1
|
|
""",
|
|
(ingest_sync_type, source_region, target_region, str(source_record_id), projection_hash),
|
|
)
|
|
existing = cur.fetchone()
|
|
if existing:
|
|
if sync_type == "runtime_projection":
|
|
_refresh_remote_runtime_node(source_region=source_region, projection=projection, received_at=received_at)
|
|
cur.execute(
|
|
"""
|
|
UPDATE detect_sync_records
|
|
SET updated_at = CURRENT_TIMESTAMP
|
|
WHERE id = %s
|
|
""",
|
|
(int(existing[0]),),
|
|
)
|
|
conn.commit()
|
|
return True, "同步投影已存在，已按幂等处理", {"record_id": int(existing[0]), "deduplicated": True}
|
|
|
|
stored_payload = {
|
|
"sync_type": sync_type,
|
|
"source_record_id": source_record_id,
|
|
"projection_hash": projection_hash,
|
|
"projection": projection,
|
|
"received_at": _format_time(received_at),
|
|
}
|
|
cur.execute(
|
|
"""
|
|
INSERT INTO detect_sync_records (
|
|
sync_type, source_region, target_region, status, payload_json, error_message, created_at, updated_at
|
|
) VALUES (%s, %s, %s, %s, %s::jsonb, %s, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)
|
|
RETURNING id
|
|
""",
|
|
(
|
|
ingest_sync_type,
|
|
source_region,
|
|
target_region,
|
|
"received",
|
|
json.dumps(stored_payload, ensure_ascii=False),
|
|
"",
|
|
),
|
|
)
|
|
record_id = int(cur.fetchone()[0])
|
|
conn.commit()
|
|
if sync_type == "runtime_projection":
|
|
_refresh_remote_runtime_node(source_region=source_region, projection=projection, received_at=received_at)
|
|
return True, "同步投影接收成功", {"record_id": record_id, "deduplicated": False}
|
|
|
|
|
|
def _push_projection_now(sync_type: str, ingest_url: str) -> tuple[bool, str, dict]:
|
|
source_record = _load_latest_projection(sync_type)
|
|
if not source_record:
|
|
return False, f"当前没有可推送的{sync_type}", {"action": "push_sync", "sync_type": sync_type}
|
|
return _push_projection_record(source_record, sync_type, ingest_url)
|
|
|
|
|
|
def _push_projection_batch(sync_type: str, ingest_url: str) -> tuple[bool, str, dict]:
|
|
pending_records = _load_pushable_projections(sync_type, limit=max(1, int(settings.sync_batch_size or 200)))
|
|
if not pending_records:
|
|
latest = _load_latest_projection(sync_type)
|
|
if latest:
|
|
latest_attempt = _latest_push_attempt(latest["id"], latest["target_region"], sync_type)
|
|
if latest_attempt and latest_attempt["status"] == "success":
|
|
return True, "当前批次已全部同步完成", {
|
|
"action": "push_sync",
|
|
"sync_type": sync_type,
|
|
"deduplicated": True,
|
|
"batch_count": 0,
|
|
}
|
|
return False, f"当前没有可推送的{sync_type}", {"action": "push_sync", "sync_type": sync_type, "batch_count": 0}
|
|
|
|
results: list[dict] = []
|
|
success_count = 0
|
|
for record in pending_records:
|
|
ok, message, data = _push_projection_record(record, sync_type, ingest_url)
|
|
results.append(
|
|
{
|
|
"source_record_id": record["id"],
|
|
"ok": ok,
|
|
"message": message,
|
|
"data": data,
|
|
}
|
|
)
|
|
if ok:
|
|
success_count += 1
|
|
|
|
overall_ok = success_count > 0
|
|
message = f"{sync_type} 批量推送完成，成功 {success_count}/{len(results)}"
|
|
return overall_ok, message, {
|
|
"action": "push_sync",
|
|
"sync_type": sync_type,
|
|
"batch_count": len(results),
|
|
"success_count": success_count,
|
|
"results": results,
|
|
}
|
|
|
|
|
|
def _push_projection_record(source_record: dict, sync_type: str, ingest_url: str) -> tuple[bool, str, dict]:
|
|
latest_attempt = _latest_push_attempt(source_record["id"], source_record["target_region"], sync_type)
|
|
if latest_attempt and latest_attempt["status"] == "success":
|
|
last_created_at = latest_attempt.get("created_at")
|
|
if sync_type != "runtime_projection" or not isinstance(last_created_at, datetime):
|
|
return True, "该投影已推送，无需重复发送", {
|
|
"action": "push_sync",
|
|
"sync_type": sync_type,
|
|
"source_record_id": source_record["id"],
|
|
"deduplicated": True,
|
|
}
|
|
now = datetime.now(last_created_at.tzinfo) if last_created_at.tzinfo else datetime.now()
|
|
if now - last_created_at < timedelta(seconds=max(20, int(settings.sync_poll_interval_seconds or 30))):
|
|
return True, "该投影已推送，无需重复发送", {
|
|
"action": "push_sync",
|
|
"sync_type": sync_type,
|
|
"source_record_id": source_record["id"],
|
|
"deduplicated": True,
|
|
}
|
|
if latest_attempt and latest_attempt["status"] == "pending":
|
|
return True, "该投影已有同步推送进行中，暂不重复发送", {
|
|
"action": "push_sync",
|
|
"sync_type": sync_type,
|
|
"source_record_id": source_record["id"],
|
|
"deduplicated": True,
|
|
}
|
|
if latest_attempt and latest_attempt["status"] == "failed":
|
|
last_created_at = latest_attempt.get("created_at")
|
|
if isinstance(last_created_at, datetime):
|
|
now = datetime.now(last_created_at.tzinfo) if last_created_at.tzinfo else datetime.now()
|
|
if now - last_created_at < timedelta(seconds=max(10, int(settings.sync_poll_interval_seconds or 30))):
|
|
return False, "最近一次同步推送刚失败，等待下个重试窗口", {
|
|
"action": "push_sync",
|
|
"sync_type": sync_type,
|
|
"source_record_id": source_record["id"],
|
|
"deduplicated": True,
|
|
}
|
|
|
|
attempt_id = _create_push_attempt(source_record=source_record, ingest_url=ingest_url, sync_type=sync_type)
|
|
request_payload = {
|
|
"sync_type": sync_type,
|
|
"source_region": source_record["source_region"],
|
|
"source_record_id": source_record["id"],
|
|
"projection_hash": (source_record.get("payload") or {}).get("projection_hash", ""),
|
|
"projection": (source_record.get("payload") or {}).get("projection", {}),
|
|
"created_at": _format_time(source_record.get("created_at")),
|
|
}
|
|
request_body = json.dumps(request_payload, ensure_ascii=False).encode("utf-8")
|
|
request = urllib.request.Request(
|
|
ingest_url,
|
|
data=request_body,
|
|
headers={
|
|
"Content-Type": "application/json",
|
|
**({"X-Domaincheck-Sync-Token": settings.sync_shared_token} if settings.sync_shared_token else {}),
|
|
},
|
|
method="POST",
|
|
)
|
|
try:
|
|
with urllib.request.urlopen(request, timeout=15) as response:
|
|
raw = response.read().decode("utf-8")
|
|
data = json.loads(raw) if raw else {}
|
|
status_code = getattr(response, "status", 200)
|
|
except json.JSONDecodeError as exc:
|
|
payload = {
|
|
"sync_type": sync_type,
|
|
"source_record_id": source_record["id"],
|
|
"projection_hash": request_payload["projection_hash"],
|
|
"ingest_url": ingest_url,
|
|
}
|
|
_update_push_attempt(attempt_id, status="failed", payload=payload, error_message=f"invalid json response: {exc}")
|
|
return False, f"同步推送失败: 远端响应不是合法 JSON ({exc})", {"action": "push_sync", "sync_type": sync_type, "attempt_id": attempt_id}
|
|
except urllib.error.HTTPError as exc:
|
|
error_body = exc.read().decode("utf-8", errors="ignore") if hasattr(exc, "read") else ""
|
|
payload = {
|
|
"sync_type": sync_type,
|
|
"source_record_id": source_record["id"],
|
|
"projection_hash": request_payload["projection_hash"],
|
|
"ingest_url": ingest_url,
|
|
"http_status": getattr(exc, "code", 500),
|
|
"response_text": error_body[:1000],
|
|
}
|
|
_update_push_attempt(attempt_id, status="failed", payload=payload, error_message=f"HTTP {getattr(exc, 'code', 500)}")
|
|
return False, f"同步推送失败: HTTP {getattr(exc, 'code', 500)}", {"action": "push_sync", "sync_type": sync_type, "attempt_id": attempt_id}
|
|
except Exception as exc:
|
|
payload = {
|
|
"sync_type": sync_type,
|
|
"source_record_id": source_record["id"],
|
|
"projection_hash": request_payload["projection_hash"],
|
|
"ingest_url": ingest_url,
|
|
}
|
|
_update_push_attempt(attempt_id, status="failed", payload=payload, error_message=str(exc))
|
|
return False, f"同步推送失败: {exc}", {"action": "push_sync", "sync_type": sync_type, "attempt_id": attempt_id}
|
|
|
|
payload = {
|
|
"sync_type": sync_type,
|
|
"source_record_id": source_record["id"],
|
|
"projection_hash": request_payload["projection_hash"],
|
|
"ingest_url": ingest_url,
|
|
"http_status": status_code,
|
|
"response": data,
|
|
}
|
|
response_code = data.get("code", 0)
|
|
if response_code not in (0, "0", None, ""):
|
|
_update_push_attempt(
|
|
attempt_id,
|
|
status="failed",
|
|
payload=payload,
|
|
error_message=str(data.get("message") or "remote business error"),
|
|
)
|
|
return False, f"同步推送失败: {str(data.get('message') or '远端业务返回失败')}", {
|
|
"action": "push_sync",
|
|
"sync_type": sync_type,
|
|
"attempt_id": attempt_id,
|
|
"source_record_id": source_record["id"],
|
|
"response": data,
|
|
}
|
|
_update_push_attempt(attempt_id, status="success", payload=payload, error_message="")
|
|
return True, "投影推送成功", {
|
|
"action": "push_sync",
|
|
"sync_type": sync_type,
|
|
"attempt_id": attempt_id,
|
|
"source_record_id": source_record["id"],
|
|
"response": data,
|
|
}
|
|
|
|
|
|
def push_runtime_projection_now() -> tuple[bool, str, dict]:
|
|
if not settings.sync_push_enabled:
|
|
return False, "未启用同步推送", {"action": "push_sync", "sync_state": "disabled", "ui_level": "warning", "poll_schedule_seconds": []}
|
|
|
|
ingest_url = _ingest_url(settings.sync_target_api_base_url)
|
|
if not ingest_url:
|
|
return False, "未配置同步目标地址", {"action": "push_sync", "sync_state": "misconfigured", "ui_level": "warning", "poll_schedule_seconds": []}
|
|
|
|
results = []
|
|
ok, message, data = _push_projection_now("runtime_projection", ingest_url)
|
|
results.append({"sync_type": "runtime_projection", "ok": ok, "message": message, "data": data})
|
|
|
|
ok, message, data = _push_projection_batch("detect_result_projection", ingest_url)
|
|
results.append({"sync_type": "detect_result_projection", "ok": ok, "message": message, "data": data})
|
|
|
|
success_count = sum(1 for item in results if item["ok"])
|
|
warning_count = sum(
|
|
1
|
|
for item in results
|
|
if not item["ok"] and any(keyword in str(item.get("message") or "") for keyword in ("当前没有可推送", "已全部同步完成", "无需重复发送", "进行中", "等待下个重试窗口"))
|
|
)
|
|
if success_count == 0:
|
|
if warning_count == len(results):
|
|
return False, "当前没有需要立即推送的同步投影", {
|
|
"action": "push_sync",
|
|
"sync_state": "idle",
|
|
"ui_level": "warning",
|
|
"poll_schedule_seconds": [],
|
|
"results": results,
|
|
}
|
|
return False, "同步推送未成功，请检查明细结果", {
|
|
"action": "push_sync",
|
|
"sync_state": "failed",
|
|
"ui_level": "error",
|
|
"poll_schedule_seconds": [2],
|
|
"results": results,
|
|
}
|
|
if success_count < len(results):
|
|
return True, f"同步推送部分完成，成功 {success_count}/{len(results)}", {
|
|
"action": "push_sync",
|
|
"sync_state": "partial_success",
|
|
"ui_level": "warning",
|
|
"poll_schedule_seconds": [1, 3],
|
|
"results": results,
|
|
}
|
|
return True, f"同步推送完成，成功 {success_count}/{len(results)}", {
|
|
"action": "push_sync",
|
|
"sync_state": "success",
|
|
"ui_level": "success",
|
|
"poll_schedule_seconds": [1, 3],
|
|
"results": results,
|
|
}
|
|
|
|
|
|
def pull_detect_task_batch_now(limit: int | None = None) -> tuple[bool, str, dict]:
|
|
if settings.node_region != "mainland" or settings.node_role != "control":
|
|
return False, "当前节点无需拉取待检测任务批次", {"action": "pull_tasks", "pull_state": "not_applicable", "ui_level": "warning", "poll_schedule_seconds": []}
|
|
|
|
export_url = _task_export_url(settings.sync_target_api_base_url)
|
|
ack_url = _task_ack_url(settings.sync_target_api_base_url)
|
|
if not export_url or not ack_url:
|
|
return False, "未配置任务拉取目标地址", {"action": "pull_tasks", "pull_state": "misconfigured", "ui_level": "warning", "poll_schedule_seconds": []}
|
|
|
|
safe_limit = max(1, min(int(limit or settings.sync_batch_size or 200), max(1, int(settings.sync_batch_size or 200))))
|
|
request_url = f"{export_url}?limit={safe_limit}"
|
|
request = urllib.request.Request(
|
|
request_url,
|
|
headers={**({"X-Domaincheck-Sync-Token": settings.sync_shared_token} if settings.sync_shared_token else {})},
|
|
method="GET",
|
|
)
|
|
try:
|
|
with urllib.request.urlopen(request, timeout=20) as response:
|
|
raw = response.read().decode("utf-8")
|
|
data = json.loads(raw) if raw else {}
|
|
except json.JSONDecodeError as exc:
|
|
return False, f"拉取待检测任务失败: 远端响应不是合法 JSON ({exc})", {"action": "pull_tasks", "pull_state": "failed", "ui_level": "error", "poll_schedule_seconds": [2]}
|
|
except urllib.error.HTTPError as exc:
|
|
error_body = exc.read().decode("utf-8", errors="ignore") if hasattr(exc, "read") else ""
|
|
return False, f"拉取待检测任务失败: HTTP {getattr(exc, 'code', 500)}", {
|
|
"action": "pull_tasks",
|
|
"pull_state": "failed",
|
|
"ui_level": "error",
|
|
"poll_schedule_seconds": [2],
|
|
"http_status": getattr(exc, "code", 500),
|
|
"response_text": error_body[:500],
|
|
}
|
|
except Exception as exc:
|
|
return False, f"拉取待检测任务失败: {exc}", {"action": "pull_tasks", "pull_state": "failed", "ui_level": "error", "poll_schedule_seconds": [2]}
|
|
|
|
payload = data.get("data") or {}
|
|
response_code = data.get("code", 0)
|
|
if response_code not in (0, "0", None) and not payload:
|
|
return False, str(data.get("message") or "拉取待检测任务失败"), {"action": "pull_tasks", "pull_state": "failed", "ui_level": "error", "poll_schedule_seconds": [2], "response": data}
|
|
|
|
source_record_id = int(payload.get("source_record_id") or 0)
|
|
projection_hash = str(payload.get("projection_hash") or "").strip()
|
|
projection = payload.get("projection") or {}
|
|
if source_record_id <= 0 or not projection_hash or not projection:
|
|
return False, "远端当前没有可拉取的待检测任务批次", {"action": "pull_tasks", "pull_state": "idle", "ui_level": "warning", "poll_schedule_seconds": [], "batch_size": 0}
|
|
|
|
ingest_ok, ingest_message, ingest_data = ingest_detect_task_projection(
|
|
{
|
|
"sync_type": "detect_task_projection",
|
|
"source_region": _normalize_region(settings.sync_target_region, "overseas"),
|
|
"source_record_id": source_record_id,
|
|
"projection_hash": projection_hash,
|
|
"projection": projection,
|
|
},
|
|
shared_token=settings.sync_shared_token,
|
|
)
|
|
if not ingest_ok:
|
|
return False, ingest_message, {"action": "pull_tasks", "pull_state": "failed", "ui_level": "error", "poll_schedule_seconds": [2], **(ingest_data or {})}
|
|
|
|
ack_request = urllib.request.Request(
|
|
ack_url,
|
|
data=json.dumps(
|
|
{
|
|
"source_record_id": source_record_id,
|
|
"projection_hash": projection_hash,
|
|
},
|
|
ensure_ascii=False,
|
|
).encode("utf-8"),
|
|
headers={
|
|
"Content-Type": "application/json",
|
|
**({"X-Domaincheck-Sync-Token": settings.sync_shared_token} if settings.sync_shared_token else {}),
|
|
},
|
|
method="POST",
|
|
)
|
|
ack_data = {}
|
|
try:
|
|
with urllib.request.urlopen(ack_request, timeout=15) as response:
|
|
raw = response.read().decode("utf-8")
|
|
ack_response = json.loads(raw) if raw else {}
|
|
ack_data = ack_response.get("data") or {}
|
|
ack_code = ack_response.get("code", 0)
|
|
if ack_code not in (0, "0", None, ""):
|
|
return True, f"{ingest_message}；但远端确认失败: {str(ack_response.get('message') or 'ack business error')}", {
|
|
"action": "pull_tasks",
|
|
"pull_state": "ack_warning",
|
|
"ui_level": "warning",
|
|
"poll_schedule_seconds": [1, 3],
|
|
"source_record_id": source_record_id,
|
|
"projection_hash": projection_hash,
|
|
**(ingest_data or {}),
|
|
"ack": ack_data,
|
|
"ack_response": ack_response,
|
|
}
|
|
except json.JSONDecodeError as exc:
|
|
return True, f"{ingest_message}；但远端确认失败: ack 响应不是合法 JSON ({exc})", {
|
|
"action": "pull_tasks",
|
|
"pull_state": "ack_warning",
|
|
"ui_level": "warning",
|
|
"poll_schedule_seconds": [1, 3],
|
|
"source_record_id": source_record_id,
|
|
**(ingest_data or {}),
|
|
}
|
|
except Exception as exc:
|
|
return True, f"{ingest_message}；但远端确认失败: {exc}", {
|
|
"action": "pull_tasks",
|
|
"pull_state": "ack_warning",
|
|
"ui_level": "warning",
|
|
"poll_schedule_seconds": [1, 3],
|
|
"source_record_id": source_record_id,
|
|
**(ingest_data or {}),
|
|
}
|
|
|
|
return True, "待检测任务批次拉取并入库成功", {
|
|
"action": "pull_tasks",
|
|
"pull_state": "success",
|
|
"ui_level": "success",
|
|
"poll_schedule_seconds": [1, 3],
|
|
"source_record_id": source_record_id,
|
|
"projection_hash": projection_hash,
|
|
"batch_code": str(projection.get("batch_code") or "").strip(),
|
|
**(ingest_data or {}),
|
|
"ack": ack_data,
|
|
}
|