1826 lines
75 KiB
Python
1826 lines
75 KiB
Python
from __future__ import annotations
|
||
|
||
import hashlib
|
||
import json
|
||
import socket
|
||
import urllib.error
|
||
import urllib.parse
|
||
import urllib.request
|
||
from datetime import datetime, timedelta
|
||
from uuid import uuid4
|
||
|
||
from app.core.config import settings
|
||
from app.core.db import get_db
|
||
from app.services.cluster_runtime_service import (
|
||
cleanup_imported_runtime_nodes,
|
||
cleanup_imported_runtime_nodes_many,
|
||
register_node_heartbeat,
|
||
)
|
||
from app.services.detect_job_service import (
|
||
_load_domain_pipeline_snapshot,
|
||
resolve_initial_domain_pipeline_item,
|
||
)
|
||
from app.services.settings_service import get_settings_payload
|
||
from app.services.sync_record_service import _decode_json, _normalize_region
|
||
|
||
|
||
_DETECT_RESULT_EVENT_TYPES = {
|
||
"domain_started",
|
||
"domain_completed",
|
||
"domain_failed",
|
||
"domain_blacklisted",
|
||
}
|
||
|
||
|
||
def _format_time(value: datetime | None) -> str:
|
||
return value.isoformat(sep=" ", timespec="seconds") if value else ""
|
||
|
||
|
||
def _ingest_url(base_url: str) -> str:
|
||
text = str(base_url or "").strip().rstrip("/")
|
||
if not text:
|
||
return ""
|
||
if text.endswith("/api/v1"):
|
||
return f"{text}/runtime/sync-ingest"
|
||
if text.endswith("/api/v1/runtime"):
|
||
return f"{text}/sync-ingest"
|
||
return f"{text}/api/v1/runtime/sync-ingest"
|
||
|
||
|
||
def _task_export_url(base_url: str) -> str:
|
||
text = str(base_url or "").strip().rstrip("/")
|
||
if not text:
|
||
return ""
|
||
if text.endswith("/api/v1"):
|
||
return f"{text}/runtime/task-export"
|
||
if text.endswith("/api/v1/runtime"):
|
||
return f"{text}/task-export"
|
||
return f"{text}/api/v1/runtime/task-export"
|
||
|
||
|
||
def _task_ack_url(base_url: str) -> str:
|
||
text = str(base_url or "").strip().rstrip("/")
|
||
if not text:
|
||
return ""
|
||
if text.endswith("/api/v1"):
|
||
return f"{text}/runtime/task-ack"
|
||
if text.endswith("/api/v1/runtime"):
|
||
return f"{text}/task-ack"
|
||
return f"{text}/api/v1/runtime/task-ack"
|
||
|
||
|
||
def _projection_ingest_type(sync_type: str) -> str:
|
||
if sync_type == "runtime_projection":
|
||
return "runtime_ingest"
|
||
if sync_type == "detect_result_projection":
|
||
return "detect_result_ingest"
|
||
if sync_type == "detect_task_projection":
|
||
return "detect_task_ingest"
|
||
return "sync_ingest"
|
||
|
||
|
||
def _refresh_remote_runtime_node(*, source_region: str, projection: dict, received_at: datetime | None = None) -> None:
|
||
node_info = projection.get("node") or {}
|
||
node_code = str(node_info.get("node_code") or "").strip()
|
||
region = _normalize_region(node_info.get("region"), source_region)
|
||
role = str(node_info.get("role") or "control").strip() or "control"
|
||
hostname = str(node_info.get("hostname") or "").strip() or socket.gethostname()
|
||
ip = str(node_info.get("ip") or "").strip()
|
||
if not node_code:
|
||
node_code = f"{region}-{role}-imported"
|
||
|
||
controller_current_load = max(
|
||
int(projection.get("active_thread_count", 0) or 0),
|
||
int(((projection.get("active_job") or {}).get("items_running", 0) or 0)),
|
||
)
|
||
metadata = {
|
||
"service": "runtime-ingest",
|
||
"projection_source_region": source_region,
|
||
"worker_mode": projection.get("worker_mode", ""),
|
||
"active_threads": int(projection.get("active_thread_count", 0) or 0),
|
||
"max_threads": int(projection.get("max_thread_count", 0) or 0),
|
||
"phase_label": projection.get("phase_label", ""),
|
||
"phase_detail": projection.get("phase_detail", ""),
|
||
"proxy_runtime_label": projection.get("proxy_runtime_label", ""),
|
||
"proxy_runtime_reason": projection.get("proxy_runtime_reason", ""),
|
||
"worker_online": bool(projection.get("worker_online", False)),
|
||
"detect_participating": bool(projection.get("detect_participating", False)),
|
||
"active_job_code": str((projection.get("active_job") or {}).get("job_code") or ""),
|
||
"active_job_status": str((projection.get("active_job") or {}).get("status") or ""),
|
||
"job_items_total": int(((projection.get("active_job") or {}).get("items_total", 0) or 0)),
|
||
"job_items_claimed": int(((projection.get("active_job") or {}).get("items_claimed", 0) or 0)),
|
||
"job_items_running": int(((projection.get("active_job") or {}).get("items_running", 0) or 0)),
|
||
"job_items_completed": int(((projection.get("active_job") or {}).get("items_completed", 0) or 0)),
|
||
"updated_at": _format_time(received_at or datetime.now()),
|
||
}
|
||
register_node_heartbeat(
|
||
node_code=node_code,
|
||
region=region,
|
||
role=role,
|
||
status="busy" if controller_current_load > 0 else "online",
|
||
current_load=controller_current_load,
|
||
metadata=metadata,
|
||
hostname_override=hostname,
|
||
ip_override=ip,
|
||
)
|
||
cleanup_imported_runtime_nodes(region=region, role=role, keep_node_code=node_code)
|
||
|
||
active_job = projection.get("active_job") or {}
|
||
worker_node_codes: list[str] = []
|
||
for node_stat in list(active_job.get("node_stats") or []):
|
||
worker_node_code = str(node_stat.get("node_code") or "").strip()
|
||
if not worker_node_code or worker_node_code == "unassigned":
|
||
continue
|
||
if worker_node_code == node_code:
|
||
continue
|
||
items_running = int(node_stat.get("items_running", 0) or 0)
|
||
items_claimed = int(node_stat.get("items_claimed", 0) or 0)
|
||
items_total = int(node_stat.get("items_total", 0) or 0)
|
||
worker_runtime_load = int(node_stat.get("current_load", 0) or 0)
|
||
worker_active_threads = int(node_stat.get("active_threads", worker_runtime_load) or 0)
|
||
worker_max_threads = int(node_stat.get("max_threads", 0) or 0)
|
||
worker_load = max(worker_active_threads, items_running, 0)
|
||
worker_status = "busy" if worker_load > 0 else "online"
|
||
worker_metadata = {
|
||
"service": "runtime-ingest",
|
||
"projection_source_region": source_region,
|
||
"worker_mode": projection.get("worker_mode", ""),
|
||
"phase_label": projection.get("phase_label", ""),
|
||
"phase_detail": projection.get("phase_detail", ""),
|
||
"proxy_runtime_label": projection.get("proxy_runtime_label", ""),
|
||
"proxy_runtime_reason": projection.get("proxy_runtime_reason", ""),
|
||
"active_threads": worker_active_threads,
|
||
"max_threads": worker_max_threads,
|
||
"updated_at": _format_time(received_at or datetime.now()),
|
||
"job_items_total": items_total,
|
||
"job_items_running": items_running,
|
||
"job_items_claimed": items_claimed,
|
||
"job_items_completed": int(node_stat.get("items_completed", 0) or 0),
|
||
"job_items_failed": int(node_stat.get("items_failed", 0) or 0),
|
||
"job_items_blacklisted": int(node_stat.get("items_blacklisted", 0) or 0),
|
||
"metrics_source": str(node_stat.get("metrics_source") or "runtime").strip() or "runtime",
|
||
"source_status": str(node_stat.get("status") or "").strip(),
|
||
"source_role": str(node_stat.get("role") or "worker").strip() or "worker",
|
||
"derived_from": node_code,
|
||
}
|
||
register_node_heartbeat(
|
||
node_code=worker_node_code,
|
||
region=region,
|
||
role="worker",
|
||
status=worker_status,
|
||
current_load=worker_load,
|
||
metadata=worker_metadata,
|
||
hostname_override=hostname,
|
||
ip_override=ip,
|
||
)
|
||
worker_node_codes.append(worker_node_code)
|
||
if worker_node_codes:
|
||
cleanup_imported_runtime_nodes_many(region=region, role="worker", keep_node_codes=worker_node_codes)
|
||
|
||
|
||
def _load_latest_projection(sync_type: str) -> dict | None:
|
||
source_region = _normalize_region(settings.sync_source_region, settings.node_region)
|
||
target_region = _normalize_region(settings.sync_target_region, "overseas")
|
||
with get_db() as conn:
|
||
with conn.cursor() as cur:
|
||
cur.execute(
|
||
"""
|
||
SELECT id, source_region, target_region, status, payload_json, created_at, updated_at
|
||
FROM detect_sync_records
|
||
WHERE sync_type = %s
|
||
AND source_region = %s
|
||
AND target_region = %s
|
||
ORDER BY created_at DESC, id DESC
|
||
LIMIT 1
|
||
""",
|
||
(sync_type, source_region, target_region),
|
||
)
|
||
row = cur.fetchone()
|
||
if not row:
|
||
return None
|
||
return {
|
||
"id": row[0],
|
||
"source_region": row[1],
|
||
"target_region": row[2],
|
||
"status": row[3],
|
||
"payload": _decode_json(row[4]),
|
||
"created_at": row[5],
|
||
"updated_at": row[6],
|
||
}
|
||
|
||
|
||
def _load_pushable_projections(sync_type: str, limit: int) -> list[dict]:
|
||
safe_limit = max(1, min(int(limit or 1), max(1, int(settings.sync_batch_size or 200))))
|
||
source_region = _normalize_region(settings.sync_source_region, settings.node_region)
|
||
target_region = _normalize_region(settings.sync_target_region, "overseas")
|
||
with get_db() as conn:
|
||
with conn.cursor() as cur:
|
||
cur.execute(
|
||
"""
|
||
SELECT id, source_region, target_region, status, payload_json, created_at, updated_at
|
||
FROM detect_sync_records
|
||
WHERE sync_type = %s
|
||
AND source_region = %s
|
||
AND target_region = %s
|
||
ORDER BY created_at ASC, id ASC
|
||
LIMIT %s
|
||
""",
|
||
(sync_type, source_region, target_region, safe_limit * 5),
|
||
)
|
||
rows = cur.fetchall()
|
||
|
||
selected: list[dict] = []
|
||
for row in rows:
|
||
projection = {
|
||
"id": row[0],
|
||
"source_region": row[1],
|
||
"target_region": row[2],
|
||
"status": row[3],
|
||
"payload": _decode_json(row[4]),
|
||
"created_at": row[5],
|
||
"updated_at": row[6],
|
||
}
|
||
latest_attempt = _latest_push_attempt(projection["id"], projection["target_region"], sync_type)
|
||
if latest_attempt and latest_attempt["status"] == "success":
|
||
continue
|
||
if latest_attempt and latest_attempt["status"] == "pending":
|
||
continue
|
||
if latest_attempt and latest_attempt["status"] == "failed":
|
||
last_created_at = latest_attempt.get("created_at")
|
||
if isinstance(last_created_at, datetime):
|
||
now = datetime.now(last_created_at.tzinfo) if last_created_at.tzinfo else datetime.now()
|
||
if now - last_created_at < timedelta(seconds=max(10, int(settings.sync_poll_interval_seconds or 30))):
|
||
continue
|
||
selected.append(projection)
|
||
if len(selected) >= safe_limit:
|
||
break
|
||
return selected
|
||
|
||
|
||
def _estimate_total_worker_threads(settings_payload: dict | None = None) -> int:
|
||
payload = settings_payload if isinstance(settings_payload, dict) else get_settings_payload()
|
||
default_threads = max(1, int(payload.get("thread_count", 100) or 100))
|
||
node_thread_counts = payload.get("node_thread_counts") if isinstance(payload.get("node_thread_counts"), dict) else {}
|
||
total_threads = 0
|
||
for raw_value in node_thread_counts.values():
|
||
try:
|
||
total_threads += max(0, int(raw_value or 0))
|
||
except (TypeError, ValueError):
|
||
continue
|
||
return max(total_threads, default_threads)
|
||
|
||
|
||
def _load_local_detect_backlog_snapshot() -> dict:
|
||
with get_db() as conn:
|
||
with conn.cursor() as cur:
|
||
cur.execute(
|
||
"""
|
||
SELECT
|
||
COUNT(*) FILTER (WHERE item.status = 'pending') AS pending_total,
|
||
COUNT(*) FILTER (WHERE item.status = 'claimed') AS claimed_total,
|
||
COUNT(*) FILTER (WHERE item.status = 'running') AS running_total,
|
||
COUNT(*) FILTER (WHERE item.status = 'pending' AND item.step_code = 'detect_register') AS register_pending,
|
||
COUNT(*) FILTER (WHERE item.status = 'pending' AND item.step_code <> 'detect_register') AS downstream_pending
|
||
FROM detect_job_items item
|
||
JOIN detect_jobs job ON job.id = item.job_id
|
||
WHERE job.status IN ('pending', 'running')
|
||
"""
|
||
)
|
||
row = cur.fetchone() or (0, 0, 0, 0, 0)
|
||
return {
|
||
"pending_total": int(row[0] or 0),
|
||
"claimed_total": int(row[1] or 0),
|
||
"running_total": int(row[2] or 0),
|
||
"register_pending": int(row[3] or 0),
|
||
"downstream_pending": int(row[4] or 0),
|
||
}
|
||
|
||
|
||
def _build_task_pull_backlog_limits(configured_limit: int, settings_payload: dict | None = None) -> dict:
|
||
estimated_total_threads = _estimate_total_worker_threads(settings_payload)
|
||
max_pending_total = int(settings.sync_pull_max_pending_items or 0)
|
||
if max_pending_total <= 0:
|
||
max_pending_total = max(int(configured_limit or 0), estimated_total_threads * 2)
|
||
|
||
max_register_pending = int(settings.sync_pull_max_register_pending_items or 0)
|
||
if max_register_pending <= 0:
|
||
max_register_pending = max(max(500, int(configured_limit or 0) // 2), estimated_total_threads)
|
||
|
||
max_downstream_pending = int(settings.sync_pull_max_downstream_pending_items or 0)
|
||
if max_downstream_pending <= 0:
|
||
max_downstream_pending = max(250, estimated_total_threads // 4)
|
||
|
||
return {
|
||
"estimated_total_threads": estimated_total_threads,
|
||
"max_pending_total": max_pending_total,
|
||
"max_register_pending": max_register_pending,
|
||
"max_downstream_pending": max_downstream_pending,
|
||
}
|
||
|
||
|
||
def _should_throttle_task_pull(backlog_snapshot: dict, backlog_limits: dict) -> tuple[bool, str]:
|
||
pending_total = int(backlog_snapshot.get("pending_total", 0) or 0)
|
||
register_pending = int(backlog_snapshot.get("register_pending", 0) or 0)
|
||
downstream_pending = int(backlog_snapshot.get("downstream_pending", 0) or 0)
|
||
max_pending_total = int(backlog_limits.get("max_pending_total", 0) or 0)
|
||
max_register_pending = int(backlog_limits.get("max_register_pending", 0) or 0)
|
||
max_downstream_pending = int(backlog_limits.get("max_downstream_pending", 0) or 0)
|
||
|
||
if max_pending_total > 0 and pending_total >= max_pending_total:
|
||
return True, "pending_total"
|
||
if (
|
||
downstream_pending > 0
|
||
and max_register_pending > 0
|
||
and register_pending >= max_register_pending
|
||
):
|
||
return True, "register_pending"
|
||
if max_downstream_pending > 0 and downstream_pending >= max_downstream_pending:
|
||
return True, "downstream_pending"
|
||
return False, ""
|
||
|
||
|
||
def _latest_push_attempt(source_record_id: int, target_region: str, sync_type: str) -> dict | None:
|
||
with get_db() as conn:
|
||
with conn.cursor() as cur:
|
||
cur.execute(
|
||
"""
|
||
SELECT id, status, payload_json, error_message, created_at, updated_at
|
||
FROM detect_sync_records
|
||
WHERE sync_type = 'runtime_push'
|
||
AND source_region = %s
|
||
AND target_region = %s
|
||
AND (payload_json->>'sync_type') = %s
|
||
AND (payload_json->>'source_record_id') = %s
|
||
ORDER BY created_at DESC, id DESC
|
||
LIMIT 1
|
||
""",
|
||
(
|
||
_normalize_region(settings.sync_source_region, settings.node_region),
|
||
target_region,
|
||
sync_type,
|
||
str(int(source_record_id)),
|
||
),
|
||
)
|
||
row = cur.fetchone()
|
||
if not row:
|
||
return None
|
||
return {
|
||
"id": row[0],
|
||
"status": row[1],
|
||
"payload": _decode_json(row[2]),
|
||
"error_message": row[3] or "",
|
||
"created_at": row[4],
|
||
"updated_at": row[5],
|
||
}
|
||
|
||
|
||
def _latest_ingest_attempt(source_record_id: int, target_region: str, sync_type: str) -> dict | None:
|
||
ingest_sync_type = _projection_ingest_type(sync_type)
|
||
with get_db() as conn:
|
||
with conn.cursor() as cur:
|
||
cur.execute(
|
||
"""
|
||
SELECT id, status, payload_json, error_message, created_at, updated_at
|
||
FROM detect_sync_records
|
||
WHERE sync_type = %s
|
||
AND source_region = %s
|
||
AND target_region = %s
|
||
AND (payload_json->>'source_record_id') = %s
|
||
ORDER BY created_at DESC, id DESC
|
||
LIMIT 1
|
||
""",
|
||
(
|
||
ingest_sync_type,
|
||
_normalize_region(settings.sync_source_region, settings.node_region),
|
||
target_region,
|
||
str(int(source_record_id)),
|
||
),
|
||
)
|
||
row = cur.fetchone()
|
||
if not row:
|
||
return None
|
||
return {
|
||
"id": row[0],
|
||
"status": row[1],
|
||
"payload": _decode_json(row[2]),
|
||
"error_message": row[3] or "",
|
||
"created_at": row[4],
|
||
"updated_at": row[5],
|
||
}
|
||
|
||
|
||
def _task_selection_sql() -> str:
|
||
return """
|
||
SELECT id, domain, tld, source_type, use_status, detect_status, register_status, expire_date, jucha_status, juziseo_status
|
||
FROM domains
|
||
WHERE
|
||
detect_status IN (0, 4)
|
||
OR (use_status = 0 AND detect_status = 1 AND register_status = 3 AND expire_date < CURRENT_DATE)
|
||
ORDER BY id ASC
|
||
LIMIT %s
|
||
"""
|
||
|
||
|
||
def _task_projection_limit(limit: int | None) -> int:
|
||
requested = max(1, int(limit or 5000))
|
||
configured = max(5000, int(settings.sync_batch_size or 200))
|
||
cap = max(10000, configured, 5000)
|
||
return max(1, min(requested, cap))
|
||
|
||
|
||
def _task_projection_items_total(projection: dict) -> int:
|
||
payload = projection.get("payload") or {}
|
||
projection_payload = payload.get("projection") or {}
|
||
try:
|
||
return int(projection_payload.get("items_total", 0) or 0)
|
||
except Exception:
|
||
return 0
|
||
|
||
|
||
def _task_projection_selection_limit(projection: dict) -> int:
|
||
payload = projection.get("payload") or {}
|
||
projection_payload = payload.get("projection") or {}
|
||
try:
|
||
return int(projection_payload.get("selection_limit", 0) or 0)
|
||
except Exception:
|
||
return 0
|
||
|
||
|
||
def _mark_task_projection_superseded(record_id: int, *, reason: str) -> None:
|
||
with get_db() as conn:
|
||
with conn.cursor() as cur:
|
||
cur.execute(
|
||
"""
|
||
UPDATE detect_sync_records
|
||
SET status = 'superseded',
|
||
error_message = %s,
|
||
updated_at = CURRENT_TIMESTAMP
|
||
WHERE id = %s
|
||
""",
|
||
(str(reason or "").strip()[:500], int(record_id)),
|
||
)
|
||
conn.commit()
|
||
|
||
|
||
def _load_pending_task_projection(limit: int) -> dict | None:
|
||
source_region = _normalize_region(settings.sync_source_region, settings.node_region)
|
||
target_region = _normalize_region(settings.sync_target_region, "overseas")
|
||
safe_limit = _task_projection_limit(limit)
|
||
with get_db() as conn:
|
||
with conn.cursor() as cur:
|
||
cur.execute(
|
||
"""
|
||
SELECT id, source_region, target_region, status, payload_json, created_at, updated_at
|
||
FROM detect_sync_records
|
||
WHERE sync_type = 'detect_task_projection'
|
||
AND source_region = %s
|
||
AND target_region = %s
|
||
ORDER BY created_at ASC, id ASC
|
||
LIMIT %s
|
||
""",
|
||
(source_region, target_region, safe_limit * 3),
|
||
)
|
||
rows = cur.fetchall()
|
||
for row in rows:
|
||
projection = {
|
||
"id": row[0],
|
||
"source_region": row[1],
|
||
"target_region": row[2],
|
||
"status": row[3],
|
||
"payload": _decode_json(row[4]),
|
||
"created_at": row[5],
|
||
"updated_at": row[6],
|
||
}
|
||
latest_ingest = _latest_ingest_attempt(projection["id"], projection["target_region"], "detect_task_projection")
|
||
if latest_ingest and latest_ingest["status"] == "received":
|
||
continue
|
||
items_total = _task_projection_items_total(projection)
|
||
selection_limit = _task_projection_selection_limit(projection)
|
||
created_at = projection.get("created_at")
|
||
if (
|
||
safe_limit >= 1000
|
||
and max(items_total, selection_limit) > 0
|
||
and max(items_total, selection_limit) < safe_limit
|
||
and isinstance(created_at, datetime)
|
||
):
|
||
now = datetime.now(created_at.tzinfo) if created_at.tzinfo else datetime.now()
|
||
if now - created_at >= timedelta(minutes=10):
|
||
_mark_task_projection_superseded(
|
||
projection["id"],
|
||
reason=f"stale small task projection skipped: items_total={items_total}, selection_limit={selection_limit}, requested_limit={safe_limit}",
|
||
)
|
||
continue
|
||
return projection
|
||
return None
|
||
|
||
|
||
def export_detect_task_projection(limit: int = 1000, *, shared_token: str | None = None) -> tuple[bool, str, dict]:
|
||
configured_token = str(settings.sync_shared_token or "").strip()
|
||
incoming_token = str(shared_token or "").strip()
|
||
if configured_token and incoming_token != configured_token:
|
||
return False, "同步 token 校验失败", {}
|
||
|
||
existing = _load_pending_task_projection(limit)
|
||
if existing:
|
||
return True, "已返回待确认的任务批次", {
|
||
"source_record_id": existing["id"],
|
||
"projection_hash": (existing.get("payload") or {}).get("projection_hash", ""),
|
||
"projection": (existing.get("payload") or {}).get("projection", {}),
|
||
"deduplicated": True,
|
||
}
|
||
|
||
source_region = _normalize_region(settings.sync_source_region, settings.node_region)
|
||
target_region = _normalize_region(settings.sync_target_region, "overseas")
|
||
safe_limit = _task_projection_limit(limit)
|
||
with get_db() as conn:
|
||
with conn.cursor() as cur:
|
||
cur.execute(_task_selection_sql(), (safe_limit,))
|
||
rows = cur.fetchall()
|
||
if not rows:
|
||
return False, "当前没有可下发的待检测任务", {"batch_size": 0}
|
||
|
||
items = []
|
||
for row in rows:
|
||
items.append(
|
||
{
|
||
"source_domain_id": int(row[0]),
|
||
"domain": str(row[1] or "").strip(),
|
||
"tld": str(row[2] or "").strip(),
|
||
"source_type": int(row[3] or 0),
|
||
"use_status": int(row[4] or 0),
|
||
"detect_status": int(row[5] or 0),
|
||
"register_status": int(row[6] or 0),
|
||
"expire_date": row[7].isoformat(sep=" ", timespec="seconds") if row[7] else "",
|
||
"jucha_status": int(row[8] or 0),
|
||
"juziseo_status": int(row[9] or 0),
|
||
}
|
||
)
|
||
|
||
projection = {
|
||
"batch_code": f"task-{datetime.now().strftime('%Y%m%d%H%M%S')}-{uuid4().hex[:6]}",
|
||
"items_total": len(items),
|
||
"selection_limit": safe_limit,
|
||
"items": items,
|
||
}
|
||
payload = {
|
||
"projection": projection,
|
||
"projection_hash": f"{uuid4().hex}-{len(items)}",
|
||
}
|
||
cur.execute(
|
||
"""
|
||
INSERT INTO detect_sync_records (
|
||
sync_type, source_region, target_region, status, payload_json, error_message, created_at, updated_at
|
||
) VALUES (%s, %s, %s, %s, %s::jsonb, %s, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)
|
||
RETURNING id
|
||
""",
|
||
(
|
||
"detect_task_projection",
|
||
source_region,
|
||
target_region,
|
||
"projected",
|
||
json.dumps(payload, ensure_ascii=False),
|
||
"",
|
||
),
|
||
)
|
||
record_id = int(cur.fetchone()[0])
|
||
conn.commit()
|
||
return True, "待检测任务批次已生成", {
|
||
"source_record_id": record_id,
|
||
"projection_hash": payload["projection_hash"],
|
||
"projection": projection,
|
||
"deduplicated": False,
|
||
}
|
||
|
||
|
||
def ack_detect_task_projection(payload: dict, *, shared_token: str | None = None) -> tuple[bool, str, dict]:
|
||
configured_token = str(settings.sync_shared_token or "").strip()
|
||
incoming_token = str(shared_token or "").strip()
|
||
if configured_token and incoming_token != configured_token:
|
||
return False, "同步 token 校验失败", {}
|
||
|
||
source_record_id = int(payload.get("source_record_id") or 0)
|
||
projection_hash = str(payload.get("projection_hash") or "").strip()
|
||
if source_record_id <= 0 or not projection_hash:
|
||
return False, "任务确认参数不完整", {}
|
||
|
||
source_region = _normalize_region(settings.sync_source_region, settings.node_region)
|
||
target_region = _normalize_region(settings.sync_target_region, "overseas")
|
||
with get_db() as conn:
|
||
with conn.cursor() as cur:
|
||
cur.execute(
|
||
"""
|
||
SELECT id
|
||
FROM detect_sync_records
|
||
WHERE sync_type = 'detect_task_ingest'
|
||
AND source_region = %s
|
||
AND target_region = %s
|
||
AND (payload_json->>'source_record_id') = %s
|
||
AND (payload_json->>'projection_hash') = %s
|
||
ORDER BY created_at DESC, id DESC
|
||
LIMIT 1
|
||
""",
|
||
(source_region, target_region, str(source_record_id), projection_hash),
|
||
)
|
||
existing = cur.fetchone()
|
||
if existing:
|
||
return True, "任务批次确认已存在", {"record_id": int(existing[0]), "deduplicated": True}
|
||
|
||
cur.execute(
|
||
"""
|
||
INSERT INTO detect_sync_records (
|
||
sync_type, source_region, target_region, status, payload_json, error_message, created_at, updated_at
|
||
) VALUES (%s, %s, %s, %s, %s::jsonb, %s, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)
|
||
RETURNING id
|
||
""",
|
||
(
|
||
"detect_task_ingest",
|
||
source_region,
|
||
target_region,
|
||
"received",
|
||
json.dumps(
|
||
{
|
||
"source_record_id": source_record_id,
|
||
"projection_hash": projection_hash,
|
||
"received_at": _format_time(datetime.now()),
|
||
},
|
||
ensure_ascii=False,
|
||
),
|
||
"",
|
||
),
|
||
)
|
||
record_id = int(cur.fetchone()[0])
|
||
conn.commit()
|
||
return True, "任务批次确认成功", {"record_id": record_id, "deduplicated": False}
|
||
|
||
|
||
def ingest_detect_task_projection(payload: dict, *, shared_token: str | None = None) -> tuple[bool, str, dict]:
|
||
configured_token = str(settings.sync_shared_token or "").strip()
|
||
incoming_token = str(shared_token or "").strip()
|
||
if configured_token and incoming_token != configured_token:
|
||
return False, "同步 token 校验失败", {}
|
||
|
||
sync_type = str(payload.get("sync_type") or "detect_task_projection").strip() or "detect_task_projection"
|
||
if sync_type != "detect_task_projection":
|
||
return False, "同步类型不匹配", {"sync_type": sync_type}
|
||
|
||
source_region = _normalize_region(payload.get("source_region"), "unknown")
|
||
source_record_id = int(payload.get("source_record_id") or 0)
|
||
projection_hash = str(payload.get("projection_hash") or "").strip()
|
||
projection = payload.get("projection") or {}
|
||
items = list(projection.get("items") or [])
|
||
target_region = _normalize_region(settings.node_region, "overseas")
|
||
received_at = datetime.now()
|
||
|
||
if source_record_id <= 0 or not projection_hash:
|
||
return False, "任务批次参数不完整", {}
|
||
if not items:
|
||
return False, "任务批次为空", {"source_record_id": source_record_id}
|
||
|
||
with get_db() as conn:
|
||
with conn.cursor() as cur:
|
||
cur.execute(
|
||
"""
|
||
SELECT id
|
||
FROM detect_sync_records
|
||
WHERE sync_type = 'detect_task_ingest'
|
||
AND source_region = %s
|
||
AND target_region = %s
|
||
AND (payload_json->>'source_record_id') = %s
|
||
AND (payload_json->>'projection_hash') = %s
|
||
ORDER BY created_at DESC, id DESC
|
||
LIMIT 1
|
||
""",
|
||
(source_region, target_region, str(source_record_id), projection_hash),
|
||
)
|
||
existing = cur.fetchone()
|
||
if existing:
|
||
return True, "任务批次已接收，已按幂等处理", {"record_id": int(existing[0]), "deduplicated": True}
|
||
|
||
inserted_count = 0
|
||
updated_count = 0
|
||
domain_ids: list[int] = []
|
||
for item in items:
|
||
domain = str(item.get("domain") or "").strip().lower()
|
||
if not domain:
|
||
continue
|
||
tld = str(item.get("tld") or "").strip()
|
||
if not tld and "." in domain:
|
||
tld = domain.rsplit(".", 1)[-1]
|
||
expire_date_text = str(item.get("expire_date") or "").strip()
|
||
cur.execute(
|
||
"""
|
||
INSERT INTO domains (
|
||
domain, tld, source_type, use_status, detect_status, register_status,
|
||
has_beian, company_type, website_url, beian_year, snapshot_years,
|
||
expire_date, create_time, update_time, review_status, detect_time,
|
||
backlink_count, jucha_status, juziseo_status
|
||
) VALUES (
|
||
%s, %s, %s, %s, %s, %s,
|
||
1, NULL, NULL, NULL, NULL,
|
||
NULLIF(%s, '')::timestamp, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP, 0, NULL,
|
||
0, %s, %s
|
||
)
|
||
ON CONFLICT (domain) DO UPDATE SET
|
||
tld = EXCLUDED.tld,
|
||
source_type = EXCLUDED.source_type,
|
||
use_status = EXCLUDED.use_status,
|
||
register_status = EXCLUDED.register_status,
|
||
expire_date = COALESCE(EXCLUDED.expire_date, domains.expire_date),
|
||
jucha_status = EXCLUDED.jucha_status,
|
||
juziseo_status = EXCLUDED.juziseo_status,
|
||
detect_status = CASE
|
||
WHEN domains.detect_status IN (1, 2, 3, 4) THEN domains.detect_status
|
||
ELSE EXCLUDED.detect_status
|
||
END,
|
||
update_time = CURRENT_TIMESTAMP
|
||
RETURNING id, (xmax = 0) AS inserted
|
||
""",
|
||
(
|
||
domain,
|
||
tld,
|
||
int(item.get("source_type") or 0),
|
||
int(item.get("use_status") or 0),
|
||
int(item.get("detect_status") or 0),
|
||
int(item.get("register_status") or 0),
|
||
expire_date_text,
|
||
int(item.get("jucha_status") or 0),
|
||
int(item.get("juziseo_status") or 0),
|
||
),
|
||
)
|
||
row = cur.fetchone() or [0, False]
|
||
domain_id = int(row[0] or 0)
|
||
inserted = bool(row[1])
|
||
if domain_id > 0:
|
||
domain_ids.append(domain_id)
|
||
if inserted:
|
||
inserted_count += 1
|
||
else:
|
||
updated_count += 1
|
||
|
||
target_job_code = f"sync-{source_region}-{source_record_id}"
|
||
target_job_remark = (
|
||
f"同步拉取待检测批次 {str(projection.get('batch_code') or '').strip() or source_record_id}，"
|
||
f"共 {len(domain_ids)} 个域名"
|
||
)
|
||
cur.execute(
|
||
"""
|
||
INSERT INTO detect_jobs (job_code, source, plan_hash, task_mode, step_code, status, remark, created_by)
|
||
VALUES (%s, %s, %s, 'domain_pipeline', '', 'pending', %s, %s)
|
||
ON CONFLICT (job_code) DO UPDATE SET
|
||
source = EXCLUDED.source,
|
||
plan_hash = EXCLUDED.plan_hash,
|
||
task_mode = EXCLUDED.task_mode,
|
||
step_code = EXCLUDED.step_code,
|
||
remark = EXCLUDED.remark,
|
||
created_by = EXCLUDED.created_by,
|
||
status = CASE
|
||
WHEN detect_jobs.status IN ('completed', 'failed', 'cancelled') THEN 'pending'
|
||
ELSE detect_jobs.status
|
||
END,
|
||
started_at = CASE
|
||
WHEN detect_jobs.status IN ('completed', 'failed', 'cancelled') THEN NULL
|
||
ELSE detect_jobs.started_at
|
||
END,
|
||
finished_at = CASE
|
||
WHEN detect_jobs.status IN ('completed', 'failed', 'cancelled') THEN NULL
|
||
ELSE detect_jobs.finished_at
|
||
END
|
||
RETURNING id
|
||
""",
|
||
(
|
||
target_job_code,
|
||
"sync-pull",
|
||
projection_hash,
|
||
target_job_remark,
|
||
"sync-agent",
|
||
),
|
||
)
|
||
target_job_id = int((cur.fetchone() or [0])[0] or 0)
|
||
|
||
settings_payload = get_settings_payload()
|
||
queued_count = 0
|
||
deduplicated_job_items = 0
|
||
skipped_job_items = 0
|
||
if target_job_id > 0:
|
||
for domain_id in domain_ids:
|
||
domain_snapshot = _load_domain_pipeline_snapshot(cur, int(domain_id))
|
||
if not domain_snapshot:
|
||
skipped_job_items += 1
|
||
continue
|
||
item_step_code, step_payload = resolve_initial_domain_pipeline_item(
|
||
domain_snapshot,
|
||
settings_payload=settings_payload,
|
||
)
|
||
if not item_step_code or not step_payload:
|
||
skipped_job_items += 1
|
||
continue
|
||
cur.execute(
|
||
"""
|
||
INSERT INTO detect_job_items (job_id, domain_id, step_code, status, step_payload_json)
|
||
VALUES (%s, %s, %s, 'pending', %s::jsonb)
|
||
ON CONFLICT (job_id, domain_id, step_code) DO NOTHING
|
||
RETURNING id
|
||
""",
|
||
(
|
||
target_job_id,
|
||
domain_id,
|
||
item_step_code,
|
||
json.dumps(step_payload, ensure_ascii=False),
|
||
),
|
||
)
|
||
inserted_job_item = cur.fetchone()
|
||
if inserted_job_item:
|
||
queued_count += 1
|
||
else:
|
||
deduplicated_job_items += 1
|
||
|
||
if queued_count > 0:
|
||
cur.execute(
|
||
"""
|
||
INSERT INTO detect_run_events (job_id, node_code, event_type, level, message, payload_json)
|
||
VALUES (%s, %s, %s, %s, %s, %s::jsonb)
|
||
""",
|
||
(
|
||
target_job_id,
|
||
settings.node_code,
|
||
"job_created",
|
||
"info",
|
||
f"同步拉取待检测批次 {target_job_code}，共 {queued_count} 个任务项",
|
||
json.dumps(
|
||
{
|
||
"source_region": source_region,
|
||
"source_record_id": source_record_id,
|
||
"projection_hash": projection_hash,
|
||
"batch_code": str(projection.get("batch_code") or "").strip(),
|
||
"queued_count": queued_count,
|
||
"deduplicated_job_items": deduplicated_job_items,
|
||
"skipped_job_items": skipped_job_items,
|
||
},
|
||
ensure_ascii=False,
|
||
),
|
||
),
|
||
)
|
||
|
||
cur.execute(
|
||
"""
|
||
INSERT INTO detect_sync_records (
|
||
sync_type, source_region, target_region, status, payload_json, error_message, created_at, updated_at
|
||
) VALUES (%s, %s, %s, %s, %s::jsonb, %s, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)
|
||
RETURNING id
|
||
""",
|
||
(
|
||
"detect_task_ingest",
|
||
source_region,
|
||
target_region,
|
||
"received",
|
||
json.dumps(
|
||
{
|
||
"source_record_id": source_record_id,
|
||
"projection_hash": projection_hash,
|
||
"batch_code": str(projection.get("batch_code") or "").strip(),
|
||
"items_total": len(items),
|
||
"inserted_count": inserted_count,
|
||
"updated_count": updated_count,
|
||
"target_job_id": target_job_id,
|
||
"target_job_code": target_job_code,
|
||
"queued_count": queued_count,
|
||
"deduplicated_job_items": deduplicated_job_items,
|
||
"skipped_job_items": skipped_job_items,
|
||
"received_at": _format_time(received_at),
|
||
},
|
||
ensure_ascii=False,
|
||
),
|
||
"",
|
||
),
|
||
)
|
||
record_id = int(cur.fetchone()[0])
|
||
conn.commit()
|
||
return True, "任务批次接收成功", {
|
||
"record_id": record_id,
|
||
"source_record_id": source_record_id,
|
||
"items_total": len(items),
|
||
"inserted_count": inserted_count,
|
||
"updated_count": updated_count,
|
||
"target_job_id": target_job_id,
|
||
"target_job_code": target_job_code,
|
||
"queued_count": queued_count,
|
||
"deduplicated_job_items": deduplicated_job_items,
|
||
"skipped_job_items": skipped_job_items,
|
||
"deduplicated": False,
|
||
}
|
||
|
||
|
||
def _create_push_attempt(*, source_record: dict, ingest_url: str, sync_type: str) -> int:
|
||
payload = {
|
||
"sync_type": sync_type,
|
||
"source_record_id": source_record["id"],
|
||
"projection_hash": (source_record.get("payload") or {}).get("projection_hash", ""),
|
||
"ingest_url": ingest_url,
|
||
}
|
||
with get_db() as conn:
|
||
with conn.cursor() as cur:
|
||
cur.execute(
|
||
"""
|
||
INSERT INTO detect_sync_records (
|
||
sync_type, source_region, target_region, status, payload_json, error_message, created_at, updated_at
|
||
) VALUES (%s, %s, %s, %s, %s::jsonb, %s, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)
|
||
RETURNING id
|
||
""",
|
||
(
|
||
"runtime_push",
|
||
source_record["source_region"],
|
||
source_record["target_region"],
|
||
"pending",
|
||
json.dumps(payload, ensure_ascii=False),
|
||
"",
|
||
),
|
||
)
|
||
record_id = int(cur.fetchone()[0])
|
||
conn.commit()
|
||
return record_id
|
||
|
||
|
||
def _update_push_attempt(record_id: int, *, status: str, payload: dict | None = None, error_message: str = "") -> None:
|
||
with get_db() as conn:
|
||
with conn.cursor() as cur:
|
||
cur.execute(
|
||
"""
|
||
UPDATE detect_sync_records
|
||
SET status = %s,
|
||
payload_json = %s::jsonb,
|
||
error_message = %s,
|
||
updated_at = CURRENT_TIMESTAMP
|
||
WHERE id = %s
|
||
""",
|
||
(
|
||
str(status or "").strip() or "pending",
|
||
json.dumps(payload or {}, ensure_ascii=False),
|
||
str(error_message or "").strip(),
|
||
int(record_id),
|
||
),
|
||
)
|
||
conn.commit()
|
||
|
||
|
||
def _build_detect_result_import_fingerprint(
|
||
*,
|
||
source_region: str,
|
||
source_record_id: int,
|
||
source_job_code: str,
|
||
event: dict,
|
||
) -> str:
|
||
payload = _decode_json(event.get("payload"))
|
||
seed = {
|
||
"source_region": str(source_region or "").strip(),
|
||
"source_record_id": int(source_record_id or 0),
|
||
"source_job_code": str(source_job_code or "").strip(),
|
||
"node_code": str(event.get("node_code") or "").strip(),
|
||
"event_type": str(event.get("event_type") or "").strip(),
|
||
"message": str(event.get("message") or "").strip(),
|
||
"created_at": str(event.get("created_at") or "").strip(),
|
||
"domain": str(payload.get("domain") or "").strip(),
|
||
"domain_id": str(payload.get("domain_id") or "").strip(),
|
||
"status": str(payload.get("status") or "").strip(),
|
||
"cycle_token": str(payload.get("cycle_token") or "").strip(),
|
||
}
|
||
return hashlib.sha1(json.dumps(seed, ensure_ascii=False, sort_keys=True).encode("utf-8")).hexdigest()
|
||
|
||
|
||
def _extract_detect_result_projection_events(
|
||
*,
|
||
source_region: str,
|
||
source_record_id: int,
|
||
projection: dict,
|
||
) -> list[dict]:
|
||
source_job = projection.get("job") or {}
|
||
source_job_code = str(source_job.get("job_code") or "").strip()
|
||
source_job_id = int(source_job.get("job_id") or 0)
|
||
events: list[dict] = []
|
||
for raw_event in list(projection.get("recent_domain_events") or []):
|
||
event_type = str(raw_event.get("event_type") or "").strip()
|
||
if event_type not in _DETECT_RESULT_EVENT_TYPES:
|
||
continue
|
||
payload = _decode_json(raw_event.get("payload"))
|
||
fingerprint = _build_detect_result_import_fingerprint(
|
||
source_region=source_region,
|
||
source_record_id=source_record_id,
|
||
source_job_code=source_job_code,
|
||
event=raw_event,
|
||
)
|
||
payload.update(
|
||
{
|
||
"imported_from_projection": True,
|
||
"import_source_region": source_region,
|
||
"import_source_record_id": int(source_record_id or 0),
|
||
"import_source_job_id": source_job_id,
|
||
"import_source_job_code": source_job_code,
|
||
"import_fingerprint": fingerprint,
|
||
}
|
||
)
|
||
events.append(
|
||
{
|
||
"node_code": str(raw_event.get("node_code") or "").strip(),
|
||
"event_type": event_type,
|
||
"level": str(raw_event.get("level") or "info").strip() or "info",
|
||
"message": str(raw_event.get("message") or "").strip(),
|
||
"created_at": str(raw_event.get("created_at") or "").strip(),
|
||
"payload": payload,
|
||
"fingerprint": fingerprint,
|
||
}
|
||
)
|
||
return events
|
||
|
||
|
||
def _resolve_detect_result_target_job_id(*, projection: dict) -> int:
|
||
source_job = projection.get("job") or {}
|
||
source_job_code = str(source_job.get("job_code") or "").strip()
|
||
|
||
with get_db() as conn:
|
||
with conn.cursor() as cur:
|
||
if source_job_code:
|
||
cur.execute(
|
||
"""
|
||
SELECT id
|
||
FROM detect_jobs
|
||
WHERE job_code = %s
|
||
ORDER BY id DESC
|
||
LIMIT 1
|
||
""",
|
||
(source_job_code,),
|
||
)
|
||
row = cur.fetchone()
|
||
if row:
|
||
return int(row[0] or 0)
|
||
|
||
from app.services.detect_job_service import get_active_detect_job_summary
|
||
|
||
active_job = get_active_detect_job_summary(event_limit=1) or {}
|
||
return int(active_job.get("job_id") or 0)
|
||
|
||
|
||
def _parse_event_created_at(value: str) -> datetime | None:
|
||
text = str(value or "").strip()
|
||
if not text:
|
||
return None
|
||
try:
|
||
return datetime.fromisoformat(text)
|
||
except Exception:
|
||
return None
|
||
|
||
|
||
def _extract_event_domain(payload: dict, message: str) -> str:
|
||
domain = str(payload.get("domain") or "").strip().lower()
|
||
if domain:
|
||
return domain
|
||
text = str(message or "").strip()
|
||
if ":" in text:
|
||
candidate = text.rsplit(":", 1)[-1].strip().lower()
|
||
if candidate:
|
||
return candidate
|
||
return ""
|
||
|
||
|
||
def _apply_detect_result_event_to_domain(cur, event: dict) -> None:
|
||
payload = _decode_json(event.get("payload"))
|
||
domain = _extract_event_domain(payload, str(event.get("message") or ""))
|
||
if not domain:
|
||
return
|
||
|
||
event_type = str(event.get("event_type") or "").strip()
|
||
created_at = _parse_event_created_at(str(event.get("created_at") or ""))
|
||
effective_time = created_at or datetime.now()
|
||
|
||
if event_type == "domain_started":
|
||
cur.execute(
|
||
"""
|
||
UPDATE domains
|
||
SET detect_status = CASE
|
||
WHEN detect_status IN (1, 3) THEN detect_status
|
||
ELSE 2
|
||
END,
|
||
update_time = CURRENT_TIMESTAMP
|
||
WHERE domain = %s
|
||
""",
|
||
(domain,),
|
||
)
|
||
return
|
||
|
||
if event_type == "domain_completed":
|
||
cur.execute(
|
||
"""
|
||
UPDATE domains
|
||
SET detect_status = 1,
|
||
detect_time = COALESCE(detect_time, %s),
|
||
update_time = CURRENT_TIMESTAMP
|
||
WHERE domain = %s
|
||
""",
|
||
(effective_time, domain),
|
||
)
|
||
return
|
||
|
||
if event_type == "domain_blacklisted":
|
||
cur.execute(
|
||
"""
|
||
UPDATE domains
|
||
SET detect_status = 3,
|
||
update_time = CURRENT_TIMESTAMP
|
||
WHERE domain = %s
|
||
""",
|
||
(domain,),
|
||
)
|
||
return
|
||
|
||
if event_type == "domain_failed":
|
||
cur.execute(
|
||
"""
|
||
UPDATE domains
|
||
SET detect_status = CASE
|
||
WHEN detect_status IN (1, 3) THEN detect_status
|
||
ELSE 4
|
||
END,
|
||
update_time = CURRENT_TIMESTAMP
|
||
WHERE domain = %s
|
||
""",
|
||
(domain,),
|
||
)
|
||
return
|
||
|
||
|
||
def _apply_detect_result_event_to_job_item(cur, *, target_job_id: int, event: dict) -> int:
|
||
if int(target_job_id or 0) <= 0:
|
||
return 0
|
||
|
||
payload = _decode_json(event.get("payload"))
|
||
domain = _extract_event_domain(payload, str(event.get("message") or ""))
|
||
if not domain:
|
||
return 0
|
||
|
||
event_type = str(event.get("event_type") or "").strip()
|
||
node_code = str(event.get("node_code") or "").strip()
|
||
message = str(event.get("message") or "").strip()
|
||
|
||
if event_type == "domain_started":
|
||
cur.execute(
|
||
"""
|
||
UPDATE detect_job_items AS item
|
||
SET status = CASE
|
||
WHEN item.status IN ('completed', 'blacklisted', 'failed') THEN item.status
|
||
ELSE 'running'
|
||
END,
|
||
claimed_by = CASE
|
||
WHEN %s <> '' THEN %s
|
||
ELSE item.claimed_by
|
||
END,
|
||
started_at = COALESCE(item.started_at, CURRENT_TIMESTAMP),
|
||
updated_at = CURRENT_TIMESTAMP
|
||
FROM domains AS d
|
||
WHERE item.job_id = %s
|
||
AND item.domain_id = d.id
|
||
AND d.domain = %s
|
||
AND item.status IN ('pending', 'claimed', 'running')
|
||
""",
|
||
(node_code, node_code, int(target_job_id), domain),
|
||
)
|
||
return int(cur.rowcount or 0)
|
||
|
||
if event_type not in {"domain_completed", "domain_blacklisted", "domain_failed"}:
|
||
return 0
|
||
|
||
final_status = {
|
||
"domain_completed": "completed",
|
||
"domain_blacklisted": "blacklisted",
|
||
"domain_failed": "failed",
|
||
}[event_type]
|
||
cur.execute(
|
||
"""
|
||
UPDATE detect_job_items AS item
|
||
SET status = %s,
|
||
claimed_by = CASE
|
||
WHEN %s <> '' THEN %s
|
||
ELSE item.claimed_by
|
||
END,
|
||
finished_at = COALESCE(item.finished_at, CURRENT_TIMESTAMP),
|
||
updated_at = CURRENT_TIMESTAMP,
|
||
lease_expires_at = NULL,
|
||
last_error = CASE
|
||
WHEN %s = 'failed' THEN LEFT(%s, 1000)
|
||
ELSE item.last_error
|
||
END
|
||
FROM domains AS d
|
||
WHERE item.job_id = %s
|
||
AND item.domain_id = d.id
|
||
AND d.domain = %s
|
||
AND item.status IN ('pending', 'claimed', 'running')
|
||
""",
|
||
(
|
||
final_status,
|
||
node_code,
|
||
node_code,
|
||
final_status,
|
||
message,
|
||
int(target_job_id),
|
||
domain,
|
||
),
|
||
)
|
||
return int(cur.rowcount or 0)
|
||
|
||
|
||
def _import_detect_result_projection_events(
|
||
*,
|
||
source_region: str,
|
||
source_record_id: int,
|
||
projection: dict,
|
||
) -> dict:
|
||
events = _extract_detect_result_projection_events(
|
||
source_region=source_region,
|
||
source_record_id=source_record_id,
|
||
projection=projection,
|
||
)
|
||
if not events:
|
||
return {"imported_count": 0, "deduplicated_count": 0, "target_job_id": 0}
|
||
|
||
target_job_id = _resolve_detect_result_target_job_id(projection=projection)
|
||
|
||
imported_count = 0
|
||
deduplicated_count = 0
|
||
updated_job_items = 0
|
||
with get_db() as conn:
|
||
with conn.cursor() as cur:
|
||
for event in events:
|
||
cur.execute(
|
||
"""
|
||
SELECT id
|
||
FROM detect_run_events
|
||
WHERE job_id = %s
|
||
AND (payload_json->>'import_fingerprint') = %s
|
||
ORDER BY id DESC
|
||
LIMIT 1
|
||
""",
|
||
(target_job_id, event["fingerprint"]),
|
||
)
|
||
if cur.fetchone():
|
||
deduplicated_count += 1
|
||
continue
|
||
|
||
created_at = _parse_event_created_at(event.get("created_at", ""))
|
||
if created_at:
|
||
cur.execute(
|
||
"""
|
||
INSERT INTO detect_run_events (
|
||
job_id, node_code, event_type, level, message, payload_json, created_at
|
||
) VALUES (%s, %s, %s, %s, %s, %s::jsonb, %s)
|
||
""",
|
||
(
|
||
target_job_id,
|
||
event["node_code"],
|
||
event["event_type"],
|
||
event["level"],
|
||
event["message"],
|
||
json.dumps(event["payload"], ensure_ascii=False),
|
||
created_at,
|
||
),
|
||
)
|
||
else:
|
||
cur.execute(
|
||
"""
|
||
INSERT INTO detect_run_events (
|
||
job_id, node_code, event_type, level, message, payload_json
|
||
) VALUES (%s, %s, %s, %s, %s, %s::jsonb)
|
||
""",
|
||
(
|
||
target_job_id,
|
||
event["node_code"],
|
||
event["event_type"],
|
||
event["level"],
|
||
event["message"],
|
||
json.dumps(event["payload"], ensure_ascii=False),
|
||
),
|
||
)
|
||
_apply_detect_result_event_to_domain(cur, event)
|
||
if target_job_id > 0:
|
||
updated_job_items += _apply_detect_result_event_to_job_item(
|
||
cur,
|
||
target_job_id=target_job_id,
|
||
event=event,
|
||
)
|
||
imported_count += 1
|
||
conn.commit()
|
||
if target_job_id > 0:
|
||
from app.services.detect_job_service import refresh_detect_job_status
|
||
|
||
try:
|
||
refresh_detect_job_status(target_job_id)
|
||
except Exception:
|
||
pass
|
||
return {
|
||
"imported_count": imported_count,
|
||
"deduplicated_count": deduplicated_count,
|
||
"target_job_id": target_job_id,
|
||
"updated_job_items": updated_job_items,
|
||
}
|
||
|
||
|
||
def ingest_runtime_projection(payload: dict, *, shared_token: str | None = None) -> tuple[bool, str, dict]:
|
||
configured_token = str(settings.sync_shared_token or "").strip()
|
||
incoming_token = str(shared_token or "").strip()
|
||
if configured_token and incoming_token != configured_token:
|
||
return False, "同步 token 校验失败", {}
|
||
|
||
sync_type = str(payload.get("sync_type") or "runtime_projection").strip() or "runtime_projection"
|
||
ingest_sync_type = _projection_ingest_type(sync_type)
|
||
source_region = _normalize_region(payload.get("source_region"), "unknown")
|
||
source_record_id = int(payload.get("source_record_id") or 0)
|
||
projection_hash = str(payload.get("projection_hash") or "").strip()
|
||
projection = payload.get("projection") or {}
|
||
target_region = _normalize_region(settings.node_region, "overseas")
|
||
received_at = datetime.now()
|
||
|
||
with get_db() as conn:
|
||
with conn.cursor() as cur:
|
||
cur.execute(
|
||
"""
|
||
SELECT id
|
||
FROM detect_sync_records
|
||
WHERE sync_type = %s
|
||
AND source_region = %s
|
||
AND target_region = %s
|
||
AND (payload_json->>'source_record_id') = %s
|
||
AND (payload_json->>'projection_hash') = %s
|
||
ORDER BY created_at DESC, id DESC
|
||
LIMIT 1
|
||
""",
|
||
(ingest_sync_type, source_region, target_region, str(source_record_id), projection_hash),
|
||
)
|
||
existing = cur.fetchone()
|
||
if existing:
|
||
if sync_type == "runtime_projection":
|
||
_refresh_remote_runtime_node(source_region=source_region, projection=projection, received_at=received_at)
|
||
if sync_type == "detect_result_projection":
|
||
import_result = _import_detect_result_projection_events(
|
||
source_region=source_region,
|
||
source_record_id=source_record_id,
|
||
projection=projection,
|
||
)
|
||
else:
|
||
import_result = {}
|
||
cur.execute(
|
||
"""
|
||
UPDATE detect_sync_records
|
||
SET updated_at = CURRENT_TIMESTAMP
|
||
WHERE id = %s
|
||
""",
|
||
(int(existing[0]),),
|
||
)
|
||
conn.commit()
|
||
return True, "同步投影已存在，已按幂等处理", {
|
||
"record_id": int(existing[0]),
|
||
"deduplicated": True,
|
||
"event_import": import_result,
|
||
}
|
||
|
||
stored_payload = {
|
||
"sync_type": sync_type,
|
||
"source_record_id": source_record_id,
|
||
"projection_hash": projection_hash,
|
||
"projection": projection,
|
||
"received_at": _format_time(received_at),
|
||
}
|
||
cur.execute(
|
||
"""
|
||
INSERT INTO detect_sync_records (
|
||
sync_type, source_region, target_region, status, payload_json, error_message, created_at, updated_at
|
||
) VALUES (%s, %s, %s, %s, %s::jsonb, %s, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP)
|
||
RETURNING id
|
||
""",
|
||
(
|
||
ingest_sync_type,
|
||
source_region,
|
||
target_region,
|
||
"received",
|
||
json.dumps(stored_payload, ensure_ascii=False),
|
||
"",
|
||
),
|
||
)
|
||
record_id = int(cur.fetchone()[0])
|
||
conn.commit()
|
||
if sync_type == "runtime_projection":
|
||
_refresh_remote_runtime_node(source_region=source_region, projection=projection, received_at=received_at)
|
||
if sync_type == "detect_result_projection":
|
||
import_result = _import_detect_result_projection_events(
|
||
source_region=source_region,
|
||
source_record_id=source_record_id,
|
||
projection=projection,
|
||
)
|
||
else:
|
||
import_result = {}
|
||
return True, "同步投影接收成功", {
|
||
"record_id": record_id,
|
||
"deduplicated": False,
|
||
"event_import": import_result,
|
||
}
|
||
|
||
|
||
def _push_projection_now(sync_type: str, ingest_url: str) -> tuple[bool, str, dict]:
|
||
if sync_type == "runtime_projection":
|
||
# Regenerate the runtime snapshot before every push so the sync agent
|
||
# does not keep replaying a stale projection record while the worker
|
||
# thread count / phase is still changing.
|
||
from app.services.runtime_status_service import get_runtime_status
|
||
|
||
try:
|
||
get_runtime_status()
|
||
except Exception as exc:
|
||
return False, f"刷新 runtime_projection 失败: {exc}", {"action": "push_sync", "sync_type": sync_type}
|
||
source_record = _load_latest_projection(sync_type)
|
||
if not source_record:
|
||
return False, f"当前没有可推送的{sync_type}", {"action": "push_sync", "sync_type": sync_type}
|
||
return _push_projection_record(source_record, sync_type, ingest_url)
|
||
|
||
|
||
def _push_projection_batch(sync_type: str, ingest_url: str) -> tuple[bool, str, dict]:
|
||
pending_records = _load_pushable_projections(sync_type, limit=max(1, int(settings.sync_batch_size or 200)))
|
||
if not pending_records:
|
||
latest = _load_latest_projection(sync_type)
|
||
if latest:
|
||
latest_attempt = _latest_push_attempt(latest["id"], latest["target_region"], sync_type)
|
||
if latest_attempt and latest_attempt["status"] == "success":
|
||
return True, "当前批次已全部同步完成", {
|
||
"action": "push_sync",
|
||
"sync_type": sync_type,
|
||
"deduplicated": True,
|
||
"batch_count": 0,
|
||
}
|
||
return False, f"当前没有可推送的{sync_type}", {"action": "push_sync", "sync_type": sync_type, "batch_count": 0}
|
||
|
||
results: list[dict] = []
|
||
success_count = 0
|
||
for record in pending_records:
|
||
ok, message, data = _push_projection_record(record, sync_type, ingest_url)
|
||
results.append(
|
||
{
|
||
"source_record_id": record["id"],
|
||
"ok": ok,
|
||
"message": message,
|
||
"data": data,
|
||
}
|
||
)
|
||
if ok:
|
||
success_count += 1
|
||
|
||
overall_ok = success_count > 0
|
||
message = f"{sync_type} 批量推送完成，成功 {success_count}/{len(results)}"
|
||
return overall_ok, message, {
|
||
"action": "push_sync",
|
||
"sync_type": sync_type,
|
||
"batch_count": len(results),
|
||
"success_count": success_count,
|
||
"results": results,
|
||
}
|
||
|
||
|
||
def _push_projection_record(source_record: dict, sync_type: str, ingest_url: str) -> tuple[bool, str, dict]:
|
||
latest_attempt = _latest_push_attempt(source_record["id"], source_record["target_region"], sync_type)
|
||
if latest_attempt and latest_attempt["status"] == "success":
|
||
last_created_at = latest_attempt.get("created_at")
|
||
if sync_type != "runtime_projection" or not isinstance(last_created_at, datetime):
|
||
return True, "该投影已推送，无需重复发送", {
|
||
"action": "push_sync",
|
||
"sync_type": sync_type,
|
||
"source_record_id": source_record["id"],
|
||
"deduplicated": True,
|
||
}
|
||
now = datetime.now(last_created_at.tzinfo) if last_created_at.tzinfo else datetime.now()
|
||
if now - last_created_at < timedelta(seconds=max(20, int(settings.sync_poll_interval_seconds or 30))):
|
||
return True, "该投影已推送，无需重复发送", {
|
||
"action": "push_sync",
|
||
"sync_type": sync_type,
|
||
"source_record_id": source_record["id"],
|
||
"deduplicated": True,
|
||
}
|
||
if latest_attempt and latest_attempt["status"] == "pending":
|
||
return True, "该投影已有同步推送进行中，暂不重复发送", {
|
||
"action": "push_sync",
|
||
"sync_type": sync_type,
|
||
"source_record_id": source_record["id"],
|
||
"deduplicated": True,
|
||
}
|
||
if latest_attempt and latest_attempt["status"] == "failed":
|
||
last_created_at = latest_attempt.get("created_at")
|
||
if isinstance(last_created_at, datetime):
|
||
now = datetime.now(last_created_at.tzinfo) if last_created_at.tzinfo else datetime.now()
|
||
if now - last_created_at < timedelta(seconds=max(10, int(settings.sync_poll_interval_seconds or 30))):
|
||
return False, "最近一次同步推送刚失败，等待下个重试窗口", {
|
||
"action": "push_sync",
|
||
"sync_type": sync_type,
|
||
"source_record_id": source_record["id"],
|
||
"deduplicated": True,
|
||
}
|
||
|
||
attempt_id = _create_push_attempt(source_record=source_record, ingest_url=ingest_url, sync_type=sync_type)
|
||
request_payload = {
|
||
"sync_type": sync_type,
|
||
"source_region": source_record["source_region"],
|
||
"source_record_id": source_record["id"],
|
||
"projection_hash": (source_record.get("payload") or {}).get("projection_hash", ""),
|
||
"projection": (source_record.get("payload") or {}).get("projection", {}),
|
||
"created_at": _format_time(source_record.get("created_at")),
|
||
}
|
||
request_body = json.dumps(request_payload, ensure_ascii=False).encode("utf-8")
|
||
request = urllib.request.Request(
|
||
ingest_url,
|
||
data=request_body,
|
||
headers={
|
||
"Content-Type": "application/json",
|
||
**({"X-Domaincheck-Sync-Token": settings.sync_shared_token} if settings.sync_shared_token else {}),
|
||
},
|
||
method="POST",
|
||
)
|
||
try:
|
||
with urllib.request.urlopen(request, timeout=15) as response:
|
||
raw = response.read().decode("utf-8")
|
||
data = json.loads(raw) if raw else {}
|
||
status_code = getattr(response, "status", 200)
|
||
except json.JSONDecodeError as exc:
|
||
payload = {
|
||
"sync_type": sync_type,
|
||
"source_record_id": source_record["id"],
|
||
"projection_hash": request_payload["projection_hash"],
|
||
"ingest_url": ingest_url,
|
||
}
|
||
_update_push_attempt(attempt_id, status="failed", payload=payload, error_message=f"invalid json response: {exc}")
|
||
return False, f"同步推送失败: 远端响应不是合法 JSON ({exc})", {"action": "push_sync", "sync_type": sync_type, "attempt_id": attempt_id}
|
||
except urllib.error.HTTPError as exc:
|
||
error_body = exc.read().decode("utf-8", errors="ignore") if hasattr(exc, "read") else ""
|
||
payload = {
|
||
"sync_type": sync_type,
|
||
"source_record_id": source_record["id"],
|
||
"projection_hash": request_payload["projection_hash"],
|
||
"ingest_url": ingest_url,
|
||
"http_status": getattr(exc, "code", 500),
|
||
"response_text": error_body[:1000],
|
||
}
|
||
_update_push_attempt(attempt_id, status="failed", payload=payload, error_message=f"HTTP {getattr(exc, 'code', 500)}")
|
||
return False, f"同步推送失败: HTTP {getattr(exc, 'code', 500)}", {"action": "push_sync", "sync_type": sync_type, "attempt_id": attempt_id}
|
||
except Exception as exc:
|
||
payload = {
|
||
"sync_type": sync_type,
|
||
"source_record_id": source_record["id"],
|
||
"projection_hash": request_payload["projection_hash"],
|
||
"ingest_url": ingest_url,
|
||
}
|
||
_update_push_attempt(attempt_id, status="failed", payload=payload, error_message=str(exc))
|
||
return False, f"同步推送失败: {exc}", {"action": "push_sync", "sync_type": sync_type, "attempt_id": attempt_id}
|
||
|
||
payload = {
|
||
"sync_type": sync_type,
|
||
"source_record_id": source_record["id"],
|
||
"projection_hash": request_payload["projection_hash"],
|
||
"ingest_url": ingest_url,
|
||
"http_status": status_code,
|
||
"response": data,
|
||
}
|
||
response_code = data.get("code", 0)
|
||
if response_code not in (0, "0", None, ""):
|
||
_update_push_attempt(
|
||
attempt_id,
|
||
status="failed",
|
||
payload=payload,
|
||
error_message=str(data.get("message") or "remote business error"),
|
||
)
|
||
return False, f"同步推送失败: {str(data.get('message') or '远端业务返回失败')}", {
|
||
"action": "push_sync",
|
||
"sync_type": sync_type,
|
||
"attempt_id": attempt_id,
|
||
"source_record_id": source_record["id"],
|
||
"response": data,
|
||
}
|
||
_update_push_attempt(attempt_id, status="success", payload=payload, error_message="")
|
||
return True, "投影推送成功", {
|
||
"action": "push_sync",
|
||
"sync_type": sync_type,
|
||
"attempt_id": attempt_id,
|
||
"source_record_id": source_record["id"],
|
||
"response": data,
|
||
}
|
||
|
||
|
||
def push_runtime_projection_now() -> tuple[bool, str, dict]:
|
||
if not settings.sync_push_enabled:
|
||
return False, "未启用同步推送", {"action": "push_sync", "sync_state": "disabled", "ui_level": "warning", "poll_schedule_seconds": []}
|
||
|
||
ingest_url = _ingest_url(settings.sync_target_api_base_url)
|
||
if not ingest_url:
|
||
return False, "未配置同步目标地址", {"action": "push_sync", "sync_state": "misconfigured", "ui_level": "warning", "poll_schedule_seconds": []}
|
||
|
||
results = []
|
||
ok, message, data = _push_projection_now("runtime_projection", ingest_url)
|
||
results.append({"sync_type": "runtime_projection", "ok": ok, "message": message, "data": data})
|
||
|
||
ok, message, data = _push_projection_batch("detect_result_projection", ingest_url)
|
||
results.append({"sync_type": "detect_result_projection", "ok": ok, "message": message, "data": data})
|
||
|
||
success_count = sum(1 for item in results if item["ok"])
|
||
warning_count = sum(
|
||
1
|
||
for item in results
|
||
if not item["ok"] and any(keyword in str(item.get("message") or "") for keyword in ("当前没有可推送", "已全部同步完成", "无需重复发送", "进行中", "等待下个重试窗口"))
|
||
)
|
||
if success_count == 0:
|
||
if warning_count == len(results):
|
||
return False, "当前没有需要立即推送的同步投影", {
|
||
"action": "push_sync",
|
||
"sync_state": "idle",
|
||
"ui_level": "warning",
|
||
"poll_schedule_seconds": [],
|
||
"results": results,
|
||
}
|
||
return False, "同步推送未成功，请检查明细结果", {
|
||
"action": "push_sync",
|
||
"sync_state": "failed",
|
||
"ui_level": "error",
|
||
"poll_schedule_seconds": [2],
|
||
"results": results,
|
||
}
|
||
if success_count < len(results):
|
||
return True, f"同步推送部分完成，成功 {success_count}/{len(results)}", {
|
||
"action": "push_sync",
|
||
"sync_state": "partial_success",
|
||
"ui_level": "warning",
|
||
"poll_schedule_seconds": [1, 3],
|
||
"results": results,
|
||
}
|
||
return True, f"同步推送完成，成功 {success_count}/{len(results)}", {
|
||
"action": "push_sync",
|
||
"sync_state": "success",
|
||
"ui_level": "success",
|
||
"poll_schedule_seconds": [1, 3],
|
||
"results": results,
|
||
}
|
||
|
||
|
||
def pull_detect_task_batch_now(limit: int | None = None) -> tuple[bool, str, dict]:
|
||
if settings.node_region != "mainland" or settings.node_role != "control":
|
||
return False, "当前节点无需拉取待检测任务批次", {"action": "pull_tasks", "pull_state": "not_applicable", "ui_level": "warning", "poll_schedule_seconds": []}
|
||
|
||
export_url = _task_export_url(settings.sync_target_api_base_url)
|
||
ack_url = _task_ack_url(settings.sync_target_api_base_url)
|
||
if not export_url or not ack_url:
|
||
return False, "未配置任务拉取目标地址", {"action": "pull_tasks", "pull_state": "misconfigured", "ui_level": "warning", "poll_schedule_seconds": []}
|
||
|
||
configured_limit = max(5000, int(settings.sync_batch_size or 200))
|
||
requested_limit = int(limit or configured_limit)
|
||
safe_limit = max(1, min(requested_limit, max(10000, configured_limit)))
|
||
settings_payload = get_settings_payload()
|
||
backlog_snapshot = _load_local_detect_backlog_snapshot()
|
||
backlog_limits = _build_task_pull_backlog_limits(configured_limit, settings_payload=settings_payload)
|
||
should_throttle, throttle_reason = _should_throttle_task_pull(backlog_snapshot, backlog_limits)
|
||
if should_throttle:
|
||
return True, "本地待处理积压较高，暂停拉取新批次", {
|
||
"action": "pull_tasks",
|
||
"pull_state": "throttled",
|
||
"ui_level": "info",
|
||
"poll_schedule_seconds": [1, 3],
|
||
"reason": throttle_reason,
|
||
**backlog_snapshot,
|
||
**backlog_limits,
|
||
}
|
||
|
||
request_url = f"{export_url}?limit={safe_limit}"
|
||
export_timeout = max(20, min(90, 15 + safe_limit // 40))
|
||
request = urllib.request.Request(
|
||
request_url,
|
||
headers={**({"X-Domaincheck-Sync-Token": settings.sync_shared_token} if settings.sync_shared_token else {})},
|
||
method="GET",
|
||
)
|
||
try:
|
||
with urllib.request.urlopen(request, timeout=export_timeout) as response:
|
||
raw = response.read().decode("utf-8")
|
||
data = json.loads(raw) if raw else {}
|
||
except json.JSONDecodeError as exc:
|
||
return False, f"拉取待检测任务失败: 远端响应不是合法 JSON ({exc})", {"action": "pull_tasks", "pull_state": "failed", "ui_level": "error", "poll_schedule_seconds": [2]}
|
||
except urllib.error.HTTPError as exc:
|
||
error_body = exc.read().decode("utf-8", errors="ignore") if hasattr(exc, "read") else ""
|
||
return False, f"拉取待检测任务失败: HTTP {getattr(exc, 'code', 500)}", {
|
||
"action": "pull_tasks",
|
||
"pull_state": "failed",
|
||
"ui_level": "error",
|
||
"poll_schedule_seconds": [2],
|
||
"http_status": getattr(exc, "code", 500),
|
||
"response_text": error_body[:500],
|
||
}
|
||
except Exception as exc:
|
||
return False, f"拉取待检测任务失败: {exc}", {"action": "pull_tasks", "pull_state": "failed", "ui_level": "error", "poll_schedule_seconds": [2]}
|
||
|
||
payload = data.get("data") or {}
|
||
response_code = data.get("code", 0)
|
||
if response_code not in (0, "0", None) and not payload:
|
||
return False, str(data.get("message") or "拉取待检测任务失败"), {"action": "pull_tasks", "pull_state": "failed", "ui_level": "error", "poll_schedule_seconds": [2], "response": data}
|
||
|
||
source_record_id = int(payload.get("source_record_id") or 0)
|
||
projection_hash = str(payload.get("projection_hash") or "").strip()
|
||
projection = payload.get("projection") or {}
|
||
if source_record_id <= 0 or not projection_hash or not projection:
|
||
return False, "远端当前没有可拉取的待检测任务批次", {"action": "pull_tasks", "pull_state": "idle", "ui_level": "warning", "poll_schedule_seconds": [], "batch_size": 0}
|
||
|
||
ingest_ok, ingest_message, ingest_data = ingest_detect_task_projection(
|
||
{
|
||
"sync_type": "detect_task_projection",
|
||
"source_region": _normalize_region(settings.sync_target_region, "overseas"),
|
||
"source_record_id": source_record_id,
|
||
"projection_hash": projection_hash,
|
||
"projection": projection,
|
||
},
|
||
shared_token=settings.sync_shared_token,
|
||
)
|
||
if not ingest_ok:
|
||
return False, ingest_message, {"action": "pull_tasks", "pull_state": "failed", "ui_level": "error", "poll_schedule_seconds": [2], **(ingest_data or {})}
|
||
|
||
ack_request = urllib.request.Request(
|
||
ack_url,
|
||
data=json.dumps(
|
||
{
|
||
"source_record_id": source_record_id,
|
||
"projection_hash": projection_hash,
|
||
},
|
||
ensure_ascii=False,
|
||
).encode("utf-8"),
|
||
headers={
|
||
"Content-Type": "application/json",
|
||
**({"X-Domaincheck-Sync-Token": settings.sync_shared_token} if settings.sync_shared_token else {}),
|
||
},
|
||
method="POST",
|
||
)
|
||
ack_data = {}
|
||
try:
|
||
with urllib.request.urlopen(ack_request, timeout=15) as response:
|
||
raw = response.read().decode("utf-8")
|
||
ack_response = json.loads(raw) if raw else {}
|
||
ack_data = ack_response.get("data") or {}
|
||
ack_code = ack_response.get("code", 0)
|
||
if ack_code not in (0, "0", None, ""):
|
||
return True, f"{ingest_message}；但远端确认失败: {str(ack_response.get('message') or 'ack business error')}", {
|
||
"action": "pull_tasks",
|
||
"pull_state": "ack_warning",
|
||
"ui_level": "warning",
|
||
"poll_schedule_seconds": [1, 3],
|
||
"source_record_id": source_record_id,
|
||
"projection_hash": projection_hash,
|
||
**(ingest_data or {}),
|
||
"ack": ack_data,
|
||
"ack_response": ack_response,
|
||
}
|
||
except json.JSONDecodeError as exc:
|
||
return True, f"{ingest_message}；但远端确认失败: ack 响应不是合法 JSON ({exc})", {
|
||
"action": "pull_tasks",
|
||
"pull_state": "ack_warning",
|
||
"ui_level": "warning",
|
||
"poll_schedule_seconds": [1, 3],
|
||
"source_record_id": source_record_id,
|
||
**(ingest_data or {}),
|
||
}
|
||
except Exception as exc:
|
||
return True, f"{ingest_message}；但远端确认失败: {exc}", {
|
||
"action": "pull_tasks",
|
||
"pull_state": "ack_warning",
|
||
"ui_level": "warning",
|
||
"poll_schedule_seconds": [1, 3],
|
||
"source_record_id": source_record_id,
|
||
**(ingest_data or {}),
|
||
}
|
||
|
||
result = {
|
||
"action": "pull_tasks",
|
||
"pull_state": "success",
|
||
"ui_level": "success",
|
||
"poll_schedule_seconds": [1, 3],
|
||
"source_record_id": source_record_id,
|
||
"projection_hash": projection_hash,
|
||
"batch_code": str(projection.get("batch_code") or "").strip(),
|
||
**(ingest_data or {}),
|
||
"ack": ack_data,
|
||
}
|
||
queued_count = int(result.get("queued_count", 0) or 0)
|
||
if queued_count > 0:
|
||
try:
|
||
from app.services.worker_control_service import send_worker_command
|
||
|
||
start_ok, start_message = send_worker_command(
|
||
"start_detection",
|
||
payload={
|
||
"source": "sync-pull",
|
||
"source_record_id": source_record_id,
|
||
"target_job_id": int(result.get("target_job_id", 0) or 0),
|
||
"target_job_code": str(result.get("target_job_code") or "").strip(),
|
||
},
|
||
)
|
||
result["worker_start_ok"] = bool(start_ok)
|
||
result["worker_start_message"] = str(start_message or "").strip()
|
||
except Exception as exc:
|
||
result["worker_start_ok"] = False
|
||
result["worker_start_message"] = f"同步入库后自动唤起 Worker 失败: {exc}"
|
||
result["ui_level"] = "warning"
|
||
result["pull_state"] = "worker_start_warning"
|
||
return True, "待检测任务批次拉取并入库成功；但自动唤起 Worker 失败", result
|
||
|
||
return True, "待检测任务批次拉取并入库成功", result
|