from __future__ import annotations import hashlib import json import socket import urllib.error import urllib.parse import urllib.request from datetime import datetime, timedelta from uuid import uuid4 from app.core.config import settings from app.core.db import get_db from app.services.cluster_runtime_service import ( cleanup_imported_runtime_nodes, cleanup_imported_runtime_nodes_many, register_node_heartbeat, ) from app.services.detect_job_service import ( _load_domain_pipeline_snapshot, resolve_initial_domain_pipeline_item, ) from app.services.settings_service import get_settings_payload from app.services.sync_record_service import _decode_json, _normalize_region _DETECT_RESULT_EVENT_TYPES = { "domain_started", "domain_completed", "domain_failed", "domain_blacklisted", } def _format_time(value: datetime | None) -> str: return value.isoformat(sep=" ", timespec="seconds") if value else "" def _ingest_url(base_url: str) -> str: text = str(base_url or "").strip().rstrip("/") if not text: return "" if text.endswith("/api/v1"): return f"{text}/runtime/sync-ingest" if text.endswith("/api/v1/runtime"): return f"{text}/sync-ingest" return f"{text}/api/v1/runtime/sync-ingest" def _task_export_url(base_url: str) -> str: text = str(base_url or "").strip().rstrip("/") if not text: return "" if text.endswith("/api/v1"): return f"{text}/runtime/task-export" if text.endswith("/api/v1/runtime"): return f"{text}/task-export" return f"{text}/api/v1/runtime/task-export" def _task_ack_url(base_url: str) -> str: text = str(base_url or "").strip().rstrip("/") if not text: return "" if text.endswith("/api/v1"): return f"{text}/runtime/task-ack" if text.endswith("/api/v1/runtime"): return f"{text}/task-ack" return f"{text}/api/v1/runtime/task-ack" def _projection_ingest_type(sync_type: str) -> str: if sync_type == "runtime_projection": return "runtime_ingest" if sync_type == "detect_result_projection": return "detect_result_ingest" if sync_type == "detect_task_projection": return "detect_task_ingest" return "sync_ingest" def _refresh_remote_runtime_node(*, source_region: str, projection: dict, received_at: datetime | None = None) -> None: node_info = projection.get("node") or {} node_code = str(node_info.get("node_code") or "").strip() region = _normalize_region(node_info.get("region"), source_region) role = str(node_info.get("role") or "control").strip() or "control" hostname = str(node_info.get("hostname") or "").strip() or socket.gethostname() ip = str(node_info.get("ip") or "").strip() if not node_code: node_code = f"{region}-{role}-imported" controller_current_load = max( int(projection.get("active_thread_count", 0) or 0), int(((projection.get("active_job") or {}).get("items_running", 0) or 0)), ) metadata = { "service": "runtime-ingest", "projection_source_region": source_region, "worker_mode": projection.get("worker_mode", ""), "active_threads": int(projection.get("active_thread_count", 0) or 0), "max_threads": int(projection.get("max_thread_count", 0) or 0), "phase_label": projection.get("phase_label", ""), "phase_detail": projection.get("phase_detail", ""), "proxy_runtime_label": projection.get("proxy_runtime_label", ""), "proxy_runtime_reason": projection.get("proxy_runtime_reason", ""), "worker_online": bool(projection.get("worker_online", False)), "detect_participating": bool(projection.get("detect_participating", False)), "active_job_code": str((projection.get("active_job") or {}).get("job_code") or ""), "active_job_status": str((projection.get("active_job") or {}).get("status") or ""), "job_items_total": int(((projection.get("active_job") or {}).get("items_total", 0) or 0)), "job_items_claimed": int(((projection.get("active_job") or {}).get("items_claimed", 0) or 0)), "job_items_running": int(((projection.get("active_job") or {}).get("items_running", 0) or 0)), "job_items_completed": int(((projection.get("active_job") or {}).get("items_completed", 0) or 0)), "updated_at": _format_time(received_at or datetime.now()), } register_node_heartbeat( node_code=node_code, region=region, role=role, status="busy" if controller_current_load > 0 else "online", current_load=controller_current_load, metadata=metadata, hostname_override=hostname, ip_override=ip, ) cleanup_imported_runtime_nodes(region=region, role=role, keep_node_code=node_code) active_job = projection.get("active_job") or {} worker_node_codes: list[str] = [] for node_stat in list(active_job.get("node_stats") or []): worker_node_code = str(node_stat.get("node_code") or "").strip() if not worker_node_code or worker_node_code == "unassigned": continue if worker_node_code == node_code: continue items_running = int(node_stat.get("items_running", 0) or 0) items_claimed = int(node_stat.get("items_claimed", 0) or 0) items_total = int(node_stat.get("items_total", 0) or 0) worker_runtime_load = int(node_stat.get("current_load", 0) or 0) worker_active_threads = int(node_stat.get("active_threads", worker_runtime_load) or 0) worker_max_threads = int(node_stat.get("max_threads", 0) or 0) worker_load = max(worker_active_threads, items_running, 0) worker_status = "busy" if worker_load > 0 else "online" worker_metadata = { "service": "runtime-ingest", "projection_source_region": source_region, "worker_mode": projection.get("worker_mode", ""), "phase_label": projection.get("phase_label", ""), "phase_detail": projection.get("phase_detail", ""), "proxy_runtime_label": projection.get("proxy_runtime_label", ""), "proxy_runtime_reason": projection.get("proxy_runtime_reason", ""), "active_threads": worker_active_threads, "max_threads": worker_max_threads, "updated_at": _format_time(received_at or datetime.now()), "job_items_total": items_total, "job_items_running": items_running, "job_items_claimed": items_claimed, "job_items_completed": int(node_stat.get("items_completed", 0) or 0), "job_items_failed": int(node_stat.get("items_failed", 0) or 0), "job_items_blacklisted": int(node_stat.get("items_blacklisted", 0) or 0), "metrics_source": str(node_stat.get("metrics_source") or "runtime").strip() or "runtime", "source_status": str(node_stat.get("status") or "").strip(), "source_role": str(node_stat.get("role") or "worker").strip() or "worker", "derived_from": node_code, } register_node_heartbeat( node_code=worker_node_code, region=region, role="worker", status=worker_status, current_load=worker_load, metadata=worker_metadata, hostname_override=hostname, ip_override=ip, ) worker_node_codes.append(worker_node_code) if worker_node_codes: cleanup_imported_runtime_nodes_many(region=region, role="worker", keep_node_codes=worker_node_codes) def _load_latest_projection(sync_type: str) -> dict | None: source_region = _normalize_region(settings.sync_source_region, settings.node_region) target_region = _normalize_region(settings.sync_target_region, "overseas") with get_db() as conn: with conn.cursor() as cur: cur.execute( """ SELECT id, source_region, target_region, status, payload_json, created_at, updated_at FROM detect_sync_records WHERE sync_type = %s AND source_region = %s AND target_region = %s ORDER BY created_at DESC, id DESC LIMIT 1 """, (sync_type, source_region, target_region), ) row = cur.fetchone() if not row: return None return { "id": row[0], "source_region": row[1], "target_region": row[2], "status": row[3], "payload": _decode_json(row[4]), "created_at": row[5], "updated_at": row[6], } def _load_pushable_projections(sync_type: str, limit: int) -> list[dict]: safe_limit = max(1, min(int(limit or 1), max(1, int(settings.sync_batch_size or 200)))) source_region = _normalize_region(settings.sync_source_region, settings.node_region) target_region = _normalize_region(settings.sync_target_region, "overseas") with get_db() as conn: with conn.cursor() as cur: cur.execute( """ SELECT id, source_region, target_region, status, payload_json, created_at, updated_at FROM detect_sync_records WHERE sync_type = %s AND source_region = %s AND target_region = %s ORDER BY created_at ASC, id ASC LIMIT %s """, (sync_type, source_region, target_region, safe_limit * 5), ) rows = cur.fetchall() selected: list[dict] = [] for row in rows: projection = { "id": row[0], "source_region": row[1], "target_region": row[2], "status": row[3], "payload": _decode_json(row[4]), "created_at": row[5], "updated_at": row[6], } latest_attempt = _latest_push_attempt(projection["id"], projection["target_region"], sync_type) if latest_attempt and latest_attempt["status"] == "success": continue if latest_attempt and latest_attempt["status"] == "pending": continue if latest_attempt and latest_attempt["status"] == "failed": last_created_at = latest_attempt.get("created_at") if isinstance(last_created_at, datetime): now = datetime.now(last_created_at.tzinfo) if last_created_at.tzinfo else datetime.now() if now - last_created_at < timedelta(seconds=max(10, int(settings.sync_poll_interval_seconds or 30))): continue selected.append(projection) if len(selected) >= safe_limit: break return selected def _estimate_total_worker_threads(settings_payload: dict | None = None) -> int: payload = settings_payload if isinstance(settings_payload, dict) else get_settings_payload() default_threads = max(1, int(payload.get("thread_count", 100) or 100)) node_thread_counts = payload.get("node_thread_counts") if isinstance(payload.get("node_thread_counts"), dict) else {} total_threads = 0 for raw_value in node_thread_counts.values(): try: total_threads += max(0, int(raw_value or 0)) except (TypeError, ValueError): continue return max(total_threads, default_threads) def _load_local_detect_backlog_snapshot() -> dict: with get_db() as conn: with conn.cursor() as cur: cur.execute( """ SELECT COUNT(*) FILTER (WHERE item.status = 'pending') AS pending_total, COUNT(*) FILTER (WHERE item.status = 'claimed') AS claimed_total, COUNT(*) FILTER (WHERE item.status = 'running') AS running_total, COUNT(*) FILTER (WHERE item.status = 'pending' AND item.step_code = 'detect_register') AS register_pending, COUNT(*) FILTER (WHERE item.status = 'pending' AND item.step_code <> 'detect_register') AS downstream_pending FROM detect_job_items item JOIN detect_jobs job ON job.id = item.job_id WHERE job.status IN ('pending', 'running') """ ) row = cur.fetchone() or (0, 0, 0, 0, 0) return { "pending_total": int(row[0] or 0), "claimed_total": int(row[1] or 0), "running_total": int(row[2] or 0), "register_pending": int(row[3] or 0), "downstream_pending": int(row[4] or 0), } def _build_task_pull_backlog_limits(configured_limit: int, settings_payload: dict | None = None) -> dict: estimated_total_threads = _estimate_total_worker_threads(settings_payload) max_pending_total = int(settings.sync_pull_max_pending_items or 0) if max_pending_total <= 0: max_pending_total = max(int(configured_limit or 0), estimated_total_threads * 2) max_register_pending = int(settings.sync_pull_max_register_pending_items or 0) if max_register_pending <= 0: max_register_pending = max(max(500, int(configured_limit or 0) // 2), estimated_total_threads) max_downstream_pending = int(settings.sync_pull_max_downstream_pending_items or 0) if max_downstream_pending <= 0: max_downstream_pending = max(250, estimated_total_threads // 4) return { "estimated_total_threads": estimated_total_threads, "max_pending_total": max_pending_total, "max_register_pending": max_register_pending, "max_downstream_pending": max_downstream_pending, } def _should_throttle_task_pull(backlog_snapshot: dict, backlog_limits: dict) -> tuple[bool, str]: pending_total = int(backlog_snapshot.get("pending_total", 0) or 0) register_pending = int(backlog_snapshot.get("register_pending", 0) or 0) downstream_pending = int(backlog_snapshot.get("downstream_pending", 0) or 0) max_pending_total = int(backlog_limits.get("max_pending_total", 0) or 0) max_register_pending = int(backlog_limits.get("max_register_pending", 0) or 0) max_downstream_pending = int(backlog_limits.get("max_downstream_pending", 0) or 0) if max_pending_total > 0 and pending_total >= max_pending_total: return True, "pending_total" if ( downstream_pending > 0 and max_register_pending > 0 and register_pending >= max_register_pending ): return True, "register_pending" if max_downstream_pending > 0 and downstream_pending >= max_downstream_pending: return True, "downstream_pending" return False, "" def _latest_push_attempt(source_record_id: int, target_region: str, sync_type: str) -> dict | None: with get_db() as conn: with conn.cursor() as cur: cur.execute( """ SELECT id, status, payload_json, error_message, created_at, updated_at FROM detect_sync_records WHERE sync_type = 'runtime_push' AND source_region = %s AND target_region = %s AND (payload_json->>'sync_type') = %s AND (payload_json->>'source_record_id') = %s ORDER BY created_at DESC, id DESC LIMIT 1 """, ( _normalize_region(settings.sync_source_region, settings.node_region), target_region, sync_type, str(int(source_record_id)), ), ) row = cur.fetchone() if not row: return None return { "id": row[0], "status": row[1], "payload": _decode_json(row[2]), "error_message": row[3] or "", "created_at": row[4], "updated_at": row[5], } def _latest_ingest_attempt(source_record_id: int, target_region: str, sync_type: str) -> dict | None: ingest_sync_type = _projection_ingest_type(sync_type) with get_db() as conn: with conn.cursor() as cur: cur.execute( """ SELECT id, status, payload_json, error_message, created_at, updated_at FROM detect_sync_records WHERE sync_type = %s AND source_region = %s AND target_region = %s AND (payload_json->>'source_record_id') = %s ORDER BY created_at DESC, id DESC LIMIT 1 """, ( ingest_sync_type, _normalize_region(settings.sync_source_region, settings.node_region), target_region, str(int(source_record_id)), ), ) row = cur.fetchone() if not row: return None return { "id": row[0], "status": row[1], "payload": _decode_json(row[2]), "error_message": row[3] or "", "created_at": row[4], "updated_at": row[5], } def _task_selection_sql() -> str: return """ SELECT id, domain, tld, source_type, use_status, detect_status, register_status, expire_date, jucha_status, juziseo_status FROM domains WHERE detect_status IN (0, 4) OR (use_status = 0 AND detect_status = 1 AND register_status = 3 AND expire_date < CURRENT_DATE) ORDER BY id ASC LIMIT %s """ def _task_projection_limit(limit: int | None) -> int: requested = max(1, int(limit or 5000)) configured = max(5000, int(settings.sync_batch_size or 200)) cap = max(10000, configured, 5000) return max(1, min(requested, cap)) def _task_projection_items_total(projection: dict) -> int: payload = projection.get("payload") or {} projection_payload = payload.get("projection") or {} try: return int(projection_payload.get("items_total", 0) or 0) except Exception: return 0 def _task_projection_selection_limit(projection: dict) -> int: payload = projection.get("payload") or {} projection_payload = payload.get("projection") or {} try: return int(projection_payload.get("selection_limit", 0) or 0) except Exception: return 0 def _mark_task_projection_superseded(record_id: int, *, reason: str) -> None: with get_db() as conn: with conn.cursor() as cur: cur.execute( """ UPDATE detect_sync_records SET status = 'superseded', error_message = %s, updated_at = CURRENT_TIMESTAMP WHERE id = %s """, (str(reason or "").strip()[:500], int(record_id)), ) conn.commit() def _load_pending_task_projection(limit: int) -> dict | None: source_region = _normalize_region(settings.sync_source_region, settings.node_region) target_region = _normalize_region(settings.sync_target_region, "overseas") safe_limit = _task_projection_limit(limit) with get_db() as conn: with conn.cursor() as cur: cur.execute( """ SELECT id, source_region, target_region, status, payload_json, created_at, updated_at FROM detect_sync_records WHERE sync_type = 'detect_task_projection' AND source_region = %s AND target_region = %s ORDER BY created_at ASC, id ASC LIMIT %s """, (source_region, target_region, safe_limit * 3), ) rows = cur.fetchall() for row in rows: projection = { "id": row[0], "source_region": row[1], "target_region": row[2], "status": row[3], "payload": _decode_json(row[4]), "created_at": row[5], "updated_at": row[6], } latest_ingest = _latest_ingest_attempt(projection["id"], projection["target_region"], "detect_task_projection") if latest_ingest and latest_ingest["status"] == "received": continue items_total = _task_projection_items_total(projection) selection_limit = _task_projection_selection_limit(projection) created_at = projection.get("created_at") if ( safe_limit >= 1000 and max(items_total, selection_limit) > 0 and max(items_total, selection_limit) < safe_limit and isinstance(created_at, datetime) ): now = datetime.now(created_at.tzinfo) if created_at.tzinfo else datetime.now() if now - created_at >= timedelta(minutes=10): _mark_task_projection_superseded( projection["id"], reason=f"stale small task projection skipped: items_total={items_total}, selection_limit={selection_limit}, requested_limit={safe_limit}", ) continue return projection return None def export_detect_task_projection(limit: int = 1000, *, shared_token: str | None = None) -> tuple[bool, str, dict]: configured_token = str(settings.sync_shared_token or "").strip() incoming_token = str(shared_token or "").strip() if configured_token and incoming_token != configured_token: return False, "同步 token 校验失败", {} existing = _load_pending_task_projection(limit) if existing: return True, "已返回待确认的任务批次", { "source_record_id": existing["id"], "projection_hash": (existing.get("payload") or {}).get("projection_hash", ""), "projection": (existing.get("payload") or {}).get("projection", {}), "deduplicated": True, } source_region = _normalize_region(settings.sync_source_region, settings.node_region) target_region = _normalize_region(settings.sync_target_region, "overseas") safe_limit = _task_projection_limit(limit) with get_db() as conn: with conn.cursor() as cur: cur.execute(_task_selection_sql(), (safe_limit,)) rows = cur.fetchall() if not rows: return False, "当前没有可下发的待检测任务", {"batch_size": 0} items = [] for row in rows: items.append( { "source_domain_id": int(row[0]), "domain": str(row[1] or "").strip(), "tld": str(row[2] or "").strip(), "source_type": int(row[3] or 0), "use_status": int(row[4] or 0), "detect_status": int(row[5] or 0), "register_status": int(row[6] or 0), "expire_date": row[7].isoformat(sep=" ", timespec="seconds") if row[7] else "", "jucha_status": int(row[8] or 0), "juziseo_status": int(row[9] or 0), } ) projection = { "batch_code": f"task-{datetime.now().strftime('%Y%m%d%H%M%S')}-{uuid4().hex[:6]}", "items_total": len(items), "selection_limit": safe_limit, "items": items, } payload = { "projection": projection, "projection_hash": f"{uuid4().hex}-{len(items)}", } cur.execute( """ INSERT INTO detect_sync_records ( sync_type, source_region, target_region, status, payload_json, error_message, created_at, updated_at ) VALUES (%s, %s, %s, %s, %s::jsonb, %s, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP) RETURNING id """, ( "detect_task_projection", source_region, target_region, "projected", json.dumps(payload, ensure_ascii=False), "", ), ) record_id = int(cur.fetchone()[0]) conn.commit() return True, "待检测任务批次已生成", { "source_record_id": record_id, "projection_hash": payload["projection_hash"], "projection": projection, "deduplicated": False, } def ack_detect_task_projection(payload: dict, *, shared_token: str | None = None) -> tuple[bool, str, dict]: configured_token = str(settings.sync_shared_token or "").strip() incoming_token = str(shared_token or "").strip() if configured_token and incoming_token != configured_token: return False, "同步 token 校验失败", {} source_record_id = int(payload.get("source_record_id") or 0) projection_hash = str(payload.get("projection_hash") or "").strip() if source_record_id <= 0 or not projection_hash: return False, "任务确认参数不完整", {} source_region = _normalize_region(settings.sync_source_region, settings.node_region) target_region = _normalize_region(settings.sync_target_region, "overseas") with get_db() as conn: with conn.cursor() as cur: cur.execute( """ SELECT id FROM detect_sync_records WHERE sync_type = 'detect_task_ingest' AND source_region = %s AND target_region = %s AND (payload_json->>'source_record_id') = %s AND (payload_json->>'projection_hash') = %s ORDER BY created_at DESC, id DESC LIMIT 1 """, (source_region, target_region, str(source_record_id), projection_hash), ) existing = cur.fetchone() if existing: return True, "任务批次确认已存在", {"record_id": int(existing[0]), "deduplicated": True} cur.execute( """ INSERT INTO detect_sync_records ( sync_type, source_region, target_region, status, payload_json, error_message, created_at, updated_at ) VALUES (%s, %s, %s, %s, %s::jsonb, %s, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP) RETURNING id """, ( "detect_task_ingest", source_region, target_region, "received", json.dumps( { "source_record_id": source_record_id, "projection_hash": projection_hash, "received_at": _format_time(datetime.now()), }, ensure_ascii=False, ), "", ), ) record_id = int(cur.fetchone()[0]) conn.commit() return True, "任务批次确认成功", {"record_id": record_id, "deduplicated": False} def ingest_detect_task_projection(payload: dict, *, shared_token: str | None = None) -> tuple[bool, str, dict]: configured_token = str(settings.sync_shared_token or "").strip() incoming_token = str(shared_token or "").strip() if configured_token and incoming_token != configured_token: return False, "同步 token 校验失败", {} sync_type = str(payload.get("sync_type") or "detect_task_projection").strip() or "detect_task_projection" if sync_type != "detect_task_projection": return False, "同步类型不匹配", {"sync_type": sync_type} source_region = _normalize_region(payload.get("source_region"), "unknown") source_record_id = int(payload.get("source_record_id") or 0) projection_hash = str(payload.get("projection_hash") or "").strip() projection = payload.get("projection") or {} items = list(projection.get("items") or []) target_region = _normalize_region(settings.node_region, "overseas") received_at = datetime.now() if source_record_id <= 0 or not projection_hash: return False, "任务批次参数不完整", {} if not items: return False, "任务批次为空", {"source_record_id": source_record_id} with get_db() as conn: with conn.cursor() as cur: cur.execute( """ SELECT id FROM detect_sync_records WHERE sync_type = 'detect_task_ingest' AND source_region = %s AND target_region = %s AND (payload_json->>'source_record_id') = %s AND (payload_json->>'projection_hash') = %s ORDER BY created_at DESC, id DESC LIMIT 1 """, (source_region, target_region, str(source_record_id), projection_hash), ) existing = cur.fetchone() if existing: return True, "任务批次已接收，已按幂等处理", {"record_id": int(existing[0]), "deduplicated": True} inserted_count = 0 updated_count = 0 domain_ids: list[int] = [] for item in items: domain = str(item.get("domain") or "").strip().lower() if not domain: continue tld = str(item.get("tld") or "").strip() if not tld and "." in domain: tld = domain.rsplit(".", 1)[-1] expire_date_text = str(item.get("expire_date") or "").strip() cur.execute( """ INSERT INTO domains ( domain, tld, source_type, use_status, detect_status, register_status, has_beian, company_type, website_url, beian_year, snapshot_years, expire_date, create_time, update_time, review_status, detect_time, backlink_count, jucha_status, juziseo_status ) VALUES ( %s, %s, %s, %s, %s, %s, 1, NULL, NULL, NULL, NULL, NULLIF(%s, '')::timestamp, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP, 0, NULL, 0, %s, %s ) ON CONFLICT (domain) DO UPDATE SET tld = EXCLUDED.tld, source_type = EXCLUDED.source_type, use_status = EXCLUDED.use_status, register_status = EXCLUDED.register_status, expire_date = COALESCE(EXCLUDED.expire_date, domains.expire_date), jucha_status = EXCLUDED.jucha_status, juziseo_status = EXCLUDED.juziseo_status, detect_status = CASE WHEN domains.detect_status IN (1, 2, 3, 4) THEN domains.detect_status ELSE EXCLUDED.detect_status END, update_time = CURRENT_TIMESTAMP RETURNING id, (xmax = 0) AS inserted """, ( domain, tld, int(item.get("source_type") or 0), int(item.get("use_status") or 0), int(item.get("detect_status") or 0), int(item.get("register_status") or 0), expire_date_text, int(item.get("jucha_status") or 0), int(item.get("juziseo_status") or 0), ), ) row = cur.fetchone() or [0, False] domain_id = int(row[0] or 0) inserted = bool(row[1]) if domain_id > 0: domain_ids.append(domain_id) if inserted: inserted_count += 1 else: updated_count += 1 target_job_code = f"sync-{source_region}-{source_record_id}" target_job_remark = ( f"同步拉取待检测批次 {str(projection.get('batch_code') or '').strip() or source_record_id}，" f"共 {len(domain_ids)} 个域名" ) cur.execute( """ INSERT INTO detect_jobs (job_code, source, plan_hash, task_mode, step_code, status, remark, created_by) VALUES (%s, %s, %s, 'domain_pipeline', '', 'pending', %s, %s) ON CONFLICT (job_code) DO UPDATE SET source = EXCLUDED.source, plan_hash = EXCLUDED.plan_hash, task_mode = EXCLUDED.task_mode, step_code = EXCLUDED.step_code, remark = EXCLUDED.remark, created_by = EXCLUDED.created_by, status = CASE WHEN detect_jobs.status IN ('completed', 'failed', 'cancelled') THEN 'pending' ELSE detect_jobs.status END, started_at = CASE WHEN detect_jobs.status IN ('completed', 'failed', 'cancelled') THEN NULL ELSE detect_jobs.started_at END, finished_at = CASE WHEN detect_jobs.status IN ('completed', 'failed', 'cancelled') THEN NULL ELSE detect_jobs.finished_at END RETURNING id """, ( target_job_code, "sync-pull", projection_hash, target_job_remark, "sync-agent", ), ) target_job_id = int((cur.fetchone() or [0])[0] or 0) settings_payload = get_settings_payload() queued_count = 0 deduplicated_job_items = 0 skipped_job_items = 0 if target_job_id > 0: for domain_id in domain_ids: domain_snapshot = _load_domain_pipeline_snapshot(cur, int(domain_id)) if not domain_snapshot: skipped_job_items += 1 continue item_step_code, step_payload = resolve_initial_domain_pipeline_item( domain_snapshot, settings_payload=settings_payload, ) if not item_step_code or not step_payload: skipped_job_items += 1 continue cur.execute( """ INSERT INTO detect_job_items (job_id, domain_id, step_code, status, step_payload_json) VALUES (%s, %s, %s, 'pending', %s::jsonb) ON CONFLICT (job_id, domain_id, step_code) DO NOTHING RETURNING id """, ( target_job_id, domain_id, item_step_code, json.dumps(step_payload, ensure_ascii=False), ), ) inserted_job_item = cur.fetchone() if inserted_job_item: queued_count += 1 else: deduplicated_job_items += 1 if queued_count > 0: cur.execute( """ INSERT INTO detect_run_events (job_id, node_code, event_type, level, message, payload_json) VALUES (%s, %s, %s, %s, %s, %s::jsonb) """, ( target_job_id, settings.node_code, "job_created", "info", f"同步拉取待检测批次 {target_job_code}，共 {queued_count} 个任务项", json.dumps( { "source_region": source_region, "source_record_id": source_record_id, "projection_hash": projection_hash, "batch_code": str(projection.get("batch_code") or "").strip(), "queued_count": queued_count, "deduplicated_job_items": deduplicated_job_items, "skipped_job_items": skipped_job_items, }, ensure_ascii=False, ), ), ) cur.execute( """ INSERT INTO detect_sync_records ( sync_type, source_region, target_region, status, payload_json, error_message, created_at, updated_at ) VALUES (%s, %s, %s, %s, %s::jsonb, %s, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP) RETURNING id """, ( "detect_task_ingest", source_region, target_region, "received", json.dumps( { "source_record_id": source_record_id, "projection_hash": projection_hash, "batch_code": str(projection.get("batch_code") or "").strip(), "items_total": len(items), "inserted_count": inserted_count, "updated_count": updated_count, "target_job_id": target_job_id, "target_job_code": target_job_code, "queued_count": queued_count, "deduplicated_job_items": deduplicated_job_items, "skipped_job_items": skipped_job_items, "received_at": _format_time(received_at), }, ensure_ascii=False, ), "", ), ) record_id = int(cur.fetchone()[0]) conn.commit() return True, "任务批次接收成功", { "record_id": record_id, "source_record_id": source_record_id, "items_total": len(items), "inserted_count": inserted_count, "updated_count": updated_count, "target_job_id": target_job_id, "target_job_code": target_job_code, "queued_count": queued_count, "deduplicated_job_items": deduplicated_job_items, "skipped_job_items": skipped_job_items, "deduplicated": False, } def _create_push_attempt(*, source_record: dict, ingest_url: str, sync_type: str) -> int: payload = { "sync_type": sync_type, "source_record_id": source_record["id"], "projection_hash": (source_record.get("payload") or {}).get("projection_hash", ""), "ingest_url": ingest_url, } with get_db() as conn: with conn.cursor() as cur: cur.execute( """ INSERT INTO detect_sync_records ( sync_type, source_region, target_region, status, payload_json, error_message, created_at, updated_at ) VALUES (%s, %s, %s, %s, %s::jsonb, %s, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP) RETURNING id """, ( "runtime_push", source_record["source_region"], source_record["target_region"], "pending", json.dumps(payload, ensure_ascii=False), "", ), ) record_id = int(cur.fetchone()[0]) conn.commit() return record_id def _update_push_attempt(record_id: int, *, status: str, payload: dict | None = None, error_message: str = "") -> None: with get_db() as conn: with conn.cursor() as cur: cur.execute( """ UPDATE detect_sync_records SET status = %s, payload_json = %s::jsonb, error_message = %s, updated_at = CURRENT_TIMESTAMP WHERE id = %s """, ( str(status or "").strip() or "pending", json.dumps(payload or {}, ensure_ascii=False), str(error_message or "").strip(), int(record_id), ), ) conn.commit() def _build_detect_result_import_fingerprint( *, source_region: str, source_record_id: int, source_job_code: str, event: dict, ) -> str: payload = _decode_json(event.get("payload")) seed = { "source_region": str(source_region or "").strip(), "source_record_id": int(source_record_id or 0), "source_job_code": str(source_job_code or "").strip(), "node_code": str(event.get("node_code") or "").strip(), "event_type": str(event.get("event_type") or "").strip(), "message": str(event.get("message") or "").strip(), "created_at": str(event.get("created_at") or "").strip(), "domain": str(payload.get("domain") or "").strip(), "domain_id": str(payload.get("domain_id") or "").strip(), "status": str(payload.get("status") or "").strip(), "cycle_token": str(payload.get("cycle_token") or "").strip(), } return hashlib.sha1(json.dumps(seed, ensure_ascii=False, sort_keys=True).encode("utf-8")).hexdigest() def _extract_detect_result_projection_events( *, source_region: str, source_record_id: int, projection: dict, ) -> list[dict]: source_job = projection.get("job") or {} source_job_code = str(source_job.get("job_code") or "").strip() source_job_id = int(source_job.get("job_id") or 0) events: list[dict] = [] for raw_event in list(projection.get("recent_domain_events") or []): event_type = str(raw_event.get("event_type") or "").strip() if event_type not in _DETECT_RESULT_EVENT_TYPES: continue payload = _decode_json(raw_event.get("payload")) fingerprint = _build_detect_result_import_fingerprint( source_region=source_region, source_record_id=source_record_id, source_job_code=source_job_code, event=raw_event, ) payload.update( { "imported_from_projection": True, "import_source_region": source_region, "import_source_record_id": int(source_record_id or 0), "import_source_job_id": source_job_id, "import_source_job_code": source_job_code, "import_fingerprint": fingerprint, } ) events.append( { "node_code": str(raw_event.get("node_code") or "").strip(), "event_type": event_type, "level": str(raw_event.get("level") or "info").strip() or "info", "message": str(raw_event.get("message") or "").strip(), "created_at": str(raw_event.get("created_at") or "").strip(), "payload": payload, "fingerprint": fingerprint, } ) return events def _resolve_detect_result_target_job_id(*, projection: dict) -> int: source_job = projection.get("job") or {} source_job_code = str(source_job.get("job_code") or "").strip() with get_db() as conn: with conn.cursor() as cur: if source_job_code: cur.execute( """ SELECT id FROM detect_jobs WHERE job_code = %s ORDER BY id DESC LIMIT 1 """, (source_job_code,), ) row = cur.fetchone() if row: return int(row[0] or 0) from app.services.detect_job_service import get_active_detect_job_summary active_job = get_active_detect_job_summary(event_limit=1) or {} return int(active_job.get("job_id") or 0) def _parse_event_created_at(value: str) -> datetime | None: text = str(value or "").strip() if not text: return None try: return datetime.fromisoformat(text) except Exception: return None def _extract_event_domain(payload: dict, message: str) -> str: domain = str(payload.get("domain") or "").strip().lower() if domain: return domain text = str(message or "").strip() if ":" in text: candidate = text.rsplit(":", 1)[-1].strip().lower() if candidate: return candidate return "" def _apply_detect_result_event_to_domain(cur, event: dict) -> None: payload = _decode_json(event.get("payload")) domain = _extract_event_domain(payload, str(event.get("message") or "")) if not domain: return event_type = str(event.get("event_type") or "").strip() created_at = _parse_event_created_at(str(event.get("created_at") or "")) effective_time = created_at or datetime.now() if event_type == "domain_started": cur.execute( """ UPDATE domains SET detect_status = CASE WHEN detect_status IN (1, 3) THEN detect_status ELSE 2 END, update_time = CURRENT_TIMESTAMP WHERE domain = %s """, (domain,), ) return if event_type == "domain_completed": cur.execute( """ UPDATE domains SET detect_status = 1, detect_time = COALESCE(detect_time, %s), update_time = CURRENT_TIMESTAMP WHERE domain = %s """, (effective_time, domain), ) return if event_type == "domain_blacklisted": cur.execute( """ UPDATE domains SET detect_status = 3, update_time = CURRENT_TIMESTAMP WHERE domain = %s """, (domain,), ) return if event_type == "domain_failed": cur.execute( """ UPDATE domains SET detect_status = CASE WHEN detect_status IN (1, 3) THEN detect_status ELSE 4 END, update_time = CURRENT_TIMESTAMP WHERE domain = %s """, (domain,), ) return def _apply_detect_result_event_to_job_item(cur, *, target_job_id: int, event: dict) -> int: if int(target_job_id or 0) <= 0: return 0 payload = _decode_json(event.get("payload")) domain = _extract_event_domain(payload, str(event.get("message") or "")) if not domain: return 0 event_type = str(event.get("event_type") or "").strip() node_code = str(event.get("node_code") or "").strip() message = str(event.get("message") or "").strip() if event_type == "domain_started": cur.execute( """ UPDATE detect_job_items AS item SET status = CASE WHEN item.status IN ('completed', 'blacklisted', 'failed') THEN item.status ELSE 'running' END, claimed_by = CASE WHEN %s <> '' THEN %s ELSE item.claimed_by END, started_at = COALESCE(item.started_at, CURRENT_TIMESTAMP), updated_at = CURRENT_TIMESTAMP FROM domains AS d WHERE item.job_id = %s AND item.domain_id = d.id AND d.domain = %s AND item.status IN ('pending', 'claimed', 'running') """, (node_code, node_code, int(target_job_id), domain), ) return int(cur.rowcount or 0) if event_type not in {"domain_completed", "domain_blacklisted", "domain_failed"}: return 0 final_status = { "domain_completed": "completed", "domain_blacklisted": "blacklisted", "domain_failed": "failed", }[event_type] cur.execute( """ UPDATE detect_job_items AS item SET status = %s, claimed_by = CASE WHEN %s <> '' THEN %s ELSE item.claimed_by END, finished_at = COALESCE(item.finished_at, CURRENT_TIMESTAMP), updated_at = CURRENT_TIMESTAMP, lease_expires_at = NULL, last_error = CASE WHEN %s = 'failed' THEN LEFT(%s, 1000) ELSE item.last_error END FROM domains AS d WHERE item.job_id = %s AND item.domain_id = d.id AND d.domain = %s AND item.status IN ('pending', 'claimed', 'running') """, ( final_status, node_code, node_code, final_status, message, int(target_job_id), domain, ), ) return int(cur.rowcount or 0) def _import_detect_result_projection_events( *, source_region: str, source_record_id: int, projection: dict, ) -> dict: events = _extract_detect_result_projection_events( source_region=source_region, source_record_id=source_record_id, projection=projection, ) if not events: return {"imported_count": 0, "deduplicated_count": 0, "target_job_id": 0} target_job_id = _resolve_detect_result_target_job_id(projection=projection) imported_count = 0 deduplicated_count = 0 updated_job_items = 0 with get_db() as conn: with conn.cursor() as cur: for event in events: cur.execute( """ SELECT id FROM detect_run_events WHERE job_id = %s AND (payload_json->>'import_fingerprint') = %s ORDER BY id DESC LIMIT 1 """, (target_job_id, event["fingerprint"]), ) if cur.fetchone(): deduplicated_count += 1 continue created_at = _parse_event_created_at(event.get("created_at", "")) if created_at: cur.execute( """ INSERT INTO detect_run_events ( job_id, node_code, event_type, level, message, payload_json, created_at ) VALUES (%s, %s, %s, %s, %s, %s::jsonb, %s) """, ( target_job_id, event["node_code"], event["event_type"], event["level"], event["message"], json.dumps(event["payload"], ensure_ascii=False), created_at, ), ) else: cur.execute( """ INSERT INTO detect_run_events ( job_id, node_code, event_type, level, message, payload_json ) VALUES (%s, %s, %s, %s, %s, %s::jsonb) """, ( target_job_id, event["node_code"], event["event_type"], event["level"], event["message"], json.dumps(event["payload"], ensure_ascii=False), ), ) _apply_detect_result_event_to_domain(cur, event) if target_job_id > 0: updated_job_items += _apply_detect_result_event_to_job_item( cur, target_job_id=target_job_id, event=event, ) imported_count += 1 conn.commit() if target_job_id > 0: from app.services.detect_job_service import refresh_detect_job_status try: refresh_detect_job_status(target_job_id) except Exception: pass return { "imported_count": imported_count, "deduplicated_count": deduplicated_count, "target_job_id": target_job_id, "updated_job_items": updated_job_items, } def ingest_runtime_projection(payload: dict, *, shared_token: str | None = None) -> tuple[bool, str, dict]: configured_token = str(settings.sync_shared_token or "").strip() incoming_token = str(shared_token or "").strip() if configured_token and incoming_token != configured_token: return False, "同步 token 校验失败", {} sync_type = str(payload.get("sync_type") or "runtime_projection").strip() or "runtime_projection" ingest_sync_type = _projection_ingest_type(sync_type) source_region = _normalize_region(payload.get("source_region"), "unknown") source_record_id = int(payload.get("source_record_id") or 0) projection_hash = str(payload.get("projection_hash") or "").strip() projection = payload.get("projection") or {} target_region = _normalize_region(settings.node_region, "overseas") received_at = datetime.now() with get_db() as conn: with conn.cursor() as cur: cur.execute( """ SELECT id FROM detect_sync_records WHERE sync_type = %s AND source_region = %s AND target_region = %s AND (payload_json->>'source_record_id') = %s AND (payload_json->>'projection_hash') = %s ORDER BY created_at DESC, id DESC LIMIT 1 """, (ingest_sync_type, source_region, target_region, str(source_record_id), projection_hash), ) existing = cur.fetchone() if existing: if sync_type == "runtime_projection": _refresh_remote_runtime_node(source_region=source_region, projection=projection, received_at=received_at) if sync_type == "detect_result_projection": import_result = _import_detect_result_projection_events( source_region=source_region, source_record_id=source_record_id, projection=projection, ) else: import_result = {} cur.execute( """ UPDATE detect_sync_records SET updated_at = CURRENT_TIMESTAMP WHERE id = %s """, (int(existing[0]),), ) conn.commit() return True, "同步投影已存在，已按幂等处理", { "record_id": int(existing[0]), "deduplicated": True, "event_import": import_result, } stored_payload = { "sync_type": sync_type, "source_record_id": source_record_id, "projection_hash": projection_hash, "projection": projection, "received_at": _format_time(received_at), } cur.execute( """ INSERT INTO detect_sync_records ( sync_type, source_region, target_region, status, payload_json, error_message, created_at, updated_at ) VALUES (%s, %s, %s, %s, %s::jsonb, %s, CURRENT_TIMESTAMP, CURRENT_TIMESTAMP) RETURNING id """, ( ingest_sync_type, source_region, target_region, "received", json.dumps(stored_payload, ensure_ascii=False), "", ), ) record_id = int(cur.fetchone()[0]) conn.commit() if sync_type == "runtime_projection": _refresh_remote_runtime_node(source_region=source_region, projection=projection, received_at=received_at) if sync_type == "detect_result_projection": import_result = _import_detect_result_projection_events( source_region=source_region, source_record_id=source_record_id, projection=projection, ) else: import_result = {} return True, "同步投影接收成功", { "record_id": record_id, "deduplicated": False, "event_import": import_result, } def _push_projection_now(sync_type: str, ingest_url: str) -> tuple[bool, str, dict]: if sync_type == "runtime_projection": # Regenerate the runtime snapshot before every push so the sync agent # does not keep replaying a stale projection record while the worker # thread count / phase is still changing. from app.services.runtime_status_service import get_runtime_status try: get_runtime_status() except Exception as exc: return False, f"刷新 runtime_projection 失败: {exc}", {"action": "push_sync", "sync_type": sync_type} source_record = _load_latest_projection(sync_type) if not source_record: return False, f"当前没有可推送的{sync_type}", {"action": "push_sync", "sync_type": sync_type} return _push_projection_record(source_record, sync_type, ingest_url) def _push_projection_batch(sync_type: str, ingest_url: str) -> tuple[bool, str, dict]: pending_records = _load_pushable_projections(sync_type, limit=max(1, int(settings.sync_batch_size or 200))) if not pending_records: latest = _load_latest_projection(sync_type) if latest: latest_attempt = _latest_push_attempt(latest["id"], latest["target_region"], sync_type) if latest_attempt and latest_attempt["status"] == "success": return True, "当前批次已全部同步完成", { "action": "push_sync", "sync_type": sync_type, "deduplicated": True, "batch_count": 0, } return False, f"当前没有可推送的{sync_type}", {"action": "push_sync", "sync_type": sync_type, "batch_count": 0} results: list[dict] = [] success_count = 0 for record in pending_records: ok, message, data = _push_projection_record(record, sync_type, ingest_url) results.append( { "source_record_id": record["id"], "ok": ok, "message": message, "data": data, } ) if ok: success_count += 1 overall_ok = success_count > 0 message = f"{sync_type} 批量推送完成，成功 {success_count}/{len(results)}" return overall_ok, message, { "action": "push_sync", "sync_type": sync_type, "batch_count": len(results), "success_count": success_count, "results": results, } def _push_projection_record(source_record: dict, sync_type: str, ingest_url: str) -> tuple[bool, str, dict]: latest_attempt = _latest_push_attempt(source_record["id"], source_record["target_region"], sync_type) if latest_attempt and latest_attempt["status"] == "success": last_created_at = latest_attempt.get("created_at") if sync_type != "runtime_projection" or not isinstance(last_created_at, datetime): return True, "该投影已推送，无需重复发送", { "action": "push_sync", "sync_type": sync_type, "source_record_id": source_record["id"], "deduplicated": True, } now = datetime.now(last_created_at.tzinfo) if last_created_at.tzinfo else datetime.now() if now - last_created_at < timedelta(seconds=max(20, int(settings.sync_poll_interval_seconds or 30))): return True, "该投影已推送，无需重复发送", { "action": "push_sync", "sync_type": sync_type, "source_record_id": source_record["id"], "deduplicated": True, } if latest_attempt and latest_attempt["status"] == "pending": return True, "该投影已有同步推送进行中，暂不重复发送", { "action": "push_sync", "sync_type": sync_type, "source_record_id": source_record["id"], "deduplicated": True, } if latest_attempt and latest_attempt["status"] == "failed": last_created_at = latest_attempt.get("created_at") if isinstance(last_created_at, datetime): now = datetime.now(last_created_at.tzinfo) if last_created_at.tzinfo else datetime.now() if now - last_created_at < timedelta(seconds=max(10, int(settings.sync_poll_interval_seconds or 30))): return False, "最近一次同步推送刚失败，等待下个重试窗口", { "action": "push_sync", "sync_type": sync_type, "source_record_id": source_record["id"], "deduplicated": True, } attempt_id = _create_push_attempt(source_record=source_record, ingest_url=ingest_url, sync_type=sync_type) request_payload = { "sync_type": sync_type, "source_region": source_record["source_region"], "source_record_id": source_record["id"], "projection_hash": (source_record.get("payload") or {}).get("projection_hash", ""), "projection": (source_record.get("payload") or {}).get("projection", {}), "created_at": _format_time(source_record.get("created_at")), } request_body = json.dumps(request_payload, ensure_ascii=False).encode("utf-8") request = urllib.request.Request( ingest_url, data=request_body, headers={ "Content-Type": "application/json", **({"X-Domaincheck-Sync-Token": settings.sync_shared_token} if settings.sync_shared_token else {}), }, method="POST", ) try: with urllib.request.urlopen(request, timeout=15) as response: raw = response.read().decode("utf-8") data = json.loads(raw) if raw else {} status_code = getattr(response, "status", 200) except json.JSONDecodeError as exc: payload = { "sync_type": sync_type, "source_record_id": source_record["id"], "projection_hash": request_payload["projection_hash"], "ingest_url": ingest_url, } _update_push_attempt(attempt_id, status="failed", payload=payload, error_message=f"invalid json response: {exc}") return False, f"同步推送失败: 远端响应不是合法 JSON ({exc})", {"action": "push_sync", "sync_type": sync_type, "attempt_id": attempt_id} except urllib.error.HTTPError as exc: error_body = exc.read().decode("utf-8", errors="ignore") if hasattr(exc, "read") else "" payload = { "sync_type": sync_type, "source_record_id": source_record["id"], "projection_hash": request_payload["projection_hash"], "ingest_url": ingest_url, "http_status": getattr(exc, "code", 500), "response_text": error_body[:1000], } _update_push_attempt(attempt_id, status="failed", payload=payload, error_message=f"HTTP {getattr(exc, 'code', 500)}") return False, f"同步推送失败: HTTP {getattr(exc, 'code', 500)}", {"action": "push_sync", "sync_type": sync_type, "attempt_id": attempt_id} except Exception as exc: payload = { "sync_type": sync_type, "source_record_id": source_record["id"], "projection_hash": request_payload["projection_hash"], "ingest_url": ingest_url, } _update_push_attempt(attempt_id, status="failed", payload=payload, error_message=str(exc)) return False, f"同步推送失败: {exc}", {"action": "push_sync", "sync_type": sync_type, "attempt_id": attempt_id} payload = { "sync_type": sync_type, "source_record_id": source_record["id"], "projection_hash": request_payload["projection_hash"], "ingest_url": ingest_url, "http_status": status_code, "response": data, } response_code = data.get("code", 0) if response_code not in (0, "0", None, ""): _update_push_attempt( attempt_id, status="failed", payload=payload, error_message=str(data.get("message") or "remote business error"), ) return False, f"同步推送失败: {str(data.get('message') or '远端业务返回失败')}", { "action": "push_sync", "sync_type": sync_type, "attempt_id": attempt_id, "source_record_id": source_record["id"], "response": data, } _update_push_attempt(attempt_id, status="success", payload=payload, error_message="") return True, "投影推送成功", { "action": "push_sync", "sync_type": sync_type, "attempt_id": attempt_id, "source_record_id": source_record["id"], "response": data, } def push_runtime_projection_now() -> tuple[bool, str, dict]: if not settings.sync_push_enabled: return False, "未启用同步推送", {"action": "push_sync", "sync_state": "disabled", "ui_level": "warning", "poll_schedule_seconds": []} ingest_url = _ingest_url(settings.sync_target_api_base_url) if not ingest_url: return False, "未配置同步目标地址", {"action": "push_sync", "sync_state": "misconfigured", "ui_level": "warning", "poll_schedule_seconds": []} results = [] ok, message, data = _push_projection_now("runtime_projection", ingest_url) results.append({"sync_type": "runtime_projection", "ok": ok, "message": message, "data": data}) ok, message, data = _push_projection_batch("detect_result_projection", ingest_url) results.append({"sync_type": "detect_result_projection", "ok": ok, "message": message, "data": data}) success_count = sum(1 for item in results if item["ok"]) warning_count = sum( 1 for item in results if not item["ok"] and any(keyword in str(item.get("message") or "") for keyword in ("当前没有可推送", "已全部同步完成", "无需重复发送", "进行中", "等待下个重试窗口")) ) if success_count == 0: if warning_count == len(results): return False, "当前没有需要立即推送的同步投影", { "action": "push_sync", "sync_state": "idle", "ui_level": "warning", "poll_schedule_seconds": [], "results": results, } return False, "同步推送未成功，请检查明细结果", { "action": "push_sync", "sync_state": "failed", "ui_level": "error", "poll_schedule_seconds": [2], "results": results, } if success_count < len(results): return True, f"同步推送部分完成，成功 {success_count}/{len(results)}", { "action": "push_sync", "sync_state": "partial_success", "ui_level": "warning", "poll_schedule_seconds": [1, 3], "results": results, } return True, f"同步推送完成，成功 {success_count}/{len(results)}", { "action": "push_sync", "sync_state": "success", "ui_level": "success", "poll_schedule_seconds": [1, 3], "results": results, } def pull_detect_task_batch_now(limit: int | None = None) -> tuple[bool, str, dict]: if settings.node_region != "mainland" or settings.node_role != "control": return False, "当前节点无需拉取待检测任务批次", {"action": "pull_tasks", "pull_state": "not_applicable", "ui_level": "warning", "poll_schedule_seconds": []} export_url = _task_export_url(settings.sync_target_api_base_url) ack_url = _task_ack_url(settings.sync_target_api_base_url) if not export_url or not ack_url: return False, "未配置任务拉取目标地址", {"action": "pull_tasks", "pull_state": "misconfigured", "ui_level": "warning", "poll_schedule_seconds": []} configured_limit = max(5000, int(settings.sync_batch_size or 200)) requested_limit = int(limit or configured_limit) safe_limit = max(1, min(requested_limit, max(10000, configured_limit))) settings_payload = get_settings_payload() backlog_snapshot = _load_local_detect_backlog_snapshot() backlog_limits = _build_task_pull_backlog_limits(configured_limit, settings_payload=settings_payload) should_throttle, throttle_reason = _should_throttle_task_pull(backlog_snapshot, backlog_limits) if should_throttle: return True, "本地待处理积压较高，暂停拉取新批次", { "action": "pull_tasks", "pull_state": "throttled", "ui_level": "info", "poll_schedule_seconds": [1, 3], "reason": throttle_reason, **backlog_snapshot, **backlog_limits, } request_url = f"{export_url}?limit={safe_limit}" export_timeout = max(20, min(90, 15 + safe_limit // 40)) request = urllib.request.Request( request_url, headers={**({"X-Domaincheck-Sync-Token": settings.sync_shared_token} if settings.sync_shared_token else {})}, method="GET", ) try: with urllib.request.urlopen(request, timeout=export_timeout) as response: raw = response.read().decode("utf-8") data = json.loads(raw) if raw else {} except json.JSONDecodeError as exc: return False, f"拉取待检测任务失败: 远端响应不是合法 JSON ({exc})", {"action": "pull_tasks", "pull_state": "failed", "ui_level": "error", "poll_schedule_seconds": [2]} except urllib.error.HTTPError as exc: error_body = exc.read().decode("utf-8", errors="ignore") if hasattr(exc, "read") else "" return False, f"拉取待检测任务失败: HTTP {getattr(exc, 'code', 500)}", { "action": "pull_tasks", "pull_state": "failed", "ui_level": "error", "poll_schedule_seconds": [2], "http_status": getattr(exc, "code", 500), "response_text": error_body[:500], } except Exception as exc: return False, f"拉取待检测任务失败: {exc}", {"action": "pull_tasks", "pull_state": "failed", "ui_level": "error", "poll_schedule_seconds": [2]} payload = data.get("data") or {} response_code = data.get("code", 0) if response_code not in (0, "0", None) and not payload: return False, str(data.get("message") or "拉取待检测任务失败"), {"action": "pull_tasks", "pull_state": "failed", "ui_level": "error", "poll_schedule_seconds": [2], "response": data} source_record_id = int(payload.get("source_record_id") or 0) projection_hash = str(payload.get("projection_hash") or "").strip() projection = payload.get("projection") or {} if source_record_id <= 0 or not projection_hash or not projection: return False, "远端当前没有可拉取的待检测任务批次", {"action": "pull_tasks", "pull_state": "idle", "ui_level": "warning", "poll_schedule_seconds": [], "batch_size": 0} ingest_ok, ingest_message, ingest_data = ingest_detect_task_projection( { "sync_type": "detect_task_projection", "source_region": _normalize_region(settings.sync_target_region, "overseas"), "source_record_id": source_record_id, "projection_hash": projection_hash, "projection": projection, }, shared_token=settings.sync_shared_token, ) if not ingest_ok: return False, ingest_message, {"action": "pull_tasks", "pull_state": "failed", "ui_level": "error", "poll_schedule_seconds": [2], **(ingest_data or {})} ack_request = urllib.request.Request( ack_url, data=json.dumps( { "source_record_id": source_record_id, "projection_hash": projection_hash, }, ensure_ascii=False, ).encode("utf-8"), headers={ "Content-Type": "application/json", **({"X-Domaincheck-Sync-Token": settings.sync_shared_token} if settings.sync_shared_token else {}), }, method="POST", ) ack_data = {} try: with urllib.request.urlopen(ack_request, timeout=15) as response: raw = response.read().decode("utf-8") ack_response = json.loads(raw) if raw else {} ack_data = ack_response.get("data") or {} ack_code = ack_response.get("code", 0) if ack_code not in (0, "0", None, ""): return True, f"{ingest_message}；但远端确认失败: {str(ack_response.get('message') or 'ack business error')}", { "action": "pull_tasks", "pull_state": "ack_warning", "ui_level": "warning", "poll_schedule_seconds": [1, 3], "source_record_id": source_record_id, "projection_hash": projection_hash, **(ingest_data or {}), "ack": ack_data, "ack_response": ack_response, } except json.JSONDecodeError as exc: return True, f"{ingest_message}；但远端确认失败: ack 响应不是合法 JSON ({exc})", { "action": "pull_tasks", "pull_state": "ack_warning", "ui_level": "warning", "poll_schedule_seconds": [1, 3], "source_record_id": source_record_id, **(ingest_data or {}), } except Exception as exc: return True, f"{ingest_message}；但远端确认失败: {exc}", { "action": "pull_tasks", "pull_state": "ack_warning", "ui_level": "warning", "poll_schedule_seconds": [1, 3], "source_record_id": source_record_id, **(ingest_data or {}), } result = { "action": "pull_tasks", "pull_state": "success", "ui_level": "success", "poll_schedule_seconds": [1, 3], "source_record_id": source_record_id, "projection_hash": projection_hash, "batch_code": str(projection.get("batch_code") or "").strip(), **(ingest_data or {}), "ack": ack_data, } queued_count = int(result.get("queued_count", 0) or 0) if queued_count > 0: try: from app.services.worker_control_service import send_worker_command start_ok, start_message = send_worker_command( "start_detection", payload={ "source": "sync-pull", "source_record_id": source_record_id, "target_job_id": int(result.get("target_job_id", 0) or 0), "target_job_code": str(result.get("target_job_code") or "").strip(), }, ) result["worker_start_ok"] = bool(start_ok) result["worker_start_message"] = str(start_message or "").strip() except Exception as exc: result["worker_start_ok"] = False result["worker_start_message"] = f"同步入库后自动唤起 Worker 失败: {exc}" result["ui_level"] = "warning" result["pull_state"] = "worker_start_warning" return True, "待检测任务批次拉取并入库成功；但自动唤起 Worker 失败", result return True, "待检测任务批次拉取并入库成功", result