from __future__ import annotations import logging import time from app.core.config import settings from app.services.debug_event_service import push_debug_event from app.services.detect_job_service import ( get_active_detect_job_summary, get_detect_queue_health, get_latest_detect_job_summary, get_latest_unprojected_detect_job_summary, list_recent_detect_run_events, process_detect_pipeline_now, ) from app.services.sync_record_service import append_detect_result_projection_if_changed from app.services.sync_push_service import ( _load_local_detect_backlog_snapshot, pull_detect_task_batch_now, push_runtime_projection_now, ) logger = logging.getLogger("domaincheck.sync_agent") _IDLE_SYNC_KEYWORDS = ( "当前没有可推送", "当前没有需要立即推送", "已全部同步完成", "无需重复发送", "进行中", "等待下个重试窗口", "暂停拉取", ) def _append_detect_result_projection_snapshot(active_job: dict) -> None: if not active_job: return append_detect_result_projection_if_changed( detect={ "active_job": active_job, "progress": { "pending": int(active_job.get("items_pending", 0) or 0), "running": int(active_job.get("items_running", 0) or 0), "completed": int(active_job.get("items_completed", 0) or 0), "blacklisted": int(active_job.get("items_blacklisted", 0) or 0), "failed": int(active_job.get("items_failed", 0) or 0), }, "phase_label": str(active_job.get("status") or "").strip(), "phase_detail": f"sync-agent snapshot for {active_job.get('job_code', '')}", } ) def _is_idle_sync_message(message: str) -> bool: normalized = str(message or "").strip() return any(keyword in normalized for keyword in _IDLE_SYNC_KEYWORDS) def _filter_runtime_events_for_job(events: list[dict], *, job_code: str = "", job_id: int = 0, limit: int = 8) -> list[dict]: target_job_code = str(job_code or "").strip() target_job_id = int(job_id or 0) safe_limit = max(1, min(int(limit or 8), 50)) filtered: list[dict] = [] for raw_event in list(events or []): if not isinstance(raw_event, dict): continue payload = raw_event.get("payload") if isinstance(raw_event.get("payload"), dict) else {} event_job_code = str(payload.get("job_code") or "").strip() event_job_id = int(raw_event.get("job_id") or 0) if target_job_code and event_job_code != target_job_code and (target_job_id <= 0 or event_job_id != target_job_id): continue filtered.append(raw_event) if len(filtered) >= safe_limit: break return filtered def _build_aligned_queue_health_snapshot(active_job: dict, queue_health: dict | None) -> dict: snapshot = dict(queue_health or {}) if not active_job: return snapshot active_job_code = str(active_job.get("runtime_job_code") or active_job.get("job_code") or "").strip() queue_job = dict(snapshot.get("job") or {}) queue_job_code = str(queue_job.get("runtime_job_code") or queue_job.get("job_code") or "").strip() if active_job_code and queue_job_code and active_job_code == queue_job_code: return snapshot active_job_items_total = int(active_job.get("items_total", 0) or 0) active_job_pending = int(active_job.get("items_pending", 0) or 0) active_job_claimed = int(active_job.get("items_claimed", 0) or 0) active_job_running = int(active_job.get("items_running", 0) or 0) active_job_completed = int(active_job.get("items_completed", 0) or 0) active_job_blacklisted = int(active_job.get("items_blacklisted", 0) or 0) active_job_failed = int(active_job.get("items_failed", 0) or 0) active_job_terminal = int( active_job.get("items_terminal", active_job_completed + active_job_blacklisted + active_job_failed) or 0 ) display_claimed = int(active_job.get("display_items_claimed", active_job_claimed) or active_job_claimed) display_running = int(active_job.get("display_items_running", active_job_running) or active_job_running) node_entries: list[dict] = [] for node in list(active_job.get("node_stats") or []): node_entries.append( { "node_code": str(node.get("node_code") or "").strip(), "items_total": int(node.get("items_total", 0) or 0), "items_pending": int(node.get("items_pending", 0) or 0), "items_claimed": int(node.get("items_claimed", 0) or 0), "items_running": int(node.get("items_running", 0) or 0), "items_completed": int(node.get("items_completed", 0) or 0), "items_blacklisted": int(node.get("items_blacklisted", 0) or 0), "items_failed": int(node.get("items_failed", 0) or 0), "processed_recent": int(node.get("processed_recent", 0) or 0), "processed_per_minute": float(node.get("processed_per_minute", 0) or 0), "completed_recent": int(node.get("completed_recent", 0) or 0), "blacklisted_recent": int(node.get("blacklisted_recent", 0) or 0), "failed_recent": int(node.get("failed_recent", 0) or 0), "metrics_source": str(node.get("metrics_source") or "runtime"), } ) assigned_total = sum(int(item.get("items_total", 0) or 0) for item in node_entries) unassigned_total = max(0, active_job_items_total - assigned_total) if unassigned_total > 0: node_entries.append( { "node_code": "unassigned", "items_total": unassigned_total, "items_pending": active_job_pending, "items_claimed": 0, "items_running": 0, "items_completed": 0, "items_blacklisted": 0, "items_failed": 0, "processed_recent": 0, "processed_per_minute": 0.0, "completed_recent": 0, "blacklisted_recent": 0, "failed_recent": 0, "metrics_source": "central_queue", } ) snapshot["job"] = { "job_id": active_job.get("job_id"), "job_code": str(active_job.get("job_code") or "").strip(), "runtime_job_code": active_job_code, "status": str(active_job.get("status") or "").strip(), "progress_percent": float(active_job.get("progress_percent", 0) or 0), } snapshot["queue"] = { **dict(snapshot.get("queue") or {}), "items_total": active_job_items_total, "pending": active_job_pending, "claimed": active_job_claimed, "running": active_job_running, "display_claimed": display_claimed, "display_running": display_running, "completed": active_job_completed, "blacklisted": active_job_blacklisted, "failed": active_job_failed, "terminal": active_job_terminal, "terminal_percent": round((active_job_terminal / active_job_items_total) * 100, 2) if active_job_items_total else 0.0, } snapshot["nodes"] = node_entries return snapshot def _select_projection_job_snapshot() -> dict | None: active_job = get_active_detect_job_summary(event_limit=10) if active_job: return active_job return get_latest_detect_job_summary( event_limit=10, statuses=("completed", "partial_failed", "failed"), recent_minutes=20, ) def _select_projection_job_snapshots() -> list[dict]: snapshots: list[dict] = [] seen_job_ids: set[int] = set() active_job = get_active_detect_job_summary(event_limit=10) if active_job: active_job_id = int(active_job.get("job_id") or 0) if active_job_id > 0 and active_job_id not in seen_job_ids: snapshots.append(active_job) seen_job_ids.add(active_job_id) latest_finished_job = get_latest_unprojected_detect_job_summary( event_limit=10, statuses=("completed", "partial_failed", "failed"), recent_minutes=180, ) if latest_finished_job: latest_finished_job_id = int(latest_finished_job.get("job_id") or 0) if latest_finished_job_id > 0 and latest_finished_job_id not in seen_job_ids: snapshots.append(latest_finished_job) seen_job_ids.add(latest_finished_job_id) return snapshots def _emit_structured_tick( *, base_event_type: str, ok: bool, message: str, data: dict | None = None, ) -> None: payload = {"ok": ok, "data": data or {}} event_type = f"{base_event_type}_failed" level = "warning" if isinstance(data, dict) and str(data.get("pull_state") or "").strip() == "throttled": event_type = f"{base_event_type}_idle" level = "info" elif not ok and _is_idle_sync_message(message): event_type = f"{base_event_type}_idle" level = "info" elif ok: event_type = f"{base_event_type}_success" level = "info" if "但远端确认失败" in str(message or ""): event_type = f"{base_event_type}_partial" level = "warning" results = list((data or {}).get("results") or []) if results and any(not bool(item.get("ok")) for item in results): event_type = f"{base_event_type}_partial" level = "warning" push_debug_event( service="sync-agent", event_type=event_type, level=level, message=message, payload=payload, ) def _emit_sync_result_breakdown(data: dict | None) -> None: results = list((data or {}).get("results") or []) for item in results: sync_type = str(item.get("sync_type") or "").strip() or "unknown" ok = bool(item.get("ok")) message = str(item.get("message") or "").strip() or f"{sync_type} sync result" result_data = item.get("data") or {} event_type = f"{sync_type}_sync_failed" level = "warning" if not ok and _is_idle_sync_message(message): event_type = f"{sync_type}_sync_idle" level = "info" elif ok: event_type = f"{sync_type}_sync_success" level = "info" if isinstance(result_data, dict) and result_data.get("success_count") is not None: batch_count = int(result_data.get("batch_count") or 0) success_count = int(result_data.get("success_count") or 0) if batch_count > 0 and success_count < batch_count: event_type = f"{sync_type}_sync_partial" level = "warning" if "成功 1/" in message or "但" in message: event_type = f"{sync_type}_sync_partial" level = "warning" push_debug_event( service="sync-agent", event_type=event_type, level=level, message=message, payload={ "sync_type": sync_type, "ok": ok, "data": result_data, }, ) def _run_pipeline_stage_processor() -> tuple[bool, str, dict]: process_limit = max(500, min(int(settings.sync_pipeline_process_limit or 5000), 5000)) ok, message, data = process_detect_pipeline_now(limit=process_limit) push_debug_event( service="sync-agent", event_type="pipeline_tick_success" if ok else "pipeline_tick_failed", level="info" if ok else "warning", message=message, payload={ "ok": ok, "limit": process_limit, "data": data or {}, }, ) return ok, message, data def main() -> None: logging.basicConfig( level=logging.INFO, format="%(asctime)s [%(levelname)s] %(name)s: %(message)s", ) # Old env files still ship SYNC_POLL_INTERVAL_SECONDS=30. Cap the interval # so controller pull/pipeline ticks cannot be throttled into starvation. interval = max(2, min(int(settings.sync_poll_interval_seconds or 2), 5)) logger.info( "sync agent started: node=%s source=%s target=%s interval=%ss enabled=%s", settings.node_code, settings.sync_source_region, settings.sync_target_region, interval, settings.sync_push_enabled, ) while True: try: pipeline_ok, pipeline_message, pipeline_data = _run_pipeline_stage_processor() logger.info( "pipeline tick: ok=%s message=%s data=%s", pipeline_ok, pipeline_message, pipeline_data, ) active_job = get_active_detect_job_summary(event_limit=10) for projection_job in _select_projection_job_snapshots(): _append_detect_result_projection_snapshot(projection_job) ok, message, data = push_runtime_projection_now() logger.info("sync tick: ok=%s message=%s data=%s", ok, message, data) push_debug_event( service="sync-agent", event_type="sync_tick", level="info" if ok else "warning", message=message, payload={"ok": ok, "data": data}, ) _emit_structured_tick(base_event_type="sync_push", ok=ok, message=message, data=data) _emit_sync_result_breakdown(data) pull_ok, pull_message, pull_data = pull_detect_task_batch_now() logger.info("task pull tick: ok=%s message=%s data=%s", pull_ok, pull_message, pull_data) push_debug_event( service="sync-agent", event_type="task_pull_tick", level="info" if pull_ok else "warning", message=pull_message, payload={"ok": pull_ok, "data": pull_data}, ) _emit_structured_tick(base_event_type="task_pull", ok=pull_ok, message=pull_message, data=pull_data) if active_job: queue_health = _build_aligned_queue_health_snapshot( active_job, get_detect_queue_health(window_minutes=15), ) recent_events = _filter_runtime_events_for_job( list_recent_detect_run_events(limit=24), job_code=str(active_job.get("runtime_job_code") or active_job.get("job_code") or "").strip(), job_id=int(active_job.get("job_id", 0) or 0), limit=8, ) push_debug_event( service="detect-runtime", event_type="active_job_snapshot", level="info", message=f"active job {active_job.get('job_code', '')} status={active_job.get('status', '')}", payload={ "job": { "job_id": active_job.get("job_id"), "job_code": active_job.get("job_code", ""), "status": active_job.get("status", ""), "items_total": active_job.get("items_total", 0), "items_pending": active_job.get("items_pending", 0), "items_claimed": active_job.get("items_claimed", 0), "items_running": active_job.get("items_running", 0), "items_completed": active_job.get("items_completed", 0), "items_failed": active_job.get("items_failed", 0), "progress_percent": active_job.get("progress_percent", 0), "node_stats": list(active_job.get("node_stats") or []), }, "queue_health": queue_health, "backlog": _load_local_detect_backlog_snapshot(), "recent_events": recent_events, }, ) if queue_health.get("queue", {}).get("overdue_leases", 0): push_debug_event( service="detect-runtime", event_type="queue_overdue_leases", level="warning", message=f"检测队列存在过期租约 {queue_health.get('queue', {}).get('overdue_leases', 0)} 个", payload=queue_health, ) for event in recent_events: event_type = str(event.get("event_type") or "").strip() if event_type not in {"domain_started", "domain_completed", "domain_failed", "domain_blacklisted"}: continue push_debug_event( service="worker-event", event_type=event_type, level=str(event.get("level") or "info"), message=str(event.get("message") or "").strip(), payload={ "job_id": event.get("job_id"), "node_code": event.get("node_code", ""), "created_at": event.get("created_at", ""), **(event.get("payload") or {}), }, ) except Exception as exc: logger.exception("sync tick failed: %s", exc) push_debug_event( service="sync-agent", event_type="sync_tick_failed", level="error", message=str(exc), payload={}, ) time.sleep(interval) if __name__ == "__main__": main()