d

domain-api/app/sync_agent.py

@@ -5,13 +5,34 @@ import time
 
 from app.core.config import settings
 from app.services.debug_event_service import push_debug_event
-from app.services.detect_job_service import get_active_detect_job_summary, get_detect_queue_health, list_recent_detect_run_events
+from app.services.detect_job_service import (
+    get_active_detect_job_summary,
+    get_detect_queue_health,
+    get_latest_detect_job_summary,
+    get_latest_unprojected_detect_job_summary,
+    list_recent_detect_run_events,
+    process_detect_pipeline_now,
+)
 from app.services.sync_record_service import append_detect_result_projection_if_changed
-from app.services.sync_push_service import pull_detect_task_batch_now, push_runtime_projection_now
+from app.services.sync_push_service import (
+    _load_local_detect_backlog_snapshot,
+    pull_detect_task_batch_now,
+    push_runtime_projection_now,
+)
 
 
 logger = logging.getLogger("domaincheck.sync_agent")
 
+_IDLE_SYNC_KEYWORDS = (
+    "当前没有可推送",
+    "当前没有需要立即推送",
+    "已全部同步完成",
+    "无需重复发送",
+    "进行中",
+    "等待下个重试窗口",
+    "暂停拉取",
+)
+
 
 def _append_detect_result_projection_snapshot(active_job: dict) -> None:
     if not active_job:
@@ -32,6 +53,159 @@ def _append_detect_result_projection_snapshot(active_job: dict) -> None:
     )
 
 
+def _is_idle_sync_message(message: str) -> bool:
+    normalized = str(message or "").strip()
+    return any(keyword in normalized for keyword in _IDLE_SYNC_KEYWORDS)
+
+
+def _filter_runtime_events_for_job(events: list[dict], *, job_code: str = "", job_id: int = 0, limit: int = 8) -> list[dict]:
+    target_job_code = str(job_code or "").strip()
+    target_job_id = int(job_id or 0)
+    safe_limit = max(1, min(int(limit or 8), 50))
+    filtered: list[dict] = []
+    for raw_event in list(events or []):
+        if not isinstance(raw_event, dict):
+            continue
+        payload = raw_event.get("payload") if isinstance(raw_event.get("payload"), dict) else {}
+        event_job_code = str(payload.get("job_code") or "").strip()
+        event_job_id = int(raw_event.get("job_id") or 0)
+        if target_job_code and event_job_code != target_job_code and (target_job_id <= 0 or event_job_id != target_job_id):
+            continue
+        filtered.append(raw_event)
+        if len(filtered) >= safe_limit:
+            break
+    return filtered
+
+
+def _build_aligned_queue_health_snapshot(active_job: dict, queue_health: dict | None) -> dict:
+    snapshot = dict(queue_health or {})
+    if not active_job:
+        return snapshot
+
+    active_job_code = str(active_job.get("runtime_job_code") or active_job.get("job_code") or "").strip()
+    queue_job = dict(snapshot.get("job") or {})
+    queue_job_code = str(queue_job.get("runtime_job_code") or queue_job.get("job_code") or "").strip()
+
+    if active_job_code and queue_job_code and active_job_code == queue_job_code:
+        return snapshot
+
+    active_job_items_total = int(active_job.get("items_total", 0) or 0)
+    active_job_pending = int(active_job.get("items_pending", 0) or 0)
+    active_job_claimed = int(active_job.get("items_claimed", 0) or 0)
+    active_job_running = int(active_job.get("items_running", 0) or 0)
+    active_job_completed = int(active_job.get("items_completed", 0) or 0)
+    active_job_blacklisted = int(active_job.get("items_blacklisted", 0) or 0)
+    active_job_failed = int(active_job.get("items_failed", 0) or 0)
+    active_job_terminal = int(
+        active_job.get("items_terminal", active_job_completed + active_job_blacklisted + active_job_failed) or 0
+    )
+    display_claimed = int(active_job.get("display_items_claimed", active_job_claimed) or active_job_claimed)
+    display_running = int(active_job.get("display_items_running", active_job_running) or active_job_running)
+
+    node_entries: list[dict] = []
+    for node in list(active_job.get("node_stats") or []):
+        node_entries.append(
+            {
+                "node_code": str(node.get("node_code") or "").strip(),
+                "items_total": int(node.get("items_total", 0) or 0),
+                "items_pending": int(node.get("items_pending", 0) or 0),
+                "items_claimed": int(node.get("items_claimed", 0) or 0),
+                "items_running": int(node.get("items_running", 0) or 0),
+                "items_completed": int(node.get("items_completed", 0) or 0),
+                "items_blacklisted": int(node.get("items_blacklisted", 0) or 0),
+                "items_failed": int(node.get("items_failed", 0) or 0),
+                "processed_recent": int(node.get("processed_recent", 0) or 0),
+                "processed_per_minute": float(node.get("processed_per_minute", 0) or 0),
+                "completed_recent": int(node.get("completed_recent", 0) or 0),
+                "blacklisted_recent": int(node.get("blacklisted_recent", 0) or 0),
+                "failed_recent": int(node.get("failed_recent", 0) or 0),
+                "metrics_source": str(node.get("metrics_source") or "runtime"),
+            }
+        )
+
+    assigned_total = sum(int(item.get("items_total", 0) or 0) for item in node_entries)
+    unassigned_total = max(0, active_job_items_total - assigned_total)
+    if unassigned_total > 0:
+        node_entries.append(
+            {
+                "node_code": "unassigned",
+                "items_total": unassigned_total,
+                "items_pending": active_job_pending,
+                "items_claimed": 0,
+                "items_running": 0,
+                "items_completed": 0,
+                "items_blacklisted": 0,
+                "items_failed": 0,
+                "processed_recent": 0,
+                "processed_per_minute": 0.0,
+                "completed_recent": 0,
+                "blacklisted_recent": 0,
+                "failed_recent": 0,
+                "metrics_source": "central_queue",
+            }
+        )
+
+    snapshot["job"] = {
+        "job_id": active_job.get("job_id"),
+        "job_code": str(active_job.get("job_code") or "").strip(),
+        "runtime_job_code": active_job_code,
+        "status": str(active_job.get("status") or "").strip(),
+        "progress_percent": float(active_job.get("progress_percent", 0) or 0),
+    }
+    snapshot["queue"] = {
+        **dict(snapshot.get("queue") or {}),
+        "items_total": active_job_items_total,
+        "pending": active_job_pending,
+        "claimed": active_job_claimed,
+        "running": active_job_running,
+        "display_claimed": display_claimed,
+        "display_running": display_running,
+        "completed": active_job_completed,
+        "blacklisted": active_job_blacklisted,
+        "failed": active_job_failed,
+        "terminal": active_job_terminal,
+        "terminal_percent": round((active_job_terminal / active_job_items_total) * 100, 2) if active_job_items_total else 0.0,
+    }
+    snapshot["nodes"] = node_entries
+    return snapshot
+
+
+def _select_projection_job_snapshot() -> dict | None:
+    active_job = get_active_detect_job_summary(event_limit=10)
+    if active_job:
+        return active_job
+    return get_latest_detect_job_summary(
+        event_limit=10,
+        statuses=("completed", "partial_failed", "failed"),
+        recent_minutes=20,
+    )
+
+
+def _select_projection_job_snapshots() -> list[dict]:
+    snapshots: list[dict] = []
+    seen_job_ids: set[int] = set()
+
+    active_job = get_active_detect_job_summary(event_limit=10)
+    if active_job:
+        active_job_id = int(active_job.get("job_id") or 0)
+        if active_job_id > 0 and active_job_id not in seen_job_ids:
+            snapshots.append(active_job)
+            seen_job_ids.add(active_job_id)
+
+    latest_finished_job = get_latest_unprojected_detect_job_summary(
+        event_limit=10,
+        statuses=("completed", "partial_failed", "failed"),
+        recent_minutes=180,
+    )
+    if latest_finished_job:
+        latest_finished_job_id = int(latest_finished_job.get("job_id") or 0)
+        if latest_finished_job_id > 0 and latest_finished_job_id not in seen_job_ids:
+            snapshots.append(latest_finished_job)
+            seen_job_ids.add(latest_finished_job_id)
+
+    return snapshots
+
+
 def _emit_structured_tick(
     *,
     base_event_type: str,
@@ -42,7 +216,13 @@ def _emit_structured_tick(
     payload = {"ok": ok, "data": data or {}}
     event_type = f"{base_event_type}_failed"
     level = "warning"
-    if ok:
+    if isinstance(data, dict) and str(data.get("pull_state") or "").strip() == "throttled":
+        event_type = f"{base_event_type}_idle"
+        level = "info"
+    elif not ok and _is_idle_sync_message(message):
+        event_type = f"{base_event_type}_idle"
+        level = "info"
+    elif ok:
         event_type = f"{base_event_type}_success"
         level = "info"
         if "但远端确认失败" in str(message or ""):
@@ -70,7 +250,10 @@ def _emit_sync_result_breakdown(data: dict | None) -> None:
         result_data = item.get("data") or {}
         event_type = f"{sync_type}_sync_failed"
         level = "warning"
-        if ok:
+        if not ok and _is_idle_sync_message(message):
+            event_type = f"{sync_type}_sync_idle"
+            level = "info"
+        elif ok:
             event_type = f"{sync_type}_sync_success"
             level = "info"
             if isinstance(result_data, dict) and result_data.get("success_count") is not None:
@@ -95,12 +278,31 @@ def _emit_sync_result_breakdown(data: dict | None) -> None:
         )
 
 
+def _run_pipeline_stage_processor() -> tuple[bool, str, dict]:
+    process_limit = max(500, min(int(settings.sync_pipeline_process_limit or 5000), 5000))
+    ok, message, data = process_detect_pipeline_now(limit=process_limit)
+    push_debug_event(
+        service="sync-agent",
+        event_type="pipeline_tick_success" if ok else "pipeline_tick_failed",
+        level="info" if ok else "warning",
+        message=message,
+        payload={
+            "ok": ok,
+            "limit": process_limit,
+            "data": data or {},
+        },
+    )
+    return ok, message, data
+
+
 def main() -> None:
     logging.basicConfig(
         level=logging.INFO,
         format="%(asctime)s [%(levelname)s] %(name)s: %(message)s",
     )
-    interval = max(10, int(settings.sync_poll_interval_seconds or 30))
+    # Old env files still ship SYNC_POLL_INTERVAL_SECONDS=30. Cap the interval
+    # so controller pull/pipeline ticks cannot be throttled into starvation.
+    interval = max(2, min(int(settings.sync_poll_interval_seconds or 2), 5))
     logger.info(
         "sync agent started: node=%s source=%s target=%s interval=%ss enabled=%s",
         settings.node_code,
@@ -111,9 +313,16 @@ def main() -> None:
     )
     while True:
         try:
+            pipeline_ok, pipeline_message, pipeline_data = _run_pipeline_stage_processor()
+            logger.info(
+                "pipeline tick: ok=%s message=%s data=%s",
+                pipeline_ok,
+                pipeline_message,
+                pipeline_data,
+            )
             active_job = get_active_detect_job_summary(event_limit=10)
-            if active_job:
-                _append_detect_result_projection_snapshot(active_job)
+            for projection_job in _select_projection_job_snapshots():
+                _append_detect_result_projection_snapshot(projection_job)
             ok, message, data = push_runtime_projection_now()
             logger.info("sync tick: ok=%s message=%s data=%s", ok, message, data)
             push_debug_event(
@@ -136,8 +345,16 @@ def main() -> None:
             )
             _emit_structured_tick(base_event_type="task_pull", ok=pull_ok, message=pull_message, data=pull_data)
             if active_job:
-                queue_health = get_detect_queue_health(window_minutes=15)
-                recent_events = list_recent_detect_run_events(limit=8)
+                queue_health = _build_aligned_queue_health_snapshot(
+                    active_job,
+                    get_detect_queue_health(window_minutes=15),
+                )
+                recent_events = _filter_runtime_events_for_job(
+                    list_recent_detect_run_events(limit=24),
+                    job_code=str(active_job.get("runtime_job_code") or active_job.get("job_code") or "").strip(),
+                    job_id=int(active_job.get("job_id", 0) or 0),
+                    limit=8,
+                )
                 push_debug_event(
                     service="detect-runtime",
                     event_type="active_job_snapshot",
@@ -158,6 +375,7 @@ def main() -> None:
                             "node_stats": list(active_job.get("node_stats") or []),
                         },
                         "queue_health": queue_health,
+                        "backlog": _load_local_detect_backlog_snapshot(),
                         "recent_events": recent_events,
                     },
                 )