420 lines
17 KiB
Python
420 lines
17 KiB
Python
from __future__ import annotations
|
|
|
|
import logging
|
|
import time
|
|
|
|
from app.core.config import settings
|
|
from app.services.debug_event_service import push_debug_event
|
|
from app.services.detect_job_service import (
|
|
get_active_detect_job_summary,
|
|
get_detect_queue_health,
|
|
get_latest_detect_job_summary,
|
|
get_latest_unprojected_detect_job_summary,
|
|
list_recent_detect_run_events,
|
|
process_detect_pipeline_now,
|
|
)
|
|
from app.services.sync_record_service import append_detect_result_projection_if_changed
|
|
from app.services.sync_push_service import (
|
|
_load_local_detect_backlog_snapshot,
|
|
pull_detect_task_batch_now,
|
|
push_runtime_projection_now,
|
|
)
|
|
|
|
|
|
logger = logging.getLogger("domaincheck.sync_agent")
|
|
|
|
_IDLE_SYNC_KEYWORDS = (
|
|
"当前没有可推送",
|
|
"当前没有需要立即推送",
|
|
"已全部同步完成",
|
|
"无需重复发送",
|
|
"进行中",
|
|
"等待下个重试窗口",
|
|
"暂停拉取",
|
|
)
|
|
|
|
|
|
def _append_detect_result_projection_snapshot(active_job: dict) -> None:
|
|
if not active_job:
|
|
return
|
|
append_detect_result_projection_if_changed(
|
|
detect={
|
|
"active_job": active_job,
|
|
"progress": {
|
|
"pending": int(active_job.get("items_pending", 0) or 0),
|
|
"running": int(active_job.get("items_running", 0) or 0),
|
|
"completed": int(active_job.get("items_completed", 0) or 0),
|
|
"blacklisted": int(active_job.get("items_blacklisted", 0) or 0),
|
|
"failed": int(active_job.get("items_failed", 0) or 0),
|
|
},
|
|
"phase_label": str(active_job.get("status") or "").strip(),
|
|
"phase_detail": f"sync-agent snapshot for {active_job.get('job_code', '')}",
|
|
}
|
|
)
|
|
|
|
|
|
def _is_idle_sync_message(message: str) -> bool:
|
|
normalized = str(message or "").strip()
|
|
return any(keyword in normalized for keyword in _IDLE_SYNC_KEYWORDS)
|
|
|
|
|
|
def _filter_runtime_events_for_job(events: list[dict], *, job_code: str = "", job_id: int = 0, limit: int = 8) -> list[dict]:
|
|
target_job_code = str(job_code or "").strip()
|
|
target_job_id = int(job_id or 0)
|
|
safe_limit = max(1, min(int(limit or 8), 50))
|
|
filtered: list[dict] = []
|
|
for raw_event in list(events or []):
|
|
if not isinstance(raw_event, dict):
|
|
continue
|
|
payload = raw_event.get("payload") if isinstance(raw_event.get("payload"), dict) else {}
|
|
event_job_code = str(payload.get("job_code") or "").strip()
|
|
event_job_id = int(raw_event.get("job_id") or 0)
|
|
if target_job_code and event_job_code != target_job_code and (target_job_id <= 0 or event_job_id != target_job_id):
|
|
continue
|
|
filtered.append(raw_event)
|
|
if len(filtered) >= safe_limit:
|
|
break
|
|
return filtered
|
|
|
|
|
|
def _build_aligned_queue_health_snapshot(active_job: dict, queue_health: dict | None) -> dict:
|
|
snapshot = dict(queue_health or {})
|
|
if not active_job:
|
|
return snapshot
|
|
|
|
active_job_code = str(active_job.get("runtime_job_code") or active_job.get("job_code") or "").strip()
|
|
queue_job = dict(snapshot.get("job") or {})
|
|
queue_job_code = str(queue_job.get("runtime_job_code") or queue_job.get("job_code") or "").strip()
|
|
|
|
if active_job_code and queue_job_code and active_job_code == queue_job_code:
|
|
return snapshot
|
|
|
|
active_job_items_total = int(active_job.get("items_total", 0) or 0)
|
|
active_job_pending = int(active_job.get("items_pending", 0) or 0)
|
|
active_job_claimed = int(active_job.get("items_claimed", 0) or 0)
|
|
active_job_running = int(active_job.get("items_running", 0) or 0)
|
|
active_job_completed = int(active_job.get("items_completed", 0) or 0)
|
|
active_job_blacklisted = int(active_job.get("items_blacklisted", 0) or 0)
|
|
active_job_failed = int(active_job.get("items_failed", 0) or 0)
|
|
active_job_terminal = int(
|
|
active_job.get("items_terminal", active_job_completed + active_job_blacklisted + active_job_failed) or 0
|
|
)
|
|
display_claimed = int(active_job.get("display_items_claimed", active_job_claimed) or active_job_claimed)
|
|
display_running = int(active_job.get("display_items_running", active_job_running) or active_job_running)
|
|
|
|
node_entries: list[dict] = []
|
|
for node in list(active_job.get("node_stats") or []):
|
|
node_entries.append(
|
|
{
|
|
"node_code": str(node.get("node_code") or "").strip(),
|
|
"items_total": int(node.get("items_total", 0) or 0),
|
|
"items_pending": int(node.get("items_pending", 0) or 0),
|
|
"items_claimed": int(node.get("items_claimed", 0) or 0),
|
|
"items_running": int(node.get("items_running", 0) or 0),
|
|
"items_completed": int(node.get("items_completed", 0) or 0),
|
|
"items_blacklisted": int(node.get("items_blacklisted", 0) or 0),
|
|
"items_failed": int(node.get("items_failed", 0) or 0),
|
|
"processed_recent": int(node.get("processed_recent", 0) or 0),
|
|
"processed_per_minute": float(node.get("processed_per_minute", 0) or 0),
|
|
"completed_recent": int(node.get("completed_recent", 0) or 0),
|
|
"blacklisted_recent": int(node.get("blacklisted_recent", 0) or 0),
|
|
"failed_recent": int(node.get("failed_recent", 0) or 0),
|
|
"metrics_source": str(node.get("metrics_source") or "runtime"),
|
|
}
|
|
)
|
|
|
|
assigned_total = sum(int(item.get("items_total", 0) or 0) for item in node_entries)
|
|
unassigned_total = max(0, active_job_items_total - assigned_total)
|
|
if unassigned_total > 0:
|
|
node_entries.append(
|
|
{
|
|
"node_code": "unassigned",
|
|
"items_total": unassigned_total,
|
|
"items_pending": active_job_pending,
|
|
"items_claimed": 0,
|
|
"items_running": 0,
|
|
"items_completed": 0,
|
|
"items_blacklisted": 0,
|
|
"items_failed": 0,
|
|
"processed_recent": 0,
|
|
"processed_per_minute": 0.0,
|
|
"completed_recent": 0,
|
|
"blacklisted_recent": 0,
|
|
"failed_recent": 0,
|
|
"metrics_source": "central_queue",
|
|
}
|
|
)
|
|
|
|
snapshot["job"] = {
|
|
"job_id": active_job.get("job_id"),
|
|
"job_code": str(active_job.get("job_code") or "").strip(),
|
|
"runtime_job_code": active_job_code,
|
|
"status": str(active_job.get("status") or "").strip(),
|
|
"progress_percent": float(active_job.get("progress_percent", 0) or 0),
|
|
}
|
|
snapshot["queue"] = {
|
|
**dict(snapshot.get("queue") or {}),
|
|
"items_total": active_job_items_total,
|
|
"pending": active_job_pending,
|
|
"claimed": active_job_claimed,
|
|
"running": active_job_running,
|
|
"display_claimed": display_claimed,
|
|
"display_running": display_running,
|
|
"completed": active_job_completed,
|
|
"blacklisted": active_job_blacklisted,
|
|
"failed": active_job_failed,
|
|
"terminal": active_job_terminal,
|
|
"terminal_percent": round((active_job_terminal / active_job_items_total) * 100, 2) if active_job_items_total else 0.0,
|
|
}
|
|
snapshot["nodes"] = node_entries
|
|
return snapshot
|
|
|
|
|
|
def _select_projection_job_snapshot() -> dict | None:
|
|
active_job = get_active_detect_job_summary(event_limit=10)
|
|
if active_job:
|
|
return active_job
|
|
return get_latest_detect_job_summary(
|
|
event_limit=10,
|
|
statuses=("completed", "partial_failed", "failed"),
|
|
recent_minutes=20,
|
|
)
|
|
|
|
|
|
def _select_projection_job_snapshots() -> list[dict]:
|
|
snapshots: list[dict] = []
|
|
seen_job_ids: set[int] = set()
|
|
|
|
active_job = get_active_detect_job_summary(event_limit=10)
|
|
if active_job:
|
|
active_job_id = int(active_job.get("job_id") or 0)
|
|
if active_job_id > 0 and active_job_id not in seen_job_ids:
|
|
snapshots.append(active_job)
|
|
seen_job_ids.add(active_job_id)
|
|
|
|
latest_finished_job = get_latest_unprojected_detect_job_summary(
|
|
event_limit=10,
|
|
statuses=("completed", "partial_failed", "failed"),
|
|
recent_minutes=180,
|
|
)
|
|
if latest_finished_job:
|
|
latest_finished_job_id = int(latest_finished_job.get("job_id") or 0)
|
|
if latest_finished_job_id > 0 and latest_finished_job_id not in seen_job_ids:
|
|
snapshots.append(latest_finished_job)
|
|
seen_job_ids.add(latest_finished_job_id)
|
|
|
|
return snapshots
|
|
|
|
|
|
def _emit_structured_tick(
|
|
*,
|
|
base_event_type: str,
|
|
ok: bool,
|
|
message: str,
|
|
data: dict | None = None,
|
|
) -> None:
|
|
payload = {"ok": ok, "data": data or {}}
|
|
event_type = f"{base_event_type}_failed"
|
|
level = "warning"
|
|
if isinstance(data, dict) and str(data.get("pull_state") or "").strip() == "throttled":
|
|
event_type = f"{base_event_type}_idle"
|
|
level = "info"
|
|
elif not ok and _is_idle_sync_message(message):
|
|
event_type = f"{base_event_type}_idle"
|
|
level = "info"
|
|
elif ok:
|
|
event_type = f"{base_event_type}_success"
|
|
level = "info"
|
|
if "但远端确认失败" in str(message or ""):
|
|
event_type = f"{base_event_type}_partial"
|
|
level = "warning"
|
|
results = list((data or {}).get("results") or [])
|
|
if results and any(not bool(item.get("ok")) for item in results):
|
|
event_type = f"{base_event_type}_partial"
|
|
level = "warning"
|
|
push_debug_event(
|
|
service="sync-agent",
|
|
event_type=event_type,
|
|
level=level,
|
|
message=message,
|
|
payload=payload,
|
|
)
|
|
|
|
|
|
def _emit_sync_result_breakdown(data: dict | None) -> None:
|
|
results = list((data or {}).get("results") or [])
|
|
for item in results:
|
|
sync_type = str(item.get("sync_type") or "").strip() or "unknown"
|
|
ok = bool(item.get("ok"))
|
|
message = str(item.get("message") or "").strip() or f"{sync_type} sync result"
|
|
result_data = item.get("data") or {}
|
|
event_type = f"{sync_type}_sync_failed"
|
|
level = "warning"
|
|
if not ok and _is_idle_sync_message(message):
|
|
event_type = f"{sync_type}_sync_idle"
|
|
level = "info"
|
|
elif ok:
|
|
event_type = f"{sync_type}_sync_success"
|
|
level = "info"
|
|
if isinstance(result_data, dict) and result_data.get("success_count") is not None:
|
|
batch_count = int(result_data.get("batch_count") or 0)
|
|
success_count = int(result_data.get("success_count") or 0)
|
|
if batch_count > 0 and success_count < batch_count:
|
|
event_type = f"{sync_type}_sync_partial"
|
|
level = "warning"
|
|
if "成功 1/" in message or "但" in message:
|
|
event_type = f"{sync_type}_sync_partial"
|
|
level = "warning"
|
|
push_debug_event(
|
|
service="sync-agent",
|
|
event_type=event_type,
|
|
level=level,
|
|
message=message,
|
|
payload={
|
|
"sync_type": sync_type,
|
|
"ok": ok,
|
|
"data": result_data,
|
|
},
|
|
)
|
|
|
|
|
|
def _run_pipeline_stage_processor() -> tuple[bool, str, dict]:
|
|
process_limit = max(500, min(int(settings.sync_pipeline_process_limit or 5000), 5000))
|
|
ok, message, data = process_detect_pipeline_now(limit=process_limit)
|
|
push_debug_event(
|
|
service="sync-agent",
|
|
event_type="pipeline_tick_success" if ok else "pipeline_tick_failed",
|
|
level="info" if ok else "warning",
|
|
message=message,
|
|
payload={
|
|
"ok": ok,
|
|
"limit": process_limit,
|
|
"data": data or {},
|
|
},
|
|
)
|
|
return ok, message, data
|
|
|
|
|
|
def main() -> None:
|
|
logging.basicConfig(
|
|
level=logging.INFO,
|
|
format="%(asctime)s [%(levelname)s] %(name)s: %(message)s",
|
|
)
|
|
# Old env files still ship SYNC_POLL_INTERVAL_SECONDS=30. Cap the interval
|
|
# so controller pull/pipeline ticks cannot be throttled into starvation.
|
|
interval = max(2, min(int(settings.sync_poll_interval_seconds or 2), 5))
|
|
logger.info(
|
|
"sync agent started: node=%s source=%s target=%s interval=%ss enabled=%s",
|
|
settings.node_code,
|
|
settings.sync_source_region,
|
|
settings.sync_target_region,
|
|
interval,
|
|
settings.sync_push_enabled,
|
|
)
|
|
while True:
|
|
try:
|
|
pipeline_ok, pipeline_message, pipeline_data = _run_pipeline_stage_processor()
|
|
logger.info(
|
|
"pipeline tick: ok=%s message=%s data=%s",
|
|
pipeline_ok,
|
|
pipeline_message,
|
|
pipeline_data,
|
|
)
|
|
active_job = get_active_detect_job_summary(event_limit=10)
|
|
for projection_job in _select_projection_job_snapshots():
|
|
_append_detect_result_projection_snapshot(projection_job)
|
|
ok, message, data = push_runtime_projection_now()
|
|
logger.info("sync tick: ok=%s message=%s data=%s", ok, message, data)
|
|
push_debug_event(
|
|
service="sync-agent",
|
|
event_type="sync_tick",
|
|
level="info" if ok else "warning",
|
|
message=message,
|
|
payload={"ok": ok, "data": data},
|
|
)
|
|
_emit_structured_tick(base_event_type="sync_push", ok=ok, message=message, data=data)
|
|
_emit_sync_result_breakdown(data)
|
|
pull_ok, pull_message, pull_data = pull_detect_task_batch_now()
|
|
logger.info("task pull tick: ok=%s message=%s data=%s", pull_ok, pull_message, pull_data)
|
|
push_debug_event(
|
|
service="sync-agent",
|
|
event_type="task_pull_tick",
|
|
level="info" if pull_ok else "warning",
|
|
message=pull_message,
|
|
payload={"ok": pull_ok, "data": pull_data},
|
|
)
|
|
_emit_structured_tick(base_event_type="task_pull", ok=pull_ok, message=pull_message, data=pull_data)
|
|
if active_job:
|
|
queue_health = _build_aligned_queue_health_snapshot(
|
|
active_job,
|
|
get_detect_queue_health(window_minutes=15),
|
|
)
|
|
recent_events = _filter_runtime_events_for_job(
|
|
list_recent_detect_run_events(limit=24),
|
|
job_code=str(active_job.get("runtime_job_code") or active_job.get("job_code") or "").strip(),
|
|
job_id=int(active_job.get("job_id", 0) or 0),
|
|
limit=8,
|
|
)
|
|
push_debug_event(
|
|
service="detect-runtime",
|
|
event_type="active_job_snapshot",
|
|
level="info",
|
|
message=f"active job {active_job.get('job_code', '')} status={active_job.get('status', '')}",
|
|
payload={
|
|
"job": {
|
|
"job_id": active_job.get("job_id"),
|
|
"job_code": active_job.get("job_code", ""),
|
|
"status": active_job.get("status", ""),
|
|
"items_total": active_job.get("items_total", 0),
|
|
"items_pending": active_job.get("items_pending", 0),
|
|
"items_claimed": active_job.get("items_claimed", 0),
|
|
"items_running": active_job.get("items_running", 0),
|
|
"items_completed": active_job.get("items_completed", 0),
|
|
"items_failed": active_job.get("items_failed", 0),
|
|
"progress_percent": active_job.get("progress_percent", 0),
|
|
"node_stats": list(active_job.get("node_stats") or []),
|
|
},
|
|
"queue_health": queue_health,
|
|
"backlog": _load_local_detect_backlog_snapshot(),
|
|
"recent_events": recent_events,
|
|
},
|
|
)
|
|
if queue_health.get("queue", {}).get("overdue_leases", 0):
|
|
push_debug_event(
|
|
service="detect-runtime",
|
|
event_type="queue_overdue_leases",
|
|
level="warning",
|
|
message=f"检测队列存在过期租约 {queue_health.get('queue', {}).get('overdue_leases', 0)} 个",
|
|
payload=queue_health,
|
|
)
|
|
for event in recent_events:
|
|
event_type = str(event.get("event_type") or "").strip()
|
|
if event_type not in {"domain_started", "domain_completed", "domain_failed", "domain_blacklisted"}:
|
|
continue
|
|
push_debug_event(
|
|
service="worker-event",
|
|
event_type=event_type,
|
|
level=str(event.get("level") or "info"),
|
|
message=str(event.get("message") or "").strip(),
|
|
payload={
|
|
"job_id": event.get("job_id"),
|
|
"node_code": event.get("node_code", ""),
|
|
"created_at": event.get("created_at", ""),
|
|
**(event.get("payload") or {}),
|
|
},
|
|
)
|
|
except Exception as exc:
|
|
logger.exception("sync tick failed: %s", exc)
|
|
push_debug_event(
|
|
service="sync-agent",
|
|
event_type="sync_tick_failed",
|
|
level="error",
|
|
message=str(exc),
|
|
payload={},
|
|
)
|
|
time.sleep(interval)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
main()
|