first

domain-api/app/services/detect_job_service.py

@@ -0,0 +1,558 @@
+from __future__ import annotations
+
+import json
+from datetime import datetime
+import math
+from uuid import uuid4
+
+from app.core.config import settings
+from app.core.db import get_db
+
+
+ACTIVE_JOB_STATUSES = ("pending", "running")
+
+
+def _selection_sql() -> str:
+    return """
+    SELECT id
+    FROM domains
+    WHERE
+        detect_status IN (0, 4)
+        OR (use_status = 0 AND detect_status = 1 AND register_status = 3 AND expire_date < CURRENT_DATE)
+    ORDER BY id ASC
+    LIMIT %s
+    """
+
+
+def _format_time(value: datetime | None) -> str:
+    return value.isoformat(sep=" ", timespec="seconds") if value else ""
+
+
+def _decode_payload(value: object) -> dict:
+    if isinstance(value, dict):
+        return value
+    if value in (None, ""):
+        return {}
+    try:
+        return json.loads(value)
+    except Exception:
+        return {}
+
+
+def _extract_current_cycle_events(events: list[dict]) -> tuple[str, list[dict]]:
+    if not events:
+        return "", []
+
+    latest_payload = events[0].get("payload") or {}
+    latest_cycle_token = str(latest_payload.get("cycle_token") or "").strip()
+    if latest_cycle_token:
+        current_cycle = []
+        for event in events:
+            payload = event.get("payload") or {}
+            event_cycle = str(payload.get("cycle_token") or "").strip()
+            if event_cycle == latest_cycle_token:
+                current_cycle.append(event)
+        return latest_cycle_token, current_cycle
+
+    cycle_token = ""
+    anchor_index = -1
+    for index, event in enumerate(events):
+        payload = event.get("payload") or {}
+        event_type = str(event.get("event_type") or "")
+        candidate = str(payload.get("cycle_token") or "").strip()
+        if candidate and event_type in {"job_dispatch_sent", "job_dispatch_requested", "job_dispatch_failed", "job_dispatch_rejected"}:
+            cycle_token = candidate
+            anchor_index = index
+            break
+    if not cycle_token:
+        return "", events
+    current_cycle = []
+    for index, event in enumerate(events[: anchor_index + 1]):
+        payload = event.get("payload") or {}
+        event_cycle = str(payload.get("cycle_token") or "").strip()
+        if index == anchor_index or event_cycle == cycle_token:
+            current_cycle.append(event)
+    return cycle_token, current_cycle
+
+
+def _fetch_job_summary(cur, job_row, event_limit: int = 20) -> dict:
+    job_id = job_row[0]
+    cur.execute(
+        """
+        SELECT status, count(*)
+        FROM detect_job_items
+        WHERE job_id = %s
+        GROUP BY status
+        """,
+        (job_id,),
+    )
+    counts = {status: int(count) for status, count in cur.fetchall()}
+    cur.execute(
+        """
+        SELECT COALESCE(NULLIF(claimed_by, ''), 'unassigned') AS node_code,
+               status,
+               count(*)
+        FROM detect_job_items
+        WHERE job_id = %s
+        GROUP BY COALESCE(NULLIF(claimed_by, ''), 'unassigned'), status
+        ORDER BY node_code ASC, status ASC
+        """,
+        (job_id,),
+    )
+    node_buckets: dict[str, dict] = {}
+    for node_code, status, count in cur.fetchall():
+        bucket = node_buckets.setdefault(
+            node_code,
+            {
+                "node_code": node_code,
+                "items_total": 0,
+                "items_pending": 0,
+                "items_claimed": 0,
+                "items_running": 0,
+                "items_completed": 0,
+                "items_blacklisted": 0,
+                "items_failed": 0,
+            },
+        )
+        field_name = f"items_{status}"
+        if field_name in bucket:
+            bucket[field_name] += int(count)
+        bucket["items_total"] += int(count)
+
+    cur.execute(
+        """
+        SELECT node_code, event_type, level, message, payload_json, created_at
+        FROM detect_run_events
+        WHERE job_id = %s
+        ORDER BY created_at DESC, id DESC
+        LIMIT %s
+        """,
+        (job_id, max(1, int(event_limit or 20))),
+    )
+    events = [
+        {
+            "node_code": item[0],
+            "event_type": item[1],
+            "level": item[2],
+            "message": item[3],
+            "payload": _decode_payload(item[4]),
+            "created_at": _format_time(item[5]),
+        }
+        for item in cur.fetchall()
+    ]
+    cycle_token, current_cycle_events = _extract_current_cycle_events(events)
+    total = sum(counts.values())
+    terminal = int(counts.get("completed", 0)) + int(counts.get("blacklisted", 0)) + int(counts.get("failed", 0))
+    return {
+        "job_id": job_id,
+        "job_code": job_row[1],
+        "source": job_row[2],
+        "status": job_row[3],
+        "created_by": job_row[4],
+        "created_at": _format_time(job_row[5]),
+        "started_at": _format_time(job_row[6]),
+        "finished_at": _format_time(job_row[7]),
+        "items_total": total,
+        "items_pending": int(counts.get("pending", 0)),
+        "items_claimed": int(counts.get("claimed", 0)),
+        "items_running": int(counts.get("running", 0)),
+        "items_completed": int(counts.get("completed", 0)),
+        "items_blacklisted": int(counts.get("blacklisted", 0)),
+        "items_failed": int(counts.get("failed", 0)),
+        "items_terminal": terminal,
+        "progress_percent": round((terminal / total) * 100, 2) if total else 0,
+        "node_stats": list(node_buckets.values()),
+        "recent_events": events,
+        "latest_event": events[0] if events else None,
+        "current_cycle_token": cycle_token,
+        "current_cycle_events": current_cycle_events,
+        "latest_cycle_event": current_cycle_events[0] if current_cycle_events else None,
+    }
+
+
+def get_detect_job_summary(job_id: int, event_limit: int = 20) -> dict | None:
+    with get_db() as conn:
+        with conn.cursor() as cur:
+            cur.execute(
+                """
+                SELECT id, job_code, source, status, created_by, created_at, started_at, finished_at
+                FROM detect_jobs
+                WHERE id = %s
+                LIMIT 1
+                """,
+                (int(job_id),),
+            )
+            row = cur.fetchone()
+            if not row:
+                return None
+            return _fetch_job_summary(cur, row, event_limit=event_limit)
+
+
+def get_active_detect_job_summary(event_limit: int = 20) -> dict | None:
+    with get_db() as conn:
+        with conn.cursor() as cur:
+            cur.execute(
+                """
+                SELECT id, job_code, source, status, created_by, created_at, started_at, finished_at
+                FROM detect_jobs
+                WHERE status IN ('pending', 'running')
+                ORDER BY created_at DESC, id DESC
+                LIMIT 1
+                """
+            )
+            row = cur.fetchone()
+            if not row:
+                return None
+            return _fetch_job_summary(cur, row, event_limit=event_limit)
+
+
+def list_detect_jobs(limit: int = 20) -> list[dict]:
+    with get_db() as conn:
+        with conn.cursor() as cur:
+            cur.execute(
+                """
+                SELECT id, job_code, source, status, created_by, created_at, started_at, finished_at
+                FROM detect_jobs
+                ORDER BY created_at DESC, id DESC
+                LIMIT %s
+                """,
+                (max(1, min(int(limit or 20), 100)),),
+            )
+            rows = cur.fetchall()
+            return [_fetch_job_summary(cur, row, event_limit=10) for row in rows]
+
+
+def get_detect_queue_health(window_minutes: int = 15) -> dict:
+    window_minutes = max(5, min(int(window_minutes or 15), 120))
+    active_job = get_active_detect_job_summary(event_limit=10)
+    if not active_job:
+        return {
+            "window_minutes": window_minutes,
+            "has_active_job": False,
+            "job": None,
+            "queue": {
+                "items_total": 0,
+                "pending": 0,
+                "claimed": 0,
+                "running": 0,
+                "completed": 0,
+                "blacklisted": 0,
+                "failed": 0,
+                "terminal": 0,
+                "terminal_percent": 0,
+                "oldest_pending_at": "",
+                "oldest_pending_age_minutes": 0,
+                "nearest_lease_expiry_at": "",
+                "overdue_leases": 0,
+                "expiring_soon_leases": 0,
+            },
+            "throughput": {
+                "processed_recent": 0,
+                "processed_per_minute": 0,
+                "completed_recent": 0,
+                "blacklisted_recent": 0,
+                "failed_recent": 0,
+            },
+            "nodes": [],
+        }
+
+    job_id = int(active_job["job_id"])
+    with get_db() as conn:
+        with conn.cursor() as cur:
+            cur.execute(
+                """
+                SELECT
+                    MIN(create_time) FILTER (WHERE status = 'pending') AS oldest_pending_at,
+                    MIN(lease_expires_at) FILTER (WHERE status IN ('claimed', 'running') AND lease_expires_at IS NOT NULL) AS nearest_lease_expiry_at,
+                    COUNT(*) FILTER (WHERE status IN ('claimed', 'running') AND lease_expires_at IS NOT NULL AND lease_expires_at < CURRENT_TIMESTAMP) AS overdue_leases,
+                    COUNT(*) FILTER (
+                        WHERE status IN ('claimed', 'running')
+                          AND lease_expires_at IS NOT NULL
+                          AND lease_expires_at >= CURRENT_TIMESTAMP
+                          AND lease_expires_at < CURRENT_TIMESTAMP + interval '5 minutes'
+                    ) AS expiring_soon_leases
+                FROM detect_job_items
+                WHERE job_id = %s
+                """,
+                (job_id,),
+            )
+            lease_row = cur.fetchone()
+
+            cur.execute(
+                """
+                SELECT
+                    COALESCE(NULLIF(node_code, ''), 'unknown') AS node_code,
+                    COUNT(*) AS processed_recent,
+                    COUNT(*) FILTER (WHERE event_type = 'domain_completed') AS completed_recent,
+                    COUNT(*) FILTER (WHERE event_type = 'domain_blacklisted') AS blacklisted_recent,
+                    COUNT(*) FILTER (WHERE event_type = 'domain_failed') AS failed_recent
+                FROM detect_run_events
+                WHERE job_id = %s
+                  AND event_type IN ('domain_completed', 'domain_blacklisted', 'domain_failed')
+                  AND created_at >= CURRENT_TIMESTAMP - (%s || ' minutes')::interval
+                GROUP BY COALESCE(NULLIF(node_code, ''), 'unknown')
+                ORDER BY processed_recent DESC, node_code ASC
+                """,
+                (job_id, window_minutes),
+            )
+            throughput_rows = cur.fetchall()
+
+    oldest_pending_at = _format_time(lease_row[0]) if lease_row and lease_row[0] else ""
+    nearest_lease_expiry_at = _format_time(lease_row[1]) if lease_row and lease_row[1] else ""
+    oldest_pending_age_minutes = 0
+    if lease_row and lease_row[0]:
+        oldest_pending_age_minutes = max(0, int((datetime.now() - lease_row[0]).total_seconds() // 60))
+
+    node_map = {
+        str(item.get("node_code") or "unknown"): {
+            "node_code": str(item.get("node_code") or "unknown"),
+            "items_total": int(item.get("items_total", 0) or 0),
+            "items_pending": int(item.get("items_pending", 0) or 0),
+            "items_claimed": int(item.get("items_claimed", 0) or 0),
+            "items_running": int(item.get("items_running", 0) or 0),
+            "items_completed": int(item.get("items_completed", 0) or 0),
+            "items_blacklisted": int(item.get("items_blacklisted", 0) or 0),
+            "items_failed": int(item.get("items_failed", 0) or 0),
+            "processed_recent": 0,
+            "processed_per_minute": 0,
+            "completed_recent": 0,
+            "blacklisted_recent": 0,
+            "failed_recent": 0,
+        }
+        for item in active_job.get("node_stats") or []
+    }
+    total_processed_recent = 0
+    total_completed_recent = 0
+    total_blacklisted_recent = 0
+    total_failed_recent = 0
+    for row in throughput_rows:
+        node_code = str(row[0] or "unknown")
+        bucket = node_map.setdefault(
+            node_code,
+            {
+                "node_code": node_code,
+                "items_total": 0,
+                "items_pending": 0,
+                "items_claimed": 0,
+                "items_running": 0,
+                "items_completed": 0,
+                "items_blacklisted": 0,
+                "items_failed": 0,
+                "processed_recent": 0,
+                "processed_per_minute": 0,
+                "completed_recent": 0,
+                "blacklisted_recent": 0,
+                "failed_recent": 0,
+            },
+        )
+        processed_recent = int(row[1] or 0)
+        completed_recent = int(row[2] or 0)
+        blacklisted_recent = int(row[3] or 0)
+        failed_recent = int(row[4] or 0)
+        bucket["processed_recent"] = processed_recent
+        bucket["processed_per_minute"] = round(processed_recent / window_minutes, 2)
+        bucket["completed_recent"] = completed_recent
+        bucket["blacklisted_recent"] = blacklisted_recent
+        bucket["failed_recent"] = failed_recent
+        total_processed_recent += processed_recent
+        total_completed_recent += completed_recent
+        total_blacklisted_recent += blacklisted_recent
+        total_failed_recent += failed_recent
+
+    nodes = sorted(
+        node_map.values(),
+        key=lambda item: (
+            -int(item.get("processed_recent", 0) or 0),
+            -int(item.get("items_running", 0) or 0),
+            str(item.get("node_code") or ""),
+        ),
+    )
+    items_total = int(active_job.get("items_total", 0) or 0)
+    terminal = int(active_job.get("items_terminal", 0) or 0)
+    return {
+        "window_minutes": window_minutes,
+        "has_active_job": True,
+        "job": {
+            "job_id": job_id,
+            "job_code": active_job.get("job_code", ""),
+            "status": active_job.get("status", ""),
+            "progress_percent": active_job.get("progress_percent", 0),
+        },
+        "queue": {
+            "items_total": items_total,
+            "pending": int(active_job.get("items_pending", 0) or 0),
+            "claimed": int(active_job.get("items_claimed", 0) or 0),
+            "running": int(active_job.get("items_running", 0) or 0),
+            "completed": int(active_job.get("items_completed", 0) or 0),
+            "blacklisted": int(active_job.get("items_blacklisted", 0) or 0),
+            "failed": int(active_job.get("items_failed", 0) or 0),
+            "terminal": terminal,
+            "terminal_percent": round((terminal / items_total) * 100, 2) if items_total else 0,
+            "oldest_pending_at": oldest_pending_at,
+            "oldest_pending_age_minutes": oldest_pending_age_minutes,
+            "nearest_lease_expiry_at": nearest_lease_expiry_at,
+            "overdue_leases": int(lease_row[2] or 0) if lease_row else 0,
+            "expiring_soon_leases": int(lease_row[3] or 0) if lease_row else 0,
+        },
+        "throughput": {
+            "processed_recent": total_processed_recent,
+            "processed_per_minute": round(total_processed_recent / window_minutes, 2),
+            "completed_recent": total_completed_recent,
+            "blacklisted_recent": total_blacklisted_recent,
+            "failed_recent": total_failed_recent,
+        },
+        "nodes": nodes,
+    }
+
+
+def get_detect_capacity_plan(*, queue_health: dict | None = None, online_worker_nodes: int = 0, target_finish_hours: int = 6) -> dict:
+    queue_health = queue_health or get_detect_queue_health(window_minutes=15)
+    target_finish_hours = max(1, min(int(target_finish_hours or 6), 72))
+    online_worker_nodes = max(0, int(online_worker_nodes or 0))
+
+    if not queue_health.get("has_active_job"):
+        return {
+            "has_active_job": False,
+            "online_worker_nodes": online_worker_nodes,
+            "target_finish_hours": target_finish_hours,
+            "estimated_hours_remaining": 0,
+            "recommended_total_workers": max(1, online_worker_nodes),
+            "recommended_additional_workers": 0,
+            "current_processed_per_hour": 0,
+            "pending_items": 0,
+            "terminal_items": 0,
+            "summary": "当前没有活跃任务，无需扩容建议。",
+        }
+
+    queue = queue_health.get("queue") or {}
+    throughput = queue_health.get("throughput") or {}
+    pending_items = int(queue.get("pending", 0) or 0)
+    claimed_items = int(queue.get("claimed", 0) or 0)
+    running_items = int(queue.get("running", 0) or 0)
+    remaining_items = pending_items + claimed_items + running_items
+    current_processed_per_hour = round(float(throughput.get("processed_per_minute", 0) or 0) * 60, 2)
+    estimated_hours_remaining = round((remaining_items / current_processed_per_hour), 2) if current_processed_per_hour > 0 else 0
+
+    recommended_total_workers = max(1, online_worker_nodes or 1)
+    recommended_additional_workers = 0
+    if remaining_items > 0 and target_finish_hours > 0:
+        required_per_hour = remaining_items / target_finish_hours
+        if current_processed_per_hour > 0 and max(1, online_worker_nodes) > 0:
+            per_worker_per_hour = current_processed_per_hour / max(1, online_worker_nodes)
+            recommended_total_workers = max(1, int(math.ceil(required_per_hour / per_worker_per_hour)))
+            recommended_additional_workers = max(0, recommended_total_workers - online_worker_nodes)
+        elif remaining_items > 0:
+            recommended_total_workers = max(1, online_worker_nodes or 1)
+            recommended_additional_workers = 0
+
+    summary = (
+        f"当前在线 Worker {online_worker_nodes} 台，近窗吞吐约 {current_processed_per_hour} 项/小时，"
+        f"剩余待处理约 {remaining_items} 项，预计还需 {estimated_hours_remaining} 小时。"
+    )
+    if recommended_additional_workers > 0:
+        summary = (
+            f"{summary} 若希望在 {target_finish_hours} 小时内收敛，建议总 Worker 数达到 "
+            f"{recommended_total_workers} 台，至少再加 {recommended_additional_workers} 台。"
+        )
+    else:
+        summary = f"{summary} 按当前目标 {target_finish_hours} 小时看，现有 Worker 数量暂时够用。"
+
+    return {
+        "has_active_job": True,
+        "online_worker_nodes": online_worker_nodes,
+        "target_finish_hours": target_finish_hours,
+        "estimated_hours_remaining": estimated_hours_remaining,
+        "recommended_total_workers": recommended_total_workers,
+        "recommended_additional_workers": recommended_additional_workers,
+        "current_processed_per_hour": current_processed_per_hour,
+        "pending_items": pending_items,
+        "remaining_items": remaining_items,
+        "terminal_items": int(queue.get("terminal", 0) or 0),
+        "summary": summary,
+    }
+
+
+def append_detect_job_event(
+    job_id: int,
+    *,
+    event_type: str,
+    message: str,
+    level: str = "info",
+    payload: dict | None = None,
+    node_code: str | None = None,
+) -> None:
+    with get_db() as conn:
+        with conn.cursor() as cur:
+            cur.execute(
+                """
+                INSERT INTO detect_run_events (job_id, node_code, event_type, level, message, payload_json)
+                VALUES (%s, %s, %s, %s, %s, %s::jsonb)
+                """,
+                (
+                    int(job_id),
+                    node_code or settings.node_code,
+                    str(event_type or "").strip() or "info",
+                    str(level or "info").strip() or "info",
+                    str(message or "").strip(),
+                    json.dumps(payload or {}, ensure_ascii=False),
+                ),
+            )
+        conn.commit()
+
+
+def create_detect_job_if_needed(limit: int = 1000, created_by: str = "system") -> dict | None:
+    existing = get_active_detect_job_summary()
+    if existing:
+        return existing
+
+    with get_db() as conn:
+        conn.autocommit = False
+        with conn.cursor() as cur:
+            cur.execute(_selection_sql(), (max(1, int(limit or 1000)),))
+            domain_ids = [row[0] for row in cur.fetchall()]
+            if not domain_ids:
+                conn.rollback()
+                return None
+
+            job_code = f"detect-{datetime.now().strftime('%Y%m%d%H%M%S')}-{uuid4().hex[:6]}"
+            plan_hash = uuid4().hex
+            cur.execute(
+                """
+                INSERT INTO detect_jobs (job_code, source, plan_hash, status, remark, created_by)
+                VALUES (%s, %s, %s, 'pending', %s, %s)
+                RETURNING id
+                """,
+                (
+                    job_code,
+                    "api-start",
+                    plan_hash,
+                    f"API 创建检测任务，待检测域名 {len(domain_ids)} 个",
+                    created_by,
+                ),
+            )
+            job_id = cur.fetchone()[0]
+            item_rows = [(job_id, domain_id) for domain_id in domain_ids]
+            cur.executemany(
+                """
+                INSERT INTO detect_job_items (job_id, domain_id, status)
+                VALUES (%s, %s, 'pending')
+                ON CONFLICT (job_id, domain_id) DO NOTHING
+                """,
+                item_rows,
+            )
+            cur.execute(
+                """
+                INSERT INTO detect_run_events (job_id, node_code, event_type, level, message, payload_json)
+                VALUES (%s, %s, %s, %s, %s, %s::jsonb)
+                """,
+                (
+                    job_id,
+                    settings.node_code,
+                    "job_created",
+                    "info",
+                    f"创建检测任务 {job_code}，共 {len(domain_ids)} 个域名",
+                    '{"count": %s}' % len(domain_ids),
+                ),
+            )
+        conn.commit()
+    return get_active_detect_job_summary()