from __future__ import annotations import json from datetime import datetime import math from uuid import uuid4 from app.core.config import settings from app.core.db import get_db from app.services.debug_event_service import push_debug_event ACTIVE_JOB_STATUSES = ("pending", "running") def _selection_sql() -> str: return """ SELECT id FROM domains WHERE detect_status IN (0, 4) OR (use_status = 0 AND detect_status = 1 AND register_status = 3 AND expire_date < CURRENT_DATE) ORDER BY id ASC LIMIT %s """ def _format_time(value: datetime | None) -> str: return value.isoformat(sep=" ", timespec="seconds") if value else "" def _decode_payload(value: object) -> dict: if isinstance(value, dict): return value if value in (None, ""): return {} try: return json.loads(value) except Exception: return {} def _extract_current_cycle_events(events: list[dict]) -> tuple[str, list[dict]]: if not events: return "", [] latest_payload = events[0].get("payload") or {} latest_cycle_token = str(latest_payload.get("cycle_token") or "").strip() if latest_cycle_token: current_cycle = [] for event in events: payload = event.get("payload") or {} event_cycle = str(payload.get("cycle_token") or "").strip() if event_cycle == latest_cycle_token: current_cycle.append(event) return latest_cycle_token, current_cycle cycle_token = "" anchor_index = -1 for index, event in enumerate(events): payload = event.get("payload") or {} event_type = str(event.get("event_type") or "") candidate = str(payload.get("cycle_token") or "").strip() if candidate and event_type in {"job_dispatch_sent", "job_dispatch_requested", "job_dispatch_failed", "job_dispatch_rejected"}: cycle_token = candidate anchor_index = index break if not cycle_token: return "", events current_cycle = [] for index, event in enumerate(events[: anchor_index + 1]): payload = event.get("payload") or {} event_cycle = str(payload.get("cycle_token") or "").strip() if index == anchor_index or event_cycle == cycle_token: current_cycle.append(event) return cycle_token, current_cycle def _fetch_job_summary(cur, job_row, event_limit: int = 20) -> dict: job_id = job_row[0] cur.execute( """ SELECT status, count(*) FROM detect_job_items WHERE job_id = %s GROUP BY status """, (job_id,), ) counts = {status: int(count) for status, count in cur.fetchall()} cur.execute( """ SELECT COALESCE(NULLIF(claimed_by, ''), 'unassigned') AS node_code, status, count(*) FROM detect_job_items WHERE job_id = %s GROUP BY COALESCE(NULLIF(claimed_by, ''), 'unassigned'), status ORDER BY node_code ASC, status ASC """, (job_id,), ) node_buckets: dict[str, dict] = {} for node_code, status, count in cur.fetchall(): bucket = node_buckets.setdefault( node_code, { "node_code": node_code, "items_total": 0, "items_pending": 0, "items_claimed": 0, "items_running": 0, "items_completed": 0, "items_blacklisted": 0, "items_failed": 0, }, ) field_name = f"items_{status}" if field_name in bucket: bucket[field_name] += int(count) bucket["items_total"] += int(count) cur.execute( """ SELECT node_code, event_type, level, message, payload_json, created_at FROM detect_run_events WHERE job_id = %s ORDER BY created_at DESC, id DESC LIMIT %s """, (job_id, max(1, int(event_limit or 20))), ) events = [ { "node_code": item[0], "event_type": item[1], "level": item[2], "message": item[3], "payload": _decode_payload(item[4]), "created_at": _format_time(item[5]), } for item in cur.fetchall() ] cycle_token, current_cycle_events = _extract_current_cycle_events(events) total = sum(counts.values()) terminal = int(counts.get("completed", 0)) + int(counts.get("blacklisted", 0)) + int(counts.get("failed", 0)) return { "job_id": job_id, "job_code": job_row[1], "source": job_row[2], "status": job_row[3], "created_by": job_row[4], "created_at": _format_time(job_row[5]), "started_at": _format_time(job_row[6]), "finished_at": _format_time(job_row[7]), "items_total": total, "items_pending": int(counts.get("pending", 0)), "items_claimed": int(counts.get("claimed", 0)), "items_running": int(counts.get("running", 0)), "items_completed": int(counts.get("completed", 0)), "items_blacklisted": int(counts.get("blacklisted", 0)), "items_failed": int(counts.get("failed", 0)), "items_terminal": terminal, "progress_percent": round((terminal / total) * 100, 2) if total else 0, "node_stats": list(node_buckets.values()), "recent_events": events, "latest_event": events[0] if events else None, "current_cycle_token": cycle_token, "current_cycle_events": current_cycle_events, "latest_cycle_event": current_cycle_events[0] if current_cycle_events else None, } def get_detect_job_summary(job_id: int, event_limit: int = 20) -> dict | None: with get_db() as conn: with conn.cursor() as cur: cur.execute( """ SELECT id, job_code, source, status, created_by, created_at, started_at, finished_at FROM detect_jobs WHERE id = %s LIMIT 1 """, (int(job_id),), ) row = cur.fetchone() if not row: return None return _fetch_job_summary(cur, row, event_limit=event_limit) def get_active_detect_job_summary(event_limit: int = 20) -> dict | None: with get_db() as conn: with conn.cursor() as cur: cur.execute( """ SELECT id, job_code, source, status, created_by, created_at, started_at, finished_at FROM detect_jobs WHERE status IN ('pending', 'running') ORDER BY created_at DESC, id DESC LIMIT 1 """ ) row = cur.fetchone() if not row: return None return _fetch_job_summary(cur, row, event_limit=event_limit) def list_detect_jobs(limit: int = 20) -> list[dict]: with get_db() as conn: with conn.cursor() as cur: cur.execute( """ SELECT id, job_code, source, status, created_by, created_at, started_at, finished_at FROM detect_jobs ORDER BY created_at DESC, id DESC LIMIT %s """, (max(1, min(int(limit or 20), 100)),), ) rows = cur.fetchall() return [_fetch_job_summary(cur, row, event_limit=10) for row in rows] def list_recent_detect_run_events(limit: int = 20) -> list[dict]: safe_limit = max(1, min(int(limit or 20), 200)) with get_db() as conn: with conn.cursor() as cur: cur.execute( """ SELECT job_id, node_code, event_type, level, message, payload_json, created_at FROM detect_run_events ORDER BY created_at DESC, id DESC LIMIT %s """, (safe_limit,), ) rows = cur.fetchall() return [ { "job_id": int(row[0]) if row[0] is not None else None, "node_code": row[1] or "", "event_type": row[2] or "", "level": row[3] or "info", "message": row[4] or "", "payload": _decode_payload(row[5]), "created_at": _format_time(row[6]), } for row in rows ] def get_detect_queue_health(window_minutes: int = 15) -> dict: window_minutes = max(5, min(int(window_minutes or 15), 120)) active_job = get_active_detect_job_summary(event_limit=10) if not active_job: return { "window_minutes": window_minutes, "has_active_job": False, "job": None, "queue": { "items_total": 0, "pending": 0, "claimed": 0, "running": 0, "completed": 0, "blacklisted": 0, "failed": 0, "terminal": 0, "terminal_percent": 0, "oldest_pending_at": "", "oldest_pending_age_minutes": 0, "nearest_lease_expiry_at": "", "overdue_leases": 0, "expiring_soon_leases": 0, }, "throughput": { "processed_recent": 0, "processed_per_minute": 0, "completed_recent": 0, "blacklisted_recent": 0, "failed_recent": 0, }, "nodes": [], } job_id = int(active_job["job_id"]) with get_db() as conn: with conn.cursor() as cur: cur.execute( """ SELECT MIN(create_time) FILTER (WHERE status = 'pending') AS oldest_pending_at, MIN(lease_expires_at) FILTER (WHERE status IN ('claimed', 'running') AND lease_expires_at IS NOT NULL) AS nearest_lease_expiry_at, COUNT(*) FILTER (WHERE status IN ('claimed', 'running') AND lease_expires_at IS NOT NULL AND lease_expires_at < CURRENT_TIMESTAMP) AS overdue_leases, COUNT(*) FILTER ( WHERE status IN ('claimed', 'running') AND lease_expires_at IS NOT NULL AND lease_expires_at >= CURRENT_TIMESTAMP AND lease_expires_at < CURRENT_TIMESTAMP + interval '5 minutes' ) AS expiring_soon_leases FROM detect_job_items WHERE job_id = %s """, (job_id,), ) lease_row = cur.fetchone() cur.execute( """ SELECT COALESCE(NULLIF(node_code, ''), 'unknown') AS node_code, COUNT(*) AS processed_recent, COUNT(*) FILTER (WHERE event_type = 'domain_completed') AS completed_recent, COUNT(*) FILTER (WHERE event_type = 'domain_blacklisted') AS blacklisted_recent, COUNT(*) FILTER (WHERE event_type = 'domain_failed') AS failed_recent FROM detect_run_events WHERE job_id = %s AND event_type IN ('domain_completed', 'domain_blacklisted', 'domain_failed') AND created_at >= CURRENT_TIMESTAMP - (%s || ' minutes')::interval GROUP BY COALESCE(NULLIF(node_code, ''), 'unknown') ORDER BY processed_recent DESC, node_code ASC """, (job_id, window_minutes), ) throughput_rows = cur.fetchall() oldest_pending_at = _format_time(lease_row[0]) if lease_row and lease_row[0] else "" nearest_lease_expiry_at = _format_time(lease_row[1]) if lease_row and lease_row[1] else "" oldest_pending_age_minutes = 0 if lease_row and lease_row[0]: oldest_pending_age_minutes = max(0, int((datetime.now() - lease_row[0]).total_seconds() // 60)) node_map = { str(item.get("node_code") or "unknown"): { "node_code": str(item.get("node_code") or "unknown"), "items_total": int(item.get("items_total", 0) or 0), "items_pending": int(item.get("items_pending", 0) or 0), "items_claimed": int(item.get("items_claimed", 0) or 0), "items_running": int(item.get("items_running", 0) or 0), "items_completed": int(item.get("items_completed", 0) or 0), "items_blacklisted": int(item.get("items_blacklisted", 0) or 0), "items_failed": int(item.get("items_failed", 0) or 0), "processed_recent": 0, "processed_per_minute": 0, "completed_recent": 0, "blacklisted_recent": 0, "failed_recent": 0, } for item in active_job.get("node_stats") or [] } total_processed_recent = 0 total_completed_recent = 0 total_blacklisted_recent = 0 total_failed_recent = 0 for row in throughput_rows: node_code = str(row[0] or "unknown") bucket = node_map.setdefault( node_code, { "node_code": node_code, "items_total": 0, "items_pending": 0, "items_claimed": 0, "items_running": 0, "items_completed": 0, "items_blacklisted": 0, "items_failed": 0, "processed_recent": 0, "processed_per_minute": 0, "completed_recent": 0, "blacklisted_recent": 0, "failed_recent": 0, }, ) processed_recent = int(row[1] or 0) completed_recent = int(row[2] or 0) blacklisted_recent = int(row[3] or 0) failed_recent = int(row[4] or 0) bucket["processed_recent"] = processed_recent bucket["processed_per_minute"] = round(processed_recent / window_minutes, 2) bucket["completed_recent"] = completed_recent bucket["blacklisted_recent"] = blacklisted_recent bucket["failed_recent"] = failed_recent total_processed_recent += processed_recent total_completed_recent += completed_recent total_blacklisted_recent += blacklisted_recent total_failed_recent += failed_recent nodes = sorted( node_map.values(), key=lambda item: ( -int(item.get("processed_recent", 0) or 0), -int(item.get("items_running", 0) or 0), str(item.get("node_code") or ""), ), ) items_total = int(active_job.get("items_total", 0) or 0) terminal = int(active_job.get("items_terminal", 0) or 0) return { "window_minutes": window_minutes, "has_active_job": True, "job": { "job_id": job_id, "job_code": active_job.get("job_code", ""), "status": active_job.get("status", ""), "progress_percent": active_job.get("progress_percent", 0), }, "queue": { "items_total": items_total, "pending": int(active_job.get("items_pending", 0) or 0), "claimed": int(active_job.get("items_claimed", 0) or 0), "running": int(active_job.get("items_running", 0) or 0), "completed": int(active_job.get("items_completed", 0) or 0), "blacklisted": int(active_job.get("items_blacklisted", 0) or 0), "failed": int(active_job.get("items_failed", 0) or 0), "terminal": terminal, "terminal_percent": round((terminal / items_total) * 100, 2) if items_total else 0, "oldest_pending_at": oldest_pending_at, "oldest_pending_age_minutes": oldest_pending_age_minutes, "nearest_lease_expiry_at": nearest_lease_expiry_at, "overdue_leases": int(lease_row[2] or 0) if lease_row else 0, "expiring_soon_leases": int(lease_row[3] or 0) if lease_row else 0, }, "throughput": { "processed_recent": total_processed_recent, "processed_per_minute": round(total_processed_recent / window_minutes, 2), "completed_recent": total_completed_recent, "blacklisted_recent": total_blacklisted_recent, "failed_recent": total_failed_recent, }, "nodes": nodes, } def get_detect_capacity_plan(*, queue_health: dict | None = None, online_worker_nodes: int = 0, target_finish_hours: int = 6) -> dict: queue_health = queue_health or get_detect_queue_health(window_minutes=15) target_finish_hours = max(1, min(int(target_finish_hours or 6), 72)) online_worker_nodes = max(0, int(online_worker_nodes or 0)) if not queue_health.get("has_active_job"): return { "has_active_job": False, "online_worker_nodes": online_worker_nodes, "target_finish_hours": target_finish_hours, "estimated_hours_remaining": 0, "recommended_total_workers": max(1, online_worker_nodes), "recommended_additional_workers": 0, "current_processed_per_hour": 0, "pending_items": 0, "terminal_items": 0, "summary": "当前没有活跃任务,无需扩容建议。", } queue = queue_health.get("queue") or {} throughput = queue_health.get("throughput") or {} pending_items = int(queue.get("pending", 0) or 0) claimed_items = int(queue.get("claimed", 0) or 0) running_items = int(queue.get("running", 0) or 0) remaining_items = pending_items + claimed_items + running_items current_processed_per_hour = round(float(throughput.get("processed_per_minute", 0) or 0) * 60, 2) estimated_hours_remaining = round((remaining_items / current_processed_per_hour), 2) if current_processed_per_hour > 0 else 0 recommended_total_workers = max(1, online_worker_nodes or 1) recommended_additional_workers = 0 if remaining_items > 0 and target_finish_hours > 0: required_per_hour = remaining_items / target_finish_hours if current_processed_per_hour > 0 and max(1, online_worker_nodes) > 0: per_worker_per_hour = current_processed_per_hour / max(1, online_worker_nodes) recommended_total_workers = max(1, int(math.ceil(required_per_hour / per_worker_per_hour))) recommended_additional_workers = max(0, recommended_total_workers - online_worker_nodes) elif remaining_items > 0: recommended_total_workers = max(1, online_worker_nodes or 1) recommended_additional_workers = 0 summary = ( f"当前在线 Worker {online_worker_nodes} 台,近窗吞吐约 {current_processed_per_hour} 项/小时," f"剩余待处理约 {remaining_items} 项,预计还需 {estimated_hours_remaining} 小时。" ) if recommended_additional_workers > 0: summary = ( f"{summary} 若希望在 {target_finish_hours} 小时内收敛,建议总 Worker 数达到 " f"{recommended_total_workers} 台,至少再加 {recommended_additional_workers} 台。" ) else: summary = f"{summary} 按当前目标 {target_finish_hours} 小时看,现有 Worker 数量暂时够用。" return { "has_active_job": True, "online_worker_nodes": online_worker_nodes, "target_finish_hours": target_finish_hours, "estimated_hours_remaining": estimated_hours_remaining, "recommended_total_workers": recommended_total_workers, "recommended_additional_workers": recommended_additional_workers, "current_processed_per_hour": current_processed_per_hour, "pending_items": pending_items, "remaining_items": remaining_items, "terminal_items": int(queue.get("terminal", 0) or 0), "summary": summary, } def append_detect_job_event( job_id: int, *, event_type: str, message: str, level: str = "info", payload: dict | None = None, node_code: str | None = None, ) -> None: with get_db() as conn: with conn.cursor() as cur: cur.execute( """ INSERT INTO detect_run_events (job_id, node_code, event_type, level, message, payload_json) VALUES (%s, %s, %s, %s, %s, %s::jsonb) """, ( int(job_id), node_code or settings.node_code, str(event_type or "").strip() or "info", str(level or "info").strip() or "info", str(message or "").strip(), json.dumps(payload or {}, ensure_ascii=False), ), ) conn.commit() try: push_debug_event( service="detect-job", event_type=str(event_type or "").strip() or "info", level=str(level or "info").strip() or "info", message=str(message or "").strip(), payload={ "job_id": int(job_id), "node_code": node_code or settings.node_code, **(payload or {}), }, ) except Exception: pass def create_detect_job_if_needed(limit: int = 1000, created_by: str = "system") -> dict | None: existing = get_active_detect_job_summary() if existing: return existing with get_db() as conn: conn.autocommit = False with conn.cursor() as cur: cur.execute(_selection_sql(), (max(1, int(limit or 1000)),)) domain_ids = [row[0] for row in cur.fetchall()] if not domain_ids: conn.rollback() if not domain_ids: try: from app.services.sync_push_service import pull_detect_task_batch_now pull_detect_task_batch_now(limit=max(1, int(limit or 1000))) except Exception: pass with get_db() as conn: conn.autocommit = False with conn.cursor() as cur: cur.execute(_selection_sql(), (max(1, int(limit or 1000)),)) domain_ids = [row[0] for row in cur.fetchall()] if not domain_ids: conn.rollback() return None with get_db() as conn: conn.autocommit = False with conn.cursor() as cur: job_code = f"detect-{datetime.now().strftime('%Y%m%d%H%M%S')}-{uuid4().hex[:6]}" plan_hash = uuid4().hex cur.execute( """ INSERT INTO detect_jobs (job_code, source, plan_hash, status, remark, created_by) VALUES (%s, %s, %s, 'pending', %s, %s) RETURNING id """, ( job_code, "api-start", plan_hash, f"API 创建检测任务,待检测域名 {len(domain_ids)} 个", created_by, ), ) job_id = cur.fetchone()[0] item_rows = [(job_id, domain_id) for domain_id in domain_ids] cur.executemany( """ INSERT INTO detect_job_items (job_id, domain_id, status) VALUES (%s, %s, 'pending') ON CONFLICT (job_id, domain_id) DO NOTHING """, item_rows, ) cur.execute( """ INSERT INTO detect_run_events (job_id, node_code, event_type, level, message, payload_json) VALUES (%s, %s, %s, %s, %s, %s::jsonb) """, ( job_id, settings.node_code, "job_created", "info", f"创建检测任务 {job_code},共 {len(domain_ids)} 个域名", '{"count": %s}' % len(domain_ids), ), ) conn.commit() return get_active_detect_job_summary()