This commit is contained in:
Your Name
2026-04-22 14:13:21 +08:00
parent e0406b5d0e
commit 7cbde2aa78
145 changed files with 23086 additions and 2243 deletions

View File

@@ -4,6 +4,7 @@ from uuid import uuid4
from fastapi import APIRouter
from app.core.config import settings
from app.schemas.common import ApiResponse
from app.services.detect_job_service import (
append_detect_job_event,
@@ -12,6 +13,7 @@ from app.services.detect_job_service import (
get_detect_job_summary,
get_detect_queue_health,
list_detect_jobs,
normalize_detect_step_code,
)
from app.services.detect_service import get_detect_status
from app.services.detect_run_service import create_detect_run_snapshot, finalize_detect_run, mark_detect_run_stopping
@@ -22,6 +24,13 @@ from app.services.worker_control_service import send_worker_command, start_worke
router = APIRouter(tags=["detect"])
def _local_worker_expected_on_this_node() -> bool:
return not (
str(settings.node_region or "").strip() == "overseas"
and str(settings.node_role or "").strip() == "control"
)
def _build_detect_action_result(
*,
action: str,
@@ -255,6 +264,11 @@ def detect_queue_summary(window_minutes: int = 15) -> ApiResponse:
return ApiResponse(data=get_detect_queue_health(window_minutes=window_minutes))
@router.get("/detect/queue-health", response_model=ApiResponse)
def detect_queue_health_alias(window_minutes: int = 15) -> ApiResponse:
return ApiResponse(data=get_detect_queue_health(window_minutes=window_minutes))
@router.get("/detect/jobs/{job_id}", response_model=ApiResponse)
def detect_job_detail(job_id: int) -> ApiResponse:
data = get_detect_job_summary(job_id, event_limit=100)
@@ -264,8 +278,17 @@ def detect_job_detail(job_id: int) -> ApiResponse:
@router.post("/detect/start", response_model=ApiResponse)
def start_detect() -> ApiResponse:
job_summary = create_detect_job_if_needed(limit=1000, created_by="api")
def start_detect(step_code: str | None = None) -> ApiResponse:
normalized_step_code = normalize_detect_step_code(step_code)
if step_code and not normalized_step_code:
result = _build_detect_action_result(
action="start",
ok=False,
message=f"暂不支持步骤任务: {step_code}",
data={"job": None, "step_code": str(step_code or "").strip()},
)
return ApiResponse(code=1, message=f"暂不支持步骤任务: {step_code}", data=result)
job_summary = create_detect_job_if_needed(limit=10000, created_by="api", step_code=step_code)
if not job_summary:
result = _build_detect_action_result(
action="start",
@@ -292,42 +315,58 @@ def start_detect() -> ApiResponse:
},
)
ok, message = start_worker()
if not ok:
result = _build_detect_action_result(
action="start",
ok=False,
message=message,
data={"job": job_summary},
local_worker_expected = _local_worker_expected_on_this_node()
if local_worker_expected:
ok, message = start_worker()
if not ok:
result = _build_detect_action_result(
action="start",
ok=False,
message=message,
data={"job": job_summary},
)
append_detect_job_event(
job_summary["job_id"],
event_type="job_dispatch_failed",
level="error",
message=f"启动 Worker 失败: {message}",
payload={"cycle_token": cycle_token},
)
return ApiResponse(
code=1,
message=message,
data=result,
)
command_ok, command_message = send_worker_command(
"start_detection",
payload={
"cycle_token": cycle_token,
"job_id": job_summary["job_id"],
"job_code": job_summary["job_code"],
"task_mode": job_summary.get("task_mode", ""),
"step_code": job_summary.get("step_code", ""),
},
)
append_detect_job_event(
job_summary["job_id"],
event_type="job_dispatch_failed",
level="error",
message=f"启动 Worker 失败: {message}",
event_type="job_dispatch_sent" if command_ok else "job_dispatch_rejected",
level="info" if command_ok else "error",
message=command_message,
payload={"cycle_token": cycle_token},
)
return ApiResponse(
code=1,
message=message,
data=result,
else:
ok = True
message = "当前节点为海外控制面,仅负责派单,不启动本机 Worker"
command_ok = True
command_message = "已跳过本机 Worker 启动,转为仅向大陆执行节点派发检测动作"
append_detect_job_event(
job_summary["job_id"],
event_type="job_dispatch_skipped_local",
level="info",
message=command_message,
payload={"cycle_token": cycle_token},
)
command_ok, command_message = send_worker_command(
"start_detection",
payload={
"cycle_token": cycle_token,
"job_id": job_summary["job_id"],
"job_code": job_summary["job_code"],
},
)
append_detect_job_event(
job_summary["job_id"],
event_type="job_dispatch_sent" if command_ok else "job_dispatch_rejected",
level="info" if command_ok else "error",
message=command_message,
payload={"cycle_token": cycle_token},
)
snapshot = get_detect_status()
settings_payload = get_settings_payload()
settings_summary = _build_settings_summary(settings_payload)

View File

@@ -9,6 +9,7 @@ from app.services.ops_agent_service import (
agent_heartbeat,
agent_mark_job_started,
agent_pull_jobs,
agent_pull_runtime_config,
agent_register,
build_node_agent_bootstrap_plan,
issue_node_agent_token,
@@ -72,6 +73,12 @@ def ops_agent_pull(payload: dict, limit: int = 1, x_domaincheck_agent_token: Opt
return _build_agent_response(ok, message, data)
@router.post("/ops/agent/runtime-config", response_model=ApiResponse)
def ops_agent_runtime_config(payload: dict, x_domaincheck_agent_token: Optional[str] = Header(default=None)) -> ApiResponse:
ok, message, data = agent_pull_runtime_config(payload, token=_resolve_agent_token(x_domaincheck_agent_token))
return _build_agent_response(ok, message, data)
@router.post("/ops/agent/jobs/{job_id}/start", response_model=ApiResponse)
def ops_agent_job_start(job_id: int, payload: dict, x_domaincheck_agent_token: Optional[str] = Header(default=None)) -> ApiResponse:
ok, message, data = agent_mark_job_started(job_id, payload, token=_resolve_agent_token(x_domaincheck_agent_token))

View File

@@ -1,11 +1,12 @@
from typing import Optional
from fastapi import APIRouter, Header
from fastapi import APIRouter, Body, Header
from app.schemas.common import ApiResponse
from app.services.build_info_service import get_runtime_build_info
from app.services.cluster_runtime_service import get_cluster_snapshot
from app.services.debug_event_service import get_debug_diagnosis, get_debug_event_overview, get_debug_handoff_report, ingest_debug_event, list_debug_events
from app.services.detect_job_service import get_detect_queue_health
from app.services.runtime_control_service import runtime_action
from app.services.runtime_status_service import get_runtime_preflight, get_runtime_status
from app.services.sync_push_service import (
@@ -103,6 +104,36 @@ def runtime_debug_handoff(window_minutes: int = 10, source_region: Optional[str]
)
@router.get("/runtime/queue-health", response_model=ApiResponse)
def runtime_queue_health(window_minutes: int = 15) -> ApiResponse:
return ApiResponse(data=get_detect_queue_health(window_minutes=window_minutes))
@router.get("/runtime/health-handover", response_model=ApiResponse)
def runtime_health_handover(
node_code: Optional[str] = None,
window_minutes: int = 10,
source_region: Optional[str] = None,
) -> ApiResponse:
sync_summary = get_sync_summary()
runtime_status_payload = get_runtime_status()
data = get_debug_handoff_report(
window_minutes=window_minutes,
source_region=source_region,
sync_summary=sync_summary,
readiness=runtime_status_payload.get("readiness") or {},
)
normalized_node_code = str(node_code or "").strip()
if normalized_node_code:
data = dict(data)
data["nodes"] = [
item
for item in list(data.get("nodes") or [])
if str(item.get("node_code") or "").strip() == normalized_node_code
]
return ApiResponse(data=data)
@router.post("/runtime/sync-ingest", response_model=ApiResponse)
def runtime_sync_ingest(payload: dict, x_domaincheck_sync_token: Optional[str] = Header(default=None)) -> ApiResponse:
ok, message, data = ingest_runtime_projection(payload, shared_token=x_domaincheck_sync_token)
@@ -110,7 +141,7 @@ def runtime_sync_ingest(payload: dict, x_domaincheck_sync_token: Optional[str] =
@router.get("/runtime/task-export", response_model=ApiResponse)
def runtime_task_export(limit: int = 200, x_domaincheck_sync_token: Optional[str] = Header(default=None)) -> ApiResponse:
def runtime_task_export(limit: int = 1000, x_domaincheck_sync_token: Optional[str] = Header(default=None)) -> ApiResponse:
ok, message, data = export_detect_task_projection(limit=limit, shared_token=x_domaincheck_sync_token)
return ApiResponse(code=0 if ok else 1, message=message, data=data)
@@ -128,6 +159,6 @@ def runtime_debug_ingest(payload: dict, x_domaincheck_sync_token: Optional[str]
@router.post("/runtime/actions/{action}", response_model=ApiResponse)
def runtime_action_trigger(action: str) -> ApiResponse:
ok, message, data = runtime_action(action)
def runtime_action_trigger(action: str, payload: Optional[dict] = Body(default=None)) -> ApiResponse:
ok, message, data = runtime_action(action, payload=payload)
return ApiResponse(code=0 if ok else 1, message=message, data=data)

View File

@@ -40,8 +40,12 @@ class Settings(BaseSettings):
sync_target_region: str = "overseas"
sync_target_api_base_url: str = ""
sync_shared_token: str = ""
sync_batch_size: int = 200
sync_poll_interval_seconds: int = 30
sync_batch_size: int = 5000
sync_poll_interval_seconds: int = 2
sync_pipeline_process_limit: int = 5000
sync_pull_max_pending_items: int = 0
sync_pull_max_register_pending_items: int = 0
sync_pull_max_downstream_pending_items: int = 0
build_manifest_path: str = ""
build_commit_sha: str = ""
build_commit_ref: str = ""

View File

@@ -1,6 +1,9 @@
from contextlib import contextmanager
from functools import wraps
import time
import psycopg2
from psycopg2 import errors
from app.core.config import settings
@@ -18,3 +21,39 @@ def get_db():
yield conn
finally:
conn.close()
_RETRYABLE_READ_ERRORS = (
errors.DeadlockDetected,
errors.SerializationFailure,
errors.LockNotAvailable,
)
def is_retryable_read_error(exc: Exception) -> bool:
return isinstance(exc, _RETRYABLE_READ_ERRORS)
def is_retryable_db_error(exc: Exception) -> bool:
return isinstance(exc, _RETRYABLE_READ_ERRORS)
def db_read_retry(*, attempts: int = 3, initial_delay_seconds: float = 0.05, backoff: float = 2.0):
def decorator(func):
@wraps(func)
def wrapper(*args, **kwargs):
delay = max(0.0, float(initial_delay_seconds or 0.0))
for attempt in range(1, max(1, int(attempts or 1)) + 1):
try:
return func(*args, **kwargs)
except Exception as exc:
if not is_retryable_read_error(exc) or attempt >= max(1, int(attempts or 1)):
raise
if delay > 0:
time.sleep(delay)
delay *= max(1.0, float(backoff or 1.0))
return func(*args, **kwargs)
return wrapper
return decorator

View File

@@ -1,6 +1,7 @@
from __future__ import annotations
import json
import os
from pathlib import Path
from datetime import datetime
@@ -63,9 +64,35 @@ def tail_lines(relative_path: str, max_lines: int = 120) -> list[str]:
def runtime_root() -> Path:
path = Path(__file__).resolve().parents[2] / "runtime"
path.mkdir(parents=True, exist_ok=True)
return path
env_override = str(os.getenv("DOMAIN_API_RUNTIME_ROOT", "") or "").strip()
candidates: list[Path] = []
if env_override:
candidates.append(Path(env_override))
base_dir = Path(__file__).resolve().parents[2]
for parent in base_dir.parents:
if parent.name != "releases":
continue
# Released builds live under /opt/domaincheck/releases/<release>/domain-api.
# Runtime state must not be written back into the immutable release tree,
# otherwise sync-agent / detect runtime snapshots fail with permission errors.
candidates.append(parent.parent / "runtime" / "domain-api")
break
candidates.append(base_dir / "runtime")
last_error: OSError | None = None
for candidate in candidates:
try:
candidate.mkdir(parents=True, exist_ok=True)
return candidate
except OSError as exc:
last_error = exc
continue
if last_error is not None:
raise last_error
raise RuntimeError("failed to resolve runtime root")
def read_runtime_json(filename: str, default: dict | list | None = None):
@@ -164,7 +191,16 @@ def load_detect_records() -> list[dict]:
try:
with path.open("r", encoding="utf-8") as handle:
return json.load(handle)
except (json.JSONDecodeError, OSError):
except (json.JSONDecodeError, UnicodeDecodeError, OSError):
try:
raw = path.read_bytes()
text = raw.decode("utf-8", errors="replace")
decoder = json.JSONDecoder()
payload, _ = decoder.raw_decode(text)
if isinstance(payload, list):
return payload
except Exception:
pass
return []

View File

@@ -1,13 +1,16 @@
from __future__ import annotations
import hashlib
import json
import os
import socket
import subprocess
import traceback
import time
import urllib.error
import urllib.request
from datetime import datetime
from urllib.parse import urlparse
from uuid import uuid4
from app.services.ops_action_executor_core import (
@@ -28,6 +31,44 @@ NODE_CODE = str(os.getenv("NODE_CODE", "")).strip()
NODE_REGION = str(os.getenv("NODE_REGION", "mainland")).strip() or "mainland"
NODE_ROLE = str(os.getenv("NODE_ROLE", "worker")).strip() or "worker"
AGENT_POLL_INTERVAL_SECONDS = max(2, int(os.getenv("OPS_AGENT_POLL_INTERVAL_SECONDS", "5") or 5))
AGENT_RUNTIME_CONFIG_SYNC_INTERVAL_SECONDS = max(
10,
int(os.getenv("OPS_AGENT_RUNTIME_CONFIG_SYNC_INTERVAL_SECONDS", "30") or 30),
)
AGENT_HTTP_TIMEOUT_SECONDS = max(10, int(os.getenv("OPS_AGENT_HTTP_TIMEOUT_SECONDS", "60") or 60))
AGENT_REGISTER_TIMEOUT_SECONDS = max(
10,
int(os.getenv("OPS_AGENT_REGISTER_TIMEOUT_SECONDS", str(AGENT_HTTP_TIMEOUT_SECONDS)) or AGENT_HTTP_TIMEOUT_SECONDS),
)
AGENT_HEARTBEAT_TIMEOUT_SECONDS = max(
10,
int(os.getenv("OPS_AGENT_HEARTBEAT_TIMEOUT_SECONDS", str(AGENT_HTTP_TIMEOUT_SECONDS)) or AGENT_HTTP_TIMEOUT_SECONDS),
)
AGENT_PULL_TIMEOUT_SECONDS = max(
10,
int(os.getenv("OPS_AGENT_PULL_TIMEOUT_SECONDS", str(AGENT_HTTP_TIMEOUT_SECONDS)) or AGENT_HTTP_TIMEOUT_SECONDS),
)
AGENT_RUNTIME_CONFIG_TIMEOUT_SECONDS = max(
10,
int(
os.getenv("OPS_AGENT_RUNTIME_CONFIG_TIMEOUT_SECONDS", str(AGENT_HTTP_TIMEOUT_SECONDS))
or AGENT_HTTP_TIMEOUT_SECONDS
),
)
AGENT_JOB_COMPLETE_TIMEOUT_SECONDS = max(
10,
int(
os.getenv("OPS_AGENT_JOB_COMPLETE_TIMEOUT_SECONDS", str(AGENT_HTTP_TIMEOUT_SECONDS))
or AGENT_HTTP_TIMEOUT_SECONDS
),
)
AGENT_JOB_EVENT_TIMEOUT_SECONDS = max(
10,
int(
os.getenv("OPS_AGENT_JOB_EVENT_TIMEOUT_SECONDS", str(max(15, AGENT_HTTP_TIMEOUT_SECONDS // 2)))
or max(15, AGENT_HTTP_TIMEOUT_SECONDS // 2)
),
)
WORKER_SERVICE_NAME = str(os.getenv("WORKER_SERVICE_NAME", os.getenv("WORKER_SERVICE", "domaincheck-worker"))).strip() or "domaincheck-worker"
API_SERVICE_NAME = str(os.getenv("API_SERVICE_NAME", "domaincheck-api")).strip() or "domaincheck-api"
SYNC_AGENT_SERVICE_NAME = str(os.getenv("SYNC_AGENT_SERVICE_NAME", "domaincheck-sync-agent")).strip() or "domaincheck-sync-agent"
@@ -50,6 +91,7 @@ _LAST_QUEUE_FLUSH_SUMMARY = {
"dead_letter": 0,
"last_flush_at": "",
}
_LAST_RUNTIME_CONFIG_HASH = ""
def _normalize_text_list(raw_value: object) -> list[str]:
@@ -98,6 +140,7 @@ AGENT_CAPABILITIES = _json_env(
"runtime.restart_api",
"runtime.start_sync_agent",
"runtime.stop_sync_agent",
"runtime.reset_lab_state",
"health.snapshot",
"logs.collect",
"diagnostics.collect",
@@ -601,7 +644,7 @@ def _headers() -> dict[str, str]:
}
def _request(method: str, path: str, payload: dict | None = None, timeout: int = 30) -> dict:
def _request(method: str, path: str, payload: dict | None = None, timeout: int = AGENT_HTTP_TIMEOUT_SECONDS) -> dict:
if not CONTROL_PLANE_BASE_URL:
raise RuntimeError("OPS_CONTROL_PLANE_BASE_URL 未配置")
if not AGENT_TOKEN:
@@ -614,22 +657,286 @@ def _request(method: str, path: str, payload: dict | None = None, timeout: int =
return json.loads(body or "{}")
def _post(path: str, payload: dict, timeout: int = 30) -> dict:
def _post(path: str, payload: dict, timeout: int = AGENT_HTTP_TIMEOUT_SECONDS) -> dict:
return _request("POST", path, payload, timeout=timeout)
def _hostname() -> str:
def _pull_runtime_config(timeout: int = AGENT_RUNTIME_CONFIG_TIMEOUT_SECONDS) -> dict:
response = _post("/api/v1/ops/agent/runtime-config", {"node_code": NODE_CODE}, timeout=timeout)
_ensure_ok_response(response, "agent runtime config pull failed")
data = response.get("data") or {}
return dict(data.get("bundle") or {})
def _publish_local_config_update(config_type: str) -> None:
try:
return socket.gethostname()
from app.core.redis_client import get_redis
redis_client = get_redis()
redis_client.publish("domain_tool:config_update", str(config_type or "").strip() or "config")
except Exception:
return ""
return
def _apply_runtime_config(bundle: dict) -> bool:
global _LAST_RUNTIME_CONFIG_HASH
normalized_bundle = dict(bundle or {})
bundle_hash = str(normalized_bundle.get("config_hash") or "").strip()
if not bundle_hash:
bundle_hash = hashlib.sha256(
json.dumps(normalized_bundle, ensure_ascii=False, sort_keys=True).encode("utf-8")
).hexdigest()
if bundle_hash and bundle_hash == _LAST_RUNTIME_CONFIG_HASH:
return False
from app.core.files import write_json
from app.services.runtime_settings_service import update_runtime_settings
from app.services.sensitive_words_service import save_sensitive_words_payload
detect_options = dict(normalized_bundle.get("detect_options") or {})
proxy_config = dict(normalized_bundle.get("proxy_config") or {})
thread_count = int(normalized_bundle.get("thread_count", 2) or 2)
node_thread_counts = dict(normalized_bundle.get("node_thread_counts") or {})
runtime_settings = dict(normalized_bundle.get("runtime_settings") or {})
sensitive_words = dict(normalized_bundle.get("sensitive_words") or {})
sensitive_words_text = str(sensitive_words.get("text") or "")
sensitive_word_items = list(sensitive_words.get("items") or [])
write_json("detect_options.json", detect_options)
write_json("proxy_config.json", proxy_config)
write_json("thread_count.json", {"thread_count": str(thread_count)})
write_json("node_thread_counts.json", node_thread_counts)
write_json("runtime_settings.json", runtime_settings)
write_json("runtime/runtime_settings.json", runtime_settings)
write_json(
"runtime/sensitive_words.json",
{
"items": sensitive_word_items,
"text": sensitive_words_text,
"total": int(sensitive_words.get("total", 0) or 0),
},
)
try:
update_runtime_settings(runtime_settings)
except Exception as exc:
_log(f"runtime settings local api sync skipped: {exc}")
try:
save_sensitive_words_payload({"text": sensitive_words_text})
except Exception as exc:
_log(f"sensitive words db sync skipped: {exc}")
try:
from app.core.redis_client import get_redis
redis_client = get_redis()
redis_client.set("domain_tool:detect_options", json.dumps(detect_options, ensure_ascii=False))
redis_client.set("domain_tool:proxy_config", json.dumps(proxy_config, ensure_ascii=False))
redis_client.set("domain_tool:thread_count", thread_count)
redis_client.set("domain_tool:node_thread_counts", json.dumps(node_thread_counts, ensure_ascii=False))
redis_client.set("domain_tool:runtime_settings", json.dumps(runtime_settings, ensure_ascii=False))
redis_client.set("domain_tool:sensitive_words", json.dumps(sensitive_word_items, ensure_ascii=False))
redis_client.publish("domain_tool:config_update", "detect_options")
redis_client.publish("domain_tool:config_update", "proxy_config")
redis_client.publish("domain_tool:config_update", "thread_count")
redis_client.publish("domain_tool:config_update", "node_thread_counts")
redis_client.publish("domain_tool:config_update", "runtime_settings")
redis_client.publish("domain_tool:config_update", "sensitive_words")
except Exception:
_publish_local_config_update("detect_options")
_publish_local_config_update("proxy_config")
_publish_local_config_update("thread_count")
_publish_local_config_update("node_thread_counts")
_publish_local_config_update("runtime_settings")
_publish_local_config_update("sensitive_words")
_LAST_RUNTIME_CONFIG_HASH = bundle_hash
_log(
"runtime config applied: "
f"thread_count={thread_count} "
f"node_override={node_thread_counts.get(NODE_CODE)} "
f"sensitive_words={int(sensitive_words.get('total', 0) or 0)}"
)
return True
def _hostname() -> str:
generic_values = {"localhost", "localhost.localdomain", "ip6-localhost", "localhost6"}
candidates: list[str] = []
try:
candidates.append(socket.gethostname())
except Exception:
pass
try:
candidates.append(socket.getfqdn())
except Exception:
pass
try:
candidates.append(os.uname().nodename)
except Exception:
pass
candidates.append(str(os.getenv("HOSTNAME", "")).strip())
for value in candidates:
normalized = str(value or "").strip()
if not normalized:
continue
if normalized.lower() in generic_values:
continue
return normalized
return NODE_CODE or ""
def _control_plane_host_port() -> tuple[str, int]:
parsed = urlparse(CONTROL_PLANE_BASE_URL if "://" in CONTROL_PLANE_BASE_URL else f"http://{CONTROL_PLANE_BASE_URL}")
host = str(parsed.hostname or "").strip()
if not host:
return "", 0
port = int(parsed.port or (443 if parsed.scheme == "https" else 80))
return host, port
def _first_non_loopback_ip(values: list[str]) -> str:
for value in values:
normalized = str(value or "").strip()
if not normalized or normalized.startswith("127.") or normalized == "::1":
continue
return normalized
return ""
def _ip() -> str:
explicit_ip = str(os.getenv("OPS_AGENT_IP", "")).strip()
if explicit_ip:
return explicit_ip
host, port = _control_plane_host_port()
if host and port:
for family in (socket.AF_INET, socket.AF_INET6):
sock = None
try:
sock = socket.socket(family, socket.SOCK_DGRAM)
sock.connect((host, port))
local_ip = str(sock.getsockname()[0] or "").strip()
if local_ip and not local_ip.startswith("127.") and local_ip != "::1":
return local_ip
except Exception:
pass
finally:
if sock is not None:
try:
sock.close()
except Exception:
pass
hostname = _hostname()
if hostname:
try:
addrinfo = socket.getaddrinfo(hostname, None)
resolved_ips = [str(item[4][0] or "").strip() for item in addrinfo if item and len(item) >= 5]
best_ip = _first_non_loopback_ip(resolved_ips)
if best_ip:
return best_ip
except Exception:
pass
try:
return socket.gethostbyname(socket.gethostname())
fallback_ip = str(socket.gethostbyname(socket.gethostname()) or "").strip()
if fallback_ip and not fallback_ip.startswith("127."):
return fallback_ip
except Exception:
return ""
pass
return ""
def _detect_runtime_snapshot() -> dict:
worker_runtime = {}
try:
from app.services.worker_control_service import detect_worker_runtime
worker_runtime = detect_worker_runtime() or {}
except Exception as exc:
worker_runtime = {
"running": False,
"process_count": 0,
"latest_start_time": "",
"message": "",
"error": str(exc),
}
try:
from app.services.detect_service import get_detect_status
detect_status = get_detect_status() or {}
runtime_state = detect_status.get("runtime_state") if isinstance(detect_status.get("runtime_state"), dict) else {}
active_job = detect_status.get("active_job") if isinstance(detect_status.get("active_job"), dict) else {}
active_threads = max(0, int(detect_status.get("active_thread_count", 0) or 0))
max_threads = max(0, int(detect_status.get("max_thread_count", 0) or 0))
current_load = max(
active_threads,
int(active_job.get("items_running", 0) or 0),
)
phase_label = str(detect_status.get("phase_label") or "").strip()
phase_detail = str(
detect_status.get("phase_detail")
or detect_status.get("recent_event")
or detect_status.get("worker_runtime_message")
or ""
).strip()
inferred_worker_online = bool(
worker_runtime.get("running", False)
or detect_status.get("worker_online", False)
or runtime_state.get("service_running", False)
or current_load > 0
or active_threads > 0
)
return {
"worker_online": inferred_worker_online,
"service_running": bool(
runtime_state.get("service_running", False)
or worker_runtime.get("running", False)
or inferred_worker_online
),
"detecting": bool(
detect_status.get("detecting", False)
or runtime_state.get("detecting", False)
or current_load > 0
or active_threads > 0
),
"active_threads": active_threads,
"max_threads": max_threads,
"current_load": current_load,
"phase_label": phase_label,
"phase_detail": phase_detail,
"recent_warning": str(detect_status.get("recent_warning") or "").strip(),
"updated_at": str(runtime_state.get("updated_at") or "").strip(),
"detect_participating": bool(
detect_status.get("detect_participating", False)
or current_load > 0
or active_threads > 0
),
}
except Exception as exc:
worker_online = bool(worker_runtime.get("running", False))
worker_message = str(worker_runtime.get("message") or "").strip()
return {
"worker_online": worker_online,
"service_running": worker_online,
"detecting": False,
"active_threads": 0,
"max_threads": 0,
"current_load": 0,
"phase_label": "",
"phase_detail": worker_message,
"recent_warning": "",
"updated_at": str(worker_runtime.get("latest_start_time") or "").strip(),
"detect_participating": False,
"error": str(exc),
}
def _base_payload() -> dict:
@@ -651,12 +958,37 @@ def _base_payload() -> dict:
"node_agent": NODE_AGENT_SERVICE_NAME,
},
"delivery_queue": _delivery_queue_snapshot(),
"detect_runtime": _detect_runtime_snapshot(),
},
}
def _run(command: list[str], timeout: int = 60) -> tuple[int, str, str]:
completed = subprocess.run(command, capture_output=True, text=True, timeout=timeout)
normalized_command = [str(part or "").strip() for part in command]
combined_output = f"{completed.stdout or ''}\n{completed.stderr or ''}".lower()
needs_sudo_retry = (
normalized_command
and normalized_command[0] == "systemctl"
and completed.returncode != 0
and "sudo" not in normalized_command
and any(
marker in combined_output
for marker in (
"interactive authentication required",
"authentication is required",
"authorization not available",
"polkit",
)
)
)
if needs_sudo_retry:
completed = subprocess.run(
["sudo", "-n", *normalized_command],
capture_output=True,
text=True,
timeout=timeout,
)
return completed.returncode, completed.stdout.strip(), completed.stderr.strip()
@@ -727,18 +1059,22 @@ def _execute_action(
def _register() -> None:
response = _post("/api/v1/ops/agent/register", _base_payload())
response = _post("/api/v1/ops/agent/register", _base_payload(), timeout=AGENT_REGISTER_TIMEOUT_SECONDS)
_ensure_ok_response(response, "agent register failed")
_log(f"registered: {response.get('message')}")
def _heartbeat() -> None:
response = _post("/api/v1/ops/agent/heartbeat", _base_payload())
response = _post("/api/v1/ops/agent/heartbeat", _base_payload(), timeout=AGENT_HEARTBEAT_TIMEOUT_SECONDS)
_ensure_ok_response(response, "agent heartbeat failed")
def _pull_jobs() -> list[dict]:
response = _post(f"/api/v1/ops/agent/pull?limit=1", {"node_code": NODE_CODE})
response = _post(
f"/api/v1/ops/agent/pull?limit=1",
{"node_code": NODE_CODE},
timeout=AGENT_PULL_TIMEOUT_SECONDS,
)
_ensure_ok_response(response, "agent pull failed")
data = response.get("data") or {}
return list(data.get("jobs") or [])
@@ -847,7 +1183,7 @@ def _job_complete(
path=f"/api/v1/ops/agent/jobs/{job_id}/complete",
payload=payload,
request_id=request_id,
timeout=30,
timeout=AGENT_JOB_COMPLETE_TIMEOUT_SECONDS,
)
@@ -883,7 +1219,7 @@ def _job_event(
path=f"/api/v1/ops/agent/jobs/{job_id}/events",
payload=delivery_payload,
request_id=request_id,
timeout=15,
timeout=AGENT_JOB_EVENT_TIMEOUT_SECONDS,
)
@@ -931,7 +1267,22 @@ def _process_job(job: dict) -> None:
"start_delivery_error": start_delivery_error,
},
)
ok, message, result = _execute_action(action, payload, job_id=job_id, job_context=normalized_job)
try:
ok, message, result = _execute_action(action, payload, job_id=job_id, job_context=normalized_job)
except Exception as exc:
ok = False
message = f"executor exception: {exc}"
result = {
"stdout": "",
"stderr": traceback.format_exc(),
"summary_text": message,
"exception_type": exc.__class__.__name__,
}
_log(
"job execute exception: "
f"id={job_id} code={normalized_job.get('job_code') or '-'} "
f"action={action} error={exc}"
)
stdout = str(result.get("stdout") or "")
stderr = str(result.get("stderr") or "")
duration_ms = max(0, int((time.monotonic() - started_at) * 1000))
@@ -956,6 +1307,57 @@ def _process_job(job: dict) -> None:
)
def _process_job_with_guard(job: dict) -> None:
normalized_job = _normalize_agent_job(job)
job_id = int(normalized_job.get("job_id") or 0)
action = str(normalized_job.get("action") or "").strip()
started_at = time.monotonic()
try:
_process_job(normalized_job)
except Exception as exc:
duration_ms = max(0, int((time.monotonic() - started_at) * 1000))
traceback_text = traceback.format_exc()
summary_text = f"node-agent fatal exception: {exc}"
_log(
"job fatal exception: "
f"id={job_id} code={normalized_job.get('job_code') or '-'} "
f"action={action or '-'} error={exc}"
)
if job_id <= 0:
raise
try:
delivery = _job_complete(
job_id,
status="failed",
stdout="",
stderr=traceback_text,
result={
"stdout": "",
"stderr": traceback_text,
"summary_text": summary_text,
"exception_type": exc.__class__.__name__,
"fatal_loop_exception": True,
},
error_message=summary_text,
duration_ms=duration_ms,
summary_text=summary_text,
focus_ref=dict(normalized_job.get("focus_ref") or {}),
step_ref=dict(normalized_job.get("step_ref") or {}),
release_context=dict(normalized_job.get("release_context") or {}),
)
_log(
"job fatal exception completion: "
f"id={job_id} code={normalized_job.get('job_code') or '-'} "
f"action={action or '-'} delivery={delivery.get('state')}"
)
except Exception as completion_exc:
_log(
"job fatal exception completion failed: "
f"id={job_id} code={normalized_job.get('job_code') or '-'} "
f"action={action or '-'} error={completion_exc}"
)
def main() -> None:
if not NODE_CODE:
raise RuntimeError("NODE_CODE 未配置")
@@ -963,6 +1365,7 @@ def main() -> None:
_ensure_queue_dirs()
_register()
last_heartbeat_at = 0.0
last_runtime_config_sync_at = 0.0
while True:
now = time.time()
@@ -970,13 +1373,17 @@ def main() -> None:
delivery_summary = _flush_delivery_queue(limit=AGENT_QUEUE_FLUSH_LIMIT)
if delivery_summary["delivered"] or delivery_summary["dead_letter"]:
_log(f"delivery queue flush: {delivery_summary}")
if now - last_runtime_config_sync_at >= AGENT_RUNTIME_CONFIG_SYNC_INTERVAL_SECONDS:
bundle = _pull_runtime_config()
_apply_runtime_config(bundle)
last_runtime_config_sync_at = now
if now - last_heartbeat_at >= 15:
_heartbeat()
last_heartbeat_at = now
jobs = _pull_jobs()
if jobs:
for job in jobs:
_process_job(job)
_process_job_with_guard(job)
else:
time.sleep(AGENT_POLL_INTERVAL_SECONDS)
except urllib.error.HTTPError as exc:

View File

@@ -2,10 +2,11 @@ from __future__ import annotations
import json
import socket
import threading
from datetime import datetime, timedelta
from app.core.config import settings
from app.core.db import get_db
from app.core.db import db_read_retry, get_db
_RUNTIME_SCHEMA_SQL = """
@@ -29,6 +30,8 @@ CREATE TABLE IF NOT EXISTS detect_jobs (
job_code VARCHAR(64) NOT NULL UNIQUE,
source VARCHAR(64) NOT NULL DEFAULT 'manual',
plan_hash VARCHAR(128) NOT NULL DEFAULT '',
task_mode VARCHAR(32) NOT NULL DEFAULT 'domain_pipeline',
step_code VARCHAR(64) NOT NULL DEFAULT '',
status VARCHAR(32) NOT NULL DEFAULT 'pending',
remark TEXT NOT NULL DEFAULT '',
created_by VARCHAR(64) NOT NULL DEFAULT '',
@@ -41,6 +44,7 @@ CREATE TABLE IF NOT EXISTS detect_job_items (
id BIGSERIAL PRIMARY KEY,
job_id BIGINT NOT NULL REFERENCES detect_jobs(id) ON DELETE CASCADE,
domain_id BIGINT NOT NULL,
step_code VARCHAR(64) NOT NULL DEFAULT '',
status VARCHAR(32) NOT NULL DEFAULT 'pending',
claimed_by VARCHAR(64) NOT NULL DEFAULT '',
claim_token VARCHAR(64) NOT NULL DEFAULT '',
@@ -48,16 +52,32 @@ CREATE TABLE IF NOT EXISTS detect_job_items (
attempt_count INTEGER NOT NULL DEFAULT 0,
last_error TEXT NOT NULL DEFAULT '',
result_version VARCHAR(64) NOT NULL DEFAULT '',
step_payload_json JSONB,
result_payload_json JSONB,
started_at TIMESTAMP,
finished_at TIMESTAMP,
updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
create_time TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP,
CONSTRAINT uq_detect_job_items_job_domain UNIQUE (job_id, domain_id)
create_time TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP
);
CREATE INDEX IF NOT EXISTS idx_detect_job_items_status_lease
ON detect_job_items(status, lease_expires_at);
ALTER TABLE detect_jobs
ADD COLUMN IF NOT EXISTS task_mode VARCHAR(32) NOT NULL DEFAULT 'domain_pipeline',
ADD COLUMN IF NOT EXISTS step_code VARCHAR(64) NOT NULL DEFAULT '';
ALTER TABLE detect_job_items
ADD COLUMN IF NOT EXISTS step_code VARCHAR(64) NOT NULL DEFAULT '',
ADD COLUMN IF NOT EXISTS step_payload_json JSONB,
ADD COLUMN IF NOT EXISTS result_payload_json JSONB;
ALTER TABLE detect_job_items
DROP CONSTRAINT IF EXISTS uq_detect_job_items_job_domain;
CREATE UNIQUE INDEX IF NOT EXISTS idx_detect_job_items_job_domain_step
ON detect_job_items(job_id, domain_id, step_code);
CREATE TABLE IF NOT EXISTS detect_run_events (
id BIGSERIAL PRIMARY KEY,
job_id BIGINT REFERENCES detect_jobs(id) ON DELETE SET NULL,
@@ -90,6 +110,9 @@ _STALE_AFTER_SECONDS = 90
_OFFLINE_AFTER_MINUTES = 5
_PRUNE_IMPORTED_AFTER_MINUTES = 30
_PRUNE_GENERAL_AFTER_HOURS = 6
_RUNTIME_SCHEMA_READY = False
_RUNTIME_SCHEMA_LOCK = threading.Lock()
_RUNTIME_SCHEMA_ADVISORY_LOCK_ID = 62021001
def _resolve_local_ip() -> str:
@@ -110,12 +133,85 @@ def _decode_json(value: object) -> dict:
return {}
def _control_node_supports_worker(*, region: object, metadata: dict | None) -> bool:
normalized_region = str(region or "").strip()
runtime_metadata = dict(metadata or {})
active_threads = int(runtime_metadata.get("active_threads", 0) or 0)
max_threads = int(runtime_metadata.get("max_threads", 0) or 0)
if normalized_region != "mainland":
return False
return bool(
runtime_metadata.get("worker_online", False)
or runtime_metadata.get("detect_participating", False)
or active_threads > 0
or max_threads > 0
)
def _metadata_idle_without_runtime_work(metadata: dict | None) -> bool:
runtime_metadata = dict(metadata or {})
phase = str(
runtime_metadata.get("phase")
or runtime_metadata.get("phase_label")
or ""
).strip().lower()
if phase not in {"idle", "completed", "stopped"}:
return False
active_job_code = str(runtime_metadata.get("active_job_code") or "").strip()
counters = (
int(runtime_metadata.get("job_items_total", 0) or 0),
int(runtime_metadata.get("job_items_claimed", 0) or 0),
int(runtime_metadata.get("job_items_running", 0) or 0),
int(runtime_metadata.get("job_items_completed", 0) or 0),
int(runtime_metadata.get("job_items_failed", 0) or 0),
)
if active_job_code:
return False
return not any(value > 0 for value in counters)
def _load_managed_node_overlays() -> dict[str, dict]:
try:
with get_db() as conn:
with conn.cursor() as cur:
cur.execute(
"""
SELECT node_code, metadata_json, last_seen_at
FROM ops_managed_nodes
WHERE is_enabled = TRUE
"""
)
rows = cur.fetchall()
except Exception:
return {}
overlays: dict[str, dict] = {}
for row in rows:
node_code = str(row[0] or "").strip()
if not node_code:
continue
overlays[node_code] = {
"metadata": _decode_json(row[1]),
"last_seen_at": row[2],
}
return overlays
def ensure_runtime_schema() -> None:
with get_db() as conn:
conn.autocommit = False
with conn.cursor() as cur:
cur.execute(_RUNTIME_SCHEMA_SQL)
conn.commit()
global _RUNTIME_SCHEMA_READY
if _RUNTIME_SCHEMA_READY:
return
with _RUNTIME_SCHEMA_LOCK:
if _RUNTIME_SCHEMA_READY:
return
with get_db() as conn:
conn.autocommit = False
with conn.cursor() as cur:
cur.execute("SELECT pg_advisory_xact_lock(%s)", (_RUNTIME_SCHEMA_ADVISORY_LOCK_ID,))
cur.execute(_RUNTIME_SCHEMA_SQL)
conn.commit()
_RUNTIME_SCHEMA_READY = True
def register_node_heartbeat(
@@ -195,6 +291,43 @@ def cleanup_imported_runtime_nodes(*, region: str, role: str, keep_node_code: st
conn.commit()
def cleanup_imported_runtime_nodes_many(*, region: str, role: str, keep_node_codes: list[str] | tuple[str, ...] | set[str]) -> None:
normalized_region = str(region or "").strip() or "unknown"
normalized_role = str(role or "").strip() or "unknown"
preserved_node_codes = sorted(
{
str(node_code or "").strip()
for node_code in (keep_node_codes or [])
if str(node_code or "").strip()
}
)
if not preserved_node_codes:
return
placeholders = ", ".join(["%s"] * len(preserved_node_codes))
with get_db() as conn:
with conn.cursor() as cur:
cur.execute(
f"""
DELETE FROM detect_worker_nodes
WHERE region = %s
AND role = %s
AND (
node_code = %s
OR (metadata_json->>'service') = 'runtime-ingest'
)
AND node_code NOT IN ({placeholders})
""",
(
normalized_region,
normalized_role,
f"{normalized_region}-{normalized_role}-imported",
*preserved_node_codes,
),
)
conn.commit()
def prune_expired_runtime_nodes() -> None:
imported_cutoff = datetime.now() - timedelta(minutes=_PRUNE_IMPORTED_AFTER_MINUTES)
general_cutoff = datetime.now() - timedelta(hours=_PRUNE_GENERAL_AFTER_HOURS)
@@ -219,10 +352,12 @@ def prune_expired_runtime_nodes() -> None:
def register_local_control_heartbeat() -> None:
from app.services.detect_job_service import get_active_detect_job_summary
from app.services.detect_service import get_detect_status
from app.services.worker_control_service import detect_worker_runtime
worker_runtime = detect_worker_runtime()
worker_online = bool(worker_runtime.get("running", False))
detect_status = get_detect_status()
active_job = get_active_detect_job_summary(event_limit=5) or {}
node_stats = list(active_job.get("node_stats") or [])
local_bucket = next(
@@ -233,7 +368,9 @@ def register_local_control_heartbeat() -> None:
items_claimed = int(local_bucket.get("items_claimed", 0) or 0)
items_running = int(local_bucket.get("items_running", 0) or 0)
items_completed = int(local_bucket.get("items_completed", 0) or 0)
current_load = max(items_running, items_claimed, 0)
active_threads = int(detect_status.get("active_thread_count", 0) or 0)
max_threads = int(detect_status.get("max_thread_count", 0) or 0)
current_load = max(items_running, active_threads, 0)
detect_participating = bool(worker_online and (items_total > 0 or current_load > 0))
node_status = "busy" if current_load > 0 else "online"
register_node_heartbeat(
@@ -256,6 +393,10 @@ def register_local_control_heartbeat() -> None:
"job_items_claimed": items_claimed,
"job_items_running": items_running,
"job_items_completed": items_completed,
"active_threads": active_threads,
"max_threads": max_threads,
"phase_label": str(detect_status.get("phase_label") or ""),
"phase_detail": str(detect_status.get("phase_detail") or ""),
"updated_at": datetime.now().isoformat(timespec="seconds"),
},
)
@@ -274,9 +415,11 @@ def _normalize_node_status(raw_status: str, last_heartbeat_at: datetime | None)
return status
@db_read_retry()
def get_cluster_snapshot() -> dict:
prune_expired_runtime_nodes()
register_local_control_heartbeat()
managed_overlays = _load_managed_node_overlays()
with get_db() as conn:
with conn.cursor() as cur:
cur.execute(
@@ -297,21 +440,50 @@ def get_cluster_snapshot() -> dict:
cur.execute("SELECT count(*) FROM detect_job_items WHERE status IN ('pending', 'claimed', 'running')")
active_items = cur.fetchone()[0]
nodes = [
{
"node_code": row[0],
"region": row[1],
"role": row[2],
"hostname": row[3],
"ip": row[4],
"status": _normalize_node_status(row[5], row[9]),
"worker_version": row[6],
"current_load": row[7],
"metadata": _decode_json(row[8]),
"last_heartbeat_at": row[9].isoformat(sep=" ", timespec="seconds") if row[9] else "",
}
for row in rows
]
nodes = []
for row in rows:
node_code = str(row[0] or "").strip()
metadata = _decode_json(row[8])
current_load = int(row[7] or 0)
sanitized_idle_runtime = _metadata_idle_without_runtime_work(metadata)
if sanitized_idle_runtime:
current_load = 0
metadata["active_threads"] = 0
metadata["detect_participating"] = False
metadata["sanitized_runtime_state"] = "idle_phase_zeroed"
runtime_last_heartbeat = row[9]
managed_overlay = managed_overlays.get(node_code) or {}
managed_last_seen = managed_overlay.get("last_seen_at")
overlay_is_newer = bool(
managed_last_seen
and (not runtime_last_heartbeat or managed_last_seen > runtime_last_heartbeat)
)
effective_last_heartbeat = managed_last_seen if overlay_is_newer else runtime_last_heartbeat
normalized_status = _normalize_node_status(row[5], effective_last_heartbeat)
if sanitized_idle_runtime and normalized_status == "busy":
normalized_status = "online"
if overlay_is_newer and normalized_status in {"offline", "stale"}:
normalized_status = "busy" if current_load > 0 else "online"
if managed_last_seen:
metadata["agent_last_seen_at"] = managed_last_seen.isoformat(sep=" ", timespec="seconds")
if overlay_is_newer:
metadata["cluster_status_source"] = "managed-agent-overlay"
nodes.append(
{
"node_code": node_code,
"region": row[1],
"role": row[2],
"hostname": row[3],
"ip": row[4],
"status": normalized_status,
"worker_version": row[6],
"current_load": current_load,
"metadata": metadata,
"last_heartbeat_at": effective_last_heartbeat.isoformat(sep=" ", timespec="seconds")
if effective_last_heartbeat
else "",
}
)
status_counts: dict[str, int] = {}
role_counts: dict[str, int] = {}
region_counts: dict[str, int] = {}
@@ -330,6 +502,12 @@ def get_cluster_snapshot() -> dict:
metadata = node.get("metadata") or {}
node_current_load = int(node.get("current_load", 0) or 0)
effective_worker = False
if node_role == "control" and not _control_node_supports_worker(region=node_region, metadata=metadata):
node_current_load = 0
node["current_load"] = 0
if node_status == "busy":
node_status = "online"
node["status"] = "online"
status_counts[node_status] = status_counts.get(node_status, 0) + 1
role_counts[node_role] = role_counts.get(node_role, 0) + 1
@@ -345,14 +523,16 @@ def get_cluster_snapshot() -> dict:
dedicated_online_worker_nodes += 1
effective_worker = True
elif node_role == "control" and node_status in {"online", "busy"}:
if bool(metadata.get("worker_online", False)) or bool(metadata.get("detect_participating", False)) or node_current_load > 0:
if _control_node_supports_worker(region=node_region, metadata=metadata):
effective_worker = True
if effective_worker:
online_worker_nodes += 1
if node_role == "control" and node_status in {"online", "busy"}:
online_control_nodes += 1
node["is_effective_worker"] = effective_worker
node["detect_participating"] = bool(metadata.get("detect_participating", False) or node_current_load > 0)
node["detect_participating"] = bool(
effective_worker and (metadata.get("detect_participating", False) or node_current_load > 0)
)
return {
"nodes": nodes,

View File

@@ -1,10 +1,390 @@
from __future__ import annotations
from app.core.db import get_db
from app.services.detect_job_service import (
_build_step_bucket,
get_active_detect_job_summary,
get_detect_capacity_plan,
get_detect_queue_health,
)
from app.services.runtime_status_service import get_runtime_status
def _empty_active_jobs_aggregate(window_minutes: int) -> dict:
return {
"window_minutes": int(window_minutes or 15),
"active_jobs_total": 0,
"queue": {
"items_total": 0,
"pending": 0,
"claimed": 0,
"running": 0,
"completed": 0,
"blacklisted": 0,
"failed": 0,
"terminal": 0,
},
"throughput": {
"processed_recent": 0,
"processed_per_minute": 0,
"completed_recent": 0,
"blacklisted_recent": 0,
"failed_recent": 0,
},
"steps": [],
"nodes": [],
"retry_total": 0,
}
def _merge_step_queues_with_runtime_activity(
base_steps: list[dict] | None,
*,
runtime_activity: dict | None = None,
window_minutes: int = 15,
limit: int = 8,
) -> list[dict]:
safe_window_minutes = max(1, int(window_minutes or 15))
normalized_limit = max(1, int(limit or 8))
step_map: dict[str, dict] = {}
for item in list(base_steps or []):
step_code = str(item.get("step_code") or "").strip()
if not step_code:
continue
bucket = _build_step_bucket(step_code)
bucket.update(
{
"items_total": int(item.get("items_total", 0) or 0),
"items_pending": int(item.get("items_pending", 0) or 0),
"items_claimed": int(item.get("items_claimed", 0) or 0),
"items_running": int(item.get("items_running", 0) or 0),
"items_completed": int(item.get("items_completed", 0) or 0),
"items_blacklisted": int(item.get("items_blacklisted", 0) or 0),
"items_failed": int(item.get("items_failed", 0) or 0),
"started_recent": int(item.get("started_recent", 0) or 0),
"processed_recent": int(item.get("processed_recent", 0) or 0),
"processed_per_minute": float(item.get("processed_per_minute", 0) or 0),
"completed_recent": int(item.get("completed_recent", 0) or 0),
"blacklisted_recent": int(item.get("blacklisted_recent", 0) or 0),
"failed_recent": int(item.get("failed_recent", 0) or 0),
}
)
step_map[step_code] = bucket
runtime_step_stats = dict((runtime_activity or {}).get("step_stats") or {})
for step_code, stats in runtime_step_stats.items():
normalized_step_code = str(step_code or "").strip()
if not normalized_step_code:
continue
bucket = step_map.setdefault(normalized_step_code, _build_step_bucket(normalized_step_code))
started_recent = int((stats or {}).get("started_recent", 0) or 0)
processed_recent = int((stats or {}).get("processed_recent", 0) or 0)
completed_recent = int((stats or {}).get("completed_recent", 0) or 0)
blacklisted_recent = int((stats or {}).get("blacklisted_recent", 0) or 0)
failed_recent = int((stats or {}).get("failed_recent", 0) or 0)
bucket["started_recent"] = max(int(bucket.get("started_recent", 0) or 0), started_recent)
bucket["processed_recent"] = max(int(bucket.get("processed_recent", 0) or 0), processed_recent)
bucket["completed_recent"] = max(int(bucket.get("completed_recent", 0) or 0), completed_recent)
bucket["blacklisted_recent"] = max(int(bucket.get("blacklisted_recent", 0) or 0), blacklisted_recent)
bucket["failed_recent"] = max(int(bucket.get("failed_recent", 0) or 0), failed_recent)
bucket["processed_per_minute"] = max(
float(bucket.get("processed_per_minute", 0) or 0),
round(processed_recent / safe_window_minutes, 2),
)
return sorted(
step_map.values(),
key=lambda item: (
-int(item.get("items_pending", 0) or 0),
-int(item.get("items_running", 0) or 0),
-int(item.get("started_recent", 0) or 0),
-int(item.get("processed_recent", 0) or 0),
str(item.get("step_code") or ""),
),
)[:normalized_limit]
def _align_active_jobs_aggregate_with_runtime(
aggregate: dict,
*,
runtime: dict,
queue_health: dict,
) -> dict:
normalized = dict(aggregate or {})
node_payload = dict((runtime or {}).get("node") or {})
if str(node_payload.get("region") or "").strip() != "overseas" or str(node_payload.get("role") or "").strip() != "control":
return normalized
backlog = dict(((runtime or {}).get("detect") or {}).get("backlog") or {})
snapshot_backlog = dict(queue_health.get("runtime_snapshot_backlog") or {})
def _backlog_value(key: str) -> int:
return max(int(backlog.get(key, 0) or 0), int(snapshot_backlog.get(key, 0) or 0))
pending_total = _backlog_value("pending_total")
claimed_total = _backlog_value("claimed_total")
running_total = _backlog_value("running_total")
completed_total = _backlog_value("completed_total")
blacklisted_total = _backlog_value("blacklisted_total")
failed_total = _backlog_value("failed_total")
queue = dict(queue_health.get("queue") or {})
throughput = dict(queue_health.get("throughput") or {})
terminal_total = max(
completed_total + blacklisted_total + failed_total,
int(queue.get("completed", 0) or 0) + int(queue.get("blacklisted", 0) or 0) + int(queue.get("failed", 0) or 0),
)
items_total = pending_total + claimed_total + running_total + terminal_total
has_runtime_work = items_total > 0 or bool(queue_health.get("has_active_job"))
normalized["active_jobs_total"] = max(
int(normalized.get("active_jobs_total", 0) or 0),
1 if has_runtime_work else 0,
)
normalized["queue"] = {
"items_total": items_total,
"pending": pending_total,
"claimed": claimed_total,
"running": running_total,
"completed": max(completed_total, int(queue.get("completed", 0) or 0)),
"blacklisted": max(blacklisted_total, int(queue.get("blacklisted", 0) or 0)),
"failed": max(failed_total, int(queue.get("failed", 0) or 0)),
"terminal": terminal_total,
}
normalized["throughput"] = {
"processed_recent": int(throughput.get("processed_recent", 0) or 0),
"processed_per_minute": float(throughput.get("processed_per_minute", 0) or 0),
"completed_recent": int(throughput.get("completed_recent", 0) or 0),
"blacklisted_recent": int(throughput.get("blacklisted_recent", 0) or 0),
"failed_recent": int(throughput.get("failed_recent", 0) or 0),
}
normalized["steps"] = _merge_step_queues_with_runtime_activity(
list(queue_health.get("steps") or []),
runtime_activity=dict(queue_health.get("runtime_activity") or {}),
window_minutes=int(queue_health.get("window_minutes", normalized.get("window_minutes", 15)) or 15),
limit=8,
)
normalized["nodes"] = list(queue_health.get("nodes") or [])
return normalized
def _fetch_active_jobs_aggregate(window_minutes: int = 15) -> dict:
safe_window_minutes = max(5, min(int(window_minutes or 15), 120))
payload = _empty_active_jobs_aggregate(safe_window_minutes)
with get_db() as conn:
with conn.cursor() as cur:
cur.execute(
"""
SELECT id
FROM detect_jobs
WHERE status IN ('pending', 'running', 'partial_failed')
ORDER BY
CASE
WHEN status = 'running' THEN 0
WHEN status = 'pending' THEN 1
WHEN status = 'partial_failed' THEN 2
ELSE 3
END ASC,
COALESCE(started_at, created_at) DESC,
id DESC
"""
)
job_ids = [int(row[0]) for row in cur.fetchall() if row and row[0] is not None]
if not job_ids:
return payload
payload["active_jobs_total"] = len(job_ids)
cur.execute(
"""
SELECT
COUNT(*) AS items_total,
COUNT(*) FILTER (WHERE status = 'pending') AS items_pending,
COUNT(*) FILTER (WHERE status = 'claimed') AS items_claimed,
COUNT(*) FILTER (WHERE status = 'running') AS items_running,
COUNT(*) FILTER (WHERE status = 'completed') AS items_completed,
COUNT(*) FILTER (WHERE status = 'blacklisted') AS items_blacklisted,
COUNT(*) FILTER (WHERE status = 'failed') AS items_failed
FROM detect_job_items
WHERE job_id = ANY(%s)
""",
(job_ids,),
)
queue_row = cur.fetchone() or (0, 0, 0, 0, 0, 0, 0)
payload["queue"] = {
"items_total": int(queue_row[0] or 0),
"pending": int(queue_row[1] or 0),
"claimed": int(queue_row[2] or 0),
"running": int(queue_row[3] or 0),
"completed": int(queue_row[4] or 0),
"blacklisted": int(queue_row[5] or 0),
"failed": int(queue_row[6] or 0),
"terminal": int(queue_row[4] or 0) + int(queue_row[5] or 0) + int(queue_row[6] or 0),
}
cur.execute(
"""
SELECT
COUNT(*) AS processed_recent,
COUNT(*) FILTER (WHERE event_type = 'domain_completed') AS completed_recent,
COUNT(*) FILTER (WHERE event_type = 'domain_blacklisted') AS blacklisted_recent,
COUNT(*) FILTER (WHERE event_type = 'domain_failed') AS failed_recent
FROM detect_run_events
WHERE job_id = ANY(%s)
AND event_type IN ('domain_completed', 'domain_blacklisted', 'domain_failed')
AND created_at >= CURRENT_TIMESTAMP - (%s || ' minutes')::interval
""",
(job_ids, safe_window_minutes),
)
throughput_row = cur.fetchone() or (0, 0, 0, 0)
processed_recent = int(throughput_row[0] or 0)
payload["throughput"] = {
"processed_recent": processed_recent,
"processed_per_minute": round(processed_recent / safe_window_minutes, 2),
"completed_recent": int(throughput_row[1] or 0),
"blacklisted_recent": int(throughput_row[2] or 0),
"failed_recent": int(throughput_row[3] or 0),
}
cur.execute(
"""
SELECT COUNT(*)
FROM detect_job_items
WHERE job_id = ANY(%s)
AND attempt_count > 1
""",
(job_ids,),
)
payload["retry_total"] = int((cur.fetchone() or [0])[0] or 0)
cur.execute(
"""
SELECT
COALESCE(NULLIF(step_code, ''), 'domain_pipeline') AS step_code,
COUNT(*) AS items_total,
COUNT(*) FILTER (WHERE status = 'pending') AS items_pending,
COUNT(*) FILTER (WHERE status = 'claimed') AS items_claimed,
COUNT(*) FILTER (WHERE status = 'running') AS items_running,
COUNT(*) FILTER (WHERE status = 'completed') AS items_completed,
COUNT(*) FILTER (WHERE status = 'blacklisted') AS items_blacklisted,
COUNT(*) FILTER (WHERE status = 'failed') AS items_failed,
COUNT(*) FILTER (
WHERE status IN ('completed', 'blacklisted', 'failed')
AND finished_at >= CURRENT_TIMESTAMP - (%s || ' minutes')::interval
) AS processed_recent
FROM detect_job_items
WHERE job_id = ANY(%s)
GROUP BY COALESCE(NULLIF(step_code, ''), 'domain_pipeline')
ORDER BY
COUNT(*) FILTER (WHERE status = 'pending') DESC,
COUNT(*) FILTER (WHERE status = 'running') DESC,
COUNT(*) DESC,
COALESCE(NULLIF(step_code, ''), 'domain_pipeline') ASC
LIMIT 8
""",
(safe_window_minutes, job_ids),
)
steps: list[dict] = []
for row in cur.fetchall():
step_code = str(row[0] or "domain_pipeline")
bucket = _build_step_bucket(step_code)
bucket.update(
{
"items_total": int(row[1] or 0),
"items_pending": int(row[2] or 0),
"items_claimed": int(row[3] or 0),
"items_running": int(row[4] or 0),
"items_completed": int(row[5] or 0),
"items_blacklisted": int(row[6] or 0),
"items_failed": int(row[7] or 0),
"processed_recent": int(row[8] or 0),
"processed_per_minute": round(int(row[8] or 0) / safe_window_minutes, 2),
}
)
steps.append(bucket)
payload["steps"] = steps
cur.execute(
"""
SELECT
COALESCE(NULLIF(claimed_by, ''), 'unassigned') AS node_code,
COUNT(*) FILTER (WHERE status = 'claimed') AS items_claimed,
COUNT(*) FILTER (WHERE status = 'running') AS items_running,
COUNT(*) FILTER (WHERE status = 'completed') AS items_completed_total
FROM detect_job_items
WHERE job_id = ANY(%s)
GROUP BY COALESCE(NULLIF(claimed_by, ''), 'unassigned')
""",
(job_ids,),
)
node_map = {
str(row[0] or "unassigned"): {
"node_code": str(row[0] or "unassigned"),
"items_running": int(row[2] or 0),
"items_claimed": int(row[1] or 0),
"items_completed": int(row[3] or 0),
"processed_recent": 0,
"processed_per_minute": 0,
"completed_recent": 0,
"failed_recent": 0,
"blacklisted_recent": 0,
}
for row in cur.fetchall()
}
cur.execute(
"""
SELECT
COALESCE(NULLIF(node_code, ''), 'unassigned') AS node_code,
COUNT(*) AS processed_recent,
COUNT(*) FILTER (WHERE event_type = 'domain_completed') AS completed_recent,
COUNT(*) FILTER (WHERE event_type = 'domain_blacklisted') AS blacklisted_recent,
COUNT(*) FILTER (WHERE event_type = 'domain_failed') AS failed_recent
FROM detect_run_events
WHERE job_id = ANY(%s)
AND event_type IN ('domain_completed', 'domain_blacklisted', 'domain_failed')
AND created_at >= CURRENT_TIMESTAMP - (%s || ' minutes')::interval
GROUP BY COALESCE(NULLIF(node_code, ''), 'unassigned')
""",
(job_ids, safe_window_minutes),
)
for row in cur.fetchall():
node_code = str(row[0] or "unassigned")
bucket = node_map.setdefault(
node_code,
{
"node_code": node_code,
"items_running": 0,
"items_claimed": 0,
"items_completed": 0,
"processed_recent": 0,
"processed_per_minute": 0,
"completed_recent": 0,
"failed_recent": 0,
"blacklisted_recent": 0,
},
)
processed_recent = int(row[1] or 0)
bucket["processed_recent"] = processed_recent
bucket["processed_per_minute"] = round(processed_recent / safe_window_minutes, 2)
bucket["completed_recent"] = int(row[2] or 0)
bucket["blacklisted_recent"] = int(row[3] or 0)
bucket["failed_recent"] = int(row[4] or 0)
payload["nodes"] = sorted(
node_map.values(),
key=lambda item: (
-int(item.get("items_running", 0) or 0),
-int(item.get("processed_recent", 0) or 0),
-int(item.get("items_claimed", 0) or 0),
str(item.get("node_code") or ""),
),
)[:8]
return payload
def fetch_overview() -> dict:
window_minutes = 15
queries = {
"domains_total": "select count(*) from domains",
"pending_total": "select count(*) from domains where detect_status = 0",
@@ -12,6 +392,8 @@ def fetch_overview() -> dict:
"running_total": "select count(*) from domains where detect_status = 2",
"blacklist_total": "select count(*) from domains where detect_status = 3",
"failed_total": "select count(*) from domains where detect_status = 4",
"registerable_total": "select count(*) from domains where detect_status = 1 and register_status = 2",
"purchasable_total": "select count(*) from domains where detect_status = 1 and register_status = 2 and coalesce(use_status, 0) = 0",
"sensitive_words_total": "select count(*) from sensitive_words",
}
result: dict[str, int | str] = {}
@@ -23,6 +405,11 @@ def fetch_overview() -> dict:
result[key] = cur.fetchone()[0]
except Exception:
result[key] = 0
active_jobs_aggregate = _fetch_active_jobs_aggregate(window_minutes=window_minutes)
active_job = get_active_detect_job_summary(event_limit=20) or {}
aggregate_queue = active_jobs_aggregate.get("queue") or {}
runtime = get_runtime_status()
cluster_summary = ((runtime.get("cluster") or {}).get("summary") or {})
online_worker_nodes = int(cluster_summary.get("online_worker_nodes", 0) or 0)
@@ -37,4 +424,274 @@ def fetch_overview() -> dict:
result["worker_mode"] = runtime["worker"]["mode"]
result["node_region"] = runtime["node"]["region"]
result["node_role"] = runtime["node"]["role"]
queue_health = get_detect_queue_health(window_minutes=window_minutes)
active_jobs_aggregate = _align_active_jobs_aggregate_with_runtime(
active_jobs_aggregate,
runtime=runtime,
queue_health=queue_health,
)
runtime_snapshot_backlog = dict(queue_health.get("runtime_snapshot_backlog") or {})
aggregate_queue = active_jobs_aggregate.get("queue") or {}
aggregate_queue_health = {
"has_active_job": bool(int(active_jobs_aggregate.get("active_jobs_total", 0) or 0) > 0),
"queue": aggregate_queue,
"throughput": active_jobs_aggregate.get("throughput") or {},
}
selected_queue_health = aggregate_queue_health if aggregate_queue_health["has_active_job"] else queue_health
if float((queue_health.get("throughput") or {}).get("processed_per_minute", 0) or 0) > float(
(selected_queue_health.get("throughput") or {}).get("processed_per_minute", 0) or 0
):
selected_queue_health = queue_health
capacity_plan = get_detect_capacity_plan(
queue_health=selected_queue_health,
online_worker_nodes=online_worker_nodes,
)
retry_total = int(active_jobs_aggregate.get("retry_total", 0) or 0)
step_queue: list[dict] = []
node_throughput: list[dict] = []
bottleneck_step: dict | None = None
active_job_summary: dict | None = None
if queue_health.get("has_active_job"):
job_payload = queue_health.get("job") or {}
queue_payload = queue_health.get("queue") or {}
throughput_payload = queue_health.get("throughput") or {}
runtime_job_code = str(job_payload.get("runtime_job_code") or "").strip()
display_job_code = runtime_job_code or str(job_payload.get("job_code") or "")
active_job_summary = {
"job_id": int(job_payload.get("job_id", 0) or 0),
"job_code": display_job_code,
"db_job_code": str(job_payload.get("job_code") or ""),
"runtime_job_code": runtime_job_code,
"status": str(job_payload.get("status") or ""),
"progress_percent": float(job_payload.get("progress_percent", 0) or 0),
"items_total": int(queue_payload.get("items_total", 0) or 0),
"items_pending": int(queue_payload.get("pending", 0) or 0),
"items_claimed": int(queue_payload.get("display_claimed", queue_payload.get("claimed", 0)) or 0),
"items_running": int(queue_payload.get("running", 0) or 0),
"items_display_running": int(queue_payload.get("display_running", queue_payload.get("running", 0)) or 0),
"items_completed": int(queue_payload.get("completed", 0) or 0),
"items_blacklisted": int(queue_payload.get("blacklisted", 0) or 0),
"items_failed": int(queue_payload.get("failed", 0) or 0),
"processed_per_minute": float(throughput_payload.get("processed_per_minute", 0) or 0),
"processed_recent": int(throughput_payload.get("processed_recent", 0) or 0),
"completed_recent": int(throughput_payload.get("completed_recent", 0) or 0),
"failed_recent": int(throughput_payload.get("failed_recent", 0) or 0),
"blacklisted_recent": int(throughput_payload.get("blacklisted_recent", 0) or 0),
"active_jobs_total": int(active_jobs_aggregate.get("active_jobs_total", 0) or 0),
}
queue_pending_total = 0
queue_claimed_total = 0
queue_running_total = 0
queue_display_running_total = 0
queue_completed_total = 0
queue_blacklist_total = 0
queue_failed_total = 0
backlog_payload = ((runtime.get("detect") or {}).get("backlog") or {})
backlog_pending_total = max(
int(backlog_payload.get("pending_total", 0) or 0),
int(runtime_snapshot_backlog.get("pending_total", 0) or 0),
)
backlog_claimed_total = max(
int(backlog_payload.get("claimed_total", 0) or 0),
int(runtime_snapshot_backlog.get("claimed_total", 0) or 0),
)
backlog_running_total = max(
int(backlog_payload.get("running_total", 0) or 0),
int(runtime_snapshot_backlog.get("running_total", 0) or 0),
)
backlog_register_pending_total = max(
int(backlog_payload.get("register_pending", 0) or 0),
int(runtime_snapshot_backlog.get("register_pending", 0) or 0),
)
backlog_downstream_pending_total = max(
int(backlog_payload.get("downstream_pending", 0) or 0),
int(runtime_snapshot_backlog.get("downstream_pending", 0) or 0),
)
if queue_health.get("has_active_job"):
queue_payload = queue_health.get("queue") or {}
queue_pending_total = int(queue_payload.get("pending", 0) or 0)
queue_claimed_total = int(queue_payload.get("display_claimed", queue_payload.get("claimed", 0)) or 0)
queue_running_total = int(queue_payload.get("running", 0) or 0)
queue_display_running_total = int(queue_payload.get("display_running", queue_payload.get("running", 0)) or 0)
queue_completed_total = int(queue_payload.get("completed", 0) or 0)
queue_blacklist_total = int(queue_payload.get("blacklisted", 0) or 0)
queue_failed_total = int(queue_payload.get("failed", 0) or 0)
elif int(active_jobs_aggregate.get("active_jobs_total", 0) or 0) > 0:
queue_pending_total = int(aggregate_queue.get("pending", 0) or 0)
queue_claimed_total = int(aggregate_queue.get("claimed", 0) or 0)
queue_running_total = int(aggregate_queue.get("running", 0) or 0)
queue_display_running_total = queue_running_total
queue_completed_total = int(aggregate_queue.get("completed", 0) or 0)
queue_blacklist_total = int(aggregate_queue.get("blacklisted", 0) or 0)
queue_failed_total = int(aggregate_queue.get("failed", 0) or 0)
elif active_job:
queue_pending_total = int(active_job.get("items_pending", 0) or 0)
queue_claimed_total = int(active_job.get("items_claimed", 0) or 0)
queue_running_total = int(active_job.get("items_running", 0) or 0)
queue_display_running_total = int(
active_job.get("display_items_running", active_job.get("items_running", 0)) or 0
)
queue_completed_total = int(active_job.get("items_completed", 0) or 0)
queue_blacklist_total = int(active_job.get("items_blacklisted", 0) or 0)
queue_failed_total = int(active_job.get("items_failed", 0) or 0)
aggregate_step_queue = [
{
"step_code": str(item.get("step_code") or ""),
"step_name": str(item.get("step_name") or ""),
"items_pending": int(item.get("items_pending", 0) or 0),
"items_running": int(item.get("items_running", 0) or 0),
"items_claimed": int(item.get("items_claimed", 0) or 0),
"items_completed": int(item.get("items_completed", 0) or 0),
"items_blacklisted": int(item.get("items_blacklisted", 0) or 0),
"items_failed": int(item.get("items_failed", 0) or 0),
"started_recent": int(item.get("started_recent", 0) or 0),
"processed_per_minute": float(item.get("processed_per_minute", 0) or 0),
"processed_recent": int(item.get("processed_recent", 0) or 0),
"completed_recent": int(item.get("completed_recent", 0) or 0),
"blacklisted_recent": int(item.get("blacklisted_recent", 0) or 0),
"failed_recent": int(item.get("failed_recent", 0) or 0),
}
for item in list(active_jobs_aggregate.get("steps") or [])[:8]
]
aggregate_node_throughput = [
{
"node_code": str(item.get("node_code") or ""),
"items_pending": int(item.get("items_pending", 0) or 0),
"items_running": int(item.get("items_running", 0) or 0),
"display_running": int(item.get("display_running", item.get("items_running", 0)) or 0),
"items_claimed": int(item.get("items_claimed", 0) or 0),
"current_load": int(item.get("current_load", item.get("display_running", 0)) or 0),
"active_threads": int(item.get("active_threads", 0) or 0),
"max_threads": int(item.get("max_threads", 0) or 0),
"processed_recent": int(item.get("processed_recent", 0) or 0),
"processed_per_minute": float(item.get("processed_per_minute", 0) or 0),
"completed_recent": int(item.get("completed_recent", 0) or 0),
"failed_recent": int(item.get("failed_recent", 0) or 0),
"blacklisted_recent": int(item.get("blacklisted_recent", 0) or 0),
}
for item in list(active_jobs_aggregate.get("nodes") or [])[:8]
]
queue_step_queue = [
{
"step_code": str(item.get("step_code") or ""),
"step_name": str(item.get("step_name") or ""),
"items_pending": int(item.get("items_pending", 0) or 0),
"items_running": int(item.get("items_running", 0) or 0),
"items_claimed": int(item.get("items_claimed", 0) or 0),
"items_completed": int(item.get("items_completed", 0) or 0),
"items_blacklisted": int(item.get("items_blacklisted", 0) or 0),
"items_failed": int(item.get("items_failed", 0) or 0),
"started_recent": int(item.get("started_recent", 0) or 0),
"processed_per_minute": float(item.get("processed_per_minute", 0) or 0),
"processed_recent": int(item.get("processed_recent", 0) or 0),
"completed_recent": int(item.get("completed_recent", 0) or 0),
"blacklisted_recent": int(item.get("blacklisted_recent", 0) or 0),
"failed_recent": int(item.get("failed_recent", 0) or 0),
}
for item in _merge_step_queues_with_runtime_activity(
list(queue_health.get("steps") or []),
runtime_activity=dict(queue_health.get("runtime_activity") or {}),
window_minutes=window_minutes,
limit=8,
)
]
queue_node_throughput = [
{
"node_code": str(item.get("node_code") or ""),
"items_pending": int(item.get("items_pending", 0) or 0),
"items_running": int(item.get("items_running", 0) or 0),
"display_running": int(item.get("display_running", item.get("items_running", 0)) or 0),
"items_claimed": int(item.get("items_claimed", 0) or 0),
"current_load": int(item.get("current_load", item.get("display_running", 0)) or 0),
"active_threads": int(item.get("active_threads", 0) or 0),
"max_threads": int(item.get("max_threads", 0) or 0),
"processed_recent": int(item.get("processed_recent", 0) or 0),
"processed_per_minute": float(item.get("processed_per_minute", 0) or 0),
"completed_recent": int(item.get("completed_recent", 0) or 0),
"failed_recent": int(item.get("failed_recent", 0) or 0),
"blacklisted_recent": int(item.get("blacklisted_recent", 0) or 0),
}
for item in list(queue_health.get("nodes") or [])[:8]
]
step_queue = aggregate_step_queue
node_throughput = aggregate_node_throughput
aggregate_ppm = float((active_jobs_aggregate.get("throughput") or {}).get("processed_per_minute", 0) or 0)
queue_ppm = float((queue_health.get("throughput") or {}).get("processed_per_minute", 0) or 0)
if queue_health.get("has_active_job") or queue_ppm > aggregate_ppm:
step_queue = queue_step_queue
node_throughput = queue_node_throughput
if step_queue:
bottleneck_step = max(
step_queue,
key=lambda item: (
int(item.get("items_pending", 0) or 0),
int(item.get("items_running", 0) or 0),
-float(item.get("processed_per_minute", 0) or 0),
),
)
result["active_job"] = active_job_summary or {}
result["queue_health"] = queue_health
result["active_jobs_aggregate"] = active_jobs_aggregate
result["capacity_plan"] = capacity_plan
result["step_queue"] = step_queue
result["node_throughput"] = node_throughput
result["retry_total"] = retry_total
result["bottleneck_step"] = bottleneck_step or {}
aggregate_throughput = active_jobs_aggregate.get("throughput") or {}
queue_throughput = queue_health.get("throughput") or {}
ops_processed_per_minute = float(aggregate_throughput.get("processed_per_minute", 0) or 0)
ops_processed_recent = int(aggregate_throughput.get("processed_recent", 0) or 0)
ops_completed_recent = int(aggregate_throughput.get("completed_recent", 0) or 0)
ops_failed_recent = int(aggregate_throughput.get("failed_recent", 0) or 0)
ops_blacklisted_recent = int(aggregate_throughput.get("blacklisted_recent", 0) or 0)
if queue_health.get("has_active_job") or float(queue_throughput.get("processed_per_minute", 0) or 0) > ops_processed_per_minute:
ops_processed_per_minute = float(queue_throughput.get("processed_per_minute", 0) or 0)
ops_processed_recent = int(queue_throughput.get("processed_recent", 0) or 0)
ops_completed_recent = int(queue_throughput.get("completed_recent", 0) or 0)
ops_failed_recent = int(queue_throughput.get("failed_recent", 0) or 0)
ops_blacklisted_recent = int(queue_throughput.get("blacklisted_recent", 0) or 0)
result["ops_summary"] = {
"active_jobs_total": int(active_jobs_aggregate.get("active_jobs_total", 0) or 0),
"processed_per_minute": ops_processed_per_minute,
"processed_recent": ops_processed_recent,
"completed_recent": ops_completed_recent,
"failed_recent": ops_failed_recent,
"blacklisted_recent": ops_blacklisted_recent,
"estimated_hours_remaining": float(capacity_plan.get("estimated_hours_remaining", 0) or 0),
"remaining_items": int(capacity_plan.get("remaining_items", 0) or 0),
"recommended_additional_workers": int(capacity_plan.get("recommended_additional_workers", 0) or 0),
"online_worker_nodes": online_worker_nodes,
"dedicated_online_worker_nodes": dedicated_online_worker_nodes,
"active_execution_nodes": sum(
1
for item in node_throughput
if int(item.get("items_running", 0) or 0) > 0
or int(item.get("items_claimed", 0) or 0) > 0
or int(item.get("processed_recent", 0) or 0) > 0
),
}
result["processed_per_minute"] = ops_processed_per_minute
result["processed_recent"] = ops_processed_recent
result["completed_recent"] = ops_completed_recent
result["failed_recent"] = ops_failed_recent
result["blacklisted_recent"] = ops_blacklisted_recent
result["active_execution_nodes"] = int(result["ops_summary"]["active_execution_nodes"] or 0)
result["queue_pending_total"] = queue_pending_total
result["queue_claimed_total"] = queue_claimed_total
result["queue_running_total"] = queue_running_total
result["queue_display_running_total"] = max(queue_display_running_total, queue_running_total)
result["queue_completed_total"] = queue_completed_total
result["queue_blacklist_total"] = queue_blacklist_total
result["queue_failed_total"] = queue_failed_total
result["backlog_pending_total"] = max(backlog_pending_total, queue_pending_total)
result["backlog_claimed_total"] = max(backlog_claimed_total, queue_claimed_total)
result["backlog_running_total"] = max(backlog_running_total, queue_running_total)
result["backlog_register_pending_total"] = backlog_register_pending_total
result["backlog_downstream_pending_total"] = backlog_downstream_pending_total
return result

View File

@@ -2,12 +2,13 @@ from __future__ import annotations
import json
import socket
import threading
import urllib.error
import urllib.request
from datetime import datetime, timedelta
from app.core.config import settings
from app.core.db import get_db
from app.core.db import db_read_retry, get_db
_DEBUG_SCHEMA_SQL = """
@@ -27,6 +28,9 @@ CREATE INDEX IF NOT EXISTS idx_detect_debug_events_created
ON detect_debug_events(created_at DESC);
"""
_DEBUG_SCHEMA_READY = False
_DEBUG_SCHEMA_LOCK = threading.Lock()
def _format_time(value: datetime | None) -> str:
return value.isoformat(sep=" ", timespec="seconds") if value else ""
@@ -82,11 +86,18 @@ def _debug_ingest_url(base_url: str) -> str:
def ensure_debug_event_schema() -> None:
with get_db() as conn:
conn.autocommit = False
with conn.cursor() as cur:
cur.execute(_DEBUG_SCHEMA_SQL)
conn.commit()
global _DEBUG_SCHEMA_READY
if _DEBUG_SCHEMA_READY:
return
with _DEBUG_SCHEMA_LOCK:
if _DEBUG_SCHEMA_READY:
return
with get_db() as conn:
conn.autocommit = False
with conn.cursor() as cur:
cur.execute(_DEBUG_SCHEMA_SQL)
conn.commit()
_DEBUG_SCHEMA_READY = True
def append_debug_event(
@@ -159,12 +170,185 @@ def append_debug_event(
return record_id
def _load_debug_event_record(record_id: int) -> dict | None:
with get_db() as conn:
with conn.cursor() as cur:
cur.execute(
"""
SELECT id, source_region, node_code, service, event_type, level, message, payload_json, created_at
FROM detect_debug_events
WHERE id = %s
LIMIT 1
""",
(int(record_id),),
)
row = cur.fetchone()
if not row:
return None
return {
"id": int(row[0]),
"source_region": str(row[1] or ""),
"node_code": str(row[2] or ""),
"service": str(row[3] or ""),
"event_type": str(row[4] or ""),
"level": str(row[5] or "info"),
"message": str(row[6] or ""),
"payload": row[7] if isinstance(row[7], dict) else {},
"created_at": _format_time(row[8]),
}
def _normalize_worker_log_event(debug_event: dict) -> dict | None:
payload = dict(debug_event.get("payload") or {})
message = _normalize_message(debug_event.get("message"), fallback="worker_log")
domain = str(payload.get("domain") or "").strip().lower()
status = str(payload.get("status") or "").strip().lower()
if not domain and ":" in message:
domain = message.rsplit(":", 1)[-1].strip().lower()
event_type = ""
if status == "completed":
event_type = "domain_completed"
elif status == "failed":
event_type = "domain_failed"
elif status == "blacklisted":
event_type = "domain_blacklisted"
elif "开始检测域名" in message:
event_type = "domain_started"
elif "域名检测完成" in message:
event_type = "domain_completed"
payload.setdefault("status", "completed")
elif "域名已命中黑名单" in message or "命中黑名单" in message:
event_type = "domain_blacklisted"
payload.setdefault("status", "blacklisted")
elif "域名检测失败" in message:
event_type = "domain_failed"
payload.setdefault("status", "failed")
if not event_type or not domain:
return None
payload.setdefault("domain", domain)
payload["imported_from_debug_event"] = True
payload["debug_event_record_id"] = int(debug_event.get("id") or 0)
payload["debug_event_source_region"] = str(debug_event.get("source_region") or "")
return {
"node_code": str(debug_event.get("node_code") or "").strip(),
"event_type": event_type,
"level": str(debug_event.get("level") or "info"),
"message": message,
"payload": payload,
"created_at": str(debug_event.get("created_at") or "").strip(),
}
def _ingest_worker_log_into_active_job(debug_event: dict) -> dict:
if str(debug_event.get("event_type") or "").strip() != "worker_log":
return {"imported": False, "reason": "not_worker_log"}
normalized_event = _normalize_worker_log_event(debug_event)
if not normalized_event:
return {"imported": False, "reason": "not_domain_progress_event"}
from app.services.detect_job_service import get_active_detect_job_summary
from app.services.sync_push_service import (
_apply_detect_result_event_to_domain,
_apply_detect_result_event_to_job_item,
)
active_job = get_active_detect_job_summary(event_limit=1) or {}
target_job_id = int(active_job.get("job_id") or 0)
if target_job_id <= 0:
return {"imported": False, "reason": "no_active_job"}
debug_event_record_id = int(debug_event.get("id") or 0)
updated_job_items = 0
with get_db() as conn:
with conn.cursor() as cur:
cur.execute(
"""
SELECT id
FROM detect_run_events
WHERE job_id = %s
AND (payload_json->>'debug_event_record_id') = %s
ORDER BY id DESC
LIMIT 1
""",
(target_job_id, str(debug_event_record_id)),
)
existing = cur.fetchone()
if existing:
return {
"imported": False,
"reason": "deduplicated",
"target_job_id": target_job_id,
"detect_run_event_id": int(existing[0]),
}
created_at = _parse_time(normalized_event.get("created_at"))
payload_json = _safe_json_dumps(normalized_event.get("payload") or {})
if created_at:
cur.execute(
"""
INSERT INTO detect_run_events (
job_id, node_code, event_type, level, message, payload_json, created_at
) VALUES (%s, %s, %s, %s, %s, %s::jsonb, %s)
RETURNING id
""",
(
target_job_id,
normalized_event["node_code"],
normalized_event["event_type"],
normalized_event["level"],
normalized_event["message"],
payload_json,
created_at,
),
)
else:
cur.execute(
"""
INSERT INTO detect_run_events (
job_id, node_code, event_type, level, message, payload_json
) VALUES (%s, %s, %s, %s, %s, %s::jsonb)
RETURNING id
""",
(
target_job_id,
normalized_event["node_code"],
normalized_event["event_type"],
normalized_event["level"],
normalized_event["message"],
payload_json,
),
)
detect_run_event_id = int((cur.fetchone() or [0])[0] or 0)
_apply_detect_result_event_to_domain(cur, normalized_event)
updated_job_items = _apply_detect_result_event_to_job_item(
cur,
target_job_id=target_job_id,
event=normalized_event,
)
conn.commit()
return {
"imported": True,
"reason": "imported",
"target_job_id": target_job_id,
"detect_run_event_id": detect_run_event_id,
"updated_job_items": updated_job_items,
"event_type": normalized_event["event_type"],
"domain": str((normalized_event.get("payload") or {}).get("domain") or ""),
}
@db_read_retry()
def list_debug_events(
limit: int = 50,
*,
service: str | None = None,
event_type: str | None = None,
source_region: str | None = None,
node_code: str | None = None,
level: str | None = None,
before_id: int | None = None,
after_id: int | None = None,
@@ -183,6 +367,9 @@ def list_debug_events(
if str(source_region or "").strip():
conditions.append("source_region = %s")
params.append(str(source_region).strip())
if str(node_code or "").strip():
conditions.append("node_code = %s")
params.append(str(node_code).strip())
if str(level or "").strip():
conditions.append("level = %s")
params.append(str(level).strip())
@@ -237,6 +424,7 @@ def list_debug_events(
}
@db_read_retry()
def get_debug_event_overview(*, window_minutes: int = 10, source_region: str | None = None) -> dict:
ensure_debug_event_schema()
safe_window = max(1, min(int(window_minutes or 10), 180))
@@ -671,7 +859,24 @@ def ingest_debug_event(payload: dict, *, shared_token: str | None = None) -> tup
message=_normalize_message(payload.get("message"), fallback="remote debug event"),
payload=payload.get("payload") or {},
)
return True, "调试事件接收成功", {"record_id": record_id}
debug_event = _load_debug_event_record(record_id) or {
"id": int(record_id),
"source_region": str(payload.get("source_region") or settings.node_region),
"node_code": str(payload.get("node_code") or ""),
"service": str(payload.get("service") or "unknown"),
"event_type": str(payload.get("event_type") or "event"),
"level": str(payload.get("level") or "info"),
"message": _normalize_message(payload.get("message"), fallback="remote debug event"),
"payload": payload.get("payload") or {},
"created_at": "",
}
job_import = {}
if str(debug_event.get("service") or "").strip() == "worker-event":
try:
job_import = _ingest_worker_log_into_active_job(debug_event)
except Exception as exc:
job_import = {"imported": False, "reason": f"job_import_failed: {exc}"}
return True, "调试事件接收成功", {"record_id": record_id, "job_import": job_import}
def push_debug_event(

File diff suppressed because it is too large Load Diff

View File

@@ -2,10 +2,14 @@ from __future__ import annotations
import json
import re
from datetime import datetime, timezone
import subprocess
from datetime import datetime, timedelta, timezone
from app.core.config import settings
from app.core.db import get_db
from app.core.files import resolve_domain_path, tail_lines
from app.core.redis_client import get_redis
from app.services.debug_event_service import list_debug_events
from app.services.cluster_runtime_service import ensure_runtime_schema
from app.services.runtime_settings_service import get_runtime_settings
from app.services.detect_run_service import sync_detect_runs
from app.services.detect_job_service import get_active_detect_job_summary
@@ -16,9 +20,79 @@ from app.services.worker_control_service import detect_worker_runtime
_PROXY_COUNT_RE = re.compile(r"当前可用代理数[:]\s*(\d+)")
_THREAD_COUNT_RE = re.compile(r"当前实际线程数量[:]\s*(\d+)\s*/\s*(\d+)")
_STEP_TRACE_DOMAIN_RE = re.compile(r"domain=([^\s|]+)")
_REGISTER_DOMAIN_RE = re.compile(r"检测注册状态[:]\s*([^\s]+)")
_RUNTIME_STATE_KEY = "domain_tool:detect_runtime_state"
_TIMESTAMP_FORMATS = ("%Y-%m-%d %H:%M:%S.%f", "%Y-%m-%d %H:%M:%S")
_SYSLOG_TIMESTAMP_FORMAT = "%b %d %H:%M:%S"
_REMOTE_LOG_MAX_CHARS = 500
_REMOTE_DEBUG_EVENT_TYPES = {
"worker_log",
"active_job_snapshot",
"domain_started",
"domain_completed",
"domain_failed",
"domain_blacklisted",
"task_pull_success",
"task_pull_partial",
"task_pull_failed",
"queue_overdue_leases",
}
def _extract_remote_log_node_code(line: str) -> str:
text = str(line or "").strip()
if not text.startswith("["):
return ""
first_close = text.find("]")
if first_close < 0:
return ""
second_open = text.find("[", first_close + 1)
second_close = text.find("]", second_open + 1) if second_open >= 0 else -1
if second_open < 0 or second_close < 0:
return ""
return text[second_open + 1:second_close].strip()
def _slice_remote_log_lines_fairly(lines: list[str], *, limit: int = 240, min_per_node: int = 12) -> list[str]:
safe_limit = max(1, int(limit or 240))
if len(lines) <= safe_limit:
return list(lines or [])
normalized_lines = [str(line or "").strip() for line in list(lines or []) if str(line or "").strip()]
if len(normalized_lines) <= safe_limit:
return normalized_lines
if min_per_node <= 0:
return normalized_lines[-safe_limit:]
kept_indexes: set[int] = set()
per_node_counts: dict[str, int] = {}
for index in range(len(normalized_lines) - 1, -1, -1):
node_code = _extract_remote_log_node_code(normalized_lines[index])
if not node_code:
continue
current_count = int(per_node_counts.get(node_code, 0) or 0)
if current_count >= min_per_node:
continue
kept_indexes.add(index)
per_node_counts[node_code] = current_count + 1
if len(kept_indexes) >= safe_limit:
break
for index in range(len(normalized_lines) - 1, -1, -1):
if len(kept_indexes) >= safe_limit:
break
kept_indexes.add(index)
return [normalized_lines[index] for index in sorted(kept_indexes)]
def _runtime_state_key(node_code: str | None = None) -> str:
normalized_node_code = str(node_code or settings.node_code or "").strip()
if not normalized_node_code:
return _RUNTIME_STATE_KEY
return f"{_RUNTIME_STATE_KEY}:{normalized_node_code}"
def _extract_dependency_alerts(lines: list[str]) -> list[dict]:
@@ -83,6 +157,26 @@ def _extract_active_thread_snapshot(lines: list[str]) -> dict:
return {"active": 0, "max": 0}
def _estimate_active_threads_from_recent_lines(lines: list[str], *, limit: int) -> int:
if not lines:
return 0
active_domains: list[str] = []
seen_domains: set[str] = set()
for line in reversed(lines[-80:]):
match = _STEP_TRACE_DOMAIN_RE.search(line) or _REGISTER_DOMAIN_RE.search(line)
if not match:
continue
domain = str(match.group(1) or "").strip()
if not domain or domain in seen_domains:
continue
seen_domains.add(domain)
active_domains.append(domain)
if len(active_domains) >= int(limit):
break
return len(active_domains)
def _parse_time(raw: str | None) -> datetime | None:
if not raw:
return None
@@ -101,10 +195,12 @@ def _parse_time(raw: str | None) -> datetime | None:
return None
def _extract_log_time(line: str) -> datetime | None:
def _extract_log_time(line: str, *, reference_year: int | None = None) -> datetime | None:
if len(line) < 19:
return None
candidates = [line[:26], line[:19]]
text = str(line or "").strip()
else:
text = str(line or "")
candidates = [text[:26], text[:19]]
for candidate in candidates:
for fmt in _TIMESTAMP_FORMATS:
if len(candidate) != len(datetime.now().strftime(fmt)):
@@ -113,14 +209,79 @@ def _extract_log_time(line: str) -> datetime | None:
return datetime.strptime(candidate, fmt)
except ValueError:
continue
syslog_candidate = str(text[:15] or "").strip()
if syslog_candidate:
try:
parsed = datetime.strptime(syslog_candidate, _SYSLOG_TIMESTAMP_FORMAT)
return parsed.replace(year=int(reference_year or datetime.now().year))
except ValueError:
pass
return None
def _read_worker_journal_lines(service_name: str, *, max_lines: int) -> tuple[list[str], str | None]:
normalized_service_name = str(service_name or "").strip()
if not normalized_service_name:
return [], None
try:
completed = subprocess.run(
["journalctl", "-u", normalized_service_name, "-n", str(max_lines), "--no-pager"],
capture_output=True,
text=True,
timeout=12,
)
except Exception:
return [], None
output = str(completed.stdout or "").strip()
if completed.returncode != 0 or not output:
return [], None
lines = [str(line or "").rstrip() for line in output.splitlines() if str(line or "").strip()]
if not lines:
return [], None
return lines[-max_lines:], datetime.now(timezone.utc).isoformat()
def _load_recent_worker_lines(runtime_settings: dict, *, max_lines: int = 160) -> tuple[bool, str | None, list[str]]:
worker_log = resolve_domain_path("detect_worker.log", "logs/detect_worker.log")
worker_online = False
last_log_time: str | None = None
recent_lines = tail_lines("detect_worker.log", max_lines=max_lines)
if worker_log and worker_log.exists():
modified = datetime.fromtimestamp(worker_log.stat().st_mtime, tz=timezone.utc)
last_log_time = modified.isoformat()
worker_online = (datetime.now(timezone.utc) - modified).total_seconds() < 180
if str(runtime_settings.get("worker_mode") or "").strip() == "linux-systemd":
service_name = str(runtime_settings.get("worker_service_name") or "").strip() or "domaincheck-worker"
journal_lines, journal_last_time = _read_worker_journal_lines(service_name, max_lines=max_lines)
if journal_lines:
recent_lines = journal_lines
worker_online = True
if journal_last_time:
last_log_time = journal_last_time
return worker_online, last_log_time, recent_lines
def _filter_lines_since(lines: list[str], started_at: str | None) -> list[str]:
started_time = _parse_time(started_at)
if not started_time:
return lines
filtered = [line for line in lines if (_extract_log_time(line) or started_time) >= started_time]
filtered: list[str] = []
parsed_any = False
for line in lines:
line_time = _extract_log_time(line, reference_year=started_time.year)
if line_time is None:
continue
parsed_any = True
if line_time >= started_time:
filtered.append(line)
if not parsed_any:
return lines
return filtered or lines
@@ -258,6 +419,204 @@ def _build_remote_log_snapshot(
}
def _build_remote_log_snapshot_from_debug_events(
active_job: dict | None,
*,
enabled: bool,
mode: str,
limit: int = 240,
) -> dict:
if not enabled:
return {
"lines": [],
"line_count": 0,
"last_at": "",
"last_line": "",
"source_nodes": [],
"source_node_count": 0,
"source_node_summaries": [],
}
normalized_mode = str(mode or "key").strip().lower()
if normalized_mode not in {"key", "full"}:
normalized_mode = "key"
participating_node_codes = {
str(item.get("node_code") or "").strip()
for item in list((active_job or {}).get("node_stats") or [])
if str(item.get("node_code") or "").strip() and str(item.get("node_code") or "").strip() != "unassigned"
}
created_after = (datetime.now() - timedelta(hours=6)).strftime("%Y-%m-%d %H:%M:%S")
safe_limit = max(1, int(limit or 240))
node_limit = max(40, min(200, safe_limit))
records: list[dict] = []
if participating_node_codes:
for node_code in sorted(participating_node_codes):
payload = list_debug_events(
limit=node_limit,
created_after=created_after,
node_code=node_code,
)
records.extend(list(payload.get("records") or []))
records.sort(
key=lambda item: (
str(item.get("created_at") or ""),
int(item.get("id") or 0),
),
reverse=True,
)
else:
payload = list_debug_events(limit=max(safe_limit * 4, 240), created_after=created_after)
records = list(payload.get("records") or [])
if not records:
return {
"lines": [],
"line_count": 0,
"last_at": "",
"last_line": "",
"source_nodes": [],
"source_node_count": 0,
"source_node_summaries": [],
}
lines: list[str] = []
source_nodes: set[str] = set()
source_node_summaries: dict[str, dict] = {}
last_at = ""
last_line = ""
for record in reversed(records):
event_type = str(record.get("event_type") or "").strip()
if event_type not in _REMOTE_DEBUG_EVENT_TYPES:
continue
node_code = str(record.get("node_code") or "").strip() or "unknown"
if participating_node_codes and node_code not in participating_node_codes:
continue
message = str(record.get("message") or "").strip()
if not message:
continue
created_at = str(record.get("created_at") or "").strip()
payload = record.get("payload") if isinstance(record.get("payload"), dict) else {}
event_mode = str(payload.get("log_mode") or "key").strip().lower()
if event_mode not in {"key", "full"}:
event_mode = "key"
if normalized_mode != "full" and event_mode == "full":
continue
if len(message) > _REMOTE_LOG_MAX_CHARS:
message = f"{message[:_REMOTE_LOG_MAX_CHARS]}..."
formatted_line = f"[{created_at}] [{node_code}] {message}"
lines.append(formatted_line)
source_nodes.add(node_code)
node_summary = source_node_summaries.setdefault(
node_code,
{
"node_code": node_code,
"line_count": 0,
"key_line_count": 0,
"full_line_count": 0,
"last_at": "",
"last_line": "",
},
)
node_summary["line_count"] += 1
if event_mode == "full":
node_summary["full_line_count"] += 1
else:
node_summary["key_line_count"] += 1
node_summary["last_at"] = created_at
node_summary["last_line"] = formatted_line
last_at = created_at
last_line = formatted_line
sliced_lines = _slice_remote_log_lines_fairly(lines, limit=safe_limit)
sorted_source_node_summaries = sorted(
source_node_summaries.values(),
key=lambda item: (
str(item.get("last_at") or ""),
str(item.get("node_code") or ""),
),
reverse=True,
)
return {
"lines": sliced_lines,
"line_count": len(sliced_lines),
"last_at": last_at,
"last_line": last_line,
"source_nodes": sorted(source_nodes),
"source_node_count": len(source_nodes),
"source_node_summaries": sorted_source_node_summaries,
}
def _merge_remote_log_snapshots(primary: dict, secondary: dict, *, limit: int = 240) -> dict:
merged_lines: list[str] = []
seen_lines: set[str] = set()
for raw_line in list(primary.get("lines") or []) + list(secondary.get("lines") or []):
line = str(raw_line or "").strip()
if not line or line in seen_lines:
continue
seen_lines.add(line)
merged_lines.append(line)
if limit > 0:
merged_lines = _slice_remote_log_lines_fairly(merged_lines, limit=limit)
summaries: dict[str, dict] = {}
for snapshot in (primary, secondary):
for raw_summary in list(snapshot.get("source_node_summaries") or []):
if not isinstance(raw_summary, dict):
continue
node_code = str(raw_summary.get("node_code") or "").strip()
if not node_code:
continue
summary = summaries.setdefault(
node_code,
{
"node_code": node_code,
"line_count": 0,
"key_line_count": 0,
"full_line_count": 0,
"last_at": "",
"last_line": "",
},
)
summary["line_count"] = max(int(summary.get("line_count", 0) or 0), int(raw_summary.get("line_count", 0) or 0))
summary["key_line_count"] = max(int(summary.get("key_line_count", 0) or 0), int(raw_summary.get("key_line_count", 0) or 0))
summary["full_line_count"] = max(int(summary.get("full_line_count", 0) or 0), int(raw_summary.get("full_line_count", 0) or 0))
raw_last_at = str(raw_summary.get("last_at") or "")
if raw_last_at >= str(summary.get("last_at") or ""):
summary["last_at"] = raw_last_at
summary["last_line"] = str(raw_summary.get("last_line") or "")
source_nodes = sorted(
{
str(node_code or "").strip()
for node_code in list(primary.get("source_nodes") or []) + list(secondary.get("source_nodes") or [])
if str(node_code or "").strip()
}
)
last_at = max(str(primary.get("last_at") or ""), str(secondary.get("last_at") or ""))
last_line = str(primary.get("last_line") or "")
if str(secondary.get("last_at") or "") >= str(primary.get("last_at") or ""):
last_line = str(secondary.get("last_line") or last_line)
return {
"lines": merged_lines,
"line_count": len(merged_lines),
"last_at": last_at,
"last_line": last_line,
"source_nodes": source_nodes,
"source_node_count": len(source_nodes),
"source_node_summaries": sorted(
summaries.values(),
key=lambda item: (
str(item.get("last_at") or ""),
str(item.get("node_code") or ""),
),
reverse=True,
),
}
def _resolve_remote_log_lines(
active_job: dict | None,
runs: list[dict],
@@ -277,17 +636,72 @@ def _resolve_remote_log_snapshot(
mode: str,
limit: int = 240,
) -> dict:
return _build_remote_log_snapshot(active_job, enabled=enabled, mode=mode, limit=limit)
primary_snapshot = _build_remote_log_snapshot(active_job, enabled=enabled, mode=mode, limit=limit)
debug_snapshot = _build_remote_log_snapshot_from_debug_events(active_job, enabled=enabled, mode=mode, limit=limit)
if int(primary_snapshot.get("line_count", 0) or 0) <= 0:
return debug_snapshot
if int(debug_snapshot.get("line_count", 0) or 0) <= 0:
return primary_snapshot
return _merge_remote_log_snapshots(primary_snapshot, debug_snapshot, limit=limit)
def _load_runtime_state() -> dict:
try:
redis_client = get_redis()
raw = redis_client.get(_RUNTIME_STATE_KEY)
if not raw:
for key in (_runtime_state_key(), _RUNTIME_STATE_KEY):
raw = redis_client.get(key)
if not raw:
continue
data = json.loads(raw)
if not isinstance(data, dict):
continue
if key == _RUNTIME_STATE_KEY:
payload_node_code = str(data.get("node_code") or "").strip()
if payload_node_code and payload_node_code != str(settings.node_code or "").strip():
continue
return data
return {}
except Exception:
return {}
def _load_runtime_state_from_cluster_node() -> dict:
try:
with get_db() as conn:
with conn.cursor() as cur:
cur.execute(
"""
SELECT current_load, metadata_json, last_heartbeat_at
FROM detect_worker_nodes
WHERE node_code = %s
LIMIT 1
""",
(settings.node_code,),
)
row = cur.fetchone()
if not row:
return {}
data = json.loads(raw)
return data if isinstance(data, dict) else {}
current_load, metadata_json, last_heartbeat_at = row
metadata = metadata_json if isinstance(metadata_json, dict) else {}
if not metadata:
return {}
runtime_state = {
"node_code": settings.node_code,
"phase": str(metadata.get("phase") or metadata.get("phase_label") or "").strip(),
"detail": str(metadata.get("detail") or metadata.get("phase_detail") or "").strip(),
"service_running": True,
"detecting": bool(metadata.get("detecting", False) or int(current_load or 0) > 0),
"stop_requested": False,
"available_proxy_count": int(metadata.get("available_proxy_count", 0) or 0),
"active_threads": int(metadata.get("active_threads", 0) or 0),
"max_threads": int(metadata.get("max_threads", 0) or 0),
"job_id": metadata.get("job_id"),
"job_code": str(metadata.get("job_code") or metadata.get("active_job_code") or "").strip(),
"updated_at": str(metadata.get("updated_at") or (_format_time(last_heartbeat_at) if last_heartbeat_at else "")).strip(),
}
if runtime_state["detail"] or runtime_state["active_threads"] > 0 or runtime_state["max_threads"] > 0:
return runtime_state
return {}
except Exception:
return {}
@@ -361,12 +775,51 @@ def _build_proxy_runtime_snapshot(settings_payload: dict, runtime_state: dict, a
"supplier_empty": False,
}
if refresh_status in {"", "未刷新"} and source_count > 0:
if allow_direct:
return {
"state": "degraded_direct",
"label": "等待首刷",
"detail": f"代理配置已下发,但代理池尚未完成首轮刷新;当前先按直连继续执行;最近状态:{refresh_status or '未刷新'}",
"direct_fallback_active": True,
"reason": "proxy_not_refreshed_yet",
"last_refresh_status": refresh_status or "未刷新",
"last_refresh_time": refresh_time,
"source_count": source_count,
"raw_items": raw_items,
"validated_count": validated,
"available_count": available,
"source_stats": source_stats,
"supplier_empty": False,
}
return {
"state": "warming_up",
"label": "等待首刷",
"detail": "代理配置已下发,但代理池尚未完成首轮刷新;由于未允许直连,检测链路会等待代理刷新完成",
"direct_fallback_active": False,
"reason": "proxy_not_refreshed_yet",
"last_refresh_status": refresh_status or "未刷新",
"last_refresh_time": refresh_time,
"source_count": source_count,
"raw_items": raw_items,
"validated_count": validated,
"available_count": available,
"source_stats": source_stats,
"supplier_empty": False,
}
if allow_direct:
detail = "代理池当前无可用代理,已自动降级为直连继续执行"
reason = "no_available_proxy"
if supplier_empty:
reason = "supplier_empty_pool"
detail = "代理源最近都返回正常响应,但原始代理数为 0当前判断为供应池为空系统已自动降级为直连继续执行"
elif raw_items > 0 and validated > 0:
reason = "proxy_validation_zero"
detail = (
f"代理源最近返回了 {raw_items} 个代理,已验证 {validated} 个,但当前 0 个可用;"
"系统已自动降级为直连继续执行"
)
if refresh_status:
detail = f"{detail};最近状态:{refresh_status}"
return {
@@ -390,6 +843,12 @@ def _build_proxy_runtime_snapshot(settings_payload: dict, runtime_state: dict, a
if supplier_empty:
reason = "supplier_empty_pool"
detail = "代理源最近都返回正常响应,但原始代理数为 0当前判断为供应池为空由于未允许直连检测链路会等待代理恢复"
elif raw_items > 0 and validated > 0:
reason = "proxy_validation_zero"
detail = (
f"代理源最近返回了 {raw_items} 个代理,已验证 {validated} 个,但当前 0 个可用;"
"由于未允许直连,检测链路会等待代理恢复"
)
if refresh_status:
detail = f"{detail};最近状态:{refresh_status}"
return {
@@ -410,36 +869,45 @@ def _build_proxy_runtime_snapshot(settings_payload: dict, runtime_state: dict, a
def get_detect_status() -> dict:
try:
ensure_runtime_schema()
except Exception:
# Node agent heartbeats should degrade gracefully even if runtime schema
# initialization is temporarily unavailable.
pass
queries = {
"pending": "select count(*) from domains where detect_status = 0",
"completed": "select count(*) from domains where detect_status = 1",
"running": "select count(*) from domains where detect_status = 2",
"blacklisted": "select count(*) from domains where detect_status = 3",
"failed": "select count(*) from domains where detect_status = 4",
"registerable": "select count(*) from domains where detect_status = 1 and register_status = 2",
}
progress: dict[str, int] = {}
with get_db() as conn:
with conn.cursor() as cur:
for key, query in queries.items():
try:
cur.execute(query)
progress[key] = cur.fetchone()[0]
except Exception:
progress[key] = 0
progress: dict[str, int] = {key: 0 for key in queries}
try:
with get_db() as conn:
with conn.cursor() as cur:
for key, query in queries.items():
try:
cur.execute(query)
progress[key] = cur.fetchone()[0]
except Exception:
progress[key] = 0
except Exception:
# Worker runtime/status pages should still render using runtime-state and
# cluster fallbacks even when the local DB endpoint is temporarily wrong
# or unreachable (for example remote worker nodes without direct DB access).
pass
settings_payload = get_settings_payload()
worker_log = resolve_domain_path("detect_worker.log", "logs/detect_worker.log")
worker_online = False
last_log_time = None
if worker_log and worker_log.exists():
modified = datetime.fromtimestamp(worker_log.stat().st_mtime, tz=timezone.utc)
last_log_time = modified.isoformat()
worker_online = (datetime.now(timezone.utc) - modified).total_seconds() < 180
recent_lines = tail_lines("detect_worker.log", max_lines=160)
runtime_settings = get_runtime_settings()
worker_online, last_log_time, recent_lines = _load_recent_worker_lines(runtime_settings, max_lines=160)
runtime = detect_worker_runtime()
runtime_state = _load_runtime_state()
runtime_started_at = runtime.get("latest_start_time", "") if 'runtime' in locals() else ""
if not runtime_state:
runtime_state = _load_runtime_state_from_cluster_node()
runtime_started_at = runtime.get("latest_start_time", "")
recent_lines = _filter_lines_since(recent_lines, runtime_started_at)
available_proxy_count = _extract_available_proxy_count(recent_lines)
active_thread_snapshot = _extract_active_thread_snapshot(recent_lines)
@@ -449,21 +917,35 @@ def get_detect_status() -> dict:
"active": int(runtime_state.get("active_threads", active_thread_snapshot["active"]) or active_thread_snapshot["active"]),
"max": int(runtime_state.get("max_threads", active_thread_snapshot["max"]) or active_thread_snapshot["max"]),
}
progress_total = sum(progress.values())
registerable_count = int(progress.get("registerable", 0) or 0)
progress_total = sum(
int(progress.get(key, 0) or 0)
for key in ("pending", "completed", "running", "blacklisted", "failed")
)
progress_done = progress.get("completed", 0) + progress.get("blacklisted", 0) + progress.get("failed", 0)
progress_percent = round((progress_done / progress_total) * 100, 2) if progress_total > 0 else 0
runtime = detect_worker_runtime()
runtime_started_at = runtime.get("latest_start_time", "")
recent_lines = _filter_lines_since(recent_lines, runtime_started_at)
recent_proxy_warning = _normalize_recent_warning(runtime_state, recent_lines, available_proxy_count)
proxy_runtime = _build_proxy_runtime_snapshot(settings_payload, runtime_state, available_proxy_count)
thread_count_resolution = resolve_thread_count(settings_payload=settings_payload)
effective_thread_count = int(thread_count_resolution["effective_thread_count"])
runtime_settings = get_runtime_settings()
worker_online = worker_online or runtime.get("running", False)
if runtime_state.get("service_running") is True:
worker_online = True
if not runtime_state.get("detecting", False) and not progress.get("running", 0):
if active_thread_snapshot["active"] <= 0 and (runtime_state.get("detecting", False) or runtime.get("running", False)):
estimated_active_threads = _estimate_active_threads_from_recent_lines(
recent_lines,
limit=max(1, effective_thread_count),
)
if estimated_active_threads > 0:
active_thread_snapshot["active"] = estimated_active_threads
inferred_detecting = bool(
runtime_state.get("detecting", False)
or int(progress.get("running", 0) or 0) > 0
or int(active_thread_snapshot.get("active", 0) or 0) > 0
)
if not inferred_detecting:
active_thread_snapshot = {"active": 0, "max": active_thread_snapshot["max"] or effective_thread_count}
settings_summary = {
"thread_count": effective_thread_count,
@@ -475,10 +957,59 @@ def get_detect_status() -> dict:
"allow_direct": settings_payload["proxy_config"].get("allow_direct", False),
"proxy_pool_count": len(settings_payload["proxy_config"].get("proxy_urls", [])),
}
active_job = get_active_detect_job_summary(event_limit=240)
try:
active_job = get_active_detect_job_summary(event_limit=240)
except Exception:
active_job = None
if settings.node_region == "overseas" and settings.node_role == "control" and active_job:
progress = {
"pending": int(active_job.get("items_pending", 0) or 0),
"running": int(
active_job.get("display_active_threads", active_job.get("display_items_running", active_job.get("items_running", 0)))
or 0
),
"completed": int(active_job.get("items_completed", 0) or 0),
"failed": int(active_job.get("items_failed", 0) or 0),
"blacklisted": int(active_job.get("items_blacklisted", 0) or 0),
"registerable": registerable_count,
}
progress_percent = float(active_job.get("progress_percent", 0) or 0)
local_node_bucket = {}
for item in list((active_job or {}).get("node_stats") or []):
if str(item.get("node_code") or "").strip() == str(settings.node_code or "").strip():
local_node_bucket = item
break
local_runtime_load = int(
local_node_bucket.get("active_threads")
or local_node_bucket.get("items_running")
or 0
)
local_runtime_max_threads = int(local_node_bucket.get("max_threads", 0) or 0)
if active_thread_snapshot["active"] <= 0 and local_runtime_load > 0:
active_thread_snapshot["active"] = local_runtime_load
if active_thread_snapshot["max"] <= 0:
active_thread_snapshot["max"] = local_runtime_max_threads or effective_thread_count
if settings.node_region == "overseas" and settings.node_role == "control" and active_job:
distributed_node_stats = list(active_job.get("distributed_node_stats") or active_job.get("node_stats") or [])
aggregated_active_threads = 0
aggregated_max_threads = 0
for item in distributed_node_stats:
node_code = str(item.get("node_code") or "").strip()
if not node_code or node_code == "unassigned":
continue
aggregated_active_threads += int(
item.get("active_threads")
or item.get("items_running")
or 0
)
aggregated_max_threads += int(item.get("max_threads", 0) or 0)
if aggregated_active_threads > 0:
active_thread_snapshot["active"] = aggregated_active_threads
if aggregated_max_threads > 0:
active_thread_snapshot["max"] = aggregated_max_threads
runtime_snapshot = {
**runtime,
"detecting": runtime_state.get("detecting", False),
"detecting": inferred_detecting,
"proxy_runtime_state": proxy_runtime["state"],
"proxy_runtime_label": proxy_runtime["label"],
"proxy_runtime_detail": proxy_runtime["detail"],
@@ -518,7 +1049,7 @@ def get_detect_status() -> dict:
"runtime_state": runtime_state,
"phase_label": runtime_state.get("phase", ""),
"phase_detail": runtime_state.get("detail", ""),
"detecting": runtime_state.get("detecting", False),
"detecting": inferred_detecting,
"thread_count": effective_thread_count,
"thread_count_default": int(thread_count_resolution["default_thread_count"]),
"thread_count_source": str(thread_count_resolution["source"]),
@@ -549,6 +1080,7 @@ def get_detect_status() -> dict:
"progress_percent": progress_percent,
"recent_event": runtime_state.get("detail") or _recent_event(recent_lines),
"recent_warning": recent_proxy_warning,
"aggregate_detect_view": bool(settings.node_region == "overseas" and settings.node_role == "control" and active_job),
"log_lines": recent_lines,
"remote_log_lines": remote_log_lines,
"remote_log_line_count": int(remote_log_snapshot.get("line_count", 0) or 0),

View File

@@ -145,7 +145,8 @@ def _write_txt(path, rows: list[dict]) -> None:
def _write_csv(path, rows: list[dict]) -> None:
with path.open("w", encoding="utf-8", newline="") as handle:
# Add BOM so Excel on Chinese Windows opens CSV without mojibake.
with path.open("w", encoding="utf-8-sig", newline="") as handle:
writer = csv.writer(handle)
writer.writerow([label for _, label in EXPORT_HEADERS])
for row in rows:

View File

@@ -1,6 +1,8 @@
from __future__ import annotations
import ast
import hashlib
from io import StringIO
import pickle
import re
import sys
@@ -35,6 +37,9 @@ LEGACY_JUMING_COOKIE_FILES = [
DELETE_LIST_SOURCE_TYPE = 2
FIXED_PRICE_SOURCE_TYPE = 1
JUMING_PREFERENCES_FILE = "juming_preferences.json"
JUMING_DELETE_IMPORT_STATE_FILE = "juming_delete_import_state.json"
IMPORT_BATCH_SIZE = 50000
IMPORT_PROGRESS_EVERY = 100000
class TaskStoppedError(RuntimeError):
@@ -125,6 +130,118 @@ def _persist_juming_cookie(cookie_jar: RequestsCookieJar) -> None:
pass
def _empty_import_stats() -> dict[str, int]:
return {
"total": 0,
"valid": 0,
"added": 0,
"exists": 0,
"invalid": 0,
"failed": 0,
}
def _merge_import_stats(base: dict[str, int], delta: dict[str, int]) -> dict[str, int]:
merged = dict(base or _empty_import_stats())
for key in ("total", "valid", "added", "exists", "invalid", "failed"):
merged[key] = int(merged.get(key, 0) or 0) + int((delta or {}).get(key, 0) or 0)
return merged
def _load_delete_import_state() -> dict[str, dict]:
payload = read_runtime_json(JUMING_DELETE_IMPORT_STATE_FILE, default={})
return payload if isinstance(payload, dict) else {}
def _save_delete_import_state(payload: dict[str, dict]) -> None:
write_runtime_json(JUMING_DELETE_IMPORT_STATE_FILE, payload)
def _compute_domains_signature(domains: list[str]) -> str:
digest = hashlib.sha1()
for domain in domains:
digest.update(str(domain).strip().encode("utf-8", errors="ignore"))
digest.update(b"\n")
return digest.hexdigest()
def _looks_like_login_redirect(location: str) -> bool:
normalized = str(location or "").strip().lower()
if not normalized:
return False
return any(
marker in normalized
for marker in (
"/login",
"user_zh",
"p_login",
"passport",
"sign",
)
)
def _looks_like_login_body(body: str) -> bool:
normalized = str(body or "").strip().lower()
if not normalized:
return False
return any(
marker in normalized
for marker in (
"账号登录",
"请先登录",
"登录后查看",
"登录聚名",
"user_zh",
"p_login",
)
)
def _validate_juming_cookie(
cookie_jar: RequestsCookieJar | None,
*,
probe_date: str | None = None,
) -> tuple[bool, str]:
if cookie_jar is None or not _cookie_jar_to_dict(cookie_jar):
return False, "未检测到有效 Cookie"
jm = JM()
jm.cookie = cookie_jar
probe_date = str(probe_date or date.today().isoformat())
url = f"{jm.base_url}/newcha/del_down?scsj={probe_date}"
try:
response = jm.session.get(
url,
headers=jm.headers,
cookies=jm.cookie,
allow_redirects=False,
timeout=10,
)
except Exception as exc:
return False, f"登录态校验失败: {exc}"
location = str(response.headers.get("Location") or "").strip()
if response.status_code in {301, 302, 303, 307, 308}:
if _looks_like_login_redirect(location):
return False, "聚名登录态已失效,请重新登录"
if location:
return True, f"删除列表下载链路校验通过: {probe_date}"
try:
body = response.text
except Exception:
body = ""
if _looks_like_login_body(body):
return False, "聚名登录态已失效,请重新登录"
if response.ok:
return True, f"聚名 Cookie 已通过远端校验: {probe_date}"
return False, f"聚名登录态校验失败HTTP {response.status_code}"
def _jucha_cookie_status() -> dict:
if JUCHA_COOKIE_FILE.exists():
return {
@@ -186,11 +303,16 @@ def _load_juming_cookie() -> tuple[RequestsCookieJar | None, str]:
def get_juming_status() -> dict:
cookie_jar, storage = _load_juming_cookie()
cookie_valid, cookie_message = _validate_juming_cookie(cookie_jar)
cookie_count = len(_cookie_jar_to_dict(cookie_jar)) if cookie_jar is not None else 0
status = {
"cookie_ready": cookie_jar is not None,
"cookie_ready": bool(cookie_jar is not None and cookie_valid),
"cookie_present": cookie_jar is not None,
"cookie_valid": cookie_valid,
"cookie_message": cookie_message,
"cookie_storage": storage,
"cookie_file": str(JUMING_COOKIE_FILE),
"cookie_count": len(_cookie_jar_to_dict(cookie_jar)) if cookie_jar is not None else 0,
"cookie_count": cookie_count,
"jucha": _jucha_cookie_status(),
"supported_modes": [
{"label": "聚名一口价", "value": "fixed_price", "source_type": FIXED_PRICE_SOURCE_TYPE},
@@ -308,80 +430,146 @@ def _insert_domains(
source_type: int,
log: Callable[[str], None] | None = None,
should_stop: Callable[[], bool] | None = None,
*,
announce_total: bool = True,
progress_label: str = "",
) -> dict:
total = len(domains)
normalized_rows: list[tuple[str, str]] = []
invalid = 0
_emit_log(log, f"开始入库处理,共收到 {total} 个原始域名")
for value in domains:
_check_stop(should_stop)
normalized = normalize_domain(value)
if not normalized:
invalid += 1
continue
tld = normalized.rsplit(".", 1)[-1]
normalized_rows.append((normalized, tld))
existing_set: set[str] = set()
progress_prefix = f"{progress_label} " if str(progress_label or "").strip() else ""
if announce_total:
_emit_log(log, f"{progress_prefix}开始入库处理,共收到 {total} 个原始域名")
inserted = 0
existing = 0
processed = 0
valid = 0
last_progress_at = 0
pending_batch: list[tuple[str, str]] = []
pending_seen: set[str] = set()
stage_ready = False
def emit_progress(force: bool = False) -> None:
nonlocal last_progress_at
if not force and processed - last_progress_at < IMPORT_PROGRESS_EVERY:
return
last_progress_at = processed
_emit_log(
log,
(
f"{progress_prefix}入库进度:已处理 {processed}/{total}"
f"有效 {valid},新增 {inserted},已存在 {existing},无效 {invalid}"
),
)
def ensure_stage_table(cur) -> None:
nonlocal stage_ready
if stage_ready:
return
cur.execute(
"""
create temporary table if not exists juming_import_stage (
domain text primary key,
tld text not null
) on commit preserve rows
"""
)
stage_ready = True
def stage_rows(cur, rows: list[tuple[str, str]]) -> None:
buffer = StringIO()
for domain, tld in rows:
buffer.write(f"{domain}\t{tld}\n")
buffer.seek(0)
cur.copy_from(buffer, "juming_import_stage", columns=("domain", "tld"))
def flush_batch(cur, conn) -> None:
nonlocal inserted, existing
if not pending_batch:
return
ensure_stage_table(cur)
cur.execute("set local synchronous_commit = off")
cur.execute("truncate table juming_import_stage")
stage_rows(cur, pending_batch)
cur.execute(
"""
with existing_rows as (
select count(*)
from juming_import_stage stage
join domains existing on existing.domain = stage.domain
),
inserted as (
insert into domains (
domain, tld, source_type, use_status, detect_status, register_status,
has_beian, company_type, website_url, beian_year, snapshot_years,
expire_date, create_time, update_time, review_status, detect_time,
backlink_count, jucha_status, juziseo_status
)
select
stage.domain,
stage.tld,
%s,
0, 0, 0,
1, null, null, null, null,
null, now(), now(), 0, null,
0, 0, 0
from juming_import_stage stage
left join domains existing on existing.domain = stage.domain
where existing.id is null
returning id
),
task_insert as (
insert into detect_tasks (
domain_id, task_type, status, priority, retry_count, create_time, update_time
)
select id, 1, 1, 5, 0, now(), now()
from inserted
returning 1
)
select
(select count(*) from inserted),
(select count(*) from task_insert),
(select count(*) from existing_rows)
""",
(source_type,),
)
inserted_count, _task_count, existing_count = cur.fetchone()
inserted += int(inserted_count or 0)
existing += int(existing_count or 0)
conn.commit()
pending_batch.clear()
pending_seen.clear()
with get_db() as conn:
with conn.cursor() as cur:
normalized_domains = [row[0] for row in normalized_rows]
if normalized_domains:
cur.execute("select domain from domains where domain = any(%s)", (normalized_domains,))
existing_set = {row[0] for row in cur.fetchall()}
if existing_set:
_emit_log(log, f"检测到 {len(existing_set)} 个已存在域名,将自动跳过")
inserted_since_commit = 0
for domain, tld in normalized_rows:
for value in domains:
_check_stop(should_stop)
if domain in existing_set:
processed += 1
normalized = normalize_domain(value)
if not normalized:
invalid += 1
emit_progress()
continue
cur.execute(
"""
insert into domains (
domain, tld, source_type, use_status, detect_status, register_status,
has_beian, company_type, website_url, beian_year, snapshot_years,
expire_date, create_time, update_time, review_status, detect_time,
backlink_count, jucha_status, juziseo_status
) values (
%s, %s, %s, 0, 0, 0,
1, null, null, null, null,
null, now(), now(), 0, null,
0, 0, 0
)
returning id
""",
(domain, tld, source_type),
)
domain_id = cur.fetchone()[0]
cur.execute(
"""
insert into detect_tasks (domain_id, task_type, status, priority, retry_count, create_time, update_time)
values (%s, 1, 1, 5, 0, now(), now())
""",
(domain_id,),
)
inserted += 1
inserted_since_commit += 1
if inserted_since_commit >= 500:
conn.commit()
inserted_since_commit = 0
conn.commit()
valid += 1
if normalized in pending_seen:
existing += 1
emit_progress()
continue
pending_seen.add(normalized)
pending_batch.append((normalized, normalized.rsplit(".", 1)[-1]))
if len(pending_batch) >= IMPORT_BATCH_SIZE:
flush_batch(cur, conn)
emit_progress(force=True)
flush_batch(cur, conn)
emit_progress(force=True)
valid = len(normalized_rows)
exists = len(existing_set)
_emit_log(log, f"入库完成:有效 {valid},新增 {inserted},已存在 {exists},无效 {invalid}")
_emit_log(log, f"{progress_prefix}入库完成:有效 {valid},新增 {inserted},已存在 {existing},无效 {invalid}")
return {
"total": total,
"valid": valid,
"added": inserted,
"exists": exists,
"exists": existing,
"invalid": invalid,
"failed": max(valid - exists - inserted, 0),
"failed": max(valid - existing - inserted, 0),
}
@@ -450,6 +638,98 @@ def _crawl_delete_list(
return domains, dates
def _crawl_delete_list_and_import(
crawl_date: str,
auto_date: bool,
log: Callable[[str], None] | None = None,
should_stop: Callable[[], bool] | None = None,
) -> dict:
cookie_jar, _ = _load_juming_cookie()
jm = JM()
jm.cookie = cookie_jar or RequestsCookieJar()
start_date = datetime.strptime(crawl_date, "%Y-%m-%d").date()
end_date = date.today() + timedelta(days=4)
current_date = start_date
dates: list[dict[str, int]] = []
domains_found = 0
sample_domains: list[str] = []
stats = _empty_import_stats()
import_state = _load_delete_import_state()
_emit_log(log, f"开始采集删除列表:起始日期 {crawl_date},自动追加日期 {'开启' if auto_date else '关闭'}")
while current_date <= end_date:
_check_stop(should_stop)
current_date_text = current_date.isoformat()
_emit_log(log, f"正在抓取 {current_date_text} 的删除列表")
domains_for_date = [item.strip() for item in jm.new_cha_del(current_date_text) if item.strip()]
domains_found += len(domains_for_date)
dates.append({"date": current_date_text, "count": len(domains_for_date)})
_emit_log(log, f"{current_date_text} 抓取到 {len(domains_for_date)} 个域名,累计 {domains_found}")
if domains_for_date:
if len(sample_domains) < 20:
sample_domains.extend(domains_for_date[: max(0, 20 - len(sample_domains))])
signature = _compute_domains_signature(domains_for_date)
cached = import_state.get(current_date_text) or {}
if (
cached.get("signature") == signature
and int(cached.get("total", 0) or 0) == len(domains_for_date)
):
cached_valid = int(cached.get("valid", 0) or 0)
cached_invalid = int(cached.get("invalid", 0) or 0)
date_stats = {
"total": len(domains_for_date),
"valid": cached_valid,
"added": 0,
"exists": cached_valid,
"invalid": cached_invalid,
"failed": 0,
}
_emit_log(
log,
(
f"{current_date_text} 删除列表内容未变化,跳过重复入库:"
f"有效 {cached_valid},视为已存在 {cached_valid},无效 {cached_invalid}"
),
)
else:
date_stats = _insert_domains(
domains_for_date,
DELETE_LIST_SOURCE_TYPE,
log=log,
should_stop=should_stop,
announce_total=False,
progress_label=current_date_text,
)
import_state[current_date_text] = {
"signature": signature,
"total": int(date_stats.get("total", 0) or 0),
"valid": int(date_stats.get("valid", 0) or 0),
"invalid": int(date_stats.get("invalid", 0) or 0),
"updated_at": datetime.now().isoformat(sep=" ", timespec="seconds"),
}
_save_delete_import_state(import_state)
stats = _merge_import_stats(stats, date_stats)
if not auto_date:
break
current_date += timedelta(days=1)
_emit_log(
log,
(
f"删除列表采集+入库完成:抓取 {domains_found} 个域名,"
f"新增 {stats['added']},已存在 {stats['exists']},无效 {stats['invalid']}"
),
)
return {
"mode": "delete_list",
"dates": dates,
"domains_found": domains_found,
"stats": stats,
"sample_domains": sample_domains[:20],
}
def crawl_juming(
payload: dict,
log: Callable[[str], None] | None = None,
@@ -462,7 +742,10 @@ def crawl_juming(
cookie_jar, storage = _load_juming_cookie()
if cookie_jar is None:
raise ValueError("未找到聚名 Cookie请先在桌面版系统设置完成聚名登录或将 Cookie 同步到服务器")
_emit_log(log, f"检测到聚名登录态,来源:{storage}")
cookie_valid, cookie_message = _validate_juming_cookie(cookie_jar)
if not cookie_valid:
raise ValueError(cookie_message)
_emit_log(log, f"检测到聚名登录态,来源:{storage},远端校验通过")
_check_stop(should_stop)
if mode == "fixed_price":
@@ -482,13 +765,6 @@ def crawl_juming(
crawl_date = str(payload.get("crawl_date") or date.today().isoformat())
auto_date = bool(payload.get("auto_date", True))
domains, dates = _crawl_delete_list(crawl_date, auto_date, log=log, should_stop=should_stop)
stats = _insert_domains(domains, DELETE_LIST_SOURCE_TYPE, log=log, should_stop=should_stop)
return {
"mode": mode,
"cookie_storage": storage,
"dates": dates,
"domains_found": len(domains),
"stats": stats,
"sample_domains": domains[:20],
}
result = _crawl_delete_list_and_import(crawl_date, auto_date, log=log, should_stop=should_stop)
result["cookie_storage"] = storage
return result

View File

@@ -10,6 +10,7 @@ from app.services.juming_service import TaskStoppedError, crawl_juming
_JUMING_TASK_LOCK = threading.Lock()
_MAX_LOG_LINES = 400
_ACTIVE_TASK_IDS: set[str] = set()
def _now() -> str:
@@ -17,6 +18,7 @@ def _now() -> str:
def list_juming_tasks() -> list[dict]:
_cleanup_orphaned_tasks()
return load_juming_records()
@@ -55,6 +57,43 @@ def _is_stop_requested(task_id: str) -> bool:
return bool(target and target.get("cancel_requested"))
def _cleanup_orphaned_tasks() -> None:
with _JUMING_TASK_LOCK:
tasks = load_juming_records()
changed = False
for item in tasks:
status = str(item.get("status") or "").strip()
task_id = str(item.get("task_id") or "").strip()
if status == "running" and task_id and task_id not in _ACTIVE_TASK_IDS:
item["status"] = "failed"
item["phase"] = "failed"
item["phase_label"] = "失败"
item["cancel_requested"] = False
item["message"] = "任务因服务重启或进程中断而终止,请重新执行"
item["updated_at"] = _now()
logs = list(item.get("logs") or [])
logs.append(f"[{_now()}] 任务因服务重启或进程中断而终止,请重新执行")
item["logs"] = logs[-_MAX_LOG_LINES:]
changed = True
if changed:
_save_tasks(tasks)
def _ensure_no_active_task() -> None:
_cleanup_orphaned_tasks()
tasks = load_juming_records()
active = next(
(
item
for item in tasks
if str(item.get("status") or "").strip() == "running"
),
None,
)
if active:
raise ValueError(f"已有聚名采集任务正在运行:{active.get('task_id')}")
def _set_phase(task_id: str, phase: str, message: str | None = None) -> None:
phase_labels = {
"queued": "排队中",
@@ -78,12 +117,18 @@ def _set_phase(task_id: str, phase: str, message: str | None = None) -> None:
def _log_and_track_phase(task_id: str, message: str) -> None:
if "开始采集" in message or "正在抓取" in message:
_set_phase(task_id, "fetching", message)
elif "开始入库处理" in message or "入库完成" in message or "已存在域名" in message:
elif (
"开始入库处理" in message
or "入库进度" in message
or "入库完成" in message
or "已存在域名" in message
):
_set_phase(task_id, "importing", message)
_append_log(task_id, message)
def _run_juming_task(task_id: str, payload: dict) -> None:
_ACTIVE_TASK_IDS.add(task_id)
_update_task(task_id, status="running", started_at=_now(), message="聚名采集任务开始执行", cancel_requested=False)
_set_phase(task_id, "starting", "聚名采集任务开始执行")
_append_log(task_id, "任务已启动,正在准备读取聚名登录态")
@@ -127,9 +172,12 @@ def _run_juming_task(task_id: str, payload: dict) -> None:
)
_set_phase(task_id, "failed")
_append_log(task_id, f"任务执行失败:{exc}")
finally:
_ACTIVE_TASK_IDS.discard(task_id)
def create_juming_task(payload: dict) -> dict:
_ensure_no_active_task()
task_id = uuid4().hex
mode = str(payload.get("mode") or "delete_list").strip() or "delete_list"
record = {
@@ -154,12 +202,14 @@ def create_juming_task(payload: dict) -> dict:
tasks.insert(0, record)
_save_tasks(tasks)
_ACTIVE_TASK_IDS.add(task_id)
worker = threading.Thread(target=_run_juming_task, args=(task_id, dict(payload or {})), daemon=True)
worker.start()
return record
def retry_juming_task(task_id: str) -> dict:
_ensure_no_active_task()
with _JUMING_TASK_LOCK:
tasks = load_juming_records()
target = next((item for item in tasks if item["task_id"] == task_id), None)

View File

@@ -1,9 +1,15 @@
from __future__ import annotations
import os
import shutil
from pathlib import Path
from typing import Protocol
import psycopg2
import redis
from app.core.config import settings
from app.core.files import runtime_root as api_runtime_root
STRUCTURED_ACTIONS = {
@@ -22,6 +28,7 @@ STRUCTURED_ACTIONS = {
"runtime.restart_api",
"runtime.start_sync_agent",
"runtime.stop_sync_agent",
"runtime.reset_lab_state",
}
@@ -140,6 +147,233 @@ def systemctl_action_name(action: str) -> str:
return ""
def _safe_bool(value: object, default: bool) -> bool:
if value is None:
return bool(default)
if isinstance(value, bool):
return value
text = str(value).strip().lower()
if not text:
return bool(default)
if text in {"1", "true", "yes", "on"}:
return True
if text in {"0", "false", "no", "off"}:
return False
return bool(default)
def _node_agent_queue_dir() -> Path:
explicit = str(os.getenv("OPS_AGENT_QUEUE_DIR", "") or "").strip()
if explicit:
return Path(explicit)
project_dir = Path(__file__).resolve().parents[2]
return project_dir / "runtime" / "node-agent-queue" / (settings.node_code or "unbound")
def _detect_install_root(base_path: Path) -> Path | None:
normalized = str(base_path.resolve())
for marker in (f"{os.sep}releases{os.sep}", f"{os.sep}current{os.sep}"):
if marker in normalized:
return Path(normalized.split(marker, 1)[0])
return None
def _domaincheck_runtime_root() -> Path:
explicit = str(os.getenv("DOMAINCHECK_RUNTIME_ROOT", "") or "").strip()
if explicit:
return Path(explicit)
domain_root = Path(settings.domain_root).resolve()
install_root = _detect_install_root(domain_root)
if install_root is not None:
return install_root / "runtime" / "domainCheck"
return domain_root
def _clear_path_contents(path: Path, *, preserve_names: set[str] | None = None) -> list[str]:
if not path.exists():
return []
cleared: list[str] = []
preserved = preserve_names or set()
for child in path.iterdir():
if child.name in preserved:
continue
if child.is_dir() and not child.is_symlink():
shutil.rmtree(child, ignore_errors=False)
else:
child.unlink(missing_ok=True)
cleared.append(str(child))
return cleared
def _reset_runtime_files(
*,
include_api_runtime: bool,
include_worker_runtime: bool,
include_node_agent_queue: bool,
) -> dict:
cleared_targets: dict[str, list[str]] = {}
if include_api_runtime:
api_root = api_runtime_root()
cleared_targets["api_runtime"] = _clear_path_contents(api_root, preserve_names={".env"})
if include_worker_runtime:
worker_root = _domaincheck_runtime_root()
worker_targets: dict[str, list[str]] = {}
for name in ("data", "logs"):
target = worker_root / name
worker_targets[name] = _clear_path_contents(target, preserve_names={".env"}) if target.exists() else []
detect_worker_log = Path(settings.domain_root) / "detect_worker.log"
if detect_worker_log.exists():
detect_worker_log.unlink(missing_ok=True)
worker_targets["files"] = [str(detect_worker_log)]
cleared_targets["worker_runtime"] = [
item for values in worker_targets.values() for item in values
]
if include_node_agent_queue:
queue_dir = _node_agent_queue_dir()
cleared_targets["node_agent_queue"] = _clear_path_contents(queue_dir) if queue_dir.exists() else []
return cleared_targets
def _truncate_detect_runtime_tables(*, include_domains: bool) -> dict:
table_names = [
"detect_debug_events",
"detect_run_events",
"detect_job_items",
"detect_jobs",
"detect_sync_records",
"detect_tasks",
"detect_worker_nodes",
"domain_detections",
"domain_blacklist",
]
if include_domains:
table_names.append("domains")
conn = psycopg2.connect(
host=settings.db_host,
port=settings.db_port,
dbname=settings.db_database,
user=settings.db_user,
password=settings.db_password,
)
try:
conn.autocommit = False
with conn.cursor() as cur:
cur.execute(
f"TRUNCATE TABLE {', '.join(table_names)} RESTART IDENTITY CASCADE"
)
conn.commit()
finally:
conn.close()
return {"tables": table_names, "include_domains": bool(include_domains)}
def _flush_runtime_redis() -> dict:
client = redis.Redis(
host=settings.redis_host,
port=settings.redis_port,
password=settings.redis_password or None,
db=settings.redis_db,
decode_responses=True,
socket_connect_timeout=5,
socket_timeout=5,
)
try:
size_before = int(client.dbsize() or 0)
client.flushdb()
size_after = int(client.dbsize() or 0)
finally:
try:
client.close()
except Exception:
pass
return {"db": int(settings.redis_db), "size_before": size_before, "size_after": size_after}
def _run_service_command(
runner: CommandRunner,
service_name: str,
action: str,
) -> dict:
code, stdout, stderr = runner(["systemctl", action, service_name], timeout=60)
return {
"service_name": service_name,
"action": action,
"returncode": int(code or 0),
"stdout": trim_output(stdout, 4000),
"stderr": trim_output(stderr, 4000),
"ok": int(code or 0) == 0,
}
def _reset_lab_state(
payload: dict,
*,
runner: CommandRunner,
) -> tuple[bool, str, dict]:
normalized_payload = dict(payload or {})
stop_services = [
str(item).strip()
for item in list(normalized_payload.get("stop_services") or [])
if str(item).strip()
]
start_services = [
str(item).strip()
for item in list(normalized_payload.get("start_services") or [])
if str(item).strip()
]
clear_database = _safe_bool(normalized_payload.get("clear_database"), False)
clear_domains = _safe_bool(normalized_payload.get("clear_domains"), False)
clear_redis = _safe_bool(normalized_payload.get("clear_redis"), False)
clear_api_runtime = _safe_bool(normalized_payload.get("clear_api_runtime"), False)
clear_worker_runtime = _safe_bool(normalized_payload.get("clear_worker_runtime"), True)
clear_node_agent_queue = _safe_bool(normalized_payload.get("clear_node_agent_queue"), True)
result: dict[str, object] = {
"stop_services": [],
"start_services": [],
"database": {},
"redis": {},
"runtime": {},
}
errors: list[str] = []
for service_name in stop_services:
service_result = _run_service_command(runner, service_name, "stop")
result["stop_services"].append(service_result)
if not service_result["ok"]:
errors.append(f"stop {service_name} failed")
try:
if clear_database:
result["database"] = _truncate_detect_runtime_tables(include_domains=clear_domains)
if clear_redis:
result["redis"] = _flush_runtime_redis()
result["runtime"] = _reset_runtime_files(
include_api_runtime=clear_api_runtime,
include_worker_runtime=clear_worker_runtime,
include_node_agent_queue=clear_node_agent_queue,
)
except Exception as exc:
errors.append(str(exc))
for service_name in start_services:
service_result = _run_service_command(runner, service_name, "start")
result["start_services"].append(service_result)
if not service_result["ok"]:
errors.append(f"start {service_name} failed")
ok = not errors
message = "lab runtime reset completed" if ok else "lab runtime reset finished with errors"
if errors:
result["errors"] = errors
return ok, message, result
def execute_structured_action(
action: str,
payload: dict | None,
@@ -291,4 +525,12 @@ def execute_structured_action(
}
return True, "diagnostics collected", diagnostics
if normalized_action == "runtime.reset_lab_state":
ok, message, result = _reset_lab_state(
normalized_payload,
runner=runner,
)
result.update(host_details)
return ok, message, result
return False, f"unsupported action: {normalized_action}", {"action": normalized_action}

View File

@@ -12,6 +12,9 @@ from app.core.db import get_db
from app.services.ops_command_service import build_bash_command
from app.services.ops_job_service import create_ops_job, ensure_ops_schema, get_ops_job
from app.services.ops_template_service import build_ops_template_payload, get_ops_action_template
from app.services.runtime_settings_service import get_runtime_settings
from app.services.sensitive_words_service import get_sensitive_words_payload
from app.services.settings_service import get_settings_payload
_AGENT_SCHEMA_SQL = """
@@ -117,6 +120,48 @@ def _format_time(value: object) -> str:
return ""
def _prefer_non_loopback_identity(primary: object, fallback: object) -> str:
primary_value = str(primary or "").strip()
fallback_value = str(fallback or "").strip()
invalid_values = {"", "localhost", "localhost.localdomain", "127.0.0.1", "::1"}
if primary_value and primary_value.lower() not in invalid_values:
return primary_value
return fallback_value
def _merge_detect_runtime_snapshot(cluster_metadata: dict, metadata: dict, current_load: int) -> dict:
active_threads = int(cluster_metadata.get("active_threads", metadata.get("active_threads", 0)) or 0)
max_threads = int(cluster_metadata.get("max_threads", metadata.get("max_threads", 0)) or 0)
inferred_worker_online = bool(
cluster_metadata.get("worker_online", metadata.get("worker_online", False))
or cluster_metadata.get("service_running", metadata.get("service_running", False))
or current_load > 0
or active_threads > 0
)
inferred_detect_participating = bool(
cluster_metadata.get("detect_participating", metadata.get("detect_participating", False))
or current_load > 0
or active_threads > 0
)
return {
"worker_online": inferred_worker_online,
"detect_participating": inferred_detect_participating,
"active_threads": active_threads,
"max_threads": max_threads,
"current_load": current_load,
"phase_label": str(cluster_metadata.get("phase_label", metadata.get("phase_label", "")) or "").strip(),
"phase_detail": str(cluster_metadata.get("phase_detail", metadata.get("phase_detail", "")) or "").strip(),
"recent_warning": str(cluster_metadata.get("recent_warning", metadata.get("recent_warning", "")) or "").strip(),
"proxy_runtime_label": str(
cluster_metadata.get("proxy_runtime_label", metadata.get("proxy_runtime_label", "")) or ""
).strip(),
"proxy_runtime_reason": str(
cluster_metadata.get("proxy_runtime_reason", metadata.get("proxy_runtime_reason", "")) or ""
).strip(),
"updated_at": str(cluster_metadata.get("updated_at", metadata.get("updated_at", "")) or "").strip(),
}
def _ops_job_event_level_label(level: object) -> str:
normalized_level = str(level or "").strip().lower()
mapping = {
@@ -588,6 +633,7 @@ def list_managed_nodes_with_agent_state(*, participation_payload: dict | None =
seen_node_codes.add(node_code)
metadata = dict(node.get("metadata") or {})
cluster_node = cluster_map.get(node_code, {})
cluster_metadata = dict(cluster_node.get("metadata") or {})
latest_token = latest_tokens.get(node_code, {})
latest_job = latest_jobs.get(node_code, {})
participation_row = dict(detect_participation_map.get(node_code) or {})
@@ -607,6 +653,15 @@ def list_managed_nodes_with_agent_state(*, participation_payload: dict | None =
current_load=current_load,
)
capabilities = list(metadata.get("capabilities") or [])
detect_runtime = _merge_detect_runtime_snapshot(cluster_metadata, metadata, current_load)
active_job_snapshot = {
"job_code": str(cluster_metadata.get("active_job_code", metadata.get("active_job_code", "")) or "").strip(),
"status": str(cluster_metadata.get("active_job_status", metadata.get("active_job_status", "")) or "").strip(),
"items_total": int(cluster_metadata.get("job_items_total", metadata.get("job_items_total", 0)) or 0),
"items_claimed": int(cluster_metadata.get("job_items_claimed", metadata.get("job_items_claimed", 0)) or 0),
"items_running": int(cluster_metadata.get("job_items_running", metadata.get("job_items_running", 0)) or 0),
"items_completed": int(cluster_metadata.get("job_items_completed", metadata.get("job_items_completed", 0)) or 0),
}
merged_node = {
**node,
"is_managed": True,
@@ -620,10 +675,14 @@ def list_managed_nodes_with_agent_state(*, participation_payload: dict | None =
"agent_version": str(metadata.get("agent_version") or "").strip(),
"agent_hostname": str(metadata.get("hostname") or "").strip(),
"agent_ip": str(metadata.get("ip") or "").strip(),
"cluster_hostname": str(cluster_node.get("hostname") or "").strip(),
"cluster_ip": str(cluster_node.get("ip") or "").strip(),
"cluster_hostname": _prefer_non_loopback_identity(cluster_node.get("hostname"), metadata.get("hostname")),
"cluster_ip": _prefer_non_loopback_identity(cluster_node.get("ip"), metadata.get("ip")),
"cluster_status": cluster_status,
"cluster_current_load": current_load,
"current_load": current_load,
"detect_runtime": detect_runtime,
"runtime_state": detect_runtime,
"active_job": active_job_snapshot,
"cluster_last_heartbeat_at": str(cluster_node.get("last_heartbeat_at") or metadata.get("last_heartbeat_at") or "").strip(),
"cluster_is_effective_worker": bool(cluster_node.get("is_effective_worker", metadata.get("is_effective_worker", False))),
"cluster_detect_participating": bool(
@@ -688,6 +747,7 @@ def list_managed_nodes_with_agent_state(*, participation_payload: dict | None =
if not node_code or node_code in seen_node_codes:
continue
metadata = dict(cluster_node.get("metadata") or {})
cluster_metadata = dict(cluster_node.get("metadata") or {})
latest_token = latest_tokens.get(node_code, {})
latest_job = latest_jobs.get(node_code, {})
participation_row = dict(detect_participation_map.get(node_code) or {})
@@ -706,6 +766,15 @@ def list_managed_nodes_with_agent_state(*, participation_payload: dict | None =
current_load=current_load,
)
capabilities = list(metadata.get("capabilities") or [])
detect_runtime = _merge_detect_runtime_snapshot(cluster_metadata, metadata, current_load)
active_job_snapshot = {
"job_code": str(cluster_metadata.get("active_job_code", metadata.get("active_job_code", "")) or "").strip(),
"status": str(cluster_metadata.get("active_job_status", metadata.get("active_job_status", "")) or "").strip(),
"items_total": int(cluster_metadata.get("job_items_total", metadata.get("job_items_total", 0)) or 0),
"items_claimed": int(cluster_metadata.get("job_items_claimed", metadata.get("job_items_claimed", 0)) or 0),
"items_running": int(cluster_metadata.get("job_items_running", metadata.get("job_items_running", 0)) or 0),
"items_completed": int(cluster_metadata.get("job_items_completed", metadata.get("job_items_completed", 0)) or 0),
}
fallback_node = {
"node_code": node_code,
"region": str(cluster_node.get("region") or "").strip(),
@@ -732,12 +801,16 @@ def list_managed_nodes_with_agent_state(*, participation_payload: dict | None =
"capabilities": capabilities,
"capabilities_count": len(capabilities),
"agent_version": str(metadata.get("agent_version") or "").strip(),
"agent_hostname": str(cluster_node.get("hostname") or metadata.get("hostname") or "").strip(),
"agent_ip": str(cluster_node.get("ip") or metadata.get("ip") or "").strip(),
"cluster_hostname": str(cluster_node.get("hostname") or "").strip(),
"cluster_ip": str(cluster_node.get("ip") or "").strip(),
"agent_hostname": _prefer_non_loopback_identity(cluster_node.get("hostname"), metadata.get("hostname")),
"agent_ip": _prefer_non_loopback_identity(cluster_node.get("ip"), metadata.get("ip")),
"cluster_hostname": _prefer_non_loopback_identity(cluster_node.get("hostname"), metadata.get("hostname")),
"cluster_ip": _prefer_non_loopback_identity(cluster_node.get("ip"), metadata.get("ip")),
"cluster_status": cluster_status,
"cluster_current_load": current_load,
"current_load": current_load,
"detect_runtime": detect_runtime,
"runtime_state": detect_runtime,
"active_job": active_job_snapshot,
"cluster_last_heartbeat_at": str(cluster_node.get("last_heartbeat_at") or "").strip(),
"cluster_is_effective_worker": bool(cluster_node.get("is_effective_worker", False)),
"cluster_detect_participating": bool(
@@ -2359,7 +2432,7 @@ def build_node_agent_bootstrap_plan(
f"SYNC_AGENT_SERVICE_NAME={settings.sync_agent_service_name}",
"NODE_AGENT_SERVICE_NAME=domaincheck-node-agent",
"",
'OPS_AGENT_CAPABILITIES=["service.start","service.stop","service.restart","service.status","runtime.start_worker","runtime.stop_worker","runtime.restart_api","runtime.start_sync_agent","runtime.stop_sync_agent","health.snapshot","logs.collect","diagnostics.collect","deploy.release"]',
'OPS_AGENT_CAPABILITIES=["service.start","service.stop","service.restart","service.status","runtime.start_worker","runtime.stop_worker","runtime.start_detection","runtime.stop_detection","runtime.pull_tasks","runtime.restart_api","runtime.start_sync_agent","runtime.stop_sync_agent","runtime.reset_lab_state","health.snapshot","logs.collect","diagnostics.collect","deploy.release"]',
"OPS_AGENT_LABELS={}",
]
env_content = "\n".join(env_lines)
@@ -2529,6 +2602,135 @@ def _upsert_agent_runtime(node_code: str, payload: dict) -> None:
conn.commit()
def _load_existing_detect_node_runtime(node_code: str) -> dict:
normalized_node_code = str(node_code or "").strip()
if not normalized_node_code:
return {}
try:
with get_db() as conn:
with conn.cursor() as cur:
cur.execute(
"""
SELECT status, current_load, metadata_json, last_heartbeat_at
FROM detect_worker_nodes
WHERE node_code = %s
LIMIT 1
""",
(normalized_node_code,),
)
row = cur.fetchone()
except Exception:
return {}
if not row:
return {}
metadata = row[2] if isinstance(row[2], dict) else {}
return {
"status": str(row[0] or "").strip(),
"current_load": int(row[1] or 0),
"metadata": metadata,
"last_heartbeat_at": row[3],
}
def _upsert_agent_detect_runtime(node_code: str, payload: dict) -> None:
normalized_node_code = str(node_code or "").strip()
if not normalized_node_code:
return
metadata = payload.get("metadata") if isinstance(payload.get("metadata"), dict) else {}
detect_runtime = metadata.get("detect_runtime") if isinstance(metadata.get("detect_runtime"), dict) else {}
if not detect_runtime:
return
try:
from app.services.cluster_runtime_service import register_node_heartbeat
region = str(payload.get("region") or "unknown").strip() or "unknown"
role = str(payload.get("role") or "worker").strip() or "worker"
active_threads = max(0, int(detect_runtime.get("active_threads") or 0))
max_threads = max(0, int(detect_runtime.get("max_threads") or 0))
current_load = max(
0,
int(detect_runtime.get("current_load") or 0),
active_threads,
)
worker_online = bool(detect_runtime.get("worker_online", False) or detect_runtime.get("service_running", False))
detect_participating = bool(detect_runtime.get("detect_participating", False) or current_load > 0)
phase_label = str(detect_runtime.get("phase_label") or "").strip()
phase_detail = str(detect_runtime.get("phase_detail") or "").strip()
recent_warning = str(detect_runtime.get("recent_warning") or "").strip()
has_runtime_signal = any(
[
worker_online,
detect_participating,
current_load > 0,
active_threads > 0,
max_threads > 0,
bool(phase_label),
bool(phase_detail),
bool(recent_warning),
]
)
if not has_runtime_signal:
return
existing_runtime = _load_existing_detect_node_runtime(normalized_node_code)
existing_metadata = existing_runtime.get("metadata") if isinstance(existing_runtime.get("metadata"), dict) else {}
existing_active_threads = max(0, int(existing_metadata.get("active_threads", 0) or 0))
existing_current_load = max(0, int(existing_runtime.get("current_load", 0) or 0), existing_active_threads)
existing_status = str(existing_runtime.get("status") or "").strip()
existing_last_heartbeat_at = existing_runtime.get("last_heartbeat_at")
existing_is_recent = False
if isinstance(existing_last_heartbeat_at, datetime):
current_time = (
datetime.now(existing_last_heartbeat_at.tzinfo)
if existing_last_heartbeat_at.tzinfo
else datetime.now()
)
existing_is_recent = (current_time - existing_last_heartbeat_at) <= timedelta(seconds=120)
normalized_phase_detail = phase_detail.lower()
generic_phase_detail = bool(normalized_phase_detail) and "/" in normalized_phase_detail and " " not in normalized_phase_detail
weak_agent_snapshot = bool(
worker_online
and current_load <= 0
and active_threads <= 0
and max_threads > 0
and not phase_label
and (not phase_detail or generic_phase_detail)
and not recent_warning
)
if weak_agent_snapshot and existing_is_recent and (
existing_current_load > 0 or existing_status == "busy"
):
return
status = "busy" if current_load > 0 else ("online" if worker_online else "unknown")
heartbeat_metadata = {
"service": "agent-heartbeat",
"worker_online": worker_online,
"detect_participating": detect_participating,
"active_threads": active_threads,
"max_threads": max_threads,
"phase_label": phase_label,
"phase_detail": phase_detail,
"recent_warning": recent_warning,
"updated_at": str(detect_runtime.get("updated_at") or "").strip(),
"agent_heartbeat_at": datetime.now().isoformat(timespec="seconds"),
}
register_node_heartbeat(
node_code=normalized_node_code,
region=region,
role=role,
status=status,
current_load=current_load,
metadata=heartbeat_metadata,
hostname_override=str(payload.get("hostname") or "").strip(),
ip_override=str(payload.get("ip") or "").strip(),
)
except Exception:
return
def agent_register(payload: dict, *, token: str) -> tuple[bool, str, dict]:
node_code = str(payload.get("node_code") or "").strip()
if not node_code:
@@ -2537,6 +2739,7 @@ def agent_register(payload: dict, *, token: str) -> tuple[bool, str, dict]:
if not ok:
return False, message, auth
_upsert_agent_runtime(node_code, payload)
_upsert_agent_detect_runtime(node_code, payload)
return True, "Agent 注册成功", {
"node_code": node_code,
"expires_at": auth.get("expires_at", ""),
@@ -2552,6 +2755,7 @@ def agent_heartbeat(payload: dict, *, token: str) -> tuple[bool, str, dict]:
if not ok:
return False, message, auth
_upsert_agent_runtime(node_code, payload)
_upsert_agent_detect_runtime(node_code, payload)
return True, "heartbeat ok", {
"node_code": node_code,
"server_time": _format_time(datetime.now()),
@@ -2559,6 +2763,57 @@ def agent_heartbeat(payload: dict, *, token: str) -> tuple[bool, str, dict]:
}
def _runtime_config_bundle_hash_payload(bundle: dict | None) -> dict:
normalized = dict(bundle or {})
normalized.pop("config_hash", None)
normalized.pop("generated_at", None)
return normalized
def _build_agent_runtime_config_bundle(node_code: str) -> dict:
settings_payload = get_settings_payload()
runtime_settings = get_runtime_settings()
sensitive_words_payload = get_sensitive_words_payload()
bundle = {
"node_code": str(node_code or "").strip(),
"detect_options": dict(settings_payload.get("detect_options") or {}),
"proxy_config": dict(settings_payload.get("proxy_config") or {}),
"thread_count": int(settings_payload.get("thread_count", 2) or 2),
"node_thread_counts": dict(settings_payload.get("node_thread_counts") or {}),
"runtime_settings": dict(runtime_settings or {}),
"sensitive_words": {
"text": str(sensitive_words_payload.get("text") or ""),
"total": int(sensitive_words_payload.get("total", 0) or 0),
"items": list(sensitive_words_payload.get("items") or []),
},
"generated_at": _format_time(datetime.now()),
}
bundle["config_hash"] = hashlib.sha256(
json.dumps(
_runtime_config_bundle_hash_payload(bundle),
ensure_ascii=False,
sort_keys=True,
).encode("utf-8")
).hexdigest()
return bundle
def agent_pull_runtime_config(payload: dict, *, token: str) -> tuple[bool, str, dict]:
node_code = str(payload.get("node_code") or "").strip()
if not node_code:
return _agent_error("node_code 不能为空", "agent_node_code_required")
ok, message, auth = _authenticate_agent_token(token, expected_node_code=node_code)
if not ok:
return False, message, auth
bundle = _build_agent_runtime_config_bundle(node_code)
return True, "runtime config ok", {
"node_code": node_code,
"server_time": _format_time(datetime.now()),
"expires_at": auth.get("expires_at", ""),
"bundle": bundle,
}
def agent_pull_jobs(payload: dict, *, token: str, limit: int = 1) -> tuple[bool, str, dict]:
node_code = str(payload.get("node_code") or "").strip()
if not node_code:
@@ -2568,6 +2823,7 @@ def agent_pull_jobs(payload: dict, *, token: str, limit: int = 1) -> tuple[bool,
return False, message, _auth
safe_limit = min(max(int(limit or 1), 1), 10)
dispatched_events: list[dict] = []
with get_db() as conn:
conn.autocommit = False
with conn.cursor() as cur:
@@ -2604,15 +2860,19 @@ def agent_pull_jobs(payload: dict, *, token: str, limit: int = 1) -> tuple[bool,
""",
(job_id,),
)
append_ops_job_event(
job_id=job_id,
node_code=node_code,
event_type="agent_dispatched",
message=f"任务已派发给节点 {node_code}",
payload={"node_code": node_code},
dispatched_events.append(
{
"job_id": job_id,
"node_code": node_code,
"event_type": "agent_dispatched",
"message": f"任务已派发给节点 {node_code}",
"payload": {"node_code": node_code},
}
)
jobs.append(_agent_job_envelope(get_ops_job(job_id)))
conn.commit()
for event in dispatched_events:
append_ops_job_event(**event)
return True, "ok", {
"jobs": jobs,
"count": len(jobs),
@@ -2631,6 +2891,7 @@ def agent_mark_job_started(job_id: int, payload: dict, *, token: str) -> tuple[b
if not ok:
return False, message, _auth
started_event: dict | None = None
with get_db() as conn:
conn.autocommit = False
with conn.cursor() as cur:
@@ -2662,14 +2923,16 @@ def agent_mark_job_started(job_id: int, payload: dict, *, token: str) -> tuple[b
)
step_rows = cur.fetchall()
step_ids = [int(item[0]) for item in step_rows]
append_ops_job_event(
job_id=int(job_id),
node_code=node_code,
event_type="agent_started",
message=f"节点 {node_code} 已开始执行任务",
payload={"step_ids": step_ids},
)
started_event = {
"job_id": int(job_id),
"node_code": node_code,
"event_type": "agent_started",
"message": f"节点 {node_code} 已开始执行任务",
"payload": {"step_ids": step_ids},
}
conn.commit()
if started_event:
append_ops_job_event(**started_event)
job = get_ops_job(int(job_id))
return True, "任务已标记为运行中", {
"job": job,
@@ -2710,6 +2973,7 @@ def agent_complete_job(job_id: int, payload: dict, *, token: str) -> tuple[bool,
result["focus_ref"] = focus_ref
event_level = "info" if job_status == "success" else ("warning" if job_status == "partially_succeeded" else "error")
completed_event: dict | None = None
with get_db() as conn:
conn.autocommit = False
with conn.cursor() as cur:
@@ -2789,22 +3053,24 @@ def agent_complete_job(job_id: int, payload: dict, *, token: str) -> tuple[bool,
)
step_rows = cur.fetchall()
step_ids = [int(item[0]) for item in step_rows]
append_ops_job_event(
job_id=int(job_id),
node_code=node_code,
client_event_id=(f"complete:{client_request_id}" if client_request_id else ""),
event_type="agent_completed",
message=f"节点 {node_code} 已完成任务,状态: {job_status}",
level=event_level,
payload={
completed_event = {
"job_id": int(job_id),
"node_code": node_code,
"client_event_id": (f"complete:{client_request_id}" if client_request_id else ""),
"event_type": "agent_completed",
"message": f"节点 {node_code} 已完成任务,状态: {job_status}",
"level": event_level,
"payload": {
"step_ids": step_ids,
"result": result,
"duration_ms": duration_ms,
"summary_text": str(result.get("summary_text") or result.get("summary") or summary_text).strip(),
"focus_ref": focus_ref,
},
)
}
conn.commit()
if completed_event:
append_ops_job_event(**completed_event)
from app.services.ops_release_service import refresh_release_rollout_for_job
refresh_release_rollout_for_job(int(job_id))

View File

@@ -39,6 +39,13 @@ CREATE TABLE IF NOT EXISTS ops_managed_nodes (
updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP
);
CREATE TABLE IF NOT EXISTS ops_managed_node_secrets (
node_code VARCHAR(64) PRIMARY KEY REFERENCES ops_managed_nodes(node_code) ON DELETE CASCADE,
ssh_password TEXT NOT NULL DEFAULT '',
ssh_private_key TEXT NOT NULL DEFAULT '',
updated_at TIMESTAMP NOT NULL DEFAULT CURRENT_TIMESTAMP
);
CREATE TABLE IF NOT EXISTS ops_jobs (
id BIGSERIAL PRIMARY KEY,
job_code VARCHAR(64) NOT NULL UNIQUE,
@@ -321,6 +328,107 @@ def _serialize_node_row(row: tuple) -> dict:
}
def _load_node_secret_flags(node_codes: list[str]) -> dict[str, dict]:
normalized_codes = [str(item or "").strip() for item in node_codes if str(item or "").strip()]
if not normalized_codes:
return {}
result: dict[str, dict] = {}
with get_db() as conn:
with conn.cursor() as cur:
cur.execute(
"""
SELECT node_code, ssh_password, ssh_private_key
FROM ops_managed_node_secrets
WHERE node_code = ANY(%s)
""",
(normalized_codes,),
)
rows = cur.fetchall()
for row in rows:
node_code = str(row[0] or "").strip()
result[node_code] = {
"ssh_password_configured": bool(str(row[1] or "").strip()),
"ssh_private_key_configured": bool(str(row[2] or "").strip()),
}
return result
def _parse_ssh_entry(raw_value: object) -> dict:
raw = str(raw_value or "").strip()
if not raw:
return {}
parts = raw.split(maxsplit=2)
if len(parts) < 2:
return {}
host_port = str(parts[0] or "").strip()
ssh_user = str(parts[1] or "").strip()
secret = str(parts[2] or "").strip() if len(parts) >= 3 else ""
ssh_host = host_port
ssh_port = 22
if ":" in host_port:
host_candidate, port_candidate = host_port.rsplit(":", 1)
if host_candidate and port_candidate.isdigit():
ssh_host = host_candidate
ssh_port = max(int(port_candidate), 1)
if secret.startswith("<") and secret.endswith(">") and len(secret) >= 2:
secret = secret[1:-1].strip()
payload = {
"ssh_host": ssh_host,
"ssh_port": ssh_port,
"ssh_user": ssh_user,
}
if secret:
payload["auth_mode"] = "password"
payload["ssh_password"] = secret
return payload
def _upsert_managed_node_secret(
*,
node_code: str,
ssh_password: str = "",
ssh_private_key: str = "",
clear_ssh_password: bool = False,
clear_ssh_private_key: bool = False,
) -> None:
normalized_node_code = str(node_code or "").strip()
if not normalized_node_code:
return
with get_db() as conn:
with conn.cursor() as cur:
cur.execute(
"""
INSERT INTO ops_managed_node_secrets (
node_code, ssh_password, ssh_private_key, updated_at
) VALUES (%s, %s, %s, CURRENT_TIMESTAMP)
ON CONFLICT (node_code) DO UPDATE SET
ssh_password = CASE
WHEN %s THEN ''
WHEN %s <> '' THEN %s
ELSE ops_managed_node_secrets.ssh_password
END,
ssh_private_key = CASE
WHEN %s THEN ''
WHEN %s <> '' THEN %s
ELSE ops_managed_node_secrets.ssh_private_key
END,
updated_at = CURRENT_TIMESTAMP
""",
(
normalized_node_code,
"" if clear_ssh_password else ssh_password,
"" if clear_ssh_private_key else ssh_private_key,
clear_ssh_password,
ssh_password,
ssh_password,
clear_ssh_private_key,
ssh_private_key,
ssh_private_key,
),
)
conn.commit()
def _pick_text_value(payload: dict, key: str, fallback: str = "", *, default: str = "") -> str:
if key in payload:
normalized = str(payload.get(key) or "").strip()
@@ -397,6 +505,15 @@ def upsert_managed_node(payload: dict) -> tuple[bool, str, dict]:
node_code = str(payload.get("node_code") or "").strip()
if not node_code:
return False, "node_code 不能为空", {}
parsed_ssh_entry = _parse_ssh_entry(payload.get("ssh_entry"))
merged_payload = {
**dict(payload or {}),
**{key: value for key, value in parsed_ssh_entry.items() if value not in ("", None)},
}
ssh_password = str(merged_payload.get("ssh_password") or "").strip()
ssh_private_key = str(merged_payload.get("ssh_private_key") or "")
clear_ssh_password = bool(merged_payload.get("clear_ssh_password", False))
clear_ssh_private_key = bool(merged_payload.get("clear_ssh_private_key", False))
with get_db() as conn:
with conn.cursor() as cur:
@@ -413,22 +530,22 @@ def upsert_managed_node(payload: dict) -> tuple[bool, str, dict]:
existing_row = cur.fetchone()
existing_node = _serialize_node_row(existing_row) if existing_row else {}
existing_metadata = dict(existing_node.get("metadata") or {})
incoming_metadata = dict(payload.get("metadata") or {})
incoming_metadata = dict(merged_payload.get("metadata") or {})
region = _pick_text_value(payload, "region", str(existing_node.get("region") or ""), default="unknown") or "unknown"
role = _pick_text_value(payload, "role", str(existing_node.get("role") or ""), default="worker") or "worker"
title = _pick_text_value(payload, "title", str(existing_node.get("title") or ""), default=node_code) or node_code
ssh_host = _pick_text_value(payload, "ssh_host", str(existing_node.get("ssh_host") or ""))
ssh_port = _pick_int_value(payload, "ssh_port", int(existing_node.get("ssh_port") or 22), default=22, minimum=1)
ssh_user = _pick_text_value(payload, "ssh_user", str(existing_node.get("ssh_user") or ""))
auth_mode = _pick_text_value(payload, "auth_mode", str(existing_node.get("auth_mode") or ""), default="key") or "key"
region = _pick_text_value(merged_payload, "region", str(existing_node.get("region") or ""), default="unknown") or "unknown"
role = _pick_text_value(merged_payload, "role", str(existing_node.get("role") or ""), default="worker") or "worker"
title = _pick_text_value(merged_payload, "title", str(existing_node.get("title") or ""), default=node_code) or node_code
ssh_host = _pick_text_value(merged_payload, "ssh_host", str(existing_node.get("ssh_host") or ""))
ssh_port = _pick_int_value(merged_payload, "ssh_port", int(existing_node.get("ssh_port") or 22), default=22, minimum=1)
ssh_user = _pick_text_value(merged_payload, "ssh_user", str(existing_node.get("ssh_user") or ""))
auth_mode = _pick_text_value(merged_payload, "auth_mode", str(existing_node.get("auth_mode") or ""), default="key") or "key"
deploy_channel = _pick_text_value(
payload,
merged_payload,
"deploy_channel",
str(existing_node.get("deploy_channel") or ""),
default="stable",
) or "stable"
is_enabled = bool(payload["is_enabled"]) if "is_enabled" in payload else bool(existing_node.get("is_enabled", True))
is_enabled = bool(merged_payload["is_enabled"]) if "is_enabled" in merged_payload else bool(existing_node.get("is_enabled", True))
metadata = {
**existing_metadata,
**incoming_metadata,
@@ -469,7 +586,16 @@ def upsert_managed_node(payload: dict) -> tuple[bool, str, dict]:
)
row = cur.fetchone()
conn.commit()
return True, "托管节点已保存", {"node": _serialize_node_row(row)}
_upsert_managed_node_secret(
node_code=node_code,
ssh_password=ssh_password,
ssh_private_key=ssh_private_key,
clear_ssh_password=clear_ssh_password,
clear_ssh_private_key=clear_ssh_private_key,
)
node = _serialize_node_row(row)
node.update(_load_node_secret_flags([node_code]).get(node_code, {}))
return True, "托管节点已保存", {"node": node}
def list_managed_nodes() -> list[dict]:
@@ -485,7 +611,11 @@ def list_managed_nodes() -> list[dict]:
"""
)
rows = cur.fetchall()
return [_serialize_node_row(row) for row in rows]
items = [_serialize_node_row(row) for row in rows]
secret_flags = _load_node_secret_flags([str(item.get("node_code") or "") for item in items])
for item in items:
item.update(secret_flags.get(str(item.get("node_code") or "").strip(), {}))
return items
def sync_managed_nodes_from_cluster(*, dry_run: bool = False) -> dict:

View File

@@ -42,6 +42,7 @@ _CRITICAL_RISK_ACTIONS = {
"deploy.rollback",
"node.bootstrap",
"cluster.reconfigure",
"runtime.reset_lab_state",
}
@@ -166,11 +167,23 @@ def _preview_single_node_policy(
node_detect_participating = bool(target_node.get("detect_participating", False))
if target_node:
if node_status in {"busy"} and action in {"runtime.stop_worker", "runtime.restart_api", "deploy.release", "deploy.rollback", "service.restart"}:
blocking_reasons.append("目标节点当前处于 busy 状态,不适合直接执行中断类动作。")
interrupt_actions = {"runtime.stop_worker", "runtime.restart_api", "service.restart"}
restart_like_actions = {"service.restart"}
rolling_deploy_actions = {"deploy.release", "deploy.rollback"}
if node_detect_participating and action in {"runtime.stop_worker", "runtime.restart_api", "deploy.release", "deploy.rollback", "service.restart"}:
if node_status in {"busy"} and action in interrupt_actions - restart_like_actions:
blocking_reasons.append("目标节点当前处于 busy 状态,不适合直接执行中断类动作。")
elif node_status in {"busy"} and action in restart_like_actions:
warnings.append("目标节点当前处于 busy 状态,重启会带来瞬时抖动,请确认当前窗口可接受。")
elif node_status in {"busy"} and action in rolling_deploy_actions:
warnings.append("目标节点当前处于 busy 状态,滚动发布会触发服务重启,请确认当前窗口可接受短暂抖动。")
if node_detect_participating and action in interrupt_actions - restart_like_actions:
blocking_reasons.append("目标节点正在参与检测,需先迁移负载或人工确认后再执行。")
elif node_detect_participating and action in restart_like_actions:
warnings.append("目标节点正在参与检测,重启会中断当前任务,请确认剩余节点仍可承接负载。")
elif node_detect_participating and action in rolling_deploy_actions:
warnings.append("目标节点正在参与检测,建议优先采用单节点滚动发布,并确认其余节点仍可承接负载。")
if node_role == "control" and action in {"runtime.restart_api", "deploy.release", "deploy.rollback", "service.restart"}:
approval_reasons.append("目标节点是 control 节点,建议强制走审批或维护窗口。")

View File

@@ -1,8 +1,11 @@
from __future__ import annotations
import grp
import hashlib
import inspect
import json
import os
import pwd
import shutil
import tarfile
import textwrap
@@ -13,6 +16,22 @@ from datetime import datetime
from pathlib import Path
_SYSTEMD_TEMPLATE_SPECS = {
"domaincheck-api": {
"template": Path("domain-api/deploy/systemd/domain-api.service"),
},
"domaincheck-worker": {
"template": Path("domain-api/deploy/systemd/domain-worker.service"),
},
"domaincheck-sync-agent": {
"template": Path("domain-api/deploy/systemd/domain-sync-agent.service"),
},
"domaincheck-node-agent": {
"template": Path("domain-api/deploy/systemd/domain-node-agent.service"),
},
}
def normalize_text_list(raw_value: object) -> list[str]:
if isinstance(raw_value, list):
return [str(item).strip() for item in raw_value if str(item).strip()]
@@ -147,6 +166,219 @@ def run_release_health_checks(
}
def ensure_directory_ready(path: Path) -> tuple[bool, dict]:
normalized_path = Path(path).resolve()
try:
normalized_path.mkdir(parents=True, exist_ok=True)
except Exception as exc:
return False, {
"path": str(normalized_path),
"error": str(exc),
"exception_type": exc.__class__.__name__,
}
return True, {"path": str(normalized_path)}
def _resolve_path_owner_group(path: Path) -> tuple[str, str]:
normalized_path = Path(path).resolve()
stat_info = normalized_path.stat()
owner_user = ""
owner_group = ""
try:
owner_user = pwd.getpwuid(stat_info.st_uid).pw_name
except Exception:
owner_user = ""
try:
owner_group = grp.getgrgid(stat_info.st_gid).gr_name
except Exception:
owner_group = ""
return owner_user, owner_group
def collect_service_identity(run_command, service_name: str) -> dict:
code, stdout, stderr = run_command(
["systemctl", "show", service_name, "-p", "User", "-p", "Group", "--value"],
timeout=15,
)
lines = [line.strip() for line in (stdout or stderr or "").splitlines()]
user = lines[0] if len(lines) >= 1 else ""
group = lines[1] if len(lines) >= 2 else ""
return {
"service_name": service_name,
"returncode": int(code or 0),
"user": user,
"group": group,
"ok": int(code or 0) == 0,
}
def _pick_release_owner_group(
releases_dir: Path,
service_identities: list[dict],
) -> tuple[str, str]:
for item in service_identities:
user = str(item.get("user") or "").strip()
group = str(item.get("group") or "").strip()
if user or group:
return user, group
owner_user, owner_group = _resolve_path_owner_group(releases_dir)
if owner_user or owner_group:
return owner_user, owner_group
return "", ""
def apply_release_permissions(
run_command,
*,
release_dir: Path,
owner_user: str,
owner_group: str,
) -> dict:
normalized_release_dir = Path(release_dir).resolve()
normalized_user = str(owner_user or "").strip()
normalized_group = str(owner_group or "").strip()
if not normalized_user and not normalized_group:
return {
"attempted": False,
"release_dir": str(normalized_release_dir),
"owner_user": normalized_user,
"owner_group": normalized_group,
"returncode": 0,
"stdout": "",
"stderr": "",
"ok": True,
}
owner_spec = f"{normalized_user}:{normalized_group}" if normalized_group else normalized_user
code, stdout, stderr = run_command(
["chown", "-R", owner_spec, str(normalized_release_dir)],
timeout=180,
)
return {
"attempted": True,
"release_dir": str(normalized_release_dir),
"owner_user": normalized_user,
"owner_group": normalized_group,
"owner_spec": owner_spec,
"returncode": int(code or 0),
"stdout": stdout,
"stderr": stderr,
"ok": int(code or 0) == 0,
}
def collect_service_execstart(run_command, service_name: str) -> dict:
code, stdout, stderr = run_command(
["systemctl", "show", service_name, "-p", "ExecStart", "--value"],
timeout=15,
)
execstart_value = stdout or stderr
return {
"service_name": service_name,
"returncode": int(code or 0),
"execstart": execstart_value,
"ok": int(code or 0) == 0,
}
def _write_text_file(path: Path, content: str) -> None:
normalized_path = Path(path).resolve()
normalized_path.parent.mkdir(parents=True, exist_ok=True)
normalized_path.write_text(str(content or "").rstrip() + "\n", encoding="utf-8")
def _systemd_dropin_content(service_name: str, install_root: str) -> str:
normalized_service_name = str(service_name or "").strip()
normalized_install_root = str(install_root or "").rstrip("/")
if normalized_service_name == "domaincheck-api":
return "\n".join(
[
"[Service]",
f"WorkingDirectory={normalized_install_root}/current/domain-api",
"ExecStart=",
f"ExecStart={normalized_install_root}/domainCheck/.venv/bin/python -m uvicorn app.main:app --host 0.0.0.0 --port 8100",
]
)
if normalized_service_name == "domaincheck-worker":
return "\n".join(
[
"[Service]",
f"WorkingDirectory={normalized_install_root}/current/domainCheck",
"ExecStart=",
f"ExecStart={normalized_install_root}/domainCheck/.venv/bin/python {normalized_install_root}/current/domainCheck/detect_worker.py",
]
)
if normalized_service_name == "domaincheck-sync-agent":
return "\n".join(
[
"[Service]",
f"WorkingDirectory={normalized_install_root}/current/domain-api",
"ExecStart=",
f"ExecStart={normalized_install_root}/domainCheck/.venv/bin/python -m app.sync_agent",
]
)
if normalized_service_name == "domaincheck-node-agent":
return "\n".join(
[
"[Service]",
"User=root",
"Group=root",
f"WorkingDirectory={normalized_install_root}/current/domain-api",
"ExecStart=",
f"ExecStart={normalized_install_root}/domainCheck/.venv/bin/python -m app.node_agent",
]
)
return ""
def _sync_release_systemd_units(
*,
release_dir: Path,
install_root: Path,
systemd_unit_root: Path,
switch_current: bool,
) -> dict:
normalized_release_dir = Path(release_dir).resolve()
normalized_systemd_root = Path(systemd_unit_root).resolve()
results: list[dict] = []
synced_units: list[str] = []
for service_name, spec in _SYSTEMD_TEMPLATE_SPECS.items():
template_path = normalized_release_dir / Path(spec["template"])
if not template_path.exists():
continue
target_unit_path = normalized_systemd_root / f"{service_name}.service"
target_unit_path.parent.mkdir(parents=True, exist_ok=True)
shutil.copyfile(template_path, target_unit_path)
os.chmod(target_unit_path, 0o644)
dropin_path = normalized_systemd_root / f"{service_name}.service.d" / "current-path.conf"
dropin_written = False
if switch_current:
dropin_content = _systemd_dropin_content(service_name, str(install_root))
if dropin_content:
_write_text_file(dropin_path, dropin_content)
dropin_written = True
results.append(
{
"service_name": service_name,
"template_path": str(template_path),
"target_unit_path": str(target_unit_path),
"dropin_path": str(dropin_path) if dropin_written else "",
"dropin_written": dropin_written,
}
)
synced_units.append(service_name)
return {
"systemd_unit_root": str(normalized_systemd_root),
"synced_units": synced_units,
"results": results,
"daemon_reload_required": bool(synced_units),
}
def safe_extract_tar(archive: tarfile.TarFile, target_dir: Path) -> None:
target_dir_resolved = target_dir.resolve()
members = archive.getmembers()
@@ -189,6 +421,7 @@ def execute_release_action(
health_check_retries = max(0, int(normalized_payload.get("health_check_retries") or 2))
health_check_interval_seconds = max(0, int(normalized_payload.get("health_check_interval_seconds") or 2))
rollback_on_failure = coerce_bool(normalized_payload.get("rollback_on_failure", True), default=True)
systemd_unit_root = Path(str(normalized_payload.get("systemd_unit_root") or "/etc/systemd/system")).resolve()
if not release_version:
return False, "release_version missing", {}
@@ -202,8 +435,57 @@ def execute_release_action(
artifact_path = downloads_dir / f"{release_version}.tar.gz"
current_link = install_root / "current"
previous_current_target = ""
downloads_dir.mkdir(parents=True, exist_ok=True)
releases_dir.mkdir(parents=True, exist_ok=True)
runtime_dirs = [downloads_dir, releases_dir]
prepared_dirs: list[dict] = []
for directory in runtime_dirs:
ok, preparation = ensure_directory_ready(directory)
prepared_dirs.append(preparation)
if not ok:
event_callback(
"deploy_preflight_failed",
f"发布目录不可写: {preparation.get('path') or directory}",
level="error",
payload={
"release_version": release_version,
"install_root": str(install_root),
"prepared_dirs": prepared_dirs,
},
)
return False, f"install_root not writable: {preparation.get('path') or directory}", {
"release_version": release_version,
"install_root": str(install_root),
"prepared_dirs": prepared_dirs,
}
service_execstarts = [
collect_service_execstart(run_command, service_name)
for service_name in restart_services
if str(service_name or "").strip()
]
service_identities = [
collect_service_identity(run_command, service_name)
for service_name in restart_services
if str(service_name or "").strip()
]
current_link_text = str(current_link)
execstart_alignment = {
"switch_current": switch_current,
"current_link": current_link_text,
"services": service_execstarts,
"mismatched_services": [
item.get("service_name")
for item in service_execstarts
if str(item.get("execstart") or "").strip()
and current_link_text not in str(item.get("execstart") or "")
],
}
if switch_current and execstart_alignment["mismatched_services"]:
event_callback(
"deploy_execstart_mismatch",
"检测到目标服务 ExecStart 未引用 current 软链,发布后可能不会切到新版本",
level="warning",
payload=execstart_alignment,
)
if current_link.exists():
try:
@@ -259,6 +541,38 @@ def execute_release_action(
temp_dir.rename(target_dir)
extracted_target = target_dir
release_owner_user, release_owner_group = _pick_release_owner_group(releases_dir, service_identities)
permission_result = apply_release_permissions(
run_command,
release_dir=extracted_target,
owner_user=release_owner_user,
owner_group=release_owner_group,
)
if not permission_result.get("ok", False):
event_callback(
"deploy_permission_fix_failed",
f"发布目录权限修正失败: {release_version}",
level="error",
payload=permission_result,
)
return False, "release permission fix failed", {
"release_version": release_version,
"release_dir": str(extracted_target),
"artifact_path": str(artifact_path),
"checksum": calculated_checksum,
"previous_current_target": previous_current_target,
"prepared_dirs": prepared_dirs,
"execstart_alignment": execstart_alignment,
"service_identities": service_identities,
"permission_result": permission_result,
}
if permission_result.get("attempted"):
event_callback(
"deploy_permissions_aligned",
f"发布目录权限已对齐: {release_version}",
payload=permission_result,
)
meta_path = extracted_target / ".release-meta.json"
meta_path.write_text(
json.dumps(
@@ -274,6 +588,50 @@ def execute_release_action(
encoding="utf-8",
)
systemd_sync_result = _sync_release_systemd_units(
release_dir=extracted_target,
install_root=install_root,
systemd_unit_root=systemd_unit_root,
switch_current=switch_current,
)
if systemd_sync_result.get("synced_units"):
event_callback(
"deploy_systemd_units_synced",
f"systemd 单元已同步: {', '.join(systemd_sync_result.get('synced_units') or [])}",
payload=systemd_sync_result,
)
daemon_reload_result = {
"returncode": 0,
"stdout": "",
"stderr": "",
}
if systemd_sync_result.get("daemon_reload_required"):
code, stdout, stderr = run_command(["systemctl", "daemon-reload"], timeout=45)
daemon_reload_result = {
"returncode": int(code or 0),
"stdout": stdout,
"stderr": stderr,
}
if int(code or 0) != 0:
return False, "systemd daemon-reload failed", {
"release_version": release_version,
"release_dir": str(extracted_target),
"artifact_path": str(artifact_path),
"checksum": calculated_checksum,
"previous_current_target": previous_current_target,
"prepared_dirs": prepared_dirs,
"execstart_alignment": execstart_alignment,
"service_identities": service_identities,
"permission_result": permission_result,
"systemd_sync": systemd_sync_result,
"daemon_reload": daemon_reload_result,
}
event_callback(
"deploy_systemd_reloaded",
"systemd daemon-reload 完成",
payload=daemon_reload_result,
)
if switch_current:
if current_link.is_symlink() or current_link.is_file():
current_link.unlink(missing_ok=True)
@@ -312,6 +670,12 @@ def execute_release_action(
"artifact_path": str(artifact_path),
"checksum": calculated_checksum,
"previous_current_target": previous_current_target,
"prepared_dirs": prepared_dirs,
"execstart_alignment": execstart_alignment,
"service_identities": service_identities,
"permission_result": permission_result,
"systemd_sync": systemd_sync_result,
"daemon_reload": daemon_reload_result,
"restart_results": restarted,
}
@@ -380,6 +744,10 @@ def execute_release_action(
"checksum": calculated_checksum,
"current_link": str(current_link),
"previous_current_target": previous_current_target,
"prepared_dirs": prepared_dirs,
"execstart_alignment": execstart_alignment,
"systemd_sync": systemd_sync_result,
"daemon_reload": daemon_reload_result,
"restart_results": restarted,
"health_check": health_result,
"rollback": rollback_result,
@@ -397,6 +765,10 @@ def execute_release_action(
"checksum": calculated_checksum,
"current_link": str(current_link),
"previous_current_target": previous_current_target,
"prepared_dirs": prepared_dirs,
"execstart_alignment": execstart_alignment,
"systemd_sync": systemd_sync_result,
"daemon_reload": daemon_reload_result,
"restart_results": restarted,
"health_check": health_result,
}
@@ -415,6 +787,15 @@ def build_remote_release_action_script(
collect_service_state,
check_health_url,
run_release_health_checks,
ensure_directory_ready,
_resolve_path_owner_group,
collect_service_identity,
_pick_release_owner_group,
apply_release_permissions,
collect_service_execstart,
_write_text_file,
_systemd_dropin_content,
_sync_release_systemd_units,
safe_extract_tar,
execute_release_action,
]
@@ -422,6 +803,17 @@ def build_remote_release_action_script(
textwrap.dedent(inspect.getsource(func)).strip("\n")
for func in helper_functions
)
systemd_template_specs_source = (
"_SYSTEMD_TEMPLATE_SPECS = "
+ repr(
{
service_name: {
"template": str(spec["template"]),
}
for service_name, spec in _SYSTEMD_TEMPLATE_SPECS.items()
}
)
)
return f"""from __future__ import annotations
import hashlib
@@ -435,12 +827,15 @@ from datetime import datetime
from pathlib import Path
{systemd_template_specs_source}
{helper_source}
PAYLOAD = {json.dumps(dict(payload or {{}}), ensure_ascii=False)}
DEFAULT_API_SERVICE_NAME = {json.dumps(str(default_api_service_name or 'domaincheck-api'), ensure_ascii=False)}
USER_AGENT = {json.dumps(str(user_agent or 'domaincheck-ssh/0.1'), ensure_ascii=False)}
PAYLOAD = {repr(dict(payload or {}))}
DEFAULT_API_SERVICE_NAME = {repr(str(default_api_service_name or 'domaincheck-api'))}
USER_AGENT = {repr(str(user_agent or 'domaincheck-ssh/0.1'))}
def _run(command, timeout=60):

View File

@@ -3,11 +3,13 @@ from __future__ import annotations
import json
import os
import re
import socket
import subprocess
from datetime import datetime
from math import ceil
from pathlib import Path
from threading import Lock
from urllib.parse import urlsplit, urlunsplit
from uuid import uuid4
from app.core.db import get_db
@@ -490,7 +492,119 @@ def _normalize_public_base_url(raw_value: str) -> str:
normalized = str(raw_value or "").strip().rstrip("/")
if normalized.endswith("/api/v1"):
normalized = normalized[: -len("/api/v1")]
return normalized
return _rewrite_loopback_control_plane_url(normalized)
def _is_loopback_hostname(hostname: str) -> bool:
normalized = str(hostname or "").strip().lower().strip("[]")
return normalized in {"127.0.0.1", "localhost", "0.0.0.0", "::1"}
def _build_url_with_host(raw_url: str, *, host: str, scheme: str = "", port: int | None = None) -> str:
normalized_url = str(raw_url or "").strip()
if not normalized_url:
return ""
parsed = urlsplit(normalized_url)
if not parsed.scheme or not parsed.netloc:
return normalized_url
normalized_host = str(host or "").strip().strip("[]")
if not normalized_host:
return normalized_url
final_scheme = str(scheme or parsed.scheme or "http").strip() or "http"
final_port = parsed.port if port is None else int(port)
netloc = f"{normalized_host}:{final_port}" if final_port else normalized_host
return urlunsplit((final_scheme, netloc, parsed.path, parsed.query, parsed.fragment))
def _resolve_public_control_plane_origin(loopback_url: str) -> str:
normalized_loopback_url = str(loopback_url or "").strip()
if not normalized_loopback_url:
return ""
parsed_loopback = urlsplit(normalized_loopback_url)
default_scheme = str(parsed_loopback.scheme or "http").strip() or "http"
default_port = parsed_loopback.port
env_candidates = [
os.getenv("OPS_CONTROL_PLANE_PUBLIC_BASE_URL", ""),
os.getenv("CONTROL_PLANE_PUBLIC_BASE_URL", ""),
os.getenv("OPS_CONTROL_PLANE_BASE_URL", ""),
]
for candidate in env_candidates:
normalized_candidate = str(candidate or "").strip().rstrip("/")
if not normalized_candidate:
continue
parsed_candidate = urlsplit(
normalized_candidate if "://" in normalized_candidate else f"{default_scheme}://{normalized_candidate}"
)
candidate_host = str(parsed_candidate.hostname or "").strip()
if candidate_host and not _is_loopback_hostname(candidate_host):
return _build_url_with_host(
normalized_loopback_url,
host=candidate_host,
scheme=str(parsed_candidate.scheme or default_scheme),
port=parsed_candidate.port if parsed_candidate.port is not None else default_port,
)
try:
from app.services.cluster_runtime_service import get_cluster_snapshot
snapshot = get_cluster_snapshot()
local_hostnames = {
str(socket.gethostname() or "").strip().lower(),
str(socket.getfqdn() or "").strip().lower(),
}
fallback_control_hosts: list[str] = []
for item in list(snapshot.get("nodes") or []):
if str(item.get("role") or "").strip() != "control":
continue
control_host = str(item.get("hostname") or "").strip().lower()
control_ip = str(item.get("ip") or "").strip()
if not control_ip or _is_loopback_hostname(control_ip):
continue
if control_host and control_host in local_hostnames:
return _build_url_with_host(
normalized_loopback_url,
host=control_ip,
scheme=default_scheme,
port=default_port,
)
fallback_control_hosts.append(control_ip)
for control_ip in fallback_control_hosts:
if control_ip and not _is_loopback_hostname(control_ip):
return _build_url_with_host(
normalized_loopback_url,
host=control_ip,
scheme=default_scheme,
port=default_port,
)
except Exception:
pass
try:
resolved_host = str(socket.gethostbyname(socket.gethostname()) or "").strip()
if resolved_host and not _is_loopback_hostname(resolved_host):
return _build_url_with_host(
normalized_loopback_url,
host=resolved_host,
scheme=default_scheme,
port=default_port,
)
except Exception:
pass
return ""
def _rewrite_loopback_control_plane_url(raw_url: str) -> str:
normalized = str(raw_url or "").strip()
if not normalized:
return ""
parsed = urlsplit(normalized)
if not parsed.scheme or not parsed.netloc:
return normalized
if not _is_loopback_hostname(str(parsed.hostname or "").strip()):
return normalized
resolved = _resolve_public_control_plane_origin(normalized)
return resolved or normalized
def _build_absolute_release_package_url(base_url: str, raw_path: str) -> str:
@@ -1684,7 +1798,7 @@ def _resolve_rollout_targets(selector: dict) -> list[dict]:
continue
if only_effective_workers and not bool(item.get("is_effective_worker", False)):
continue
if only_online and str(item.get("status") or "") != "online":
if only_online and str(item.get("status") or "") not in {"online", "busy"}:
continue
targets.append(item)
@@ -2037,9 +2151,12 @@ def _build_smart_rollout_role_policy(mode: str, *, execution_mode: str = "remote
"restart_services": ["domaincheck-api", "domaincheck-worker", "domaincheck-sync-agent"],
"health_check_urls": ["http://127.0.0.1:8100/health"],
"health_check_services": ["domaincheck-api", "domaincheck-worker", "domaincheck-sync-agent"],
"health_check_timeout_seconds": 10,
"health_check_retries": 2,
"health_check_interval_seconds": 2,
# Control 节点启动期间会先经历较长的 import / startup hook
# systemd 已经 active 但 /health 仍可能在 15-20 秒内拒绝连接。
# 这里把健康检查窗口放宽到约 40 秒,避免被误回滚。
"health_check_timeout_seconds": 20,
"health_check_retries": 9,
"health_check_interval_seconds": 4,
"rollback_on_failure": True,
"switch_current": True,
}
@@ -3254,18 +3371,56 @@ def refresh_release_rollout_for_job(job_id: int) -> dict:
return refresh_release_rollout(rollout_id)
def _build_release_job_payload(release: dict, rollout: dict) -> dict:
def _default_release_deploy_payload_for_target(target: dict) -> dict:
role = str((target or {}).get("role") or "").strip().lower()
if role == "control":
return {
"restart_services": ["domaincheck-api", "domaincheck-worker", "domaincheck-sync-agent"],
"health_check_urls": ["http://127.0.0.1:8100/health"],
"health_check_services": ["domaincheck-api", "domaincheck-worker", "domaincheck-sync-agent"],
"health_check_timeout_seconds": 20,
"health_check_retries": 9,
"health_check_interval_seconds": 4,
}
return {
"restart_services": ["domaincheck-worker"],
"health_check_urls": [],
"health_check_services": ["domaincheck-worker"],
"health_check_timeout_seconds": 10,
"health_check_retries": 2,
"health_check_interval_seconds": 2,
}
def _build_release_job_payload(release: dict, rollout: dict, *, target: dict | None = None) -> dict:
policy = dict(rollout.get("policy") or {})
deploy_payload = dict(policy.get("deploy_payload") or {})
target_defaults = _default_release_deploy_payload_for_target(target or {})
if not [str(item).strip() for item in list(deploy_payload.get("restart_services") or []) if str(item).strip()]:
deploy_payload["restart_services"] = list(target_defaults.get("restart_services") or [])
if not [str(item).strip() for item in list(deploy_payload.get("health_check_services") or []) if str(item).strip()]:
deploy_payload["health_check_services"] = list(target_defaults.get("health_check_services") or [])
if not [str(item).strip() for item in list(deploy_payload.get("health_check_urls") or []) if str(item).strip()]:
deploy_payload["health_check_urls"] = list(target_defaults.get("health_check_urls") or [])
if deploy_payload.get("health_check_timeout_seconds") in (None, "", 0, "0"):
deploy_payload["health_check_timeout_seconds"] = int(target_defaults.get("health_check_timeout_seconds") or 10)
if deploy_payload.get("health_check_retries") in (None, "", 0, "0"):
deploy_payload["health_check_retries"] = int(target_defaults.get("health_check_retries") or 2)
if deploy_payload.get("health_check_interval_seconds") in (None, "", 0, "0"):
deploy_payload["health_check_interval_seconds"] = int(target_defaults.get("health_check_interval_seconds") or 2)
artifact_url = _rewrite_loopback_control_plane_url(str(release.get("artifact_url") or "").strip())
return {
"release_id": int(release.get("id") or 0),
"rollout_id": int(rollout.get("id") or 0),
"release_version": str(release.get("release_version") or ""),
"artifact_url": str(release.get("artifact_url") or ""),
"artifact_url": artifact_url,
"checksum": str(release.get("checksum") or ""),
"channel": str(release.get("channel") or ""),
"commit_sha": str(release.get("commit_sha") or ""),
"notes": str(release.get("notes") or ""),
"target_node_role": str((target or {}).get("role") or "").strip(),
**deploy_payload,
}
@@ -3308,12 +3463,11 @@ def _enqueue_rollout_batch(rollout_id: int, *, created_by: str, reason: str = "m
auto_dispatch = bool(policy.get("auto_dispatch", False))
auto_approve = bool(policy.get("auto_approve", False))
execution_mode = str(policy.get("execution_mode") or "remote-agent").strip() or "remote-agent"
job_payload = _build_release_job_payload(release, rollout)
for target in batch_targets:
target_node_code = str(target.get("node_code") or "").strip()
if not target_node_code:
continue
job_payload = _build_release_job_payload(release, rollout, target=target)
job_ok, _job_message, job_data = create_ops_job(
{
"action": "deploy.release",

View File

@@ -1,8 +1,16 @@
from __future__ import annotations
import json
import shlex
import subprocess
from io import StringIO
try:
import paramiko
except ImportError: # pragma: no cover - exercised via graceful fallback tests
paramiko = None
from app.core.db import get_db
from app.core.config import settings
from app.services.ops_action_executor_core import (
STRUCTURED_ACTIONS,
@@ -76,6 +84,7 @@ def execute_ssh_action(node: dict, action: str, payload: dict | None = None) ->
}
normalized_payload = dict(payload or {})
node_secret = _load_ssh_secret(node_code)
service_names = _service_name_map()
if normalized_action == "deploy.release":
remote_script = build_remote_release_action_script(
@@ -105,28 +114,69 @@ def execute_ssh_action(node: dict, action: str, payload: dict | None = None) ->
"PY",
]
)
ssh_command = [
"ssh",
"-o",
"BatchMode=yes",
"-o",
"PreferredAuthentications=publickey",
"-o",
"StrictHostKeyChecking=accept-new",
"-o",
f"ConnectTimeout={_SSH_CONNECT_TIMEOUT_SECONDS}",
"-p",
str(ssh_port),
f"{ssh_user}@{ssh_host}",
remote_command,
]
timeout_seconds = int(_SSH_REMOTE_TIMEOUT_SECONDS.get(normalized_action, 45) or 45) + _SSH_CONNECT_TIMEOUT_SECONDS
completed = subprocess.run(
ssh_command,
capture_output=True,
text=True,
timeout=timeout_seconds,
)
auth_mode = str(node.get("auth_mode") or "").strip() or ("password" if node_secret.get("ssh_password") else "key")
ssh_password = str(node_secret.get("ssh_password") or "").strip()
ssh_private_key = str(node_secret.get("ssh_private_key") or "").strip()
if auth_mode == "key" and not ssh_private_key and ssh_password:
auth_mode = "password"
elif auth_mode == "password" and not ssh_password and ssh_private_key:
auth_mode = "key"
if (auth_mode == "password" and ssh_password) or ssh_private_key:
if paramiko is None:
return False, "当前环境未安装 paramiko无法使用密码或私钥 SSH 执行", {
"executor": "ssh",
"action": normalized_action,
"transport": {
"executor": "ssh",
"node_code": node_code,
"ssh_host": ssh_host,
"ssh_user": ssh_user,
"ssh_port": ssh_port,
"auth_mode": auth_mode,
"action": normalized_action,
},
}
if auth_mode == "password" and ssh_password:
completed = _run_paramiko_command(
ssh_host=ssh_host,
ssh_port=ssh_port,
ssh_user=ssh_user,
remote_command=remote_command,
timeout_seconds=timeout_seconds,
ssh_password=ssh_password,
)
elif ssh_private_key:
completed = _run_paramiko_command(
ssh_host=ssh_host,
ssh_port=ssh_port,
ssh_user=ssh_user,
remote_command=remote_command,
timeout_seconds=timeout_seconds,
ssh_private_key=ssh_private_key,
)
else:
ssh_command = [
"ssh",
"-o",
"BatchMode=yes",
"-o",
"PreferredAuthentications=publickey",
"-o",
"StrictHostKeyChecking=accept-new",
"-o",
f"ConnectTimeout={_SSH_CONNECT_TIMEOUT_SECONDS}",
"-p",
str(ssh_port),
f"{ssh_user}@{ssh_host}",
remote_command,
]
completed = subprocess.run(
ssh_command,
capture_output=True,
text=True,
timeout=timeout_seconds,
)
transport = {
"executor": "ssh",
@@ -134,6 +184,7 @@ def execute_ssh_action(node: dict, action: str, payload: dict | None = None) ->
"ssh_host": ssh_host,
"ssh_user": ssh_user,
"ssh_port": ssh_port,
"auth_mode": auth_mode,
"action": normalized_action,
"returncode": int(completed.returncode or 0),
}
@@ -166,6 +217,93 @@ def execute_ssh_action(node: dict, action: str, payload: dict | None = None) ->
}
def _load_ssh_secret(node_code: str) -> dict:
normalized_node_code = str(node_code or "").strip()
if not normalized_node_code:
return {}
try:
with get_db() as conn:
with conn.cursor() as cur:
cur.execute(
"""
SELECT ssh_password, ssh_private_key
FROM ops_managed_node_secrets
WHERE node_code = %s
LIMIT 1
""",
(normalized_node_code,),
)
row = cur.fetchone()
except Exception:
return {}
if not row:
return {}
return {
"ssh_password": str(row[0] or ""),
"ssh_private_key": str(row[1] or ""),
}
def _load_private_key(private_key_text: str) -> paramiko.PKey:
if paramiko is None:
raise RuntimeError("paramiko is not installed")
key_text = str(private_key_text or "")
for key_cls in (paramiko.Ed25519Key, paramiko.RSAKey, paramiko.ECDSAKey, paramiko.DSSKey):
try:
return key_cls.from_private_key(StringIO(key_text))
except Exception:
continue
raise ValueError("无法识别 SSH 私钥格式")
def _run_paramiko_command(
*,
ssh_host: str,
ssh_port: int,
ssh_user: str,
remote_command: str,
timeout_seconds: int,
ssh_password: str = "",
ssh_private_key: str = "",
) -> subprocess.CompletedProcess:
if paramiko is None:
raise RuntimeError("paramiko is not installed")
client = paramiko.SSHClient()
client.set_missing_host_key_policy(paramiko.AutoAddPolicy())
connect_kwargs = {
"hostname": ssh_host,
"port": int(ssh_port),
"username": ssh_user,
"timeout": _SSH_CONNECT_TIMEOUT_SECONDS,
"banner_timeout": _SSH_CONNECT_TIMEOUT_SECONDS,
"auth_timeout": _SSH_CONNECT_TIMEOUT_SECONDS,
"look_for_keys": False,
"allow_agent": False,
}
if ssh_password:
connect_kwargs["password"] = ssh_password
elif ssh_private_key:
connect_kwargs["pkey"] = _load_private_key(ssh_private_key)
else:
connect_kwargs["look_for_keys"] = True
connect_kwargs["allow_agent"] = True
try:
client.connect(**connect_kwargs)
wrapped_command = f"bash -lc {shlex.quote(remote_command)}"
_, stdout, stderr = client.exec_command(wrapped_command, timeout=timeout_seconds)
returncode = int(stdout.channel.recv_exit_status())
stdout_text = stdout.read().decode("utf-8", errors="replace")
stderr_text = stderr.read().decode("utf-8", errors="replace")
return subprocess.CompletedProcess(
args=["paramiko", f"{ssh_user}@{ssh_host}"],
returncode=returncode,
stdout=stdout_text,
stderr=stderr_text,
)
finally:
client.close()
def _service_name_map() -> dict[str, str]:
return build_service_name_map(
api_service_name=settings.api_service_name,
@@ -199,6 +337,25 @@ def trim_output(text, limit):
def run(cmd, timeout=60):
completed = subprocess.run(cmd, capture_output=True, text=True, timeout=timeout)
normalized_cmd = [str(part or "").strip() for part in cmd]
combined_output = f"{{completed.stdout or ''}}\\n{{completed.stderr or ''}}".lower()
needs_sudo_retry = (
normalized_cmd
and normalized_cmd[0] == "systemctl"
and completed.returncode != 0
and "sudo" not in normalized_cmd
and any(
marker in combined_output
for marker in (
"interactive authentication required",
"authentication is required",
"authorization not available",
"polkit",
)
)
)
if needs_sudo_retry:
completed = subprocess.run(["sudo", "-n", *normalized_cmd], capture_output=True, text=True, timeout=timeout)
return int(completed.returncode or 0), str(completed.stdout or "").strip(), str(completed.stderr or "").strip()

View File

@@ -6,6 +6,7 @@ from pathlib import Path
from app.core.config import settings
from app.services.debug_event_service import push_debug_event
from app.services.detect_job_service import process_detect_pipeline_now
from app.services.sync_push_service import pull_detect_task_batch_now, push_runtime_projection_now
from app.services.runtime_settings_service import get_runtime_settings
from app.services.worker_control_service import _run_systemctl, normalize_systemctl_error, send_worker_command, start_worker, stop_worker
@@ -180,6 +181,16 @@ def runtime_action(action: str, payload: dict | None = None) -> tuple[bool, str,
result = _build_runtime_action_result(action=normalized_action, poll_after_seconds=2, refresh_runtime=True, ok=ok, message=message, data=data)
_emit_runtime_action_event(normalized_action, stage="finished", ok=ok, message=message, data=result)
return ok, message, result
if normalized_action == "process_pipeline":
process_limit = normalized_payload.get("limit")
process_job_id = normalized_payload.get("job_id")
ok, message, data = process_detect_pipeline_now(
limit=int(process_limit or 0) or None,
job_id=int(process_job_id or 0) or None,
)
result = _build_runtime_action_result(action=normalized_action, poll_after_seconds=1, refresh_runtime=True, ok=ok, message=message, data=data)
_emit_runtime_action_event(normalized_action, stage="finished", ok=ok, message=message, data=result)
return ok, message, result
if normalized_action == "start_detection":
command_ok, command_message = send_worker_command(
"start_detection",

View File

@@ -10,12 +10,151 @@ from app.core.redis_client import get_redis
from app.services.build_info_service import get_runtime_build_info
from app.services.cluster_runtime_service import get_cluster_snapshot
from app.services.detect_service import get_detect_status
from app.services.detect_job_service import get_detect_capacity_plan, get_detect_queue_health
from app.services.detect_job_service import (
_load_latest_runtime_active_job_snapshot,
get_detect_capacity_plan,
get_detect_queue_health,
)
from app.services.sync_record_service import append_runtime_projection_if_changed, get_sync_summary
from app.services.runtime_settings_service import get_runtime_settings
from app.services.worker_control_service import detect_sync_agent_runtime, detect_worker_runtime
def _align_queue_health_with_backlog(queue_health: dict | None, backlog_snapshot: dict | None) -> dict:
normalized = dict(queue_health or {})
queue = dict(normalized.get("queue") or {})
backlog = dict(backlog_snapshot or {})
pending_total = max(int(queue.get("pending", 0) or 0), int(backlog.get("pending_total", 0) or 0))
claimed_total = max(int(queue.get("claimed", 0) or 0), int(backlog.get("claimed_total", 0) or 0))
running_total = max(int(queue.get("running", 0) or 0), int(backlog.get("running_total", 0) or 0))
completed_total = max(int(queue.get("completed", 0) or 0), int(backlog.get("completed_total", 0) or 0))
blacklisted_total = max(int(queue.get("blacklisted", 0) or 0), int(backlog.get("blacklisted_total", 0) or 0))
failed_total = max(int(queue.get("failed", 0) or 0), int(backlog.get("failed_total", 0) or 0))
terminal_total = max(
int(queue.get("terminal", 0) or 0),
completed_total + blacklisted_total + failed_total,
)
normalized["has_active_job"] = bool(
normalized.get("has_active_job")
or pending_total > 0
or claimed_total > 0
or running_total > 0
or terminal_total > 0
)
normalized["queue"] = {
**queue,
"items_total": pending_total + claimed_total + running_total + terminal_total,
"pending": pending_total,
"claimed": claimed_total,
"running": running_total,
"completed": completed_total,
"blacklisted": blacklisted_total,
"failed": failed_total,
"terminal": terminal_total,
}
return normalized
def _decode_projection_payload(value: object) -> dict:
if isinstance(value, dict):
return dict(value)
if value in (None, ""):
return {}
try:
import json
return dict(json.loads(value))
except Exception:
return {}
def _load_detect_backlog_snapshot() -> dict:
with get_db() as conn:
with conn.cursor() as cur:
cur.execute(
"""
SELECT
COUNT(*) FILTER (WHERE item.status = 'pending') AS pending_total,
COUNT(*) FILTER (WHERE item.status = 'claimed') AS claimed_total,
COUNT(*) FILTER (WHERE item.status = 'running') AS running_total,
COUNT(*) FILTER (WHERE item.status = 'completed') AS completed_total,
COUNT(*) FILTER (WHERE item.status = 'blacklisted') AS blacklisted_total,
COUNT(*) FILTER (WHERE item.status = 'failed') AS failed_total,
COUNT(*) FILTER (WHERE item.status = 'pending' AND item.step_code = 'detect_register') AS register_pending,
COUNT(*) FILTER (WHERE item.status = 'pending' AND item.step_code <> 'detect_register') AS downstream_pending
FROM detect_job_items item
JOIN detect_jobs job ON job.id = item.job_id
WHERE job.status IN ('pending', 'running')
"""
)
row = cur.fetchone() or (0, 0, 0, 0, 0, 0, 0, 0)
return {
"pending_total": int(row[0] or 0),
"claimed_total": int(row[1] or 0),
"running_total": int(row[2] or 0),
"completed_total": int(row[3] or 0),
"blacklisted_total": int(row[4] or 0),
"failed_total": int(row[5] or 0),
"register_pending": int(row[6] or 0),
"downstream_pending": int(row[7] or 0),
}
def _load_latest_remote_runtime_projection_backlog() -> dict:
if not (settings.node_region == "overseas" and settings.node_role == "control"):
return {}
with get_db() as conn:
with conn.cursor() as cur:
cur.execute(
"""
SELECT payload_json
FROM detect_sync_records
WHERE sync_type = 'runtime_projection'
AND source_region = 'mainland'
AND target_region = 'overseas'
AND status IN ('projected', 'pushing', 'synced')
ORDER BY updated_at DESC, id DESC
LIMIT 1
"""
)
row = cur.fetchone()
if not row:
return {}
payload = _decode_projection_payload(row[0])
projection = payload.get("projection") if isinstance(payload, dict) else {}
backlog = projection.get("backlog") if isinstance(projection, dict) else {}
if not isinstance(backlog, dict):
return {}
return {
"pending_total": int(backlog.get("pending_total", 0) or 0),
"claimed_total": int(backlog.get("claimed_total", 0) or 0),
"running_total": int(backlog.get("running_total", 0) or 0),
"completed_total": int(backlog.get("completed_total", 0) or 0),
"blacklisted_total": int(backlog.get("blacklisted_total", 0) or 0),
"failed_total": int(backlog.get("failed_total", 0) or 0),
"register_pending": int(backlog.get("register_pending", 0) or 0),
"downstream_pending": int(backlog.get("downstream_pending", 0) or 0),
}
def _merge_backlog_snapshots(primary: dict, secondary: dict) -> dict:
merged = dict(primary or {})
for key in (
"pending_total",
"claimed_total",
"running_total",
"completed_total",
"blacklisted_total",
"failed_total",
"register_pending",
"downstream_pending",
):
merged[key] = max(int(merged.get(key, 0) or 0), int((secondary or {}).get(key, 0) or 0))
return merged
def _runtime_log_path(filename: str) -> str:
path = Path(__file__).resolve().parents[2] / "runtime" / "logs" / filename
return str(path)
@@ -191,6 +330,8 @@ def _build_multi_region_readiness(
def _detect_participation_snapshot(*, row: dict) -> dict:
items_running = int(row.get("items_running", 0) or 0)
items_claimed = int(row.get("items_claimed", 0) or 0)
active_threads = int(row.get("active_threads", 0) or 0)
max_threads = int(row.get("max_threads", 0) or 0)
processed_recent = int(row.get("processed_recent", 0) or 0)
current_load = int(row.get("current_load", 0) or 0)
status = str(row.get("status") or "").strip().lower()
@@ -211,6 +352,17 @@ def _detect_participation_snapshot(*, row: dict) -> dict:
"is_current_participant": True,
"is_dispatch_active": True,
}
if active_threads > 0:
detail = f"当前活跃线程 {active_threads}"
if max_threads > 0:
detail = f"{detail}/{max_threads}"
return {
"participation_state": "runtime_active",
"participation_label": "执行中",
"participation_reason": detail,
"is_current_participant": True,
"is_dispatch_active": True,
}
if processed_recent > 0:
return {
"participation_state": "recent_throughput",
@@ -242,11 +394,52 @@ def _build_detect_node_row(*, node_code: str, cluster_node: dict, job_node: dict
role = str(cluster_node.get("role") or job_node.get("role") or "worker")
region = str(cluster_node.get("region") or settings.node_region)
is_effective_worker = bool(cluster_node.get("is_effective_worker", False) or role == "worker")
items_total = int(job_node.get("items_total", metadata.get("job_items_total", 0)) or 0)
items_claimed = int(job_node.get("items_claimed", metadata.get("job_items_claimed", 0)) or 0)
items_running = int(job_node.get("items_running", metadata.get("job_items_running", 0)) or 0)
items_completed = int(job_node.get("items_completed", metadata.get("job_items_completed", 0)) or 0)
items_failed = int(job_node.get("items_failed", metadata.get("job_items_failed", 0)) or 0)
items_total = max(
int(job_node.get("items_total", 0) or 0),
int(queue_node.get("items_total", 0) or 0),
int(metadata.get("job_items_total", 0) or 0),
)
items_claimed = max(
int(job_node.get("items_claimed", 0) or 0),
int(queue_node.get("items_claimed", 0) or 0),
int(metadata.get("job_items_claimed", 0) or 0),
)
items_running = max(
int(job_node.get("items_running", 0) or 0),
int(queue_node.get("items_running", 0) or 0),
int(metadata.get("job_items_running", 0) or 0),
)
items_completed = max(
int(job_node.get("items_completed", 0) or 0),
int(queue_node.get("items_completed", 0) or 0),
int(metadata.get("job_items_completed", 0) or 0),
)
items_failed = max(
int(job_node.get("items_failed", 0) or 0),
int(queue_node.get("items_failed", 0) or 0),
int(metadata.get("job_items_failed", 0) or 0),
)
items_blacklisted = max(
int(job_node.get("items_blacklisted", 0) or 0),
int(queue_node.get("items_blacklisted", 0) or 0),
)
active_threads = max(
int((cluster_node.get("metadata") or {}).get("active_threads", 0) or 0),
int(job_node.get("active_threads", 0) or 0),
int(queue_node.get("active_threads", 0) or 0),
)
max_threads = max(
int((cluster_node.get("metadata") or {}).get("max_threads", 0) or 0),
int(job_node.get("max_threads", 0) or 0),
int(queue_node.get("max_threads", 0) or 0),
)
current_load = max(items_running, active_threads, 0)
derived_pending = max(items_total - items_claimed - items_running - items_completed - items_failed - items_blacklisted, 0)
items_pending = max(
int(job_node.get("items_pending", 0) or 0),
int(queue_node.get("items_pending", 0) or 0),
derived_pending,
)
row = {
"node_code": node_code,
"role": role,
@@ -254,13 +447,16 @@ def _build_detect_node_row(*, node_code: str, cluster_node: dict, job_node: dict
"status": str(cluster_node.get("status") or "unknown"),
"is_effective_worker": is_effective_worker,
"detect_participating": False,
"current_load": int(cluster_node.get("current_load", 0) or 0),
"current_load": current_load,
"items_total": items_total,
"items_pending": int(job_node.get("items_pending", max(items_total - items_claimed - items_completed - items_failed, 0)) or 0),
"items_pending": items_pending,
"items_claimed": items_claimed,
"items_running": items_running,
"items_completed": items_completed,
"items_blacklisted": items_blacklisted,
"items_failed": items_failed,
"active_threads": active_threads,
"max_threads": max_threads,
"processed_recent": int(queue_node.get("processed_recent", 0) or 0),
"processed_per_minute": float(queue_node.get("processed_per_minute", 0) or 0),
"last_heartbeat_at": str(cluster_node.get("last_heartbeat_at") or ""),
@@ -465,8 +661,13 @@ def get_runtime_status() -> dict:
effective_online_worker_nodes = int((cluster_snapshot.get("summary") or {}).get("online_worker_nodes", 0) or 0)
if effective_online_worker_nodes <= 0 and worker_runtime.get("running", False):
effective_online_worker_nodes = max(1, worker_runtime.get("process_count", 1) or 1)
backlog_snapshot = _load_detect_backlog_snapshot()
remote_backlog_snapshot = _load_latest_remote_runtime_projection_backlog()
runtime_snapshot_backlog = dict(_load_latest_runtime_active_job_snapshot(15).get("backlog") or {})
backlog_snapshot = _merge_backlog_snapshots(backlog_snapshot, remote_backlog_snapshot)
backlog_snapshot = _merge_backlog_snapshots(backlog_snapshot, runtime_snapshot_backlog)
capacity_plan = get_detect_capacity_plan(
queue_health=queue_health,
queue_health=_align_queue_health_with_backlog(queue_health, backlog_snapshot),
online_worker_nodes=effective_online_worker_nodes,
target_finish_hours=6,
)
@@ -498,6 +699,7 @@ def get_runtime_status() -> dict:
"worker_online": worker_runtime.get("running", False),
"worker_mode": worker_runtime.get("mode", runtime_settings.get("worker_mode", "windows-local")),
"queue_health": queue_health,
"backlog": backlog_snapshot,
"capacity_plan": capacity_plan,
"log_sync": {
"enabled": bool(runtime_settings.get("worker_log_sync_enabled", False)),
@@ -565,8 +767,7 @@ def get_runtime_status() -> dict:
)
build_info = get_runtime_build_info()
return {
"api": {
api_payload = {
"service": "domain-api",
"version": "0.1.0",
"api_prefix": settings.api_prefix,
@@ -579,13 +780,13 @@ def get_runtime_status() -> dict:
"stdout_log": _runtime_log_path("domain-api.stdout.log"),
"stderr_log": _runtime_log_path("domain-api.stderr.log"),
"build": build_info,
},
"node": {
}
node_payload = {
"code": settings.node_code,
"region": settings.node_region,
"role": settings.node_role,
},
"worker": {
}
worker_payload = {
"mode": worker_runtime.get("mode", runtime_settings.get("worker_mode", "windows-local")),
"service_name": runtime_settings.get("worker_service_name", settings.worker_service_name),
"running": worker_runtime.get("running", False),
@@ -594,8 +795,8 @@ def get_runtime_status() -> dict:
"latest_start_time": worker_runtime.get("latest_start_time", ""),
"message": worker_runtime.get("message", ""),
"log_path": str(Path(settings.domain_root) / "detect_worker.log"),
},
"sync_agent": {
}
sync_agent_payload = {
"mode": sync_agent_runtime.get("mode", runtime_settings.get("worker_mode", "windows-local")),
"service_name": runtime_settings.get("sync_agent_service_name", settings.sync_agent_service_name),
"running": sync_agent_runtime.get("running", False),
@@ -603,7 +804,44 @@ def get_runtime_status() -> dict:
"latest_start_time": sync_agent_runtime.get("latest_start_time", ""),
"message": sync_agent_runtime.get("message", ""),
"expected_on_this_node": settings.node_region == "mainland" and settings.node_role == "control",
},
}
compatibility_payload = {
# Backward-compatible flat fields for older pages / stale built assets.
"api_online": bool(api_payload.get("pid")),
"api_service_name": api_payload.get("service_name", ""),
"worker_online": bool(worker_payload.get("running", False)),
"worker_mode": worker_payload.get("mode", ""),
"worker_service_name": worker_payload.get("service_name", ""),
"worker_process_count": worker_payload.get("process_count", 0),
"worker_latest_start_time": worker_payload.get("latest_start_time", ""),
"worker_runtime_message": worker_payload.get("message", ""),
"thread_count": detect_snapshot.get("thread_count", 0),
"thread_count_default": detect_snapshot.get("thread_count_default", 0),
"thread_count_source": detect_snapshot.get("thread_count_source", ""),
"thread_count_override": detect_snapshot.get("thread_count_override"),
"active_thread_count": detect_payload.get("active_thread_count", 0),
"max_thread_count": detect_payload.get("max_thread_count", 0),
"progress": detect_payload.get("progress", {}),
"backlog": detect_payload.get("backlog", {}),
"progress_percent": detect_payload.get("progress_percent", 0),
"available_proxy_count": detect_payload.get("available_proxy_count", 0),
"proxy_pool_count": detect_payload.get("proxy_pool_count", 0),
"proxy_runtime_label": detect_payload.get("proxy_runtime_label", ""),
"proxy_runtime_detail": detect_payload.get("proxy_runtime_detail", ""),
"proxy_runtime_reason": detect_payload.get("proxy_runtime_reason", ""),
"proxy_last_refresh_time": detect_payload.get("proxy_last_refresh_time", ""),
"recent_event": detect_payload.get("recent_event", ""),
"recent_warning": detect_payload.get("recent_warning", ""),
"runtime_state": worker_runtime.get("runtime_state") or {},
"cluster_summary": cluster_snapshot.get("summary") or {},
}
return {
**compatibility_payload,
"api": api_payload,
"node": node_payload,
"worker": worker_payload,
"sync_agent": sync_agent_payload,
"detect": detect_payload,
"cluster": cluster_snapshot,
"sync": sync_summary,

View File

@@ -1,6 +1,9 @@
from __future__ import annotations
import json
from app.core.db import get_db
from app.core.redis_client import get_redis
def get_sensitive_words_payload() -> dict:
@@ -54,6 +57,13 @@ def save_sensitive_words_payload(payload: dict) -> dict:
)
conn.commit()
try:
redis_client = get_redis()
redis_client.set("domain_tool:sensitive_words", json.dumps(words, ensure_ascii=False))
redis_client.publish("domain_tool:config_update", "sensitive_words")
except Exception:
pass
return {
"total": len(words),
"text": "\n".join(words),

View File

@@ -35,8 +35,8 @@ def _normalize_thread_count(value: object, *, field_name: str = "thread_count")
thread_count = int(value)
except Exception as exc:
raise ValueError(f"{field_name} must be an integer") from exc
if thread_count < 1 or thread_count > 256:
raise ValueError(f"{field_name} out of range")
if thread_count < 1:
raise ValueError(f"{field_name} must be >= 1")
return thread_count

View File

@@ -11,7 +11,16 @@ from uuid import uuid4
from app.core.config import settings
from app.core.db import get_db
from app.services.cluster_runtime_service import cleanup_imported_runtime_nodes, register_node_heartbeat
from app.services.cluster_runtime_service import (
cleanup_imported_runtime_nodes,
cleanup_imported_runtime_nodes_many,
register_node_heartbeat,
)
from app.services.detect_job_service import (
_load_domain_pipeline_snapshot,
resolve_initial_domain_pipeline_item,
)
from app.services.settings_service import get_settings_payload
from app.services.sync_record_service import _decode_json, _normalize_region
@@ -80,10 +89,16 @@ def _refresh_remote_runtime_node(*, source_region: str, projection: dict, receiv
if not node_code:
node_code = f"{region}-{role}-imported"
controller_current_load = max(
int(projection.get("active_thread_count", 0) or 0),
int(((projection.get("active_job") or {}).get("items_running", 0) or 0)),
)
metadata = {
"service": "runtime-ingest",
"projection_source_region": source_region,
"worker_mode": projection.get("worker_mode", ""),
"active_threads": int(projection.get("active_thread_count", 0) or 0),
"max_threads": int(projection.get("max_thread_count", 0) or 0),
"phase_label": projection.get("phase_label", ""),
"phase_detail": projection.get("phase_detail", ""),
"proxy_runtime_label": projection.get("proxy_runtime_label", ""),
@@ -102,8 +117,8 @@ def _refresh_remote_runtime_node(*, source_region: str, projection: dict, receiv
node_code=node_code,
region=region,
role=role,
status="online",
current_load=int(((projection.get("progress") or {}).get("running", 0) or 0)),
status="busy" if controller_current_load > 0 else "online",
current_load=controller_current_load,
metadata=metadata,
hostname_override=hostname,
ip_override=ip,
@@ -111,15 +126,21 @@ def _refresh_remote_runtime_node(*, source_region: str, projection: dict, receiv
cleanup_imported_runtime_nodes(region=region, role=role, keep_node_code=node_code)
active_job = projection.get("active_job") or {}
worker_node_codes: list[str] = []
for node_stat in list(active_job.get("node_stats") or []):
worker_node_code = str(node_stat.get("node_code") or "").strip()
if not worker_node_code or worker_node_code == "unassigned":
continue
if worker_node_code == node_code:
continue
items_running = int(node_stat.get("items_running", 0) or 0)
items_claimed = int(node_stat.get("items_claimed", 0) or 0)
items_total = int(node_stat.get("items_total", 0) or 0)
worker_status = "busy" if (items_running > 0 or items_claimed > 0) else "online"
worker_load = max(items_running, items_claimed, 0)
worker_runtime_load = int(node_stat.get("current_load", 0) or 0)
worker_active_threads = int(node_stat.get("active_threads", worker_runtime_load) or 0)
worker_max_threads = int(node_stat.get("max_threads", 0) or 0)
worker_load = max(worker_active_threads, items_running, 0)
worker_status = "busy" if worker_load > 0 else "online"
worker_metadata = {
"service": "runtime-ingest",
"projection_source_region": source_region,
@@ -128,10 +149,18 @@ def _refresh_remote_runtime_node(*, source_region: str, projection: dict, receiv
"phase_detail": projection.get("phase_detail", ""),
"proxy_runtime_label": projection.get("proxy_runtime_label", ""),
"proxy_runtime_reason": projection.get("proxy_runtime_reason", ""),
"active_threads": worker_active_threads,
"max_threads": worker_max_threads,
"updated_at": _format_time(received_at or datetime.now()),
"job_items_total": items_total,
"job_items_running": items_running,
"job_items_claimed": items_claimed,
"job_items_completed": int(node_stat.get("items_completed", 0) or 0),
"job_items_failed": int(node_stat.get("items_failed", 0) or 0),
"job_items_blacklisted": int(node_stat.get("items_blacklisted", 0) or 0),
"metrics_source": str(node_stat.get("metrics_source") or "runtime").strip() or "runtime",
"source_status": str(node_stat.get("status") or "").strip(),
"source_role": str(node_stat.get("role") or "worker").strip() or "worker",
"derived_from": node_code,
}
register_node_heartbeat(
@@ -144,7 +173,9 @@ def _refresh_remote_runtime_node(*, source_region: str, projection: dict, receiv
hostname_override=hostname,
ip_override=ip,
)
cleanup_imported_runtime_nodes(region=region, role="worker", keep_node_code=worker_node_code)
worker_node_codes.append(worker_node_code)
if worker_node_codes:
cleanup_imported_runtime_nodes_many(region=region, role="worker", keep_node_codes=worker_node_codes)
def _load_latest_projection(sync_type: str) -> dict | None:
@@ -226,6 +257,88 @@ def _load_pushable_projections(sync_type: str, limit: int) -> list[dict]:
return selected
def _estimate_total_worker_threads(settings_payload: dict | None = None) -> int:
payload = settings_payload if isinstance(settings_payload, dict) else get_settings_payload()
default_threads = max(1, int(payload.get("thread_count", 100) or 100))
node_thread_counts = payload.get("node_thread_counts") if isinstance(payload.get("node_thread_counts"), dict) else {}
total_threads = 0
for raw_value in node_thread_counts.values():
try:
total_threads += max(0, int(raw_value or 0))
except (TypeError, ValueError):
continue
return max(total_threads, default_threads)
def _load_local_detect_backlog_snapshot() -> dict:
with get_db() as conn:
with conn.cursor() as cur:
cur.execute(
"""
SELECT
COUNT(*) FILTER (WHERE item.status = 'pending') AS pending_total,
COUNT(*) FILTER (WHERE item.status = 'claimed') AS claimed_total,
COUNT(*) FILTER (WHERE item.status = 'running') AS running_total,
COUNT(*) FILTER (WHERE item.status = 'pending' AND item.step_code = 'detect_register') AS register_pending,
COUNT(*) FILTER (WHERE item.status = 'pending' AND item.step_code <> 'detect_register') AS downstream_pending
FROM detect_job_items item
JOIN detect_jobs job ON job.id = item.job_id
WHERE job.status IN ('pending', 'running')
"""
)
row = cur.fetchone() or (0, 0, 0, 0, 0)
return {
"pending_total": int(row[0] or 0),
"claimed_total": int(row[1] or 0),
"running_total": int(row[2] or 0),
"register_pending": int(row[3] or 0),
"downstream_pending": int(row[4] or 0),
}
def _build_task_pull_backlog_limits(configured_limit: int, settings_payload: dict | None = None) -> dict:
estimated_total_threads = _estimate_total_worker_threads(settings_payload)
max_pending_total = int(settings.sync_pull_max_pending_items or 0)
if max_pending_total <= 0:
max_pending_total = max(int(configured_limit or 0), estimated_total_threads * 2)
max_register_pending = int(settings.sync_pull_max_register_pending_items or 0)
if max_register_pending <= 0:
max_register_pending = max(max(500, int(configured_limit or 0) // 2), estimated_total_threads)
max_downstream_pending = int(settings.sync_pull_max_downstream_pending_items or 0)
if max_downstream_pending <= 0:
max_downstream_pending = max(250, estimated_total_threads // 4)
return {
"estimated_total_threads": estimated_total_threads,
"max_pending_total": max_pending_total,
"max_register_pending": max_register_pending,
"max_downstream_pending": max_downstream_pending,
}
def _should_throttle_task_pull(backlog_snapshot: dict, backlog_limits: dict) -> tuple[bool, str]:
pending_total = int(backlog_snapshot.get("pending_total", 0) or 0)
register_pending = int(backlog_snapshot.get("register_pending", 0) or 0)
downstream_pending = int(backlog_snapshot.get("downstream_pending", 0) or 0)
max_pending_total = int(backlog_limits.get("max_pending_total", 0) or 0)
max_register_pending = int(backlog_limits.get("max_register_pending", 0) or 0)
max_downstream_pending = int(backlog_limits.get("max_downstream_pending", 0) or 0)
if max_pending_total > 0 and pending_total >= max_pending_total:
return True, "pending_total"
if (
downstream_pending > 0
and max_register_pending > 0
and register_pending >= max_register_pending
):
return True, "register_pending"
if max_downstream_pending > 0 and downstream_pending >= max_downstream_pending:
return True, "downstream_pending"
return False, ""
def _latest_push_attempt(source_record_id: int, target_region: str, sync_type: str) -> dict | None:
with get_db() as conn:
with conn.cursor() as cur:
@@ -308,10 +421,51 @@ def _task_selection_sql() -> str:
"""
def _task_projection_limit(limit: int | None) -> int:
requested = max(1, int(limit or 5000))
configured = max(5000, int(settings.sync_batch_size or 200))
cap = max(10000, configured, 5000)
return max(1, min(requested, cap))
def _task_projection_items_total(projection: dict) -> int:
payload = projection.get("payload") or {}
projection_payload = payload.get("projection") or {}
try:
return int(projection_payload.get("items_total", 0) or 0)
except Exception:
return 0
def _task_projection_selection_limit(projection: dict) -> int:
payload = projection.get("payload") or {}
projection_payload = payload.get("projection") or {}
try:
return int(projection_payload.get("selection_limit", 0) or 0)
except Exception:
return 0
def _mark_task_projection_superseded(record_id: int, *, reason: str) -> None:
with get_db() as conn:
with conn.cursor() as cur:
cur.execute(
"""
UPDATE detect_sync_records
SET status = 'superseded',
error_message = %s,
updated_at = CURRENT_TIMESTAMP
WHERE id = %s
""",
(str(reason or "").strip()[:500], int(record_id)),
)
conn.commit()
def _load_pending_task_projection(limit: int) -> dict | None:
source_region = _normalize_region(settings.sync_source_region, settings.node_region)
target_region = _normalize_region(settings.sync_target_region, "overseas")
safe_limit = max(1, min(int(limit or 1000), max(1, int(settings.sync_batch_size or 200))))
safe_limit = _task_projection_limit(limit)
with get_db() as conn:
with conn.cursor() as cur:
cur.execute(
@@ -340,6 +494,22 @@ def _load_pending_task_projection(limit: int) -> dict | None:
latest_ingest = _latest_ingest_attempt(projection["id"], projection["target_region"], "detect_task_projection")
if latest_ingest and latest_ingest["status"] == "received":
continue
items_total = _task_projection_items_total(projection)
selection_limit = _task_projection_selection_limit(projection)
created_at = projection.get("created_at")
if (
safe_limit >= 1000
and max(items_total, selection_limit) > 0
and max(items_total, selection_limit) < safe_limit
and isinstance(created_at, datetime)
):
now = datetime.now(created_at.tzinfo) if created_at.tzinfo else datetime.now()
if now - created_at >= timedelta(minutes=10):
_mark_task_projection_superseded(
projection["id"],
reason=f"stale small task projection skipped: items_total={items_total}, selection_limit={selection_limit}, requested_limit={safe_limit}",
)
continue
return projection
return None
@@ -361,7 +531,7 @@ def export_detect_task_projection(limit: int = 1000, *, shared_token: str | None
source_region = _normalize_region(settings.sync_source_region, settings.node_region)
target_region = _normalize_region(settings.sync_target_region, "overseas")
safe_limit = max(1, min(int(limit or 1000), max(1, int(settings.sync_batch_size or 200))))
safe_limit = _task_projection_limit(limit)
with get_db() as conn:
with conn.cursor() as cur:
cur.execute(_task_selection_sql(), (safe_limit,))
@@ -528,6 +698,7 @@ def ingest_detect_task_projection(payload: dict, *, shared_token: str | None = N
inserted_count = 0
updated_count = 0
domain_ids: list[int] = []
for item in items:
domain = str(item.get("domain") or "").strip().lower()
if not domain:
@@ -562,7 +733,7 @@ def ingest_detect_task_projection(payload: dict, *, shared_token: str | None = N
ELSE EXCLUDED.detect_status
END,
update_time = CURRENT_TIMESTAMP
RETURNING (xmax = 0) AS inserted
RETURNING id, (xmax = 0) AS inserted
""",
(
domain,
@@ -576,12 +747,120 @@ def ingest_detect_task_projection(payload: dict, *, shared_token: str | None = N
int(item.get("juziseo_status") or 0),
),
)
inserted = bool((cur.fetchone() or [False])[0])
row = cur.fetchone() or [0, False]
domain_id = int(row[0] or 0)
inserted = bool(row[1])
if domain_id > 0:
domain_ids.append(domain_id)
if inserted:
inserted_count += 1
else:
updated_count += 1
target_job_code = f"sync-{source_region}-{source_record_id}"
target_job_remark = (
f"同步拉取待检测批次 {str(projection.get('batch_code') or '').strip() or source_record_id}"
f"{len(domain_ids)} 个域名"
)
cur.execute(
"""
INSERT INTO detect_jobs (job_code, source, plan_hash, task_mode, step_code, status, remark, created_by)
VALUES (%s, %s, %s, 'domain_pipeline', '', 'pending', %s, %s)
ON CONFLICT (job_code) DO UPDATE SET
source = EXCLUDED.source,
plan_hash = EXCLUDED.plan_hash,
task_mode = EXCLUDED.task_mode,
step_code = EXCLUDED.step_code,
remark = EXCLUDED.remark,
created_by = EXCLUDED.created_by,
status = CASE
WHEN detect_jobs.status IN ('completed', 'failed', 'cancelled') THEN 'pending'
ELSE detect_jobs.status
END,
started_at = CASE
WHEN detect_jobs.status IN ('completed', 'failed', 'cancelled') THEN NULL
ELSE detect_jobs.started_at
END,
finished_at = CASE
WHEN detect_jobs.status IN ('completed', 'failed', 'cancelled') THEN NULL
ELSE detect_jobs.finished_at
END
RETURNING id
""",
(
target_job_code,
"sync-pull",
projection_hash,
target_job_remark,
"sync-agent",
),
)
target_job_id = int((cur.fetchone() or [0])[0] or 0)
settings_payload = get_settings_payload()
queued_count = 0
deduplicated_job_items = 0
skipped_job_items = 0
if target_job_id > 0:
for domain_id in domain_ids:
domain_snapshot = _load_domain_pipeline_snapshot(cur, int(domain_id))
if not domain_snapshot:
skipped_job_items += 1
continue
item_step_code, step_payload = resolve_initial_domain_pipeline_item(
domain_snapshot,
settings_payload=settings_payload,
)
if not item_step_code or not step_payload:
skipped_job_items += 1
continue
cur.execute(
"""
INSERT INTO detect_job_items (job_id, domain_id, step_code, status, step_payload_json)
VALUES (%s, %s, %s, 'pending', %s::jsonb)
ON CONFLICT (job_id, domain_id, step_code) DO NOTHING
RETURNING id
""",
(
target_job_id,
domain_id,
item_step_code,
json.dumps(step_payload, ensure_ascii=False),
),
)
inserted_job_item = cur.fetchone()
if inserted_job_item:
queued_count += 1
else:
deduplicated_job_items += 1
if queued_count > 0:
cur.execute(
"""
INSERT INTO detect_run_events (job_id, node_code, event_type, level, message, payload_json)
VALUES (%s, %s, %s, %s, %s, %s::jsonb)
""",
(
target_job_id,
settings.node_code,
"job_created",
"info",
f"同步拉取待检测批次 {target_job_code},共 {queued_count} 个任务项",
json.dumps(
{
"source_region": source_region,
"source_record_id": source_record_id,
"projection_hash": projection_hash,
"batch_code": str(projection.get("batch_code") or "").strip(),
"queued_count": queued_count,
"deduplicated_job_items": deduplicated_job_items,
"skipped_job_items": skipped_job_items,
},
ensure_ascii=False,
),
),
)
cur.execute(
"""
INSERT INTO detect_sync_records (
@@ -602,6 +881,11 @@ def ingest_detect_task_projection(payload: dict, *, shared_token: str | None = N
"items_total": len(items),
"inserted_count": inserted_count,
"updated_count": updated_count,
"target_job_id": target_job_id,
"target_job_code": target_job_code,
"queued_count": queued_count,
"deduplicated_job_items": deduplicated_job_items,
"skipped_job_items": skipped_job_items,
"received_at": _format_time(received_at),
},
ensure_ascii=False,
@@ -617,6 +901,11 @@ def ingest_detect_task_projection(payload: dict, *, shared_token: str | None = N
"items_total": len(items),
"inserted_count": inserted_count,
"updated_count": updated_count,
"target_job_id": target_job_id,
"target_job_code": target_job_code,
"queued_count": queued_count,
"deduplicated_job_items": deduplicated_job_items,
"skipped_job_items": skipped_job_items,
"deduplicated": False,
}
@@ -742,7 +1031,27 @@ def _extract_detect_result_projection_events(
return events
def _resolve_detect_result_target_job_id() -> int:
def _resolve_detect_result_target_job_id(*, projection: dict) -> int:
source_job = projection.get("job") or {}
source_job_code = str(source_job.get("job_code") or "").strip()
with get_db() as conn:
with conn.cursor() as cur:
if source_job_code:
cur.execute(
"""
SELECT id
FROM detect_jobs
WHERE job_code = %s
ORDER BY id DESC
LIMIT 1
""",
(source_job_code,),
)
row = cur.fetchone()
if row:
return int(row[0] or 0)
from app.services.detect_job_service import get_active_detect_job_summary
active_job = get_active_detect_job_summary(event_limit=1) or {}
@@ -759,6 +1068,163 @@ def _parse_event_created_at(value: str) -> datetime | None:
return None
def _extract_event_domain(payload: dict, message: str) -> str:
domain = str(payload.get("domain") or "").strip().lower()
if domain:
return domain
text = str(message or "").strip()
if ":" in text:
candidate = text.rsplit(":", 1)[-1].strip().lower()
if candidate:
return candidate
return ""
def _apply_detect_result_event_to_domain(cur, event: dict) -> None:
payload = _decode_json(event.get("payload"))
domain = _extract_event_domain(payload, str(event.get("message") or ""))
if not domain:
return
event_type = str(event.get("event_type") or "").strip()
created_at = _parse_event_created_at(str(event.get("created_at") or ""))
effective_time = created_at or datetime.now()
if event_type == "domain_started":
cur.execute(
"""
UPDATE domains
SET detect_status = CASE
WHEN detect_status IN (1, 3) THEN detect_status
ELSE 2
END,
update_time = CURRENT_TIMESTAMP
WHERE domain = %s
""",
(domain,),
)
return
if event_type == "domain_completed":
cur.execute(
"""
UPDATE domains
SET detect_status = 1,
detect_time = COALESCE(detect_time, %s),
update_time = CURRENT_TIMESTAMP
WHERE domain = %s
""",
(effective_time, domain),
)
return
if event_type == "domain_blacklisted":
cur.execute(
"""
UPDATE domains
SET detect_status = 3,
update_time = CURRENT_TIMESTAMP
WHERE domain = %s
""",
(domain,),
)
return
if event_type == "domain_failed":
cur.execute(
"""
UPDATE domains
SET detect_status = CASE
WHEN detect_status IN (1, 3) THEN detect_status
ELSE 4
END,
update_time = CURRENT_TIMESTAMP
WHERE domain = %s
""",
(domain,),
)
return
def _apply_detect_result_event_to_job_item(cur, *, target_job_id: int, event: dict) -> int:
if int(target_job_id or 0) <= 0:
return 0
payload = _decode_json(event.get("payload"))
domain = _extract_event_domain(payload, str(event.get("message") or ""))
if not domain:
return 0
event_type = str(event.get("event_type") or "").strip()
node_code = str(event.get("node_code") or "").strip()
message = str(event.get("message") or "").strip()
if event_type == "domain_started":
cur.execute(
"""
UPDATE detect_job_items AS item
SET status = CASE
WHEN item.status IN ('completed', 'blacklisted', 'failed') THEN item.status
ELSE 'running'
END,
claimed_by = CASE
WHEN %s <> '' THEN %s
ELSE item.claimed_by
END,
started_at = COALESCE(item.started_at, CURRENT_TIMESTAMP),
updated_at = CURRENT_TIMESTAMP
FROM domains AS d
WHERE item.job_id = %s
AND item.domain_id = d.id
AND d.domain = %s
AND item.status IN ('pending', 'claimed', 'running')
""",
(node_code, node_code, int(target_job_id), domain),
)
return int(cur.rowcount or 0)
if event_type not in {"domain_completed", "domain_blacklisted", "domain_failed"}:
return 0
final_status = {
"domain_completed": "completed",
"domain_blacklisted": "blacklisted",
"domain_failed": "failed",
}[event_type]
cur.execute(
"""
UPDATE detect_job_items AS item
SET status = %s,
claimed_by = CASE
WHEN %s <> '' THEN %s
ELSE item.claimed_by
END,
finished_at = COALESCE(item.finished_at, CURRENT_TIMESTAMP),
updated_at = CURRENT_TIMESTAMP,
lease_expires_at = NULL,
last_error = CASE
WHEN %s = 'failed' THEN LEFT(%s, 1000)
ELSE item.last_error
END
FROM domains AS d
WHERE item.job_id = %s
AND item.domain_id = d.id
AND d.domain = %s
AND item.status IN ('pending', 'claimed', 'running')
""",
(
final_status,
node_code,
node_code,
final_status,
message,
int(target_job_id),
domain,
),
)
return int(cur.rowcount or 0)
def _import_detect_result_projection_events(
*,
source_region: str,
@@ -773,12 +1239,11 @@ def _import_detect_result_projection_events(
if not events:
return {"imported_count": 0, "deduplicated_count": 0, "target_job_id": 0}
target_job_id = _resolve_detect_result_target_job_id()
if target_job_id <= 0:
return {"imported_count": 0, "deduplicated_count": 0, "target_job_id": 0}
target_job_id = _resolve_detect_result_target_job_id(projection=projection)
imported_count = 0
deduplicated_count = 0
updated_job_items = 0
with get_db() as conn:
with conn.cursor() as cur:
for event in events:
@@ -831,12 +1296,27 @@ def _import_detect_result_projection_events(
json.dumps(event["payload"], ensure_ascii=False),
),
)
_apply_detect_result_event_to_domain(cur, event)
if target_job_id > 0:
updated_job_items += _apply_detect_result_event_to_job_item(
cur,
target_job_id=target_job_id,
event=event,
)
imported_count += 1
conn.commit()
if target_job_id > 0:
from app.services.detect_job_service import refresh_detect_job_status
try:
refresh_detect_job_status(target_job_id)
except Exception:
pass
return {
"imported_count": imported_count,
"deduplicated_count": deduplicated_count,
"target_job_id": target_job_id,
"updated_job_items": updated_job_items,
}
@@ -941,6 +1421,16 @@ def ingest_runtime_projection(payload: dict, *, shared_token: str | None = None)
def _push_projection_now(sync_type: str, ingest_url: str) -> tuple[bool, str, dict]:
if sync_type == "runtime_projection":
# Regenerate the runtime snapshot before every push so the sync agent
# does not keep replaying a stale projection record while the worker
# thread count / phase is still changing.
from app.services.runtime_status_service import get_runtime_status
try:
get_runtime_status()
except Exception as exc:
return False, f"刷新 runtime_projection 失败: {exc}", {"action": "push_sync", "sync_type": sync_type}
source_record = _load_latest_projection(sync_type)
if not source_record:
return False, f"当前没有可推送的{sync_type}", {"action": "push_sync", "sync_type": sync_type}
@@ -1177,15 +1667,33 @@ def pull_detect_task_batch_now(limit: int | None = None) -> tuple[bool, str, dic
if not export_url or not ack_url:
return False, "未配置任务拉取目标地址", {"action": "pull_tasks", "pull_state": "misconfigured", "ui_level": "warning", "poll_schedule_seconds": []}
safe_limit = max(1, min(int(limit or settings.sync_batch_size or 200), max(1, int(settings.sync_batch_size or 200))))
configured_limit = max(5000, int(settings.sync_batch_size or 200))
requested_limit = int(limit or configured_limit)
safe_limit = max(1, min(requested_limit, max(10000, configured_limit)))
settings_payload = get_settings_payload()
backlog_snapshot = _load_local_detect_backlog_snapshot()
backlog_limits = _build_task_pull_backlog_limits(configured_limit, settings_payload=settings_payload)
should_throttle, throttle_reason = _should_throttle_task_pull(backlog_snapshot, backlog_limits)
if should_throttle:
return True, "本地待处理积压较高,暂停拉取新批次", {
"action": "pull_tasks",
"pull_state": "throttled",
"ui_level": "info",
"poll_schedule_seconds": [1, 3],
"reason": throttle_reason,
**backlog_snapshot,
**backlog_limits,
}
request_url = f"{export_url}?limit={safe_limit}"
export_timeout = max(20, min(90, 15 + safe_limit // 40))
request = urllib.request.Request(
request_url,
headers={**({"X-Domaincheck-Sync-Token": settings.sync_shared_token} if settings.sync_shared_token else {})},
method="GET",
)
try:
with urllib.request.urlopen(request, timeout=20) as response:
with urllib.request.urlopen(request, timeout=export_timeout) as response:
raw = response.read().decode("utf-8")
data = json.loads(raw) if raw else {}
except json.JSONDecodeError as exc:
@@ -1280,7 +1788,7 @@ def pull_detect_task_batch_now(limit: int | None = None) -> tuple[bool, str, dic
**(ingest_data or {}),
}
return True, "待检测任务批次拉取并入库成功", {
result = {
"action": "pull_tasks",
"pull_state": "success",
"ui_level": "success",
@@ -1291,3 +1799,27 @@ def pull_detect_task_batch_now(limit: int | None = None) -> tuple[bool, str, dic
**(ingest_data or {}),
"ack": ack_data,
}
queued_count = int(result.get("queued_count", 0) or 0)
if queued_count > 0:
try:
from app.services.worker_control_service import send_worker_command
start_ok, start_message = send_worker_command(
"start_detection",
payload={
"source": "sync-pull",
"source_record_id": source_record_id,
"target_job_id": int(result.get("target_job_id", 0) or 0),
"target_job_code": str(result.get("target_job_code") or "").strip(),
},
)
result["worker_start_ok"] = bool(start_ok)
result["worker_start_message"] = str(start_message or "").strip()
except Exception as exc:
result["worker_start_ok"] = False
result["worker_start_message"] = f"同步入库后自动唤起 Worker 失败: {exc}"
result["ui_level"] = "warning"
result["pull_state"] = "worker_start_warning"
return True, "待检测任务批次拉取并入库成功;但自动唤起 Worker 失败", result
return True, "待检测任务批次拉取并入库成功", result

View File

@@ -6,7 +6,7 @@ import socket
from datetime import datetime, timedelta
from app.core.config import settings
from app.core.db import get_db
from app.core.db import db_read_retry, get_db
def _format_time(value: datetime | None) -> str:
@@ -45,28 +45,94 @@ _DETECT_RESULT_EVENT_TYPES = {
"domain_blacklisted",
}
_TERMINAL_DETECT_RESULT_EVENT_TYPES = {
"domain_completed",
"domain_failed",
"domain_blacklisted",
}
def _collect_recent_domain_events(active_job: dict, limit: int = 30) -> list[dict]:
events = list(active_job.get("current_cycle_events") or active_job.get("recent_events") or [])
safe_limit = max(1, int(limit or 30))
seen: set[tuple[str, str, str, str]] = set()
normalized: list[dict] = []
for event in reversed(events):
event_type = str(event.get("event_type") or "").strip()
def _append_event(raw_event: dict) -> None:
event_type = str(raw_event.get("event_type") or "").strip()
if event_type not in _DETECT_RESULT_EVENT_TYPES:
continue
payload = _decode_json(event.get("payload"))
normalized.append(
{
"node_code": str(event.get("node_code") or "").strip(),
"event_type": event_type,
"level": str(event.get("level") or "info").strip() or "info",
"message": str(event.get("message") or "").strip(),
"created_at": str(event.get("created_at") or "").strip(),
"payload": payload,
}
return
normalized_event = {
"node_code": str(raw_event.get("node_code") or "").strip(),
"event_type": event_type,
"level": str(raw_event.get("level") or "info").strip() or "info",
"message": str(raw_event.get("message") or "").strip(),
"created_at": str(raw_event.get("created_at") or "").strip(),
"payload": _decode_json(raw_event.get("payload")),
}
event_key = (
normalized_event["node_code"],
normalized_event["event_type"],
normalized_event["message"],
normalized_event["created_at"],
)
if limit <= 0:
return normalized
return normalized[-int(limit):]
if event_key in seen:
return
seen.add(event_key)
normalized.append(normalized_event)
# Keep a small slice of the current-cycle `domain_started` events so the
# remote log / live activity view still reflects the node's latest work.
for event in reversed(list(active_job.get("current_cycle_events") or active_job.get("recent_events") or [])):
if str(event.get("event_type") or "").strip() != "domain_started":
continue
_append_event(event)
if len(normalized) >= min(10, max(1, safe_limit // 3)):
break
# Always pull the most recent terminal result events from the full job
# history. Otherwise a flood of newer `domain_started` events can hide
# terminal completions, and overseas will never advance completed counts.
job_id = int(active_job.get("job_id") or 0)
if job_id > 0:
with get_db() as conn:
with conn.cursor() as cur:
cur.execute(
"""
SELECT node_code, event_type, level, message, payload_json, created_at
FROM detect_run_events
WHERE job_id = %s
AND event_type IN ('domain_completed', 'domain_failed', 'domain_blacklisted')
ORDER BY created_at DESC, id DESC
LIMIT %s
""",
(job_id, max(safe_limit * 4, 60)),
)
rows = cur.fetchall()
for row in reversed(rows):
_append_event(
{
"node_code": row[0] or "",
"event_type": row[1] or "",
"level": row[2] or "info",
"message": row[3] or "",
"payload": _decode_json(row[4]),
"created_at": _format_time(row[5]),
}
)
else:
for event in reversed(list(active_job.get("current_cycle_events") or active_job.get("recent_events") or [])):
if str(event.get("event_type") or "").strip() in _TERMINAL_DETECT_RESULT_EVENT_TYPES:
_append_event(event)
normalized.sort(
key=lambda item: (
str(item.get("created_at") or ""),
str(item.get("node_code") or ""),
str(item.get("event_type") or ""),
str(item.get("message") or ""),
)
)
return normalized[-safe_limit:]
def _build_detect_result_batch_digest(batch: dict | None) -> dict:
@@ -154,6 +220,7 @@ def _should_append_runtime_projection(previous_payload: dict, current_projection
return now - previous_created_at >= timedelta(seconds=45)
@db_read_retry()
def list_sync_records(limit: int = 20) -> list[dict]:
safe_limit = max(1, min(int(limit or 20), 200))
with get_db() as conn:
@@ -221,6 +288,7 @@ def _latest_sync_record_by_source(
}
@db_read_retry()
def get_detect_result_sync_batches(limit: int = 5) -> dict:
safe_limit = max(1, min(int(limit or 5), 20))
source_region = _normalize_region(settings.sync_source_region, settings.node_region)
@@ -364,6 +432,7 @@ def get_detect_result_sync_batches(limit: int = 5) -> dict:
}
@db_read_retry()
def get_sync_summary(record_limit: int = 10) -> dict:
source_region = _normalize_region(settings.sync_source_region, settings.node_region)
target_region = _normalize_region(settings.sync_target_region, "overseas")
@@ -465,56 +534,43 @@ def append_sync_record(
return record_id
def append_runtime_projection_if_changed(
def _local_node_expected_to_execute_worker() -> bool:
node_role = str(settings.node_role or "").strip()
node_region = str(settings.node_region or "").strip()
return node_role == "worker" or (node_region == "mainland" and node_role == "control")
def _build_runtime_projection_payload(
*,
detect: dict,
cluster: dict,
source_region: str | None = None,
target_region: str | None = None,
) -> int | None:
normalized_source_region = _normalize_region(source_region, _normalize_region(settings.sync_source_region, settings.node_region))
normalized_target_region = _normalize_region(target_region, _normalize_region(settings.sync_target_region, "overseas"))
source_region: str,
target_region: str,
) -> dict:
active_job = detect.get("active_job") or {}
local_worker_expected = _local_node_expected_to_execute_worker()
local_participating = False
for node in list(cluster.get("nodes") or []):
if str(node.get("node_code") or "").strip() != settings.node_code:
continue
local_participating = bool(node.get("detect_participating", False) or node.get("current_load", 0))
if local_worker_expected:
local_participating = bool(node.get("detect_participating", False) or node.get("current_load", 0))
break
local_job_bucket = {}
for item in list(active_job.get("node_stats") or []):
if str(item.get("node_code") or "").strip() != settings.node_code:
continue
local_job_bucket = item
break
if not local_participating:
if local_worker_expected:
for item in list(active_job.get("node_stats") or []):
if str(item.get("node_code") or "").strip() != settings.node_code:
continue
local_job_bucket = item
break
if local_worker_expected and not local_participating:
local_participating = bool(
int(local_job_bucket.get("items_running", 0) or 0) > 0
or int(local_job_bucket.get("items_claimed", 0) or 0) > 0
)
projection = {
"node": {
"node_code": settings.node_code,
"region": settings.node_region,
"role": settings.node_role,
"hostname": socket.gethostname(),
"ip": _resolve_local_ip(),
},
"worker_online": bool(detect.get("worker_online", False)),
"detect_participating": local_participating,
"worker_mode": detect.get("worker_mode", ""),
"phase_label": detect.get("phase_label", ""),
"phase_detail": detect.get("phase_detail", ""),
"proxy_runtime_label": detect.get("proxy_runtime_label", ""),
"proxy_runtime_reason": detect.get("proxy_runtime_reason", ""),
"progress": {
"pending": int((detect.get("progress") or {}).get("pending", 0) or 0),
"running": int((detect.get("progress") or {}).get("running", 0) or 0),
"completed": int((detect.get("progress") or {}).get("completed", 0) or 0),
"blacklisted": int((detect.get("progress") or {}).get("blacklisted", 0) or 0),
"failed": int((detect.get("progress") or {}).get("failed", 0) or 0),
},
"active_job": {
projection_active_job = (
{
"job_id": active_job.get("job_id"),
"job_code": active_job.get("job_code", ""),
"status": active_job.get("status", ""),
@@ -525,7 +581,58 @@ def append_runtime_projection_if_changed(
"items_running": active_job.get("items_running", 0),
"items_failed": active_job.get("items_failed", 0),
"node_stats": list(active_job.get("node_stats") or []),
}
if local_worker_expected
else {
"job_id": None,
"job_code": "",
"status": "",
"progress_percent": 0,
"items_total": 0,
"items_terminal": 0,
"items_pending": 0,
"items_running": 0,
"items_failed": 0,
"node_stats": [],
}
)
progress_payload = (
{
"pending": int((detect.get("progress") or {}).get("pending", 0) or 0),
"running": int((detect.get("progress") or {}).get("running", 0) or 0),
"completed": int((detect.get("progress") or {}).get("completed", 0) or 0),
"blacklisted": int((detect.get("progress") or {}).get("blacklisted", 0) or 0),
"failed": int((detect.get("progress") or {}).get("failed", 0) or 0),
}
if local_worker_expected
else {
"pending": 0,
"running": 0,
"completed": 0,
"blacklisted": 0,
"failed": 0,
}
)
projection = {
"node": {
"node_code": settings.node_code,
"region": settings.node_region,
"role": settings.node_role,
"hostname": socket.gethostname(),
"ip": _resolve_local_ip(),
},
"worker_online": bool(detect.get("worker_online", False)) if local_worker_expected else False,
"detect_participating": local_participating if local_worker_expected else False,
"worker_mode": detect.get("worker_mode", ""),
"active_thread_count": int(detect.get("active_thread_count", 0) or 0) if local_worker_expected else 0,
"max_thread_count": int(detect.get("max_thread_count", 0) or 0) if local_worker_expected else 0,
"phase_label": detect.get("phase_label", ""),
"phase_detail": detect.get("phase_detail", ""),
"proxy_runtime_label": detect.get("proxy_runtime_label", ""),
"proxy_runtime_reason": detect.get("proxy_runtime_reason", ""),
"progress": progress_payload,
"backlog": dict(detect.get("backlog") or {}) if local_worker_expected else {},
"active_job": projection_active_job,
"cluster_summary": {
"nodes_total": int(cluster.get("nodes_total", 0) or 0),
"online_worker_nodes": int((cluster.get("summary") or {}).get("online_worker_nodes", 0) or 0),
@@ -544,13 +651,32 @@ def append_runtime_projection_if_changed(
for item in (detect.get("dependency_alerts") or [])[:3]
],
}
payload = {
return {
"projection": projection,
"projection_hash": hashlib.sha1(
json.dumps(projection, ensure_ascii=False, sort_keys=True).encode("utf-8")
).hexdigest(),
"source_region": source_region,
"target_region": target_region,
}
def append_runtime_projection_if_changed(
*,
detect: dict,
cluster: dict,
source_region: str | None = None,
target_region: str | None = None,
) -> int | None:
normalized_source_region = _normalize_region(source_region, _normalize_region(settings.sync_source_region, settings.node_region))
normalized_target_region = _normalize_region(target_region, _normalize_region(settings.sync_target_region, "overseas"))
payload = _build_runtime_projection_payload(
detect=detect,
cluster=cluster,
source_region=normalized_source_region,
target_region=normalized_target_region,
)
with get_db() as conn:
with conn.cursor() as cur:
cur.execute(
@@ -570,7 +696,7 @@ def append_runtime_projection_if_changed(
latest_created_at = latest[1] if latest else None
if latest_payload.get("projection_hash") == payload["projection_hash"]:
return None
if not _should_append_runtime_projection(latest_payload, projection, latest_created_at):
if not _should_append_runtime_projection(latest_payload, payload["projection"], latest_created_at):
return None
cur.execute(
"""
@@ -650,6 +776,7 @@ def append_detect_result_projection_if_changed(
with get_db() as conn:
with conn.cursor() as cur:
current_job_id = int((projection.get("job") or {}).get("job_id") or 0)
cur.execute(
"""
SELECT payload_json, created_at
@@ -657,10 +784,19 @@ def append_detect_result_projection_if_changed(
WHERE sync_type = 'detect_result_projection'
AND source_region = %s
AND target_region = %s
AND (
%s <= 0
OR (payload_json->'projection'->'job'->>'job_id') = %s
)
ORDER BY created_at DESC, id DESC
LIMIT 1
""",
(normalized_source_region, normalized_target_region),
(
normalized_source_region,
normalized_target_region,
current_job_id,
str(current_job_id),
),
)
latest = cur.fetchone()
latest_payload = _decode_json(latest[0]) if latest else {}

View File

@@ -60,7 +60,12 @@ def normalize_systemctl_error(raw_message: str, *, service_name: str = "") -> st
return f"{target} 控制失败,未返回可用错误信息"
lowered = message.lower()
if "sudo: a password is required" in lowered or "authentication is required" in lowered:
if (
"sudo: a password is required" in lowered
or "authentication is required" in lowered
or "interactive authentication required" in lowered
or "authorization not available" in lowered
):
target = normalized_service_name or "systemd 服务"
return f"{target} 控制失败:当前运行用户没有免密 systemctl 权限,请为 API 进程授予对应 sudo/systemd 权限"
if "unit " in lowered and " could not be found" in lowered:

View File

@@ -5,13 +5,34 @@ import time
from app.core.config import settings
from app.services.debug_event_service import push_debug_event
from app.services.detect_job_service import get_active_detect_job_summary, get_detect_queue_health, list_recent_detect_run_events
from app.services.detect_job_service import (
get_active_detect_job_summary,
get_detect_queue_health,
get_latest_detect_job_summary,
get_latest_unprojected_detect_job_summary,
list_recent_detect_run_events,
process_detect_pipeline_now,
)
from app.services.sync_record_service import append_detect_result_projection_if_changed
from app.services.sync_push_service import pull_detect_task_batch_now, push_runtime_projection_now
from app.services.sync_push_service import (
_load_local_detect_backlog_snapshot,
pull_detect_task_batch_now,
push_runtime_projection_now,
)
logger = logging.getLogger("domaincheck.sync_agent")
_IDLE_SYNC_KEYWORDS = (
"当前没有可推送",
"当前没有需要立即推送",
"已全部同步完成",
"无需重复发送",
"进行中",
"等待下个重试窗口",
"暂停拉取",
)
def _append_detect_result_projection_snapshot(active_job: dict) -> None:
if not active_job:
@@ -32,6 +53,159 @@ def _append_detect_result_projection_snapshot(active_job: dict) -> None:
)
def _is_idle_sync_message(message: str) -> bool:
normalized = str(message or "").strip()
return any(keyword in normalized for keyword in _IDLE_SYNC_KEYWORDS)
def _filter_runtime_events_for_job(events: list[dict], *, job_code: str = "", job_id: int = 0, limit: int = 8) -> list[dict]:
target_job_code = str(job_code or "").strip()
target_job_id = int(job_id or 0)
safe_limit = max(1, min(int(limit or 8), 50))
filtered: list[dict] = []
for raw_event in list(events or []):
if not isinstance(raw_event, dict):
continue
payload = raw_event.get("payload") if isinstance(raw_event.get("payload"), dict) else {}
event_job_code = str(payload.get("job_code") or "").strip()
event_job_id = int(raw_event.get("job_id") or 0)
if target_job_code and event_job_code != target_job_code and (target_job_id <= 0 or event_job_id != target_job_id):
continue
filtered.append(raw_event)
if len(filtered) >= safe_limit:
break
return filtered
def _build_aligned_queue_health_snapshot(active_job: dict, queue_health: dict | None) -> dict:
snapshot = dict(queue_health or {})
if not active_job:
return snapshot
active_job_code = str(active_job.get("runtime_job_code") or active_job.get("job_code") or "").strip()
queue_job = dict(snapshot.get("job") or {})
queue_job_code = str(queue_job.get("runtime_job_code") or queue_job.get("job_code") or "").strip()
if active_job_code and queue_job_code and active_job_code == queue_job_code:
return snapshot
active_job_items_total = int(active_job.get("items_total", 0) or 0)
active_job_pending = int(active_job.get("items_pending", 0) or 0)
active_job_claimed = int(active_job.get("items_claimed", 0) or 0)
active_job_running = int(active_job.get("items_running", 0) or 0)
active_job_completed = int(active_job.get("items_completed", 0) or 0)
active_job_blacklisted = int(active_job.get("items_blacklisted", 0) or 0)
active_job_failed = int(active_job.get("items_failed", 0) or 0)
active_job_terminal = int(
active_job.get("items_terminal", active_job_completed + active_job_blacklisted + active_job_failed) or 0
)
display_claimed = int(active_job.get("display_items_claimed", active_job_claimed) or active_job_claimed)
display_running = int(active_job.get("display_items_running", active_job_running) or active_job_running)
node_entries: list[dict] = []
for node in list(active_job.get("node_stats") or []):
node_entries.append(
{
"node_code": str(node.get("node_code") or "").strip(),
"items_total": int(node.get("items_total", 0) or 0),
"items_pending": int(node.get("items_pending", 0) or 0),
"items_claimed": int(node.get("items_claimed", 0) or 0),
"items_running": int(node.get("items_running", 0) or 0),
"items_completed": int(node.get("items_completed", 0) or 0),
"items_blacklisted": int(node.get("items_blacklisted", 0) or 0),
"items_failed": int(node.get("items_failed", 0) or 0),
"processed_recent": int(node.get("processed_recent", 0) or 0),
"processed_per_minute": float(node.get("processed_per_minute", 0) or 0),
"completed_recent": int(node.get("completed_recent", 0) or 0),
"blacklisted_recent": int(node.get("blacklisted_recent", 0) or 0),
"failed_recent": int(node.get("failed_recent", 0) or 0),
"metrics_source": str(node.get("metrics_source") or "runtime"),
}
)
assigned_total = sum(int(item.get("items_total", 0) or 0) for item in node_entries)
unassigned_total = max(0, active_job_items_total - assigned_total)
if unassigned_total > 0:
node_entries.append(
{
"node_code": "unassigned",
"items_total": unassigned_total,
"items_pending": active_job_pending,
"items_claimed": 0,
"items_running": 0,
"items_completed": 0,
"items_blacklisted": 0,
"items_failed": 0,
"processed_recent": 0,
"processed_per_minute": 0.0,
"completed_recent": 0,
"blacklisted_recent": 0,
"failed_recent": 0,
"metrics_source": "central_queue",
}
)
snapshot["job"] = {
"job_id": active_job.get("job_id"),
"job_code": str(active_job.get("job_code") or "").strip(),
"runtime_job_code": active_job_code,
"status": str(active_job.get("status") or "").strip(),
"progress_percent": float(active_job.get("progress_percent", 0) or 0),
}
snapshot["queue"] = {
**dict(snapshot.get("queue") or {}),
"items_total": active_job_items_total,
"pending": active_job_pending,
"claimed": active_job_claimed,
"running": active_job_running,
"display_claimed": display_claimed,
"display_running": display_running,
"completed": active_job_completed,
"blacklisted": active_job_blacklisted,
"failed": active_job_failed,
"terminal": active_job_terminal,
"terminal_percent": round((active_job_terminal / active_job_items_total) * 100, 2) if active_job_items_total else 0.0,
}
snapshot["nodes"] = node_entries
return snapshot
def _select_projection_job_snapshot() -> dict | None:
active_job = get_active_detect_job_summary(event_limit=10)
if active_job:
return active_job
return get_latest_detect_job_summary(
event_limit=10,
statuses=("completed", "partial_failed", "failed"),
recent_minutes=20,
)
def _select_projection_job_snapshots() -> list[dict]:
snapshots: list[dict] = []
seen_job_ids: set[int] = set()
active_job = get_active_detect_job_summary(event_limit=10)
if active_job:
active_job_id = int(active_job.get("job_id") or 0)
if active_job_id > 0 and active_job_id not in seen_job_ids:
snapshots.append(active_job)
seen_job_ids.add(active_job_id)
latest_finished_job = get_latest_unprojected_detect_job_summary(
event_limit=10,
statuses=("completed", "partial_failed", "failed"),
recent_minutes=180,
)
if latest_finished_job:
latest_finished_job_id = int(latest_finished_job.get("job_id") or 0)
if latest_finished_job_id > 0 and latest_finished_job_id not in seen_job_ids:
snapshots.append(latest_finished_job)
seen_job_ids.add(latest_finished_job_id)
return snapshots
def _emit_structured_tick(
*,
base_event_type: str,
@@ -42,7 +216,13 @@ def _emit_structured_tick(
payload = {"ok": ok, "data": data or {}}
event_type = f"{base_event_type}_failed"
level = "warning"
if ok:
if isinstance(data, dict) and str(data.get("pull_state") or "").strip() == "throttled":
event_type = f"{base_event_type}_idle"
level = "info"
elif not ok and _is_idle_sync_message(message):
event_type = f"{base_event_type}_idle"
level = "info"
elif ok:
event_type = f"{base_event_type}_success"
level = "info"
if "但远端确认失败" in str(message or ""):
@@ -70,7 +250,10 @@ def _emit_sync_result_breakdown(data: dict | None) -> None:
result_data = item.get("data") or {}
event_type = f"{sync_type}_sync_failed"
level = "warning"
if ok:
if not ok and _is_idle_sync_message(message):
event_type = f"{sync_type}_sync_idle"
level = "info"
elif ok:
event_type = f"{sync_type}_sync_success"
level = "info"
if isinstance(result_data, dict) and result_data.get("success_count") is not None:
@@ -95,12 +278,31 @@ def _emit_sync_result_breakdown(data: dict | None) -> None:
)
def _run_pipeline_stage_processor() -> tuple[bool, str, dict]:
process_limit = max(500, min(int(settings.sync_pipeline_process_limit or 5000), 5000))
ok, message, data = process_detect_pipeline_now(limit=process_limit)
push_debug_event(
service="sync-agent",
event_type="pipeline_tick_success" if ok else "pipeline_tick_failed",
level="info" if ok else "warning",
message=message,
payload={
"ok": ok,
"limit": process_limit,
"data": data or {},
},
)
return ok, message, data
def main() -> None:
logging.basicConfig(
level=logging.INFO,
format="%(asctime)s [%(levelname)s] %(name)s: %(message)s",
)
interval = max(10, int(settings.sync_poll_interval_seconds or 30))
# Old env files still ship SYNC_POLL_INTERVAL_SECONDS=30. Cap the interval
# so controller pull/pipeline ticks cannot be throttled into starvation.
interval = max(2, min(int(settings.sync_poll_interval_seconds or 2), 5))
logger.info(
"sync agent started: node=%s source=%s target=%s interval=%ss enabled=%s",
settings.node_code,
@@ -111,9 +313,16 @@ def main() -> None:
)
while True:
try:
pipeline_ok, pipeline_message, pipeline_data = _run_pipeline_stage_processor()
logger.info(
"pipeline tick: ok=%s message=%s data=%s",
pipeline_ok,
pipeline_message,
pipeline_data,
)
active_job = get_active_detect_job_summary(event_limit=10)
if active_job:
_append_detect_result_projection_snapshot(active_job)
for projection_job in _select_projection_job_snapshots():
_append_detect_result_projection_snapshot(projection_job)
ok, message, data = push_runtime_projection_now()
logger.info("sync tick: ok=%s message=%s data=%s", ok, message, data)
push_debug_event(
@@ -136,8 +345,16 @@ def main() -> None:
)
_emit_structured_tick(base_event_type="task_pull", ok=pull_ok, message=pull_message, data=pull_data)
if active_job:
queue_health = get_detect_queue_health(window_minutes=15)
recent_events = list_recent_detect_run_events(limit=8)
queue_health = _build_aligned_queue_health_snapshot(
active_job,
get_detect_queue_health(window_minutes=15),
)
recent_events = _filter_runtime_events_for_job(
list_recent_detect_run_events(limit=24),
job_code=str(active_job.get("runtime_job_code") or active_job.get("job_code") or "").strip(),
job_id=int(active_job.get("job_id", 0) or 0),
limit=8,
)
push_debug_event(
service="detect-runtime",
event_type="active_job_snapshot",
@@ -158,6 +375,7 @@ def main() -> None:
"node_stats": list(active_job.get("node_stats") or []),
},
"queue_health": queue_health,
"backlog": _load_local_detect_backlog_snapshot(),
"recent_events": recent_events,
},
)