admin/getDomain
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: stabilize multi-region runtime sync and worker orchestration

Browse Source
This commit is contained in:
root
2026-04-27 15:48:12 +08:00
parent 7cbde2aa78
commit 215a364891
137 changed files with 31931 additions and 1943 deletions
Download Patch File Download Diff File Expand all files Collapse all files

621
domain-api/app/services/sync_push_service.py
View File

@@ -2,6 +2,7 @@ from __future__ import annotations
import hashlib
import json
import os
import socket
import urllib.error
import urllib.parse
@@ -11,17 +12,30 @@ from uuid import uuid4
from app.core.config import settings
from app.core.db import get_db
from app.core.redis_client import get_redis
from app.services.cluster_runtime_service import (
cleanup_imported_runtime_nodes,
cleanup_imported_runtime_nodes_many,
get_cluster_snapshot,
register_node_heartbeat,
)
from app.services.detect_job_service import (
_load_domain_pipeline_snapshot,
get_active_detect_job_summary,
resolve_initial_domain_pipeline_item,
)
from app.services.settings_service import get_settings_payload
from app.services.sync_record_service import _decode_json, _normalize_region
from app.services.settings_service import (
get_settings_payload,
resolve_process_count,
resolve_thread_count,
)
from app.services.sync_record_service import (
_RUNTIME_PROJECTION_FUTURE_SKEW_GRACE,
_decode_json,
_normalize_region,
_pick_latest_projection_row,
append_runtime_projection_if_changed,
)
_DETECT_RESULT_EVENT_TYPES = {
@@ -30,6 +44,222 @@ _DETECT_RESULT_EVENT_TYPES = {
"domain_failed",
"domain_blacklisted",
}
_LOCAL_BACKLOG_PENDING_FRESHNESS_HOURS = 6
_LOCAL_BACKLOG_MAX_JOBS = 4
_SYNC_PULL_WORKER_WAKE_TTL_SECONDS = 20
_SYNC_PULL_WORKER_WAKE_KEY_PREFIX = "domain_tool:sync_pull_worker_wake"
def _flag_enabled(raw_value: object, *, default: bool = False) -> bool:
if raw_value is None:
return bool(default)
if isinstance(raw_value, bool):
return raw_value
return str(raw_value or "").strip().lower() not in {"", "0", "false", "no", "off"}
def _fast_runtime_projection_enabled() -> bool:
return _flag_enabled(
os.getenv("DOMAINCHECK_SYNC_RUNTIME_FAST_PROJECTION"),
default=False,
)
def _local_projection_node_code(node_code: str) -> bool:
normalized_node_code = str(node_code or "").strip()
local_node_code = str(settings.node_code or "").strip()
if not normalized_node_code or not local_node_code:
return False
return normalized_node_code == local_node_code or normalized_node_code.startswith(f"{local_node_code}-")
def _append_fast_runtime_projection_snapshot() -> int | None:
local_node_code = str(settings.node_code or "").strip()
if not local_node_code:
return None
with get_db() as conn:
with conn.cursor() as cur:
cur.execute(
"""
SELECT node_code, region, role, status, current_load, metadata_json
FROM detect_worker_nodes
WHERE node_code = %s OR node_code LIKE %s
ORDER BY node_code ASC
""",
(local_node_code, f"{local_node_code}-%"),
)
raw_rows = list(cur.fetchall() or [])
if not raw_rows:
return None
cluster_nodes: list[dict] = []
queue_nodes: list[dict] = []
busy_nodes: list[str] = []
stale_nodes: list[str] = []
offline_nodes: list[str] = []
online_worker_nodes = 0
dedicated_online_worker_nodes = 0
online_control_nodes = 0
display_running = 0
display_max_threads = 0
controller_metadata: dict = {}
for node_code, region, role, status, current_load, metadata_json in raw_rows:
metadata = dict(metadata_json or {})
normalized_node_code = str(node_code or "").strip()
normalized_role = str(role or metadata.get("source_role") or "").strip() or "worker"
normalized_status = str(status or metadata.get("source_status") or "").strip() or "unknown"
normalized_region = _normalize_region(region, settings.node_region)
normalized_current_load = int(current_load or 0)
active_threads = int(metadata.get("active_threads", 0) or 0)
max_threads = int(metadata.get("max_threads", 0) or 0)
detect_participating = bool(metadata.get("detect_participating", False) or normalized_current_load > 0 or active_threads > 0)
runtime_running = max(active_threads, normalized_current_load)
if normalized_status == "busy":
busy_nodes.append(normalized_node_code)
elif normalized_status == "stale":
stale_nodes.append(normalized_node_code)
elif normalized_status == "offline":
offline_nodes.append(normalized_node_code)
if normalized_status not in {"stale", "offline"}:
if normalized_role == "worker":
online_worker_nodes += 1
if normalized_node_code != local_node_code:
dedicated_online_worker_nodes += 1
elif normalized_role == "control":
online_control_nodes += 1
cluster_nodes.append(
{
"node_code": normalized_node_code,
"role": normalized_role,
"status": normalized_status,
"current_load": normalized_current_load,
"active_threads": active_threads,
"max_threads": max_threads,
"detect_participating": detect_participating,
}
)
queue_nodes.append(
{
"node_code": normalized_node_code,
"items_total": 0,
"items_pending": 0,
"items_claimed": 0,
"items_running": runtime_running,
"items_completed": 0,
"items_blacklisted": 0,
"items_failed": 0,
"display_running": runtime_running,
"display_claimed": 0,
"current_load": normalized_current_load,
"active_threads": active_threads,
"max_threads": max_threads,
"role": normalized_role,
"status": normalized_status,
"detect_participating": detect_participating,
}
)
display_running += runtime_running
display_max_threads += max_threads
if normalized_node_code == local_node_code:
controller_metadata = metadata
controller_job_code = str(controller_metadata.get("active_job_code") or "").strip()
controller_job_status = str(controller_metadata.get("active_job_status") or "").strip()
detect_payload = {
"worker_online": True,
"worker_mode": str(controller_metadata.get("worker_mode") or "linux-systemd").strip() or "linux-systemd",
"phase_label": str(controller_metadata.get("phase_label") or "集群执行中").strip() or "集群执行中",
"phase_detail": str(controller_metadata.get("phase_detail") or "").strip(),
"proxy_runtime_label": str(controller_metadata.get("proxy_runtime_label") or "").strip(),
"proxy_runtime_reason": str(controller_metadata.get("proxy_runtime_reason") or "").strip(),
"detect_participating": bool(display_running > 0),
"progress": {
"pending": 0,
"running": display_running,
"completed": 0,
"blacklisted": 0,
"failed": 0,
},
"queue_health": {
"queue": {
"items_total": 0,
"pending": 0,
"claimed": 0,
"running": display_running,
"completed": 0,
"blacklisted": 0,
"failed": 0,
"terminal": 0,
"display_claimed": 0,
"display_running": display_running,
"display_max_threads": display_max_threads,
},
"nodes": list(queue_nodes),
},
"active_job": {
"job_id": None,
"job_code": controller_job_code,
"status": controller_job_status,
"progress_percent": 0,
"items_total": 0,
"items_terminal": 0,
"items_pending": 0,
"items_claimed": 0,
"items_running": display_running,
"items_failed": 0,
"items_completed": 0,
"display_items_claimed": 0,
"display_items_running": display_running,
"display_active_threads": display_running,
"display_max_threads": display_max_threads,
"node_stats": list(queue_nodes),
"distributed_node_stats": list(queue_nodes),
},
"backlog": {},
"dependency_alerts": [],
}
cluster_payload = {
"nodes": [
{
"node_code": item["node_code"],
"role": item["role"],
"status": item["status"],
"current_load": item["current_load"],
"metadata": {
"active_threads": item["active_threads"],
"max_threads": item["max_threads"],
"detect_participating": item["detect_participating"],
"source_role": item["role"],
"source_status": item["status"],
},
}
for item in cluster_nodes
],
"nodes_total": len(cluster_nodes),
"summary": {
"busy_nodes": busy_nodes,
"stale_nodes": stale_nodes,
"offline_nodes": offline_nodes,
"online_worker_nodes": online_worker_nodes,
"dedicated_online_worker_nodes": dedicated_online_worker_nodes,
"online_control_nodes": online_control_nodes,
},
}
return append_runtime_projection_if_changed(
detect=detect_payload,
cluster=cluster_payload,
source_region=_normalize_region(settings.sync_source_region, settings.node_region),
target_region=_normalize_region(settings.sync_target_region, "overseas"),
)
def _format_time(value: datetime | None) -> str:
@@ -69,6 +299,42 @@ def _task_ack_url(base_url: str) -> str:
return f"{text}/api/v1/runtime/task-ack"
def _build_sync_pull_worker_wake_key(
*,
projection_job_code: str = "",
projection_cycle_token: str = "",
target_job_code: str = "",
source_record_id: int = 0,
) -> str:
scope = (
str(projection_cycle_token or "").strip()
or str(projection_job_code or "").strip()
or str(target_job_code or "").strip()
or f"record-{int(source_record_id or 0)}"
)
return f"{_SYNC_PULL_WORKER_WAKE_KEY_PREFIX}:{scope}"
def _acquire_sync_pull_worker_wake_guard(key: str, ttl_seconds: int = _SYNC_PULL_WORKER_WAKE_TTL_SECONDS) -> bool:
normalized_key = str(key or "").strip()
if not normalized_key:
return True
try:
redis_client = get_redis()
return bool(
redis_client.set(
normalized_key,
datetime.now().isoformat(timespec="seconds"),
ex=max(1, int(ttl_seconds or 1)),
nx=True,
)
)
except Exception:
# Wake dedupe is a throughput optimization; fall back to legacy behavior
# if Redis is temporarily unavailable.
return True
def _projection_ingest_type(sync_type: str) -> str:
if sync_type == "runtime_projection":
return "runtime_ingest"
@@ -126,21 +392,82 @@ def _refresh_remote_runtime_node(*, source_region: str, projection: dict, receiv
cleanup_imported_runtime_nodes(region=region, role=role, keep_node_code=node_code)
active_job = projection.get("active_job") or {}
worker_node_codes: list[str] = []
worker_rows_by_code: dict[str, dict] = {}
for cluster_node in list(projection.get("cluster_nodes") or []):
if not isinstance(cluster_node, dict):
continue
worker_node_code = str(cluster_node.get("node_code") or "").strip()
if not worker_node_code or worker_node_code == node_code:
continue
worker_rows_by_code[worker_node_code] = {
"node_code": worker_node_code,
"role": str(cluster_node.get("role") or "worker").strip() or "worker",
"status": str(cluster_node.get("status") or "").strip(),
"current_load": int(cluster_node.get("current_load", 0) or 0),
"active_threads": int(cluster_node.get("active_threads", 0) or 0),
"max_threads": int(cluster_node.get("max_threads", 0) or 0),
"detect_participating": bool(cluster_node.get("detect_participating", False)),
"items_total": 0,
"items_running": 0,
"items_claimed": 0,
"items_completed": 0,
"items_failed": 0,
"items_blacklisted": 0,
"metrics_source": "runtime",
}
for node_stat in list(active_job.get("node_stats") or []):
worker_node_code = str(node_stat.get("node_code") or "").strip()
if not worker_node_code or worker_node_code == "unassigned":
if not worker_node_code or worker_node_code == "unassigned" or worker_node_code == node_code:
continue
if worker_node_code == node_code:
continue
items_running = int(node_stat.get("items_running", 0) or 0)
items_claimed = int(node_stat.get("items_claimed", 0) or 0)
items_total = int(node_stat.get("items_total", 0) or 0)
worker_runtime_load = int(node_stat.get("current_load", 0) or 0)
worker_active_threads = int(node_stat.get("active_threads", worker_runtime_load) or 0)
worker_max_threads = int(node_stat.get("max_threads", 0) or 0)
worker_load = max(worker_active_threads, items_running, 0)
worker_status = "busy" if worker_load > 0 else "online"
worker_row = worker_rows_by_code.setdefault(
worker_node_code,
{
"node_code": worker_node_code,
"role": str(node_stat.get("role") or "worker").strip() or "worker",
"status": str(node_stat.get("status") or "").strip(),
"current_load": int(node_stat.get("current_load", 0) or 0),
"active_threads": int(node_stat.get("active_threads", 0) or 0),
"max_threads": int(node_stat.get("max_threads", 0) or 0),
"detect_participating": False,
"items_total": 0,
"items_running": 0,
"items_claimed": 0,
"items_completed": 0,
"items_failed": 0,
"items_blacklisted": 0,
"metrics_source": "runtime",
},
)
worker_row["role"] = str(node_stat.get("role") or worker_row.get("role") or "worker").strip() or "worker"
worker_row["status"] = str(node_stat.get("status") or worker_row.get("status") or "").strip()
worker_row["current_load"] = max(int(worker_row.get("current_load", 0) or 0), int(node_stat.get("current_load", 0) or 0))
worker_row["active_threads"] = max(int(worker_row.get("active_threads", 0) or 0), int(node_stat.get("active_threads", 0) or 0))
worker_row["max_threads"] = max(int(worker_row.get("max_threads", 0) or 0), int(node_stat.get("max_threads", 0) or 0))
worker_row["detect_participating"] = bool(
worker_row.get("detect_participating", False)
or int(node_stat.get("items_running", 0) or 0) > 0
or int(node_stat.get("items_claimed", 0) or 0) > 0
or int(node_stat.get("active_threads", 0) or 0) > 0
)
worker_row["items_total"] = int(node_stat.get("items_total", worker_row.get("items_total", 0)) or 0)
worker_row["items_running"] = int(node_stat.get("items_running", worker_row.get("items_running", 0)) or 0)
worker_row["items_claimed"] = int(node_stat.get("items_claimed", worker_row.get("items_claimed", 0)) or 0)
worker_row["items_completed"] = int(node_stat.get("items_completed", worker_row.get("items_completed", 0)) or 0)
worker_row["items_failed"] = int(node_stat.get("items_failed", worker_row.get("items_failed", 0)) or 0)
worker_row["items_blacklisted"] = int(node_stat.get("items_blacklisted", worker_row.get("items_blacklisted", 0)) or 0)
worker_row["metrics_source"] = str(node_stat.get("metrics_source") or worker_row.get("metrics_source") or "runtime").strip() or "runtime"
worker_node_codes: list[str] = []
for worker_node_code, worker_row in sorted(worker_rows_by_code.items()):
items_running = int(worker_row.get("items_running", 0) or 0)
items_claimed = int(worker_row.get("items_claimed", 0) or 0)
items_total = int(worker_row.get("items_total", 0) or 0)
worker_runtime_load = int(worker_row.get("current_load", 0) or 0)
worker_active_threads = int(worker_row.get("active_threads", worker_runtime_load) or 0)
worker_max_threads = int(worker_row.get("max_threads", 0) or 0)
worker_load = max(worker_active_threads, items_running, worker_runtime_load, 0)
worker_status = str(worker_row.get("status") or "").strip() or ("busy" if worker_load > 0 else "online")
worker_metadata = {
"service": "runtime-ingest",
"projection_source_region": source_region,
@@ -155,12 +482,13 @@ def _refresh_remote_runtime_node(*, source_region: str, projection: dict, receiv
"job_items_total": items_total,
"job_items_running": items_running,
"job_items_claimed": items_claimed,
"job_items_completed": int(node_stat.get("items_completed", 0) or 0),
"job_items_failed": int(node_stat.get("items_failed", 0) or 0),
"job_items_blacklisted": int(node_stat.get("items_blacklisted", 0) or 0),
"metrics_source": str(node_stat.get("metrics_source") or "runtime").strip() or "runtime",
"source_status": str(node_stat.get("status") or "").strip(),
"source_role": str(node_stat.get("role") or "worker").strip() or "worker",
"job_items_completed": int(worker_row.get("items_completed", 0) or 0),
"job_items_failed": int(worker_row.get("items_failed", 0) or 0),
"job_items_blacklisted": int(worker_row.get("items_blacklisted", 0) or 0),
"metrics_source": str(worker_row.get("metrics_source") or "runtime").strip() or "runtime",
"source_status": str(worker_row.get("status") or "").strip(),
"source_role": str(worker_row.get("role") or "worker").strip() or "worker",
"detect_participating": bool(worker_row.get("detect_participating", False)),
"derived_from": node_code,
}
register_node_heartbeat(
@@ -181,6 +509,7 @@ def _refresh_remote_runtime_node(*, source_region: str, projection: dict, receiv
def _load_latest_projection(sync_type: str) -> dict | None:
source_region = _normalize_region(settings.sync_source_region, settings.node_region)
target_region = _normalize_region(settings.sync_target_region, "overseas")
future_cutoff = datetime.now() + _RUNTIME_PROJECTION_FUTURE_SKEW_GRACE
with get_db() as conn:
with conn.cursor() as cur:
cur.execute(
@@ -190,12 +519,15 @@ def _load_latest_projection(sync_type: str) -> dict | None:
WHERE sync_type = %s
AND source_region = %s
AND target_region = %s
ORDER BY created_at DESC, id DESC
LIMIT 1
ORDER BY
CASE WHEN created_at <= %s THEN 0 ELSE 1 END ASC,
created_at DESC,
id DESC
LIMIT 200
""",
(sync_type, source_region, target_region),
(sync_type, source_region, target_region, future_cutoff),
)
row = cur.fetchone()
row = _pick_latest_projection_row(list(cur.fetchall() or []), created_at_index=5)
if not row:
return None
return {
@@ -222,7 +554,7 @@ def _load_pushable_projections(sync_type: str, limit: int) -> list[dict]:
WHERE sync_type = %s
AND source_region = %s
AND target_region = %s
ORDER BY created_at ASC, id ASC
ORDER BY created_at DESC, id DESC
LIMIT %s
""",
(sync_type, source_region, target_region, safe_limit * 5),
@@ -259,20 +591,158 @@ def _load_pushable_projections(sync_type: str, limit: int) -> list[dict]:
def _estimate_total_worker_threads(settings_payload: dict | None = None) -> int:
payload = settings_payload if isinstance(settings_payload, dict) else get_settings_payload()
default_threads = max(1, int(payload.get("thread_count", 100) or 100))
node_thread_counts = payload.get("node_thread_counts") if isinstance(payload.get("node_thread_counts"), dict) else {}
total_threads = 0
for raw_value in node_thread_counts.values():
thread_info = resolve_thread_count(settings.node_code, settings_payload=payload)
process_info = resolve_process_count(settings.node_code, settings_payload=payload)
effective_threads = max(
1,
int(thread_info.get("effective_thread_count", payload.get("thread_count", 100)) or 100),
)
effective_process_count = max(
1,
int(process_info.get("effective_process_count", payload.get("process_count", 1)) or 1),
)
return effective_threads * effective_process_count
def _sync_task_projection_limit_cap() -> int:
configured_cap = int(os.getenv("DOMAINCHECK_SYNC_TASK_LIMIT_CAP", "200000") or 200000)
return max(10000, configured_cap)
def _resolve_task_pull_request_limit(limit: int | None, settings_payload: dict | None = None) -> int:
configured = max(5000, int(settings.sync_batch_size or 200))
estimated_total_threads = _estimate_total_worker_threads(settings_payload)
cap = _sync_task_projection_limit_cap()
default_limit = max(
configured,
min(cap, max(10000, estimated_total_threads * 2)),
)
requested = int(limit or default_limit)
return max(1, min(requested, cap))
def _select_relevant_backlog_job_ids_from_rows(
job_rows: list[tuple[object, object, object]] | tuple[tuple[object, object, object], ...],
*,
freshness_hours: int = _LOCAL_BACKLOG_PENDING_FRESHNESS_HOURS,
limit: int = _LOCAL_BACKLOG_MAX_JOBS,
) -> list[int]:
safe_limit = max(1, min(int(limit or _LOCAL_BACKLOG_MAX_JOBS), 16))
safe_freshness_hours = max(1, min(int(freshness_hours or _LOCAL_BACKLOG_PENDING_FRESHNESS_HOURS), 168))
selected: list[int] = []
fallback_job_id = 0
for raw_job_id, raw_status, raw_activity_at in list(job_rows or []):
try:
total_threads += max(0, int(raw_value or 0))
job_id = int(raw_job_id or 0)
except (TypeError, ValueError):
continue
return max(total_threads, default_threads)
if job_id <= 0:
continue
if fallback_job_id <= 0:
fallback_job_id = job_id
if job_id in selected:
continue
status = str(raw_status or "").strip().lower()
keep = status == "running"
if not keep and raw_activity_at is not None:
now = datetime.now(raw_activity_at.tzinfo) if getattr(raw_activity_at, "tzinfo", None) else datetime.now()
keep = now - raw_activity_at <= timedelta(hours=safe_freshness_hours)
if not keep:
continue
selected.append(job_id)
if len(selected) >= safe_limit:
break
if not selected and fallback_job_id > 0:
selected.append(fallback_job_id)
return selected
def _build_backlog_snapshot_from_active_job(active_job: dict | None) -> dict:
normalized_job = dict(active_job or {})
if not normalized_job:
return {}
pending_total = max(0, int(normalized_job.get("items_pending", 0) or 0))
claimed_total = max(
max(
int(normalized_job.get("items_claimed", 0) or 0),
int(normalized_job.get("display_items_claimed", 0) or 0),
),
0,
)
running_total = max(
max(
int(normalized_job.get("items_running", 0) or 0),
int(normalized_job.get("display_items_running", 0) or 0),
),
int(normalized_job.get("display_active_threads", 0) or 0),
0,
)
register_pending = 0
downstream_pending = 0
for raw_step in list(normalized_job.get("step_stats") or normalized_job.get("raw_step_stats") or []):
if not isinstance(raw_step, dict):
continue
step_code = str(raw_step.get("step_code") or raw_step.get("code") or "").strip()
step_pending = max(
int(raw_step.get("items_pending", raw_step.get("pending", 0)) or 0),
0,
)
if step_pending <= 0:
continue
if step_code == "detect_register":
register_pending += step_pending
else:
downstream_pending += step_pending
if register_pending <= 0 and downstream_pending <= 0 and pending_total > 0:
downstream_pending = pending_total
if pending_total <= 0 and claimed_total <= 0 and running_total <= 0:
return {}
return {
"pending_total": pending_total,
"claimed_total": claimed_total,
"running_total": running_total,
"register_pending": register_pending,
"downstream_pending": downstream_pending,
}
def _load_local_detect_backlog_snapshot() -> dict:
active_job_snapshot = _build_backlog_snapshot_from_active_job(
get_active_detect_job_summary(event_limit=1)
)
if active_job_snapshot:
return active_job_snapshot
with get_db() as conn:
with conn.cursor() as cur:
cur.execute(
"""
SELECT id, status, COALESCE(started_at, created_at) AS activity_at
FROM detect_jobs
WHERE status IN ('pending', 'running')
ORDER BY
CASE WHEN status = 'running' THEN 0 ELSE 1 END,
COALESCE(started_at, created_at) DESC,
id DESC
LIMIT %s
""",
(_LOCAL_BACKLOG_MAX_JOBS * 8,),
)
selected_job_ids = _select_relevant_backlog_job_ids_from_rows(cur.fetchall())
if not selected_job_ids:
return {
"pending_total": 0,
"claimed_total": 0,
"running_total": 0,
"register_pending": 0,
"downstream_pending": 0,
}
cur.execute(
"""
SELECT
@@ -282,9 +752,9 @@ def _load_local_detect_backlog_snapshot() -> dict:
COUNT(*) FILTER (WHERE item.status = 'pending' AND item.step_code = 'detect_register') AS register_pending,
COUNT(*) FILTER (WHERE item.status = 'pending' AND item.step_code <> 'detect_register') AS downstream_pending
FROM detect_job_items item
JOIN detect_jobs job ON job.id = item.job_id
WHERE job.status IN ('pending', 'running')
"""
WHERE item.job_id = ANY(%s)
""",
(selected_job_ids,),
)
row = cur.fetchone() or (0, 0, 0, 0, 0)
return {
@@ -422,9 +892,9 @@ def _task_selection_sql() -> str:
def _task_projection_limit(limit: int | None) -> int:
requested = max(1, int(limit or 5000))
requested = max(1, int(limit or max(5000, int(settings.sync_batch_size or 200))))
configured = max(5000, int(settings.sync_batch_size or 200))
cap = max(10000, configured, 5000)
cap = max(configured, _sync_task_projection_limit_cap())
return max(1, min(requested, cap))
@@ -1422,15 +1892,21 @@ def ingest_runtime_projection(payload: dict, *, shared_token: str | None = None)
def _push_projection_now(sync_type: str, ingest_url: str) -> tuple[bool, str, dict]:
if sync_type == "runtime_projection":
# Regenerate the runtime snapshot before every push so the sync agent
# does not keep replaying a stale projection record while the worker
# thread count / phase is still changing.
from app.services.runtime_status_service import get_runtime_status
if _fast_runtime_projection_enabled():
try:
_append_fast_runtime_projection_snapshot()
except Exception as exc:
return False, f"快速刷新 runtime_projection 失败: {exc}", {"action": "push_sync", "sync_type": sync_type}
else:
# Refresh only the lightweight runtime projection snapshot before every
# push so the sync agent does not keep replaying a stale record while
# avoiding the full runtime/status assembly cost.
from app.services.runtime_status_service import refresh_runtime_projection_snapshot
try:
get_runtime_status()
except Exception as exc:
return False, f"刷新 runtime_projection 失败: {exc}", {"action": "push_sync", "sync_type": sync_type}
try:
refresh_runtime_projection_snapshot(window_minutes=15)
except Exception as exc:
return False, f"刷新 runtime_projection 失败: {exc}", {"action": "push_sync", "sync_type": sync_type}
source_record = _load_latest_projection(sync_type)
if not source_record:
return False, f"当前没有可推送的{sync_type}", {"action": "push_sync", "sync_type": sync_type}
@@ -1667,12 +2143,10 @@ def pull_detect_task_batch_now(limit: int | None = None) -> tuple[bool, str, dic
if not export_url or not ack_url:
return False, "未配置任务拉取目标地址", {"action": "pull_tasks", "pull_state": "misconfigured", "ui_level": "warning", "poll_schedule_seconds": []}
configured_limit = max(5000, int(settings.sync_batch_size or 200))
requested_limit = int(limit or configured_limit)
safe_limit = max(1, min(requested_limit, max(10000, configured_limit)))
settings_payload = get_settings_payload()
safe_limit = _resolve_task_pull_request_limit(limit, settings_payload=settings_payload)
backlog_snapshot = _load_local_detect_backlog_snapshot()
backlog_limits = _build_task_pull_backlog_limits(configured_limit, settings_payload=settings_payload)
backlog_limits = _build_task_pull_backlog_limits(safe_limit, settings_payload=settings_payload)
should_throttle, throttle_reason = _should_throttle_task_pull(backlog_snapshot, backlog_limits)
if should_throttle:
return True, "本地待处理积压较高，暂停拉取新批次", {
@@ -1804,17 +2278,48 @@ def pull_detect_task_batch_now(limit: int | None = None) -> tuple[bool, str, dic
try:
from app.services.worker_control_service import send_worker_command
start_ok, start_message = send_worker_command(
"start_detection",
payload={
"source": "sync-pull",
"source_record_id": source_record_id,
"target_job_id": int(result.get("target_job_id", 0) or 0),
"target_job_code": str(result.get("target_job_code") or "").strip(),
},
projection_active_job = dict(projection.get("active_job") or {})
projection_job_id = int(projection_active_job.get("job_id", 0) or 0)
projection_job_code = str(projection_active_job.get("job_code") or "").strip()
projection_cycle_token = str(
projection_active_job.get("current_cycle_token")
or projection_active_job.get("cycle_token")
or ""
).strip()
start_payload = {
"source": "sync-pull",
"source_record_id": source_record_id,
"target_job_id": int(result.get("target_job_id", 0) or 0),
"target_job_code": str(result.get("target_job_code") or "").strip(),
}
# Mainland ingest creates a local target_job_* for queue ownership, but
# worker runtime/log identity should still follow the upstream active
# detect job so cluster aggregation keeps controller activity attached
# to the real pipeline job instead of the local sync-pull surrogate.
if projection_job_id > 0:
start_payload["job_id"] = projection_job_id
if projection_job_code:
start_payload["job_code"] = projection_job_code
if projection_cycle_token:
start_payload["cycle_token"] = projection_cycle_token
wake_guard_key = _build_sync_pull_worker_wake_key(
projection_job_code=projection_job_code,
projection_cycle_token=projection_cycle_token,
target_job_code=str(result.get("target_job_code") or "").strip(),
source_record_id=source_record_id,
)
result["worker_start_ok"] = bool(start_ok)
result["worker_start_message"] = str(start_message or "").strip()
if _acquire_sync_pull_worker_wake_guard(wake_guard_key):
start_ok, start_message = send_worker_command(
"start_detection",
payload=start_payload,
)
result["worker_start_ok"] = bool(start_ok)
result["worker_start_message"] = str(start_message or "").strip()
result["worker_start_skipped"] = False
else:
result["worker_start_ok"] = True
result["worker_start_skipped"] = True
result["worker_start_message"] = "已跳过同任务短窗内重复 Worker 唤起"
except Exception as exc:
result["worker_start_ok"] = False
result["worker_start_message"] = f"同步入库后自动唤起 Worker 失败: {exc}"