admin/getDomain
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

d

Browse Source
This commit is contained in:
Your Name
2026-04-22 14:13:21 +08:00
parent e0406b5d0e
commit 7cbde2aa78
145 changed files with 23086 additions and 2243 deletions
Download Patch File Download Diff File Expand all files Collapse all files

566
domain-api/app/services/sync_push_service.py
View File

@@ -11,7 +11,16 @@ from uuid import uuid4
from app.core.config import settings
from app.core.db import get_db
from app.services.cluster_runtime_service import cleanup_imported_runtime_nodes, register_node_heartbeat
from app.services.cluster_runtime_service import (
cleanup_imported_runtime_nodes,
cleanup_imported_runtime_nodes_many,
register_node_heartbeat,
)
from app.services.detect_job_service import (
_load_domain_pipeline_snapshot,
resolve_initial_domain_pipeline_item,
)
from app.services.settings_service import get_settings_payload
from app.services.sync_record_service import _decode_json, _normalize_region
@@ -80,10 +89,16 @@ def _refresh_remote_runtime_node(*, source_region: str, projection: dict, receiv
if not node_code:
node_code = f"{region}-{role}-imported"
controller_current_load = max(
int(projection.get("active_thread_count", 0) or 0),
int(((projection.get("active_job") or {}).get("items_running", 0) or 0)),
)
metadata = {
"service": "runtime-ingest",
"projection_source_region": source_region,
"worker_mode": projection.get("worker_mode", ""),
"active_threads": int(projection.get("active_thread_count", 0) or 0),
"max_threads": int(projection.get("max_thread_count", 0) or 0),
"phase_label": projection.get("phase_label", ""),
"phase_detail": projection.get("phase_detail", ""),
"proxy_runtime_label": projection.get("proxy_runtime_label", ""),
@@ -102,8 +117,8 @@ def _refresh_remote_runtime_node(*, source_region: str, projection: dict, receiv
node_code=node_code,
region=region,
role=role,
status="online",
current_load=int(((projection.get("progress") or {}).get("running", 0) or 0)),
status="busy" if controller_current_load > 0 else "online",
current_load=controller_current_load,
metadata=metadata,
hostname_override=hostname,
ip_override=ip,
@@ -111,15 +126,21 @@ def _refresh_remote_runtime_node(*, source_region: str, projection: dict, receiv
cleanup_imported_runtime_nodes(region=region, role=role, keep_node_code=node_code)
active_job = projection.get("active_job") or {}
worker_node_codes: list[str] = []
for node_stat in list(active_job.get("node_stats") or []):
worker_node_code = str(node_stat.get("node_code") or "").strip()
if not worker_node_code or worker_node_code == "unassigned":
continue
if worker_node_code == node_code:
continue
items_running = int(node_stat.get("items_running", 0) or 0)
items_claimed = int(node_stat.get("items_claimed", 0) or 0)
items_total = int(node_stat.get("items_total", 0) or 0)
worker_status = "busy" if (items_running > 0 or items_claimed > 0) else "online"
worker_load = max(items_running, items_claimed, 0)
worker_runtime_load = int(node_stat.get("current_load", 0) or 0)
worker_active_threads = int(node_stat.get("active_threads", worker_runtime_load) or 0)
worker_max_threads = int(node_stat.get("max_threads", 0) or 0)
worker_load = max(worker_active_threads, items_running, 0)
worker_status = "busy" if worker_load > 0 else "online"
worker_metadata = {
"service": "runtime-ingest",
"projection_source_region": source_region,
@@ -128,10 +149,18 @@ def _refresh_remote_runtime_node(*, source_region: str, projection: dict, receiv
"phase_detail": projection.get("phase_detail", ""),
"proxy_runtime_label": projection.get("proxy_runtime_label", ""),
"proxy_runtime_reason": projection.get("proxy_runtime_reason", ""),
"active_threads": worker_active_threads,
"max_threads": worker_max_threads,
"updated_at": _format_time(received_at or datetime.now()),
"job_items_total": items_total,
"job_items_running": items_running,
"job_items_claimed": items_claimed,
"job_items_completed": int(node_stat.get("items_completed", 0) or 0),
"job_items_failed": int(node_stat.get("items_failed", 0) or 0),
"job_items_blacklisted": int(node_stat.get("items_blacklisted", 0) or 0),
"metrics_source": str(node_stat.get("metrics_source") or "runtime").strip() or "runtime",
"source_status": str(node_stat.get("status") or "").strip(),
"source_role": str(node_stat.get("role") or "worker").strip() or "worker",
"derived_from": node_code,
}
register_node_heartbeat(
@@ -144,7 +173,9 @@ def _refresh_remote_runtime_node(*, source_region: str, projection: dict, receiv
hostname_override=hostname,
ip_override=ip,
)
cleanup_imported_runtime_nodes(region=region, role="worker", keep_node_code=worker_node_code)
worker_node_codes.append(worker_node_code)
if worker_node_codes:
cleanup_imported_runtime_nodes_many(region=region, role="worker", keep_node_codes=worker_node_codes)
def _load_latest_projection(sync_type: str) -> dict | None:
@@ -226,6 +257,88 @@ def _load_pushable_projections(sync_type: str, limit: int) -> list[dict]:
return selected
def _estimate_total_worker_threads(settings_payload: dict | None = None) -> int:
payload = settings_payload if isinstance(settings_payload, dict) else get_settings_payload()
default_threads = max(1, int(payload.get("thread_count", 100) or 100))
node_thread_counts = payload.get("node_thread_counts") if isinstance(payload.get("node_thread_counts"), dict) else {}
total_threads = 0
for raw_value in node_thread_counts.values():
try:
total_threads += max(0, int(raw_value or 0))
except (TypeError, ValueError):
continue
return max(total_threads, default_threads)
def _load_local_detect_backlog_snapshot() -> dict:
with get_db() as conn:
with conn.cursor() as cur:
cur.execute(
"""
SELECT
COUNT(*) FILTER (WHERE item.status = 'pending') AS pending_total,
COUNT(*) FILTER (WHERE item.status = 'claimed') AS claimed_total,
COUNT(*) FILTER (WHERE item.status = 'running') AS running_total,
COUNT(*) FILTER (WHERE item.status = 'pending' AND item.step_code = 'detect_register') AS register_pending,
COUNT(*) FILTER (WHERE item.status = 'pending' AND item.step_code <> 'detect_register') AS downstream_pending
FROM detect_job_items item
JOIN detect_jobs job ON job.id = item.job_id
WHERE job.status IN ('pending', 'running')
"""
)
row = cur.fetchone() or (0, 0, 0, 0, 0)
return {
"pending_total": int(row[0] or 0),
"claimed_total": int(row[1] or 0),
"running_total": int(row[2] or 0),
"register_pending": int(row[3] or 0),
"downstream_pending": int(row[4] or 0),
}
def _build_task_pull_backlog_limits(configured_limit: int, settings_payload: dict | None = None) -> dict:
estimated_total_threads = _estimate_total_worker_threads(settings_payload)
max_pending_total = int(settings.sync_pull_max_pending_items or 0)
if max_pending_total <= 0:
max_pending_total = max(int(configured_limit or 0), estimated_total_threads * 2)
max_register_pending = int(settings.sync_pull_max_register_pending_items or 0)
if max_register_pending <= 0:
max_register_pending = max(max(500, int(configured_limit or 0) // 2), estimated_total_threads)
max_downstream_pending = int(settings.sync_pull_max_downstream_pending_items or 0)
if max_downstream_pending <= 0:
max_downstream_pending = max(250, estimated_total_threads // 4)
return {
"estimated_total_threads": estimated_total_threads,
"max_pending_total": max_pending_total,
"max_register_pending": max_register_pending,
"max_downstream_pending": max_downstream_pending,
}
def _should_throttle_task_pull(backlog_snapshot: dict, backlog_limits: dict) -> tuple[bool, str]:
pending_total = int(backlog_snapshot.get("pending_total", 0) or 0)
register_pending = int(backlog_snapshot.get("register_pending", 0) or 0)
downstream_pending = int(backlog_snapshot.get("downstream_pending", 0) or 0)
max_pending_total = int(backlog_limits.get("max_pending_total", 0) or 0)
max_register_pending = int(backlog_limits.get("max_register_pending", 0) or 0)
max_downstream_pending = int(backlog_limits.get("max_downstream_pending", 0) or 0)
if max_pending_total > 0 and pending_total >= max_pending_total:
return True, "pending_total"
if (
downstream_pending > 0
and max_register_pending > 0
and register_pending >= max_register_pending
):
return True, "register_pending"
if max_downstream_pending > 0 and downstream_pending >= max_downstream_pending:
return True, "downstream_pending"
return False, ""
def _latest_push_attempt(source_record_id: int, target_region: str, sync_type: str) -> dict | None:
with get_db() as conn:
with conn.cursor() as cur:
@@ -308,10 +421,51 @@ def _task_selection_sql() -> str:
"""
def _task_projection_limit(limit: int | None) -> int:
requested = max(1, int(limit or 5000))
configured = max(5000, int(settings.sync_batch_size or 200))
cap = max(10000, configured, 5000)
return max(1, min(requested, cap))
def _task_projection_items_total(projection: dict) -> int:
payload = projection.get("payload") or {}
projection_payload = payload.get("projection") or {}
try:
return int(projection_payload.get("items_total", 0) or 0)
except Exception:
return 0
def _task_projection_selection_limit(projection: dict) -> int:
payload = projection.get("payload") or {}
projection_payload = payload.get("projection") or {}
try:
return int(projection_payload.get("selection_limit", 0) or 0)
except Exception:
return 0
def _mark_task_projection_superseded(record_id: int, *, reason: str) -> None:
with get_db() as conn:
with conn.cursor() as cur:
cur.execute(
"""
UPDATE detect_sync_records
SET status = 'superseded',
error_message = %s,
updated_at = CURRENT_TIMESTAMP
WHERE id = %s
""",
(str(reason or "").strip()[:500], int(record_id)),
)
conn.commit()
def _load_pending_task_projection(limit: int) -> dict | None:
source_region = _normalize_region(settings.sync_source_region, settings.node_region)
target_region = _normalize_region(settings.sync_target_region, "overseas")
safe_limit = max(1, min(int(limit or 1000), max(1, int(settings.sync_batch_size or 200))))
safe_limit = _task_projection_limit(limit)
with get_db() as conn:
with conn.cursor() as cur:
cur.execute(
@@ -340,6 +494,22 @@ def _load_pending_task_projection(limit: int) -> dict | None:
latest_ingest = _latest_ingest_attempt(projection["id"], projection["target_region"], "detect_task_projection")
if latest_ingest and latest_ingest["status"] == "received":
continue
items_total = _task_projection_items_total(projection)
selection_limit = _task_projection_selection_limit(projection)
created_at = projection.get("created_at")
if (
safe_limit >= 1000
and max(items_total, selection_limit) > 0
and max(items_total, selection_limit) < safe_limit
and isinstance(created_at, datetime)
):
now = datetime.now(created_at.tzinfo) if created_at.tzinfo else datetime.now()
if now - created_at >= timedelta(minutes=10):
_mark_task_projection_superseded(
projection["id"],
reason=f"stale small task projection skipped: items_total={items_total}, selection_limit={selection_limit}, requested_limit={safe_limit}",
)
continue
return projection
return None
@@ -361,7 +531,7 @@ def export_detect_task_projection(limit: int = 1000, *, shared_token: str | None
source_region = _normalize_region(settings.sync_source_region, settings.node_region)
target_region = _normalize_region(settings.sync_target_region, "overseas")
safe_limit = max(1, min(int(limit or 1000), max(1, int(settings.sync_batch_size or 200))))
safe_limit = _task_projection_limit(limit)
with get_db() as conn:
with conn.cursor() as cur:
cur.execute(_task_selection_sql(), (safe_limit,))
@@ -528,6 +698,7 @@ def ingest_detect_task_projection(payload: dict, *, shared_token: str | None = N
inserted_count = 0
updated_count = 0
domain_ids: list[int] = []
for item in items:
domain = str(item.get("domain") or "").strip().lower()
if not domain:
@@ -562,7 +733,7 @@ def ingest_detect_task_projection(payload: dict, *, shared_token: str | None = N
ELSE EXCLUDED.detect_status
END,
update_time = CURRENT_TIMESTAMP
RETURNING (xmax = 0) AS inserted
RETURNING id, (xmax = 0) AS inserted
""",
(
domain,
@@ -576,12 +747,120 @@ def ingest_detect_task_projection(payload: dict, *, shared_token: str | None = N
int(item.get("juziseo_status") or 0),
),
)
inserted = bool((cur.fetchone() or [False])[0])
row = cur.fetchone() or [0, False]
domain_id = int(row[0] or 0)
inserted = bool(row[1])
if domain_id > 0:
domain_ids.append(domain_id)
if inserted:
inserted_count += 1
else:
updated_count += 1
target_job_code = f"sync-{source_region}-{source_record_id}"
target_job_remark = (
f"同步拉取待检测批次 {str(projection.get('batch_code') or '').strip() or source_record_id}，"
f"共 {len(domain_ids)} 个域名"
)
cur.execute(
"""
INSERT INTO detect_jobs (job_code, source, plan_hash, task_mode, step_code, status, remark, created_by)
VALUES (%s, %s, %s, 'domain_pipeline', '', 'pending', %s, %s)
ON CONFLICT (job_code) DO UPDATE SET
source = EXCLUDED.source,
plan_hash = EXCLUDED.plan_hash,
task_mode = EXCLUDED.task_mode,
step_code = EXCLUDED.step_code,
remark = EXCLUDED.remark,
created_by = EXCLUDED.created_by,
status = CASE
WHEN detect_jobs.status IN ('completed', 'failed', 'cancelled') THEN 'pending'
ELSE detect_jobs.status
END,
started_at = CASE
WHEN detect_jobs.status IN ('completed', 'failed', 'cancelled') THEN NULL
ELSE detect_jobs.started_at
END,
finished_at = CASE
WHEN detect_jobs.status IN ('completed', 'failed', 'cancelled') THEN NULL
ELSE detect_jobs.finished_at
END
RETURNING id
""",
(
target_job_code,
"sync-pull",
projection_hash,
target_job_remark,
"sync-agent",
),
)
target_job_id = int((cur.fetchone() or [0])[0] or 0)
settings_payload = get_settings_payload()
queued_count = 0
deduplicated_job_items = 0
skipped_job_items = 0
if target_job_id > 0:
for domain_id in domain_ids:
domain_snapshot = _load_domain_pipeline_snapshot(cur, int(domain_id))
if not domain_snapshot:
skipped_job_items += 1
continue
item_step_code, step_payload = resolve_initial_domain_pipeline_item(
domain_snapshot,
settings_payload=settings_payload,
)
if not item_step_code or not step_payload:
skipped_job_items += 1
continue
cur.execute(
"""
INSERT INTO detect_job_items (job_id, domain_id, step_code, status, step_payload_json)
VALUES (%s, %s, %s, 'pending', %s::jsonb)
ON CONFLICT (job_id, domain_id, step_code) DO NOTHING
RETURNING id
""",
(
target_job_id,
domain_id,
item_step_code,
json.dumps(step_payload, ensure_ascii=False),
),
)
inserted_job_item = cur.fetchone()
if inserted_job_item:
queued_count += 1
else:
deduplicated_job_items += 1
if queued_count > 0:
cur.execute(
"""
INSERT INTO detect_run_events (job_id, node_code, event_type, level, message, payload_json)
VALUES (%s, %s, %s, %s, %s, %s::jsonb)
""",
(
target_job_id,
settings.node_code,
"job_created",
"info",
f"同步拉取待检测批次 {target_job_code}，共 {queued_count} 个任务项",
json.dumps(
{
"source_region": source_region,
"source_record_id": source_record_id,
"projection_hash": projection_hash,
"batch_code": str(projection.get("batch_code") or "").strip(),
"queued_count": queued_count,
"deduplicated_job_items": deduplicated_job_items,
"skipped_job_items": skipped_job_items,
},
ensure_ascii=False,
),
),
)
cur.execute(
"""
INSERT INTO detect_sync_records (
@@ -602,6 +881,11 @@ def ingest_detect_task_projection(payload: dict, *, shared_token: str | None = N
"items_total": len(items),
"inserted_count": inserted_count,
"updated_count": updated_count,
"target_job_id": target_job_id,
"target_job_code": target_job_code,
"queued_count": queued_count,
"deduplicated_job_items": deduplicated_job_items,
"skipped_job_items": skipped_job_items,
"received_at": _format_time(received_at),
},
ensure_ascii=False,
@@ -617,6 +901,11 @@ def ingest_detect_task_projection(payload: dict, *, shared_token: str | None = N
"items_total": len(items),
"inserted_count": inserted_count,
"updated_count": updated_count,
"target_job_id": target_job_id,
"target_job_code": target_job_code,
"queued_count": queued_count,
"deduplicated_job_items": deduplicated_job_items,
"skipped_job_items": skipped_job_items,
"deduplicated": False,
}
@@ -742,7 +1031,27 @@ def _extract_detect_result_projection_events(
return events
def _resolve_detect_result_target_job_id() -> int:
def _resolve_detect_result_target_job_id(*, projection: dict) -> int:
source_job = projection.get("job") or {}
source_job_code = str(source_job.get("job_code") or "").strip()
with get_db() as conn:
with conn.cursor() as cur:
if source_job_code:
cur.execute(
"""
SELECT id
FROM detect_jobs
WHERE job_code = %s
ORDER BY id DESC
LIMIT 1
""",
(source_job_code,),
)
row = cur.fetchone()
if row:
return int(row[0] or 0)
from app.services.detect_job_service import get_active_detect_job_summary
active_job = get_active_detect_job_summary(event_limit=1) or {}
@@ -759,6 +1068,163 @@ def _parse_event_created_at(value: str) -> datetime | None:
return None
def _extract_event_domain(payload: dict, message: str) -> str:
domain = str(payload.get("domain") or "").strip().lower()
if domain:
return domain
text = str(message or "").strip()
if ":" in text:
candidate = text.rsplit(":", 1)[-1].strip().lower()
if candidate:
return candidate
return ""
def _apply_detect_result_event_to_domain(cur, event: dict) -> None:
payload = _decode_json(event.get("payload"))
domain = _extract_event_domain(payload, str(event.get("message") or ""))
if not domain:
return
event_type = str(event.get("event_type") or "").strip()
created_at = _parse_event_created_at(str(event.get("created_at") or ""))
effective_time = created_at or datetime.now()
if event_type == "domain_started":
cur.execute(
"""
UPDATE domains
SET detect_status = CASE
WHEN detect_status IN (1, 3) THEN detect_status
ELSE 2
END,
update_time = CURRENT_TIMESTAMP
WHERE domain = %s
""",
(domain,),
)
return
if event_type == "domain_completed":
cur.execute(
"""
UPDATE domains
SET detect_status = 1,
detect_time = COALESCE(detect_time, %s),
update_time = CURRENT_TIMESTAMP
WHERE domain = %s
""",
(effective_time, domain),
)
return
if event_type == "domain_blacklisted":
cur.execute(
"""
UPDATE domains
SET detect_status = 3,
update_time = CURRENT_TIMESTAMP
WHERE domain = %s
""",
(domain,),
)
return
if event_type == "domain_failed":
cur.execute(
"""
UPDATE domains
SET detect_status = CASE
WHEN detect_status IN (1, 3) THEN detect_status
ELSE 4
END,
update_time = CURRENT_TIMESTAMP
WHERE domain = %s
""",
(domain,),
)
return
def _apply_detect_result_event_to_job_item(cur, *, target_job_id: int, event: dict) -> int:
if int(target_job_id or 0) <= 0:
return 0
payload = _decode_json(event.get("payload"))
domain = _extract_event_domain(payload, str(event.get("message") or ""))
if not domain:
return 0
event_type = str(event.get("event_type") or "").strip()
node_code = str(event.get("node_code") or "").strip()
message = str(event.get("message") or "").strip()
if event_type == "domain_started":
cur.execute(
"""
UPDATE detect_job_items AS item
SET status = CASE
WHEN item.status IN ('completed', 'blacklisted', 'failed') THEN item.status
ELSE 'running'
END,
claimed_by = CASE
WHEN %s <> '' THEN %s
ELSE item.claimed_by
END,
started_at = COALESCE(item.started_at, CURRENT_TIMESTAMP),
updated_at = CURRENT_TIMESTAMP
FROM domains AS d
WHERE item.job_id = %s
AND item.domain_id = d.id
AND d.domain = %s
AND item.status IN ('pending', 'claimed', 'running')
""",
(node_code, node_code, int(target_job_id), domain),
)
return int(cur.rowcount or 0)
if event_type not in {"domain_completed", "domain_blacklisted", "domain_failed"}:
return 0
final_status = {
"domain_completed": "completed",
"domain_blacklisted": "blacklisted",
"domain_failed": "failed",
}[event_type]
cur.execute(
"""
UPDATE detect_job_items AS item
SET status = %s,
claimed_by = CASE
WHEN %s <> '' THEN %s
ELSE item.claimed_by
END,
finished_at = COALESCE(item.finished_at, CURRENT_TIMESTAMP),
updated_at = CURRENT_TIMESTAMP,
lease_expires_at = NULL,
last_error = CASE
WHEN %s = 'failed' THEN LEFT(%s, 1000)
ELSE item.last_error
END
FROM domains AS d
WHERE item.job_id = %s
AND item.domain_id = d.id
AND d.domain = %s
AND item.status IN ('pending', 'claimed', 'running')
""",
(
final_status,
node_code,
node_code,
final_status,
message,
int(target_job_id),
domain,
),
)
return int(cur.rowcount or 0)
def _import_detect_result_projection_events(
*,
source_region: str,
@@ -773,12 +1239,11 @@ def _import_detect_result_projection_events(
if not events:
return {"imported_count": 0, "deduplicated_count": 0, "target_job_id": 0}
target_job_id = _resolve_detect_result_target_job_id()
if target_job_id <= 0:
return {"imported_count": 0, "deduplicated_count": 0, "target_job_id": 0}
target_job_id = _resolve_detect_result_target_job_id(projection=projection)
imported_count = 0
deduplicated_count = 0
updated_job_items = 0
with get_db() as conn:
with conn.cursor() as cur:
for event in events:
@@ -831,12 +1296,27 @@ def _import_detect_result_projection_events(
json.dumps(event["payload"], ensure_ascii=False),
),
)
_apply_detect_result_event_to_domain(cur, event)
if target_job_id > 0:
updated_job_items += _apply_detect_result_event_to_job_item(
cur,
target_job_id=target_job_id,
event=event,
)
imported_count += 1
conn.commit()
if target_job_id > 0:
from app.services.detect_job_service import refresh_detect_job_status
try:
refresh_detect_job_status(target_job_id)
except Exception:
pass
return {
"imported_count": imported_count,
"deduplicated_count": deduplicated_count,
"target_job_id": target_job_id,
"updated_job_items": updated_job_items,
}
@@ -941,6 +1421,16 @@ def ingest_runtime_projection(payload: dict, *, shared_token: str | None = None)
def _push_projection_now(sync_type: str, ingest_url: str) -> tuple[bool, str, dict]:
if sync_type == "runtime_projection":
# Regenerate the runtime snapshot before every push so the sync agent
# does not keep replaying a stale projection record while the worker
# thread count / phase is still changing.
from app.services.runtime_status_service import get_runtime_status
try:
get_runtime_status()
except Exception as exc:
return False, f"刷新 runtime_projection 失败: {exc}", {"action": "push_sync", "sync_type": sync_type}
source_record = _load_latest_projection(sync_type)
if not source_record:
return False, f"当前没有可推送的{sync_type}", {"action": "push_sync", "sync_type": sync_type}
@@ -1177,15 +1667,33 @@ def pull_detect_task_batch_now(limit: int | None = None) -> tuple[bool, str, dic
if not export_url or not ack_url:
return False, "未配置任务拉取目标地址", {"action": "pull_tasks", "pull_state": "misconfigured", "ui_level": "warning", "poll_schedule_seconds": []}
safe_limit = max(1, min(int(limit or settings.sync_batch_size or 200), max(1, int(settings.sync_batch_size or 200))))
configured_limit = max(5000, int(settings.sync_batch_size or 200))
requested_limit = int(limit or configured_limit)
safe_limit = max(1, min(requested_limit, max(10000, configured_limit)))
settings_payload = get_settings_payload()
backlog_snapshot = _load_local_detect_backlog_snapshot()
backlog_limits = _build_task_pull_backlog_limits(configured_limit, settings_payload=settings_payload)
should_throttle, throttle_reason = _should_throttle_task_pull(backlog_snapshot, backlog_limits)
if should_throttle:
return True, "本地待处理积压较高，暂停拉取新批次", {
"action": "pull_tasks",
"pull_state": "throttled",
"ui_level": "info",
"poll_schedule_seconds": [1, 3],
"reason": throttle_reason,
**backlog_snapshot,
**backlog_limits,
}
request_url = f"{export_url}?limit={safe_limit}"
export_timeout = max(20, min(90, 15 + safe_limit // 40))
request = urllib.request.Request(
request_url,
headers={**({"X-Domaincheck-Sync-Token": settings.sync_shared_token} if settings.sync_shared_token else {})},
method="GET",
)
try:
with urllib.request.urlopen(request, timeout=20) as response:
with urllib.request.urlopen(request, timeout=export_timeout) as response:
raw = response.read().decode("utf-8")
data = json.loads(raw) if raw else {}
except json.JSONDecodeError as exc:
@@ -1280,7 +1788,7 @@ def pull_detect_task_batch_now(limit: int | None = None) -> tuple[bool, str, dic
**(ingest_data or {}),
}
return True, "待检测任务批次拉取并入库成功", {
result = {
"action": "pull_tasks",
"pull_state": "success",
"ui_level": "success",
@@ -1291,3 +1799,27 @@ def pull_detect_task_batch_now(limit: int | None = None) -> tuple[bool, str, dic
**(ingest_data or {}),
"ack": ack_data,
}
queued_count = int(result.get("queued_count", 0) or 0)
if queued_count > 0:
try:
from app.services.worker_control_service import send_worker_command
start_ok, start_message = send_worker_command(
"start_detection",
payload={
"source": "sync-pull",
"source_record_id": source_record_id,
"target_job_id": int(result.get("target_job_id", 0) or 0),
"target_job_code": str(result.get("target_job_code") or "").strip(),
},
)
result["worker_start_ok"] = bool(start_ok)
result["worker_start_message"] = str(start_message or "").strip()
except Exception as exc:
result["worker_start_ok"] = False
result["worker_start_message"] = f"同步入库后自动唤起 Worker 失败: {exc}"
result["ui_level"] = "warning"
result["pull_state"] = "worker_start_warning"
return True, "待检测任务批次拉取并入库成功；但自动唤起 Worker 失败", result
return True, "待检测任务批次拉取并入库成功", result