feat: stabilize multi-region runtime sync and worker orchestration

This commit is contained in:
root
2026-04-27 15:48:12 +08:00
parent 7cbde2aa78
commit 215a364891
137 changed files with 31931 additions and 1943 deletions

View File

@@ -242,6 +242,105 @@ def _normalize_worker_log_event(debug_event: dict) -> dict | None:
}
def _normalize_debug_event_job_identity(payload: dict | None) -> dict:
normalized_payload = dict(payload or {}) if isinstance(payload, dict) else {}
nested_job = normalized_payload.get("job") if isinstance(normalized_payload.get("job"), dict) else {}
raw_job_id = normalized_payload.get("job_id")
if raw_job_id in (None, "", 0, "0"):
raw_job_id = normalized_payload.get("target_job_id")
if raw_job_id in (None, "", 0, "0"):
raw_job_id = nested_job.get("job_id")
try:
job_id = int(raw_job_id or 0)
except Exception:
job_id = 0
job_code = str(
normalized_payload.get("job_code")
or normalized_payload.get("target_job_code")
or nested_job.get("job_code")
or ""
).strip()
cycle_token = str(normalized_payload.get("cycle_token") or nested_job.get("cycle_token") or "").strip()
return {
"job_id": job_id,
"job_code": job_code,
"cycle_token": cycle_token,
"has_identity": bool(job_id > 0 or job_code),
}
def _load_detect_job_summary_by_job_code(job_code: str, *, event_limit: int = 1) -> dict | None:
normalized_job_code = str(job_code or "").strip()
if not normalized_job_code:
return None
from app.services.detect_job_service import get_detect_job_summary
with get_db() as conn:
with conn.cursor() as cur:
cur.execute(
"""
SELECT id
FROM detect_jobs
WHERE job_code = %s
ORDER BY id DESC
LIMIT 1
""",
(normalized_job_code,),
)
row = cur.fetchone()
if not row:
return None
return get_detect_job_summary(int(row[0]), event_limit=event_limit)
def _resolve_target_job_for_debug_event(event_payload: dict | None) -> tuple[dict | None, str]:
from app.services.detect_job_service import get_active_detect_job_summary, get_detect_job_summary
identity = _normalize_debug_event_job_identity(event_payload)
if not identity["has_identity"]:
return None, "missing_job_identity"
payload_job_id = int(identity["job_id"] or 0)
payload_job_code = str(identity["job_code"] or "").strip()
payload_cycle_token = str(identity["cycle_token"] or "").strip()
active_job = get_active_detect_job_summary(event_limit=1) or {}
active_job_id = int(active_job.get("job_id") or 0)
active_job_code = str(active_job.get("job_code") or active_job.get("runtime_job_code") or "").strip()
target_job: dict | None = None
if payload_job_id > 0:
if active_job_id == payload_job_id:
target_job = active_job
else:
target_job = get_detect_job_summary(payload_job_id, event_limit=1)
elif payload_job_code:
if active_job_code and active_job_code == payload_job_code:
target_job = active_job
else:
target_job = _load_detect_job_summary_by_job_code(payload_job_code, event_limit=1)
if not target_job:
return None, "job_not_found"
target_job_id = int(target_job.get("job_id") or 0)
target_job_code = str(target_job.get("job_code") or target_job.get("runtime_job_code") or "").strip()
target_cycle_token = str(target_job.get("current_cycle_token") or "").strip()
if payload_job_id > 0 and target_job_id > 0 and target_job_id != payload_job_id:
return None, "job_mismatch"
if payload_job_code and target_job_code and target_job_code != payload_job_code:
return None, "job_mismatch"
if payload_cycle_token and target_cycle_token and payload_cycle_token != target_cycle_token:
return None, "cycle_mismatch"
return target_job, "matched"
def _ingest_worker_log_into_active_job(debug_event: dict) -> dict:
if str(debug_event.get("event_type") or "").strip() != "worker_log":
return {"imported": False, "reason": "not_worker_log"}
@@ -249,15 +348,16 @@ def _ingest_worker_log_into_active_job(debug_event: dict) -> dict:
normalized_event = _normalize_worker_log_event(debug_event)
if not normalized_event:
return {"imported": False, "reason": "not_domain_progress_event"}
from app.services.detect_job_service import get_active_detect_job_summary
from app.services.sync_push_service import (
_apply_detect_result_event_to_domain,
_apply_detect_result_event_to_job_item,
)
active_job = get_active_detect_job_summary(event_limit=1) or {}
target_job_id = int(active_job.get("job_id") or 0)
target_job, resolve_reason = _resolve_target_job_for_debug_event(normalized_event.get("payload"))
if not target_job:
return {"imported": False, "reason": resolve_reason}
target_job_id = int(target_job.get("job_id") or 0)
if target_job_id <= 0:
return {"imported": False, "reason": "no_active_job"}
@@ -334,6 +434,7 @@ def _ingest_worker_log_into_active_job(debug_event: dict) -> dict:
"imported": True,
"reason": "imported",
"target_job_id": target_job_id,
"target_job_code": str(target_job.get("job_code") or ""),
"detect_run_event_id": detect_run_event_id,
"updated_job_items": updated_job_items,
"event_type": normalized_event["event_type"],
@@ -847,7 +948,9 @@ def get_debug_handoff_report(
def ingest_debug_event(payload: dict, *, shared_token: str | None = None) -> tuple[bool, str, dict]:
configured_token = str(settings.sync_shared_token or "").strip()
incoming_token = str(shared_token or "").strip()
if configured_token and incoming_token != configured_token:
if not configured_token:
return False, "调试事件共享 token 未配置,拒绝远端写入", {"configuration_required": True}
if incoming_token != configured_token:
return False, "调试事件 token 校验失败", {}
record_id = append_debug_event(