feat: stabilize multi-region runtime sync and worker orchestration

domain-api/app/services/detect_run_service.py

@@ -1,5 +1,6 @@
 from __future__ import annotations
 
+import threading
 from datetime import datetime
 from uuid import uuid4
 
@@ -10,6 +11,7 @@ _MAX_LOG_LINES = 240
 _LOG_TAIL_LINES = 1200
 _ACTIVE_STATUSES = {"starting", "running", "stopping"}
 _TIMESTAMP_FORMATS = ("%Y-%m-%d %H:%M:%S.%f", "%Y-%m-%d %H:%M:%S")
+_DETECT_RUNS_LOCK = threading.RLock()
 
 
 def _now() -> str:
@@ -234,192 +236,196 @@ def _sync_record(
 
 
 def create_detect_run_snapshot(message: str, runtime: dict, progress: dict, settings_summary: dict) -> dict:
-    records = _load()
-    active = _find_active(records)
-    current_logs = _capture_worker_logs()
-    if active:
-        if active.get("status") == "stopping" and runtime.get("running"):
-            active["status"] = "running"
+    with _DETECT_RUNS_LOCK:
+        records = _load()
+        active = _find_active(records)
+        current_logs = _capture_worker_logs()
+        if active:
+            if active.get("status") == "stopping" and runtime.get("running"):
+                active["status"] = "running"
+            _sync_record(
+                active,
+                status=active.get("status", "starting"),
+                message=message,
+                runtime=runtime,
+                progress=progress,
+                settings_summary=settings_summary,
+                log_lines=current_logs,
+            )
+            _save(records)
+            return dict(active)
+
+        initial_started_at = runtime.get("latest_start_time") or _now()
+        record = {
+            "run_id": uuid4().hex,
+            "status": "starting",
+            "message": message,
+            "created_at": _now(),
+            "updated_at": _now(),
+            "started_at": initial_started_at,
+            "completed_at": "",
+            "runtime": runtime,
+            "progress": progress,
+            "settings_summary": settings_summary,
+            "phase_label": "",
+            "phase_detail": "",
+            "phase_history": [],
+            "logs": [],
+        }
         _sync_record(
-            active,
-            status=active.get("status", "starting"),
+            record,
+            status="starting",
             message=message,
             runtime=runtime,
             progress=progress,
             settings_summary=settings_summary,
             log_lines=current_logs,
         )
+        records.insert(0, record)
         _save(records)
-        return dict(active)
-
-    initial_started_at = runtime.get("latest_start_time") or _now()
-    record = {
-        "run_id": uuid4().hex,
-        "status": "starting",
-        "message": message,
-        "created_at": _now(),
-        "updated_at": _now(),
-        "started_at": initial_started_at,
-        "completed_at": "",
-        "runtime": runtime,
-        "progress": progress,
-        "settings_summary": settings_summary,
-        "phase_label": "",
-        "phase_detail": "",
-        "phase_history": [],
-        "logs": [],
-    }
-    _sync_record(
-        record,
-        status="starting",
-        message=message,
-        runtime=runtime,
-        progress=progress,
-        settings_summary=settings_summary,
-        log_lines=current_logs,
-    )
-    records.insert(0, record)
-    _save(records)
-    return dict(record)
+        return dict(record)
 
 
 def finalize_detect_run(message: str, runtime: dict, progress: dict, settings_summary: dict, active_job: dict | None = None) -> dict | None:
-    records = _load()
-    target = _find_active(records)
-    if not target:
-        return None
-    final_status = "stopped" if target.get("status") == "stopping" else "failed"
-    _sync_record(
-        target,
-        status=final_status,
-        message=message,
-        runtime=runtime,
-        progress=progress,
-        settings_summary=settings_summary,
-        log_lines=_capture_worker_logs(),
-        active_job=active_job,
-    )
-    _save(records)
-    return dict(target)
-
-
-def sync_detect_runs(runtime: dict, progress: dict, settings_summary: dict, active_job: dict | None = None) -> list[dict]:
-    records = _load()
-    active = _find_active(records)
-    current_logs = _capture_worker_logs()
-    active_job = active_job or {}
-    runtime_detecting = bool(runtime.get("detecting", False))
-    active_job_status = str(active_job.get("status", "") or "").strip()
-    active_job_open = active_job_status in {"pending", "running"}
-    execution_active = runtime_detecting or active_job_open or int((progress or {}).get("running", 0) or 0) > 0
-
-    if runtime.get("running") and execution_active:
-        if active and not _same_session(active, runtime):
-            _sync_record(
-                active,
-                status="stopped",
-                message="检测服务已重启，上一轮会话已归档",
-                runtime=active.get("runtime") or runtime,
-                progress=active.get("progress") or progress,
-                settings_summary=active.get("settings_summary") or settings_summary,
-                log_lines=current_logs,
-                active_job=active.get("active_job") or active_job,
-            )
-            active = None
-        if active:
-            next_status = "running" if active.get("status") != "stopping" else "stopping"
-            _sync_record(
-                active,
-                status=next_status,
-                message=runtime.get("message") or active.get("message") or "检测服务运行中",
-                runtime=runtime,
-                progress=progress,
-                settings_summary=settings_summary,
-                log_lines=current_logs,
-                active_job=active_job,
-            )
-        else:
-            started_at = runtime.get("latest_start_time") or _now()
-            record = {
-                "run_id": uuid4().hex,
-                "status": "running",
-                "message": runtime.get("message") or "检测服务运行中",
-                "created_at": _now(),
-                "updated_at": _now(),
-                "started_at": started_at,
-                "completed_at": "",
-                "runtime": runtime,
-                "progress": progress,
-                "settings_summary": settings_summary,
-                "phase_label": "",
-                "phase_detail": "",
-                "phase_history": [],
-                "logs": [],
-                "active_job": active_job,
-            }
-            _sync_record(
-                record,
-                status="running",
-                message=record["message"],
-                runtime=runtime,
-                progress=progress,
-                settings_summary=settings_summary,
-                log_lines=current_logs,
-                active_job=active_job,
-            )
-            records.insert(0, record)
-    elif active:
-        if runtime.get("running") and not execution_active:
-            if active.get("status") == "stopping":
-                final_status = "stopped"
-                final_message = runtime.get("message") or "检测任务已停止，Worker 保持待命"
-            elif active_job_status == "partial_failed":
-                final_status = "partial_failed"
-                final_message = "检测任务已结束，存在部分失败项"
-            elif active_job_status == "failed":
-                final_status = "failed"
-                final_message = "检测任务已结束，任务结果为失败"
-            else:
-                final_status = "completed"
-                final_message = "检测任务已自然完成，Worker 保持待命"
-        else:
-            final_status = "stopped" if active.get("status") == "stopping" else "failed"
-            final_message = runtime.get("message") or ("检测服务已停止" if final_status == "stopped" else "检测服务异常退出")
+    with _DETECT_RUNS_LOCK:
+        records = _load()
+        target = _find_active(records)
+        if not target:
+            return None
+        final_status = "stopped" if target.get("status") == "stopping" else "failed"
         _sync_record(
-            active,
+            target,
             status=final_status,
-            message=final_message,
+            message=message,
             runtime=runtime,
             progress=progress,
             settings_summary=settings_summary,
-            log_lines=current_logs,
+            log_lines=_capture_worker_logs(),
             active_job=active_job,
         )
+        _save(records)
+        return dict(target)
 
-    if records:
-        records[0]["logs"] = _merge_logs(
-            records[0].get("logs"),
-            _filter_logs_since(current_logs, records[0].get("started_at")),
-        )
 
-    _save(records)
-    return records
+def sync_detect_runs(runtime: dict, progress: dict, settings_summary: dict, active_job: dict | None = None) -> list[dict]:
+    with _DETECT_RUNS_LOCK:
+        records = _load()
+        active = _find_active(records)
+        current_logs = _capture_worker_logs()
+        active_job = active_job or {}
+        runtime_detecting = bool(runtime.get("detecting", False))
+        active_job_status = str(active_job.get("status", "") or "").strip()
+        active_job_open = active_job_status in {"pending", "running"}
+        execution_active = runtime_detecting or active_job_open or int((progress or {}).get("running", 0) or 0) > 0
+
+        if runtime.get("running") and execution_active:
+            if active and not _same_session(active, runtime):
+                _sync_record(
+                    active,
+                    status="stopped",
+                    message="检测服务已重启，上一轮会话已归档",
+                    runtime=active.get("runtime") or runtime,
+                    progress=active.get("progress") or progress,
+                    settings_summary=active.get("settings_summary") or settings_summary,
+                    log_lines=current_logs,
+                    active_job=active.get("active_job") or active_job,
+                )
+                active = None
+            if active:
+                next_status = "running" if active.get("status") != "stopping" else "stopping"
+                _sync_record(
+                    active,
+                    status=next_status,
+                    message=runtime.get("message") or active.get("message") or "检测服务运行中",
+                    runtime=runtime,
+                    progress=progress,
+                    settings_summary=settings_summary,
+                    log_lines=current_logs,
+                    active_job=active_job,
+                )
+            else:
+                started_at = runtime.get("latest_start_time") or _now()
+                record = {
+                    "run_id": uuid4().hex,
+                    "status": "running",
+                    "message": runtime.get("message") or "检测服务运行中",
+                    "created_at": _now(),
+                    "updated_at": _now(),
+                    "started_at": started_at,
+                    "completed_at": "",
+                    "runtime": runtime,
+                    "progress": progress,
+                    "settings_summary": settings_summary,
+                    "phase_label": "",
+                    "phase_detail": "",
+                    "phase_history": [],
+                    "logs": [],
+                    "active_job": active_job,
+                }
+                _sync_record(
+                    record,
+                    status="running",
+                    message=record["message"],
+                    runtime=runtime,
+                    progress=progress,
+                    settings_summary=settings_summary,
+                    log_lines=current_logs,
+                    active_job=active_job,
+                )
+                records.insert(0, record)
+        elif active:
+            if runtime.get("running") and not execution_active:
+                if active.get("status") == "stopping":
+                    final_status = "stopped"
+                    final_message = runtime.get("message") or "检测任务已停止，Worker 保持待命"
+                elif active_job_status == "partial_failed":
+                    final_status = "partial_failed"
+                    final_message = "检测任务已结束，存在部分失败项"
+                elif active_job_status == "failed":
+                    final_status = "failed"
+                    final_message = "检测任务已结束，任务结果为失败"
+                else:
+                    final_status = "completed"
+                    final_message = "检测任务已自然完成，Worker 保持待命"
+            else:
+                final_status = "stopped" if active.get("status") == "stopping" else "failed"
+                final_message = runtime.get("message") or ("检测服务已停止" if final_status == "stopped" else "检测服务异常退出")
+            _sync_record(
+                active,
+                status=final_status,
+                message=final_message,
+                runtime=runtime,
+                progress=progress,
+                settings_summary=settings_summary,
+                log_lines=current_logs,
+                active_job=active_job,
+            )
+
+        if records:
+            records[0]["logs"] = _merge_logs(
+                records[0].get("logs"),
+                _filter_logs_since(current_logs, records[0].get("started_at")),
+            )
+
+        _save(records)
+        return records
 
 
 def mark_detect_run_stopping(message: str, runtime: dict, progress: dict, settings_summary: dict) -> dict | None:
-    records = _load()
-    target = _find_active(records)
-    if not target:
-        return None
-    _sync_record(
-        target,
-        status="stopping",
-        message=message,
-        runtime=runtime,
-        progress=progress,
-        settings_summary=settings_summary,
-        log_lines=_capture_worker_logs(),
-        active_job=target.get("active_job") or {},
-    )
-    _save(records)
-    return dict(target)
+    with _DETECT_RUNS_LOCK:
+        records = _load()
+        target = _find_active(records)
+        if not target:
+            return None
+        _sync_record(
+            target,
+            status="stopping",
+            message=message,
+            runtime=runtime,
+            progress=progress,
+            settings_summary=settings_summary,
+            log_lines=_capture_worker_logs(),
+            active_job=target.get("active_job") or {},
+        )
+        _save(records)
+        return dict(target)