feat: stabilize multi-region runtime sync and worker orchestration

domain-api/tests/test_sync_agent.py

@@ -4,6 +4,9 @@ from unittest.mock import patch
 from app.sync_agent import (
     _append_detect_result_projection_snapshot,
     _build_aligned_queue_health_snapshot,
+    _emit_runtime_debug_snapshots,
+    _maybe_trigger_overlap_start,
+    _run_sync_tick_once,
     _emit_structured_tick,
     _emit_sync_result_breakdown,
     _filter_runtime_events_for_job,
@@ -187,6 +190,140 @@ class SyncAgentTests(unittest.TestCase):
         mock_process_detect_pipeline_now.assert_called_once_with(limit=5000)
         self.assertEqual("pipeline_tick_success", mock_push_debug_event.call_args.kwargs["event_type"])
 
+    @patch("app.sync_agent._load_local_detect_backlog_snapshot", return_value={"pending_total": 12})
+    @patch("app.sync_agent.list_recent_detect_run_events", return_value=[])
+    @patch("app.sync_agent.get_detect_queue_health", return_value={"queue": {"overdue_leases": 0}})
+    @patch("app.sync_agent.push_debug_event")
+    @patch("app.sync_agent._append_detect_result_projection_snapshot")
+    @patch("app.sync_agent._select_projection_job_snapshots", return_value=[{"job_id": 1, "job_code": "finished-job"}])
+    @patch(
+        "app.sync_agent.get_active_detect_job_summary",
+        return_value={
+            "job_id": 2,
+            "job_code": "running-job",
+            "status": "running",
+            "items_total": 100,
+            "items_pending": 10,
+            "items_claimed": 5,
+            "items_running": 7,
+            "items_completed": 70,
+            "items_failed": 8,
+            "progress_percent": 70.0,
+            "node_stats": [{"node_code": "mainland-controller-01", "items_total": 90}],
+        },
+    )
+    def test_emit_runtime_debug_snapshots_emits_projection_and_active_job(
+        self,
+        mock_get_active_detect_job_summary,
+        mock_select_projection_job_snapshots,
+        mock_append_detect_result_projection_snapshot,
+        mock_push_debug_event,
+        mock_get_detect_queue_health,
+        mock_list_recent_detect_run_events,
+        mock_load_local_detect_backlog_snapshot,
+    ) -> None:
+        _emit_runtime_debug_snapshots()
+
+        mock_get_active_detect_job_summary.assert_called_once_with(event_limit=10)
+        mock_select_projection_job_snapshots.assert_called_once_with()
+        mock_append_detect_result_projection_snapshot.assert_called_once()
+        self.assertTrue(mock_push_debug_event.called)
+        self.assertEqual("active_job_snapshot", mock_push_debug_event.call_args_list[0].kwargs["event_type"])
+
+    @patch("app.sync_agent._emit_runtime_debug_snapshots")
+    @patch("app.sync_agent._run_pipeline_stage_processor", return_value=(True, "pipeline ok", {"stage": "pipeline"}))
+    @patch("app.sync_agent.pull_detect_task_batch_now", return_value=(True, "pull ok", {"stage": "pull"}))
+    @patch("app.sync_agent.push_runtime_projection_now", return_value=(True, "sync ok", {"stage": "sync"}))
+    def test_run_sync_tick_once_prioritizes_sync_before_pipeline(
+        self,
+        mock_push_runtime_projection_now,
+        mock_pull_detect_task_batch_now,
+        mock_run_pipeline_stage_processor,
+        mock_emit_runtime_debug_snapshots,
+    ) -> None:
+        with patch("app.sync_agent._maybe_trigger_overlap_start", return_value=(False, "no overlap", {})) as mock_overlap:
+            tick = _run_sync_tick_once()
+
+        self.assertTrue(tick["sync"]["ok"])
+        self.assertEqual("sync ok", tick["sync"]["message"])
+        mock_push_runtime_projection_now.assert_called_once_with()
+        mock_pull_detect_task_batch_now.assert_called_once_with()
+        mock_run_pipeline_stage_processor.assert_called_once_with()
+        mock_overlap.assert_called_once_with()
+        mock_emit_runtime_debug_snapshots.assert_called_once_with()
+        self.assertEqual("no overlap", tick["overlap"]["message"])
+
+    @patch("app.sync_agent.send_worker_command")
+    @patch("app.sync_agent._select_overlap_target_node_codes")
+    @patch("app.sync_agent._select_overlap_start_candidate")
+    @patch("app.sync_agent.push_debug_event")
+    def test_maybe_trigger_overlap_start_dispatches_pending_job(
+        self,
+        mock_push_debug_event,
+        mock_select_overlap_start_candidate,
+        mock_select_overlap_target_node_codes,
+        mock_send_worker_command,
+    ) -> None:
+        mock_select_overlap_start_candidate.return_value = {
+            "job_id": 885,
+            "job_code": "sync-overseas-20592",
+            "task_mode": "domain_pipeline",
+            "items_pending": 64000,
+            "items_claimed": 0,
+            "items_running": 0,
+            "selection_reason": "overlap_tail_handoff",
+        }
+        mock_select_overlap_target_node_codes.return_value = [
+            "mainland-controller-01-a",
+            "mainland-controller-01-da",
+        ]
+        mock_send_worker_command.return_value = (True, "已发送 Worker 控制指令: start_detection -> mainland-controller-01-a,mainland-controller-01-da")
+
+        with patch("app.sync_agent._LAST_OVERLAP_JOB_ID", 0), patch("app.sync_agent._LAST_OVERLAP_TRIGGERED_AT", 0.0):
+            ok, message, data = _maybe_trigger_overlap_start()
+
+        self.assertTrue(ok)
+        self.assertIn("mainland-controller-01-a", message)
+        self.assertEqual(885, data["job_id"])
+        mock_send_worker_command.assert_called_once_with(
+            "start_detection",
+            payload={
+                "job_id": 885,
+                "job_code": "sync-overseas-20592",
+                "target_job_id": 885,
+                "target_job_code": "sync-overseas-20592",
+                "task_mode": "domain_pipeline",
+                "source": "overlap-handoff",
+                "selection_reason": "overlap_tail_handoff",
+                "tail_handoff_candidate": True,
+                "target_node_codes": [
+                    "mainland-controller-01-a",
+                    "mainland-controller-01-da",
+                ],
+            },
+        )
+        self.assertEqual("overlap_handoff_started", mock_push_debug_event.call_args.kwargs["event_type"])
+
+    @patch("app.sync_agent._select_overlap_start_candidate")
+    def test_maybe_trigger_overlap_start_honors_cooldown_for_same_job(
+        self,
+        mock_select_overlap_start_candidate,
+    ) -> None:
+        mock_select_overlap_start_candidate.return_value = {
+            "job_id": 885,
+            "job_code": "sync-overseas-20592",
+            "task_mode": "domain_pipeline",
+            "items_pending": 64000,
+            "selection_reason": "overlap_tail_handoff",
+        }
+
+        with patch("app.sync_agent._LAST_OVERLAP_JOB_ID", 885), patch("app.sync_agent._LAST_OVERLAP_TRIGGERED_AT", __import__('time').time()):
+            ok, message, data = _maybe_trigger_overlap_start()
+
+        self.assertFalse(ok)
+        self.assertIn("冷却中", message)
+        self.assertEqual(885, data["job_id"])
+
 
 if __name__ == "__main__":
     unittest.main()