feat: stabilize multi-region runtime sync and worker orchestration

domain-api/tests/test_detect_api_routes.py

@@ -7,6 +7,153 @@ from app.api.routes import detect as detect_route
 
 
 class DetectApiRoutesTestCase(unittest.TestCase):
+    @patch("app.api.routes.detect.create_detect_run_snapshot")
+    @patch("app.api.routes.detect._dispatch_remote_detect_start")
+    @patch("app.api.routes.detect.get_settings_payload")
+    @patch("app.api.routes.detect.get_detect_status")
+    @patch("app.api.routes.detect.send_worker_command")
+    @patch("app.api.routes.detect.start_worker")
+    @patch("app.api.routes.detect.append_detect_job_event")
+    @patch("app.api.routes.detect.create_detect_job_if_needed")
+    def test_start_detect_skips_redundant_systemctl_when_worker_already_online(
+        self,
+        mock_create_job,
+        mock_append_event,
+        mock_start_worker,
+        mock_send_worker_command,
+        mock_get_detect_status,
+        mock_get_settings_payload,
+        mock_dispatch_remote,
+        mock_create_snapshot,
+    ) -> None:
+        mock_create_job.return_value = {
+            "job_id": 30,
+            "job_code": "sync-overseas-30",
+            "status": "running",
+            "items_pending": 100,
+            "items_claimed": 0,
+            "items_running": 0,
+            "task_mode": "domain_pipeline",
+            "step_code": "",
+        }
+        mock_get_detect_status.return_value = {
+            "worker_mode": "linux-systemd",
+            "worker_online": True,
+            "worker_process_count": 60,
+            "worker_latest_start_time": "2026-04-24 23:30:00",
+            "worker_runtime_message": "running",
+            "progress": {},
+        }
+        mock_send_worker_command.return_value = (True, "已发送 Worker 控制指令: start_detection")
+        mock_get_settings_payload.return_value = {
+            "runtime": {"thread_count": 1000},
+            "proxy_config": {"proxy_enable": True, "allow_direct": True, "proxy_urls": ["a"]},
+        }
+        mock_dispatch_remote.return_value = {
+            "queued_jobs": [],
+            "queued_total": 0,
+            "failed_total": 0,
+            "target_summary": {"controller_nodes": [], "worker_nodes": []},
+        }
+
+        with patch.object(detect_route.settings, "node_region", "mainland"), patch.object(
+            detect_route.settings, "node_role", "control"
+        ):
+            response = detect_route.start_detect()
+
+        self.assertEqual(0, response.code)
+        self.assertIn("检测端已在运行，跳过重复启动", response.message)
+        mock_start_worker.assert_not_called()
+        mock_send_worker_command.assert_called_once()
+        event_types = [call.kwargs.get("event_type") for call in mock_append_event.call_args_list]
+        self.assertIn("job_dispatch_start_skipped", event_types)
+        self.assertIn("job_dispatch_sent", event_types)
+        mock_create_snapshot.assert_called_once()
+
+    @patch("app.api.routes.detect.create_detect_run_snapshot")
+    @patch("app.api.routes.detect._dispatch_remote_detect_start")
+    @patch("app.api.routes.detect.get_settings_payload")
+    @patch("app.api.routes.detect.get_detect_status")
+    @patch("app.api.routes.detect.send_worker_command")
+    @patch("app.api.routes.detect.start_worker")
+    @patch("app.api.routes.detect.append_detect_job_event")
+    @patch("app.api.routes.detect.create_detect_job_if_needed")
+    def test_start_detect_falls_back_to_direct_command_when_worker_already_running(
+        self,
+        mock_create_job,
+        mock_append_event,
+        mock_start_worker,
+        mock_send_worker_command,
+        mock_get_detect_status,
+        mock_get_settings_payload,
+        mock_dispatch_remote,
+        mock_create_snapshot,
+    ) -> None:
+        mock_create_job.return_value = {
+            "job_id": 31,
+            "job_code": "sync-overseas-31",
+            "status": "running",
+            "items_pending": 100,
+            "items_claimed": 0,
+            "items_running": 0,
+            "task_mode": "domain_pipeline",
+            "step_code": "",
+        }
+        mock_start_worker.return_value = (
+            False,
+            "domaincheck-worker 控制失败：当前运行用户没有免密 systemctl 权限，请为 API 进程授予对应 sudo/systemd 权限",
+        )
+        mock_send_worker_command.return_value = (True, "已发送 Worker 控制指令: start_detection")
+        mock_get_detect_status.side_effect = [
+            {
+                "worker_mode": "linux-systemd",
+                "worker_online": False,
+                "worker_process_count": 0,
+                "worker_latest_start_time": "",
+                "worker_runtime_message": "starting",
+                "progress": {},
+            },
+            {
+                "worker_mode": "linux-systemd",
+                "worker_online": True,
+                "worker_process_count": 60,
+                "worker_latest_start_time": "2026-04-24 23:30:00",
+                "worker_runtime_message": "running",
+                "progress": {},
+            },
+            {
+                "worker_mode": "linux-systemd",
+                "worker_online": True,
+                "worker_process_count": 60,
+                "worker_latest_start_time": "2026-04-24 23:30:00",
+                "worker_runtime_message": "running",
+                "progress": {},
+            },
+        ]
+        mock_get_settings_payload.return_value = {
+            "runtime": {"thread_count": 1000},
+            "proxy_config": {"proxy_enable": True, "allow_direct": True, "proxy_urls": ["a"]},
+        }
+        mock_dispatch_remote.return_value = {
+            "queued_jobs": [],
+            "queued_total": 0,
+            "failed_total": 0,
+            "target_summary": {"controller_nodes": [], "worker_nodes": []},
+        }
+
+        with patch.object(detect_route.settings, "node_region", "mainland"), patch.object(
+            detect_route.settings, "node_role", "control"
+        ):
+            response = detect_route.start_detect()
+
+        self.assertEqual(0, response.code)
+        self.assertIn("检测端已在运行，改为直接发送控制指令", response.message)
+        mock_send_worker_command.assert_called_once()
+        event_types = [call.kwargs.get("event_type") for call in mock_append_event.call_args_list]
+        self.assertIn("job_dispatch_start_degraded", event_types)
+        self.assertIn("job_dispatch_sent", event_types)
+        mock_create_snapshot.assert_called_once()
+
     @patch("app.api.routes.detect.create_detect_run_snapshot")
     @patch("app.api.routes.detect._dispatch_remote_detect_start")
     @patch("app.api.routes.detect.get_settings_payload")
@@ -65,11 +212,94 @@ class DetectApiRoutesTestCase(unittest.TestCase):
         mock_start_worker.assert_not_called()
         mock_send_worker_command.assert_not_called()
         mock_dispatch_remote.assert_called_once()
-        mock_create_snapshot.assert_called_once()
+        mock_create_snapshot.assert_not_called()
         event_types = [call.kwargs.get("event_type") for call in mock_append_event.call_args_list]
         self.assertIn("job_dispatch_requested", event_types)
         self.assertIn("job_dispatch_skipped_local", event_types)
 
+    @patch("app.api.routes.detect.finalize_detect_run")
+    @patch("app.api.routes.detect.mark_detect_run_stopping")
+    @patch("app.api.routes.detect.get_settings_payload")
+    @patch("app.api.routes.detect.get_detect_status")
+    @patch("app.api.routes.detect._dispatch_remote_detect_stop")
+    @patch("app.api.routes.detect.send_worker_command")
+    @patch("app.api.routes.detect.get_active_detect_job_summary")
+    def test_stop_detect_forwards_target_payload(
+        self,
+        mock_get_active_job,
+        mock_send_worker_command,
+        mock_dispatch_remote_stop,
+        mock_get_detect_status,
+        mock_get_settings_payload,
+        mock_mark_detect_run_stopping,
+        mock_finalize_detect_run,
+    ) -> None:
+        mock_get_active_job.return_value = {
+            "job_id": 29,
+            "job_code": "sync-overseas-29",
+            "status": "running",
+            "current_cycle_token": "cycle-29",
+        }
+        mock_send_worker_command.return_value = (True, "已发送 Worker 控制指令")
+        mock_dispatch_remote_stop.return_value = {
+            "queued_jobs": [{"node_code": "mainland-controller-01"}],
+            "queued_total": 1,
+            "failed_total": 0,
+            "target_summary": {"controller_nodes": ["mainland-controller-01"], "worker_nodes": []},
+        }
+        mock_get_detect_status.return_value = {
+            "worker_mode": "linux-systemd",
+            "worker_online": True,
+            "worker_process_count": 1,
+            "worker_latest_start_time": "",
+            "worker_runtime_message": "running",
+            "progress": {},
+        }
+        mock_get_settings_payload.return_value = {
+            "thread_count": 1000,
+            "node_thread_counts": {},
+            "process_count": 80,
+            "node_process_counts": {},
+            "proxy_config": {"proxy_enable": True, "allow_direct": False, "proxy_urls": ["a"]},
+        }
+
+        response = detect_route.stop_detect(payload={"target_node_codes": ["mainland-controller-01"]})
+
+        self.assertEqual(0, response.code)
+        mock_send_worker_command.assert_called_once_with(
+            "stop_detection",
+            payload={"target_node_codes": ["mainland-controller-01"]},
+        )
+        mock_dispatch_remote_stop.assert_called_once_with(
+            active_job=mock_get_active_job.return_value,
+            cycle_token="cycle-29",
+            payload={"target_node_codes": ["mainland-controller-01"]},
+        )
+        mock_mark_detect_run_stopping.assert_called_once()
+        mock_finalize_detect_run.assert_not_called()
+
+    @patch("app.api.routes.detect.create_ops_job")
+    @patch("app.api.routes.detect.list_managed_nodes")
+    def test_dispatch_remote_detect_stop_filters_target_nodes(self, mock_list_managed_nodes, mock_create_ops_job) -> None:
+        mock_list_managed_nodes.return_value = [
+            {"node_code": "mainland-controller-01", "region": "mainland", "role": "control", "last_seen_at": "2026-04-23T14:00:00", "is_enabled": True},
+            {"node_code": "mainland-worker-01", "region": "mainland", "role": "worker", "last_seen_at": "2026-04-23T14:00:00", "is_enabled": True},
+        ]
+        mock_create_ops_job.return_value = (True, "queued", {})
+
+        result = detect_route._dispatch_remote_detect_stop(
+            active_job={"job_id": 29, "job_code": "sync-overseas-29"},
+            cycle_token="cycle-29",
+            payload={"target_node_codes": ["mainland-worker-01"]},
+        )
+
+        self.assertEqual(1, result["queued_total"])
+        self.assertEqual(["mainland-controller-01"], result["target_summary"]["controller_nodes"])
+        self.assertEqual(["mainland-worker-01"], result["target_summary"]["worker_nodes"])
+        mock_create_ops_job.assert_called_once()
+        create_payload = mock_create_ops_job.call_args.args[0]
+        self.assertEqual("mainland-worker-01", create_payload["target_node_code"])
+
 
 if __name__ == "__main__":
     unittest.main()