admin/getDomain
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: stabilize multi-region runtime sync and worker orchestration

Browse Source
This commit is contained in:
root
2026-04-27 15:48:12 +08:00
parent 7cbde2aa78
commit 215a364891
137 changed files with 31931 additions and 1943 deletions
Download Patch File Download Diff File Expand all files Collapse all files

607
domain-api/tests/test_sync_push_service.py
View File

@@ -1,10 +1,20 @@
import unittest
from datetime import datetime, timedelta
from unittest.mock import patch
from app.services.sync_push_service import (
_build_task_pull_backlog_limits,
_extract_detect_result_projection_events,
_load_latest_projection,
_load_pushable_projections,
_refresh_remote_runtime_node,
_load_local_detect_backlog_snapshot,
_push_projection_now,
_resolve_task_pull_request_limit,
_task_projection_limit,
pull_detect_task_batch_now,
_resolve_detect_result_target_job_id,
_select_relevant_backlog_job_ids_from_rows,
_should_throttle_task_pull,
ingest_runtime_projection,
)
@@ -23,6 +33,12 @@ class _FakeCursor:
return self._rows.pop(0)
return None
def fetchall(self):
if self._rows:
value = self._rows.pop(0)
return list(value or [])
return []
def __enter__(self):
return self
@@ -48,23 +64,311 @@ class _FakeConnection:
return False
class _FakeUrlopenResponse:
def __init__(self, payload):
self._payload = payload
def read(self):
import json
return json.dumps(self._payload, ensure_ascii=False).encode("utf-8")
def __enter__(self):
return self
def __exit__(self, exc_type, exc, tb):
return False
class SyncPushServiceTests(unittest.TestCase):
def test_build_task_pull_backlog_limits_scales_with_thread_configuration(self) -> None:
@patch("app.services.sync_push_service.cleanup_imported_runtime_nodes_many")
@patch("app.services.sync_push_service.cleanup_imported_runtime_nodes")
@patch("app.services.sync_push_service.register_node_heartbeat")
def test_refresh_remote_runtime_node_imports_cluster_worker_rows_without_active_job(
self,
mock_register_node_heartbeat,
_mock_cleanup_imported_runtime_nodes,
mock_cleanup_imported_runtime_nodes_many,
) -> None:
_refresh_remote_runtime_node(
source_region="mainland",
projection={
"node": {
"node_code": "mainland-controller-01",
"region": "mainland",
"role": "control",
"hostname": "localhost",
"ip": "127.0.0.1",
},
"worker_mode": "linux-systemd",
"phase_label": "等待中",
"phase_detail": "等待任务",
"proxy_runtime_label": "正常",
"proxy_runtime_reason": "healthy",
"active_thread_count": 0,
"max_thread_count": 60000,
"worker_online": True,
"detect_participating": False,
"active_job": {
"job_code": "",
"status": "",
"node_stats": [],
},
"cluster_nodes": [
{
"node_code": "mainland-controller-01",
"role": "control",
"status": "online",
"current_load": 0,
"active_threads": 0,
"max_threads": 60000,
"detect_participating": False,
},
{
"node_code": "mainland-controller-01-a",
"role": "worker",
"status": "online",
"current_load": 0,
"active_threads": 0,
"max_threads": 1000,
"detect_participating": False,
},
{
"node_code": "mainland-controller-01-b",
"role": "worker",
"status": "busy",
"current_load": 12,
"active_threads": 12,
"max_threads": 1000,
"detect_participating": True,
},
],
},
)
self.assertEqual(3, mock_register_node_heartbeat.call_count)
worker_calls = [call.kwargs for call in mock_register_node_heartbeat.call_args_list[1:]]
self.assertEqual(
["mainland-controller-01-a", "mainland-controller-01-b"],
[item["node_code"] for item in worker_calls],
)
self.assertEqual(12, worker_calls[1]["current_load"])
self.assertEqual(
["mainland-controller-01-a", "mainland-controller-01-b"],
mock_cleanup_imported_runtime_nodes_many.call_args.kwargs["keep_node_codes"],
)
@patch("app.services.sync_push_service.settings")
def test_task_pull_request_limit_scales_with_estimated_threads(self, mock_settings) -> None:
mock_settings.node_code = "mainland-controller-01"
mock_settings.sync_batch_size = 5000
safe_limit = _resolve_task_pull_request_limit(
None,
settings_payload={
"thread_count": 1000,
"process_count": 1,
"node_thread_counts": {"mainland-controller-01": 1000},
"node_process_counts": {"mainland-controller-01": 60},
},
)
self.assertEqual(120000, safe_limit)
@patch("app.services.sync_push_service.settings")
def test_task_projection_limit_allows_large_explicit_pull_request(self, mock_settings) -> None:
mock_settings.sync_batch_size = 5000
self.assertEqual(120000, _task_projection_limit(120000))
@patch("app.services.sync_push_service.settings")
@patch("app.services.sync_push_service.get_db")
def test_load_latest_projection_prefers_non_future_row_over_clock_skewed_history(
self,
mock_get_db,
mock_settings,
) -> None:
now = datetime.now()
future_row = (
10081,
"mainland",
"overseas",
"projected",
{"projection": {"active_thread_count": 138}},
now + timedelta(hours=4),
now + timedelta(hours=4),
)
valid_row = (
11633,
"mainland",
"overseas",
"projected",
{"projection": {"active_thread_count": 6431}},
now - timedelta(minutes=2),
now - timedelta(minutes=2),
)
fake_conn = _FakeConnection(rows=[[future_row, valid_row]])
mock_get_db.return_value = fake_conn
mock_settings.sync_source_region = "mainland"
mock_settings.node_region = "mainland"
mock_settings.sync_target_region = "overseas"
with patch("app.services.sync_push_service.datetime") as mock_datetime:
mock_datetime.now.return_value = now
record = _load_latest_projection("runtime_projection")
self.assertIsNotNone(record)
self.assertEqual(11633, record["id"])
executed_sql, executed_params = fake_conn.cursor_obj.executed[0]
self.assertIn("CASE WHEN created_at <= %s THEN 0 ELSE 1 END", executed_sql)
self.assertEqual("runtime_projection", executed_params[0])
self.assertEqual("mainland", executed_params[1])
self.assertEqual("overseas", executed_params[2])
self.assertEqual(now + timedelta(minutes=5), executed_params[3])
@patch("app.services.sync_push_service._latest_push_attempt")
@patch("app.services.sync_push_service.settings")
@patch("app.services.sync_push_service.get_db")
def test_load_pushable_projections_prefers_latest_unsent_records(
self,
mock_get_db,
mock_settings,
mock_latest_push_attempt,
) -> None:
now = datetime.now()
old_success_rows = [
(
10000 + index,
"mainland",
"overseas",
"projected",
{"projection_hash": f"old-{index}", "projection": {"job": {"job_id": index}}},
now - timedelta(minutes=120 - index),
now - timedelta(minutes=120 - index),
)
for index in range(100)
]
newest_unsent = (
23360,
"mainland",
"overseas",
"projected",
{"projection_hash": "new-hash", "projection": {"job": {"job_id": 909, "job_code": "sync-overseas-28618"}}},
now,
now,
)
fake_conn = _FakeConnection(rows=[old_success_rows + [newest_unsent]])
mock_get_db.return_value = fake_conn
mock_settings.sync_source_region = "mainland"
mock_settings.node_region = "mainland"
mock_settings.sync_target_region = "overseas"
mock_settings.sync_batch_size = 20
def _attempt_side_effect(source_record_id, target_region, sync_type):
if source_record_id == 23360:
return None
return {"status": "success", "created_at": now}
mock_latest_push_attempt.side_effect = _attempt_side_effect
records = _load_pushable_projections("detect_result_projection", limit=20)
self.assertEqual([23360], [item["id"] for item in records])
executed_sql, _ = fake_conn.cursor_obj.executed[0]
self.assertIn("ORDER BY created_at DESC, id DESC", executed_sql)
@patch("app.services.sync_push_service.settings.node_code", "mainland-controller-01")
def test_build_task_pull_backlog_limits_scales_with_local_server_capacity(self) -> None:
limits = _build_task_pull_backlog_limits(
5000,
settings_payload={
"thread_count": 100,
"process_count": 80,
"node_thread_counts": {
"mainland-controller-01": 2000,
"mainland-controller-01": 1000,
"mainland-controller-01-a": 1000,
"mainland-worker-01": 1200,
},
"node_process_counts": {
"mainland-controller-01": 80,
"mainland-worker-01": 60,
},
},
)
self.assertEqual(3200, limits["estimated_total_threads"])
self.assertEqual(6400, limits["max_pending_total"])
self.assertEqual(3200, limits["max_register_pending"])
self.assertEqual(800, limits["max_downstream_pending"])
self.assertEqual(80000, limits["estimated_total_threads"])
self.assertEqual(160000, limits["max_pending_total"])
self.assertEqual(80000, limits["max_register_pending"])
self.assertEqual(20000, limits["max_downstream_pending"])
def test_select_relevant_backlog_job_ids_from_rows_skips_stale_pending_jobs(self) -> None:
now = datetime.now()
job_ids = _select_relevant_backlog_job_ids_from_rows(
[
(11, "running", now - timedelta(hours=10)),
(10, "pending", now - timedelta(minutes=30)),
(9, "pending", now - timedelta(hours=7)),
],
freshness_hours=6,
limit=4,
)
self.assertEqual([11, 10], job_ids)
@patch("app.services.sync_push_service.get_active_detect_job_summary")
@patch("app.services.sync_push_service.get_db")
def test_load_local_detect_backlog_snapshot_prefers_active_job_snapshot(
self,
mock_get_db,
mock_get_active_detect_job_summary,
) -> None:
mock_get_active_detect_job_summary.return_value = {
"items_pending": 123,
"items_claimed": 4,
"items_running": 7,
"display_items_running": 9,
"step_stats": [
{"step_code": "detect_register", "items_pending": 90},
{"step_code": "detect_baidu", "items_pending": 33},
],
}
backlog = _load_local_detect_backlog_snapshot()
self.assertEqual(123, backlog["pending_total"])
self.assertEqual(4, backlog["claimed_total"])
self.assertEqual(9, backlog["running_total"])
self.assertEqual(90, backlog["register_pending"])
self.assertEqual(33, backlog["downstream_pending"])
mock_get_db.assert_not_called()
@patch("app.services.sync_push_service.get_active_detect_job_summary", return_value={})
@patch("app.services.sync_push_service.get_db")
def test_load_local_detect_backlog_snapshot_only_counts_relevant_jobs(
self,
mock_get_db,
_mock_get_active_detect_job_summary,
) -> None:
fake_conn = _FakeConnection(
rows=[
[
(11, "running", datetime.now() - timedelta(hours=2)),
(10, "pending", datetime.now() - timedelta(minutes=30)),
(9, "pending", datetime.now() - timedelta(hours=7)),
],
(123, 4, 7, 90, 33),
]
)
mock_get_db.return_value = fake_conn
backlog = _load_local_detect_backlog_snapshot()
self.assertEqual(123, backlog["pending_total"])
self.assertEqual(90, backlog["register_pending"])
executed_sql, executed_params = fake_conn.cursor_obj.executed[1]
self.assertIn("WHERE item.job_id = ANY(%s)", executed_sql)
self.assertEqual([11, 10], list(executed_params[0]))
def test_should_throttle_task_pull_when_register_backlog_overwhelms_downstream(self) -> None:
should_throttle, reason = _should_throttle_task_pull(
@@ -104,6 +408,224 @@ class SyncPushServiceTests(unittest.TestCase):
self.assertFalse(should_throttle)
self.assertEqual("", reason)
@patch("app.services.sync_push_service._should_throttle_task_pull", return_value=(False, ""))
@patch("app.services.sync_push_service._build_task_pull_backlog_limits", return_value={})
@patch("app.services.sync_push_service._load_local_detect_backlog_snapshot", return_value={})
@patch("app.services.sync_push_service._acquire_sync_pull_worker_wake_guard", return_value=True)
@patch("app.services.sync_push_service.ingest_detect_task_projection")
@patch("app.services.worker_control_service.send_worker_command")
@patch("app.services.sync_push_service.urllib.request.urlopen")
@patch("app.services.sync_push_service.settings")
@patch("app.services.sync_push_service.get_settings_payload")
def test_pull_detect_task_batch_now_starts_worker_with_projection_active_job_identity(
self,
mock_get_settings_payload,
mock_settings,
mock_urlopen,
mock_send_worker_command,
mock_ingest_detect_task_projection,
_mock_acquire_sync_pull_worker_wake_guard,
_mock_load_local_detect_backlog_snapshot,
_mock_build_task_pull_backlog_limits,
_mock_should_throttle_task_pull,
) -> None:
mock_settings.node_region = "mainland"
mock_settings.node_role = "control"
mock_settings.node_code = "mainland-controller-01"
mock_settings.sync_target_api_base_url = "http://example.com"
mock_settings.sync_target_region = "overseas"
mock_settings.sync_batch_size = 200
mock_settings.sync_shared_token = ""
mock_get_settings_payload.return_value = {
"thread_count": 1000,
"process_count": 1,
"node_thread_counts": {"mainland-controller-01": 1000},
"node_process_counts": {"mainland-controller-01": 60},
}
projection = {
"batch_code": "task-20260423210000-aa11bb",
"active_job": {
"job_id": 376,
"job_code": "sync-overseas-376",
"current_cycle_token": "cycle-376",
},
}
export_response = {
"code": 0,
"data": {
"source_record_id": 15164,
"projection_hash": "hash-15164",
"projection": projection,
},
}
ack_response = {"code": 0, "data": {"acknowledged": True}}
mock_urlopen.side_effect = [
_FakeUrlopenResponse(export_response),
_FakeUrlopenResponse(ack_response),
]
mock_ingest_detect_task_projection.return_value = (
True,
"任务批次接收成功",
{
"target_job_id": 1902,
"target_job_code": "sync-overseas-15164",
"queued_count": 1000,
},
)
mock_send_worker_command.return_value = (True, "started")
ok, message, payload = pull_detect_task_batch_now(limit=1000)
self.assertTrue(ok)
self.assertEqual("待检测任务批次拉取并入库成功", message)
self.assertTrue(payload["worker_start_ok"])
export_request = mock_urlopen.call_args_list[0][0][0]
self.assertIn("limit=1000", export_request.full_url)
mock_send_worker_command.assert_called_once()
args, kwargs = mock_send_worker_command.call_args
self.assertEqual("start_detection", args[0])
sent_payload = kwargs["payload"]
self.assertEqual(15164, sent_payload["source_record_id"])
self.assertEqual(1902, sent_payload["target_job_id"])
self.assertEqual("sync-overseas-15164", sent_payload["target_job_code"])
self.assertEqual(376, sent_payload["job_id"])
self.assertEqual("sync-overseas-376", sent_payload["job_code"])
self.assertEqual("cycle-376", sent_payload["cycle_token"])
@patch("app.services.sync_push_service._should_throttle_task_pull", return_value=(False, ""))
@patch("app.services.sync_push_service._build_task_pull_backlog_limits", return_value={})
@patch("app.services.sync_push_service._load_local_detect_backlog_snapshot", return_value={})
@patch("app.services.sync_push_service._acquire_sync_pull_worker_wake_guard", return_value=False)
@patch("app.services.sync_push_service.ingest_detect_task_projection")
@patch("app.services.worker_control_service.send_worker_command")
@patch("app.services.sync_push_service.urllib.request.urlopen")
@patch("app.services.sync_push_service.settings")
@patch("app.services.sync_push_service.get_settings_payload")
def test_pull_detect_task_batch_now_skips_duplicate_worker_wake_within_short_window(
self,
mock_get_settings_payload,
mock_settings,
mock_urlopen,
mock_send_worker_command,
mock_ingest_detect_task_projection,
_mock_acquire_sync_pull_worker_wake_guard,
_mock_load_local_detect_backlog_snapshot,
_mock_build_task_pull_backlog_limits,
_mock_should_throttle_task_pull,
) -> None:
mock_settings.node_region = "mainland"
mock_settings.node_role = "control"
mock_settings.node_code = "mainland-controller-01"
mock_settings.sync_target_api_base_url = "http://example.com"
mock_settings.sync_target_region = "overseas"
mock_settings.sync_batch_size = 200
mock_settings.sync_shared_token = ""
mock_get_settings_payload.return_value = {
"thread_count": 1000,
"process_count": 1,
"node_thread_counts": {"mainland-controller-01": 1000},
"node_process_counts": {"mainland-controller-01": 60},
}
projection = {
"batch_code": "task-20260423210000-aa11bb",
"active_job": {
"job_id": 376,
"job_code": "sync-overseas-376",
"current_cycle_token": "cycle-376",
},
}
export_response = {
"code": 0,
"data": {
"source_record_id": 15164,
"projection_hash": "hash-15164",
"projection": projection,
},
}
ack_response = {"code": 0, "data": {"acknowledged": True}}
mock_urlopen.side_effect = [
_FakeUrlopenResponse(export_response),
_FakeUrlopenResponse(ack_response),
]
mock_ingest_detect_task_projection.return_value = (
True,
"任务批次接收成功",
{
"target_job_id": 1902,
"target_job_code": "sync-overseas-15164",
"queued_count": 1000,
},
)
ok, message, payload = pull_detect_task_batch_now(limit=1000)
self.assertTrue(ok)
self.assertEqual("待检测任务批次拉取并入库成功", message)
self.assertTrue(payload["worker_start_ok"])
self.assertTrue(payload["worker_start_skipped"])
self.assertIn("重复 Worker 唤起", payload["worker_start_message"])
mock_send_worker_command.assert_not_called()
@patch("app.services.sync_push_service._should_throttle_task_pull", return_value=(False, ""))
@patch("app.services.sync_push_service._build_task_pull_backlog_limits", return_value={})
@patch("app.services.sync_push_service._load_local_detect_backlog_snapshot", return_value={})
@patch("app.services.sync_push_service._acquire_sync_pull_worker_wake_guard", return_value=False)
@patch("app.services.sync_push_service.ingest_detect_task_projection")
@patch("app.services.sync_push_service.urllib.request.urlopen")
@patch("app.services.sync_push_service.settings")
@patch("app.services.sync_push_service.get_settings_payload")
def test_pull_detect_task_batch_now_uses_adaptive_limit_when_unspecified(
self,
mock_get_settings_payload,
mock_settings,
mock_urlopen,
mock_ingest_detect_task_projection,
_mock_acquire_sync_pull_worker_wake_guard,
_mock_load_local_detect_backlog_snapshot,
_mock_build_task_pull_backlog_limits,
_mock_should_throttle_task_pull,
) -> None:
mock_settings.node_region = "mainland"
mock_settings.node_role = "control"
mock_settings.node_code = "mainland-controller-01"
mock_settings.sync_target_api_base_url = "http://example.com"
mock_settings.sync_target_region = "overseas"
mock_settings.sync_batch_size = 5000
mock_settings.sync_shared_token = ""
mock_get_settings_payload.return_value = {
"thread_count": 1000,
"process_count": 1,
"node_thread_counts": {"mainland-controller-01": 1000},
"node_process_counts": {"mainland-controller-01": 60},
}
export_response = {
"code": 0,
"data": {
"source_record_id": 15164,
"projection_hash": "hash-15164",
"projection": {"batch_code": "task-20260423210000-aa11bb", "active_job": {}},
},
}
ack_response = {"code": 0, "data": {"acknowledged": True}}
mock_urlopen.side_effect = [
_FakeUrlopenResponse(export_response),
_FakeUrlopenResponse(ack_response),
]
mock_ingest_detect_task_projection.return_value = (
True,
"任务批次接收成功",
{"target_job_id": 1902, "target_job_code": "sync-overseas-15164", "queued_count": 1000},
)
ok, _message, _payload = pull_detect_task_batch_now(limit=None)
self.assertTrue(ok)
export_request = mock_urlopen.call_args_list[0][0][0]
self.assertIn("limit=120000", export_request.full_url)
def test_extract_detect_result_projection_events_adds_import_metadata(self) -> None:
projection = {
"job": {
@@ -137,6 +659,79 @@ class SyncPushServiceTests(unittest.TestCase):
self.assertTrue(event["payload"]["imported_from_projection"])
self.assertTrue(event["payload"]["import_fingerprint"])
@patch("app.services.sync_push_service._push_projection_record")
@patch("app.services.sync_push_service._load_latest_projection")
@patch("app.services.runtime_status_service.refresh_runtime_projection_snapshot")
def test_push_projection_now_refreshes_runtime_projection_with_lightweight_snapshot(
self,
mock_refresh_runtime_projection_snapshot,
mock_load_latest_projection,
mock_push_projection_record,
) -> None:
mock_refresh_runtime_projection_snapshot.return_value = {
"record_id": 10082,
"active_thread_count": 1972,
"max_thread_count": 80000,
"queue_display_running": 1972,
}
mock_load_latest_projection.return_value = {
"id": 10082,
"source_region": "mainland",
"target_region": "overseas",
"created_at": None,
"payload": {"projection_hash": "hash-10082", "projection": {}},
}
mock_push_projection_record.return_value = (
True,
"投影推送成功",
{"action": "push_sync", "sync_type": "runtime_projection", "source_record_id": 10082},
)
ok, message, data = _push_projection_now("runtime_projection", "https://example.com/api/v1/runtime/sync-ingest")
self.assertTrue(ok)
self.assertEqual("投影推送成功", message)
self.assertEqual(10082, data["source_record_id"])
mock_refresh_runtime_projection_snapshot.assert_called_once_with(window_minutes=15)
mock_load_latest_projection.assert_called_once_with("runtime_projection")
mock_push_projection_record.assert_called_once()
@patch("app.services.sync_push_service._push_projection_record")
@patch("app.services.sync_push_service._load_latest_projection")
@patch("app.services.sync_push_service._append_fast_runtime_projection_snapshot")
@patch("app.services.runtime_status_service.refresh_runtime_projection_snapshot")
def test_push_projection_now_can_use_fast_runtime_projection_path(
self,
mock_refresh_runtime_projection_snapshot,
mock_append_fast_runtime_projection_snapshot,
mock_load_latest_projection,
mock_push_projection_record,
) -> None:
mock_append_fast_runtime_projection_snapshot.return_value = 10091
mock_load_latest_projection.return_value = {
"id": 10091,
"source_region": "mainland",
"target_region": "overseas",
"created_at": None,
"payload": {"projection_hash": "hash-10091", "projection": {}},
}
mock_push_projection_record.return_value = (
True,
"投影推送成功",
{"action": "push_sync", "sync_type": "runtime_projection", "source_record_id": 10091},
)
with patch.dict("os.environ", {"DOMAINCHECK_SYNC_RUNTIME_FAST_PROJECTION": "1"}, clear=False):
ok, message, data = _push_projection_now("runtime_projection", "https://example.com/api/v1/runtime/sync-ingest")
self.assertTrue(ok)
self.assertEqual("投影推送成功", message)
self.assertEqual(10091, data["source_record_id"])
mock_append_fast_runtime_projection_snapshot.assert_called_once()
mock_refresh_runtime_projection_snapshot.assert_not_called()
mock_load_latest_projection.assert_called_once_with("runtime_projection")
mock_push_projection_record.assert_called_once()
@patch("app.services.sync_push_service.get_db")
def test_resolve_detect_result_target_job_id_prefers_matching_job_code(self, mock_get_db) -> None:
fake_conn = _FakeConnection(rows=[(456,)])