feat: stabilize multi-region runtime sync and worker orchestration
This commit is contained in:
root
@@ -1,12 +1,61 @@
|
||||
import json
|
||||
import unittest
|
||||
from datetime import datetime, timedelta
|
||||
from unittest.mock import patch
|
||||
|
||||
from app.services.sync_record_service import (
|
||||
_build_runtime_projection_payload,
|
||||
_collect_recent_domain_events,
|
||||
_pick_latest_projection_row,
|
||||
append_runtime_projection_if_changed,
|
||||
get_detect_result_sync_batches,
|
||||
)
|
||||
|
||||
|
||||
class _FakeCursor:
|
||||
def __init__(self, fetchone_values):
|
||||
self.fetchone_values = list(fetchone_values or [])
|
||||
self.executed = []
|
||||
|
||||
def execute(self, sql, params=None):
|
||||
self.executed.append((sql, params))
|
||||
|
||||
def fetchone(self):
|
||||
if self.fetchone_values:
|
||||
return self.fetchone_values.pop(0)
|
||||
return None
|
||||
|
||||
def fetchall(self):
|
||||
if self.fetchone_values:
|
||||
value = self.fetchone_values.pop(0)
|
||||
return list(value or [])
|
||||
return []
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, exc_type, exc, tb):
|
||||
return False
|
||||
|
||||
|
||||
class _FakeConnection:
|
||||
def __init__(self, fetchone_values):
|
||||
self.cursor_obj = _FakeCursor(fetchone_values)
|
||||
self.committed = False
|
||||
|
||||
def cursor(self):
|
||||
return self.cursor_obj
|
||||
|
||||
def commit(self):
|
||||
self.committed = True
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, exc_type, exc, tb):
|
||||
return False
|
||||
|
||||
|
||||
class SyncRecordServiceTests(unittest.TestCase):
|
||||
def test_collect_recent_domain_events_filters_and_keeps_latest_slice(self) -> None:
|
||||
active_job = {
|
||||
@@ -130,6 +179,682 @@ class SyncRecordServiceTests(unittest.TestCase):
|
||||
self.assertEqual("", projection["active_job"]["job_code"])
|
||||
self.assertEqual([], projection["active_job"]["node_stats"])
|
||||
|
||||
@patch("app.services.sync_record_service._resolve_local_ip", return_value="121.204.244.188")
|
||||
@patch("app.services.sync_record_service.socket.gethostname", return_value="mainland-controller-01")
|
||||
@patch("app.services.sync_record_service.settings")
|
||||
def test_build_runtime_projection_payload_prefers_runtime_queue_nodes_for_multi_process_controller(
|
||||
self,
|
||||
mock_settings,
|
||||
_mock_hostname,
|
||||
_mock_resolve_ip,
|
||||
) -> None:
|
||||
mock_settings.node_code = "mainland-controller-01"
|
||||
mock_settings.node_region = "mainland"
|
||||
mock_settings.node_role = "control"
|
||||
mock_settings.sync_source_region = "mainland"
|
||||
mock_settings.sync_target_region = "overseas"
|
||||
|
||||
payload = _build_runtime_projection_payload(
|
||||
detect={
|
||||
"worker_online": True,
|
||||
"worker_mode": "linux-systemd",
|
||||
"active_thread_count": 1,
|
||||
"max_thread_count": 1000,
|
||||
"aggregate_max_thread_count": 80000,
|
||||
"phase_label": "运行中",
|
||||
"phase_detail": "80 实例运行",
|
||||
"proxy_runtime_label": "正常",
|
||||
"proxy_runtime_reason": "healthy",
|
||||
"progress": {
|
||||
"pending": 5001,
|
||||
"running": 0,
|
||||
"completed": 0,
|
||||
"blacklisted": 0,
|
||||
"failed": 0,
|
||||
},
|
||||
"backlog": {"pending_total": 1157325},
|
||||
"queue_health": {
|
||||
"queue": {
|
||||
"items_total": 5292,
|
||||
"pending": 4578,
|
||||
"claimed": 0,
|
||||
"display_claimed": 120,
|
||||
"running": 1972,
|
||||
"display_running": 1972,
|
||||
"completed": 690,
|
||||
"blacklisted": 0,
|
||||
"failed": 24,
|
||||
"terminal": 714,
|
||||
},
|
||||
"nodes": [
|
||||
{
|
||||
"node_code": "mainland-controller-01-a",
|
||||
"items_running": 1000,
|
||||
"items_claimed": 0,
|
||||
"display_running": 1000,
|
||||
"active_threads": 1000,
|
||||
"max_threads": 1000,
|
||||
"region": "mainland",
|
||||
"role": "control",
|
||||
"status": "busy",
|
||||
},
|
||||
{
|
||||
"node_code": "mainland-controller-01-b",
|
||||
"items_running": 972,
|
||||
"items_claimed": 120,
|
||||
"display_running": 972,
|
||||
"active_threads": 972,
|
||||
"max_threads": 1000,
|
||||
"region": "mainland",
|
||||
"role": "control",
|
||||
"status": "busy",
|
||||
},
|
||||
],
|
||||
},
|
||||
"active_job": {
|
||||
"job_id": 55,
|
||||
"job_code": "sync-overseas-55",
|
||||
"status": "running",
|
||||
"progress_percent": 13.49,
|
||||
"items_total": 5292,
|
||||
"items_terminal": 714,
|
||||
"items_pending": 4578,
|
||||
"items_running": 0,
|
||||
"items_failed": 24,
|
||||
"node_stats": [
|
||||
{
|
||||
"node_code": "mainland-controller-01",
|
||||
"items_running": 1,
|
||||
"items_claimed": 0,
|
||||
"items_total": 5292,
|
||||
}
|
||||
],
|
||||
},
|
||||
"dependency_alerts": [],
|
||||
},
|
||||
cluster={
|
||||
"nodes_total": 80,
|
||||
"nodes": [
|
||||
{
|
||||
"node_code": "mainland-controller-01",
|
||||
"current_load": 1,
|
||||
"detect_participating": True,
|
||||
}
|
||||
],
|
||||
"summary": {
|
||||
"online_worker_nodes": 65,
|
||||
"dedicated_online_worker_nodes": 1,
|
||||
"online_control_nodes": 65,
|
||||
"busy_nodes": ["mainland-controller-01-a", "mainland-controller-01-b"],
|
||||
"stale_nodes": [],
|
||||
"offline_nodes": [],
|
||||
},
|
||||
},
|
||||
source_region="mainland",
|
||||
target_region="overseas",
|
||||
)
|
||||
|
||||
projection = payload["projection"]
|
||||
self.assertTrue(projection["detect_participating"])
|
||||
self.assertEqual(1972, projection["active_thread_count"])
|
||||
self.assertEqual(80000, projection["max_thread_count"])
|
||||
self.assertEqual(1972, projection["progress"]["running"])
|
||||
self.assertEqual(2, len(projection["active_job"]["node_stats"]))
|
||||
self.assertEqual(2, len(projection["active_job"]["distributed_node_stats"]))
|
||||
self.assertEqual(1, len(projection["cluster_nodes"]))
|
||||
self.assertEqual("mainland-controller-01", projection["cluster_nodes"][0]["node_code"])
|
||||
self.assertEqual(1972, projection["active_job"]["items_running"])
|
||||
self.assertEqual(1972, projection["active_job"]["display_items_running"])
|
||||
self.assertEqual(80000, projection["active_job"]["display_max_threads"])
|
||||
|
||||
@patch("app.services.sync_record_service._resolve_local_ip", return_value="121.204.244.188")
|
||||
@patch("app.services.sync_record_service.socket.gethostname", return_value="mainland-controller-01")
|
||||
@patch("app.services.sync_record_service.settings")
|
||||
@patch("app.services.sync_record_service.get_db")
|
||||
def test_append_runtime_projection_if_changed_writes_heartbeat_for_unchanged_projection_after_interval(
|
||||
self,
|
||||
mock_get_db,
|
||||
mock_settings,
|
||||
_mock_hostname,
|
||||
_mock_resolve_ip,
|
||||
) -> None:
|
||||
mock_settings.node_code = "mainland-controller-01"
|
||||
mock_settings.node_region = "mainland"
|
||||
mock_settings.node_role = "control"
|
||||
mock_settings.sync_source_region = "mainland"
|
||||
mock_settings.sync_target_region = "overseas"
|
||||
|
||||
detect = {
|
||||
"worker_online": True,
|
||||
"worker_mode": "linux-systemd",
|
||||
"active_thread_count": 3200,
|
||||
"max_thread_count": 80000,
|
||||
"phase_label": "运行中",
|
||||
"phase_detail": "80 实例运行",
|
||||
"proxy_runtime_label": "正常",
|
||||
"proxy_runtime_reason": "healthy",
|
||||
"progress": {
|
||||
"pending": 1200000,
|
||||
"running": 6400,
|
||||
"completed": 50000,
|
||||
"blacklisted": 1200,
|
||||
"failed": 88,
|
||||
},
|
||||
"backlog": {"pending_total": 1200000},
|
||||
"active_job": {
|
||||
"job_id": 55,
|
||||
"job_code": "sync-overseas-55",
|
||||
"status": "running",
|
||||
"progress_percent": 12.5,
|
||||
"items_total": 1300000,
|
||||
"items_terminal": 51288,
|
||||
"items_pending": 1200000,
|
||||
"items_running": 6400,
|
||||
"items_failed": 88,
|
||||
"node_stats": [
|
||||
{
|
||||
"node_code": "mainland-controller-01",
|
||||
"items_running": 6400,
|
||||
"items_claimed": 7000,
|
||||
"items_total": 1300000,
|
||||
}
|
||||
],
|
||||
},
|
||||
"dependency_alerts": [],
|
||||
}
|
||||
cluster = {
|
||||
"nodes_total": 1,
|
||||
"nodes": [
|
||||
{
|
||||
"node_code": "mainland-controller-01",
|
||||
"current_load": 6400,
|
||||
"detect_participating": True,
|
||||
}
|
||||
],
|
||||
"summary": {
|
||||
"online_worker_nodes": 1,
|
||||
"dedicated_online_worker_nodes": 0,
|
||||
"online_control_nodes": 1,
|
||||
"busy_nodes": ["mainland-controller-01"],
|
||||
"stale_nodes": [],
|
||||
"offline_nodes": [],
|
||||
},
|
||||
}
|
||||
|
||||
previous_payload = _build_runtime_projection_payload(
|
||||
detect=detect,
|
||||
cluster=cluster,
|
||||
source_region="mainland",
|
||||
target_region="overseas",
|
||||
)
|
||||
fake_conn = _FakeConnection(
|
||||
[
|
||||
[(previous_payload, datetime.now() - timedelta(seconds=90))],
|
||||
(321,),
|
||||
]
|
||||
)
|
||||
mock_get_db.return_value = fake_conn
|
||||
|
||||
record_id = append_runtime_projection_if_changed(
|
||||
detect=detect,
|
||||
cluster=cluster,
|
||||
source_region="mainland",
|
||||
target_region="overseas",
|
||||
)
|
||||
|
||||
self.assertEqual(321, record_id)
|
||||
self.assertTrue(fake_conn.committed)
|
||||
self.assertTrue(
|
||||
any("INSERT INTO detect_sync_records" in sql for sql, _params in fake_conn.cursor_obj.executed)
|
||||
)
|
||||
|
||||
@patch("app.services.sync_record_service._resolve_local_ip", return_value="121.204.244.188")
|
||||
@patch("app.services.sync_record_service.socket.gethostname", return_value="mainland-controller-01")
|
||||
@patch("app.services.sync_record_service.settings")
|
||||
@patch("app.services.sync_record_service.get_db")
|
||||
def test_append_runtime_projection_if_changed_writes_when_cluster_nodes_change_within_window(
|
||||
self,
|
||||
mock_get_db,
|
||||
mock_settings,
|
||||
_mock_hostname,
|
||||
_mock_resolve_ip,
|
||||
) -> None:
|
||||
mock_settings.node_code = "mainland-controller-01"
|
||||
mock_settings.node_region = "mainland"
|
||||
mock_settings.node_role = "control"
|
||||
mock_settings.sync_source_region = "mainland"
|
||||
mock_settings.sync_target_region = "overseas"
|
||||
|
||||
previous_payload = {
|
||||
"projection": {
|
||||
"worker_online": True,
|
||||
"worker_mode": "linux-systemd",
|
||||
"phase_label": "运行中",
|
||||
"phase_detail": "等待中",
|
||||
"proxy_runtime_label": "正常",
|
||||
"proxy_runtime_reason": "healthy",
|
||||
"active_thread_count": 0,
|
||||
"max_thread_count": 60000,
|
||||
"progress": {"pending": 10000, "running": 0, "completed": 0, "blacklisted": 0, "failed": 0},
|
||||
"active_job": {
|
||||
"job_id": None,
|
||||
"job_code": "",
|
||||
"status": "",
|
||||
"items_total": 0,
|
||||
"items_running": 0,
|
||||
"items_claimed": 0,
|
||||
"display_items_running": 0,
|
||||
"display_items_claimed": 0,
|
||||
"display_max_threads": 0,
|
||||
"node_stats": [],
|
||||
"distributed_node_stats": [],
|
||||
},
|
||||
"cluster_summary": {
|
||||
"nodes_total": 43,
|
||||
"online_worker_nodes": 43,
|
||||
"dedicated_online_worker_nodes": 42,
|
||||
"online_control_nodes": 1,
|
||||
"busy_nodes": [],
|
||||
"stale_nodes": [],
|
||||
"offline_nodes": [],
|
||||
},
|
||||
"cluster_nodes": [
|
||||
{
|
||||
"node_code": "mainland-controller-01",
|
||||
"role": "control",
|
||||
"status": "online",
|
||||
"current_load": 0,
|
||||
"active_threads": 0,
|
||||
"max_threads": 60000,
|
||||
"detect_participating": False,
|
||||
}
|
||||
],
|
||||
"dependency_alerts": [],
|
||||
}
|
||||
}
|
||||
fake_conn = _FakeConnection(
|
||||
[
|
||||
[(previous_payload, datetime.now() - timedelta(seconds=10))],
|
||||
(911,),
|
||||
]
|
||||
)
|
||||
mock_get_db.return_value = fake_conn
|
||||
|
||||
record_id = append_runtime_projection_if_changed(
|
||||
detect={
|
||||
"worker_online": True,
|
||||
"worker_mode": "linux-systemd",
|
||||
"phase_label": "运行中",
|
||||
"phase_detail": "等待中",
|
||||
"proxy_runtime_label": "正常",
|
||||
"proxy_runtime_reason": "healthy",
|
||||
"progress": {"pending": 10000, "running": 0, "completed": 0, "blacklisted": 0, "failed": 0},
|
||||
"backlog": {},
|
||||
"active_job": {},
|
||||
"dependency_alerts": [],
|
||||
},
|
||||
cluster={
|
||||
"nodes_total": 61,
|
||||
"nodes": [
|
||||
{
|
||||
"node_code": "mainland-controller-01",
|
||||
"role": "control",
|
||||
"status": "online",
|
||||
"current_load": 0,
|
||||
"metadata": {"active_threads": 0, "max_threads": 60000},
|
||||
},
|
||||
{
|
||||
"node_code": "mainland-controller-01-a",
|
||||
"role": "worker",
|
||||
"status": "online",
|
||||
"current_load": 0,
|
||||
"metadata": {"active_threads": 0, "max_threads": 1000},
|
||||
},
|
||||
],
|
||||
"summary": {
|
||||
"online_worker_nodes": 61,
|
||||
"dedicated_online_worker_nodes": 60,
|
||||
"online_control_nodes": 1,
|
||||
"busy_nodes": [],
|
||||
"stale_nodes": [],
|
||||
"offline_nodes": [],
|
||||
},
|
||||
},
|
||||
source_region="mainland",
|
||||
target_region="overseas",
|
||||
)
|
||||
|
||||
self.assertEqual(911, record_id)
|
||||
self.assertTrue(fake_conn.committed)
|
||||
self.assertTrue(
|
||||
any("INSERT INTO detect_sync_records" in sql for sql, _params in fake_conn.cursor_obj.executed)
|
||||
)
|
||||
|
||||
@patch("app.services.sync_record_service._resolve_local_ip", return_value="121.204.244.188")
|
||||
@patch("app.services.sync_record_service.socket.gethostname", return_value="mainland-controller-01")
|
||||
@patch("app.services.sync_record_service.settings")
|
||||
@patch("app.services.sync_record_service.get_db")
|
||||
def test_append_runtime_projection_if_changed_skips_unchanged_projection_within_heartbeat_window(
|
||||
self,
|
||||
mock_get_db,
|
||||
mock_settings,
|
||||
_mock_hostname,
|
||||
_mock_resolve_ip,
|
||||
) -> None:
|
||||
mock_settings.node_code = "mainland-controller-01"
|
||||
mock_settings.node_region = "mainland"
|
||||
mock_settings.node_role = "control"
|
||||
mock_settings.sync_source_region = "mainland"
|
||||
mock_settings.sync_target_region = "overseas"
|
||||
|
||||
detect = {
|
||||
"worker_online": True,
|
||||
"worker_mode": "linux-systemd",
|
||||
"active_thread_count": 3200,
|
||||
"max_thread_count": 80000,
|
||||
"phase_label": "运行中",
|
||||
"phase_detail": "80 实例运行",
|
||||
"proxy_runtime_label": "正常",
|
||||
"proxy_runtime_reason": "healthy",
|
||||
"progress": {
|
||||
"pending": 1200000,
|
||||
"running": 6400,
|
||||
"completed": 50000,
|
||||
"blacklisted": 1200,
|
||||
"failed": 88,
|
||||
},
|
||||
"backlog": {"pending_total": 1200000},
|
||||
"active_job": {
|
||||
"job_id": 55,
|
||||
"job_code": "sync-overseas-55",
|
||||
"status": "running",
|
||||
"progress_percent": 12.5,
|
||||
"items_total": 1300000,
|
||||
"items_terminal": 51288,
|
||||
"items_pending": 1200000,
|
||||
"items_running": 6400,
|
||||
"items_failed": 88,
|
||||
"node_stats": [
|
||||
{
|
||||
"node_code": "mainland-controller-01",
|
||||
"items_running": 6400,
|
||||
"items_claimed": 7000,
|
||||
"items_total": 1300000,
|
||||
}
|
||||
],
|
||||
},
|
||||
"dependency_alerts": [],
|
||||
}
|
||||
cluster = {
|
||||
"nodes_total": 1,
|
||||
"nodes": [
|
||||
{
|
||||
"node_code": "mainland-controller-01",
|
||||
"current_load": 6400,
|
||||
"detect_participating": True,
|
||||
}
|
||||
],
|
||||
"summary": {
|
||||
"online_worker_nodes": 1,
|
||||
"dedicated_online_worker_nodes": 0,
|
||||
"online_control_nodes": 1,
|
||||
"busy_nodes": ["mainland-controller-01"],
|
||||
"stale_nodes": [],
|
||||
"offline_nodes": [],
|
||||
},
|
||||
}
|
||||
|
||||
previous_payload = _build_runtime_projection_payload(
|
||||
detect=detect,
|
||||
cluster=cluster,
|
||||
source_region="mainland",
|
||||
target_region="overseas",
|
||||
)
|
||||
fake_conn = _FakeConnection(
|
||||
[
|
||||
[(previous_payload, datetime.now() - timedelta(seconds=10))],
|
||||
]
|
||||
)
|
||||
mock_get_db.return_value = fake_conn
|
||||
|
||||
record_id = append_runtime_projection_if_changed(
|
||||
detect=detect,
|
||||
cluster=cluster,
|
||||
source_region="mainland",
|
||||
target_region="overseas",
|
||||
)
|
||||
|
||||
self.assertIsNone(record_id)
|
||||
self.assertFalse(fake_conn.committed)
|
||||
self.assertFalse(
|
||||
any("INSERT INTO detect_sync_records" in sql for sql, _params in fake_conn.cursor_obj.executed)
|
||||
)
|
||||
|
||||
@patch("app.services.sync_record_service._resolve_local_ip", return_value="121.204.244.188")
|
||||
@patch("app.services.sync_record_service.socket.gethostname", return_value="mainland-controller-01")
|
||||
@patch("app.services.sync_record_service.settings")
|
||||
@patch("app.services.sync_record_service.get_db")
|
||||
def test_append_runtime_projection_if_changed_writes_when_activity_signature_changes_within_window(
|
||||
self,
|
||||
mock_get_db,
|
||||
mock_settings,
|
||||
_mock_hostname,
|
||||
_mock_resolve_ip,
|
||||
) -> None:
|
||||
mock_settings.node_code = "mainland-controller-01"
|
||||
mock_settings.node_region = "mainland"
|
||||
mock_settings.node_role = "control"
|
||||
mock_settings.sync_source_region = "mainland"
|
||||
mock_settings.sync_target_region = "overseas"
|
||||
|
||||
previous_detect = {
|
||||
"worker_online": True,
|
||||
"worker_mode": "linux-systemd",
|
||||
"active_thread_count": 300,
|
||||
"max_thread_count": 60000,
|
||||
"phase_label": "运行中",
|
||||
"phase_detail": "60 实例运行",
|
||||
"proxy_runtime_label": "正常",
|
||||
"proxy_runtime_reason": "healthy",
|
||||
"progress": {
|
||||
"pending": 5000,
|
||||
"running": 600,
|
||||
"completed": 1000,
|
||||
"blacklisted": 0,
|
||||
"failed": 10,
|
||||
},
|
||||
"queue_health": {
|
||||
"queue": {
|
||||
"items_total": 6610,
|
||||
"pending": 5000,
|
||||
"claimed": 10,
|
||||
"display_claimed": 10,
|
||||
"running": 600,
|
||||
"display_running": 600,
|
||||
"completed": 1000,
|
||||
"blacklisted": 0,
|
||||
"failed": 10,
|
||||
"terminal": 1010,
|
||||
"display_max_threads": 60000,
|
||||
},
|
||||
"nodes": [
|
||||
{
|
||||
"node_code": "mainland-controller-01-a",
|
||||
"items_running": 300,
|
||||
"items_claimed": 10,
|
||||
"display_running": 300,
|
||||
"active_threads": 300,
|
||||
"max_threads": 1000,
|
||||
"region": "mainland",
|
||||
"role": "control",
|
||||
"status": "busy",
|
||||
}
|
||||
],
|
||||
},
|
||||
"active_job": {
|
||||
"job_id": 867,
|
||||
"job_code": "sync-overseas-19835",
|
||||
"status": "running",
|
||||
"progress_percent": 15.0,
|
||||
"items_total": 6610,
|
||||
"items_terminal": 1010,
|
||||
"items_pending": 5000,
|
||||
"items_claimed": 10,
|
||||
"items_running": 600,
|
||||
"items_failed": 10,
|
||||
"node_stats": [
|
||||
{
|
||||
"node_code": "mainland-controller-01-a",
|
||||
"items_running": 300,
|
||||
"items_claimed": 10,
|
||||
"items_total": 6610,
|
||||
}
|
||||
],
|
||||
},
|
||||
"dependency_alerts": [],
|
||||
}
|
||||
current_detect = {
|
||||
**previous_detect,
|
||||
"active_thread_count": 900,
|
||||
"queue_health": {
|
||||
"queue": {
|
||||
"items_total": 6610,
|
||||
"pending": 4300,
|
||||
"claimed": 30,
|
||||
"display_claimed": 30,
|
||||
"running": 900,
|
||||
"display_running": 900,
|
||||
"completed": 1370,
|
||||
"blacklisted": 0,
|
||||
"failed": 10,
|
||||
"terminal": 1380,
|
||||
"display_max_threads": 60000,
|
||||
},
|
||||
"nodes": [
|
||||
{
|
||||
"node_code": "mainland-controller-01-a",
|
||||
"items_running": 420,
|
||||
"items_claimed": 20,
|
||||
"display_running": 420,
|
||||
"active_threads": 420,
|
||||
"max_threads": 1000,
|
||||
"region": "mainland",
|
||||
"role": "control",
|
||||
"status": "busy",
|
||||
},
|
||||
{
|
||||
"node_code": "mainland-controller-01-b",
|
||||
"items_running": 480,
|
||||
"items_claimed": 10,
|
||||
"display_running": 480,
|
||||
"active_threads": 480,
|
||||
"max_threads": 1000,
|
||||
"region": "mainland",
|
||||
"role": "control",
|
||||
"status": "busy",
|
||||
},
|
||||
],
|
||||
},
|
||||
"active_job": {
|
||||
**previous_detect["active_job"],
|
||||
"items_pending": 4300,
|
||||
"items_claimed": 30,
|
||||
"items_running": 900,
|
||||
"node_stats": [
|
||||
{
|
||||
"node_code": "mainland-controller-01-a",
|
||||
"items_running": 420,
|
||||
"items_claimed": 20,
|
||||
"items_total": 6610,
|
||||
},
|
||||
{
|
||||
"node_code": "mainland-controller-01-b",
|
||||
"items_running": 480,
|
||||
"items_claimed": 10,
|
||||
"items_total": 6610,
|
||||
},
|
||||
],
|
||||
},
|
||||
}
|
||||
cluster = {
|
||||
"nodes_total": 2,
|
||||
"nodes": [
|
||||
{
|
||||
"node_code": "mainland-controller-01",
|
||||
"current_load": 900,
|
||||
"detect_participating": True,
|
||||
}
|
||||
],
|
||||
"summary": {
|
||||
"online_worker_nodes": 60,
|
||||
"dedicated_online_worker_nodes": 0,
|
||||
"online_control_nodes": 60,
|
||||
"busy_nodes": ["mainland-controller-01-a", "mainland-controller-01-b"],
|
||||
"stale_nodes": [],
|
||||
"offline_nodes": [],
|
||||
},
|
||||
}
|
||||
|
||||
previous_payload = _build_runtime_projection_payload(
|
||||
detect=previous_detect,
|
||||
cluster=cluster,
|
||||
source_region="mainland",
|
||||
target_region="overseas",
|
||||
)
|
||||
fake_conn = _FakeConnection(
|
||||
[
|
||||
[(previous_payload, datetime.now() - timedelta(seconds=10))],
|
||||
(654,),
|
||||
]
|
||||
)
|
||||
mock_get_db.return_value = fake_conn
|
||||
|
||||
record_id = append_runtime_projection_if_changed(
|
||||
detect=current_detect,
|
||||
cluster=cluster,
|
||||
source_region="mainland",
|
||||
target_region="overseas",
|
||||
)
|
||||
|
||||
self.assertEqual(654, record_id)
|
||||
self.assertTrue(fake_conn.committed)
|
||||
self.assertTrue(
|
||||
any("INSERT INTO detect_sync_records" in sql for sql, _params in fake_conn.cursor_obj.executed)
|
||||
)
|
||||
|
||||
def test_pick_latest_projection_row_skips_future_dated_rows(self) -> None:
|
||||
rows = [
|
||||
("future", datetime.now() + timedelta(hours=6)),
|
||||
("recent", datetime.now() - timedelta(seconds=10)),
|
||||
("older", datetime.now() - timedelta(minutes=2)),
|
||||
]
|
||||
|
||||
selected = _pick_latest_projection_row(rows, created_at_index=1)
|
||||
|
||||
self.assertEqual("recent", selected[0])
|
||||
|
||||
@patch("app.services.sync_record_service.get_db")
|
||||
@patch("app.services.sync_record_service.settings")
|
||||
def test_get_detect_result_sync_batches_marks_overseas_control_as_not_applicable(
|
||||
self,
|
||||
mock_settings,
|
||||
mock_get_db,
|
||||
) -> None:
|
||||
mock_settings.node_code = "overseas-control-01"
|
||||
mock_settings.node_region = "overseas"
|
||||
mock_settings.node_role = "control"
|
||||
mock_settings.sync_source_region = "overseas"
|
||||
mock_settings.sync_target_region = "mainland"
|
||||
|
||||
payload = get_detect_result_sync_batches(limit=5)
|
||||
|
||||
self.assertFalse(payload["applicable"])
|
||||
self.assertFalse(payload["local_worker_expected"])
|
||||
self.assertEqual(0, payload["jobs_total"])
|
||||
self.assertEqual([], payload["batches"])
|
||||
self.assertIn("不适用", payload["reason"])
|
||||
mock_get_db.assert_not_called()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
Reference in New Issue
Block a user