feat: stabilize multi-region runtime sync and worker orchestration
This commit is contained in:
@@ -81,6 +81,9 @@ class BuildInfoServiceTests(unittest.TestCase):
|
||||
self.assertEqual("env-sha256", payload["checksum"])
|
||||
self.assertFalse(payload["route_surface"]["surface_complete"])
|
||||
self.assertIn("ops_stack_diagnosis", payload["route_surface"]["missing_keys"])
|
||||
self.assertIn("ops_migration_source_profile", payload["route_surface"]["missing_keys"])
|
||||
self.assertIn("ops_migration_preview", payload["route_surface"]["missing_keys"])
|
||||
self.assertIn("ops_migration_execute", payload["route_surface"]["missing_keys"])
|
||||
self.assertIn("ops_node_onboarding_bootstrap_preview", payload["route_surface"]["missing_keys"])
|
||||
self.assertIn("ops_node_onboarding_bootstrap_execute", payload["route_surface"]["missing_keys"])
|
||||
self.assertIn("ops_node_onboarding_acceptance_preview", payload["route_surface"]["missing_keys"])
|
||||
|
||||
@@ -1,12 +1,65 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import unittest
|
||||
from datetime import datetime, timedelta
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
from psycopg2 import errors
|
||||
|
||||
from app.services import cluster_runtime_service
|
||||
|
||||
|
||||
class RuntimeSchemaGuardTests(unittest.TestCase):
|
||||
def test_ensure_runtime_schema_does_not_toggle_autocommit_after_queries(self) -> None:
|
||||
class _Connection:
|
||||
def __init__(self) -> None:
|
||||
self._autocommit = False
|
||||
self.touched = False
|
||||
self.committed = False
|
||||
|
||||
@property
|
||||
def autocommit(self):
|
||||
return self._autocommit
|
||||
|
||||
@autocommit.setter
|
||||
def autocommit(self, value):
|
||||
if self.touched:
|
||||
raise AssertionError("autocommit should not be reassigned after queries start")
|
||||
self._autocommit = value
|
||||
|
||||
def cursor(self):
|
||||
conn = self
|
||||
|
||||
class _CursorContext:
|
||||
def __enter__(self_inner):
|
||||
conn.touched = True
|
||||
return cursor
|
||||
|
||||
def __exit__(self_inner, exc_type, exc, tb):
|
||||
return False
|
||||
|
||||
return _CursorContext()
|
||||
|
||||
def commit(self):
|
||||
self.committed = True
|
||||
|
||||
cursor = MagicMock()
|
||||
conn = _Connection()
|
||||
db_cm = MagicMock()
|
||||
db_cm.__enter__.return_value = conn
|
||||
|
||||
with patch.object(cluster_runtime_service, "_RUNTIME_SCHEMA_READY", False):
|
||||
with patch.object(cluster_runtime_service, "get_db", return_value=db_cm):
|
||||
with patch.object(cluster_runtime_service, "_runtime_schema_basics_present", return_value=False):
|
||||
cluster_runtime_service.ensure_runtime_schema()
|
||||
|
||||
cursor.execute.assert_any_call(
|
||||
"SELECT pg_advisory_xact_lock(%s)",
|
||||
(cluster_runtime_service._RUNTIME_SCHEMA_ADVISORY_LOCK_ID,),
|
||||
)
|
||||
cursor.execute.assert_any_call(cluster_runtime_service._RUNTIME_SCHEMA_SQL)
|
||||
self.assertTrue(conn.committed)
|
||||
|
||||
def test_ensure_runtime_schema_executes_only_once_per_process(self) -> None:
|
||||
conn = MagicMock()
|
||||
cursor_cm = MagicMock()
|
||||
@@ -18,11 +71,11 @@ class RuntimeSchemaGuardTests(unittest.TestCase):
|
||||
|
||||
with patch.object(cluster_runtime_service, "_RUNTIME_SCHEMA_READY", False):
|
||||
with patch.object(cluster_runtime_service, "get_db", return_value=db_cm) as mocked_get_db:
|
||||
cluster_runtime_service.ensure_runtime_schema()
|
||||
cluster_runtime_service.ensure_runtime_schema()
|
||||
with patch.object(cluster_runtime_service, "_runtime_schema_basics_present", return_value=False):
|
||||
cluster_runtime_service.ensure_runtime_schema()
|
||||
cluster_runtime_service.ensure_runtime_schema()
|
||||
|
||||
mocked_get_db.assert_called_once()
|
||||
self.assertEqual(cursor.execute.call_count, 2)
|
||||
cursor.execute.assert_any_call(
|
||||
"SELECT pg_advisory_xact_lock(%s)",
|
||||
(cluster_runtime_service._RUNTIME_SCHEMA_ADVISORY_LOCK_ID,),
|
||||
@@ -30,6 +83,118 @@ class RuntimeSchemaGuardTests(unittest.TestCase):
|
||||
cursor.execute.assert_any_call(cluster_runtime_service._RUNTIME_SCHEMA_SQL)
|
||||
conn.commit.assert_called_once()
|
||||
|
||||
def test_ensure_runtime_schema_skips_ddl_when_required_schema_already_exists(self) -> None:
|
||||
conn = MagicMock()
|
||||
cursor_cm = MagicMock()
|
||||
cursor = MagicMock()
|
||||
conn.cursor.return_value = cursor_cm
|
||||
cursor_cm.__enter__.return_value = cursor
|
||||
db_cm = MagicMock()
|
||||
db_cm.__enter__.return_value = conn
|
||||
|
||||
with patch.object(cluster_runtime_service, "_RUNTIME_SCHEMA_READY", False):
|
||||
with patch.object(cluster_runtime_service, "get_db", return_value=db_cm):
|
||||
with patch.object(cluster_runtime_service, "_runtime_schema_basics_present", return_value=True):
|
||||
with patch.object(cluster_runtime_service, "_runtime_missing_indexes", return_value=iter(())):
|
||||
cluster_runtime_service.ensure_runtime_schema()
|
||||
|
||||
self.assertFalse(
|
||||
any(sql == cluster_runtime_service._RUNTIME_SCHEMA_SQL for sql, _params in cursor.execute.call_args_list)
|
||||
)
|
||||
conn.commit.assert_not_called()
|
||||
|
||||
def test_ensure_runtime_schema_accepts_deadlock_when_required_schema_already_exists(self) -> None:
|
||||
class _Cursor:
|
||||
def __init__(self, *, raise_on_schema=False, fetchone_values=None, fetchall_values=None) -> None:
|
||||
self.raise_on_schema = raise_on_schema
|
||||
self.fetchone_values = list(fetchone_values or [])
|
||||
self.fetchall_values = list(fetchall_values or [])
|
||||
self.executed = []
|
||||
|
||||
def execute(self, sql, params=None):
|
||||
self.executed.append((sql, params))
|
||||
if self.raise_on_schema and sql == cluster_runtime_service._RUNTIME_SCHEMA_SQL:
|
||||
raise errors.DeadlockDetected()
|
||||
|
||||
def fetchone(self):
|
||||
if self.fetchone_values:
|
||||
return self.fetchone_values.pop(0)
|
||||
return None
|
||||
|
||||
def fetchall(self):
|
||||
if self.fetchall_values:
|
||||
return self.fetchall_values.pop(0)
|
||||
return []
|
||||
|
||||
class _CursorContext:
|
||||
def __init__(self, cursor) -> None:
|
||||
self.cursor = cursor
|
||||
|
||||
def __enter__(self):
|
||||
return self.cursor
|
||||
|
||||
def __exit__(self, exc_type, exc, tb):
|
||||
return False
|
||||
|
||||
conn = MagicMock()
|
||||
first_cursor = _Cursor()
|
||||
second_cursor = _Cursor(raise_on_schema=True)
|
||||
third_cursor = _Cursor()
|
||||
conn.cursor.side_effect = [
|
||||
_CursorContext(first_cursor),
|
||||
_CursorContext(second_cursor),
|
||||
_CursorContext(third_cursor),
|
||||
]
|
||||
db_cm = MagicMock()
|
||||
db_cm.__enter__.return_value = conn
|
||||
|
||||
with patch.object(cluster_runtime_service, "_RUNTIME_SCHEMA_READY", False):
|
||||
with patch.object(cluster_runtime_service, "get_db", return_value=db_cm):
|
||||
with patch.object(cluster_runtime_service, "_runtime_schema_basics_present", side_effect=[False, True]):
|
||||
with patch.object(cluster_runtime_service, "_runtime_missing_indexes", return_value=iter(())):
|
||||
cluster_runtime_service.ensure_runtime_schema()
|
||||
|
||||
conn.rollback.assert_called_once()
|
||||
conn.commit.assert_not_called()
|
||||
|
||||
def test_ensure_runtime_schema_repairs_missing_indexes_without_full_ddl(self) -> None:
|
||||
conn = MagicMock()
|
||||
cursor_cm = MagicMock()
|
||||
cursor = MagicMock()
|
||||
conn.cursor.return_value = cursor_cm
|
||||
cursor_cm.__enter__.return_value = cursor
|
||||
db_cm = MagicMock()
|
||||
db_cm.__enter__.return_value = conn
|
||||
|
||||
with patch.object(cluster_runtime_service, "_RUNTIME_SCHEMA_READY", False):
|
||||
with patch.object(cluster_runtime_service, "get_db", return_value=db_cm):
|
||||
with patch.object(cluster_runtime_service, "_runtime_schema_basics_present", return_value=True):
|
||||
with patch.object(
|
||||
cluster_runtime_service,
|
||||
"_runtime_missing_indexes",
|
||||
side_effect=[iter(("idx_detect_job_items_claim_step_ready",)), iter(())],
|
||||
):
|
||||
with patch.object(cluster_runtime_service, "_ensure_runtime_schema_indexes") as mocked_ensure_indexes:
|
||||
cluster_runtime_service.ensure_runtime_schema()
|
||||
|
||||
mocked_ensure_indexes.assert_called_once()
|
||||
self.assertFalse(
|
||||
any(sql == cluster_runtime_service._RUNTIME_SCHEMA_SQL for sql, _params in cursor.execute.call_args_list)
|
||||
)
|
||||
|
||||
def test_runtime_missing_indexes_treats_invalid_indexes_as_missing(self) -> None:
|
||||
cur = MagicMock()
|
||||
cur.fetchall.return_value = [
|
||||
("idx_detect_job_items_job_domain_step", True, True, True),
|
||||
("idx_detect_job_items_claim_step_ready", False, True, True),
|
||||
]
|
||||
|
||||
missing = list(cluster_runtime_service._runtime_missing_indexes(cur))
|
||||
|
||||
self.assertIn("idx_detect_job_items_claim_step_ready", missing)
|
||||
self.assertIn("idx_detect_job_items_claim_job_step_ready", missing)
|
||||
self.assertNotIn("idx_detect_job_items_job_domain_step", missing)
|
||||
|
||||
def test_control_node_supports_worker_only_on_mainland_with_worker_signals(self) -> None:
|
||||
self.assertFalse(
|
||||
cluster_runtime_service._control_node_supports_worker(
|
||||
@@ -88,6 +253,187 @@ class RuntimeSchemaGuardTests(unittest.TestCase):
|
||||
)
|
||||
)
|
||||
|
||||
def test_get_cluster_snapshot_counts_all_rows_even_when_display_nodes_are_limited(self) -> None:
|
||||
now = datetime.now()
|
||||
display_rows = [
|
||||
(
|
||||
f"mainland-worker-{index:03d}",
|
||||
"mainland",
|
||||
"worker",
|
||||
f"worker-{index:03d}",
|
||||
f"10.0.0.{index}",
|
||||
"online",
|
||||
"test",
|
||||
0,
|
||||
{},
|
||||
now,
|
||||
now,
|
||||
)
|
||||
for index in range(1, 101)
|
||||
]
|
||||
summary_rows = display_rows + [
|
||||
(
|
||||
"mainland-worker-101",
|
||||
"mainland",
|
||||
"worker",
|
||||
"worker-101",
|
||||
"10.0.0.101",
|
||||
"online",
|
||||
"test",
|
||||
0,
|
||||
{},
|
||||
now,
|
||||
now,
|
||||
)
|
||||
]
|
||||
conn = MagicMock()
|
||||
cursor_cm = MagicMock()
|
||||
cursor = MagicMock()
|
||||
cursor.fetchall.side_effect = [display_rows, summary_rows]
|
||||
cursor.fetchone.side_effect = [(12,), (34,)]
|
||||
conn.cursor.return_value = cursor_cm
|
||||
cursor_cm.__enter__.return_value = cursor
|
||||
db_cm = MagicMock()
|
||||
db_cm.__enter__.return_value = conn
|
||||
|
||||
with patch.object(cluster_runtime_service, "get_db", return_value=db_cm):
|
||||
with patch.object(cluster_runtime_service, "prune_expired_runtime_nodes"):
|
||||
with patch.object(cluster_runtime_service, "register_local_control_heartbeat"):
|
||||
with patch.object(cluster_runtime_service, "_load_managed_node_overlays", return_value={}):
|
||||
with patch.object(cluster_runtime_service, "_load_disabled_managed_node_codes", return_value=set()):
|
||||
snapshot = cluster_runtime_service.get_cluster_snapshot()
|
||||
|
||||
self.assertEqual(100, len(snapshot["nodes"]))
|
||||
self.assertEqual(101, snapshot["nodes_total"])
|
||||
self.assertEqual(101, snapshot["summary"]["online_worker_nodes"])
|
||||
self.assertEqual(12, snapshot["jobs_total"])
|
||||
self.assertEqual(34, snapshot["active_job_items"])
|
||||
|
||||
def test_get_cluster_snapshot_excludes_disabled_managed_nodes(self) -> None:
|
||||
now = datetime.now()
|
||||
display_rows = [
|
||||
("mainland-controller-01", "mainland", "control", "controller", "10.0.0.1", "online", "test", 1, {}, now, now),
|
||||
("mainland-worker-01", "mainland", "worker", "worker-01", "10.0.0.2", "busy", "test", 1, {}, now, now),
|
||||
]
|
||||
summary_rows = list(display_rows)
|
||||
conn = MagicMock()
|
||||
cursor_cm = MagicMock()
|
||||
cursor = MagicMock()
|
||||
cursor.fetchall.side_effect = [display_rows, summary_rows]
|
||||
cursor.fetchone.side_effect = [(1,), (2,)]
|
||||
conn.cursor.return_value = cursor_cm
|
||||
cursor_cm.__enter__.return_value = cursor
|
||||
db_cm = MagicMock()
|
||||
db_cm.__enter__.return_value = conn
|
||||
|
||||
with patch.object(cluster_runtime_service, "get_db", return_value=db_cm):
|
||||
with patch.object(cluster_runtime_service, "prune_expired_runtime_nodes"):
|
||||
with patch.object(cluster_runtime_service, "register_local_control_heartbeat"):
|
||||
with patch.object(cluster_runtime_service, "_load_managed_node_overlays", return_value={}):
|
||||
with patch.object(cluster_runtime_service, "_load_disabled_managed_node_codes", return_value={"mainland-worker-01"}):
|
||||
snapshot = cluster_runtime_service.get_cluster_snapshot()
|
||||
|
||||
self.assertEqual(["mainland-controller-01"], [item["node_code"] for item in snapshot["nodes"]])
|
||||
self.assertEqual(1, snapshot["nodes_total"])
|
||||
self.assertEqual(1, snapshot["summary"]["online_control_nodes"])
|
||||
self.assertEqual(0, snapshot["summary"]["online_worker_nodes"])
|
||||
|
||||
def test_get_cluster_snapshot_prefers_imported_runtime_update_time_for_freshness(self) -> None:
|
||||
now = datetime.now()
|
||||
stale_heartbeat = now - timedelta(minutes=8)
|
||||
fresh_ingest = now - timedelta(seconds=20)
|
||||
display_rows = [
|
||||
(
|
||||
"mainland-controller-01-da",
|
||||
"mainland",
|
||||
"worker",
|
||||
"controller",
|
||||
"10.0.0.1",
|
||||
"busy",
|
||||
"test",
|
||||
42,
|
||||
{
|
||||
"service": "runtime-ingest",
|
||||
"updated_at": fresh_ingest.isoformat(timespec="seconds"),
|
||||
"active_threads": 42,
|
||||
},
|
||||
stale_heartbeat,
|
||||
fresh_ingest,
|
||||
)
|
||||
]
|
||||
summary_rows = list(display_rows)
|
||||
conn = MagicMock()
|
||||
cursor_cm = MagicMock()
|
||||
cursor = MagicMock()
|
||||
cursor.fetchall.side_effect = [display_rows, summary_rows]
|
||||
cursor.fetchone.side_effect = [(1,), (1,)]
|
||||
conn.cursor.return_value = cursor_cm
|
||||
cursor_cm.__enter__.return_value = cursor
|
||||
db_cm = MagicMock()
|
||||
db_cm.__enter__.return_value = conn
|
||||
|
||||
with patch.object(cluster_runtime_service, "get_db", return_value=db_cm):
|
||||
with patch.object(cluster_runtime_service, "prune_expired_runtime_nodes"):
|
||||
with patch.object(cluster_runtime_service, "register_local_control_heartbeat"):
|
||||
with patch.object(cluster_runtime_service, "_load_managed_node_overlays", return_value={}):
|
||||
with patch.object(cluster_runtime_service, "_load_disabled_managed_node_codes", return_value=set()):
|
||||
snapshot = cluster_runtime_service.get_cluster_snapshot()
|
||||
|
||||
self.assertEqual(1, snapshot["summary"]["online_worker_nodes"])
|
||||
self.assertEqual([], snapshot["summary"]["offline_nodes"])
|
||||
self.assertEqual("busy", snapshot["nodes"][0]["status"])
|
||||
|
||||
def test_register_local_control_heartbeat_uses_detect_status_snapshot(self) -> None:
|
||||
detect_status = {
|
||||
"worker_online": True,
|
||||
"active_thread_count": 17,
|
||||
"max_thread_count": 320,
|
||||
"available_proxy_count": 41,
|
||||
"proxy_runtime_label": "代理可用",
|
||||
"proxy_runtime_reason": "pool_ready",
|
||||
"proxy_last_refresh_status": "ok",
|
||||
"proxy_last_refresh_time": "2026-04-24 01:30:00",
|
||||
"proxy_last_refresh_source_count": 3,
|
||||
"proxy_last_refresh_total_items": 120,
|
||||
"proxy_last_validated_count": 110,
|
||||
"proxy_last_available_count": 41,
|
||||
"phase_label": "running",
|
||||
"phase_detail": "正在执行检测",
|
||||
"active_job": {
|
||||
"job_code": "detect-20260424013000-abcd12",
|
||||
"status": "running",
|
||||
"node_stats": [
|
||||
{
|
||||
"node_code": "mainland-controller-01",
|
||||
"items_total": 56,
|
||||
"items_claimed": 5,
|
||||
"items_running": 9,
|
||||
"items_completed": 42,
|
||||
}
|
||||
],
|
||||
},
|
||||
}
|
||||
|
||||
with patch.object(cluster_runtime_service.settings, "node_code", "mainland-controller-01"):
|
||||
with patch.object(cluster_runtime_service.settings, "node_region", "mainland"):
|
||||
with patch.object(cluster_runtime_service.settings, "node_role", "control"):
|
||||
with patch("app.services.detect_service.get_detect_status", return_value=detect_status):
|
||||
with patch.object(cluster_runtime_service, "register_node_heartbeat") as mock_register:
|
||||
cluster_runtime_service.register_local_control_heartbeat()
|
||||
|
||||
mock_register.assert_called_once()
|
||||
payload = mock_register.call_args.kwargs
|
||||
self.assertEqual("mainland-controller-01", payload["node_code"])
|
||||
self.assertEqual("busy", payload["status"])
|
||||
self.assertEqual(17, payload["current_load"])
|
||||
self.assertTrue(payload["metadata"]["worker_online"])
|
||||
self.assertTrue(payload["metadata"]["detect_participating"])
|
||||
self.assertEqual("detect-20260424013000-abcd12", payload["metadata"]["active_job_code"])
|
||||
self.assertEqual(17, payload["metadata"]["active_threads"])
|
||||
self.assertEqual(320, payload["metadata"]["max_threads"])
|
||||
self.assertEqual(41, payload["metadata"]["available_proxy_count"])
|
||||
self.assertEqual("running", payload["metadata"]["phase_label"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
@@ -39,20 +39,22 @@ class _FakeConnection:
|
||||
|
||||
|
||||
class DashboardServiceTests(unittest.TestCase):
|
||||
@patch("app.services.dashboard.get_detect_status")
|
||||
@patch("app.services.dashboard._fetch_active_jobs_aggregate")
|
||||
@patch("app.services.dashboard.get_detect_capacity_plan")
|
||||
@patch("app.services.dashboard.get_detect_queue_health")
|
||||
@patch("app.services.dashboard.get_runtime_status")
|
||||
@patch("app.services.dashboard._build_dashboard_runtime_summary")
|
||||
@patch("app.services.dashboard.get_active_detect_job_summary")
|
||||
@patch("app.services.dashboard.get_db")
|
||||
def test_fetch_overview_includes_ops_metrics(
|
||||
self,
|
||||
mock_get_db,
|
||||
mock_get_active_detect_job_summary,
|
||||
mock_get_runtime_status,
|
||||
mock_build_dashboard_runtime_summary,
|
||||
mock_get_detect_queue_health,
|
||||
mock_get_detect_capacity_plan,
|
||||
mock_fetch_active_jobs_aggregate,
|
||||
mock_get_detect_status,
|
||||
) -> None:
|
||||
mock_get_db.return_value = _FakeConnection(
|
||||
responses=[
|
||||
@@ -122,7 +124,7 @@ class DashboardServiceTests(unittest.TestCase):
|
||||
"items_blacklisted": 0,
|
||||
"items_failed": 0,
|
||||
}
|
||||
mock_get_runtime_status.return_value = {
|
||||
mock_build_dashboard_runtime_summary.return_value = {
|
||||
"worker": {"running": True, "mode": "linux-systemd", "expected_on_this_node": True},
|
||||
"node": {"region": "overseas", "role": "control"},
|
||||
"cluster": {"summary": {"online_worker_nodes": 2, "dedicated_online_worker_nodes": 1, "online_control_nodes": 1}},
|
||||
@@ -198,6 +200,15 @@ class DashboardServiceTests(unittest.TestCase):
|
||||
"remaining_items": 193,
|
||||
"recommended_additional_workers": 1,
|
||||
}
|
||||
mock_get_detect_status.return_value = {
|
||||
"available_proxy_count": 1011,
|
||||
"proxy_runtime_label": "集群代理正常",
|
||||
"proxy_runtime_detail": "参与服务器 2 台,共可用 1011 个代理",
|
||||
"proxy_last_refresh_status": "mainland-controller-01:201;mainland-worker-01:810",
|
||||
"aggregate_process_count": 10,
|
||||
"aggregate_participating_node_count": 10,
|
||||
"active_thread_count": 165,
|
||||
}
|
||||
|
||||
data = fetch_overview()
|
||||
|
||||
@@ -218,6 +229,10 @@ class DashboardServiceTests(unittest.TestCase):
|
||||
self.assertEqual(165, data["queue_display_running_total"])
|
||||
self.assertEqual(7, data["queue_completed_total"])
|
||||
self.assertEqual(9438, data["backlog_pending_total"])
|
||||
self.assertEqual(1011, data["cluster_proxy_available_count"])
|
||||
self.assertEqual("集群代理正常", data["cluster_proxy_runtime_label"])
|
||||
self.assertEqual(10, data["aggregate_process_count"])
|
||||
self.assertEqual(1, data["ops_summary"]["active_execution_nodes"])
|
||||
self.assertEqual(8487, data["backlog_register_pending_total"])
|
||||
self.assertEqual(951, data["backlog_downstream_pending_total"])
|
||||
self.assertEqual(12, data["retry_total"])
|
||||
@@ -228,10 +243,256 @@ class DashboardServiceTests(unittest.TestCase):
|
||||
self.assertEqual(1.5, data["ops_summary"]["estimated_hours_remaining"])
|
||||
self.assertEqual(1, data["active_execution_nodes"])
|
||||
self.assertEqual(1, data["ops_summary"]["active_execution_nodes"])
|
||||
self.assertEqual(0, data["current_job_blacklisted"])
|
||||
self.assertEqual(0, data["recent_blacklisted_total"])
|
||||
self.assertEqual(0, data["cumulative_blacklisted_total"])
|
||||
self.assertEqual(2, len(data["step_queue"]))
|
||||
self.assertTrue(any(item["step_code"] == "detect_360_site" for item in data["step_queue"]))
|
||||
self.assertEqual(1, len(data["node_throughput"]))
|
||||
|
||||
@patch("app.services.dashboard._fetch_active_jobs_aggregate")
|
||||
@patch("app.services.dashboard.get_detect_capacity_plan")
|
||||
@patch("app.services.dashboard.get_detect_queue_health")
|
||||
@patch("app.services.dashboard._build_dashboard_runtime_summary")
|
||||
@patch("app.services.dashboard.get_active_detect_job_summary")
|
||||
@patch("app.services.dashboard.get_db")
|
||||
def test_fetch_overview_prefers_active_job_display_running_when_queue_snapshot_is_stale(
|
||||
self,
|
||||
mock_get_db,
|
||||
mock_get_active_detect_job_summary,
|
||||
mock_build_dashboard_runtime_summary,
|
||||
mock_get_detect_queue_health,
|
||||
mock_get_detect_capacity_plan,
|
||||
mock_fetch_active_jobs_aggregate,
|
||||
) -> None:
|
||||
mock_get_db.return_value = _FakeConnection(
|
||||
responses=[
|
||||
(1000,),
|
||||
(900,),
|
||||
(10,),
|
||||
(5,),
|
||||
(0,),
|
||||
(1,),
|
||||
(430,),
|
||||
(420,),
|
||||
(17,),
|
||||
]
|
||||
)
|
||||
mock_fetch_active_jobs_aggregate.return_value = {
|
||||
"active_jobs_total": 1,
|
||||
"queue": {"items_total": 8402, "pending": 5001, "claimed": 0, "running": 1, "completed": 0, "blacklisted": 0, "failed": 0},
|
||||
"throughput": {"processed_recent": 0, "processed_per_minute": 0.0},
|
||||
"steps": [],
|
||||
"nodes": [],
|
||||
"retry_total": 0,
|
||||
}
|
||||
mock_get_active_detect_job_summary.return_value = {
|
||||
"job_id": 11,
|
||||
"job_code": "sync-overseas-51",
|
||||
"status": "running",
|
||||
"items_total": 8402,
|
||||
"items_pending": 5001,
|
||||
"items_claimed": 0,
|
||||
"items_running": 0,
|
||||
"items_completed": 0,
|
||||
"items_blacklisted": 0,
|
||||
"items_failed": 0,
|
||||
"display_items_running": 1432,
|
||||
"display_active_threads": 138,
|
||||
"display_max_threads": 80000,
|
||||
"distributed_node_stats": [
|
||||
{"node_code": "mainland-controller-01-a", "display_running": 1000, "active_threads": 1000, "max_threads": 1000},
|
||||
{"node_code": "mainland-controller-01-b", "display_running": 432, "active_threads": 432, "max_threads": 1000},
|
||||
],
|
||||
}
|
||||
mock_build_dashboard_runtime_summary.return_value = {
|
||||
"worker": {"running": False, "mode": "linux-systemd", "expected_on_this_node": False},
|
||||
"node": {"region": "overseas", "role": "control"},
|
||||
"cluster": {"summary": {"online_worker_nodes": 1, "dedicated_online_worker_nodes": 0, "online_control_nodes": 2}},
|
||||
"detect": {
|
||||
"backlog": {
|
||||
"pending_total": 5001,
|
||||
"claimed_total": 0,
|
||||
"running_total": 0,
|
||||
"completed_total": 0,
|
||||
"blacklisted_total": 0,
|
||||
"failed_total": 0,
|
||||
"register_pending": 3495,
|
||||
"downstream_pending": 1506,
|
||||
}
|
||||
},
|
||||
}
|
||||
mock_get_detect_queue_health.return_value = {
|
||||
"has_active_job": True,
|
||||
"job": {"job_id": 11, "job_code": "sync-overseas-51", "runtime_job_code": "sync-overseas-51", "status": "running", "progress_percent": 40.49},
|
||||
"queue": {
|
||||
"items_total": 8402,
|
||||
"pending": 5001,
|
||||
"claimed": 0,
|
||||
"display_claimed": 0,
|
||||
"running": 1,
|
||||
"display_running": 1,
|
||||
"completed": 0,
|
||||
"blacklisted": 0,
|
||||
"failed": 0,
|
||||
},
|
||||
"throughput": {"processed_recent": 0, "processed_per_minute": 0.0},
|
||||
"steps": [],
|
||||
"runtime_activity": {},
|
||||
"nodes": [],
|
||||
}
|
||||
mock_get_detect_capacity_plan.return_value = {
|
||||
"estimated_hours_remaining": 0,
|
||||
"remaining_items": 5001,
|
||||
"recommended_additional_workers": 0,
|
||||
}
|
||||
|
||||
data = fetch_overview()
|
||||
|
||||
self.assertEqual(1432, data["queue_display_running_total"])
|
||||
self.assertEqual(1432, data["active_job"]["items_display_running"])
|
||||
self.assertEqual(1432, data["active_job"]["display_items_running"])
|
||||
self.assertEqual(138, data["active_job"]["display_active_threads"])
|
||||
self.assertEqual(80000, data["active_job"]["display_max_threads"])
|
||||
self.assertEqual(2, len(data["active_job"]["distributed_node_stats"]))
|
||||
self.assertEqual(80000, data["queue_display_max_threads"])
|
||||
self.assertEqual(0, data["current_job_blacklisted"])
|
||||
self.assertEqual(0, data["recent_blacklisted_total"])
|
||||
self.assertEqual(0, data["cumulative_blacklisted_total"])
|
||||
|
||||
@patch("app.services.dashboard._fetch_active_jobs_aggregate")
|
||||
@patch("app.services.dashboard.get_detect_capacity_plan")
|
||||
@patch("app.services.dashboard.get_detect_queue_health")
|
||||
@patch("app.services.dashboard._build_dashboard_runtime_summary")
|
||||
@patch("app.services.dashboard.get_active_detect_job_summary")
|
||||
@patch("app.services.dashboard.get_db")
|
||||
def test_fetch_overview_counts_active_execution_nodes_from_full_node_set(
|
||||
self,
|
||||
mock_get_db,
|
||||
mock_get_active_detect_job_summary,
|
||||
mock_build_dashboard_runtime_summary,
|
||||
mock_get_detect_queue_health,
|
||||
mock_get_detect_capacity_plan,
|
||||
mock_fetch_active_jobs_aggregate,
|
||||
) -> None:
|
||||
mock_get_db.return_value = _FakeConnection(responses=[(0,)] * 9)
|
||||
mock_fetch_active_jobs_aggregate.return_value = {
|
||||
"active_jobs_total": 1,
|
||||
"queue": {"items_total": 9, "pending": 0, "claimed": 0, "running": 9, "completed": 0, "blacklisted": 0, "failed": 0},
|
||||
"throughput": {"processed_recent": 0, "processed_per_minute": 0.0, "completed_recent": 0, "blacklisted_recent": 0, "failed_recent": 0},
|
||||
"steps": [],
|
||||
"nodes": [
|
||||
{
|
||||
"node_code": f"mainland-controller-01-{index:02d}",
|
||||
"items_running": 1,
|
||||
"items_claimed": 0,
|
||||
"processed_recent": 0,
|
||||
"processed_per_minute": 0.0,
|
||||
"completed_recent": 0,
|
||||
"failed_recent": 0,
|
||||
"blacklisted_recent": 0,
|
||||
}
|
||||
for index in range(9)
|
||||
],
|
||||
"retry_total": 0,
|
||||
}
|
||||
mock_get_active_detect_job_summary.return_value = {}
|
||||
mock_build_dashboard_runtime_summary.return_value = {
|
||||
"worker": {"running": True, "mode": "linux-systemd", "expected_on_this_node": True},
|
||||
"node": {"region": "mainland", "role": "worker"},
|
||||
"cluster": {"summary": {"online_worker_nodes": 1, "dedicated_online_worker_nodes": 0, "online_control_nodes": 1}},
|
||||
"detect": {"backlog": {}},
|
||||
}
|
||||
mock_get_detect_queue_health.return_value = {
|
||||
"has_active_job": False,
|
||||
"queue": {},
|
||||
"throughput": {"processed_recent": 0, "processed_per_minute": 0.0},
|
||||
"steps": [],
|
||||
"runtime_activity": {},
|
||||
"nodes": [],
|
||||
"runtime_snapshot_backlog": {},
|
||||
}
|
||||
mock_get_detect_capacity_plan.return_value = {
|
||||
"estimated_hours_remaining": 0,
|
||||
"remaining_items": 0,
|
||||
"recommended_additional_workers": 0,
|
||||
}
|
||||
|
||||
data = fetch_overview()
|
||||
|
||||
self.assertEqual(8, len(data["node_throughput"]))
|
||||
self.assertEqual(9, data["active_execution_nodes"])
|
||||
self.assertEqual(9, data["ops_summary"]["active_execution_nodes"])
|
||||
|
||||
@patch("app.services.dashboard._fetch_active_jobs_aggregate")
|
||||
@patch("app.services.dashboard.get_detect_capacity_plan")
|
||||
@patch("app.services.dashboard.get_detect_queue_health")
|
||||
@patch("app.services.dashboard._build_dashboard_runtime_summary")
|
||||
@patch("app.services.dashboard.get_active_detect_job_summary")
|
||||
@patch("app.services.dashboard.get_db")
|
||||
def test_fetch_overview_keeps_active_job_summary_when_queue_health_temporarily_empty(
|
||||
self,
|
||||
mock_get_db,
|
||||
mock_get_active_detect_job_summary,
|
||||
mock_build_dashboard_runtime_summary,
|
||||
mock_get_detect_queue_health,
|
||||
mock_get_detect_capacity_plan,
|
||||
mock_fetch_active_jobs_aggregate,
|
||||
) -> None:
|
||||
mock_get_db.return_value = _FakeConnection(responses=[(0,)] * 9)
|
||||
mock_fetch_active_jobs_aggregate.return_value = {
|
||||
"active_jobs_total": 1,
|
||||
"queue": {"items_total": 8402, "pending": 5001, "claimed": 0, "running": 537, "completed": 0, "blacklisted": 0, "failed": 0},
|
||||
"throughput": {"processed_recent": 12, "processed_per_minute": 0.8, "completed_recent": 10, "blacklisted_recent": 1, "failed_recent": 1},
|
||||
"steps": [],
|
||||
"nodes": [],
|
||||
"retry_total": 0,
|
||||
}
|
||||
mock_get_active_detect_job_summary.return_value = {
|
||||
"job_id": 255,
|
||||
"job_code": "sync-overseas-255",
|
||||
"runtime_job_code": "sync-overseas-255",
|
||||
"status": "running",
|
||||
"progress_percent": 22.4,
|
||||
"items_total": 8402,
|
||||
"items_pending": 5001,
|
||||
"items_claimed": 0,
|
||||
"items_running": 537,
|
||||
"items_completed": 0,
|
||||
"items_blacklisted": 0,
|
||||
"items_failed": 0,
|
||||
"display_items_running": 11799,
|
||||
"display_active_threads": 11799,
|
||||
"display_max_threads": 75400,
|
||||
"distributed_node_stats": [{"node_code": "mainland-controller-01-a", "display_running": 537, "active_threads": 537, "max_threads": 1000}],
|
||||
}
|
||||
mock_build_dashboard_runtime_summary.return_value = {
|
||||
"worker": {"running": False, "mode": "linux-systemd", "expected_on_this_node": False},
|
||||
"node": {"region": "overseas", "role": "control"},
|
||||
"cluster": {"summary": {"online_worker_nodes": 1, "dedicated_online_worker_nodes": 0, "online_control_nodes": 1}},
|
||||
"detect": {"backlog": {}},
|
||||
}
|
||||
mock_get_detect_queue_health.return_value = {
|
||||
"has_active_job": False,
|
||||
"queue": {},
|
||||
"throughput": {"processed_recent": 0, "processed_per_minute": 0.0},
|
||||
"steps": [],
|
||||
"runtime_activity": {},
|
||||
"nodes": [],
|
||||
"runtime_snapshot_backlog": {},
|
||||
}
|
||||
mock_get_detect_capacity_plan.return_value = {
|
||||
"estimated_hours_remaining": 0,
|
||||
"remaining_items": 5001,
|
||||
"recommended_additional_workers": 0,
|
||||
}
|
||||
|
||||
data = fetch_overview()
|
||||
|
||||
self.assertEqual("sync-overseas-255", data["active_job"]["job_code"])
|
||||
self.assertEqual(11799, data["active_job"]["display_items_running"])
|
||||
self.assertEqual(75400, data["active_job"]["display_max_threads"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
83
domain-api/tests/test_debug_event_service.py
Normal file
83
domain-api/tests/test_debug_event_service.py
Normal file
@@ -0,0 +1,83 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
from app.services import debug_event_service
|
||||
|
||||
|
||||
class DebugEventServiceTests(unittest.TestCase):
|
||||
def test_ingest_debug_event_requires_configured_shared_token(self) -> None:
|
||||
with patch.object(debug_event_service.settings, "sync_shared_token", ""):
|
||||
ok, message, data = debug_event_service.ingest_debug_event(
|
||||
{
|
||||
"source_region": "mainland",
|
||||
"node_code": "mainland-controller-01",
|
||||
"service": "worker-event",
|
||||
"event_type": "worker_log",
|
||||
"message": "开始检测域名: a.com",
|
||||
"payload": {"job_id": 1, "job_code": "sync-overseas-1"},
|
||||
},
|
||||
shared_token=None,
|
||||
)
|
||||
|
||||
self.assertFalse(ok)
|
||||
self.assertIn("token 未配置", message)
|
||||
self.assertTrue(data["configuration_required"])
|
||||
|
||||
def test_resolve_target_job_for_debug_event_rejects_missing_identity(self) -> None:
|
||||
job, reason = debug_event_service._resolve_target_job_for_debug_event({"domain": "a.com"})
|
||||
|
||||
self.assertIsNone(job)
|
||||
self.assertEqual("missing_job_identity", reason)
|
||||
|
||||
@patch("app.services.debug_event_service._load_detect_job_summary_by_job_code")
|
||||
@patch("app.services.detect_job_service.get_detect_job_summary")
|
||||
@patch("app.services.detect_job_service.get_active_detect_job_summary")
|
||||
def test_resolve_target_job_for_debug_event_prefers_payload_job_id_over_current_active_job(
|
||||
self,
|
||||
mock_get_active_detect_job_summary,
|
||||
mock_get_detect_job_summary,
|
||||
mock_load_detect_job_summary_by_job_code,
|
||||
) -> None:
|
||||
mock_get_active_detect_job_summary.return_value = {
|
||||
"job_id": 12,
|
||||
"job_code": "sync-overseas-12",
|
||||
"current_cycle_token": "cycle-12",
|
||||
}
|
||||
mock_get_detect_job_summary.return_value = {
|
||||
"job_id": 11,
|
||||
"job_code": "sync-overseas-11",
|
||||
"current_cycle_token": "cycle-11",
|
||||
}
|
||||
|
||||
job, reason = debug_event_service._resolve_target_job_for_debug_event(
|
||||
{"job_id": 11, "job_code": "sync-overseas-11", "cycle_token": "cycle-11"}
|
||||
)
|
||||
|
||||
self.assertEqual("matched", reason)
|
||||
self.assertEqual(11, job["job_id"])
|
||||
mock_get_detect_job_summary.assert_called_once_with(11, event_limit=1)
|
||||
mock_load_detect_job_summary_by_job_code.assert_not_called()
|
||||
|
||||
@patch("app.services.detect_job_service.get_active_detect_job_summary")
|
||||
def test_resolve_target_job_for_debug_event_rejects_cycle_mismatch(
|
||||
self,
|
||||
mock_get_active_detect_job_summary,
|
||||
) -> None:
|
||||
mock_get_active_detect_job_summary.return_value = {
|
||||
"job_id": 12,
|
||||
"job_code": "sync-overseas-12",
|
||||
"current_cycle_token": "cycle-current",
|
||||
}
|
||||
|
||||
job, reason = debug_event_service._resolve_target_job_for_debug_event(
|
||||
{"job_id": 12, "job_code": "sync-overseas-12", "cycle_token": "cycle-old"}
|
||||
)
|
||||
|
||||
self.assertIsNone(job)
|
||||
self.assertEqual("cycle_mismatch", reason)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -7,6 +7,153 @@ from app.api.routes import detect as detect_route
|
||||
|
||||
|
||||
class DetectApiRoutesTestCase(unittest.TestCase):
|
||||
@patch("app.api.routes.detect.create_detect_run_snapshot")
|
||||
@patch("app.api.routes.detect._dispatch_remote_detect_start")
|
||||
@patch("app.api.routes.detect.get_settings_payload")
|
||||
@patch("app.api.routes.detect.get_detect_status")
|
||||
@patch("app.api.routes.detect.send_worker_command")
|
||||
@patch("app.api.routes.detect.start_worker")
|
||||
@patch("app.api.routes.detect.append_detect_job_event")
|
||||
@patch("app.api.routes.detect.create_detect_job_if_needed")
|
||||
def test_start_detect_skips_redundant_systemctl_when_worker_already_online(
|
||||
self,
|
||||
mock_create_job,
|
||||
mock_append_event,
|
||||
mock_start_worker,
|
||||
mock_send_worker_command,
|
||||
mock_get_detect_status,
|
||||
mock_get_settings_payload,
|
||||
mock_dispatch_remote,
|
||||
mock_create_snapshot,
|
||||
) -> None:
|
||||
mock_create_job.return_value = {
|
||||
"job_id": 30,
|
||||
"job_code": "sync-overseas-30",
|
||||
"status": "running",
|
||||
"items_pending": 100,
|
||||
"items_claimed": 0,
|
||||
"items_running": 0,
|
||||
"task_mode": "domain_pipeline",
|
||||
"step_code": "",
|
||||
}
|
||||
mock_get_detect_status.return_value = {
|
||||
"worker_mode": "linux-systemd",
|
||||
"worker_online": True,
|
||||
"worker_process_count": 60,
|
||||
"worker_latest_start_time": "2026-04-24 23:30:00",
|
||||
"worker_runtime_message": "running",
|
||||
"progress": {},
|
||||
}
|
||||
mock_send_worker_command.return_value = (True, "已发送 Worker 控制指令: start_detection")
|
||||
mock_get_settings_payload.return_value = {
|
||||
"runtime": {"thread_count": 1000},
|
||||
"proxy_config": {"proxy_enable": True, "allow_direct": True, "proxy_urls": ["a"]},
|
||||
}
|
||||
mock_dispatch_remote.return_value = {
|
||||
"queued_jobs": [],
|
||||
"queued_total": 0,
|
||||
"failed_total": 0,
|
||||
"target_summary": {"controller_nodes": [], "worker_nodes": []},
|
||||
}
|
||||
|
||||
with patch.object(detect_route.settings, "node_region", "mainland"), patch.object(
|
||||
detect_route.settings, "node_role", "control"
|
||||
):
|
||||
response = detect_route.start_detect()
|
||||
|
||||
self.assertEqual(0, response.code)
|
||||
self.assertIn("检测端已在运行,跳过重复启动", response.message)
|
||||
mock_start_worker.assert_not_called()
|
||||
mock_send_worker_command.assert_called_once()
|
||||
event_types = [call.kwargs.get("event_type") for call in mock_append_event.call_args_list]
|
||||
self.assertIn("job_dispatch_start_skipped", event_types)
|
||||
self.assertIn("job_dispatch_sent", event_types)
|
||||
mock_create_snapshot.assert_called_once()
|
||||
|
||||
@patch("app.api.routes.detect.create_detect_run_snapshot")
|
||||
@patch("app.api.routes.detect._dispatch_remote_detect_start")
|
||||
@patch("app.api.routes.detect.get_settings_payload")
|
||||
@patch("app.api.routes.detect.get_detect_status")
|
||||
@patch("app.api.routes.detect.send_worker_command")
|
||||
@patch("app.api.routes.detect.start_worker")
|
||||
@patch("app.api.routes.detect.append_detect_job_event")
|
||||
@patch("app.api.routes.detect.create_detect_job_if_needed")
|
||||
def test_start_detect_falls_back_to_direct_command_when_worker_already_running(
|
||||
self,
|
||||
mock_create_job,
|
||||
mock_append_event,
|
||||
mock_start_worker,
|
||||
mock_send_worker_command,
|
||||
mock_get_detect_status,
|
||||
mock_get_settings_payload,
|
||||
mock_dispatch_remote,
|
||||
mock_create_snapshot,
|
||||
) -> None:
|
||||
mock_create_job.return_value = {
|
||||
"job_id": 31,
|
||||
"job_code": "sync-overseas-31",
|
||||
"status": "running",
|
||||
"items_pending": 100,
|
||||
"items_claimed": 0,
|
||||
"items_running": 0,
|
||||
"task_mode": "domain_pipeline",
|
||||
"step_code": "",
|
||||
}
|
||||
mock_start_worker.return_value = (
|
||||
False,
|
||||
"domaincheck-worker 控制失败:当前运行用户没有免密 systemctl 权限,请为 API 进程授予对应 sudo/systemd 权限",
|
||||
)
|
||||
mock_send_worker_command.return_value = (True, "已发送 Worker 控制指令: start_detection")
|
||||
mock_get_detect_status.side_effect = [
|
||||
{
|
||||
"worker_mode": "linux-systemd",
|
||||
"worker_online": False,
|
||||
"worker_process_count": 0,
|
||||
"worker_latest_start_time": "",
|
||||
"worker_runtime_message": "starting",
|
||||
"progress": {},
|
||||
},
|
||||
{
|
||||
"worker_mode": "linux-systemd",
|
||||
"worker_online": True,
|
||||
"worker_process_count": 60,
|
||||
"worker_latest_start_time": "2026-04-24 23:30:00",
|
||||
"worker_runtime_message": "running",
|
||||
"progress": {},
|
||||
},
|
||||
{
|
||||
"worker_mode": "linux-systemd",
|
||||
"worker_online": True,
|
||||
"worker_process_count": 60,
|
||||
"worker_latest_start_time": "2026-04-24 23:30:00",
|
||||
"worker_runtime_message": "running",
|
||||
"progress": {},
|
||||
},
|
||||
]
|
||||
mock_get_settings_payload.return_value = {
|
||||
"runtime": {"thread_count": 1000},
|
||||
"proxy_config": {"proxy_enable": True, "allow_direct": True, "proxy_urls": ["a"]},
|
||||
}
|
||||
mock_dispatch_remote.return_value = {
|
||||
"queued_jobs": [],
|
||||
"queued_total": 0,
|
||||
"failed_total": 0,
|
||||
"target_summary": {"controller_nodes": [], "worker_nodes": []},
|
||||
}
|
||||
|
||||
with patch.object(detect_route.settings, "node_region", "mainland"), patch.object(
|
||||
detect_route.settings, "node_role", "control"
|
||||
):
|
||||
response = detect_route.start_detect()
|
||||
|
||||
self.assertEqual(0, response.code)
|
||||
self.assertIn("检测端已在运行,改为直接发送控制指令", response.message)
|
||||
mock_send_worker_command.assert_called_once()
|
||||
event_types = [call.kwargs.get("event_type") for call in mock_append_event.call_args_list]
|
||||
self.assertIn("job_dispatch_start_degraded", event_types)
|
||||
self.assertIn("job_dispatch_sent", event_types)
|
||||
mock_create_snapshot.assert_called_once()
|
||||
|
||||
@patch("app.api.routes.detect.create_detect_run_snapshot")
|
||||
@patch("app.api.routes.detect._dispatch_remote_detect_start")
|
||||
@patch("app.api.routes.detect.get_settings_payload")
|
||||
@@ -65,11 +212,94 @@ class DetectApiRoutesTestCase(unittest.TestCase):
|
||||
mock_start_worker.assert_not_called()
|
||||
mock_send_worker_command.assert_not_called()
|
||||
mock_dispatch_remote.assert_called_once()
|
||||
mock_create_snapshot.assert_called_once()
|
||||
mock_create_snapshot.assert_not_called()
|
||||
event_types = [call.kwargs.get("event_type") for call in mock_append_event.call_args_list]
|
||||
self.assertIn("job_dispatch_requested", event_types)
|
||||
self.assertIn("job_dispatch_skipped_local", event_types)
|
||||
|
||||
@patch("app.api.routes.detect.finalize_detect_run")
|
||||
@patch("app.api.routes.detect.mark_detect_run_stopping")
|
||||
@patch("app.api.routes.detect.get_settings_payload")
|
||||
@patch("app.api.routes.detect.get_detect_status")
|
||||
@patch("app.api.routes.detect._dispatch_remote_detect_stop")
|
||||
@patch("app.api.routes.detect.send_worker_command")
|
||||
@patch("app.api.routes.detect.get_active_detect_job_summary")
|
||||
def test_stop_detect_forwards_target_payload(
|
||||
self,
|
||||
mock_get_active_job,
|
||||
mock_send_worker_command,
|
||||
mock_dispatch_remote_stop,
|
||||
mock_get_detect_status,
|
||||
mock_get_settings_payload,
|
||||
mock_mark_detect_run_stopping,
|
||||
mock_finalize_detect_run,
|
||||
) -> None:
|
||||
mock_get_active_job.return_value = {
|
||||
"job_id": 29,
|
||||
"job_code": "sync-overseas-29",
|
||||
"status": "running",
|
||||
"current_cycle_token": "cycle-29",
|
||||
}
|
||||
mock_send_worker_command.return_value = (True, "已发送 Worker 控制指令")
|
||||
mock_dispatch_remote_stop.return_value = {
|
||||
"queued_jobs": [{"node_code": "mainland-controller-01"}],
|
||||
"queued_total": 1,
|
||||
"failed_total": 0,
|
||||
"target_summary": {"controller_nodes": ["mainland-controller-01"], "worker_nodes": []},
|
||||
}
|
||||
mock_get_detect_status.return_value = {
|
||||
"worker_mode": "linux-systemd",
|
||||
"worker_online": True,
|
||||
"worker_process_count": 1,
|
||||
"worker_latest_start_time": "",
|
||||
"worker_runtime_message": "running",
|
||||
"progress": {},
|
||||
}
|
||||
mock_get_settings_payload.return_value = {
|
||||
"thread_count": 1000,
|
||||
"node_thread_counts": {},
|
||||
"process_count": 80,
|
||||
"node_process_counts": {},
|
||||
"proxy_config": {"proxy_enable": True, "allow_direct": False, "proxy_urls": ["a"]},
|
||||
}
|
||||
|
||||
response = detect_route.stop_detect(payload={"target_node_codes": ["mainland-controller-01"]})
|
||||
|
||||
self.assertEqual(0, response.code)
|
||||
mock_send_worker_command.assert_called_once_with(
|
||||
"stop_detection",
|
||||
payload={"target_node_codes": ["mainland-controller-01"]},
|
||||
)
|
||||
mock_dispatch_remote_stop.assert_called_once_with(
|
||||
active_job=mock_get_active_job.return_value,
|
||||
cycle_token="cycle-29",
|
||||
payload={"target_node_codes": ["mainland-controller-01"]},
|
||||
)
|
||||
mock_mark_detect_run_stopping.assert_called_once()
|
||||
mock_finalize_detect_run.assert_not_called()
|
||||
|
||||
@patch("app.api.routes.detect.create_ops_job")
|
||||
@patch("app.api.routes.detect.list_managed_nodes")
|
||||
def test_dispatch_remote_detect_stop_filters_target_nodes(self, mock_list_managed_nodes, mock_create_ops_job) -> None:
|
||||
mock_list_managed_nodes.return_value = [
|
||||
{"node_code": "mainland-controller-01", "region": "mainland", "role": "control", "last_seen_at": "2026-04-23T14:00:00", "is_enabled": True},
|
||||
{"node_code": "mainland-worker-01", "region": "mainland", "role": "worker", "last_seen_at": "2026-04-23T14:00:00", "is_enabled": True},
|
||||
]
|
||||
mock_create_ops_job.return_value = (True, "queued", {})
|
||||
|
||||
result = detect_route._dispatch_remote_detect_stop(
|
||||
active_job={"job_id": 29, "job_code": "sync-overseas-29"},
|
||||
cycle_token="cycle-29",
|
||||
payload={"target_node_codes": ["mainland-worker-01"]},
|
||||
)
|
||||
|
||||
self.assertEqual(1, result["queued_total"])
|
||||
self.assertEqual(["mainland-controller-01"], result["target_summary"]["controller_nodes"])
|
||||
self.assertEqual(["mainland-worker-01"], result["target_summary"]["worker_nodes"])
|
||||
mock_create_ops_job.assert_called_once()
|
||||
create_payload = mock_create_ops_job.call_args.args[0]
|
||||
self.assertEqual("mainland-worker-01", create_payload["target_node_code"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
File diff suppressed because it is too large
Load Diff
65
domain-api/tests/test_detect_run_service.py
Normal file
65
domain-api/tests/test_detect_run_service.py
Normal file
@@ -0,0 +1,65 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
import app.services.detect_run_service as detect_run_service
|
||||
|
||||
|
||||
class _TraceLock:
|
||||
def __init__(self, order: list[str]) -> None:
|
||||
self.order = order
|
||||
|
||||
def __enter__(self):
|
||||
self.order.append("enter")
|
||||
return self
|
||||
|
||||
def __exit__(self, exc_type, exc, tb):
|
||||
self.order.append("exit")
|
||||
return False
|
||||
|
||||
|
||||
class DetectRunServiceTests(unittest.TestCase):
|
||||
def test_create_detect_run_snapshot_acquires_lock_before_loading_records(self) -> None:
|
||||
order: list[str] = []
|
||||
|
||||
with patch.object(detect_run_service, "_DETECT_RUNS_LOCK", _TraceLock(order)):
|
||||
with patch.object(detect_run_service, "_load", side_effect=lambda: order.append("load") or []):
|
||||
with patch.object(detect_run_service, "_capture_worker_logs", return_value=[]):
|
||||
with patch.object(detect_run_service, "_save", side_effect=lambda records: order.append("save")):
|
||||
record = detect_run_service.create_detect_run_snapshot(
|
||||
"start",
|
||||
{"latest_start_time": "2026-04-23 15:00:00", "running": True},
|
||||
{"running": 1},
|
||||
{"thread_count": 1000},
|
||||
)
|
||||
|
||||
self.assertEqual("enter", order[0])
|
||||
self.assertIn("load", order)
|
||||
self.assertIn("save", order)
|
||||
self.assertEqual("exit", order[-1])
|
||||
self.assertEqual("starting", record["status"])
|
||||
|
||||
def test_sync_detect_runs_acquires_lock_before_mutating_records(self) -> None:
|
||||
order: list[str] = []
|
||||
|
||||
with patch.object(detect_run_service, "_DETECT_RUNS_LOCK", _TraceLock(order)):
|
||||
with patch.object(detect_run_service, "_load", side_effect=lambda: order.append("load") or []):
|
||||
with patch.object(detect_run_service, "_capture_worker_logs", return_value=[]):
|
||||
with patch.object(detect_run_service, "_save", side_effect=lambda records: order.append("save")):
|
||||
records = detect_run_service.sync_detect_runs(
|
||||
{"running": True, "detecting": True, "latest_start_time": "2026-04-23 15:00:00"},
|
||||
{"running": 1, "pending": 0},
|
||||
{"thread_count": 1000},
|
||||
active_job={"status": "running", "items_running": 1},
|
||||
)
|
||||
|
||||
self.assertEqual("enter", order[0])
|
||||
self.assertIn("load", order)
|
||||
self.assertIn("save", order)
|
||||
self.assertEqual("exit", order[-1])
|
||||
self.assertEqual("running", records[0]["status"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
62
domain-api/tests/test_detect_service_remote_logs.py
Normal file
62
domain-api/tests/test_detect_service_remote_logs.py
Normal file
@@ -0,0 +1,62 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
from app.services import detect_service
|
||||
|
||||
|
||||
class DetectServiceRemoteLogsTests(unittest.TestCase):
|
||||
@patch("app.services.detect_service.list_debug_events")
|
||||
def test_build_remote_log_snapshot_from_debug_events_filters_to_active_job_identity(self, mock_list_debug_events) -> None:
|
||||
mock_list_debug_events.return_value = {
|
||||
"records": [
|
||||
{
|
||||
"id": 3,
|
||||
"node_code": "mainland-controller-01-a",
|
||||
"event_type": "worker_log",
|
||||
"message": "旧任务日志",
|
||||
"created_at": "2026-04-23 16:00:00",
|
||||
"payload": {
|
||||
"job_id": 10,
|
||||
"job_code": "sync-overseas-10",
|
||||
"cycle_token": "cycle-old",
|
||||
"log_mode": "key",
|
||||
},
|
||||
},
|
||||
{
|
||||
"id": 4,
|
||||
"node_code": "mainland-controller-01-a",
|
||||
"event_type": "worker_log",
|
||||
"message": "当前任务日志",
|
||||
"created_at": "2026-04-23 16:01:00",
|
||||
"payload": {
|
||||
"job_id": 12,
|
||||
"job_code": "sync-overseas-12",
|
||||
"cycle_token": "cycle-12",
|
||||
"log_mode": "key",
|
||||
},
|
||||
},
|
||||
]
|
||||
}
|
||||
|
||||
snapshot = detect_service._build_remote_log_snapshot_from_debug_events(
|
||||
{
|
||||
"job_id": 12,
|
||||
"job_code": "sync-overseas-12",
|
||||
"runtime_job_code": "sync-overseas-12",
|
||||
"current_cycle_token": "cycle-12",
|
||||
"node_stats": [{"node_code": "mainland-controller-01-a"}],
|
||||
},
|
||||
enabled=True,
|
||||
mode="key",
|
||||
limit=20,
|
||||
)
|
||||
|
||||
self.assertEqual(1, snapshot["line_count"])
|
||||
self.assertIn("当前任务日志", snapshot["lines"][0])
|
||||
self.assertNotIn("旧任务日志", "\n".join(snapshot["lines"]))
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -1,10 +1,47 @@
|
||||
import unittest
|
||||
from contextlib import ExitStack
|
||||
from datetime import datetime, timedelta
|
||||
from unittest.mock import patch
|
||||
|
||||
from app.services import detect_service
|
||||
|
||||
|
||||
class DetectServiceStatusFallbackTests(unittest.TestCase):
|
||||
def test_filter_live_aggregate_runtime_nodes_excludes_disabled_managed_nodes(self) -> None:
|
||||
now_text = datetime.now().isoformat(sep=" ", timespec="seconds")
|
||||
rows = [
|
||||
{
|
||||
"node_code": "mainland-controller-01",
|
||||
"status": "busy",
|
||||
"last_heartbeat_at": now_text,
|
||||
"display_running": 1,
|
||||
},
|
||||
{
|
||||
"node_code": "mainland-worker-01",
|
||||
"status": "busy",
|
||||
"last_heartbeat_at": now_text,
|
||||
"display_running": 1,
|
||||
},
|
||||
]
|
||||
|
||||
with patch("app.services.detect_service._load_disabled_managed_node_codes", return_value={"mainland-worker-01"}):
|
||||
filtered = detect_service._filter_live_aggregate_runtime_nodes(rows)
|
||||
|
||||
self.assertEqual(["mainland-controller-01"], [item["node_code"] for item in filtered])
|
||||
|
||||
def setUp(self) -> None:
|
||||
detect_service._DETECT_STATUS_CACHE_VALUE = None
|
||||
detect_service._DETECT_STATUS_CACHE_EXPIRES_AT = 0.0
|
||||
|
||||
def test_resolve_capacity_node_code_treats_child_instance_suffix_as_single_process(self) -> None:
|
||||
capacity_node_code, is_child_instance = detect_service._resolve_capacity_node_code(
|
||||
"mainland-controller-01-ae",
|
||||
{"node_thread_counts": {"overseas-control-01": 1}, "node_process_counts": {}},
|
||||
)
|
||||
|
||||
self.assertEqual("mainland-controller-01", capacity_node_code)
|
||||
self.assertTrue(is_child_instance)
|
||||
|
||||
def test_get_detect_status_keeps_runtime_snapshot_when_db_is_unreachable(self) -> None:
|
||||
runtime_state = {
|
||||
"service_running": True,
|
||||
@@ -17,24 +54,39 @@ class DetectServiceStatusFallbackTests(unittest.TestCase):
|
||||
"available_proxy_count": 18,
|
||||
}
|
||||
|
||||
with patch("app.services.detect_service.ensure_runtime_schema"), \
|
||||
patch("app.services.detect_service.get_db", side_effect=RuntimeError("db down")), \
|
||||
patch("app.services.detect_service.get_settings_payload", return_value={"proxy_config": {"proxy_enable": True, "allow_direct": False, "proxy_urls": ["a"]}}), \
|
||||
patch("app.services.detect_service.get_runtime_settings", return_value={"worker_log_sync_enabled": False, "worker_log_sync_mode": "full"}), \
|
||||
patch("app.services.detect_service._load_recent_worker_lines", return_value=(True, "", [])), \
|
||||
patch("app.services.detect_service.detect_worker_runtime", return_value={"mode": "linux-systemd", "running": True, "process_count": 1, "latest_start_time": "2026-04-20 23:58:00", "message": "active/running"}), \
|
||||
patch("app.services.detect_service._load_runtime_state", return_value=runtime_state), \
|
||||
patch("app.services.detect_service._load_runtime_state_from_cluster_node", return_value={}), \
|
||||
patch("app.services.detect_service._extract_available_proxy_count", return_value=0), \
|
||||
patch("app.services.detect_service._extract_active_thread_snapshot", return_value={"active": 0, "max": 0}), \
|
||||
patch("app.services.detect_service._normalize_recent_warning", return_value=""), \
|
||||
patch("app.services.detect_service._build_proxy_runtime_snapshot", return_value={"state": "healthy", "label": "代理正常", "detail": "healthy", "direct_fallback_active": False, "reason": "healthy", "last_refresh_status": "ok", "last_refresh_time": "", "source_count": 2, "raw_items": 18, "validated_count": 18, "available_count": 18, "source_stats": [], "supplier_empty": False}), \
|
||||
patch("app.services.detect_service.resolve_thread_count", return_value={"effective_thread_count": 120, "default_thread_count": 5, "source": "node_override", "override_thread_count": 120, "node_code": "mainland-worker-01"}), \
|
||||
patch("app.services.detect_service.get_active_detect_job_summary", side_effect=RuntimeError("db down")), \
|
||||
patch("app.services.detect_service.sync_detect_runs", return_value=[]), \
|
||||
patch("app.services.detect_service._resolve_remote_log_snapshot", return_value={}), \
|
||||
patch("app.services.detect_service._extract_dependency_alerts", return_value=[]), \
|
||||
patch("app.services.detect_service.append_detect_result_projection_if_changed"):
|
||||
with ExitStack() as stack:
|
||||
stack.enter_context(patch.object(detect_service.settings, "node_region", "mainland"))
|
||||
stack.enter_context(patch.object(detect_service.settings, "node_role", "worker"))
|
||||
stack.enter_context(patch.object(detect_service.settings, "node_code", "mainland-worker-01"))
|
||||
stack.enter_context(patch("app.services.detect_service.ensure_runtime_schema"))
|
||||
stack.enter_context(patch("app.services.detect_service.get_db", side_effect=RuntimeError("db down")))
|
||||
stack.enter_context(
|
||||
patch("app.services.detect_service.get_settings_payload", return_value={"proxy_config": {"proxy_enable": True, "allow_direct": False, "proxy_urls": ["a"]}})
|
||||
)
|
||||
stack.enter_context(patch("app.services.detect_service.get_runtime_settings", return_value={"worker_log_sync_enabled": False, "worker_log_sync_mode": "full"}))
|
||||
stack.enter_context(patch("app.services.detect_service._load_recent_worker_lines", return_value=(True, "", [])))
|
||||
stack.enter_context(
|
||||
patch("app.services.detect_service.detect_worker_runtime", return_value={"mode": "linux-systemd", "running": True, "process_count": 1, "latest_start_time": "2026-04-20 23:58:00", "message": "active/running"})
|
||||
)
|
||||
stack.enter_context(patch("app.services.detect_service._load_runtime_state", return_value=runtime_state))
|
||||
stack.enter_context(patch("app.services.detect_service._load_runtime_state_from_cluster_node", return_value={}))
|
||||
stack.enter_context(patch("app.services.detect_service._extract_available_proxy_count", return_value=0))
|
||||
stack.enter_context(patch("app.services.detect_service._extract_active_thread_snapshot", return_value={"active": 0, "max": 0}))
|
||||
stack.enter_context(patch("app.services.detect_service._normalize_recent_warning", return_value=""))
|
||||
stack.enter_context(
|
||||
patch(
|
||||
"app.services.detect_service._build_proxy_runtime_snapshot",
|
||||
return_value={"state": "healthy", "label": "代理正常", "detail": "healthy", "direct_fallback_active": False, "reason": "healthy", "last_refresh_status": "ok", "last_refresh_time": "", "source_count": 2, "raw_items": 18, "validated_count": 18, "available_count": 18, "source_stats": [], "supplier_empty": False},
|
||||
)
|
||||
)
|
||||
stack.enter_context(
|
||||
patch("app.services.detect_service.resolve_thread_count", return_value={"effective_thread_count": 120, "default_thread_count": 5, "source": "node_override", "override_thread_count": 120, "node_code": "mainland-worker-01"})
|
||||
)
|
||||
stack.enter_context(patch("app.services.detect_service.get_active_detect_job_summary", side_effect=RuntimeError("db down")))
|
||||
stack.enter_context(patch("app.services.detect_service.sync_detect_runs", return_value=[]))
|
||||
stack.enter_context(patch("app.services.detect_service._resolve_remote_log_snapshot", return_value={}))
|
||||
stack.enter_context(patch("app.services.detect_service._extract_dependency_alerts", return_value=[]))
|
||||
stack.enter_context(patch("app.services.detect_service.append_detect_result_projection_if_changed"))
|
||||
payload = detect_service.get_detect_status()
|
||||
|
||||
self.assertTrue(payload["worker_online"])
|
||||
@@ -44,6 +96,56 @@ class DetectServiceStatusFallbackTests(unittest.TestCase):
|
||||
self.assertEqual(0, payload["progress"]["pending"])
|
||||
self.assertEqual(0, payload["progress"]["completed"])
|
||||
|
||||
def test_get_detect_status_skips_local_result_projection_on_overseas_control(self) -> None:
|
||||
runtime_state = {
|
||||
"service_running": False,
|
||||
"detecting": False,
|
||||
"active_threads": 0,
|
||||
"max_threads": 1,
|
||||
"phase": "idle",
|
||||
"detail": "当前节点不承载本地检测执行",
|
||||
"updated_at": "2026-04-24 02:10:00",
|
||||
"available_proxy_count": 0,
|
||||
}
|
||||
|
||||
with ExitStack() as stack:
|
||||
stack.enter_context(patch.object(detect_service.settings, "node_region", "overseas"))
|
||||
stack.enter_context(patch.object(detect_service.settings, "node_role", "control"))
|
||||
stack.enter_context(patch.object(detect_service.settings, "node_code", "overseas-control-01"))
|
||||
stack.enter_context(patch("app.services.detect_service.ensure_runtime_schema"))
|
||||
stack.enter_context(patch("app.services.detect_service.get_db", side_effect=RuntimeError("db down")))
|
||||
stack.enter_context(
|
||||
patch("app.services.detect_service.get_settings_payload", return_value={"proxy_config": {"proxy_enable": False, "allow_direct": True, "proxy_urls": []}})
|
||||
)
|
||||
stack.enter_context(patch("app.services.detect_service.get_runtime_settings", return_value={"worker_log_sync_enabled": False, "worker_log_sync_mode": "key"}))
|
||||
stack.enter_context(patch("app.services.detect_service._load_recent_worker_lines", return_value=(False, "", [])))
|
||||
stack.enter_context(
|
||||
patch("app.services.detect_service.detect_worker_runtime", return_value={"mode": "linux-systemd", "running": False, "process_count": 0, "latest_start_time": "", "message": "inactive"})
|
||||
)
|
||||
stack.enter_context(patch("app.services.detect_service._load_runtime_state", return_value=runtime_state))
|
||||
stack.enter_context(patch("app.services.detect_service._load_runtime_state_from_cluster_node", return_value={}))
|
||||
stack.enter_context(patch("app.services.detect_service._extract_available_proxy_count", return_value=0))
|
||||
stack.enter_context(patch("app.services.detect_service._extract_active_thread_snapshot", return_value={"active": 0, "max": 0}))
|
||||
stack.enter_context(patch("app.services.detect_service._normalize_recent_warning", return_value=""))
|
||||
stack.enter_context(
|
||||
patch(
|
||||
"app.services.detect_service._build_proxy_runtime_snapshot",
|
||||
return_value={"state": "disabled", "label": "不适用", "detail": "-", "direct_fallback_active": False, "reason": "not_applicable", "last_refresh_status": "", "last_refresh_time": "", "source_count": 0, "raw_items": 0, "validated_count": 0, "available_count": 0, "source_stats": [], "supplier_empty": False},
|
||||
)
|
||||
)
|
||||
stack.enter_context(
|
||||
patch("app.services.detect_service.resolve_thread_count", return_value={"effective_thread_count": 1, "default_thread_count": 1, "source": "default", "override_thread_count": None, "node_code": "overseas-control-01"})
|
||||
)
|
||||
stack.enter_context(patch("app.services.detect_service.get_active_detect_job_summary", return_value=None))
|
||||
stack.enter_context(patch("app.services.detect_service.sync_detect_runs", return_value=[]))
|
||||
stack.enter_context(patch("app.services.detect_service._resolve_remote_log_snapshot", return_value={}))
|
||||
stack.enter_context(patch("app.services.detect_service._extract_dependency_alerts", return_value=[]))
|
||||
mock_append = stack.enter_context(patch("app.services.detect_service.append_detect_result_projection_if_changed"))
|
||||
|
||||
detect_service.get_detect_status()
|
||||
|
||||
mock_append.assert_not_called()
|
||||
|
||||
def test_filter_lines_since_supports_journalctl_syslog_timestamps(self) -> None:
|
||||
lines = [
|
||||
"Apr 21 20:12:42 mainland-controller python[1]: 当前实际线程数量: 323/4",
|
||||
@@ -64,6 +166,784 @@ class DetectServiceStatusFallbackTests(unittest.TestCase):
|
||||
|
||||
self.assertEqual(lines, filtered)
|
||||
|
||||
def test_get_detect_status_uses_aggregate_capacity_for_overseas_control(self) -> None:
|
||||
active_job = {
|
||||
"job_id": 11,
|
||||
"job_code": "sync-overseas-51",
|
||||
"status": "running",
|
||||
"items_pending": 5001,
|
||||
"items_completed": 1788,
|
||||
"items_failed": 0,
|
||||
"items_blacklisted": 0,
|
||||
"progress_percent": 40.49,
|
||||
"display_items_running": 1432,
|
||||
"display_active_threads": 138,
|
||||
"distributed_node_stats": [
|
||||
{
|
||||
"node_code": "mainland-controller-01",
|
||||
"items_claimed": 73,
|
||||
"items_running": 1432,
|
||||
"display_running": 1432,
|
||||
"active_threads": 138,
|
||||
"max_threads": 2000,
|
||||
},
|
||||
{
|
||||
"node_code": "mainland-worker-01",
|
||||
"items_completed": 1614,
|
||||
"items_running": 0,
|
||||
"active_threads": 0,
|
||||
"max_threads": 0,
|
||||
},
|
||||
{
|
||||
"node_code": "unassigned",
|
||||
"items_pending": 3495,
|
||||
},
|
||||
],
|
||||
}
|
||||
aggregate_queue_health = {
|
||||
"has_active_job": True,
|
||||
"job": {
|
||||
"job_id": 11,
|
||||
"job_code": "sync-overseas-51",
|
||||
"status": "running",
|
||||
"progress_percent": 40.49,
|
||||
},
|
||||
"queue": {
|
||||
"items_total": 8402,
|
||||
"pending": 5001,
|
||||
"claimed": 0,
|
||||
"running": 5909,
|
||||
"display_running": 5909,
|
||||
"completed": 0,
|
||||
"blacklisted": 0,
|
||||
"failed": 0,
|
||||
},
|
||||
"nodes": [
|
||||
{
|
||||
"node_code": "mainland-controller-01-a",
|
||||
"items_running": 1000,
|
||||
"display_running": 1000,
|
||||
"active_threads": 1000,
|
||||
"max_threads": 1000,
|
||||
},
|
||||
{
|
||||
"node_code": "mainland-controller-01-b",
|
||||
"items_running": 972,
|
||||
"display_running": 972,
|
||||
"active_threads": 972,
|
||||
"max_threads": 1000,
|
||||
},
|
||||
{
|
||||
"node_code": "mainland-controller-01-c",
|
||||
"items_running": 3937,
|
||||
"display_running": 3937,
|
||||
"active_threads": 3937,
|
||||
"max_threads": 36000,
|
||||
},
|
||||
],
|
||||
}
|
||||
|
||||
def _resolve_thread_count(*, node_code=None, settings_payload=None):
|
||||
if node_code == "mainland-controller-01" or str(node_code or "").startswith("mainland-controller-01-"):
|
||||
return {
|
||||
"effective_thread_count": 1000,
|
||||
"default_thread_count": 1000,
|
||||
"source": "default",
|
||||
"override_thread_count": None,
|
||||
"node_code": str(node_code or "mainland-controller-01"),
|
||||
}
|
||||
return {
|
||||
"effective_thread_count": 1,
|
||||
"default_thread_count": 1000,
|
||||
"source": "node_override",
|
||||
"override_thread_count": 1,
|
||||
"node_code": "overseas-control-01",
|
||||
}
|
||||
|
||||
def _resolve_process_count(*, node_code=None, settings_payload=None):
|
||||
if node_code == "mainland-controller-01":
|
||||
return {
|
||||
"effective_process_count": 80,
|
||||
"default_process_count": 80,
|
||||
"source": "default",
|
||||
"override_process_count": None,
|
||||
"node_code": "mainland-controller-01",
|
||||
}
|
||||
if str(node_code or "").startswith("mainland-controller-01-"):
|
||||
return {
|
||||
"effective_process_count": 1,
|
||||
"default_process_count": 80,
|
||||
"source": "child_instance",
|
||||
"override_process_count": None,
|
||||
"node_code": str(node_code or ""),
|
||||
}
|
||||
return {
|
||||
"effective_process_count": 1,
|
||||
"default_process_count": 80,
|
||||
"source": "node_override",
|
||||
"override_process_count": 1,
|
||||
"node_code": "overseas-control-01",
|
||||
}
|
||||
|
||||
with ExitStack() as stack:
|
||||
stack.enter_context(patch.object(detect_service.settings, "node_region", "overseas"))
|
||||
stack.enter_context(patch.object(detect_service.settings, "node_role", "control"))
|
||||
stack.enter_context(patch.object(detect_service.settings, "node_code", "overseas-control-01"))
|
||||
stack.enter_context(patch("app.services.detect_service.ensure_runtime_schema"))
|
||||
stack.enter_context(patch("app.services.detect_service.get_db", side_effect=RuntimeError("db down")))
|
||||
stack.enter_context(
|
||||
patch(
|
||||
"app.services.detect_service.get_settings_payload",
|
||||
return_value={
|
||||
"proxy_config": {"proxy_enable": False, "allow_direct": True, "proxy_urls": []},
|
||||
"process_count": 80,
|
||||
"node_process_counts": {},
|
||||
"thread_count": 1000,
|
||||
"node_thread_counts": {"overseas-control-01": 1},
|
||||
},
|
||||
)
|
||||
)
|
||||
stack.enter_context(
|
||||
patch("app.services.detect_service.get_runtime_settings", return_value={"worker_log_sync_enabled": False, "worker_log_sync_mode": "key"})
|
||||
)
|
||||
stack.enter_context(
|
||||
patch("app.services.detect_service._load_recent_worker_lines", return_value=(False, "2026-04-23 00:00:00", ["stale line"]))
|
||||
)
|
||||
stack.enter_context(
|
||||
patch("app.services.detect_service.detect_worker_runtime", return_value={"mode": "linux-systemd", "running": False, "process_count": 0, "latest_start_time": "", "message": "inactive"})
|
||||
)
|
||||
stack.enter_context(
|
||||
patch("app.services.detect_service._load_runtime_state", return_value={"service_running": True, "detecting": True, "active_threads": 2, "max_threads": 1, "detail": "当前实际线程数量: 2/1"})
|
||||
)
|
||||
stack.enter_context(patch("app.services.detect_service._load_runtime_state_from_cluster_node", return_value={}))
|
||||
stack.enter_context(patch("app.services.detect_service._extract_available_proxy_count", return_value=0))
|
||||
stack.enter_context(patch("app.services.detect_service._extract_active_thread_snapshot", return_value={"active": 0, "max": 0}))
|
||||
stack.enter_context(patch("app.services.detect_service._normalize_recent_warning", return_value=""))
|
||||
stack.enter_context(
|
||||
patch(
|
||||
"app.services.detect_service._build_proxy_runtime_snapshot",
|
||||
return_value={
|
||||
"state": "disabled",
|
||||
"label": "未启用代理",
|
||||
"detail": "-",
|
||||
"direct_fallback_active": True,
|
||||
"reason": "proxy_disabled",
|
||||
"last_refresh_status": "",
|
||||
"last_refresh_time": "",
|
||||
"source_count": 0,
|
||||
"raw_items": 0,
|
||||
"validated_count": 0,
|
||||
"available_count": 0,
|
||||
"source_stats": [],
|
||||
"supplier_empty": False,
|
||||
},
|
||||
)
|
||||
)
|
||||
stack.enter_context(patch("app.services.detect_service.resolve_thread_count", side_effect=_resolve_thread_count))
|
||||
stack.enter_context(patch("app.services.detect_service.resolve_process_count", side_effect=_resolve_process_count))
|
||||
stack.enter_context(patch("app.services.detect_service.get_active_detect_job_summary", return_value=active_job))
|
||||
stack.enter_context(patch("app.services.detect_service.get_detect_queue_health", return_value=aggregate_queue_health))
|
||||
stack.enter_context(
|
||||
patch(
|
||||
"app.services.detect_service._load_runtime_states_from_cluster_nodes",
|
||||
return_value={
|
||||
"mainland-controller-01": {
|
||||
"node_code": "mainland-controller-01",
|
||||
"available_proxy_count": 486,
|
||||
"proxy_runtime_label": "代理正常",
|
||||
"proxy_runtime_reason": "healthy",
|
||||
"proxy_last_refresh_status": "复用共享代理快照 486 个",
|
||||
"proxy_last_refresh_time": "2026-04-23 19:07:12",
|
||||
"proxy_last_refresh_source_count": 6,
|
||||
"proxy_last_refresh_total_items": 120,
|
||||
"proxy_last_validated_count": 0,
|
||||
}
|
||||
},
|
||||
)
|
||||
)
|
||||
stack.enter_context(patch("app.services.detect_service.sync_detect_runs", return_value=[]))
|
||||
stack.enter_context(patch("app.services.detect_service._resolve_remote_log_snapshot", return_value={}))
|
||||
stack.enter_context(patch("app.services.detect_service._extract_dependency_alerts", return_value=[]))
|
||||
stack.enter_context(patch("app.services.detect_service.append_detect_result_projection_if_changed"))
|
||||
payload = detect_service.get_detect_status()
|
||||
|
||||
self.assertFalse(payload["worker_online"])
|
||||
self.assertEqual({}, payload["runtime_state"])
|
||||
self.assertEqual(3, payload["worker_process_count"])
|
||||
self.assertEqual(3, payload["aggregate_process_count"])
|
||||
self.assertEqual(3, payload["aggregate_participating_node_count"])
|
||||
self.assertEqual(
|
||||
["mainland-controller-01-a", "mainland-controller-01-b", "mainland-controller-01-c"],
|
||||
payload["aggregate_participating_node_codes"],
|
||||
)
|
||||
self.assertEqual(3000, payload["aggregate_max_thread_count"])
|
||||
self.assertEqual(38000, payload["max_thread_count"])
|
||||
self.assertEqual(5909, payload["active_thread_count"])
|
||||
self.assertEqual(486, payload["available_proxy_count"])
|
||||
self.assertEqual("集群代理正常", payload["proxy_runtime_label"])
|
||||
self.assertIn("486", payload["proxy_last_refresh_status"])
|
||||
|
||||
def test_get_detect_status_uses_queue_running_when_aggregate_display_fields_are_zero(self) -> None:
|
||||
active_job = {
|
||||
"job_id": 60,
|
||||
"job_code": "sync-overseas-255",
|
||||
"status": "running",
|
||||
"items_total": 8402,
|
||||
"items_pending": 7865,
|
||||
"items_running": 537,
|
||||
"items_completed": 0,
|
||||
"items_failed": 0,
|
||||
"items_blacklisted": 0,
|
||||
"progress_percent": 6.39,
|
||||
"display_items_running": 0,
|
||||
"display_active_threads": 0,
|
||||
"display_max_threads": 0,
|
||||
"distributed_node_stats": [],
|
||||
}
|
||||
aggregate_queue_health = {
|
||||
"has_active_job": True,
|
||||
"job": {
|
||||
"job_id": 60,
|
||||
"job_code": "sync-overseas-255",
|
||||
"status": "running",
|
||||
"progress_percent": 6.39,
|
||||
},
|
||||
"queue": {
|
||||
"items_total": 8402,
|
||||
"pending": 7865,
|
||||
"claimed": 0,
|
||||
"running": 537,
|
||||
"display_running": 0,
|
||||
"completed": 0,
|
||||
"blacklisted": 0,
|
||||
"failed": 0,
|
||||
},
|
||||
"nodes": [
|
||||
{
|
||||
"node_code": "mainland-controller-01-a",
|
||||
"items_running": 537,
|
||||
"display_running": 537,
|
||||
"active_threads": 537,
|
||||
"max_threads": 1000,
|
||||
}
|
||||
],
|
||||
}
|
||||
|
||||
with ExitStack() as stack:
|
||||
stack.enter_context(patch.object(detect_service.settings, "node_region", "overseas"))
|
||||
stack.enter_context(patch.object(detect_service.settings, "node_role", "control"))
|
||||
stack.enter_context(patch.object(detect_service.settings, "node_code", "overseas-control-01"))
|
||||
stack.enter_context(patch("app.services.detect_service.ensure_runtime_schema"))
|
||||
stack.enter_context(patch("app.services.detect_service.get_db", side_effect=RuntimeError("db down")))
|
||||
stack.enter_context(
|
||||
patch(
|
||||
"app.services.detect_service.get_settings_payload",
|
||||
return_value={
|
||||
"proxy_config": {"proxy_enable": False, "allow_direct": True, "proxy_urls": []},
|
||||
"process_count": 80,
|
||||
"node_process_counts": {"mainland-controller-01": 24},
|
||||
"thread_count": 1000,
|
||||
"node_thread_counts": {"overseas-control-01": 1, "mainland-controller-01-a": 1000},
|
||||
},
|
||||
)
|
||||
)
|
||||
stack.enter_context(
|
||||
patch("app.services.detect_service.get_runtime_settings", return_value={"worker_log_sync_enabled": False, "worker_log_sync_mode": "key"})
|
||||
)
|
||||
stack.enter_context(
|
||||
patch("app.services.detect_service._load_recent_worker_lines", return_value=(False, "2026-04-23 00:00:00", ["stale line"]))
|
||||
)
|
||||
stack.enter_context(
|
||||
patch("app.services.detect_service.detect_worker_runtime", return_value={"mode": "linux-systemd", "running": False, "process_count": 0, "latest_start_time": "", "message": "inactive"})
|
||||
)
|
||||
stack.enter_context(
|
||||
patch("app.services.detect_service._load_runtime_state", return_value={"service_running": False, "detecting": False, "active_threads": 0, "max_threads": 1, "detail": ""})
|
||||
)
|
||||
stack.enter_context(patch("app.services.detect_service._load_runtime_state_from_cluster_node", return_value={}))
|
||||
stack.enter_context(patch("app.services.detect_service._extract_available_proxy_count", return_value=0))
|
||||
stack.enter_context(patch("app.services.detect_service._extract_active_thread_snapshot", return_value={"active": 0, "max": 0}))
|
||||
stack.enter_context(patch("app.services.detect_service._normalize_recent_warning", return_value=""))
|
||||
stack.enter_context(
|
||||
patch(
|
||||
"app.services.detect_service._build_proxy_runtime_snapshot",
|
||||
return_value={
|
||||
"state": "disabled",
|
||||
"label": "未启用代理",
|
||||
"detail": "-",
|
||||
"direct_fallback_active": True,
|
||||
"reason": "proxy_disabled",
|
||||
"last_refresh_status": "",
|
||||
"last_refresh_time": "",
|
||||
"source_count": 0,
|
||||
"raw_items": 0,
|
||||
"validated_count": 0,
|
||||
"available_count": 0,
|
||||
"source_stats": [],
|
||||
"supplier_empty": False,
|
||||
},
|
||||
)
|
||||
)
|
||||
stack.enter_context(
|
||||
patch(
|
||||
"app.services.detect_service.resolve_thread_count",
|
||||
return_value={
|
||||
"effective_thread_count": 1,
|
||||
"default_thread_count": 1000,
|
||||
"source": "node_override",
|
||||
"override_thread_count": 1,
|
||||
"node_code": "overseas-control-01",
|
||||
},
|
||||
)
|
||||
)
|
||||
stack.enter_context(
|
||||
patch(
|
||||
"app.services.detect_service.resolve_process_count",
|
||||
return_value={
|
||||
"effective_process_count": 1,
|
||||
"default_process_count": 80,
|
||||
"source": "node_override",
|
||||
"override_process_count": 1,
|
||||
"node_code": "overseas-control-01",
|
||||
},
|
||||
)
|
||||
)
|
||||
stack.enter_context(patch("app.services.detect_service.get_active_detect_job_summary", return_value=active_job))
|
||||
stack.enter_context(patch("app.services.detect_service.get_detect_queue_health", return_value=aggregate_queue_health))
|
||||
stack.enter_context(patch("app.services.detect_service.sync_detect_runs", return_value=[]))
|
||||
stack.enter_context(patch("app.services.detect_service._resolve_remote_log_snapshot", return_value={}))
|
||||
stack.enter_context(patch("app.services.detect_service._extract_dependency_alerts", return_value=[]))
|
||||
stack.enter_context(patch("app.services.detect_service.append_detect_result_projection_if_changed"))
|
||||
payload = detect_service.get_detect_status()
|
||||
|
||||
self.assertEqual(537, payload["progress"]["running"])
|
||||
self.assertEqual(537, payload["active_thread_count"])
|
||||
self.assertEqual(1000, payload["max_thread_count"])
|
||||
self.assertEqual(1, payload["aggregate_process_count"])
|
||||
|
||||
def test_get_detect_status_drops_stale_aggregate_nodes_from_running_and_process_counts(self) -> None:
|
||||
live_heartbeat = datetime.now().isoformat(sep=" ", timespec="seconds")
|
||||
stale_heartbeat = (datetime.now() - timedelta(minutes=8)).isoformat(sep=" ", timespec="seconds")
|
||||
active_job = {
|
||||
"job_id": 88,
|
||||
"job_code": "sync-overseas-688",
|
||||
"status": "running",
|
||||
"items_total": 2600,
|
||||
"items_pending": 500,
|
||||
"items_running": 2100,
|
||||
"items_completed": 0,
|
||||
"items_failed": 0,
|
||||
"items_blacklisted": 0,
|
||||
"progress_percent": 80.77,
|
||||
"display_items_running": 2100,
|
||||
"display_active_threads": 2100,
|
||||
"display_max_threads": 2000,
|
||||
"distributed_node_stats": [
|
||||
{
|
||||
"node_code": "mainland-controller-01-a",
|
||||
"items_running": 300,
|
||||
"display_running": 300,
|
||||
"active_threads": 300,
|
||||
"max_threads": 1000,
|
||||
"status": "busy",
|
||||
"last_heartbeat_at": live_heartbeat,
|
||||
},
|
||||
{
|
||||
"node_code": "mainland-controller-01-b",
|
||||
"items_running": 1800,
|
||||
"display_running": 1800,
|
||||
"active_threads": 1800,
|
||||
"max_threads": 1000,
|
||||
"status": "stale",
|
||||
"last_heartbeat_at": stale_heartbeat,
|
||||
},
|
||||
],
|
||||
}
|
||||
aggregate_queue_health = {
|
||||
"has_active_job": True,
|
||||
"job": {
|
||||
"job_id": 88,
|
||||
"job_code": "sync-overseas-688",
|
||||
"status": "running",
|
||||
"progress_percent": 80.77,
|
||||
},
|
||||
"queue": {
|
||||
"items_total": 2600,
|
||||
"pending": 500,
|
||||
"claimed": 0,
|
||||
"running": 2100,
|
||||
"display_running": 2100,
|
||||
"completed": 0,
|
||||
"blacklisted": 0,
|
||||
"failed": 0,
|
||||
},
|
||||
"nodes": [
|
||||
{
|
||||
"node_code": "mainland-controller-01-a",
|
||||
"items_running": 300,
|
||||
"display_running": 300,
|
||||
"active_threads": 300,
|
||||
"max_threads": 1000,
|
||||
"status": "busy",
|
||||
"last_heartbeat_at": live_heartbeat,
|
||||
},
|
||||
{
|
||||
"node_code": "mainland-controller-01-b",
|
||||
"items_running": 1800,
|
||||
"display_running": 1800,
|
||||
"active_threads": 1800,
|
||||
"max_threads": 1000,
|
||||
"status": "stale",
|
||||
"last_heartbeat_at": stale_heartbeat,
|
||||
},
|
||||
],
|
||||
}
|
||||
|
||||
with ExitStack() as stack:
|
||||
stack.enter_context(patch.object(detect_service.settings, "node_region", "overseas"))
|
||||
stack.enter_context(patch.object(detect_service.settings, "node_role", "control"))
|
||||
stack.enter_context(patch.object(detect_service.settings, "node_code", "overseas-control-01"))
|
||||
stack.enter_context(patch("app.services.detect_service.ensure_runtime_schema"))
|
||||
stack.enter_context(patch("app.services.detect_service.get_db", side_effect=RuntimeError("db down")))
|
||||
stack.enter_context(
|
||||
patch(
|
||||
"app.services.detect_service.get_settings_payload",
|
||||
return_value={
|
||||
"proxy_config": {"proxy_enable": False, "allow_direct": True, "proxy_urls": []},
|
||||
"process_count": 80,
|
||||
"node_process_counts": {"mainland-controller-01": 70},
|
||||
"thread_count": 1000,
|
||||
"node_thread_counts": {"overseas-control-01": 1},
|
||||
},
|
||||
)
|
||||
)
|
||||
stack.enter_context(
|
||||
patch("app.services.detect_service.get_runtime_settings", return_value={"worker_log_sync_enabled": False, "worker_log_sync_mode": "key"})
|
||||
)
|
||||
stack.enter_context(
|
||||
patch("app.services.detect_service._load_recent_worker_lines", return_value=(False, "2026-04-23 00:00:00", ["stale line"]))
|
||||
)
|
||||
stack.enter_context(
|
||||
patch("app.services.detect_service.detect_worker_runtime", return_value={"mode": "linux-systemd", "running": False, "process_count": 0, "latest_start_time": "", "message": "inactive"})
|
||||
)
|
||||
stack.enter_context(
|
||||
patch("app.services.detect_service._load_runtime_state", return_value={"service_running": False, "detecting": False, "active_threads": 0, "max_threads": 1, "detail": ""})
|
||||
)
|
||||
stack.enter_context(patch("app.services.detect_service._load_runtime_state_from_cluster_node", return_value={}))
|
||||
stack.enter_context(patch("app.services.detect_service._extract_available_proxy_count", return_value=0))
|
||||
stack.enter_context(patch("app.services.detect_service._extract_active_thread_snapshot", return_value={"active": 0, "max": 0}))
|
||||
stack.enter_context(patch("app.services.detect_service._normalize_recent_warning", return_value=""))
|
||||
stack.enter_context(
|
||||
patch(
|
||||
"app.services.detect_service._build_proxy_runtime_snapshot",
|
||||
return_value={
|
||||
"state": "disabled",
|
||||
"label": "未启用代理",
|
||||
"detail": "-",
|
||||
"direct_fallback_active": True,
|
||||
"reason": "proxy_disabled",
|
||||
"last_refresh_status": "",
|
||||
"last_refresh_time": "",
|
||||
"source_count": 0,
|
||||
"raw_items": 0,
|
||||
"validated_count": 0,
|
||||
"available_count": 0,
|
||||
"source_stats": [],
|
||||
"supplier_empty": False,
|
||||
},
|
||||
)
|
||||
)
|
||||
stack.enter_context(
|
||||
patch(
|
||||
"app.services.detect_service.resolve_thread_count",
|
||||
return_value={
|
||||
"effective_thread_count": 1,
|
||||
"default_thread_count": 1000,
|
||||
"source": "node_override",
|
||||
"override_thread_count": 1,
|
||||
"node_code": "overseas-control-01",
|
||||
},
|
||||
)
|
||||
)
|
||||
stack.enter_context(
|
||||
patch(
|
||||
"app.services.detect_service.resolve_process_count",
|
||||
side_effect=[
|
||||
{
|
||||
"effective_process_count": 1,
|
||||
"default_process_count": 80,
|
||||
"source": "child_instance",
|
||||
"override_process_count": None,
|
||||
"node_code": "mainland-controller-01-a",
|
||||
}
|
||||
],
|
||||
)
|
||||
)
|
||||
stack.enter_context(patch("app.services.detect_service.get_active_detect_job_summary", return_value=active_job))
|
||||
stack.enter_context(patch("app.services.detect_service.get_detect_queue_health", return_value=aggregate_queue_health))
|
||||
stack.enter_context(patch("app.services.detect_service.sync_detect_runs", return_value=[]))
|
||||
stack.enter_context(patch("app.services.detect_service._resolve_remote_log_snapshot", return_value={}))
|
||||
stack.enter_context(patch("app.services.detect_service._extract_dependency_alerts", return_value=[]))
|
||||
stack.enter_context(patch("app.services.detect_service.append_detect_result_projection_if_changed"))
|
||||
payload = detect_service.get_detect_status()
|
||||
|
||||
self.assertEqual(300, payload["progress"]["running"])
|
||||
self.assertEqual(300, payload["active_thread_count"])
|
||||
self.assertEqual(1000, payload["max_thread_count"])
|
||||
self.assertEqual(1, payload["aggregate_process_count"])
|
||||
self.assertEqual(1, payload["aggregate_participating_node_count"])
|
||||
self.assertEqual(["mainland-controller-01-a"], payload["aggregate_participating_node_codes"])
|
||||
|
||||
def test_get_detect_status_uses_event_proxy_counts_when_cluster_runtime_proxy_counts_are_missing(self) -> None:
|
||||
active_job = {
|
||||
"job_id": 77,
|
||||
"job_code": "sync-overseas-482",
|
||||
"status": "running",
|
||||
"items_pending": 410,
|
||||
"items_completed": 100,
|
||||
"items_failed": 0,
|
||||
"items_blacklisted": 0,
|
||||
"progress_percent": 55.0,
|
||||
"display_items_running": 807,
|
||||
"display_active_threads": 807,
|
||||
"current_cycle_token": "cycle-1",
|
||||
"distributed_node_stats": [
|
||||
{
|
||||
"node_code": "mainland-controller-01-u",
|
||||
"items_running": 300,
|
||||
"display_running": 300,
|
||||
"active_threads": 300,
|
||||
"max_threads": 1000,
|
||||
},
|
||||
{
|
||||
"node_code": "mainland-controller-01-v",
|
||||
"items_running": 301,
|
||||
"display_running": 301,
|
||||
"active_threads": 301,
|
||||
"max_threads": 1000,
|
||||
},
|
||||
{
|
||||
"node_code": "mainland-worker-01-a",
|
||||
"items_running": 206,
|
||||
"display_running": 206,
|
||||
"active_threads": 206,
|
||||
"max_threads": 1000,
|
||||
},
|
||||
],
|
||||
"current_cycle_events": [
|
||||
{
|
||||
"event_type": "worker_log",
|
||||
"created_at": "2026-04-23 19:40:00",
|
||||
"node_code": "mainland-controller-01-u",
|
||||
"message": "当前可用代理数: 486",
|
||||
"payload": {"cycle_token": "cycle-1", "log_mode": "key"},
|
||||
},
|
||||
{
|
||||
"event_type": "worker_log",
|
||||
"created_at": "2026-04-23 19:40:10",
|
||||
"node_code": "mainland-controller-01-v",
|
||||
"message": "代理池刷新完成,共 486 个可用代理,来源链接 6 个,原始 520 个",
|
||||
"payload": {"cycle_token": "cycle-1", "log_mode": "key"},
|
||||
},
|
||||
{
|
||||
"event_type": "worker_log",
|
||||
"created_at": "2026-04-23 19:40:20",
|
||||
"node_code": "mainland-worker-01-a",
|
||||
"message": "共享刷新进行中,继续沿用缓存 321 个",
|
||||
"payload": {"cycle_token": "cycle-1", "log_mode": "key"},
|
||||
},
|
||||
],
|
||||
}
|
||||
aggregate_queue_health = {
|
||||
"has_active_job": True,
|
||||
"job": {
|
||||
"job_id": 77,
|
||||
"job_code": "sync-overseas-482",
|
||||
"status": "running",
|
||||
"progress_percent": 55.0,
|
||||
},
|
||||
"queue": {
|
||||
"items_total": 1317,
|
||||
"pending": 410,
|
||||
"claimed": 0,
|
||||
"running": 807,
|
||||
"display_running": 807,
|
||||
"completed": 100,
|
||||
"blacklisted": 0,
|
||||
"failed": 0,
|
||||
},
|
||||
"nodes": [
|
||||
{
|
||||
"node_code": "mainland-controller-01-u",
|
||||
"items_running": 300,
|
||||
"display_running": 300,
|
||||
"active_threads": 300,
|
||||
"max_threads": 1000,
|
||||
},
|
||||
{
|
||||
"node_code": "mainland-controller-01-v",
|
||||
"items_running": 301,
|
||||
"display_running": 301,
|
||||
"active_threads": 301,
|
||||
"max_threads": 1000,
|
||||
},
|
||||
{
|
||||
"node_code": "mainland-worker-01-a",
|
||||
"items_running": 206,
|
||||
"display_running": 206,
|
||||
"active_threads": 206,
|
||||
"max_threads": 1000,
|
||||
},
|
||||
],
|
||||
}
|
||||
|
||||
def _resolve_thread_count(*, node_code=None, settings_payload=None):
|
||||
if str(node_code or "").startswith("mainland-"):
|
||||
return {
|
||||
"effective_thread_count": 1000,
|
||||
"default_thread_count": 1000,
|
||||
"source": "default",
|
||||
"override_thread_count": None,
|
||||
"node_code": str(node_code or ""),
|
||||
}
|
||||
return {
|
||||
"effective_thread_count": 1,
|
||||
"default_thread_count": 1000,
|
||||
"source": "node_override",
|
||||
"override_thread_count": 1,
|
||||
"node_code": "overseas-control-01",
|
||||
}
|
||||
|
||||
def _resolve_process_count(*, node_code=None, settings_payload=None):
|
||||
normalized_node_code = str(node_code or "")
|
||||
if normalized_node_code == "mainland-controller-01":
|
||||
return {
|
||||
"effective_process_count": 80,
|
||||
"default_process_count": 80,
|
||||
"source": "default",
|
||||
"override_process_count": None,
|
||||
"node_code": normalized_node_code,
|
||||
}
|
||||
if normalized_node_code == "mainland-worker-01":
|
||||
return {
|
||||
"effective_process_count": 60,
|
||||
"default_process_count": 60,
|
||||
"source": "default",
|
||||
"override_process_count": None,
|
||||
"node_code": normalized_node_code,
|
||||
}
|
||||
if normalized_node_code.startswith("mainland-"):
|
||||
return {
|
||||
"effective_process_count": 1,
|
||||
"default_process_count": 80,
|
||||
"source": "child_instance",
|
||||
"override_process_count": None,
|
||||
"node_code": normalized_node_code,
|
||||
}
|
||||
return {
|
||||
"effective_process_count": 1,
|
||||
"default_process_count": 80,
|
||||
"source": "node_override",
|
||||
"override_process_count": 1,
|
||||
"node_code": "overseas-control-01",
|
||||
}
|
||||
|
||||
with ExitStack() as stack:
|
||||
stack.enter_context(patch.object(detect_service.settings, "node_region", "overseas"))
|
||||
stack.enter_context(patch.object(detect_service.settings, "node_role", "control"))
|
||||
stack.enter_context(patch.object(detect_service.settings, "node_code", "overseas-control-01"))
|
||||
stack.enter_context(patch("app.services.detect_service.ensure_runtime_schema"))
|
||||
stack.enter_context(patch("app.services.detect_service.get_db", side_effect=RuntimeError("db down")))
|
||||
stack.enter_context(
|
||||
patch(
|
||||
"app.services.detect_service.get_settings_payload",
|
||||
return_value={
|
||||
"proxy_config": {"proxy_enable": False, "allow_direct": True, "proxy_urls": []},
|
||||
"process_count": 80,
|
||||
"node_process_counts": {"mainland-worker-01": 60},
|
||||
"thread_count": 1000,
|
||||
"node_thread_counts": {"overseas-control-01": 1},
|
||||
},
|
||||
)
|
||||
)
|
||||
stack.enter_context(
|
||||
patch("app.services.detect_service.get_runtime_settings", return_value={"worker_log_sync_enabled": False, "worker_log_sync_mode": "key"})
|
||||
)
|
||||
stack.enter_context(
|
||||
patch("app.services.detect_service._load_recent_worker_lines", return_value=(False, "2026-04-23 00:00:00", ["stale line"]))
|
||||
)
|
||||
stack.enter_context(
|
||||
patch("app.services.detect_service.detect_worker_runtime", return_value={"mode": "linux-systemd", "running": False, "process_count": 0, "latest_start_time": "", "message": "inactive"})
|
||||
)
|
||||
stack.enter_context(
|
||||
patch("app.services.detect_service._load_runtime_state", return_value={"service_running": False, "detecting": False, "active_threads": 0, "max_threads": 1, "detail": ""})
|
||||
)
|
||||
stack.enter_context(patch("app.services.detect_service._load_runtime_state_from_cluster_node", return_value={}))
|
||||
stack.enter_context(patch("app.services.detect_service._extract_available_proxy_count", return_value=0))
|
||||
stack.enter_context(patch("app.services.detect_service._extract_active_thread_snapshot", return_value={"active": 0, "max": 0}))
|
||||
stack.enter_context(patch("app.services.detect_service._normalize_recent_warning", return_value=""))
|
||||
stack.enter_context(
|
||||
patch(
|
||||
"app.services.detect_service._build_proxy_runtime_snapshot",
|
||||
return_value={
|
||||
"state": "warming_up",
|
||||
"label": "等待首刷",
|
||||
"detail": "-",
|
||||
"direct_fallback_active": False,
|
||||
"reason": "waiting_for_first_refresh",
|
||||
"last_refresh_status": "未刷新",
|
||||
"last_refresh_time": "",
|
||||
"source_count": 0,
|
||||
"raw_items": 0,
|
||||
"validated_count": 0,
|
||||
"available_count": 0,
|
||||
"source_stats": [],
|
||||
"supplier_empty": False,
|
||||
},
|
||||
)
|
||||
)
|
||||
stack.enter_context(patch("app.services.detect_service.resolve_thread_count", side_effect=_resolve_thread_count))
|
||||
stack.enter_context(patch("app.services.detect_service.resolve_process_count", side_effect=_resolve_process_count))
|
||||
stack.enter_context(patch("app.services.detect_service.get_active_detect_job_summary", return_value=active_job))
|
||||
stack.enter_context(patch("app.services.detect_service.get_detect_queue_health", return_value=aggregate_queue_health))
|
||||
stack.enter_context(
|
||||
patch(
|
||||
"app.services.detect_service._load_runtime_states_from_cluster_nodes",
|
||||
return_value={
|
||||
"mainland-controller-01": {
|
||||
"node_code": "mainland-controller-01",
|
||||
"available_proxy_count": 0,
|
||||
"proxy_runtime_label": "",
|
||||
"proxy_runtime_reason": "",
|
||||
"proxy_last_refresh_status": "",
|
||||
"proxy_last_refresh_time": "",
|
||||
"proxy_last_refresh_source_count": 0,
|
||||
"proxy_last_refresh_total_items": 0,
|
||||
"proxy_last_validated_count": 0,
|
||||
},
|
||||
"mainland-worker-01": {
|
||||
"node_code": "mainland-worker-01",
|
||||
"available_proxy_count": 0,
|
||||
"proxy_runtime_label": "",
|
||||
"proxy_runtime_reason": "",
|
||||
"proxy_last_refresh_status": "",
|
||||
"proxy_last_refresh_time": "",
|
||||
"proxy_last_refresh_source_count": 0,
|
||||
"proxy_last_refresh_total_items": 0,
|
||||
"proxy_last_validated_count": 0,
|
||||
},
|
||||
},
|
||||
)
|
||||
)
|
||||
stack.enter_context(patch("app.services.detect_service.sync_detect_runs", return_value=[]))
|
||||
stack.enter_context(patch("app.services.detect_service._resolve_remote_log_snapshot", return_value={}))
|
||||
stack.enter_context(patch("app.services.detect_service._extract_dependency_alerts", return_value=[]))
|
||||
stack.enter_context(patch("app.services.detect_service.append_detect_result_projection_if_changed"))
|
||||
payload = detect_service.get_detect_status()
|
||||
|
||||
self.assertEqual(807, payload["available_proxy_count"])
|
||||
self.assertEqual("集群代理正常", payload["proxy_runtime_label"])
|
||||
self.assertIn("参与服务器 2 台", payload["proxy_runtime_detail"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
143
domain-api/tests/test_domains_service.py
Normal file
143
domain-api/tests/test_domains_service.py
Normal file
@@ -0,0 +1,143 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
from app.services import domains_service
|
||||
|
||||
|
||||
class _FakeDomainsCursor:
|
||||
def __init__(self) -> None:
|
||||
self._fetchone_result = None
|
||||
self._fetchall_result = []
|
||||
self.executed: list[tuple[str, tuple]] = []
|
||||
self.updated_detection_params = None
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, exc_type, exc, tb):
|
||||
return False
|
||||
|
||||
def execute(self, sql: str, params=None) -> None:
|
||||
normalized_sql = " ".join(str(sql or "").split()).lower()
|
||||
tuple_params = tuple(params or ())
|
||||
self.executed.append((normalized_sql, tuple_params))
|
||||
if normalized_sql.startswith("select count(*)"):
|
||||
self._fetchone_result = (1,)
|
||||
return
|
||||
if normalized_sql.startswith("select d.id, d.domain"):
|
||||
self._fetchall_result = [
|
||||
(
|
||||
1,
|
||||
"a.com",
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
0,
|
||||
1,
|
||||
"",
|
||||
None,
|
||||
"",
|
||||
0,
|
||||
None,
|
||||
7,
|
||||
False,
|
||||
None,
|
||||
"",
|
||||
{"status": True, "state": "passed"},
|
||||
{"status": True, "state": "passed"},
|
||||
True,
|
||||
{"status": True, "state": "passed"},
|
||||
{"status": True, "state": "passed"},
|
||||
{"status": True, "state": "passed"},
|
||||
{"status": True, "state": "passed"},
|
||||
{"status": True, "state": "passed"},
|
||||
{"status": False, "state": "failed", "message": "juziseo failed"},
|
||||
{"status": False, "state": "blacklisted", "message": "jucha blacklisted"},
|
||||
)
|
||||
]
|
||||
return
|
||||
if normalized_sql.startswith("update domains set"):
|
||||
return
|
||||
if normalized_sql.startswith("select id, baidu_history"):
|
||||
self._fetchone_result = (
|
||||
7,
|
||||
{"status": False, "state": "failed", "message": "timeout", "checked_at": "2026-04-20 12:00:00", "step": "baidu_site"},
|
||||
{"status": True, "state": "passed", "message": "ok", "checked_at": "2026-04-20 12:00:00", "step": "baidu_site"},
|
||||
False,
|
||||
{"status": False, "state": "failed", "message": "old", "checked_at": "2026-04-20 12:00:00", "step": "qihu360_site"},
|
||||
{"status": False, "state": "failed", "message": "old", "checked_at": "2026-04-20 12:00:00", "step": "google_site"},
|
||||
False,
|
||||
)
|
||||
return
|
||||
if normalized_sql.startswith("update domain_detections set"):
|
||||
self.updated_detection_params = tuple_params
|
||||
return
|
||||
raise AssertionError(f"unexpected sql: {sql}")
|
||||
|
||||
def fetchone(self):
|
||||
return self._fetchone_result
|
||||
|
||||
def fetchall(self):
|
||||
return list(self._fetchall_result)
|
||||
|
||||
|
||||
class _FakeDomainsConnection:
|
||||
def __init__(self) -> None:
|
||||
self.cursor_instance = _FakeDomainsCursor()
|
||||
self.committed = False
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, exc_type, exc, tb):
|
||||
return False
|
||||
|
||||
def cursor(self):
|
||||
return self.cursor_instance
|
||||
|
||||
def commit(self) -> None:
|
||||
self.committed = True
|
||||
|
||||
|
||||
class DomainsServiceTests(unittest.TestCase):
|
||||
def test_build_domain_query_parts_supports_false_backlink_filter(self) -> None:
|
||||
_from_clause, where_clause, params = domains_service._build_domain_query_parts({"backlink_gt_10": False})
|
||||
|
||||
self.assertIn("coalesce(dd.backlink_count_gt_10, false) = %s", where_clause)
|
||||
self.assertEqual([False], params)
|
||||
|
||||
@patch("app.services.domains_service.get_db")
|
||||
def test_fetch_domains_step_summary_counts_juziseo_and_jucha_results(self, mock_get_db) -> None:
|
||||
fake_conn = _FakeDomainsConnection()
|
||||
mock_get_db.return_value = fake_conn
|
||||
|
||||
result = domains_service.fetch_domains(page=1, page_size=20)
|
||||
|
||||
summary = result["list"][0]["step_summary"]
|
||||
self.assertEqual(1, summary["failed_count"])
|
||||
self.assertEqual(1, summary["blacklisted_count"])
|
||||
self.assertTrue(summary["has_failed"])
|
||||
self.assertTrue(summary["has_blacklisted_step"])
|
||||
|
||||
@patch("app.services.domains_service.get_db")
|
||||
def test_batch_update_domains_preserves_detection_metadata_shape(self, mock_get_db) -> None:
|
||||
fake_conn = _FakeDomainsConnection()
|
||||
mock_get_db.return_value = fake_conn
|
||||
|
||||
with patch("app.services.domains_service._now_text", return_value="2026-04-23 15:30:00"):
|
||||
result = domains_service.batch_update_domains([42], {"baidu_site": "否"})
|
||||
|
||||
self.assertEqual(1, result["updated_count"])
|
||||
self.assertIsNotNone(fake_conn.cursor_instance.updated_detection_params)
|
||||
updated_payload = fake_conn.cursor_instance.updated_detection_params[0]
|
||||
self.assertEqual(False, updated_payload["status"])
|
||||
self.assertEqual("failed", updated_payload["state"])
|
||||
self.assertEqual("人工批量更新", updated_payload["message"])
|
||||
self.assertEqual("2026-04-23 15:30:00", updated_payload["checked_at"])
|
||||
self.assertTrue(updated_payload["manual_override"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
52
domain-api/tests/test_import_task_service.py
Normal file
52
domain-api/tests/test_import_task_service.py
Normal file
@@ -0,0 +1,52 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
import app.services.import_task_service as import_task_service
|
||||
|
||||
|
||||
class _TraceLock:
|
||||
def __init__(self, order: list[str]) -> None:
|
||||
self.order = order
|
||||
|
||||
def __enter__(self):
|
||||
self.order.append("enter")
|
||||
return self
|
||||
|
||||
def __exit__(self, exc_type, exc, tb):
|
||||
self.order.append("exit")
|
||||
return False
|
||||
|
||||
|
||||
class ImportTaskServiceTests(unittest.TestCase):
|
||||
@patch("app.services.import_task_service.import_domains_from_path")
|
||||
@patch("app.services.import_task_service._update_task_with_log")
|
||||
def test_run_import_task_acquires_execution_lock_before_marking_running(
|
||||
self,
|
||||
mock_update_task_with_log,
|
||||
mock_import_domains_from_path,
|
||||
) -> None:
|
||||
order: list[str] = []
|
||||
mock_update_task_with_log.side_effect = lambda *args, **kwargs: order.append("update")
|
||||
mock_import_domains_from_path.return_value = {
|
||||
"source_label": "TXT 导入",
|
||||
"stats": {"total": 1, "valid": 1, "added": 1, "exists": 0, "invalid": 0},
|
||||
}
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
path = Path(tmpdir) / "domains.txt"
|
||||
path.write_text("a.com\n", encoding="utf-8")
|
||||
|
||||
with patch.object(import_task_service, "_IMPORT_EXECUTION_LOCK", _TraceLock(order)):
|
||||
import_task_service._run_import_task("task-1", str(path), source_type=7)
|
||||
|
||||
self.assertEqual("enter", order[0])
|
||||
self.assertIn("update", order[1:])
|
||||
self.assertEqual("exit", order[-1])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
94
domain-api/tests/test_import_worker_service.py
Normal file
94
domain-api/tests/test_import_worker_service.py
Normal file
@@ -0,0 +1,94 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import tempfile
|
||||
import unittest
|
||||
from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
from app.services.import_worker_service import import_domains_from_path
|
||||
|
||||
|
||||
class _FakeCursor:
|
||||
def __init__(self) -> None:
|
||||
self._fetchall_result = []
|
||||
self._fetchone_result = None
|
||||
self.inserted_domains: list[str] = []
|
||||
self.inserted_detect_tasks: list[int] = []
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, exc_type, exc, tb):
|
||||
return False
|
||||
|
||||
def execute(self, sql: str, params=None) -> None:
|
||||
normalized_sql = " ".join(str(sql or "").split()).lower()
|
||||
params = params or ()
|
||||
if normalized_sql.startswith("select domain from domains where domain = any"):
|
||||
self._fetchall_result = []
|
||||
return
|
||||
if normalized_sql.startswith("insert into domains"):
|
||||
domain = params[0]
|
||||
self.inserted_domains.append(domain)
|
||||
self._fetchone_result = (len(self.inserted_domains),)
|
||||
return
|
||||
if normalized_sql.startswith("insert into detect_tasks"):
|
||||
self.inserted_detect_tasks.append(int(params[0]))
|
||||
return
|
||||
raise AssertionError(f"unexpected sql: {sql}")
|
||||
|
||||
def fetchall(self):
|
||||
return list(self._fetchall_result)
|
||||
|
||||
def fetchone(self):
|
||||
return self._fetchone_result
|
||||
|
||||
|
||||
class _FakeConnection:
|
||||
def __init__(self) -> None:
|
||||
self.cursor_instance = _FakeCursor()
|
||||
self.committed = False
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, exc_type, exc, tb):
|
||||
return False
|
||||
|
||||
def cursor(self):
|
||||
return self.cursor_instance
|
||||
|
||||
def commit(self) -> None:
|
||||
self.committed = True
|
||||
|
||||
|
||||
class ImportWorkerServiceTests(unittest.TestCase):
|
||||
@patch("app.services.import_worker_service.get_db")
|
||||
def test_import_domains_from_path_skips_duplicate_domains_in_same_batch(self, mock_get_db) -> None:
|
||||
fake_conn = _FakeConnection()
|
||||
mock_get_db.return_value = fake_conn
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
path = Path(tmpdir) / "domains.txt"
|
||||
path.write_text("a.com\na.com\nb.net\ninvalid-domain\n", encoding="utf-8")
|
||||
|
||||
result = import_domains_from_path(path, source_type=7)
|
||||
|
||||
self.assertEqual(["a.com", "b.net"], fake_conn.cursor_instance.inserted_domains)
|
||||
self.assertEqual([1, 2], fake_conn.cursor_instance.inserted_detect_tasks)
|
||||
self.assertTrue(fake_conn.committed)
|
||||
self.assertEqual(
|
||||
{
|
||||
"total": 4,
|
||||
"valid": 3,
|
||||
"added": 2,
|
||||
"exists": 1,
|
||||
"invalid": 1,
|
||||
"failed": 0,
|
||||
},
|
||||
result["stats"],
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -4,6 +4,9 @@ import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
from app.api.routes.ops import (
|
||||
ops_migration_execute,
|
||||
ops_migration_preview,
|
||||
ops_migration_source_profile,
|
||||
ops_doctor_decision,
|
||||
ops_go_live_bundle,
|
||||
ops_go_live_review,
|
||||
@@ -13,6 +16,39 @@ from app.api.routes.ops import (
|
||||
|
||||
|
||||
class OpsApiRoutesTestCase(unittest.TestCase):
|
||||
@patch("app.api.routes.ops.get_ops_migration_source_profile")
|
||||
def test_migration_source_profile_route_returns_payload(self, mock_source_profile) -> None:
|
||||
mock_source_profile.return_value = {"source_node": {"node_code": "overseas-control-01"}}
|
||||
|
||||
response = ops_migration_source_profile()
|
||||
|
||||
self.assertEqual(0, response.code)
|
||||
self.assertEqual("overseas-control-01", response.data["source_node"]["node_code"])
|
||||
mock_source_profile.assert_called_once_with()
|
||||
|
||||
@patch("app.api.routes.ops.preview_ops_migration")
|
||||
def test_migration_preview_route_passes_payload(self, mock_preview_ops_migration) -> None:
|
||||
mock_preview_ops_migration.return_value = (True, "ok", {"plan_steps": [{"key": "sync_env_files"}]})
|
||||
|
||||
payload = {"target_node_code": "node-a"}
|
||||
response = ops_migration_preview(payload)
|
||||
|
||||
self.assertEqual(0, response.code)
|
||||
self.assertEqual("sync_env_files", response.data["plan_steps"][0]["key"])
|
||||
mock_preview_ops_migration.assert_called_once_with(payload)
|
||||
|
||||
@patch("app.api.routes.ops.execute_ops_migration")
|
||||
def test_migration_execute_route_returns_error_payload(self, mock_execute_ops_migration) -> None:
|
||||
mock_execute_ops_migration.return_value = (False, "failed", {"blocking_reasons": ["ssh missing"]})
|
||||
|
||||
payload = {"target_node_code": "node-a"}
|
||||
response = ops_migration_execute(payload)
|
||||
|
||||
self.assertEqual(1, response.code)
|
||||
self.assertEqual("failed", response.message)
|
||||
self.assertEqual(["ssh missing"], response.data["blocking_reasons"])
|
||||
mock_execute_ops_migration.assert_called_once_with(payload)
|
||||
|
||||
@patch("app.api.routes.ops.get_ops_go_live_signoff")
|
||||
def test_go_live_signoff_route_uses_service_payload(self, mock_get_ops_go_live_signoff) -> None:
|
||||
mock_get_ops_go_live_signoff.return_value = {
|
||||
|
||||
290
domain-api/tests/test_ops_migration_service.py
Normal file
290
domain-api/tests/test_ops_migration_service.py
Normal file
@@ -0,0 +1,290 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
from app.services.ops_migration_service import execute_ops_migration, preview_ops_migration
|
||||
|
||||
|
||||
class OpsMigrationServiceTestCase(unittest.TestCase):
|
||||
@patch("app.services.ops_migration_service._collect_remote_checks")
|
||||
@patch("app.services.ops_migration_service._inspect_target_database")
|
||||
@patch("app.services.ops_migration_service._find_managed_node")
|
||||
def test_preview_returns_blocking_reason_when_remote_tools_missing(
|
||||
self,
|
||||
mock_find_managed_node,
|
||||
mock_inspect_target_database,
|
||||
mock_collect_remote_checks,
|
||||
) -> None:
|
||||
mock_find_managed_node.return_value = {
|
||||
"node_code": "target-a",
|
||||
"ssh_host": "10.0.0.8",
|
||||
"ssh_user": "root",
|
||||
"ssh_port": 22,
|
||||
"auth_mode": "key",
|
||||
}
|
||||
mock_collect_remote_checks.return_value = {
|
||||
"tools": {
|
||||
"python3": True,
|
||||
"node": False,
|
||||
"npm": False,
|
||||
"systemctl": True,
|
||||
"psql": True,
|
||||
"pg_dump": True,
|
||||
"curl": True,
|
||||
},
|
||||
"paths": {
|
||||
"repo_exists": True,
|
||||
"repo_git": True,
|
||||
"domain_root_exists": True,
|
||||
"api_root_exists": True,
|
||||
"web_root_exists": True,
|
||||
},
|
||||
"remote_db_config": {
|
||||
"DB_HOST": "127.0.0.1",
|
||||
"DB_PORT": "5432",
|
||||
"DB_DATABASE": "domain",
|
||||
"DB_USER": "domainuser",
|
||||
},
|
||||
"blocking_reasons": ["目标机缺少 node。", "目标机缺少 npm。"],
|
||||
"warnings": [],
|
||||
}
|
||||
mock_inspect_target_database.return_value = {"available": False}
|
||||
|
||||
ok, message, data = preview_ops_migration({"target_node_code": "target-a"})
|
||||
|
||||
self.assertFalse(ok)
|
||||
self.assertEqual("迁移预检查未通过", message)
|
||||
self.assertIn("目标机缺少 node。", data["blocking_reasons"])
|
||||
self.assertIn("目标机缺少 npm。", data["blocking_reasons"])
|
||||
|
||||
@patch("app.services.ops_migration_service._collect_remote_checks")
|
||||
@patch("app.services.ops_migration_service._inspect_target_database")
|
||||
@patch("app.services.ops_migration_service._find_managed_node")
|
||||
def test_preview_uses_remote_env_as_target_db_default(
|
||||
self,
|
||||
mock_find_managed_node,
|
||||
mock_inspect_target_database,
|
||||
mock_collect_remote_checks,
|
||||
) -> None:
|
||||
mock_find_managed_node.return_value = {
|
||||
"node_code": "target-a",
|
||||
"ssh_host": "10.0.0.8",
|
||||
"ssh_user": "root",
|
||||
"ssh_port": 22,
|
||||
"auth_mode": "key",
|
||||
}
|
||||
mock_collect_remote_checks.return_value = {
|
||||
"tools": {
|
||||
"python3": True,
|
||||
"node": True,
|
||||
"npm": True,
|
||||
"systemctl": True,
|
||||
"psql": True,
|
||||
"pg_dump": True,
|
||||
"curl": True,
|
||||
},
|
||||
"paths": {
|
||||
"repo_exists": True,
|
||||
"repo_git": True,
|
||||
"domain_root_exists": True,
|
||||
"api_root_exists": True,
|
||||
"web_root_exists": True,
|
||||
},
|
||||
"git_commit": "abc123",
|
||||
"remote_db_config": {
|
||||
"DB_HOST": "127.0.0.1",
|
||||
"DB_PORT": "5433",
|
||||
"DB_DATABASE": "domain_remote",
|
||||
"DB_USER": "remote_user",
|
||||
},
|
||||
"blocking_reasons": [],
|
||||
"warnings": [],
|
||||
}
|
||||
mock_inspect_target_database.return_value = {
|
||||
"available": True,
|
||||
"has_business_data": False,
|
||||
}
|
||||
|
||||
ok, message, data = preview_ops_migration({"target_node_code": "target-a", "overwrite_database": True})
|
||||
|
||||
self.assertTrue(ok)
|
||||
self.assertEqual("迁移预检查完成", message)
|
||||
self.assertEqual("domain_remote", data["target_db_config"]["database"])
|
||||
self.assertEqual("remote_user", data["target_db_config"]["user"])
|
||||
self.assertEqual(5433, data["target_db_config"]["port"])
|
||||
self.assertTrue(bool(data["execution_guard"]["token"]))
|
||||
|
||||
@patch("app.services.ops_migration_service._run_remote_health_check")
|
||||
@patch("app.services.ops_migration_service._restart_remote_services")
|
||||
@patch("app.services.ops_migration_service._build_remote_frontend")
|
||||
@patch("app.services.ops_migration_service._inspect_target_database")
|
||||
@patch("app.services.ops_migration_service._collect_remote_checks")
|
||||
@patch("app.services.ops_migration_service._find_managed_node")
|
||||
def test_execute_rejects_missing_confirmation_text_for_nonempty_target_db(
|
||||
self,
|
||||
mock_find_managed_node,
|
||||
mock_collect_remote_checks,
|
||||
mock_inspect_target_database,
|
||||
mock_build_remote_frontend,
|
||||
mock_restart_remote_services,
|
||||
mock_run_remote_health_check,
|
||||
) -> None:
|
||||
mock_find_managed_node.return_value = {
|
||||
"node_code": "target-a",
|
||||
"ssh_host": "10.0.0.8",
|
||||
"ssh_user": "root",
|
||||
"ssh_port": 22,
|
||||
"auth_mode": "key",
|
||||
}
|
||||
mock_collect_remote_checks.return_value = {
|
||||
"tools": {
|
||||
"python3": True,
|
||||
"node": True,
|
||||
"npm": True,
|
||||
"systemctl": True,
|
||||
"psql": True,
|
||||
"pg_dump": True,
|
||||
"curl": True,
|
||||
},
|
||||
"paths": {
|
||||
"repo_exists": True,
|
||||
"repo_git": True,
|
||||
"domain_root_exists": True,
|
||||
"api_root_exists": True,
|
||||
"web_root_exists": True,
|
||||
},
|
||||
"git_commit": "abc123",
|
||||
"remote_db_config": {
|
||||
"DB_HOST": "127.0.0.1",
|
||||
"DB_PORT": "5432",
|
||||
"DB_DATABASE": "domain_remote",
|
||||
"DB_USER": "remote_user",
|
||||
},
|
||||
"blocking_reasons": [],
|
||||
"warnings": [],
|
||||
}
|
||||
mock_inspect_target_database.return_value = {
|
||||
"available": True,
|
||||
"database": "domain_remote",
|
||||
"has_business_data": True,
|
||||
"public_table_count": 10,
|
||||
"business_table_count": 5,
|
||||
}
|
||||
mock_build_remote_frontend.return_value = (True, "ok", {})
|
||||
mock_restart_remote_services.return_value = (True, "ok", {})
|
||||
mock_run_remote_health_check.return_value = (True, "ok", {})
|
||||
|
||||
preview_ok, _preview_message, preview_data = preview_ops_migration(
|
||||
{"target_node_code": "target-a", "overwrite_database": True}
|
||||
)
|
||||
|
||||
self.assertTrue(preview_ok)
|
||||
token = preview_data["execution_guard"]["token"]
|
||||
required_confirmation_text = preview_data["execution_guard"]["required_confirmation_text"]
|
||||
self.assertEqual("OVERWRITE domain_remote", required_confirmation_text)
|
||||
|
||||
execute_ok, execute_message, execute_data = execute_ops_migration(
|
||||
{
|
||||
"target_node_code": "target-a",
|
||||
"overwrite_database": True,
|
||||
"execute_confirmation_token": token,
|
||||
}
|
||||
)
|
||||
|
||||
self.assertFalse(execute_ok)
|
||||
self.assertEqual("缺少数据库覆盖确认文案,执行被拒绝。", execute_message)
|
||||
self.assertIn("missing execute_confirmation_text", execute_data["blocking_reasons"])
|
||||
|
||||
@patch("app.services.ops_migration_service._start_migration_dispatch_thread")
|
||||
@patch("app.services.ops_migration_service.create_ops_job")
|
||||
@patch("app.services.ops_migration_service._collect_remote_checks")
|
||||
@patch("app.services.ops_migration_service._inspect_target_database")
|
||||
@patch("app.services.ops_migration_service._find_managed_node")
|
||||
def test_execute_creates_async_job_for_long_running_migration(
|
||||
self,
|
||||
mock_find_managed_node,
|
||||
mock_inspect_target_database,
|
||||
mock_collect_remote_checks,
|
||||
mock_create_ops_job,
|
||||
mock_start_thread,
|
||||
) -> None:
|
||||
mock_find_managed_node.return_value = {
|
||||
"node_code": "target-a",
|
||||
"ssh_host": "10.0.0.8",
|
||||
"ssh_user": "root",
|
||||
"ssh_port": 22,
|
||||
"auth_mode": "key",
|
||||
}
|
||||
mock_collect_remote_checks.return_value = {
|
||||
"tools": {
|
||||
"python3": True,
|
||||
"node": True,
|
||||
"npm": True,
|
||||
"systemctl": True,
|
||||
"psql": True,
|
||||
"pg_dump": True,
|
||||
"curl": True,
|
||||
},
|
||||
"paths": {
|
||||
"repo_exists": True,
|
||||
"repo_git": True,
|
||||
"domain_root_exists": True,
|
||||
"api_root_exists": True,
|
||||
"web_root_exists": True,
|
||||
},
|
||||
"git_commit": "abc123",
|
||||
"remote_db_config": {
|
||||
"DB_HOST": "127.0.0.1",
|
||||
"DB_PORT": "5432",
|
||||
"DB_DATABASE": "domain_remote",
|
||||
"DB_USER": "remote_user",
|
||||
},
|
||||
"blocking_reasons": [],
|
||||
"warnings": [],
|
||||
}
|
||||
mock_inspect_target_database.return_value = {
|
||||
"available": True,
|
||||
"database": "domain_remote",
|
||||
"has_business_data": False,
|
||||
"public_table_count": 0,
|
||||
"business_table_count": 0,
|
||||
}
|
||||
mock_create_ops_job.return_value = (
|
||||
True,
|
||||
"ok",
|
||||
{
|
||||
"job": {
|
||||
"id": 88,
|
||||
"job_code": "ops-20260422160000-abc123",
|
||||
"action": "migration.execute",
|
||||
"status": "queued",
|
||||
"target_node_code": "target-a",
|
||||
}
|
||||
},
|
||||
)
|
||||
|
||||
preview_ok, _preview_message, preview_data = preview_ops_migration({"target_node_code": "target-a"})
|
||||
|
||||
self.assertTrue(preview_ok)
|
||||
token = preview_data["execution_guard"]["token"]
|
||||
|
||||
execute_ok, execute_message, execute_data = execute_ops_migration(
|
||||
{
|
||||
"target_node_code": "target-a",
|
||||
"execute_confirmation_token": token,
|
||||
}
|
||||
)
|
||||
|
||||
self.assertTrue(execute_ok)
|
||||
self.assertEqual("迁移任务已创建,后台开始执行。", execute_message)
|
||||
self.assertEqual(88, execute_data["job"]["id"])
|
||||
self.assertEqual("migration.execute", mock_create_ops_job.call_args.args[0]["action"])
|
||||
self.assertFalse(bool(mock_create_ops_job.call_args.args[0]["run_now"]))
|
||||
self.assertEqual("control-plane", mock_create_ops_job.call_args.args[0]["execution_mode"])
|
||||
self.assertEqual("", mock_create_ops_job.call_args.args[0]["payload"]["target_db_password"])
|
||||
mock_start_thread.assert_called_once_with(88)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -9,9 +9,11 @@ from pathlib import Path
|
||||
from unittest.mock import patch
|
||||
|
||||
from app.services.ops_release_executor_core import (
|
||||
_systemd_dropin_content,
|
||||
_pick_release_owner_group,
|
||||
build_remote_release_action_script,
|
||||
execute_release_action,
|
||||
normalize_release_health_check_urls,
|
||||
)
|
||||
|
||||
|
||||
@@ -36,6 +38,7 @@ def _build_release_archive() -> bytes:
|
||||
"README.txt": b"hello-release",
|
||||
"domain-api/deploy/systemd/domain-node-agent.service": b"[Service]\nEnvironmentFile=-/etc/default/domaincheck-worker\n",
|
||||
"domain-api/deploy/systemd/domain-worker.service": b"[Service]\nEnvironmentFile=-/etc/default/domaincheck-worker\n",
|
||||
"domain-api/deploy/systemd/domain-worker@.service": b"[Service]\nEnvironmentFile=-/etc/default/domaincheck-worker-%i\n",
|
||||
"domain-api/deploy/systemd/domain-api.service": b"[Service]\nEnvironmentFile=-/etc/default/domaincheck-api\n",
|
||||
"domain-api/deploy/systemd/domain-sync-agent.service": b"[Service]\nEnvironmentFile=-/etc/default/domaincheck-worker\n",
|
||||
}
|
||||
@@ -47,6 +50,42 @@ def _build_release_archive() -> bytes:
|
||||
|
||||
|
||||
class OpsReleaseExecutorCoreTests(unittest.TestCase):
|
||||
def test_normalize_release_health_check_urls_rewrites_runtime_status_probe(self) -> None:
|
||||
self.assertEqual(
|
||||
[
|
||||
"http://127.0.0.1:8100/health",
|
||||
"http://127.0.0.1:8100/health",
|
||||
"https://example.com/custom-health",
|
||||
],
|
||||
normalize_release_health_check_urls(
|
||||
[
|
||||
"http://127.0.0.1:8100/api/v1/runtime/status",
|
||||
"http://127.0.0.1:8100/runtime/status?full=1",
|
||||
"https://example.com/custom-health",
|
||||
]
|
||||
),
|
||||
)
|
||||
|
||||
def test_api_service_template_limits_graceful_shutdown(self) -> None:
|
||||
service_text = Path("domain-api/deploy/systemd/domain-api.service").read_text(encoding="utf-8")
|
||||
|
||||
self.assertIn("--timeout-graceful-shutdown 15", service_text)
|
||||
self.assertIn("TimeoutStopSec=20", service_text)
|
||||
|
||||
def test_worker_service_template_uses_current_symlink(self) -> None:
|
||||
service_text = Path("domain-api/deploy/systemd/domain-worker.service").read_text(encoding="utf-8")
|
||||
|
||||
self.assertIn("WorkingDirectory=/opt/domaincheck/current/domainCheck", service_text)
|
||||
self.assertIn(
|
||||
"ExecStart=/opt/domaincheck/domainCheck/.venv/bin/python /opt/domaincheck/current/domainCheck/detect_worker.py",
|
||||
service_text,
|
||||
)
|
||||
|
||||
def test_api_service_dropin_uses_graceful_shutdown_timeout(self) -> None:
|
||||
dropin_text = _systemd_dropin_content("domaincheck-api", "/opt/domaincheck")
|
||||
|
||||
self.assertIn("--timeout-graceful-shutdown 15", dropin_text)
|
||||
|
||||
def test_pick_release_owner_group_prefers_service_identity_over_path_owner(self) -> None:
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
with patch(
|
||||
@@ -65,16 +104,29 @@ class OpsReleaseExecutorCoreTests(unittest.TestCase):
|
||||
|
||||
def run_command(command: list[str], *, timeout: int = 60):
|
||||
commands.append(tuple(command))
|
||||
if command[:5] == ["systemctl", "list-units", "--type=service", "--all", "domaincheck-worker@*"]:
|
||||
return 0, (
|
||||
"domaincheck-worker@a.service loaded active running domainCheck Worker Instance a\n"
|
||||
"domaincheck-worker@ah.service loaded inactive dead domainCheck Worker Instance ah\n"
|
||||
), ""
|
||||
if command[:3] == ["systemctl", "show", "domaincheck-worker"] and "-p" in command:
|
||||
current_user = pwd.getpwuid(os.getuid()).pw_name
|
||||
current_group = grp.getgrgid(os.getgid()).gr_name
|
||||
return 0, f"{current_user}\n{current_group}\n", ""
|
||||
if command[:3] == ["systemctl", "show", "domaincheck-worker@a.service"] and "-p" in command:
|
||||
current_user = pwd.getpwuid(os.getuid()).pw_name
|
||||
current_group = grp.getgrgid(os.getgid()).gr_name
|
||||
return 0, f"{current_user}\n{current_group}\n", ""
|
||||
if command[:3] == ["systemctl", "show", "domaincheck-worker@ah.service"] and "-p" in command:
|
||||
current_user = pwd.getpwuid(os.getuid()).pw_name
|
||||
current_group = grp.getgrgid(os.getgid()).gr_name
|
||||
return 0, f"{current_user}\n{current_group}\n", ""
|
||||
if command[:2] == ["chown", "-R"]:
|
||||
return 0, "", ""
|
||||
if command[:2] == ["systemctl", "restart"]:
|
||||
return 0, "", ""
|
||||
if command[:2] == ["systemctl", "is-active"]:
|
||||
return 0, "active", ""
|
||||
return 0, "\n".join("active" for _ in command[2:]), ""
|
||||
return 0, "", ""
|
||||
|
||||
with tempfile.TemporaryDirectory() as tmpdir:
|
||||
@@ -108,8 +160,13 @@ class OpsReleaseExecutorCoreTests(unittest.TestCase):
|
||||
self.assertTrue(any(cmd[:2] == ("chown", "-R") for cmd in commands))
|
||||
self.assertIn(("systemctl", "daemon-reload"), commands)
|
||||
self.assertIn(("systemctl", "restart", "domaincheck-worker"), commands)
|
||||
self.assertIn(("systemctl", "restart", "domaincheck-worker@a.service"), commands)
|
||||
self.assertIn(("systemctl", "restart", "domaincheck-worker@ah.service"), commands)
|
||||
self.assertIn(("systemctl", "is-active", "domaincheck-worker"), commands)
|
||||
self.assertIn(("systemctl", "is-active", "domaincheck-worker@a.service"), commands)
|
||||
self.assertIn(("systemctl", "is-active", "domaincheck-worker@ah.service"), commands)
|
||||
self.assertTrue((systemd_root / "domaincheck-node-agent.service").exists())
|
||||
self.assertTrue((systemd_root / "domaincheck-worker@.service").exists())
|
||||
self.assertTrue((systemd_root / "domaincheck-node-agent.service.d" / "current-path.conf").exists())
|
||||
|
||||
def test_build_remote_release_action_script_is_valid_python(self) -> None:
|
||||
@@ -128,6 +185,9 @@ class OpsReleaseExecutorCoreTests(unittest.TestCase):
|
||||
self.assertIn("False", script)
|
||||
self.assertIn("def collect_service_identity(", script)
|
||||
self.assertIn("def apply_release_permissions(", script)
|
||||
self.assertIn("import grp", script)
|
||||
self.assertIn("import os", script)
|
||||
self.assertIn("import pwd", script)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
|
||||
@@ -38,6 +38,97 @@ class _EmptyConnection:
|
||||
|
||||
|
||||
class OpsReleaseServiceExecutionModeTests(unittest.TestCase):
|
||||
@patch("app.services.ops_release_service.get_db")
|
||||
@patch("app.services.ops_agent_service.ensure_ops_agent_schema")
|
||||
@patch("app.services.ops_agent_service.get_managed_node_onboarding")
|
||||
@patch("app.services.ops_agent_service.list_managed_nodes_with_agent_state")
|
||||
@patch("app.services.ops_job_service.list_managed_nodes")
|
||||
@patch("app.services.cluster_runtime_service.get_cluster_snapshot")
|
||||
def test_build_rollout_target_operational_readiness_reuses_managed_node_snapshot_for_onboarding(
|
||||
self,
|
||||
mock_get_cluster_snapshot,
|
||||
mock_list_managed_nodes,
|
||||
mock_list_managed_nodes_with_agent_state,
|
||||
mock_get_managed_node_onboarding,
|
||||
mock_ensure_ops_agent_schema,
|
||||
mock_get_db,
|
||||
) -> None:
|
||||
captured_nodes_payloads = []
|
||||
|
||||
def _fake_onboarding(node_code, **kwargs):
|
||||
captured_nodes_payloads.append(kwargs.get("nodes_payload"))
|
||||
return {
|
||||
"onboarding_stage": {"code": "ready", "label": "已接管"},
|
||||
"summary": f"{node_code} ready",
|
||||
"recovery_decision": {
|
||||
"action": "noop",
|
||||
"label": "当前无需额外恢复动作",
|
||||
"summary": "当前节点暂无需要执行的接管恢复动作。",
|
||||
"command_hint": "",
|
||||
"window": "none",
|
||||
},
|
||||
}
|
||||
|
||||
mock_ensure_ops_agent_schema.return_value = None
|
||||
mock_get_db.return_value = _EmptyConnection()
|
||||
mock_get_cluster_snapshot.return_value = {
|
||||
"nodes": [
|
||||
{
|
||||
"node_code": "mainland-worker-01",
|
||||
"region": "mainland",
|
||||
"role": "worker",
|
||||
"status": "online",
|
||||
"current_load": 0,
|
||||
"is_effective_worker": True,
|
||||
},
|
||||
{
|
||||
"node_code": "mainland-worker-02",
|
||||
"region": "mainland",
|
||||
"role": "worker",
|
||||
"status": "online",
|
||||
"current_load": 0,
|
||||
"is_effective_worker": True,
|
||||
},
|
||||
]
|
||||
}
|
||||
mock_list_managed_nodes.return_value = [
|
||||
{
|
||||
"node_code": "mainland-worker-01",
|
||||
"region": "mainland",
|
||||
"role": "worker",
|
||||
"is_enabled": True,
|
||||
"ssh_host": "121.204.244.248",
|
||||
"ssh_user": "root",
|
||||
"metadata": {},
|
||||
"last_seen_at": "",
|
||||
},
|
||||
{
|
||||
"node_code": "mainland-worker-02",
|
||||
"region": "mainland",
|
||||
"role": "worker",
|
||||
"is_enabled": True,
|
||||
"ssh_host": "121.204.244.249",
|
||||
"ssh_user": "root",
|
||||
"metadata": {},
|
||||
"last_seen_at": "",
|
||||
},
|
||||
]
|
||||
mock_list_managed_nodes_with_agent_state.return_value = {"nodes": [{"node_code": "mainland-worker-01"}, {"node_code": "mainland-worker-02"}]}
|
||||
mock_get_managed_node_onboarding.side_effect = _fake_onboarding
|
||||
|
||||
readiness = build_rollout_target_operational_readiness(
|
||||
[
|
||||
{"node_code": "mainland-worker-01", "region": "mainland", "role": "worker"},
|
||||
{"node_code": "mainland-worker-02", "region": "mainland", "role": "worker"},
|
||||
],
|
||||
execution_mode="remote-agent",
|
||||
)
|
||||
|
||||
self.assertEqual(2, len(readiness["rows"]))
|
||||
mock_list_managed_nodes_with_agent_state.assert_called_once()
|
||||
self.assertEqual(2, len(captured_nodes_payloads))
|
||||
self.assertTrue(all(payload == {"nodes": [{"node_code": "mainland-worker-01"}, {"node_code": "mainland-worker-02"}]} for payload in captured_nodes_payloads))
|
||||
|
||||
@patch("app.services.ops_release_service.get_db")
|
||||
@patch("app.services.ops_agent_service.ensure_ops_agent_schema")
|
||||
@patch("app.services.ops_agent_service.get_managed_node_onboarding")
|
||||
|
||||
@@ -3,6 +3,8 @@ from unittest.mock import MagicMock, patch
|
||||
|
||||
import app.services.ops_agent_service as ops_agent_service
|
||||
import app.services.ops_job_service as ops_job_service
|
||||
import app.services.ops_release_service as ops_release_service
|
||||
from psycopg2 import errors
|
||||
|
||||
|
||||
class OpsSchemaInitTests(unittest.TestCase):
|
||||
@@ -10,6 +12,7 @@ class OpsSchemaInitTests(unittest.TestCase):
|
||||
def test_ensure_ops_schema_uses_advisory_lock_and_skips_after_ready(self, mock_get_db) -> None:
|
||||
conn = MagicMock()
|
||||
cursor = MagicMock()
|
||||
cursor.fetchone.return_value = None
|
||||
db_ctx = MagicMock()
|
||||
cursor_ctx = MagicMock()
|
||||
db_ctx.__enter__.return_value = conn
|
||||
@@ -35,6 +38,94 @@ class OpsSchemaInitTests(unittest.TestCase):
|
||||
cursor.execute.assert_any_call(ops_job_service._OPS_SCHEMA_SQL)
|
||||
conn.commit.assert_called_once()
|
||||
|
||||
@patch("app.services.ops_job_service.get_db")
|
||||
def test_ensure_ops_schema_skips_ddl_when_required_schema_already_exists(self, mock_get_db) -> None:
|
||||
conn = MagicMock()
|
||||
cursor = MagicMock()
|
||||
cursor.fetchone.side_effect = [(f"public.{name}",) for name in ops_job_service._OPS_REQUIRED_TABLES]
|
||||
cursor.fetchall.side_effect = [
|
||||
[(column,) for column in ops_job_service._OPS_REQUIRED_COLUMNS["ops_jobs"]],
|
||||
[(column,) for column in ops_job_service._OPS_REQUIRED_COLUMNS["ops_job_steps"]],
|
||||
]
|
||||
db_ctx = MagicMock()
|
||||
cursor_ctx = MagicMock()
|
||||
db_ctx.__enter__.return_value = conn
|
||||
db_ctx.__exit__.return_value = False
|
||||
cursor_ctx.__enter__.return_value = cursor
|
||||
cursor_ctx.__exit__.return_value = False
|
||||
conn.cursor.return_value = cursor_ctx
|
||||
mock_get_db.return_value = db_ctx
|
||||
|
||||
previous_ready = ops_job_service._OPS_SCHEMA_READY
|
||||
ops_job_service._OPS_SCHEMA_READY = False
|
||||
try:
|
||||
ops_job_service.ensure_ops_schema()
|
||||
finally:
|
||||
ops_job_service._OPS_SCHEMA_READY = previous_ready
|
||||
|
||||
self.assertFalse(any(call.args[0] == ops_job_service._OPS_SCHEMA_SQL for call in cursor.execute.call_args_list))
|
||||
conn.commit.assert_not_called()
|
||||
|
||||
@patch("app.services.ops_job_service.get_db")
|
||||
def test_ensure_ops_schema_accepts_deadlock_when_required_schema_already_exists(self, mock_get_db) -> None:
|
||||
class _Cursor:
|
||||
def __init__(self, *, raise_on_schema=False, fetchone_values=None, fetchall_values=None) -> None:
|
||||
self.raise_on_schema = raise_on_schema
|
||||
self.fetchone_values = list(fetchone_values or [])
|
||||
self.fetchall_values = list(fetchall_values or [])
|
||||
|
||||
def execute(self, sql, params=None):
|
||||
if self.raise_on_schema and sql == ops_job_service._OPS_SCHEMA_SQL:
|
||||
raise errors.DeadlockDetected()
|
||||
|
||||
def fetchone(self):
|
||||
if self.fetchone_values:
|
||||
return self.fetchone_values.pop(0)
|
||||
return None
|
||||
|
||||
def fetchall(self):
|
||||
if self.fetchall_values:
|
||||
return self.fetchall_values.pop(0)
|
||||
return []
|
||||
|
||||
class _CursorContext:
|
||||
def __init__(self, cursor) -> None:
|
||||
self.cursor = cursor
|
||||
|
||||
def __enter__(self):
|
||||
return self.cursor
|
||||
|
||||
def __exit__(self, exc_type, exc, tb):
|
||||
return False
|
||||
|
||||
conn = MagicMock()
|
||||
conn.cursor.side_effect = [
|
||||
_CursorContext(_Cursor(fetchone_values=[None])),
|
||||
_CursorContext(_Cursor(raise_on_schema=True)),
|
||||
_CursorContext(
|
||||
_Cursor(
|
||||
fetchone_values=[(f"public.{name}",) for name in ops_job_service._OPS_REQUIRED_TABLES],
|
||||
fetchall_values=[
|
||||
[(column,) for column in ops_job_service._OPS_REQUIRED_COLUMNS["ops_jobs"]],
|
||||
[(column,) for column in ops_job_service._OPS_REQUIRED_COLUMNS["ops_job_steps"]],
|
||||
],
|
||||
)
|
||||
),
|
||||
]
|
||||
db_ctx = MagicMock()
|
||||
db_ctx.__enter__.return_value = conn
|
||||
mock_get_db.return_value = db_ctx
|
||||
|
||||
previous_ready = ops_job_service._OPS_SCHEMA_READY
|
||||
ops_job_service._OPS_SCHEMA_READY = False
|
||||
try:
|
||||
ops_job_service.ensure_ops_schema()
|
||||
finally:
|
||||
ops_job_service._OPS_SCHEMA_READY = previous_ready
|
||||
|
||||
conn.rollback.assert_called_once()
|
||||
conn.commit.assert_not_called()
|
||||
|
||||
@patch("app.services.ops_agent_service.ensure_ops_schema")
|
||||
@patch("app.services.ops_agent_service.get_db")
|
||||
def test_ensure_ops_agent_schema_uses_advisory_lock_and_skips_after_ready(
|
||||
@@ -44,6 +135,7 @@ class OpsSchemaInitTests(unittest.TestCase):
|
||||
) -> None:
|
||||
conn = MagicMock()
|
||||
cursor = MagicMock()
|
||||
cursor.fetchone.return_value = None
|
||||
db_ctx = MagicMock()
|
||||
cursor_ctx = MagicMock()
|
||||
db_ctx.__enter__.return_value = conn
|
||||
@@ -70,6 +162,224 @@ class OpsSchemaInitTests(unittest.TestCase):
|
||||
cursor.execute.assert_any_call(ops_agent_service._AGENT_SCHEMA_SQL)
|
||||
conn.commit.assert_called_once()
|
||||
|
||||
@patch("app.services.ops_agent_service.ensure_ops_schema")
|
||||
@patch("app.services.ops_agent_service.get_db")
|
||||
def test_ensure_ops_agent_schema_skips_ddl_when_required_schema_already_exists(
|
||||
self,
|
||||
mock_get_db,
|
||||
mock_ensure_ops_schema,
|
||||
) -> None:
|
||||
conn = MagicMock()
|
||||
cursor = MagicMock()
|
||||
cursor.fetchone.side_effect = [(f"public.{name}",) for name in ops_agent_service._OPS_AGENT_REQUIRED_TABLES]
|
||||
cursor.fetchall.side_effect = [
|
||||
[(column,) for column in ops_agent_service._OPS_AGENT_REQUIRED_COLUMNS["ops_node_tokens"]],
|
||||
[(column,) for column in ops_agent_service._OPS_AGENT_REQUIRED_COLUMNS["ops_job_events"]],
|
||||
[(column,) for column in ops_agent_service._OPS_AGENT_REQUIRED_COLUMNS["ops_jobs"]],
|
||||
]
|
||||
db_ctx = MagicMock()
|
||||
cursor_ctx = MagicMock()
|
||||
db_ctx.__enter__.return_value = conn
|
||||
db_ctx.__exit__.return_value = False
|
||||
cursor_ctx.__enter__.return_value = cursor
|
||||
cursor_ctx.__exit__.return_value = False
|
||||
conn.cursor.return_value = cursor_ctx
|
||||
mock_get_db.return_value = db_ctx
|
||||
|
||||
previous_ready = ops_agent_service._OPS_AGENT_SCHEMA_READY
|
||||
ops_agent_service._OPS_AGENT_SCHEMA_READY = False
|
||||
try:
|
||||
ops_agent_service.ensure_ops_agent_schema()
|
||||
finally:
|
||||
ops_agent_service._OPS_AGENT_SCHEMA_READY = previous_ready
|
||||
|
||||
self.assertFalse(any(call.args[0] == ops_agent_service._AGENT_SCHEMA_SQL for call in cursor.execute.call_args_list))
|
||||
conn.commit.assert_not_called()
|
||||
self.assertEqual(1, mock_ensure_ops_schema.call_count)
|
||||
|
||||
@patch("app.services.ops_agent_service.ensure_ops_schema")
|
||||
@patch("app.services.ops_agent_service.get_db")
|
||||
def test_ensure_ops_agent_schema_accepts_deadlock_when_required_schema_already_exists(
|
||||
self,
|
||||
mock_get_db,
|
||||
mock_ensure_ops_schema,
|
||||
) -> None:
|
||||
class _Cursor:
|
||||
def __init__(self, *, raise_on_schema=False, fetchone_values=None, fetchall_values=None) -> None:
|
||||
self.raise_on_schema = raise_on_schema
|
||||
self.fetchone_values = list(fetchone_values or [])
|
||||
self.fetchall_values = list(fetchall_values or [])
|
||||
|
||||
def execute(self, sql, params=None):
|
||||
if self.raise_on_schema and sql == ops_agent_service._AGENT_SCHEMA_SQL:
|
||||
raise errors.DeadlockDetected()
|
||||
|
||||
def fetchone(self):
|
||||
if self.fetchone_values:
|
||||
return self.fetchone_values.pop(0)
|
||||
return None
|
||||
|
||||
def fetchall(self):
|
||||
if self.fetchall_values:
|
||||
return self.fetchall_values.pop(0)
|
||||
return []
|
||||
|
||||
class _CursorContext:
|
||||
def __init__(self, cursor) -> None:
|
||||
self.cursor = cursor
|
||||
|
||||
def __enter__(self):
|
||||
return self.cursor
|
||||
|
||||
def __exit__(self, exc_type, exc, tb):
|
||||
return False
|
||||
|
||||
conn = MagicMock()
|
||||
conn.cursor.side_effect = [
|
||||
_CursorContext(_Cursor(fetchone_values=[None, None])),
|
||||
_CursorContext(_Cursor(raise_on_schema=True)),
|
||||
_CursorContext(
|
||||
_Cursor(
|
||||
fetchone_values=[(f"public.{name}",) for name in ops_agent_service._OPS_AGENT_REQUIRED_TABLES],
|
||||
fetchall_values=[
|
||||
[(column,) for column in ops_agent_service._OPS_AGENT_REQUIRED_COLUMNS["ops_node_tokens"]],
|
||||
[(column,) for column in ops_agent_service._OPS_AGENT_REQUIRED_COLUMNS["ops_job_events"]],
|
||||
[(column,) for column in ops_agent_service._OPS_AGENT_REQUIRED_COLUMNS["ops_jobs"]],
|
||||
],
|
||||
)
|
||||
),
|
||||
]
|
||||
db_ctx = MagicMock()
|
||||
db_ctx.__enter__.return_value = conn
|
||||
mock_get_db.return_value = db_ctx
|
||||
|
||||
previous_ready = ops_agent_service._OPS_AGENT_SCHEMA_READY
|
||||
ops_agent_service._OPS_AGENT_SCHEMA_READY = False
|
||||
try:
|
||||
ops_agent_service.ensure_ops_agent_schema()
|
||||
finally:
|
||||
ops_agent_service._OPS_AGENT_SCHEMA_READY = previous_ready
|
||||
|
||||
conn.rollback.assert_called_once()
|
||||
conn.commit.assert_not_called()
|
||||
self.assertEqual(1, mock_ensure_ops_schema.call_count)
|
||||
|
||||
@patch("app.services.ops_release_service.get_db")
|
||||
def test_ensure_ops_release_schema_uses_advisory_lock_and_skips_after_ready(self, mock_get_db) -> None:
|
||||
conn = MagicMock()
|
||||
cursor = MagicMock()
|
||||
cursor.fetchone.return_value = None
|
||||
db_ctx = MagicMock()
|
||||
cursor_ctx = MagicMock()
|
||||
db_ctx.__enter__.return_value = conn
|
||||
db_ctx.__exit__.return_value = False
|
||||
cursor_ctx.__enter__.return_value = cursor
|
||||
cursor_ctx.__exit__.return_value = False
|
||||
conn.cursor.return_value = cursor_ctx
|
||||
mock_get_db.return_value = db_ctx
|
||||
|
||||
previous_ready = ops_release_service._RELEASE_SCHEMA_READY
|
||||
ops_release_service._RELEASE_SCHEMA_READY = False
|
||||
try:
|
||||
ops_release_service.ensure_ops_release_schema()
|
||||
ops_release_service.ensure_ops_release_schema()
|
||||
finally:
|
||||
ops_release_service._RELEASE_SCHEMA_READY = previous_ready
|
||||
|
||||
self.assertEqual(1, mock_get_db.call_count)
|
||||
cursor.execute.assert_any_call(
|
||||
"SELECT pg_advisory_xact_lock(%s)",
|
||||
(ops_release_service._RELEASE_SCHEMA_ADVISORY_LOCK_KEY,),
|
||||
)
|
||||
cursor.execute.assert_any_call(ops_release_service._RELEASE_SCHEMA_SQL)
|
||||
conn.commit.assert_called_once()
|
||||
|
||||
@patch("app.services.ops_release_service.get_db")
|
||||
def test_ensure_ops_release_schema_skips_ddl_when_required_schema_already_exists(self, mock_get_db) -> None:
|
||||
conn = MagicMock()
|
||||
cursor = MagicMock()
|
||||
cursor.fetchone.side_effect = [(f"public.{name}",) for name in ops_release_service._RELEASE_REQUIRED_TABLES]
|
||||
cursor.fetchall.side_effect = [
|
||||
[(column,) for column in ops_release_service._RELEASE_REQUIRED_COLUMNS["ops_release_rollouts"]],
|
||||
]
|
||||
db_ctx = MagicMock()
|
||||
cursor_ctx = MagicMock()
|
||||
db_ctx.__enter__.return_value = conn
|
||||
db_ctx.__exit__.return_value = False
|
||||
cursor_ctx.__enter__.return_value = cursor
|
||||
cursor_ctx.__exit__.return_value = False
|
||||
conn.cursor.return_value = cursor_ctx
|
||||
mock_get_db.return_value = db_ctx
|
||||
|
||||
previous_ready = ops_release_service._RELEASE_SCHEMA_READY
|
||||
ops_release_service._RELEASE_SCHEMA_READY = False
|
||||
try:
|
||||
ops_release_service.ensure_ops_release_schema()
|
||||
finally:
|
||||
ops_release_service._RELEASE_SCHEMA_READY = previous_ready
|
||||
|
||||
self.assertFalse(any(call.args[0] == ops_release_service._RELEASE_SCHEMA_SQL for call in cursor.execute.call_args_list))
|
||||
conn.commit.assert_not_called()
|
||||
|
||||
@patch("app.services.ops_release_service.get_db")
|
||||
def test_ensure_ops_release_schema_accepts_deadlock_when_required_schema_already_exists(self, mock_get_db) -> None:
|
||||
class _Cursor:
|
||||
def __init__(self, *, raise_on_schema=False, fetchone_values=None, fetchall_values=None) -> None:
|
||||
self.raise_on_schema = raise_on_schema
|
||||
self.fetchone_values = list(fetchone_values or [])
|
||||
self.fetchall_values = list(fetchall_values or [])
|
||||
|
||||
def execute(self, sql, params=None):
|
||||
if self.raise_on_schema and sql == ops_release_service._RELEASE_SCHEMA_SQL:
|
||||
raise errors.DeadlockDetected()
|
||||
|
||||
def fetchone(self):
|
||||
if self.fetchone_values:
|
||||
return self.fetchone_values.pop(0)
|
||||
return None
|
||||
|
||||
def fetchall(self):
|
||||
if self.fetchall_values:
|
||||
return self.fetchall_values.pop(0)
|
||||
return []
|
||||
|
||||
class _CursorContext:
|
||||
def __init__(self, cursor) -> None:
|
||||
self.cursor = cursor
|
||||
|
||||
def __enter__(self):
|
||||
return self.cursor
|
||||
|
||||
def __exit__(self, exc_type, exc, tb):
|
||||
return False
|
||||
|
||||
conn = MagicMock()
|
||||
conn.cursor.side_effect = [
|
||||
_CursorContext(_Cursor(fetchone_values=[None])),
|
||||
_CursorContext(_Cursor(raise_on_schema=True)),
|
||||
_CursorContext(
|
||||
_Cursor(
|
||||
fetchone_values=[(f"public.{name}",) for name in ops_release_service._RELEASE_REQUIRED_TABLES],
|
||||
fetchall_values=[
|
||||
[(column,) for column in ops_release_service._RELEASE_REQUIRED_COLUMNS["ops_release_rollouts"]],
|
||||
],
|
||||
)
|
||||
),
|
||||
]
|
||||
db_ctx = MagicMock()
|
||||
db_ctx.__enter__.return_value = conn
|
||||
mock_get_db.return_value = db_ctx
|
||||
|
||||
previous_ready = ops_release_service._RELEASE_SCHEMA_READY
|
||||
ops_release_service._RELEASE_SCHEMA_READY = False
|
||||
try:
|
||||
ops_release_service.ensure_ops_release_schema()
|
||||
finally:
|
||||
ops_release_service._RELEASE_SCHEMA_READY = previous_ready
|
||||
|
||||
conn.rollback.assert_called_once()
|
||||
conn.commit.assert_not_called()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
@@ -10,6 +10,35 @@ class OpsServiceActivityTests(unittest.TestCase):
|
||||
def _runtime_status(detect: Optional[dict] = None) -> dict:
|
||||
return {"detect": dict(detect or {})}
|
||||
|
||||
@patch("app.services.ops_service.get_ops_runbook", return_value={"control_sequences": []})
|
||||
@patch("app.services.ops_service.list_release_rollouts", return_value=[])
|
||||
@patch("app.services.ops_service.list_ops_jobs", return_value=[])
|
||||
@patch("app.services.ops_service.get_recent_ops_playbook_runs", return_value={"runs": []})
|
||||
@patch("app.services.ops_service.list_managed_nodes_with_agent_state", side_effect=AssertionError("should reuse managed nodes"))
|
||||
@patch("app.services.ops_service.get_runtime_status", side_effect=AssertionError("should reuse runtime status"))
|
||||
def test_activity_stream_reuses_provided_runtime_and_managed_snapshots(
|
||||
self,
|
||||
_mock_get_runtime_status,
|
||||
_mock_list_managed_nodes_with_agent_state,
|
||||
_mock_get_recent_ops_playbook_runs,
|
||||
_mock_list_ops_jobs,
|
||||
_mock_list_release_rollouts,
|
||||
mock_get_ops_runbook,
|
||||
) -> None:
|
||||
runtime_snapshot = self._runtime_status()
|
||||
managed_snapshot = {"nodes": []}
|
||||
payload = get_ops_activity_stream(
|
||||
limit=10,
|
||||
scan_limit=20,
|
||||
runtime_status=runtime_snapshot,
|
||||
managed_nodes_payload=managed_snapshot,
|
||||
)
|
||||
|
||||
self.assertEqual([], payload["items"])
|
||||
self.assertEqual(0, payload["summary"]["total"])
|
||||
self.assertEqual(runtime_snapshot, mock_get_ops_runbook.call_args.kwargs["runtime_status"])
|
||||
self.assertEqual(managed_snapshot, mock_get_ops_runbook.call_args.kwargs["managed_nodes_payload"])
|
||||
|
||||
@patch("app.services.ops_service.get_ops_runbook")
|
||||
@patch("app.services.ops_service.list_release_rollouts")
|
||||
@patch("app.services.ops_service.list_ops_jobs")
|
||||
|
||||
27
domain-api/tests/test_redis_client.py
Normal file
27
domain-api/tests/test_redis_client.py
Normal file
@@ -0,0 +1,27 @@
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
from app.core.redis_client import get_redis, reset_redis_client_for_tests
|
||||
|
||||
|
||||
class ApiRedisClientTests(unittest.TestCase):
|
||||
def tearDown(self) -> None:
|
||||
reset_redis_client_for_tests()
|
||||
|
||||
@patch("app.core.redis_client.redis.Redis")
|
||||
@patch("app.core.redis_client.redis.BlockingConnectionPool")
|
||||
def test_get_redis_reuses_singleton_client(self, mock_pool, mock_redis) -> None:
|
||||
singleton = object()
|
||||
mock_redis.return_value = singleton
|
||||
|
||||
client_a = get_redis()
|
||||
client_b = get_redis()
|
||||
|
||||
self.assertIs(client_a, singleton)
|
||||
self.assertIs(client_b, singleton)
|
||||
mock_pool.assert_called_once()
|
||||
mock_redis.assert_called_once()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
93
domain-api/tests/test_runtime_api_routes.py
Normal file
93
domain-api/tests/test_runtime_api_routes.py
Normal file
@@ -0,0 +1,93 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
from app.api.routes import runtime as runtime_route
|
||||
|
||||
|
||||
class RuntimeApiRoutesTests(unittest.TestCase):
|
||||
@patch("app.api.routes.runtime.list_debug_events")
|
||||
def test_runtime_debug_events_forwards_node_code_filter(self, mock_list_debug_events) -> None:
|
||||
mock_list_debug_events.return_value = {"records": [], "has_more": False}
|
||||
|
||||
response = runtime_route.runtime_debug_events(node_code="mainland-controller-01-a", limit=20)
|
||||
|
||||
self.assertEqual(0, response.code)
|
||||
mock_list_debug_events.assert_called_once_with(
|
||||
limit=20,
|
||||
service=None,
|
||||
event_type=None,
|
||||
source_region=None,
|
||||
node_code="mainland-controller-01-a",
|
||||
level=None,
|
||||
before_id=None,
|
||||
after_id=None,
|
||||
created_after=None,
|
||||
)
|
||||
|
||||
@patch("app.api.routes.runtime.get_runtime_status")
|
||||
@patch("app.api.routes.runtime.get_sync_summary")
|
||||
@patch("app.api.routes.runtime.get_debug_handoff_report")
|
||||
def test_runtime_health_handover_filters_recent_issues_and_issue_groups_by_node_code(
|
||||
self,
|
||||
mock_get_debug_handoff_report,
|
||||
mock_get_sync_summary,
|
||||
mock_get_runtime_status,
|
||||
) -> None:
|
||||
mock_get_sync_summary.return_value = {"latest_record": {}}
|
||||
mock_get_runtime_status.return_value = {"readiness": {"status": "ready"}}
|
||||
mock_get_debug_handoff_report.return_value = {
|
||||
"overview": {
|
||||
"recent_issues": [
|
||||
{"node_code": "mainland-controller-01", "message": "keep"},
|
||||
{"node_code": "mainland-worker-01", "message": "drop"},
|
||||
]
|
||||
},
|
||||
"recent_issues": [
|
||||
{"node_code": "mainland-controller-01", "message": "keep"},
|
||||
{"node_code": "mainland-worker-01", "message": "drop"},
|
||||
],
|
||||
"issue_groups": [
|
||||
{"node_code": "mainland-controller-01", "message": "keep"},
|
||||
{"node_code": "mainland-worker-01", "message": "drop"},
|
||||
],
|
||||
"failure_handoff": {
|
||||
"recent_issues": [
|
||||
{"node_code": "mainland-controller-01", "message": "keep"},
|
||||
{"node_code": "mainland-worker-01", "message": "drop"},
|
||||
],
|
||||
"issue_groups": [
|
||||
{"node_code": "mainland-controller-01", "message": "keep"},
|
||||
{"node_code": "mainland-worker-01", "message": "drop"},
|
||||
],
|
||||
},
|
||||
}
|
||||
|
||||
response = runtime_route.runtime_health_handover(node_code="mainland-controller-01")
|
||||
|
||||
self.assertEqual(0, response.code)
|
||||
self.assertEqual(
|
||||
[{"node_code": "mainland-controller-01", "message": "keep"}],
|
||||
response.data["recent_issues"],
|
||||
)
|
||||
self.assertEqual(
|
||||
[{"node_code": "mainland-controller-01", "message": "keep"}],
|
||||
response.data["issue_groups"],
|
||||
)
|
||||
self.assertEqual(
|
||||
[{"node_code": "mainland-controller-01", "message": "keep"}],
|
||||
response.data["overview"]["recent_issues"],
|
||||
)
|
||||
self.assertEqual(
|
||||
[{"node_code": "mainland-controller-01", "message": "keep"}],
|
||||
response.data["failure_handoff"]["recent_issues"],
|
||||
)
|
||||
self.assertEqual(
|
||||
[{"node_code": "mainland-controller-01", "message": "keep"}],
|
||||
response.data["failure_handoff"]["issue_groups"],
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
37
domain-api/tests/test_runtime_control_service.py
Normal file
37
domain-api/tests/test_runtime_control_service.py
Normal file
@@ -0,0 +1,37 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import unittest
|
||||
from unittest.mock import patch
|
||||
|
||||
from app.services.runtime_control_service import runtime_action
|
||||
|
||||
|
||||
class RuntimeControlServiceTests(unittest.TestCase):
|
||||
@patch("app.services.runtime_control_service._emit_runtime_action_event")
|
||||
@patch("app.services.runtime_control_service.send_worker_command")
|
||||
def test_runtime_action_stop_detection_forwards_target_payload(
|
||||
self,
|
||||
mock_send_worker_command,
|
||||
_mock_emit_runtime_action_event,
|
||||
) -> None:
|
||||
mock_send_worker_command.return_value = (True, "已发送 Worker 控制指令")
|
||||
|
||||
ok, message, data = runtime_action(
|
||||
"stop_detection",
|
||||
payload={"target_node_codes": ["mainland-controller-01-a", "mainland-controller-01-b"]},
|
||||
)
|
||||
|
||||
self.assertTrue(ok)
|
||||
self.assertEqual("已发送 Worker 控制指令", message)
|
||||
mock_send_worker_command.assert_called_once_with(
|
||||
"stop_detection",
|
||||
payload={"target_node_codes": ["mainland-controller-01-a", "mainland-controller-01-b"]},
|
||||
)
|
||||
self.assertEqual(
|
||||
["mainland-controller-01-a", "mainland-controller-01-b"],
|
||||
data["payload"]["target_node_codes"],
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
77
domain-api/tests/test_runtime_settings_service.py
Normal file
77
domain-api/tests/test_runtime_settings_service.py
Normal file
@@ -0,0 +1,77 @@
|
||||
from __future__ import annotations
|
||||
|
||||
import json
|
||||
import unittest
|
||||
from unittest.mock import Mock, patch
|
||||
|
||||
from app.services.runtime_settings_service import update_runtime_settings
|
||||
|
||||
|
||||
class RuntimeSettingsServiceTests(unittest.TestCase):
|
||||
@patch("app.services.runtime_settings_service.get_redis")
|
||||
@patch("app.services.runtime_settings_service.write_runtime_json")
|
||||
@patch("app.services.runtime_settings_service.get_runtime_settings")
|
||||
def test_update_runtime_settings_publishes_runtime_settings_refresh(
|
||||
self,
|
||||
mock_get_runtime_settings,
|
||||
mock_write_runtime_json,
|
||||
mock_get_redis,
|
||||
) -> None:
|
||||
redis_client = Mock()
|
||||
mock_get_redis.return_value = redis_client
|
||||
mock_get_runtime_settings.return_value = {
|
||||
"worker_mode": "linux-systemd",
|
||||
"worker_service_name": "domaincheck-worker",
|
||||
"api_service_name": "domaincheck-api",
|
||||
"sync_agent_service_name": "domaincheck-sync-agent",
|
||||
"worker_log_sync_enabled": False,
|
||||
"worker_log_sync_mode": "key",
|
||||
"control_node_autoresume_enabled": False,
|
||||
"claim_recent_jobs_first": False,
|
||||
"claim_recent_jobs_limit": 8,
|
||||
"claim_recent_jobs_window_hours": 24,
|
||||
"claim_batch_floor": 0,
|
||||
"claim_batch_ceil": 0,
|
||||
"submit_backlog_floor": 0,
|
||||
"submit_backlog_ceil": 0,
|
||||
"dispatch_cap_multiplier": 1,
|
||||
"pending_buffer_cap_multiplier": 1,
|
||||
}
|
||||
|
||||
updated = update_runtime_settings(
|
||||
{
|
||||
"worker_log_sync_enabled": True,
|
||||
"worker_log_sync_mode": "full",
|
||||
"control_node_autoresume_enabled": True,
|
||||
"claim_recent_jobs_first": True,
|
||||
"claim_recent_jobs_limit": 6,
|
||||
"claim_recent_jobs_window_hours": 72,
|
||||
"claim_batch_floor": 200,
|
||||
"claim_batch_ceil": 800,
|
||||
"submit_backlog_floor": 500,
|
||||
"submit_backlog_ceil": 1500,
|
||||
"dispatch_cap_multiplier": 2,
|
||||
"pending_buffer_cap_multiplier": 2,
|
||||
}
|
||||
)
|
||||
|
||||
mock_write_runtime_json.assert_called_once_with("runtime_settings.json", updated)
|
||||
redis_client.set.assert_called_once()
|
||||
redis_key, serialized = redis_client.set.call_args.args
|
||||
self.assertEqual("domain_tool:runtime_settings", redis_key)
|
||||
self.assertEqual(updated, json.loads(serialized))
|
||||
self.assertTrue(updated["control_node_autoresume_enabled"])
|
||||
self.assertTrue(updated["claim_recent_jobs_first"])
|
||||
self.assertEqual(6, updated["claim_recent_jobs_limit"])
|
||||
self.assertEqual(72, updated["claim_recent_jobs_window_hours"])
|
||||
self.assertEqual(200, updated["claim_batch_floor"])
|
||||
self.assertEqual(800, updated["claim_batch_ceil"])
|
||||
self.assertEqual(500, updated["submit_backlog_floor"])
|
||||
self.assertEqual(1500, updated["submit_backlog_ceil"])
|
||||
self.assertEqual(2, updated["dispatch_cap_multiplier"])
|
||||
self.assertEqual(2, updated["pending_buffer_cap_multiplier"])
|
||||
redis_client.publish.assert_called_once_with("domain_tool:config_update", "runtime_settings")
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
@@ -1,10 +1,273 @@
|
||||
import unittest
|
||||
from datetime import datetime, timedelta
|
||||
from unittest.mock import MagicMock, patch
|
||||
|
||||
import app.services.runtime_status_service as runtime_status_service_module
|
||||
from app.services.detect_service import _slice_remote_log_lines_fairly
|
||||
from app.services.runtime_status_service import _align_queue_health_with_backlog, _build_detect_node_row
|
||||
from app.services.runtime_status_service import (
|
||||
_align_queue_health_with_backlog,
|
||||
_build_detect_node_row,
|
||||
_build_detect_observation_summary,
|
||||
refresh_runtime_projection_snapshot,
|
||||
get_runtime_status,
|
||||
)
|
||||
|
||||
|
||||
class RuntimeStatusServiceTests(unittest.TestCase):
|
||||
def setUp(self) -> None:
|
||||
runtime_status_service_module._DOMAIN_INVENTORY_CACHE_VALUE = None
|
||||
runtime_status_service_module._DOMAIN_INVENTORY_CACHE_EXPIRES_AT = 0.0
|
||||
runtime_status_service_module._RUNTIME_STATUS_CACHE_VALUE = None
|
||||
runtime_status_service_module._RUNTIME_STATUS_CACHE_EXPIRES_AT = 0.0
|
||||
runtime_status_service_module._RUNTIME_STATUS_CACHE_SIGNATURE = ()
|
||||
|
||||
def test_build_detect_observation_summary_marks_real_running_state(self) -> None:
|
||||
summary = _build_detect_observation_summary(
|
||||
detect_payload={
|
||||
"active_job": {
|
||||
"job_id": 308,
|
||||
"job_code": "sync-overseas-16432",
|
||||
"status": "running",
|
||||
"progress_percent": 24.6,
|
||||
"items_pending": 3999,
|
||||
"items_claimed": 52,
|
||||
"items_running": 136,
|
||||
"items_completed": 971,
|
||||
"items_failed": 0,
|
||||
"items_blacklisted": 0,
|
||||
"display_items_running": 222,
|
||||
"display_active_threads": 222,
|
||||
"display_max_threads": 2000,
|
||||
"processed_recent": 172,
|
||||
"processed_per_minute": 11.47,
|
||||
"completed_recent": 2,
|
||||
"failed_recent": 169,
|
||||
"blacklisted_recent": 1,
|
||||
},
|
||||
"active_thread_count": 222,
|
||||
"max_thread_count": 2000,
|
||||
"queue_health": {
|
||||
"queue": {
|
||||
"pending": 3999,
|
||||
"claimed": 52,
|
||||
"running": 136,
|
||||
"display_running": 222,
|
||||
"completed": 971,
|
||||
"failed": 0,
|
||||
"blacklisted": 0,
|
||||
},
|
||||
"throughput": {
|
||||
"processed_recent": 172,
|
||||
"processed_per_minute": 11.47,
|
||||
"completed_recent": 2,
|
||||
"failed_recent": 169,
|
||||
"blacklisted_recent": 1,
|
||||
},
|
||||
"steps": [
|
||||
{
|
||||
"step_code": "detect_register",
|
||||
"step_name": "注册检测",
|
||||
"items_pending": 2100,
|
||||
"items_running": 40,
|
||||
"processed_recent": 88,
|
||||
"failed_recent": 100,
|
||||
"blacklisted_recent": 0,
|
||||
}
|
||||
],
|
||||
},
|
||||
"participating_nodes": [
|
||||
{
|
||||
"node_code": "mainland-controller-01-a",
|
||||
"participation_label": "执行中",
|
||||
"is_dispatch_active": True,
|
||||
"items_claimed": 12,
|
||||
"items_running": 100,
|
||||
"active_threads": 100,
|
||||
"max_threads": 1000,
|
||||
"processed_recent": 80,
|
||||
"processed_per_minute": 5.33,
|
||||
},
|
||||
{
|
||||
"node_code": "mainland-controller-01-b",
|
||||
"participation_label": "执行中",
|
||||
"is_dispatch_active": True,
|
||||
"items_claimed": 8,
|
||||
"items_running": 122,
|
||||
"active_threads": 122,
|
||||
"max_threads": 1000,
|
||||
"processed_recent": 92,
|
||||
"processed_per_minute": 6.13,
|
||||
},
|
||||
],
|
||||
"participation_summary": {
|
||||
"dispatch_active_nodes": 2,
|
||||
"participating_nodes": 2,
|
||||
},
|
||||
"backlog": {
|
||||
"pending_total": 3999,
|
||||
"claimed_total": 52,
|
||||
"running_total": 136,
|
||||
"completed_total": 971,
|
||||
},
|
||||
},
|
||||
cluster_snapshot={"summary": {"online_worker_nodes": 1}},
|
||||
inventory_summary={
|
||||
"scope_label": "海外主库总盘子",
|
||||
"authoritative": True,
|
||||
"domains_total": 5690937,
|
||||
"pending_total": 5690918,
|
||||
"completed_total": 0,
|
||||
"running_total": 7,
|
||||
"blacklist_total": 0,
|
||||
"failed_total": 12,
|
||||
"processed_total": 12,
|
||||
"remaining_total": 5690925,
|
||||
},
|
||||
)
|
||||
|
||||
self.assertEqual("running", summary["state"])
|
||||
self.assertEqual("真跑中", summary["state_label"])
|
||||
self.assertEqual(2, summary["execution"]["active_processes"])
|
||||
self.assertEqual(222, summary["execution"]["active_threads"])
|
||||
self.assertEqual(169, summary["throughput"]["failed_recent"])
|
||||
self.assertEqual("detect_register", summary["top_steps"][0]["step_code"])
|
||||
self.assertEqual(5690937, summary["source_inventory"]["domains_total"])
|
||||
self.assertEqual(5244, summary["active_batch"]["effective_items_total"])
|
||||
self.assertIn("不等于全盘累计", summary["scope_hint"])
|
||||
|
||||
def test_build_detect_observation_summary_marks_not_running_when_only_backlog_left(self) -> None:
|
||||
summary = _build_detect_observation_summary(
|
||||
detect_payload={
|
||||
"active_job": {
|
||||
"job_id": 401,
|
||||
"job_code": "sync-overseas-404",
|
||||
"status": "running",
|
||||
"items_pending": 5000,
|
||||
"items_claimed": 0,
|
||||
"items_running": 0,
|
||||
"items_completed": 0,
|
||||
"items_failed": 0,
|
||||
"items_blacklisted": 0,
|
||||
},
|
||||
"queue_health": {
|
||||
"queue": {
|
||||
"pending": 5000,
|
||||
"claimed": 0,
|
||||
"running": 0,
|
||||
"completed": 0,
|
||||
"failed": 0,
|
||||
"blacklisted": 0,
|
||||
},
|
||||
"throughput": {
|
||||
"processed_recent": 0,
|
||||
"processed_per_minute": 0,
|
||||
"completed_recent": 0,
|
||||
"failed_recent": 0,
|
||||
"blacklisted_recent": 0,
|
||||
},
|
||||
"steps": [
|
||||
{
|
||||
"step_code": "detect_register",
|
||||
"step_name": "注册检测",
|
||||
"items_pending": 5000,
|
||||
"items_running": 0,
|
||||
}
|
||||
],
|
||||
},
|
||||
"participating_nodes": [],
|
||||
"participation_summary": {
|
||||
"dispatch_active_nodes": 0,
|
||||
"participating_nodes": 0,
|
||||
},
|
||||
"backlog": {
|
||||
"pending_total": 5000,
|
||||
"claimed_total": 0,
|
||||
"running_total": 0,
|
||||
},
|
||||
},
|
||||
cluster_snapshot={"summary": {"online_worker_nodes": 1}},
|
||||
inventory_summary={
|
||||
"scope_label": "海外主库总盘子",
|
||||
"authoritative": True,
|
||||
"domains_total": 5690937,
|
||||
"pending_total": 5690918,
|
||||
"completed_total": 0,
|
||||
"running_total": 7,
|
||||
"blacklist_total": 0,
|
||||
"failed_total": 12,
|
||||
"processed_total": 12,
|
||||
"remaining_total": 5690925,
|
||||
},
|
||||
)
|
||||
|
||||
self.assertEqual("not_running", summary["state"])
|
||||
self.assertEqual("没跑起来", summary["state_label"])
|
||||
self.assertEqual(0, summary["execution"]["active_processes"])
|
||||
self.assertIn("队列", summary["focus_hint"])
|
||||
self.assertEqual(5000, summary["active_batch"]["effective_items_total"])
|
||||
self.assertEqual(5690937, summary["source_inventory"]["domains_total"])
|
||||
|
||||
@patch("app.services.runtime_status_service.get_active_detect_job_summary")
|
||||
@patch("app.services.runtime_status_service.get_db")
|
||||
def test_load_detect_backlog_snapshot_prefers_active_job_snapshot(
|
||||
self,
|
||||
mock_get_db,
|
||||
mock_get_active_detect_job_summary,
|
||||
) -> None:
|
||||
mock_get_active_detect_job_summary.return_value = {
|
||||
"items_pending": 321,
|
||||
"items_claimed": 2,
|
||||
"items_running": 9,
|
||||
"items_completed": 50,
|
||||
"items_failed": 3,
|
||||
"display_active_threads": 11,
|
||||
"step_stats": [
|
||||
{"step_code": "detect_register", "items_pending": 210},
|
||||
{"step_code": "detect_baidu", "items_pending": 111},
|
||||
],
|
||||
}
|
||||
|
||||
backlog = runtime_status_service_module._load_detect_backlog_snapshot()
|
||||
|
||||
self.assertEqual(321, backlog["pending_total"])
|
||||
self.assertEqual(2, backlog["claimed_total"])
|
||||
self.assertEqual(11, backlog["running_total"])
|
||||
self.assertEqual(50, backlog["completed_total"])
|
||||
self.assertEqual(3, backlog["failed_total"])
|
||||
self.assertEqual(210, backlog["register_pending"])
|
||||
self.assertEqual(111, backlog["downstream_pending"])
|
||||
mock_get_db.assert_not_called()
|
||||
|
||||
@patch("app.services.runtime_status_service.get_active_detect_job_summary", return_value={})
|
||||
@patch("app.services.runtime_status_service.get_db")
|
||||
def test_load_detect_backlog_snapshot_ignores_stale_pending_jobs(
|
||||
self,
|
||||
mock_get_db,
|
||||
_mock_get_active_detect_job_summary,
|
||||
) -> None:
|
||||
conn = MagicMock()
|
||||
cursor_cm = MagicMock()
|
||||
cursor = MagicMock()
|
||||
cursor.fetchall.return_value = [
|
||||
(11, "running", datetime.now() - timedelta(hours=2)),
|
||||
(10, "pending", datetime.now() - timedelta(minutes=30)),
|
||||
(9, "pending", datetime.now() - timedelta(hours=8)),
|
||||
]
|
||||
cursor.fetchone.return_value = (321, 2, 9, 50, 1, 3, 210, 111)
|
||||
conn.cursor.return_value = cursor_cm
|
||||
cursor_cm.__enter__.return_value = cursor
|
||||
db_cm = MagicMock()
|
||||
db_cm.__enter__.return_value = conn
|
||||
mock_get_db.return_value = db_cm
|
||||
|
||||
backlog = runtime_status_service_module._load_detect_backlog_snapshot()
|
||||
|
||||
self.assertEqual(321, backlog["pending_total"])
|
||||
self.assertEqual(210, backlog["register_pending"])
|
||||
executed_sql, executed_params = cursor.execute.call_args_list[1][0]
|
||||
self.assertIn("WHERE item.job_id = ANY(%s)", executed_sql)
|
||||
self.assertEqual([11, 10], list(executed_params[0]))
|
||||
|
||||
def test_build_detect_node_row_merges_queue_running_metrics(self) -> None:
|
||||
row = _build_detect_node_row(
|
||||
node_code="mainland-worker-01",
|
||||
@@ -92,6 +355,42 @@ class RuntimeStatusServiceTests(unittest.TestCase):
|
||||
self.assertTrue(any("[mainland-controller-01]" in line for line in sliced))
|
||||
self.assertTrue(any("[mainland-worker-01]" in line for line in sliced))
|
||||
|
||||
def test_build_multi_region_readiness_skips_local_batch_warning_when_batches_are_not_applicable(self) -> None:
|
||||
with patch("app.services.runtime_status_service.settings.node_region", "overseas"), \
|
||||
patch("app.services.runtime_status_service.settings.node_role", "control"):
|
||||
readiness = runtime_status_service_module._build_multi_region_readiness(
|
||||
cluster_snapshot={
|
||||
"nodes": [
|
||||
{"node_code": "overseas-control-01", "region": "overseas", "role": "control", "status": "online"},
|
||||
{"node_code": "mainland-controller-01", "region": "mainland", "role": "control", "status": "online"},
|
||||
{"node_code": "mainland-worker-01", "region": "mainland", "role": "worker", "status": "busy", "is_effective_worker": True},
|
||||
],
|
||||
"summary": {
|
||||
"online_control_nodes": 2,
|
||||
"online_worker_nodes": 1,
|
||||
},
|
||||
},
|
||||
sync_summary={
|
||||
"enabled": False,
|
||||
"source_region": "overseas",
|
||||
"target_region": "mainland",
|
||||
"detect_result_batches": {
|
||||
"applicable": False,
|
||||
"reason": "当前节点不承载本地检测执行,结果批次推送概览不适用。",
|
||||
"state_counts": {"projected": 3, "failed": 1, "pushing": 1, "synced": 0},
|
||||
},
|
||||
},
|
||||
worker_runtime={"running": False},
|
||||
sync_agent_runtime={"running": False},
|
||||
)
|
||||
|
||||
warning_text = " ".join(readiness["warnings"])
|
||||
info_text = " ".join(readiness["info"])
|
||||
self.assertNotIn("结果批次仍待推送", warning_text)
|
||||
self.assertNotIn("结果批次同步失败", warning_text)
|
||||
self.assertIn("不适用", info_text)
|
||||
self.assertFalse(readiness["sync"]["applicable"])
|
||||
|
||||
def test_align_queue_health_with_backlog_prefers_larger_runtime_snapshot(self) -> None:
|
||||
aligned = _align_queue_health_with_backlog(
|
||||
{
|
||||
@@ -126,6 +425,227 @@ class RuntimeStatusServiceTests(unittest.TestCase):
|
||||
self.assertEqual(8, aligned["queue"]["failed"])
|
||||
self.assertEqual(1790, aligned["queue"]["items_total"])
|
||||
|
||||
def test_load_latest_remote_runtime_projection_backlog_skips_future_dated_rows(self) -> None:
|
||||
now = datetime.now()
|
||||
conn = MagicMock()
|
||||
cursor_cm = MagicMock()
|
||||
cursor = MagicMock()
|
||||
cursor.fetchall.return_value = [
|
||||
({"projection": {"backlog": {"pending_total": 9999}}}, now + timedelta(hours=4)),
|
||||
({"projection": {"backlog": {"pending_total": 321, "running_total": 8}}}, now - timedelta(minutes=2)),
|
||||
]
|
||||
conn.cursor.return_value = cursor_cm
|
||||
cursor_cm.__enter__.return_value = cursor
|
||||
db_cm = MagicMock()
|
||||
db_cm.__enter__.return_value = conn
|
||||
|
||||
with patch("app.services.runtime_status_service.settings.node_region", "overseas"), \
|
||||
patch("app.services.runtime_status_service.settings.node_role", "control"), \
|
||||
patch("app.services.runtime_status_service.get_db", return_value=db_cm):
|
||||
backlog = runtime_status_service_module._load_latest_remote_runtime_projection_backlog()
|
||||
|
||||
self.assertEqual(321, backlog["pending_total"])
|
||||
self.assertEqual(8, backlog["running_total"])
|
||||
|
||||
def test_get_runtime_status_keeps_remote_aggregate_detect_metrics_on_overseas_control(self) -> None:
|
||||
with patch("app.services.runtime_status_service.settings.node_region", "overseas"), \
|
||||
patch("app.services.runtime_status_service.settings.node_role", "control"), \
|
||||
patch("app.services.runtime_status_service.detect_worker_runtime", return_value={"running": False, "mode": "linux-systemd", "process_count": 0, "message": "inactive", "latest_start_time": ""}), \
|
||||
patch("app.services.runtime_status_service.detect_sync_agent_runtime", return_value={"running": False, "mode": "linux-systemd", "process_count": 0, "message": "inactive", "latest_start_time": ""}), \
|
||||
patch("app.services.runtime_status_service.get_runtime_settings", return_value={"worker_mode": "linux-systemd"}), \
|
||||
patch("app.services.runtime_status_service.get_detect_status", return_value={
|
||||
"phase_label": "",
|
||||
"phase_detail": "",
|
||||
"recent_event": "远端执行中",
|
||||
"recent_warning": "",
|
||||
"progress_percent": 40.49,
|
||||
"progress": {"pending": 5001, "running": 138, "completed": 0, "failed": 0, "blacklisted": 0},
|
||||
"active_thread_count": 138,
|
||||
"max_thread_count": 80000,
|
||||
"aggregate_process_count": 80,
|
||||
"aggregate_participating_node_count": 1,
|
||||
"aggregate_participating_node_codes": ["mainland-controller-01"],
|
||||
"aggregate_max_thread_count": 80000,
|
||||
"aggregate_thread_count_per_process": 1000,
|
||||
"available_proxy_count": 0,
|
||||
"proxy_pool_count": 0,
|
||||
"proxy_runtime_label": "",
|
||||
"proxy_runtime_detail": "",
|
||||
"proxy_runtime_reason": "",
|
||||
"proxy_supplier_empty": False,
|
||||
"proxy_last_refresh_status": "",
|
||||
"proxy_last_refresh_time": "",
|
||||
"proxy_last_refresh_source_count": 0,
|
||||
"proxy_last_refresh_total_items": 0,
|
||||
"proxy_last_validated_count": 0,
|
||||
"proxy_last_available_count": 0,
|
||||
"proxy_source_stats": [],
|
||||
"dependency_alerts": [],
|
||||
"active_job": {"job_code": "sync-overseas-51", "items_pending": 5001, "items_running": 0},
|
||||
"runs": [],
|
||||
"remote_log_line_count": 0,
|
||||
"remote_log_node_count": 0,
|
||||
"remote_log_nodes": [],
|
||||
"remote_log_node_summaries": [],
|
||||
"remote_log_last_at": "",
|
||||
"remote_log_last_line": "",
|
||||
"remote_log_lines": [],
|
||||
"aggregate_detect_view": True,
|
||||
}), \
|
||||
patch("app.services.runtime_status_service.get_cluster_snapshot", return_value={
|
||||
"nodes": [{"node_code": "mainland-controller-01", "role": "control", "region": "mainland", "status": "busy", "is_effective_worker": True, "current_load": 1432, "metadata": {"active_threads": 138, "max_threads": 80000}}],
|
||||
"summary": {"online_worker_nodes": 1, "dedicated_online_worker_nodes": 0, "online_control_nodes": 2},
|
||||
}), \
|
||||
patch("app.services.runtime_status_service.get_detect_queue_health", return_value={
|
||||
"has_active_job": True,
|
||||
"job": {"job_id": 11, "job_code": "sync-overseas-51", "status": "running", "progress_percent": 40.49},
|
||||
"queue": {"items_total": 8402, "pending": 5001, "claimed": 0, "running": 1432, "display_running": 1432},
|
||||
"nodes": [{"node_code": "mainland-controller-01-a", "items_running": 1432, "display_running": 1432, "active_threads": 1432, "max_threads": 1000}],
|
||||
"steps": [],
|
||||
"runtime_activity": {},
|
||||
"window_minutes": 15,
|
||||
}), \
|
||||
patch("app.services.runtime_status_service._load_detect_backlog_snapshot", return_value={"pending_total": 5001, "claimed_total": 0, "running_total": 0, "completed_total": 0, "blacklisted_total": 0, "failed_total": 0, "register_pending": 3495, "downstream_pending": 1506}), \
|
||||
patch("app.services.runtime_status_service._load_latest_remote_runtime_projection_backlog", return_value={}), \
|
||||
patch("app.services.runtime_status_service._load_latest_runtime_active_job_snapshot", return_value={}), \
|
||||
patch("app.services.runtime_status_service._load_domain_inventory_summary", return_value={"scope_label": "海外主库总盘子", "authoritative": True, "domains_total": 5690937, "pending_total": 5690918, "completed_total": 0, "running_total": 7, "blacklist_total": 0, "failed_total": 12, "processed_total": 12, "remaining_total": 5690925}), \
|
||||
patch("app.services.runtime_status_service.get_detect_capacity_plan", return_value={}), \
|
||||
patch("app.services.runtime_status_service.append_runtime_projection_if_changed"), \
|
||||
patch("app.services.runtime_status_service.get_sync_summary", return_value={}), \
|
||||
patch("app.services.runtime_status_service._build_multi_region_readiness", return_value={"status": "ready", "ready": True}), \
|
||||
patch("app.services.runtime_status_service.get_runtime_build_info", return_value={}):
|
||||
payload = get_runtime_status()
|
||||
|
||||
self.assertEqual(1, payload["detect"]["aggregate_process_count"])
|
||||
self.assertEqual(1432, payload["detect"]["active_thread_count"])
|
||||
self.assertEqual(1000, payload["detect"]["max_thread_count"])
|
||||
self.assertEqual("集群执行中", payload["detect"]["phase_label"])
|
||||
self.assertFalse(payload["detect"]["worker_online"])
|
||||
self.assertEqual("sync-overseas-51", payload["detect"]["active_job"]["job_code"])
|
||||
self.assertIn("observation_summary", payload["detect"])
|
||||
self.assertEqual("在跑但偏慢", payload["detect"]["observation_summary"]["state_label"])
|
||||
self.assertEqual(5690937, payload["detect"]["observation_summary"]["source_inventory"]["domains_total"])
|
||||
|
||||
def test_get_runtime_status_reuses_short_ttl_cache(self) -> None:
|
||||
context_payload = {
|
||||
"runtime_settings": {
|
||||
"worker_mode": "linux-systemd",
|
||||
"api_service_name": "domaincheck-api",
|
||||
"worker_service_name": "domaincheck-worker",
|
||||
"sync_agent_service_name": "domaincheck-sync-agent",
|
||||
},
|
||||
"worker_runtime": {
|
||||
"running": True,
|
||||
"mode": "linux-systemd",
|
||||
"process_count": 60,
|
||||
"latest_start_time": "2026-04-24 17:10:28",
|
||||
"message": "running",
|
||||
"runtime_state": {"phase": "detecting"},
|
||||
},
|
||||
"worker_expected_on_this_node": True,
|
||||
"detect_payload": {
|
||||
"_detect_snapshot": {"thread_count": 1000},
|
||||
"active_thread_count": 321,
|
||||
"max_thread_count": 60000,
|
||||
"progress": {"pending": 10, "running": 3},
|
||||
"backlog": {"pending_total": 10, "running_total": 3},
|
||||
"progress_percent": 25.5,
|
||||
"available_proxy_count": 12,
|
||||
"proxy_pool_count": 20,
|
||||
"proxy_runtime_label": "代理正常",
|
||||
"proxy_runtime_detail": "ok",
|
||||
"proxy_runtime_reason": "healthy",
|
||||
"proxy_last_refresh_time": "2026-04-24 17:12:55",
|
||||
"recent_event": "running",
|
||||
"recent_warning": "",
|
||||
},
|
||||
"cluster_snapshot": {"summary": {"online_worker_nodes": 1}},
|
||||
}
|
||||
sync_summary = {"enabled": True}
|
||||
readiness = {"status": "ready", "ready": True}
|
||||
|
||||
with patch("app.services.runtime_status_service._build_runtime_detect_context", return_value=context_payload) as mock_context, \
|
||||
patch("app.services.runtime_status_service.detect_sync_agent_runtime", return_value={"running": True, "mode": "linux-systemd", "process_count": 1, "latest_start_time": "", "message": "ok"}), \
|
||||
patch("app.services.runtime_status_service.append_runtime_projection_if_changed") as mock_append, \
|
||||
patch("app.services.runtime_status_service.get_sync_summary", return_value=sync_summary) as mock_sync_summary, \
|
||||
patch("app.services.runtime_status_service._build_multi_region_readiness", return_value=readiness), \
|
||||
patch("app.services.runtime_status_service.get_runtime_build_info", return_value={"package_name": "pkg"}), \
|
||||
patch("app.services.runtime_status_service.time.monotonic", side_effect=[100.0, 100.1, 100.2]):
|
||||
first = get_runtime_status()
|
||||
second = get_runtime_status()
|
||||
|
||||
self.assertEqual(first, second)
|
||||
self.assertEqual(1, mock_context.call_count)
|
||||
self.assertEqual(1, mock_append.call_count)
|
||||
self.assertEqual(1, mock_sync_summary.call_count)
|
||||
|
||||
@patch("app.services.runtime_status_service.append_runtime_projection_if_changed", return_value=321)
|
||||
@patch(
|
||||
"app.services.runtime_status_service.get_cluster_snapshot",
|
||||
return_value={"nodes": [], "summary": {"online_worker_nodes": 3}},
|
||||
)
|
||||
@patch(
|
||||
"app.services.runtime_status_service.get_active_detect_job_summary",
|
||||
return_value={
|
||||
"job_id": 867,
|
||||
"job_code": "sync-overseas-19835",
|
||||
"status": "running",
|
||||
"progress_percent": 99.1,
|
||||
"items_total": 10000,
|
||||
"items_pending": 4,
|
||||
"items_claimed": 0,
|
||||
"items_running": 5,
|
||||
"items_completed": 55402,
|
||||
"items_blacklisted": 0,
|
||||
"items_failed": 0,
|
||||
"display_items_running": 5,
|
||||
"display_active_threads": 5,
|
||||
"display_items_claimed": 0,
|
||||
"display_max_threads": 1000,
|
||||
"node_stats": [{"node_code": "mainland-controller-01", "items_running": 5}],
|
||||
"raw_step_stats": [{"step_code": "detect_wayback", "items_pending": 4}],
|
||||
},
|
||||
)
|
||||
@patch(
|
||||
"app.services.runtime_status_service.detect_worker_runtime",
|
||||
return_value={"running": True, "mode": "linux-systemd", "process_count": 60, "thread_count": 1000, "max_threads": 60000, "message": "ok"},
|
||||
)
|
||||
@patch("app.services.runtime_status_service.get_runtime_settings", return_value={"thread_count": 1000, "worker_mode": "linux-systemd"})
|
||||
@patch(
|
||||
"app.services.runtime_status_service._load_detect_backlog_snapshot",
|
||||
return_value={
|
||||
"pending_total": 4,
|
||||
"claimed_total": 0,
|
||||
"running_total": 5,
|
||||
"completed_total": 55402,
|
||||
"blacklisted_total": 0,
|
||||
"failed_total": 0,
|
||||
"register_pending": 0,
|
||||
"downstream_pending": 4,
|
||||
},
|
||||
)
|
||||
@patch("app.services.runtime_status_service._build_runtime_detect_context", side_effect=AssertionError("heavy path should not run"))
|
||||
def test_refresh_runtime_projection_snapshot_uses_lightweight_context(
|
||||
self,
|
||||
mock_heavy_context,
|
||||
mock_load_detect_backlog_snapshot,
|
||||
mock_get_runtime_settings,
|
||||
mock_detect_worker_runtime,
|
||||
mock_get_active_detect_job_summary,
|
||||
mock_get_cluster_snapshot,
|
||||
mock_append_runtime_projection_if_changed,
|
||||
) -> None:
|
||||
result = refresh_runtime_projection_snapshot()
|
||||
|
||||
self.assertEqual(321, result["record_id"])
|
||||
mock_append_runtime_projection_if_changed.assert_called_once()
|
||||
detect = mock_append_runtime_projection_if_changed.call_args.kwargs["detect"]
|
||||
self.assertEqual("sync-overseas-19835", detect["active_job"]["job_code"])
|
||||
self.assertEqual(5, detect["active_thread_count"])
|
||||
self.assertEqual(4, detect["backlog"]["pending_total"])
|
||||
self.assertEqual(5, detect["queue_health"]["queue"]["running"])
|
||||
mock_heavy_context.assert_not_called()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
@@ -1,6 +1,11 @@
|
||||
import unittest
|
||||
|
||||
from app.services.settings_service import _normalize_thread_count, resolve_thread_count
|
||||
from app.services.settings_service import (
|
||||
_normalize_process_count,
|
||||
_normalize_thread_count,
|
||||
resolve_process_count,
|
||||
resolve_thread_count,
|
||||
)
|
||||
|
||||
|
||||
class SettingsServiceTests(unittest.TestCase):
|
||||
@@ -26,6 +31,25 @@ class SettingsServiceTests(unittest.TestCase):
|
||||
self.assertEqual(512, resolved["effective_thread_count"])
|
||||
self.assertEqual("node_override", resolved["source"])
|
||||
|
||||
def test_normalize_process_count_rejects_non_positive_values(self) -> None:
|
||||
with self.assertRaises(ValueError):
|
||||
_normalize_process_count(0)
|
||||
|
||||
def test_resolve_process_count_uses_large_node_override(self) -> None:
|
||||
payload = {
|
||||
"process_count": 80,
|
||||
"node_process_counts": {
|
||||
"mainland-controller-01": 96,
|
||||
},
|
||||
}
|
||||
|
||||
resolved = resolve_process_count(node_code="mainland-controller-01", settings_payload=payload)
|
||||
|
||||
self.assertEqual(80, resolved["default_process_count"])
|
||||
self.assertEqual(96, resolved["override_process_count"])
|
||||
self.assertEqual(96, resolved["effective_process_count"])
|
||||
self.assertEqual("node_override", resolved["source"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
@@ -4,6 +4,9 @@ from unittest.mock import patch
|
||||
from app.sync_agent import (
|
||||
_append_detect_result_projection_snapshot,
|
||||
_build_aligned_queue_health_snapshot,
|
||||
_emit_runtime_debug_snapshots,
|
||||
_maybe_trigger_overlap_start,
|
||||
_run_sync_tick_once,
|
||||
_emit_structured_tick,
|
||||
_emit_sync_result_breakdown,
|
||||
_filter_runtime_events_for_job,
|
||||
@@ -187,6 +190,140 @@ class SyncAgentTests(unittest.TestCase):
|
||||
mock_process_detect_pipeline_now.assert_called_once_with(limit=5000)
|
||||
self.assertEqual("pipeline_tick_success", mock_push_debug_event.call_args.kwargs["event_type"])
|
||||
|
||||
@patch("app.sync_agent._load_local_detect_backlog_snapshot", return_value={"pending_total": 12})
|
||||
@patch("app.sync_agent.list_recent_detect_run_events", return_value=[])
|
||||
@patch("app.sync_agent.get_detect_queue_health", return_value={"queue": {"overdue_leases": 0}})
|
||||
@patch("app.sync_agent.push_debug_event")
|
||||
@patch("app.sync_agent._append_detect_result_projection_snapshot")
|
||||
@patch("app.sync_agent._select_projection_job_snapshots", return_value=[{"job_id": 1, "job_code": "finished-job"}])
|
||||
@patch(
|
||||
"app.sync_agent.get_active_detect_job_summary",
|
||||
return_value={
|
||||
"job_id": 2,
|
||||
"job_code": "running-job",
|
||||
"status": "running",
|
||||
"items_total": 100,
|
||||
"items_pending": 10,
|
||||
"items_claimed": 5,
|
||||
"items_running": 7,
|
||||
"items_completed": 70,
|
||||
"items_failed": 8,
|
||||
"progress_percent": 70.0,
|
||||
"node_stats": [{"node_code": "mainland-controller-01", "items_total": 90}],
|
||||
},
|
||||
)
|
||||
def test_emit_runtime_debug_snapshots_emits_projection_and_active_job(
|
||||
self,
|
||||
mock_get_active_detect_job_summary,
|
||||
mock_select_projection_job_snapshots,
|
||||
mock_append_detect_result_projection_snapshot,
|
||||
mock_push_debug_event,
|
||||
mock_get_detect_queue_health,
|
||||
mock_list_recent_detect_run_events,
|
||||
mock_load_local_detect_backlog_snapshot,
|
||||
) -> None:
|
||||
_emit_runtime_debug_snapshots()
|
||||
|
||||
mock_get_active_detect_job_summary.assert_called_once_with(event_limit=10)
|
||||
mock_select_projection_job_snapshots.assert_called_once_with()
|
||||
mock_append_detect_result_projection_snapshot.assert_called_once()
|
||||
self.assertTrue(mock_push_debug_event.called)
|
||||
self.assertEqual("active_job_snapshot", mock_push_debug_event.call_args_list[0].kwargs["event_type"])
|
||||
|
||||
@patch("app.sync_agent._emit_runtime_debug_snapshots")
|
||||
@patch("app.sync_agent._run_pipeline_stage_processor", return_value=(True, "pipeline ok", {"stage": "pipeline"}))
|
||||
@patch("app.sync_agent.pull_detect_task_batch_now", return_value=(True, "pull ok", {"stage": "pull"}))
|
||||
@patch("app.sync_agent.push_runtime_projection_now", return_value=(True, "sync ok", {"stage": "sync"}))
|
||||
def test_run_sync_tick_once_prioritizes_sync_before_pipeline(
|
||||
self,
|
||||
mock_push_runtime_projection_now,
|
||||
mock_pull_detect_task_batch_now,
|
||||
mock_run_pipeline_stage_processor,
|
||||
mock_emit_runtime_debug_snapshots,
|
||||
) -> None:
|
||||
with patch("app.sync_agent._maybe_trigger_overlap_start", return_value=(False, "no overlap", {})) as mock_overlap:
|
||||
tick = _run_sync_tick_once()
|
||||
|
||||
self.assertTrue(tick["sync"]["ok"])
|
||||
self.assertEqual("sync ok", tick["sync"]["message"])
|
||||
mock_push_runtime_projection_now.assert_called_once_with()
|
||||
mock_pull_detect_task_batch_now.assert_called_once_with()
|
||||
mock_run_pipeline_stage_processor.assert_called_once_with()
|
||||
mock_overlap.assert_called_once_with()
|
||||
mock_emit_runtime_debug_snapshots.assert_called_once_with()
|
||||
self.assertEqual("no overlap", tick["overlap"]["message"])
|
||||
|
||||
@patch("app.sync_agent.send_worker_command")
|
||||
@patch("app.sync_agent._select_overlap_target_node_codes")
|
||||
@patch("app.sync_agent._select_overlap_start_candidate")
|
||||
@patch("app.sync_agent.push_debug_event")
|
||||
def test_maybe_trigger_overlap_start_dispatches_pending_job(
|
||||
self,
|
||||
mock_push_debug_event,
|
||||
mock_select_overlap_start_candidate,
|
||||
mock_select_overlap_target_node_codes,
|
||||
mock_send_worker_command,
|
||||
) -> None:
|
||||
mock_select_overlap_start_candidate.return_value = {
|
||||
"job_id": 885,
|
||||
"job_code": "sync-overseas-20592",
|
||||
"task_mode": "domain_pipeline",
|
||||
"items_pending": 64000,
|
||||
"items_claimed": 0,
|
||||
"items_running": 0,
|
||||
"selection_reason": "overlap_tail_handoff",
|
||||
}
|
||||
mock_select_overlap_target_node_codes.return_value = [
|
||||
"mainland-controller-01-a",
|
||||
"mainland-controller-01-da",
|
||||
]
|
||||
mock_send_worker_command.return_value = (True, "已发送 Worker 控制指令: start_detection -> mainland-controller-01-a,mainland-controller-01-da")
|
||||
|
||||
with patch("app.sync_agent._LAST_OVERLAP_JOB_ID", 0), patch("app.sync_agent._LAST_OVERLAP_TRIGGERED_AT", 0.0):
|
||||
ok, message, data = _maybe_trigger_overlap_start()
|
||||
|
||||
self.assertTrue(ok)
|
||||
self.assertIn("mainland-controller-01-a", message)
|
||||
self.assertEqual(885, data["job_id"])
|
||||
mock_send_worker_command.assert_called_once_with(
|
||||
"start_detection",
|
||||
payload={
|
||||
"job_id": 885,
|
||||
"job_code": "sync-overseas-20592",
|
||||
"target_job_id": 885,
|
||||
"target_job_code": "sync-overseas-20592",
|
||||
"task_mode": "domain_pipeline",
|
||||
"source": "overlap-handoff",
|
||||
"selection_reason": "overlap_tail_handoff",
|
||||
"tail_handoff_candidate": True,
|
||||
"target_node_codes": [
|
||||
"mainland-controller-01-a",
|
||||
"mainland-controller-01-da",
|
||||
],
|
||||
},
|
||||
)
|
||||
self.assertEqual("overlap_handoff_started", mock_push_debug_event.call_args.kwargs["event_type"])
|
||||
|
||||
@patch("app.sync_agent._select_overlap_start_candidate")
|
||||
def test_maybe_trigger_overlap_start_honors_cooldown_for_same_job(
|
||||
self,
|
||||
mock_select_overlap_start_candidate,
|
||||
) -> None:
|
||||
mock_select_overlap_start_candidate.return_value = {
|
||||
"job_id": 885,
|
||||
"job_code": "sync-overseas-20592",
|
||||
"task_mode": "domain_pipeline",
|
||||
"items_pending": 64000,
|
||||
"selection_reason": "overlap_tail_handoff",
|
||||
}
|
||||
|
||||
with patch("app.sync_agent._LAST_OVERLAP_JOB_ID", 885), patch("app.sync_agent._LAST_OVERLAP_TRIGGERED_AT", __import__('time').time()):
|
||||
ok, message, data = _maybe_trigger_overlap_start()
|
||||
|
||||
self.assertFalse(ok)
|
||||
self.assertIn("冷却中", message)
|
||||
self.assertEqual(885, data["job_id"])
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
@@ -1,10 +1,20 @@
|
||||
import unittest
|
||||
from datetime import datetime, timedelta
|
||||
from unittest.mock import patch
|
||||
|
||||
from app.services.sync_push_service import (
|
||||
_build_task_pull_backlog_limits,
|
||||
_extract_detect_result_projection_events,
|
||||
_load_latest_projection,
|
||||
_load_pushable_projections,
|
||||
_refresh_remote_runtime_node,
|
||||
_load_local_detect_backlog_snapshot,
|
||||
_push_projection_now,
|
||||
_resolve_task_pull_request_limit,
|
||||
_task_projection_limit,
|
||||
pull_detect_task_batch_now,
|
||||
_resolve_detect_result_target_job_id,
|
||||
_select_relevant_backlog_job_ids_from_rows,
|
||||
_should_throttle_task_pull,
|
||||
ingest_runtime_projection,
|
||||
)
|
||||
@@ -23,6 +33,12 @@ class _FakeCursor:
|
||||
return self._rows.pop(0)
|
||||
return None
|
||||
|
||||
def fetchall(self):
|
||||
if self._rows:
|
||||
value = self._rows.pop(0)
|
||||
return list(value or [])
|
||||
return []
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
@@ -48,23 +64,311 @@ class _FakeConnection:
|
||||
return False
|
||||
|
||||
|
||||
class _FakeUrlopenResponse:
|
||||
def __init__(self, payload):
|
||||
self._payload = payload
|
||||
|
||||
def read(self):
|
||||
import json
|
||||
|
||||
return json.dumps(self._payload, ensure_ascii=False).encode("utf-8")
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, exc_type, exc, tb):
|
||||
return False
|
||||
|
||||
|
||||
class SyncPushServiceTests(unittest.TestCase):
|
||||
def test_build_task_pull_backlog_limits_scales_with_thread_configuration(self) -> None:
|
||||
@patch("app.services.sync_push_service.cleanup_imported_runtime_nodes_many")
|
||||
@patch("app.services.sync_push_service.cleanup_imported_runtime_nodes")
|
||||
@patch("app.services.sync_push_service.register_node_heartbeat")
|
||||
def test_refresh_remote_runtime_node_imports_cluster_worker_rows_without_active_job(
|
||||
self,
|
||||
mock_register_node_heartbeat,
|
||||
_mock_cleanup_imported_runtime_nodes,
|
||||
mock_cleanup_imported_runtime_nodes_many,
|
||||
) -> None:
|
||||
_refresh_remote_runtime_node(
|
||||
source_region="mainland",
|
||||
projection={
|
||||
"node": {
|
||||
"node_code": "mainland-controller-01",
|
||||
"region": "mainland",
|
||||
"role": "control",
|
||||
"hostname": "localhost",
|
||||
"ip": "127.0.0.1",
|
||||
},
|
||||
"worker_mode": "linux-systemd",
|
||||
"phase_label": "等待中",
|
||||
"phase_detail": "等待任务",
|
||||
"proxy_runtime_label": "正常",
|
||||
"proxy_runtime_reason": "healthy",
|
||||
"active_thread_count": 0,
|
||||
"max_thread_count": 60000,
|
||||
"worker_online": True,
|
||||
"detect_participating": False,
|
||||
"active_job": {
|
||||
"job_code": "",
|
||||
"status": "",
|
||||
"node_stats": [],
|
||||
},
|
||||
"cluster_nodes": [
|
||||
{
|
||||
"node_code": "mainland-controller-01",
|
||||
"role": "control",
|
||||
"status": "online",
|
||||
"current_load": 0,
|
||||
"active_threads": 0,
|
||||
"max_threads": 60000,
|
||||
"detect_participating": False,
|
||||
},
|
||||
{
|
||||
"node_code": "mainland-controller-01-a",
|
||||
"role": "worker",
|
||||
"status": "online",
|
||||
"current_load": 0,
|
||||
"active_threads": 0,
|
||||
"max_threads": 1000,
|
||||
"detect_participating": False,
|
||||
},
|
||||
{
|
||||
"node_code": "mainland-controller-01-b",
|
||||
"role": "worker",
|
||||
"status": "busy",
|
||||
"current_load": 12,
|
||||
"active_threads": 12,
|
||||
"max_threads": 1000,
|
||||
"detect_participating": True,
|
||||
},
|
||||
],
|
||||
},
|
||||
)
|
||||
|
||||
self.assertEqual(3, mock_register_node_heartbeat.call_count)
|
||||
worker_calls = [call.kwargs for call in mock_register_node_heartbeat.call_args_list[1:]]
|
||||
self.assertEqual(
|
||||
["mainland-controller-01-a", "mainland-controller-01-b"],
|
||||
[item["node_code"] for item in worker_calls],
|
||||
)
|
||||
self.assertEqual(12, worker_calls[1]["current_load"])
|
||||
self.assertEqual(
|
||||
["mainland-controller-01-a", "mainland-controller-01-b"],
|
||||
mock_cleanup_imported_runtime_nodes_many.call_args.kwargs["keep_node_codes"],
|
||||
)
|
||||
|
||||
@patch("app.services.sync_push_service.settings")
|
||||
def test_task_pull_request_limit_scales_with_estimated_threads(self, mock_settings) -> None:
|
||||
mock_settings.node_code = "mainland-controller-01"
|
||||
mock_settings.sync_batch_size = 5000
|
||||
|
||||
safe_limit = _resolve_task_pull_request_limit(
|
||||
None,
|
||||
settings_payload={
|
||||
"thread_count": 1000,
|
||||
"process_count": 1,
|
||||
"node_thread_counts": {"mainland-controller-01": 1000},
|
||||
"node_process_counts": {"mainland-controller-01": 60},
|
||||
},
|
||||
)
|
||||
|
||||
self.assertEqual(120000, safe_limit)
|
||||
|
||||
@patch("app.services.sync_push_service.settings")
|
||||
def test_task_projection_limit_allows_large_explicit_pull_request(self, mock_settings) -> None:
|
||||
mock_settings.sync_batch_size = 5000
|
||||
|
||||
self.assertEqual(120000, _task_projection_limit(120000))
|
||||
|
||||
@patch("app.services.sync_push_service.settings")
|
||||
@patch("app.services.sync_push_service.get_db")
|
||||
def test_load_latest_projection_prefers_non_future_row_over_clock_skewed_history(
|
||||
self,
|
||||
mock_get_db,
|
||||
mock_settings,
|
||||
) -> None:
|
||||
now = datetime.now()
|
||||
future_row = (
|
||||
10081,
|
||||
"mainland",
|
||||
"overseas",
|
||||
"projected",
|
||||
{"projection": {"active_thread_count": 138}},
|
||||
now + timedelta(hours=4),
|
||||
now + timedelta(hours=4),
|
||||
)
|
||||
valid_row = (
|
||||
11633,
|
||||
"mainland",
|
||||
"overseas",
|
||||
"projected",
|
||||
{"projection": {"active_thread_count": 6431}},
|
||||
now - timedelta(minutes=2),
|
||||
now - timedelta(minutes=2),
|
||||
)
|
||||
fake_conn = _FakeConnection(rows=[[future_row, valid_row]])
|
||||
mock_get_db.return_value = fake_conn
|
||||
mock_settings.sync_source_region = "mainland"
|
||||
mock_settings.node_region = "mainland"
|
||||
mock_settings.sync_target_region = "overseas"
|
||||
|
||||
with patch("app.services.sync_push_service.datetime") as mock_datetime:
|
||||
mock_datetime.now.return_value = now
|
||||
record = _load_latest_projection("runtime_projection")
|
||||
|
||||
self.assertIsNotNone(record)
|
||||
self.assertEqual(11633, record["id"])
|
||||
executed_sql, executed_params = fake_conn.cursor_obj.executed[0]
|
||||
self.assertIn("CASE WHEN created_at <= %s THEN 0 ELSE 1 END", executed_sql)
|
||||
self.assertEqual("runtime_projection", executed_params[0])
|
||||
self.assertEqual("mainland", executed_params[1])
|
||||
self.assertEqual("overseas", executed_params[2])
|
||||
self.assertEqual(now + timedelta(minutes=5), executed_params[3])
|
||||
|
||||
@patch("app.services.sync_push_service._latest_push_attempt")
|
||||
@patch("app.services.sync_push_service.settings")
|
||||
@patch("app.services.sync_push_service.get_db")
|
||||
def test_load_pushable_projections_prefers_latest_unsent_records(
|
||||
self,
|
||||
mock_get_db,
|
||||
mock_settings,
|
||||
mock_latest_push_attempt,
|
||||
) -> None:
|
||||
now = datetime.now()
|
||||
old_success_rows = [
|
||||
(
|
||||
10000 + index,
|
||||
"mainland",
|
||||
"overseas",
|
||||
"projected",
|
||||
{"projection_hash": f"old-{index}", "projection": {"job": {"job_id": index}}},
|
||||
now - timedelta(minutes=120 - index),
|
||||
now - timedelta(minutes=120 - index),
|
||||
)
|
||||
for index in range(100)
|
||||
]
|
||||
newest_unsent = (
|
||||
23360,
|
||||
"mainland",
|
||||
"overseas",
|
||||
"projected",
|
||||
{"projection_hash": "new-hash", "projection": {"job": {"job_id": 909, "job_code": "sync-overseas-28618"}}},
|
||||
now,
|
||||
now,
|
||||
)
|
||||
fake_conn = _FakeConnection(rows=[old_success_rows + [newest_unsent]])
|
||||
mock_get_db.return_value = fake_conn
|
||||
mock_settings.sync_source_region = "mainland"
|
||||
mock_settings.node_region = "mainland"
|
||||
mock_settings.sync_target_region = "overseas"
|
||||
mock_settings.sync_batch_size = 20
|
||||
|
||||
def _attempt_side_effect(source_record_id, target_region, sync_type):
|
||||
if source_record_id == 23360:
|
||||
return None
|
||||
return {"status": "success", "created_at": now}
|
||||
|
||||
mock_latest_push_attempt.side_effect = _attempt_side_effect
|
||||
|
||||
records = _load_pushable_projections("detect_result_projection", limit=20)
|
||||
|
||||
self.assertEqual([23360], [item["id"] for item in records])
|
||||
executed_sql, _ = fake_conn.cursor_obj.executed[0]
|
||||
self.assertIn("ORDER BY created_at DESC, id DESC", executed_sql)
|
||||
|
||||
@patch("app.services.sync_push_service.settings.node_code", "mainland-controller-01")
|
||||
def test_build_task_pull_backlog_limits_scales_with_local_server_capacity(self) -> None:
|
||||
limits = _build_task_pull_backlog_limits(
|
||||
5000,
|
||||
settings_payload={
|
||||
"thread_count": 100,
|
||||
"process_count": 80,
|
||||
"node_thread_counts": {
|
||||
"mainland-controller-01": 2000,
|
||||
"mainland-controller-01": 1000,
|
||||
"mainland-controller-01-a": 1000,
|
||||
"mainland-worker-01": 1200,
|
||||
},
|
||||
"node_process_counts": {
|
||||
"mainland-controller-01": 80,
|
||||
"mainland-worker-01": 60,
|
||||
},
|
||||
},
|
||||
)
|
||||
|
||||
self.assertEqual(3200, limits["estimated_total_threads"])
|
||||
self.assertEqual(6400, limits["max_pending_total"])
|
||||
self.assertEqual(3200, limits["max_register_pending"])
|
||||
self.assertEqual(800, limits["max_downstream_pending"])
|
||||
self.assertEqual(80000, limits["estimated_total_threads"])
|
||||
self.assertEqual(160000, limits["max_pending_total"])
|
||||
self.assertEqual(80000, limits["max_register_pending"])
|
||||
self.assertEqual(20000, limits["max_downstream_pending"])
|
||||
|
||||
def test_select_relevant_backlog_job_ids_from_rows_skips_stale_pending_jobs(self) -> None:
|
||||
now = datetime.now()
|
||||
|
||||
job_ids = _select_relevant_backlog_job_ids_from_rows(
|
||||
[
|
||||
(11, "running", now - timedelta(hours=10)),
|
||||
(10, "pending", now - timedelta(minutes=30)),
|
||||
(9, "pending", now - timedelta(hours=7)),
|
||||
],
|
||||
freshness_hours=6,
|
||||
limit=4,
|
||||
)
|
||||
|
||||
self.assertEqual([11, 10], job_ids)
|
||||
|
||||
@patch("app.services.sync_push_service.get_active_detect_job_summary")
|
||||
@patch("app.services.sync_push_service.get_db")
|
||||
def test_load_local_detect_backlog_snapshot_prefers_active_job_snapshot(
|
||||
self,
|
||||
mock_get_db,
|
||||
mock_get_active_detect_job_summary,
|
||||
) -> None:
|
||||
mock_get_active_detect_job_summary.return_value = {
|
||||
"items_pending": 123,
|
||||
"items_claimed": 4,
|
||||
"items_running": 7,
|
||||
"display_items_running": 9,
|
||||
"step_stats": [
|
||||
{"step_code": "detect_register", "items_pending": 90},
|
||||
{"step_code": "detect_baidu", "items_pending": 33},
|
||||
],
|
||||
}
|
||||
|
||||
backlog = _load_local_detect_backlog_snapshot()
|
||||
|
||||
self.assertEqual(123, backlog["pending_total"])
|
||||
self.assertEqual(4, backlog["claimed_total"])
|
||||
self.assertEqual(9, backlog["running_total"])
|
||||
self.assertEqual(90, backlog["register_pending"])
|
||||
self.assertEqual(33, backlog["downstream_pending"])
|
||||
mock_get_db.assert_not_called()
|
||||
|
||||
@patch("app.services.sync_push_service.get_active_detect_job_summary", return_value={})
|
||||
@patch("app.services.sync_push_service.get_db")
|
||||
def test_load_local_detect_backlog_snapshot_only_counts_relevant_jobs(
|
||||
self,
|
||||
mock_get_db,
|
||||
_mock_get_active_detect_job_summary,
|
||||
) -> None:
|
||||
fake_conn = _FakeConnection(
|
||||
rows=[
|
||||
[
|
||||
(11, "running", datetime.now() - timedelta(hours=2)),
|
||||
(10, "pending", datetime.now() - timedelta(minutes=30)),
|
||||
(9, "pending", datetime.now() - timedelta(hours=7)),
|
||||
],
|
||||
(123, 4, 7, 90, 33),
|
||||
]
|
||||
)
|
||||
mock_get_db.return_value = fake_conn
|
||||
|
||||
backlog = _load_local_detect_backlog_snapshot()
|
||||
|
||||
self.assertEqual(123, backlog["pending_total"])
|
||||
self.assertEqual(90, backlog["register_pending"])
|
||||
executed_sql, executed_params = fake_conn.cursor_obj.executed[1]
|
||||
self.assertIn("WHERE item.job_id = ANY(%s)", executed_sql)
|
||||
self.assertEqual([11, 10], list(executed_params[0]))
|
||||
|
||||
def test_should_throttle_task_pull_when_register_backlog_overwhelms_downstream(self) -> None:
|
||||
should_throttle, reason = _should_throttle_task_pull(
|
||||
@@ -104,6 +408,224 @@ class SyncPushServiceTests(unittest.TestCase):
|
||||
self.assertFalse(should_throttle)
|
||||
self.assertEqual("", reason)
|
||||
|
||||
@patch("app.services.sync_push_service._should_throttle_task_pull", return_value=(False, ""))
|
||||
@patch("app.services.sync_push_service._build_task_pull_backlog_limits", return_value={})
|
||||
@patch("app.services.sync_push_service._load_local_detect_backlog_snapshot", return_value={})
|
||||
@patch("app.services.sync_push_service._acquire_sync_pull_worker_wake_guard", return_value=True)
|
||||
@patch("app.services.sync_push_service.ingest_detect_task_projection")
|
||||
@patch("app.services.worker_control_service.send_worker_command")
|
||||
@patch("app.services.sync_push_service.urllib.request.urlopen")
|
||||
@patch("app.services.sync_push_service.settings")
|
||||
@patch("app.services.sync_push_service.get_settings_payload")
|
||||
def test_pull_detect_task_batch_now_starts_worker_with_projection_active_job_identity(
|
||||
self,
|
||||
mock_get_settings_payload,
|
||||
mock_settings,
|
||||
mock_urlopen,
|
||||
mock_send_worker_command,
|
||||
mock_ingest_detect_task_projection,
|
||||
_mock_acquire_sync_pull_worker_wake_guard,
|
||||
_mock_load_local_detect_backlog_snapshot,
|
||||
_mock_build_task_pull_backlog_limits,
|
||||
_mock_should_throttle_task_pull,
|
||||
) -> None:
|
||||
mock_settings.node_region = "mainland"
|
||||
mock_settings.node_role = "control"
|
||||
mock_settings.node_code = "mainland-controller-01"
|
||||
mock_settings.sync_target_api_base_url = "http://example.com"
|
||||
mock_settings.sync_target_region = "overseas"
|
||||
mock_settings.sync_batch_size = 200
|
||||
mock_settings.sync_shared_token = ""
|
||||
mock_get_settings_payload.return_value = {
|
||||
"thread_count": 1000,
|
||||
"process_count": 1,
|
||||
"node_thread_counts": {"mainland-controller-01": 1000},
|
||||
"node_process_counts": {"mainland-controller-01": 60},
|
||||
}
|
||||
|
||||
projection = {
|
||||
"batch_code": "task-20260423210000-aa11bb",
|
||||
"active_job": {
|
||||
"job_id": 376,
|
||||
"job_code": "sync-overseas-376",
|
||||
"current_cycle_token": "cycle-376",
|
||||
},
|
||||
}
|
||||
export_response = {
|
||||
"code": 0,
|
||||
"data": {
|
||||
"source_record_id": 15164,
|
||||
"projection_hash": "hash-15164",
|
||||
"projection": projection,
|
||||
},
|
||||
}
|
||||
ack_response = {"code": 0, "data": {"acknowledged": True}}
|
||||
mock_urlopen.side_effect = [
|
||||
_FakeUrlopenResponse(export_response),
|
||||
_FakeUrlopenResponse(ack_response),
|
||||
]
|
||||
mock_ingest_detect_task_projection.return_value = (
|
||||
True,
|
||||
"任务批次接收成功",
|
||||
{
|
||||
"target_job_id": 1902,
|
||||
"target_job_code": "sync-overseas-15164",
|
||||
"queued_count": 1000,
|
||||
},
|
||||
)
|
||||
mock_send_worker_command.return_value = (True, "started")
|
||||
|
||||
ok, message, payload = pull_detect_task_batch_now(limit=1000)
|
||||
|
||||
self.assertTrue(ok)
|
||||
self.assertEqual("待检测任务批次拉取并入库成功", message)
|
||||
self.assertTrue(payload["worker_start_ok"])
|
||||
export_request = mock_urlopen.call_args_list[0][0][0]
|
||||
self.assertIn("limit=1000", export_request.full_url)
|
||||
mock_send_worker_command.assert_called_once()
|
||||
args, kwargs = mock_send_worker_command.call_args
|
||||
self.assertEqual("start_detection", args[0])
|
||||
sent_payload = kwargs["payload"]
|
||||
self.assertEqual(15164, sent_payload["source_record_id"])
|
||||
self.assertEqual(1902, sent_payload["target_job_id"])
|
||||
self.assertEqual("sync-overseas-15164", sent_payload["target_job_code"])
|
||||
self.assertEqual(376, sent_payload["job_id"])
|
||||
self.assertEqual("sync-overseas-376", sent_payload["job_code"])
|
||||
self.assertEqual("cycle-376", sent_payload["cycle_token"])
|
||||
|
||||
@patch("app.services.sync_push_service._should_throttle_task_pull", return_value=(False, ""))
|
||||
@patch("app.services.sync_push_service._build_task_pull_backlog_limits", return_value={})
|
||||
@patch("app.services.sync_push_service._load_local_detect_backlog_snapshot", return_value={})
|
||||
@patch("app.services.sync_push_service._acquire_sync_pull_worker_wake_guard", return_value=False)
|
||||
@patch("app.services.sync_push_service.ingest_detect_task_projection")
|
||||
@patch("app.services.worker_control_service.send_worker_command")
|
||||
@patch("app.services.sync_push_service.urllib.request.urlopen")
|
||||
@patch("app.services.sync_push_service.settings")
|
||||
@patch("app.services.sync_push_service.get_settings_payload")
|
||||
def test_pull_detect_task_batch_now_skips_duplicate_worker_wake_within_short_window(
|
||||
self,
|
||||
mock_get_settings_payload,
|
||||
mock_settings,
|
||||
mock_urlopen,
|
||||
mock_send_worker_command,
|
||||
mock_ingest_detect_task_projection,
|
||||
_mock_acquire_sync_pull_worker_wake_guard,
|
||||
_mock_load_local_detect_backlog_snapshot,
|
||||
_mock_build_task_pull_backlog_limits,
|
||||
_mock_should_throttle_task_pull,
|
||||
) -> None:
|
||||
mock_settings.node_region = "mainland"
|
||||
mock_settings.node_role = "control"
|
||||
mock_settings.node_code = "mainland-controller-01"
|
||||
mock_settings.sync_target_api_base_url = "http://example.com"
|
||||
mock_settings.sync_target_region = "overseas"
|
||||
mock_settings.sync_batch_size = 200
|
||||
mock_settings.sync_shared_token = ""
|
||||
mock_get_settings_payload.return_value = {
|
||||
"thread_count": 1000,
|
||||
"process_count": 1,
|
||||
"node_thread_counts": {"mainland-controller-01": 1000},
|
||||
"node_process_counts": {"mainland-controller-01": 60},
|
||||
}
|
||||
|
||||
projection = {
|
||||
"batch_code": "task-20260423210000-aa11bb",
|
||||
"active_job": {
|
||||
"job_id": 376,
|
||||
"job_code": "sync-overseas-376",
|
||||
"current_cycle_token": "cycle-376",
|
||||
},
|
||||
}
|
||||
export_response = {
|
||||
"code": 0,
|
||||
"data": {
|
||||
"source_record_id": 15164,
|
||||
"projection_hash": "hash-15164",
|
||||
"projection": projection,
|
||||
},
|
||||
}
|
||||
ack_response = {"code": 0, "data": {"acknowledged": True}}
|
||||
mock_urlopen.side_effect = [
|
||||
_FakeUrlopenResponse(export_response),
|
||||
_FakeUrlopenResponse(ack_response),
|
||||
]
|
||||
mock_ingest_detect_task_projection.return_value = (
|
||||
True,
|
||||
"任务批次接收成功",
|
||||
{
|
||||
"target_job_id": 1902,
|
||||
"target_job_code": "sync-overseas-15164",
|
||||
"queued_count": 1000,
|
||||
},
|
||||
)
|
||||
|
||||
ok, message, payload = pull_detect_task_batch_now(limit=1000)
|
||||
|
||||
self.assertTrue(ok)
|
||||
self.assertEqual("待检测任务批次拉取并入库成功", message)
|
||||
self.assertTrue(payload["worker_start_ok"])
|
||||
self.assertTrue(payload["worker_start_skipped"])
|
||||
self.assertIn("重复 Worker 唤起", payload["worker_start_message"])
|
||||
mock_send_worker_command.assert_not_called()
|
||||
|
||||
@patch("app.services.sync_push_service._should_throttle_task_pull", return_value=(False, ""))
|
||||
@patch("app.services.sync_push_service._build_task_pull_backlog_limits", return_value={})
|
||||
@patch("app.services.sync_push_service._load_local_detect_backlog_snapshot", return_value={})
|
||||
@patch("app.services.sync_push_service._acquire_sync_pull_worker_wake_guard", return_value=False)
|
||||
@patch("app.services.sync_push_service.ingest_detect_task_projection")
|
||||
@patch("app.services.sync_push_service.urllib.request.urlopen")
|
||||
@patch("app.services.sync_push_service.settings")
|
||||
@patch("app.services.sync_push_service.get_settings_payload")
|
||||
def test_pull_detect_task_batch_now_uses_adaptive_limit_when_unspecified(
|
||||
self,
|
||||
mock_get_settings_payload,
|
||||
mock_settings,
|
||||
mock_urlopen,
|
||||
mock_ingest_detect_task_projection,
|
||||
_mock_acquire_sync_pull_worker_wake_guard,
|
||||
_mock_load_local_detect_backlog_snapshot,
|
||||
_mock_build_task_pull_backlog_limits,
|
||||
_mock_should_throttle_task_pull,
|
||||
) -> None:
|
||||
mock_settings.node_region = "mainland"
|
||||
mock_settings.node_role = "control"
|
||||
mock_settings.node_code = "mainland-controller-01"
|
||||
mock_settings.sync_target_api_base_url = "http://example.com"
|
||||
mock_settings.sync_target_region = "overseas"
|
||||
mock_settings.sync_batch_size = 5000
|
||||
mock_settings.sync_shared_token = ""
|
||||
mock_get_settings_payload.return_value = {
|
||||
"thread_count": 1000,
|
||||
"process_count": 1,
|
||||
"node_thread_counts": {"mainland-controller-01": 1000},
|
||||
"node_process_counts": {"mainland-controller-01": 60},
|
||||
}
|
||||
|
||||
export_response = {
|
||||
"code": 0,
|
||||
"data": {
|
||||
"source_record_id": 15164,
|
||||
"projection_hash": "hash-15164",
|
||||
"projection": {"batch_code": "task-20260423210000-aa11bb", "active_job": {}},
|
||||
},
|
||||
}
|
||||
ack_response = {"code": 0, "data": {"acknowledged": True}}
|
||||
mock_urlopen.side_effect = [
|
||||
_FakeUrlopenResponse(export_response),
|
||||
_FakeUrlopenResponse(ack_response),
|
||||
]
|
||||
mock_ingest_detect_task_projection.return_value = (
|
||||
True,
|
||||
"任务批次接收成功",
|
||||
{"target_job_id": 1902, "target_job_code": "sync-overseas-15164", "queued_count": 1000},
|
||||
)
|
||||
|
||||
ok, _message, _payload = pull_detect_task_batch_now(limit=None)
|
||||
|
||||
self.assertTrue(ok)
|
||||
export_request = mock_urlopen.call_args_list[0][0][0]
|
||||
self.assertIn("limit=120000", export_request.full_url)
|
||||
|
||||
def test_extract_detect_result_projection_events_adds_import_metadata(self) -> None:
|
||||
projection = {
|
||||
"job": {
|
||||
@@ -137,6 +659,79 @@ class SyncPushServiceTests(unittest.TestCase):
|
||||
self.assertTrue(event["payload"]["imported_from_projection"])
|
||||
self.assertTrue(event["payload"]["import_fingerprint"])
|
||||
|
||||
@patch("app.services.sync_push_service._push_projection_record")
|
||||
@patch("app.services.sync_push_service._load_latest_projection")
|
||||
@patch("app.services.runtime_status_service.refresh_runtime_projection_snapshot")
|
||||
def test_push_projection_now_refreshes_runtime_projection_with_lightweight_snapshot(
|
||||
self,
|
||||
mock_refresh_runtime_projection_snapshot,
|
||||
mock_load_latest_projection,
|
||||
mock_push_projection_record,
|
||||
) -> None:
|
||||
mock_refresh_runtime_projection_snapshot.return_value = {
|
||||
"record_id": 10082,
|
||||
"active_thread_count": 1972,
|
||||
"max_thread_count": 80000,
|
||||
"queue_display_running": 1972,
|
||||
}
|
||||
mock_load_latest_projection.return_value = {
|
||||
"id": 10082,
|
||||
"source_region": "mainland",
|
||||
"target_region": "overseas",
|
||||
"created_at": None,
|
||||
"payload": {"projection_hash": "hash-10082", "projection": {}},
|
||||
}
|
||||
mock_push_projection_record.return_value = (
|
||||
True,
|
||||
"投影推送成功",
|
||||
{"action": "push_sync", "sync_type": "runtime_projection", "source_record_id": 10082},
|
||||
)
|
||||
|
||||
ok, message, data = _push_projection_now("runtime_projection", "https://example.com/api/v1/runtime/sync-ingest")
|
||||
|
||||
self.assertTrue(ok)
|
||||
self.assertEqual("投影推送成功", message)
|
||||
self.assertEqual(10082, data["source_record_id"])
|
||||
mock_refresh_runtime_projection_snapshot.assert_called_once_with(window_minutes=15)
|
||||
mock_load_latest_projection.assert_called_once_with("runtime_projection")
|
||||
mock_push_projection_record.assert_called_once()
|
||||
|
||||
@patch("app.services.sync_push_service._push_projection_record")
|
||||
@patch("app.services.sync_push_service._load_latest_projection")
|
||||
@patch("app.services.sync_push_service._append_fast_runtime_projection_snapshot")
|
||||
@patch("app.services.runtime_status_service.refresh_runtime_projection_snapshot")
|
||||
def test_push_projection_now_can_use_fast_runtime_projection_path(
|
||||
self,
|
||||
mock_refresh_runtime_projection_snapshot,
|
||||
mock_append_fast_runtime_projection_snapshot,
|
||||
mock_load_latest_projection,
|
||||
mock_push_projection_record,
|
||||
) -> None:
|
||||
mock_append_fast_runtime_projection_snapshot.return_value = 10091
|
||||
mock_load_latest_projection.return_value = {
|
||||
"id": 10091,
|
||||
"source_region": "mainland",
|
||||
"target_region": "overseas",
|
||||
"created_at": None,
|
||||
"payload": {"projection_hash": "hash-10091", "projection": {}},
|
||||
}
|
||||
mock_push_projection_record.return_value = (
|
||||
True,
|
||||
"投影推送成功",
|
||||
{"action": "push_sync", "sync_type": "runtime_projection", "source_record_id": 10091},
|
||||
)
|
||||
|
||||
with patch.dict("os.environ", {"DOMAINCHECK_SYNC_RUNTIME_FAST_PROJECTION": "1"}, clear=False):
|
||||
ok, message, data = _push_projection_now("runtime_projection", "https://example.com/api/v1/runtime/sync-ingest")
|
||||
|
||||
self.assertTrue(ok)
|
||||
self.assertEqual("投影推送成功", message)
|
||||
self.assertEqual(10091, data["source_record_id"])
|
||||
mock_append_fast_runtime_projection_snapshot.assert_called_once()
|
||||
mock_refresh_runtime_projection_snapshot.assert_not_called()
|
||||
mock_load_latest_projection.assert_called_once_with("runtime_projection")
|
||||
mock_push_projection_record.assert_called_once()
|
||||
|
||||
@patch("app.services.sync_push_service.get_db")
|
||||
def test_resolve_detect_result_target_job_id_prefers_matching_job_code(self, mock_get_db) -> None:
|
||||
fake_conn = _FakeConnection(rows=[(456,)])
|
||||
|
||||
@@ -1,12 +1,61 @@
|
||||
import json
|
||||
import unittest
|
||||
from datetime import datetime, timedelta
|
||||
from unittest.mock import patch
|
||||
|
||||
from app.services.sync_record_service import (
|
||||
_build_runtime_projection_payload,
|
||||
_collect_recent_domain_events,
|
||||
_pick_latest_projection_row,
|
||||
append_runtime_projection_if_changed,
|
||||
get_detect_result_sync_batches,
|
||||
)
|
||||
|
||||
|
||||
class _FakeCursor:
|
||||
def __init__(self, fetchone_values):
|
||||
self.fetchone_values = list(fetchone_values or [])
|
||||
self.executed = []
|
||||
|
||||
def execute(self, sql, params=None):
|
||||
self.executed.append((sql, params))
|
||||
|
||||
def fetchone(self):
|
||||
if self.fetchone_values:
|
||||
return self.fetchone_values.pop(0)
|
||||
return None
|
||||
|
||||
def fetchall(self):
|
||||
if self.fetchone_values:
|
||||
value = self.fetchone_values.pop(0)
|
||||
return list(value or [])
|
||||
return []
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, exc_type, exc, tb):
|
||||
return False
|
||||
|
||||
|
||||
class _FakeConnection:
|
||||
def __init__(self, fetchone_values):
|
||||
self.cursor_obj = _FakeCursor(fetchone_values)
|
||||
self.committed = False
|
||||
|
||||
def cursor(self):
|
||||
return self.cursor_obj
|
||||
|
||||
def commit(self):
|
||||
self.committed = True
|
||||
|
||||
def __enter__(self):
|
||||
return self
|
||||
|
||||
def __exit__(self, exc_type, exc, tb):
|
||||
return False
|
||||
|
||||
|
||||
class SyncRecordServiceTests(unittest.TestCase):
|
||||
def test_collect_recent_domain_events_filters_and_keeps_latest_slice(self) -> None:
|
||||
active_job = {
|
||||
@@ -130,6 +179,682 @@ class SyncRecordServiceTests(unittest.TestCase):
|
||||
self.assertEqual("", projection["active_job"]["job_code"])
|
||||
self.assertEqual([], projection["active_job"]["node_stats"])
|
||||
|
||||
@patch("app.services.sync_record_service._resolve_local_ip", return_value="121.204.244.188")
|
||||
@patch("app.services.sync_record_service.socket.gethostname", return_value="mainland-controller-01")
|
||||
@patch("app.services.sync_record_service.settings")
|
||||
def test_build_runtime_projection_payload_prefers_runtime_queue_nodes_for_multi_process_controller(
|
||||
self,
|
||||
mock_settings,
|
||||
_mock_hostname,
|
||||
_mock_resolve_ip,
|
||||
) -> None:
|
||||
mock_settings.node_code = "mainland-controller-01"
|
||||
mock_settings.node_region = "mainland"
|
||||
mock_settings.node_role = "control"
|
||||
mock_settings.sync_source_region = "mainland"
|
||||
mock_settings.sync_target_region = "overseas"
|
||||
|
||||
payload = _build_runtime_projection_payload(
|
||||
detect={
|
||||
"worker_online": True,
|
||||
"worker_mode": "linux-systemd",
|
||||
"active_thread_count": 1,
|
||||
"max_thread_count": 1000,
|
||||
"aggregate_max_thread_count": 80000,
|
||||
"phase_label": "运行中",
|
||||
"phase_detail": "80 实例运行",
|
||||
"proxy_runtime_label": "正常",
|
||||
"proxy_runtime_reason": "healthy",
|
||||
"progress": {
|
||||
"pending": 5001,
|
||||
"running": 0,
|
||||
"completed": 0,
|
||||
"blacklisted": 0,
|
||||
"failed": 0,
|
||||
},
|
||||
"backlog": {"pending_total": 1157325},
|
||||
"queue_health": {
|
||||
"queue": {
|
||||
"items_total": 5292,
|
||||
"pending": 4578,
|
||||
"claimed": 0,
|
||||
"display_claimed": 120,
|
||||
"running": 1972,
|
||||
"display_running": 1972,
|
||||
"completed": 690,
|
||||
"blacklisted": 0,
|
||||
"failed": 24,
|
||||
"terminal": 714,
|
||||
},
|
||||
"nodes": [
|
||||
{
|
||||
"node_code": "mainland-controller-01-a",
|
||||
"items_running": 1000,
|
||||
"items_claimed": 0,
|
||||
"display_running": 1000,
|
||||
"active_threads": 1000,
|
||||
"max_threads": 1000,
|
||||
"region": "mainland",
|
||||
"role": "control",
|
||||
"status": "busy",
|
||||
},
|
||||
{
|
||||
"node_code": "mainland-controller-01-b",
|
||||
"items_running": 972,
|
||||
"items_claimed": 120,
|
||||
"display_running": 972,
|
||||
"active_threads": 972,
|
||||
"max_threads": 1000,
|
||||
"region": "mainland",
|
||||
"role": "control",
|
||||
"status": "busy",
|
||||
},
|
||||
],
|
||||
},
|
||||
"active_job": {
|
||||
"job_id": 55,
|
||||
"job_code": "sync-overseas-55",
|
||||
"status": "running",
|
||||
"progress_percent": 13.49,
|
||||
"items_total": 5292,
|
||||
"items_terminal": 714,
|
||||
"items_pending": 4578,
|
||||
"items_running": 0,
|
||||
"items_failed": 24,
|
||||
"node_stats": [
|
||||
{
|
||||
"node_code": "mainland-controller-01",
|
||||
"items_running": 1,
|
||||
"items_claimed": 0,
|
||||
"items_total": 5292,
|
||||
}
|
||||
],
|
||||
},
|
||||
"dependency_alerts": [],
|
||||
},
|
||||
cluster={
|
||||
"nodes_total": 80,
|
||||
"nodes": [
|
||||
{
|
||||
"node_code": "mainland-controller-01",
|
||||
"current_load": 1,
|
||||
"detect_participating": True,
|
||||
}
|
||||
],
|
||||
"summary": {
|
||||
"online_worker_nodes": 65,
|
||||
"dedicated_online_worker_nodes": 1,
|
||||
"online_control_nodes": 65,
|
||||
"busy_nodes": ["mainland-controller-01-a", "mainland-controller-01-b"],
|
||||
"stale_nodes": [],
|
||||
"offline_nodes": [],
|
||||
},
|
||||
},
|
||||
source_region="mainland",
|
||||
target_region="overseas",
|
||||
)
|
||||
|
||||
projection = payload["projection"]
|
||||
self.assertTrue(projection["detect_participating"])
|
||||
self.assertEqual(1972, projection["active_thread_count"])
|
||||
self.assertEqual(80000, projection["max_thread_count"])
|
||||
self.assertEqual(1972, projection["progress"]["running"])
|
||||
self.assertEqual(2, len(projection["active_job"]["node_stats"]))
|
||||
self.assertEqual(2, len(projection["active_job"]["distributed_node_stats"]))
|
||||
self.assertEqual(1, len(projection["cluster_nodes"]))
|
||||
self.assertEqual("mainland-controller-01", projection["cluster_nodes"][0]["node_code"])
|
||||
self.assertEqual(1972, projection["active_job"]["items_running"])
|
||||
self.assertEqual(1972, projection["active_job"]["display_items_running"])
|
||||
self.assertEqual(80000, projection["active_job"]["display_max_threads"])
|
||||
|
||||
@patch("app.services.sync_record_service._resolve_local_ip", return_value="121.204.244.188")
|
||||
@patch("app.services.sync_record_service.socket.gethostname", return_value="mainland-controller-01")
|
||||
@patch("app.services.sync_record_service.settings")
|
||||
@patch("app.services.sync_record_service.get_db")
|
||||
def test_append_runtime_projection_if_changed_writes_heartbeat_for_unchanged_projection_after_interval(
|
||||
self,
|
||||
mock_get_db,
|
||||
mock_settings,
|
||||
_mock_hostname,
|
||||
_mock_resolve_ip,
|
||||
) -> None:
|
||||
mock_settings.node_code = "mainland-controller-01"
|
||||
mock_settings.node_region = "mainland"
|
||||
mock_settings.node_role = "control"
|
||||
mock_settings.sync_source_region = "mainland"
|
||||
mock_settings.sync_target_region = "overseas"
|
||||
|
||||
detect = {
|
||||
"worker_online": True,
|
||||
"worker_mode": "linux-systemd",
|
||||
"active_thread_count": 3200,
|
||||
"max_thread_count": 80000,
|
||||
"phase_label": "运行中",
|
||||
"phase_detail": "80 实例运行",
|
||||
"proxy_runtime_label": "正常",
|
||||
"proxy_runtime_reason": "healthy",
|
||||
"progress": {
|
||||
"pending": 1200000,
|
||||
"running": 6400,
|
||||
"completed": 50000,
|
||||
"blacklisted": 1200,
|
||||
"failed": 88,
|
||||
},
|
||||
"backlog": {"pending_total": 1200000},
|
||||
"active_job": {
|
||||
"job_id": 55,
|
||||
"job_code": "sync-overseas-55",
|
||||
"status": "running",
|
||||
"progress_percent": 12.5,
|
||||
"items_total": 1300000,
|
||||
"items_terminal": 51288,
|
||||
"items_pending": 1200000,
|
||||
"items_running": 6400,
|
||||
"items_failed": 88,
|
||||
"node_stats": [
|
||||
{
|
||||
"node_code": "mainland-controller-01",
|
||||
"items_running": 6400,
|
||||
"items_claimed": 7000,
|
||||
"items_total": 1300000,
|
||||
}
|
||||
],
|
||||
},
|
||||
"dependency_alerts": [],
|
||||
}
|
||||
cluster = {
|
||||
"nodes_total": 1,
|
||||
"nodes": [
|
||||
{
|
||||
"node_code": "mainland-controller-01",
|
||||
"current_load": 6400,
|
||||
"detect_participating": True,
|
||||
}
|
||||
],
|
||||
"summary": {
|
||||
"online_worker_nodes": 1,
|
||||
"dedicated_online_worker_nodes": 0,
|
||||
"online_control_nodes": 1,
|
||||
"busy_nodes": ["mainland-controller-01"],
|
||||
"stale_nodes": [],
|
||||
"offline_nodes": [],
|
||||
},
|
||||
}
|
||||
|
||||
previous_payload = _build_runtime_projection_payload(
|
||||
detect=detect,
|
||||
cluster=cluster,
|
||||
source_region="mainland",
|
||||
target_region="overseas",
|
||||
)
|
||||
fake_conn = _FakeConnection(
|
||||
[
|
||||
[(previous_payload, datetime.now() - timedelta(seconds=90))],
|
||||
(321,),
|
||||
]
|
||||
)
|
||||
mock_get_db.return_value = fake_conn
|
||||
|
||||
record_id = append_runtime_projection_if_changed(
|
||||
detect=detect,
|
||||
cluster=cluster,
|
||||
source_region="mainland",
|
||||
target_region="overseas",
|
||||
)
|
||||
|
||||
self.assertEqual(321, record_id)
|
||||
self.assertTrue(fake_conn.committed)
|
||||
self.assertTrue(
|
||||
any("INSERT INTO detect_sync_records" in sql for sql, _params in fake_conn.cursor_obj.executed)
|
||||
)
|
||||
|
||||
@patch("app.services.sync_record_service._resolve_local_ip", return_value="121.204.244.188")
|
||||
@patch("app.services.sync_record_service.socket.gethostname", return_value="mainland-controller-01")
|
||||
@patch("app.services.sync_record_service.settings")
|
||||
@patch("app.services.sync_record_service.get_db")
|
||||
def test_append_runtime_projection_if_changed_writes_when_cluster_nodes_change_within_window(
|
||||
self,
|
||||
mock_get_db,
|
||||
mock_settings,
|
||||
_mock_hostname,
|
||||
_mock_resolve_ip,
|
||||
) -> None:
|
||||
mock_settings.node_code = "mainland-controller-01"
|
||||
mock_settings.node_region = "mainland"
|
||||
mock_settings.node_role = "control"
|
||||
mock_settings.sync_source_region = "mainland"
|
||||
mock_settings.sync_target_region = "overseas"
|
||||
|
||||
previous_payload = {
|
||||
"projection": {
|
||||
"worker_online": True,
|
||||
"worker_mode": "linux-systemd",
|
||||
"phase_label": "运行中",
|
||||
"phase_detail": "等待中",
|
||||
"proxy_runtime_label": "正常",
|
||||
"proxy_runtime_reason": "healthy",
|
||||
"active_thread_count": 0,
|
||||
"max_thread_count": 60000,
|
||||
"progress": {"pending": 10000, "running": 0, "completed": 0, "blacklisted": 0, "failed": 0},
|
||||
"active_job": {
|
||||
"job_id": None,
|
||||
"job_code": "",
|
||||
"status": "",
|
||||
"items_total": 0,
|
||||
"items_running": 0,
|
||||
"items_claimed": 0,
|
||||
"display_items_running": 0,
|
||||
"display_items_claimed": 0,
|
||||
"display_max_threads": 0,
|
||||
"node_stats": [],
|
||||
"distributed_node_stats": [],
|
||||
},
|
||||
"cluster_summary": {
|
||||
"nodes_total": 43,
|
||||
"online_worker_nodes": 43,
|
||||
"dedicated_online_worker_nodes": 42,
|
||||
"online_control_nodes": 1,
|
||||
"busy_nodes": [],
|
||||
"stale_nodes": [],
|
||||
"offline_nodes": [],
|
||||
},
|
||||
"cluster_nodes": [
|
||||
{
|
||||
"node_code": "mainland-controller-01",
|
||||
"role": "control",
|
||||
"status": "online",
|
||||
"current_load": 0,
|
||||
"active_threads": 0,
|
||||
"max_threads": 60000,
|
||||
"detect_participating": False,
|
||||
}
|
||||
],
|
||||
"dependency_alerts": [],
|
||||
}
|
||||
}
|
||||
fake_conn = _FakeConnection(
|
||||
[
|
||||
[(previous_payload, datetime.now() - timedelta(seconds=10))],
|
||||
(911,),
|
||||
]
|
||||
)
|
||||
mock_get_db.return_value = fake_conn
|
||||
|
||||
record_id = append_runtime_projection_if_changed(
|
||||
detect={
|
||||
"worker_online": True,
|
||||
"worker_mode": "linux-systemd",
|
||||
"phase_label": "运行中",
|
||||
"phase_detail": "等待中",
|
||||
"proxy_runtime_label": "正常",
|
||||
"proxy_runtime_reason": "healthy",
|
||||
"progress": {"pending": 10000, "running": 0, "completed": 0, "blacklisted": 0, "failed": 0},
|
||||
"backlog": {},
|
||||
"active_job": {},
|
||||
"dependency_alerts": [],
|
||||
},
|
||||
cluster={
|
||||
"nodes_total": 61,
|
||||
"nodes": [
|
||||
{
|
||||
"node_code": "mainland-controller-01",
|
||||
"role": "control",
|
||||
"status": "online",
|
||||
"current_load": 0,
|
||||
"metadata": {"active_threads": 0, "max_threads": 60000},
|
||||
},
|
||||
{
|
||||
"node_code": "mainland-controller-01-a",
|
||||
"role": "worker",
|
||||
"status": "online",
|
||||
"current_load": 0,
|
||||
"metadata": {"active_threads": 0, "max_threads": 1000},
|
||||
},
|
||||
],
|
||||
"summary": {
|
||||
"online_worker_nodes": 61,
|
||||
"dedicated_online_worker_nodes": 60,
|
||||
"online_control_nodes": 1,
|
||||
"busy_nodes": [],
|
||||
"stale_nodes": [],
|
||||
"offline_nodes": [],
|
||||
},
|
||||
},
|
||||
source_region="mainland",
|
||||
target_region="overseas",
|
||||
)
|
||||
|
||||
self.assertEqual(911, record_id)
|
||||
self.assertTrue(fake_conn.committed)
|
||||
self.assertTrue(
|
||||
any("INSERT INTO detect_sync_records" in sql for sql, _params in fake_conn.cursor_obj.executed)
|
||||
)
|
||||
|
||||
@patch("app.services.sync_record_service._resolve_local_ip", return_value="121.204.244.188")
|
||||
@patch("app.services.sync_record_service.socket.gethostname", return_value="mainland-controller-01")
|
||||
@patch("app.services.sync_record_service.settings")
|
||||
@patch("app.services.sync_record_service.get_db")
|
||||
def test_append_runtime_projection_if_changed_skips_unchanged_projection_within_heartbeat_window(
|
||||
self,
|
||||
mock_get_db,
|
||||
mock_settings,
|
||||
_mock_hostname,
|
||||
_mock_resolve_ip,
|
||||
) -> None:
|
||||
mock_settings.node_code = "mainland-controller-01"
|
||||
mock_settings.node_region = "mainland"
|
||||
mock_settings.node_role = "control"
|
||||
mock_settings.sync_source_region = "mainland"
|
||||
mock_settings.sync_target_region = "overseas"
|
||||
|
||||
detect = {
|
||||
"worker_online": True,
|
||||
"worker_mode": "linux-systemd",
|
||||
"active_thread_count": 3200,
|
||||
"max_thread_count": 80000,
|
||||
"phase_label": "运行中",
|
||||
"phase_detail": "80 实例运行",
|
||||
"proxy_runtime_label": "正常",
|
||||
"proxy_runtime_reason": "healthy",
|
||||
"progress": {
|
||||
"pending": 1200000,
|
||||
"running": 6400,
|
||||
"completed": 50000,
|
||||
"blacklisted": 1200,
|
||||
"failed": 88,
|
||||
},
|
||||
"backlog": {"pending_total": 1200000},
|
||||
"active_job": {
|
||||
"job_id": 55,
|
||||
"job_code": "sync-overseas-55",
|
||||
"status": "running",
|
||||
"progress_percent": 12.5,
|
||||
"items_total": 1300000,
|
||||
"items_terminal": 51288,
|
||||
"items_pending": 1200000,
|
||||
"items_running": 6400,
|
||||
"items_failed": 88,
|
||||
"node_stats": [
|
||||
{
|
||||
"node_code": "mainland-controller-01",
|
||||
"items_running": 6400,
|
||||
"items_claimed": 7000,
|
||||
"items_total": 1300000,
|
||||
}
|
||||
],
|
||||
},
|
||||
"dependency_alerts": [],
|
||||
}
|
||||
cluster = {
|
||||
"nodes_total": 1,
|
||||
"nodes": [
|
||||
{
|
||||
"node_code": "mainland-controller-01",
|
||||
"current_load": 6400,
|
||||
"detect_participating": True,
|
||||
}
|
||||
],
|
||||
"summary": {
|
||||
"online_worker_nodes": 1,
|
||||
"dedicated_online_worker_nodes": 0,
|
||||
"online_control_nodes": 1,
|
||||
"busy_nodes": ["mainland-controller-01"],
|
||||
"stale_nodes": [],
|
||||
"offline_nodes": [],
|
||||
},
|
||||
}
|
||||
|
||||
previous_payload = _build_runtime_projection_payload(
|
||||
detect=detect,
|
||||
cluster=cluster,
|
||||
source_region="mainland",
|
||||
target_region="overseas",
|
||||
)
|
||||
fake_conn = _FakeConnection(
|
||||
[
|
||||
[(previous_payload, datetime.now() - timedelta(seconds=10))],
|
||||
]
|
||||
)
|
||||
mock_get_db.return_value = fake_conn
|
||||
|
||||
record_id = append_runtime_projection_if_changed(
|
||||
detect=detect,
|
||||
cluster=cluster,
|
||||
source_region="mainland",
|
||||
target_region="overseas",
|
||||
)
|
||||
|
||||
self.assertIsNone(record_id)
|
||||
self.assertFalse(fake_conn.committed)
|
||||
self.assertFalse(
|
||||
any("INSERT INTO detect_sync_records" in sql for sql, _params in fake_conn.cursor_obj.executed)
|
||||
)
|
||||
|
||||
@patch("app.services.sync_record_service._resolve_local_ip", return_value="121.204.244.188")
|
||||
@patch("app.services.sync_record_service.socket.gethostname", return_value="mainland-controller-01")
|
||||
@patch("app.services.sync_record_service.settings")
|
||||
@patch("app.services.sync_record_service.get_db")
|
||||
def test_append_runtime_projection_if_changed_writes_when_activity_signature_changes_within_window(
|
||||
self,
|
||||
mock_get_db,
|
||||
mock_settings,
|
||||
_mock_hostname,
|
||||
_mock_resolve_ip,
|
||||
) -> None:
|
||||
mock_settings.node_code = "mainland-controller-01"
|
||||
mock_settings.node_region = "mainland"
|
||||
mock_settings.node_role = "control"
|
||||
mock_settings.sync_source_region = "mainland"
|
||||
mock_settings.sync_target_region = "overseas"
|
||||
|
||||
previous_detect = {
|
||||
"worker_online": True,
|
||||
"worker_mode": "linux-systemd",
|
||||
"active_thread_count": 300,
|
||||
"max_thread_count": 60000,
|
||||
"phase_label": "运行中",
|
||||
"phase_detail": "60 实例运行",
|
||||
"proxy_runtime_label": "正常",
|
||||
"proxy_runtime_reason": "healthy",
|
||||
"progress": {
|
||||
"pending": 5000,
|
||||
"running": 600,
|
||||
"completed": 1000,
|
||||
"blacklisted": 0,
|
||||
"failed": 10,
|
||||
},
|
||||
"queue_health": {
|
||||
"queue": {
|
||||
"items_total": 6610,
|
||||
"pending": 5000,
|
||||
"claimed": 10,
|
||||
"display_claimed": 10,
|
||||
"running": 600,
|
||||
"display_running": 600,
|
||||
"completed": 1000,
|
||||
"blacklisted": 0,
|
||||
"failed": 10,
|
||||
"terminal": 1010,
|
||||
"display_max_threads": 60000,
|
||||
},
|
||||
"nodes": [
|
||||
{
|
||||
"node_code": "mainland-controller-01-a",
|
||||
"items_running": 300,
|
||||
"items_claimed": 10,
|
||||
"display_running": 300,
|
||||
"active_threads": 300,
|
||||
"max_threads": 1000,
|
||||
"region": "mainland",
|
||||
"role": "control",
|
||||
"status": "busy",
|
||||
}
|
||||
],
|
||||
},
|
||||
"active_job": {
|
||||
"job_id": 867,
|
||||
"job_code": "sync-overseas-19835",
|
||||
"status": "running",
|
||||
"progress_percent": 15.0,
|
||||
"items_total": 6610,
|
||||
"items_terminal": 1010,
|
||||
"items_pending": 5000,
|
||||
"items_claimed": 10,
|
||||
"items_running": 600,
|
||||
"items_failed": 10,
|
||||
"node_stats": [
|
||||
{
|
||||
"node_code": "mainland-controller-01-a",
|
||||
"items_running": 300,
|
||||
"items_claimed": 10,
|
||||
"items_total": 6610,
|
||||
}
|
||||
],
|
||||
},
|
||||
"dependency_alerts": [],
|
||||
}
|
||||
current_detect = {
|
||||
**previous_detect,
|
||||
"active_thread_count": 900,
|
||||
"queue_health": {
|
||||
"queue": {
|
||||
"items_total": 6610,
|
||||
"pending": 4300,
|
||||
"claimed": 30,
|
||||
"display_claimed": 30,
|
||||
"running": 900,
|
||||
"display_running": 900,
|
||||
"completed": 1370,
|
||||
"blacklisted": 0,
|
||||
"failed": 10,
|
||||
"terminal": 1380,
|
||||
"display_max_threads": 60000,
|
||||
},
|
||||
"nodes": [
|
||||
{
|
||||
"node_code": "mainland-controller-01-a",
|
||||
"items_running": 420,
|
||||
"items_claimed": 20,
|
||||
"display_running": 420,
|
||||
"active_threads": 420,
|
||||
"max_threads": 1000,
|
||||
"region": "mainland",
|
||||
"role": "control",
|
||||
"status": "busy",
|
||||
},
|
||||
{
|
||||
"node_code": "mainland-controller-01-b",
|
||||
"items_running": 480,
|
||||
"items_claimed": 10,
|
||||
"display_running": 480,
|
||||
"active_threads": 480,
|
||||
"max_threads": 1000,
|
||||
"region": "mainland",
|
||||
"role": "control",
|
||||
"status": "busy",
|
||||
},
|
||||
],
|
||||
},
|
||||
"active_job": {
|
||||
**previous_detect["active_job"],
|
||||
"items_pending": 4300,
|
||||
"items_claimed": 30,
|
||||
"items_running": 900,
|
||||
"node_stats": [
|
||||
{
|
||||
"node_code": "mainland-controller-01-a",
|
||||
"items_running": 420,
|
||||
"items_claimed": 20,
|
||||
"items_total": 6610,
|
||||
},
|
||||
{
|
||||
"node_code": "mainland-controller-01-b",
|
||||
"items_running": 480,
|
||||
"items_claimed": 10,
|
||||
"items_total": 6610,
|
||||
},
|
||||
],
|
||||
},
|
||||
}
|
||||
cluster = {
|
||||
"nodes_total": 2,
|
||||
"nodes": [
|
||||
{
|
||||
"node_code": "mainland-controller-01",
|
||||
"current_load": 900,
|
||||
"detect_participating": True,
|
||||
}
|
||||
],
|
||||
"summary": {
|
||||
"online_worker_nodes": 60,
|
||||
"dedicated_online_worker_nodes": 0,
|
||||
"online_control_nodes": 60,
|
||||
"busy_nodes": ["mainland-controller-01-a", "mainland-controller-01-b"],
|
||||
"stale_nodes": [],
|
||||
"offline_nodes": [],
|
||||
},
|
||||
}
|
||||
|
||||
previous_payload = _build_runtime_projection_payload(
|
||||
detect=previous_detect,
|
||||
cluster=cluster,
|
||||
source_region="mainland",
|
||||
target_region="overseas",
|
||||
)
|
||||
fake_conn = _FakeConnection(
|
||||
[
|
||||
[(previous_payload, datetime.now() - timedelta(seconds=10))],
|
||||
(654,),
|
||||
]
|
||||
)
|
||||
mock_get_db.return_value = fake_conn
|
||||
|
||||
record_id = append_runtime_projection_if_changed(
|
||||
detect=current_detect,
|
||||
cluster=cluster,
|
||||
source_region="mainland",
|
||||
target_region="overseas",
|
||||
)
|
||||
|
||||
self.assertEqual(654, record_id)
|
||||
self.assertTrue(fake_conn.committed)
|
||||
self.assertTrue(
|
||||
any("INSERT INTO detect_sync_records" in sql for sql, _params in fake_conn.cursor_obj.executed)
|
||||
)
|
||||
|
||||
def test_pick_latest_projection_row_skips_future_dated_rows(self) -> None:
|
||||
rows = [
|
||||
("future", datetime.now() + timedelta(hours=6)),
|
||||
("recent", datetime.now() - timedelta(seconds=10)),
|
||||
("older", datetime.now() - timedelta(minutes=2)),
|
||||
]
|
||||
|
||||
selected = _pick_latest_projection_row(rows, created_at_index=1)
|
||||
|
||||
self.assertEqual("recent", selected[0])
|
||||
|
||||
@patch("app.services.sync_record_service.get_db")
|
||||
@patch("app.services.sync_record_service.settings")
|
||||
def test_get_detect_result_sync_batches_marks_overseas_control_as_not_applicable(
|
||||
self,
|
||||
mock_settings,
|
||||
mock_get_db,
|
||||
) -> None:
|
||||
mock_settings.node_code = "overseas-control-01"
|
||||
mock_settings.node_region = "overseas"
|
||||
mock_settings.node_role = "control"
|
||||
mock_settings.sync_source_region = "overseas"
|
||||
mock_settings.sync_target_region = "mainland"
|
||||
|
||||
payload = get_detect_result_sync_batches(limit=5)
|
||||
|
||||
self.assertFalse(payload["applicable"])
|
||||
self.assertFalse(payload["local_worker_expected"])
|
||||
self.assertEqual(0, payload["jobs_total"])
|
||||
self.assertEqual([], payload["batches"])
|
||||
self.assertIn("不适用", payload["reason"])
|
||||
mock_get_db.assert_not_called()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
@@ -1,8 +1,17 @@
|
||||
import json
|
||||
import subprocess
|
||||
import unittest
|
||||
from unittest.mock import Mock, patch
|
||||
|
||||
from app.services.worker_control_service import WORKER_CONTROL_CHANNEL, WORKER_PENDING_COMMAND_KEY, send_worker_command
|
||||
from app.core.config import settings
|
||||
from app.services.worker_control_service import (
|
||||
WORKER_CONTROL_CHANNEL,
|
||||
WORKER_PENDING_COMMAND_KEY,
|
||||
detect_worker_runtime,
|
||||
send_worker_command,
|
||||
start_worker,
|
||||
stop_worker,
|
||||
)
|
||||
|
||||
|
||||
class WorkerControlServiceTests(unittest.TestCase):
|
||||
@@ -11,10 +20,11 @@ class WorkerControlServiceTests(unittest.TestCase):
|
||||
redis_client = Mock()
|
||||
mock_get_redis.return_value = redis_client
|
||||
|
||||
ok, message = send_worker_command(
|
||||
"start_detection",
|
||||
payload={"job_id": 1, "job_code": "detect-20260419030000-abc123"},
|
||||
)
|
||||
with patch("app.services.worker_control_service._runtime_config", return_value={"worker_mode": "windows-local", "worker_service_name": "domaincheck-worker"}):
|
||||
ok, message = send_worker_command(
|
||||
"start_detection",
|
||||
payload={"job_id": 1, "job_code": "detect-20260419030000-abc123"},
|
||||
)
|
||||
|
||||
self.assertTrue(ok)
|
||||
self.assertIn("已发送 Worker 控制指令", message)
|
||||
@@ -33,6 +43,309 @@ class WorkerControlServiceTests(unittest.TestCase):
|
||||
|
||||
redis_client.publish.assert_called_once_with(WORKER_CONTROL_CHANNEL, serialized)
|
||||
|
||||
@patch("app.services.worker_control_service.get_redis")
|
||||
def test_send_worker_command_scopes_pending_command_to_target_worker_instances(self, mock_get_redis) -> None:
|
||||
redis_client = Mock()
|
||||
mock_get_redis.return_value = redis_client
|
||||
|
||||
with patch("app.services.worker_control_service._runtime_config", return_value={"worker_mode": "linux-systemd", "worker_service_name": "domaincheck-worker"}):
|
||||
ok, message = send_worker_command(
|
||||
"start_detection",
|
||||
payload={
|
||||
"job_id": 2,
|
||||
"target_node_codes": ["mainland-controller-01-a", "mainland-controller-01-b"],
|
||||
},
|
||||
)
|
||||
|
||||
self.assertTrue(ok)
|
||||
self.assertIn("mainland-controller-01-a,mainland-controller-01-b", message)
|
||||
self.assertEqual(2, redis_client.set.call_count)
|
||||
|
||||
set_keys = [call.args[0] for call in redis_client.set.call_args_list]
|
||||
self.assertEqual(
|
||||
[
|
||||
f"{WORKER_PENDING_COMMAND_KEY}:mainland-controller-01-a",
|
||||
f"{WORKER_PENDING_COMMAND_KEY}:mainland-controller-01-b",
|
||||
],
|
||||
set_keys,
|
||||
)
|
||||
|
||||
serialized = redis_client.set.call_args_list[0].args[1]
|
||||
payload = json.loads(serialized)
|
||||
self.assertEqual(["mainland-controller-01-a", "mainland-controller-01-b"], payload["target_node_codes"])
|
||||
redis_client.publish.assert_called_once_with(WORKER_CONTROL_CHANNEL, serialized)
|
||||
|
||||
@patch("app.services.worker_control_service._expand_linux_worker_control_units")
|
||||
@patch("app.services.worker_control_service.get_redis")
|
||||
def test_send_worker_command_expands_local_linux_worker_instances_when_targets_unspecified(
|
||||
self,
|
||||
mock_get_redis,
|
||||
mock_expand_linux_worker_control_units,
|
||||
) -> None:
|
||||
redis_client = Mock()
|
||||
mock_get_redis.return_value = redis_client
|
||||
mock_expand_linux_worker_control_units.return_value = [
|
||||
"domaincheck-worker",
|
||||
"domaincheck-worker@a.service",
|
||||
"domaincheck-worker@b.service",
|
||||
]
|
||||
|
||||
with patch("app.services.worker_control_service._runtime_config", return_value={"worker_mode": "linux-systemd", "worker_service_name": "domaincheck-worker"}), \
|
||||
patch.object(settings, "node_code", "mainland-controller-01"):
|
||||
ok, message = send_worker_command("start_detection", payload={"job_id": 3})
|
||||
|
||||
self.assertTrue(ok)
|
||||
self.assertIn("mainland-controller-01,mainland-controller-01-a,mainland-controller-01-b", message)
|
||||
self.assertEqual(3, redis_client.set.call_count)
|
||||
set_keys = [call.args[0] for call in redis_client.set.call_args_list]
|
||||
self.assertEqual(
|
||||
[
|
||||
f"{WORKER_PENDING_COMMAND_KEY}:mainland-controller-01",
|
||||
f"{WORKER_PENDING_COMMAND_KEY}:mainland-controller-01-a",
|
||||
f"{WORKER_PENDING_COMMAND_KEY}:mainland-controller-01-b",
|
||||
],
|
||||
set_keys,
|
||||
)
|
||||
serialized = redis_client.set.call_args_list[0].args[1]
|
||||
payload = json.loads(serialized)
|
||||
self.assertEqual(
|
||||
["mainland-controller-01", "mainland-controller-01-a", "mainland-controller-01-b"],
|
||||
payload["target_node_codes"],
|
||||
)
|
||||
redis_client.publish.assert_called_once_with(WORKER_CONTROL_CHANNEL, serialized)
|
||||
|
||||
@patch("app.services.worker_control_service._build_direct_redis_client")
|
||||
@patch("app.services.worker_control_service.get_redis")
|
||||
def test_send_worker_command_falls_back_to_direct_redis_client(
|
||||
self,
|
||||
mock_get_redis,
|
||||
mock_build_direct_redis_client,
|
||||
) -> None:
|
||||
mock_get_redis.side_effect = RecursionError("maximum recursion depth exceeded")
|
||||
direct_client = Mock()
|
||||
mock_build_direct_redis_client.return_value = direct_client
|
||||
|
||||
with patch("app.services.worker_control_service._runtime_config", return_value={"worker_mode": "windows-local", "worker_service_name": "domaincheck-worker"}):
|
||||
ok, message = send_worker_command("start_detection", payload={"job_id": 9})
|
||||
|
||||
self.assertTrue(ok)
|
||||
self.assertIn("已发送 Worker 控制指令", message)
|
||||
direct_client.set.assert_called_once()
|
||||
serialized = direct_client.set.call_args.args[1]
|
||||
payload = json.loads(serialized)
|
||||
self.assertEqual("start_detection", payload["action"])
|
||||
self.assertEqual(9, payload["job_id"])
|
||||
direct_client.publish.assert_called_once_with(WORKER_CONTROL_CHANNEL, serialized)
|
||||
direct_client.close.assert_called_once()
|
||||
|
||||
@patch("app.services.worker_control_service._probe_linux_worker_instance_count", return_value=0)
|
||||
@patch("app.services.worker_control_service._run_shell")
|
||||
@patch("app.services.worker_control_service.probe_systemd_service")
|
||||
@patch("app.services.worker_control_service._runtime_config")
|
||||
def test_detect_worker_runtime_prefers_fast_pgrep_probe(
|
||||
self,
|
||||
mock_runtime_config,
|
||||
mock_probe_systemd_service,
|
||||
mock_run_shell,
|
||||
_mock_instance_count,
|
||||
) -> None:
|
||||
mock_runtime_config.return_value = {
|
||||
"worker_mode": "linux-systemd",
|
||||
"worker_service_name": "domaincheck-worker",
|
||||
}
|
||||
mock_probe_systemd_service.return_value = {
|
||||
"mode": "linux-systemd",
|
||||
"service_name": "domaincheck-worker",
|
||||
"running": True,
|
||||
"process_count": 1,
|
||||
"latest_start_time": "2026-04-22 23:00:00",
|
||||
"message": "active/running",
|
||||
}
|
||||
mock_run_shell.return_value = subprocess.CompletedProcess(
|
||||
args=["bash", "-lc", "pgrep -fc '[d]etect_worker.py' || true"],
|
||||
returncode=0,
|
||||
stdout="80\n",
|
||||
stderr="",
|
||||
)
|
||||
|
||||
runtime = detect_worker_runtime()
|
||||
|
||||
self.assertTrue(runtime["running"])
|
||||
self.assertEqual(80, runtime["process_count"])
|
||||
self.assertEqual(1, mock_run_shell.call_count)
|
||||
|
||||
@patch("app.services.worker_control_service._probe_linux_worker_instance_count", return_value=0)
|
||||
@patch("app.services.worker_control_service._run_shell")
|
||||
@patch("app.services.worker_control_service.probe_systemd_service")
|
||||
@patch("app.services.worker_control_service._runtime_config")
|
||||
def test_detect_worker_runtime_falls_back_when_pgrep_probe_is_unavailable(
|
||||
self,
|
||||
mock_runtime_config,
|
||||
mock_probe_systemd_service,
|
||||
mock_run_shell,
|
||||
_mock_instance_count,
|
||||
) -> None:
|
||||
mock_runtime_config.return_value = {
|
||||
"worker_mode": "linux-systemd",
|
||||
"worker_service_name": "domaincheck-worker",
|
||||
}
|
||||
mock_probe_systemd_service.return_value = {
|
||||
"mode": "linux-systemd",
|
||||
"service_name": "domaincheck-worker",
|
||||
"running": True,
|
||||
"process_count": 1,
|
||||
"latest_start_time": "2026-04-22 23:00:00",
|
||||
"message": "active/running",
|
||||
}
|
||||
mock_run_shell.side_effect = [
|
||||
subprocess.CompletedProcess(
|
||||
args=["bash", "-lc", "pgrep -fc '[d]etect_worker.py' || true"],
|
||||
returncode=0,
|
||||
stdout="",
|
||||
stderr="pgrep: command not found\n",
|
||||
),
|
||||
subprocess.CompletedProcess(
|
||||
args=["bash", "-lc", "ps -eo args= | grep '[d]etect_worker.py' | wc -l"],
|
||||
returncode=0,
|
||||
stdout="12\n",
|
||||
stderr="",
|
||||
),
|
||||
]
|
||||
|
||||
runtime = detect_worker_runtime()
|
||||
|
||||
self.assertTrue(runtime["running"])
|
||||
self.assertEqual(12, runtime["process_count"])
|
||||
self.assertEqual(2, mock_run_shell.call_count)
|
||||
|
||||
@patch("app.services.worker_control_service._probe_linux_worker_process_count", return_value=7)
|
||||
@patch("app.services.worker_control_service._probe_linux_worker_instance_count", return_value=0)
|
||||
@patch("app.services.worker_control_service.probe_systemd_service")
|
||||
@patch("app.services.worker_control_service._runtime_config")
|
||||
def test_detect_worker_runtime_keeps_service_offline_when_only_unmanaged_processes_exist(
|
||||
self,
|
||||
mock_runtime_config,
|
||||
mock_probe_systemd_service,
|
||||
_mock_instance_count,
|
||||
_mock_process_count,
|
||||
) -> None:
|
||||
mock_runtime_config.return_value = {
|
||||
"worker_mode": "linux-systemd",
|
||||
"worker_service_name": "domaincheck-worker",
|
||||
}
|
||||
mock_probe_systemd_service.return_value = {
|
||||
"mode": "linux-systemd",
|
||||
"service_name": "domaincheck-worker",
|
||||
"running": False,
|
||||
"process_count": 0,
|
||||
"latest_start_time": "",
|
||||
"message": "inactive/dead",
|
||||
}
|
||||
|
||||
runtime = detect_worker_runtime()
|
||||
|
||||
self.assertFalse(runtime["running"])
|
||||
self.assertEqual(7, runtime["process_count"])
|
||||
self.assertIn("unmanaged worker processes", runtime["message"])
|
||||
|
||||
@patch("app.services.worker_control_service._probe_linux_worker_process_count", return_value=30)
|
||||
@patch("app.services.worker_control_service._probe_linux_worker_instance_count", return_value=3)
|
||||
@patch("app.services.worker_control_service.probe_systemd_service")
|
||||
@patch("app.services.worker_control_service._runtime_config")
|
||||
def test_detect_worker_runtime_accepts_active_template_instances_when_base_service_is_inactive(
|
||||
self,
|
||||
mock_runtime_config,
|
||||
mock_probe_systemd_service,
|
||||
_mock_instance_count,
|
||||
_mock_process_count,
|
||||
) -> None:
|
||||
mock_runtime_config.return_value = {
|
||||
"worker_mode": "linux-systemd",
|
||||
"worker_service_name": "domaincheck-worker",
|
||||
}
|
||||
mock_probe_systemd_service.return_value = {
|
||||
"mode": "linux-systemd",
|
||||
"service_name": "domaincheck-worker",
|
||||
"running": False,
|
||||
"process_count": 0,
|
||||
"latest_start_time": "",
|
||||
"message": "inactive/dead",
|
||||
}
|
||||
|
||||
runtime = detect_worker_runtime()
|
||||
|
||||
self.assertTrue(runtime["running"])
|
||||
self.assertEqual(30, runtime["process_count"])
|
||||
self.assertEqual("template instances active (3)", runtime["message"])
|
||||
|
||||
@patch("app.services.worker_control_service._expand_linux_worker_control_units")
|
||||
@patch("app.services.worker_control_service._run_systemctl")
|
||||
@patch("app.services.worker_control_service._runtime_config")
|
||||
def test_start_worker_includes_template_instances(
|
||||
self,
|
||||
mock_runtime_config,
|
||||
mock_run_systemctl,
|
||||
mock_expand_units,
|
||||
) -> None:
|
||||
mock_runtime_config.return_value = {
|
||||
"worker_mode": "linux-systemd",
|
||||
"worker_service_name": "domaincheck-worker",
|
||||
}
|
||||
mock_expand_units.return_value = [
|
||||
"domaincheck-worker",
|
||||
"domaincheck-worker@a",
|
||||
"domaincheck-worker@b",
|
||||
]
|
||||
mock_run_systemctl.return_value = subprocess.CompletedProcess(
|
||||
args=["systemctl", "start", "domaincheck-worker", "domaincheck-worker@a", "domaincheck-worker@b"],
|
||||
returncode=0,
|
||||
stdout="",
|
||||
stderr="",
|
||||
)
|
||||
|
||||
ok, message = start_worker()
|
||||
|
||||
self.assertTrue(ok)
|
||||
self.assertIn("附带 2 个实例", message)
|
||||
mock_run_systemctl.assert_called_once_with(
|
||||
["start", "domaincheck-worker", "domaincheck-worker@a", "domaincheck-worker@b"],
|
||||
timeout=45,
|
||||
)
|
||||
|
||||
@patch("app.services.worker_control_service._expand_linux_worker_control_units")
|
||||
@patch("app.services.worker_control_service._run_systemctl")
|
||||
@patch("app.services.worker_control_service._runtime_config")
|
||||
def test_stop_worker_includes_template_instances(
|
||||
self,
|
||||
mock_runtime_config,
|
||||
mock_run_systemctl,
|
||||
mock_expand_units,
|
||||
) -> None:
|
||||
mock_runtime_config.return_value = {
|
||||
"worker_mode": "linux-systemd",
|
||||
"worker_service_name": "domaincheck-worker",
|
||||
}
|
||||
mock_expand_units.return_value = [
|
||||
"domaincheck-worker",
|
||||
"domaincheck-worker@a",
|
||||
"domaincheck-worker@b",
|
||||
]
|
||||
mock_run_systemctl.return_value = subprocess.CompletedProcess(
|
||||
args=["systemctl", "stop", "domaincheck-worker", "domaincheck-worker@a", "domaincheck-worker@b"],
|
||||
returncode=0,
|
||||
stdout="",
|
||||
stderr="",
|
||||
)
|
||||
|
||||
ok, message = stop_worker()
|
||||
|
||||
self.assertTrue(ok)
|
||||
self.assertIn("附带 2 个实例", message)
|
||||
mock_run_systemctl.assert_called_once_with(
|
||||
["stop", "domaincheck-worker", "domaincheck-worker@a", "domaincheck-worker@b"],
|
||||
timeout=45,
|
||||
)
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
unittest.main()
|
||||
|
||||
Reference in New Issue
Block a user