feat: stabilize multi-region runtime sync and worker orchestration

This commit is contained in:
root
2026-04-27 15:48:12 +08:00
parent 7cbde2aa78
commit 215a364891
137 changed files with 31931 additions and 1943 deletions

View File

@@ -81,6 +81,9 @@ class BuildInfoServiceTests(unittest.TestCase):
self.assertEqual("env-sha256", payload["checksum"])
self.assertFalse(payload["route_surface"]["surface_complete"])
self.assertIn("ops_stack_diagnosis", payload["route_surface"]["missing_keys"])
self.assertIn("ops_migration_source_profile", payload["route_surface"]["missing_keys"])
self.assertIn("ops_migration_preview", payload["route_surface"]["missing_keys"])
self.assertIn("ops_migration_execute", payload["route_surface"]["missing_keys"])
self.assertIn("ops_node_onboarding_bootstrap_preview", payload["route_surface"]["missing_keys"])
self.assertIn("ops_node_onboarding_bootstrap_execute", payload["route_surface"]["missing_keys"])
self.assertIn("ops_node_onboarding_acceptance_preview", payload["route_surface"]["missing_keys"])

View File

@@ -1,12 +1,65 @@
from __future__ import annotations
import unittest
from datetime import datetime, timedelta
from unittest.mock import MagicMock, patch
from psycopg2 import errors
from app.services import cluster_runtime_service
class RuntimeSchemaGuardTests(unittest.TestCase):
def test_ensure_runtime_schema_does_not_toggle_autocommit_after_queries(self) -> None:
class _Connection:
def __init__(self) -> None:
self._autocommit = False
self.touched = False
self.committed = False
@property
def autocommit(self):
return self._autocommit
@autocommit.setter
def autocommit(self, value):
if self.touched:
raise AssertionError("autocommit should not be reassigned after queries start")
self._autocommit = value
def cursor(self):
conn = self
class _CursorContext:
def __enter__(self_inner):
conn.touched = True
return cursor
def __exit__(self_inner, exc_type, exc, tb):
return False
return _CursorContext()
def commit(self):
self.committed = True
cursor = MagicMock()
conn = _Connection()
db_cm = MagicMock()
db_cm.__enter__.return_value = conn
with patch.object(cluster_runtime_service, "_RUNTIME_SCHEMA_READY", False):
with patch.object(cluster_runtime_service, "get_db", return_value=db_cm):
with patch.object(cluster_runtime_service, "_runtime_schema_basics_present", return_value=False):
cluster_runtime_service.ensure_runtime_schema()
cursor.execute.assert_any_call(
"SELECT pg_advisory_xact_lock(%s)",
(cluster_runtime_service._RUNTIME_SCHEMA_ADVISORY_LOCK_ID,),
)
cursor.execute.assert_any_call(cluster_runtime_service._RUNTIME_SCHEMA_SQL)
self.assertTrue(conn.committed)
def test_ensure_runtime_schema_executes_only_once_per_process(self) -> None:
conn = MagicMock()
cursor_cm = MagicMock()
@@ -18,11 +71,11 @@ class RuntimeSchemaGuardTests(unittest.TestCase):
with patch.object(cluster_runtime_service, "_RUNTIME_SCHEMA_READY", False):
with patch.object(cluster_runtime_service, "get_db", return_value=db_cm) as mocked_get_db:
cluster_runtime_service.ensure_runtime_schema()
cluster_runtime_service.ensure_runtime_schema()
with patch.object(cluster_runtime_service, "_runtime_schema_basics_present", return_value=False):
cluster_runtime_service.ensure_runtime_schema()
cluster_runtime_service.ensure_runtime_schema()
mocked_get_db.assert_called_once()
self.assertEqual(cursor.execute.call_count, 2)
cursor.execute.assert_any_call(
"SELECT pg_advisory_xact_lock(%s)",
(cluster_runtime_service._RUNTIME_SCHEMA_ADVISORY_LOCK_ID,),
@@ -30,6 +83,118 @@ class RuntimeSchemaGuardTests(unittest.TestCase):
cursor.execute.assert_any_call(cluster_runtime_service._RUNTIME_SCHEMA_SQL)
conn.commit.assert_called_once()
def test_ensure_runtime_schema_skips_ddl_when_required_schema_already_exists(self) -> None:
conn = MagicMock()
cursor_cm = MagicMock()
cursor = MagicMock()
conn.cursor.return_value = cursor_cm
cursor_cm.__enter__.return_value = cursor
db_cm = MagicMock()
db_cm.__enter__.return_value = conn
with patch.object(cluster_runtime_service, "_RUNTIME_SCHEMA_READY", False):
with patch.object(cluster_runtime_service, "get_db", return_value=db_cm):
with patch.object(cluster_runtime_service, "_runtime_schema_basics_present", return_value=True):
with patch.object(cluster_runtime_service, "_runtime_missing_indexes", return_value=iter(())):
cluster_runtime_service.ensure_runtime_schema()
self.assertFalse(
any(sql == cluster_runtime_service._RUNTIME_SCHEMA_SQL for sql, _params in cursor.execute.call_args_list)
)
conn.commit.assert_not_called()
def test_ensure_runtime_schema_accepts_deadlock_when_required_schema_already_exists(self) -> None:
class _Cursor:
def __init__(self, *, raise_on_schema=False, fetchone_values=None, fetchall_values=None) -> None:
self.raise_on_schema = raise_on_schema
self.fetchone_values = list(fetchone_values or [])
self.fetchall_values = list(fetchall_values or [])
self.executed = []
def execute(self, sql, params=None):
self.executed.append((sql, params))
if self.raise_on_schema and sql == cluster_runtime_service._RUNTIME_SCHEMA_SQL:
raise errors.DeadlockDetected()
def fetchone(self):
if self.fetchone_values:
return self.fetchone_values.pop(0)
return None
def fetchall(self):
if self.fetchall_values:
return self.fetchall_values.pop(0)
return []
class _CursorContext:
def __init__(self, cursor) -> None:
self.cursor = cursor
def __enter__(self):
return self.cursor
def __exit__(self, exc_type, exc, tb):
return False
conn = MagicMock()
first_cursor = _Cursor()
second_cursor = _Cursor(raise_on_schema=True)
third_cursor = _Cursor()
conn.cursor.side_effect = [
_CursorContext(first_cursor),
_CursorContext(second_cursor),
_CursorContext(third_cursor),
]
db_cm = MagicMock()
db_cm.__enter__.return_value = conn
with patch.object(cluster_runtime_service, "_RUNTIME_SCHEMA_READY", False):
with patch.object(cluster_runtime_service, "get_db", return_value=db_cm):
with patch.object(cluster_runtime_service, "_runtime_schema_basics_present", side_effect=[False, True]):
with patch.object(cluster_runtime_service, "_runtime_missing_indexes", return_value=iter(())):
cluster_runtime_service.ensure_runtime_schema()
conn.rollback.assert_called_once()
conn.commit.assert_not_called()
def test_ensure_runtime_schema_repairs_missing_indexes_without_full_ddl(self) -> None:
conn = MagicMock()
cursor_cm = MagicMock()
cursor = MagicMock()
conn.cursor.return_value = cursor_cm
cursor_cm.__enter__.return_value = cursor
db_cm = MagicMock()
db_cm.__enter__.return_value = conn
with patch.object(cluster_runtime_service, "_RUNTIME_SCHEMA_READY", False):
with patch.object(cluster_runtime_service, "get_db", return_value=db_cm):
with patch.object(cluster_runtime_service, "_runtime_schema_basics_present", return_value=True):
with patch.object(
cluster_runtime_service,
"_runtime_missing_indexes",
side_effect=[iter(("idx_detect_job_items_claim_step_ready",)), iter(())],
):
with patch.object(cluster_runtime_service, "_ensure_runtime_schema_indexes") as mocked_ensure_indexes:
cluster_runtime_service.ensure_runtime_schema()
mocked_ensure_indexes.assert_called_once()
self.assertFalse(
any(sql == cluster_runtime_service._RUNTIME_SCHEMA_SQL for sql, _params in cursor.execute.call_args_list)
)
def test_runtime_missing_indexes_treats_invalid_indexes_as_missing(self) -> None:
cur = MagicMock()
cur.fetchall.return_value = [
("idx_detect_job_items_job_domain_step", True, True, True),
("idx_detect_job_items_claim_step_ready", False, True, True),
]
missing = list(cluster_runtime_service._runtime_missing_indexes(cur))
self.assertIn("idx_detect_job_items_claim_step_ready", missing)
self.assertIn("idx_detect_job_items_claim_job_step_ready", missing)
self.assertNotIn("idx_detect_job_items_job_domain_step", missing)
def test_control_node_supports_worker_only_on_mainland_with_worker_signals(self) -> None:
self.assertFalse(
cluster_runtime_service._control_node_supports_worker(
@@ -88,6 +253,187 @@ class RuntimeSchemaGuardTests(unittest.TestCase):
)
)
def test_get_cluster_snapshot_counts_all_rows_even_when_display_nodes_are_limited(self) -> None:
now = datetime.now()
display_rows = [
(
f"mainland-worker-{index:03d}",
"mainland",
"worker",
f"worker-{index:03d}",
f"10.0.0.{index}",
"online",
"test",
0,
{},
now,
now,
)
for index in range(1, 101)
]
summary_rows = display_rows + [
(
"mainland-worker-101",
"mainland",
"worker",
"worker-101",
"10.0.0.101",
"online",
"test",
0,
{},
now,
now,
)
]
conn = MagicMock()
cursor_cm = MagicMock()
cursor = MagicMock()
cursor.fetchall.side_effect = [display_rows, summary_rows]
cursor.fetchone.side_effect = [(12,), (34,)]
conn.cursor.return_value = cursor_cm
cursor_cm.__enter__.return_value = cursor
db_cm = MagicMock()
db_cm.__enter__.return_value = conn
with patch.object(cluster_runtime_service, "get_db", return_value=db_cm):
with patch.object(cluster_runtime_service, "prune_expired_runtime_nodes"):
with patch.object(cluster_runtime_service, "register_local_control_heartbeat"):
with patch.object(cluster_runtime_service, "_load_managed_node_overlays", return_value={}):
with patch.object(cluster_runtime_service, "_load_disabled_managed_node_codes", return_value=set()):
snapshot = cluster_runtime_service.get_cluster_snapshot()
self.assertEqual(100, len(snapshot["nodes"]))
self.assertEqual(101, snapshot["nodes_total"])
self.assertEqual(101, snapshot["summary"]["online_worker_nodes"])
self.assertEqual(12, snapshot["jobs_total"])
self.assertEqual(34, snapshot["active_job_items"])
def test_get_cluster_snapshot_excludes_disabled_managed_nodes(self) -> None:
now = datetime.now()
display_rows = [
("mainland-controller-01", "mainland", "control", "controller", "10.0.0.1", "online", "test", 1, {}, now, now),
("mainland-worker-01", "mainland", "worker", "worker-01", "10.0.0.2", "busy", "test", 1, {}, now, now),
]
summary_rows = list(display_rows)
conn = MagicMock()
cursor_cm = MagicMock()
cursor = MagicMock()
cursor.fetchall.side_effect = [display_rows, summary_rows]
cursor.fetchone.side_effect = [(1,), (2,)]
conn.cursor.return_value = cursor_cm
cursor_cm.__enter__.return_value = cursor
db_cm = MagicMock()
db_cm.__enter__.return_value = conn
with patch.object(cluster_runtime_service, "get_db", return_value=db_cm):
with patch.object(cluster_runtime_service, "prune_expired_runtime_nodes"):
with patch.object(cluster_runtime_service, "register_local_control_heartbeat"):
with patch.object(cluster_runtime_service, "_load_managed_node_overlays", return_value={}):
with patch.object(cluster_runtime_service, "_load_disabled_managed_node_codes", return_value={"mainland-worker-01"}):
snapshot = cluster_runtime_service.get_cluster_snapshot()
self.assertEqual(["mainland-controller-01"], [item["node_code"] for item in snapshot["nodes"]])
self.assertEqual(1, snapshot["nodes_total"])
self.assertEqual(1, snapshot["summary"]["online_control_nodes"])
self.assertEqual(0, snapshot["summary"]["online_worker_nodes"])
def test_get_cluster_snapshot_prefers_imported_runtime_update_time_for_freshness(self) -> None:
now = datetime.now()
stale_heartbeat = now - timedelta(minutes=8)
fresh_ingest = now - timedelta(seconds=20)
display_rows = [
(
"mainland-controller-01-da",
"mainland",
"worker",
"controller",
"10.0.0.1",
"busy",
"test",
42,
{
"service": "runtime-ingest",
"updated_at": fresh_ingest.isoformat(timespec="seconds"),
"active_threads": 42,
},
stale_heartbeat,
fresh_ingest,
)
]
summary_rows = list(display_rows)
conn = MagicMock()
cursor_cm = MagicMock()
cursor = MagicMock()
cursor.fetchall.side_effect = [display_rows, summary_rows]
cursor.fetchone.side_effect = [(1,), (1,)]
conn.cursor.return_value = cursor_cm
cursor_cm.__enter__.return_value = cursor
db_cm = MagicMock()
db_cm.__enter__.return_value = conn
with patch.object(cluster_runtime_service, "get_db", return_value=db_cm):
with patch.object(cluster_runtime_service, "prune_expired_runtime_nodes"):
with patch.object(cluster_runtime_service, "register_local_control_heartbeat"):
with patch.object(cluster_runtime_service, "_load_managed_node_overlays", return_value={}):
with patch.object(cluster_runtime_service, "_load_disabled_managed_node_codes", return_value=set()):
snapshot = cluster_runtime_service.get_cluster_snapshot()
self.assertEqual(1, snapshot["summary"]["online_worker_nodes"])
self.assertEqual([], snapshot["summary"]["offline_nodes"])
self.assertEqual("busy", snapshot["nodes"][0]["status"])
def test_register_local_control_heartbeat_uses_detect_status_snapshot(self) -> None:
detect_status = {
"worker_online": True,
"active_thread_count": 17,
"max_thread_count": 320,
"available_proxy_count": 41,
"proxy_runtime_label": "代理可用",
"proxy_runtime_reason": "pool_ready",
"proxy_last_refresh_status": "ok",
"proxy_last_refresh_time": "2026-04-24 01:30:00",
"proxy_last_refresh_source_count": 3,
"proxy_last_refresh_total_items": 120,
"proxy_last_validated_count": 110,
"proxy_last_available_count": 41,
"phase_label": "running",
"phase_detail": "正在执行检测",
"active_job": {
"job_code": "detect-20260424013000-abcd12",
"status": "running",
"node_stats": [
{
"node_code": "mainland-controller-01",
"items_total": 56,
"items_claimed": 5,
"items_running": 9,
"items_completed": 42,
}
],
},
}
with patch.object(cluster_runtime_service.settings, "node_code", "mainland-controller-01"):
with patch.object(cluster_runtime_service.settings, "node_region", "mainland"):
with patch.object(cluster_runtime_service.settings, "node_role", "control"):
with patch("app.services.detect_service.get_detect_status", return_value=detect_status):
with patch.object(cluster_runtime_service, "register_node_heartbeat") as mock_register:
cluster_runtime_service.register_local_control_heartbeat()
mock_register.assert_called_once()
payload = mock_register.call_args.kwargs
self.assertEqual("mainland-controller-01", payload["node_code"])
self.assertEqual("busy", payload["status"])
self.assertEqual(17, payload["current_load"])
self.assertTrue(payload["metadata"]["worker_online"])
self.assertTrue(payload["metadata"]["detect_participating"])
self.assertEqual("detect-20260424013000-abcd12", payload["metadata"]["active_job_code"])
self.assertEqual(17, payload["metadata"]["active_threads"])
self.assertEqual(320, payload["metadata"]["max_threads"])
self.assertEqual(41, payload["metadata"]["available_proxy_count"])
self.assertEqual("running", payload["metadata"]["phase_label"])
if __name__ == "__main__":
unittest.main()

View File

@@ -39,20 +39,22 @@ class _FakeConnection:
class DashboardServiceTests(unittest.TestCase):
@patch("app.services.dashboard.get_detect_status")
@patch("app.services.dashboard._fetch_active_jobs_aggregate")
@patch("app.services.dashboard.get_detect_capacity_plan")
@patch("app.services.dashboard.get_detect_queue_health")
@patch("app.services.dashboard.get_runtime_status")
@patch("app.services.dashboard._build_dashboard_runtime_summary")
@patch("app.services.dashboard.get_active_detect_job_summary")
@patch("app.services.dashboard.get_db")
def test_fetch_overview_includes_ops_metrics(
self,
mock_get_db,
mock_get_active_detect_job_summary,
mock_get_runtime_status,
mock_build_dashboard_runtime_summary,
mock_get_detect_queue_health,
mock_get_detect_capacity_plan,
mock_fetch_active_jobs_aggregate,
mock_get_detect_status,
) -> None:
mock_get_db.return_value = _FakeConnection(
responses=[
@@ -122,7 +124,7 @@ class DashboardServiceTests(unittest.TestCase):
"items_blacklisted": 0,
"items_failed": 0,
}
mock_get_runtime_status.return_value = {
mock_build_dashboard_runtime_summary.return_value = {
"worker": {"running": True, "mode": "linux-systemd", "expected_on_this_node": True},
"node": {"region": "overseas", "role": "control"},
"cluster": {"summary": {"online_worker_nodes": 2, "dedicated_online_worker_nodes": 1, "online_control_nodes": 1}},
@@ -198,6 +200,15 @@ class DashboardServiceTests(unittest.TestCase):
"remaining_items": 193,
"recommended_additional_workers": 1,
}
mock_get_detect_status.return_value = {
"available_proxy_count": 1011,
"proxy_runtime_label": "集群代理正常",
"proxy_runtime_detail": "参与服务器 2 台,共可用 1011 个代理",
"proxy_last_refresh_status": "mainland-controller-01:201mainland-worker-01:810",
"aggregate_process_count": 10,
"aggregate_participating_node_count": 10,
"active_thread_count": 165,
}
data = fetch_overview()
@@ -218,6 +229,10 @@ class DashboardServiceTests(unittest.TestCase):
self.assertEqual(165, data["queue_display_running_total"])
self.assertEqual(7, data["queue_completed_total"])
self.assertEqual(9438, data["backlog_pending_total"])
self.assertEqual(1011, data["cluster_proxy_available_count"])
self.assertEqual("集群代理正常", data["cluster_proxy_runtime_label"])
self.assertEqual(10, data["aggregate_process_count"])
self.assertEqual(1, data["ops_summary"]["active_execution_nodes"])
self.assertEqual(8487, data["backlog_register_pending_total"])
self.assertEqual(951, data["backlog_downstream_pending_total"])
self.assertEqual(12, data["retry_total"])
@@ -228,10 +243,256 @@ class DashboardServiceTests(unittest.TestCase):
self.assertEqual(1.5, data["ops_summary"]["estimated_hours_remaining"])
self.assertEqual(1, data["active_execution_nodes"])
self.assertEqual(1, data["ops_summary"]["active_execution_nodes"])
self.assertEqual(0, data["current_job_blacklisted"])
self.assertEqual(0, data["recent_blacklisted_total"])
self.assertEqual(0, data["cumulative_blacklisted_total"])
self.assertEqual(2, len(data["step_queue"]))
self.assertTrue(any(item["step_code"] == "detect_360_site" for item in data["step_queue"]))
self.assertEqual(1, len(data["node_throughput"]))
@patch("app.services.dashboard._fetch_active_jobs_aggregate")
@patch("app.services.dashboard.get_detect_capacity_plan")
@patch("app.services.dashboard.get_detect_queue_health")
@patch("app.services.dashboard._build_dashboard_runtime_summary")
@patch("app.services.dashboard.get_active_detect_job_summary")
@patch("app.services.dashboard.get_db")
def test_fetch_overview_prefers_active_job_display_running_when_queue_snapshot_is_stale(
self,
mock_get_db,
mock_get_active_detect_job_summary,
mock_build_dashboard_runtime_summary,
mock_get_detect_queue_health,
mock_get_detect_capacity_plan,
mock_fetch_active_jobs_aggregate,
) -> None:
mock_get_db.return_value = _FakeConnection(
responses=[
(1000,),
(900,),
(10,),
(5,),
(0,),
(1,),
(430,),
(420,),
(17,),
]
)
mock_fetch_active_jobs_aggregate.return_value = {
"active_jobs_total": 1,
"queue": {"items_total": 8402, "pending": 5001, "claimed": 0, "running": 1, "completed": 0, "blacklisted": 0, "failed": 0},
"throughput": {"processed_recent": 0, "processed_per_minute": 0.0},
"steps": [],
"nodes": [],
"retry_total": 0,
}
mock_get_active_detect_job_summary.return_value = {
"job_id": 11,
"job_code": "sync-overseas-51",
"status": "running",
"items_total": 8402,
"items_pending": 5001,
"items_claimed": 0,
"items_running": 0,
"items_completed": 0,
"items_blacklisted": 0,
"items_failed": 0,
"display_items_running": 1432,
"display_active_threads": 138,
"display_max_threads": 80000,
"distributed_node_stats": [
{"node_code": "mainland-controller-01-a", "display_running": 1000, "active_threads": 1000, "max_threads": 1000},
{"node_code": "mainland-controller-01-b", "display_running": 432, "active_threads": 432, "max_threads": 1000},
],
}
mock_build_dashboard_runtime_summary.return_value = {
"worker": {"running": False, "mode": "linux-systemd", "expected_on_this_node": False},
"node": {"region": "overseas", "role": "control"},
"cluster": {"summary": {"online_worker_nodes": 1, "dedicated_online_worker_nodes": 0, "online_control_nodes": 2}},
"detect": {
"backlog": {
"pending_total": 5001,
"claimed_total": 0,
"running_total": 0,
"completed_total": 0,
"blacklisted_total": 0,
"failed_total": 0,
"register_pending": 3495,
"downstream_pending": 1506,
}
},
}
mock_get_detect_queue_health.return_value = {
"has_active_job": True,
"job": {"job_id": 11, "job_code": "sync-overseas-51", "runtime_job_code": "sync-overseas-51", "status": "running", "progress_percent": 40.49},
"queue": {
"items_total": 8402,
"pending": 5001,
"claimed": 0,
"display_claimed": 0,
"running": 1,
"display_running": 1,
"completed": 0,
"blacklisted": 0,
"failed": 0,
},
"throughput": {"processed_recent": 0, "processed_per_minute": 0.0},
"steps": [],
"runtime_activity": {},
"nodes": [],
}
mock_get_detect_capacity_plan.return_value = {
"estimated_hours_remaining": 0,
"remaining_items": 5001,
"recommended_additional_workers": 0,
}
data = fetch_overview()
self.assertEqual(1432, data["queue_display_running_total"])
self.assertEqual(1432, data["active_job"]["items_display_running"])
self.assertEqual(1432, data["active_job"]["display_items_running"])
self.assertEqual(138, data["active_job"]["display_active_threads"])
self.assertEqual(80000, data["active_job"]["display_max_threads"])
self.assertEqual(2, len(data["active_job"]["distributed_node_stats"]))
self.assertEqual(80000, data["queue_display_max_threads"])
self.assertEqual(0, data["current_job_blacklisted"])
self.assertEqual(0, data["recent_blacklisted_total"])
self.assertEqual(0, data["cumulative_blacklisted_total"])
@patch("app.services.dashboard._fetch_active_jobs_aggregate")
@patch("app.services.dashboard.get_detect_capacity_plan")
@patch("app.services.dashboard.get_detect_queue_health")
@patch("app.services.dashboard._build_dashboard_runtime_summary")
@patch("app.services.dashboard.get_active_detect_job_summary")
@patch("app.services.dashboard.get_db")
def test_fetch_overview_counts_active_execution_nodes_from_full_node_set(
self,
mock_get_db,
mock_get_active_detect_job_summary,
mock_build_dashboard_runtime_summary,
mock_get_detect_queue_health,
mock_get_detect_capacity_plan,
mock_fetch_active_jobs_aggregate,
) -> None:
mock_get_db.return_value = _FakeConnection(responses=[(0,)] * 9)
mock_fetch_active_jobs_aggregate.return_value = {
"active_jobs_total": 1,
"queue": {"items_total": 9, "pending": 0, "claimed": 0, "running": 9, "completed": 0, "blacklisted": 0, "failed": 0},
"throughput": {"processed_recent": 0, "processed_per_minute": 0.0, "completed_recent": 0, "blacklisted_recent": 0, "failed_recent": 0},
"steps": [],
"nodes": [
{
"node_code": f"mainland-controller-01-{index:02d}",
"items_running": 1,
"items_claimed": 0,
"processed_recent": 0,
"processed_per_minute": 0.0,
"completed_recent": 0,
"failed_recent": 0,
"blacklisted_recent": 0,
}
for index in range(9)
],
"retry_total": 0,
}
mock_get_active_detect_job_summary.return_value = {}
mock_build_dashboard_runtime_summary.return_value = {
"worker": {"running": True, "mode": "linux-systemd", "expected_on_this_node": True},
"node": {"region": "mainland", "role": "worker"},
"cluster": {"summary": {"online_worker_nodes": 1, "dedicated_online_worker_nodes": 0, "online_control_nodes": 1}},
"detect": {"backlog": {}},
}
mock_get_detect_queue_health.return_value = {
"has_active_job": False,
"queue": {},
"throughput": {"processed_recent": 0, "processed_per_minute": 0.0},
"steps": [],
"runtime_activity": {},
"nodes": [],
"runtime_snapshot_backlog": {},
}
mock_get_detect_capacity_plan.return_value = {
"estimated_hours_remaining": 0,
"remaining_items": 0,
"recommended_additional_workers": 0,
}
data = fetch_overview()
self.assertEqual(8, len(data["node_throughput"]))
self.assertEqual(9, data["active_execution_nodes"])
self.assertEqual(9, data["ops_summary"]["active_execution_nodes"])
@patch("app.services.dashboard._fetch_active_jobs_aggregate")
@patch("app.services.dashboard.get_detect_capacity_plan")
@patch("app.services.dashboard.get_detect_queue_health")
@patch("app.services.dashboard._build_dashboard_runtime_summary")
@patch("app.services.dashboard.get_active_detect_job_summary")
@patch("app.services.dashboard.get_db")
def test_fetch_overview_keeps_active_job_summary_when_queue_health_temporarily_empty(
self,
mock_get_db,
mock_get_active_detect_job_summary,
mock_build_dashboard_runtime_summary,
mock_get_detect_queue_health,
mock_get_detect_capacity_plan,
mock_fetch_active_jobs_aggregate,
) -> None:
mock_get_db.return_value = _FakeConnection(responses=[(0,)] * 9)
mock_fetch_active_jobs_aggregate.return_value = {
"active_jobs_total": 1,
"queue": {"items_total": 8402, "pending": 5001, "claimed": 0, "running": 537, "completed": 0, "blacklisted": 0, "failed": 0},
"throughput": {"processed_recent": 12, "processed_per_minute": 0.8, "completed_recent": 10, "blacklisted_recent": 1, "failed_recent": 1},
"steps": [],
"nodes": [],
"retry_total": 0,
}
mock_get_active_detect_job_summary.return_value = {
"job_id": 255,
"job_code": "sync-overseas-255",
"runtime_job_code": "sync-overseas-255",
"status": "running",
"progress_percent": 22.4,
"items_total": 8402,
"items_pending": 5001,
"items_claimed": 0,
"items_running": 537,
"items_completed": 0,
"items_blacklisted": 0,
"items_failed": 0,
"display_items_running": 11799,
"display_active_threads": 11799,
"display_max_threads": 75400,
"distributed_node_stats": [{"node_code": "mainland-controller-01-a", "display_running": 537, "active_threads": 537, "max_threads": 1000}],
}
mock_build_dashboard_runtime_summary.return_value = {
"worker": {"running": False, "mode": "linux-systemd", "expected_on_this_node": False},
"node": {"region": "overseas", "role": "control"},
"cluster": {"summary": {"online_worker_nodes": 1, "dedicated_online_worker_nodes": 0, "online_control_nodes": 1}},
"detect": {"backlog": {}},
}
mock_get_detect_queue_health.return_value = {
"has_active_job": False,
"queue": {},
"throughput": {"processed_recent": 0, "processed_per_minute": 0.0},
"steps": [],
"runtime_activity": {},
"nodes": [],
"runtime_snapshot_backlog": {},
}
mock_get_detect_capacity_plan.return_value = {
"estimated_hours_remaining": 0,
"remaining_items": 5001,
"recommended_additional_workers": 0,
}
data = fetch_overview()
self.assertEqual("sync-overseas-255", data["active_job"]["job_code"])
self.assertEqual(11799, data["active_job"]["display_items_running"])
self.assertEqual(75400, data["active_job"]["display_max_threads"])
if __name__ == "__main__":
unittest.main()

View File

@@ -0,0 +1,83 @@
from __future__ import annotations
import unittest
from unittest.mock import patch
from app.services import debug_event_service
class DebugEventServiceTests(unittest.TestCase):
def test_ingest_debug_event_requires_configured_shared_token(self) -> None:
with patch.object(debug_event_service.settings, "sync_shared_token", ""):
ok, message, data = debug_event_service.ingest_debug_event(
{
"source_region": "mainland",
"node_code": "mainland-controller-01",
"service": "worker-event",
"event_type": "worker_log",
"message": "开始检测域名: a.com",
"payload": {"job_id": 1, "job_code": "sync-overseas-1"},
},
shared_token=None,
)
self.assertFalse(ok)
self.assertIn("token 未配置", message)
self.assertTrue(data["configuration_required"])
def test_resolve_target_job_for_debug_event_rejects_missing_identity(self) -> None:
job, reason = debug_event_service._resolve_target_job_for_debug_event({"domain": "a.com"})
self.assertIsNone(job)
self.assertEqual("missing_job_identity", reason)
@patch("app.services.debug_event_service._load_detect_job_summary_by_job_code")
@patch("app.services.detect_job_service.get_detect_job_summary")
@patch("app.services.detect_job_service.get_active_detect_job_summary")
def test_resolve_target_job_for_debug_event_prefers_payload_job_id_over_current_active_job(
self,
mock_get_active_detect_job_summary,
mock_get_detect_job_summary,
mock_load_detect_job_summary_by_job_code,
) -> None:
mock_get_active_detect_job_summary.return_value = {
"job_id": 12,
"job_code": "sync-overseas-12",
"current_cycle_token": "cycle-12",
}
mock_get_detect_job_summary.return_value = {
"job_id": 11,
"job_code": "sync-overseas-11",
"current_cycle_token": "cycle-11",
}
job, reason = debug_event_service._resolve_target_job_for_debug_event(
{"job_id": 11, "job_code": "sync-overseas-11", "cycle_token": "cycle-11"}
)
self.assertEqual("matched", reason)
self.assertEqual(11, job["job_id"])
mock_get_detect_job_summary.assert_called_once_with(11, event_limit=1)
mock_load_detect_job_summary_by_job_code.assert_not_called()
@patch("app.services.detect_job_service.get_active_detect_job_summary")
def test_resolve_target_job_for_debug_event_rejects_cycle_mismatch(
self,
mock_get_active_detect_job_summary,
) -> None:
mock_get_active_detect_job_summary.return_value = {
"job_id": 12,
"job_code": "sync-overseas-12",
"current_cycle_token": "cycle-current",
}
job, reason = debug_event_service._resolve_target_job_for_debug_event(
{"job_id": 12, "job_code": "sync-overseas-12", "cycle_token": "cycle-old"}
)
self.assertIsNone(job)
self.assertEqual("cycle_mismatch", reason)
if __name__ == "__main__":
unittest.main()

View File

@@ -7,6 +7,153 @@ from app.api.routes import detect as detect_route
class DetectApiRoutesTestCase(unittest.TestCase):
@patch("app.api.routes.detect.create_detect_run_snapshot")
@patch("app.api.routes.detect._dispatch_remote_detect_start")
@patch("app.api.routes.detect.get_settings_payload")
@patch("app.api.routes.detect.get_detect_status")
@patch("app.api.routes.detect.send_worker_command")
@patch("app.api.routes.detect.start_worker")
@patch("app.api.routes.detect.append_detect_job_event")
@patch("app.api.routes.detect.create_detect_job_if_needed")
def test_start_detect_skips_redundant_systemctl_when_worker_already_online(
self,
mock_create_job,
mock_append_event,
mock_start_worker,
mock_send_worker_command,
mock_get_detect_status,
mock_get_settings_payload,
mock_dispatch_remote,
mock_create_snapshot,
) -> None:
mock_create_job.return_value = {
"job_id": 30,
"job_code": "sync-overseas-30",
"status": "running",
"items_pending": 100,
"items_claimed": 0,
"items_running": 0,
"task_mode": "domain_pipeline",
"step_code": "",
}
mock_get_detect_status.return_value = {
"worker_mode": "linux-systemd",
"worker_online": True,
"worker_process_count": 60,
"worker_latest_start_time": "2026-04-24 23:30:00",
"worker_runtime_message": "running",
"progress": {},
}
mock_send_worker_command.return_value = (True, "已发送 Worker 控制指令: start_detection")
mock_get_settings_payload.return_value = {
"runtime": {"thread_count": 1000},
"proxy_config": {"proxy_enable": True, "allow_direct": True, "proxy_urls": ["a"]},
}
mock_dispatch_remote.return_value = {
"queued_jobs": [],
"queued_total": 0,
"failed_total": 0,
"target_summary": {"controller_nodes": [], "worker_nodes": []},
}
with patch.object(detect_route.settings, "node_region", "mainland"), patch.object(
detect_route.settings, "node_role", "control"
):
response = detect_route.start_detect()
self.assertEqual(0, response.code)
self.assertIn("检测端已在运行,跳过重复启动", response.message)
mock_start_worker.assert_not_called()
mock_send_worker_command.assert_called_once()
event_types = [call.kwargs.get("event_type") for call in mock_append_event.call_args_list]
self.assertIn("job_dispatch_start_skipped", event_types)
self.assertIn("job_dispatch_sent", event_types)
mock_create_snapshot.assert_called_once()
@patch("app.api.routes.detect.create_detect_run_snapshot")
@patch("app.api.routes.detect._dispatch_remote_detect_start")
@patch("app.api.routes.detect.get_settings_payload")
@patch("app.api.routes.detect.get_detect_status")
@patch("app.api.routes.detect.send_worker_command")
@patch("app.api.routes.detect.start_worker")
@patch("app.api.routes.detect.append_detect_job_event")
@patch("app.api.routes.detect.create_detect_job_if_needed")
def test_start_detect_falls_back_to_direct_command_when_worker_already_running(
self,
mock_create_job,
mock_append_event,
mock_start_worker,
mock_send_worker_command,
mock_get_detect_status,
mock_get_settings_payload,
mock_dispatch_remote,
mock_create_snapshot,
) -> None:
mock_create_job.return_value = {
"job_id": 31,
"job_code": "sync-overseas-31",
"status": "running",
"items_pending": 100,
"items_claimed": 0,
"items_running": 0,
"task_mode": "domain_pipeline",
"step_code": "",
}
mock_start_worker.return_value = (
False,
"domaincheck-worker 控制失败:当前运行用户没有免密 systemctl 权限,请为 API 进程授予对应 sudo/systemd 权限",
)
mock_send_worker_command.return_value = (True, "已发送 Worker 控制指令: start_detection")
mock_get_detect_status.side_effect = [
{
"worker_mode": "linux-systemd",
"worker_online": False,
"worker_process_count": 0,
"worker_latest_start_time": "",
"worker_runtime_message": "starting",
"progress": {},
},
{
"worker_mode": "linux-systemd",
"worker_online": True,
"worker_process_count": 60,
"worker_latest_start_time": "2026-04-24 23:30:00",
"worker_runtime_message": "running",
"progress": {},
},
{
"worker_mode": "linux-systemd",
"worker_online": True,
"worker_process_count": 60,
"worker_latest_start_time": "2026-04-24 23:30:00",
"worker_runtime_message": "running",
"progress": {},
},
]
mock_get_settings_payload.return_value = {
"runtime": {"thread_count": 1000},
"proxy_config": {"proxy_enable": True, "allow_direct": True, "proxy_urls": ["a"]},
}
mock_dispatch_remote.return_value = {
"queued_jobs": [],
"queued_total": 0,
"failed_total": 0,
"target_summary": {"controller_nodes": [], "worker_nodes": []},
}
with patch.object(detect_route.settings, "node_region", "mainland"), patch.object(
detect_route.settings, "node_role", "control"
):
response = detect_route.start_detect()
self.assertEqual(0, response.code)
self.assertIn("检测端已在运行,改为直接发送控制指令", response.message)
mock_send_worker_command.assert_called_once()
event_types = [call.kwargs.get("event_type") for call in mock_append_event.call_args_list]
self.assertIn("job_dispatch_start_degraded", event_types)
self.assertIn("job_dispatch_sent", event_types)
mock_create_snapshot.assert_called_once()
@patch("app.api.routes.detect.create_detect_run_snapshot")
@patch("app.api.routes.detect._dispatch_remote_detect_start")
@patch("app.api.routes.detect.get_settings_payload")
@@ -65,11 +212,94 @@ class DetectApiRoutesTestCase(unittest.TestCase):
mock_start_worker.assert_not_called()
mock_send_worker_command.assert_not_called()
mock_dispatch_remote.assert_called_once()
mock_create_snapshot.assert_called_once()
mock_create_snapshot.assert_not_called()
event_types = [call.kwargs.get("event_type") for call in mock_append_event.call_args_list]
self.assertIn("job_dispatch_requested", event_types)
self.assertIn("job_dispatch_skipped_local", event_types)
@patch("app.api.routes.detect.finalize_detect_run")
@patch("app.api.routes.detect.mark_detect_run_stopping")
@patch("app.api.routes.detect.get_settings_payload")
@patch("app.api.routes.detect.get_detect_status")
@patch("app.api.routes.detect._dispatch_remote_detect_stop")
@patch("app.api.routes.detect.send_worker_command")
@patch("app.api.routes.detect.get_active_detect_job_summary")
def test_stop_detect_forwards_target_payload(
self,
mock_get_active_job,
mock_send_worker_command,
mock_dispatch_remote_stop,
mock_get_detect_status,
mock_get_settings_payload,
mock_mark_detect_run_stopping,
mock_finalize_detect_run,
) -> None:
mock_get_active_job.return_value = {
"job_id": 29,
"job_code": "sync-overseas-29",
"status": "running",
"current_cycle_token": "cycle-29",
}
mock_send_worker_command.return_value = (True, "已发送 Worker 控制指令")
mock_dispatch_remote_stop.return_value = {
"queued_jobs": [{"node_code": "mainland-controller-01"}],
"queued_total": 1,
"failed_total": 0,
"target_summary": {"controller_nodes": ["mainland-controller-01"], "worker_nodes": []},
}
mock_get_detect_status.return_value = {
"worker_mode": "linux-systemd",
"worker_online": True,
"worker_process_count": 1,
"worker_latest_start_time": "",
"worker_runtime_message": "running",
"progress": {},
}
mock_get_settings_payload.return_value = {
"thread_count": 1000,
"node_thread_counts": {},
"process_count": 80,
"node_process_counts": {},
"proxy_config": {"proxy_enable": True, "allow_direct": False, "proxy_urls": ["a"]},
}
response = detect_route.stop_detect(payload={"target_node_codes": ["mainland-controller-01"]})
self.assertEqual(0, response.code)
mock_send_worker_command.assert_called_once_with(
"stop_detection",
payload={"target_node_codes": ["mainland-controller-01"]},
)
mock_dispatch_remote_stop.assert_called_once_with(
active_job=mock_get_active_job.return_value,
cycle_token="cycle-29",
payload={"target_node_codes": ["mainland-controller-01"]},
)
mock_mark_detect_run_stopping.assert_called_once()
mock_finalize_detect_run.assert_not_called()
@patch("app.api.routes.detect.create_ops_job")
@patch("app.api.routes.detect.list_managed_nodes")
def test_dispatch_remote_detect_stop_filters_target_nodes(self, mock_list_managed_nodes, mock_create_ops_job) -> None:
mock_list_managed_nodes.return_value = [
{"node_code": "mainland-controller-01", "region": "mainland", "role": "control", "last_seen_at": "2026-04-23T14:00:00", "is_enabled": True},
{"node_code": "mainland-worker-01", "region": "mainland", "role": "worker", "last_seen_at": "2026-04-23T14:00:00", "is_enabled": True},
]
mock_create_ops_job.return_value = (True, "queued", {})
result = detect_route._dispatch_remote_detect_stop(
active_job={"job_id": 29, "job_code": "sync-overseas-29"},
cycle_token="cycle-29",
payload={"target_node_codes": ["mainland-worker-01"]},
)
self.assertEqual(1, result["queued_total"])
self.assertEqual(["mainland-controller-01"], result["target_summary"]["controller_nodes"])
self.assertEqual(["mainland-worker-01"], result["target_summary"]["worker_nodes"])
mock_create_ops_job.assert_called_once()
create_payload = mock_create_ops_job.call_args.args[0]
self.assertEqual("mainland-worker-01", create_payload["target_node_code"])
if __name__ == "__main__":
unittest.main()

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,65 @@
from __future__ import annotations
import unittest
from unittest.mock import patch
import app.services.detect_run_service as detect_run_service
class _TraceLock:
def __init__(self, order: list[str]) -> None:
self.order = order
def __enter__(self):
self.order.append("enter")
return self
def __exit__(self, exc_type, exc, tb):
self.order.append("exit")
return False
class DetectRunServiceTests(unittest.TestCase):
def test_create_detect_run_snapshot_acquires_lock_before_loading_records(self) -> None:
order: list[str] = []
with patch.object(detect_run_service, "_DETECT_RUNS_LOCK", _TraceLock(order)):
with patch.object(detect_run_service, "_load", side_effect=lambda: order.append("load") or []):
with patch.object(detect_run_service, "_capture_worker_logs", return_value=[]):
with patch.object(detect_run_service, "_save", side_effect=lambda records: order.append("save")):
record = detect_run_service.create_detect_run_snapshot(
"start",
{"latest_start_time": "2026-04-23 15:00:00", "running": True},
{"running": 1},
{"thread_count": 1000},
)
self.assertEqual("enter", order[0])
self.assertIn("load", order)
self.assertIn("save", order)
self.assertEqual("exit", order[-1])
self.assertEqual("starting", record["status"])
def test_sync_detect_runs_acquires_lock_before_mutating_records(self) -> None:
order: list[str] = []
with patch.object(detect_run_service, "_DETECT_RUNS_LOCK", _TraceLock(order)):
with patch.object(detect_run_service, "_load", side_effect=lambda: order.append("load") or []):
with patch.object(detect_run_service, "_capture_worker_logs", return_value=[]):
with patch.object(detect_run_service, "_save", side_effect=lambda records: order.append("save")):
records = detect_run_service.sync_detect_runs(
{"running": True, "detecting": True, "latest_start_time": "2026-04-23 15:00:00"},
{"running": 1, "pending": 0},
{"thread_count": 1000},
active_job={"status": "running", "items_running": 1},
)
self.assertEqual("enter", order[0])
self.assertIn("load", order)
self.assertIn("save", order)
self.assertEqual("exit", order[-1])
self.assertEqual("running", records[0]["status"])
if __name__ == "__main__":
unittest.main()

View File

@@ -0,0 +1,62 @@
from __future__ import annotations
import unittest
from unittest.mock import patch
from app.services import detect_service
class DetectServiceRemoteLogsTests(unittest.TestCase):
@patch("app.services.detect_service.list_debug_events")
def test_build_remote_log_snapshot_from_debug_events_filters_to_active_job_identity(self, mock_list_debug_events) -> None:
mock_list_debug_events.return_value = {
"records": [
{
"id": 3,
"node_code": "mainland-controller-01-a",
"event_type": "worker_log",
"message": "旧任务日志",
"created_at": "2026-04-23 16:00:00",
"payload": {
"job_id": 10,
"job_code": "sync-overseas-10",
"cycle_token": "cycle-old",
"log_mode": "key",
},
},
{
"id": 4,
"node_code": "mainland-controller-01-a",
"event_type": "worker_log",
"message": "当前任务日志",
"created_at": "2026-04-23 16:01:00",
"payload": {
"job_id": 12,
"job_code": "sync-overseas-12",
"cycle_token": "cycle-12",
"log_mode": "key",
},
},
]
}
snapshot = detect_service._build_remote_log_snapshot_from_debug_events(
{
"job_id": 12,
"job_code": "sync-overseas-12",
"runtime_job_code": "sync-overseas-12",
"current_cycle_token": "cycle-12",
"node_stats": [{"node_code": "mainland-controller-01-a"}],
},
enabled=True,
mode="key",
limit=20,
)
self.assertEqual(1, snapshot["line_count"])
self.assertIn("当前任务日志", snapshot["lines"][0])
self.assertNotIn("旧任务日志", "\n".join(snapshot["lines"]))
if __name__ == "__main__":
unittest.main()

View File

@@ -1,10 +1,47 @@
import unittest
from contextlib import ExitStack
from datetime import datetime, timedelta
from unittest.mock import patch
from app.services import detect_service
class DetectServiceStatusFallbackTests(unittest.TestCase):
def test_filter_live_aggregate_runtime_nodes_excludes_disabled_managed_nodes(self) -> None:
now_text = datetime.now().isoformat(sep=" ", timespec="seconds")
rows = [
{
"node_code": "mainland-controller-01",
"status": "busy",
"last_heartbeat_at": now_text,
"display_running": 1,
},
{
"node_code": "mainland-worker-01",
"status": "busy",
"last_heartbeat_at": now_text,
"display_running": 1,
},
]
with patch("app.services.detect_service._load_disabled_managed_node_codes", return_value={"mainland-worker-01"}):
filtered = detect_service._filter_live_aggregate_runtime_nodes(rows)
self.assertEqual(["mainland-controller-01"], [item["node_code"] for item in filtered])
def setUp(self) -> None:
detect_service._DETECT_STATUS_CACHE_VALUE = None
detect_service._DETECT_STATUS_CACHE_EXPIRES_AT = 0.0
def test_resolve_capacity_node_code_treats_child_instance_suffix_as_single_process(self) -> None:
capacity_node_code, is_child_instance = detect_service._resolve_capacity_node_code(
"mainland-controller-01-ae",
{"node_thread_counts": {"overseas-control-01": 1}, "node_process_counts": {}},
)
self.assertEqual("mainland-controller-01", capacity_node_code)
self.assertTrue(is_child_instance)
def test_get_detect_status_keeps_runtime_snapshot_when_db_is_unreachable(self) -> None:
runtime_state = {
"service_running": True,
@@ -17,24 +54,39 @@ class DetectServiceStatusFallbackTests(unittest.TestCase):
"available_proxy_count": 18,
}
with patch("app.services.detect_service.ensure_runtime_schema"), \
patch("app.services.detect_service.get_db", side_effect=RuntimeError("db down")), \
patch("app.services.detect_service.get_settings_payload", return_value={"proxy_config": {"proxy_enable": True, "allow_direct": False, "proxy_urls": ["a"]}}), \
patch("app.services.detect_service.get_runtime_settings", return_value={"worker_log_sync_enabled": False, "worker_log_sync_mode": "full"}), \
patch("app.services.detect_service._load_recent_worker_lines", return_value=(True, "", [])), \
patch("app.services.detect_service.detect_worker_runtime", return_value={"mode": "linux-systemd", "running": True, "process_count": 1, "latest_start_time": "2026-04-20 23:58:00", "message": "active/running"}), \
patch("app.services.detect_service._load_runtime_state", return_value=runtime_state), \
patch("app.services.detect_service._load_runtime_state_from_cluster_node", return_value={}), \
patch("app.services.detect_service._extract_available_proxy_count", return_value=0), \
patch("app.services.detect_service._extract_active_thread_snapshot", return_value={"active": 0, "max": 0}), \
patch("app.services.detect_service._normalize_recent_warning", return_value=""), \
patch("app.services.detect_service._build_proxy_runtime_snapshot", return_value={"state": "healthy", "label": "代理正常", "detail": "healthy", "direct_fallback_active": False, "reason": "healthy", "last_refresh_status": "ok", "last_refresh_time": "", "source_count": 2, "raw_items": 18, "validated_count": 18, "available_count": 18, "source_stats": [], "supplier_empty": False}), \
patch("app.services.detect_service.resolve_thread_count", return_value={"effective_thread_count": 120, "default_thread_count": 5, "source": "node_override", "override_thread_count": 120, "node_code": "mainland-worker-01"}), \
patch("app.services.detect_service.get_active_detect_job_summary", side_effect=RuntimeError("db down")), \
patch("app.services.detect_service.sync_detect_runs", return_value=[]), \
patch("app.services.detect_service._resolve_remote_log_snapshot", return_value={}), \
patch("app.services.detect_service._extract_dependency_alerts", return_value=[]), \
patch("app.services.detect_service.append_detect_result_projection_if_changed"):
with ExitStack() as stack:
stack.enter_context(patch.object(detect_service.settings, "node_region", "mainland"))
stack.enter_context(patch.object(detect_service.settings, "node_role", "worker"))
stack.enter_context(patch.object(detect_service.settings, "node_code", "mainland-worker-01"))
stack.enter_context(patch("app.services.detect_service.ensure_runtime_schema"))
stack.enter_context(patch("app.services.detect_service.get_db", side_effect=RuntimeError("db down")))
stack.enter_context(
patch("app.services.detect_service.get_settings_payload", return_value={"proxy_config": {"proxy_enable": True, "allow_direct": False, "proxy_urls": ["a"]}})
)
stack.enter_context(patch("app.services.detect_service.get_runtime_settings", return_value={"worker_log_sync_enabled": False, "worker_log_sync_mode": "full"}))
stack.enter_context(patch("app.services.detect_service._load_recent_worker_lines", return_value=(True, "", [])))
stack.enter_context(
patch("app.services.detect_service.detect_worker_runtime", return_value={"mode": "linux-systemd", "running": True, "process_count": 1, "latest_start_time": "2026-04-20 23:58:00", "message": "active/running"})
)
stack.enter_context(patch("app.services.detect_service._load_runtime_state", return_value=runtime_state))
stack.enter_context(patch("app.services.detect_service._load_runtime_state_from_cluster_node", return_value={}))
stack.enter_context(patch("app.services.detect_service._extract_available_proxy_count", return_value=0))
stack.enter_context(patch("app.services.detect_service._extract_active_thread_snapshot", return_value={"active": 0, "max": 0}))
stack.enter_context(patch("app.services.detect_service._normalize_recent_warning", return_value=""))
stack.enter_context(
patch(
"app.services.detect_service._build_proxy_runtime_snapshot",
return_value={"state": "healthy", "label": "代理正常", "detail": "healthy", "direct_fallback_active": False, "reason": "healthy", "last_refresh_status": "ok", "last_refresh_time": "", "source_count": 2, "raw_items": 18, "validated_count": 18, "available_count": 18, "source_stats": [], "supplier_empty": False},
)
)
stack.enter_context(
patch("app.services.detect_service.resolve_thread_count", return_value={"effective_thread_count": 120, "default_thread_count": 5, "source": "node_override", "override_thread_count": 120, "node_code": "mainland-worker-01"})
)
stack.enter_context(patch("app.services.detect_service.get_active_detect_job_summary", side_effect=RuntimeError("db down")))
stack.enter_context(patch("app.services.detect_service.sync_detect_runs", return_value=[]))
stack.enter_context(patch("app.services.detect_service._resolve_remote_log_snapshot", return_value={}))
stack.enter_context(patch("app.services.detect_service._extract_dependency_alerts", return_value=[]))
stack.enter_context(patch("app.services.detect_service.append_detect_result_projection_if_changed"))
payload = detect_service.get_detect_status()
self.assertTrue(payload["worker_online"])
@@ -44,6 +96,56 @@ class DetectServiceStatusFallbackTests(unittest.TestCase):
self.assertEqual(0, payload["progress"]["pending"])
self.assertEqual(0, payload["progress"]["completed"])
def test_get_detect_status_skips_local_result_projection_on_overseas_control(self) -> None:
runtime_state = {
"service_running": False,
"detecting": False,
"active_threads": 0,
"max_threads": 1,
"phase": "idle",
"detail": "当前节点不承载本地检测执行",
"updated_at": "2026-04-24 02:10:00",
"available_proxy_count": 0,
}
with ExitStack() as stack:
stack.enter_context(patch.object(detect_service.settings, "node_region", "overseas"))
stack.enter_context(patch.object(detect_service.settings, "node_role", "control"))
stack.enter_context(patch.object(detect_service.settings, "node_code", "overseas-control-01"))
stack.enter_context(patch("app.services.detect_service.ensure_runtime_schema"))
stack.enter_context(patch("app.services.detect_service.get_db", side_effect=RuntimeError("db down")))
stack.enter_context(
patch("app.services.detect_service.get_settings_payload", return_value={"proxy_config": {"proxy_enable": False, "allow_direct": True, "proxy_urls": []}})
)
stack.enter_context(patch("app.services.detect_service.get_runtime_settings", return_value={"worker_log_sync_enabled": False, "worker_log_sync_mode": "key"}))
stack.enter_context(patch("app.services.detect_service._load_recent_worker_lines", return_value=(False, "", [])))
stack.enter_context(
patch("app.services.detect_service.detect_worker_runtime", return_value={"mode": "linux-systemd", "running": False, "process_count": 0, "latest_start_time": "", "message": "inactive"})
)
stack.enter_context(patch("app.services.detect_service._load_runtime_state", return_value=runtime_state))
stack.enter_context(patch("app.services.detect_service._load_runtime_state_from_cluster_node", return_value={}))
stack.enter_context(patch("app.services.detect_service._extract_available_proxy_count", return_value=0))
stack.enter_context(patch("app.services.detect_service._extract_active_thread_snapshot", return_value={"active": 0, "max": 0}))
stack.enter_context(patch("app.services.detect_service._normalize_recent_warning", return_value=""))
stack.enter_context(
patch(
"app.services.detect_service._build_proxy_runtime_snapshot",
return_value={"state": "disabled", "label": "不适用", "detail": "-", "direct_fallback_active": False, "reason": "not_applicable", "last_refresh_status": "", "last_refresh_time": "", "source_count": 0, "raw_items": 0, "validated_count": 0, "available_count": 0, "source_stats": [], "supplier_empty": False},
)
)
stack.enter_context(
patch("app.services.detect_service.resolve_thread_count", return_value={"effective_thread_count": 1, "default_thread_count": 1, "source": "default", "override_thread_count": None, "node_code": "overseas-control-01"})
)
stack.enter_context(patch("app.services.detect_service.get_active_detect_job_summary", return_value=None))
stack.enter_context(patch("app.services.detect_service.sync_detect_runs", return_value=[]))
stack.enter_context(patch("app.services.detect_service._resolve_remote_log_snapshot", return_value={}))
stack.enter_context(patch("app.services.detect_service._extract_dependency_alerts", return_value=[]))
mock_append = stack.enter_context(patch("app.services.detect_service.append_detect_result_projection_if_changed"))
detect_service.get_detect_status()
mock_append.assert_not_called()
def test_filter_lines_since_supports_journalctl_syslog_timestamps(self) -> None:
lines = [
"Apr 21 20:12:42 mainland-controller python[1]: 当前实际线程数量: 323/4",
@@ -64,6 +166,784 @@ class DetectServiceStatusFallbackTests(unittest.TestCase):
self.assertEqual(lines, filtered)
def test_get_detect_status_uses_aggregate_capacity_for_overseas_control(self) -> None:
active_job = {
"job_id": 11,
"job_code": "sync-overseas-51",
"status": "running",
"items_pending": 5001,
"items_completed": 1788,
"items_failed": 0,
"items_blacklisted": 0,
"progress_percent": 40.49,
"display_items_running": 1432,
"display_active_threads": 138,
"distributed_node_stats": [
{
"node_code": "mainland-controller-01",
"items_claimed": 73,
"items_running": 1432,
"display_running": 1432,
"active_threads": 138,
"max_threads": 2000,
},
{
"node_code": "mainland-worker-01",
"items_completed": 1614,
"items_running": 0,
"active_threads": 0,
"max_threads": 0,
},
{
"node_code": "unassigned",
"items_pending": 3495,
},
],
}
aggregate_queue_health = {
"has_active_job": True,
"job": {
"job_id": 11,
"job_code": "sync-overseas-51",
"status": "running",
"progress_percent": 40.49,
},
"queue": {
"items_total": 8402,
"pending": 5001,
"claimed": 0,
"running": 5909,
"display_running": 5909,
"completed": 0,
"blacklisted": 0,
"failed": 0,
},
"nodes": [
{
"node_code": "mainland-controller-01-a",
"items_running": 1000,
"display_running": 1000,
"active_threads": 1000,
"max_threads": 1000,
},
{
"node_code": "mainland-controller-01-b",
"items_running": 972,
"display_running": 972,
"active_threads": 972,
"max_threads": 1000,
},
{
"node_code": "mainland-controller-01-c",
"items_running": 3937,
"display_running": 3937,
"active_threads": 3937,
"max_threads": 36000,
},
],
}
def _resolve_thread_count(*, node_code=None, settings_payload=None):
if node_code == "mainland-controller-01" or str(node_code or "").startswith("mainland-controller-01-"):
return {
"effective_thread_count": 1000,
"default_thread_count": 1000,
"source": "default",
"override_thread_count": None,
"node_code": str(node_code or "mainland-controller-01"),
}
return {
"effective_thread_count": 1,
"default_thread_count": 1000,
"source": "node_override",
"override_thread_count": 1,
"node_code": "overseas-control-01",
}
def _resolve_process_count(*, node_code=None, settings_payload=None):
if node_code == "mainland-controller-01":
return {
"effective_process_count": 80,
"default_process_count": 80,
"source": "default",
"override_process_count": None,
"node_code": "mainland-controller-01",
}
if str(node_code or "").startswith("mainland-controller-01-"):
return {
"effective_process_count": 1,
"default_process_count": 80,
"source": "child_instance",
"override_process_count": None,
"node_code": str(node_code or ""),
}
return {
"effective_process_count": 1,
"default_process_count": 80,
"source": "node_override",
"override_process_count": 1,
"node_code": "overseas-control-01",
}
with ExitStack() as stack:
stack.enter_context(patch.object(detect_service.settings, "node_region", "overseas"))
stack.enter_context(patch.object(detect_service.settings, "node_role", "control"))
stack.enter_context(patch.object(detect_service.settings, "node_code", "overseas-control-01"))
stack.enter_context(patch("app.services.detect_service.ensure_runtime_schema"))
stack.enter_context(patch("app.services.detect_service.get_db", side_effect=RuntimeError("db down")))
stack.enter_context(
patch(
"app.services.detect_service.get_settings_payload",
return_value={
"proxy_config": {"proxy_enable": False, "allow_direct": True, "proxy_urls": []},
"process_count": 80,
"node_process_counts": {},
"thread_count": 1000,
"node_thread_counts": {"overseas-control-01": 1},
},
)
)
stack.enter_context(
patch("app.services.detect_service.get_runtime_settings", return_value={"worker_log_sync_enabled": False, "worker_log_sync_mode": "key"})
)
stack.enter_context(
patch("app.services.detect_service._load_recent_worker_lines", return_value=(False, "2026-04-23 00:00:00", ["stale line"]))
)
stack.enter_context(
patch("app.services.detect_service.detect_worker_runtime", return_value={"mode": "linux-systemd", "running": False, "process_count": 0, "latest_start_time": "", "message": "inactive"})
)
stack.enter_context(
patch("app.services.detect_service._load_runtime_state", return_value={"service_running": True, "detecting": True, "active_threads": 2, "max_threads": 1, "detail": "当前实际线程数量: 2/1"})
)
stack.enter_context(patch("app.services.detect_service._load_runtime_state_from_cluster_node", return_value={}))
stack.enter_context(patch("app.services.detect_service._extract_available_proxy_count", return_value=0))
stack.enter_context(patch("app.services.detect_service._extract_active_thread_snapshot", return_value={"active": 0, "max": 0}))
stack.enter_context(patch("app.services.detect_service._normalize_recent_warning", return_value=""))
stack.enter_context(
patch(
"app.services.detect_service._build_proxy_runtime_snapshot",
return_value={
"state": "disabled",
"label": "未启用代理",
"detail": "-",
"direct_fallback_active": True,
"reason": "proxy_disabled",
"last_refresh_status": "",
"last_refresh_time": "",
"source_count": 0,
"raw_items": 0,
"validated_count": 0,
"available_count": 0,
"source_stats": [],
"supplier_empty": False,
},
)
)
stack.enter_context(patch("app.services.detect_service.resolve_thread_count", side_effect=_resolve_thread_count))
stack.enter_context(patch("app.services.detect_service.resolve_process_count", side_effect=_resolve_process_count))
stack.enter_context(patch("app.services.detect_service.get_active_detect_job_summary", return_value=active_job))
stack.enter_context(patch("app.services.detect_service.get_detect_queue_health", return_value=aggregate_queue_health))
stack.enter_context(
patch(
"app.services.detect_service._load_runtime_states_from_cluster_nodes",
return_value={
"mainland-controller-01": {
"node_code": "mainland-controller-01",
"available_proxy_count": 486,
"proxy_runtime_label": "代理正常",
"proxy_runtime_reason": "healthy",
"proxy_last_refresh_status": "复用共享代理快照 486 个",
"proxy_last_refresh_time": "2026-04-23 19:07:12",
"proxy_last_refresh_source_count": 6,
"proxy_last_refresh_total_items": 120,
"proxy_last_validated_count": 0,
}
},
)
)
stack.enter_context(patch("app.services.detect_service.sync_detect_runs", return_value=[]))
stack.enter_context(patch("app.services.detect_service._resolve_remote_log_snapshot", return_value={}))
stack.enter_context(patch("app.services.detect_service._extract_dependency_alerts", return_value=[]))
stack.enter_context(patch("app.services.detect_service.append_detect_result_projection_if_changed"))
payload = detect_service.get_detect_status()
self.assertFalse(payload["worker_online"])
self.assertEqual({}, payload["runtime_state"])
self.assertEqual(3, payload["worker_process_count"])
self.assertEqual(3, payload["aggregate_process_count"])
self.assertEqual(3, payload["aggregate_participating_node_count"])
self.assertEqual(
["mainland-controller-01-a", "mainland-controller-01-b", "mainland-controller-01-c"],
payload["aggregate_participating_node_codes"],
)
self.assertEqual(3000, payload["aggregate_max_thread_count"])
self.assertEqual(38000, payload["max_thread_count"])
self.assertEqual(5909, payload["active_thread_count"])
self.assertEqual(486, payload["available_proxy_count"])
self.assertEqual("集群代理正常", payload["proxy_runtime_label"])
self.assertIn("486", payload["proxy_last_refresh_status"])
def test_get_detect_status_uses_queue_running_when_aggregate_display_fields_are_zero(self) -> None:
active_job = {
"job_id": 60,
"job_code": "sync-overseas-255",
"status": "running",
"items_total": 8402,
"items_pending": 7865,
"items_running": 537,
"items_completed": 0,
"items_failed": 0,
"items_blacklisted": 0,
"progress_percent": 6.39,
"display_items_running": 0,
"display_active_threads": 0,
"display_max_threads": 0,
"distributed_node_stats": [],
}
aggregate_queue_health = {
"has_active_job": True,
"job": {
"job_id": 60,
"job_code": "sync-overseas-255",
"status": "running",
"progress_percent": 6.39,
},
"queue": {
"items_total": 8402,
"pending": 7865,
"claimed": 0,
"running": 537,
"display_running": 0,
"completed": 0,
"blacklisted": 0,
"failed": 0,
},
"nodes": [
{
"node_code": "mainland-controller-01-a",
"items_running": 537,
"display_running": 537,
"active_threads": 537,
"max_threads": 1000,
}
],
}
with ExitStack() as stack:
stack.enter_context(patch.object(detect_service.settings, "node_region", "overseas"))
stack.enter_context(patch.object(detect_service.settings, "node_role", "control"))
stack.enter_context(patch.object(detect_service.settings, "node_code", "overseas-control-01"))
stack.enter_context(patch("app.services.detect_service.ensure_runtime_schema"))
stack.enter_context(patch("app.services.detect_service.get_db", side_effect=RuntimeError("db down")))
stack.enter_context(
patch(
"app.services.detect_service.get_settings_payload",
return_value={
"proxy_config": {"proxy_enable": False, "allow_direct": True, "proxy_urls": []},
"process_count": 80,
"node_process_counts": {"mainland-controller-01": 24},
"thread_count": 1000,
"node_thread_counts": {"overseas-control-01": 1, "mainland-controller-01-a": 1000},
},
)
)
stack.enter_context(
patch("app.services.detect_service.get_runtime_settings", return_value={"worker_log_sync_enabled": False, "worker_log_sync_mode": "key"})
)
stack.enter_context(
patch("app.services.detect_service._load_recent_worker_lines", return_value=(False, "2026-04-23 00:00:00", ["stale line"]))
)
stack.enter_context(
patch("app.services.detect_service.detect_worker_runtime", return_value={"mode": "linux-systemd", "running": False, "process_count": 0, "latest_start_time": "", "message": "inactive"})
)
stack.enter_context(
patch("app.services.detect_service._load_runtime_state", return_value={"service_running": False, "detecting": False, "active_threads": 0, "max_threads": 1, "detail": ""})
)
stack.enter_context(patch("app.services.detect_service._load_runtime_state_from_cluster_node", return_value={}))
stack.enter_context(patch("app.services.detect_service._extract_available_proxy_count", return_value=0))
stack.enter_context(patch("app.services.detect_service._extract_active_thread_snapshot", return_value={"active": 0, "max": 0}))
stack.enter_context(patch("app.services.detect_service._normalize_recent_warning", return_value=""))
stack.enter_context(
patch(
"app.services.detect_service._build_proxy_runtime_snapshot",
return_value={
"state": "disabled",
"label": "未启用代理",
"detail": "-",
"direct_fallback_active": True,
"reason": "proxy_disabled",
"last_refresh_status": "",
"last_refresh_time": "",
"source_count": 0,
"raw_items": 0,
"validated_count": 0,
"available_count": 0,
"source_stats": [],
"supplier_empty": False,
},
)
)
stack.enter_context(
patch(
"app.services.detect_service.resolve_thread_count",
return_value={
"effective_thread_count": 1,
"default_thread_count": 1000,
"source": "node_override",
"override_thread_count": 1,
"node_code": "overseas-control-01",
},
)
)
stack.enter_context(
patch(
"app.services.detect_service.resolve_process_count",
return_value={
"effective_process_count": 1,
"default_process_count": 80,
"source": "node_override",
"override_process_count": 1,
"node_code": "overseas-control-01",
},
)
)
stack.enter_context(patch("app.services.detect_service.get_active_detect_job_summary", return_value=active_job))
stack.enter_context(patch("app.services.detect_service.get_detect_queue_health", return_value=aggregate_queue_health))
stack.enter_context(patch("app.services.detect_service.sync_detect_runs", return_value=[]))
stack.enter_context(patch("app.services.detect_service._resolve_remote_log_snapshot", return_value={}))
stack.enter_context(patch("app.services.detect_service._extract_dependency_alerts", return_value=[]))
stack.enter_context(patch("app.services.detect_service.append_detect_result_projection_if_changed"))
payload = detect_service.get_detect_status()
self.assertEqual(537, payload["progress"]["running"])
self.assertEqual(537, payload["active_thread_count"])
self.assertEqual(1000, payload["max_thread_count"])
self.assertEqual(1, payload["aggregate_process_count"])
def test_get_detect_status_drops_stale_aggregate_nodes_from_running_and_process_counts(self) -> None:
live_heartbeat = datetime.now().isoformat(sep=" ", timespec="seconds")
stale_heartbeat = (datetime.now() - timedelta(minutes=8)).isoformat(sep=" ", timespec="seconds")
active_job = {
"job_id": 88,
"job_code": "sync-overseas-688",
"status": "running",
"items_total": 2600,
"items_pending": 500,
"items_running": 2100,
"items_completed": 0,
"items_failed": 0,
"items_blacklisted": 0,
"progress_percent": 80.77,
"display_items_running": 2100,
"display_active_threads": 2100,
"display_max_threads": 2000,
"distributed_node_stats": [
{
"node_code": "mainland-controller-01-a",
"items_running": 300,
"display_running": 300,
"active_threads": 300,
"max_threads": 1000,
"status": "busy",
"last_heartbeat_at": live_heartbeat,
},
{
"node_code": "mainland-controller-01-b",
"items_running": 1800,
"display_running": 1800,
"active_threads": 1800,
"max_threads": 1000,
"status": "stale",
"last_heartbeat_at": stale_heartbeat,
},
],
}
aggregate_queue_health = {
"has_active_job": True,
"job": {
"job_id": 88,
"job_code": "sync-overseas-688",
"status": "running",
"progress_percent": 80.77,
},
"queue": {
"items_total": 2600,
"pending": 500,
"claimed": 0,
"running": 2100,
"display_running": 2100,
"completed": 0,
"blacklisted": 0,
"failed": 0,
},
"nodes": [
{
"node_code": "mainland-controller-01-a",
"items_running": 300,
"display_running": 300,
"active_threads": 300,
"max_threads": 1000,
"status": "busy",
"last_heartbeat_at": live_heartbeat,
},
{
"node_code": "mainland-controller-01-b",
"items_running": 1800,
"display_running": 1800,
"active_threads": 1800,
"max_threads": 1000,
"status": "stale",
"last_heartbeat_at": stale_heartbeat,
},
],
}
with ExitStack() as stack:
stack.enter_context(patch.object(detect_service.settings, "node_region", "overseas"))
stack.enter_context(patch.object(detect_service.settings, "node_role", "control"))
stack.enter_context(patch.object(detect_service.settings, "node_code", "overseas-control-01"))
stack.enter_context(patch("app.services.detect_service.ensure_runtime_schema"))
stack.enter_context(patch("app.services.detect_service.get_db", side_effect=RuntimeError("db down")))
stack.enter_context(
patch(
"app.services.detect_service.get_settings_payload",
return_value={
"proxy_config": {"proxy_enable": False, "allow_direct": True, "proxy_urls": []},
"process_count": 80,
"node_process_counts": {"mainland-controller-01": 70},
"thread_count": 1000,
"node_thread_counts": {"overseas-control-01": 1},
},
)
)
stack.enter_context(
patch("app.services.detect_service.get_runtime_settings", return_value={"worker_log_sync_enabled": False, "worker_log_sync_mode": "key"})
)
stack.enter_context(
patch("app.services.detect_service._load_recent_worker_lines", return_value=(False, "2026-04-23 00:00:00", ["stale line"]))
)
stack.enter_context(
patch("app.services.detect_service.detect_worker_runtime", return_value={"mode": "linux-systemd", "running": False, "process_count": 0, "latest_start_time": "", "message": "inactive"})
)
stack.enter_context(
patch("app.services.detect_service._load_runtime_state", return_value={"service_running": False, "detecting": False, "active_threads": 0, "max_threads": 1, "detail": ""})
)
stack.enter_context(patch("app.services.detect_service._load_runtime_state_from_cluster_node", return_value={}))
stack.enter_context(patch("app.services.detect_service._extract_available_proxy_count", return_value=0))
stack.enter_context(patch("app.services.detect_service._extract_active_thread_snapshot", return_value={"active": 0, "max": 0}))
stack.enter_context(patch("app.services.detect_service._normalize_recent_warning", return_value=""))
stack.enter_context(
patch(
"app.services.detect_service._build_proxy_runtime_snapshot",
return_value={
"state": "disabled",
"label": "未启用代理",
"detail": "-",
"direct_fallback_active": True,
"reason": "proxy_disabled",
"last_refresh_status": "",
"last_refresh_time": "",
"source_count": 0,
"raw_items": 0,
"validated_count": 0,
"available_count": 0,
"source_stats": [],
"supplier_empty": False,
},
)
)
stack.enter_context(
patch(
"app.services.detect_service.resolve_thread_count",
return_value={
"effective_thread_count": 1,
"default_thread_count": 1000,
"source": "node_override",
"override_thread_count": 1,
"node_code": "overseas-control-01",
},
)
)
stack.enter_context(
patch(
"app.services.detect_service.resolve_process_count",
side_effect=[
{
"effective_process_count": 1,
"default_process_count": 80,
"source": "child_instance",
"override_process_count": None,
"node_code": "mainland-controller-01-a",
}
],
)
)
stack.enter_context(patch("app.services.detect_service.get_active_detect_job_summary", return_value=active_job))
stack.enter_context(patch("app.services.detect_service.get_detect_queue_health", return_value=aggregate_queue_health))
stack.enter_context(patch("app.services.detect_service.sync_detect_runs", return_value=[]))
stack.enter_context(patch("app.services.detect_service._resolve_remote_log_snapshot", return_value={}))
stack.enter_context(patch("app.services.detect_service._extract_dependency_alerts", return_value=[]))
stack.enter_context(patch("app.services.detect_service.append_detect_result_projection_if_changed"))
payload = detect_service.get_detect_status()
self.assertEqual(300, payload["progress"]["running"])
self.assertEqual(300, payload["active_thread_count"])
self.assertEqual(1000, payload["max_thread_count"])
self.assertEqual(1, payload["aggregate_process_count"])
self.assertEqual(1, payload["aggregate_participating_node_count"])
self.assertEqual(["mainland-controller-01-a"], payload["aggregate_participating_node_codes"])
def test_get_detect_status_uses_event_proxy_counts_when_cluster_runtime_proxy_counts_are_missing(self) -> None:
active_job = {
"job_id": 77,
"job_code": "sync-overseas-482",
"status": "running",
"items_pending": 410,
"items_completed": 100,
"items_failed": 0,
"items_blacklisted": 0,
"progress_percent": 55.0,
"display_items_running": 807,
"display_active_threads": 807,
"current_cycle_token": "cycle-1",
"distributed_node_stats": [
{
"node_code": "mainland-controller-01-u",
"items_running": 300,
"display_running": 300,
"active_threads": 300,
"max_threads": 1000,
},
{
"node_code": "mainland-controller-01-v",
"items_running": 301,
"display_running": 301,
"active_threads": 301,
"max_threads": 1000,
},
{
"node_code": "mainland-worker-01-a",
"items_running": 206,
"display_running": 206,
"active_threads": 206,
"max_threads": 1000,
},
],
"current_cycle_events": [
{
"event_type": "worker_log",
"created_at": "2026-04-23 19:40:00",
"node_code": "mainland-controller-01-u",
"message": "当前可用代理数: 486",
"payload": {"cycle_token": "cycle-1", "log_mode": "key"},
},
{
"event_type": "worker_log",
"created_at": "2026-04-23 19:40:10",
"node_code": "mainland-controller-01-v",
"message": "代理池刷新完成,共 486 个可用代理,来源链接 6 个,原始 520 个",
"payload": {"cycle_token": "cycle-1", "log_mode": "key"},
},
{
"event_type": "worker_log",
"created_at": "2026-04-23 19:40:20",
"node_code": "mainland-worker-01-a",
"message": "共享刷新进行中,继续沿用缓存 321 个",
"payload": {"cycle_token": "cycle-1", "log_mode": "key"},
},
],
}
aggregate_queue_health = {
"has_active_job": True,
"job": {
"job_id": 77,
"job_code": "sync-overseas-482",
"status": "running",
"progress_percent": 55.0,
},
"queue": {
"items_total": 1317,
"pending": 410,
"claimed": 0,
"running": 807,
"display_running": 807,
"completed": 100,
"blacklisted": 0,
"failed": 0,
},
"nodes": [
{
"node_code": "mainland-controller-01-u",
"items_running": 300,
"display_running": 300,
"active_threads": 300,
"max_threads": 1000,
},
{
"node_code": "mainland-controller-01-v",
"items_running": 301,
"display_running": 301,
"active_threads": 301,
"max_threads": 1000,
},
{
"node_code": "mainland-worker-01-a",
"items_running": 206,
"display_running": 206,
"active_threads": 206,
"max_threads": 1000,
},
],
}
def _resolve_thread_count(*, node_code=None, settings_payload=None):
if str(node_code or "").startswith("mainland-"):
return {
"effective_thread_count": 1000,
"default_thread_count": 1000,
"source": "default",
"override_thread_count": None,
"node_code": str(node_code or ""),
}
return {
"effective_thread_count": 1,
"default_thread_count": 1000,
"source": "node_override",
"override_thread_count": 1,
"node_code": "overseas-control-01",
}
def _resolve_process_count(*, node_code=None, settings_payload=None):
normalized_node_code = str(node_code or "")
if normalized_node_code == "mainland-controller-01":
return {
"effective_process_count": 80,
"default_process_count": 80,
"source": "default",
"override_process_count": None,
"node_code": normalized_node_code,
}
if normalized_node_code == "mainland-worker-01":
return {
"effective_process_count": 60,
"default_process_count": 60,
"source": "default",
"override_process_count": None,
"node_code": normalized_node_code,
}
if normalized_node_code.startswith("mainland-"):
return {
"effective_process_count": 1,
"default_process_count": 80,
"source": "child_instance",
"override_process_count": None,
"node_code": normalized_node_code,
}
return {
"effective_process_count": 1,
"default_process_count": 80,
"source": "node_override",
"override_process_count": 1,
"node_code": "overseas-control-01",
}
with ExitStack() as stack:
stack.enter_context(patch.object(detect_service.settings, "node_region", "overseas"))
stack.enter_context(patch.object(detect_service.settings, "node_role", "control"))
stack.enter_context(patch.object(detect_service.settings, "node_code", "overseas-control-01"))
stack.enter_context(patch("app.services.detect_service.ensure_runtime_schema"))
stack.enter_context(patch("app.services.detect_service.get_db", side_effect=RuntimeError("db down")))
stack.enter_context(
patch(
"app.services.detect_service.get_settings_payload",
return_value={
"proxy_config": {"proxy_enable": False, "allow_direct": True, "proxy_urls": []},
"process_count": 80,
"node_process_counts": {"mainland-worker-01": 60},
"thread_count": 1000,
"node_thread_counts": {"overseas-control-01": 1},
},
)
)
stack.enter_context(
patch("app.services.detect_service.get_runtime_settings", return_value={"worker_log_sync_enabled": False, "worker_log_sync_mode": "key"})
)
stack.enter_context(
patch("app.services.detect_service._load_recent_worker_lines", return_value=(False, "2026-04-23 00:00:00", ["stale line"]))
)
stack.enter_context(
patch("app.services.detect_service.detect_worker_runtime", return_value={"mode": "linux-systemd", "running": False, "process_count": 0, "latest_start_time": "", "message": "inactive"})
)
stack.enter_context(
patch("app.services.detect_service._load_runtime_state", return_value={"service_running": False, "detecting": False, "active_threads": 0, "max_threads": 1, "detail": ""})
)
stack.enter_context(patch("app.services.detect_service._load_runtime_state_from_cluster_node", return_value={}))
stack.enter_context(patch("app.services.detect_service._extract_available_proxy_count", return_value=0))
stack.enter_context(patch("app.services.detect_service._extract_active_thread_snapshot", return_value={"active": 0, "max": 0}))
stack.enter_context(patch("app.services.detect_service._normalize_recent_warning", return_value=""))
stack.enter_context(
patch(
"app.services.detect_service._build_proxy_runtime_snapshot",
return_value={
"state": "warming_up",
"label": "等待首刷",
"detail": "-",
"direct_fallback_active": False,
"reason": "waiting_for_first_refresh",
"last_refresh_status": "未刷新",
"last_refresh_time": "",
"source_count": 0,
"raw_items": 0,
"validated_count": 0,
"available_count": 0,
"source_stats": [],
"supplier_empty": False,
},
)
)
stack.enter_context(patch("app.services.detect_service.resolve_thread_count", side_effect=_resolve_thread_count))
stack.enter_context(patch("app.services.detect_service.resolve_process_count", side_effect=_resolve_process_count))
stack.enter_context(patch("app.services.detect_service.get_active_detect_job_summary", return_value=active_job))
stack.enter_context(patch("app.services.detect_service.get_detect_queue_health", return_value=aggregate_queue_health))
stack.enter_context(
patch(
"app.services.detect_service._load_runtime_states_from_cluster_nodes",
return_value={
"mainland-controller-01": {
"node_code": "mainland-controller-01",
"available_proxy_count": 0,
"proxy_runtime_label": "",
"proxy_runtime_reason": "",
"proxy_last_refresh_status": "",
"proxy_last_refresh_time": "",
"proxy_last_refresh_source_count": 0,
"proxy_last_refresh_total_items": 0,
"proxy_last_validated_count": 0,
},
"mainland-worker-01": {
"node_code": "mainland-worker-01",
"available_proxy_count": 0,
"proxy_runtime_label": "",
"proxy_runtime_reason": "",
"proxy_last_refresh_status": "",
"proxy_last_refresh_time": "",
"proxy_last_refresh_source_count": 0,
"proxy_last_refresh_total_items": 0,
"proxy_last_validated_count": 0,
},
},
)
)
stack.enter_context(patch("app.services.detect_service.sync_detect_runs", return_value=[]))
stack.enter_context(patch("app.services.detect_service._resolve_remote_log_snapshot", return_value={}))
stack.enter_context(patch("app.services.detect_service._extract_dependency_alerts", return_value=[]))
stack.enter_context(patch("app.services.detect_service.append_detect_result_projection_if_changed"))
payload = detect_service.get_detect_status()
self.assertEqual(807, payload["available_proxy_count"])
self.assertEqual("集群代理正常", payload["proxy_runtime_label"])
self.assertIn("参与服务器 2 台", payload["proxy_runtime_detail"])
if __name__ == "__main__":
unittest.main()

View File

@@ -0,0 +1,143 @@
from __future__ import annotations
import unittest
from unittest.mock import patch
from app.services import domains_service
class _FakeDomainsCursor:
def __init__(self) -> None:
self._fetchone_result = None
self._fetchall_result = []
self.executed: list[tuple[str, tuple]] = []
self.updated_detection_params = None
def __enter__(self):
return self
def __exit__(self, exc_type, exc, tb):
return False
def execute(self, sql: str, params=None) -> None:
normalized_sql = " ".join(str(sql or "").split()).lower()
tuple_params = tuple(params or ())
self.executed.append((normalized_sql, tuple_params))
if normalized_sql.startswith("select count(*)"):
self._fetchone_result = (1,)
return
if normalized_sql.startswith("select d.id, d.domain"):
self._fetchall_result = [
(
1,
"a.com",
0,
0,
0,
0,
1,
"",
None,
"",
0,
None,
7,
False,
None,
"",
{"status": True, "state": "passed"},
{"status": True, "state": "passed"},
True,
{"status": True, "state": "passed"},
{"status": True, "state": "passed"},
{"status": True, "state": "passed"},
{"status": True, "state": "passed"},
{"status": True, "state": "passed"},
{"status": False, "state": "failed", "message": "juziseo failed"},
{"status": False, "state": "blacklisted", "message": "jucha blacklisted"},
)
]
return
if normalized_sql.startswith("update domains set"):
return
if normalized_sql.startswith("select id, baidu_history"):
self._fetchone_result = (
7,
{"status": False, "state": "failed", "message": "timeout", "checked_at": "2026-04-20 12:00:00", "step": "baidu_site"},
{"status": True, "state": "passed", "message": "ok", "checked_at": "2026-04-20 12:00:00", "step": "baidu_site"},
False,
{"status": False, "state": "failed", "message": "old", "checked_at": "2026-04-20 12:00:00", "step": "qihu360_site"},
{"status": False, "state": "failed", "message": "old", "checked_at": "2026-04-20 12:00:00", "step": "google_site"},
False,
)
return
if normalized_sql.startswith("update domain_detections set"):
self.updated_detection_params = tuple_params
return
raise AssertionError(f"unexpected sql: {sql}")
def fetchone(self):
return self._fetchone_result
def fetchall(self):
return list(self._fetchall_result)
class _FakeDomainsConnection:
def __init__(self) -> None:
self.cursor_instance = _FakeDomainsCursor()
self.committed = False
def __enter__(self):
return self
def __exit__(self, exc_type, exc, tb):
return False
def cursor(self):
return self.cursor_instance
def commit(self) -> None:
self.committed = True
class DomainsServiceTests(unittest.TestCase):
def test_build_domain_query_parts_supports_false_backlink_filter(self) -> None:
_from_clause, where_clause, params = domains_service._build_domain_query_parts({"backlink_gt_10": False})
self.assertIn("coalesce(dd.backlink_count_gt_10, false) = %s", where_clause)
self.assertEqual([False], params)
@patch("app.services.domains_service.get_db")
def test_fetch_domains_step_summary_counts_juziseo_and_jucha_results(self, mock_get_db) -> None:
fake_conn = _FakeDomainsConnection()
mock_get_db.return_value = fake_conn
result = domains_service.fetch_domains(page=1, page_size=20)
summary = result["list"][0]["step_summary"]
self.assertEqual(1, summary["failed_count"])
self.assertEqual(1, summary["blacklisted_count"])
self.assertTrue(summary["has_failed"])
self.assertTrue(summary["has_blacklisted_step"])
@patch("app.services.domains_service.get_db")
def test_batch_update_domains_preserves_detection_metadata_shape(self, mock_get_db) -> None:
fake_conn = _FakeDomainsConnection()
mock_get_db.return_value = fake_conn
with patch("app.services.domains_service._now_text", return_value="2026-04-23 15:30:00"):
result = domains_service.batch_update_domains([42], {"baidu_site": ""})
self.assertEqual(1, result["updated_count"])
self.assertIsNotNone(fake_conn.cursor_instance.updated_detection_params)
updated_payload = fake_conn.cursor_instance.updated_detection_params[0]
self.assertEqual(False, updated_payload["status"])
self.assertEqual("failed", updated_payload["state"])
self.assertEqual("人工批量更新", updated_payload["message"])
self.assertEqual("2026-04-23 15:30:00", updated_payload["checked_at"])
self.assertTrue(updated_payload["manual_override"])
if __name__ == "__main__":
unittest.main()

View File

@@ -0,0 +1,52 @@
from __future__ import annotations
import tempfile
import unittest
from pathlib import Path
from unittest.mock import patch
import app.services.import_task_service as import_task_service
class _TraceLock:
def __init__(self, order: list[str]) -> None:
self.order = order
def __enter__(self):
self.order.append("enter")
return self
def __exit__(self, exc_type, exc, tb):
self.order.append("exit")
return False
class ImportTaskServiceTests(unittest.TestCase):
@patch("app.services.import_task_service.import_domains_from_path")
@patch("app.services.import_task_service._update_task_with_log")
def test_run_import_task_acquires_execution_lock_before_marking_running(
self,
mock_update_task_with_log,
mock_import_domains_from_path,
) -> None:
order: list[str] = []
mock_update_task_with_log.side_effect = lambda *args, **kwargs: order.append("update")
mock_import_domains_from_path.return_value = {
"source_label": "TXT 导入",
"stats": {"total": 1, "valid": 1, "added": 1, "exists": 0, "invalid": 0},
}
with tempfile.TemporaryDirectory() as tmpdir:
path = Path(tmpdir) / "domains.txt"
path.write_text("a.com\n", encoding="utf-8")
with patch.object(import_task_service, "_IMPORT_EXECUTION_LOCK", _TraceLock(order)):
import_task_service._run_import_task("task-1", str(path), source_type=7)
self.assertEqual("enter", order[0])
self.assertIn("update", order[1:])
self.assertEqual("exit", order[-1])
if __name__ == "__main__":
unittest.main()

View File

@@ -0,0 +1,94 @@
from __future__ import annotations
import tempfile
import unittest
from pathlib import Path
from unittest.mock import patch
from app.services.import_worker_service import import_domains_from_path
class _FakeCursor:
def __init__(self) -> None:
self._fetchall_result = []
self._fetchone_result = None
self.inserted_domains: list[str] = []
self.inserted_detect_tasks: list[int] = []
def __enter__(self):
return self
def __exit__(self, exc_type, exc, tb):
return False
def execute(self, sql: str, params=None) -> None:
normalized_sql = " ".join(str(sql or "").split()).lower()
params = params or ()
if normalized_sql.startswith("select domain from domains where domain = any"):
self._fetchall_result = []
return
if normalized_sql.startswith("insert into domains"):
domain = params[0]
self.inserted_domains.append(domain)
self._fetchone_result = (len(self.inserted_domains),)
return
if normalized_sql.startswith("insert into detect_tasks"):
self.inserted_detect_tasks.append(int(params[0]))
return
raise AssertionError(f"unexpected sql: {sql}")
def fetchall(self):
return list(self._fetchall_result)
def fetchone(self):
return self._fetchone_result
class _FakeConnection:
def __init__(self) -> None:
self.cursor_instance = _FakeCursor()
self.committed = False
def __enter__(self):
return self
def __exit__(self, exc_type, exc, tb):
return False
def cursor(self):
return self.cursor_instance
def commit(self) -> None:
self.committed = True
class ImportWorkerServiceTests(unittest.TestCase):
@patch("app.services.import_worker_service.get_db")
def test_import_domains_from_path_skips_duplicate_domains_in_same_batch(self, mock_get_db) -> None:
fake_conn = _FakeConnection()
mock_get_db.return_value = fake_conn
with tempfile.TemporaryDirectory() as tmpdir:
path = Path(tmpdir) / "domains.txt"
path.write_text("a.com\na.com\nb.net\ninvalid-domain\n", encoding="utf-8")
result = import_domains_from_path(path, source_type=7)
self.assertEqual(["a.com", "b.net"], fake_conn.cursor_instance.inserted_domains)
self.assertEqual([1, 2], fake_conn.cursor_instance.inserted_detect_tasks)
self.assertTrue(fake_conn.committed)
self.assertEqual(
{
"total": 4,
"valid": 3,
"added": 2,
"exists": 1,
"invalid": 1,
"failed": 0,
},
result["stats"],
)
if __name__ == "__main__":
unittest.main()

View File

@@ -4,6 +4,9 @@ import unittest
from unittest.mock import patch
from app.api.routes.ops import (
ops_migration_execute,
ops_migration_preview,
ops_migration_source_profile,
ops_doctor_decision,
ops_go_live_bundle,
ops_go_live_review,
@@ -13,6 +16,39 @@ from app.api.routes.ops import (
class OpsApiRoutesTestCase(unittest.TestCase):
@patch("app.api.routes.ops.get_ops_migration_source_profile")
def test_migration_source_profile_route_returns_payload(self, mock_source_profile) -> None:
mock_source_profile.return_value = {"source_node": {"node_code": "overseas-control-01"}}
response = ops_migration_source_profile()
self.assertEqual(0, response.code)
self.assertEqual("overseas-control-01", response.data["source_node"]["node_code"])
mock_source_profile.assert_called_once_with()
@patch("app.api.routes.ops.preview_ops_migration")
def test_migration_preview_route_passes_payload(self, mock_preview_ops_migration) -> None:
mock_preview_ops_migration.return_value = (True, "ok", {"plan_steps": [{"key": "sync_env_files"}]})
payload = {"target_node_code": "node-a"}
response = ops_migration_preview(payload)
self.assertEqual(0, response.code)
self.assertEqual("sync_env_files", response.data["plan_steps"][0]["key"])
mock_preview_ops_migration.assert_called_once_with(payload)
@patch("app.api.routes.ops.execute_ops_migration")
def test_migration_execute_route_returns_error_payload(self, mock_execute_ops_migration) -> None:
mock_execute_ops_migration.return_value = (False, "failed", {"blocking_reasons": ["ssh missing"]})
payload = {"target_node_code": "node-a"}
response = ops_migration_execute(payload)
self.assertEqual(1, response.code)
self.assertEqual("failed", response.message)
self.assertEqual(["ssh missing"], response.data["blocking_reasons"])
mock_execute_ops_migration.assert_called_once_with(payload)
@patch("app.api.routes.ops.get_ops_go_live_signoff")
def test_go_live_signoff_route_uses_service_payload(self, mock_get_ops_go_live_signoff) -> None:
mock_get_ops_go_live_signoff.return_value = {

View File

@@ -0,0 +1,290 @@
from __future__ import annotations
import unittest
from unittest.mock import patch
from app.services.ops_migration_service import execute_ops_migration, preview_ops_migration
class OpsMigrationServiceTestCase(unittest.TestCase):
@patch("app.services.ops_migration_service._collect_remote_checks")
@patch("app.services.ops_migration_service._inspect_target_database")
@patch("app.services.ops_migration_service._find_managed_node")
def test_preview_returns_blocking_reason_when_remote_tools_missing(
self,
mock_find_managed_node,
mock_inspect_target_database,
mock_collect_remote_checks,
) -> None:
mock_find_managed_node.return_value = {
"node_code": "target-a",
"ssh_host": "10.0.0.8",
"ssh_user": "root",
"ssh_port": 22,
"auth_mode": "key",
}
mock_collect_remote_checks.return_value = {
"tools": {
"python3": True,
"node": False,
"npm": False,
"systemctl": True,
"psql": True,
"pg_dump": True,
"curl": True,
},
"paths": {
"repo_exists": True,
"repo_git": True,
"domain_root_exists": True,
"api_root_exists": True,
"web_root_exists": True,
},
"remote_db_config": {
"DB_HOST": "127.0.0.1",
"DB_PORT": "5432",
"DB_DATABASE": "domain",
"DB_USER": "domainuser",
},
"blocking_reasons": ["目标机缺少 node。", "目标机缺少 npm。"],
"warnings": [],
}
mock_inspect_target_database.return_value = {"available": False}
ok, message, data = preview_ops_migration({"target_node_code": "target-a"})
self.assertFalse(ok)
self.assertEqual("迁移预检查未通过", message)
self.assertIn("目标机缺少 node。", data["blocking_reasons"])
self.assertIn("目标机缺少 npm。", data["blocking_reasons"])
@patch("app.services.ops_migration_service._collect_remote_checks")
@patch("app.services.ops_migration_service._inspect_target_database")
@patch("app.services.ops_migration_service._find_managed_node")
def test_preview_uses_remote_env_as_target_db_default(
self,
mock_find_managed_node,
mock_inspect_target_database,
mock_collect_remote_checks,
) -> None:
mock_find_managed_node.return_value = {
"node_code": "target-a",
"ssh_host": "10.0.0.8",
"ssh_user": "root",
"ssh_port": 22,
"auth_mode": "key",
}
mock_collect_remote_checks.return_value = {
"tools": {
"python3": True,
"node": True,
"npm": True,
"systemctl": True,
"psql": True,
"pg_dump": True,
"curl": True,
},
"paths": {
"repo_exists": True,
"repo_git": True,
"domain_root_exists": True,
"api_root_exists": True,
"web_root_exists": True,
},
"git_commit": "abc123",
"remote_db_config": {
"DB_HOST": "127.0.0.1",
"DB_PORT": "5433",
"DB_DATABASE": "domain_remote",
"DB_USER": "remote_user",
},
"blocking_reasons": [],
"warnings": [],
}
mock_inspect_target_database.return_value = {
"available": True,
"has_business_data": False,
}
ok, message, data = preview_ops_migration({"target_node_code": "target-a", "overwrite_database": True})
self.assertTrue(ok)
self.assertEqual("迁移预检查完成", message)
self.assertEqual("domain_remote", data["target_db_config"]["database"])
self.assertEqual("remote_user", data["target_db_config"]["user"])
self.assertEqual(5433, data["target_db_config"]["port"])
self.assertTrue(bool(data["execution_guard"]["token"]))
@patch("app.services.ops_migration_service._run_remote_health_check")
@patch("app.services.ops_migration_service._restart_remote_services")
@patch("app.services.ops_migration_service._build_remote_frontend")
@patch("app.services.ops_migration_service._inspect_target_database")
@patch("app.services.ops_migration_service._collect_remote_checks")
@patch("app.services.ops_migration_service._find_managed_node")
def test_execute_rejects_missing_confirmation_text_for_nonempty_target_db(
self,
mock_find_managed_node,
mock_collect_remote_checks,
mock_inspect_target_database,
mock_build_remote_frontend,
mock_restart_remote_services,
mock_run_remote_health_check,
) -> None:
mock_find_managed_node.return_value = {
"node_code": "target-a",
"ssh_host": "10.0.0.8",
"ssh_user": "root",
"ssh_port": 22,
"auth_mode": "key",
}
mock_collect_remote_checks.return_value = {
"tools": {
"python3": True,
"node": True,
"npm": True,
"systemctl": True,
"psql": True,
"pg_dump": True,
"curl": True,
},
"paths": {
"repo_exists": True,
"repo_git": True,
"domain_root_exists": True,
"api_root_exists": True,
"web_root_exists": True,
},
"git_commit": "abc123",
"remote_db_config": {
"DB_HOST": "127.0.0.1",
"DB_PORT": "5432",
"DB_DATABASE": "domain_remote",
"DB_USER": "remote_user",
},
"blocking_reasons": [],
"warnings": [],
}
mock_inspect_target_database.return_value = {
"available": True,
"database": "domain_remote",
"has_business_data": True,
"public_table_count": 10,
"business_table_count": 5,
}
mock_build_remote_frontend.return_value = (True, "ok", {})
mock_restart_remote_services.return_value = (True, "ok", {})
mock_run_remote_health_check.return_value = (True, "ok", {})
preview_ok, _preview_message, preview_data = preview_ops_migration(
{"target_node_code": "target-a", "overwrite_database": True}
)
self.assertTrue(preview_ok)
token = preview_data["execution_guard"]["token"]
required_confirmation_text = preview_data["execution_guard"]["required_confirmation_text"]
self.assertEqual("OVERWRITE domain_remote", required_confirmation_text)
execute_ok, execute_message, execute_data = execute_ops_migration(
{
"target_node_code": "target-a",
"overwrite_database": True,
"execute_confirmation_token": token,
}
)
self.assertFalse(execute_ok)
self.assertEqual("缺少数据库覆盖确认文案,执行被拒绝。", execute_message)
self.assertIn("missing execute_confirmation_text", execute_data["blocking_reasons"])
@patch("app.services.ops_migration_service._start_migration_dispatch_thread")
@patch("app.services.ops_migration_service.create_ops_job")
@patch("app.services.ops_migration_service._collect_remote_checks")
@patch("app.services.ops_migration_service._inspect_target_database")
@patch("app.services.ops_migration_service._find_managed_node")
def test_execute_creates_async_job_for_long_running_migration(
self,
mock_find_managed_node,
mock_inspect_target_database,
mock_collect_remote_checks,
mock_create_ops_job,
mock_start_thread,
) -> None:
mock_find_managed_node.return_value = {
"node_code": "target-a",
"ssh_host": "10.0.0.8",
"ssh_user": "root",
"ssh_port": 22,
"auth_mode": "key",
}
mock_collect_remote_checks.return_value = {
"tools": {
"python3": True,
"node": True,
"npm": True,
"systemctl": True,
"psql": True,
"pg_dump": True,
"curl": True,
},
"paths": {
"repo_exists": True,
"repo_git": True,
"domain_root_exists": True,
"api_root_exists": True,
"web_root_exists": True,
},
"git_commit": "abc123",
"remote_db_config": {
"DB_HOST": "127.0.0.1",
"DB_PORT": "5432",
"DB_DATABASE": "domain_remote",
"DB_USER": "remote_user",
},
"blocking_reasons": [],
"warnings": [],
}
mock_inspect_target_database.return_value = {
"available": True,
"database": "domain_remote",
"has_business_data": False,
"public_table_count": 0,
"business_table_count": 0,
}
mock_create_ops_job.return_value = (
True,
"ok",
{
"job": {
"id": 88,
"job_code": "ops-20260422160000-abc123",
"action": "migration.execute",
"status": "queued",
"target_node_code": "target-a",
}
},
)
preview_ok, _preview_message, preview_data = preview_ops_migration({"target_node_code": "target-a"})
self.assertTrue(preview_ok)
token = preview_data["execution_guard"]["token"]
execute_ok, execute_message, execute_data = execute_ops_migration(
{
"target_node_code": "target-a",
"execute_confirmation_token": token,
}
)
self.assertTrue(execute_ok)
self.assertEqual("迁移任务已创建,后台开始执行。", execute_message)
self.assertEqual(88, execute_data["job"]["id"])
self.assertEqual("migration.execute", mock_create_ops_job.call_args.args[0]["action"])
self.assertFalse(bool(mock_create_ops_job.call_args.args[0]["run_now"]))
self.assertEqual("control-plane", mock_create_ops_job.call_args.args[0]["execution_mode"])
self.assertEqual("", mock_create_ops_job.call_args.args[0]["payload"]["target_db_password"])
mock_start_thread.assert_called_once_with(88)
if __name__ == "__main__":
unittest.main()

View File

@@ -9,9 +9,11 @@ from pathlib import Path
from unittest.mock import patch
from app.services.ops_release_executor_core import (
_systemd_dropin_content,
_pick_release_owner_group,
build_remote_release_action_script,
execute_release_action,
normalize_release_health_check_urls,
)
@@ -36,6 +38,7 @@ def _build_release_archive() -> bytes:
"README.txt": b"hello-release",
"domain-api/deploy/systemd/domain-node-agent.service": b"[Service]\nEnvironmentFile=-/etc/default/domaincheck-worker\n",
"domain-api/deploy/systemd/domain-worker.service": b"[Service]\nEnvironmentFile=-/etc/default/domaincheck-worker\n",
"domain-api/deploy/systemd/domain-worker@.service": b"[Service]\nEnvironmentFile=-/etc/default/domaincheck-worker-%i\n",
"domain-api/deploy/systemd/domain-api.service": b"[Service]\nEnvironmentFile=-/etc/default/domaincheck-api\n",
"domain-api/deploy/systemd/domain-sync-agent.service": b"[Service]\nEnvironmentFile=-/etc/default/domaincheck-worker\n",
}
@@ -47,6 +50,42 @@ def _build_release_archive() -> bytes:
class OpsReleaseExecutorCoreTests(unittest.TestCase):
def test_normalize_release_health_check_urls_rewrites_runtime_status_probe(self) -> None:
self.assertEqual(
[
"http://127.0.0.1:8100/health",
"http://127.0.0.1:8100/health",
"https://example.com/custom-health",
],
normalize_release_health_check_urls(
[
"http://127.0.0.1:8100/api/v1/runtime/status",
"http://127.0.0.1:8100/runtime/status?full=1",
"https://example.com/custom-health",
]
),
)
def test_api_service_template_limits_graceful_shutdown(self) -> None:
service_text = Path("domain-api/deploy/systemd/domain-api.service").read_text(encoding="utf-8")
self.assertIn("--timeout-graceful-shutdown 15", service_text)
self.assertIn("TimeoutStopSec=20", service_text)
def test_worker_service_template_uses_current_symlink(self) -> None:
service_text = Path("domain-api/deploy/systemd/domain-worker.service").read_text(encoding="utf-8")
self.assertIn("WorkingDirectory=/opt/domaincheck/current/domainCheck", service_text)
self.assertIn(
"ExecStart=/opt/domaincheck/domainCheck/.venv/bin/python /opt/domaincheck/current/domainCheck/detect_worker.py",
service_text,
)
def test_api_service_dropin_uses_graceful_shutdown_timeout(self) -> None:
dropin_text = _systemd_dropin_content("domaincheck-api", "/opt/domaincheck")
self.assertIn("--timeout-graceful-shutdown 15", dropin_text)
def test_pick_release_owner_group_prefers_service_identity_over_path_owner(self) -> None:
with tempfile.TemporaryDirectory() as tmpdir:
with patch(
@@ -65,16 +104,29 @@ class OpsReleaseExecutorCoreTests(unittest.TestCase):
def run_command(command: list[str], *, timeout: int = 60):
commands.append(tuple(command))
if command[:5] == ["systemctl", "list-units", "--type=service", "--all", "domaincheck-worker@*"]:
return 0, (
"domaincheck-worker@a.service loaded active running domainCheck Worker Instance a\n"
"domaincheck-worker@ah.service loaded inactive dead domainCheck Worker Instance ah\n"
), ""
if command[:3] == ["systemctl", "show", "domaincheck-worker"] and "-p" in command:
current_user = pwd.getpwuid(os.getuid()).pw_name
current_group = grp.getgrgid(os.getgid()).gr_name
return 0, f"{current_user}\n{current_group}\n", ""
if command[:3] == ["systemctl", "show", "domaincheck-worker@a.service"] and "-p" in command:
current_user = pwd.getpwuid(os.getuid()).pw_name
current_group = grp.getgrgid(os.getgid()).gr_name
return 0, f"{current_user}\n{current_group}\n", ""
if command[:3] == ["systemctl", "show", "domaincheck-worker@ah.service"] and "-p" in command:
current_user = pwd.getpwuid(os.getuid()).pw_name
current_group = grp.getgrgid(os.getgid()).gr_name
return 0, f"{current_user}\n{current_group}\n", ""
if command[:2] == ["chown", "-R"]:
return 0, "", ""
if command[:2] == ["systemctl", "restart"]:
return 0, "", ""
if command[:2] == ["systemctl", "is-active"]:
return 0, "active", ""
return 0, "\n".join("active" for _ in command[2:]), ""
return 0, "", ""
with tempfile.TemporaryDirectory() as tmpdir:
@@ -108,8 +160,13 @@ class OpsReleaseExecutorCoreTests(unittest.TestCase):
self.assertTrue(any(cmd[:2] == ("chown", "-R") for cmd in commands))
self.assertIn(("systemctl", "daemon-reload"), commands)
self.assertIn(("systemctl", "restart", "domaincheck-worker"), commands)
self.assertIn(("systemctl", "restart", "domaincheck-worker@a.service"), commands)
self.assertIn(("systemctl", "restart", "domaincheck-worker@ah.service"), commands)
self.assertIn(("systemctl", "is-active", "domaincheck-worker"), commands)
self.assertIn(("systemctl", "is-active", "domaincheck-worker@a.service"), commands)
self.assertIn(("systemctl", "is-active", "domaincheck-worker@ah.service"), commands)
self.assertTrue((systemd_root / "domaincheck-node-agent.service").exists())
self.assertTrue((systemd_root / "domaincheck-worker@.service").exists())
self.assertTrue((systemd_root / "domaincheck-node-agent.service.d" / "current-path.conf").exists())
def test_build_remote_release_action_script_is_valid_python(self) -> None:
@@ -128,6 +185,9 @@ class OpsReleaseExecutorCoreTests(unittest.TestCase):
self.assertIn("False", script)
self.assertIn("def collect_service_identity(", script)
self.assertIn("def apply_release_permissions(", script)
self.assertIn("import grp", script)
self.assertIn("import os", script)
self.assertIn("import pwd", script)
if __name__ == "__main__":

View File

@@ -38,6 +38,97 @@ class _EmptyConnection:
class OpsReleaseServiceExecutionModeTests(unittest.TestCase):
@patch("app.services.ops_release_service.get_db")
@patch("app.services.ops_agent_service.ensure_ops_agent_schema")
@patch("app.services.ops_agent_service.get_managed_node_onboarding")
@patch("app.services.ops_agent_service.list_managed_nodes_with_agent_state")
@patch("app.services.ops_job_service.list_managed_nodes")
@patch("app.services.cluster_runtime_service.get_cluster_snapshot")
def test_build_rollout_target_operational_readiness_reuses_managed_node_snapshot_for_onboarding(
self,
mock_get_cluster_snapshot,
mock_list_managed_nodes,
mock_list_managed_nodes_with_agent_state,
mock_get_managed_node_onboarding,
mock_ensure_ops_agent_schema,
mock_get_db,
) -> None:
captured_nodes_payloads = []
def _fake_onboarding(node_code, **kwargs):
captured_nodes_payloads.append(kwargs.get("nodes_payload"))
return {
"onboarding_stage": {"code": "ready", "label": "已接管"},
"summary": f"{node_code} ready",
"recovery_decision": {
"action": "noop",
"label": "当前无需额外恢复动作",
"summary": "当前节点暂无需要执行的接管恢复动作。",
"command_hint": "",
"window": "none",
},
}
mock_ensure_ops_agent_schema.return_value = None
mock_get_db.return_value = _EmptyConnection()
mock_get_cluster_snapshot.return_value = {
"nodes": [
{
"node_code": "mainland-worker-01",
"region": "mainland",
"role": "worker",
"status": "online",
"current_load": 0,
"is_effective_worker": True,
},
{
"node_code": "mainland-worker-02",
"region": "mainland",
"role": "worker",
"status": "online",
"current_load": 0,
"is_effective_worker": True,
},
]
}
mock_list_managed_nodes.return_value = [
{
"node_code": "mainland-worker-01",
"region": "mainland",
"role": "worker",
"is_enabled": True,
"ssh_host": "121.204.244.248",
"ssh_user": "root",
"metadata": {},
"last_seen_at": "",
},
{
"node_code": "mainland-worker-02",
"region": "mainland",
"role": "worker",
"is_enabled": True,
"ssh_host": "121.204.244.249",
"ssh_user": "root",
"metadata": {},
"last_seen_at": "",
},
]
mock_list_managed_nodes_with_agent_state.return_value = {"nodes": [{"node_code": "mainland-worker-01"}, {"node_code": "mainland-worker-02"}]}
mock_get_managed_node_onboarding.side_effect = _fake_onboarding
readiness = build_rollout_target_operational_readiness(
[
{"node_code": "mainland-worker-01", "region": "mainland", "role": "worker"},
{"node_code": "mainland-worker-02", "region": "mainland", "role": "worker"},
],
execution_mode="remote-agent",
)
self.assertEqual(2, len(readiness["rows"]))
mock_list_managed_nodes_with_agent_state.assert_called_once()
self.assertEqual(2, len(captured_nodes_payloads))
self.assertTrue(all(payload == {"nodes": [{"node_code": "mainland-worker-01"}, {"node_code": "mainland-worker-02"}]} for payload in captured_nodes_payloads))
@patch("app.services.ops_release_service.get_db")
@patch("app.services.ops_agent_service.ensure_ops_agent_schema")
@patch("app.services.ops_agent_service.get_managed_node_onboarding")

View File

@@ -3,6 +3,8 @@ from unittest.mock import MagicMock, patch
import app.services.ops_agent_service as ops_agent_service
import app.services.ops_job_service as ops_job_service
import app.services.ops_release_service as ops_release_service
from psycopg2 import errors
class OpsSchemaInitTests(unittest.TestCase):
@@ -10,6 +12,7 @@ class OpsSchemaInitTests(unittest.TestCase):
def test_ensure_ops_schema_uses_advisory_lock_and_skips_after_ready(self, mock_get_db) -> None:
conn = MagicMock()
cursor = MagicMock()
cursor.fetchone.return_value = None
db_ctx = MagicMock()
cursor_ctx = MagicMock()
db_ctx.__enter__.return_value = conn
@@ -35,6 +38,94 @@ class OpsSchemaInitTests(unittest.TestCase):
cursor.execute.assert_any_call(ops_job_service._OPS_SCHEMA_SQL)
conn.commit.assert_called_once()
@patch("app.services.ops_job_service.get_db")
def test_ensure_ops_schema_skips_ddl_when_required_schema_already_exists(self, mock_get_db) -> None:
conn = MagicMock()
cursor = MagicMock()
cursor.fetchone.side_effect = [(f"public.{name}",) for name in ops_job_service._OPS_REQUIRED_TABLES]
cursor.fetchall.side_effect = [
[(column,) for column in ops_job_service._OPS_REQUIRED_COLUMNS["ops_jobs"]],
[(column,) for column in ops_job_service._OPS_REQUIRED_COLUMNS["ops_job_steps"]],
]
db_ctx = MagicMock()
cursor_ctx = MagicMock()
db_ctx.__enter__.return_value = conn
db_ctx.__exit__.return_value = False
cursor_ctx.__enter__.return_value = cursor
cursor_ctx.__exit__.return_value = False
conn.cursor.return_value = cursor_ctx
mock_get_db.return_value = db_ctx
previous_ready = ops_job_service._OPS_SCHEMA_READY
ops_job_service._OPS_SCHEMA_READY = False
try:
ops_job_service.ensure_ops_schema()
finally:
ops_job_service._OPS_SCHEMA_READY = previous_ready
self.assertFalse(any(call.args[0] == ops_job_service._OPS_SCHEMA_SQL for call in cursor.execute.call_args_list))
conn.commit.assert_not_called()
@patch("app.services.ops_job_service.get_db")
def test_ensure_ops_schema_accepts_deadlock_when_required_schema_already_exists(self, mock_get_db) -> None:
class _Cursor:
def __init__(self, *, raise_on_schema=False, fetchone_values=None, fetchall_values=None) -> None:
self.raise_on_schema = raise_on_schema
self.fetchone_values = list(fetchone_values or [])
self.fetchall_values = list(fetchall_values or [])
def execute(self, sql, params=None):
if self.raise_on_schema and sql == ops_job_service._OPS_SCHEMA_SQL:
raise errors.DeadlockDetected()
def fetchone(self):
if self.fetchone_values:
return self.fetchone_values.pop(0)
return None
def fetchall(self):
if self.fetchall_values:
return self.fetchall_values.pop(0)
return []
class _CursorContext:
def __init__(self, cursor) -> None:
self.cursor = cursor
def __enter__(self):
return self.cursor
def __exit__(self, exc_type, exc, tb):
return False
conn = MagicMock()
conn.cursor.side_effect = [
_CursorContext(_Cursor(fetchone_values=[None])),
_CursorContext(_Cursor(raise_on_schema=True)),
_CursorContext(
_Cursor(
fetchone_values=[(f"public.{name}",) for name in ops_job_service._OPS_REQUIRED_TABLES],
fetchall_values=[
[(column,) for column in ops_job_service._OPS_REQUIRED_COLUMNS["ops_jobs"]],
[(column,) for column in ops_job_service._OPS_REQUIRED_COLUMNS["ops_job_steps"]],
],
)
),
]
db_ctx = MagicMock()
db_ctx.__enter__.return_value = conn
mock_get_db.return_value = db_ctx
previous_ready = ops_job_service._OPS_SCHEMA_READY
ops_job_service._OPS_SCHEMA_READY = False
try:
ops_job_service.ensure_ops_schema()
finally:
ops_job_service._OPS_SCHEMA_READY = previous_ready
conn.rollback.assert_called_once()
conn.commit.assert_not_called()
@patch("app.services.ops_agent_service.ensure_ops_schema")
@patch("app.services.ops_agent_service.get_db")
def test_ensure_ops_agent_schema_uses_advisory_lock_and_skips_after_ready(
@@ -44,6 +135,7 @@ class OpsSchemaInitTests(unittest.TestCase):
) -> None:
conn = MagicMock()
cursor = MagicMock()
cursor.fetchone.return_value = None
db_ctx = MagicMock()
cursor_ctx = MagicMock()
db_ctx.__enter__.return_value = conn
@@ -70,6 +162,224 @@ class OpsSchemaInitTests(unittest.TestCase):
cursor.execute.assert_any_call(ops_agent_service._AGENT_SCHEMA_SQL)
conn.commit.assert_called_once()
@patch("app.services.ops_agent_service.ensure_ops_schema")
@patch("app.services.ops_agent_service.get_db")
def test_ensure_ops_agent_schema_skips_ddl_when_required_schema_already_exists(
self,
mock_get_db,
mock_ensure_ops_schema,
) -> None:
conn = MagicMock()
cursor = MagicMock()
cursor.fetchone.side_effect = [(f"public.{name}",) for name in ops_agent_service._OPS_AGENT_REQUIRED_TABLES]
cursor.fetchall.side_effect = [
[(column,) for column in ops_agent_service._OPS_AGENT_REQUIRED_COLUMNS["ops_node_tokens"]],
[(column,) for column in ops_agent_service._OPS_AGENT_REQUIRED_COLUMNS["ops_job_events"]],
[(column,) for column in ops_agent_service._OPS_AGENT_REQUIRED_COLUMNS["ops_jobs"]],
]
db_ctx = MagicMock()
cursor_ctx = MagicMock()
db_ctx.__enter__.return_value = conn
db_ctx.__exit__.return_value = False
cursor_ctx.__enter__.return_value = cursor
cursor_ctx.__exit__.return_value = False
conn.cursor.return_value = cursor_ctx
mock_get_db.return_value = db_ctx
previous_ready = ops_agent_service._OPS_AGENT_SCHEMA_READY
ops_agent_service._OPS_AGENT_SCHEMA_READY = False
try:
ops_agent_service.ensure_ops_agent_schema()
finally:
ops_agent_service._OPS_AGENT_SCHEMA_READY = previous_ready
self.assertFalse(any(call.args[0] == ops_agent_service._AGENT_SCHEMA_SQL for call in cursor.execute.call_args_list))
conn.commit.assert_not_called()
self.assertEqual(1, mock_ensure_ops_schema.call_count)
@patch("app.services.ops_agent_service.ensure_ops_schema")
@patch("app.services.ops_agent_service.get_db")
def test_ensure_ops_agent_schema_accepts_deadlock_when_required_schema_already_exists(
self,
mock_get_db,
mock_ensure_ops_schema,
) -> None:
class _Cursor:
def __init__(self, *, raise_on_schema=False, fetchone_values=None, fetchall_values=None) -> None:
self.raise_on_schema = raise_on_schema
self.fetchone_values = list(fetchone_values or [])
self.fetchall_values = list(fetchall_values or [])
def execute(self, sql, params=None):
if self.raise_on_schema and sql == ops_agent_service._AGENT_SCHEMA_SQL:
raise errors.DeadlockDetected()
def fetchone(self):
if self.fetchone_values:
return self.fetchone_values.pop(0)
return None
def fetchall(self):
if self.fetchall_values:
return self.fetchall_values.pop(0)
return []
class _CursorContext:
def __init__(self, cursor) -> None:
self.cursor = cursor
def __enter__(self):
return self.cursor
def __exit__(self, exc_type, exc, tb):
return False
conn = MagicMock()
conn.cursor.side_effect = [
_CursorContext(_Cursor(fetchone_values=[None, None])),
_CursorContext(_Cursor(raise_on_schema=True)),
_CursorContext(
_Cursor(
fetchone_values=[(f"public.{name}",) for name in ops_agent_service._OPS_AGENT_REQUIRED_TABLES],
fetchall_values=[
[(column,) for column in ops_agent_service._OPS_AGENT_REQUIRED_COLUMNS["ops_node_tokens"]],
[(column,) for column in ops_agent_service._OPS_AGENT_REQUIRED_COLUMNS["ops_job_events"]],
[(column,) for column in ops_agent_service._OPS_AGENT_REQUIRED_COLUMNS["ops_jobs"]],
],
)
),
]
db_ctx = MagicMock()
db_ctx.__enter__.return_value = conn
mock_get_db.return_value = db_ctx
previous_ready = ops_agent_service._OPS_AGENT_SCHEMA_READY
ops_agent_service._OPS_AGENT_SCHEMA_READY = False
try:
ops_agent_service.ensure_ops_agent_schema()
finally:
ops_agent_service._OPS_AGENT_SCHEMA_READY = previous_ready
conn.rollback.assert_called_once()
conn.commit.assert_not_called()
self.assertEqual(1, mock_ensure_ops_schema.call_count)
@patch("app.services.ops_release_service.get_db")
def test_ensure_ops_release_schema_uses_advisory_lock_and_skips_after_ready(self, mock_get_db) -> None:
conn = MagicMock()
cursor = MagicMock()
cursor.fetchone.return_value = None
db_ctx = MagicMock()
cursor_ctx = MagicMock()
db_ctx.__enter__.return_value = conn
db_ctx.__exit__.return_value = False
cursor_ctx.__enter__.return_value = cursor
cursor_ctx.__exit__.return_value = False
conn.cursor.return_value = cursor_ctx
mock_get_db.return_value = db_ctx
previous_ready = ops_release_service._RELEASE_SCHEMA_READY
ops_release_service._RELEASE_SCHEMA_READY = False
try:
ops_release_service.ensure_ops_release_schema()
ops_release_service.ensure_ops_release_schema()
finally:
ops_release_service._RELEASE_SCHEMA_READY = previous_ready
self.assertEqual(1, mock_get_db.call_count)
cursor.execute.assert_any_call(
"SELECT pg_advisory_xact_lock(%s)",
(ops_release_service._RELEASE_SCHEMA_ADVISORY_LOCK_KEY,),
)
cursor.execute.assert_any_call(ops_release_service._RELEASE_SCHEMA_SQL)
conn.commit.assert_called_once()
@patch("app.services.ops_release_service.get_db")
def test_ensure_ops_release_schema_skips_ddl_when_required_schema_already_exists(self, mock_get_db) -> None:
conn = MagicMock()
cursor = MagicMock()
cursor.fetchone.side_effect = [(f"public.{name}",) for name in ops_release_service._RELEASE_REQUIRED_TABLES]
cursor.fetchall.side_effect = [
[(column,) for column in ops_release_service._RELEASE_REQUIRED_COLUMNS["ops_release_rollouts"]],
]
db_ctx = MagicMock()
cursor_ctx = MagicMock()
db_ctx.__enter__.return_value = conn
db_ctx.__exit__.return_value = False
cursor_ctx.__enter__.return_value = cursor
cursor_ctx.__exit__.return_value = False
conn.cursor.return_value = cursor_ctx
mock_get_db.return_value = db_ctx
previous_ready = ops_release_service._RELEASE_SCHEMA_READY
ops_release_service._RELEASE_SCHEMA_READY = False
try:
ops_release_service.ensure_ops_release_schema()
finally:
ops_release_service._RELEASE_SCHEMA_READY = previous_ready
self.assertFalse(any(call.args[0] == ops_release_service._RELEASE_SCHEMA_SQL for call in cursor.execute.call_args_list))
conn.commit.assert_not_called()
@patch("app.services.ops_release_service.get_db")
def test_ensure_ops_release_schema_accepts_deadlock_when_required_schema_already_exists(self, mock_get_db) -> None:
class _Cursor:
def __init__(self, *, raise_on_schema=False, fetchone_values=None, fetchall_values=None) -> None:
self.raise_on_schema = raise_on_schema
self.fetchone_values = list(fetchone_values or [])
self.fetchall_values = list(fetchall_values or [])
def execute(self, sql, params=None):
if self.raise_on_schema and sql == ops_release_service._RELEASE_SCHEMA_SQL:
raise errors.DeadlockDetected()
def fetchone(self):
if self.fetchone_values:
return self.fetchone_values.pop(0)
return None
def fetchall(self):
if self.fetchall_values:
return self.fetchall_values.pop(0)
return []
class _CursorContext:
def __init__(self, cursor) -> None:
self.cursor = cursor
def __enter__(self):
return self.cursor
def __exit__(self, exc_type, exc, tb):
return False
conn = MagicMock()
conn.cursor.side_effect = [
_CursorContext(_Cursor(fetchone_values=[None])),
_CursorContext(_Cursor(raise_on_schema=True)),
_CursorContext(
_Cursor(
fetchone_values=[(f"public.{name}",) for name in ops_release_service._RELEASE_REQUIRED_TABLES],
fetchall_values=[
[(column,) for column in ops_release_service._RELEASE_REQUIRED_COLUMNS["ops_release_rollouts"]],
],
)
),
]
db_ctx = MagicMock()
db_ctx.__enter__.return_value = conn
mock_get_db.return_value = db_ctx
previous_ready = ops_release_service._RELEASE_SCHEMA_READY
ops_release_service._RELEASE_SCHEMA_READY = False
try:
ops_release_service.ensure_ops_release_schema()
finally:
ops_release_service._RELEASE_SCHEMA_READY = previous_ready
conn.rollback.assert_called_once()
conn.commit.assert_not_called()
if __name__ == "__main__":
unittest.main()

View File

@@ -10,6 +10,35 @@ class OpsServiceActivityTests(unittest.TestCase):
def _runtime_status(detect: Optional[dict] = None) -> dict:
return {"detect": dict(detect or {})}
@patch("app.services.ops_service.get_ops_runbook", return_value={"control_sequences": []})
@patch("app.services.ops_service.list_release_rollouts", return_value=[])
@patch("app.services.ops_service.list_ops_jobs", return_value=[])
@patch("app.services.ops_service.get_recent_ops_playbook_runs", return_value={"runs": []})
@patch("app.services.ops_service.list_managed_nodes_with_agent_state", side_effect=AssertionError("should reuse managed nodes"))
@patch("app.services.ops_service.get_runtime_status", side_effect=AssertionError("should reuse runtime status"))
def test_activity_stream_reuses_provided_runtime_and_managed_snapshots(
self,
_mock_get_runtime_status,
_mock_list_managed_nodes_with_agent_state,
_mock_get_recent_ops_playbook_runs,
_mock_list_ops_jobs,
_mock_list_release_rollouts,
mock_get_ops_runbook,
) -> None:
runtime_snapshot = self._runtime_status()
managed_snapshot = {"nodes": []}
payload = get_ops_activity_stream(
limit=10,
scan_limit=20,
runtime_status=runtime_snapshot,
managed_nodes_payload=managed_snapshot,
)
self.assertEqual([], payload["items"])
self.assertEqual(0, payload["summary"]["total"])
self.assertEqual(runtime_snapshot, mock_get_ops_runbook.call_args.kwargs["runtime_status"])
self.assertEqual(managed_snapshot, mock_get_ops_runbook.call_args.kwargs["managed_nodes_payload"])
@patch("app.services.ops_service.get_ops_runbook")
@patch("app.services.ops_service.list_release_rollouts")
@patch("app.services.ops_service.list_ops_jobs")

View File

@@ -0,0 +1,27 @@
import unittest
from unittest.mock import patch
from app.core.redis_client import get_redis, reset_redis_client_for_tests
class ApiRedisClientTests(unittest.TestCase):
def tearDown(self) -> None:
reset_redis_client_for_tests()
@patch("app.core.redis_client.redis.Redis")
@patch("app.core.redis_client.redis.BlockingConnectionPool")
def test_get_redis_reuses_singleton_client(self, mock_pool, mock_redis) -> None:
singleton = object()
mock_redis.return_value = singleton
client_a = get_redis()
client_b = get_redis()
self.assertIs(client_a, singleton)
self.assertIs(client_b, singleton)
mock_pool.assert_called_once()
mock_redis.assert_called_once()
if __name__ == "__main__":
unittest.main()

View File

@@ -0,0 +1,93 @@
from __future__ import annotations
import unittest
from unittest.mock import patch
from app.api.routes import runtime as runtime_route
class RuntimeApiRoutesTests(unittest.TestCase):
@patch("app.api.routes.runtime.list_debug_events")
def test_runtime_debug_events_forwards_node_code_filter(self, mock_list_debug_events) -> None:
mock_list_debug_events.return_value = {"records": [], "has_more": False}
response = runtime_route.runtime_debug_events(node_code="mainland-controller-01-a", limit=20)
self.assertEqual(0, response.code)
mock_list_debug_events.assert_called_once_with(
limit=20,
service=None,
event_type=None,
source_region=None,
node_code="mainland-controller-01-a",
level=None,
before_id=None,
after_id=None,
created_after=None,
)
@patch("app.api.routes.runtime.get_runtime_status")
@patch("app.api.routes.runtime.get_sync_summary")
@patch("app.api.routes.runtime.get_debug_handoff_report")
def test_runtime_health_handover_filters_recent_issues_and_issue_groups_by_node_code(
self,
mock_get_debug_handoff_report,
mock_get_sync_summary,
mock_get_runtime_status,
) -> None:
mock_get_sync_summary.return_value = {"latest_record": {}}
mock_get_runtime_status.return_value = {"readiness": {"status": "ready"}}
mock_get_debug_handoff_report.return_value = {
"overview": {
"recent_issues": [
{"node_code": "mainland-controller-01", "message": "keep"},
{"node_code": "mainland-worker-01", "message": "drop"},
]
},
"recent_issues": [
{"node_code": "mainland-controller-01", "message": "keep"},
{"node_code": "mainland-worker-01", "message": "drop"},
],
"issue_groups": [
{"node_code": "mainland-controller-01", "message": "keep"},
{"node_code": "mainland-worker-01", "message": "drop"},
],
"failure_handoff": {
"recent_issues": [
{"node_code": "mainland-controller-01", "message": "keep"},
{"node_code": "mainland-worker-01", "message": "drop"},
],
"issue_groups": [
{"node_code": "mainland-controller-01", "message": "keep"},
{"node_code": "mainland-worker-01", "message": "drop"},
],
},
}
response = runtime_route.runtime_health_handover(node_code="mainland-controller-01")
self.assertEqual(0, response.code)
self.assertEqual(
[{"node_code": "mainland-controller-01", "message": "keep"}],
response.data["recent_issues"],
)
self.assertEqual(
[{"node_code": "mainland-controller-01", "message": "keep"}],
response.data["issue_groups"],
)
self.assertEqual(
[{"node_code": "mainland-controller-01", "message": "keep"}],
response.data["overview"]["recent_issues"],
)
self.assertEqual(
[{"node_code": "mainland-controller-01", "message": "keep"}],
response.data["failure_handoff"]["recent_issues"],
)
self.assertEqual(
[{"node_code": "mainland-controller-01", "message": "keep"}],
response.data["failure_handoff"]["issue_groups"],
)
if __name__ == "__main__":
unittest.main()

View File

@@ -0,0 +1,37 @@
from __future__ import annotations
import unittest
from unittest.mock import patch
from app.services.runtime_control_service import runtime_action
class RuntimeControlServiceTests(unittest.TestCase):
@patch("app.services.runtime_control_service._emit_runtime_action_event")
@patch("app.services.runtime_control_service.send_worker_command")
def test_runtime_action_stop_detection_forwards_target_payload(
self,
mock_send_worker_command,
_mock_emit_runtime_action_event,
) -> None:
mock_send_worker_command.return_value = (True, "已发送 Worker 控制指令")
ok, message, data = runtime_action(
"stop_detection",
payload={"target_node_codes": ["mainland-controller-01-a", "mainland-controller-01-b"]},
)
self.assertTrue(ok)
self.assertEqual("已发送 Worker 控制指令", message)
mock_send_worker_command.assert_called_once_with(
"stop_detection",
payload={"target_node_codes": ["mainland-controller-01-a", "mainland-controller-01-b"]},
)
self.assertEqual(
["mainland-controller-01-a", "mainland-controller-01-b"],
data["payload"]["target_node_codes"],
)
if __name__ == "__main__":
unittest.main()

View File

@@ -0,0 +1,77 @@
from __future__ import annotations
import json
import unittest
from unittest.mock import Mock, patch
from app.services.runtime_settings_service import update_runtime_settings
class RuntimeSettingsServiceTests(unittest.TestCase):
@patch("app.services.runtime_settings_service.get_redis")
@patch("app.services.runtime_settings_service.write_runtime_json")
@patch("app.services.runtime_settings_service.get_runtime_settings")
def test_update_runtime_settings_publishes_runtime_settings_refresh(
self,
mock_get_runtime_settings,
mock_write_runtime_json,
mock_get_redis,
) -> None:
redis_client = Mock()
mock_get_redis.return_value = redis_client
mock_get_runtime_settings.return_value = {
"worker_mode": "linux-systemd",
"worker_service_name": "domaincheck-worker",
"api_service_name": "domaincheck-api",
"sync_agent_service_name": "domaincheck-sync-agent",
"worker_log_sync_enabled": False,
"worker_log_sync_mode": "key",
"control_node_autoresume_enabled": False,
"claim_recent_jobs_first": False,
"claim_recent_jobs_limit": 8,
"claim_recent_jobs_window_hours": 24,
"claim_batch_floor": 0,
"claim_batch_ceil": 0,
"submit_backlog_floor": 0,
"submit_backlog_ceil": 0,
"dispatch_cap_multiplier": 1,
"pending_buffer_cap_multiplier": 1,
}
updated = update_runtime_settings(
{
"worker_log_sync_enabled": True,
"worker_log_sync_mode": "full",
"control_node_autoresume_enabled": True,
"claim_recent_jobs_first": True,
"claim_recent_jobs_limit": 6,
"claim_recent_jobs_window_hours": 72,
"claim_batch_floor": 200,
"claim_batch_ceil": 800,
"submit_backlog_floor": 500,
"submit_backlog_ceil": 1500,
"dispatch_cap_multiplier": 2,
"pending_buffer_cap_multiplier": 2,
}
)
mock_write_runtime_json.assert_called_once_with("runtime_settings.json", updated)
redis_client.set.assert_called_once()
redis_key, serialized = redis_client.set.call_args.args
self.assertEqual("domain_tool:runtime_settings", redis_key)
self.assertEqual(updated, json.loads(serialized))
self.assertTrue(updated["control_node_autoresume_enabled"])
self.assertTrue(updated["claim_recent_jobs_first"])
self.assertEqual(6, updated["claim_recent_jobs_limit"])
self.assertEqual(72, updated["claim_recent_jobs_window_hours"])
self.assertEqual(200, updated["claim_batch_floor"])
self.assertEqual(800, updated["claim_batch_ceil"])
self.assertEqual(500, updated["submit_backlog_floor"])
self.assertEqual(1500, updated["submit_backlog_ceil"])
self.assertEqual(2, updated["dispatch_cap_multiplier"])
self.assertEqual(2, updated["pending_buffer_cap_multiplier"])
redis_client.publish.assert_called_once_with("domain_tool:config_update", "runtime_settings")
if __name__ == "__main__":
unittest.main()

View File

@@ -1,10 +1,273 @@
import unittest
from datetime import datetime, timedelta
from unittest.mock import MagicMock, patch
import app.services.runtime_status_service as runtime_status_service_module
from app.services.detect_service import _slice_remote_log_lines_fairly
from app.services.runtime_status_service import _align_queue_health_with_backlog, _build_detect_node_row
from app.services.runtime_status_service import (
_align_queue_health_with_backlog,
_build_detect_node_row,
_build_detect_observation_summary,
refresh_runtime_projection_snapshot,
get_runtime_status,
)
class RuntimeStatusServiceTests(unittest.TestCase):
def setUp(self) -> None:
runtime_status_service_module._DOMAIN_INVENTORY_CACHE_VALUE = None
runtime_status_service_module._DOMAIN_INVENTORY_CACHE_EXPIRES_AT = 0.0
runtime_status_service_module._RUNTIME_STATUS_CACHE_VALUE = None
runtime_status_service_module._RUNTIME_STATUS_CACHE_EXPIRES_AT = 0.0
runtime_status_service_module._RUNTIME_STATUS_CACHE_SIGNATURE = ()
def test_build_detect_observation_summary_marks_real_running_state(self) -> None:
summary = _build_detect_observation_summary(
detect_payload={
"active_job": {
"job_id": 308,
"job_code": "sync-overseas-16432",
"status": "running",
"progress_percent": 24.6,
"items_pending": 3999,
"items_claimed": 52,
"items_running": 136,
"items_completed": 971,
"items_failed": 0,
"items_blacklisted": 0,
"display_items_running": 222,
"display_active_threads": 222,
"display_max_threads": 2000,
"processed_recent": 172,
"processed_per_minute": 11.47,
"completed_recent": 2,
"failed_recent": 169,
"blacklisted_recent": 1,
},
"active_thread_count": 222,
"max_thread_count": 2000,
"queue_health": {
"queue": {
"pending": 3999,
"claimed": 52,
"running": 136,
"display_running": 222,
"completed": 971,
"failed": 0,
"blacklisted": 0,
},
"throughput": {
"processed_recent": 172,
"processed_per_minute": 11.47,
"completed_recent": 2,
"failed_recent": 169,
"blacklisted_recent": 1,
},
"steps": [
{
"step_code": "detect_register",
"step_name": "注册检测",
"items_pending": 2100,
"items_running": 40,
"processed_recent": 88,
"failed_recent": 100,
"blacklisted_recent": 0,
}
],
},
"participating_nodes": [
{
"node_code": "mainland-controller-01-a",
"participation_label": "执行中",
"is_dispatch_active": True,
"items_claimed": 12,
"items_running": 100,
"active_threads": 100,
"max_threads": 1000,
"processed_recent": 80,
"processed_per_minute": 5.33,
},
{
"node_code": "mainland-controller-01-b",
"participation_label": "执行中",
"is_dispatch_active": True,
"items_claimed": 8,
"items_running": 122,
"active_threads": 122,
"max_threads": 1000,
"processed_recent": 92,
"processed_per_minute": 6.13,
},
],
"participation_summary": {
"dispatch_active_nodes": 2,
"participating_nodes": 2,
},
"backlog": {
"pending_total": 3999,
"claimed_total": 52,
"running_total": 136,
"completed_total": 971,
},
},
cluster_snapshot={"summary": {"online_worker_nodes": 1}},
inventory_summary={
"scope_label": "海外主库总盘子",
"authoritative": True,
"domains_total": 5690937,
"pending_total": 5690918,
"completed_total": 0,
"running_total": 7,
"blacklist_total": 0,
"failed_total": 12,
"processed_total": 12,
"remaining_total": 5690925,
},
)
self.assertEqual("running", summary["state"])
self.assertEqual("真跑中", summary["state_label"])
self.assertEqual(2, summary["execution"]["active_processes"])
self.assertEqual(222, summary["execution"]["active_threads"])
self.assertEqual(169, summary["throughput"]["failed_recent"])
self.assertEqual("detect_register", summary["top_steps"][0]["step_code"])
self.assertEqual(5690937, summary["source_inventory"]["domains_total"])
self.assertEqual(5244, summary["active_batch"]["effective_items_total"])
self.assertIn("不等于全盘累计", summary["scope_hint"])
def test_build_detect_observation_summary_marks_not_running_when_only_backlog_left(self) -> None:
summary = _build_detect_observation_summary(
detect_payload={
"active_job": {
"job_id": 401,
"job_code": "sync-overseas-404",
"status": "running",
"items_pending": 5000,
"items_claimed": 0,
"items_running": 0,
"items_completed": 0,
"items_failed": 0,
"items_blacklisted": 0,
},
"queue_health": {
"queue": {
"pending": 5000,
"claimed": 0,
"running": 0,
"completed": 0,
"failed": 0,
"blacklisted": 0,
},
"throughput": {
"processed_recent": 0,
"processed_per_minute": 0,
"completed_recent": 0,
"failed_recent": 0,
"blacklisted_recent": 0,
},
"steps": [
{
"step_code": "detect_register",
"step_name": "注册检测",
"items_pending": 5000,
"items_running": 0,
}
],
},
"participating_nodes": [],
"participation_summary": {
"dispatch_active_nodes": 0,
"participating_nodes": 0,
},
"backlog": {
"pending_total": 5000,
"claimed_total": 0,
"running_total": 0,
},
},
cluster_snapshot={"summary": {"online_worker_nodes": 1}},
inventory_summary={
"scope_label": "海外主库总盘子",
"authoritative": True,
"domains_total": 5690937,
"pending_total": 5690918,
"completed_total": 0,
"running_total": 7,
"blacklist_total": 0,
"failed_total": 12,
"processed_total": 12,
"remaining_total": 5690925,
},
)
self.assertEqual("not_running", summary["state"])
self.assertEqual("没跑起来", summary["state_label"])
self.assertEqual(0, summary["execution"]["active_processes"])
self.assertIn("队列", summary["focus_hint"])
self.assertEqual(5000, summary["active_batch"]["effective_items_total"])
self.assertEqual(5690937, summary["source_inventory"]["domains_total"])
@patch("app.services.runtime_status_service.get_active_detect_job_summary")
@patch("app.services.runtime_status_service.get_db")
def test_load_detect_backlog_snapshot_prefers_active_job_snapshot(
self,
mock_get_db,
mock_get_active_detect_job_summary,
) -> None:
mock_get_active_detect_job_summary.return_value = {
"items_pending": 321,
"items_claimed": 2,
"items_running": 9,
"items_completed": 50,
"items_failed": 3,
"display_active_threads": 11,
"step_stats": [
{"step_code": "detect_register", "items_pending": 210},
{"step_code": "detect_baidu", "items_pending": 111},
],
}
backlog = runtime_status_service_module._load_detect_backlog_snapshot()
self.assertEqual(321, backlog["pending_total"])
self.assertEqual(2, backlog["claimed_total"])
self.assertEqual(11, backlog["running_total"])
self.assertEqual(50, backlog["completed_total"])
self.assertEqual(3, backlog["failed_total"])
self.assertEqual(210, backlog["register_pending"])
self.assertEqual(111, backlog["downstream_pending"])
mock_get_db.assert_not_called()
@patch("app.services.runtime_status_service.get_active_detect_job_summary", return_value={})
@patch("app.services.runtime_status_service.get_db")
def test_load_detect_backlog_snapshot_ignores_stale_pending_jobs(
self,
mock_get_db,
_mock_get_active_detect_job_summary,
) -> None:
conn = MagicMock()
cursor_cm = MagicMock()
cursor = MagicMock()
cursor.fetchall.return_value = [
(11, "running", datetime.now() - timedelta(hours=2)),
(10, "pending", datetime.now() - timedelta(minutes=30)),
(9, "pending", datetime.now() - timedelta(hours=8)),
]
cursor.fetchone.return_value = (321, 2, 9, 50, 1, 3, 210, 111)
conn.cursor.return_value = cursor_cm
cursor_cm.__enter__.return_value = cursor
db_cm = MagicMock()
db_cm.__enter__.return_value = conn
mock_get_db.return_value = db_cm
backlog = runtime_status_service_module._load_detect_backlog_snapshot()
self.assertEqual(321, backlog["pending_total"])
self.assertEqual(210, backlog["register_pending"])
executed_sql, executed_params = cursor.execute.call_args_list[1][0]
self.assertIn("WHERE item.job_id = ANY(%s)", executed_sql)
self.assertEqual([11, 10], list(executed_params[0]))
def test_build_detect_node_row_merges_queue_running_metrics(self) -> None:
row = _build_detect_node_row(
node_code="mainland-worker-01",
@@ -92,6 +355,42 @@ class RuntimeStatusServiceTests(unittest.TestCase):
self.assertTrue(any("[mainland-controller-01]" in line for line in sliced))
self.assertTrue(any("[mainland-worker-01]" in line for line in sliced))
def test_build_multi_region_readiness_skips_local_batch_warning_when_batches_are_not_applicable(self) -> None:
with patch("app.services.runtime_status_service.settings.node_region", "overseas"), \
patch("app.services.runtime_status_service.settings.node_role", "control"):
readiness = runtime_status_service_module._build_multi_region_readiness(
cluster_snapshot={
"nodes": [
{"node_code": "overseas-control-01", "region": "overseas", "role": "control", "status": "online"},
{"node_code": "mainland-controller-01", "region": "mainland", "role": "control", "status": "online"},
{"node_code": "mainland-worker-01", "region": "mainland", "role": "worker", "status": "busy", "is_effective_worker": True},
],
"summary": {
"online_control_nodes": 2,
"online_worker_nodes": 1,
},
},
sync_summary={
"enabled": False,
"source_region": "overseas",
"target_region": "mainland",
"detect_result_batches": {
"applicable": False,
"reason": "当前节点不承载本地检测执行,结果批次推送概览不适用。",
"state_counts": {"projected": 3, "failed": 1, "pushing": 1, "synced": 0},
},
},
worker_runtime={"running": False},
sync_agent_runtime={"running": False},
)
warning_text = " ".join(readiness["warnings"])
info_text = " ".join(readiness["info"])
self.assertNotIn("结果批次仍待推送", warning_text)
self.assertNotIn("结果批次同步失败", warning_text)
self.assertIn("不适用", info_text)
self.assertFalse(readiness["sync"]["applicable"])
def test_align_queue_health_with_backlog_prefers_larger_runtime_snapshot(self) -> None:
aligned = _align_queue_health_with_backlog(
{
@@ -126,6 +425,227 @@ class RuntimeStatusServiceTests(unittest.TestCase):
self.assertEqual(8, aligned["queue"]["failed"])
self.assertEqual(1790, aligned["queue"]["items_total"])
def test_load_latest_remote_runtime_projection_backlog_skips_future_dated_rows(self) -> None:
now = datetime.now()
conn = MagicMock()
cursor_cm = MagicMock()
cursor = MagicMock()
cursor.fetchall.return_value = [
({"projection": {"backlog": {"pending_total": 9999}}}, now + timedelta(hours=4)),
({"projection": {"backlog": {"pending_total": 321, "running_total": 8}}}, now - timedelta(minutes=2)),
]
conn.cursor.return_value = cursor_cm
cursor_cm.__enter__.return_value = cursor
db_cm = MagicMock()
db_cm.__enter__.return_value = conn
with patch("app.services.runtime_status_service.settings.node_region", "overseas"), \
patch("app.services.runtime_status_service.settings.node_role", "control"), \
patch("app.services.runtime_status_service.get_db", return_value=db_cm):
backlog = runtime_status_service_module._load_latest_remote_runtime_projection_backlog()
self.assertEqual(321, backlog["pending_total"])
self.assertEqual(8, backlog["running_total"])
def test_get_runtime_status_keeps_remote_aggregate_detect_metrics_on_overseas_control(self) -> None:
with patch("app.services.runtime_status_service.settings.node_region", "overseas"), \
patch("app.services.runtime_status_service.settings.node_role", "control"), \
patch("app.services.runtime_status_service.detect_worker_runtime", return_value={"running": False, "mode": "linux-systemd", "process_count": 0, "message": "inactive", "latest_start_time": ""}), \
patch("app.services.runtime_status_service.detect_sync_agent_runtime", return_value={"running": False, "mode": "linux-systemd", "process_count": 0, "message": "inactive", "latest_start_time": ""}), \
patch("app.services.runtime_status_service.get_runtime_settings", return_value={"worker_mode": "linux-systemd"}), \
patch("app.services.runtime_status_service.get_detect_status", return_value={
"phase_label": "",
"phase_detail": "",
"recent_event": "远端执行中",
"recent_warning": "",
"progress_percent": 40.49,
"progress": {"pending": 5001, "running": 138, "completed": 0, "failed": 0, "blacklisted": 0},
"active_thread_count": 138,
"max_thread_count": 80000,
"aggregate_process_count": 80,
"aggregate_participating_node_count": 1,
"aggregate_participating_node_codes": ["mainland-controller-01"],
"aggregate_max_thread_count": 80000,
"aggregate_thread_count_per_process": 1000,
"available_proxy_count": 0,
"proxy_pool_count": 0,
"proxy_runtime_label": "",
"proxy_runtime_detail": "",
"proxy_runtime_reason": "",
"proxy_supplier_empty": False,
"proxy_last_refresh_status": "",
"proxy_last_refresh_time": "",
"proxy_last_refresh_source_count": 0,
"proxy_last_refresh_total_items": 0,
"proxy_last_validated_count": 0,
"proxy_last_available_count": 0,
"proxy_source_stats": [],
"dependency_alerts": [],
"active_job": {"job_code": "sync-overseas-51", "items_pending": 5001, "items_running": 0},
"runs": [],
"remote_log_line_count": 0,
"remote_log_node_count": 0,
"remote_log_nodes": [],
"remote_log_node_summaries": [],
"remote_log_last_at": "",
"remote_log_last_line": "",
"remote_log_lines": [],
"aggregate_detect_view": True,
}), \
patch("app.services.runtime_status_service.get_cluster_snapshot", return_value={
"nodes": [{"node_code": "mainland-controller-01", "role": "control", "region": "mainland", "status": "busy", "is_effective_worker": True, "current_load": 1432, "metadata": {"active_threads": 138, "max_threads": 80000}}],
"summary": {"online_worker_nodes": 1, "dedicated_online_worker_nodes": 0, "online_control_nodes": 2},
}), \
patch("app.services.runtime_status_service.get_detect_queue_health", return_value={
"has_active_job": True,
"job": {"job_id": 11, "job_code": "sync-overseas-51", "status": "running", "progress_percent": 40.49},
"queue": {"items_total": 8402, "pending": 5001, "claimed": 0, "running": 1432, "display_running": 1432},
"nodes": [{"node_code": "mainland-controller-01-a", "items_running": 1432, "display_running": 1432, "active_threads": 1432, "max_threads": 1000}],
"steps": [],
"runtime_activity": {},
"window_minutes": 15,
}), \
patch("app.services.runtime_status_service._load_detect_backlog_snapshot", return_value={"pending_total": 5001, "claimed_total": 0, "running_total": 0, "completed_total": 0, "blacklisted_total": 0, "failed_total": 0, "register_pending": 3495, "downstream_pending": 1506}), \
patch("app.services.runtime_status_service._load_latest_remote_runtime_projection_backlog", return_value={}), \
patch("app.services.runtime_status_service._load_latest_runtime_active_job_snapshot", return_value={}), \
patch("app.services.runtime_status_service._load_domain_inventory_summary", return_value={"scope_label": "海外主库总盘子", "authoritative": True, "domains_total": 5690937, "pending_total": 5690918, "completed_total": 0, "running_total": 7, "blacklist_total": 0, "failed_total": 12, "processed_total": 12, "remaining_total": 5690925}), \
patch("app.services.runtime_status_service.get_detect_capacity_plan", return_value={}), \
patch("app.services.runtime_status_service.append_runtime_projection_if_changed"), \
patch("app.services.runtime_status_service.get_sync_summary", return_value={}), \
patch("app.services.runtime_status_service._build_multi_region_readiness", return_value={"status": "ready", "ready": True}), \
patch("app.services.runtime_status_service.get_runtime_build_info", return_value={}):
payload = get_runtime_status()
self.assertEqual(1, payload["detect"]["aggregate_process_count"])
self.assertEqual(1432, payload["detect"]["active_thread_count"])
self.assertEqual(1000, payload["detect"]["max_thread_count"])
self.assertEqual("集群执行中", payload["detect"]["phase_label"])
self.assertFalse(payload["detect"]["worker_online"])
self.assertEqual("sync-overseas-51", payload["detect"]["active_job"]["job_code"])
self.assertIn("observation_summary", payload["detect"])
self.assertEqual("在跑但偏慢", payload["detect"]["observation_summary"]["state_label"])
self.assertEqual(5690937, payload["detect"]["observation_summary"]["source_inventory"]["domains_total"])
def test_get_runtime_status_reuses_short_ttl_cache(self) -> None:
context_payload = {
"runtime_settings": {
"worker_mode": "linux-systemd",
"api_service_name": "domaincheck-api",
"worker_service_name": "domaincheck-worker",
"sync_agent_service_name": "domaincheck-sync-agent",
},
"worker_runtime": {
"running": True,
"mode": "linux-systemd",
"process_count": 60,
"latest_start_time": "2026-04-24 17:10:28",
"message": "running",
"runtime_state": {"phase": "detecting"},
},
"worker_expected_on_this_node": True,
"detect_payload": {
"_detect_snapshot": {"thread_count": 1000},
"active_thread_count": 321,
"max_thread_count": 60000,
"progress": {"pending": 10, "running": 3},
"backlog": {"pending_total": 10, "running_total": 3},
"progress_percent": 25.5,
"available_proxy_count": 12,
"proxy_pool_count": 20,
"proxy_runtime_label": "代理正常",
"proxy_runtime_detail": "ok",
"proxy_runtime_reason": "healthy",
"proxy_last_refresh_time": "2026-04-24 17:12:55",
"recent_event": "running",
"recent_warning": "",
},
"cluster_snapshot": {"summary": {"online_worker_nodes": 1}},
}
sync_summary = {"enabled": True}
readiness = {"status": "ready", "ready": True}
with patch("app.services.runtime_status_service._build_runtime_detect_context", return_value=context_payload) as mock_context, \
patch("app.services.runtime_status_service.detect_sync_agent_runtime", return_value={"running": True, "mode": "linux-systemd", "process_count": 1, "latest_start_time": "", "message": "ok"}), \
patch("app.services.runtime_status_service.append_runtime_projection_if_changed") as mock_append, \
patch("app.services.runtime_status_service.get_sync_summary", return_value=sync_summary) as mock_sync_summary, \
patch("app.services.runtime_status_service._build_multi_region_readiness", return_value=readiness), \
patch("app.services.runtime_status_service.get_runtime_build_info", return_value={"package_name": "pkg"}), \
patch("app.services.runtime_status_service.time.monotonic", side_effect=[100.0, 100.1, 100.2]):
first = get_runtime_status()
second = get_runtime_status()
self.assertEqual(first, second)
self.assertEqual(1, mock_context.call_count)
self.assertEqual(1, mock_append.call_count)
self.assertEqual(1, mock_sync_summary.call_count)
@patch("app.services.runtime_status_service.append_runtime_projection_if_changed", return_value=321)
@patch(
"app.services.runtime_status_service.get_cluster_snapshot",
return_value={"nodes": [], "summary": {"online_worker_nodes": 3}},
)
@patch(
"app.services.runtime_status_service.get_active_detect_job_summary",
return_value={
"job_id": 867,
"job_code": "sync-overseas-19835",
"status": "running",
"progress_percent": 99.1,
"items_total": 10000,
"items_pending": 4,
"items_claimed": 0,
"items_running": 5,
"items_completed": 55402,
"items_blacklisted": 0,
"items_failed": 0,
"display_items_running": 5,
"display_active_threads": 5,
"display_items_claimed": 0,
"display_max_threads": 1000,
"node_stats": [{"node_code": "mainland-controller-01", "items_running": 5}],
"raw_step_stats": [{"step_code": "detect_wayback", "items_pending": 4}],
},
)
@patch(
"app.services.runtime_status_service.detect_worker_runtime",
return_value={"running": True, "mode": "linux-systemd", "process_count": 60, "thread_count": 1000, "max_threads": 60000, "message": "ok"},
)
@patch("app.services.runtime_status_service.get_runtime_settings", return_value={"thread_count": 1000, "worker_mode": "linux-systemd"})
@patch(
"app.services.runtime_status_service._load_detect_backlog_snapshot",
return_value={
"pending_total": 4,
"claimed_total": 0,
"running_total": 5,
"completed_total": 55402,
"blacklisted_total": 0,
"failed_total": 0,
"register_pending": 0,
"downstream_pending": 4,
},
)
@patch("app.services.runtime_status_service._build_runtime_detect_context", side_effect=AssertionError("heavy path should not run"))
def test_refresh_runtime_projection_snapshot_uses_lightweight_context(
self,
mock_heavy_context,
mock_load_detect_backlog_snapshot,
mock_get_runtime_settings,
mock_detect_worker_runtime,
mock_get_active_detect_job_summary,
mock_get_cluster_snapshot,
mock_append_runtime_projection_if_changed,
) -> None:
result = refresh_runtime_projection_snapshot()
self.assertEqual(321, result["record_id"])
mock_append_runtime_projection_if_changed.assert_called_once()
detect = mock_append_runtime_projection_if_changed.call_args.kwargs["detect"]
self.assertEqual("sync-overseas-19835", detect["active_job"]["job_code"])
self.assertEqual(5, detect["active_thread_count"])
self.assertEqual(4, detect["backlog"]["pending_total"])
self.assertEqual(5, detect["queue_health"]["queue"]["running"])
mock_heavy_context.assert_not_called()
if __name__ == "__main__":
unittest.main()

View File

@@ -1,6 +1,11 @@
import unittest
from app.services.settings_service import _normalize_thread_count, resolve_thread_count
from app.services.settings_service import (
_normalize_process_count,
_normalize_thread_count,
resolve_process_count,
resolve_thread_count,
)
class SettingsServiceTests(unittest.TestCase):
@@ -26,6 +31,25 @@ class SettingsServiceTests(unittest.TestCase):
self.assertEqual(512, resolved["effective_thread_count"])
self.assertEqual("node_override", resolved["source"])
def test_normalize_process_count_rejects_non_positive_values(self) -> None:
with self.assertRaises(ValueError):
_normalize_process_count(0)
def test_resolve_process_count_uses_large_node_override(self) -> None:
payload = {
"process_count": 80,
"node_process_counts": {
"mainland-controller-01": 96,
},
}
resolved = resolve_process_count(node_code="mainland-controller-01", settings_payload=payload)
self.assertEqual(80, resolved["default_process_count"])
self.assertEqual(96, resolved["override_process_count"])
self.assertEqual(96, resolved["effective_process_count"])
self.assertEqual("node_override", resolved["source"])
if __name__ == "__main__":
unittest.main()

View File

@@ -4,6 +4,9 @@ from unittest.mock import patch
from app.sync_agent import (
_append_detect_result_projection_snapshot,
_build_aligned_queue_health_snapshot,
_emit_runtime_debug_snapshots,
_maybe_trigger_overlap_start,
_run_sync_tick_once,
_emit_structured_tick,
_emit_sync_result_breakdown,
_filter_runtime_events_for_job,
@@ -187,6 +190,140 @@ class SyncAgentTests(unittest.TestCase):
mock_process_detect_pipeline_now.assert_called_once_with(limit=5000)
self.assertEqual("pipeline_tick_success", mock_push_debug_event.call_args.kwargs["event_type"])
@patch("app.sync_agent._load_local_detect_backlog_snapshot", return_value={"pending_total": 12})
@patch("app.sync_agent.list_recent_detect_run_events", return_value=[])
@patch("app.sync_agent.get_detect_queue_health", return_value={"queue": {"overdue_leases": 0}})
@patch("app.sync_agent.push_debug_event")
@patch("app.sync_agent._append_detect_result_projection_snapshot")
@patch("app.sync_agent._select_projection_job_snapshots", return_value=[{"job_id": 1, "job_code": "finished-job"}])
@patch(
"app.sync_agent.get_active_detect_job_summary",
return_value={
"job_id": 2,
"job_code": "running-job",
"status": "running",
"items_total": 100,
"items_pending": 10,
"items_claimed": 5,
"items_running": 7,
"items_completed": 70,
"items_failed": 8,
"progress_percent": 70.0,
"node_stats": [{"node_code": "mainland-controller-01", "items_total": 90}],
},
)
def test_emit_runtime_debug_snapshots_emits_projection_and_active_job(
self,
mock_get_active_detect_job_summary,
mock_select_projection_job_snapshots,
mock_append_detect_result_projection_snapshot,
mock_push_debug_event,
mock_get_detect_queue_health,
mock_list_recent_detect_run_events,
mock_load_local_detect_backlog_snapshot,
) -> None:
_emit_runtime_debug_snapshots()
mock_get_active_detect_job_summary.assert_called_once_with(event_limit=10)
mock_select_projection_job_snapshots.assert_called_once_with()
mock_append_detect_result_projection_snapshot.assert_called_once()
self.assertTrue(mock_push_debug_event.called)
self.assertEqual("active_job_snapshot", mock_push_debug_event.call_args_list[0].kwargs["event_type"])
@patch("app.sync_agent._emit_runtime_debug_snapshots")
@patch("app.sync_agent._run_pipeline_stage_processor", return_value=(True, "pipeline ok", {"stage": "pipeline"}))
@patch("app.sync_agent.pull_detect_task_batch_now", return_value=(True, "pull ok", {"stage": "pull"}))
@patch("app.sync_agent.push_runtime_projection_now", return_value=(True, "sync ok", {"stage": "sync"}))
def test_run_sync_tick_once_prioritizes_sync_before_pipeline(
self,
mock_push_runtime_projection_now,
mock_pull_detect_task_batch_now,
mock_run_pipeline_stage_processor,
mock_emit_runtime_debug_snapshots,
) -> None:
with patch("app.sync_agent._maybe_trigger_overlap_start", return_value=(False, "no overlap", {})) as mock_overlap:
tick = _run_sync_tick_once()
self.assertTrue(tick["sync"]["ok"])
self.assertEqual("sync ok", tick["sync"]["message"])
mock_push_runtime_projection_now.assert_called_once_with()
mock_pull_detect_task_batch_now.assert_called_once_with()
mock_run_pipeline_stage_processor.assert_called_once_with()
mock_overlap.assert_called_once_with()
mock_emit_runtime_debug_snapshots.assert_called_once_with()
self.assertEqual("no overlap", tick["overlap"]["message"])
@patch("app.sync_agent.send_worker_command")
@patch("app.sync_agent._select_overlap_target_node_codes")
@patch("app.sync_agent._select_overlap_start_candidate")
@patch("app.sync_agent.push_debug_event")
def test_maybe_trigger_overlap_start_dispatches_pending_job(
self,
mock_push_debug_event,
mock_select_overlap_start_candidate,
mock_select_overlap_target_node_codes,
mock_send_worker_command,
) -> None:
mock_select_overlap_start_candidate.return_value = {
"job_id": 885,
"job_code": "sync-overseas-20592",
"task_mode": "domain_pipeline",
"items_pending": 64000,
"items_claimed": 0,
"items_running": 0,
"selection_reason": "overlap_tail_handoff",
}
mock_select_overlap_target_node_codes.return_value = [
"mainland-controller-01-a",
"mainland-controller-01-da",
]
mock_send_worker_command.return_value = (True, "已发送 Worker 控制指令: start_detection -> mainland-controller-01-a,mainland-controller-01-da")
with patch("app.sync_agent._LAST_OVERLAP_JOB_ID", 0), patch("app.sync_agent._LAST_OVERLAP_TRIGGERED_AT", 0.0):
ok, message, data = _maybe_trigger_overlap_start()
self.assertTrue(ok)
self.assertIn("mainland-controller-01-a", message)
self.assertEqual(885, data["job_id"])
mock_send_worker_command.assert_called_once_with(
"start_detection",
payload={
"job_id": 885,
"job_code": "sync-overseas-20592",
"target_job_id": 885,
"target_job_code": "sync-overseas-20592",
"task_mode": "domain_pipeline",
"source": "overlap-handoff",
"selection_reason": "overlap_tail_handoff",
"tail_handoff_candidate": True,
"target_node_codes": [
"mainland-controller-01-a",
"mainland-controller-01-da",
],
},
)
self.assertEqual("overlap_handoff_started", mock_push_debug_event.call_args.kwargs["event_type"])
@patch("app.sync_agent._select_overlap_start_candidate")
def test_maybe_trigger_overlap_start_honors_cooldown_for_same_job(
self,
mock_select_overlap_start_candidate,
) -> None:
mock_select_overlap_start_candidate.return_value = {
"job_id": 885,
"job_code": "sync-overseas-20592",
"task_mode": "domain_pipeline",
"items_pending": 64000,
"selection_reason": "overlap_tail_handoff",
}
with patch("app.sync_agent._LAST_OVERLAP_JOB_ID", 885), patch("app.sync_agent._LAST_OVERLAP_TRIGGERED_AT", __import__('time').time()):
ok, message, data = _maybe_trigger_overlap_start()
self.assertFalse(ok)
self.assertIn("冷却中", message)
self.assertEqual(885, data["job_id"])
if __name__ == "__main__":
unittest.main()

View File

@@ -1,10 +1,20 @@
import unittest
from datetime import datetime, timedelta
from unittest.mock import patch
from app.services.sync_push_service import (
_build_task_pull_backlog_limits,
_extract_detect_result_projection_events,
_load_latest_projection,
_load_pushable_projections,
_refresh_remote_runtime_node,
_load_local_detect_backlog_snapshot,
_push_projection_now,
_resolve_task_pull_request_limit,
_task_projection_limit,
pull_detect_task_batch_now,
_resolve_detect_result_target_job_id,
_select_relevant_backlog_job_ids_from_rows,
_should_throttle_task_pull,
ingest_runtime_projection,
)
@@ -23,6 +33,12 @@ class _FakeCursor:
return self._rows.pop(0)
return None
def fetchall(self):
if self._rows:
value = self._rows.pop(0)
return list(value or [])
return []
def __enter__(self):
return self
@@ -48,23 +64,311 @@ class _FakeConnection:
return False
class _FakeUrlopenResponse:
def __init__(self, payload):
self._payload = payload
def read(self):
import json
return json.dumps(self._payload, ensure_ascii=False).encode("utf-8")
def __enter__(self):
return self
def __exit__(self, exc_type, exc, tb):
return False
class SyncPushServiceTests(unittest.TestCase):
def test_build_task_pull_backlog_limits_scales_with_thread_configuration(self) -> None:
@patch("app.services.sync_push_service.cleanup_imported_runtime_nodes_many")
@patch("app.services.sync_push_service.cleanup_imported_runtime_nodes")
@patch("app.services.sync_push_service.register_node_heartbeat")
def test_refresh_remote_runtime_node_imports_cluster_worker_rows_without_active_job(
self,
mock_register_node_heartbeat,
_mock_cleanup_imported_runtime_nodes,
mock_cleanup_imported_runtime_nodes_many,
) -> None:
_refresh_remote_runtime_node(
source_region="mainland",
projection={
"node": {
"node_code": "mainland-controller-01",
"region": "mainland",
"role": "control",
"hostname": "localhost",
"ip": "127.0.0.1",
},
"worker_mode": "linux-systemd",
"phase_label": "等待中",
"phase_detail": "等待任务",
"proxy_runtime_label": "正常",
"proxy_runtime_reason": "healthy",
"active_thread_count": 0,
"max_thread_count": 60000,
"worker_online": True,
"detect_participating": False,
"active_job": {
"job_code": "",
"status": "",
"node_stats": [],
},
"cluster_nodes": [
{
"node_code": "mainland-controller-01",
"role": "control",
"status": "online",
"current_load": 0,
"active_threads": 0,
"max_threads": 60000,
"detect_participating": False,
},
{
"node_code": "mainland-controller-01-a",
"role": "worker",
"status": "online",
"current_load": 0,
"active_threads": 0,
"max_threads": 1000,
"detect_participating": False,
},
{
"node_code": "mainland-controller-01-b",
"role": "worker",
"status": "busy",
"current_load": 12,
"active_threads": 12,
"max_threads": 1000,
"detect_participating": True,
},
],
},
)
self.assertEqual(3, mock_register_node_heartbeat.call_count)
worker_calls = [call.kwargs for call in mock_register_node_heartbeat.call_args_list[1:]]
self.assertEqual(
["mainland-controller-01-a", "mainland-controller-01-b"],
[item["node_code"] for item in worker_calls],
)
self.assertEqual(12, worker_calls[1]["current_load"])
self.assertEqual(
["mainland-controller-01-a", "mainland-controller-01-b"],
mock_cleanup_imported_runtime_nodes_many.call_args.kwargs["keep_node_codes"],
)
@patch("app.services.sync_push_service.settings")
def test_task_pull_request_limit_scales_with_estimated_threads(self, mock_settings) -> None:
mock_settings.node_code = "mainland-controller-01"
mock_settings.sync_batch_size = 5000
safe_limit = _resolve_task_pull_request_limit(
None,
settings_payload={
"thread_count": 1000,
"process_count": 1,
"node_thread_counts": {"mainland-controller-01": 1000},
"node_process_counts": {"mainland-controller-01": 60},
},
)
self.assertEqual(120000, safe_limit)
@patch("app.services.sync_push_service.settings")
def test_task_projection_limit_allows_large_explicit_pull_request(self, mock_settings) -> None:
mock_settings.sync_batch_size = 5000
self.assertEqual(120000, _task_projection_limit(120000))
@patch("app.services.sync_push_service.settings")
@patch("app.services.sync_push_service.get_db")
def test_load_latest_projection_prefers_non_future_row_over_clock_skewed_history(
self,
mock_get_db,
mock_settings,
) -> None:
now = datetime.now()
future_row = (
10081,
"mainland",
"overseas",
"projected",
{"projection": {"active_thread_count": 138}},
now + timedelta(hours=4),
now + timedelta(hours=4),
)
valid_row = (
11633,
"mainland",
"overseas",
"projected",
{"projection": {"active_thread_count": 6431}},
now - timedelta(minutes=2),
now - timedelta(minutes=2),
)
fake_conn = _FakeConnection(rows=[[future_row, valid_row]])
mock_get_db.return_value = fake_conn
mock_settings.sync_source_region = "mainland"
mock_settings.node_region = "mainland"
mock_settings.sync_target_region = "overseas"
with patch("app.services.sync_push_service.datetime") as mock_datetime:
mock_datetime.now.return_value = now
record = _load_latest_projection("runtime_projection")
self.assertIsNotNone(record)
self.assertEqual(11633, record["id"])
executed_sql, executed_params = fake_conn.cursor_obj.executed[0]
self.assertIn("CASE WHEN created_at <= %s THEN 0 ELSE 1 END", executed_sql)
self.assertEqual("runtime_projection", executed_params[0])
self.assertEqual("mainland", executed_params[1])
self.assertEqual("overseas", executed_params[2])
self.assertEqual(now + timedelta(minutes=5), executed_params[3])
@patch("app.services.sync_push_service._latest_push_attempt")
@patch("app.services.sync_push_service.settings")
@patch("app.services.sync_push_service.get_db")
def test_load_pushable_projections_prefers_latest_unsent_records(
self,
mock_get_db,
mock_settings,
mock_latest_push_attempt,
) -> None:
now = datetime.now()
old_success_rows = [
(
10000 + index,
"mainland",
"overseas",
"projected",
{"projection_hash": f"old-{index}", "projection": {"job": {"job_id": index}}},
now - timedelta(minutes=120 - index),
now - timedelta(minutes=120 - index),
)
for index in range(100)
]
newest_unsent = (
23360,
"mainland",
"overseas",
"projected",
{"projection_hash": "new-hash", "projection": {"job": {"job_id": 909, "job_code": "sync-overseas-28618"}}},
now,
now,
)
fake_conn = _FakeConnection(rows=[old_success_rows + [newest_unsent]])
mock_get_db.return_value = fake_conn
mock_settings.sync_source_region = "mainland"
mock_settings.node_region = "mainland"
mock_settings.sync_target_region = "overseas"
mock_settings.sync_batch_size = 20
def _attempt_side_effect(source_record_id, target_region, sync_type):
if source_record_id == 23360:
return None
return {"status": "success", "created_at": now}
mock_latest_push_attempt.side_effect = _attempt_side_effect
records = _load_pushable_projections("detect_result_projection", limit=20)
self.assertEqual([23360], [item["id"] for item in records])
executed_sql, _ = fake_conn.cursor_obj.executed[0]
self.assertIn("ORDER BY created_at DESC, id DESC", executed_sql)
@patch("app.services.sync_push_service.settings.node_code", "mainland-controller-01")
def test_build_task_pull_backlog_limits_scales_with_local_server_capacity(self) -> None:
limits = _build_task_pull_backlog_limits(
5000,
settings_payload={
"thread_count": 100,
"process_count": 80,
"node_thread_counts": {
"mainland-controller-01": 2000,
"mainland-controller-01": 1000,
"mainland-controller-01-a": 1000,
"mainland-worker-01": 1200,
},
"node_process_counts": {
"mainland-controller-01": 80,
"mainland-worker-01": 60,
},
},
)
self.assertEqual(3200, limits["estimated_total_threads"])
self.assertEqual(6400, limits["max_pending_total"])
self.assertEqual(3200, limits["max_register_pending"])
self.assertEqual(800, limits["max_downstream_pending"])
self.assertEqual(80000, limits["estimated_total_threads"])
self.assertEqual(160000, limits["max_pending_total"])
self.assertEqual(80000, limits["max_register_pending"])
self.assertEqual(20000, limits["max_downstream_pending"])
def test_select_relevant_backlog_job_ids_from_rows_skips_stale_pending_jobs(self) -> None:
now = datetime.now()
job_ids = _select_relevant_backlog_job_ids_from_rows(
[
(11, "running", now - timedelta(hours=10)),
(10, "pending", now - timedelta(minutes=30)),
(9, "pending", now - timedelta(hours=7)),
],
freshness_hours=6,
limit=4,
)
self.assertEqual([11, 10], job_ids)
@patch("app.services.sync_push_service.get_active_detect_job_summary")
@patch("app.services.sync_push_service.get_db")
def test_load_local_detect_backlog_snapshot_prefers_active_job_snapshot(
self,
mock_get_db,
mock_get_active_detect_job_summary,
) -> None:
mock_get_active_detect_job_summary.return_value = {
"items_pending": 123,
"items_claimed": 4,
"items_running": 7,
"display_items_running": 9,
"step_stats": [
{"step_code": "detect_register", "items_pending": 90},
{"step_code": "detect_baidu", "items_pending": 33},
],
}
backlog = _load_local_detect_backlog_snapshot()
self.assertEqual(123, backlog["pending_total"])
self.assertEqual(4, backlog["claimed_total"])
self.assertEqual(9, backlog["running_total"])
self.assertEqual(90, backlog["register_pending"])
self.assertEqual(33, backlog["downstream_pending"])
mock_get_db.assert_not_called()
@patch("app.services.sync_push_service.get_active_detect_job_summary", return_value={})
@patch("app.services.sync_push_service.get_db")
def test_load_local_detect_backlog_snapshot_only_counts_relevant_jobs(
self,
mock_get_db,
_mock_get_active_detect_job_summary,
) -> None:
fake_conn = _FakeConnection(
rows=[
[
(11, "running", datetime.now() - timedelta(hours=2)),
(10, "pending", datetime.now() - timedelta(minutes=30)),
(9, "pending", datetime.now() - timedelta(hours=7)),
],
(123, 4, 7, 90, 33),
]
)
mock_get_db.return_value = fake_conn
backlog = _load_local_detect_backlog_snapshot()
self.assertEqual(123, backlog["pending_total"])
self.assertEqual(90, backlog["register_pending"])
executed_sql, executed_params = fake_conn.cursor_obj.executed[1]
self.assertIn("WHERE item.job_id = ANY(%s)", executed_sql)
self.assertEqual([11, 10], list(executed_params[0]))
def test_should_throttle_task_pull_when_register_backlog_overwhelms_downstream(self) -> None:
should_throttle, reason = _should_throttle_task_pull(
@@ -104,6 +408,224 @@ class SyncPushServiceTests(unittest.TestCase):
self.assertFalse(should_throttle)
self.assertEqual("", reason)
@patch("app.services.sync_push_service._should_throttle_task_pull", return_value=(False, ""))
@patch("app.services.sync_push_service._build_task_pull_backlog_limits", return_value={})
@patch("app.services.sync_push_service._load_local_detect_backlog_snapshot", return_value={})
@patch("app.services.sync_push_service._acquire_sync_pull_worker_wake_guard", return_value=True)
@patch("app.services.sync_push_service.ingest_detect_task_projection")
@patch("app.services.worker_control_service.send_worker_command")
@patch("app.services.sync_push_service.urllib.request.urlopen")
@patch("app.services.sync_push_service.settings")
@patch("app.services.sync_push_service.get_settings_payload")
def test_pull_detect_task_batch_now_starts_worker_with_projection_active_job_identity(
self,
mock_get_settings_payload,
mock_settings,
mock_urlopen,
mock_send_worker_command,
mock_ingest_detect_task_projection,
_mock_acquire_sync_pull_worker_wake_guard,
_mock_load_local_detect_backlog_snapshot,
_mock_build_task_pull_backlog_limits,
_mock_should_throttle_task_pull,
) -> None:
mock_settings.node_region = "mainland"
mock_settings.node_role = "control"
mock_settings.node_code = "mainland-controller-01"
mock_settings.sync_target_api_base_url = "http://example.com"
mock_settings.sync_target_region = "overseas"
mock_settings.sync_batch_size = 200
mock_settings.sync_shared_token = ""
mock_get_settings_payload.return_value = {
"thread_count": 1000,
"process_count": 1,
"node_thread_counts": {"mainland-controller-01": 1000},
"node_process_counts": {"mainland-controller-01": 60},
}
projection = {
"batch_code": "task-20260423210000-aa11bb",
"active_job": {
"job_id": 376,
"job_code": "sync-overseas-376",
"current_cycle_token": "cycle-376",
},
}
export_response = {
"code": 0,
"data": {
"source_record_id": 15164,
"projection_hash": "hash-15164",
"projection": projection,
},
}
ack_response = {"code": 0, "data": {"acknowledged": True}}
mock_urlopen.side_effect = [
_FakeUrlopenResponse(export_response),
_FakeUrlopenResponse(ack_response),
]
mock_ingest_detect_task_projection.return_value = (
True,
"任务批次接收成功",
{
"target_job_id": 1902,
"target_job_code": "sync-overseas-15164",
"queued_count": 1000,
},
)
mock_send_worker_command.return_value = (True, "started")
ok, message, payload = pull_detect_task_batch_now(limit=1000)
self.assertTrue(ok)
self.assertEqual("待检测任务批次拉取并入库成功", message)
self.assertTrue(payload["worker_start_ok"])
export_request = mock_urlopen.call_args_list[0][0][0]
self.assertIn("limit=1000", export_request.full_url)
mock_send_worker_command.assert_called_once()
args, kwargs = mock_send_worker_command.call_args
self.assertEqual("start_detection", args[0])
sent_payload = kwargs["payload"]
self.assertEqual(15164, sent_payload["source_record_id"])
self.assertEqual(1902, sent_payload["target_job_id"])
self.assertEqual("sync-overseas-15164", sent_payload["target_job_code"])
self.assertEqual(376, sent_payload["job_id"])
self.assertEqual("sync-overseas-376", sent_payload["job_code"])
self.assertEqual("cycle-376", sent_payload["cycle_token"])
@patch("app.services.sync_push_service._should_throttle_task_pull", return_value=(False, ""))
@patch("app.services.sync_push_service._build_task_pull_backlog_limits", return_value={})
@patch("app.services.sync_push_service._load_local_detect_backlog_snapshot", return_value={})
@patch("app.services.sync_push_service._acquire_sync_pull_worker_wake_guard", return_value=False)
@patch("app.services.sync_push_service.ingest_detect_task_projection")
@patch("app.services.worker_control_service.send_worker_command")
@patch("app.services.sync_push_service.urllib.request.urlopen")
@patch("app.services.sync_push_service.settings")
@patch("app.services.sync_push_service.get_settings_payload")
def test_pull_detect_task_batch_now_skips_duplicate_worker_wake_within_short_window(
self,
mock_get_settings_payload,
mock_settings,
mock_urlopen,
mock_send_worker_command,
mock_ingest_detect_task_projection,
_mock_acquire_sync_pull_worker_wake_guard,
_mock_load_local_detect_backlog_snapshot,
_mock_build_task_pull_backlog_limits,
_mock_should_throttle_task_pull,
) -> None:
mock_settings.node_region = "mainland"
mock_settings.node_role = "control"
mock_settings.node_code = "mainland-controller-01"
mock_settings.sync_target_api_base_url = "http://example.com"
mock_settings.sync_target_region = "overseas"
mock_settings.sync_batch_size = 200
mock_settings.sync_shared_token = ""
mock_get_settings_payload.return_value = {
"thread_count": 1000,
"process_count": 1,
"node_thread_counts": {"mainland-controller-01": 1000},
"node_process_counts": {"mainland-controller-01": 60},
}
projection = {
"batch_code": "task-20260423210000-aa11bb",
"active_job": {
"job_id": 376,
"job_code": "sync-overseas-376",
"current_cycle_token": "cycle-376",
},
}
export_response = {
"code": 0,
"data": {
"source_record_id": 15164,
"projection_hash": "hash-15164",
"projection": projection,
},
}
ack_response = {"code": 0, "data": {"acknowledged": True}}
mock_urlopen.side_effect = [
_FakeUrlopenResponse(export_response),
_FakeUrlopenResponse(ack_response),
]
mock_ingest_detect_task_projection.return_value = (
True,
"任务批次接收成功",
{
"target_job_id": 1902,
"target_job_code": "sync-overseas-15164",
"queued_count": 1000,
},
)
ok, message, payload = pull_detect_task_batch_now(limit=1000)
self.assertTrue(ok)
self.assertEqual("待检测任务批次拉取并入库成功", message)
self.assertTrue(payload["worker_start_ok"])
self.assertTrue(payload["worker_start_skipped"])
self.assertIn("重复 Worker 唤起", payload["worker_start_message"])
mock_send_worker_command.assert_not_called()
@patch("app.services.sync_push_service._should_throttle_task_pull", return_value=(False, ""))
@patch("app.services.sync_push_service._build_task_pull_backlog_limits", return_value={})
@patch("app.services.sync_push_service._load_local_detect_backlog_snapshot", return_value={})
@patch("app.services.sync_push_service._acquire_sync_pull_worker_wake_guard", return_value=False)
@patch("app.services.sync_push_service.ingest_detect_task_projection")
@patch("app.services.sync_push_service.urllib.request.urlopen")
@patch("app.services.sync_push_service.settings")
@patch("app.services.sync_push_service.get_settings_payload")
def test_pull_detect_task_batch_now_uses_adaptive_limit_when_unspecified(
self,
mock_get_settings_payload,
mock_settings,
mock_urlopen,
mock_ingest_detect_task_projection,
_mock_acquire_sync_pull_worker_wake_guard,
_mock_load_local_detect_backlog_snapshot,
_mock_build_task_pull_backlog_limits,
_mock_should_throttle_task_pull,
) -> None:
mock_settings.node_region = "mainland"
mock_settings.node_role = "control"
mock_settings.node_code = "mainland-controller-01"
mock_settings.sync_target_api_base_url = "http://example.com"
mock_settings.sync_target_region = "overseas"
mock_settings.sync_batch_size = 5000
mock_settings.sync_shared_token = ""
mock_get_settings_payload.return_value = {
"thread_count": 1000,
"process_count": 1,
"node_thread_counts": {"mainland-controller-01": 1000},
"node_process_counts": {"mainland-controller-01": 60},
}
export_response = {
"code": 0,
"data": {
"source_record_id": 15164,
"projection_hash": "hash-15164",
"projection": {"batch_code": "task-20260423210000-aa11bb", "active_job": {}},
},
}
ack_response = {"code": 0, "data": {"acknowledged": True}}
mock_urlopen.side_effect = [
_FakeUrlopenResponse(export_response),
_FakeUrlopenResponse(ack_response),
]
mock_ingest_detect_task_projection.return_value = (
True,
"任务批次接收成功",
{"target_job_id": 1902, "target_job_code": "sync-overseas-15164", "queued_count": 1000},
)
ok, _message, _payload = pull_detect_task_batch_now(limit=None)
self.assertTrue(ok)
export_request = mock_urlopen.call_args_list[0][0][0]
self.assertIn("limit=120000", export_request.full_url)
def test_extract_detect_result_projection_events_adds_import_metadata(self) -> None:
projection = {
"job": {
@@ -137,6 +659,79 @@ class SyncPushServiceTests(unittest.TestCase):
self.assertTrue(event["payload"]["imported_from_projection"])
self.assertTrue(event["payload"]["import_fingerprint"])
@patch("app.services.sync_push_service._push_projection_record")
@patch("app.services.sync_push_service._load_latest_projection")
@patch("app.services.runtime_status_service.refresh_runtime_projection_snapshot")
def test_push_projection_now_refreshes_runtime_projection_with_lightweight_snapshot(
self,
mock_refresh_runtime_projection_snapshot,
mock_load_latest_projection,
mock_push_projection_record,
) -> None:
mock_refresh_runtime_projection_snapshot.return_value = {
"record_id": 10082,
"active_thread_count": 1972,
"max_thread_count": 80000,
"queue_display_running": 1972,
}
mock_load_latest_projection.return_value = {
"id": 10082,
"source_region": "mainland",
"target_region": "overseas",
"created_at": None,
"payload": {"projection_hash": "hash-10082", "projection": {}},
}
mock_push_projection_record.return_value = (
True,
"投影推送成功",
{"action": "push_sync", "sync_type": "runtime_projection", "source_record_id": 10082},
)
ok, message, data = _push_projection_now("runtime_projection", "https://example.com/api/v1/runtime/sync-ingest")
self.assertTrue(ok)
self.assertEqual("投影推送成功", message)
self.assertEqual(10082, data["source_record_id"])
mock_refresh_runtime_projection_snapshot.assert_called_once_with(window_minutes=15)
mock_load_latest_projection.assert_called_once_with("runtime_projection")
mock_push_projection_record.assert_called_once()
@patch("app.services.sync_push_service._push_projection_record")
@patch("app.services.sync_push_service._load_latest_projection")
@patch("app.services.sync_push_service._append_fast_runtime_projection_snapshot")
@patch("app.services.runtime_status_service.refresh_runtime_projection_snapshot")
def test_push_projection_now_can_use_fast_runtime_projection_path(
self,
mock_refresh_runtime_projection_snapshot,
mock_append_fast_runtime_projection_snapshot,
mock_load_latest_projection,
mock_push_projection_record,
) -> None:
mock_append_fast_runtime_projection_snapshot.return_value = 10091
mock_load_latest_projection.return_value = {
"id": 10091,
"source_region": "mainland",
"target_region": "overseas",
"created_at": None,
"payload": {"projection_hash": "hash-10091", "projection": {}},
}
mock_push_projection_record.return_value = (
True,
"投影推送成功",
{"action": "push_sync", "sync_type": "runtime_projection", "source_record_id": 10091},
)
with patch.dict("os.environ", {"DOMAINCHECK_SYNC_RUNTIME_FAST_PROJECTION": "1"}, clear=False):
ok, message, data = _push_projection_now("runtime_projection", "https://example.com/api/v1/runtime/sync-ingest")
self.assertTrue(ok)
self.assertEqual("投影推送成功", message)
self.assertEqual(10091, data["source_record_id"])
mock_append_fast_runtime_projection_snapshot.assert_called_once()
mock_refresh_runtime_projection_snapshot.assert_not_called()
mock_load_latest_projection.assert_called_once_with("runtime_projection")
mock_push_projection_record.assert_called_once()
@patch("app.services.sync_push_service.get_db")
def test_resolve_detect_result_target_job_id_prefers_matching_job_code(self, mock_get_db) -> None:
fake_conn = _FakeConnection(rows=[(456,)])

View File

@@ -1,12 +1,61 @@
import json
import unittest
from datetime import datetime, timedelta
from unittest.mock import patch
from app.services.sync_record_service import (
_build_runtime_projection_payload,
_collect_recent_domain_events,
_pick_latest_projection_row,
append_runtime_projection_if_changed,
get_detect_result_sync_batches,
)
class _FakeCursor:
def __init__(self, fetchone_values):
self.fetchone_values = list(fetchone_values or [])
self.executed = []
def execute(self, sql, params=None):
self.executed.append((sql, params))
def fetchone(self):
if self.fetchone_values:
return self.fetchone_values.pop(0)
return None
def fetchall(self):
if self.fetchone_values:
value = self.fetchone_values.pop(0)
return list(value or [])
return []
def __enter__(self):
return self
def __exit__(self, exc_type, exc, tb):
return False
class _FakeConnection:
def __init__(self, fetchone_values):
self.cursor_obj = _FakeCursor(fetchone_values)
self.committed = False
def cursor(self):
return self.cursor_obj
def commit(self):
self.committed = True
def __enter__(self):
return self
def __exit__(self, exc_type, exc, tb):
return False
class SyncRecordServiceTests(unittest.TestCase):
def test_collect_recent_domain_events_filters_and_keeps_latest_slice(self) -> None:
active_job = {
@@ -130,6 +179,682 @@ class SyncRecordServiceTests(unittest.TestCase):
self.assertEqual("", projection["active_job"]["job_code"])
self.assertEqual([], projection["active_job"]["node_stats"])
@patch("app.services.sync_record_service._resolve_local_ip", return_value="121.204.244.188")
@patch("app.services.sync_record_service.socket.gethostname", return_value="mainland-controller-01")
@patch("app.services.sync_record_service.settings")
def test_build_runtime_projection_payload_prefers_runtime_queue_nodes_for_multi_process_controller(
self,
mock_settings,
_mock_hostname,
_mock_resolve_ip,
) -> None:
mock_settings.node_code = "mainland-controller-01"
mock_settings.node_region = "mainland"
mock_settings.node_role = "control"
mock_settings.sync_source_region = "mainland"
mock_settings.sync_target_region = "overseas"
payload = _build_runtime_projection_payload(
detect={
"worker_online": True,
"worker_mode": "linux-systemd",
"active_thread_count": 1,
"max_thread_count": 1000,
"aggregate_max_thread_count": 80000,
"phase_label": "运行中",
"phase_detail": "80 实例运行",
"proxy_runtime_label": "正常",
"proxy_runtime_reason": "healthy",
"progress": {
"pending": 5001,
"running": 0,
"completed": 0,
"blacklisted": 0,
"failed": 0,
},
"backlog": {"pending_total": 1157325},
"queue_health": {
"queue": {
"items_total": 5292,
"pending": 4578,
"claimed": 0,
"display_claimed": 120,
"running": 1972,
"display_running": 1972,
"completed": 690,
"blacklisted": 0,
"failed": 24,
"terminal": 714,
},
"nodes": [
{
"node_code": "mainland-controller-01-a",
"items_running": 1000,
"items_claimed": 0,
"display_running": 1000,
"active_threads": 1000,
"max_threads": 1000,
"region": "mainland",
"role": "control",
"status": "busy",
},
{
"node_code": "mainland-controller-01-b",
"items_running": 972,
"items_claimed": 120,
"display_running": 972,
"active_threads": 972,
"max_threads": 1000,
"region": "mainland",
"role": "control",
"status": "busy",
},
],
},
"active_job": {
"job_id": 55,
"job_code": "sync-overseas-55",
"status": "running",
"progress_percent": 13.49,
"items_total": 5292,
"items_terminal": 714,
"items_pending": 4578,
"items_running": 0,
"items_failed": 24,
"node_stats": [
{
"node_code": "mainland-controller-01",
"items_running": 1,
"items_claimed": 0,
"items_total": 5292,
}
],
},
"dependency_alerts": [],
},
cluster={
"nodes_total": 80,
"nodes": [
{
"node_code": "mainland-controller-01",
"current_load": 1,
"detect_participating": True,
}
],
"summary": {
"online_worker_nodes": 65,
"dedicated_online_worker_nodes": 1,
"online_control_nodes": 65,
"busy_nodes": ["mainland-controller-01-a", "mainland-controller-01-b"],
"stale_nodes": [],
"offline_nodes": [],
},
},
source_region="mainland",
target_region="overseas",
)
projection = payload["projection"]
self.assertTrue(projection["detect_participating"])
self.assertEqual(1972, projection["active_thread_count"])
self.assertEqual(80000, projection["max_thread_count"])
self.assertEqual(1972, projection["progress"]["running"])
self.assertEqual(2, len(projection["active_job"]["node_stats"]))
self.assertEqual(2, len(projection["active_job"]["distributed_node_stats"]))
self.assertEqual(1, len(projection["cluster_nodes"]))
self.assertEqual("mainland-controller-01", projection["cluster_nodes"][0]["node_code"])
self.assertEqual(1972, projection["active_job"]["items_running"])
self.assertEqual(1972, projection["active_job"]["display_items_running"])
self.assertEqual(80000, projection["active_job"]["display_max_threads"])
@patch("app.services.sync_record_service._resolve_local_ip", return_value="121.204.244.188")
@patch("app.services.sync_record_service.socket.gethostname", return_value="mainland-controller-01")
@patch("app.services.sync_record_service.settings")
@patch("app.services.sync_record_service.get_db")
def test_append_runtime_projection_if_changed_writes_heartbeat_for_unchanged_projection_after_interval(
self,
mock_get_db,
mock_settings,
_mock_hostname,
_mock_resolve_ip,
) -> None:
mock_settings.node_code = "mainland-controller-01"
mock_settings.node_region = "mainland"
mock_settings.node_role = "control"
mock_settings.sync_source_region = "mainland"
mock_settings.sync_target_region = "overseas"
detect = {
"worker_online": True,
"worker_mode": "linux-systemd",
"active_thread_count": 3200,
"max_thread_count": 80000,
"phase_label": "运行中",
"phase_detail": "80 实例运行",
"proxy_runtime_label": "正常",
"proxy_runtime_reason": "healthy",
"progress": {
"pending": 1200000,
"running": 6400,
"completed": 50000,
"blacklisted": 1200,
"failed": 88,
},
"backlog": {"pending_total": 1200000},
"active_job": {
"job_id": 55,
"job_code": "sync-overseas-55",
"status": "running",
"progress_percent": 12.5,
"items_total": 1300000,
"items_terminal": 51288,
"items_pending": 1200000,
"items_running": 6400,
"items_failed": 88,
"node_stats": [
{
"node_code": "mainland-controller-01",
"items_running": 6400,
"items_claimed": 7000,
"items_total": 1300000,
}
],
},
"dependency_alerts": [],
}
cluster = {
"nodes_total": 1,
"nodes": [
{
"node_code": "mainland-controller-01",
"current_load": 6400,
"detect_participating": True,
}
],
"summary": {
"online_worker_nodes": 1,
"dedicated_online_worker_nodes": 0,
"online_control_nodes": 1,
"busy_nodes": ["mainland-controller-01"],
"stale_nodes": [],
"offline_nodes": [],
},
}
previous_payload = _build_runtime_projection_payload(
detect=detect,
cluster=cluster,
source_region="mainland",
target_region="overseas",
)
fake_conn = _FakeConnection(
[
[(previous_payload, datetime.now() - timedelta(seconds=90))],
(321,),
]
)
mock_get_db.return_value = fake_conn
record_id = append_runtime_projection_if_changed(
detect=detect,
cluster=cluster,
source_region="mainland",
target_region="overseas",
)
self.assertEqual(321, record_id)
self.assertTrue(fake_conn.committed)
self.assertTrue(
any("INSERT INTO detect_sync_records" in sql for sql, _params in fake_conn.cursor_obj.executed)
)
@patch("app.services.sync_record_service._resolve_local_ip", return_value="121.204.244.188")
@patch("app.services.sync_record_service.socket.gethostname", return_value="mainland-controller-01")
@patch("app.services.sync_record_service.settings")
@patch("app.services.sync_record_service.get_db")
def test_append_runtime_projection_if_changed_writes_when_cluster_nodes_change_within_window(
self,
mock_get_db,
mock_settings,
_mock_hostname,
_mock_resolve_ip,
) -> None:
mock_settings.node_code = "mainland-controller-01"
mock_settings.node_region = "mainland"
mock_settings.node_role = "control"
mock_settings.sync_source_region = "mainland"
mock_settings.sync_target_region = "overseas"
previous_payload = {
"projection": {
"worker_online": True,
"worker_mode": "linux-systemd",
"phase_label": "运行中",
"phase_detail": "等待中",
"proxy_runtime_label": "正常",
"proxy_runtime_reason": "healthy",
"active_thread_count": 0,
"max_thread_count": 60000,
"progress": {"pending": 10000, "running": 0, "completed": 0, "blacklisted": 0, "failed": 0},
"active_job": {
"job_id": None,
"job_code": "",
"status": "",
"items_total": 0,
"items_running": 0,
"items_claimed": 0,
"display_items_running": 0,
"display_items_claimed": 0,
"display_max_threads": 0,
"node_stats": [],
"distributed_node_stats": [],
},
"cluster_summary": {
"nodes_total": 43,
"online_worker_nodes": 43,
"dedicated_online_worker_nodes": 42,
"online_control_nodes": 1,
"busy_nodes": [],
"stale_nodes": [],
"offline_nodes": [],
},
"cluster_nodes": [
{
"node_code": "mainland-controller-01",
"role": "control",
"status": "online",
"current_load": 0,
"active_threads": 0,
"max_threads": 60000,
"detect_participating": False,
}
],
"dependency_alerts": [],
}
}
fake_conn = _FakeConnection(
[
[(previous_payload, datetime.now() - timedelta(seconds=10))],
(911,),
]
)
mock_get_db.return_value = fake_conn
record_id = append_runtime_projection_if_changed(
detect={
"worker_online": True,
"worker_mode": "linux-systemd",
"phase_label": "运行中",
"phase_detail": "等待中",
"proxy_runtime_label": "正常",
"proxy_runtime_reason": "healthy",
"progress": {"pending": 10000, "running": 0, "completed": 0, "blacklisted": 0, "failed": 0},
"backlog": {},
"active_job": {},
"dependency_alerts": [],
},
cluster={
"nodes_total": 61,
"nodes": [
{
"node_code": "mainland-controller-01",
"role": "control",
"status": "online",
"current_load": 0,
"metadata": {"active_threads": 0, "max_threads": 60000},
},
{
"node_code": "mainland-controller-01-a",
"role": "worker",
"status": "online",
"current_load": 0,
"metadata": {"active_threads": 0, "max_threads": 1000},
},
],
"summary": {
"online_worker_nodes": 61,
"dedicated_online_worker_nodes": 60,
"online_control_nodes": 1,
"busy_nodes": [],
"stale_nodes": [],
"offline_nodes": [],
},
},
source_region="mainland",
target_region="overseas",
)
self.assertEqual(911, record_id)
self.assertTrue(fake_conn.committed)
self.assertTrue(
any("INSERT INTO detect_sync_records" in sql for sql, _params in fake_conn.cursor_obj.executed)
)
@patch("app.services.sync_record_service._resolve_local_ip", return_value="121.204.244.188")
@patch("app.services.sync_record_service.socket.gethostname", return_value="mainland-controller-01")
@patch("app.services.sync_record_service.settings")
@patch("app.services.sync_record_service.get_db")
def test_append_runtime_projection_if_changed_skips_unchanged_projection_within_heartbeat_window(
self,
mock_get_db,
mock_settings,
_mock_hostname,
_mock_resolve_ip,
) -> None:
mock_settings.node_code = "mainland-controller-01"
mock_settings.node_region = "mainland"
mock_settings.node_role = "control"
mock_settings.sync_source_region = "mainland"
mock_settings.sync_target_region = "overseas"
detect = {
"worker_online": True,
"worker_mode": "linux-systemd",
"active_thread_count": 3200,
"max_thread_count": 80000,
"phase_label": "运行中",
"phase_detail": "80 实例运行",
"proxy_runtime_label": "正常",
"proxy_runtime_reason": "healthy",
"progress": {
"pending": 1200000,
"running": 6400,
"completed": 50000,
"blacklisted": 1200,
"failed": 88,
},
"backlog": {"pending_total": 1200000},
"active_job": {
"job_id": 55,
"job_code": "sync-overseas-55",
"status": "running",
"progress_percent": 12.5,
"items_total": 1300000,
"items_terminal": 51288,
"items_pending": 1200000,
"items_running": 6400,
"items_failed": 88,
"node_stats": [
{
"node_code": "mainland-controller-01",
"items_running": 6400,
"items_claimed": 7000,
"items_total": 1300000,
}
],
},
"dependency_alerts": [],
}
cluster = {
"nodes_total": 1,
"nodes": [
{
"node_code": "mainland-controller-01",
"current_load": 6400,
"detect_participating": True,
}
],
"summary": {
"online_worker_nodes": 1,
"dedicated_online_worker_nodes": 0,
"online_control_nodes": 1,
"busy_nodes": ["mainland-controller-01"],
"stale_nodes": [],
"offline_nodes": [],
},
}
previous_payload = _build_runtime_projection_payload(
detect=detect,
cluster=cluster,
source_region="mainland",
target_region="overseas",
)
fake_conn = _FakeConnection(
[
[(previous_payload, datetime.now() - timedelta(seconds=10))],
]
)
mock_get_db.return_value = fake_conn
record_id = append_runtime_projection_if_changed(
detect=detect,
cluster=cluster,
source_region="mainland",
target_region="overseas",
)
self.assertIsNone(record_id)
self.assertFalse(fake_conn.committed)
self.assertFalse(
any("INSERT INTO detect_sync_records" in sql for sql, _params in fake_conn.cursor_obj.executed)
)
@patch("app.services.sync_record_service._resolve_local_ip", return_value="121.204.244.188")
@patch("app.services.sync_record_service.socket.gethostname", return_value="mainland-controller-01")
@patch("app.services.sync_record_service.settings")
@patch("app.services.sync_record_service.get_db")
def test_append_runtime_projection_if_changed_writes_when_activity_signature_changes_within_window(
self,
mock_get_db,
mock_settings,
_mock_hostname,
_mock_resolve_ip,
) -> None:
mock_settings.node_code = "mainland-controller-01"
mock_settings.node_region = "mainland"
mock_settings.node_role = "control"
mock_settings.sync_source_region = "mainland"
mock_settings.sync_target_region = "overseas"
previous_detect = {
"worker_online": True,
"worker_mode": "linux-systemd",
"active_thread_count": 300,
"max_thread_count": 60000,
"phase_label": "运行中",
"phase_detail": "60 实例运行",
"proxy_runtime_label": "正常",
"proxy_runtime_reason": "healthy",
"progress": {
"pending": 5000,
"running": 600,
"completed": 1000,
"blacklisted": 0,
"failed": 10,
},
"queue_health": {
"queue": {
"items_total": 6610,
"pending": 5000,
"claimed": 10,
"display_claimed": 10,
"running": 600,
"display_running": 600,
"completed": 1000,
"blacklisted": 0,
"failed": 10,
"terminal": 1010,
"display_max_threads": 60000,
},
"nodes": [
{
"node_code": "mainland-controller-01-a",
"items_running": 300,
"items_claimed": 10,
"display_running": 300,
"active_threads": 300,
"max_threads": 1000,
"region": "mainland",
"role": "control",
"status": "busy",
}
],
},
"active_job": {
"job_id": 867,
"job_code": "sync-overseas-19835",
"status": "running",
"progress_percent": 15.0,
"items_total": 6610,
"items_terminal": 1010,
"items_pending": 5000,
"items_claimed": 10,
"items_running": 600,
"items_failed": 10,
"node_stats": [
{
"node_code": "mainland-controller-01-a",
"items_running": 300,
"items_claimed": 10,
"items_total": 6610,
}
],
},
"dependency_alerts": [],
}
current_detect = {
**previous_detect,
"active_thread_count": 900,
"queue_health": {
"queue": {
"items_total": 6610,
"pending": 4300,
"claimed": 30,
"display_claimed": 30,
"running": 900,
"display_running": 900,
"completed": 1370,
"blacklisted": 0,
"failed": 10,
"terminal": 1380,
"display_max_threads": 60000,
},
"nodes": [
{
"node_code": "mainland-controller-01-a",
"items_running": 420,
"items_claimed": 20,
"display_running": 420,
"active_threads": 420,
"max_threads": 1000,
"region": "mainland",
"role": "control",
"status": "busy",
},
{
"node_code": "mainland-controller-01-b",
"items_running": 480,
"items_claimed": 10,
"display_running": 480,
"active_threads": 480,
"max_threads": 1000,
"region": "mainland",
"role": "control",
"status": "busy",
},
],
},
"active_job": {
**previous_detect["active_job"],
"items_pending": 4300,
"items_claimed": 30,
"items_running": 900,
"node_stats": [
{
"node_code": "mainland-controller-01-a",
"items_running": 420,
"items_claimed": 20,
"items_total": 6610,
},
{
"node_code": "mainland-controller-01-b",
"items_running": 480,
"items_claimed": 10,
"items_total": 6610,
},
],
},
}
cluster = {
"nodes_total": 2,
"nodes": [
{
"node_code": "mainland-controller-01",
"current_load": 900,
"detect_participating": True,
}
],
"summary": {
"online_worker_nodes": 60,
"dedicated_online_worker_nodes": 0,
"online_control_nodes": 60,
"busy_nodes": ["mainland-controller-01-a", "mainland-controller-01-b"],
"stale_nodes": [],
"offline_nodes": [],
},
}
previous_payload = _build_runtime_projection_payload(
detect=previous_detect,
cluster=cluster,
source_region="mainland",
target_region="overseas",
)
fake_conn = _FakeConnection(
[
[(previous_payload, datetime.now() - timedelta(seconds=10))],
(654,),
]
)
mock_get_db.return_value = fake_conn
record_id = append_runtime_projection_if_changed(
detect=current_detect,
cluster=cluster,
source_region="mainland",
target_region="overseas",
)
self.assertEqual(654, record_id)
self.assertTrue(fake_conn.committed)
self.assertTrue(
any("INSERT INTO detect_sync_records" in sql for sql, _params in fake_conn.cursor_obj.executed)
)
def test_pick_latest_projection_row_skips_future_dated_rows(self) -> None:
rows = [
("future", datetime.now() + timedelta(hours=6)),
("recent", datetime.now() - timedelta(seconds=10)),
("older", datetime.now() - timedelta(minutes=2)),
]
selected = _pick_latest_projection_row(rows, created_at_index=1)
self.assertEqual("recent", selected[0])
@patch("app.services.sync_record_service.get_db")
@patch("app.services.sync_record_service.settings")
def test_get_detect_result_sync_batches_marks_overseas_control_as_not_applicable(
self,
mock_settings,
mock_get_db,
) -> None:
mock_settings.node_code = "overseas-control-01"
mock_settings.node_region = "overseas"
mock_settings.node_role = "control"
mock_settings.sync_source_region = "overseas"
mock_settings.sync_target_region = "mainland"
payload = get_detect_result_sync_batches(limit=5)
self.assertFalse(payload["applicable"])
self.assertFalse(payload["local_worker_expected"])
self.assertEqual(0, payload["jobs_total"])
self.assertEqual([], payload["batches"])
self.assertIn("不适用", payload["reason"])
mock_get_db.assert_not_called()
if __name__ == "__main__":
unittest.main()

View File

@@ -1,8 +1,17 @@
import json
import subprocess
import unittest
from unittest.mock import Mock, patch
from app.services.worker_control_service import WORKER_CONTROL_CHANNEL, WORKER_PENDING_COMMAND_KEY, send_worker_command
from app.core.config import settings
from app.services.worker_control_service import (
WORKER_CONTROL_CHANNEL,
WORKER_PENDING_COMMAND_KEY,
detect_worker_runtime,
send_worker_command,
start_worker,
stop_worker,
)
class WorkerControlServiceTests(unittest.TestCase):
@@ -11,10 +20,11 @@ class WorkerControlServiceTests(unittest.TestCase):
redis_client = Mock()
mock_get_redis.return_value = redis_client
ok, message = send_worker_command(
"start_detection",
payload={"job_id": 1, "job_code": "detect-20260419030000-abc123"},
)
with patch("app.services.worker_control_service._runtime_config", return_value={"worker_mode": "windows-local", "worker_service_name": "domaincheck-worker"}):
ok, message = send_worker_command(
"start_detection",
payload={"job_id": 1, "job_code": "detect-20260419030000-abc123"},
)
self.assertTrue(ok)
self.assertIn("已发送 Worker 控制指令", message)
@@ -33,6 +43,309 @@ class WorkerControlServiceTests(unittest.TestCase):
redis_client.publish.assert_called_once_with(WORKER_CONTROL_CHANNEL, serialized)
@patch("app.services.worker_control_service.get_redis")
def test_send_worker_command_scopes_pending_command_to_target_worker_instances(self, mock_get_redis) -> None:
redis_client = Mock()
mock_get_redis.return_value = redis_client
with patch("app.services.worker_control_service._runtime_config", return_value={"worker_mode": "linux-systemd", "worker_service_name": "domaincheck-worker"}):
ok, message = send_worker_command(
"start_detection",
payload={
"job_id": 2,
"target_node_codes": ["mainland-controller-01-a", "mainland-controller-01-b"],
},
)
self.assertTrue(ok)
self.assertIn("mainland-controller-01-a,mainland-controller-01-b", message)
self.assertEqual(2, redis_client.set.call_count)
set_keys = [call.args[0] for call in redis_client.set.call_args_list]
self.assertEqual(
[
f"{WORKER_PENDING_COMMAND_KEY}:mainland-controller-01-a",
f"{WORKER_PENDING_COMMAND_KEY}:mainland-controller-01-b",
],
set_keys,
)
serialized = redis_client.set.call_args_list[0].args[1]
payload = json.loads(serialized)
self.assertEqual(["mainland-controller-01-a", "mainland-controller-01-b"], payload["target_node_codes"])
redis_client.publish.assert_called_once_with(WORKER_CONTROL_CHANNEL, serialized)
@patch("app.services.worker_control_service._expand_linux_worker_control_units")
@patch("app.services.worker_control_service.get_redis")
def test_send_worker_command_expands_local_linux_worker_instances_when_targets_unspecified(
self,
mock_get_redis,
mock_expand_linux_worker_control_units,
) -> None:
redis_client = Mock()
mock_get_redis.return_value = redis_client
mock_expand_linux_worker_control_units.return_value = [
"domaincheck-worker",
"domaincheck-worker@a.service",
"domaincheck-worker@b.service",
]
with patch("app.services.worker_control_service._runtime_config", return_value={"worker_mode": "linux-systemd", "worker_service_name": "domaincheck-worker"}), \
patch.object(settings, "node_code", "mainland-controller-01"):
ok, message = send_worker_command("start_detection", payload={"job_id": 3})
self.assertTrue(ok)
self.assertIn("mainland-controller-01,mainland-controller-01-a,mainland-controller-01-b", message)
self.assertEqual(3, redis_client.set.call_count)
set_keys = [call.args[0] for call in redis_client.set.call_args_list]
self.assertEqual(
[
f"{WORKER_PENDING_COMMAND_KEY}:mainland-controller-01",
f"{WORKER_PENDING_COMMAND_KEY}:mainland-controller-01-a",
f"{WORKER_PENDING_COMMAND_KEY}:mainland-controller-01-b",
],
set_keys,
)
serialized = redis_client.set.call_args_list[0].args[1]
payload = json.loads(serialized)
self.assertEqual(
["mainland-controller-01", "mainland-controller-01-a", "mainland-controller-01-b"],
payload["target_node_codes"],
)
redis_client.publish.assert_called_once_with(WORKER_CONTROL_CHANNEL, serialized)
@patch("app.services.worker_control_service._build_direct_redis_client")
@patch("app.services.worker_control_service.get_redis")
def test_send_worker_command_falls_back_to_direct_redis_client(
self,
mock_get_redis,
mock_build_direct_redis_client,
) -> None:
mock_get_redis.side_effect = RecursionError("maximum recursion depth exceeded")
direct_client = Mock()
mock_build_direct_redis_client.return_value = direct_client
with patch("app.services.worker_control_service._runtime_config", return_value={"worker_mode": "windows-local", "worker_service_name": "domaincheck-worker"}):
ok, message = send_worker_command("start_detection", payload={"job_id": 9})
self.assertTrue(ok)
self.assertIn("已发送 Worker 控制指令", message)
direct_client.set.assert_called_once()
serialized = direct_client.set.call_args.args[1]
payload = json.loads(serialized)
self.assertEqual("start_detection", payload["action"])
self.assertEqual(9, payload["job_id"])
direct_client.publish.assert_called_once_with(WORKER_CONTROL_CHANNEL, serialized)
direct_client.close.assert_called_once()
@patch("app.services.worker_control_service._probe_linux_worker_instance_count", return_value=0)
@patch("app.services.worker_control_service._run_shell")
@patch("app.services.worker_control_service.probe_systemd_service")
@patch("app.services.worker_control_service._runtime_config")
def test_detect_worker_runtime_prefers_fast_pgrep_probe(
self,
mock_runtime_config,
mock_probe_systemd_service,
mock_run_shell,
_mock_instance_count,
) -> None:
mock_runtime_config.return_value = {
"worker_mode": "linux-systemd",
"worker_service_name": "domaincheck-worker",
}
mock_probe_systemd_service.return_value = {
"mode": "linux-systemd",
"service_name": "domaincheck-worker",
"running": True,
"process_count": 1,
"latest_start_time": "2026-04-22 23:00:00",
"message": "active/running",
}
mock_run_shell.return_value = subprocess.CompletedProcess(
args=["bash", "-lc", "pgrep -fc '[d]etect_worker.py' || true"],
returncode=0,
stdout="80\n",
stderr="",
)
runtime = detect_worker_runtime()
self.assertTrue(runtime["running"])
self.assertEqual(80, runtime["process_count"])
self.assertEqual(1, mock_run_shell.call_count)
@patch("app.services.worker_control_service._probe_linux_worker_instance_count", return_value=0)
@patch("app.services.worker_control_service._run_shell")
@patch("app.services.worker_control_service.probe_systemd_service")
@patch("app.services.worker_control_service._runtime_config")
def test_detect_worker_runtime_falls_back_when_pgrep_probe_is_unavailable(
self,
mock_runtime_config,
mock_probe_systemd_service,
mock_run_shell,
_mock_instance_count,
) -> None:
mock_runtime_config.return_value = {
"worker_mode": "linux-systemd",
"worker_service_name": "domaincheck-worker",
}
mock_probe_systemd_service.return_value = {
"mode": "linux-systemd",
"service_name": "domaincheck-worker",
"running": True,
"process_count": 1,
"latest_start_time": "2026-04-22 23:00:00",
"message": "active/running",
}
mock_run_shell.side_effect = [
subprocess.CompletedProcess(
args=["bash", "-lc", "pgrep -fc '[d]etect_worker.py' || true"],
returncode=0,
stdout="",
stderr="pgrep: command not found\n",
),
subprocess.CompletedProcess(
args=["bash", "-lc", "ps -eo args= | grep '[d]etect_worker.py' | wc -l"],
returncode=0,
stdout="12\n",
stderr="",
),
]
runtime = detect_worker_runtime()
self.assertTrue(runtime["running"])
self.assertEqual(12, runtime["process_count"])
self.assertEqual(2, mock_run_shell.call_count)
@patch("app.services.worker_control_service._probe_linux_worker_process_count", return_value=7)
@patch("app.services.worker_control_service._probe_linux_worker_instance_count", return_value=0)
@patch("app.services.worker_control_service.probe_systemd_service")
@patch("app.services.worker_control_service._runtime_config")
def test_detect_worker_runtime_keeps_service_offline_when_only_unmanaged_processes_exist(
self,
mock_runtime_config,
mock_probe_systemd_service,
_mock_instance_count,
_mock_process_count,
) -> None:
mock_runtime_config.return_value = {
"worker_mode": "linux-systemd",
"worker_service_name": "domaincheck-worker",
}
mock_probe_systemd_service.return_value = {
"mode": "linux-systemd",
"service_name": "domaincheck-worker",
"running": False,
"process_count": 0,
"latest_start_time": "",
"message": "inactive/dead",
}
runtime = detect_worker_runtime()
self.assertFalse(runtime["running"])
self.assertEqual(7, runtime["process_count"])
self.assertIn("unmanaged worker processes", runtime["message"])
@patch("app.services.worker_control_service._probe_linux_worker_process_count", return_value=30)
@patch("app.services.worker_control_service._probe_linux_worker_instance_count", return_value=3)
@patch("app.services.worker_control_service.probe_systemd_service")
@patch("app.services.worker_control_service._runtime_config")
def test_detect_worker_runtime_accepts_active_template_instances_when_base_service_is_inactive(
self,
mock_runtime_config,
mock_probe_systemd_service,
_mock_instance_count,
_mock_process_count,
) -> None:
mock_runtime_config.return_value = {
"worker_mode": "linux-systemd",
"worker_service_name": "domaincheck-worker",
}
mock_probe_systemd_service.return_value = {
"mode": "linux-systemd",
"service_name": "domaincheck-worker",
"running": False,
"process_count": 0,
"latest_start_time": "",
"message": "inactive/dead",
}
runtime = detect_worker_runtime()
self.assertTrue(runtime["running"])
self.assertEqual(30, runtime["process_count"])
self.assertEqual("template instances active (3)", runtime["message"])
@patch("app.services.worker_control_service._expand_linux_worker_control_units")
@patch("app.services.worker_control_service._run_systemctl")
@patch("app.services.worker_control_service._runtime_config")
def test_start_worker_includes_template_instances(
self,
mock_runtime_config,
mock_run_systemctl,
mock_expand_units,
) -> None:
mock_runtime_config.return_value = {
"worker_mode": "linux-systemd",
"worker_service_name": "domaincheck-worker",
}
mock_expand_units.return_value = [
"domaincheck-worker",
"domaincheck-worker@a",
"domaincheck-worker@b",
]
mock_run_systemctl.return_value = subprocess.CompletedProcess(
args=["systemctl", "start", "domaincheck-worker", "domaincheck-worker@a", "domaincheck-worker@b"],
returncode=0,
stdout="",
stderr="",
)
ok, message = start_worker()
self.assertTrue(ok)
self.assertIn("附带 2 个实例", message)
mock_run_systemctl.assert_called_once_with(
["start", "domaincheck-worker", "domaincheck-worker@a", "domaincheck-worker@b"],
timeout=45,
)
@patch("app.services.worker_control_service._expand_linux_worker_control_units")
@patch("app.services.worker_control_service._run_systemctl")
@patch("app.services.worker_control_service._runtime_config")
def test_stop_worker_includes_template_instances(
self,
mock_runtime_config,
mock_run_systemctl,
mock_expand_units,
) -> None:
mock_runtime_config.return_value = {
"worker_mode": "linux-systemd",
"worker_service_name": "domaincheck-worker",
}
mock_expand_units.return_value = [
"domaincheck-worker",
"domaincheck-worker@a",
"domaincheck-worker@b",
]
mock_run_systemctl.return_value = subprocess.CompletedProcess(
args=["systemctl", "stop", "domaincheck-worker", "domaincheck-worker@a", "domaincheck-worker@b"],
returncode=0,
stdout="",
stderr="",
)
ok, message = stop_worker()
self.assertTrue(ok)
self.assertIn("附带 2 个实例", message)
mock_run_systemctl.assert_called_once_with(
["stop", "domaincheck-worker", "domaincheck-worker@a", "domaincheck-worker@b"],
timeout=45,
)
if __name__ == "__main__":
unittest.main()