admin/getDomain
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: stabilize multi-region runtime sync and worker orchestration

Browse Source
This commit is contained in:
root
2026-04-27 15:48:12 +08:00
parent 7cbde2aa78
commit 215a364891
137 changed files with 31931 additions and 1943 deletions
Download Patch File Download Diff File Expand all files Collapse all files

916
domain-api/tests/test_detect_service_status_fallback.py
View File

@@ -1,10 +1,47 @@
import unittest
from contextlib import ExitStack
from datetime import datetime, timedelta
from unittest.mock import patch
from app.services import detect_service
class DetectServiceStatusFallbackTests(unittest.TestCase):
def test_filter_live_aggregate_runtime_nodes_excludes_disabled_managed_nodes(self) -> None:
now_text = datetime.now().isoformat(sep=" ", timespec="seconds")
rows = [
{
"node_code": "mainland-controller-01",
"status": "busy",
"last_heartbeat_at": now_text,
"display_running": 1,
},
{
"node_code": "mainland-worker-01",
"status": "busy",
"last_heartbeat_at": now_text,
"display_running": 1,
},
]
with patch("app.services.detect_service._load_disabled_managed_node_codes", return_value={"mainland-worker-01"}):
filtered = detect_service._filter_live_aggregate_runtime_nodes(rows)
self.assertEqual(["mainland-controller-01"], [item["node_code"] for item in filtered])
def setUp(self) -> None:
detect_service._DETECT_STATUS_CACHE_VALUE = None
detect_service._DETECT_STATUS_CACHE_EXPIRES_AT = 0.0
def test_resolve_capacity_node_code_treats_child_instance_suffix_as_single_process(self) -> None:
capacity_node_code, is_child_instance = detect_service._resolve_capacity_node_code(
"mainland-controller-01-ae",
{"node_thread_counts": {"overseas-control-01": 1}, "node_process_counts": {}},
)
self.assertEqual("mainland-controller-01", capacity_node_code)
self.assertTrue(is_child_instance)
def test_get_detect_status_keeps_runtime_snapshot_when_db_is_unreachable(self) -> None:
runtime_state = {
"service_running": True,
@@ -17,24 +54,39 @@ class DetectServiceStatusFallbackTests(unittest.TestCase):
"available_proxy_count": 18,
}
with patch("app.services.detect_service.ensure_runtime_schema"), \
patch("app.services.detect_service.get_db", side_effect=RuntimeError("db down")), \
patch("app.services.detect_service.get_settings_payload", return_value={"proxy_config": {"proxy_enable": True, "allow_direct": False, "proxy_urls": ["a"]}}), \
patch("app.services.detect_service.get_runtime_settings", return_value={"worker_log_sync_enabled": False, "worker_log_sync_mode": "full"}), \
patch("app.services.detect_service._load_recent_worker_lines", return_value=(True, "", [])), \
patch("app.services.detect_service.detect_worker_runtime", return_value={"mode": "linux-systemd", "running": True, "process_count": 1, "latest_start_time": "2026-04-20 23:58:00", "message": "active/running"}), \
patch("app.services.detect_service._load_runtime_state", return_value=runtime_state), \
patch("app.services.detect_service._load_runtime_state_from_cluster_node", return_value={}), \
patch("app.services.detect_service._extract_available_proxy_count", return_value=0), \
patch("app.services.detect_service._extract_active_thread_snapshot", return_value={"active": 0, "max": 0}), \
patch("app.services.detect_service._normalize_recent_warning", return_value=""), \
patch("app.services.detect_service._build_proxy_runtime_snapshot", return_value={"state": "healthy", "label": "代理正常", "detail": "healthy", "direct_fallback_active": False, "reason": "healthy", "last_refresh_status": "ok", "last_refresh_time": "", "source_count": 2, "raw_items": 18, "validated_count": 18, "available_count": 18, "source_stats": [], "supplier_empty": False}), \
patch("app.services.detect_service.resolve_thread_count", return_value={"effective_thread_count": 120, "default_thread_count": 5, "source": "node_override", "override_thread_count": 120, "node_code": "mainland-worker-01"}), \
patch("app.services.detect_service.get_active_detect_job_summary", side_effect=RuntimeError("db down")), \
patch("app.services.detect_service.sync_detect_runs", return_value=[]), \
patch("app.services.detect_service._resolve_remote_log_snapshot", return_value={}), \
patch("app.services.detect_service._extract_dependency_alerts", return_value=[]), \
patch("app.services.detect_service.append_detect_result_projection_if_changed"):
with ExitStack() as stack:
stack.enter_context(patch.object(detect_service.settings, "node_region", "mainland"))
stack.enter_context(patch.object(detect_service.settings, "node_role", "worker"))
stack.enter_context(patch.object(detect_service.settings, "node_code", "mainland-worker-01"))
stack.enter_context(patch("app.services.detect_service.ensure_runtime_schema"))
stack.enter_context(patch("app.services.detect_service.get_db", side_effect=RuntimeError("db down")))
stack.enter_context(
patch("app.services.detect_service.get_settings_payload", return_value={"proxy_config": {"proxy_enable": True, "allow_direct": False, "proxy_urls": ["a"]}})
)
stack.enter_context(patch("app.services.detect_service.get_runtime_settings", return_value={"worker_log_sync_enabled": False, "worker_log_sync_mode": "full"}))
stack.enter_context(patch("app.services.detect_service._load_recent_worker_lines", return_value=(True, "", [])))
stack.enter_context(
patch("app.services.detect_service.detect_worker_runtime", return_value={"mode": "linux-systemd", "running": True, "process_count": 1, "latest_start_time": "2026-04-20 23:58:00", "message": "active/running"})
)
stack.enter_context(patch("app.services.detect_service._load_runtime_state", return_value=runtime_state))
stack.enter_context(patch("app.services.detect_service._load_runtime_state_from_cluster_node", return_value={}))
stack.enter_context(patch("app.services.detect_service._extract_available_proxy_count", return_value=0))
stack.enter_context(patch("app.services.detect_service._extract_active_thread_snapshot", return_value={"active": 0, "max": 0}))
stack.enter_context(patch("app.services.detect_service._normalize_recent_warning", return_value=""))
stack.enter_context(
patch(
"app.services.detect_service._build_proxy_runtime_snapshot",
return_value={"state": "healthy", "label": "代理正常", "detail": "healthy", "direct_fallback_active": False, "reason": "healthy", "last_refresh_status": "ok", "last_refresh_time": "", "source_count": 2, "raw_items": 18, "validated_count": 18, "available_count": 18, "source_stats": [], "supplier_empty": False},
)
)
stack.enter_context(
patch("app.services.detect_service.resolve_thread_count", return_value={"effective_thread_count": 120, "default_thread_count": 5, "source": "node_override", "override_thread_count": 120, "node_code": "mainland-worker-01"})
)
stack.enter_context(patch("app.services.detect_service.get_active_detect_job_summary", side_effect=RuntimeError("db down")))
stack.enter_context(patch("app.services.detect_service.sync_detect_runs", return_value=[]))
stack.enter_context(patch("app.services.detect_service._resolve_remote_log_snapshot", return_value={}))
stack.enter_context(patch("app.services.detect_service._extract_dependency_alerts", return_value=[]))
stack.enter_context(patch("app.services.detect_service.append_detect_result_projection_if_changed"))
payload = detect_service.get_detect_status()
self.assertTrue(payload["worker_online"])
@@ -44,6 +96,56 @@ class DetectServiceStatusFallbackTests(unittest.TestCase):
self.assertEqual(0, payload["progress"]["pending"])
self.assertEqual(0, payload["progress"]["completed"])
def test_get_detect_status_skips_local_result_projection_on_overseas_control(self) -> None:
runtime_state = {
"service_running": False,
"detecting": False,
"active_threads": 0,
"max_threads": 1,
"phase": "idle",
"detail": "当前节点不承载本地检测执行",
"updated_at": "2026-04-24 02:10:00",
"available_proxy_count": 0,
}
with ExitStack() as stack:
stack.enter_context(patch.object(detect_service.settings, "node_region", "overseas"))
stack.enter_context(patch.object(detect_service.settings, "node_role", "control"))
stack.enter_context(patch.object(detect_service.settings, "node_code", "overseas-control-01"))
stack.enter_context(patch("app.services.detect_service.ensure_runtime_schema"))
stack.enter_context(patch("app.services.detect_service.get_db", side_effect=RuntimeError("db down")))
stack.enter_context(
patch("app.services.detect_service.get_settings_payload", return_value={"proxy_config": {"proxy_enable": False, "allow_direct": True, "proxy_urls": []}})
)
stack.enter_context(patch("app.services.detect_service.get_runtime_settings", return_value={"worker_log_sync_enabled": False, "worker_log_sync_mode": "key"}))
stack.enter_context(patch("app.services.detect_service._load_recent_worker_lines", return_value=(False, "", [])))
stack.enter_context(
patch("app.services.detect_service.detect_worker_runtime", return_value={"mode": "linux-systemd", "running": False, "process_count": 0, "latest_start_time": "", "message": "inactive"})
)
stack.enter_context(patch("app.services.detect_service._load_runtime_state", return_value=runtime_state))
stack.enter_context(patch("app.services.detect_service._load_runtime_state_from_cluster_node", return_value={}))
stack.enter_context(patch("app.services.detect_service._extract_available_proxy_count", return_value=0))
stack.enter_context(patch("app.services.detect_service._extract_active_thread_snapshot", return_value={"active": 0, "max": 0}))
stack.enter_context(patch("app.services.detect_service._normalize_recent_warning", return_value=""))
stack.enter_context(
patch(
"app.services.detect_service._build_proxy_runtime_snapshot",
return_value={"state": "disabled", "label": "不适用", "detail": "-", "direct_fallback_active": False, "reason": "not_applicable", "last_refresh_status": "", "last_refresh_time": "", "source_count": 0, "raw_items": 0, "validated_count": 0, "available_count": 0, "source_stats": [], "supplier_empty": False},
)
)
stack.enter_context(
patch("app.services.detect_service.resolve_thread_count", return_value={"effective_thread_count": 1, "default_thread_count": 1, "source": "default", "override_thread_count": None, "node_code": "overseas-control-01"})
)
stack.enter_context(patch("app.services.detect_service.get_active_detect_job_summary", return_value=None))
stack.enter_context(patch("app.services.detect_service.sync_detect_runs", return_value=[]))
stack.enter_context(patch("app.services.detect_service._resolve_remote_log_snapshot", return_value={}))
stack.enter_context(patch("app.services.detect_service._extract_dependency_alerts", return_value=[]))
mock_append = stack.enter_context(patch("app.services.detect_service.append_detect_result_projection_if_changed"))
detect_service.get_detect_status()
mock_append.assert_not_called()
def test_filter_lines_since_supports_journalctl_syslog_timestamps(self) -> None:
lines = [
"Apr 21 20:12:42 mainland-controller python[1]: 当前实际线程数量: 323/4",
@@ -64,6 +166,784 @@ class DetectServiceStatusFallbackTests(unittest.TestCase):
self.assertEqual(lines, filtered)
def test_get_detect_status_uses_aggregate_capacity_for_overseas_control(self) -> None:
active_job = {
"job_id": 11,
"job_code": "sync-overseas-51",
"status": "running",
"items_pending": 5001,
"items_completed": 1788,
"items_failed": 0,
"items_blacklisted": 0,
"progress_percent": 40.49,
"display_items_running": 1432,
"display_active_threads": 138,
"distributed_node_stats": [
{
"node_code": "mainland-controller-01",
"items_claimed": 73,
"items_running": 1432,
"display_running": 1432,
"active_threads": 138,
"max_threads": 2000,
},
{
"node_code": "mainland-worker-01",
"items_completed": 1614,
"items_running": 0,
"active_threads": 0,
"max_threads": 0,
},
{
"node_code": "unassigned",
"items_pending": 3495,
},
],
}
aggregate_queue_health = {
"has_active_job": True,
"job": {
"job_id": 11,
"job_code": "sync-overseas-51",
"status": "running",
"progress_percent": 40.49,
},
"queue": {
"items_total": 8402,
"pending": 5001,
"claimed": 0,
"running": 5909,
"display_running": 5909,
"completed": 0,
"blacklisted": 0,
"failed": 0,
},
"nodes": [
{
"node_code": "mainland-controller-01-a",
"items_running": 1000,
"display_running": 1000,
"active_threads": 1000,
"max_threads": 1000,
},
{
"node_code": "mainland-controller-01-b",
"items_running": 972,
"display_running": 972,
"active_threads": 972,
"max_threads": 1000,
},
{
"node_code": "mainland-controller-01-c",
"items_running": 3937,
"display_running": 3937,
"active_threads": 3937,
"max_threads": 36000,
},
],
}
def _resolve_thread_count(*, node_code=None, settings_payload=None):
if node_code == "mainland-controller-01" or str(node_code or "").startswith("mainland-controller-01-"):
return {
"effective_thread_count": 1000,
"default_thread_count": 1000,
"source": "default",
"override_thread_count": None,
"node_code": str(node_code or "mainland-controller-01"),
}
return {
"effective_thread_count": 1,
"default_thread_count": 1000,
"source": "node_override",
"override_thread_count": 1,
"node_code": "overseas-control-01",
}
def _resolve_process_count(*, node_code=None, settings_payload=None):
if node_code == "mainland-controller-01":
return {
"effective_process_count": 80,
"default_process_count": 80,
"source": "default",
"override_process_count": None,
"node_code": "mainland-controller-01",
}
if str(node_code or "").startswith("mainland-controller-01-"):
return {
"effective_process_count": 1,
"default_process_count": 80,
"source": "child_instance",
"override_process_count": None,
"node_code": str(node_code or ""),
}
return {
"effective_process_count": 1,
"default_process_count": 80,
"source": "node_override",
"override_process_count": 1,
"node_code": "overseas-control-01",
}
with ExitStack() as stack:
stack.enter_context(patch.object(detect_service.settings, "node_region", "overseas"))
stack.enter_context(patch.object(detect_service.settings, "node_role", "control"))
stack.enter_context(patch.object(detect_service.settings, "node_code", "overseas-control-01"))
stack.enter_context(patch("app.services.detect_service.ensure_runtime_schema"))
stack.enter_context(patch("app.services.detect_service.get_db", side_effect=RuntimeError("db down")))
stack.enter_context(
patch(
"app.services.detect_service.get_settings_payload",
return_value={
"proxy_config": {"proxy_enable": False, "allow_direct": True, "proxy_urls": []},
"process_count": 80,
"node_process_counts": {},
"thread_count": 1000,
"node_thread_counts": {"overseas-control-01": 1},
},
)
)
stack.enter_context(
patch("app.services.detect_service.get_runtime_settings", return_value={"worker_log_sync_enabled": False, "worker_log_sync_mode": "key"})
)
stack.enter_context(
patch("app.services.detect_service._load_recent_worker_lines", return_value=(False, "2026-04-23 00:00:00", ["stale line"]))
)
stack.enter_context(
patch("app.services.detect_service.detect_worker_runtime", return_value={"mode": "linux-systemd", "running": False, "process_count": 0, "latest_start_time": "", "message": "inactive"})
)
stack.enter_context(
patch("app.services.detect_service._load_runtime_state", return_value={"service_running": True, "detecting": True, "active_threads": 2, "max_threads": 1, "detail": "当前实际线程数量: 2/1"})
)
stack.enter_context(patch("app.services.detect_service._load_runtime_state_from_cluster_node", return_value={}))
stack.enter_context(patch("app.services.detect_service._extract_available_proxy_count", return_value=0))
stack.enter_context(patch("app.services.detect_service._extract_active_thread_snapshot", return_value={"active": 0, "max": 0}))
stack.enter_context(patch("app.services.detect_service._normalize_recent_warning", return_value=""))
stack.enter_context(
patch(
"app.services.detect_service._build_proxy_runtime_snapshot",
return_value={
"state": "disabled",
"label": "未启用代理",
"detail": "-",
"direct_fallback_active": True,
"reason": "proxy_disabled",
"last_refresh_status": "",
"last_refresh_time": "",
"source_count": 0,
"raw_items": 0,
"validated_count": 0,
"available_count": 0,
"source_stats": [],
"supplier_empty": False,
},
)
)
stack.enter_context(patch("app.services.detect_service.resolve_thread_count", side_effect=_resolve_thread_count))
stack.enter_context(patch("app.services.detect_service.resolve_process_count", side_effect=_resolve_process_count))
stack.enter_context(patch("app.services.detect_service.get_active_detect_job_summary", return_value=active_job))
stack.enter_context(patch("app.services.detect_service.get_detect_queue_health", return_value=aggregate_queue_health))
stack.enter_context(
patch(
"app.services.detect_service._load_runtime_states_from_cluster_nodes",
return_value={
"mainland-controller-01": {
"node_code": "mainland-controller-01",
"available_proxy_count": 486,
"proxy_runtime_label": "代理正常",
"proxy_runtime_reason": "healthy",
"proxy_last_refresh_status": "复用共享代理快照 486 个",
"proxy_last_refresh_time": "2026-04-23 19:07:12",
"proxy_last_refresh_source_count": 6,
"proxy_last_refresh_total_items": 120,
"proxy_last_validated_count": 0,
}
},
)
)
stack.enter_context(patch("app.services.detect_service.sync_detect_runs", return_value=[]))
stack.enter_context(patch("app.services.detect_service._resolve_remote_log_snapshot", return_value={}))
stack.enter_context(patch("app.services.detect_service._extract_dependency_alerts", return_value=[]))
stack.enter_context(patch("app.services.detect_service.append_detect_result_projection_if_changed"))
payload = detect_service.get_detect_status()
self.assertFalse(payload["worker_online"])
self.assertEqual({}, payload["runtime_state"])
self.assertEqual(3, payload["worker_process_count"])
self.assertEqual(3, payload["aggregate_process_count"])
self.assertEqual(3, payload["aggregate_participating_node_count"])
self.assertEqual(
["mainland-controller-01-a", "mainland-controller-01-b", "mainland-controller-01-c"],
payload["aggregate_participating_node_codes"],
)
self.assertEqual(3000, payload["aggregate_max_thread_count"])
self.assertEqual(38000, payload["max_thread_count"])
self.assertEqual(5909, payload["active_thread_count"])
self.assertEqual(486, payload["available_proxy_count"])
self.assertEqual("集群代理正常", payload["proxy_runtime_label"])
self.assertIn("486", payload["proxy_last_refresh_status"])
def test_get_detect_status_uses_queue_running_when_aggregate_display_fields_are_zero(self) -> None:
active_job = {
"job_id": 60,
"job_code": "sync-overseas-255",
"status": "running",
"items_total": 8402,
"items_pending": 7865,
"items_running": 537,
"items_completed": 0,
"items_failed": 0,
"items_blacklisted": 0,
"progress_percent": 6.39,
"display_items_running": 0,
"display_active_threads": 0,
"display_max_threads": 0,
"distributed_node_stats": [],
}
aggregate_queue_health = {
"has_active_job": True,
"job": {
"job_id": 60,
"job_code": "sync-overseas-255",
"status": "running",
"progress_percent": 6.39,
},
"queue": {
"items_total": 8402,
"pending": 7865,
"claimed": 0,
"running": 537,
"display_running": 0,
"completed": 0,
"blacklisted": 0,
"failed": 0,
},
"nodes": [
{
"node_code": "mainland-controller-01-a",
"items_running": 537,
"display_running": 537,
"active_threads": 537,
"max_threads": 1000,
}
],
}
with ExitStack() as stack:
stack.enter_context(patch.object(detect_service.settings, "node_region", "overseas"))
stack.enter_context(patch.object(detect_service.settings, "node_role", "control"))
stack.enter_context(patch.object(detect_service.settings, "node_code", "overseas-control-01"))
stack.enter_context(patch("app.services.detect_service.ensure_runtime_schema"))
stack.enter_context(patch("app.services.detect_service.get_db", side_effect=RuntimeError("db down")))
stack.enter_context(
patch(
"app.services.detect_service.get_settings_payload",
return_value={
"proxy_config": {"proxy_enable": False, "allow_direct": True, "proxy_urls": []},
"process_count": 80,
"node_process_counts": {"mainland-controller-01": 24},
"thread_count": 1000,
"node_thread_counts": {"overseas-control-01": 1, "mainland-controller-01-a": 1000},
},
)
)
stack.enter_context(
patch("app.services.detect_service.get_runtime_settings", return_value={"worker_log_sync_enabled": False, "worker_log_sync_mode": "key"})
)
stack.enter_context(
patch("app.services.detect_service._load_recent_worker_lines", return_value=(False, "2026-04-23 00:00:00", ["stale line"]))
)
stack.enter_context(
patch("app.services.detect_service.detect_worker_runtime", return_value={"mode": "linux-systemd", "running": False, "process_count": 0, "latest_start_time": "", "message": "inactive"})
)
stack.enter_context(
patch("app.services.detect_service._load_runtime_state", return_value={"service_running": False, "detecting": False, "active_threads": 0, "max_threads": 1, "detail": ""})
)
stack.enter_context(patch("app.services.detect_service._load_runtime_state_from_cluster_node", return_value={}))
stack.enter_context(patch("app.services.detect_service._extract_available_proxy_count", return_value=0))
stack.enter_context(patch("app.services.detect_service._extract_active_thread_snapshot", return_value={"active": 0, "max": 0}))
stack.enter_context(patch("app.services.detect_service._normalize_recent_warning", return_value=""))
stack.enter_context(
patch(
"app.services.detect_service._build_proxy_runtime_snapshot",
return_value={
"state": "disabled",
"label": "未启用代理",
"detail": "-",
"direct_fallback_active": True,
"reason": "proxy_disabled",
"last_refresh_status": "",
"last_refresh_time": "",
"source_count": 0,
"raw_items": 0,
"validated_count": 0,
"available_count": 0,
"source_stats": [],
"supplier_empty": False,
},
)
)
stack.enter_context(
patch(
"app.services.detect_service.resolve_thread_count",
return_value={
"effective_thread_count": 1,
"default_thread_count": 1000,
"source": "node_override",
"override_thread_count": 1,
"node_code": "overseas-control-01",
},
)
)
stack.enter_context(
patch(
"app.services.detect_service.resolve_process_count",
return_value={
"effective_process_count": 1,
"default_process_count": 80,
"source": "node_override",
"override_process_count": 1,
"node_code": "overseas-control-01",
},
)
)
stack.enter_context(patch("app.services.detect_service.get_active_detect_job_summary", return_value=active_job))
stack.enter_context(patch("app.services.detect_service.get_detect_queue_health", return_value=aggregate_queue_health))
stack.enter_context(patch("app.services.detect_service.sync_detect_runs", return_value=[]))
stack.enter_context(patch("app.services.detect_service._resolve_remote_log_snapshot", return_value={}))
stack.enter_context(patch("app.services.detect_service._extract_dependency_alerts", return_value=[]))
stack.enter_context(patch("app.services.detect_service.append_detect_result_projection_if_changed"))
payload = detect_service.get_detect_status()
self.assertEqual(537, payload["progress"]["running"])
self.assertEqual(537, payload["active_thread_count"])
self.assertEqual(1000, payload["max_thread_count"])
self.assertEqual(1, payload["aggregate_process_count"])
def test_get_detect_status_drops_stale_aggregate_nodes_from_running_and_process_counts(self) -> None:
live_heartbeat = datetime.now().isoformat(sep=" ", timespec="seconds")
stale_heartbeat = (datetime.now() - timedelta(minutes=8)).isoformat(sep=" ", timespec="seconds")
active_job = {
"job_id": 88,
"job_code": "sync-overseas-688",
"status": "running",
"items_total": 2600,
"items_pending": 500,
"items_running": 2100,
"items_completed": 0,
"items_failed": 0,
"items_blacklisted": 0,
"progress_percent": 80.77,
"display_items_running": 2100,
"display_active_threads": 2100,
"display_max_threads": 2000,
"distributed_node_stats": [
{
"node_code": "mainland-controller-01-a",
"items_running": 300,
"display_running": 300,
"active_threads": 300,
"max_threads": 1000,
"status": "busy",
"last_heartbeat_at": live_heartbeat,
},
{
"node_code": "mainland-controller-01-b",
"items_running": 1800,
"display_running": 1800,
"active_threads": 1800,
"max_threads": 1000,
"status": "stale",
"last_heartbeat_at": stale_heartbeat,
},
],
}
aggregate_queue_health = {
"has_active_job": True,
"job": {
"job_id": 88,
"job_code": "sync-overseas-688",
"status": "running",
"progress_percent": 80.77,
},
"queue": {
"items_total": 2600,
"pending": 500,
"claimed": 0,
"running": 2100,
"display_running": 2100,
"completed": 0,
"blacklisted": 0,
"failed": 0,
},
"nodes": [
{
"node_code": "mainland-controller-01-a",
"items_running": 300,
"display_running": 300,
"active_threads": 300,
"max_threads": 1000,
"status": "busy",
"last_heartbeat_at": live_heartbeat,
},
{
"node_code": "mainland-controller-01-b",
"items_running": 1800,
"display_running": 1800,
"active_threads": 1800,
"max_threads": 1000,
"status": "stale",
"last_heartbeat_at": stale_heartbeat,
},
],
}
with ExitStack() as stack:
stack.enter_context(patch.object(detect_service.settings, "node_region", "overseas"))
stack.enter_context(patch.object(detect_service.settings, "node_role", "control"))
stack.enter_context(patch.object(detect_service.settings, "node_code", "overseas-control-01"))
stack.enter_context(patch("app.services.detect_service.ensure_runtime_schema"))
stack.enter_context(patch("app.services.detect_service.get_db", side_effect=RuntimeError("db down")))
stack.enter_context(
patch(
"app.services.detect_service.get_settings_payload",
return_value={
"proxy_config": {"proxy_enable": False, "allow_direct": True, "proxy_urls": []},
"process_count": 80,
"node_process_counts": {"mainland-controller-01": 70},
"thread_count": 1000,
"node_thread_counts": {"overseas-control-01": 1},
},
)
)
stack.enter_context(
patch("app.services.detect_service.get_runtime_settings", return_value={"worker_log_sync_enabled": False, "worker_log_sync_mode": "key"})
)
stack.enter_context(
patch("app.services.detect_service._load_recent_worker_lines", return_value=(False, "2026-04-23 00:00:00", ["stale line"]))
)
stack.enter_context(
patch("app.services.detect_service.detect_worker_runtime", return_value={"mode": "linux-systemd", "running": False, "process_count": 0, "latest_start_time": "", "message": "inactive"})
)
stack.enter_context(
patch("app.services.detect_service._load_runtime_state", return_value={"service_running": False, "detecting": False, "active_threads": 0, "max_threads": 1, "detail": ""})
)
stack.enter_context(patch("app.services.detect_service._load_runtime_state_from_cluster_node", return_value={}))
stack.enter_context(patch("app.services.detect_service._extract_available_proxy_count", return_value=0))
stack.enter_context(patch("app.services.detect_service._extract_active_thread_snapshot", return_value={"active": 0, "max": 0}))
stack.enter_context(patch("app.services.detect_service._normalize_recent_warning", return_value=""))
stack.enter_context(
patch(
"app.services.detect_service._build_proxy_runtime_snapshot",
return_value={
"state": "disabled",
"label": "未启用代理",
"detail": "-",
"direct_fallback_active": True,
"reason": "proxy_disabled",
"last_refresh_status": "",
"last_refresh_time": "",
"source_count": 0,
"raw_items": 0,
"validated_count": 0,
"available_count": 0,
"source_stats": [],
"supplier_empty": False,
},
)
)
stack.enter_context(
patch(
"app.services.detect_service.resolve_thread_count",
return_value={
"effective_thread_count": 1,
"default_thread_count": 1000,
"source": "node_override",
"override_thread_count": 1,
"node_code": "overseas-control-01",
},
)
)
stack.enter_context(
patch(
"app.services.detect_service.resolve_process_count",
side_effect=[
{
"effective_process_count": 1,
"default_process_count": 80,
"source": "child_instance",
"override_process_count": None,
"node_code": "mainland-controller-01-a",
}
],
)
)
stack.enter_context(patch("app.services.detect_service.get_active_detect_job_summary", return_value=active_job))
stack.enter_context(patch("app.services.detect_service.get_detect_queue_health", return_value=aggregate_queue_health))
stack.enter_context(patch("app.services.detect_service.sync_detect_runs", return_value=[]))
stack.enter_context(patch("app.services.detect_service._resolve_remote_log_snapshot", return_value={}))
stack.enter_context(patch("app.services.detect_service._extract_dependency_alerts", return_value=[]))
stack.enter_context(patch("app.services.detect_service.append_detect_result_projection_if_changed"))
payload = detect_service.get_detect_status()
self.assertEqual(300, payload["progress"]["running"])
self.assertEqual(300, payload["active_thread_count"])
self.assertEqual(1000, payload["max_thread_count"])
self.assertEqual(1, payload["aggregate_process_count"])
self.assertEqual(1, payload["aggregate_participating_node_count"])
self.assertEqual(["mainland-controller-01-a"], payload["aggregate_participating_node_codes"])
def test_get_detect_status_uses_event_proxy_counts_when_cluster_runtime_proxy_counts_are_missing(self) -> None:
active_job = {
"job_id": 77,
"job_code": "sync-overseas-482",
"status": "running",
"items_pending": 410,
"items_completed": 100,
"items_failed": 0,
"items_blacklisted": 0,
"progress_percent": 55.0,
"display_items_running": 807,
"display_active_threads": 807,
"current_cycle_token": "cycle-1",
"distributed_node_stats": [
{
"node_code": "mainland-controller-01-u",
"items_running": 300,
"display_running": 300,
"active_threads": 300,
"max_threads": 1000,
},
{
"node_code": "mainland-controller-01-v",
"items_running": 301,
"display_running": 301,
"active_threads": 301,
"max_threads": 1000,
},
{
"node_code": "mainland-worker-01-a",
"items_running": 206,
"display_running": 206,
"active_threads": 206,
"max_threads": 1000,
},
],
"current_cycle_events": [
{
"event_type": "worker_log",
"created_at": "2026-04-23 19:40:00",
"node_code": "mainland-controller-01-u",
"message": "当前可用代理数: 486",
"payload": {"cycle_token": "cycle-1", "log_mode": "key"},
},
{
"event_type": "worker_log",
"created_at": "2026-04-23 19:40:10",
"node_code": "mainland-controller-01-v",
"message": "代理池刷新完成，共 486 个可用代理，来源链接 6 个，原始 520 个",
"payload": {"cycle_token": "cycle-1", "log_mode": "key"},
},
{
"event_type": "worker_log",
"created_at": "2026-04-23 19:40:20",
"node_code": "mainland-worker-01-a",
"message": "共享刷新进行中，继续沿用缓存 321 个",
"payload": {"cycle_token": "cycle-1", "log_mode": "key"},
},
],
}
aggregate_queue_health = {
"has_active_job": True,
"job": {
"job_id": 77,
"job_code": "sync-overseas-482",
"status": "running",
"progress_percent": 55.0,
},
"queue": {
"items_total": 1317,
"pending": 410,
"claimed": 0,
"running": 807,
"display_running": 807,
"completed": 100,
"blacklisted": 0,
"failed": 0,
},
"nodes": [
{
"node_code": "mainland-controller-01-u",
"items_running": 300,
"display_running": 300,
"active_threads": 300,
"max_threads": 1000,
},
{
"node_code": "mainland-controller-01-v",
"items_running": 301,
"display_running": 301,
"active_threads": 301,
"max_threads": 1000,
},
{
"node_code": "mainland-worker-01-a",
"items_running": 206,
"display_running": 206,
"active_threads": 206,
"max_threads": 1000,
},
],
}
def _resolve_thread_count(*, node_code=None, settings_payload=None):
if str(node_code or "").startswith("mainland-"):
return {
"effective_thread_count": 1000,
"default_thread_count": 1000,
"source": "default",
"override_thread_count": None,
"node_code": str(node_code or ""),
}
return {
"effective_thread_count": 1,
"default_thread_count": 1000,
"source": "node_override",
"override_thread_count": 1,
"node_code": "overseas-control-01",
}
def _resolve_process_count(*, node_code=None, settings_payload=None):
normalized_node_code = str(node_code or "")
if normalized_node_code == "mainland-controller-01":
return {
"effective_process_count": 80,
"default_process_count": 80,
"source": "default",
"override_process_count": None,
"node_code": normalized_node_code,
}
if normalized_node_code == "mainland-worker-01":
return {
"effective_process_count": 60,
"default_process_count": 60,
"source": "default",
"override_process_count": None,
"node_code": normalized_node_code,
}
if normalized_node_code.startswith("mainland-"):
return {
"effective_process_count": 1,
"default_process_count": 80,
"source": "child_instance",
"override_process_count": None,
"node_code": normalized_node_code,
}
return {
"effective_process_count": 1,
"default_process_count": 80,
"source": "node_override",
"override_process_count": 1,
"node_code": "overseas-control-01",
}
with ExitStack() as stack:
stack.enter_context(patch.object(detect_service.settings, "node_region", "overseas"))
stack.enter_context(patch.object(detect_service.settings, "node_role", "control"))
stack.enter_context(patch.object(detect_service.settings, "node_code", "overseas-control-01"))
stack.enter_context(patch("app.services.detect_service.ensure_runtime_schema"))
stack.enter_context(patch("app.services.detect_service.get_db", side_effect=RuntimeError("db down")))
stack.enter_context(
patch(
"app.services.detect_service.get_settings_payload",
return_value={
"proxy_config": {"proxy_enable": False, "allow_direct": True, "proxy_urls": []},
"process_count": 80,
"node_process_counts": {"mainland-worker-01": 60},
"thread_count": 1000,
"node_thread_counts": {"overseas-control-01": 1},
},
)
)
stack.enter_context(
patch("app.services.detect_service.get_runtime_settings", return_value={"worker_log_sync_enabled": False, "worker_log_sync_mode": "key"})
)
stack.enter_context(
patch("app.services.detect_service._load_recent_worker_lines", return_value=(False, "2026-04-23 00:00:00", ["stale line"]))
)
stack.enter_context(
patch("app.services.detect_service.detect_worker_runtime", return_value={"mode": "linux-systemd", "running": False, "process_count": 0, "latest_start_time": "", "message": "inactive"})
)
stack.enter_context(
patch("app.services.detect_service._load_runtime_state", return_value={"service_running": False, "detecting": False, "active_threads": 0, "max_threads": 1, "detail": ""})
)
stack.enter_context(patch("app.services.detect_service._load_runtime_state_from_cluster_node", return_value={}))
stack.enter_context(patch("app.services.detect_service._extract_available_proxy_count", return_value=0))
stack.enter_context(patch("app.services.detect_service._extract_active_thread_snapshot", return_value={"active": 0, "max": 0}))
stack.enter_context(patch("app.services.detect_service._normalize_recent_warning", return_value=""))
stack.enter_context(
patch(
"app.services.detect_service._build_proxy_runtime_snapshot",
return_value={
"state": "warming_up",
"label": "等待首刷",
"detail": "-",
"direct_fallback_active": False,
"reason": "waiting_for_first_refresh",
"last_refresh_status": "未刷新",
"last_refresh_time": "",
"source_count": 0,
"raw_items": 0,
"validated_count": 0,
"available_count": 0,
"source_stats": [],
"supplier_empty": False,
},
)
)
stack.enter_context(patch("app.services.detect_service.resolve_thread_count", side_effect=_resolve_thread_count))
stack.enter_context(patch("app.services.detect_service.resolve_process_count", side_effect=_resolve_process_count))
stack.enter_context(patch("app.services.detect_service.get_active_detect_job_summary", return_value=active_job))
stack.enter_context(patch("app.services.detect_service.get_detect_queue_health", return_value=aggregate_queue_health))
stack.enter_context(
patch(
"app.services.detect_service._load_runtime_states_from_cluster_nodes",
return_value={
"mainland-controller-01": {
"node_code": "mainland-controller-01",
"available_proxy_count": 0,
"proxy_runtime_label": "",
"proxy_runtime_reason": "",
"proxy_last_refresh_status": "",
"proxy_last_refresh_time": "",
"proxy_last_refresh_source_count": 0,
"proxy_last_refresh_total_items": 0,
"proxy_last_validated_count": 0,
},
"mainland-worker-01": {
"node_code": "mainland-worker-01",
"available_proxy_count": 0,
"proxy_runtime_label": "",
"proxy_runtime_reason": "",
"proxy_last_refresh_status": "",
"proxy_last_refresh_time": "",
"proxy_last_refresh_source_count": 0,
"proxy_last_refresh_total_items": 0,
"proxy_last_validated_count": 0,
},
},
)
)
stack.enter_context(patch("app.services.detect_service.sync_detect_runs", return_value=[]))
stack.enter_context(patch("app.services.detect_service._resolve_remote_log_snapshot", return_value={}))
stack.enter_context(patch("app.services.detect_service._extract_dependency_alerts", return_value=[]))
stack.enter_context(patch("app.services.detect_service.append_detect_result_projection_if_changed"))
payload = detect_service.get_detect_status()
self.assertEqual(807, payload["available_proxy_count"])
self.assertEqual("集群代理正常", payload["proxy_runtime_label"])
self.assertIn("参与服务器 2 台", payload["proxy_runtime_detail"])
if __name__ == "__main__":
unittest.main()