950 lines
48 KiB
Python
950 lines
48 KiB
Python
import unittest
|
|
from contextlib import ExitStack
|
|
from datetime import datetime, timedelta
|
|
from unittest.mock import patch
|
|
|
|
from app.services import detect_service
|
|
|
|
|
|
class DetectServiceStatusFallbackTests(unittest.TestCase):
|
|
def test_filter_live_aggregate_runtime_nodes_excludes_disabled_managed_nodes(self) -> None:
|
|
now_text = datetime.now().isoformat(sep=" ", timespec="seconds")
|
|
rows = [
|
|
{
|
|
"node_code": "mainland-controller-01",
|
|
"status": "busy",
|
|
"last_heartbeat_at": now_text,
|
|
"display_running": 1,
|
|
},
|
|
{
|
|
"node_code": "mainland-worker-01",
|
|
"status": "busy",
|
|
"last_heartbeat_at": now_text,
|
|
"display_running": 1,
|
|
},
|
|
]
|
|
|
|
with patch("app.services.detect_service._load_disabled_managed_node_codes", return_value={"mainland-worker-01"}):
|
|
filtered = detect_service._filter_live_aggregate_runtime_nodes(rows)
|
|
|
|
self.assertEqual(["mainland-controller-01"], [item["node_code"] for item in filtered])
|
|
|
|
def setUp(self) -> None:
|
|
detect_service._DETECT_STATUS_CACHE_VALUE = None
|
|
detect_service._DETECT_STATUS_CACHE_EXPIRES_AT = 0.0
|
|
|
|
def test_resolve_capacity_node_code_treats_child_instance_suffix_as_single_process(self) -> None:
|
|
capacity_node_code, is_child_instance = detect_service._resolve_capacity_node_code(
|
|
"mainland-controller-01-ae",
|
|
{"node_thread_counts": {"overseas-control-01": 1}, "node_process_counts": {}},
|
|
)
|
|
|
|
self.assertEqual("mainland-controller-01", capacity_node_code)
|
|
self.assertTrue(is_child_instance)
|
|
|
|
def test_get_detect_status_keeps_runtime_snapshot_when_db_is_unreachable(self) -> None:
|
|
runtime_state = {
|
|
"service_running": True,
|
|
"detecting": True,
|
|
"active_threads": 7,
|
|
"max_threads": 120,
|
|
"phase": "running",
|
|
"detail": "Worker 正在处理 7 个检测任务",
|
|
"updated_at": "2026-04-20 23:59:00",
|
|
"available_proxy_count": 18,
|
|
}
|
|
|
|
with ExitStack() as stack:
|
|
stack.enter_context(patch.object(detect_service.settings, "node_region", "mainland"))
|
|
stack.enter_context(patch.object(detect_service.settings, "node_role", "worker"))
|
|
stack.enter_context(patch.object(detect_service.settings, "node_code", "mainland-worker-01"))
|
|
stack.enter_context(patch("app.services.detect_service.ensure_runtime_schema"))
|
|
stack.enter_context(patch("app.services.detect_service.get_db", side_effect=RuntimeError("db down")))
|
|
stack.enter_context(
|
|
patch("app.services.detect_service.get_settings_payload", return_value={"proxy_config": {"proxy_enable": True, "allow_direct": False, "proxy_urls": ["a"]}})
|
|
)
|
|
stack.enter_context(patch("app.services.detect_service.get_runtime_settings", return_value={"worker_log_sync_enabled": False, "worker_log_sync_mode": "full"}))
|
|
stack.enter_context(patch("app.services.detect_service._load_recent_worker_lines", return_value=(True, "", [])))
|
|
stack.enter_context(
|
|
patch("app.services.detect_service.detect_worker_runtime", return_value={"mode": "linux-systemd", "running": True, "process_count": 1, "latest_start_time": "2026-04-20 23:58:00", "message": "active/running"})
|
|
)
|
|
stack.enter_context(patch("app.services.detect_service._load_runtime_state", return_value=runtime_state))
|
|
stack.enter_context(patch("app.services.detect_service._load_runtime_state_from_cluster_node", return_value={}))
|
|
stack.enter_context(patch("app.services.detect_service._extract_available_proxy_count", return_value=0))
|
|
stack.enter_context(patch("app.services.detect_service._extract_active_thread_snapshot", return_value={"active": 0, "max": 0}))
|
|
stack.enter_context(patch("app.services.detect_service._normalize_recent_warning", return_value=""))
|
|
stack.enter_context(
|
|
patch(
|
|
"app.services.detect_service._build_proxy_runtime_snapshot",
|
|
return_value={"state": "healthy", "label": "代理正常", "detail": "healthy", "direct_fallback_active": False, "reason": "healthy", "last_refresh_status": "ok", "last_refresh_time": "", "source_count": 2, "raw_items": 18, "validated_count": 18, "available_count": 18, "source_stats": [], "supplier_empty": False},
|
|
)
|
|
)
|
|
stack.enter_context(
|
|
patch("app.services.detect_service.resolve_thread_count", return_value={"effective_thread_count": 120, "default_thread_count": 5, "source": "node_override", "override_thread_count": 120, "node_code": "mainland-worker-01"})
|
|
)
|
|
stack.enter_context(patch("app.services.detect_service.get_active_detect_job_summary", side_effect=RuntimeError("db down")))
|
|
stack.enter_context(patch("app.services.detect_service.sync_detect_runs", return_value=[]))
|
|
stack.enter_context(patch("app.services.detect_service._resolve_remote_log_snapshot", return_value={}))
|
|
stack.enter_context(patch("app.services.detect_service._extract_dependency_alerts", return_value=[]))
|
|
stack.enter_context(patch("app.services.detect_service.append_detect_result_projection_if_changed"))
|
|
payload = detect_service.get_detect_status()
|
|
|
|
self.assertTrue(payload["worker_online"])
|
|
self.assertTrue(payload["detecting"])
|
|
self.assertEqual(7, payload["active_thread_count"])
|
|
self.assertEqual(120, payload["max_thread_count"])
|
|
self.assertEqual(0, payload["progress"]["pending"])
|
|
self.assertEqual(0, payload["progress"]["completed"])
|
|
|
|
def test_get_detect_status_skips_local_result_projection_on_overseas_control(self) -> None:
|
|
runtime_state = {
|
|
"service_running": False,
|
|
"detecting": False,
|
|
"active_threads": 0,
|
|
"max_threads": 1,
|
|
"phase": "idle",
|
|
"detail": "当前节点不承载本地检测执行",
|
|
"updated_at": "2026-04-24 02:10:00",
|
|
"available_proxy_count": 0,
|
|
}
|
|
|
|
with ExitStack() as stack:
|
|
stack.enter_context(patch.object(detect_service.settings, "node_region", "overseas"))
|
|
stack.enter_context(patch.object(detect_service.settings, "node_role", "control"))
|
|
stack.enter_context(patch.object(detect_service.settings, "node_code", "overseas-control-01"))
|
|
stack.enter_context(patch("app.services.detect_service.ensure_runtime_schema"))
|
|
stack.enter_context(patch("app.services.detect_service.get_db", side_effect=RuntimeError("db down")))
|
|
stack.enter_context(
|
|
patch("app.services.detect_service.get_settings_payload", return_value={"proxy_config": {"proxy_enable": False, "allow_direct": True, "proxy_urls": []}})
|
|
)
|
|
stack.enter_context(patch("app.services.detect_service.get_runtime_settings", return_value={"worker_log_sync_enabled": False, "worker_log_sync_mode": "key"}))
|
|
stack.enter_context(patch("app.services.detect_service._load_recent_worker_lines", return_value=(False, "", [])))
|
|
stack.enter_context(
|
|
patch("app.services.detect_service.detect_worker_runtime", return_value={"mode": "linux-systemd", "running": False, "process_count": 0, "latest_start_time": "", "message": "inactive"})
|
|
)
|
|
stack.enter_context(patch("app.services.detect_service._load_runtime_state", return_value=runtime_state))
|
|
stack.enter_context(patch("app.services.detect_service._load_runtime_state_from_cluster_node", return_value={}))
|
|
stack.enter_context(patch("app.services.detect_service._extract_available_proxy_count", return_value=0))
|
|
stack.enter_context(patch("app.services.detect_service._extract_active_thread_snapshot", return_value={"active": 0, "max": 0}))
|
|
stack.enter_context(patch("app.services.detect_service._normalize_recent_warning", return_value=""))
|
|
stack.enter_context(
|
|
patch(
|
|
"app.services.detect_service._build_proxy_runtime_snapshot",
|
|
return_value={"state": "disabled", "label": "不适用", "detail": "-", "direct_fallback_active": False, "reason": "not_applicable", "last_refresh_status": "", "last_refresh_time": "", "source_count": 0, "raw_items": 0, "validated_count": 0, "available_count": 0, "source_stats": [], "supplier_empty": False},
|
|
)
|
|
)
|
|
stack.enter_context(
|
|
patch("app.services.detect_service.resolve_thread_count", return_value={"effective_thread_count": 1, "default_thread_count": 1, "source": "default", "override_thread_count": None, "node_code": "overseas-control-01"})
|
|
)
|
|
stack.enter_context(patch("app.services.detect_service.get_active_detect_job_summary", return_value=None))
|
|
stack.enter_context(patch("app.services.detect_service.sync_detect_runs", return_value=[]))
|
|
stack.enter_context(patch("app.services.detect_service._resolve_remote_log_snapshot", return_value={}))
|
|
stack.enter_context(patch("app.services.detect_service._extract_dependency_alerts", return_value=[]))
|
|
mock_append = stack.enter_context(patch("app.services.detect_service.append_detect_result_projection_if_changed"))
|
|
|
|
detect_service.get_detect_status()
|
|
|
|
mock_append.assert_not_called()
|
|
|
|
def test_filter_lines_since_supports_journalctl_syslog_timestamps(self) -> None:
|
|
lines = [
|
|
"Apr 21 20:12:42 mainland-controller python[1]: 当前实际线程数量: 323/4",
|
|
"Apr 21 20:17:20 mainland-controller python[2]: Worker 已启动，等待检测指令",
|
|
]
|
|
|
|
filtered = detect_service._filter_lines_since(lines, "2026-04-21 20:17:00")
|
|
|
|
self.assertEqual(
|
|
["Apr 21 20:17:20 mainland-controller python[2]: Worker 已启动，等待检测指令"],
|
|
filtered,
|
|
)
|
|
|
|
def test_filter_lines_since_falls_back_when_no_timestamp_is_parseable(self) -> None:
|
|
lines = ["no timestamp line 1", "no timestamp line 2"]
|
|
|
|
filtered = detect_service._filter_lines_since(lines, "2026-04-21 20:17:00")
|
|
|
|
self.assertEqual(lines, filtered)
|
|
|
|
def test_get_detect_status_uses_aggregate_capacity_for_overseas_control(self) -> None:
|
|
active_job = {
|
|
"job_id": 11,
|
|
"job_code": "sync-overseas-51",
|
|
"status": "running",
|
|
"items_pending": 5001,
|
|
"items_completed": 1788,
|
|
"items_failed": 0,
|
|
"items_blacklisted": 0,
|
|
"progress_percent": 40.49,
|
|
"display_items_running": 1432,
|
|
"display_active_threads": 138,
|
|
"distributed_node_stats": [
|
|
{
|
|
"node_code": "mainland-controller-01",
|
|
"items_claimed": 73,
|
|
"items_running": 1432,
|
|
"display_running": 1432,
|
|
"active_threads": 138,
|
|
"max_threads": 2000,
|
|
},
|
|
{
|
|
"node_code": "mainland-worker-01",
|
|
"items_completed": 1614,
|
|
"items_running": 0,
|
|
"active_threads": 0,
|
|
"max_threads": 0,
|
|
},
|
|
{
|
|
"node_code": "unassigned",
|
|
"items_pending": 3495,
|
|
},
|
|
],
|
|
}
|
|
aggregate_queue_health = {
|
|
"has_active_job": True,
|
|
"job": {
|
|
"job_id": 11,
|
|
"job_code": "sync-overseas-51",
|
|
"status": "running",
|
|
"progress_percent": 40.49,
|
|
},
|
|
"queue": {
|
|
"items_total": 8402,
|
|
"pending": 5001,
|
|
"claimed": 0,
|
|
"running": 5909,
|
|
"display_running": 5909,
|
|
"completed": 0,
|
|
"blacklisted": 0,
|
|
"failed": 0,
|
|
},
|
|
"nodes": [
|
|
{
|
|
"node_code": "mainland-controller-01-a",
|
|
"items_running": 1000,
|
|
"display_running": 1000,
|
|
"active_threads": 1000,
|
|
"max_threads": 1000,
|
|
},
|
|
{
|
|
"node_code": "mainland-controller-01-b",
|
|
"items_running": 972,
|
|
"display_running": 972,
|
|
"active_threads": 972,
|
|
"max_threads": 1000,
|
|
},
|
|
{
|
|
"node_code": "mainland-controller-01-c",
|
|
"items_running": 3937,
|
|
"display_running": 3937,
|
|
"active_threads": 3937,
|
|
"max_threads": 36000,
|
|
},
|
|
],
|
|
}
|
|
|
|
def _resolve_thread_count(*, node_code=None, settings_payload=None):
|
|
if node_code == "mainland-controller-01" or str(node_code or "").startswith("mainland-controller-01-"):
|
|
return {
|
|
"effective_thread_count": 1000,
|
|
"default_thread_count": 1000,
|
|
"source": "default",
|
|
"override_thread_count": None,
|
|
"node_code": str(node_code or "mainland-controller-01"),
|
|
}
|
|
return {
|
|
"effective_thread_count": 1,
|
|
"default_thread_count": 1000,
|
|
"source": "node_override",
|
|
"override_thread_count": 1,
|
|
"node_code": "overseas-control-01",
|
|
}
|
|
|
|
def _resolve_process_count(*, node_code=None, settings_payload=None):
|
|
if node_code == "mainland-controller-01":
|
|
return {
|
|
"effective_process_count": 80,
|
|
"default_process_count": 80,
|
|
"source": "default",
|
|
"override_process_count": None,
|
|
"node_code": "mainland-controller-01",
|
|
}
|
|
if str(node_code or "").startswith("mainland-controller-01-"):
|
|
return {
|
|
"effective_process_count": 1,
|
|
"default_process_count": 80,
|
|
"source": "child_instance",
|
|
"override_process_count": None,
|
|
"node_code": str(node_code or ""),
|
|
}
|
|
return {
|
|
"effective_process_count": 1,
|
|
"default_process_count": 80,
|
|
"source": "node_override",
|
|
"override_process_count": 1,
|
|
"node_code": "overseas-control-01",
|
|
}
|
|
|
|
with ExitStack() as stack:
|
|
stack.enter_context(patch.object(detect_service.settings, "node_region", "overseas"))
|
|
stack.enter_context(patch.object(detect_service.settings, "node_role", "control"))
|
|
stack.enter_context(patch.object(detect_service.settings, "node_code", "overseas-control-01"))
|
|
stack.enter_context(patch("app.services.detect_service.ensure_runtime_schema"))
|
|
stack.enter_context(patch("app.services.detect_service.get_db", side_effect=RuntimeError("db down")))
|
|
stack.enter_context(
|
|
patch(
|
|
"app.services.detect_service.get_settings_payload",
|
|
return_value={
|
|
"proxy_config": {"proxy_enable": False, "allow_direct": True, "proxy_urls": []},
|
|
"process_count": 80,
|
|
"node_process_counts": {},
|
|
"thread_count": 1000,
|
|
"node_thread_counts": {"overseas-control-01": 1},
|
|
},
|
|
)
|
|
)
|
|
stack.enter_context(
|
|
patch("app.services.detect_service.get_runtime_settings", return_value={"worker_log_sync_enabled": False, "worker_log_sync_mode": "key"})
|
|
)
|
|
stack.enter_context(
|
|
patch("app.services.detect_service._load_recent_worker_lines", return_value=(False, "2026-04-23 00:00:00", ["stale line"]))
|
|
)
|
|
stack.enter_context(
|
|
patch("app.services.detect_service.detect_worker_runtime", return_value={"mode": "linux-systemd", "running": False, "process_count": 0, "latest_start_time": "", "message": "inactive"})
|
|
)
|
|
stack.enter_context(
|
|
patch("app.services.detect_service._load_runtime_state", return_value={"service_running": True, "detecting": True, "active_threads": 2, "max_threads": 1, "detail": "当前实际线程数量: 2/1"})
|
|
)
|
|
stack.enter_context(patch("app.services.detect_service._load_runtime_state_from_cluster_node", return_value={}))
|
|
stack.enter_context(patch("app.services.detect_service._extract_available_proxy_count", return_value=0))
|
|
stack.enter_context(patch("app.services.detect_service._extract_active_thread_snapshot", return_value={"active": 0, "max": 0}))
|
|
stack.enter_context(patch("app.services.detect_service._normalize_recent_warning", return_value=""))
|
|
stack.enter_context(
|
|
patch(
|
|
"app.services.detect_service._build_proxy_runtime_snapshot",
|
|
return_value={
|
|
"state": "disabled",
|
|
"label": "未启用代理",
|
|
"detail": "-",
|
|
"direct_fallback_active": True,
|
|
"reason": "proxy_disabled",
|
|
"last_refresh_status": "",
|
|
"last_refresh_time": "",
|
|
"source_count": 0,
|
|
"raw_items": 0,
|
|
"validated_count": 0,
|
|
"available_count": 0,
|
|
"source_stats": [],
|
|
"supplier_empty": False,
|
|
},
|
|
)
|
|
)
|
|
stack.enter_context(patch("app.services.detect_service.resolve_thread_count", side_effect=_resolve_thread_count))
|
|
stack.enter_context(patch("app.services.detect_service.resolve_process_count", side_effect=_resolve_process_count))
|
|
stack.enter_context(patch("app.services.detect_service.get_active_detect_job_summary", return_value=active_job))
|
|
stack.enter_context(patch("app.services.detect_service.get_detect_queue_health", return_value=aggregate_queue_health))
|
|
stack.enter_context(
|
|
patch(
|
|
"app.services.detect_service._load_runtime_states_from_cluster_nodes",
|
|
return_value={
|
|
"mainland-controller-01": {
|
|
"node_code": "mainland-controller-01",
|
|
"available_proxy_count": 486,
|
|
"proxy_runtime_label": "代理正常",
|
|
"proxy_runtime_reason": "healthy",
|
|
"proxy_last_refresh_status": "复用共享代理快照 486 个",
|
|
"proxy_last_refresh_time": "2026-04-23 19:07:12",
|
|
"proxy_last_refresh_source_count": 6,
|
|
"proxy_last_refresh_total_items": 120,
|
|
"proxy_last_validated_count": 0,
|
|
}
|
|
},
|
|
)
|
|
)
|
|
stack.enter_context(patch("app.services.detect_service.sync_detect_runs", return_value=[]))
|
|
stack.enter_context(patch("app.services.detect_service._resolve_remote_log_snapshot", return_value={}))
|
|
stack.enter_context(patch("app.services.detect_service._extract_dependency_alerts", return_value=[]))
|
|
stack.enter_context(patch("app.services.detect_service.append_detect_result_projection_if_changed"))
|
|
payload = detect_service.get_detect_status()
|
|
|
|
self.assertFalse(payload["worker_online"])
|
|
self.assertEqual({}, payload["runtime_state"])
|
|
self.assertEqual(3, payload["worker_process_count"])
|
|
self.assertEqual(3, payload["aggregate_process_count"])
|
|
self.assertEqual(3, payload["aggregate_participating_node_count"])
|
|
self.assertEqual(
|
|
["mainland-controller-01-a", "mainland-controller-01-b", "mainland-controller-01-c"],
|
|
payload["aggregate_participating_node_codes"],
|
|
)
|
|
self.assertEqual(3000, payload["aggregate_max_thread_count"])
|
|
self.assertEqual(38000, payload["max_thread_count"])
|
|
self.assertEqual(5909, payload["active_thread_count"])
|
|
self.assertEqual(486, payload["available_proxy_count"])
|
|
self.assertEqual("集群代理正常", payload["proxy_runtime_label"])
|
|
self.assertIn("486", payload["proxy_last_refresh_status"])
|
|
|
|
def test_get_detect_status_uses_queue_running_when_aggregate_display_fields_are_zero(self) -> None:
|
|
active_job = {
|
|
"job_id": 60,
|
|
"job_code": "sync-overseas-255",
|
|
"status": "running",
|
|
"items_total": 8402,
|
|
"items_pending": 7865,
|
|
"items_running": 537,
|
|
"items_completed": 0,
|
|
"items_failed": 0,
|
|
"items_blacklisted": 0,
|
|
"progress_percent": 6.39,
|
|
"display_items_running": 0,
|
|
"display_active_threads": 0,
|
|
"display_max_threads": 0,
|
|
"distributed_node_stats": [],
|
|
}
|
|
aggregate_queue_health = {
|
|
"has_active_job": True,
|
|
"job": {
|
|
"job_id": 60,
|
|
"job_code": "sync-overseas-255",
|
|
"status": "running",
|
|
"progress_percent": 6.39,
|
|
},
|
|
"queue": {
|
|
"items_total": 8402,
|
|
"pending": 7865,
|
|
"claimed": 0,
|
|
"running": 537,
|
|
"display_running": 0,
|
|
"completed": 0,
|
|
"blacklisted": 0,
|
|
"failed": 0,
|
|
},
|
|
"nodes": [
|
|
{
|
|
"node_code": "mainland-controller-01-a",
|
|
"items_running": 537,
|
|
"display_running": 537,
|
|
"active_threads": 537,
|
|
"max_threads": 1000,
|
|
}
|
|
],
|
|
}
|
|
|
|
with ExitStack() as stack:
|
|
stack.enter_context(patch.object(detect_service.settings, "node_region", "overseas"))
|
|
stack.enter_context(patch.object(detect_service.settings, "node_role", "control"))
|
|
stack.enter_context(patch.object(detect_service.settings, "node_code", "overseas-control-01"))
|
|
stack.enter_context(patch("app.services.detect_service.ensure_runtime_schema"))
|
|
stack.enter_context(patch("app.services.detect_service.get_db", side_effect=RuntimeError("db down")))
|
|
stack.enter_context(
|
|
patch(
|
|
"app.services.detect_service.get_settings_payload",
|
|
return_value={
|
|
"proxy_config": {"proxy_enable": False, "allow_direct": True, "proxy_urls": []},
|
|
"process_count": 80,
|
|
"node_process_counts": {"mainland-controller-01": 24},
|
|
"thread_count": 1000,
|
|
"node_thread_counts": {"overseas-control-01": 1, "mainland-controller-01-a": 1000},
|
|
},
|
|
)
|
|
)
|
|
stack.enter_context(
|
|
patch("app.services.detect_service.get_runtime_settings", return_value={"worker_log_sync_enabled": False, "worker_log_sync_mode": "key"})
|
|
)
|
|
stack.enter_context(
|
|
patch("app.services.detect_service._load_recent_worker_lines", return_value=(False, "2026-04-23 00:00:00", ["stale line"]))
|
|
)
|
|
stack.enter_context(
|
|
patch("app.services.detect_service.detect_worker_runtime", return_value={"mode": "linux-systemd", "running": False, "process_count": 0, "latest_start_time": "", "message": "inactive"})
|
|
)
|
|
stack.enter_context(
|
|
patch("app.services.detect_service._load_runtime_state", return_value={"service_running": False, "detecting": False, "active_threads": 0, "max_threads": 1, "detail": ""})
|
|
)
|
|
stack.enter_context(patch("app.services.detect_service._load_runtime_state_from_cluster_node", return_value={}))
|
|
stack.enter_context(patch("app.services.detect_service._extract_available_proxy_count", return_value=0))
|
|
stack.enter_context(patch("app.services.detect_service._extract_active_thread_snapshot", return_value={"active": 0, "max": 0}))
|
|
stack.enter_context(patch("app.services.detect_service._normalize_recent_warning", return_value=""))
|
|
stack.enter_context(
|
|
patch(
|
|
"app.services.detect_service._build_proxy_runtime_snapshot",
|
|
return_value={
|
|
"state": "disabled",
|
|
"label": "未启用代理",
|
|
"detail": "-",
|
|
"direct_fallback_active": True,
|
|
"reason": "proxy_disabled",
|
|
"last_refresh_status": "",
|
|
"last_refresh_time": "",
|
|
"source_count": 0,
|
|
"raw_items": 0,
|
|
"validated_count": 0,
|
|
"available_count": 0,
|
|
"source_stats": [],
|
|
"supplier_empty": False,
|
|
},
|
|
)
|
|
)
|
|
stack.enter_context(
|
|
patch(
|
|
"app.services.detect_service.resolve_thread_count",
|
|
return_value={
|
|
"effective_thread_count": 1,
|
|
"default_thread_count": 1000,
|
|
"source": "node_override",
|
|
"override_thread_count": 1,
|
|
"node_code": "overseas-control-01",
|
|
},
|
|
)
|
|
)
|
|
stack.enter_context(
|
|
patch(
|
|
"app.services.detect_service.resolve_process_count",
|
|
return_value={
|
|
"effective_process_count": 1,
|
|
"default_process_count": 80,
|
|
"source": "node_override",
|
|
"override_process_count": 1,
|
|
"node_code": "overseas-control-01",
|
|
},
|
|
)
|
|
)
|
|
stack.enter_context(patch("app.services.detect_service.get_active_detect_job_summary", return_value=active_job))
|
|
stack.enter_context(patch("app.services.detect_service.get_detect_queue_health", return_value=aggregate_queue_health))
|
|
stack.enter_context(patch("app.services.detect_service.sync_detect_runs", return_value=[]))
|
|
stack.enter_context(patch("app.services.detect_service._resolve_remote_log_snapshot", return_value={}))
|
|
stack.enter_context(patch("app.services.detect_service._extract_dependency_alerts", return_value=[]))
|
|
stack.enter_context(patch("app.services.detect_service.append_detect_result_projection_if_changed"))
|
|
payload = detect_service.get_detect_status()
|
|
|
|
self.assertEqual(537, payload["progress"]["running"])
|
|
self.assertEqual(537, payload["active_thread_count"])
|
|
self.assertEqual(1000, payload["max_thread_count"])
|
|
self.assertEqual(1, payload["aggregate_process_count"])
|
|
|
|
def test_get_detect_status_drops_stale_aggregate_nodes_from_running_and_process_counts(self) -> None:
|
|
live_heartbeat = datetime.now().isoformat(sep=" ", timespec="seconds")
|
|
stale_heartbeat = (datetime.now() - timedelta(minutes=8)).isoformat(sep=" ", timespec="seconds")
|
|
active_job = {
|
|
"job_id": 88,
|
|
"job_code": "sync-overseas-688",
|
|
"status": "running",
|
|
"items_total": 2600,
|
|
"items_pending": 500,
|
|
"items_running": 2100,
|
|
"items_completed": 0,
|
|
"items_failed": 0,
|
|
"items_blacklisted": 0,
|
|
"progress_percent": 80.77,
|
|
"display_items_running": 2100,
|
|
"display_active_threads": 2100,
|
|
"display_max_threads": 2000,
|
|
"distributed_node_stats": [
|
|
{
|
|
"node_code": "mainland-controller-01-a",
|
|
"items_running": 300,
|
|
"display_running": 300,
|
|
"active_threads": 300,
|
|
"max_threads": 1000,
|
|
"status": "busy",
|
|
"last_heartbeat_at": live_heartbeat,
|
|
},
|
|
{
|
|
"node_code": "mainland-controller-01-b",
|
|
"items_running": 1800,
|
|
"display_running": 1800,
|
|
"active_threads": 1800,
|
|
"max_threads": 1000,
|
|
"status": "stale",
|
|
"last_heartbeat_at": stale_heartbeat,
|
|
},
|
|
],
|
|
}
|
|
aggregate_queue_health = {
|
|
"has_active_job": True,
|
|
"job": {
|
|
"job_id": 88,
|
|
"job_code": "sync-overseas-688",
|
|
"status": "running",
|
|
"progress_percent": 80.77,
|
|
},
|
|
"queue": {
|
|
"items_total": 2600,
|
|
"pending": 500,
|
|
"claimed": 0,
|
|
"running": 2100,
|
|
"display_running": 2100,
|
|
"completed": 0,
|
|
"blacklisted": 0,
|
|
"failed": 0,
|
|
},
|
|
"nodes": [
|
|
{
|
|
"node_code": "mainland-controller-01-a",
|
|
"items_running": 300,
|
|
"display_running": 300,
|
|
"active_threads": 300,
|
|
"max_threads": 1000,
|
|
"status": "busy",
|
|
"last_heartbeat_at": live_heartbeat,
|
|
},
|
|
{
|
|
"node_code": "mainland-controller-01-b",
|
|
"items_running": 1800,
|
|
"display_running": 1800,
|
|
"active_threads": 1800,
|
|
"max_threads": 1000,
|
|
"status": "stale",
|
|
"last_heartbeat_at": stale_heartbeat,
|
|
},
|
|
],
|
|
}
|
|
|
|
with ExitStack() as stack:
|
|
stack.enter_context(patch.object(detect_service.settings, "node_region", "overseas"))
|
|
stack.enter_context(patch.object(detect_service.settings, "node_role", "control"))
|
|
stack.enter_context(patch.object(detect_service.settings, "node_code", "overseas-control-01"))
|
|
stack.enter_context(patch("app.services.detect_service.ensure_runtime_schema"))
|
|
stack.enter_context(patch("app.services.detect_service.get_db", side_effect=RuntimeError("db down")))
|
|
stack.enter_context(
|
|
patch(
|
|
"app.services.detect_service.get_settings_payload",
|
|
return_value={
|
|
"proxy_config": {"proxy_enable": False, "allow_direct": True, "proxy_urls": []},
|
|
"process_count": 80,
|
|
"node_process_counts": {"mainland-controller-01": 70},
|
|
"thread_count": 1000,
|
|
"node_thread_counts": {"overseas-control-01": 1},
|
|
},
|
|
)
|
|
)
|
|
stack.enter_context(
|
|
patch("app.services.detect_service.get_runtime_settings", return_value={"worker_log_sync_enabled": False, "worker_log_sync_mode": "key"})
|
|
)
|
|
stack.enter_context(
|
|
patch("app.services.detect_service._load_recent_worker_lines", return_value=(False, "2026-04-23 00:00:00", ["stale line"]))
|
|
)
|
|
stack.enter_context(
|
|
patch("app.services.detect_service.detect_worker_runtime", return_value={"mode": "linux-systemd", "running": False, "process_count": 0, "latest_start_time": "", "message": "inactive"})
|
|
)
|
|
stack.enter_context(
|
|
patch("app.services.detect_service._load_runtime_state", return_value={"service_running": False, "detecting": False, "active_threads": 0, "max_threads": 1, "detail": ""})
|
|
)
|
|
stack.enter_context(patch("app.services.detect_service._load_runtime_state_from_cluster_node", return_value={}))
|
|
stack.enter_context(patch("app.services.detect_service._extract_available_proxy_count", return_value=0))
|
|
stack.enter_context(patch("app.services.detect_service._extract_active_thread_snapshot", return_value={"active": 0, "max": 0}))
|
|
stack.enter_context(patch("app.services.detect_service._normalize_recent_warning", return_value=""))
|
|
stack.enter_context(
|
|
patch(
|
|
"app.services.detect_service._build_proxy_runtime_snapshot",
|
|
return_value={
|
|
"state": "disabled",
|
|
"label": "未启用代理",
|
|
"detail": "-",
|
|
"direct_fallback_active": True,
|
|
"reason": "proxy_disabled",
|
|
"last_refresh_status": "",
|
|
"last_refresh_time": "",
|
|
"source_count": 0,
|
|
"raw_items": 0,
|
|
"validated_count": 0,
|
|
"available_count": 0,
|
|
"source_stats": [],
|
|
"supplier_empty": False,
|
|
},
|
|
)
|
|
)
|
|
stack.enter_context(
|
|
patch(
|
|
"app.services.detect_service.resolve_thread_count",
|
|
return_value={
|
|
"effective_thread_count": 1,
|
|
"default_thread_count": 1000,
|
|
"source": "node_override",
|
|
"override_thread_count": 1,
|
|
"node_code": "overseas-control-01",
|
|
},
|
|
)
|
|
)
|
|
stack.enter_context(
|
|
patch(
|
|
"app.services.detect_service.resolve_process_count",
|
|
side_effect=[
|
|
{
|
|
"effective_process_count": 1,
|
|
"default_process_count": 80,
|
|
"source": "child_instance",
|
|
"override_process_count": None,
|
|
"node_code": "mainland-controller-01-a",
|
|
}
|
|
],
|
|
)
|
|
)
|
|
stack.enter_context(patch("app.services.detect_service.get_active_detect_job_summary", return_value=active_job))
|
|
stack.enter_context(patch("app.services.detect_service.get_detect_queue_health", return_value=aggregate_queue_health))
|
|
stack.enter_context(patch("app.services.detect_service.sync_detect_runs", return_value=[]))
|
|
stack.enter_context(patch("app.services.detect_service._resolve_remote_log_snapshot", return_value={}))
|
|
stack.enter_context(patch("app.services.detect_service._extract_dependency_alerts", return_value=[]))
|
|
stack.enter_context(patch("app.services.detect_service.append_detect_result_projection_if_changed"))
|
|
payload = detect_service.get_detect_status()
|
|
|
|
self.assertEqual(300, payload["progress"]["running"])
|
|
self.assertEqual(300, payload["active_thread_count"])
|
|
self.assertEqual(1000, payload["max_thread_count"])
|
|
self.assertEqual(1, payload["aggregate_process_count"])
|
|
self.assertEqual(1, payload["aggregate_participating_node_count"])
|
|
self.assertEqual(["mainland-controller-01-a"], payload["aggregate_participating_node_codes"])
|
|
|
|
def test_get_detect_status_uses_event_proxy_counts_when_cluster_runtime_proxy_counts_are_missing(self) -> None:
|
|
active_job = {
|
|
"job_id": 77,
|
|
"job_code": "sync-overseas-482",
|
|
"status": "running",
|
|
"items_pending": 410,
|
|
"items_completed": 100,
|
|
"items_failed": 0,
|
|
"items_blacklisted": 0,
|
|
"progress_percent": 55.0,
|
|
"display_items_running": 807,
|
|
"display_active_threads": 807,
|
|
"current_cycle_token": "cycle-1",
|
|
"distributed_node_stats": [
|
|
{
|
|
"node_code": "mainland-controller-01-u",
|
|
"items_running": 300,
|
|
"display_running": 300,
|
|
"active_threads": 300,
|
|
"max_threads": 1000,
|
|
},
|
|
{
|
|
"node_code": "mainland-controller-01-v",
|
|
"items_running": 301,
|
|
"display_running": 301,
|
|
"active_threads": 301,
|
|
"max_threads": 1000,
|
|
},
|
|
{
|
|
"node_code": "mainland-worker-01-a",
|
|
"items_running": 206,
|
|
"display_running": 206,
|
|
"active_threads": 206,
|
|
"max_threads": 1000,
|
|
},
|
|
],
|
|
"current_cycle_events": [
|
|
{
|
|
"event_type": "worker_log",
|
|
"created_at": "2026-04-23 19:40:00",
|
|
"node_code": "mainland-controller-01-u",
|
|
"message": "当前可用代理数: 486",
|
|
"payload": {"cycle_token": "cycle-1", "log_mode": "key"},
|
|
},
|
|
{
|
|
"event_type": "worker_log",
|
|
"created_at": "2026-04-23 19:40:10",
|
|
"node_code": "mainland-controller-01-v",
|
|
"message": "代理池刷新完成，共 486 个可用代理，来源链接 6 个，原始 520 个",
|
|
"payload": {"cycle_token": "cycle-1", "log_mode": "key"},
|
|
},
|
|
{
|
|
"event_type": "worker_log",
|
|
"created_at": "2026-04-23 19:40:20",
|
|
"node_code": "mainland-worker-01-a",
|
|
"message": "共享刷新进行中，继续沿用缓存 321 个",
|
|
"payload": {"cycle_token": "cycle-1", "log_mode": "key"},
|
|
},
|
|
],
|
|
}
|
|
aggregate_queue_health = {
|
|
"has_active_job": True,
|
|
"job": {
|
|
"job_id": 77,
|
|
"job_code": "sync-overseas-482",
|
|
"status": "running",
|
|
"progress_percent": 55.0,
|
|
},
|
|
"queue": {
|
|
"items_total": 1317,
|
|
"pending": 410,
|
|
"claimed": 0,
|
|
"running": 807,
|
|
"display_running": 807,
|
|
"completed": 100,
|
|
"blacklisted": 0,
|
|
"failed": 0,
|
|
},
|
|
"nodes": [
|
|
{
|
|
"node_code": "mainland-controller-01-u",
|
|
"items_running": 300,
|
|
"display_running": 300,
|
|
"active_threads": 300,
|
|
"max_threads": 1000,
|
|
},
|
|
{
|
|
"node_code": "mainland-controller-01-v",
|
|
"items_running": 301,
|
|
"display_running": 301,
|
|
"active_threads": 301,
|
|
"max_threads": 1000,
|
|
},
|
|
{
|
|
"node_code": "mainland-worker-01-a",
|
|
"items_running": 206,
|
|
"display_running": 206,
|
|
"active_threads": 206,
|
|
"max_threads": 1000,
|
|
},
|
|
],
|
|
}
|
|
|
|
def _resolve_thread_count(*, node_code=None, settings_payload=None):
|
|
if str(node_code or "").startswith("mainland-"):
|
|
return {
|
|
"effective_thread_count": 1000,
|
|
"default_thread_count": 1000,
|
|
"source": "default",
|
|
"override_thread_count": None,
|
|
"node_code": str(node_code or ""),
|
|
}
|
|
return {
|
|
"effective_thread_count": 1,
|
|
"default_thread_count": 1000,
|
|
"source": "node_override",
|
|
"override_thread_count": 1,
|
|
"node_code": "overseas-control-01",
|
|
}
|
|
|
|
def _resolve_process_count(*, node_code=None, settings_payload=None):
|
|
normalized_node_code = str(node_code or "")
|
|
if normalized_node_code == "mainland-controller-01":
|
|
return {
|
|
"effective_process_count": 80,
|
|
"default_process_count": 80,
|
|
"source": "default",
|
|
"override_process_count": None,
|
|
"node_code": normalized_node_code,
|
|
}
|
|
if normalized_node_code == "mainland-worker-01":
|
|
return {
|
|
"effective_process_count": 60,
|
|
"default_process_count": 60,
|
|
"source": "default",
|
|
"override_process_count": None,
|
|
"node_code": normalized_node_code,
|
|
}
|
|
if normalized_node_code.startswith("mainland-"):
|
|
return {
|
|
"effective_process_count": 1,
|
|
"default_process_count": 80,
|
|
"source": "child_instance",
|
|
"override_process_count": None,
|
|
"node_code": normalized_node_code,
|
|
}
|
|
return {
|
|
"effective_process_count": 1,
|
|
"default_process_count": 80,
|
|
"source": "node_override",
|
|
"override_process_count": 1,
|
|
"node_code": "overseas-control-01",
|
|
}
|
|
|
|
with ExitStack() as stack:
|
|
stack.enter_context(patch.object(detect_service.settings, "node_region", "overseas"))
|
|
stack.enter_context(patch.object(detect_service.settings, "node_role", "control"))
|
|
stack.enter_context(patch.object(detect_service.settings, "node_code", "overseas-control-01"))
|
|
stack.enter_context(patch("app.services.detect_service.ensure_runtime_schema"))
|
|
stack.enter_context(patch("app.services.detect_service.get_db", side_effect=RuntimeError("db down")))
|
|
stack.enter_context(
|
|
patch(
|
|
"app.services.detect_service.get_settings_payload",
|
|
return_value={
|
|
"proxy_config": {"proxy_enable": False, "allow_direct": True, "proxy_urls": []},
|
|
"process_count": 80,
|
|
"node_process_counts": {"mainland-worker-01": 60},
|
|
"thread_count": 1000,
|
|
"node_thread_counts": {"overseas-control-01": 1},
|
|
},
|
|
)
|
|
)
|
|
stack.enter_context(
|
|
patch("app.services.detect_service.get_runtime_settings", return_value={"worker_log_sync_enabled": False, "worker_log_sync_mode": "key"})
|
|
)
|
|
stack.enter_context(
|
|
patch("app.services.detect_service._load_recent_worker_lines", return_value=(False, "2026-04-23 00:00:00", ["stale line"]))
|
|
)
|
|
stack.enter_context(
|
|
patch("app.services.detect_service.detect_worker_runtime", return_value={"mode": "linux-systemd", "running": False, "process_count": 0, "latest_start_time": "", "message": "inactive"})
|
|
)
|
|
stack.enter_context(
|
|
patch("app.services.detect_service._load_runtime_state", return_value={"service_running": False, "detecting": False, "active_threads": 0, "max_threads": 1, "detail": ""})
|
|
)
|
|
stack.enter_context(patch("app.services.detect_service._load_runtime_state_from_cluster_node", return_value={}))
|
|
stack.enter_context(patch("app.services.detect_service._extract_available_proxy_count", return_value=0))
|
|
stack.enter_context(patch("app.services.detect_service._extract_active_thread_snapshot", return_value={"active": 0, "max": 0}))
|
|
stack.enter_context(patch("app.services.detect_service._normalize_recent_warning", return_value=""))
|
|
stack.enter_context(
|
|
patch(
|
|
"app.services.detect_service._build_proxy_runtime_snapshot",
|
|
return_value={
|
|
"state": "warming_up",
|
|
"label": "等待首刷",
|
|
"detail": "-",
|
|
"direct_fallback_active": False,
|
|
"reason": "waiting_for_first_refresh",
|
|
"last_refresh_status": "未刷新",
|
|
"last_refresh_time": "",
|
|
"source_count": 0,
|
|
"raw_items": 0,
|
|
"validated_count": 0,
|
|
"available_count": 0,
|
|
"source_stats": [],
|
|
"supplier_empty": False,
|
|
},
|
|
)
|
|
)
|
|
stack.enter_context(patch("app.services.detect_service.resolve_thread_count", side_effect=_resolve_thread_count))
|
|
stack.enter_context(patch("app.services.detect_service.resolve_process_count", side_effect=_resolve_process_count))
|
|
stack.enter_context(patch("app.services.detect_service.get_active_detect_job_summary", return_value=active_job))
|
|
stack.enter_context(patch("app.services.detect_service.get_detect_queue_health", return_value=aggregate_queue_health))
|
|
stack.enter_context(
|
|
patch(
|
|
"app.services.detect_service._load_runtime_states_from_cluster_nodes",
|
|
return_value={
|
|
"mainland-controller-01": {
|
|
"node_code": "mainland-controller-01",
|
|
"available_proxy_count": 0,
|
|
"proxy_runtime_label": "",
|
|
"proxy_runtime_reason": "",
|
|
"proxy_last_refresh_status": "",
|
|
"proxy_last_refresh_time": "",
|
|
"proxy_last_refresh_source_count": 0,
|
|
"proxy_last_refresh_total_items": 0,
|
|
"proxy_last_validated_count": 0,
|
|
},
|
|
"mainland-worker-01": {
|
|
"node_code": "mainland-worker-01",
|
|
"available_proxy_count": 0,
|
|
"proxy_runtime_label": "",
|
|
"proxy_runtime_reason": "",
|
|
"proxy_last_refresh_status": "",
|
|
"proxy_last_refresh_time": "",
|
|
"proxy_last_refresh_source_count": 0,
|
|
"proxy_last_refresh_total_items": 0,
|
|
"proxy_last_validated_count": 0,
|
|
},
|
|
},
|
|
)
|
|
)
|
|
stack.enter_context(patch("app.services.detect_service.sync_detect_runs", return_value=[]))
|
|
stack.enter_context(patch("app.services.detect_service._resolve_remote_log_snapshot", return_value={}))
|
|
stack.enter_context(patch("app.services.detect_service._extract_dependency_alerts", return_value=[]))
|
|
stack.enter_context(patch("app.services.detect_service.append_detect_result_projection_if_changed"))
|
|
payload = detect_service.get_detect_status()
|
|
|
|
self.assertEqual(807, payload["available_proxy_count"])
|
|
self.assertEqual("集群代理正常", payload["proxy_runtime_label"])
|
|
self.assertIn("参与服务器 2 台", payload["proxy_runtime_detail"])
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|