499 lines
20 KiB
Python
499 lines
20 KiB
Python
import unittest
|
||
from unittest.mock import patch
|
||
|
||
from app.services.dashboard import fetch_overview
|
||
|
||
|
||
class _FakeCursor:
|
||
def __init__(self, responses):
|
||
self._responses = list(responses)
|
||
|
||
def execute(self, sql, params=None):
|
||
self._last_sql = sql
|
||
self._last_params = params
|
||
|
||
def fetchone(self):
|
||
if self._responses:
|
||
return self._responses.pop(0)
|
||
return (0,)
|
||
|
||
def __enter__(self):
|
||
return self
|
||
|
||
def __exit__(self, exc_type, exc, tb):
|
||
return False
|
||
|
||
|
||
class _FakeConnection:
|
||
def __init__(self, responses):
|
||
self._cursor = _FakeCursor(responses)
|
||
|
||
def cursor(self):
|
||
return self._cursor
|
||
|
||
def __enter__(self):
|
||
return self
|
||
|
||
def __exit__(self, exc_type, exc, tb):
|
||
return False
|
||
|
||
|
||
class DashboardServiceTests(unittest.TestCase):
|
||
@patch("app.services.dashboard.get_detect_status")
|
||
@patch("app.services.dashboard._fetch_active_jobs_aggregate")
|
||
@patch("app.services.dashboard.get_detect_capacity_plan")
|
||
@patch("app.services.dashboard.get_detect_queue_health")
|
||
@patch("app.services.dashboard._build_dashboard_runtime_summary")
|
||
@patch("app.services.dashboard.get_active_detect_job_summary")
|
||
@patch("app.services.dashboard.get_db")
|
||
def test_fetch_overview_includes_ops_metrics(
|
||
self,
|
||
mock_get_db,
|
||
mock_get_active_detect_job_summary,
|
||
mock_build_dashboard_runtime_summary,
|
||
mock_get_detect_queue_health,
|
||
mock_get_detect_capacity_plan,
|
||
mock_fetch_active_jobs_aggregate,
|
||
mock_get_detect_status,
|
||
) -> None:
|
||
mock_get_db.return_value = _FakeConnection(
|
||
responses=[
|
||
(1000,),
|
||
(900,),
|
||
(10,),
|
||
(5,),
|
||
(0,),
|
||
(1,),
|
||
(430,),
|
||
(420,),
|
||
(17,),
|
||
]
|
||
)
|
||
mock_fetch_active_jobs_aggregate.return_value = {
|
||
"active_jobs_total": 3,
|
||
"queue": {
|
||
"items_total": 5200,
|
||
"pending": 4700,
|
||
"claimed": 11,
|
||
"running": 165,
|
||
"completed": 300,
|
||
"blacklisted": 12,
|
||
"failed": 7,
|
||
},
|
||
"throughput": {
|
||
"processed_recent": 30,
|
||
"processed_per_minute": 2.0,
|
||
"completed_recent": 20,
|
||
"blacklisted_recent": 1,
|
||
"failed_recent": 2,
|
||
},
|
||
"retry_total": 12,
|
||
"steps": [
|
||
{
|
||
"step_code": "detect_register",
|
||
"step_name": "注册状态检测",
|
||
"items_pending": 185,
|
||
"items_running": 3,
|
||
"items_claimed": 0,
|
||
"items_completed": 7,
|
||
"items_blacklisted": 0,
|
||
"items_failed": 0,
|
||
"processed_recent": 20,
|
||
"processed_per_minute": 1.33,
|
||
}
|
||
],
|
||
"nodes": [
|
||
{
|
||
"node_code": "mainland-controller-01",
|
||
"items_running": 120,
|
||
"items_claimed": 0,
|
||
"processed_recent": 20,
|
||
"processed_per_minute": 1.33,
|
||
"completed_recent": 15,
|
||
"failed_recent": 1,
|
||
"blacklisted_recent": 0,
|
||
}
|
||
],
|
||
}
|
||
mock_get_active_detect_job_summary.return_value = {
|
||
"job_id": 76,
|
||
"job_code": "sync-overseas-3612",
|
||
"items_pending": 185,
|
||
"items_running": 8,
|
||
"items_completed": 7,
|
||
"items_blacklisted": 0,
|
||
"items_failed": 0,
|
||
}
|
||
mock_build_dashboard_runtime_summary.return_value = {
|
||
"worker": {"running": True, "mode": "linux-systemd", "expected_on_this_node": True},
|
||
"node": {"region": "overseas", "role": "control"},
|
||
"cluster": {"summary": {"online_worker_nodes": 2, "dedicated_online_worker_nodes": 1, "online_control_nodes": 1}},
|
||
"detect": {
|
||
"backlog": {
|
||
"pending_total": 9438,
|
||
"claimed_total": 410,
|
||
"running_total": 15,
|
||
"completed_total": 951,
|
||
"blacklisted_total": 0,
|
||
"failed_total": 137,
|
||
"register_pending": 8487,
|
||
"downstream_pending": 951,
|
||
}
|
||
},
|
||
}
|
||
mock_get_detect_queue_health.return_value = {
|
||
"has_active_job": True,
|
||
"job": {"job_id": 76, "job_code": "sync-overseas-3612", "status": "running", "progress_percent": 3.5},
|
||
"queue": {
|
||
"items_total": 200,
|
||
"pending": 185,
|
||
"claimed": 0,
|
||
"display_claimed": 0,
|
||
"running": 8,
|
||
"display_running": 165,
|
||
"completed": 7,
|
||
"blacklisted": 0,
|
||
"failed": 0,
|
||
},
|
||
"throughput": {"processed_recent": 0, "processed_per_minute": 0},
|
||
"steps": [
|
||
{
|
||
"step_code": "detect_register",
|
||
"step_name": "注册状态检测",
|
||
"items_pending": 185,
|
||
"items_running": 165,
|
||
"items_claimed": 0,
|
||
"items_completed": 7,
|
||
"items_blacklisted": 0,
|
||
"items_failed": 0,
|
||
"processed_recent": 0,
|
||
"processed_per_minute": 0.0,
|
||
}
|
||
],
|
||
"runtime_activity": {
|
||
"step_stats": {
|
||
"detect_360_site": {
|
||
"step_code": "detect_360_site",
|
||
"started_recent": 269,
|
||
"processed_recent": 6,
|
||
"completed_recent": 6,
|
||
"failed_recent": 0,
|
||
"blacklisted_recent": 0,
|
||
}
|
||
}
|
||
},
|
||
"nodes": [
|
||
{
|
||
"node_code": "mainland-controller-01",
|
||
"items_running": 165,
|
||
"items_claimed": 0,
|
||
"processed_recent": 0,
|
||
"processed_per_minute": 0.0,
|
||
"completed_recent": 0,
|
||
"failed_recent": 0,
|
||
"blacklisted_recent": 0,
|
||
}
|
||
],
|
||
}
|
||
mock_get_detect_capacity_plan.return_value = {
|
||
"estimated_hours_remaining": 1.5,
|
||
"remaining_items": 193,
|
||
"recommended_additional_workers": 1,
|
||
}
|
||
mock_get_detect_status.return_value = {
|
||
"available_proxy_count": 1011,
|
||
"proxy_runtime_label": "集群代理正常",
|
||
"proxy_runtime_detail": "参与服务器 2 台，共可用 1011 个代理",
|
||
"proxy_last_refresh_status": "mainland-controller-01:201；mainland-worker-01:810",
|
||
"aggregate_process_count": 10,
|
||
"aggregate_participating_node_count": 10,
|
||
"active_thread_count": 165,
|
||
}
|
||
|
||
data = fetch_overview()
|
||
|
||
self.assertEqual("sync-overseas-3612", data["active_job"]["job_code"])
|
||
self.assertEqual(3, data["active_jobs_aggregate"]["active_jobs_total"])
|
||
self.assertEqual(9438, data["active_jobs_aggregate"]["queue"]["pending"])
|
||
self.assertEqual(410, data["active_jobs_aggregate"]["queue"]["claimed"])
|
||
self.assertEqual(15, data["active_jobs_aggregate"]["queue"]["running"])
|
||
self.assertEqual(951, data["active_jobs_aggregate"]["queue"]["completed"])
|
||
self.assertEqual(137, data["active_jobs_aggregate"]["queue"]["failed"])
|
||
self.assertEqual("mainland-controller-01", data["active_jobs_aggregate"]["nodes"][0]["node_code"])
|
||
self.assertEqual(900, data["pending_total"])
|
||
self.assertEqual(10, data["completed_total"])
|
||
self.assertEqual(430, data["registerable_total"])
|
||
self.assertEqual(420, data["purchasable_total"])
|
||
self.assertEqual(185, data["queue_pending_total"])
|
||
self.assertEqual(8, data["queue_running_total"])
|
||
self.assertEqual(165, data["queue_display_running_total"])
|
||
self.assertEqual(7, data["queue_completed_total"])
|
||
self.assertEqual(9438, data["backlog_pending_total"])
|
||
self.assertEqual(1011, data["cluster_proxy_available_count"])
|
||
self.assertEqual("集群代理正常", data["cluster_proxy_runtime_label"])
|
||
self.assertEqual(10, data["aggregate_process_count"])
|
||
self.assertEqual(1, data["ops_summary"]["active_execution_nodes"])
|
||
self.assertEqual(8487, data["backlog_register_pending_total"])
|
||
self.assertEqual(951, data["backlog_downstream_pending_total"])
|
||
self.assertEqual(12, data["retry_total"])
|
||
self.assertEqual("注册状态检测", data["bottleneck_step"]["step_name"])
|
||
self.assertEqual(0.0, data["processed_per_minute"])
|
||
self.assertEqual(0, data["processed_recent"])
|
||
self.assertEqual(0.0, data["ops_summary"]["processed_per_minute"])
|
||
self.assertEqual(1.5, data["ops_summary"]["estimated_hours_remaining"])
|
||
self.assertEqual(1, data["active_execution_nodes"])
|
||
self.assertEqual(1, data["ops_summary"]["active_execution_nodes"])
|
||
self.assertEqual(0, data["current_job_blacklisted"])
|
||
self.assertEqual(0, data["recent_blacklisted_total"])
|
||
self.assertEqual(0, data["cumulative_blacklisted_total"])
|
||
self.assertEqual(2, len(data["step_queue"]))
|
||
self.assertTrue(any(item["step_code"] == "detect_360_site" for item in data["step_queue"]))
|
||
self.assertEqual(1, len(data["node_throughput"]))
|
||
|
||
@patch("app.services.dashboard._fetch_active_jobs_aggregate")
|
||
@patch("app.services.dashboard.get_detect_capacity_plan")
|
||
@patch("app.services.dashboard.get_detect_queue_health")
|
||
@patch("app.services.dashboard._build_dashboard_runtime_summary")
|
||
@patch("app.services.dashboard.get_active_detect_job_summary")
|
||
@patch("app.services.dashboard.get_db")
|
||
def test_fetch_overview_prefers_active_job_display_running_when_queue_snapshot_is_stale(
|
||
self,
|
||
mock_get_db,
|
||
mock_get_active_detect_job_summary,
|
||
mock_build_dashboard_runtime_summary,
|
||
mock_get_detect_queue_health,
|
||
mock_get_detect_capacity_plan,
|
||
mock_fetch_active_jobs_aggregate,
|
||
) -> None:
|
||
mock_get_db.return_value = _FakeConnection(
|
||
responses=[
|
||
(1000,),
|
||
(900,),
|
||
(10,),
|
||
(5,),
|
||
(0,),
|
||
(1,),
|
||
(430,),
|
||
(420,),
|
||
(17,),
|
||
]
|
||
)
|
||
mock_fetch_active_jobs_aggregate.return_value = {
|
||
"active_jobs_total": 1,
|
||
"queue": {"items_total": 8402, "pending": 5001, "claimed": 0, "running": 1, "completed": 0, "blacklisted": 0, "failed": 0},
|
||
"throughput": {"processed_recent": 0, "processed_per_minute": 0.0},
|
||
"steps": [],
|
||
"nodes": [],
|
||
"retry_total": 0,
|
||
}
|
||
mock_get_active_detect_job_summary.return_value = {
|
||
"job_id": 11,
|
||
"job_code": "sync-overseas-51",
|
||
"status": "running",
|
||
"items_total": 8402,
|
||
"items_pending": 5001,
|
||
"items_claimed": 0,
|
||
"items_running": 0,
|
||
"items_completed": 0,
|
||
"items_blacklisted": 0,
|
||
"items_failed": 0,
|
||
"display_items_running": 1432,
|
||
"display_active_threads": 138,
|
||
"display_max_threads": 80000,
|
||
"distributed_node_stats": [
|
||
{"node_code": "mainland-controller-01-a", "display_running": 1000, "active_threads": 1000, "max_threads": 1000},
|
||
{"node_code": "mainland-controller-01-b", "display_running": 432, "active_threads": 432, "max_threads": 1000},
|
||
],
|
||
}
|
||
mock_build_dashboard_runtime_summary.return_value = {
|
||
"worker": {"running": False, "mode": "linux-systemd", "expected_on_this_node": False},
|
||
"node": {"region": "overseas", "role": "control"},
|
||
"cluster": {"summary": {"online_worker_nodes": 1, "dedicated_online_worker_nodes": 0, "online_control_nodes": 2}},
|
||
"detect": {
|
||
"backlog": {
|
||
"pending_total": 5001,
|
||
"claimed_total": 0,
|
||
"running_total": 0,
|
||
"completed_total": 0,
|
||
"blacklisted_total": 0,
|
||
"failed_total": 0,
|
||
"register_pending": 3495,
|
||
"downstream_pending": 1506,
|
||
}
|
||
},
|
||
}
|
||
mock_get_detect_queue_health.return_value = {
|
||
"has_active_job": True,
|
||
"job": {"job_id": 11, "job_code": "sync-overseas-51", "runtime_job_code": "sync-overseas-51", "status": "running", "progress_percent": 40.49},
|
||
"queue": {
|
||
"items_total": 8402,
|
||
"pending": 5001,
|
||
"claimed": 0,
|
||
"display_claimed": 0,
|
||
"running": 1,
|
||
"display_running": 1,
|
||
"completed": 0,
|
||
"blacklisted": 0,
|
||
"failed": 0,
|
||
},
|
||
"throughput": {"processed_recent": 0, "processed_per_minute": 0.0},
|
||
"steps": [],
|
||
"runtime_activity": {},
|
||
"nodes": [],
|
||
}
|
||
mock_get_detect_capacity_plan.return_value = {
|
||
"estimated_hours_remaining": 0,
|
||
"remaining_items": 5001,
|
||
"recommended_additional_workers": 0,
|
||
}
|
||
|
||
data = fetch_overview()
|
||
|
||
self.assertEqual(1432, data["queue_display_running_total"])
|
||
self.assertEqual(1432, data["active_job"]["items_display_running"])
|
||
self.assertEqual(1432, data["active_job"]["display_items_running"])
|
||
self.assertEqual(138, data["active_job"]["display_active_threads"])
|
||
self.assertEqual(80000, data["active_job"]["display_max_threads"])
|
||
self.assertEqual(2, len(data["active_job"]["distributed_node_stats"]))
|
||
self.assertEqual(80000, data["queue_display_max_threads"])
|
||
self.assertEqual(0, data["current_job_blacklisted"])
|
||
self.assertEqual(0, data["recent_blacklisted_total"])
|
||
self.assertEqual(0, data["cumulative_blacklisted_total"])
|
||
|
||
@patch("app.services.dashboard._fetch_active_jobs_aggregate")
|
||
@patch("app.services.dashboard.get_detect_capacity_plan")
|
||
@patch("app.services.dashboard.get_detect_queue_health")
|
||
@patch("app.services.dashboard._build_dashboard_runtime_summary")
|
||
@patch("app.services.dashboard.get_active_detect_job_summary")
|
||
@patch("app.services.dashboard.get_db")
|
||
def test_fetch_overview_counts_active_execution_nodes_from_full_node_set(
|
||
self,
|
||
mock_get_db,
|
||
mock_get_active_detect_job_summary,
|
||
mock_build_dashboard_runtime_summary,
|
||
mock_get_detect_queue_health,
|
||
mock_get_detect_capacity_plan,
|
||
mock_fetch_active_jobs_aggregate,
|
||
) -> None:
|
||
mock_get_db.return_value = _FakeConnection(responses=[(0,)] * 9)
|
||
mock_fetch_active_jobs_aggregate.return_value = {
|
||
"active_jobs_total": 1,
|
||
"queue": {"items_total": 9, "pending": 0, "claimed": 0, "running": 9, "completed": 0, "blacklisted": 0, "failed": 0},
|
||
"throughput": {"processed_recent": 0, "processed_per_minute": 0.0, "completed_recent": 0, "blacklisted_recent": 0, "failed_recent": 0},
|
||
"steps": [],
|
||
"nodes": [
|
||
{
|
||
"node_code": f"mainland-controller-01-{index:02d}",
|
||
"items_running": 1,
|
||
"items_claimed": 0,
|
||
"processed_recent": 0,
|
||
"processed_per_minute": 0.0,
|
||
"completed_recent": 0,
|
||
"failed_recent": 0,
|
||
"blacklisted_recent": 0,
|
||
}
|
||
for index in range(9)
|
||
],
|
||
"retry_total": 0,
|
||
}
|
||
mock_get_active_detect_job_summary.return_value = {}
|
||
mock_build_dashboard_runtime_summary.return_value = {
|
||
"worker": {"running": True, "mode": "linux-systemd", "expected_on_this_node": True},
|
||
"node": {"region": "mainland", "role": "worker"},
|
||
"cluster": {"summary": {"online_worker_nodes": 1, "dedicated_online_worker_nodes": 0, "online_control_nodes": 1}},
|
||
"detect": {"backlog": {}},
|
||
}
|
||
mock_get_detect_queue_health.return_value = {
|
||
"has_active_job": False,
|
||
"queue": {},
|
||
"throughput": {"processed_recent": 0, "processed_per_minute": 0.0},
|
||
"steps": [],
|
||
"runtime_activity": {},
|
||
"nodes": [],
|
||
"runtime_snapshot_backlog": {},
|
||
}
|
||
mock_get_detect_capacity_plan.return_value = {
|
||
"estimated_hours_remaining": 0,
|
||
"remaining_items": 0,
|
||
"recommended_additional_workers": 0,
|
||
}
|
||
|
||
data = fetch_overview()
|
||
|
||
self.assertEqual(8, len(data["node_throughput"]))
|
||
self.assertEqual(9, data["active_execution_nodes"])
|
||
self.assertEqual(9, data["ops_summary"]["active_execution_nodes"])
|
||
|
||
@patch("app.services.dashboard._fetch_active_jobs_aggregate")
|
||
@patch("app.services.dashboard.get_detect_capacity_plan")
|
||
@patch("app.services.dashboard.get_detect_queue_health")
|
||
@patch("app.services.dashboard._build_dashboard_runtime_summary")
|
||
@patch("app.services.dashboard.get_active_detect_job_summary")
|
||
@patch("app.services.dashboard.get_db")
|
||
def test_fetch_overview_keeps_active_job_summary_when_queue_health_temporarily_empty(
|
||
self,
|
||
mock_get_db,
|
||
mock_get_active_detect_job_summary,
|
||
mock_build_dashboard_runtime_summary,
|
||
mock_get_detect_queue_health,
|
||
mock_get_detect_capacity_plan,
|
||
mock_fetch_active_jobs_aggregate,
|
||
) -> None:
|
||
mock_get_db.return_value = _FakeConnection(responses=[(0,)] * 9)
|
||
mock_fetch_active_jobs_aggregate.return_value = {
|
||
"active_jobs_total": 1,
|
||
"queue": {"items_total": 8402, "pending": 5001, "claimed": 0, "running": 537, "completed": 0, "blacklisted": 0, "failed": 0},
|
||
"throughput": {"processed_recent": 12, "processed_per_minute": 0.8, "completed_recent": 10, "blacklisted_recent": 1, "failed_recent": 1},
|
||
"steps": [],
|
||
"nodes": [],
|
||
"retry_total": 0,
|
||
}
|
||
mock_get_active_detect_job_summary.return_value = {
|
||
"job_id": 255,
|
||
"job_code": "sync-overseas-255",
|
||
"runtime_job_code": "sync-overseas-255",
|
||
"status": "running",
|
||
"progress_percent": 22.4,
|
||
"items_total": 8402,
|
||
"items_pending": 5001,
|
||
"items_claimed": 0,
|
||
"items_running": 537,
|
||
"items_completed": 0,
|
||
"items_blacklisted": 0,
|
||
"items_failed": 0,
|
||
"display_items_running": 11799,
|
||
"display_active_threads": 11799,
|
||
"display_max_threads": 75400,
|
||
"distributed_node_stats": [{"node_code": "mainland-controller-01-a", "display_running": 537, "active_threads": 537, "max_threads": 1000}],
|
||
}
|
||
mock_build_dashboard_runtime_summary.return_value = {
|
||
"worker": {"running": False, "mode": "linux-systemd", "expected_on_this_node": False},
|
||
"node": {"region": "overseas", "role": "control"},
|
||
"cluster": {"summary": {"online_worker_nodes": 1, "dedicated_online_worker_nodes": 0, "online_control_nodes": 1}},
|
||
"detect": {"backlog": {}},
|
||
}
|
||
mock_get_detect_queue_health.return_value = {
|
||
"has_active_job": False,
|
||
"queue": {},
|
||
"throughput": {"processed_recent": 0, "processed_per_minute": 0.0},
|
||
"steps": [],
|
||
"runtime_activity": {},
|
||
"nodes": [],
|
||
"runtime_snapshot_backlog": {},
|
||
}
|
||
mock_get_detect_capacity_plan.return_value = {
|
||
"estimated_hours_remaining": 0,
|
||
"remaining_items": 5001,
|
||
"recommended_additional_workers": 0,
|
||
}
|
||
|
||
data = fetch_overview()
|
||
|
||
self.assertEqual("sync-overseas-255", data["active_job"]["job_code"])
|
||
self.assertEqual(11799, data["active_job"]["display_items_running"])
|
||
self.assertEqual(75400, data["active_job"]["display_max_threads"])
|
||
|
||
|
||
if __name__ == "__main__":
|
||
unittest.main()
|