admin/getDomain
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: stabilize multi-region runtime sync and worker orchestration

Browse Source
This commit is contained in:
root
2026-04-27 15:48:12 +08:00
parent 7cbde2aa78
commit 215a364891
137 changed files with 31931 additions and 1943 deletions
Download Patch File Download Diff File Expand all files Collapse all files

267
domain-api/tests/test_dashboard_service.py
View File

@@ -39,20 +39,22 @@ class _FakeConnection:
class DashboardServiceTests(unittest.TestCase):
@patch("app.services.dashboard.get_detect_status")
@patch("app.services.dashboard._fetch_active_jobs_aggregate")
@patch("app.services.dashboard.get_detect_capacity_plan")
@patch("app.services.dashboard.get_detect_queue_health")
@patch("app.services.dashboard.get_runtime_status")
@patch("app.services.dashboard._build_dashboard_runtime_summary")
@patch("app.services.dashboard.get_active_detect_job_summary")
@patch("app.services.dashboard.get_db")
def test_fetch_overview_includes_ops_metrics(
self,
mock_get_db,
mock_get_active_detect_job_summary,
mock_get_runtime_status,
mock_build_dashboard_runtime_summary,
mock_get_detect_queue_health,
mock_get_detect_capacity_plan,
mock_fetch_active_jobs_aggregate,
mock_get_detect_status,
) -> None:
mock_get_db.return_value = _FakeConnection(
responses=[
@@ -122,7 +124,7 @@ class DashboardServiceTests(unittest.TestCase):
"items_blacklisted": 0,
"items_failed": 0,
}
mock_get_runtime_status.return_value = {
mock_build_dashboard_runtime_summary.return_value = {
"worker": {"running": True, "mode": "linux-systemd", "expected_on_this_node": True},
"node": {"region": "overseas", "role": "control"},
"cluster": {"summary": {"online_worker_nodes": 2, "dedicated_online_worker_nodes": 1, "online_control_nodes": 1}},
@@ -198,6 +200,15 @@ class DashboardServiceTests(unittest.TestCase):
"remaining_items": 193,
"recommended_additional_workers": 1,
}
mock_get_detect_status.return_value = {
"available_proxy_count": 1011,
"proxy_runtime_label": "集群代理正常",
"proxy_runtime_detail": "参与服务器 2 台，共可用 1011 个代理",
"proxy_last_refresh_status": "mainland-controller-01:201；mainland-worker-01:810",
"aggregate_process_count": 10,
"aggregate_participating_node_count": 10,
"active_thread_count": 165,
}
data = fetch_overview()
@@ -218,6 +229,10 @@ class DashboardServiceTests(unittest.TestCase):
self.assertEqual(165, data["queue_display_running_total"])
self.assertEqual(7, data["queue_completed_total"])
self.assertEqual(9438, data["backlog_pending_total"])
self.assertEqual(1011, data["cluster_proxy_available_count"])
self.assertEqual("集群代理正常", data["cluster_proxy_runtime_label"])
self.assertEqual(10, data["aggregate_process_count"])
self.assertEqual(1, data["ops_summary"]["active_execution_nodes"])
self.assertEqual(8487, data["backlog_register_pending_total"])
self.assertEqual(951, data["backlog_downstream_pending_total"])
self.assertEqual(12, data["retry_total"])
@@ -228,10 +243,256 @@ class DashboardServiceTests(unittest.TestCase):
self.assertEqual(1.5, data["ops_summary"]["estimated_hours_remaining"])
self.assertEqual(1, data["active_execution_nodes"])
self.assertEqual(1, data["ops_summary"]["active_execution_nodes"])
self.assertEqual(0, data["current_job_blacklisted"])
self.assertEqual(0, data["recent_blacklisted_total"])
self.assertEqual(0, data["cumulative_blacklisted_total"])
self.assertEqual(2, len(data["step_queue"]))
self.assertTrue(any(item["step_code"] == "detect_360_site" for item in data["step_queue"]))
self.assertEqual(1, len(data["node_throughput"]))
@patch("app.services.dashboard._fetch_active_jobs_aggregate")
@patch("app.services.dashboard.get_detect_capacity_plan")
@patch("app.services.dashboard.get_detect_queue_health")
@patch("app.services.dashboard._build_dashboard_runtime_summary")
@patch("app.services.dashboard.get_active_detect_job_summary")
@patch("app.services.dashboard.get_db")
def test_fetch_overview_prefers_active_job_display_running_when_queue_snapshot_is_stale(
self,
mock_get_db,
mock_get_active_detect_job_summary,
mock_build_dashboard_runtime_summary,
mock_get_detect_queue_health,
mock_get_detect_capacity_plan,
mock_fetch_active_jobs_aggregate,
) -> None:
mock_get_db.return_value = _FakeConnection(
responses=[
(1000,),
(900,),
(10,),
(5,),
(0,),
(1,),
(430,),
(420,),
(17,),
]
)
mock_fetch_active_jobs_aggregate.return_value = {
"active_jobs_total": 1,
"queue": {"items_total": 8402, "pending": 5001, "claimed": 0, "running": 1, "completed": 0, "blacklisted": 0, "failed": 0},
"throughput": {"processed_recent": 0, "processed_per_minute": 0.0},
"steps": [],
"nodes": [],
"retry_total": 0,
}
mock_get_active_detect_job_summary.return_value = {
"job_id": 11,
"job_code": "sync-overseas-51",
"status": "running",
"items_total": 8402,
"items_pending": 5001,
"items_claimed": 0,
"items_running": 0,
"items_completed": 0,
"items_blacklisted": 0,
"items_failed": 0,
"display_items_running": 1432,
"display_active_threads": 138,
"display_max_threads": 80000,
"distributed_node_stats": [
{"node_code": "mainland-controller-01-a", "display_running": 1000, "active_threads": 1000, "max_threads": 1000},
{"node_code": "mainland-controller-01-b", "display_running": 432, "active_threads": 432, "max_threads": 1000},
],
}
mock_build_dashboard_runtime_summary.return_value = {
"worker": {"running": False, "mode": "linux-systemd", "expected_on_this_node": False},
"node": {"region": "overseas", "role": "control"},
"cluster": {"summary": {"online_worker_nodes": 1, "dedicated_online_worker_nodes": 0, "online_control_nodes": 2}},
"detect": {
"backlog": {
"pending_total": 5001,
"claimed_total": 0,
"running_total": 0,
"completed_total": 0,
"blacklisted_total": 0,
"failed_total": 0,
"register_pending": 3495,
"downstream_pending": 1506,
}
},
}
mock_get_detect_queue_health.return_value = {
"has_active_job": True,
"job": {"job_id": 11, "job_code": "sync-overseas-51", "runtime_job_code": "sync-overseas-51", "status": "running", "progress_percent": 40.49},
"queue": {
"items_total": 8402,
"pending": 5001,
"claimed": 0,
"display_claimed": 0,
"running": 1,
"display_running": 1,
"completed": 0,
"blacklisted": 0,
"failed": 0,
},
"throughput": {"processed_recent": 0, "processed_per_minute": 0.0},
"steps": [],
"runtime_activity": {},
"nodes": [],
}
mock_get_detect_capacity_plan.return_value = {
"estimated_hours_remaining": 0,
"remaining_items": 5001,
"recommended_additional_workers": 0,
}
data = fetch_overview()
self.assertEqual(1432, data["queue_display_running_total"])
self.assertEqual(1432, data["active_job"]["items_display_running"])
self.assertEqual(1432, data["active_job"]["display_items_running"])
self.assertEqual(138, data["active_job"]["display_active_threads"])
self.assertEqual(80000, data["active_job"]["display_max_threads"])
self.assertEqual(2, len(data["active_job"]["distributed_node_stats"]))
self.assertEqual(80000, data["queue_display_max_threads"])
self.assertEqual(0, data["current_job_blacklisted"])
self.assertEqual(0, data["recent_blacklisted_total"])
self.assertEqual(0, data["cumulative_blacklisted_total"])
@patch("app.services.dashboard._fetch_active_jobs_aggregate")
@patch("app.services.dashboard.get_detect_capacity_plan")
@patch("app.services.dashboard.get_detect_queue_health")
@patch("app.services.dashboard._build_dashboard_runtime_summary")
@patch("app.services.dashboard.get_active_detect_job_summary")
@patch("app.services.dashboard.get_db")
def test_fetch_overview_counts_active_execution_nodes_from_full_node_set(
self,
mock_get_db,
mock_get_active_detect_job_summary,
mock_build_dashboard_runtime_summary,
mock_get_detect_queue_health,
mock_get_detect_capacity_plan,
mock_fetch_active_jobs_aggregate,
) -> None:
mock_get_db.return_value = _FakeConnection(responses=[(0,)] * 9)
mock_fetch_active_jobs_aggregate.return_value = {
"active_jobs_total": 1,
"queue": {"items_total": 9, "pending": 0, "claimed": 0, "running": 9, "completed": 0, "blacklisted": 0, "failed": 0},
"throughput": {"processed_recent": 0, "processed_per_minute": 0.0, "completed_recent": 0, "blacklisted_recent": 0, "failed_recent": 0},
"steps": [],
"nodes": [
{
"node_code": f"mainland-controller-01-{index:02d}",
"items_running": 1,
"items_claimed": 0,
"processed_recent": 0,
"processed_per_minute": 0.0,
"completed_recent": 0,
"failed_recent": 0,
"blacklisted_recent": 0,
}
for index in range(9)
],
"retry_total": 0,
}
mock_get_active_detect_job_summary.return_value = {}
mock_build_dashboard_runtime_summary.return_value = {
"worker": {"running": True, "mode": "linux-systemd", "expected_on_this_node": True},
"node": {"region": "mainland", "role": "worker"},
"cluster": {"summary": {"online_worker_nodes": 1, "dedicated_online_worker_nodes": 0, "online_control_nodes": 1}},
"detect": {"backlog": {}},
}
mock_get_detect_queue_health.return_value = {
"has_active_job": False,
"queue": {},
"throughput": {"processed_recent": 0, "processed_per_minute": 0.0},
"steps": [],
"runtime_activity": {},
"nodes": [],
"runtime_snapshot_backlog": {},
}
mock_get_detect_capacity_plan.return_value = {
"estimated_hours_remaining": 0,
"remaining_items": 0,
"recommended_additional_workers": 0,
}
data = fetch_overview()
self.assertEqual(8, len(data["node_throughput"]))
self.assertEqual(9, data["active_execution_nodes"])
self.assertEqual(9, data["ops_summary"]["active_execution_nodes"])
@patch("app.services.dashboard._fetch_active_jobs_aggregate")
@patch("app.services.dashboard.get_detect_capacity_plan")
@patch("app.services.dashboard.get_detect_queue_health")
@patch("app.services.dashboard._build_dashboard_runtime_summary")
@patch("app.services.dashboard.get_active_detect_job_summary")
@patch("app.services.dashboard.get_db")
def test_fetch_overview_keeps_active_job_summary_when_queue_health_temporarily_empty(
self,
mock_get_db,
mock_get_active_detect_job_summary,
mock_build_dashboard_runtime_summary,
mock_get_detect_queue_health,
mock_get_detect_capacity_plan,
mock_fetch_active_jobs_aggregate,
) -> None:
mock_get_db.return_value = _FakeConnection(responses=[(0,)] * 9)
mock_fetch_active_jobs_aggregate.return_value = {
"active_jobs_total": 1,
"queue": {"items_total": 8402, "pending": 5001, "claimed": 0, "running": 537, "completed": 0, "blacklisted": 0, "failed": 0},
"throughput": {"processed_recent": 12, "processed_per_minute": 0.8, "completed_recent": 10, "blacklisted_recent": 1, "failed_recent": 1},
"steps": [],
"nodes": [],
"retry_total": 0,
}
mock_get_active_detect_job_summary.return_value = {
"job_id": 255,
"job_code": "sync-overseas-255",
"runtime_job_code": "sync-overseas-255",
"status": "running",
"progress_percent": 22.4,
"items_total": 8402,
"items_pending": 5001,
"items_claimed": 0,
"items_running": 537,
"items_completed": 0,
"items_blacklisted": 0,
"items_failed": 0,
"display_items_running": 11799,
"display_active_threads": 11799,
"display_max_threads": 75400,
"distributed_node_stats": [{"node_code": "mainland-controller-01-a", "display_running": 537, "active_threads": 537, "max_threads": 1000}],
}
mock_build_dashboard_runtime_summary.return_value = {
"worker": {"running": False, "mode": "linux-systemd", "expected_on_this_node": False},
"node": {"region": "overseas", "role": "control"},
"cluster": {"summary": {"online_worker_nodes": 1, "dedicated_online_worker_nodes": 0, "online_control_nodes": 1}},
"detect": {"backlog": {}},
}
mock_get_detect_queue_health.return_value = {
"has_active_job": False,
"queue": {},
"throughput": {"processed_recent": 0, "processed_per_minute": 0.0},
"steps": [],
"runtime_activity": {},
"nodes": [],
"runtime_snapshot_backlog": {},
}
mock_get_detect_capacity_plan.return_value = {
"estimated_hours_remaining": 0,
"remaining_items": 5001,
"recommended_additional_workers": 0,
}
data = fetch_overview()
self.assertEqual("sync-overseas-255", data["active_job"]["job_code"])
self.assertEqual(11799, data["active_job"]["display_items_running"])
self.assertEqual(75400, data["active_job"]["display_max_threads"])
if __name__ == "__main__":
unittest.main()