admin/getDomain
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: stabilize multi-region runtime sync and worker orchestration

Browse Source
This commit is contained in:
root
2026-04-27 15:48:12 +08:00
parent 7cbde2aa78
commit 215a364891
137 changed files with 31931 additions and 1943 deletions
Download Patch File Download Diff File Expand all files Collapse all files

522
domain-api/tests/test_runtime_status_service.py
View File

@@ -1,10 +1,273 @@
import unittest
from datetime import datetime, timedelta
from unittest.mock import MagicMock, patch
import app.services.runtime_status_service as runtime_status_service_module
from app.services.detect_service import _slice_remote_log_lines_fairly
from app.services.runtime_status_service import _align_queue_health_with_backlog, _build_detect_node_row
from app.services.runtime_status_service import (
_align_queue_health_with_backlog,
_build_detect_node_row,
_build_detect_observation_summary,
refresh_runtime_projection_snapshot,
get_runtime_status,
)
class RuntimeStatusServiceTests(unittest.TestCase):
def setUp(self) -> None:
runtime_status_service_module._DOMAIN_INVENTORY_CACHE_VALUE = None
runtime_status_service_module._DOMAIN_INVENTORY_CACHE_EXPIRES_AT = 0.0
runtime_status_service_module._RUNTIME_STATUS_CACHE_VALUE = None
runtime_status_service_module._RUNTIME_STATUS_CACHE_EXPIRES_AT = 0.0
runtime_status_service_module._RUNTIME_STATUS_CACHE_SIGNATURE = ()
def test_build_detect_observation_summary_marks_real_running_state(self) -> None:
summary = _build_detect_observation_summary(
detect_payload={
"active_job": {
"job_id": 308,
"job_code": "sync-overseas-16432",
"status": "running",
"progress_percent": 24.6,
"items_pending": 3999,
"items_claimed": 52,
"items_running": 136,
"items_completed": 971,
"items_failed": 0,
"items_blacklisted": 0,
"display_items_running": 222,
"display_active_threads": 222,
"display_max_threads": 2000,
"processed_recent": 172,
"processed_per_minute": 11.47,
"completed_recent": 2,
"failed_recent": 169,
"blacklisted_recent": 1,
},
"active_thread_count": 222,
"max_thread_count": 2000,
"queue_health": {
"queue": {
"pending": 3999,
"claimed": 52,
"running": 136,
"display_running": 222,
"completed": 971,
"failed": 0,
"blacklisted": 0,
},
"throughput": {
"processed_recent": 172,
"processed_per_minute": 11.47,
"completed_recent": 2,
"failed_recent": 169,
"blacklisted_recent": 1,
},
"steps": [
{
"step_code": "detect_register",
"step_name": "注册检测",
"items_pending": 2100,
"items_running": 40,
"processed_recent": 88,
"failed_recent": 100,
"blacklisted_recent": 0,
}
],
},
"participating_nodes": [
{
"node_code": "mainland-controller-01-a",
"participation_label": "执行中",
"is_dispatch_active": True,
"items_claimed": 12,
"items_running": 100,
"active_threads": 100,
"max_threads": 1000,
"processed_recent": 80,
"processed_per_minute": 5.33,
},
{
"node_code": "mainland-controller-01-b",
"participation_label": "执行中",
"is_dispatch_active": True,
"items_claimed": 8,
"items_running": 122,
"active_threads": 122,
"max_threads": 1000,
"processed_recent": 92,
"processed_per_minute": 6.13,
},
],
"participation_summary": {
"dispatch_active_nodes": 2,
"participating_nodes": 2,
},
"backlog": {
"pending_total": 3999,
"claimed_total": 52,
"running_total": 136,
"completed_total": 971,
},
},
cluster_snapshot={"summary": {"online_worker_nodes": 1}},
inventory_summary={
"scope_label": "海外主库总盘子",
"authoritative": True,
"domains_total": 5690937,
"pending_total": 5690918,
"completed_total": 0,
"running_total": 7,
"blacklist_total": 0,
"failed_total": 12,
"processed_total": 12,
"remaining_total": 5690925,
},
)
self.assertEqual("running", summary["state"])
self.assertEqual("真跑中", summary["state_label"])
self.assertEqual(2, summary["execution"]["active_processes"])
self.assertEqual(222, summary["execution"]["active_threads"])
self.assertEqual(169, summary["throughput"]["failed_recent"])
self.assertEqual("detect_register", summary["top_steps"][0]["step_code"])
self.assertEqual(5690937, summary["source_inventory"]["domains_total"])
self.assertEqual(5244, summary["active_batch"]["effective_items_total"])
self.assertIn("不等于全盘累计", summary["scope_hint"])
def test_build_detect_observation_summary_marks_not_running_when_only_backlog_left(self) -> None:
summary = _build_detect_observation_summary(
detect_payload={
"active_job": {
"job_id": 401,
"job_code": "sync-overseas-404",
"status": "running",
"items_pending": 5000,
"items_claimed": 0,
"items_running": 0,
"items_completed": 0,
"items_failed": 0,
"items_blacklisted": 0,
},
"queue_health": {
"queue": {
"pending": 5000,
"claimed": 0,
"running": 0,
"completed": 0,
"failed": 0,
"blacklisted": 0,
},
"throughput": {
"processed_recent": 0,
"processed_per_minute": 0,
"completed_recent": 0,
"failed_recent": 0,
"blacklisted_recent": 0,
},
"steps": [
{
"step_code": "detect_register",
"step_name": "注册检测",
"items_pending": 5000,
"items_running": 0,
}
],
},
"participating_nodes": [],
"participation_summary": {
"dispatch_active_nodes": 0,
"participating_nodes": 0,
},
"backlog": {
"pending_total": 5000,
"claimed_total": 0,
"running_total": 0,
},
},
cluster_snapshot={"summary": {"online_worker_nodes": 1}},
inventory_summary={
"scope_label": "海外主库总盘子",
"authoritative": True,
"domains_total": 5690937,
"pending_total": 5690918,
"completed_total": 0,
"running_total": 7,
"blacklist_total": 0,
"failed_total": 12,
"processed_total": 12,
"remaining_total": 5690925,
},
)
self.assertEqual("not_running", summary["state"])
self.assertEqual("没跑起来", summary["state_label"])
self.assertEqual(0, summary["execution"]["active_processes"])
self.assertIn("队列", summary["focus_hint"])
self.assertEqual(5000, summary["active_batch"]["effective_items_total"])
self.assertEqual(5690937, summary["source_inventory"]["domains_total"])
@patch("app.services.runtime_status_service.get_active_detect_job_summary")
@patch("app.services.runtime_status_service.get_db")
def test_load_detect_backlog_snapshot_prefers_active_job_snapshot(
self,
mock_get_db,
mock_get_active_detect_job_summary,
) -> None:
mock_get_active_detect_job_summary.return_value = {
"items_pending": 321,
"items_claimed": 2,
"items_running": 9,
"items_completed": 50,
"items_failed": 3,
"display_active_threads": 11,
"step_stats": [
{"step_code": "detect_register", "items_pending": 210},
{"step_code": "detect_baidu", "items_pending": 111},
],
}
backlog = runtime_status_service_module._load_detect_backlog_snapshot()
self.assertEqual(321, backlog["pending_total"])
self.assertEqual(2, backlog["claimed_total"])
self.assertEqual(11, backlog["running_total"])
self.assertEqual(50, backlog["completed_total"])
self.assertEqual(3, backlog["failed_total"])
self.assertEqual(210, backlog["register_pending"])
self.assertEqual(111, backlog["downstream_pending"])
mock_get_db.assert_not_called()
@patch("app.services.runtime_status_service.get_active_detect_job_summary", return_value={})
@patch("app.services.runtime_status_service.get_db")
def test_load_detect_backlog_snapshot_ignores_stale_pending_jobs(
self,
mock_get_db,
_mock_get_active_detect_job_summary,
) -> None:
conn = MagicMock()
cursor_cm = MagicMock()
cursor = MagicMock()
cursor.fetchall.return_value = [
(11, "running", datetime.now() - timedelta(hours=2)),
(10, "pending", datetime.now() - timedelta(minutes=30)),
(9, "pending", datetime.now() - timedelta(hours=8)),
]
cursor.fetchone.return_value = (321, 2, 9, 50, 1, 3, 210, 111)
conn.cursor.return_value = cursor_cm
cursor_cm.__enter__.return_value = cursor
db_cm = MagicMock()
db_cm.__enter__.return_value = conn
mock_get_db.return_value = db_cm
backlog = runtime_status_service_module._load_detect_backlog_snapshot()
self.assertEqual(321, backlog["pending_total"])
self.assertEqual(210, backlog["register_pending"])
executed_sql, executed_params = cursor.execute.call_args_list[1][0]
self.assertIn("WHERE item.job_id = ANY(%s)", executed_sql)
self.assertEqual([11, 10], list(executed_params[0]))
def test_build_detect_node_row_merges_queue_running_metrics(self) -> None:
row = _build_detect_node_row(
node_code="mainland-worker-01",
@@ -92,6 +355,42 @@ class RuntimeStatusServiceTests(unittest.TestCase):
self.assertTrue(any("[mainland-controller-01]" in line for line in sliced))
self.assertTrue(any("[mainland-worker-01]" in line for line in sliced))
def test_build_multi_region_readiness_skips_local_batch_warning_when_batches_are_not_applicable(self) -> None:
with patch("app.services.runtime_status_service.settings.node_region", "overseas"), \
patch("app.services.runtime_status_service.settings.node_role", "control"):
readiness = runtime_status_service_module._build_multi_region_readiness(
cluster_snapshot={
"nodes": [
{"node_code": "overseas-control-01", "region": "overseas", "role": "control", "status": "online"},
{"node_code": "mainland-controller-01", "region": "mainland", "role": "control", "status": "online"},
{"node_code": "mainland-worker-01", "region": "mainland", "role": "worker", "status": "busy", "is_effective_worker": True},
],
"summary": {
"online_control_nodes": 2,
"online_worker_nodes": 1,
},
},
sync_summary={
"enabled": False,
"source_region": "overseas",
"target_region": "mainland",
"detect_result_batches": {
"applicable": False,
"reason": "当前节点不承载本地检测执行，结果批次推送概览不适用。",
"state_counts": {"projected": 3, "failed": 1, "pushing": 1, "synced": 0},
},
},
worker_runtime={"running": False},
sync_agent_runtime={"running": False},
)
warning_text = " ".join(readiness["warnings"])
info_text = " ".join(readiness["info"])
self.assertNotIn("结果批次仍待推送", warning_text)
self.assertNotIn("结果批次同步失败", warning_text)
self.assertIn("不适用", info_text)
self.assertFalse(readiness["sync"]["applicable"])
def test_align_queue_health_with_backlog_prefers_larger_runtime_snapshot(self) -> None:
aligned = _align_queue_health_with_backlog(
{
@@ -126,6 +425,227 @@ class RuntimeStatusServiceTests(unittest.TestCase):
self.assertEqual(8, aligned["queue"]["failed"])
self.assertEqual(1790, aligned["queue"]["items_total"])
def test_load_latest_remote_runtime_projection_backlog_skips_future_dated_rows(self) -> None:
now = datetime.now()
conn = MagicMock()
cursor_cm = MagicMock()
cursor = MagicMock()
cursor.fetchall.return_value = [
({"projection": {"backlog": {"pending_total": 9999}}}, now + timedelta(hours=4)),
({"projection": {"backlog": {"pending_total": 321, "running_total": 8}}}, now - timedelta(minutes=2)),
]
conn.cursor.return_value = cursor_cm
cursor_cm.__enter__.return_value = cursor
db_cm = MagicMock()
db_cm.__enter__.return_value = conn
with patch("app.services.runtime_status_service.settings.node_region", "overseas"), \
patch("app.services.runtime_status_service.settings.node_role", "control"), \
patch("app.services.runtime_status_service.get_db", return_value=db_cm):
backlog = runtime_status_service_module._load_latest_remote_runtime_projection_backlog()
self.assertEqual(321, backlog["pending_total"])
self.assertEqual(8, backlog["running_total"])
def test_get_runtime_status_keeps_remote_aggregate_detect_metrics_on_overseas_control(self) -> None:
with patch("app.services.runtime_status_service.settings.node_region", "overseas"), \
patch("app.services.runtime_status_service.settings.node_role", "control"), \
patch("app.services.runtime_status_service.detect_worker_runtime", return_value={"running": False, "mode": "linux-systemd", "process_count": 0, "message": "inactive", "latest_start_time": ""}), \
patch("app.services.runtime_status_service.detect_sync_agent_runtime", return_value={"running": False, "mode": "linux-systemd", "process_count": 0, "message": "inactive", "latest_start_time": ""}), \
patch("app.services.runtime_status_service.get_runtime_settings", return_value={"worker_mode": "linux-systemd"}), \
patch("app.services.runtime_status_service.get_detect_status", return_value={
"phase_label": "",
"phase_detail": "",
"recent_event": "远端执行中",
"recent_warning": "",
"progress_percent": 40.49,
"progress": {"pending": 5001, "running": 138, "completed": 0, "failed": 0, "blacklisted": 0},
"active_thread_count": 138,
"max_thread_count": 80000,
"aggregate_process_count": 80,
"aggregate_participating_node_count": 1,
"aggregate_participating_node_codes": ["mainland-controller-01"],
"aggregate_max_thread_count": 80000,
"aggregate_thread_count_per_process": 1000,
"available_proxy_count": 0,
"proxy_pool_count": 0,
"proxy_runtime_label": "",
"proxy_runtime_detail": "",
"proxy_runtime_reason": "",
"proxy_supplier_empty": False,
"proxy_last_refresh_status": "",
"proxy_last_refresh_time": "",
"proxy_last_refresh_source_count": 0,
"proxy_last_refresh_total_items": 0,
"proxy_last_validated_count": 0,
"proxy_last_available_count": 0,
"proxy_source_stats": [],
"dependency_alerts": [],
"active_job": {"job_code": "sync-overseas-51", "items_pending": 5001, "items_running": 0},
"runs": [],
"remote_log_line_count": 0,
"remote_log_node_count": 0,
"remote_log_nodes": [],
"remote_log_node_summaries": [],
"remote_log_last_at": "",
"remote_log_last_line": "",
"remote_log_lines": [],
"aggregate_detect_view": True,
}), \
patch("app.services.runtime_status_service.get_cluster_snapshot", return_value={
"nodes": [{"node_code": "mainland-controller-01", "role": "control", "region": "mainland", "status": "busy", "is_effective_worker": True, "current_load": 1432, "metadata": {"active_threads": 138, "max_threads": 80000}}],
"summary": {"online_worker_nodes": 1, "dedicated_online_worker_nodes": 0, "online_control_nodes": 2},
}), \
patch("app.services.runtime_status_service.get_detect_queue_health", return_value={
"has_active_job": True,
"job": {"job_id": 11, "job_code": "sync-overseas-51", "status": "running", "progress_percent": 40.49},
"queue": {"items_total": 8402, "pending": 5001, "claimed": 0, "running": 1432, "display_running": 1432},
"nodes": [{"node_code": "mainland-controller-01-a", "items_running": 1432, "display_running": 1432, "active_threads": 1432, "max_threads": 1000}],
"steps": [],
"runtime_activity": {},
"window_minutes": 15,
}), \
patch("app.services.runtime_status_service._load_detect_backlog_snapshot", return_value={"pending_total": 5001, "claimed_total": 0, "running_total": 0, "completed_total": 0, "blacklisted_total": 0, "failed_total": 0, "register_pending": 3495, "downstream_pending": 1506}), \
patch("app.services.runtime_status_service._load_latest_remote_runtime_projection_backlog", return_value={}), \
patch("app.services.runtime_status_service._load_latest_runtime_active_job_snapshot", return_value={}), \
patch("app.services.runtime_status_service._load_domain_inventory_summary", return_value={"scope_label": "海外主库总盘子", "authoritative": True, "domains_total": 5690937, "pending_total": 5690918, "completed_total": 0, "running_total": 7, "blacklist_total": 0, "failed_total": 12, "processed_total": 12, "remaining_total": 5690925}), \
patch("app.services.runtime_status_service.get_detect_capacity_plan", return_value={}), \
patch("app.services.runtime_status_service.append_runtime_projection_if_changed"), \
patch("app.services.runtime_status_service.get_sync_summary", return_value={}), \
patch("app.services.runtime_status_service._build_multi_region_readiness", return_value={"status": "ready", "ready": True}), \
patch("app.services.runtime_status_service.get_runtime_build_info", return_value={}):
payload = get_runtime_status()
self.assertEqual(1, payload["detect"]["aggregate_process_count"])
self.assertEqual(1432, payload["detect"]["active_thread_count"])
self.assertEqual(1000, payload["detect"]["max_thread_count"])
self.assertEqual("集群执行中", payload["detect"]["phase_label"])
self.assertFalse(payload["detect"]["worker_online"])
self.assertEqual("sync-overseas-51", payload["detect"]["active_job"]["job_code"])
self.assertIn("observation_summary", payload["detect"])
self.assertEqual("在跑但偏慢", payload["detect"]["observation_summary"]["state_label"])
self.assertEqual(5690937, payload["detect"]["observation_summary"]["source_inventory"]["domains_total"])
def test_get_runtime_status_reuses_short_ttl_cache(self) -> None:
context_payload = {
"runtime_settings": {
"worker_mode": "linux-systemd",
"api_service_name": "domaincheck-api",
"worker_service_name": "domaincheck-worker",
"sync_agent_service_name": "domaincheck-sync-agent",
},
"worker_runtime": {
"running": True,
"mode": "linux-systemd",
"process_count": 60,
"latest_start_time": "2026-04-24 17:10:28",
"message": "running",
"runtime_state": {"phase": "detecting"},
},
"worker_expected_on_this_node": True,
"detect_payload": {
"_detect_snapshot": {"thread_count": 1000},
"active_thread_count": 321,
"max_thread_count": 60000,
"progress": {"pending": 10, "running": 3},
"backlog": {"pending_total": 10, "running_total": 3},
"progress_percent": 25.5,
"available_proxy_count": 12,
"proxy_pool_count": 20,
"proxy_runtime_label": "代理正常",
"proxy_runtime_detail": "ok",
"proxy_runtime_reason": "healthy",
"proxy_last_refresh_time": "2026-04-24 17:12:55",
"recent_event": "running",
"recent_warning": "",
},
"cluster_snapshot": {"summary": {"online_worker_nodes": 1}},
}
sync_summary = {"enabled": True}
readiness = {"status": "ready", "ready": True}
with patch("app.services.runtime_status_service._build_runtime_detect_context", return_value=context_payload) as mock_context, \
patch("app.services.runtime_status_service.detect_sync_agent_runtime", return_value={"running": True, "mode": "linux-systemd", "process_count": 1, "latest_start_time": "", "message": "ok"}), \
patch("app.services.runtime_status_service.append_runtime_projection_if_changed") as mock_append, \
patch("app.services.runtime_status_service.get_sync_summary", return_value=sync_summary) as mock_sync_summary, \
patch("app.services.runtime_status_service._build_multi_region_readiness", return_value=readiness), \
patch("app.services.runtime_status_service.get_runtime_build_info", return_value={"package_name": "pkg"}), \
patch("app.services.runtime_status_service.time.monotonic", side_effect=[100.0, 100.1, 100.2]):
first = get_runtime_status()
second = get_runtime_status()
self.assertEqual(first, second)
self.assertEqual(1, mock_context.call_count)
self.assertEqual(1, mock_append.call_count)
self.assertEqual(1, mock_sync_summary.call_count)
@patch("app.services.runtime_status_service.append_runtime_projection_if_changed", return_value=321)
@patch(
"app.services.runtime_status_service.get_cluster_snapshot",
return_value={"nodes": [], "summary": {"online_worker_nodes": 3}},
)
@patch(
"app.services.runtime_status_service.get_active_detect_job_summary",
return_value={
"job_id": 867,
"job_code": "sync-overseas-19835",
"status": "running",
"progress_percent": 99.1,
"items_total": 10000,
"items_pending": 4,
"items_claimed": 0,
"items_running": 5,
"items_completed": 55402,
"items_blacklisted": 0,
"items_failed": 0,
"display_items_running": 5,
"display_active_threads": 5,
"display_items_claimed": 0,
"display_max_threads": 1000,
"node_stats": [{"node_code": "mainland-controller-01", "items_running": 5}],
"raw_step_stats": [{"step_code": "detect_wayback", "items_pending": 4}],
},
)
@patch(
"app.services.runtime_status_service.detect_worker_runtime",
return_value={"running": True, "mode": "linux-systemd", "process_count": 60, "thread_count": 1000, "max_threads": 60000, "message": "ok"},
)
@patch("app.services.runtime_status_service.get_runtime_settings", return_value={"thread_count": 1000, "worker_mode": "linux-systemd"})
@patch(
"app.services.runtime_status_service._load_detect_backlog_snapshot",
return_value={
"pending_total": 4,
"claimed_total": 0,
"running_total": 5,
"completed_total": 55402,
"blacklisted_total": 0,
"failed_total": 0,
"register_pending": 0,
"downstream_pending": 4,
},
)
@patch("app.services.runtime_status_service._build_runtime_detect_context", side_effect=AssertionError("heavy path should not run"))
def test_refresh_runtime_projection_snapshot_uses_lightweight_context(
self,
mock_heavy_context,
mock_load_detect_backlog_snapshot,
mock_get_runtime_settings,
mock_detect_worker_runtime,
mock_get_active_detect_job_summary,
mock_get_cluster_snapshot,
mock_append_runtime_projection_if_changed,
) -> None:
result = refresh_runtime_projection_snapshot()
self.assertEqual(321, result["record_id"])
mock_append_runtime_projection_if_changed.assert_called_once()
detect = mock_append_runtime_projection_if_changed.call_args.kwargs["detect"]
self.assertEqual("sync-overseas-19835", detect["active_job"]["job_code"])
self.assertEqual(5, detect["active_thread_count"])
self.assertEqual(4, detect["backlog"]["pending_total"])
self.assertEqual(5, detect["queue_health"]["queue"]["running"])
mock_heavy_context.assert_not_called()
if __name__ == "__main__":
unittest.main()