import unittest from datetime import datetime, timedelta from unittest.mock import MagicMock, patch import app.services.runtime_status_service as runtime_status_service_module from app.services.detect_service import _slice_remote_log_lines_fairly from app.services.runtime_status_service import ( _align_queue_health_with_backlog, _build_detect_node_row, _build_detect_observation_summary, refresh_runtime_projection_snapshot, get_runtime_status, ) class RuntimeStatusServiceTests(unittest.TestCase): def setUp(self) -> None: runtime_status_service_module._DOMAIN_INVENTORY_CACHE_VALUE = None runtime_status_service_module._DOMAIN_INVENTORY_CACHE_EXPIRES_AT = 0.0 runtime_status_service_module._RUNTIME_STATUS_CACHE_VALUE = None runtime_status_service_module._RUNTIME_STATUS_CACHE_EXPIRES_AT = 0.0 runtime_status_service_module._RUNTIME_STATUS_CACHE_SIGNATURE = () def test_build_detect_observation_summary_marks_real_running_state(self) -> None: summary = _build_detect_observation_summary( detect_payload={ "active_job": { "job_id": 308, "job_code": "sync-overseas-16432", "status": "running", "progress_percent": 24.6, "items_pending": 3999, "items_claimed": 52, "items_running": 136, "items_completed": 971, "items_failed": 0, "items_blacklisted": 0, "display_items_running": 222, "display_active_threads": 222, "display_max_threads": 2000, "processed_recent": 172, "processed_per_minute": 11.47, "completed_recent": 2, "failed_recent": 169, "blacklisted_recent": 1, }, "active_thread_count": 222, "max_thread_count": 2000, "queue_health": { "queue": { "pending": 3999, "claimed": 52, "running": 136, "display_running": 222, "completed": 971, "failed": 0, "blacklisted": 0, }, "throughput": { "processed_recent": 172, "processed_per_minute": 11.47, "completed_recent": 2, "failed_recent": 169, "blacklisted_recent": 1, }, "steps": [ { "step_code": "detect_register", "step_name": "注册检测", "items_pending": 2100, "items_running": 40, "processed_recent": 88, "failed_recent": 100, "blacklisted_recent": 0, } ], }, "participating_nodes": [ { "node_code": "mainland-controller-01-a", "participation_label": "执行中", "is_dispatch_active": True, "items_claimed": 12, "items_running": 100, "active_threads": 100, "max_threads": 1000, "processed_recent": 80, "processed_per_minute": 5.33, }, { "node_code": "mainland-controller-01-b", "participation_label": "执行中", "is_dispatch_active": True, "items_claimed": 8, "items_running": 122, "active_threads": 122, "max_threads": 1000, "processed_recent": 92, "processed_per_minute": 6.13, }, ], "participation_summary": { "dispatch_active_nodes": 2, "participating_nodes": 2, }, "backlog": { "pending_total": 3999, "claimed_total": 52, "running_total": 136, "completed_total": 971, }, }, cluster_snapshot={"summary": {"online_worker_nodes": 1}}, inventory_summary={ "scope_label": "海外主库总盘子", "authoritative": True, "domains_total": 5690937, "pending_total": 5690918, "completed_total": 0, "running_total": 7, "blacklist_total": 0, "failed_total": 12, "processed_total": 12, "remaining_total": 5690925, }, ) self.assertEqual("running", summary["state"]) self.assertEqual("真跑中", summary["state_label"]) self.assertEqual(2, summary["execution"]["active_processes"]) self.assertEqual(222, summary["execution"]["active_threads"]) self.assertEqual(169, summary["throughput"]["failed_recent"]) self.assertEqual("detect_register", summary["top_steps"][0]["step_code"]) self.assertEqual(5690937, summary["source_inventory"]["domains_total"]) self.assertEqual(5244, summary["active_batch"]["effective_items_total"]) self.assertIn("不等于全盘累计", summary["scope_hint"]) def test_build_detect_observation_summary_marks_not_running_when_only_backlog_left(self) -> None: summary = _build_detect_observation_summary( detect_payload={ "active_job": { "job_id": 401, "job_code": "sync-overseas-404", "status": "running", "items_pending": 5000, "items_claimed": 0, "items_running": 0, "items_completed": 0, "items_failed": 0, "items_blacklisted": 0, }, "queue_health": { "queue": { "pending": 5000, "claimed": 0, "running": 0, "completed": 0, "failed": 0, "blacklisted": 0, }, "throughput": { "processed_recent": 0, "processed_per_minute": 0, "completed_recent": 0, "failed_recent": 0, "blacklisted_recent": 0, }, "steps": [ { "step_code": "detect_register", "step_name": "注册检测", "items_pending": 5000, "items_running": 0, } ], }, "participating_nodes": [], "participation_summary": { "dispatch_active_nodes": 0, "participating_nodes": 0, }, "backlog": { "pending_total": 5000, "claimed_total": 0, "running_total": 0, }, }, cluster_snapshot={"summary": {"online_worker_nodes": 1}}, inventory_summary={ "scope_label": "海外主库总盘子", "authoritative": True, "domains_total": 5690937, "pending_total": 5690918, "completed_total": 0, "running_total": 7, "blacklist_total": 0, "failed_total": 12, "processed_total": 12, "remaining_total": 5690925, }, ) self.assertEqual("not_running", summary["state"]) self.assertEqual("没跑起来", summary["state_label"]) self.assertEqual(0, summary["execution"]["active_processes"]) self.assertIn("队列", summary["focus_hint"]) self.assertEqual(5000, summary["active_batch"]["effective_items_total"]) self.assertEqual(5690937, summary["source_inventory"]["domains_total"]) @patch("app.services.runtime_status_service.get_active_detect_job_summary") @patch("app.services.runtime_status_service.get_db") def test_load_detect_backlog_snapshot_prefers_active_job_snapshot( self, mock_get_db, mock_get_active_detect_job_summary, ) -> None: mock_get_active_detect_job_summary.return_value = { "items_pending": 321, "items_claimed": 2, "items_running": 9, "items_completed": 50, "items_failed": 3, "display_active_threads": 11, "step_stats": [ {"step_code": "detect_register", "items_pending": 210}, {"step_code": "detect_baidu", "items_pending": 111}, ], } backlog = runtime_status_service_module._load_detect_backlog_snapshot() self.assertEqual(321, backlog["pending_total"]) self.assertEqual(2, backlog["claimed_total"]) self.assertEqual(11, backlog["running_total"]) self.assertEqual(50, backlog["completed_total"]) self.assertEqual(3, backlog["failed_total"]) self.assertEqual(210, backlog["register_pending"]) self.assertEqual(111, backlog["downstream_pending"]) mock_get_db.assert_not_called() @patch("app.services.runtime_status_service.get_active_detect_job_summary", return_value={}) @patch("app.services.runtime_status_service.get_db") def test_load_detect_backlog_snapshot_ignores_stale_pending_jobs( self, mock_get_db, _mock_get_active_detect_job_summary, ) -> None: conn = MagicMock() cursor_cm = MagicMock() cursor = MagicMock() cursor.fetchall.return_value = [ (11, "running", datetime.now() - timedelta(hours=2)), (10, "pending", datetime.now() - timedelta(minutes=30)), (9, "pending", datetime.now() - timedelta(hours=8)), ] cursor.fetchone.return_value = (321, 2, 9, 50, 1, 3, 210, 111) conn.cursor.return_value = cursor_cm cursor_cm.__enter__.return_value = cursor db_cm = MagicMock() db_cm.__enter__.return_value = conn mock_get_db.return_value = db_cm backlog = runtime_status_service_module._load_detect_backlog_snapshot() self.assertEqual(321, backlog["pending_total"]) self.assertEqual(210, backlog["register_pending"]) executed_sql, executed_params = cursor.execute.call_args_list[1][0] self.assertIn("WHERE item.job_id = ANY(%s)", executed_sql) self.assertEqual([11, 10], list(executed_params[0])) def test_build_detect_node_row_merges_queue_running_metrics(self) -> None: row = _build_detect_node_row( node_code="mainland-worker-01", cluster_node={ "node_code": "mainland-worker-01", "region": "mainland", "role": "worker", "status": "busy", "is_effective_worker": True, "current_load": 61, "metadata": { "job_items_total": 1000, "job_items_claimed": 0, "job_items_running": 61, }, }, job_node={ "node_code": "mainland-worker-01", "items_total": 0, "items_pending": 0, "items_claimed": 0, "items_running": 0, "items_completed": 0, "items_failed": 0, }, queue_node={ "node_code": "mainland-worker-01", "items_total": 1000, "items_pending": 739, "items_claimed": 200, "items_running": 61, "items_completed": 191, "items_blacklisted": 0, "items_failed": 0, "processed_recent": 632, "processed_per_minute": 42.13, }, ) self.assertEqual(1000, row["items_total"]) self.assertEqual(200, row["items_claimed"]) self.assertEqual(61, row["items_running"]) self.assertEqual("running", row["participation_state"]) self.assertTrue(row["is_current_participant"]) self.assertTrue(row["is_dispatch_active"]) def test_build_detect_node_row_uses_active_threads_as_current_execution_signal(self) -> None: row = _build_detect_node_row( node_code="mainland-worker-01", cluster_node={ "node_code": "mainland-worker-01", "region": "mainland", "role": "worker", "status": "busy", "is_effective_worker": True, "current_load": 61, "metadata": { "active_threads": 61, "max_threads": 400, }, }, job_node={}, queue_node={ "node_code": "mainland-worker-01", "processed_recent": 632, "processed_per_minute": 42.13, }, ) self.assertEqual(61, row["active_threads"]) self.assertEqual(400, row["max_threads"]) self.assertEqual("runtime_active", row["participation_state"]) self.assertEqual("执行中", row["participation_label"]) self.assertTrue(row["is_dispatch_active"]) def test_slice_remote_log_lines_fairly_keeps_secondary_node_visible(self) -> None: lines = [f"[2026-04-19 17:30:{i:02d}] [mainland-controller-01] controller-{i}" for i in range(20)] lines.extend( [f"[2026-04-19 17:31:{i:02d}] [mainland-worker-01] worker-{i}" for i in range(2)] ) sliced = _slice_remote_log_lines_fairly(lines, limit=6, min_per_node=2) self.assertEqual(6, len(sliced)) self.assertTrue(any("[mainland-controller-01]" in line for line in sliced)) self.assertTrue(any("[mainland-worker-01]" in line for line in sliced)) def test_build_multi_region_readiness_skips_local_batch_warning_when_batches_are_not_applicable(self) -> None: with patch("app.services.runtime_status_service.settings.node_region", "overseas"), \ patch("app.services.runtime_status_service.settings.node_role", "control"): readiness = runtime_status_service_module._build_multi_region_readiness( cluster_snapshot={ "nodes": [ {"node_code": "overseas-control-01", "region": "overseas", "role": "control", "status": "online"}, {"node_code": "mainland-controller-01", "region": "mainland", "role": "control", "status": "online"}, {"node_code": "mainland-worker-01", "region": "mainland", "role": "worker", "status": "busy", "is_effective_worker": True}, ], "summary": { "online_control_nodes": 2, "online_worker_nodes": 1, }, }, sync_summary={ "enabled": False, "source_region": "overseas", "target_region": "mainland", "detect_result_batches": { "applicable": False, "reason": "当前节点不承载本地检测执行，结果批次推送概览不适用。", "state_counts": {"projected": 3, "failed": 1, "pushing": 1, "synced": 0}, }, }, worker_runtime={"running": False}, sync_agent_runtime={"running": False}, ) warning_text = " ".join(readiness["warnings"]) info_text = " ".join(readiness["info"]) self.assertNotIn("结果批次仍待推送", warning_text) self.assertNotIn("结果批次同步失败", warning_text) self.assertIn("不适用", info_text) self.assertFalse(readiness["sync"]["applicable"]) def test_align_queue_health_with_backlog_prefers_larger_runtime_snapshot(self) -> None: aligned = _align_queue_health_with_backlog( { "has_active_job": True, "queue": { "items_total": 120, "pending": 100, "claimed": 10, "running": 5, "completed": 5, "blacklisted": 0, "failed": 0, "terminal": 5, }, }, { "pending_total": 1200, "claimed_total": 230, "running_total": 40, "completed_total": 300, "blacklisted_total": 12, "failed_total": 8, }, ) self.assertTrue(aligned["has_active_job"]) self.assertEqual(1200, aligned["queue"]["pending"]) self.assertEqual(230, aligned["queue"]["claimed"]) self.assertEqual(40, aligned["queue"]["running"]) self.assertEqual(300, aligned["queue"]["completed"]) self.assertEqual(12, aligned["queue"]["blacklisted"]) self.assertEqual(8, aligned["queue"]["failed"]) self.assertEqual(1790, aligned["queue"]["items_total"]) def test_load_latest_remote_runtime_projection_backlog_skips_future_dated_rows(self) -> None: now = datetime.now() conn = MagicMock() cursor_cm = MagicMock() cursor = MagicMock() cursor.fetchall.return_value = [ ({"projection": {"backlog": {"pending_total": 9999}}}, now + timedelta(hours=4)), ({"projection": {"backlog": {"pending_total": 321, "running_total": 8}}}, now - timedelta(minutes=2)), ] conn.cursor.return_value = cursor_cm cursor_cm.__enter__.return_value = cursor db_cm = MagicMock() db_cm.__enter__.return_value = conn with patch("app.services.runtime_status_service.settings.node_region", "overseas"), \ patch("app.services.runtime_status_service.settings.node_role", "control"), \ patch("app.services.runtime_status_service.get_db", return_value=db_cm): backlog = runtime_status_service_module._load_latest_remote_runtime_projection_backlog() self.assertEqual(321, backlog["pending_total"]) self.assertEqual(8, backlog["running_total"]) def test_get_runtime_status_keeps_remote_aggregate_detect_metrics_on_overseas_control(self) -> None: with patch("app.services.runtime_status_service.settings.node_region", "overseas"), \ patch("app.services.runtime_status_service.settings.node_role", "control"), \ patch("app.services.runtime_status_service.detect_worker_runtime", return_value={"running": False, "mode": "linux-systemd", "process_count": 0, "message": "inactive", "latest_start_time": ""}), \ patch("app.services.runtime_status_service.detect_sync_agent_runtime", return_value={"running": False, "mode": "linux-systemd", "process_count": 0, "message": "inactive", "latest_start_time": ""}), \ patch("app.services.runtime_status_service.get_runtime_settings", return_value={"worker_mode": "linux-systemd"}), \ patch("app.services.runtime_status_service.get_detect_status", return_value={ "phase_label": "", "phase_detail": "", "recent_event": "远端执行中", "recent_warning": "", "progress_percent": 40.49, "progress": {"pending": 5001, "running": 138, "completed": 0, "failed": 0, "blacklisted": 0}, "active_thread_count": 138, "max_thread_count": 80000, "aggregate_process_count": 80, "aggregate_participating_node_count": 1, "aggregate_participating_node_codes": ["mainland-controller-01"], "aggregate_max_thread_count": 80000, "aggregate_thread_count_per_process": 1000, "available_proxy_count": 0, "proxy_pool_count": 0, "proxy_runtime_label": "", "proxy_runtime_detail": "", "proxy_runtime_reason": "", "proxy_supplier_empty": False, "proxy_last_refresh_status": "", "proxy_last_refresh_time": "", "proxy_last_refresh_source_count": 0, "proxy_last_refresh_total_items": 0, "proxy_last_validated_count": 0, "proxy_last_available_count": 0, "proxy_source_stats": [], "dependency_alerts": [], "active_job": {"job_code": "sync-overseas-51", "items_pending": 5001, "items_running": 0}, "runs": [], "remote_log_line_count": 0, "remote_log_node_count": 0, "remote_log_nodes": [], "remote_log_node_summaries": [], "remote_log_last_at": "", "remote_log_last_line": "", "remote_log_lines": [], "aggregate_detect_view": True, }), \ patch("app.services.runtime_status_service.get_cluster_snapshot", return_value={ "nodes": [{"node_code": "mainland-controller-01", "role": "control", "region": "mainland", "status": "busy", "is_effective_worker": True, "current_load": 1432, "metadata": {"active_threads": 138, "max_threads": 80000}}], "summary": {"online_worker_nodes": 1, "dedicated_online_worker_nodes": 0, "online_control_nodes": 2}, }), \ patch("app.services.runtime_status_service.get_detect_queue_health", return_value={ "has_active_job": True, "job": {"job_id": 11, "job_code": "sync-overseas-51", "status": "running", "progress_percent": 40.49}, "queue": {"items_total": 8402, "pending": 5001, "claimed": 0, "running": 1432, "display_running": 1432}, "nodes": [{"node_code": "mainland-controller-01-a", "items_running": 1432, "display_running": 1432, "active_threads": 1432, "max_threads": 1000}], "steps": [], "runtime_activity": {}, "window_minutes": 15, }), \ patch("app.services.runtime_status_service._load_detect_backlog_snapshot", return_value={"pending_total": 5001, "claimed_total": 0, "running_total": 0, "completed_total": 0, "blacklisted_total": 0, "failed_total": 0, "register_pending": 3495, "downstream_pending": 1506}), \ patch("app.services.runtime_status_service._load_latest_remote_runtime_projection_backlog", return_value={}), \ patch("app.services.runtime_status_service._load_latest_runtime_active_job_snapshot", return_value={}), \ patch("app.services.runtime_status_service._load_domain_inventory_summary", return_value={"scope_label": "海外主库总盘子", "authoritative": True, "domains_total": 5690937, "pending_total": 5690918, "completed_total": 0, "running_total": 7, "blacklist_total": 0, "failed_total": 12, "processed_total": 12, "remaining_total": 5690925}), \ patch("app.services.runtime_status_service.get_detect_capacity_plan", return_value={}), \ patch("app.services.runtime_status_service.append_runtime_projection_if_changed"), \ patch("app.services.runtime_status_service.get_sync_summary", return_value={}), \ patch("app.services.runtime_status_service._build_multi_region_readiness", return_value={"status": "ready", "ready": True}), \ patch("app.services.runtime_status_service.get_runtime_build_info", return_value={}): payload = get_runtime_status() self.assertEqual(1, payload["detect"]["aggregate_process_count"]) self.assertEqual(1432, payload["detect"]["active_thread_count"]) self.assertEqual(1000, payload["detect"]["max_thread_count"]) self.assertEqual("集群执行中", payload["detect"]["phase_label"]) self.assertFalse(payload["detect"]["worker_online"]) self.assertEqual("sync-overseas-51", payload["detect"]["active_job"]["job_code"]) self.assertIn("observation_summary", payload["detect"]) self.assertEqual("在跑但偏慢", payload["detect"]["observation_summary"]["state_label"]) self.assertEqual(5690937, payload["detect"]["observation_summary"]["source_inventory"]["domains_total"]) def test_get_runtime_status_reuses_short_ttl_cache(self) -> None: context_payload = { "runtime_settings": { "worker_mode": "linux-systemd", "api_service_name": "domaincheck-api", "worker_service_name": "domaincheck-worker", "sync_agent_service_name": "domaincheck-sync-agent", }, "worker_runtime": { "running": True, "mode": "linux-systemd", "process_count": 60, "latest_start_time": "2026-04-24 17:10:28", "message": "running", "runtime_state": {"phase": "detecting"}, }, "worker_expected_on_this_node": True, "detect_payload": { "_detect_snapshot": {"thread_count": 1000}, "active_thread_count": 321, "max_thread_count": 60000, "progress": {"pending": 10, "running": 3}, "backlog": {"pending_total": 10, "running_total": 3}, "progress_percent": 25.5, "available_proxy_count": 12, "proxy_pool_count": 20, "proxy_runtime_label": "代理正常", "proxy_runtime_detail": "ok", "proxy_runtime_reason": "healthy", "proxy_last_refresh_time": "2026-04-24 17:12:55", "recent_event": "running", "recent_warning": "", }, "cluster_snapshot": {"summary": {"online_worker_nodes": 1}}, } sync_summary = {"enabled": True} readiness = {"status": "ready", "ready": True} with patch("app.services.runtime_status_service._build_runtime_detect_context", return_value=context_payload) as mock_context, \ patch("app.services.runtime_status_service.detect_sync_agent_runtime", return_value={"running": True, "mode": "linux-systemd", "process_count": 1, "latest_start_time": "", "message": "ok"}), \ patch("app.services.runtime_status_service.append_runtime_projection_if_changed") as mock_append, \ patch("app.services.runtime_status_service.get_sync_summary", return_value=sync_summary) as mock_sync_summary, \ patch("app.services.runtime_status_service._build_multi_region_readiness", return_value=readiness), \ patch("app.services.runtime_status_service.get_runtime_build_info", return_value={"package_name": "pkg"}), \ patch("app.services.runtime_status_service.time.monotonic", side_effect=[100.0, 100.1, 100.2]): first = get_runtime_status() second = get_runtime_status() self.assertEqual(first, second) self.assertEqual(1, mock_context.call_count) self.assertEqual(1, mock_append.call_count) self.assertEqual(1, mock_sync_summary.call_count) @patch("app.services.runtime_status_service.append_runtime_projection_if_changed", return_value=321) @patch( "app.services.runtime_status_service.get_cluster_snapshot", return_value={"nodes": [], "summary": {"online_worker_nodes": 3}}, ) @patch( "app.services.runtime_status_service.get_active_detect_job_summary", return_value={ "job_id": 867, "job_code": "sync-overseas-19835", "status": "running", "progress_percent": 99.1, "items_total": 10000, "items_pending": 4, "items_claimed": 0, "items_running": 5, "items_completed": 55402, "items_blacklisted": 0, "items_failed": 0, "display_items_running": 5, "display_active_threads": 5, "display_items_claimed": 0, "display_max_threads": 1000, "node_stats": [{"node_code": "mainland-controller-01", "items_running": 5}], "raw_step_stats": [{"step_code": "detect_wayback", "items_pending": 4}], }, ) @patch( "app.services.runtime_status_service.detect_worker_runtime", return_value={"running": True, "mode": "linux-systemd", "process_count": 60, "thread_count": 1000, "max_threads": 60000, "message": "ok"}, ) @patch("app.services.runtime_status_service.get_runtime_settings", return_value={"thread_count": 1000, "worker_mode": "linux-systemd"}) @patch( "app.services.runtime_status_service._load_detect_backlog_snapshot", return_value={ "pending_total": 4, "claimed_total": 0, "running_total": 5, "completed_total": 55402, "blacklisted_total": 0, "failed_total": 0, "register_pending": 0, "downstream_pending": 4, }, ) @patch("app.services.runtime_status_service._build_runtime_detect_context", side_effect=AssertionError("heavy path should not run")) def test_refresh_runtime_projection_snapshot_uses_lightweight_context( self, mock_heavy_context, mock_load_detect_backlog_snapshot, mock_get_runtime_settings, mock_detect_worker_runtime, mock_get_active_detect_job_summary, mock_get_cluster_snapshot, mock_append_runtime_projection_if_changed, ) -> None: result = refresh_runtime_projection_snapshot() self.assertEqual(321, result["record_id"]) mock_append_runtime_projection_if_changed.assert_called_once() detect = mock_append_runtime_projection_if_changed.call_args.kwargs["detect"] self.assertEqual("sync-overseas-19835", detect["active_job"]["job_code"]) self.assertEqual(5, detect["active_thread_count"]) self.assertEqual(4, detect["backlog"]["pending_total"]) self.assertEqual(5, detect["queue_health"]["queue"]["running"]) mock_heavy_context.assert_not_called() if __name__ == "__main__": unittest.main()