import collections import json import os import threading import tempfile import unittest import sys import time from queue import Queue from unittest.mock import MagicMock from unittest.mock import patch sys.path.insert(0, "/www/wwwroot/getDomain/domainCheck") import detect_worker as detect_worker_module # noqa: E402 from app.config import config # noqa: E402 from detect_worker import DetectWorker, control_targets_current_worker, pending_control_key # noqa: E402 from app.utils.status_codes import REGISTER_STATUS_CLIENT_HOLD, REGISTER_STATUS_SERVER_HOLD # noqa: E402 class _FakeDb: def __init__(self, method_name: str): self.method_name = method_name self.calls = 0 def _run(self, batch): self.calls += 1 return -1 if self.calls == 1 else len(batch) def mark_detect_job_items_running_batch(self, batch): return self._run(batch) def release_detect_job_items_batch(self, batch): return self._run(batch) def finalize_detect_job_items_batch(self, batch): return self._run(batch) class _FakeRedis: def __init__(self): self._store = {} self._expires = {} def _cleanup(self, key): expires_at = self._expires.get(key) if expires_at is not None and expires_at <= time.time(): self._store.pop(key, None) self._expires.pop(key, None) def get(self, key): self._cleanup(key) return self._store.get(key) def set(self, key, value, nx=False, ex=None): self._cleanup(key) if nx and key in self._store: return False self._store[key] = value if ex is not None: self._expires[key] = time.time() + float(ex) else: self._expires.pop(key, None) return True def delete(self, key): existed = key in self._store self._store.pop(key, None) self._expires.pop(key, None) return 1 if existed else 0 class DetectWorkerFlushRetryTest(unittest.TestCase): def _build_worker(self, db): worker = DetectWorker.__new__(DetectWorker) worker.db = db worker._task_local = threading.local() worker._running_mark_lock = threading.Lock() worker._pending_running_marks = collections.deque() worker._last_running_mark_flush_at = 0.0 worker._job_release_lock = threading.Lock() worker._pending_job_releases = collections.deque() worker._pending_job_release_reasons = {} worker._last_job_release_flush_at = 0.0 worker._job_finalize_lock = threading.Lock() worker._pending_job_finalizations = collections.deque() worker._last_job_finalize_flush_at = 0.0 worker.proxy_pool = [] worker.proxy_pool_lock = threading.Lock() worker.proxy_refresh_lock = threading.Lock() worker.proxy_refresh_cooldown_seconds = 2 worker.proxy_next_refresh_time = 0.0 worker._last_proxy_refresh_reason_at = {} worker.proxy_failure_lock = threading.Lock() worker.proxy_failure_counts = {} worker.proxy_quarantine_until = {} worker.proxy_source_by_key = {} worker.proxy_source_failure_counts = {} worker.proxy_source_quarantine_until = {} worker.proxy_active_leases = {} worker.proxy_config = {"proxy_enable": True, "allow_direct": True} worker.proxy_last_refresh_time = None worker.proxy_last_refresh_status = "test" worker.proxy_last_refresh_source_count = 0 worker.proxy_last_refresh_total_items = 0 worker.proxy_last_validated_count = 0 worker.proxy_last_available_count = 0 worker.proxy_last_source_stats = [] worker._last_proxy_clock_skew_warning_at = 0.0 worker.proxy_max_reuse_count = 1000 worker.proxy_step_wait_timeout_seconds = 1.5 worker.proxy_direct_fallback_grace_seconds = 0.35 worker.proxy_direct_fallback_grace_overrides = {} worker.proxy_direct_retry_wait_seconds = 1.2 worker.proxy_direct_retry_wait_overrides = {} worker.register_single_machine_mode_enabled = False worker.register_single_machine_direct_streak_attempts = 0 worker.trigger_proxy_refresh = MagicMock() worker._update_runtime_state = MagicMock() worker._sync_worker_log_event = MagicMock() worker._emit_step_trace = MagicMock() worker.runtime_settings = {} worker.thread_count = 600 worker.process_count = 1 worker.refresh_process_count_runtime = MagicMock(side_effect=lambda *args, **kwargs: worker.process_count) worker.use_redis = False worker.redis_client = None worker.detect_lock = threading.Lock() worker.running = True worker.detecting = True worker.stop_requested = False worker.detect_command_thread = None worker.detect_command_started_at = 0.0 worker.detect_command_last_activity_at = 0.0 worker._last_domain_started_at = 0.0 worker._last_domain_result_at = 0.0 worker._last_job_item_release_at = 0.0 worker._active_domain_threads = 0 worker._domain_thread_counter_lock = threading.Lock() worker.current_cycle_token = "" worker.current_job_id = None worker.current_job_code = "" worker.current_job_task_mode = "" worker._detect_session_owner = 0 worker._detect_session_seq = 0 worker._pending_restart_source = "" worker._pending_restart_payload = None worker._pending_restart_reason = "" worker._worker_started_at = time.time() worker._last_worker_activity_at = worker._worker_started_at worker._idle_recycle_requested = False worker._last_explicit_start_signal_at = 0.0 worker._last_explicit_start_payload = {} worker._last_detection_start_at = 0.0 worker._last_detection_start_source = "" worker._last_detection_start_job_id = None worker._last_detection_start_job_code = "" worker._last_detection_start_task_mode = "" worker._cached_active_detect_job = None worker._last_active_detect_job_refresh_at = 0.0 worker._explicit_claim_scope_job_id = None worker._explicit_claim_scope_job_code = "" worker._explicit_claim_scope_until = 0.0 worker._explicit_claim_scope_source = "" worker._ignored_targeted_job_id = None worker._ignored_targeted_job_code = "" worker._ignored_targeted_job_until = 0.0 worker._ignored_targeted_job_targets = "" worker._get_active_domain_threads = MagicMock(return_value=0) worker._worker_log_sync_queue = Queue(maxsize=50000) worker._last_worker_log_sync_degrade_notice_at = 0.0 worker._remote_debug_event_failure_streak = 0 worker._remote_debug_event_cooldown_until = 0.0 worker._last_remote_debug_event_notice_at = 0.0 worker._last_runtime_extra = {} worker._last_runtime_phase = "idle" worker._last_runtime_detail = "" worker._runtime_heartbeat_stop = threading.Event() worker._worker_log_sync_stop = threading.Event() worker._flush_pending_domain_status_updates = MagicMock(return_value=0) worker._flush_pending_domain_completions = MagicMock(return_value=0) worker._flush_pending_review_status_updates = MagicMock(return_value=0) worker._flush_completed_futures = MagicMock(return_value=0) worker.single_machine_site_direct_fallback_enabled = False return worker def test_running_mark_flush_requeues_batch_when_db_write_fails(self): worker = self._build_worker(_FakeDb("running")) worker._pending_running_marks.extend([(101, "token-a"), (102, "token-b")]) updated = worker._flush_pending_running_marks(force=True, batch_limit=10) self.assertEqual(0, updated) self.assertEqual([(101, "token-a"), (102, "token-b")], list(worker._pending_running_marks)) updated = worker._flush_pending_running_marks(force=True, batch_limit=10) self.assertEqual(2, updated) self.assertEqual([], list(worker._pending_running_marks)) def test_job_finalization_flush_requeues_batch_when_db_write_fails(self): worker = self._build_worker(_FakeDb("finalize")) worker._pending_job_finalizations.extend( [ {"job_item_id": 201, "claim_token": "token-a", "final_status": "completed"}, {"job_item_id": 202, "claim_token": "token-b", "final_status": "failed"}, ] ) updated = worker._flush_pending_job_finalizations(force=True, batch_limit=10) self.assertEqual(0, updated) self.assertEqual(2, len(worker._pending_job_finalizations)) updated = worker._flush_pending_job_finalizations(force=True, batch_limit=10) self.assertEqual(2, updated) self.assertEqual(0, len(worker._pending_job_finalizations)) def test_job_release_flush_requeues_batch_when_db_write_fails(self): worker = self._build_worker(_FakeDb("release")) worker._enqueue_job_release(201, "token-a", "session_replaced:7") worker._enqueue_job_release(202, "token-b", "queued_before_start") updated = worker._flush_pending_job_releases(force=True, batch_limit=10) self.assertEqual(0, updated) self.assertEqual(2, len(worker._pending_job_releases)) updated = worker._flush_pending_job_releases(force=True, batch_limit=10) self.assertEqual(2, updated) self.assertEqual(0, len(worker._pending_job_releases)) def test_detection_session_abort_reason_rejects_replaced_owner(self): worker = self._build_worker(_FakeDb("noop")) worker._detect_session_owner = 7 self.assertEqual("session_replaced:7", worker._detection_session_abort_reason(6)) self.assertEqual("", worker._detection_session_abort_reason(7)) def test_release_job_item_for_session_abort_enqueues_current_item(self): db = MagicMock() worker = self._build_worker(db) worker._detect_session_owner = 7 released = worker._release_job_item_for_session_abort( 201, "token-a", session_id=6, domain_name="example.com", detail="prepare", ) self.assertTrue(released) self.assertEqual(1, len(worker._pending_job_releases)) job_item_id, claim_token = worker._pending_job_releases[0] reason = worker._pending_job_release_reasons[(job_item_id, claim_token)] self.assertEqual(201, job_item_id) self.assertEqual("token-a", claim_token) self.assertIn("session_replaced:7", reason) self.assertIn("prepare", reason) def test_enqueue_job_release_deduplicates_same_job_item_and_claim_token(self): worker = self._build_worker(_FakeDb("release")) self.assertTrue(worker._enqueue_job_release(201, "token-a", "queued_before_start")) self.assertTrue(worker._enqueue_job_release(201, "token-a", "session_replaced:9")) self.assertEqual(1, len(worker._pending_job_releases)) self.assertEqual((201, "token-a"), worker._pending_job_releases[0]) self.assertEqual( "session_replaced:9", worker._pending_job_release_reasons[(201, "token-a")], ) def test_incoming_job_id_prefers_target_job_id_for_local_queue_scope(self): worker = self._build_worker(_FakeDb("noop")) payload = { "job_id": 26025, "job_code": "sync-overseas-26025", "target_job_id": 1, "target_job_code": "sync-overseas-26025-local", "task_mode": "domain_pipeline", } self.assertEqual(1, worker._incoming_job_id(payload)) def test_schedule_explicit_claim_scope_uses_target_job_id_when_present(self): worker = self._build_worker(_FakeDb("noop")) payload = { "job_id": 26025, "job_code": "sync-overseas-26025", "target_job_id": 1, "target_job_code": "sync-overseas-26025-local", "task_mode": "domain_pipeline", } worker._schedule_explicit_claim_scope(payload, source="sync-pull") self.assertEqual(1, worker._explicit_claim_scope_job_id) self.assertEqual("sync-overseas-26025", worker._explicit_claim_scope_job_code) self.assertEqual("sync-pull", worker._explicit_claim_scope_source) def test_detect_domain_releases_job_item_when_session_replaced(self): db = MagicMock() worker = self._build_worker(db) worker._detect_session_owner = 7 worker._set_current_task_context = MagicMock() worker._clear_current_task_context = MagicMock() worker._mark_detection_phase = MagicMock() worker._release_job_item_for_session_abort = MagicMock(return_value=True) worker.detect_domain( 101, {"id": 101, "domain": "example.com"}, { "job_item_id": 301, "job_id": 1, "claim_token": "token-x", "task_mode": "domain_pipeline", "session_id": 6, }, ) worker._release_job_item_for_session_abort.assert_called_once() args, kwargs = worker._release_job_item_for_session_abort.call_args self.assertEqual(301, args[0]) self.assertEqual("token-x", args[1]) self.assertEqual(6, kwargs["session_id"]) self.assertEqual("example.com", kwargs["domain_name"]) def test_detect_domain_step_task_writes_domain_started_event(self): db = MagicMock() worker = self._build_worker(db) worker.sensitive_words = [] worker._set_current_task_context = MagicMock() worker._clear_current_task_context = MagicMock() worker._assert_detection_session_active = MagicMock() worker._change_active_domain_threads = MagicMock(return_value=1) worker._mark_detection_phase = MagicMock() worker._enqueue_running_mark = MagicMock() worker._run_detect_register = MagicMock(return_value=True) worker._get_detect_execution_order = MagicMock(return_value=["detect_register"]) worker._build_single_step_job_result = MagicMock(return_value={"state": "completed", "message": "ok"}) worker._resolve_single_step_finalization = MagicMock(return_value=("completed", "ok")) worker._enqueue_job_finalization = MagicMock() worker.detect_domain( 101, {"id": 101, "domain": "example.com"}, { "job_item_id": 301, "job_id": 1, "job_code": "sync-overseas-26118", "claim_token": "token-x", "task_mode": "domain_pipeline", "step_code": "detect_register", "step_payload": {"step_code": "detect_register"}, "session_id": 7, }, ) db.append_detect_run_event.assert_called_once() self.assertEqual("domain_started", db.append_detect_run_event.call_args.kwargs["event_type"]) payload = db.append_detect_run_event.call_args.kwargs["payload"] self.assertEqual("domain_pipeline", payload["task_mode"]) self.assertEqual("detect_register", payload["step_code"]) def test_detect_domain_step_task_finalize_emits_domain_terminal_event(self): db = MagicMock() worker = self._build_worker(db) worker.sensitive_words = [] worker._set_current_task_context = MagicMock() worker._clear_current_task_context = MagicMock() worker._assert_detection_session_active = MagicMock() worker._change_active_domain_threads = MagicMock(return_value=1) worker._mark_detection_phase = MagicMock() worker._enqueue_running_mark = MagicMock() worker._run_detect_register = MagicMock(return_value=True) worker._get_detect_execution_order = MagicMock(return_value=["detect_register"]) worker._build_single_step_job_result = MagicMock( return_value={"state": "blocked", "message": "client hold", "field_name": "register_status"} ) worker._resolve_single_step_finalization = MagicMock(return_value=("blacklisted", "client hold")) worker._enqueue_job_finalization = MagicMock() worker.detect_domain( 101, {"id": 101, "domain": "example.com"}, { "job_item_id": 301, "job_id": 1, "job_code": "sync-overseas-26118", "claim_token": "token-x", "task_mode": "domain_pipeline", "step_code": "detect_register", "step_payload": {"step_code": "detect_register"}, "session_id": 7, }, ) self.assertTrue(worker._enqueue_job_finalization.called) self.assertEqual( "domain_blacklisted", worker._enqueue_job_finalization.call_args.kwargs["event_type"], ) event_payload = worker._enqueue_job_finalization.call_args.kwargs["event_payload"] self.assertEqual("example.com", event_payload["domain"]) self.assertEqual("blacklisted", event_payload["status"]) def test_start_detection_async_ignores_duplicate_start_for_same_job_and_cycle(self): worker = self._build_worker(_FakeDb("noop")) live_thread = MagicMock() live_thread.is_alive.return_value = True worker.detect_command_thread = live_thread worker.current_job_id = 321 worker.current_job_code = "sync-overseas-321" worker.current_cycle_token = "cycle-abc" worker.detect_command_started_at = time.time() - 75 worker.detect_command_last_activity_at = time.time() - 60 worker._last_domain_result_at = time.time() - 60 started = worker.start_detection_async( source="redis-control", control_payload={ "job_id": 321, "job_code": "sync-overseas-321", "cycle_token": "cycle-abc", }, ) self.assertTrue(started) self.assertIs(worker.detect_command_thread, live_thread) self.assertFalse(worker.stop_requested) worker._update_runtime_state.assert_called() def test_start_detection_async_does_not_coalesce_recent_duplicate_without_live_thread(self): worker = self._build_worker(_FakeDb("noop")) worker.detect_command_thread = None worker.detecting = False worker.current_job_id = 321 worker.current_job_code = "sync-overseas-321" worker.current_job_task_mode = "domain_pipeline" worker._last_detection_start_at = time.time() - 10 worker._last_detection_start_source = "auto-resume" worker._last_detection_start_job_id = 321 worker._last_detection_start_job_code = "sync-overseas-321" worker._last_detection_start_task_mode = "domain_pipeline" with patch("detect_worker.threading.Thread") as mock_thread: started = worker.start_detection_async( source="redis-control", control_payload={}, ) self.assertTrue(started) mock_thread.assert_called_once() worker._update_runtime_state.assert_called() def test_start_detection_async_does_not_coalesce_recent_duplicate_for_stale_session(self): worker = self._build_worker(_FakeDb("noop")) live_thread = MagicMock() live_thread.is_alive.return_value = True worker.detect_command_thread = live_thread worker.detecting = True worker.current_job_id = 874 worker.current_job_code = "sync-overseas-19897" worker.current_job_task_mode = "domain_pipeline" worker._last_detection_start_at = time.time() - 10 worker._last_detection_start_source = "auto-resume" worker._last_detection_start_job_id = 874 worker._last_detection_start_job_code = "sync-overseas-19897" worker._last_detection_start_task_mode = "domain_pipeline" worker._is_stale_detect_session = MagicMock(return_value=(True, "stale")) worker._should_force_takeover_stale_session = MagicMock(return_value=True) with patch("detect_worker.threading.Thread") as mock_thread: started = worker.start_detection_async( source="auto-resume", control_payload={ "job_id": 874, "job_code": "sync-overseas-19897", "task_mode": "domain_pipeline", }, ) self.assertTrue(started) mock_thread.assert_called_once() worker._update_runtime_state.assert_called() def test_run_detection_session_clears_context_before_idle_runtime_update(self): worker = self._build_worker(_FakeDb("noop")) worker.start_detection = MagicMock(return_value=None) worker._consume_pending_restart_request = MagicMock(return_value=False) worker._maybe_recycle_idle_worker = MagicMock(return_value=False) control_payload = { "job_id": 876, "job_code": "sync-overseas-19961", "task_mode": "domain_pipeline", "cycle_token": "cycle-876", } worker._run_detection_session(source="redis-control", control_payload=control_payload, session_id=1) phase, detail = worker._update_runtime_state.call_args_list[-1].args[:2] kwargs = worker._update_runtime_state.call_args_list[-1].kwargs self.assertEqual("idle", phase) self.assertEqual("检测任务结束，Worker 保持待命", detail) self.assertNotIn("job_id", kwargs) self.assertNotIn("job_code", kwargs) self.assertEqual("", worker.current_cycle_token) self.assertIsNone(worker.current_job_id) self.assertEqual("", worker.current_job_code) def test_start_detection_async_schedules_pending_restart_for_explicit_pipeline_switch(self): worker = self._build_worker(_FakeDb("noop")) live_thread = MagicMock() live_thread.is_alive.return_value = True worker.detect_command_thread = live_thread worker.current_job_id = 867 worker.current_job_code = "sync-overseas-19835" worker.current_job_task_mode = "domain_pipeline" worker._explicit_claim_scope_job_id = 884 worker._explicit_claim_scope_job_code = "sync-overseas-20554" worker._explicit_claim_scope_until = time.time() + 120 started = worker.start_detection_async( source="redis-control", control_payload={ "job_id": 884, "job_code": "sync-overseas-20554", "task_mode": "domain_pipeline", }, ) self.assertTrue(started) self.assertTrue(worker.stop_requested) self.assertEqual("redis-control", worker._pending_restart_source) self.assertEqual(884, worker._pending_restart_payload.get("job_id")) worker._update_runtime_state.assert_called() def test_start_detection_async_ignores_overlap_handoff_when_current_session_has_active_threads(self): worker = self._build_worker(_FakeDb("noop")) live_thread = MagicMock() live_thread.is_alive.return_value = True worker.detect_command_thread = live_thread worker.current_job_id = 884 worker.current_job_code = "sync-overseas-20554" worker.current_job_task_mode = "domain_pipeline" worker._explicit_claim_scope_job_id = 885 worker._explicit_claim_scope_job_code = "sync-overseas-20592" worker._explicit_claim_scope_until = time.time() + 120 worker._get_active_domain_threads.return_value = 128 started = worker.start_detection_async( source="redis-control", control_payload={ "job_id": 885, "job_code": "sync-overseas-20592", "task_mode": "domain_pipeline", "source": "overlap-handoff", "tail_handoff_candidate": True, }, ) self.assertTrue(started) self.assertFalse(worker.stop_requested) self.assertEqual("", worker._pending_restart_source) self.assertIsNone(worker._explicit_claim_scope_job_id) worker._update_runtime_state.assert_called() def test_single_machine_mode_allows_direct_fallback_for_360_and_baidu(self): worker = self._build_worker(_FakeDb("noop")) worker.proxy_config["allow_direct"] = False worker.single_machine_site_direct_fallback_enabled = True self.assertTrue(worker._allow_direct_connection_for_step("360检测")) self.assertTrue(worker._allow_direct_connection_for_step("百度site检测")) self.assertFalse(worker._allow_direct_connection_for_step("时光机检测")) def test_retry_budget_allows_one_pending_direct_fallback_after_proxy_failure(self): worker = self._build_worker(_FakeDb("noop")) worker.proxy_config["allow_direct"] = False worker.single_machine_site_direct_fallback_enabled = True worker.proxy_step_retry_budget_overrides = {"360检测": {"max_attempts": 2, "max_seconds": 5}} worker._set_step_force_direct_once("360检测", True) exhausted, reason = worker._check_proxy_step_retry_budget( "360检测", "example.com", time.perf_counter() - 15, 1, {"proxy-a"}, last_reason="proxy timeout", ) self.assertFalse(exhausted) self.assertEqual("", reason) def test_retry_budget_still_exhausts_when_no_direct_fallback_is_pending(self): worker = self._build_worker(_FakeDb("noop")) worker.proxy_config["allow_direct"] = False worker.single_machine_site_direct_fallback_enabled = True worker.proxy_step_retry_budget_overrides = {"360检测": {"max_attempts": 2, "max_seconds": 5}} exhausted, reason = worker._check_proxy_step_retry_budget( "360检测", "example.com", time.perf_counter() - 15, 1, {"proxy-a"}, last_reason="proxy timeout", ) self.assertTrue(exhausted) self.assertIn("attempts=1", reason) def test_auto_resume_skips_when_recent_explicit_start_signal_exists(self): worker = self._build_worker(MagicMock()) worker.db.get_active_detect_job = MagicMock( return_value={ "id": 1, "job_code": "sync-overseas-123", "task_mode": "domain_pipeline", "items_pending": 30, "items_claimed": 4, "items_running": 2, } ) worker.detecting = False worker.detect_command_thread = None worker._last_explicit_start_signal_at = time.time() worker.start_detection_async = MagicMock(return_value=True) started = worker._resume_active_detect_job_if_needed(reason="heartbeat_autoresume") self.assertFalse(started) worker.db.get_active_detect_job.assert_not_called() worker.start_detection_async.assert_not_called() def test_auto_resume_skips_for_non_primary_worker_on_heartbeat(self): worker = self._build_worker(MagicMock()) worker.db.get_active_detect_job = MagicMock(return_value={}) worker.detecting = False worker.detect_command_thread = None with patch.object(config, "NODE_CODE", "mainland-controller-01-b"), patch.dict( os.environ, {"WORKER_PARENT_NODE_CODE": "mainland-controller-01"}, clear=False, ): started = worker._resume_active_detect_job_if_needed(reason="heartbeat_autoresume") self.assertFalse(started) worker.db.get_active_detect_job.assert_called_once() def test_auto_resume_heartbeat_reuses_recent_active_job_snapshot(self): worker = self._build_worker(MagicMock()) worker.db.get_active_detect_job = MagicMock( return_value={ "id": 1, "job_code": "sync-overseas-123", "task_mode": "domain_pipeline", "items_pending": 30, "items_claimed": 4, "items_running": 2, } ) worker.detecting = False worker.detect_command_thread = None worker.start_detection_async = MagicMock(return_value=True) with patch.dict( os.environ, { "DOMAINCHECK_AUTORESUME_ACTIVE_JOB_MIN_REFRESH_SECONDS": "60", "WORKER_PARENT_NODE_CODE": "mainland-controller-01", }, clear=False, ), patch.object(config, "NODE_CODE", "mainland-controller-01-a"): first = worker._resume_active_detect_job_if_needed(reason="heartbeat_autoresume") second = worker._resume_active_detect_job_if_needed(reason="heartbeat_autoresume") self.assertTrue(first) self.assertTrue(second) worker.db.get_active_detect_job.assert_called_once() self.assertEqual(2, worker.start_detection_async.call_count) def test_auto_resume_skips_same_job_when_threads_already_active(self): worker = self._build_worker(MagicMock()) worker.db.get_active_detect_job = MagicMock( return_value={ "id": 895, "job_code": "sync-overseas-25520", "task_mode": "domain_pipeline", "items_pending": 73298, "items_claimed": 89865, "items_running": 31396, "selection_reason": "running_job_active", } ) worker.detecting = False worker.detect_command_thread = None worker.current_job_id = 895 worker.current_job_code = "sync-overseas-25520" worker.current_job_task_mode = "domain_pipeline" worker._get_active_domain_threads.return_value = 800 worker.start_detection_async = MagicMock(return_value=True) started = worker._resume_active_detect_job_if_needed(reason="heartbeat_autoresume") self.assertFalse(started) worker.db.get_active_detect_job.assert_called_once() worker.start_detection_async.assert_not_called() def test_auto_resume_skips_same_job_during_restart_handoff_window(self): worker = self._build_worker(MagicMock()) worker.db.get_active_detect_job = MagicMock( return_value={ "id": 895, "job_code": "sync-overseas-25520", "task_mode": "domain_pipeline", "items_pending": 73298, "items_claimed": 89865, "items_running": 31396, "selection_reason": "running_job_active", } ) worker.detecting = False worker.detect_command_thread = None worker.current_job_id = 895 worker.current_job_code = "sync-overseas-25520" worker.current_job_task_mode = "domain_pipeline" worker._restart_release_handoff_job_id = 895 worker._restart_release_handoff_job_code = "sync-overseas-25520" worker._restart_release_handoff_until = time.time() + 120 worker.start_detection_async = MagicMock(return_value=True) started = worker._resume_active_detect_job_if_needed(reason="heartbeat_autoresume") self.assertFalse(started) worker.db.get_active_detect_job.assert_called_once() worker.start_detection_async.assert_not_called() def test_auto_resume_skips_when_explicit_claim_scope_matches_active_job(self): worker = self._build_worker(MagicMock()) worker.db.get_active_detect_job = MagicMock( return_value={ "id": 871, "job_code": "sync-overseas-19873", "task_mode": "domain_pipeline", "items_pending": 2134, "items_claimed": 1936, "items_running": 3402, "selection_reason": "running_job_active", } ) worker.detecting = False worker.detect_command_thread = None worker._explicit_claim_scope_job_id = 871 worker._explicit_claim_scope_job_code = "sync-overseas-19873" worker._explicit_claim_scope_until = time.time() + 120 worker._explicit_claim_scope_source = "redis-control" worker.start_detection_async = MagicMock(return_value=True) started = worker._resume_active_detect_job_if_needed(reason="heartbeat_autoresume") self.assertFalse(started) worker.db.get_active_detect_job.assert_called_once() worker.start_detection_async.assert_not_called() def test_auto_resume_allows_non_primary_worker_for_tail_handoff_pending_job(self): worker = self._build_worker(MagicMock()) worker.db.get_active_detect_job = MagicMock( return_value={ "id": 885, "job_code": "sync-overseas-20592", "task_mode": "domain_pipeline", "items_pending": 64000, "items_claimed": 0, "items_running": 0, "tail_handoff_candidate": True, "selection_reason": "tail_handoff_pending", } ) worker.detecting = False worker.detect_command_thread = None worker.start_detection_async = MagicMock(return_value=True) with patch.object(config, "NODE_CODE", "mainland-controller-01-b"), patch.dict( os.environ, {"WORKER_PARENT_NODE_CODE": "mainland-controller-01"}, clear=False, ): started = worker._resume_active_detect_job_if_needed(reason="heartbeat_autoresume") self.assertTrue(started) worker.db.get_active_detect_job.assert_called_once() worker.start_detection_async.assert_called_once() def test_auto_resume_skips_stalled_running_job(self): worker = self._build_worker(MagicMock()) worker.db.get_active_detect_job = MagicMock( return_value={ "id": 876, "job_code": "sync-overseas-19961", "task_mode": "domain_pipeline", "items_pending": 4, "items_claimed": 235, "items_running": 266, "selection_reason": "running_job_stalled", "running_job_stalled": True, } ) worker.db.recycle_stalled_detect_job_items = MagicMock(return_value=7) worker.detecting = False worker.detect_command_thread = None worker.start_detection_async = MagicMock(return_value=True) worker._acquire_shared_job_maintenance_lock = MagicMock(return_value="lock-token") worker._release_shared_job_maintenance_lock = MagicMock() started = worker._resume_active_detect_job_if_needed(reason="heartbeat_autoresume") self.assertFalse(started) worker.db.get_active_detect_job.assert_called_once() worker.db.recycle_stalled_detect_job_items.assert_called_once_with(876, stall_seconds=900) worker._release_shared_job_maintenance_lock.assert_called_once() worker.start_detection_async.assert_not_called() def test_pull_sync_tasks_skips_when_shared_lock_not_acquired(self): worker = self._build_worker(MagicMock()) worker._acquire_shared_job_maintenance_lock = MagicMock(return_value="") worker._release_shared_job_maintenance_lock = MagicMock() with patch.object(config, "NODE_REGION", "mainland"), patch.object(config, "NODE_ROLE", "control"), patch( "detect_worker.urllib.request.urlopen" ) as mock_urlopen: domains = worker._pull_sync_tasks_until_available(thread_limit=1000) self.assertEqual([], domains) mock_urlopen.assert_not_called() worker._release_shared_job_maintenance_lock.assert_not_called() def test_process_pipeline_skips_when_shared_lock_not_acquired(self): worker = self._build_worker(MagicMock()) worker._acquire_shared_job_maintenance_lock = MagicMock(return_value="") worker._release_shared_job_maintenance_lock = MagicMock() with patch.object(config, "NODE_REGION", "mainland"), patch.object(config, "NODE_ROLE", "control"), patch( "detect_worker.urllib.request.urlopen" ) as mock_urlopen: domains = worker._process_pipeline_tasks_until_available(thread_limit=1000) self.assertEqual([], domains) mock_urlopen.assert_not_called() worker._release_shared_job_maintenance_lock.assert_not_called() def test_runtime_heartbeat_does_not_fake_current_load_when_detecting_but_idle(self): class _SpyRedis: def __init__(self): self.calls = [] def set(self, *args, **kwargs): self.calls.append((args, kwargs)) return True db = MagicMock() worker = self._build_worker(db) worker.use_redis = True worker.redis_client = _SpyRedis() worker.detecting = True worker.running = True worker.stop_requested = False worker.proxy_last_refresh_status = "ok" worker.proxy_last_refresh_time = None worker.proxy_last_source_stats = [] worker._active_domain_threads = 0 worker._last_runtime_state_push_phase = "" worker._last_runtime_state_push_at = 0.0 DetectWorker._update_runtime_state(worker, "running", "等待线程派发") register_calls = [call for call in db.mock_calls if call[0] == "register_cluster_node"] self.assertEqual(1, len(register_calls)) _, args, kwargs = register_calls[0] self.assertEqual(config.NODE_CODE, args[0]) self.assertEqual("busy", kwargs["status"]) self.assertEqual(0, kwargs["current_load"]) self.assertTrue(kwargs["metadata"]["detecting"]) self.assertTrue(kwargs["metadata"]["waiting_for_dispatch"]) self.assertEqual(0, kwargs["metadata"]["active_threads"]) def test_release_node_job_items_is_throttled_between_detection_restarts(self): db = MagicMock() db.release_detect_job_items_for_node.return_value = 7 worker = self._build_worker(db) first = worker._maybe_release_node_job_items(reason="test", min_interval_seconds=300) second = worker._maybe_release_node_job_items(reason="test", min_interval_seconds=300) self.assertEqual(7, first) self.assertEqual(0, second) db.release_detect_job_items_for_node.assert_called_once_with(config.NODE_CODE) def test_control_node_can_release_node_job_items(self): db = MagicMock() db.release_detect_job_items_for_node.return_value = 7 worker = self._build_worker(db) with patch.object(config, "NODE_ROLE", "control"): released = worker._maybe_release_node_job_items(reason="control_start") self.assertEqual(7, released) db.release_detect_job_items_for_node.assert_called_once_with(config.NODE_CODE) def test_release_node_job_items_records_restart_release_handoff_for_current_job(self): db = MagicMock() db.release_detect_job_items_for_node.return_value = 7 worker = self._build_worker(db) worker.current_job_id = 880 worker.current_job_code = "sync-overseas-20249" released = worker._maybe_release_node_job_items(reason="start_detection", min_interval_seconds=0) self.assertEqual(7, released) self.assertEqual(880, worker._restart_release_handoff_job_id) self.assertEqual("sync-overseas-20249", worker._restart_release_handoff_job_code) self.assertGreater(worker._restart_release_handoff_until, time.time()) def test_claim_detect_job_items_prioritizes_restart_released_handoff_before_global_queue(self): db = MagicMock() db.claim_restart_released_detect_job_items.return_value = [{"job_item_id": 1}] db.claim_detect_job_items.return_value = [{"job_item_id": 2}] worker = self._build_worker(db) worker.current_job_id = 880 worker.current_job_code = "sync-overseas-20249" worker.current_job_task_mode = "domain_pipeline" worker._restart_release_handoff_job_id = 880 worker._restart_release_handoff_job_code = "sync-overseas-20249" worker._restart_release_handoff_until = time.time() + 120 worker._restart_release_handoff_reason = "start_detection" with patch.object(config, "NODE_CODE", "mainland-controller-01-a"), patch.dict( os.environ, {"WORKER_PARENT_NODE_CODE": "mainland-controller-01"}, clear=False, ): rows = worker._claim_detect_job_items(limit=320, lease_seconds=900, job_id=None) self.assertEqual([{"job_item_id": 1}, {"job_item_id": 2}], rows) db.claim_restart_released_detect_job_items.assert_called_once_with( "mainland-controller-01-a", 880, limit=128, lease_seconds=900, ) db.claim_detect_job_items.assert_called_once_with( "mainland-controller-01-a", limit=319, lease_seconds=900, job_id=None, prefer_recent_jobs=False, preferred_recent_job_limit=8, preferred_recent_job_window_hours=24, ) def test_claim_detect_job_items_skips_restart_handoff_query_for_non_primary_worker(self): db = MagicMock() worker = self._build_worker(db) worker.current_job_id = 880 worker.current_job_code = "sync-overseas-20249" worker.current_job_task_mode = "domain_pipeline" worker._restart_release_handoff_job_id = 880 worker._restart_release_handoff_job_code = "sync-overseas-20249" worker._restart_release_handoff_until = time.time() + 120 worker._restart_release_handoff_reason = "start_detection" with patch.object(config, "NODE_CODE", "mainland-controller-01-b"), patch.dict( os.environ, {"WORKER_PARENT_NODE_CODE": "mainland-controller-01"}, clear=False, ): rows = worker._claim_detect_job_items(limit=320, lease_seconds=900, job_id=None) self.assertEqual([], rows) db.claim_restart_released_detect_job_items.assert_not_called() db.claim_detect_job_items.assert_not_called() def test_mark_explicit_start_signal_records_pipeline_claim_scope(self): db = MagicMock() worker = self._build_worker(db) worker._mark_explicit_start_signal( { "job_id": 884, "job_code": "sync-overseas-20554", "task_mode": "domain_pipeline", }, source="redis-control", ) self.assertEqual(884, worker._explicit_claim_scope_job_id) self.assertEqual("sync-overseas-20554", worker._explicit_claim_scope_job_code) self.assertEqual("redis-control", worker._explicit_claim_scope_source) self.assertGreater(worker._explicit_claim_scope_until, time.time()) def test_current_scoped_claim_job_id_prefers_explicit_pipeline_scope(self): db = MagicMock() worker = self._build_worker(db) worker.current_job_id = 867 worker.current_job_code = "sync-overseas-19835" worker.current_job_task_mode = "domain_pipeline" worker._explicit_claim_scope_job_id = 884 worker._explicit_claim_scope_job_code = "sync-overseas-20554" worker._explicit_claim_scope_until = time.time() + 120 self.assertEqual(884, worker._current_scoped_claim_job_id()) def test_explicit_claim_scope_window_expires(self): db = MagicMock() worker = self._build_worker(db) worker._explicit_claim_scope_job_id = 884 worker._explicit_claim_scope_job_code = "sync-overseas-20554" worker._explicit_claim_scope_until = time.time() - 1 self.assertIsNone(worker._explicit_claim_scope_job_id_active()) self.assertIsNone(worker._explicit_claim_scope_job_id) def test_claim_detect_job_items_honors_explicit_pipeline_scope(self): db = MagicMock() db.claim_detect_job_items.return_value = [{"job_item_id": 9}] worker = self._build_worker(db) worker._explicit_claim_scope_job_id = 884 worker._explicit_claim_scope_job_code = "sync-overseas-20554" worker._explicit_claim_scope_until = time.time() + 120 with patch.object(config, "NODE_CODE", "mainland-controller-01-da"): rows = worker._claim_detect_job_items(limit=250, lease_seconds=900, job_id=None) self.assertEqual([{"job_item_id": 9}], rows) db.claim_detect_job_items.assert_called_once_with( "mainland-controller-01-da", limit=250, lease_seconds=900, job_id=884, prefer_recent_jobs=False, preferred_recent_job_limit=8, preferred_recent_job_window_hours=24, ) def test_shared_job_maintenance_lock_uses_single_worker_with_same_scope(self): redis_client = _FakeRedis() db = MagicMock() worker_a = self._build_worker(db) worker_a.use_redis = True worker_a.redis_client = redis_client worker_b = self._build_worker(db) worker_b.use_redis = True worker_b.redis_client = redis_client with patch.object(config, "NODE_CODE", "mainland-controller-01-a"), patch.dict( os.environ, {"WORKER_PARENT_NODE_CODE": "mainland-controller-01"}, clear=False, ): token_a = worker_a._acquire_shared_job_maintenance_lock("recycle-expired-job-items", ttl_seconds=10) with patch.object(config, "NODE_CODE", "mainland-controller-01-b"), patch.dict( os.environ, {"WORKER_PARENT_NODE_CODE": "mainland-controller-01"}, clear=False, ): token_b = worker_b._acquire_shared_job_maintenance_lock("recycle-expired-job-items", ttl_seconds=10) self.assertTrue(token_a) self.assertEqual("", token_b) def test_control_node_skips_local_auto_resume(self): worker = self._build_worker(MagicMock()) worker.db.get_active_detect_job = MagicMock(return_value={"id": 1, "job_code": "sync-overseas-123"}) worker.detecting = False worker.detect_command_thread = None with patch.object(config, "NODE_ROLE", "control"): started = worker._resume_active_detect_job_if_needed(reason="service_runtime_bootstrap") self.assertFalse(started) worker.db.get_active_detect_job.assert_not_called() def test_control_node_can_auto_resume_when_override_enabled(self): worker = self._build_worker(MagicMock()) worker.runtime_settings = {"control_node_autoresume_enabled": True} worker.db.get_active_detect_job = MagicMock( return_value={ "id": 1, "job_code": "sync-overseas-123", "task_mode": "domain_pipeline", "items_pending": 30, "items_claimed": 4, "items_running": 2, } ) worker.detecting = False worker.detect_command_thread = None worker.start_detection_async = MagicMock(return_value=True) with patch.object(config, "NODE_ROLE", "control"): started = worker._resume_active_detect_job_if_needed(reason="service_runtime_bootstrap") self.assertTrue(started) worker.db.get_active_detect_job.assert_called_once() worker.start_detection_async.assert_called_once() def test_bootstrap_autoresume_lock_allows_only_one_worker_for_same_job(self): redis_client = _FakeRedis() db = MagicMock() active_job = { "id": 895, "job_code": "sync-overseas-25520", "task_mode": "domain_pipeline", "items_pending": 73298, "items_claimed": 89865, "items_running": 31396, "selection_reason": "running_job_active", } worker_a = self._build_worker(db) worker_a.use_redis = True worker_a.redis_client = redis_client worker_a.runtime_settings = {"control_node_autoresume_enabled": True} worker_a.db.get_active_detect_job = MagicMock(return_value=active_job) worker_a.detecting = False worker_a.start_detection_async = MagicMock(return_value=True) worker_b = self._build_worker(db) worker_b.use_redis = True worker_b.redis_client = redis_client worker_b.runtime_settings = {"control_node_autoresume_enabled": True} worker_b.db.get_active_detect_job = MagicMock(return_value=active_job) worker_b.detecting = False worker_b.start_detection_async = MagicMock(return_value=True) with patch.object(config, "NODE_ROLE", "control"), patch.object(config, "NODE_CODE", "mainland-controller-01-a"), patch.dict( os.environ, {"WORKER_PARENT_NODE_CODE": "mainland-controller-01"}, clear=False, ): started_a = worker_a._resume_active_detect_job_if_needed(reason="service_runtime_bootstrap") with patch.object(config, "NODE_ROLE", "control"), patch.object(config, "NODE_CODE", "mainland-controller-01-b"), patch.dict( os.environ, {"WORKER_PARENT_NODE_CODE": "mainland-controller-01"}, clear=False, ): started_b = worker_b._resume_active_detect_job_if_needed(reason="service_runtime_bootstrap") self.assertTrue(started_a) self.assertFalse(started_b) worker_a.start_detection_async.assert_called_once() worker_b.start_detection_async.assert_not_called() def test_same_job_recent_start_skip_detail_never_goes_negative(self): worker = self._build_worker(MagicMock()) worker.current_job_id = 895 worker.current_job_code = "sync-overseas-25520" worker._last_detection_start_at = time.time() + 900 should_skip, detail = worker._should_skip_same_job_autoresume( active_job={"id": 895, "job_code": "sync-overseas-25520"}, now_ts=time.time(), ) self.assertTrue(should_skip) self.assertEqual("same_job_recent_start:0s", detail) def test_claim_detect_job_items_prefers_recent_jobs_when_runtime_enabled(self): db = MagicMock() db.claim_detect_job_items.return_value = [] worker = self._build_worker(db) worker.runtime_settings = { "claim_recent_jobs_first": True, "claim_recent_jobs_limit": 6, "claim_recent_jobs_window_hours": 72, } worker._claim_detect_job_items(limit=320, lease_seconds=900, job_id=None) db.claim_detect_job_items.assert_called_once_with( config.NODE_CODE, limit=320, lease_seconds=900, job_id=None, prefer_recent_jobs=True, preferred_recent_job_limit=6, preferred_recent_job_window_hours=72, ) def test_claim_detect_job_items_does_not_enable_recent_priority_for_scoped_job(self): db = MagicMock() db.claim_detect_job_items.return_value = [] worker = self._build_worker(db) worker.runtime_settings = { "claim_recent_jobs_first": True, "claim_recent_jobs_limit": 6, "claim_recent_jobs_window_hours": 72, } worker._claim_detect_job_items(limit=320, lease_seconds=900, job_id=99) db.claim_detect_job_items.assert_called_once_with( config.NODE_CODE, limit=320, lease_seconds=900, job_id=99, prefer_recent_jobs=False, preferred_recent_job_limit=6, preferred_recent_job_window_hours=72, ) def test_resolve_sync_pull_limit_scales_with_process_pool_capacity(self): worker = self._build_worker(MagicMock()) worker.process_count = 60 worker.refresh_process_count_runtime = MagicMock(return_value=60) pull_limit = worker._resolve_sync_pull_limit(1000) self.assertEqual(120000, pull_limit) def test_proxy_auth_failure_quarantines_whole_source_group(self): worker = self._build_worker(_FakeDb("noop")) bad_proxy = {"http": "http://bad-proxy", "https": "http://bad-proxy"} good_proxy = {"http": "http://good-proxy", "https": "http://good-proxy"} worker.proxy_pool = [ {"proxy": bad_proxy, "usage_count": 0, "source_tag": "D"}, {"proxy": good_proxy, "usage_count": 0, "source_tag": "E"}, ] worker.proxy_source_by_key[worker._proxy_key(bad_proxy)] = "D" worker.proxy_source_by_key[worker._proxy_key(good_proxy)] = "E" source_tag, failure_count, cooldown_seconds, removed = worker._mark_proxy_source_failure( bad_proxy, "Tunnel connection failed: 407 Proxy Authentication Required", ) self.assertEqual("D", source_tag) self.assertEqual(1, failure_count) self.assertGreaterEqual(cooldown_seconds, 90) self.assertEqual(1, removed) self.assertTrue(worker._is_proxy_source_quarantined("D")) self.assertEqual( [{"proxy": good_proxy, "usage_count": 0, "source_tag": "E"}], worker.proxy_pool, ) def test_get_proxies_skips_quarantined_entries_without_dropping_pool_items(self): worker = self._build_worker(_FakeDb("noop")) bad_proxy = {"http": "http://bad-proxy", "https": "http://bad-proxy"} good_proxy = {"http": "http://good-proxy", "https": "http://good-proxy"} worker.proxy_pool = [ {"proxy": bad_proxy, "usage_count": 0, "source_tag": "B"}, {"proxy": good_proxy, "usage_count": 0, "source_tag": "C"}, ] worker.proxy_quarantine_until[worker._proxy_key(bad_proxy)] = time.time() + 60 worker._schedule_proxy_refresh_if_needed = MagicMock() selected = worker.get_proxies() self.assertEqual(good_proxy, selected) self.assertEqual(2, len(worker.proxy_pool)) self.assertEqual( [entry["proxy"] for entry in worker.proxy_pool], [bad_proxy, good_proxy], ) self.assertEqual(1, worker.proxy_pool[1]["usage_count"]) def test_get_proxies_scans_multiple_batches_until_it_finds_usable_proxy(self): worker = self._build_worker(_FakeDb("noop")) worker._proxy_pool_scan_batch_size = MagicMock(return_value=2) proxies = [ {"http": f"http://proxy-{idx}", "https": f"http://proxy-{idx}"} for idx in range(5) ] worker.proxy_pool = [ {"proxy": proxy, "usage_count": 0, "source_tag": f"S{idx}"} for idx, proxy in enumerate(proxies) ] excluded_keys = {worker._proxy_key(proxy) for proxy in proxies[:4]} worker._schedule_proxy_refresh_if_needed = MagicMock() selected = worker.get_proxies(excluded_proxy_keys=excluded_keys) self.assertEqual(proxies[4], selected) self.assertEqual(5, len(worker.proxy_pool)) self.assertEqual(1, worker.proxy_pool[-1]["usage_count"]) def test_prepare_proxy_fetch_plan_respects_pool_shortage_instead_of_overfetching(self): worker = self._build_worker(_FakeDb("noop")) plan, meta = worker._prepare_proxy_fetch_plan( [ "http://provider.local/get?group=B", "http://provider.local/get?group=C", "http://provider.local/get?group=D", "http://provider.local/get?group=E", "http://provider.local/get?group=F", "http://provider.local/get?group=G", ], current_pool_size=1000, ) self.assertEqual(1, meta["rounds"]) self.assertGreaterEqual(meta["target_total"], 840) self.assertLessEqual(meta["target_total"], 1200) self.assertEqual(6, len(plan)) def test_prepare_proxy_fetch_plan_caps_active_rounds(self): worker = self._build_worker(_FakeDb("noop")) plan, meta = worker._prepare_proxy_fetch_plan( [ "http://provider.local/get?group=B", "http://provider.local/get?group=C", "http://provider.local/get?group=D", "http://provider.local/get?group=E", "http://provider.local/get?group=F", "http://provider.local/get?group=G", ], current_pool_size=0, ) self.assertLessEqual(meta["rounds"], 2) self.assertEqual(meta["rounds"] * 6, len(plan)) def test_prepare_proxy_fetch_plan_scales_down_target_when_process_count_is_high(self): worker = self._build_worker(_FakeDb("noop")) worker.thread_count = 1000 worker.process_count = 80 worker.refresh_process_count_runtime = MagicMock(return_value=80) plan, meta = worker._prepare_proxy_fetch_plan( [ "http://provider.local/get?group=B", "http://provider.local/get?group=C", "http://provider.local/get?group=D", "http://provider.local/get?group=E", "http://provider.local/get?group=F", "http://provider.local/get?group=G", ], current_pool_size=150, ) self.assertEqual(80, meta["process_count"]) self.assertLessEqual(meta["target_total"], 360) self.assertLessEqual(meta["batch_size"], 60) self.assertEqual(6, len(plan)) def test_refresh_proxy_pool_reuses_shared_snapshot_when_refresh_lock_is_busy(self): worker = self._build_worker(_FakeDb("noop")) worker.use_redis = True worker.redis_client = _FakeRedis() worker.process_count = 80 worker.refresh_process_count_runtime = MagicMock(return_value=80) worker.proxy_config = { "proxy_enable": True, "allow_direct": True, "proxy_urls": ["http://provider.local/get?group=B"], } local_proxy = {"http": "http://local-proxy", "https": "http://local-proxy"} shared_proxy = {"http": "http://shared-proxy", "https": "http://shared-proxy"} worker.proxy_pool = [ {"proxy": local_proxy, "usage_count": 0, "source_tag": "L", "expire_at_ts": time.time() + 120} ] with patch.object(config, "NODE_CODE", "mainland-controller-01-u"): with patch.dict(os.environ, {"WORKER_PARENT_NODE_CODE": "mainland-controller-01"}, clear=False): worker.redis_client.set( worker._shared_proxy_pool_key(), json.dumps( { "node_code": "mainland-controller-01", "owner": "mainland-controller-01-k", "refreshed_at_ts": time.time(), "source_count": 6, "raw_items": 180, "available_count": 1, "proxy_pool": [ { "proxy": shared_proxy, "usage_count": 0, "source_tag": "S", "expire_at_ts": time.time() + 120, } ], }, ensure_ascii=False, ), ex=30, ) worker.redis_client.set(worker._shared_proxy_refresh_lock_key(), "other-owner", ex=10) worker.refresh_proxy_pool() self.assertEqual(1, len(worker.proxy_pool)) self.assertEqual(shared_proxy, worker.proxy_pool[0]["proxy"]) self.assertIn("共享代理快照", worker.proxy_last_refresh_status) def test_refresh_proxy_pool_publishes_shared_snapshot_after_success(self): worker = self._build_worker(_FakeDb("noop")) worker.use_redis = True worker.redis_client = _FakeRedis() worker.process_count = 80 worker.refresh_process_count_runtime = MagicMock(return_value=80) worker.proxy_config = { "proxy_enable": True, "allow_direct": True, "proxy_urls": ["http://provider.local/get?group=B"], } class _FakeResponse: status_code = 200 def json(self): return { "list": [ { "ip": "1.1.1.1", "port": "8080", "expire_at": time.time() + 120, } ] } with patch.object(config, "NODE_CODE", "mainland-controller-01-u"): with patch.dict(os.environ, {"WORKER_PARENT_NODE_CODE": "mainland-controller-01"}, clear=False): with patch("requests.get", return_value=_FakeResponse()): worker.refresh_proxy_pool() raw_payload = worker.redis_client.get(worker._shared_proxy_pool_key()) self.assertTrue(raw_payload) payload = json.loads(raw_payload) self.assertEqual("mainland-controller-01-u", payload["owner"]) self.assertEqual(1, payload["available_count"]) def test_refresh_proxy_pool_prefers_shared_snapshot_when_local_pool_lags(self): worker = self._build_worker(_FakeDb("noop")) worker.use_redis = True worker.redis_client = _FakeRedis() worker.process_count = 80 worker.refresh_process_count_runtime = MagicMock(return_value=80) worker.thread_count = 1000 worker.proxy_config = { "proxy_enable": True, "allow_direct": True, "proxy_urls": ["http://provider.local/get?group=B"], } worker.proxy_pool = [ {"proxy": {"http": f"http://local-{idx}", "https": f"http://local-{idx}"}, "usage_count": 0, "source_tag": "L", "expire_at_ts": time.time() + 120} for idx in range(20) ] shared_entries = [ {"proxy": {"http": f"http://shared-{idx}", "https": f"http://shared-{idx}"}, "usage_count": 0, "source_tag": "S", "expire_at_ts": time.time() + 120} for idx in range(80) ] with patch.object(config, "NODE_CODE", "mainland-controller-01-u"): with patch.dict(os.environ, {"WORKER_PARENT_NODE_CODE": "mainland-controller-01"}, clear=False): worker.redis_client.set( worker._shared_proxy_pool_key(), json.dumps( { "node_code": "mainland-controller-01", "owner": "mainland-controller-01-k", "refreshed_at_ts": time.time(), "source_count": 6, "raw_items": 120, "available_count": len(shared_entries), "proxy_pool": shared_entries, }, ensure_ascii=False, ), ex=30, ) worker.refresh_proxy_pool() self.assertEqual(len(shared_entries), len(worker.proxy_pool)) self.assertIn("共享代理快照", worker.proxy_last_refresh_status) def test_schedule_proxy_refresh_skips_empty_pool_during_cooldown(self): worker = self._build_worker(_FakeDb("noop")) worker.proxy_next_refresh_time = time.time() + 10 with patch("detect_worker.threading.Thread") as mock_thread: worker._schedule_proxy_refresh_if_needed(0) mock_thread.assert_not_called() def test_schedule_proxy_refresh_skips_low_water_pool_during_cooldown(self): worker = self._build_worker(_FakeDb("noop")) worker.process_count = 80 worker.refresh_process_count_runtime = MagicMock(return_value=80) worker.proxy_next_refresh_time = time.time() + 10 with patch("detect_worker.threading.Thread") as mock_thread: worker._schedule_proxy_refresh_if_needed(150) mock_thread.assert_not_called() def test_proxy_refresh_threshold_scales_down_for_multi_process_node(self): worker = self._build_worker(_FakeDb("noop")) worker.thread_count = 1000 worker._get_active_domain_threads = MagicMock(return_value=900) self.assertEqual(2700, worker._proxy_refresh_threshold()) worker.process_count = 80 worker.refresh_process_count_runtime = MagicMock(return_value=80) self.assertLess(worker._proxy_refresh_threshold(), 400) def test_trigger_proxy_refresh_respects_cooldown_without_reset(self): worker = self._build_worker(_FakeDb("noop")) worker.proxy_config["proxy_urls"] = ["http://provider.local/get?group=B&count=1"] worker.proxy_next_refresh_time = time.time() + 10 worker.trigger_proxy_refresh = DetectWorker.trigger_proxy_refresh.__get__(worker, DetectWorker) with patch("detect_worker.threading.Thread") as mock_thread: scheduled = worker.trigger_proxy_refresh(reason="cooldown-check") self.assertFalse(scheduled) mock_thread.assert_not_called() def test_trigger_proxy_refresh_debounces_config_update_burst(self): worker = self._build_worker(_FakeDb("noop")) worker.proxy_config["proxy_urls"] = ["http://provider.local/get?group=B&count=1"] worker.trigger_proxy_refresh = DetectWorker.trigger_proxy_refresh.__get__(worker, DetectWorker) with patch("detect_worker.threading.Thread") as mock_thread: first = worker.trigger_proxy_refresh(reason="config_update:thread_count") second = worker.trigger_proxy_refresh(reason="config_update:runtime_settings") self.assertTrue(first) self.assertFalse(second) mock_thread.assert_called_once() def test_config_update_proxy_refresh_ignores_runtime_settings_only_changes(self): worker = self._build_worker(_FakeDb("noop")) worker.proxy_config = { "proxy_enable": True, "allow_direct": True, "proxy_urls": ["http://provider.local/get?group=B"], } worker.runtime_settings = {"worker_log_sync_mode": "key"} worker.proxy_pool = [ {"proxy": {"http": "http://cached", "https": "http://cached"}, "usage_count": 0, "source_tag": "L", "expire_at_ts": time.time() + 120} for _ in range(160) ] should_refresh, reset_cooldown = worker._should_trigger_proxy_refresh_for_config_update( config_type="runtime_settings", previous_proxy_config=dict(worker.proxy_config), previous_thread_count=worker.thread_count, current_thread_count=worker.thread_count, ) self.assertFalse(should_refresh) self.assertFalse(reset_cooldown) def test_proxy_item_expiry_respects_recent_grace_window(self): worker = self._build_worker(_FakeDb("noop")) expired_recently = {"expire_at": time.time() - 30} self.assertTrue(worker._is_proxy_item_expired(expired_recently)) self.assertFalse(worker._is_proxy_item_expired(expired_recently, grace_seconds=90)) self.assertTrue(worker._is_proxy_item_expired(expired_recently, grace_seconds=10)) def test_proxy_expire_clock_skew_hint_detects_large_positive_offset(self): worker = self._build_worker(_FakeDb("noop")) base_now = 1_700_000_000.0 proxy_items = [ {"ip": "1.1.1.1", "port": "8001", "expire_at": base_now - 46000}, {"ip": "1.1.1.2", "port": "8002", "expire_at": base_now - 45990}, {"ip": "1.1.1.3", "port": "8003", "expire_at": base_now - 45980}, {"ip": "1.1.1.4", "port": "8004", "expire_at": base_now - 45970}, {"ip": "1.1.1.5", "port": "8005", "expire_at": base_now - 45960}, {"ip": "1.1.1.6", "port": "8006", "expire_at": base_now - 45950}, {"ip": "1.1.1.7", "port": "8007", "expire_at": base_now - 45940}, {"ip": "1.1.1.8", "port": "8008", "expire_at": base_now - 45930}, {"ip": "1.1.1.9", "port": "8009", "expire_at": base_now - 45920}, {"ip": "1.1.1.10", "port": "8010", "expire_at": base_now - 45910}, {"ip": "1.1.1.11", "port": "8011", "expire_at": base_now - 45900}, {"ip": "1.1.1.12", "port": "8012", "expire_at": base_now - 45890}, ] hint = worker._summarize_proxy_expire_clock_skew(proxy_items, now_ts=base_now) self.assertIsNotNone(hint) self.assertEqual(12, hint["timed_proxy_count"]) self.assertEqual(12, hint["expired_timed_count"]) self.assertGreater(hint["clock_ahead_seconds"], 45_000) def test_proxy_expire_clock_skew_hint_ignores_normally_fresh_entries(self): worker = self._build_worker(_FakeDb("noop")) base_now = 1_700_000_000.0 proxy_items = [ {"ip": "2.2.2.1", "port": "9001", "expire_at": base_now + 120}, {"ip": "2.2.2.2", "port": "9002", "expire_at": base_now + 180}, {"ip": "2.2.2.3", "port": "9003", "expire_at": base_now + 240}, {"ip": "2.2.2.4", "port": "9004", "expire_at": base_now + 300}, {"ip": "2.2.2.5", "port": "9005", "expire_at": base_now + 360}, {"ip": "2.2.2.6", "port": "9006", "expire_at": base_now + 420}, {"ip": "2.2.2.7", "port": "9007", "expire_at": base_now + 480}, {"ip": "2.2.2.8", "port": "9008", "expire_at": base_now + 540}, {"ip": "2.2.2.9", "port": "9009", "expire_at": base_now + 600}, {"ip": "2.2.2.10", "port": "9010", "expire_at": base_now + 660}, {"ip": "2.2.2.11", "port": "9011", "expire_at": base_now + 720}, {"ip": "2.2.2.12", "port": "9012", "expire_at": base_now + 780}, ] hint = worker._summarize_proxy_expire_clock_skew(proxy_items, now_ts=base_now) self.assertIsNone(hint) def test_recently_expired_backfill_only_kicks_in_when_pool_is_too_small(self): worker = self._build_worker(_FakeDb("noop")) worker.thread_count = 2000 worker._get_active_domain_threads = MagicMock(return_value=1800) stale_entries = [ { "proxy": {"http": f"http://stale-{idx}", "https": f"http://stale-{idx}"}, "usage_count": 0, "source_tag": f"S{idx % 3}", } for idx in range(200) ] merged_entries, added_count, floor = worker._backfill_recently_expired_proxy_entries( [ { "proxy": {"http": "http://fresh-1", "https": "http://fresh-1"}, "usage_count": 0, "source_tag": "F", } ], stale_entries, ) self.assertGreater(floor, 1) self.assertEqual(floor, len(merged_entries)) self.assertEqual(floor - 1, added_count) healthy_pool = [ { "proxy": {"http": f"http://fresh-{idx}", "https": f"http://fresh-{idx}"}, "usage_count": 0, "source_tag": "F", } for idx in range(floor + 5) ] merged_entries, added_count, reused_floor = worker._backfill_recently_expired_proxy_entries( healthy_pool, stale_entries, ) self.assertEqual(0, added_count) self.assertEqual(reused_floor, floor) self.assertEqual(len(healthy_pool), len(merged_entries)) def test_refresh_proxy_pool_can_reuse_recently_expired_entries_without_unbound_local_error(self): worker = self._build_worker(_FakeDb("noop")) worker.thread_count = 1200 worker.proxy_config = { "proxy_enable": True, "allow_direct": True, "proxy_urls": ["http://provider.local/get?group=B&count=1"], } class _Response: status_code = 200 @staticmethod def json(): return [ { "ip": "1.1.1.1", "port": "8080", "expire_at": time.time() - 30, } ] with patch("requests.get", return_value=_Response()): worker.refresh_proxy_pool() def test_pending_control_key_scopes_to_current_node_code(self): with patch.object(config, "NODE_CODE", "mainland-controller-01-a"): self.assertEqual( "domain_tool:worker_pending_command:mainland-controller-01-a", pending_control_key(), ) def test_control_targets_current_worker_rejects_other_instance(self): with patch.object(config, "NODE_CODE", "mainland-controller-01-a"): self.assertTrue(control_targets_current_worker({"action": "start_detection"})) self.assertTrue(control_targets_current_worker({"target_node_code": "mainland-controller-01-a"})) self.assertFalse(control_targets_current_worker({"target_node_code": "mainland-controller-01-b"})) def test_consume_pending_control_command_prefers_instance_key(self): worker = DetectWorker.__new__(DetectWorker) worker.use_redis = True worker._handle_control_message = MagicMock() class _FakeRedis: def __init__(self): self.values = { "domain_tool:worker_pending_command:mainland-controller-01-a": '{"action":"start_detection","target_node_code":"mainland-controller-01-a"}', "domain_tool:worker_pending_command": '{"action":"start_detection"}', } def getdel(self, key): return self.values.pop(key, None) def get(self, key): return self.values.get(key) def delete(self, key): self.values.pop(key, None) worker.redis_client = _FakeRedis() with patch.object(config, "NODE_CODE", "mainland-controller-01-a"): worker._consume_pending_control_command() worker._handle_control_message.assert_called_once_with( '{"action":"start_detection","target_node_code":"mainland-controller-01-a"}' ) def test_handle_control_message_ignores_command_for_other_instance(self): worker = DetectWorker.__new__(DetectWorker) worker.use_redis = False worker.redis_client = None worker._acknowledge_pending_control_command = MagicMock() worker._remember_ignored_targeted_control = MagicMock() worker.start_detection_async = MagicMock() worker.request_stop_detection = MagicMock() with patch.object(config, "NODE_CODE", "mainland-controller-01-a"): worker._handle_control_message( '{"action":"start_detection","target_node_code":"mainland-controller-01-b"}' ) worker._acknowledge_pending_control_command.assert_not_called() worker._remember_ignored_targeted_control.assert_called_once() worker.start_detection_async.assert_not_called() def test_handle_control_message_records_ignored_targeted_job_for_other_instance(self): worker = self._build_worker(MagicMock()) with patch.object(config, "NODE_CODE", "mainland-controller-01-a"): worker._handle_control_message( { "action": "start_detection", "job_id": 870, "job_code": "sync-overseas-19868", "target_node_codes": ["mainland-controller-01-do", "mainland-controller-01-cz"], } ) self.assertEqual(870, worker._ignored_targeted_job_id) self.assertEqual("sync-overseas-19868", worker._ignored_targeted_job_code) self.assertEqual( "mainland-controller-01-do,mainland-controller-01-cz", worker._ignored_targeted_job_targets, ) def test_auto_resume_skips_when_active_job_matches_recently_ignored_targeted_job(self): worker = self._build_worker(MagicMock()) worker._load_active_detect_job_snapshot = MagicMock( return_value={ "id": 870, "job_code": "sync-overseas-19868", "task_mode": "domain_pipeline", "items_pending": 1200, "items_claimed": 949, "items_running": 6315, "selection_reason": "running_job_active", } ) worker.start_detection_async = MagicMock(return_value=True) worker._ignored_targeted_job_id = 870 worker._ignored_targeted_job_code = "sync-overseas-19868" worker._ignored_targeted_job_until = time.time() + 120 worker._ignored_targeted_job_targets = "mainland-controller-01-do,mainland-controller-01-cz" started = worker._resume_active_detect_job_if_needed(reason="heartbeat_autoresume") self.assertFalse(started) worker.start_detection_async.assert_not_called() def test_proxy_direct_fallback_grace_shrinks_for_shortage_steps(self): worker = self._build_worker(_FakeDb("noop")) worker.proxy_direct_fallback_grace_overrides = { "注册状态检测": 0.08, "360检测": 0.12, "站长之家检测": 0.12, } worker.proxy_last_refresh_status = "代理源暂时冷却中，稍后继续补货" self.assertEqual(0.05, worker._proxy_direct_fallback_grace_for_step("注册状态检测")) self.assertEqual(0.05, worker._proxy_direct_fallback_grace_for_step("360检测")) self.assertEqual(0.05, worker._proxy_direct_fallback_grace_for_step("站长之家检测")) def test_run_detect_register_prefers_direct_when_proxy_source_is_short(self): worker = self._build_worker(_FakeDb("noop")) worker.proxy_last_refresh_status = "代理源暂时冷却中，稍后继续补货" worker._check_proxy_step_retry_budget = MagicMock(return_value=(False, "")) worker._remaining_proxy_step_retry_budget_seconds = MagicMock(return_value=2.0) worker._is_current_single_step_task = MagicMock(return_value=False) worker._clear_proxy_failure = MagicMock() worker._clear_proxy_source_failure = MagicMock() worker._get_proxy_for_step = MagicMock(return_value={"http": "http://proxy", "https": "http://proxy"}) worker.db.update_domain_register_result = MagicMock() with patch("detect_worker.register.check_register", return_value=(3, "")) as mock_check: result = worker._run_detect_register(1, {"source_type": 0}, "example.com") self.assertTrue(result) worker._get_proxy_for_step.assert_not_called() self.assertIsNone(mock_check.call_args.args[2]) def test_run_detect_register_single_machine_mode_allows_direct_even_when_global_direct_disabled(self): worker = self._build_worker(_FakeDb("noop")) worker.proxy_config = {"proxy_enable": True, "allow_direct": False} worker.register_single_machine_mode_enabled = True worker.register_single_machine_direct_streak_attempts = 2 worker._check_proxy_step_retry_budget = MagicMock(return_value=(False, "")) worker._remaining_proxy_step_retry_budget_seconds = MagicMock(return_value=2.0) worker._is_current_single_step_task = MagicMock(return_value=False) worker._clear_proxy_failure = MagicMock() worker._clear_proxy_source_failure = MagicMock() worker._get_proxy_for_step = MagicMock(return_value={"http": "http://proxy", "https": "http://proxy"}) worker.db.update_domain_register_result = MagicMock() with patch("detect_worker.register.check_register", return_value=(3, "")) as mock_check: result = worker._run_detect_register(1, {"source_type": 0}, "example.com") self.assertTrue(result) worker._get_proxy_for_step.assert_not_called() self.assertIsNone(mock_check.call_args.args[2]) def test_run_detect_register_single_machine_mode_retries_direct_before_proxy(self): worker = self._build_worker(_FakeDb("noop")) worker.proxy_config = {"proxy_enable": True, "allow_direct": False} worker.register_single_machine_mode_enabled = True worker.register_single_machine_direct_streak_attempts = 2 worker._check_proxy_step_retry_budget = MagicMock(return_value=(False, "")) worker._remaining_proxy_step_retry_budget_seconds = MagicMock(return_value=2.0) worker._is_current_single_step_task = MagicMock(return_value=False) worker._clear_proxy_failure = MagicMock() worker._clear_proxy_source_failure = MagicMock() worker._get_proxy_for_step = MagicMock(return_value={"http": "http://proxy", "https": "http://proxy"}) worker.db.update_domain_register_result = MagicMock() with patch( "detect_worker.register.check_register", side_effect=[RuntimeError("direct timeout"), (3, "")], ) as mock_check: result = worker._run_detect_register(1, {"source_type": 0}, "example.com") self.assertTrue(result) self.assertEqual(2, mock_check.call_count) self.assertTrue(all(call.args[2] is None for call in mock_check.call_args_list)) worker._get_proxy_for_step.assert_not_called() def test_run_detect_register_direct_error_keeps_fast_direct_retry_during_shortage(self): worker = self._build_worker(_FakeDb("noop")) worker.proxy_last_refresh_status = "代理源暂时冷却中，稍后继续补货" worker._check_proxy_step_retry_budget = MagicMock(side_effect=[(False, ""), (True, "budget exhausted")]) worker._get_proxy_for_step = MagicMock(return_value=None) worker._mark_detection_failed = MagicMock() with patch("detect_worker.register.check_register", side_effect=RuntimeError("direct timeout")): result = worker._run_detect_register(1, {"source_type": 0}, "example.com") self.assertFalse(result) self.assertTrue(worker._consume_step_force_direct_once("注册状态检测")) self.assertAlmostEqual(0.05, worker._consume_step_proxy_retry_wait("注册状态检测")) worker._mark_detection_failed.assert_called_once() def test_run_detect_register_blacklists_client_hold_for_single_step_task(self): worker = self._build_worker(_FakeDb("noop")) worker._check_proxy_step_retry_budget = MagicMock(return_value=(False, "")) worker._remaining_proxy_step_retry_budget_seconds = MagicMock(return_value=2.0) worker._is_current_single_step_task = MagicMock(return_value=True) worker._cache_single_step_result_payload = MagicMock() worker._mark_blacklisted = MagicMock() worker._clear_proxy_failure = MagicMock() worker._clear_proxy_source_failure = MagicMock() worker._get_proxy_for_step = MagicMock(return_value={"http": "http://proxy", "https": "http://proxy"}) worker.db.update_domain_register_result = MagicMock() with patch("detect_worker.register.check_register", return_value=(REGISTER_STATUS_CLIENT_HOLD, "2026-05-01")): result = worker._run_detect_register(1, {"source_type": 0}, "example.com") self.assertFalse(result) worker._cache_single_step_result_payload.assert_called_once() payload = worker._cache_single_step_result_payload.call_args.args[2] self.assertEqual("blacklisted", payload["state"]) self.assertEqual(REGISTER_STATUS_CLIENT_HOLD, payload["register_status"]) worker._mark_blacklisted.assert_called_once() worker.db.update_domain_register_result.assert_not_called() def test_run_detect_register_blacklists_server_hold_for_domain_task(self): worker = self._build_worker(_FakeDb("noop")) worker._check_proxy_step_retry_budget = MagicMock(return_value=(False, "")) worker._remaining_proxy_step_retry_budget_seconds = MagicMock(return_value=2.0) worker._is_current_single_step_task = MagicMock(return_value=False) worker._mark_blacklisted = MagicMock() worker._clear_proxy_failure = MagicMock() worker._clear_proxy_source_failure = MagicMock() worker._get_proxy_for_step = MagicMock(return_value={"http": "http://proxy", "https": "http://proxy"}) worker.db.update_domain_register_result = MagicMock() domain = {"source_type": 0} with patch("detect_worker.register.check_register", return_value=(REGISTER_STATUS_SERVER_HOLD, "2026-05-01")): result = worker._run_detect_register(1, domain, "example.com") self.assertFalse(result) worker.db.update_domain_register_result.assert_called_once() self.assertEqual(REGISTER_STATUS_SERVER_HOLD, domain["register_status"]) worker._mark_blacklisted.assert_called_once() def test_get_proxy_for_step_fast_direct_fallback_when_proxy_source_is_cooling_down(self): worker = self._build_worker(_FakeDb("noop")) worker.proxy_last_refresh_status = "代理源暂时冷却中，稍后继续补货" worker.proxy_direct_fallback_grace_overrides = {"360检测": 0.12} worker.get_proxies = MagicMock(return_value=None) with patch("detect_worker.time.time", side_effect=[100.0, 100.2]), patch("detect_worker.time.sleep") as mock_sleep: proxy = worker._get_proxy_for_step(1, "example.com", "360检测") self.assertIsNone(proxy) mock_sleep.assert_not_called() def test_get_proxy_for_step_register_single_machine_mode_skips_proxy_wait(self): worker = self._build_worker(_FakeDb("noop")) worker.proxy_config = {"proxy_enable": True, "allow_direct": False} worker.register_single_machine_mode_enabled = True worker.register_single_machine_direct_streak_attempts = 2 worker.get_proxies = MagicMock(return_value=None) with patch("detect_worker.time.time", side_effect=[100.0, 100.0]), patch("detect_worker.time.sleep") as mock_sleep: proxy = worker._get_proxy_for_step(1, "example.com", "注册状态检测") self.assertEqual(0.0, worker._proxy_direct_fallback_grace_for_step("注册状态检测")) self.assertEqual(0.0, worker._proxy_direct_retry_wait_for_step("注册状态检测")) self.assertIsNone(proxy) mock_sleep.assert_not_called() def test_external_issue_retry_wait_uses_step_specific_short_window(self): worker = self._build_worker(_FakeDb("noop")) worker._set_current_task_context(domain_id=1, is_step_task=True, task_mode="single_step") worker.proxy_direct_retry_wait_overrides = {"360检测": 0.25} worker.proxy_last_refresh_status = "代理源暂时冷却中，稍后继续补货" should_retry = worker._should_retry_external_issue( None, "360搜索请求超时", "360检测", "example.com", ) self.assertTrue(should_retry) self.assertAlmostEqual(0.25, worker._consume_step_proxy_retry_wait("360检测")) def test_stale_detect_session_holds_during_proxy_shortage_for_same_job(self): worker = self._build_worker(_FakeDb("noop")) worker.detect_command_thread = MagicMock() worker.detect_command_thread.is_alive.return_value = True worker.current_job_code = "sync-overseas-51" worker.proxy_last_refresh_status = "代理源暂时冷却中，稍后继续补货" worker.proxy_pool = [] with patch("detect_worker.time.time", return_value=200.0): worker.detect_command_started_at = 160.0 worker.detect_command_last_activity_at = 165.0 worker._last_domain_started_at = 165.0 worker._last_domain_result_at = 165.0 stale_detect, stale_reason = worker._is_stale_detect_session({"job_code": "sync-overseas-51"}) self.assertFalse(stale_detect) self.assertIn("同任务同周期重复唤起已忽略", stale_reason) def test_stale_detect_session_still_recovers_when_new_job_arrives(self): worker = self._build_worker(_FakeDb("noop")) worker.detect_command_thread = MagicMock() worker.detect_command_thread.is_alive.return_value = True worker.current_job_code = "sync-overseas-51" worker.current_job_task_mode = "single_step" worker.proxy_last_refresh_status = "代理源暂时冷却中，稍后继续补货" worker.proxy_pool = [] with patch("detect_worker.time.time", return_value=200.0): worker.detect_command_started_at = 150.0 worker.detect_command_last_activity_at = 170.0 worker._last_domain_started_at = 170.0 worker._last_domain_result_at = 170.0 stale_detect, stale_reason = worker._is_stale_detect_session({"job_code": "sync-overseas-99"}) self.assertTrue(stale_detect) self.assertIn("新任务", stale_reason) def test_stale_detect_session_does_not_replace_busy_domain_pipeline_for_new_job(self): worker = self._build_worker(_FakeDb("noop")) worker.detect_command_thread = MagicMock() worker.detect_command_thread.is_alive.return_value = True worker.current_job_code = "sync-overseas-51" worker.current_job_task_mode = "domain_pipeline" worker.proxy_last_refresh_status = "代理正常" worker.proxy_pool = [{}] worker._active_domain_threads = 6 with patch("detect_worker.time.time", return_value=200.0): worker.detect_command_started_at = 150.0 worker.detect_command_last_activity_at = 170.0 worker._last_domain_started_at = 170.0 worker._last_domain_result_at = 170.0 stale_detect, stale_reason = worker._is_stale_detect_session( {"job_code": "sync-overseas-99", "task_mode": "domain_pipeline"} ) self.assertFalse(stale_detect) self.assertEqual("", stale_reason) def test_submit_backlog_limit_defaults_to_thread_count_window(self): worker = self._build_worker(_FakeDb("noop")) self.assertEqual(2000, worker._resolve_submit_backlog_limit(2000, 0)) self.assertEqual(2000, worker._resolve_submit_backlog_limit(2000, 1200)) def test_dispatch_capacity_can_ramp_to_full_thread_limit(self): worker = self._build_worker(_FakeDb("noop")) self.assertEqual(2000, worker._resolve_dispatch_capacity(2000, 0, 2000)) self.assertEqual(2000, worker._resolve_dispatch_capacity(2000, 1000, 1000)) def test_claim_batch_size_defaults_to_thread_count_window(self): worker = self._build_worker(_FakeDb("noop")) self.assertEqual(2000, worker._resolve_claim_batch_size(2000, 2000, live_active=0)) self.assertEqual(1500, worker._resolve_claim_batch_size(2000, 1500, live_active=800)) def test_runtime_settings_can_shrink_claim_and_backlog_windows(self): worker = self._build_worker(_FakeDb("noop")) worker.runtime_settings = { "claim_batch_floor": 200, "claim_batch_ceil": 800, "submit_backlog_floor": 500, "submit_backlog_ceil": 1500, "dispatch_cap_multiplier": 2, "pending_buffer_cap_multiplier": 2, } self.assertEqual(500, worker._resolve_submit_backlog_limit(2000, 0)) self.assertEqual(500, worker._resolve_claim_batch_size(2000, 2000, live_active=0)) self.assertEqual(1500, worker._resolve_dispatch_capacity(2000, 0, 1500)) self.assertEqual(2, worker._runtime_positive_int_override("pending_buffer_cap_multiplier", "DOMAINCHECK_PENDING_BUFFER_CAP_MULTIPLIER")) def test_runtime_settings_override_environment_claim_and_backlog_windows(self): worker = self._build_worker(_FakeDb("noop")) worker.runtime_settings = { "claim_batch_floor": 200, "claim_batch_ceil": 800, "submit_backlog_floor": 500, "submit_backlog_ceil": 1500, "dispatch_cap_multiplier": 2, } with patch.dict( os.environ, { "DOMAINCHECK_CLAIM_BATCH_FLOOR": "4000", "DOMAINCHECK_CLAIM_BATCH_CEIL": "10000", "DOMAINCHECK_SUBMIT_BACKLOG_FLOOR": "5000", "DOMAINCHECK_SUBMIT_BACKLOG_CEIL": "10000", "DOMAINCHECK_DISPATCH_CAP_MULTIPLIER": "5", }, clear=False, ): self.assertEqual(500, worker._resolve_submit_backlog_limit(2000, 0)) self.assertEqual(500, worker._resolve_claim_batch_size(2000, 2000, live_active=0)) self.assertEqual(1500, worker._resolve_dispatch_capacity(2000, 0, 1500)) def test_pipeline_topup_triggers_when_partial_claim_leaves_large_gap(self): worker = self._build_worker(_FakeDb("noop")) self.assertTrue(worker._should_proactively_top_up_pipeline(280, 1800, 2000)) self.assertFalse(worker._should_proactively_top_up_pipeline(1700, 1800, 2000)) def test_sync_prefetch_triggers_when_local_work_falls_below_threshold(self): worker = self._build_worker(_FakeDb("noop")) self.assertTrue(worker._should_prefetch_sync_tasks(24, 18, 1000)) self.assertFalse(worker._should_prefetch_sync_tasks(420, 180, 1000)) def test_pull_sync_tasks_can_prefetch_without_claiming_after_pull(self): worker = self._build_worker(_FakeDb("noop")) worker._acquire_shared_job_maintenance_lock = MagicMock(return_value="lock-owner") worker._release_shared_job_maintenance_lock = MagicMock() worker._claim_detect_job_items = MagicMock(return_value=[{"domain_name": "should-not-claim.com"}]) class _FakeResponse: def __enter__(self_inner): return self_inner def __exit__(self_inner, exc_type, exc, tb): return False @staticmethod def read(): return json.dumps( { "code": 0, "message": "ok", "data": {"pull_state": "queued", "queued_count": 500}, }, ensure_ascii=False, ).encode("utf-8") with patch.object(config, "NODE_REGION", "mainland"), patch.object(config, "NODE_ROLE", "control"), patch( "detect_worker.urllib.request.urlopen", return_value=_FakeResponse(), ): domains = worker._pull_sync_tasks_until_available(thread_limit=1000, claim_after_pull=False) self.assertEqual([], domains) worker._claim_detect_job_items.assert_not_called() worker._release_shared_job_maintenance_lock.assert_called_once() def test_get_proxy_for_step_can_force_single_direct_fallback_once(self): worker = self._build_worker(_FakeDb("noop")) worker._set_current_task_context(domain_id=1, is_step_task=True, task_mode="single_step") worker._set_step_force_direct_once("百度site检测", True) proxy = worker._get_proxy_for_step(1, "example.com", "百度site检测") self.assertIsNone(proxy) self.assertFalse(worker._consume_step_force_direct_once("百度site检测")) def test_load_thread_count_can_inherit_parent_node_override(self): worker = DetectWorker.__new__(DetectWorker) worker.use_redis = True class _FakeRedis: def get(self, key): if key == "domain_tool:node_thread_counts": return json.dumps({"mainland-controller-01": 1200}, ensure_ascii=False) if key == "domain_tool:thread_count": return "1000" return None worker.redis_client = _FakeRedis() with patch.object(config, "NODE_CODE", "mainland-controller-01-a"), patch.dict( os.environ, {"WORKER_PARENT_NODE_CODE": "mainland-controller-01"}, clear=False, ): thread_count = DetectWorker.load_thread_count(worker) self.assertEqual(1200, thread_count) def test_load_process_count_can_inherit_parent_node_override_from_local_file_root(self): worker = DetectWorker.__new__(DetectWorker) worker.use_redis = False with patch.object(config, "NODE_CODE", "mainland-controller-01-u"), patch.dict( os.environ, {"WORKER_PARENT_NODE_CODE": "mainland-controller-01"}, clear=False, ), patch( "detect_worker._read_worker_local_json_config", side_effect=lambda name: ( {"mainland-controller-01": 80} if name == "node_process_counts.json" else ({"process_count": "80"} if name == "process_count.json" else None) ), ): process_count = DetectWorker.load_process_count(worker) self.assertEqual(80, process_count) def test_get_proxies_with_lease_holds_entry_until_release(self): worker = self._build_worker(_FakeDb("noop")) good_proxy = {"http": "http://good-proxy", "https": "http://good-proxy"} worker.proxy_pool = [ { "proxy": good_proxy, "usage_count": 0, "source_tag": "C", "expire_at_ts": time.time() + 60, } ] worker._schedule_proxy_refresh_if_needed = MagicMock() selected = worker.get_proxies(step_name="百度site检测", lease_selected=True) self.assertEqual(good_proxy, selected) self.assertEqual(0, len(worker.proxy_pool)) self.assertIn(worker._proxy_key(good_proxy), worker.proxy_active_leases) reused = worker.release_proxy(good_proxy, step_name="百度site检测") self.assertTrue(reused) self.assertEqual(1, len(worker.proxy_pool)) self.assertNotIn(worker._proxy_key(good_proxy), worker.proxy_active_leases) def test_release_proxy_discards_entry_when_remaining_ttl_is_too_low(self): worker = self._build_worker(_FakeDb("noop")) proxy = {"http": "http://soon-expire", "https": "http://soon-expire"} worker.proxy_active_leases = { worker._proxy_key(proxy): { "proxy": proxy, "usage_count": 1, "source_tag": "D", "expire_at_ts": time.time() + 2, "leased_step_name": "百度site检测", } } worker._schedule_proxy_refresh_if_needed = MagicMock() reused = worker.release_proxy(proxy, step_name="百度site检测") self.assertFalse(reused) self.assertEqual([], worker.proxy_pool) self.assertNotIn(worker._proxy_key(proxy), worker.proxy_active_leases) def test_retry_external_issue_with_proxy_forces_next_attempt_direct_when_allowed(self): worker = self._build_worker(_FakeDb("noop")) worker._set_current_task_context(domain_id=1, is_step_task=True, task_mode="single_step") worker.remove_proxy = MagicMock() proxy = {"http": "http://bad-proxy", "https": "http://bad-proxy"} should_retry = worker._should_retry_external_issue( proxy, "curl: (28) Connection timed out after 552 milliseconds", "百度site检测", "example.com", ) self.assertTrue(should_retry) worker.remove_proxy.assert_called_once_with(proxy) worker.trigger_proxy_refresh.assert_called_once() self.assertTrue(worker._consume_step_force_direct_once("百度site检测")) def test_external_dependency_issue_recognizes_common_chinese_timeout_messages(self): worker = self._build_worker(_FakeDb("noop")) self.assertTrue(worker._is_external_dependency_issue("360搜索请求超时")) self.assertTrue(worker._is_external_dependency_issue("百度site检测失败: 连接超时")) self.assertTrue(worker._is_external_dependency_issue("站长之家检测失败: 网络异常")) def test_run_detect_aizhan_passes_remaining_budget_to_detector(self): worker = self._build_worker(_FakeDb("noop")) worker._check_proxy_step_retry_budget = MagicMock(return_value=(False, "")) worker._remaining_proxy_step_retry_budget_seconds = MagicMock(return_value=1.7) worker._get_proxy_for_step = MagicMock(return_value=None) worker._mark_detection_degraded = MagicMock() worker._should_blacklist_result = MagicMock(return_value=False) worker._record_step_result = MagicMock() worker._should_retry_external_issue = MagicMock(return_value=False) worker._is_external_dependency_issue = MagicMock(return_value=False) worker._mark_detection_rejected = MagicMock() worker._mark_detection_failed = MagicMock() with patch("detect_worker.aizhan.check_aizhan", return_value=(True, "")) as mock_check: result = worker._run_detect_aizhan(1, "example.com", []) self.assertTrue(result) self.assertEqual(1.7, mock_check.call_args.kwargs["budget_seconds"]) def test_run_detect_aizhan_uses_external_issue_retry_path(self): worker = self._build_worker(_FakeDb("noop")) worker._check_proxy_step_retry_budget = MagicMock(side_effect=[(False, ""), (False, ""), (True, "budget exhausted")]) worker._remaining_proxy_step_retry_budget_seconds = MagicMock(return_value=2.0) worker._get_proxy_for_step = MagicMock(side_effect=[ {"http": "http://proxy-1", "https": "http://proxy-1"}, None, ]) worker._mark_detection_degraded = MagicMock() worker._should_blacklist_result = MagicMock(return_value=False) worker._record_step_result = MagicMock() worker._should_retry_external_issue = MagicMock(side_effect=[True, False]) worker._is_external_dependency_issue = MagicMock(return_value=False) worker._mark_detection_rejected = MagicMock() worker._mark_detection_failed = MagicMock() worker.release_proxy = MagicMock() with patch( "detect_worker.aizhan.check_aizhan", side_effect=[(False, "proxy timeout"), (True, "ok")], ) as mock_check: result = worker._run_detect_aizhan(1, "example.com", []) self.assertTrue(result) self.assertEqual(2, mock_check.call_count) worker._should_retry_external_issue.assert_called_with( {"http": "http://proxy-1", "https": "http://proxy-1"}, "proxy timeout", "爱站网检测", "example.com", ) def test_single_machine_aizhan_direct_first_enabled_only_on_first_attempt(self): worker = self._build_worker(_FakeDb("noop")) worker.single_machine_aizhan_direct_first_enabled = True worker.single_machine_site_direct_fallback_enabled = True worker.allow_direct_connection = MagicMock(return_value=False) self.assertTrue(worker._should_force_direct_first_attempt("爱站网检测", 0)) self.assertFalse(worker._should_force_direct_first_attempt("爱站网检测", 1)) self.assertFalse(worker._should_force_direct_first_attempt("百度site检测", 0)) def test_single_machine_baidu_direct_first_enabled_only_on_first_attempt(self): worker = self._build_worker(_FakeDb("noop")) worker.single_machine_baidu_direct_first_enabled = True worker.single_machine_site_direct_fallback_enabled = True worker.allow_direct_connection = MagicMock(return_value=False) self.assertTrue(worker._should_force_direct_first_attempt("百度site检测", 0)) self.assertFalse(worker._should_force_direct_first_attempt("百度site检测", 1)) self.assertFalse(worker._should_force_direct_first_attempt("爱站网检测", 0)) def test_single_machine_chinaz_direct_first_enabled_only_on_first_attempt(self): worker = self._build_worker(_FakeDb("noop")) worker.single_machine_chinaz_direct_first_enabled = True worker.single_machine_site_direct_fallback_enabled = True worker.allow_direct_connection = MagicMock(return_value=False) self.assertTrue(worker._should_force_direct_first_attempt("站长之家检测", 0)) self.assertFalse(worker._should_force_direct_first_attempt("站长之家检测", 1)) self.assertFalse(worker._should_force_direct_first_attempt("百度site检测", 0)) def test_run_detect_aizhan_forces_direct_first_when_single_machine_flag_enabled(self): worker = self._build_worker(_FakeDb("noop")) worker.single_machine_aizhan_direct_first_enabled = True worker.single_machine_site_direct_fallback_enabled = True worker.allow_direct_connection = MagicMock(return_value=False) worker._check_proxy_step_retry_budget = MagicMock(return_value=(False, "")) worker._remaining_proxy_step_retry_budget_seconds = MagicMock(return_value=2.0) worker._get_proxy_for_step = MagicMock(return_value=None) worker._mark_detection_degraded = MagicMock() worker._should_blacklist_result = MagicMock(return_value=False) worker._record_step_result = MagicMock() worker._should_retry_external_issue = MagicMock(return_value=False) worker._is_external_dependency_issue = MagicMock(return_value=False) worker._mark_detection_rejected = MagicMock() worker._mark_detection_failed = MagicMock() worker._set_step_force_direct_once = MagicMock() with patch("detect_worker.aizhan.check_aizhan", return_value=(True, "")): result = worker._run_detect_aizhan(1, "example.com", []) self.assertTrue(result) worker._set_step_force_direct_once.assert_called_once_with("爱站网检测", True) def test_run_detect_baidu_forces_direct_first_when_single_machine_flag_enabled(self): worker = self._build_worker(_FakeDb("noop")) worker.single_machine_baidu_direct_first_enabled = True worker.single_machine_site_direct_fallback_enabled = True worker.allow_direct_connection = MagicMock(return_value=False) worker._check_proxy_step_retry_budget = MagicMock(return_value=(False, "")) worker._remaining_proxy_step_retry_budget_seconds = MagicMock(return_value=2.0) worker._get_proxy_for_step = MagicMock(return_value=None) worker._mark_detection_degraded = MagicMock() worker._should_blacklist_result = MagicMock(return_value=False) worker._record_step_result = MagicMock() worker._should_retry_external_issue = MagicMock(return_value=False) worker._is_external_dependency_issue = MagicMock(return_value=False) worker._mark_detection_rejected = MagicMock() worker._mark_detection_failed = MagicMock() worker._set_step_force_direct_once = MagicMock() with patch("detect_worker.baidu.check_site", return_value=(True, "")): result = worker._run_detect_baidu(1, "example.com", []) self.assertTrue(result) worker._set_step_force_direct_once.assert_called_once_with("百度site检测", True) def test_run_detect_chinaz_forces_direct_first_when_single_machine_flag_enabled(self): worker = self._build_worker(_FakeDb("noop")) worker.single_machine_chinaz_direct_first_enabled = True worker.single_machine_site_direct_fallback_enabled = True worker.allow_direct_connection = MagicMock(return_value=False) worker._check_proxy_step_retry_budget = MagicMock(return_value=(False, "")) worker._remaining_proxy_step_retry_budget_seconds = MagicMock(return_value=2.0) worker._get_proxy_for_step = MagicMock(return_value={"http": "http://proxy-1", "https": "http://proxy-1"}) worker._mark_detection_degraded = MagicMock() worker._should_blacklist_result = MagicMock(return_value=False) worker._record_step_result = MagicMock() worker._should_rotate_proxy_on_error = MagicMock(return_value=False) worker._is_external_dependency_issue = MagicMock(return_value=False) worker._mark_detection_rejected = MagicMock() worker._mark_detection_failed = MagicMock() worker._emit_step_trace = MagicMock() with patch("detect_worker.chinaz.check_title", return_value=(True, "", {})) as mock_check_title: result = worker._run_detect_chinaz(1, "example.com", []) self.assertTrue(result) worker._get_proxy_for_step.assert_not_called() worker._emit_step_trace.assert_called_once() self.assertIsNone(mock_check_title.call_args.args[2]) def test_run_detect_aizhan_degrades_remote_disconnected_early_when_flag_enabled(self): worker = self._build_worker(_FakeDb("noop")) worker.aizhan_remote_disconnect_degrade_enabled = True worker._check_proxy_step_retry_budget = MagicMock(return_value=(False, "")) worker._remaining_proxy_step_retry_budget_seconds = MagicMock(return_value=2.0) proxy = {"http": "http://proxy-1", "https": "http://proxy-1"} worker._get_proxy_for_step = MagicMock(return_value=proxy) worker._mark_detection_degraded = MagicMock() worker._should_blacklist_result = MagicMock(return_value=False) worker._record_step_result = MagicMock() worker._should_retry_external_issue = MagicMock(return_value=True) worker._is_external_dependency_issue = MagicMock(return_value=True) worker._mark_detection_rejected = MagicMock() worker._mark_detection_failed = MagicMock() worker.release_proxy = MagicMock() with patch( "detect_worker.aizhan.check_aizhan", return_value=(False, "('Connection aborted.', RemoteDisconnected('Remote end closed connection without response'))"), ): result = worker._run_detect_aizhan(1, "example.com", []) self.assertTrue(result) worker._should_retry_external_issue.assert_not_called() worker.release_proxy.assert_called_once() worker._mark_detection_degraded.assert_called_once() def test_run_detect_aizhan_degrades_external_issue_early_when_fast_flag_enabled(self): worker = self._build_worker(_FakeDb("noop")) worker.aizhan_external_fast_degrade_enabled = True worker._check_proxy_step_retry_budget = MagicMock(return_value=(False, "")) worker._remaining_proxy_step_retry_budget_seconds = MagicMock(return_value=2.0) proxy = {"http": "http://proxy-1", "https": "http://proxy-1"} worker._get_proxy_for_step = MagicMock(return_value=proxy) worker._mark_detection_degraded = MagicMock() worker._should_blacklist_result = MagicMock(return_value=False) worker._record_step_result = MagicMock() worker._should_retry_external_issue = MagicMock(return_value=True) worker._is_external_dependency_issue = MagicMock(return_value=True) worker._mark_detection_rejected = MagicMock() worker._mark_detection_failed = MagicMock() worker.release_proxy = MagicMock() with patch( "detect_worker.aizhan.check_aizhan", return_value=(False, "Read timed out. (read timeout=1.8)"), ): result = worker._run_detect_aizhan(1, "example.com", []) self.assertTrue(result) worker._should_retry_external_issue.assert_not_called() worker.release_proxy.assert_called_once() worker._mark_detection_degraded.assert_called_once() def test_resolve_single_step_finalization_keeps_degraded_as_completed(self): status, message = DetectWorker._resolve_single_step_finalization( { "state": "degraded", "message": "外部依赖异常，建议重试", "retry_recommended": True, } ) self.assertEqual("completed", status) self.assertEqual("外部依赖异常，建议重试", message) def test_worker_log_sync_mode_keeps_full_when_load_is_small(self): worker = self._build_worker(_FakeDb("noop")) worker.runtime_settings = {"worker_log_sync_enabled": True, "worker_log_sync_mode": "full"} worker.thread_count = 1200 worker.detecting = True worker._get_active_domain_threads = MagicMock(return_value=80) self.assertEqual("full", worker._worker_log_sync_mode()) def test_worker_log_sync_mode_downgrades_full_when_threads_are_busy(self): worker = self._build_worker(_FakeDb("noop")) worker.runtime_settings = {"worker_log_sync_enabled": True, "worker_log_sync_mode": "full"} worker.thread_count = 1200 worker.detecting = True worker._get_active_domain_threads = MagicMock(return_value=650) self.assertEqual("key", worker._worker_log_sync_mode()) def test_worker_log_sync_mode_downgrades_full_when_queue_backlog_is_high(self): worker = self._build_worker(_FakeDb("noop")) worker.runtime_settings = {"worker_log_sync_enabled": True, "worker_log_sync_mode": "full"} worker.thread_count = 1200 worker.detecting = False worker._worker_log_sync_queue = MagicMock() worker._worker_log_sync_queue.maxsize = 50000 worker._worker_log_sync_queue.qsize.return_value = 12000 self.assertEqual("key", worker._worker_log_sync_mode()) def test_push_remote_debug_event_enters_cooldown_after_failure(self): worker = self._build_worker(_FakeDb("noop")) with patch.object(worker, "_remote_debug_ingest_url", return_value="http://example.invalid/api/v1/runtime/debug-ingest"): with patch("detect_worker.urllib.request.urlopen", side_effect=TimeoutError("timed out")) as mock_urlopen: worker._push_remote_debug_event(message="test") self.assertEqual(1, worker._remote_debug_event_failure_streak) self.assertGreater(worker._remote_debug_event_cooldown_until, time.time()) self.assertEqual(1, mock_urlopen.call_count) worker._push_remote_debug_event(message="test") self.assertEqual(1, mock_urlopen.call_count) def test_push_remote_debug_event_resets_failure_streak_after_success(self): worker = self._build_worker(_FakeDb("noop")) worker._remote_debug_event_failure_streak = 3 class _FakeResponse: def __enter__(self_inner): return self_inner def __exit__(self_inner, exc_type, exc, tb): return False @staticmethod def read(): return b'{"code":0}' with patch.object(worker, "_remote_debug_ingest_url", return_value="http://example.invalid/api/v1/runtime/debug-ingest"): with patch("detect_worker.urllib.request.urlopen", return_value=_FakeResponse()): worker._push_remote_debug_event(message="test") self.assertEqual(0, worker._remote_debug_event_failure_streak) self.assertEqual(0.0, worker._remote_debug_event_cooldown_until) def test_idle_worker_recycle_skips_parent_control_worker(self): worker = self._build_worker(_FakeDb("noop")) worker.running = True worker.detecting = True worker._get_active_domain_threads = MagicMock(return_value=0) worker._last_runtime_extra = {"waiting_for_dispatch": True} worker._last_worker_activity_at = time.time() - 600 worker._worker_started_at = time.time() - 900 with patch.dict( os.environ, { "WORKER_MODE": "linux-systemd", "WORKER_PARENT_NODE_CODE": "mainland-controller-01", "DOMAINCHECK_IDLE_WORKER_RECYCLE_ENABLED": "1", }, clear=False, ): with patch.object(config, "NODE_CODE", "mainland-controller-01"): with patch.object(worker, "_trigger_idle_worker_recycle") as mock_recycle: recycled = worker._maybe_recycle_idle_worker(trigger_reason="heartbeat") self.assertFalse(recycled) mock_recycle.assert_not_called() def test_configure_worker_thread_stack_size_uses_linux_systemd_default(self): state = {"size": 0} def fake_stack_size(value=None): if value is None: return state["size"] state["size"] = int(value) return 0 with patch.dict(os.environ, {"WORKER_MODE": "linux-systemd"}, clear=False): with patch.object(detect_worker_module.threading, "stack_size", side_effect=fake_stack_size): applied = detect_worker_module._configure_worker_thread_stack_size() self.assertEqual(1024 * 1024, applied) self.assertEqual(1024 * 1024, state["size"]) def test_configure_worker_thread_stack_size_skips_non_service_default(self): state = {"size": 0} def fake_stack_size(value=None): if value is None: return state["size"] state["size"] = int(value) return 0 with patch.dict(os.environ, {"WORKER_MODE": "desktop"}, clear=False): with patch.object(detect_worker_module.threading, "stack_size", side_effect=fake_stack_size): applied = detect_worker_module._configure_worker_thread_stack_size() self.assertEqual(0, applied) self.assertEqual(0, state["size"]) def test_idle_worker_recycle_triggers_for_idle_child_worker(self): worker = self._build_worker(_FakeDb("noop")) worker.running = True worker.detecting = True worker._get_active_domain_threads = MagicMock(return_value=0) worker._last_runtime_extra = {"waiting_for_dispatch": True} worker._last_worker_activity_at = time.time() - 600 worker._worker_started_at = time.time() - 900 with patch.dict( os.environ, { "WORKER_MODE": "linux-systemd", "WORKER_PARENT_NODE_CODE": "mainland-controller-01", "DOMAINCHECK_IDLE_WORKER_RECYCLE_ENABLED": "1", "DOMAINCHECK_IDLE_WORKER_RECYCLE_AFTER_SECONDS": "120", "DOMAINCHECK_IDLE_WORKER_RECYCLE_MIN_UPTIME_SECONDS": "180", "DOMAINCHECK_IDLE_WORKER_RECYCLE_JITTER_SECONDS": "0", }, clear=False, ): with patch.object(config, "NODE_CODE", "mainland-controller-01-b"): with patch.object(worker, "_trigger_idle_worker_recycle", return_value=True) as mock_recycle: recycled = worker._maybe_recycle_idle_worker(trigger_reason="heartbeat") self.assertTrue(recycled) mock_recycle.assert_called_once() def test_idle_worker_recycle_respects_recent_explicit_start_grace(self): worker = self._build_worker(_FakeDb("noop")) worker.running = True worker.detecting = True worker._get_active_domain_threads = MagicMock(return_value=0) worker._last_runtime_extra = {"waiting_for_dispatch": True} worker._last_worker_activity_at = time.time() - 600 worker._worker_started_at = time.time() - 900 worker._last_explicit_start_signal_at = time.time() - 10 with patch.dict( os.environ, { "WORKER_MODE": "linux-systemd", "WORKER_PARENT_NODE_CODE": "mainland-controller-01", "DOMAINCHECK_IDLE_WORKER_RECYCLE_ENABLED": "1", "DOMAINCHECK_IDLE_WORKER_RECYCLE_AFTER_SECONDS": "120", "DOMAINCHECK_IDLE_WORKER_RECYCLE_MIN_UPTIME_SECONDS": "180", "DOMAINCHECK_IDLE_WORKER_RECYCLE_START_GRACE_SECONDS": "90", "DOMAINCHECK_IDLE_WORKER_RECYCLE_JITTER_SECONDS": "0", }, clear=False, ): with patch.object(config, "NODE_CODE", "mainland-controller-01-c"): with patch.object(worker, "_trigger_idle_worker_recycle") as mock_recycle: recycled = worker._maybe_recycle_idle_worker(trigger_reason="heartbeat") self.assertFalse(recycled) mock_recycle.assert_not_called() def test_prepare_proxy_pool_for_detection_start_reuses_local_pool(self): worker = self._build_worker(_FakeDb("noop")) worker.proxy_pool = [{"proxy": "http://1.1.1.1:8080"}] worker._schedule_proxy_refresh_if_needed = MagicMock() ready, count, reason = worker._prepare_proxy_pool_for_detection_start() self.assertTrue(ready) self.assertEqual(1, count) self.assertEqual("local_pool_ready", reason) worker._schedule_proxy_refresh_if_needed.assert_called_once_with(1) def test_prepare_proxy_pool_for_detection_start_reuses_shared_snapshot(self): worker = self._build_worker(_FakeDb("noop")) worker._schedule_proxy_refresh_if_needed = MagicMock() worker._proxy_should_coordinate_shared_refresh = MagicMock(return_value=True) worker._load_shared_proxy_snapshot_payload = MagicMock( return_value={"proxy_pool": [{"proxy": "http://2.2.2.2:8080"}]} ) def apply_snapshot(payload, status_prefix=""): worker.proxy_pool = list(payload.get("proxy_pool") or []) return True worker._apply_shared_proxy_snapshot = MagicMock(side_effect=apply_snapshot) ready, count, reason = worker._prepare_proxy_pool_for_detection_start() self.assertTrue(ready) self.assertEqual(1, count) self.assertEqual("shared_snapshot_ready", reason) worker._apply_shared_proxy_snapshot.assert_called_once() worker._schedule_proxy_refresh_if_needed.assert_called_once_with(1) if __name__ == "__main__": unittest.main()