getDomain/domainCheck/tests/test_detect_worker_flush_retry.py

import collections
import json
import os
import threading
import tempfile
import unittest
import sys
import time
from queue import Queue
from unittest.mock import MagicMock
from unittest.mock import patch


sys.path.insert(0, "/www/wwwroot/getDomain/domainCheck")

import detect_worker as detect_worker_module  # noqa: E402
from app.config import config  # noqa: E402
from detect_worker import DetectWorker, control_targets_current_worker, pending_control_key  # noqa: E402
from app.utils.status_codes import REGISTER_STATUS_CLIENT_HOLD, REGISTER_STATUS_SERVER_HOLD  # noqa: E402


class _FakeDb:
    def __init__(self, method_name: str):
        self.method_name = method_name
        self.calls = 0

    def _run(self, batch):
        self.calls += 1
        return -1 if self.calls == 1 else len(batch)

    def mark_detect_job_items_running_batch(self, batch):
        return self._run(batch)

    def release_detect_job_items_batch(self, batch):
        return self._run(batch)

    def finalize_detect_job_items_batch(self, batch):
        return self._run(batch)


class _FakeRedis:
    def __init__(self):
        self._store = {}
        self._expires = {}

    def _cleanup(self, key):
        expires_at = self._expires.get(key)
        if expires_at is not None and expires_at <= time.time():
            self._store.pop(key, None)
            self._expires.pop(key, None)

    def get(self, key):
        self._cleanup(key)
        return self._store.get(key)

    def set(self, key, value, nx=False, ex=None):
        self._cleanup(key)
        if nx and key in self._store:
            return False
        self._store[key] = value
        if ex is not None:
            self._expires[key] = time.time() + float(ex)
        else:
            self._expires.pop(key, None)
        return True

    def delete(self, key):
        existed = key in self._store
        self._store.pop(key, None)
        self._expires.pop(key, None)
        return 1 if existed else 0


class DetectWorkerFlushRetryTest(unittest.TestCase):
    def _build_worker(self, db):
        worker = DetectWorker.__new__(DetectWorker)
        worker.db = db
        worker._task_local = threading.local()
        worker._running_mark_lock = threading.Lock()
        worker._pending_running_marks = collections.deque()
        worker._last_running_mark_flush_at = 0.0
        worker._job_release_lock = threading.Lock()
        worker._pending_job_releases = collections.deque()
        worker._pending_job_release_reasons = {}
        worker._last_job_release_flush_at = 0.0
        worker._job_finalize_lock = threading.Lock()
        worker._pending_job_finalizations = collections.deque()
        worker._last_job_finalize_flush_at = 0.0
        worker.proxy_pool = []
        worker.proxy_pool_lock = threading.Lock()
        worker.proxy_refresh_lock = threading.Lock()
        worker.proxy_refresh_cooldown_seconds = 2
        worker.proxy_next_refresh_time = 0.0
        worker._last_proxy_refresh_reason_at = {}
        worker.proxy_failure_lock = threading.Lock()
        worker.proxy_failure_counts = {}
        worker.proxy_quarantine_until = {}
        worker.proxy_source_by_key = {}
        worker.proxy_source_failure_counts = {}
        worker.proxy_source_quarantine_until = {}
        worker.proxy_active_leases = {}
        worker.proxy_config = {"proxy_enable": True, "allow_direct": True}
        worker.proxy_last_refresh_time = None
        worker.proxy_last_refresh_status = "test"
        worker.proxy_last_refresh_source_count = 0
        worker.proxy_last_refresh_total_items = 0
        worker.proxy_last_validated_count = 0
        worker.proxy_last_available_count = 0
        worker.proxy_last_source_stats = []
        worker._last_proxy_clock_skew_warning_at = 0.0
        worker.proxy_max_reuse_count = 1000
        worker.proxy_step_wait_timeout_seconds = 1.5
        worker.proxy_direct_fallback_grace_seconds = 0.35
        worker.proxy_direct_fallback_grace_overrides = {}
        worker.proxy_direct_retry_wait_seconds = 1.2
        worker.proxy_direct_retry_wait_overrides = {}
        worker.register_single_machine_mode_enabled = False
        worker.register_single_machine_direct_streak_attempts = 0
        worker.trigger_proxy_refresh = MagicMock()
        worker._update_runtime_state = MagicMock()
        worker._sync_worker_log_event = MagicMock()
        worker._emit_step_trace = MagicMock()
        worker.runtime_settings = {}
        worker.thread_count = 600
        worker.process_count = 1
        worker.refresh_process_count_runtime = MagicMock(side_effect=lambda *args, **kwargs: worker.process_count)
        worker.use_redis = False
        worker.redis_client = None
        worker.detect_lock = threading.Lock()
        worker.running = True
        worker.detecting = True
        worker.stop_requested = False
        worker.detect_command_thread = None
        worker.detect_command_started_at = 0.0
        worker.detect_command_last_activity_at = 0.0
        worker._last_domain_started_at = 0.0
        worker._last_domain_result_at = 0.0
        worker._last_job_item_release_at = 0.0
        worker._active_domain_threads = 0
        worker._domain_thread_counter_lock = threading.Lock()
        worker.current_cycle_token = ""
        worker.current_job_id = None
        worker.current_job_code = ""
        worker.current_job_task_mode = ""
        worker._detect_session_owner = 0
        worker._detect_session_seq = 0
        worker._pending_restart_source = ""
        worker._pending_restart_payload = None
        worker._pending_restart_reason = ""
        worker._worker_started_at = time.time()
        worker._last_worker_activity_at = worker._worker_started_at
        worker._idle_recycle_requested = False
        worker._last_explicit_start_signal_at = 0.0
        worker._last_explicit_start_payload = {}
        worker._last_detection_start_at = 0.0
        worker._last_detection_start_source = ""
        worker._last_detection_start_job_id = None
        worker._last_detection_start_job_code = ""
        worker._last_detection_start_task_mode = ""
        worker._cached_active_detect_job = None
        worker._last_active_detect_job_refresh_at = 0.0
        worker._explicit_claim_scope_job_id = None
        worker._explicit_claim_scope_job_code = ""
        worker._explicit_claim_scope_until = 0.0
        worker._explicit_claim_scope_source = ""
        worker._ignored_targeted_job_id = None
        worker._ignored_targeted_job_code = ""
        worker._ignored_targeted_job_until = 0.0
        worker._ignored_targeted_job_targets = ""
        worker._get_active_domain_threads = MagicMock(return_value=0)
        worker._worker_log_sync_queue = Queue(maxsize=50000)
        worker._last_worker_log_sync_degrade_notice_at = 0.0
        worker._remote_debug_event_failure_streak = 0
        worker._remote_debug_event_cooldown_until = 0.0
        worker._last_remote_debug_event_notice_at = 0.0
        worker._last_runtime_extra = {}
        worker._last_runtime_phase = "idle"
        worker._last_runtime_detail = ""
        worker._runtime_heartbeat_stop = threading.Event()
        worker._worker_log_sync_stop = threading.Event()
        worker._flush_pending_domain_status_updates = MagicMock(return_value=0)
        worker._flush_pending_domain_completions = MagicMock(return_value=0)
        worker._flush_pending_review_status_updates = MagicMock(return_value=0)
        worker._flush_completed_futures = MagicMock(return_value=0)
        worker.single_machine_site_direct_fallback_enabled = False
        return worker

    def test_running_mark_flush_requeues_batch_when_db_write_fails(self):
        worker = self._build_worker(_FakeDb("running"))
        worker._pending_running_marks.extend([(101, "token-a"), (102, "token-b")])

        updated = worker._flush_pending_running_marks(force=True, batch_limit=10)
        self.assertEqual(0, updated)
        self.assertEqual([(101, "token-a"), (102, "token-b")], list(worker._pending_running_marks))

        updated = worker._flush_pending_running_marks(force=True, batch_limit=10)
        self.assertEqual(2, updated)
        self.assertEqual([], list(worker._pending_running_marks))

    def test_job_finalization_flush_requeues_batch_when_db_write_fails(self):
        worker = self._build_worker(_FakeDb("finalize"))
        worker._pending_job_finalizations.extend(
            [
                {"job_item_id": 201, "claim_token": "token-a", "final_status": "completed"},
                {"job_item_id": 202, "claim_token": "token-b", "final_status": "failed"},
            ]
        )

        updated = worker._flush_pending_job_finalizations(force=True, batch_limit=10)
        self.assertEqual(0, updated)
        self.assertEqual(2, len(worker._pending_job_finalizations))

        updated = worker._flush_pending_job_finalizations(force=True, batch_limit=10)
        self.assertEqual(2, updated)
        self.assertEqual(0, len(worker._pending_job_finalizations))

    def test_job_release_flush_requeues_batch_when_db_write_fails(self):
        worker = self._build_worker(_FakeDb("release"))
        worker._enqueue_job_release(201, "token-a", "session_replaced:7")
        worker._enqueue_job_release(202, "token-b", "queued_before_start")

        updated = worker._flush_pending_job_releases(force=True, batch_limit=10)
        self.assertEqual(0, updated)
        self.assertEqual(2, len(worker._pending_job_releases))

        updated = worker._flush_pending_job_releases(force=True, batch_limit=10)
        self.assertEqual(2, updated)
        self.assertEqual(0, len(worker._pending_job_releases))

    def test_detection_session_abort_reason_rejects_replaced_owner(self):
        worker = self._build_worker(_FakeDb("noop"))
        worker._detect_session_owner = 7

        self.assertEqual("session_replaced:7", worker._detection_session_abort_reason(6))
        self.assertEqual("", worker._detection_session_abort_reason(7))

    def test_release_job_item_for_session_abort_enqueues_current_item(self):
        db = MagicMock()
        worker = self._build_worker(db)
        worker._detect_session_owner = 7

        released = worker._release_job_item_for_session_abort(
            201,
            "token-a",
            session_id=6,
            domain_name="example.com",
            detail="prepare",
        )

        self.assertTrue(released)
        self.assertEqual(1, len(worker._pending_job_releases))
        job_item_id, claim_token = worker._pending_job_releases[0]
        reason = worker._pending_job_release_reasons[(job_item_id, claim_token)]
        self.assertEqual(201, job_item_id)
        self.assertEqual("token-a", claim_token)
        self.assertIn("session_replaced:7", reason)
        self.assertIn("prepare", reason)

    def test_enqueue_job_release_deduplicates_same_job_item_and_claim_token(self):
        worker = self._build_worker(_FakeDb("release"))

        self.assertTrue(worker._enqueue_job_release(201, "token-a", "queued_before_start"))
        self.assertTrue(worker._enqueue_job_release(201, "token-a", "session_replaced:9"))
        self.assertEqual(1, len(worker._pending_job_releases))
        self.assertEqual((201, "token-a"), worker._pending_job_releases[0])
        self.assertEqual(
            "session_replaced:9",
            worker._pending_job_release_reasons[(201, "token-a")],
        )

    def test_incoming_job_id_prefers_target_job_id_for_local_queue_scope(self):
        worker = self._build_worker(_FakeDb("noop"))

        payload = {
            "job_id": 26025,
            "job_code": "sync-overseas-26025",
            "target_job_id": 1,
            "target_job_code": "sync-overseas-26025-local",
            "task_mode": "domain_pipeline",
        }

        self.assertEqual(1, worker._incoming_job_id(payload))

    def test_schedule_explicit_claim_scope_uses_target_job_id_when_present(self):
        worker = self._build_worker(_FakeDb("noop"))

        payload = {
            "job_id": 26025,
            "job_code": "sync-overseas-26025",
            "target_job_id": 1,
            "target_job_code": "sync-overseas-26025-local",
            "task_mode": "domain_pipeline",
        }

        worker._schedule_explicit_claim_scope(payload, source="sync-pull")

        self.assertEqual(1, worker._explicit_claim_scope_job_id)
        self.assertEqual("sync-overseas-26025", worker._explicit_claim_scope_job_code)
        self.assertEqual("sync-pull", worker._explicit_claim_scope_source)

    def test_detect_domain_releases_job_item_when_session_replaced(self):
        db = MagicMock()
        worker = self._build_worker(db)
        worker._detect_session_owner = 7
        worker._set_current_task_context = MagicMock()
        worker._clear_current_task_context = MagicMock()
        worker._mark_detection_phase = MagicMock()
        worker._release_job_item_for_session_abort = MagicMock(return_value=True)

        worker.detect_domain(
            101,
            {"id": 101, "domain": "example.com"},
            {
                "job_item_id": 301,
                "job_id": 1,
                "claim_token": "token-x",
                "task_mode": "domain_pipeline",
                "session_id": 6,
            },
        )

        worker._release_job_item_for_session_abort.assert_called_once()
        args, kwargs = worker._release_job_item_for_session_abort.call_args
        self.assertEqual(301, args[0])
        self.assertEqual("token-x", args[1])
        self.assertEqual(6, kwargs["session_id"])
        self.assertEqual("example.com", kwargs["domain_name"])

    def test_detect_domain_step_task_writes_domain_started_event(self):
        db = MagicMock()
        worker = self._build_worker(db)
        worker.sensitive_words = []
        worker._set_current_task_context = MagicMock()
        worker._clear_current_task_context = MagicMock()
        worker._assert_detection_session_active = MagicMock()
        worker._change_active_domain_threads = MagicMock(return_value=1)
        worker._mark_detection_phase = MagicMock()
        worker._enqueue_running_mark = MagicMock()
        worker._run_detect_register = MagicMock(return_value=True)
        worker._get_detect_execution_order = MagicMock(return_value=["detect_register"])
        worker._build_single_step_job_result = MagicMock(return_value={"state": "completed", "message": "ok"})
        worker._resolve_single_step_finalization = MagicMock(return_value=("completed", "ok"))
        worker._enqueue_job_finalization = MagicMock()

        worker.detect_domain(
            101,
            {"id": 101, "domain": "example.com"},
            {
                "job_item_id": 301,
                "job_id": 1,
                "job_code": "sync-overseas-26118",
                "claim_token": "token-x",
                "task_mode": "domain_pipeline",
                "step_code": "detect_register",
                "step_payload": {"step_code": "detect_register"},
                "session_id": 7,
            },
        )

        db.append_detect_run_event.assert_called_once()
        self.assertEqual("domain_started", db.append_detect_run_event.call_args.kwargs["event_type"])
        payload = db.append_detect_run_event.call_args.kwargs["payload"]
        self.assertEqual("domain_pipeline", payload["task_mode"])
        self.assertEqual("detect_register", payload["step_code"])

    def test_detect_domain_step_task_finalize_emits_domain_terminal_event(self):
        db = MagicMock()
        worker = self._build_worker(db)
        worker.sensitive_words = []
        worker._set_current_task_context = MagicMock()
        worker._clear_current_task_context = MagicMock()
        worker._assert_detection_session_active = MagicMock()
        worker._change_active_domain_threads = MagicMock(return_value=1)
        worker._mark_detection_phase = MagicMock()
        worker._enqueue_running_mark = MagicMock()
        worker._run_detect_register = MagicMock(return_value=True)
        worker._get_detect_execution_order = MagicMock(return_value=["detect_register"])
        worker._build_single_step_job_result = MagicMock(
            return_value={"state": "blocked", "message": "client hold", "field_name": "register_status"}
        )
        worker._resolve_single_step_finalization = MagicMock(return_value=("blacklisted", "client hold"))
        worker._enqueue_job_finalization = MagicMock()

        worker.detect_domain(
            101,
            {"id": 101, "domain": "example.com"},
            {
                "job_item_id": 301,
                "job_id": 1,
                "job_code": "sync-overseas-26118",
                "claim_token": "token-x",
                "task_mode": "domain_pipeline",
                "step_code": "detect_register",
                "step_payload": {"step_code": "detect_register"},
                "session_id": 7,
            },
        )

        self.assertTrue(worker._enqueue_job_finalization.called)
        self.assertEqual(
            "domain_blacklisted",
            worker._enqueue_job_finalization.call_args.kwargs["event_type"],
        )
        event_payload = worker._enqueue_job_finalization.call_args.kwargs["event_payload"]
        self.assertEqual("example.com", event_payload["domain"])
        self.assertEqual("blacklisted", event_payload["status"])

    def test_start_detection_async_ignores_duplicate_start_for_same_job_and_cycle(self):
        worker = self._build_worker(_FakeDb("noop"))
        live_thread = MagicMock()
        live_thread.is_alive.return_value = True
        worker.detect_command_thread = live_thread
        worker.current_job_id = 321
        worker.current_job_code = "sync-overseas-321"
        worker.current_cycle_token = "cycle-abc"
        worker.detect_command_started_at = time.time() - 75
        worker.detect_command_last_activity_at = time.time() - 60
        worker._last_domain_result_at = time.time() - 60

        started = worker.start_detection_async(
            source="redis-control",
            control_payload={
                "job_id": 321,
                "job_code": "sync-overseas-321",
                "cycle_token": "cycle-abc",
            },
        )

        self.assertTrue(started)
        self.assertIs(worker.detect_command_thread, live_thread)
        self.assertFalse(worker.stop_requested)
        worker._update_runtime_state.assert_called()

    def test_start_detection_async_does_not_coalesce_recent_duplicate_without_live_thread(self):
        worker = self._build_worker(_FakeDb("noop"))
        worker.detect_command_thread = None
        worker.detecting = False
        worker.current_job_id = 321
        worker.current_job_code = "sync-overseas-321"
        worker.current_job_task_mode = "domain_pipeline"
        worker._last_detection_start_at = time.time() - 10
        worker._last_detection_start_source = "auto-resume"
        worker._last_detection_start_job_id = 321
        worker._last_detection_start_job_code = "sync-overseas-321"
        worker._last_detection_start_task_mode = "domain_pipeline"

        with patch("detect_worker.threading.Thread") as mock_thread:
            started = worker.start_detection_async(
                source="redis-control",
                control_payload={},
            )

        self.assertTrue(started)
        mock_thread.assert_called_once()
        worker._update_runtime_state.assert_called()

    def test_start_detection_async_does_not_coalesce_recent_duplicate_for_stale_session(self):
        worker = self._build_worker(_FakeDb("noop"))
        live_thread = MagicMock()
        live_thread.is_alive.return_value = True
        worker.detect_command_thread = live_thread
        worker.detecting = True
        worker.current_job_id = 874
        worker.current_job_code = "sync-overseas-19897"
        worker.current_job_task_mode = "domain_pipeline"
        worker._last_detection_start_at = time.time() - 10
        worker._last_detection_start_source = "auto-resume"
        worker._last_detection_start_job_id = 874
        worker._last_detection_start_job_code = "sync-overseas-19897"
        worker._last_detection_start_task_mode = "domain_pipeline"
        worker._is_stale_detect_session = MagicMock(return_value=(True, "stale"))
        worker._should_force_takeover_stale_session = MagicMock(return_value=True)

        with patch("detect_worker.threading.Thread") as mock_thread:
            started = worker.start_detection_async(
                source="auto-resume",
                control_payload={
                    "job_id": 874,
                    "job_code": "sync-overseas-19897",
                    "task_mode": "domain_pipeline",
                },
            )

        self.assertTrue(started)
        mock_thread.assert_called_once()
        worker._update_runtime_state.assert_called()

    def test_run_detection_session_clears_context_before_idle_runtime_update(self):
        worker = self._build_worker(_FakeDb("noop"))
        worker.start_detection = MagicMock(return_value=None)
        worker._consume_pending_restart_request = MagicMock(return_value=False)
        worker._maybe_recycle_idle_worker = MagicMock(return_value=False)
        control_payload = {
            "job_id": 876,
            "job_code": "sync-overseas-19961",
            "task_mode": "domain_pipeline",
            "cycle_token": "cycle-876",
        }

        worker._run_detection_session(source="redis-control", control_payload=control_payload, session_id=1)

        phase, detail = worker._update_runtime_state.call_args_list[-1].args[:2]
        kwargs = worker._update_runtime_state.call_args_list[-1].kwargs
        self.assertEqual("idle", phase)
        self.assertEqual("检测任务结束，Worker 保持待命", detail)
        self.assertNotIn("job_id", kwargs)
        self.assertNotIn("job_code", kwargs)
        self.assertEqual("", worker.current_cycle_token)
        self.assertIsNone(worker.current_job_id)
        self.assertEqual("", worker.current_job_code)

    def test_start_detection_async_schedules_pending_restart_for_explicit_pipeline_switch(self):
        worker = self._build_worker(_FakeDb("noop"))
        live_thread = MagicMock()
        live_thread.is_alive.return_value = True
        worker.detect_command_thread = live_thread
        worker.current_job_id = 867
        worker.current_job_code = "sync-overseas-19835"
        worker.current_job_task_mode = "domain_pipeline"
        worker._explicit_claim_scope_job_id = 884
        worker._explicit_claim_scope_job_code = "sync-overseas-20554"
        worker._explicit_claim_scope_until = time.time() + 120

        started = worker.start_detection_async(
            source="redis-control",
            control_payload={
                "job_id": 884,
                "job_code": "sync-overseas-20554",
                "task_mode": "domain_pipeline",
            },
        )

        self.assertTrue(started)
        self.assertTrue(worker.stop_requested)
        self.assertEqual("redis-control", worker._pending_restart_source)
        self.assertEqual(884, worker._pending_restart_payload.get("job_id"))
        worker._update_runtime_state.assert_called()

    def test_start_detection_async_ignores_overlap_handoff_when_current_session_has_active_threads(self):
        worker = self._build_worker(_FakeDb("noop"))
        live_thread = MagicMock()
        live_thread.is_alive.return_value = True
        worker.detect_command_thread = live_thread
        worker.current_job_id = 884
        worker.current_job_code = "sync-overseas-20554"
        worker.current_job_task_mode = "domain_pipeline"
        worker._explicit_claim_scope_job_id = 885
        worker._explicit_claim_scope_job_code = "sync-overseas-20592"
        worker._explicit_claim_scope_until = time.time() + 120
        worker._get_active_domain_threads.return_value = 128

        started = worker.start_detection_async(
            source="redis-control",
            control_payload={
                "job_id": 885,
                "job_code": "sync-overseas-20592",
                "task_mode": "domain_pipeline",
                "source": "overlap-handoff",
                "tail_handoff_candidate": True,
            },
        )

        self.assertTrue(started)
        self.assertFalse(worker.stop_requested)
        self.assertEqual("", worker._pending_restart_source)
        self.assertIsNone(worker._explicit_claim_scope_job_id)
        worker._update_runtime_state.assert_called()

    def test_single_machine_mode_allows_direct_fallback_for_360_and_baidu(self):
        worker = self._build_worker(_FakeDb("noop"))
        worker.proxy_config["allow_direct"] = False
        worker.single_machine_site_direct_fallback_enabled = True

        self.assertTrue(worker._allow_direct_connection_for_step("360检测"))
        self.assertTrue(worker._allow_direct_connection_for_step("百度site检测"))
        self.assertFalse(worker._allow_direct_connection_for_step("时光机检测"))

    def test_retry_budget_allows_one_pending_direct_fallback_after_proxy_failure(self):
        worker = self._build_worker(_FakeDb("noop"))
        worker.proxy_config["allow_direct"] = False
        worker.single_machine_site_direct_fallback_enabled = True
        worker.proxy_step_retry_budget_overrides = {"360检测": {"max_attempts": 2, "max_seconds": 5}}
        worker._set_step_force_direct_once("360检测", True)

        exhausted, reason = worker._check_proxy_step_retry_budget(
            "360检测",
            "example.com",
            time.perf_counter() - 15,
            1,
            {"proxy-a"},
            last_reason="proxy timeout",
        )

        self.assertFalse(exhausted)
        self.assertEqual("", reason)

    def test_retry_budget_still_exhausts_when_no_direct_fallback_is_pending(self):
        worker = self._build_worker(_FakeDb("noop"))
        worker.proxy_config["allow_direct"] = False
        worker.single_machine_site_direct_fallback_enabled = True
        worker.proxy_step_retry_budget_overrides = {"360检测": {"max_attempts": 2, "max_seconds": 5}}

        exhausted, reason = worker._check_proxy_step_retry_budget(
            "360检测",
            "example.com",
            time.perf_counter() - 15,
            1,
            {"proxy-a"},
            last_reason="proxy timeout",
        )

        self.assertTrue(exhausted)
        self.assertIn("attempts=1", reason)

    def test_auto_resume_skips_when_recent_explicit_start_signal_exists(self):
        worker = self._build_worker(MagicMock())
        worker.db.get_active_detect_job = MagicMock(
            return_value={
                "id": 1,
                "job_code": "sync-overseas-123",
                "task_mode": "domain_pipeline",
                "items_pending": 30,
                "items_claimed": 4,
                "items_running": 2,
            }
        )
        worker.detecting = False
        worker.detect_command_thread = None
        worker._last_explicit_start_signal_at = time.time()
        worker.start_detection_async = MagicMock(return_value=True)

        started = worker._resume_active_detect_job_if_needed(reason="heartbeat_autoresume")

        self.assertFalse(started)
        worker.db.get_active_detect_job.assert_not_called()
        worker.start_detection_async.assert_not_called()

    def test_auto_resume_skips_for_non_primary_worker_on_heartbeat(self):
        worker = self._build_worker(MagicMock())
        worker.db.get_active_detect_job = MagicMock(return_value={})
        worker.detecting = False
        worker.detect_command_thread = None

        with patch.object(config, "NODE_CODE", "mainland-controller-01-b"), patch.dict(
            os.environ,
            {"WORKER_PARENT_NODE_CODE": "mainland-controller-01"},
            clear=False,
        ):
            started = worker._resume_active_detect_job_if_needed(reason="heartbeat_autoresume")

        self.assertFalse(started)
        worker.db.get_active_detect_job.assert_called_once()

    def test_auto_resume_heartbeat_reuses_recent_active_job_snapshot(self):
        worker = self._build_worker(MagicMock())
        worker.db.get_active_detect_job = MagicMock(
            return_value={
                "id": 1,
                "job_code": "sync-overseas-123",
                "task_mode": "domain_pipeline",
                "items_pending": 30,
                "items_claimed": 4,
                "items_running": 2,
            }
        )
        worker.detecting = False
        worker.detect_command_thread = None
        worker.start_detection_async = MagicMock(return_value=True)

        with patch.dict(
            os.environ,
            {
                "DOMAINCHECK_AUTORESUME_ACTIVE_JOB_MIN_REFRESH_SECONDS": "60",
                "WORKER_PARENT_NODE_CODE": "mainland-controller-01",
            },
            clear=False,
        ), patch.object(config, "NODE_CODE", "mainland-controller-01-a"):
            first = worker._resume_active_detect_job_if_needed(reason="heartbeat_autoresume")
            second = worker._resume_active_detect_job_if_needed(reason="heartbeat_autoresume")

        self.assertTrue(first)
        self.assertTrue(second)
        worker.db.get_active_detect_job.assert_called_once()
        self.assertEqual(2, worker.start_detection_async.call_count)

    def test_auto_resume_skips_same_job_when_threads_already_active(self):
        worker = self._build_worker(MagicMock())
        worker.db.get_active_detect_job = MagicMock(
            return_value={
                "id": 895,
                "job_code": "sync-overseas-25520",
                "task_mode": "domain_pipeline",
                "items_pending": 73298,
                "items_claimed": 89865,
                "items_running": 31396,
                "selection_reason": "running_job_active",
            }
        )
        worker.detecting = False
        worker.detect_command_thread = None
        worker.current_job_id = 895
        worker.current_job_code = "sync-overseas-25520"
        worker.current_job_task_mode = "domain_pipeline"
        worker._get_active_domain_threads.return_value = 800
        worker.start_detection_async = MagicMock(return_value=True)

        started = worker._resume_active_detect_job_if_needed(reason="heartbeat_autoresume")

        self.assertFalse(started)
        worker.db.get_active_detect_job.assert_called_once()
        worker.start_detection_async.assert_not_called()

    def test_auto_resume_skips_same_job_during_restart_handoff_window(self):
        worker = self._build_worker(MagicMock())
        worker.db.get_active_detect_job = MagicMock(
            return_value={
                "id": 895,
                "job_code": "sync-overseas-25520",
                "task_mode": "domain_pipeline",
                "items_pending": 73298,
                "items_claimed": 89865,
                "items_running": 31396,
                "selection_reason": "running_job_active",
            }
        )
        worker.detecting = False
        worker.detect_command_thread = None
        worker.current_job_id = 895
        worker.current_job_code = "sync-overseas-25520"
        worker.current_job_task_mode = "domain_pipeline"
        worker._restart_release_handoff_job_id = 895
        worker._restart_release_handoff_job_code = "sync-overseas-25520"
        worker._restart_release_handoff_until = time.time() + 120
        worker.start_detection_async = MagicMock(return_value=True)

        started = worker._resume_active_detect_job_if_needed(reason="heartbeat_autoresume")

        self.assertFalse(started)
        worker.db.get_active_detect_job.assert_called_once()
        worker.start_detection_async.assert_not_called()

    def test_auto_resume_skips_when_explicit_claim_scope_matches_active_job(self):
        worker = self._build_worker(MagicMock())
        worker.db.get_active_detect_job = MagicMock(
            return_value={
                "id": 871,
                "job_code": "sync-overseas-19873",
                "task_mode": "domain_pipeline",
                "items_pending": 2134,
                "items_claimed": 1936,
                "items_running": 3402,
                "selection_reason": "running_job_active",
            }
        )
        worker.detecting = False
        worker.detect_command_thread = None
        worker._explicit_claim_scope_job_id = 871
        worker._explicit_claim_scope_job_code = "sync-overseas-19873"
        worker._explicit_claim_scope_until = time.time() + 120
        worker._explicit_claim_scope_source = "redis-control"
        worker.start_detection_async = MagicMock(return_value=True)

        started = worker._resume_active_detect_job_if_needed(reason="heartbeat_autoresume")

        self.assertFalse(started)
        worker.db.get_active_detect_job.assert_called_once()
        worker.start_detection_async.assert_not_called()

    def test_auto_resume_allows_non_primary_worker_for_tail_handoff_pending_job(self):
        worker = self._build_worker(MagicMock())
        worker.db.get_active_detect_job = MagicMock(
            return_value={
                "id": 885,
                "job_code": "sync-overseas-20592",
                "task_mode": "domain_pipeline",
                "items_pending": 64000,
                "items_claimed": 0,
                "items_running": 0,
                "tail_handoff_candidate": True,
                "selection_reason": "tail_handoff_pending",
            }
        )
        worker.detecting = False
        worker.detect_command_thread = None
        worker.start_detection_async = MagicMock(return_value=True)

        with patch.object(config, "NODE_CODE", "mainland-controller-01-b"), patch.dict(
            os.environ,
            {"WORKER_PARENT_NODE_CODE": "mainland-controller-01"},
            clear=False,
        ):
            started = worker._resume_active_detect_job_if_needed(reason="heartbeat_autoresume")

        self.assertTrue(started)
        worker.db.get_active_detect_job.assert_called_once()
        worker.start_detection_async.assert_called_once()

    def test_auto_resume_skips_stalled_running_job(self):
        worker = self._build_worker(MagicMock())
        worker.db.get_active_detect_job = MagicMock(
            return_value={
                "id": 876,
                "job_code": "sync-overseas-19961",
                "task_mode": "domain_pipeline",
                "items_pending": 4,
                "items_claimed": 235,
                "items_running": 266,
                "selection_reason": "running_job_stalled",
                "running_job_stalled": True,
            }
        )
        worker.db.recycle_stalled_detect_job_items = MagicMock(return_value=7)
        worker.detecting = False
        worker.detect_command_thread = None
        worker.start_detection_async = MagicMock(return_value=True)
        worker._acquire_shared_job_maintenance_lock = MagicMock(return_value="lock-token")
        worker._release_shared_job_maintenance_lock = MagicMock()

        started = worker._resume_active_detect_job_if_needed(reason="heartbeat_autoresume")

        self.assertFalse(started)
        worker.db.get_active_detect_job.assert_called_once()
        worker.db.recycle_stalled_detect_job_items.assert_called_once_with(876, stall_seconds=900)
        worker._release_shared_job_maintenance_lock.assert_called_once()
        worker.start_detection_async.assert_not_called()

    def test_pull_sync_tasks_skips_when_shared_lock_not_acquired(self):
        worker = self._build_worker(MagicMock())
        worker._acquire_shared_job_maintenance_lock = MagicMock(return_value="")
        worker._release_shared_job_maintenance_lock = MagicMock()

        with patch.object(config, "NODE_REGION", "mainland"), patch.object(config, "NODE_ROLE", "control"), patch(
            "detect_worker.urllib.request.urlopen"
        ) as mock_urlopen:
            domains = worker._pull_sync_tasks_until_available(thread_limit=1000)

        self.assertEqual([], domains)
        mock_urlopen.assert_not_called()
        worker._release_shared_job_maintenance_lock.assert_not_called()

    def test_process_pipeline_skips_when_shared_lock_not_acquired(self):
        worker = self._build_worker(MagicMock())
        worker._acquire_shared_job_maintenance_lock = MagicMock(return_value="")
        worker._release_shared_job_maintenance_lock = MagicMock()

        with patch.object(config, "NODE_REGION", "mainland"), patch.object(config, "NODE_ROLE", "control"), patch(
            "detect_worker.urllib.request.urlopen"
        ) as mock_urlopen:
            domains = worker._process_pipeline_tasks_until_available(thread_limit=1000)

        self.assertEqual([], domains)
        mock_urlopen.assert_not_called()
        worker._release_shared_job_maintenance_lock.assert_not_called()

    def test_runtime_heartbeat_does_not_fake_current_load_when_detecting_but_idle(self):
        class _SpyRedis:
            def __init__(self):
                self.calls = []

            def set(self, *args, **kwargs):
                self.calls.append((args, kwargs))
                return True

        db = MagicMock()
        worker = self._build_worker(db)
        worker.use_redis = True
        worker.redis_client = _SpyRedis()
        worker.detecting = True
        worker.running = True
        worker.stop_requested = False
        worker.proxy_last_refresh_status = "ok"
        worker.proxy_last_refresh_time = None
        worker.proxy_last_source_stats = []
        worker._active_domain_threads = 0
        worker._last_runtime_state_push_phase = ""
        worker._last_runtime_state_push_at = 0.0

        DetectWorker._update_runtime_state(worker, "running", "等待线程派发")

        register_calls = [call for call in db.mock_calls if call[0] == "register_cluster_node"]
        self.assertEqual(1, len(register_calls))
        _, args, kwargs = register_calls[0]
        self.assertEqual(config.NODE_CODE, args[0])
        self.assertEqual("busy", kwargs["status"])
        self.assertEqual(0, kwargs["current_load"])
        self.assertTrue(kwargs["metadata"]["detecting"])
        self.assertTrue(kwargs["metadata"]["waiting_for_dispatch"])
        self.assertEqual(0, kwargs["metadata"]["active_threads"])

    def test_release_node_job_items_is_throttled_between_detection_restarts(self):
        db = MagicMock()
        db.release_detect_job_items_for_node.return_value = 7
        worker = self._build_worker(db)

        first = worker._maybe_release_node_job_items(reason="test", min_interval_seconds=300)
        second = worker._maybe_release_node_job_items(reason="test", min_interval_seconds=300)

        self.assertEqual(7, first)
        self.assertEqual(0, second)
        db.release_detect_job_items_for_node.assert_called_once_with(config.NODE_CODE)

    def test_control_node_can_release_node_job_items(self):
        db = MagicMock()
        db.release_detect_job_items_for_node.return_value = 7
        worker = self._build_worker(db)

        with patch.object(config, "NODE_ROLE", "control"):
            released = worker._maybe_release_node_job_items(reason="control_start")

        self.assertEqual(7, released)
        db.release_detect_job_items_for_node.assert_called_once_with(config.NODE_CODE)

    def test_release_node_job_items_records_restart_release_handoff_for_current_job(self):
        db = MagicMock()
        db.release_detect_job_items_for_node.return_value = 7
        worker = self._build_worker(db)
        worker.current_job_id = 880
        worker.current_job_code = "sync-overseas-20249"

        released = worker._maybe_release_node_job_items(reason="start_detection", min_interval_seconds=0)

        self.assertEqual(7, released)
        self.assertEqual(880, worker._restart_release_handoff_job_id)
        self.assertEqual("sync-overseas-20249", worker._restart_release_handoff_job_code)
        self.assertGreater(worker._restart_release_handoff_until, time.time())

    def test_claim_detect_job_items_prioritizes_restart_released_handoff_before_global_queue(self):
        db = MagicMock()
        db.claim_restart_released_detect_job_items.return_value = [{"job_item_id": 1}]
        db.claim_detect_job_items.return_value = [{"job_item_id": 2}]
        worker = self._build_worker(db)
        worker.current_job_id = 880
        worker.current_job_code = "sync-overseas-20249"
        worker.current_job_task_mode = "domain_pipeline"
        worker._restart_release_handoff_job_id = 880
        worker._restart_release_handoff_job_code = "sync-overseas-20249"
        worker._restart_release_handoff_until = time.time() + 120
        worker._restart_release_handoff_reason = "start_detection"

        with patch.object(config, "NODE_CODE", "mainland-controller-01-a"), patch.dict(
            os.environ,
            {"WORKER_PARENT_NODE_CODE": "mainland-controller-01"},
            clear=False,
        ):
            rows = worker._claim_detect_job_items(limit=320, lease_seconds=900, job_id=None)

        self.assertEqual([{"job_item_id": 1}, {"job_item_id": 2}], rows)
        db.claim_restart_released_detect_job_items.assert_called_once_with(
            "mainland-controller-01-a",
            880,
            limit=128,
            lease_seconds=900,
        )
        db.claim_detect_job_items.assert_called_once_with(
            "mainland-controller-01-a",
            limit=319,
            lease_seconds=900,
            job_id=None,
            prefer_recent_jobs=False,
            preferred_recent_job_limit=8,
            preferred_recent_job_window_hours=24,
        )

    def test_claim_detect_job_items_skips_restart_handoff_query_for_non_primary_worker(self):
        db = MagicMock()
        worker = self._build_worker(db)
        worker.current_job_id = 880
        worker.current_job_code = "sync-overseas-20249"
        worker.current_job_task_mode = "domain_pipeline"
        worker._restart_release_handoff_job_id = 880
        worker._restart_release_handoff_job_code = "sync-overseas-20249"
        worker._restart_release_handoff_until = time.time() + 120
        worker._restart_release_handoff_reason = "start_detection"

        with patch.object(config, "NODE_CODE", "mainland-controller-01-b"), patch.dict(
            os.environ,
            {"WORKER_PARENT_NODE_CODE": "mainland-controller-01"},
            clear=False,
        ):
            rows = worker._claim_detect_job_items(limit=320, lease_seconds=900, job_id=None)

        self.assertEqual([], rows)
        db.claim_restart_released_detect_job_items.assert_not_called()
        db.claim_detect_job_items.assert_not_called()

    def test_mark_explicit_start_signal_records_pipeline_claim_scope(self):
        db = MagicMock()
        worker = self._build_worker(db)

        worker._mark_explicit_start_signal(
            {
                "job_id": 884,
                "job_code": "sync-overseas-20554",
                "task_mode": "domain_pipeline",
            },
            source="redis-control",
        )

        self.assertEqual(884, worker._explicit_claim_scope_job_id)
        self.assertEqual("sync-overseas-20554", worker._explicit_claim_scope_job_code)
        self.assertEqual("redis-control", worker._explicit_claim_scope_source)
        self.assertGreater(worker._explicit_claim_scope_until, time.time())

    def test_current_scoped_claim_job_id_prefers_explicit_pipeline_scope(self):
        db = MagicMock()
        worker = self._build_worker(db)
        worker.current_job_id = 867
        worker.current_job_code = "sync-overseas-19835"
        worker.current_job_task_mode = "domain_pipeline"
        worker._explicit_claim_scope_job_id = 884
        worker._explicit_claim_scope_job_code = "sync-overseas-20554"
        worker._explicit_claim_scope_until = time.time() + 120

        self.assertEqual(884, worker._current_scoped_claim_job_id())

    def test_explicit_claim_scope_window_expires(self):
        db = MagicMock()
        worker = self._build_worker(db)
        worker._explicit_claim_scope_job_id = 884
        worker._explicit_claim_scope_job_code = "sync-overseas-20554"
        worker._explicit_claim_scope_until = time.time() - 1

        self.assertIsNone(worker._explicit_claim_scope_job_id_active())
        self.assertIsNone(worker._explicit_claim_scope_job_id)

    def test_claim_detect_job_items_honors_explicit_pipeline_scope(self):
        db = MagicMock()
        db.claim_detect_job_items.return_value = [{"job_item_id": 9}]
        worker = self._build_worker(db)
        worker._explicit_claim_scope_job_id = 884
        worker._explicit_claim_scope_job_code = "sync-overseas-20554"
        worker._explicit_claim_scope_until = time.time() + 120

        with patch.object(config, "NODE_CODE", "mainland-controller-01-da"):
            rows = worker._claim_detect_job_items(limit=250, lease_seconds=900, job_id=None)

        self.assertEqual([{"job_item_id": 9}], rows)
        db.claim_detect_job_items.assert_called_once_with(
            "mainland-controller-01-da",
            limit=250,
            lease_seconds=900,
            job_id=884,
            prefer_recent_jobs=False,
            preferred_recent_job_limit=8,
            preferred_recent_job_window_hours=24,
        )

    def test_shared_job_maintenance_lock_uses_single_worker_with_same_scope(self):
        redis_client = _FakeRedis()
        db = MagicMock()
        worker_a = self._build_worker(db)
        worker_a.use_redis = True
        worker_a.redis_client = redis_client
        worker_b = self._build_worker(db)
        worker_b.use_redis = True
        worker_b.redis_client = redis_client

        with patch.object(config, "NODE_CODE", "mainland-controller-01-a"), patch.dict(
            os.environ,
            {"WORKER_PARENT_NODE_CODE": "mainland-controller-01"},
            clear=False,
        ):
            token_a = worker_a._acquire_shared_job_maintenance_lock("recycle-expired-job-items", ttl_seconds=10)
        with patch.object(config, "NODE_CODE", "mainland-controller-01-b"), patch.dict(
            os.environ,
            {"WORKER_PARENT_NODE_CODE": "mainland-controller-01"},
            clear=False,
        ):
            token_b = worker_b._acquire_shared_job_maintenance_lock("recycle-expired-job-items", ttl_seconds=10)

        self.assertTrue(token_a)
        self.assertEqual("", token_b)

    def test_control_node_skips_local_auto_resume(self):
        worker = self._build_worker(MagicMock())
        worker.db.get_active_detect_job = MagicMock(return_value={"id": 1, "job_code": "sync-overseas-123"})
        worker.detecting = False
        worker.detect_command_thread = None

        with patch.object(config, "NODE_ROLE", "control"):
            started = worker._resume_active_detect_job_if_needed(reason="service_runtime_bootstrap")

        self.assertFalse(started)
        worker.db.get_active_detect_job.assert_not_called()

    def test_control_node_can_auto_resume_when_override_enabled(self):
        worker = self._build_worker(MagicMock())
        worker.runtime_settings = {"control_node_autoresume_enabled": True}
        worker.db.get_active_detect_job = MagicMock(
            return_value={
                "id": 1,
                "job_code": "sync-overseas-123",
                "task_mode": "domain_pipeline",
                "items_pending": 30,
                "items_claimed": 4,
                "items_running": 2,
            }
        )
        worker.detecting = False
        worker.detect_command_thread = None
        worker.start_detection_async = MagicMock(return_value=True)

        with patch.object(config, "NODE_ROLE", "control"):
            started = worker._resume_active_detect_job_if_needed(reason="service_runtime_bootstrap")

        self.assertTrue(started)
        worker.db.get_active_detect_job.assert_called_once()
        worker.start_detection_async.assert_called_once()

    def test_bootstrap_autoresume_lock_allows_only_one_worker_for_same_job(self):
        redis_client = _FakeRedis()
        db = MagicMock()
        active_job = {
            "id": 895,
            "job_code": "sync-overseas-25520",
            "task_mode": "domain_pipeline",
            "items_pending": 73298,
            "items_claimed": 89865,
            "items_running": 31396,
            "selection_reason": "running_job_active",
        }
        worker_a = self._build_worker(db)
        worker_a.use_redis = True
        worker_a.redis_client = redis_client
        worker_a.runtime_settings = {"control_node_autoresume_enabled": True}
        worker_a.db.get_active_detect_job = MagicMock(return_value=active_job)
        worker_a.detecting = False
        worker_a.start_detection_async = MagicMock(return_value=True)
        worker_b = self._build_worker(db)
        worker_b.use_redis = True
        worker_b.redis_client = redis_client
        worker_b.runtime_settings = {"control_node_autoresume_enabled": True}
        worker_b.db.get_active_detect_job = MagicMock(return_value=active_job)
        worker_b.detecting = False
        worker_b.start_detection_async = MagicMock(return_value=True)

        with patch.object(config, "NODE_ROLE", "control"), patch.object(config, "NODE_CODE", "mainland-controller-01-a"), patch.dict(
            os.environ,
            {"WORKER_PARENT_NODE_CODE": "mainland-controller-01"},
            clear=False,
        ):
            started_a = worker_a._resume_active_detect_job_if_needed(reason="service_runtime_bootstrap")
        with patch.object(config, "NODE_ROLE", "control"), patch.object(config, "NODE_CODE", "mainland-controller-01-b"), patch.dict(
            os.environ,
            {"WORKER_PARENT_NODE_CODE": "mainland-controller-01"},
            clear=False,
        ):
            started_b = worker_b._resume_active_detect_job_if_needed(reason="service_runtime_bootstrap")

        self.assertTrue(started_a)
        self.assertFalse(started_b)
        worker_a.start_detection_async.assert_called_once()
        worker_b.start_detection_async.assert_not_called()

    def test_same_job_recent_start_skip_detail_never_goes_negative(self):
        worker = self._build_worker(MagicMock())
        worker.current_job_id = 895
        worker.current_job_code = "sync-overseas-25520"
        worker._last_detection_start_at = time.time() + 900

        should_skip, detail = worker._should_skip_same_job_autoresume(
            active_job={"id": 895, "job_code": "sync-overseas-25520"},
            now_ts=time.time(),
        )

        self.assertTrue(should_skip)
        self.assertEqual("same_job_recent_start:0s", detail)

    def test_claim_detect_job_items_prefers_recent_jobs_when_runtime_enabled(self):
        db = MagicMock()
        db.claim_detect_job_items.return_value = []
        worker = self._build_worker(db)
        worker.runtime_settings = {
            "claim_recent_jobs_first": True,
            "claim_recent_jobs_limit": 6,
            "claim_recent_jobs_window_hours": 72,
        }

        worker._claim_detect_job_items(limit=320, lease_seconds=900, job_id=None)

        db.claim_detect_job_items.assert_called_once_with(
            config.NODE_CODE,
            limit=320,
            lease_seconds=900,
            job_id=None,
            prefer_recent_jobs=True,
            preferred_recent_job_limit=6,
            preferred_recent_job_window_hours=72,
        )

    def test_claim_detect_job_items_does_not_enable_recent_priority_for_scoped_job(self):
        db = MagicMock()
        db.claim_detect_job_items.return_value = []
        worker = self._build_worker(db)
        worker.runtime_settings = {
            "claim_recent_jobs_first": True,
            "claim_recent_jobs_limit": 6,
            "claim_recent_jobs_window_hours": 72,
        }

        worker._claim_detect_job_items(limit=320, lease_seconds=900, job_id=99)

        db.claim_detect_job_items.assert_called_once_with(
            config.NODE_CODE,
            limit=320,
            lease_seconds=900,
            job_id=99,
            prefer_recent_jobs=False,
            preferred_recent_job_limit=6,
            preferred_recent_job_window_hours=72,
        )

    def test_resolve_sync_pull_limit_scales_with_process_pool_capacity(self):
        worker = self._build_worker(MagicMock())
        worker.process_count = 60
        worker.refresh_process_count_runtime = MagicMock(return_value=60)

        pull_limit = worker._resolve_sync_pull_limit(1000)

        self.assertEqual(120000, pull_limit)

    def test_proxy_auth_failure_quarantines_whole_source_group(self):
        worker = self._build_worker(_FakeDb("noop"))
        bad_proxy = {"http": "http://bad-proxy", "https": "http://bad-proxy"}
        good_proxy = {"http": "http://good-proxy", "https": "http://good-proxy"}
        worker.proxy_pool = [
            {"proxy": bad_proxy, "usage_count": 0, "source_tag": "D"},
            {"proxy": good_proxy, "usage_count": 0, "source_tag": "E"},
        ]
        worker.proxy_source_by_key[worker._proxy_key(bad_proxy)] = "D"
        worker.proxy_source_by_key[worker._proxy_key(good_proxy)] = "E"

        source_tag, failure_count, cooldown_seconds, removed = worker._mark_proxy_source_failure(
            bad_proxy,
            "Tunnel connection failed: 407 Proxy Authentication Required",
        )

        self.assertEqual("D", source_tag)
        self.assertEqual(1, failure_count)
        self.assertGreaterEqual(cooldown_seconds, 90)
        self.assertEqual(1, removed)
        self.assertTrue(worker._is_proxy_source_quarantined("D"))
        self.assertEqual(
            [{"proxy": good_proxy, "usage_count": 0, "source_tag": "E"}],
            worker.proxy_pool,
        )

    def test_get_proxies_skips_quarantined_entries_without_dropping_pool_items(self):
        worker = self._build_worker(_FakeDb("noop"))
        bad_proxy = {"http": "http://bad-proxy", "https": "http://bad-proxy"}
        good_proxy = {"http": "http://good-proxy", "https": "http://good-proxy"}
        worker.proxy_pool = [
            {"proxy": bad_proxy, "usage_count": 0, "source_tag": "B"},
            {"proxy": good_proxy, "usage_count": 0, "source_tag": "C"},
        ]
        worker.proxy_quarantine_until[worker._proxy_key(bad_proxy)] = time.time() + 60
        worker._schedule_proxy_refresh_if_needed = MagicMock()

        selected = worker.get_proxies()

        self.assertEqual(good_proxy, selected)
        self.assertEqual(2, len(worker.proxy_pool))
        self.assertEqual(
            [entry["proxy"] for entry in worker.proxy_pool],
            [bad_proxy, good_proxy],
        )
        self.assertEqual(1, worker.proxy_pool[1]["usage_count"])

    def test_get_proxies_scans_multiple_batches_until_it_finds_usable_proxy(self):
        worker = self._build_worker(_FakeDb("noop"))
        worker._proxy_pool_scan_batch_size = MagicMock(return_value=2)
        proxies = [
            {"http": f"http://proxy-{idx}", "https": f"http://proxy-{idx}"}
            for idx in range(5)
        ]
        worker.proxy_pool = [
            {"proxy": proxy, "usage_count": 0, "source_tag": f"S{idx}"}
            for idx, proxy in enumerate(proxies)
        ]
        excluded_keys = {worker._proxy_key(proxy) for proxy in proxies[:4]}
        worker._schedule_proxy_refresh_if_needed = MagicMock()

        selected = worker.get_proxies(excluded_proxy_keys=excluded_keys)

        self.assertEqual(proxies[4], selected)
        self.assertEqual(5, len(worker.proxy_pool))
        self.assertEqual(1, worker.proxy_pool[-1]["usage_count"])

    def test_prepare_proxy_fetch_plan_respects_pool_shortage_instead_of_overfetching(self):
        worker = self._build_worker(_FakeDb("noop"))

        plan, meta = worker._prepare_proxy_fetch_plan(
            [
                "http://provider.local/get?group=B",
                "http://provider.local/get?group=C",
                "http://provider.local/get?group=D",
                "http://provider.local/get?group=E",
                "http://provider.local/get?group=F",
                "http://provider.local/get?group=G",
            ],
            current_pool_size=1000,
        )

        self.assertEqual(1, meta["rounds"])
        self.assertGreaterEqual(meta["target_total"], 840)
        self.assertLessEqual(meta["target_total"], 1200)
        self.assertEqual(6, len(plan))

    def test_prepare_proxy_fetch_plan_caps_active_rounds(self):
        worker = self._build_worker(_FakeDb("noop"))

        plan, meta = worker._prepare_proxy_fetch_plan(
            [
                "http://provider.local/get?group=B",
                "http://provider.local/get?group=C",
                "http://provider.local/get?group=D",
                "http://provider.local/get?group=E",
                "http://provider.local/get?group=F",
                "http://provider.local/get?group=G",
            ],
            current_pool_size=0,
        )

        self.assertLessEqual(meta["rounds"], 2)
        self.assertEqual(meta["rounds"] * 6, len(plan))

    def test_prepare_proxy_fetch_plan_scales_down_target_when_process_count_is_high(self):
        worker = self._build_worker(_FakeDb("noop"))
        worker.thread_count = 1000
        worker.process_count = 80
        worker.refresh_process_count_runtime = MagicMock(return_value=80)

        plan, meta = worker._prepare_proxy_fetch_plan(
            [
                "http://provider.local/get?group=B",
                "http://provider.local/get?group=C",
                "http://provider.local/get?group=D",
                "http://provider.local/get?group=E",
                "http://provider.local/get?group=F",
                "http://provider.local/get?group=G",
            ],
            current_pool_size=150,
        )

        self.assertEqual(80, meta["process_count"])
        self.assertLessEqual(meta["target_total"], 360)
        self.assertLessEqual(meta["batch_size"], 60)
        self.assertEqual(6, len(plan))

    def test_refresh_proxy_pool_reuses_shared_snapshot_when_refresh_lock_is_busy(self):
        worker = self._build_worker(_FakeDb("noop"))
        worker.use_redis = True
        worker.redis_client = _FakeRedis()
        worker.process_count = 80
        worker.refresh_process_count_runtime = MagicMock(return_value=80)
        worker.proxy_config = {
            "proxy_enable": True,
            "allow_direct": True,
            "proxy_urls": ["http://provider.local/get?group=B"],
        }
        local_proxy = {"http": "http://local-proxy", "https": "http://local-proxy"}
        shared_proxy = {"http": "http://shared-proxy", "https": "http://shared-proxy"}
        worker.proxy_pool = [
            {"proxy": local_proxy, "usage_count": 0, "source_tag": "L", "expire_at_ts": time.time() + 120}
        ]

        with patch.object(config, "NODE_CODE", "mainland-controller-01-u"):
            with patch.dict(os.environ, {"WORKER_PARENT_NODE_CODE": "mainland-controller-01"}, clear=False):
                worker.redis_client.set(
                    worker._shared_proxy_pool_key(),
                    json.dumps(
                        {
                            "node_code": "mainland-controller-01",
                            "owner": "mainland-controller-01-k",
                            "refreshed_at_ts": time.time(),
                            "source_count": 6,
                            "raw_items": 180,
                            "available_count": 1,
                            "proxy_pool": [
                                {
                                    "proxy": shared_proxy,
                                    "usage_count": 0,
                                    "source_tag": "S",
                                    "expire_at_ts": time.time() + 120,
                                }
                            ],
                        },
                        ensure_ascii=False,
                    ),
                    ex=30,
                )
                worker.redis_client.set(worker._shared_proxy_refresh_lock_key(), "other-owner", ex=10)

                worker.refresh_proxy_pool()

        self.assertEqual(1, len(worker.proxy_pool))
        self.assertEqual(shared_proxy, worker.proxy_pool[0]["proxy"])
        self.assertIn("共享代理快照", worker.proxy_last_refresh_status)

    def test_refresh_proxy_pool_publishes_shared_snapshot_after_success(self):
        worker = self._build_worker(_FakeDb("noop"))
        worker.use_redis = True
        worker.redis_client = _FakeRedis()
        worker.process_count = 80
        worker.refresh_process_count_runtime = MagicMock(return_value=80)
        worker.proxy_config = {
            "proxy_enable": True,
            "allow_direct": True,
            "proxy_urls": ["http://provider.local/get?group=B"],
        }

        class _FakeResponse:
            status_code = 200

            def json(self):
                return {
                    "list": [
                        {
                            "ip": "1.1.1.1",
                            "port": "8080",
                            "expire_at": time.time() + 120,
                        }
                    ]
                }

        with patch.object(config, "NODE_CODE", "mainland-controller-01-u"):
            with patch.dict(os.environ, {"WORKER_PARENT_NODE_CODE": "mainland-controller-01"}, clear=False):
                with patch("requests.get", return_value=_FakeResponse()):
                    worker.refresh_proxy_pool()
                raw_payload = worker.redis_client.get(worker._shared_proxy_pool_key())

        self.assertTrue(raw_payload)
        payload = json.loads(raw_payload)
        self.assertEqual("mainland-controller-01-u", payload["owner"])
        self.assertEqual(1, payload["available_count"])

    def test_refresh_proxy_pool_prefers_shared_snapshot_when_local_pool_lags(self):
        worker = self._build_worker(_FakeDb("noop"))
        worker.use_redis = True
        worker.redis_client = _FakeRedis()
        worker.process_count = 80
        worker.refresh_process_count_runtime = MagicMock(return_value=80)
        worker.thread_count = 1000
        worker.proxy_config = {
            "proxy_enable": True,
            "allow_direct": True,
            "proxy_urls": ["http://provider.local/get?group=B"],
        }
        worker.proxy_pool = [
            {"proxy": {"http": f"http://local-{idx}", "https": f"http://local-{idx}"}, "usage_count": 0, "source_tag": "L", "expire_at_ts": time.time() + 120}
            for idx in range(20)
        ]
        shared_entries = [
            {"proxy": {"http": f"http://shared-{idx}", "https": f"http://shared-{idx}"}, "usage_count": 0, "source_tag": "S", "expire_at_ts": time.time() + 120}
            for idx in range(80)
        ]

        with patch.object(config, "NODE_CODE", "mainland-controller-01-u"):
            with patch.dict(os.environ, {"WORKER_PARENT_NODE_CODE": "mainland-controller-01"}, clear=False):
                worker.redis_client.set(
                    worker._shared_proxy_pool_key(),
                    json.dumps(
                        {
                            "node_code": "mainland-controller-01",
                            "owner": "mainland-controller-01-k",
                            "refreshed_at_ts": time.time(),
                            "source_count": 6,
                            "raw_items": 120,
                            "available_count": len(shared_entries),
                            "proxy_pool": shared_entries,
                        },
                        ensure_ascii=False,
                    ),
                    ex=30,
                )

                worker.refresh_proxy_pool()

        self.assertEqual(len(shared_entries), len(worker.proxy_pool))
        self.assertIn("共享代理快照", worker.proxy_last_refresh_status)

    def test_schedule_proxy_refresh_skips_empty_pool_during_cooldown(self):
        worker = self._build_worker(_FakeDb("noop"))
        worker.proxy_next_refresh_time = time.time() + 10

        with patch("detect_worker.threading.Thread") as mock_thread:
            worker._schedule_proxy_refresh_if_needed(0)

        mock_thread.assert_not_called()

    def test_schedule_proxy_refresh_skips_low_water_pool_during_cooldown(self):
        worker = self._build_worker(_FakeDb("noop"))
        worker.process_count = 80
        worker.refresh_process_count_runtime = MagicMock(return_value=80)
        worker.proxy_next_refresh_time = time.time() + 10

        with patch("detect_worker.threading.Thread") as mock_thread:
            worker._schedule_proxy_refresh_if_needed(150)

        mock_thread.assert_not_called()

    def test_proxy_refresh_threshold_scales_down_for_multi_process_node(self):
        worker = self._build_worker(_FakeDb("noop"))
        worker.thread_count = 1000
        worker._get_active_domain_threads = MagicMock(return_value=900)

        self.assertEqual(2700, worker._proxy_refresh_threshold())

        worker.process_count = 80
        worker.refresh_process_count_runtime = MagicMock(return_value=80)

        self.assertLess(worker._proxy_refresh_threshold(), 400)

    def test_trigger_proxy_refresh_respects_cooldown_without_reset(self):
        worker = self._build_worker(_FakeDb("noop"))
        worker.proxy_config["proxy_urls"] = ["http://provider.local/get?group=B&count=1"]
        worker.proxy_next_refresh_time = time.time() + 10
        worker.trigger_proxy_refresh = DetectWorker.trigger_proxy_refresh.__get__(worker, DetectWorker)

        with patch("detect_worker.threading.Thread") as mock_thread:
            scheduled = worker.trigger_proxy_refresh(reason="cooldown-check")

        self.assertFalse(scheduled)
        mock_thread.assert_not_called()

    def test_trigger_proxy_refresh_debounces_config_update_burst(self):
        worker = self._build_worker(_FakeDb("noop"))
        worker.proxy_config["proxy_urls"] = ["http://provider.local/get?group=B&count=1"]
        worker.trigger_proxy_refresh = DetectWorker.trigger_proxy_refresh.__get__(worker, DetectWorker)

        with patch("detect_worker.threading.Thread") as mock_thread:
            first = worker.trigger_proxy_refresh(reason="config_update:thread_count")
            second = worker.trigger_proxy_refresh(reason="config_update:runtime_settings")

        self.assertTrue(first)
        self.assertFalse(second)
        mock_thread.assert_called_once()

    def test_config_update_proxy_refresh_ignores_runtime_settings_only_changes(self):
        worker = self._build_worker(_FakeDb("noop"))
        worker.proxy_config = {
            "proxy_enable": True,
            "allow_direct": True,
            "proxy_urls": ["http://provider.local/get?group=B"],
        }
        worker.runtime_settings = {"worker_log_sync_mode": "key"}
        worker.proxy_pool = [
            {"proxy": {"http": "http://cached", "https": "http://cached"}, "usage_count": 0, "source_tag": "L", "expire_at_ts": time.time() + 120}
            for _ in range(160)
        ]
        should_refresh, reset_cooldown = worker._should_trigger_proxy_refresh_for_config_update(
            config_type="runtime_settings",
            previous_proxy_config=dict(worker.proxy_config),
            previous_thread_count=worker.thread_count,
            current_thread_count=worker.thread_count,
        )

        self.assertFalse(should_refresh)
        self.assertFalse(reset_cooldown)

    def test_proxy_item_expiry_respects_recent_grace_window(self):
        worker = self._build_worker(_FakeDb("noop"))
        expired_recently = {"expire_at": time.time() - 30}

        self.assertTrue(worker._is_proxy_item_expired(expired_recently))
        self.assertFalse(worker._is_proxy_item_expired(expired_recently, grace_seconds=90))
        self.assertTrue(worker._is_proxy_item_expired(expired_recently, grace_seconds=10))

    def test_proxy_expire_clock_skew_hint_detects_large_positive_offset(self):
        worker = self._build_worker(_FakeDb("noop"))
        base_now = 1_700_000_000.0
        proxy_items = [
            {"ip": "1.1.1.1", "port": "8001", "expire_at": base_now - 46000},
            {"ip": "1.1.1.2", "port": "8002", "expire_at": base_now - 45990},
            {"ip": "1.1.1.3", "port": "8003", "expire_at": base_now - 45980},
            {"ip": "1.1.1.4", "port": "8004", "expire_at": base_now - 45970},
            {"ip": "1.1.1.5", "port": "8005", "expire_at": base_now - 45960},
            {"ip": "1.1.1.6", "port": "8006", "expire_at": base_now - 45950},
            {"ip": "1.1.1.7", "port": "8007", "expire_at": base_now - 45940},
            {"ip": "1.1.1.8", "port": "8008", "expire_at": base_now - 45930},
            {"ip": "1.1.1.9", "port": "8009", "expire_at": base_now - 45920},
            {"ip": "1.1.1.10", "port": "8010", "expire_at": base_now - 45910},
            {"ip": "1.1.1.11", "port": "8011", "expire_at": base_now - 45900},
            {"ip": "1.1.1.12", "port": "8012", "expire_at": base_now - 45890},
        ]

        hint = worker._summarize_proxy_expire_clock_skew(proxy_items, now_ts=base_now)

        self.assertIsNotNone(hint)
        self.assertEqual(12, hint["timed_proxy_count"])
        self.assertEqual(12, hint["expired_timed_count"])
        self.assertGreater(hint["clock_ahead_seconds"], 45_000)

    def test_proxy_expire_clock_skew_hint_ignores_normally_fresh_entries(self):
        worker = self._build_worker(_FakeDb("noop"))
        base_now = 1_700_000_000.0
        proxy_items = [
            {"ip": "2.2.2.1", "port": "9001", "expire_at": base_now + 120},
            {"ip": "2.2.2.2", "port": "9002", "expire_at": base_now + 180},
            {"ip": "2.2.2.3", "port": "9003", "expire_at": base_now + 240},
            {"ip": "2.2.2.4", "port": "9004", "expire_at": base_now + 300},
            {"ip": "2.2.2.5", "port": "9005", "expire_at": base_now + 360},
            {"ip": "2.2.2.6", "port": "9006", "expire_at": base_now + 420},
            {"ip": "2.2.2.7", "port": "9007", "expire_at": base_now + 480},
            {"ip": "2.2.2.8", "port": "9008", "expire_at": base_now + 540},
            {"ip": "2.2.2.9", "port": "9009", "expire_at": base_now + 600},
            {"ip": "2.2.2.10", "port": "9010", "expire_at": base_now + 660},
            {"ip": "2.2.2.11", "port": "9011", "expire_at": base_now + 720},
            {"ip": "2.2.2.12", "port": "9012", "expire_at": base_now + 780},
        ]

        hint = worker._summarize_proxy_expire_clock_skew(proxy_items, now_ts=base_now)

        self.assertIsNone(hint)

    def test_recently_expired_backfill_only_kicks_in_when_pool_is_too_small(self):
        worker = self._build_worker(_FakeDb("noop"))
        worker.thread_count = 2000
        worker._get_active_domain_threads = MagicMock(return_value=1800)
        stale_entries = [
            {
                "proxy": {"http": f"http://stale-{idx}", "https": f"http://stale-{idx}"},
                "usage_count": 0,
                "source_tag": f"S{idx % 3}",
            }
            for idx in range(200)
        ]

        merged_entries, added_count, floor = worker._backfill_recently_expired_proxy_entries(
            [
                {
                    "proxy": {"http": "http://fresh-1", "https": "http://fresh-1"},
                    "usage_count": 0,
                    "source_tag": "F",
                }
            ],
            stale_entries,
        )

        self.assertGreater(floor, 1)
        self.assertEqual(floor, len(merged_entries))
        self.assertEqual(floor - 1, added_count)

        healthy_pool = [
            {
                "proxy": {"http": f"http://fresh-{idx}", "https": f"http://fresh-{idx}"},
                "usage_count": 0,
                "source_tag": "F",
            }
            for idx in range(floor + 5)
        ]
        merged_entries, added_count, reused_floor = worker._backfill_recently_expired_proxy_entries(
            healthy_pool,
            stale_entries,
        )

        self.assertEqual(0, added_count)
        self.assertEqual(reused_floor, floor)
        self.assertEqual(len(healthy_pool), len(merged_entries))

    def test_refresh_proxy_pool_can_reuse_recently_expired_entries_without_unbound_local_error(self):
        worker = self._build_worker(_FakeDb("noop"))
        worker.thread_count = 1200
        worker.proxy_config = {
            "proxy_enable": True,
            "allow_direct": True,
            "proxy_urls": ["http://provider.local/get?group=B&count=1"],
        }

        class _Response:
            status_code = 200

            @staticmethod
            def json():
                return [
                    {
                        "ip": "1.1.1.1",
                        "port": "8080",
                        "expire_at": time.time() - 30,
                    }
                ]

        with patch("requests.get", return_value=_Response()):
            worker.refresh_proxy_pool()

    def test_pending_control_key_scopes_to_current_node_code(self):
        with patch.object(config, "NODE_CODE", "mainland-controller-01-a"):
            self.assertEqual(
                "domain_tool:worker_pending_command:mainland-controller-01-a",
                pending_control_key(),
            )

    def test_control_targets_current_worker_rejects_other_instance(self):
        with patch.object(config, "NODE_CODE", "mainland-controller-01-a"):
            self.assertTrue(control_targets_current_worker({"action": "start_detection"}))
            self.assertTrue(control_targets_current_worker({"target_node_code": "mainland-controller-01-a"}))
            self.assertFalse(control_targets_current_worker({"target_node_code": "mainland-controller-01-b"}))

    def test_consume_pending_control_command_prefers_instance_key(self):
        worker = DetectWorker.__new__(DetectWorker)
        worker.use_redis = True
        worker._handle_control_message = MagicMock()

        class _FakeRedis:
            def __init__(self):
                self.values = {
                    "domain_tool:worker_pending_command:mainland-controller-01-a": '{"action":"start_detection","target_node_code":"mainland-controller-01-a"}',
                    "domain_tool:worker_pending_command": '{"action":"start_detection"}',
                }

            def getdel(self, key):
                return self.values.pop(key, None)

            def get(self, key):
                return self.values.get(key)

            def delete(self, key):
                self.values.pop(key, None)

        worker.redis_client = _FakeRedis()

        with patch.object(config, "NODE_CODE", "mainland-controller-01-a"):
            worker._consume_pending_control_command()

        worker._handle_control_message.assert_called_once_with(
            '{"action":"start_detection","target_node_code":"mainland-controller-01-a"}'
        )

    def test_handle_control_message_ignores_command_for_other_instance(self):
        worker = DetectWorker.__new__(DetectWorker)
        worker.use_redis = False
        worker.redis_client = None
        worker._acknowledge_pending_control_command = MagicMock()
        worker._remember_ignored_targeted_control = MagicMock()
        worker.start_detection_async = MagicMock()
        worker.request_stop_detection = MagicMock()

        with patch.object(config, "NODE_CODE", "mainland-controller-01-a"):
            worker._handle_control_message(
                '{"action":"start_detection","target_node_code":"mainland-controller-01-b"}'
            )

        worker._acknowledge_pending_control_command.assert_not_called()
        worker._remember_ignored_targeted_control.assert_called_once()
        worker.start_detection_async.assert_not_called()

    def test_handle_control_message_records_ignored_targeted_job_for_other_instance(self):
        worker = self._build_worker(MagicMock())

        with patch.object(config, "NODE_CODE", "mainland-controller-01-a"):
            worker._handle_control_message(
                {
                    "action": "start_detection",
                    "job_id": 870,
                    "job_code": "sync-overseas-19868",
                    "target_node_codes": ["mainland-controller-01-do", "mainland-controller-01-cz"],
                }
            )

        self.assertEqual(870, worker._ignored_targeted_job_id)
        self.assertEqual("sync-overseas-19868", worker._ignored_targeted_job_code)
        self.assertEqual(
            "mainland-controller-01-do,mainland-controller-01-cz",
            worker._ignored_targeted_job_targets,
        )

    def test_auto_resume_skips_when_active_job_matches_recently_ignored_targeted_job(self):
        worker = self._build_worker(MagicMock())
        worker._load_active_detect_job_snapshot = MagicMock(
            return_value={
                "id": 870,
                "job_code": "sync-overseas-19868",
                "task_mode": "domain_pipeline",
                "items_pending": 1200,
                "items_claimed": 949,
                "items_running": 6315,
                "selection_reason": "running_job_active",
            }
        )
        worker.start_detection_async = MagicMock(return_value=True)
        worker._ignored_targeted_job_id = 870
        worker._ignored_targeted_job_code = "sync-overseas-19868"
        worker._ignored_targeted_job_until = time.time() + 120
        worker._ignored_targeted_job_targets = "mainland-controller-01-do,mainland-controller-01-cz"

        started = worker._resume_active_detect_job_if_needed(reason="heartbeat_autoresume")

        self.assertFalse(started)
        worker.start_detection_async.assert_not_called()

    def test_proxy_direct_fallback_grace_shrinks_for_shortage_steps(self):
        worker = self._build_worker(_FakeDb("noop"))
        worker.proxy_direct_fallback_grace_overrides = {
            "注册状态检测": 0.08,
            "360检测": 0.12,
            "站长之家检测": 0.12,
        }
        worker.proxy_last_refresh_status = "代理源暂时冷却中，稍后继续补货"

        self.assertEqual(0.05, worker._proxy_direct_fallback_grace_for_step("注册状态检测"))
        self.assertEqual(0.05, worker._proxy_direct_fallback_grace_for_step("360检测"))
        self.assertEqual(0.05, worker._proxy_direct_fallback_grace_for_step("站长之家检测"))

    def test_run_detect_register_prefers_direct_when_proxy_source_is_short(self):
        worker = self._build_worker(_FakeDb("noop"))
        worker.proxy_last_refresh_status = "代理源暂时冷却中，稍后继续补货"
        worker._check_proxy_step_retry_budget = MagicMock(return_value=(False, ""))
        worker._remaining_proxy_step_retry_budget_seconds = MagicMock(return_value=2.0)
        worker._is_current_single_step_task = MagicMock(return_value=False)
        worker._clear_proxy_failure = MagicMock()
        worker._clear_proxy_source_failure = MagicMock()
        worker._get_proxy_for_step = MagicMock(return_value={"http": "http://proxy", "https": "http://proxy"})
        worker.db.update_domain_register_result = MagicMock()

        with patch("detect_worker.register.check_register", return_value=(3, "")) as mock_check:
            result = worker._run_detect_register(1, {"source_type": 0}, "example.com")

        self.assertTrue(result)
        worker._get_proxy_for_step.assert_not_called()
        self.assertIsNone(mock_check.call_args.args[2])

    def test_run_detect_register_single_machine_mode_allows_direct_even_when_global_direct_disabled(self):
        worker = self._build_worker(_FakeDb("noop"))
        worker.proxy_config = {"proxy_enable": True, "allow_direct": False}
        worker.register_single_machine_mode_enabled = True
        worker.register_single_machine_direct_streak_attempts = 2
        worker._check_proxy_step_retry_budget = MagicMock(return_value=(False, ""))
        worker._remaining_proxy_step_retry_budget_seconds = MagicMock(return_value=2.0)
        worker._is_current_single_step_task = MagicMock(return_value=False)
        worker._clear_proxy_failure = MagicMock()
        worker._clear_proxy_source_failure = MagicMock()
        worker._get_proxy_for_step = MagicMock(return_value={"http": "http://proxy", "https": "http://proxy"})
        worker.db.update_domain_register_result = MagicMock()

        with patch("detect_worker.register.check_register", return_value=(3, "")) as mock_check:
            result = worker._run_detect_register(1, {"source_type": 0}, "example.com")

        self.assertTrue(result)
        worker._get_proxy_for_step.assert_not_called()
        self.assertIsNone(mock_check.call_args.args[2])

    def test_run_detect_register_single_machine_mode_retries_direct_before_proxy(self):
        worker = self._build_worker(_FakeDb("noop"))
        worker.proxy_config = {"proxy_enable": True, "allow_direct": False}
        worker.register_single_machine_mode_enabled = True
        worker.register_single_machine_direct_streak_attempts = 2
        worker._check_proxy_step_retry_budget = MagicMock(return_value=(False, ""))
        worker._remaining_proxy_step_retry_budget_seconds = MagicMock(return_value=2.0)
        worker._is_current_single_step_task = MagicMock(return_value=False)
        worker._clear_proxy_failure = MagicMock()
        worker._clear_proxy_source_failure = MagicMock()
        worker._get_proxy_for_step = MagicMock(return_value={"http": "http://proxy", "https": "http://proxy"})
        worker.db.update_domain_register_result = MagicMock()

        with patch(
            "detect_worker.register.check_register",
            side_effect=[RuntimeError("direct timeout"), (3, "")],
        ) as mock_check:
            result = worker._run_detect_register(1, {"source_type": 0}, "example.com")

        self.assertTrue(result)
        self.assertEqual(2, mock_check.call_count)
        self.assertTrue(all(call.args[2] is None for call in mock_check.call_args_list))
        worker._get_proxy_for_step.assert_not_called()

    def test_run_detect_register_direct_error_keeps_fast_direct_retry_during_shortage(self):
        worker = self._build_worker(_FakeDb("noop"))
        worker.proxy_last_refresh_status = "代理源暂时冷却中，稍后继续补货"
        worker._check_proxy_step_retry_budget = MagicMock(side_effect=[(False, ""), (True, "budget exhausted")])
        worker._get_proxy_for_step = MagicMock(return_value=None)
        worker._mark_detection_failed = MagicMock()

        with patch("detect_worker.register.check_register", side_effect=RuntimeError("direct timeout")):
            result = worker._run_detect_register(1, {"source_type": 0}, "example.com")

        self.assertFalse(result)
        self.assertTrue(worker._consume_step_force_direct_once("注册状态检测"))
        self.assertAlmostEqual(0.05, worker._consume_step_proxy_retry_wait("注册状态检测"))
        worker._mark_detection_failed.assert_called_once()

    def test_run_detect_register_blacklists_client_hold_for_single_step_task(self):
        worker = self._build_worker(_FakeDb("noop"))
        worker._check_proxy_step_retry_budget = MagicMock(return_value=(False, ""))
        worker._remaining_proxy_step_retry_budget_seconds = MagicMock(return_value=2.0)
        worker._is_current_single_step_task = MagicMock(return_value=True)
        worker._cache_single_step_result_payload = MagicMock()
        worker._mark_blacklisted = MagicMock()
        worker._clear_proxy_failure = MagicMock()
        worker._clear_proxy_source_failure = MagicMock()
        worker._get_proxy_for_step = MagicMock(return_value={"http": "http://proxy", "https": "http://proxy"})
        worker.db.update_domain_register_result = MagicMock()

        with patch("detect_worker.register.check_register", return_value=(REGISTER_STATUS_CLIENT_HOLD, "2026-05-01")):
            result = worker._run_detect_register(1, {"source_type": 0}, "example.com")

        self.assertFalse(result)
        worker._cache_single_step_result_payload.assert_called_once()
        payload = worker._cache_single_step_result_payload.call_args.args[2]
        self.assertEqual("blacklisted", payload["state"])
        self.assertEqual(REGISTER_STATUS_CLIENT_HOLD, payload["register_status"])
        worker._mark_blacklisted.assert_called_once()
        worker.db.update_domain_register_result.assert_not_called()

    def test_run_detect_register_blacklists_server_hold_for_domain_task(self):
        worker = self._build_worker(_FakeDb("noop"))
        worker._check_proxy_step_retry_budget = MagicMock(return_value=(False, ""))
        worker._remaining_proxy_step_retry_budget_seconds = MagicMock(return_value=2.0)
        worker._is_current_single_step_task = MagicMock(return_value=False)
        worker._mark_blacklisted = MagicMock()
        worker._clear_proxy_failure = MagicMock()
        worker._clear_proxy_source_failure = MagicMock()
        worker._get_proxy_for_step = MagicMock(return_value={"http": "http://proxy", "https": "http://proxy"})
        worker.db.update_domain_register_result = MagicMock()

        domain = {"source_type": 0}
        with patch("detect_worker.register.check_register", return_value=(REGISTER_STATUS_SERVER_HOLD, "2026-05-01")):
            result = worker._run_detect_register(1, domain, "example.com")

        self.assertFalse(result)
        worker.db.update_domain_register_result.assert_called_once()
        self.assertEqual(REGISTER_STATUS_SERVER_HOLD, domain["register_status"])
        worker._mark_blacklisted.assert_called_once()

    def test_get_proxy_for_step_fast_direct_fallback_when_proxy_source_is_cooling_down(self):
        worker = self._build_worker(_FakeDb("noop"))
        worker.proxy_last_refresh_status = "代理源暂时冷却中，稍后继续补货"
        worker.proxy_direct_fallback_grace_overrides = {"360检测": 0.12}
        worker.get_proxies = MagicMock(return_value=None)

        with patch("detect_worker.time.time", side_effect=[100.0, 100.2]), patch("detect_worker.time.sleep") as mock_sleep:
            proxy = worker._get_proxy_for_step(1, "example.com", "360检测")

        self.assertIsNone(proxy)
        mock_sleep.assert_not_called()

    def test_get_proxy_for_step_register_single_machine_mode_skips_proxy_wait(self):
        worker = self._build_worker(_FakeDb("noop"))
        worker.proxy_config = {"proxy_enable": True, "allow_direct": False}
        worker.register_single_machine_mode_enabled = True
        worker.register_single_machine_direct_streak_attempts = 2
        worker.get_proxies = MagicMock(return_value=None)

        with patch("detect_worker.time.time", side_effect=[100.0, 100.0]), patch("detect_worker.time.sleep") as mock_sleep:
            proxy = worker._get_proxy_for_step(1, "example.com", "注册状态检测")

        self.assertEqual(0.0, worker._proxy_direct_fallback_grace_for_step("注册状态检测"))
        self.assertEqual(0.0, worker._proxy_direct_retry_wait_for_step("注册状态检测"))
        self.assertIsNone(proxy)
        mock_sleep.assert_not_called()

    def test_external_issue_retry_wait_uses_step_specific_short_window(self):
        worker = self._build_worker(_FakeDb("noop"))
        worker._set_current_task_context(domain_id=1, is_step_task=True, task_mode="single_step")
        worker.proxy_direct_retry_wait_overrides = {"360检测": 0.25}
        worker.proxy_last_refresh_status = "代理源暂时冷却中，稍后继续补货"

        should_retry = worker._should_retry_external_issue(
            None,
            "360搜索请求超时",
            "360检测",
            "example.com",
        )

        self.assertTrue(should_retry)
        self.assertAlmostEqual(0.25, worker._consume_step_proxy_retry_wait("360检测"))

    def test_stale_detect_session_holds_during_proxy_shortage_for_same_job(self):
        worker = self._build_worker(_FakeDb("noop"))
        worker.detect_command_thread = MagicMock()
        worker.detect_command_thread.is_alive.return_value = True
        worker.current_job_code = "sync-overseas-51"
        worker.proxy_last_refresh_status = "代理源暂时冷却中，稍后继续补货"
        worker.proxy_pool = []

        with patch("detect_worker.time.time", return_value=200.0):
            worker.detect_command_started_at = 160.0
            worker.detect_command_last_activity_at = 165.0
            worker._last_domain_started_at = 165.0
            worker._last_domain_result_at = 165.0
            stale_detect, stale_reason = worker._is_stale_detect_session({"job_code": "sync-overseas-51"})

        self.assertFalse(stale_detect)
        self.assertIn("同任务同周期重复唤起已忽略", stale_reason)

    def test_stale_detect_session_still_recovers_when_new_job_arrives(self):
        worker = self._build_worker(_FakeDb("noop"))
        worker.detect_command_thread = MagicMock()
        worker.detect_command_thread.is_alive.return_value = True
        worker.current_job_code = "sync-overseas-51"
        worker.current_job_task_mode = "single_step"
        worker.proxy_last_refresh_status = "代理源暂时冷却中，稍后继续补货"
        worker.proxy_pool = []

        with patch("detect_worker.time.time", return_value=200.0):
            worker.detect_command_started_at = 150.0
            worker.detect_command_last_activity_at = 170.0
            worker._last_domain_started_at = 170.0
            worker._last_domain_result_at = 170.0
            stale_detect, stale_reason = worker._is_stale_detect_session({"job_code": "sync-overseas-99"})

        self.assertTrue(stale_detect)
        self.assertIn("新任务", stale_reason)

    def test_stale_detect_session_does_not_replace_busy_domain_pipeline_for_new_job(self):
        worker = self._build_worker(_FakeDb("noop"))
        worker.detect_command_thread = MagicMock()
        worker.detect_command_thread.is_alive.return_value = True
        worker.current_job_code = "sync-overseas-51"
        worker.current_job_task_mode = "domain_pipeline"
        worker.proxy_last_refresh_status = "代理正常"
        worker.proxy_pool = [{}]
        worker._active_domain_threads = 6

        with patch("detect_worker.time.time", return_value=200.0):
            worker.detect_command_started_at = 150.0
            worker.detect_command_last_activity_at = 170.0
            worker._last_domain_started_at = 170.0
            worker._last_domain_result_at = 170.0
            stale_detect, stale_reason = worker._is_stale_detect_session(
                {"job_code": "sync-overseas-99", "task_mode": "domain_pipeline"}
            )

        self.assertFalse(stale_detect)
        self.assertEqual("", stale_reason)

    def test_submit_backlog_limit_defaults_to_thread_count_window(self):
        worker = self._build_worker(_FakeDb("noop"))

        self.assertEqual(2000, worker._resolve_submit_backlog_limit(2000, 0))
        self.assertEqual(2000, worker._resolve_submit_backlog_limit(2000, 1200))

    def test_dispatch_capacity_can_ramp_to_full_thread_limit(self):
        worker = self._build_worker(_FakeDb("noop"))

        self.assertEqual(2000, worker._resolve_dispatch_capacity(2000, 0, 2000))
        self.assertEqual(2000, worker._resolve_dispatch_capacity(2000, 1000, 1000))

    def test_claim_batch_size_defaults_to_thread_count_window(self):
        worker = self._build_worker(_FakeDb("noop"))

        self.assertEqual(2000, worker._resolve_claim_batch_size(2000, 2000, live_active=0))
        self.assertEqual(1500, worker._resolve_claim_batch_size(2000, 1500, live_active=800))

    def test_runtime_settings_can_shrink_claim_and_backlog_windows(self):
        worker = self._build_worker(_FakeDb("noop"))
        worker.runtime_settings = {
            "claim_batch_floor": 200,
            "claim_batch_ceil": 800,
            "submit_backlog_floor": 500,
            "submit_backlog_ceil": 1500,
            "dispatch_cap_multiplier": 2,
            "pending_buffer_cap_multiplier": 2,
        }

        self.assertEqual(500, worker._resolve_submit_backlog_limit(2000, 0))
        self.assertEqual(500, worker._resolve_claim_batch_size(2000, 2000, live_active=0))
        self.assertEqual(1500, worker._resolve_dispatch_capacity(2000, 0, 1500))
        self.assertEqual(2, worker._runtime_positive_int_override("pending_buffer_cap_multiplier", "DOMAINCHECK_PENDING_BUFFER_CAP_MULTIPLIER"))

    def test_runtime_settings_override_environment_claim_and_backlog_windows(self):
        worker = self._build_worker(_FakeDb("noop"))
        worker.runtime_settings = {
            "claim_batch_floor": 200,
            "claim_batch_ceil": 800,
            "submit_backlog_floor": 500,
            "submit_backlog_ceil": 1500,
            "dispatch_cap_multiplier": 2,
        }

        with patch.dict(
            os.environ,
            {
                "DOMAINCHECK_CLAIM_BATCH_FLOOR": "4000",
                "DOMAINCHECK_CLAIM_BATCH_CEIL": "10000",
                "DOMAINCHECK_SUBMIT_BACKLOG_FLOOR": "5000",
                "DOMAINCHECK_SUBMIT_BACKLOG_CEIL": "10000",
                "DOMAINCHECK_DISPATCH_CAP_MULTIPLIER": "5",
            },
            clear=False,
        ):
            self.assertEqual(500, worker._resolve_submit_backlog_limit(2000, 0))
            self.assertEqual(500, worker._resolve_claim_batch_size(2000, 2000, live_active=0))
            self.assertEqual(1500, worker._resolve_dispatch_capacity(2000, 0, 1500))

    def test_pipeline_topup_triggers_when_partial_claim_leaves_large_gap(self):
        worker = self._build_worker(_FakeDb("noop"))

        self.assertTrue(worker._should_proactively_top_up_pipeline(280, 1800, 2000))
        self.assertFalse(worker._should_proactively_top_up_pipeline(1700, 1800, 2000))

    def test_sync_prefetch_triggers_when_local_work_falls_below_threshold(self):
        worker = self._build_worker(_FakeDb("noop"))

        self.assertTrue(worker._should_prefetch_sync_tasks(24, 18, 1000))
        self.assertFalse(worker._should_prefetch_sync_tasks(420, 180, 1000))

    def test_pull_sync_tasks_can_prefetch_without_claiming_after_pull(self):
        worker = self._build_worker(_FakeDb("noop"))
        worker._acquire_shared_job_maintenance_lock = MagicMock(return_value="lock-owner")
        worker._release_shared_job_maintenance_lock = MagicMock()
        worker._claim_detect_job_items = MagicMock(return_value=[{"domain_name": "should-not-claim.com"}])

        class _FakeResponse:
            def __enter__(self_inner):
                return self_inner

            def __exit__(self_inner, exc_type, exc, tb):
                return False

            @staticmethod
            def read():
                return json.dumps(
                    {
                        "code": 0,
                        "message": "ok",
                        "data": {"pull_state": "queued", "queued_count": 500},
                    },
                    ensure_ascii=False,
                ).encode("utf-8")

        with patch.object(config, "NODE_REGION", "mainland"), patch.object(config, "NODE_ROLE", "control"), patch(
            "detect_worker.urllib.request.urlopen",
            return_value=_FakeResponse(),
        ):
            domains = worker._pull_sync_tasks_until_available(thread_limit=1000, claim_after_pull=False)

        self.assertEqual([], domains)
        worker._claim_detect_job_items.assert_not_called()
        worker._release_shared_job_maintenance_lock.assert_called_once()

    def test_get_proxy_for_step_can_force_single_direct_fallback_once(self):
        worker = self._build_worker(_FakeDb("noop"))
        worker._set_current_task_context(domain_id=1, is_step_task=True, task_mode="single_step")
        worker._set_step_force_direct_once("百度site检测", True)

        proxy = worker._get_proxy_for_step(1, "example.com", "百度site检测")

        self.assertIsNone(proxy)
        self.assertFalse(worker._consume_step_force_direct_once("百度site检测"))

    def test_load_thread_count_can_inherit_parent_node_override(self):
        worker = DetectWorker.__new__(DetectWorker)
        worker.use_redis = True

        class _FakeRedis:
            def get(self, key):
                if key == "domain_tool:node_thread_counts":
                    return json.dumps({"mainland-controller-01": 1200}, ensure_ascii=False)
                if key == "domain_tool:thread_count":
                    return "1000"
                return None

        worker.redis_client = _FakeRedis()

        with patch.object(config, "NODE_CODE", "mainland-controller-01-a"), patch.dict(
            os.environ,
            {"WORKER_PARENT_NODE_CODE": "mainland-controller-01"},
            clear=False,
        ):
            thread_count = DetectWorker.load_thread_count(worker)

        self.assertEqual(1200, thread_count)

    def test_load_process_count_can_inherit_parent_node_override_from_local_file_root(self):
        worker = DetectWorker.__new__(DetectWorker)
        worker.use_redis = False

        with patch.object(config, "NODE_CODE", "mainland-controller-01-u"), patch.dict(
            os.environ,
            {"WORKER_PARENT_NODE_CODE": "mainland-controller-01"},
            clear=False,
        ), patch(
            "detect_worker._read_worker_local_json_config",
            side_effect=lambda name: (
                {"mainland-controller-01": 80}
                if name == "node_process_counts.json"
                else ({"process_count": "80"} if name == "process_count.json" else None)
            ),
        ):
            process_count = DetectWorker.load_process_count(worker)

        self.assertEqual(80, process_count)

    def test_get_proxies_with_lease_holds_entry_until_release(self):
        worker = self._build_worker(_FakeDb("noop"))
        good_proxy = {"http": "http://good-proxy", "https": "http://good-proxy"}
        worker.proxy_pool = [
            {
                "proxy": good_proxy,
                "usage_count": 0,
                "source_tag": "C",
                "expire_at_ts": time.time() + 60,
            }
        ]
        worker._schedule_proxy_refresh_if_needed = MagicMock()

        selected = worker.get_proxies(step_name="百度site检测", lease_selected=True)

        self.assertEqual(good_proxy, selected)
        self.assertEqual(0, len(worker.proxy_pool))
        self.assertIn(worker._proxy_key(good_proxy), worker.proxy_active_leases)

        reused = worker.release_proxy(good_proxy, step_name="百度site检测")

        self.assertTrue(reused)
        self.assertEqual(1, len(worker.proxy_pool))
        self.assertNotIn(worker._proxy_key(good_proxy), worker.proxy_active_leases)

    def test_release_proxy_discards_entry_when_remaining_ttl_is_too_low(self):
        worker = self._build_worker(_FakeDb("noop"))
        proxy = {"http": "http://soon-expire", "https": "http://soon-expire"}
        worker.proxy_active_leases = {
            worker._proxy_key(proxy): {
                "proxy": proxy,
                "usage_count": 1,
                "source_tag": "D",
                "expire_at_ts": time.time() + 2,
                "leased_step_name": "百度site检测",
            }
        }
        worker._schedule_proxy_refresh_if_needed = MagicMock()

        reused = worker.release_proxy(proxy, step_name="百度site检测")

        self.assertFalse(reused)
        self.assertEqual([], worker.proxy_pool)
        self.assertNotIn(worker._proxy_key(proxy), worker.proxy_active_leases)

    def test_retry_external_issue_with_proxy_forces_next_attempt_direct_when_allowed(self):
        worker = self._build_worker(_FakeDb("noop"))
        worker._set_current_task_context(domain_id=1, is_step_task=True, task_mode="single_step")
        worker.remove_proxy = MagicMock()
        proxy = {"http": "http://bad-proxy", "https": "http://bad-proxy"}

        should_retry = worker._should_retry_external_issue(
            proxy,
            "curl: (28) Connection timed out after 552 milliseconds",
            "百度site检测",
            "example.com",
        )

        self.assertTrue(should_retry)
        worker.remove_proxy.assert_called_once_with(proxy)
        worker.trigger_proxy_refresh.assert_called_once()
        self.assertTrue(worker._consume_step_force_direct_once("百度site检测"))

    def test_external_dependency_issue_recognizes_common_chinese_timeout_messages(self):
        worker = self._build_worker(_FakeDb("noop"))

        self.assertTrue(worker._is_external_dependency_issue("360搜索请求超时"))
        self.assertTrue(worker._is_external_dependency_issue("百度site检测失败: 连接超时"))
        self.assertTrue(worker._is_external_dependency_issue("站长之家检测失败: 网络异常"))

    def test_run_detect_aizhan_passes_remaining_budget_to_detector(self):
        worker = self._build_worker(_FakeDb("noop"))
        worker._check_proxy_step_retry_budget = MagicMock(return_value=(False, ""))
        worker._remaining_proxy_step_retry_budget_seconds = MagicMock(return_value=1.7)
        worker._get_proxy_for_step = MagicMock(return_value=None)
        worker._mark_detection_degraded = MagicMock()
        worker._should_blacklist_result = MagicMock(return_value=False)
        worker._record_step_result = MagicMock()
        worker._should_retry_external_issue = MagicMock(return_value=False)
        worker._is_external_dependency_issue = MagicMock(return_value=False)
        worker._mark_detection_rejected = MagicMock()
        worker._mark_detection_failed = MagicMock()

        with patch("detect_worker.aizhan.check_aizhan", return_value=(True, "")) as mock_check:
            result = worker._run_detect_aizhan(1, "example.com", [])

        self.assertTrue(result)
        self.assertEqual(1.7, mock_check.call_args.kwargs["budget_seconds"])

    def test_run_detect_aizhan_uses_external_issue_retry_path(self):
        worker = self._build_worker(_FakeDb("noop"))
        worker._check_proxy_step_retry_budget = MagicMock(side_effect=[(False, ""), (False, ""), (True, "budget exhausted")])
        worker._remaining_proxy_step_retry_budget_seconds = MagicMock(return_value=2.0)
        worker._get_proxy_for_step = MagicMock(side_effect=[
            {"http": "http://proxy-1", "https": "http://proxy-1"},
            None,
        ])
        worker._mark_detection_degraded = MagicMock()
        worker._should_blacklist_result = MagicMock(return_value=False)
        worker._record_step_result = MagicMock()
        worker._should_retry_external_issue = MagicMock(side_effect=[True, False])
        worker._is_external_dependency_issue = MagicMock(return_value=False)
        worker._mark_detection_rejected = MagicMock()
        worker._mark_detection_failed = MagicMock()
        worker.release_proxy = MagicMock()

        with patch(
            "detect_worker.aizhan.check_aizhan",
            side_effect=[(False, "proxy timeout"), (True, "ok")],
        ) as mock_check:
            result = worker._run_detect_aizhan(1, "example.com", [])

        self.assertTrue(result)
        self.assertEqual(2, mock_check.call_count)
        worker._should_retry_external_issue.assert_called_with(
            {"http": "http://proxy-1", "https": "http://proxy-1"},
            "proxy timeout",
            "爱站网检测",
            "example.com",
        )

    def test_single_machine_aizhan_direct_first_enabled_only_on_first_attempt(self):
        worker = self._build_worker(_FakeDb("noop"))
        worker.single_machine_aizhan_direct_first_enabled = True
        worker.single_machine_site_direct_fallback_enabled = True
        worker.allow_direct_connection = MagicMock(return_value=False)

        self.assertTrue(worker._should_force_direct_first_attempt("爱站网检测", 0))
        self.assertFalse(worker._should_force_direct_first_attempt("爱站网检测", 1))
        self.assertFalse(worker._should_force_direct_first_attempt("百度site检测", 0))

    def test_single_machine_baidu_direct_first_enabled_only_on_first_attempt(self):
        worker = self._build_worker(_FakeDb("noop"))
        worker.single_machine_baidu_direct_first_enabled = True
        worker.single_machine_site_direct_fallback_enabled = True
        worker.allow_direct_connection = MagicMock(return_value=False)

        self.assertTrue(worker._should_force_direct_first_attempt("百度site检测", 0))
        self.assertFalse(worker._should_force_direct_first_attempt("百度site检测", 1))
        self.assertFalse(worker._should_force_direct_first_attempt("爱站网检测", 0))

    def test_single_machine_chinaz_direct_first_enabled_only_on_first_attempt(self):
        worker = self._build_worker(_FakeDb("noop"))
        worker.single_machine_chinaz_direct_first_enabled = True
        worker.single_machine_site_direct_fallback_enabled = True
        worker.allow_direct_connection = MagicMock(return_value=False)

        self.assertTrue(worker._should_force_direct_first_attempt("站长之家检测", 0))
        self.assertFalse(worker._should_force_direct_first_attempt("站长之家检测", 1))
        self.assertFalse(worker._should_force_direct_first_attempt("百度site检测", 0))

    def test_run_detect_aizhan_forces_direct_first_when_single_machine_flag_enabled(self):
        worker = self._build_worker(_FakeDb("noop"))
        worker.single_machine_aizhan_direct_first_enabled = True
        worker.single_machine_site_direct_fallback_enabled = True
        worker.allow_direct_connection = MagicMock(return_value=False)
        worker._check_proxy_step_retry_budget = MagicMock(return_value=(False, ""))
        worker._remaining_proxy_step_retry_budget_seconds = MagicMock(return_value=2.0)
        worker._get_proxy_for_step = MagicMock(return_value=None)
        worker._mark_detection_degraded = MagicMock()
        worker._should_blacklist_result = MagicMock(return_value=False)
        worker._record_step_result = MagicMock()
        worker._should_retry_external_issue = MagicMock(return_value=False)
        worker._is_external_dependency_issue = MagicMock(return_value=False)
        worker._mark_detection_rejected = MagicMock()
        worker._mark_detection_failed = MagicMock()
        worker._set_step_force_direct_once = MagicMock()

        with patch("detect_worker.aizhan.check_aizhan", return_value=(True, "")):
            result = worker._run_detect_aizhan(1, "example.com", [])

        self.assertTrue(result)
        worker._set_step_force_direct_once.assert_called_once_with("爱站网检测", True)

    def test_run_detect_baidu_forces_direct_first_when_single_machine_flag_enabled(self):
        worker = self._build_worker(_FakeDb("noop"))
        worker.single_machine_baidu_direct_first_enabled = True
        worker.single_machine_site_direct_fallback_enabled = True
        worker.allow_direct_connection = MagicMock(return_value=False)
        worker._check_proxy_step_retry_budget = MagicMock(return_value=(False, ""))
        worker._remaining_proxy_step_retry_budget_seconds = MagicMock(return_value=2.0)
        worker._get_proxy_for_step = MagicMock(return_value=None)
        worker._mark_detection_degraded = MagicMock()
        worker._should_blacklist_result = MagicMock(return_value=False)
        worker._record_step_result = MagicMock()
        worker._should_retry_external_issue = MagicMock(return_value=False)
        worker._is_external_dependency_issue = MagicMock(return_value=False)
        worker._mark_detection_rejected = MagicMock()
        worker._mark_detection_failed = MagicMock()
        worker._set_step_force_direct_once = MagicMock()

        with patch("detect_worker.baidu.check_site", return_value=(True, "")):
            result = worker._run_detect_baidu(1, "example.com", [])

        self.assertTrue(result)
        worker._set_step_force_direct_once.assert_called_once_with("百度site检测", True)

    def test_run_detect_chinaz_forces_direct_first_when_single_machine_flag_enabled(self):
        worker = self._build_worker(_FakeDb("noop"))
        worker.single_machine_chinaz_direct_first_enabled = True
        worker.single_machine_site_direct_fallback_enabled = True
        worker.allow_direct_connection = MagicMock(return_value=False)
        worker._check_proxy_step_retry_budget = MagicMock(return_value=(False, ""))
        worker._remaining_proxy_step_retry_budget_seconds = MagicMock(return_value=2.0)
        worker._get_proxy_for_step = MagicMock(return_value={"http": "http://proxy-1", "https": "http://proxy-1"})
        worker._mark_detection_degraded = MagicMock()
        worker._should_blacklist_result = MagicMock(return_value=False)
        worker._record_step_result = MagicMock()
        worker._should_rotate_proxy_on_error = MagicMock(return_value=False)
        worker._is_external_dependency_issue = MagicMock(return_value=False)
        worker._mark_detection_rejected = MagicMock()
        worker._mark_detection_failed = MagicMock()
        worker._emit_step_trace = MagicMock()

        with patch("detect_worker.chinaz.check_title", return_value=(True, "", {})) as mock_check_title:
            result = worker._run_detect_chinaz(1, "example.com", [])

        self.assertTrue(result)
        worker._get_proxy_for_step.assert_not_called()
        worker._emit_step_trace.assert_called_once()
        self.assertIsNone(mock_check_title.call_args.args[2])

    def test_run_detect_aizhan_degrades_remote_disconnected_early_when_flag_enabled(self):
        worker = self._build_worker(_FakeDb("noop"))
        worker.aizhan_remote_disconnect_degrade_enabled = True
        worker._check_proxy_step_retry_budget = MagicMock(return_value=(False, ""))
        worker._remaining_proxy_step_retry_budget_seconds = MagicMock(return_value=2.0)
        proxy = {"http": "http://proxy-1", "https": "http://proxy-1"}
        worker._get_proxy_for_step = MagicMock(return_value=proxy)
        worker._mark_detection_degraded = MagicMock()
        worker._should_blacklist_result = MagicMock(return_value=False)
        worker._record_step_result = MagicMock()
        worker._should_retry_external_issue = MagicMock(return_value=True)
        worker._is_external_dependency_issue = MagicMock(return_value=True)
        worker._mark_detection_rejected = MagicMock()
        worker._mark_detection_failed = MagicMock()
        worker.release_proxy = MagicMock()

        with patch(
            "detect_worker.aizhan.check_aizhan",
            return_value=(False, "('Connection aborted.', RemoteDisconnected('Remote end closed connection without response'))"),
        ):
            result = worker._run_detect_aizhan(1, "example.com", [])

        self.assertTrue(result)
        worker._should_retry_external_issue.assert_not_called()
        worker.release_proxy.assert_called_once()
        worker._mark_detection_degraded.assert_called_once()

    def test_run_detect_aizhan_degrades_external_issue_early_when_fast_flag_enabled(self):
        worker = self._build_worker(_FakeDb("noop"))
        worker.aizhan_external_fast_degrade_enabled = True
        worker._check_proxy_step_retry_budget = MagicMock(return_value=(False, ""))
        worker._remaining_proxy_step_retry_budget_seconds = MagicMock(return_value=2.0)
        proxy = {"http": "http://proxy-1", "https": "http://proxy-1"}
        worker._get_proxy_for_step = MagicMock(return_value=proxy)
        worker._mark_detection_degraded = MagicMock()
        worker._should_blacklist_result = MagicMock(return_value=False)
        worker._record_step_result = MagicMock()
        worker._should_retry_external_issue = MagicMock(return_value=True)
        worker._is_external_dependency_issue = MagicMock(return_value=True)
        worker._mark_detection_rejected = MagicMock()
        worker._mark_detection_failed = MagicMock()
        worker.release_proxy = MagicMock()

        with patch(
            "detect_worker.aizhan.check_aizhan",
            return_value=(False, "Read timed out. (read timeout=1.8)"),
        ):
            result = worker._run_detect_aizhan(1, "example.com", [])

        self.assertTrue(result)
        worker._should_retry_external_issue.assert_not_called()
        worker.release_proxy.assert_called_once()
        worker._mark_detection_degraded.assert_called_once()

    def test_resolve_single_step_finalization_keeps_degraded_as_completed(self):
        status, message = DetectWorker._resolve_single_step_finalization(
            {
                "state": "degraded",
                "message": "外部依赖异常，建议重试",
                "retry_recommended": True,
            }
        )

        self.assertEqual("completed", status)
        self.assertEqual("外部依赖异常，建议重试", message)

    def test_worker_log_sync_mode_keeps_full_when_load_is_small(self):
        worker = self._build_worker(_FakeDb("noop"))
        worker.runtime_settings = {"worker_log_sync_enabled": True, "worker_log_sync_mode": "full"}
        worker.thread_count = 1200
        worker.detecting = True
        worker._get_active_domain_threads = MagicMock(return_value=80)

        self.assertEqual("full", worker._worker_log_sync_mode())

    def test_worker_log_sync_mode_downgrades_full_when_threads_are_busy(self):
        worker = self._build_worker(_FakeDb("noop"))
        worker.runtime_settings = {"worker_log_sync_enabled": True, "worker_log_sync_mode": "full"}
        worker.thread_count = 1200
        worker.detecting = True
        worker._get_active_domain_threads = MagicMock(return_value=650)

        self.assertEqual("key", worker._worker_log_sync_mode())

    def test_worker_log_sync_mode_downgrades_full_when_queue_backlog_is_high(self):
        worker = self._build_worker(_FakeDb("noop"))
        worker.runtime_settings = {"worker_log_sync_enabled": True, "worker_log_sync_mode": "full"}
        worker.thread_count = 1200
        worker.detecting = False
        worker._worker_log_sync_queue = MagicMock()
        worker._worker_log_sync_queue.maxsize = 50000
        worker._worker_log_sync_queue.qsize.return_value = 12000

        self.assertEqual("key", worker._worker_log_sync_mode())

    def test_push_remote_debug_event_enters_cooldown_after_failure(self):
        worker = self._build_worker(_FakeDb("noop"))

        with patch.object(worker, "_remote_debug_ingest_url", return_value="http://example.invalid/api/v1/runtime/debug-ingest"):
            with patch("detect_worker.urllib.request.urlopen", side_effect=TimeoutError("timed out")) as mock_urlopen:
                worker._push_remote_debug_event(message="test")

                self.assertEqual(1, worker._remote_debug_event_failure_streak)
                self.assertGreater(worker._remote_debug_event_cooldown_until, time.time())
                self.assertEqual(1, mock_urlopen.call_count)

                worker._push_remote_debug_event(message="test")
                self.assertEqual(1, mock_urlopen.call_count)

    def test_push_remote_debug_event_resets_failure_streak_after_success(self):
        worker = self._build_worker(_FakeDb("noop"))
        worker._remote_debug_event_failure_streak = 3

        class _FakeResponse:
            def __enter__(self_inner):
                return self_inner

            def __exit__(self_inner, exc_type, exc, tb):
                return False

            @staticmethod
            def read():
                return b'{"code":0}'

        with patch.object(worker, "_remote_debug_ingest_url", return_value="http://example.invalid/api/v1/runtime/debug-ingest"):
            with patch("detect_worker.urllib.request.urlopen", return_value=_FakeResponse()):
                worker._push_remote_debug_event(message="test")

        self.assertEqual(0, worker._remote_debug_event_failure_streak)
        self.assertEqual(0.0, worker._remote_debug_event_cooldown_until)

    def test_idle_worker_recycle_skips_parent_control_worker(self):
        worker = self._build_worker(_FakeDb("noop"))
        worker.running = True
        worker.detecting = True
        worker._get_active_domain_threads = MagicMock(return_value=0)
        worker._last_runtime_extra = {"waiting_for_dispatch": True}
        worker._last_worker_activity_at = time.time() - 600
        worker._worker_started_at = time.time() - 900

        with patch.dict(
            os.environ,
            {
                "WORKER_MODE": "linux-systemd",
                "WORKER_PARENT_NODE_CODE": "mainland-controller-01",
                "DOMAINCHECK_IDLE_WORKER_RECYCLE_ENABLED": "1",
            },
            clear=False,
        ):
            with patch.object(config, "NODE_CODE", "mainland-controller-01"):
                with patch.object(worker, "_trigger_idle_worker_recycle") as mock_recycle:
                    recycled = worker._maybe_recycle_idle_worker(trigger_reason="heartbeat")

        self.assertFalse(recycled)
        mock_recycle.assert_not_called()

    def test_configure_worker_thread_stack_size_uses_linux_systemd_default(self):
        state = {"size": 0}

        def fake_stack_size(value=None):
            if value is None:
                return state["size"]
            state["size"] = int(value)
            return 0

        with patch.dict(os.environ, {"WORKER_MODE": "linux-systemd"}, clear=False):
            with patch.object(detect_worker_module.threading, "stack_size", side_effect=fake_stack_size):
                applied = detect_worker_module._configure_worker_thread_stack_size()

        self.assertEqual(1024 * 1024, applied)
        self.assertEqual(1024 * 1024, state["size"])

    def test_configure_worker_thread_stack_size_skips_non_service_default(self):
        state = {"size": 0}

        def fake_stack_size(value=None):
            if value is None:
                return state["size"]
            state["size"] = int(value)
            return 0

        with patch.dict(os.environ, {"WORKER_MODE": "desktop"}, clear=False):
            with patch.object(detect_worker_module.threading, "stack_size", side_effect=fake_stack_size):
                applied = detect_worker_module._configure_worker_thread_stack_size()

        self.assertEqual(0, applied)
        self.assertEqual(0, state["size"])

    def test_idle_worker_recycle_triggers_for_idle_child_worker(self):
        worker = self._build_worker(_FakeDb("noop"))
        worker.running = True
        worker.detecting = True
        worker._get_active_domain_threads = MagicMock(return_value=0)
        worker._last_runtime_extra = {"waiting_for_dispatch": True}
        worker._last_worker_activity_at = time.time() - 600
        worker._worker_started_at = time.time() - 900

        with patch.dict(
            os.environ,
            {
                "WORKER_MODE": "linux-systemd",
                "WORKER_PARENT_NODE_CODE": "mainland-controller-01",
                "DOMAINCHECK_IDLE_WORKER_RECYCLE_ENABLED": "1",
                "DOMAINCHECK_IDLE_WORKER_RECYCLE_AFTER_SECONDS": "120",
                "DOMAINCHECK_IDLE_WORKER_RECYCLE_MIN_UPTIME_SECONDS": "180",
                "DOMAINCHECK_IDLE_WORKER_RECYCLE_JITTER_SECONDS": "0",
            },
            clear=False,
        ):
            with patch.object(config, "NODE_CODE", "mainland-controller-01-b"):
                with patch.object(worker, "_trigger_idle_worker_recycle", return_value=True) as mock_recycle:
                    recycled = worker._maybe_recycle_idle_worker(trigger_reason="heartbeat")

        self.assertTrue(recycled)
        mock_recycle.assert_called_once()

    def test_idle_worker_recycle_respects_recent_explicit_start_grace(self):
        worker = self._build_worker(_FakeDb("noop"))
        worker.running = True
        worker.detecting = True
        worker._get_active_domain_threads = MagicMock(return_value=0)
        worker._last_runtime_extra = {"waiting_for_dispatch": True}
        worker._last_worker_activity_at = time.time() - 600
        worker._worker_started_at = time.time() - 900
        worker._last_explicit_start_signal_at = time.time() - 10

        with patch.dict(
            os.environ,
            {
                "WORKER_MODE": "linux-systemd",
                "WORKER_PARENT_NODE_CODE": "mainland-controller-01",
                "DOMAINCHECK_IDLE_WORKER_RECYCLE_ENABLED": "1",
                "DOMAINCHECK_IDLE_WORKER_RECYCLE_AFTER_SECONDS": "120",
                "DOMAINCHECK_IDLE_WORKER_RECYCLE_MIN_UPTIME_SECONDS": "180",
                "DOMAINCHECK_IDLE_WORKER_RECYCLE_START_GRACE_SECONDS": "90",
                "DOMAINCHECK_IDLE_WORKER_RECYCLE_JITTER_SECONDS": "0",
            },
            clear=False,
        ):
            with patch.object(config, "NODE_CODE", "mainland-controller-01-c"):
                with patch.object(worker, "_trigger_idle_worker_recycle") as mock_recycle:
                    recycled = worker._maybe_recycle_idle_worker(trigger_reason="heartbeat")

        self.assertFalse(recycled)
        mock_recycle.assert_not_called()

    def test_prepare_proxy_pool_for_detection_start_reuses_local_pool(self):
        worker = self._build_worker(_FakeDb("noop"))
        worker.proxy_pool = [{"proxy": "http://1.1.1.1:8080"}]
        worker._schedule_proxy_refresh_if_needed = MagicMock()

        ready, count, reason = worker._prepare_proxy_pool_for_detection_start()

        self.assertTrue(ready)
        self.assertEqual(1, count)
        self.assertEqual("local_pool_ready", reason)
        worker._schedule_proxy_refresh_if_needed.assert_called_once_with(1)

    def test_prepare_proxy_pool_for_detection_start_reuses_shared_snapshot(self):
        worker = self._build_worker(_FakeDb("noop"))
        worker._schedule_proxy_refresh_if_needed = MagicMock()
        worker._proxy_should_coordinate_shared_refresh = MagicMock(return_value=True)
        worker._load_shared_proxy_snapshot_payload = MagicMock(
            return_value={"proxy_pool": [{"proxy": "http://2.2.2.2:8080"}]}
        )

        def apply_snapshot(payload, status_prefix=""):
            worker.proxy_pool = list(payload.get("proxy_pool") or [])
            return True

        worker._apply_shared_proxy_snapshot = MagicMock(side_effect=apply_snapshot)

        ready, count, reason = worker._prepare_proxy_pool_for_detection_start()

        self.assertTrue(ready)
        self.assertEqual(1, count)
        self.assertEqual("shared_snapshot_ready", reason)
        worker._apply_shared_proxy_snapshot.assert_called_once()
        worker._schedule_proxy_refresh_if_needed.assert_called_once_with(1)


if __name__ == "__main__":
    unittest.main()