feat: stabilize multi-region runtime sync and worker orchestration

domainCheck/app/detectors/wayback_detector.py

@@ -17,13 +17,13 @@ import zlib
 from base64 import b64decode, b64encode
 from concurrent.futures import FIRST_COMPLETED, ThreadPoolExecutor, wait
 
-import redis
 import requests
 from requests.adapters import HTTPAdapter
 from urllib3.util.retry import Retry
 
 from app.config import config
 from app.detectors.base import BaseDetector
+from app.utils.redis_client import get_redis_client
 
 
 class WaybackDetector(BaseDetector):
@@ -51,6 +51,7 @@ class WaybackDetector(BaseDetector):
 
     def _build_session(self):
         session = requests.Session()
+        session.trust_env = False
         retry = Retry(
             total=max(0, config.WAYBACK_RETRY_COUNT),
             backoff_factor=0.5,
@@ -106,15 +107,7 @@ class WaybackDetector(BaseDetector):
 
     def _build_redis_client(self):
         try:
-            client = redis.Redis(
-                host=config.REDIS_HOST,
-                port=config.REDIS_PORT,
-                password=config.REDIS_PASSWORD,
-                db=config.REDIS_DB,
-                decode_responses=True,
-                socket_connect_timeout=5,
-                socket_timeout=5,
-            )
+            client = get_redis_client(role="standard")
             client.ping()
             return client
         except Exception:
@@ -489,6 +482,14 @@ class WaybackDetector(BaseDetector):
             trimmed.append(item)
         return trimmed
 
+    def _resolve_scan_record_fetch_limit(self):
+        max_records = max(1, int(getattr(config, "WAYBACK_MAX_RECORDS", 8) or 8))
+        # 扫描阶段最终只会保留最近的少量快照；如果每次都把整个 CDX 历史拉下来，
+        # 在快照特别多的域名上会白白浪费很多秒。这里改成“最近一小窗”，同时留出
+        # 重复 digest/标题的缓冲空间，避免把最新几条里重复记录全部裁没了。
+        recent_window = max(12, max_records * 6)
+        return -recent_window
+
     def scan_snapshots(self, domain, sensitive_words=None, stop_on_first_hit=True, recent_years=None):
         sensitive_words = sensitive_words or config.load_sensitive_words()
         request_errors = []
@@ -512,12 +513,17 @@ class WaybackDetector(BaseDetector):
                 'request_errors': [f"wayback_backoff_active:{round(transient_backoff_remaining, 2)}s"],
                 'elapsed_seconds': 0.0,
             }
+        transient_request_failures = 0
+        transient_request_failure_threshold = 2
+
         latest_fetch = self._fetch_cdx_records_with_meta(domain, limit=-1, fast_latest=True)
         latest_record = (latest_fetch.get('records') or [None])[0]
+        latest_fetch_transient_failure = False
         if latest_fetch.get('error'):
             request_errors.append(f"latest_cdx: {latest_fetch.get('error')}")
             if self._is_transient_request_error(latest_fetch.get('error')):
-                self._trip_transient_backoff()
+                latest_fetch_transient_failure = True
+                transient_request_failures += 1
         latest_timestamp = (latest_record or {}).get('timestamp')
         latest_digest = (latest_record or {}).get('digest', '')
         cutoff_year = self._resolve_recent_year_cutoff(recent_years)
@@ -547,26 +553,6 @@ class WaybackDetector(BaseDetector):
         transient_snapshot_failures = 0
         transient_snapshot_failure_threshold = max(2, domain_concurrency)
 
-        if (not latest_timestamp) and latest_fetch.get('error') and self._is_transient_request_error(latest_fetch.get('error')):
-            return {
-                'snapshot_years': [],
-                'has_sensitive_content': False,
-                'matched_word': None,
-                'matched_timestamp': None,
-                'matched_title': None,
-                'backlink_count': 0,
-                'backlink_count_gt_10': False,
-                'checked_snapshot_count': 0,
-                'fetched_snapshot_count': 0,
-                'failed_snapshot_count': max(1, failed_snapshot_count),
-                'unique_title_count': 0,
-                'duplicate_title_skipped': 0,
-                'digest_duplicate_skipped': 0,
-                'request_error_count': len(request_errors),
-                'request_errors': request_errors,
-                'elapsed_seconds': round(time.time() - started_at, 2),
-            }
-
         if latest_timestamp:
             latest_result = self._fetch_snapshot_title(domain, latest_timestamp)
             checked_snapshot_count = 1
@@ -605,40 +591,55 @@ class WaybackDetector(BaseDetector):
                 if latest_error:
                     request_errors.append(f"latest_snapshot: {latest_error}")
                 if latest_error and self._is_transient_request_error(latest_error):
-                    transient_snapshot_failures += 1
-                    self._trip_transient_backoff()
-                    return {
-                        'snapshot_years': [],
-                        'has_sensitive_content': False,
-                        'matched_word': None,
-                        'matched_timestamp': None,
-                        'matched_title': None,
-                        'backlink_count': 0,
-                        'backlink_count_gt_10': False,
-                        'checked_snapshot_count': checked_snapshot_count,
-                        'fetched_snapshot_count': fetched_snapshot_count,
-                        'failed_snapshot_count': failed_snapshot_count,
-                        'unique_title_count': unique_title_count,
-                        'duplicate_title_skipped': duplicate_title_skipped,
-                        'digest_duplicate_skipped': digest_duplicate_skipped,
-                        'request_error_count': len(request_errors),
-                        'request_errors': request_errors,
-                        'elapsed_seconds': round(time.time() - started_at, 2),
-                    }
+                    transient_request_failures += 1
 
         cached_records = self._load_cached_records(domain)
         if cached_records is not None:
             records = cached_records
         else:
-            records_fetch = self._fetch_cdx_records_with_meta(domain)
-            if records_fetch.get('error'):
-                request_errors.append(f"records_cdx: {records_fetch.get('error')}")
-                if self._is_transient_request_error(records_fetch.get('error')):
-                    self._trip_transient_backoff()
-            records = records_fetch.get('records') or []
-            if records:
-                self._save_cached_records(domain, records)
-                self._save_cached_timestamps(domain, [item['timestamp'] for item in records])
+            if latest_fetch_transient_failure and not latest_timestamp:
+                records = []
+            else:
+                records_fetch = self._fetch_cdx_records_with_meta(
+                    domain,
+                    limit=self._resolve_scan_record_fetch_limit(),
+                )
+                if records_fetch.get('error'):
+                    request_errors.append(f"records_cdx: {records_fetch.get('error')}")
+                    if self._is_transient_request_error(records_fetch.get('error')):
+                        transient_request_failures += 1
+                records = records_fetch.get('records') or []
+                if records:
+                    self._save_cached_records(domain, records)
+                    self._save_cached_timestamps(domain, [item['timestamp'] for item in records])
+
+        if (
+            not latest_timestamp
+            and not records
+            and (
+                transient_request_failures >= transient_request_failure_threshold
+                or latest_fetch_transient_failure
+            )
+        ):
+            self._trip_transient_backoff()
+            return {
+                'snapshot_years': [],
+                'has_sensitive_content': False,
+                'matched_word': None,
+                'matched_timestamp': None,
+                'matched_title': None,
+                'backlink_count': 0,
+                'backlink_count_gt_10': False,
+                'checked_snapshot_count': checked_snapshot_count,
+                'fetched_snapshot_count': fetched_snapshot_count,
+                'failed_snapshot_count': max(1, failed_snapshot_count),
+                'unique_title_count': 0,
+                'duplicate_title_skipped': duplicate_title_skipped,
+                'digest_duplicate_skipped': digest_duplicate_skipped,
+                'request_error_count': len(request_errors),
+                'request_errors': request_errors,
+                'elapsed_seconds': round(time.time() - started_at, 2),
+            }
 
         records = self._filter_records_recent_years(records, recent_years=recent_years)
         records = sorted(records, key=lambda item: item.get('timestamp', ''), reverse=True)
@@ -660,12 +661,12 @@ class WaybackDetector(BaseDetector):
                 digest_seen.add(digest)
             pending_records.append(item)
 
-        with ThreadPoolExecutor(max_workers=domain_concurrency) as executor:
-            pending = {}
-            index = 0
-            finished_count = 1 if latest_timestamp else 0
-            stop_requested = False
-
+        executor = ThreadPoolExecutor(max_workers=domain_concurrency)
+        pending = {}
+        index = 0
+        finished_count = 1 if latest_timestamp else 0
+        stop_requested = False
+        try:
             while (index < len(pending_records) or pending) and not stop_requested:
                 while index < len(pending_records) and len(pending) < domain_concurrency and not stop_requested:
                     timestamp = pending_records[index]['timestamp']
@@ -694,8 +695,8 @@ class WaybackDetector(BaseDetector):
                             request_errors.append(f"snapshot:{timestamp}: {error_message}")
                         if error_message and self._is_transient_request_error(error_message):
                             transient_snapshot_failures += 1
-                            self._trip_transient_backoff()
                             if transient_snapshot_failures >= transient_snapshot_failure_threshold:
+                                self._trip_transient_backoff()
                                 stop_requested = True
                         continue
 
@@ -722,10 +723,13 @@ class WaybackDetector(BaseDetector):
                         )
                     if config.WAYBACK_REQUEST_DELAY > 0:
                         time.sleep(config.WAYBACK_REQUEST_DELAY)
-
+        finally:
             if stop_requested:
-                for future in pending:
+                for future in list(pending.keys()):
                     future.cancel()
+                executor.shutdown(wait=False, cancel_futures=True)
+            else:
+                executor.shutdown(wait=True)
 
         return {
             'snapshot_years': years,