feat: stabilize multi-region runtime sync and worker orchestration

domainCheck/detect/c360.py

@@ -12,6 +12,7 @@ import re  # 正则表达式模块
 import os
 import threading
 import time
+from collections import OrderedDict
 import requests  # HTTP请求库
 from loguru import logger  # 日志记录
 from requests.adapters import HTTPAdapter
@@ -23,7 +24,12 @@ SO_SEARCH_URL = 'https://www.so.com/s'  # 360搜索URL
 SO_REFERER_TEMPLATE = 'https://www.so.com/s?ie=utf-8&q=site%3A{domain}'  # Referer模板
 SEARCH_PATTERN = r'target="_blank">([^<]+)</a></h3>'  # 搜索结果匹配模式
 BLOCKED_CODE = 3  # 拦截状态码
-_SESSION_LOCAL = threading.local()
+_DIRECT_HTTP = requests.Session()
+_DIRECT_HTTP.trust_env = False
+_DIRECT_HTTP.mount("http://", HTTPAdapter(pool_connections=128, pool_maxsize=256, max_retries=0))
+_DIRECT_HTTP.mount("https://", HTTPAdapter(pool_connections=128, pool_maxsize=256, max_retries=0))
+_PROXY_SESSIONS = OrderedDict()
+_PROXY_SESSION_LOCK = threading.Lock()
 SO_TIMEOUT_PROXY = max(
     1.0,
     float(os.getenv("DOMAINCHECK_360_TIMEOUT_PROXY", "1.4") or 1.4),
@@ -34,6 +40,20 @@ SO_TIMEOUT_DIRECT = max(
 )
 
 
+def _proxy_session_cache_limit() -> int:
+    try:
+        return max(1, int(os.getenv("DOMAINCHECK_360_PROXY_SESSION_CACHE_SIZE", "128") or 128))
+    except Exception:
+        return 128
+
+
+def _close_session_quietly(session) -> None:
+    try:
+        session.close()
+    except Exception:
+        pass
+
+
 def _resolve_360_timeout(proxies: Optional[Dict] = None, budget_seconds: Optional[float] = None) -> float:
     timeout = float(SO_TIMEOUT_PROXY if proxies else SO_TIMEOUT_DIRECT)
     if budget_seconds not in (None, "", 0, "0"):
@@ -41,15 +61,27 @@ def _resolve_360_timeout(proxies: Optional[Dict] = None, budget_seconds: Optiona
     return max(0.6, timeout)
 
 
-def _get_session():
-    session = getattr(_SESSION_LOCAL, "session", None)
-    if session is not None:
+def _get_session(proxies: Optional[Dict] = None):
+    proxy_url = ""
+    if proxies:
+        proxy_url = str(proxies.get("https") or proxies.get("http") or "").strip()
+    if not proxy_url:
+        return _DIRECT_HTTP
+
+    with _PROXY_SESSION_LOCK:
+        session = _PROXY_SESSIONS.get(proxy_url)
+        if session is not None:
+            _PROXY_SESSIONS.move_to_end(proxy_url)
+            return session
+        session = requests.Session()
+        session.trust_env = False
+        session.mount("http://", HTTPAdapter(pool_connections=64, pool_maxsize=128, max_retries=0))
+        session.mount("https://", HTTPAdapter(pool_connections=64, pool_maxsize=128, max_retries=0))
+        _PROXY_SESSIONS[proxy_url] = session
+        while len(_PROXY_SESSIONS) > _proxy_session_cache_limit():
+            _, stale_session = _PROXY_SESSIONS.popitem(last=False)
+            _close_session_quietly(stale_session)
         return session
-    session = requests.Session()
-    session.mount("http://", HTTPAdapter(pool_connections=64, pool_maxsize=128, max_retries=0))
-    session.mount("https://", HTTPAdapter(pool_connections=64, pool_maxsize=128, max_retries=0))
-    _SESSION_LOCAL.session = session
-    return session
 
 
 def check_domain(
@@ -95,7 +127,7 @@ def check_domain(
     try:
         session_started_at = time.perf_counter()
         logger.info(f'360阶段: domain={domain} | stage=session_init_start')
-        session = _get_session()
+        session = _get_session(proxies)
         logger.info(
             f'360阶段: domain={domain} | stage=session_init_done '
             f'| elapsed_ms={int((time.perf_counter() - session_started_at) * 1000)}'
@@ -103,20 +135,28 @@ def check_domain(
         # 发送GET请求获取搜索结果
         request_started_at = time.perf_counter()
         logger.info(f'360阶段: domain={domain} | stage=request_start | proxy={"yes" if proxies else "no"}')
-        response = session.get(
-            SO_SEARCH_URL,
-            params=params,
-            headers=headers,
-            proxies=proxies,
-            timeout=_resolve_360_timeout(proxies, budget_seconds=budget_seconds),
-        )
-        logger.info(
-            f'360阶段: domain={domain} | stage=request_done | status={response.status_code} '
-            f'| elapsed_ms={int((time.perf_counter() - request_started_at) * 1000)}'
-        )
+        response = None
+        try:
+            response = session.get(
+                SO_SEARCH_URL,
+                params=params,
+                headers=headers,
+                proxies=proxies,
+                timeout=_resolve_360_timeout(proxies, budget_seconds=budget_seconds),
+            )
+            logger.info(
+                f'360阶段: domain={domain} | stage=request_done | status={response.status_code} '
+                f'| elapsed_ms={int((time.perf_counter() - request_started_at) * 1000)}'
+            )
 
-        # 解析响应内容
-        response_html = response.content.decode('utf-8', errors='ignore')
+            # 解析响应内容
+            response_html = response.content.decode('utf-8', errors='ignore')
+        finally:
+            try:
+                if response is not None:
+                    response.close()
+            except Exception:
+                pass
 
         # 使用正则表达式提取搜索结果
         search_results = re.findall(SEARCH_PATTERN, response_html)