debug

domainCheck/detect_worker.py

@@ -13,6 +13,7 @@ import sys
 import json
 import time
 import threading
+import collections
 import schedule
 from datetime import datetime
 from loguru import logger
@@ -1357,6 +1358,11 @@ class DetectWorker:
         :param proxy_item: 代理信息
         :param result_queue: 结果队列
         """
+        result_payload = {
+            "ok": False,
+            "proxy": None,
+            "reason": "invalid_proxy_item",
+        }
         try:
             if 'ip' in proxy_item and 'port' in proxy_item:
                 ip = proxy_item['ip']
@@ -1369,23 +1375,52 @@ class DetectWorker:
                 else:
                     proxy_url = f"http://{ip}:{port}"
                 
-                # 检查代理是否可用
                 import requests
                 test_proxies = {
                     'http': proxy_url,
                     'https': proxy_url
                 }
-                test_response = requests.get('https://m.baidu.com', proxies=test_proxies, timeout=5)
+                result_payload["proxy"] = test_proxies
-                if test_response.status_code == 200:
+
-                    # logger.info(f"代理可用性检查通过: {proxy_url}")
+                test_targets = [
-                    result_queue.put(test_proxies)
+                    ("http://www.baidu.com", 8),
-                else:
+                    ("https://m.baidu.com", 8),
-                    logger.warning(f"代理可用性检查失败，状态码: {test_response.status_code}, 代理: {proxy_url}")
+                ]
+                last_reason = "unknown"
+                session = requests.Session()
+                session.headers.update(
+                    {
+                        "User-Agent": (
+                            "Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
+                            "AppleWebKit/537.36 (KHTML, like Gecko) "
+                            "Chrome/123.0.0.0 Safari/537.36"
+                        )
+                    }
+                )
+                for target_url, timeout_seconds in test_targets:
+                    try:
+                        test_response = session.get(
+                            target_url,
+                            proxies=test_proxies,
+                            timeout=timeout_seconds,
+                            allow_redirects=True,
+                            verify=False if target_url.startswith("https://") else True,
+                        )
+                        if test_response.status_code in (200, 301, 302):
+                            result_payload["ok"] = True
+                            result_payload["reason"] = f"ok:{target_url}:{test_response.status_code}"
+                            break
+                        last_reason = f"http_status_{test_response.status_code}@{target_url}"
+                    except Exception as target_error:
+                        error_text = str(target_error).strip() or target_error.__class__.__name__
+                        last_reason = f"{target_error.__class__.__name__}@{target_url}:{error_text[:160]}"
+                session.close()
+                if not result_payload["ok"]:
+                    result_payload["reason"] = last_reason
         except Exception as e:
-            # logger.error(f"代理可用性检查失败: {e}, 代理: {proxy_item}")
+            result_payload["reason"] = f"{e.__class__.__name__}:{str(e)[:160]}"
-            pass
         finally:
-            # 确保队列中添加一个标记，表示该线程已完成
+            result_queue.put(result_payload)
             result_queue.put(None)
 
     def _extract_proxy_items(self, proxy_data):
@@ -1539,15 +1574,27 @@ class DetectWorker:
                     completed = 0
                     start_time = time.time()
                     timeout = 30
+                    failure_reason_stats = collections.Counter()
+                    failure_samples = []
 
                     while completed < len(threads) and time.time() - start_time < timeout:
                         try:
                             result = result_queue.get(timeout=1)
                             if result is not None:
-                                proxy_signature = json.dumps(result, sort_keys=True)
+                                if isinstance(result, dict) and result.get("ok"):
-                                if proxy_signature not in seen_proxy_strings:
+                                    proxy_value = result.get("proxy")
-                                    seen_proxy_strings.add(proxy_signature)
+                                    proxy_signature = json.dumps(proxy_value, sort_keys=True)
-                                    new_proxies.append({'proxy': result, 'usage_count': 0})
+                                    if proxy_signature not in seen_proxy_strings:
+                                        seen_proxy_strings.add(proxy_signature)
+                                        new_proxies.append({'proxy': proxy_value, 'usage_count': 0})
+                                else:
+                                    reason = ""
+                                    if isinstance(result, dict):
+                                        reason = str(result.get("reason", "")).strip()
+                                        proxy_value = result.get("proxy")
+                                        if proxy_value and len(failure_samples) < 3:
+                                            failure_samples.append(f"{reason} -> {proxy_value}")
+                                    failure_reason_stats[reason or "unknown"] += 1
                             completed += 1
                         except Exception:
                             pass
@@ -1565,6 +1612,12 @@ class DetectWorker:
                     self.proxy_last_refresh_status = f"刷新成功，可用 {len(new_proxies)} 个"
                     self.proxy_next_refresh_time = 0.0 if new_proxies else time.time() + self.proxy_refresh_cooldown_seconds
                     logger.info(f"代理池刷新完成，共 {len(new_proxies)} 个可用代理，来源链接 {len(proxy_api_urls)} 个")
+                    if failure_reason_stats:
+                        logger.warning(
+                            f"代理校验失败汇总: {dict(failure_reason_stats.most_common(5))}"
+                        )
+                    if failure_samples:
+                        logger.warning(f"代理校验失败样例: {' | '.join(failure_samples)}")
                     self._update_runtime_state(
                         "running" if self.detecting else "idle",
                         f"代理池刷新完成，共 {len(new_proxies)} 个可用代理，来源链接 {len(proxy_api_urls)} 个，原始 {self.proxy_last_refresh_total_items} 个，验证 {self.proxy_last_validated_count} 个",