diff --git a/domainCheck/detect_worker.py b/domainCheck/detect_worker.py index e325135..5e33531 100644 --- a/domainCheck/detect_worker.py +++ b/domainCheck/detect_worker.py @@ -13,6 +13,7 @@ import sys import json import time import threading +import collections import schedule from datetime import datetime from loguru import logger @@ -1357,6 +1358,11 @@ class DetectWorker: :param proxy_item: 代理信息 :param result_queue: 结果队列 """ + result_payload = { + "ok": False, + "proxy": None, + "reason": "invalid_proxy_item", + } try: if 'ip' in proxy_item and 'port' in proxy_item: ip = proxy_item['ip'] @@ -1369,23 +1375,52 @@ class DetectWorker: else: proxy_url = f"http://{ip}:{port}" - # 检查代理是否可用 import requests test_proxies = { 'http': proxy_url, 'https': proxy_url } - test_response = requests.get('https://m.baidu.com', proxies=test_proxies, timeout=5) - if test_response.status_code == 200: - # logger.info(f"代理可用性检查通过: {proxy_url}") - result_queue.put(test_proxies) - else: - logger.warning(f"代理可用性检查失败,状态码: {test_response.status_code}, 代理: {proxy_url}") + result_payload["proxy"] = test_proxies + + test_targets = [ + ("http://www.baidu.com", 8), + ("https://m.baidu.com", 8), + ] + last_reason = "unknown" + session = requests.Session() + session.headers.update( + { + "User-Agent": ( + "Mozilla/5.0 (Windows NT 10.0; Win64; x64) " + "AppleWebKit/537.36 (KHTML, like Gecko) " + "Chrome/123.0.0.0 Safari/537.36" + ) + } + ) + for target_url, timeout_seconds in test_targets: + try: + test_response = session.get( + target_url, + proxies=test_proxies, + timeout=timeout_seconds, + allow_redirects=True, + verify=False if target_url.startswith("https://") else True, + ) + if test_response.status_code in (200, 301, 302): + result_payload["ok"] = True + result_payload["reason"] = f"ok:{target_url}:{test_response.status_code}" + break + last_reason = f"http_status_{test_response.status_code}@{target_url}" + except Exception as target_error: + error_text = str(target_error).strip() or target_error.__class__.__name__ + last_reason = f"{target_error.__class__.__name__}@{target_url}:{error_text[:160]}" + session.close() + if not result_payload["ok"]: + result_payload["reason"] = last_reason except Exception as e: - # logger.error(f"代理可用性检查失败: {e}, 代理: {proxy_item}") - pass + result_payload["reason"] = f"{e.__class__.__name__}:{str(e)[:160]}" finally: - # 确保队列中添加一个标记,表示该线程已完成 + result_queue.put(result_payload) result_queue.put(None) def _extract_proxy_items(self, proxy_data): @@ -1539,15 +1574,27 @@ class DetectWorker: completed = 0 start_time = time.time() timeout = 30 + failure_reason_stats = collections.Counter() + failure_samples = [] while completed < len(threads) and time.time() - start_time < timeout: try: result = result_queue.get(timeout=1) if result is not None: - proxy_signature = json.dumps(result, sort_keys=True) - if proxy_signature not in seen_proxy_strings: - seen_proxy_strings.add(proxy_signature) - new_proxies.append({'proxy': result, 'usage_count': 0}) + if isinstance(result, dict) and result.get("ok"): + proxy_value = result.get("proxy") + proxy_signature = json.dumps(proxy_value, sort_keys=True) + if proxy_signature not in seen_proxy_strings: + seen_proxy_strings.add(proxy_signature) + new_proxies.append({'proxy': proxy_value, 'usage_count': 0}) + else: + reason = "" + if isinstance(result, dict): + reason = str(result.get("reason", "")).strip() + proxy_value = result.get("proxy") + if proxy_value and len(failure_samples) < 3: + failure_samples.append(f"{reason} -> {proxy_value}") + failure_reason_stats[reason or "unknown"] += 1 completed += 1 except Exception: pass @@ -1565,6 +1612,12 @@ class DetectWorker: self.proxy_last_refresh_status = f"刷新成功,可用 {len(new_proxies)} 个" self.proxy_next_refresh_time = 0.0 if new_proxies else time.time() + self.proxy_refresh_cooldown_seconds logger.info(f"代理池刷新完成,共 {len(new_proxies)} 个可用代理,来源链接 {len(proxy_api_urls)} 个") + if failure_reason_stats: + logger.warning( + f"代理校验失败汇总: {dict(failure_reason_stats.most_common(5))}" + ) + if failure_samples: + logger.warning(f"代理校验失败样例: {' | '.join(failure_samples)}") self._update_runtime_state( "running" if self.detecting else "idle", f"代理池刷新完成,共 {len(new_proxies)} 个可用代理,来源链接 {len(proxy_api_urls)} 个,原始 {self.proxy_last_refresh_total_items} 个,验证 {self.proxy_last_validated_count} 个",