3158 lines
148 KiB
Python
3158 lines
148 KiB
Python
# -*- coding: UTF-8 -*-
|
||
'''
|
||
@Project :domainScanDemo
|
||
@File :detect_worker.py
|
||
@IDE :PyCharm
|
||
@Author :梦伴
|
||
@Date :2026/4/10 22:40
|
||
@explain : 域名检测端程序
|
||
'''
|
||
|
||
import os
|
||
import sys
|
||
import json
|
||
import time
|
||
import threading
|
||
import collections
|
||
import schedule
|
||
from datetime import datetime
|
||
from loguru import logger
|
||
from queue import Queue
|
||
from PySide6.QtWidgets import QApplication, QMainWindow, QTextEdit, QPushButton, QVBoxLayout, QHBoxLayout, QWidget, QLabel, QProgressBar, QGroupBox, QScrollArea, QFrame
|
||
from PySide6.QtCore import Qt, QThread, Signal, QMetaObject, Q_ARG, QCoreApplication, QEvent, QTimer
|
||
from PySide6.QtGui import QIcon
|
||
|
||
from app.utils.database import Database
|
||
from app.config import config
|
||
from app.detectors.wayback_detector import WaybackDetector
|
||
from app.utils.status_codes import (
|
||
DETECT_STATUS_BLACKLISTED,
|
||
DETECT_STATUS_COMPLETED,
|
||
DETECT_STATUS_FAILED,
|
||
DETECT_STATUS_RUNNING,
|
||
REGISTER_STATUS_AVAILABLE,
|
||
REVIEW_STATUS_PENDING,
|
||
THIRD_PARTY_STATUS_DONE,
|
||
)
|
||
import redis
|
||
from detect import aizhan, baidu, c360, chinaz, register, jucha, juziseo
|
||
|
||
CONFIG_UPDATE_CHANNEL = "domain_tool:config_update"
|
||
CONTROL_CHANNEL = "domain_tool:worker_control"
|
||
RUNTIME_STATE_KEY = "domain_tool:detect_runtime_state"
|
||
PENDING_CONTROL_KEY = "domain_tool:worker_pending_command"
|
||
RUNTIME_SETTINGS_KEY = "domain_tool:runtime_settings"
|
||
|
||
class DetectThread(QThread):
|
||
"""
|
||
检测线程
|
||
"""
|
||
log_signal = Signal(str)
|
||
progress_signal = Signal(int, int)
|
||
finished_signal = Signal()
|
||
thread_count_signal = Signal(int, int)
|
||
|
||
def __init__(self, worker):
|
||
super().__init__()
|
||
self.worker = worker
|
||
# 设置线程名称
|
||
self.setObjectName("DetectThread")
|
||
|
||
def run(self):
|
||
# 将信号传递给worker
|
||
self.worker.detect_thread = self
|
||
self.worker.start_detection()
|
||
self.finished_signal.emit()
|
||
|
||
class DetectMainWindow(QMainWindow):
|
||
"""
|
||
检测端主窗口
|
||
"""
|
||
|
||
@staticmethod
|
||
def _button_style(
|
||
normal_color: str,
|
||
hover_color: str,
|
||
pressed_color: str,
|
||
disabled_color: str = "#6c757d",
|
||
*,
|
||
fancy_effects: bool = True,
|
||
) -> str:
|
||
hover_extra = ""
|
||
pressed_extra = ""
|
||
disabled_extra = ""
|
||
if fancy_effects:
|
||
hover_extra = "\n transform: translateY(-2px);\n box-shadow: 0 4px 8px rgba(0, 0, 0, 0.2);"
|
||
pressed_extra = "\n transform: translateY(1px);\n box-shadow: 0 2px 4px rgba(0, 0, 0, 0.2);"
|
||
disabled_extra = "\n transform: none;\n box-shadow: none;"
|
||
|
||
return f"""
|
||
QPushButton {{
|
||
background-color: {normal_color};
|
||
color: white;
|
||
border: none;
|
||
border-radius: 8px;
|
||
font-size: 16px;
|
||
font-weight: bold;
|
||
padding: 10px 20px;
|
||
min-width: 120px;
|
||
}}
|
||
QPushButton:hover {{
|
||
background-color: {hover_color};{hover_extra}
|
||
}}
|
||
QPushButton:pressed {{
|
||
background-color: {pressed_color};{pressed_extra}
|
||
}}
|
||
QPushButton:disabled {{
|
||
background-color: {disabled_color};
|
||
color: #adb5bd;{disabled_extra}
|
||
}}
|
||
"""
|
||
|
||
def __init__(self):
|
||
super().__init__()
|
||
self.setWindowTitle("域名检测端")
|
||
self.setGeometry(100, 100, 980, 820)
|
||
self.setMinimumSize(860, 680)
|
||
|
||
# 设置窗口图标
|
||
# 打印当前文件路径以便诊断
|
||
current_file = os.path.abspath(__file__)
|
||
current_dir = os.path.dirname(current_file)
|
||
logger.info(f"当前文件路径: {current_file}")
|
||
logger.info(f"当前目录: {current_dir}")
|
||
|
||
# 获取PyInstaller打包后的临时目录；开发/部署态优先使用脚本所在目录。
|
||
base_dir = current_dir
|
||
if hasattr(sys, '_MEIPASS'):
|
||
base_dir = sys._MEIPASS
|
||
logger.info(f"PyInstaller临时目录: {base_dir}")
|
||
else:
|
||
logger.info(f"开发环境目录: {base_dir}")
|
||
|
||
def resolve_asset_path(filename: str) -> str:
|
||
candidates = [
|
||
os.path.join(base_dir, filename),
|
||
os.path.join(current_dir, filename),
|
||
os.path.join(os.path.dirname(current_dir), filename),
|
||
]
|
||
for candidate in candidates:
|
||
if os.path.exists(candidate):
|
||
return candidate
|
||
return candidates[0]
|
||
|
||
# 构建图标路径
|
||
icon_path = resolve_asset_path("new_logo.svg")
|
||
logger.info(f"SVG图标路径: {icon_path}")
|
||
|
||
if os.path.exists(icon_path):
|
||
logger.info(f"SVG图标文件存在: {icon_path}")
|
||
try:
|
||
icon = QIcon(icon_path)
|
||
if icon.isNull():
|
||
logger.warning(f"SVG图标加载失败，图标为空: {icon_path}")
|
||
else:
|
||
self.setWindowIcon(icon)
|
||
logger.info(f"设置SVG窗口图标成功: {icon_path}")
|
||
except Exception as e:
|
||
logger.error(f"加载SVG图标时出错: {e}")
|
||
else:
|
||
logger.warning(f"SVG图标文件不存在: {icon_path}")
|
||
|
||
# 如果SVG加载失败，使用ico文件作为备用
|
||
ico_icon_path = resolve_asset_path("favicon2.ico")
|
||
logger.info(f"ICO图标路径: {ico_icon_path}")
|
||
|
||
if os.path.exists(ico_icon_path):
|
||
logger.info(f"ICO图标文件存在: {ico_icon_path}")
|
||
try:
|
||
icon = QIcon(ico_icon_path)
|
||
if icon.isNull():
|
||
logger.warning(f"ICO图标加载失败，图标为空: {ico_icon_path}")
|
||
else:
|
||
self.setWindowIcon(icon)
|
||
logger.info(f"设置ICO窗口图标成功: {ico_icon_path}")
|
||
except Exception as e:
|
||
logger.error(f"加载ICO图标时出错: {e}")
|
||
else:
|
||
logger.warning(f"ICO图标文件不存在: {ico_icon_path}")
|
||
|
||
# 设置窗口样式
|
||
self.setStyleSheet(""
|
||
"QMainWindow {"
|
||
" background-color: #f5f5f5;"
|
||
" font-family: 'Microsoft YaHei', Arial, sans-serif;"
|
||
"}"
|
||
"QLabel {"
|
||
" font-size: 14px;"
|
||
" color: #333;"
|
||
" padding: 5px 0;"
|
||
"}"
|
||
"QGroupBox {"
|
||
" font-size: 16px;"
|
||
" font-weight: bold;"
|
||
" border: 1px solid #ddd;"
|
||
" border-radius: 8px;"
|
||
" margin-top: 15px;"
|
||
" padding: 15px;"
|
||
" background-color: #ffffff;"
|
||
"}"
|
||
"QGroupBox::title {"
|
||
" subcontrol-origin: margin;"
|
||
" subcontrol-position: top left;"
|
||
" padding: 0 15px;"
|
||
" background-color: #4CAF50;"
|
||
" color: white;"
|
||
" border-radius: 4px;"
|
||
" font-size: 14px;"
|
||
"}"
|
||
"QProgressBar {"
|
||
" border: 1px solid #ddd;"
|
||
" border-radius: 6px;"
|
||
" text-align: center;"
|
||
" background-color: #f0f0f0;"
|
||
" height: 25px;"
|
||
"}"
|
||
"QProgressBar::chunk {"
|
||
" background-color: #4CAF50;"
|
||
" border-radius: 6px;"
|
||
"}"
|
||
"QTextEdit {"
|
||
" font-family: Consolas, 'Courier New', monospace;"
|
||
" font-size: 12px;"
|
||
" border: 1px solid #ddd;"
|
||
" border-radius: 6px;"
|
||
" background-color: #2d2d2d;"
|
||
" color: #e0e0e0;"
|
||
" padding: 10px;"
|
||
"}"
|
||
"QPushButton {"
|
||
" font-family: 'PingFang SC', 'Helvetica Neue', 'Microsoft YaHei', sans-serif;"
|
||
" font-size: 12px;"
|
||
" font-weight: 600;"
|
||
" padding: 6px 12px;"
|
||
" border: none;"
|
||
" border-radius: 4px;"
|
||
" color: white;"
|
||
" min-width: 80px;"
|
||
" min-height: 28px;"
|
||
" text-align: center;"
|
||
"}"
|
||
"QPushButton:hover {"
|
||
" opacity: 0.9;"
|
||
"}"
|
||
"QPushButton:disabled {"
|
||
" opacity: 0.5;"
|
||
"}"
|
||
"QPushButton#start_button {"
|
||
" background-color: #4CAF50;"
|
||
"}"
|
||
"QPushButton#stop_button {"
|
||
" background-color: #f44336;"
|
||
"}"
|
||
"QPushButton#exit_button {"
|
||
" background-color: #2196F3;"
|
||
"}"
|
||
"")
|
||
|
||
# 创建中心部件 + 内部滚动区域
|
||
scroll_area = QScrollArea()
|
||
scroll_area.setWidgetResizable(True)
|
||
scroll_area.setFrameShape(QFrame.NoFrame)
|
||
scroll_area.setVerticalScrollBarPolicy(Qt.ScrollBarAsNeeded)
|
||
scroll_area.setHorizontalScrollBarPolicy(Qt.ScrollBarAsNeeded)
|
||
self.setCentralWidget(scroll_area)
|
||
|
||
central_widget = QWidget()
|
||
central_widget.setMinimumWidth(920)
|
||
scroll_area.setWidget(central_widget)
|
||
|
||
# 创建布局
|
||
layout = QVBoxLayout(central_widget)
|
||
layout.setSpacing(10)
|
||
layout.setContentsMargins(20, 20, 20, 20)
|
||
|
||
# 创建状态标签
|
||
self.status_label = QLabel("状态: 就绪")
|
||
self.status_label.setStyleSheet("font-family: 'PingFang SC', 'Helvetica Neue', 'Microsoft YaHei', sans-serif; font-size: 14px; font-weight: 600; color: #2c3e50;")
|
||
layout.addWidget(self.status_label)
|
||
|
||
# 创建配置信息显示区域
|
||
config_group = QGroupBox("配置信息")
|
||
config_group.setStyleSheet("""
|
||
QGroupBox {
|
||
font-size: 16px;
|
||
font-weight: 600;
|
||
color: #2c3e50;
|
||
border: 1px solid #e2e8f0;
|
||
border-radius: 10px;
|
||
margin-top: 20px;
|
||
padding: 0;
|
||
background-color: #ffffff;
|
||
}
|
||
QGroupBox::title {
|
||
subcontrol-origin: margin;
|
||
subcontrol-position: top left;
|
||
padding: 8px 20px;
|
||
background: linear-gradient(135deg, #667eea 0%, #764ba2 100%);
|
||
color: white;
|
||
border-radius: 10px 10px 0 0;
|
||
font-family: 'PingFang SC', 'Helvetica Neue', 'Microsoft YaHei', sans-serif;
|
||
font-size: 14px;
|
||
font-weight: 600;
|
||
}
|
||
""")
|
||
config_layout = QVBoxLayout()
|
||
config_layout.setSpacing(12)
|
||
config_layout.setContentsMargins(20, 20, 20, 20)
|
||
|
||
# 创建配置标签
|
||
self.config_labels = {
|
||
'detect_options': QLabel("检测选项: 加载中..."),
|
||
'proxy_config': QLabel("代理配置: 加载中..."),
|
||
'thread_count': QLabel("检测线程数: 加载中...")
|
||
}
|
||
|
||
# 添加配置标签到布局
|
||
for label in self.config_labels.values():
|
||
label.setStyleSheet("""
|
||
QLabel {
|
||
font-family: 'PingFang SC', 'Helvetica Neue', 'Microsoft YaHei', sans-serif;
|
||
font-size: 13px;
|
||
font-weight: 400;
|
||
color: #4a5568;
|
||
background-color: #f7fafc;
|
||
padding: 12px 18px;
|
||
border-radius: 8px;
|
||
border-left: 4px solid #667eea;
|
||
margin: 0;
|
||
min-height: 64px;
|
||
white-space: normal;
|
||
border: 1px solid #e2e8f0;
|
||
}
|
||
QLabel:hover {
|
||
background-color: #edf2f7;
|
||
border-left-color: #764ba2;
|
||
}
|
||
""")
|
||
label.setWordWrap(True) # 启用自动换行
|
||
label.setAlignment(Qt.AlignLeft | Qt.AlignTop)
|
||
config_layout.addWidget(label)
|
||
|
||
config_group.setLayout(config_layout)
|
||
layout.addWidget(config_group)
|
||
|
||
# 创建进度条
|
||
self.progress_bar = QProgressBar()
|
||
self.progress_bar.setValue(0)
|
||
layout.addWidget(self.progress_bar)
|
||
|
||
# 创建日志文本框
|
||
self.log_text = QTextEdit()
|
||
self.log_text.setReadOnly(True)
|
||
self.log_text.setStyleSheet("font-family: Consolas, monospace; font-size: 12px; border: 1px solid #ddd; border-radius: 5px; background-color: #2d2d2d; color: #e0e0e0;")
|
||
self.log_text.setMinimumHeight(320)
|
||
layout.addWidget(self.log_text, 1)
|
||
|
||
# 创建按钮布局
|
||
button_layout = QHBoxLayout()
|
||
button_layout.setSpacing(10)
|
||
|
||
use_fancy_button_effects = os.environ.get("QT_QPA_PLATFORM", "").lower() != "offscreen"
|
||
|
||
# 按钮样式表
|
||
start_button_style = self._button_style(
|
||
"#28a745",
|
||
"#218838",
|
||
"#1e7e34",
|
||
fancy_effects=use_fancy_button_effects,
|
||
)
|
||
|
||
refresh_button_style = """
|
||
QPushButton {
|
||
background-color: #6f42c1;
|
||
color: white;
|
||
border: none;
|
||
border-radius: 8px;
|
||
font-size: 16px;
|
||
font-weight: bold;
|
||
padding: 10px 20px;
|
||
min-width: 120px;
|
||
}
|
||
QPushButton:hover {
|
||
background-color: #5a32a3;
|
||
}
|
||
QPushButton:disabled {
|
||
background-color: #6c757d;
|
||
color: #adb5bd;
|
||
}
|
||
"""
|
||
stop_button_style = self._button_style(
|
||
"#dc3545",
|
||
"#c82333",
|
||
"#a71e2a",
|
||
fancy_effects=use_fancy_button_effects,
|
||
)
|
||
|
||
exit_button_style = self._button_style(
|
||
"#007bff",
|
||
"#0069d9",
|
||
"#0056b3",
|
||
fancy_effects=use_fancy_button_effects,
|
||
)
|
||
|
||
# 创建开始检测按钮
|
||
self.start_button = QPushButton("开始检测")
|
||
self.start_button.setObjectName("start_button")
|
||
self.start_button.clicked.connect(self.start_detection)
|
||
self.start_button.setFixedHeight(50)
|
||
self.start_button.setStyleSheet(start_button_style)
|
||
button_layout.addWidget(self.start_button)
|
||
|
||
# 创建停止按钮
|
||
self.stop_button = QPushButton("停止检测")
|
||
self.stop_button.setObjectName("stop_button")
|
||
self.stop_button.clicked.connect(self.stop_detection)
|
||
self.stop_button.setEnabled(False)
|
||
self.stop_button.setFixedHeight(50)
|
||
self.stop_button.setStyleSheet(stop_button_style)
|
||
button_layout.addWidget(self.stop_button)
|
||
|
||
self.refresh_proxy_button = QPushButton("刷新代理池")
|
||
self.refresh_proxy_button.clicked.connect(self.refresh_proxy_pool_manually)
|
||
self.refresh_proxy_button.setFixedHeight(50)
|
||
self.refresh_proxy_button.setStyleSheet(refresh_button_style)
|
||
button_layout.addWidget(self.refresh_proxy_button)
|
||
|
||
# 创建退出按钮
|
||
self.exit_button = QPushButton("退出")
|
||
self.exit_button.setObjectName("exit_button")
|
||
self.exit_button.clicked.connect(self.close)
|
||
self.exit_button.setFixedHeight(50)
|
||
self.exit_button.setStyleSheet(exit_button_style)
|
||
button_layout.addWidget(self.exit_button)
|
||
|
||
layout.addLayout(button_layout)
|
||
|
||
# 初始化检测端
|
||
self.worker = DetectWorker(self)
|
||
|
||
# 启动Redis订阅线程（如果使用Redis）
|
||
if self.worker.use_redis:
|
||
self.worker.running = True
|
||
self.worker.redis_sub_thread = threading.Thread(target=self.worker.start_redis_subscription)
|
||
self.worker.redis_sub_thread.daemon = True
|
||
self.worker.redis_sub_thread.start()
|
||
logger.debug("Redis配置更新订阅已启动")
|
||
|
||
# 连接信号
|
||
self.worker.connect_signals()
|
||
|
||
# 初始化检测线程
|
||
self.detect_thread = None
|
||
|
||
# 初始化日志队列和处理线程
|
||
self.log_queue = Queue()
|
||
self.log_processing = True
|
||
# 启动日志处理线程
|
||
self.log_thread = threading.Thread(target=self.process_log_queue)
|
||
self.log_thread.daemon = True
|
||
self.log_thread.start()
|
||
|
||
# 配置日志
|
||
logger.add(self.enqueue_log_message, level="INFO")
|
||
|
||
# 启动定时任务
|
||
self.scheduler_thread = threading.Thread(target=self.start_scheduler)
|
||
self.scheduler_thread.daemon = True
|
||
self.scheduler_thread.start()
|
||
|
||
def enqueue_log_message(self, message):
|
||
"""
|
||
日志回调函数，将日志消息放入队列
|
||
"""
|
||
self.log_queue.put(message)
|
||
|
||
def process_log_queue(self):
|
||
"""
|
||
处理日志队列中的消息
|
||
"""
|
||
while self.log_processing:
|
||
try:
|
||
message = self.log_queue.get(timeout=1)
|
||
# 提取时间和消息部分
|
||
# 日志格式: 2026-04-13 00:22:05.365 | INFO | __main__:load_thread_count:846 - 从Redis加载检测线程数成功: 500
|
||
import re
|
||
match = re.match(r'(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d{3}) \| .*? \| .*? - (.*)', message)
|
||
if match:
|
||
time_str = match.group(1)
|
||
msg_str = match.group(2)
|
||
log_text = f"{time_str} | {msg_str}"
|
||
else:
|
||
log_text = message.replace("\n", "")
|
||
# 使用QMetaObject.invokeMethod确保线程安全的GUI更新
|
||
QMetaObject.invokeMethod(self.log_text, "append", Qt.QueuedConnection, Q_ARG(str, log_text))
|
||
QMetaObject.invokeMethod(self.log_text.verticalScrollBar(), "setValue", Qt.QueuedConnection, Q_ARG(int, self.log_text.verticalScrollBar().maximum()))
|
||
except Exception as e:
|
||
pass
|
||
|
||
def log_message(self, message):
|
||
"""
|
||
日志回调函数（保留用于信号连接）
|
||
"""
|
||
self.enqueue_log_message(message)
|
||
|
||
def start_detection(self):
|
||
"""
|
||
开始检测
|
||
"""
|
||
self.status_label.setText("状态: 检测中...")
|
||
self.start_button.setEnabled(False)
|
||
self.stop_button.setEnabled(True)
|
||
|
||
# 创建并启动检测线程
|
||
self.detect_thread = DetectThread(self.worker)
|
||
self.detect_thread.log_signal.connect(self.log_message)
|
||
self.detect_thread.progress_signal.connect(self.update_progress)
|
||
self.detect_thread.thread_count_signal.connect(self.update_thread_count_display)
|
||
self.detect_thread.finished_signal.connect(self.detection_finished)
|
||
self.detect_thread.start()
|
||
|
||
def stop_detection(self):
|
||
"""
|
||
停止检测
|
||
"""
|
||
self.status_label.setText("状态: 停止中...")
|
||
self.worker.stop()
|
||
self.stop_button.setEnabled(False)
|
||
|
||
def detection_finished(self):
|
||
"""
|
||
检测完成
|
||
"""
|
||
self.status_label.setText("状态: 检测完成")
|
||
self.start_button.setEnabled(True)
|
||
self.stop_button.setEnabled(False)
|
||
|
||
def update_progress(self, current, total):
|
||
"""
|
||
更新进度条
|
||
"""
|
||
# 直接在当前线程执行，避免使用QMetaObject.invokeMethod
|
||
try:
|
||
if total > 0:
|
||
progress = int((current / total) * 100)
|
||
self.progress_bar.setValue(progress)
|
||
except Exception as e:
|
||
print(f"更新进度失败: {e}")
|
||
|
||
def update_thread_count_display(self, active_count, total_count):
|
||
"""
|
||
更新线程数量显示
|
||
|
||
:param active_count: 当前活跃线程数量
|
||
:param total_count: 总线程数量
|
||
"""
|
||
try:
|
||
# 检查是否有配置标签
|
||
if hasattr(self, 'config_labels') and 'thread_count' in self.config_labels:
|
||
self.config_labels['thread_count'].setText(f"检测线程数: {total_count} (当前: {active_count})")
|
||
except Exception as e:
|
||
print(f"更新线程数量显示失败: {e}")
|
||
|
||
def refresh_proxy_pool_manually(self):
|
||
self.status_label.setText("状态: 正在刷新代理池...")
|
||
self.refresh_proxy_button.setEnabled(False)
|
||
|
||
def run_refresh():
|
||
try:
|
||
self.worker.refresh_proxy_pool()
|
||
with self.worker.proxy_pool_lock:
|
||
proxy_count = len(self.worker.proxy_pool)
|
||
self.worker.update_config_labels()
|
||
QMetaObject.invokeMethod(
|
||
self.status_label,
|
||
"setText",
|
||
Qt.QueuedConnection,
|
||
Q_ARG(str, f"状态: 代理池刷新完成，可用代理 {proxy_count} 个")
|
||
)
|
||
except Exception as e:
|
||
QMetaObject.invokeMethod(
|
||
self.status_label,
|
||
"setText",
|
||
Qt.QueuedConnection,
|
||
Q_ARG(str, f"状态: 刷新代理池失败: {e}")
|
||
)
|
||
finally:
|
||
QMetaObject.invokeMethod(
|
||
self.refresh_proxy_button,
|
||
"setEnabled",
|
||
Qt.QueuedConnection,
|
||
Q_ARG(bool, True)
|
||
)
|
||
|
||
threading.Thread(target=run_refresh, daemon=True).start()
|
||
|
||
def start_scheduler(self):
|
||
"""
|
||
启动定时任务
|
||
"""
|
||
# 每天凌晨1点执行检测
|
||
schedule.every().day.at("01:00").do(self.scheduled_detection)
|
||
|
||
# 循环执行定时任务
|
||
while True:
|
||
schedule.run_pending()
|
||
time.sleep(60)
|
||
|
||
def scheduled_detection(self):
|
||
"""
|
||
定时检测任务
|
||
"""
|
||
self.log_message(f"[{datetime.now()}] 开始定时检测任务")
|
||
self.start_detection()
|
||
|
||
def refresh_config(self):
|
||
"""
|
||
刷新配置
|
||
"""
|
||
try:
|
||
# 重新加载配置
|
||
new_detect_options = self.worker.load_detect_options()
|
||
new_proxy_config = self.worker.load_proxy_config()
|
||
new_thread_count = self.worker.load_thread_count()
|
||
self.worker.load_cookies_from_remote()
|
||
|
||
# 检查配置是否发生变化
|
||
config_changed = False
|
||
if new_detect_options != self.worker.detect_options:
|
||
self.worker.detect_options = new_detect_options
|
||
config_changed = True
|
||
if new_proxy_config != self.worker.proxy_config:
|
||
self.worker.proxy_config = new_proxy_config
|
||
config_changed = True
|
||
if new_thread_count != self.worker.thread_count:
|
||
self.worker.thread_count = new_thread_count
|
||
config_changed = True
|
||
|
||
# 如果配置发生变化，更新标签
|
||
if config_changed:
|
||
self.worker.update_config_labels()
|
||
self.log_message(f"[{datetime.now()}] 配置已更新")
|
||
except Exception as e:
|
||
self.log_message(f"[{datetime.now()}] 刷新配置失败: {e}")
|
||
|
||
def event(self, event):
|
||
"""
|
||
处理事件
|
||
"""
|
||
if event.type() == ConfigUpdateEvent.Type:
|
||
self.update_config_labels()
|
||
return True
|
||
return super().event(event)
|
||
|
||
def update_config_labels(self):
|
||
"""
|
||
更新GUI配置标签
|
||
"""
|
||
logger.debug("开始更新GUI配置标签")
|
||
try:
|
||
if self.worker:
|
||
try:
|
||
logger.debug(f"worker对象存在，当前配置: {self.worker.detect_options}, {self.worker.proxy_config}, {self.worker.thread_count}")
|
||
# 准备更新数据
|
||
detect_options = self.worker.detect_options
|
||
free_options_display = []
|
||
paid_options_display = []
|
||
option_names = {
|
||
'detect_register': '检查注册',
|
||
'detect_baidu_site': '百度site查询',
|
||
'detect_360_site': '360的site查询',
|
||
'detect_chinaz': '站长之家查询',
|
||
'detect_aizhan': '爱站网查询',
|
||
'detect_wayback': '时光机检测',
|
||
'detect_jucha': '聚查查询',
|
||
'detect_juziseo': '桔子查询'
|
||
}
|
||
|
||
ordered_keys = detect_options.get('detect_order') or list(option_names.keys())
|
||
normalized_keys = [key for key in ordered_keys if key in option_names]
|
||
for key in option_names:
|
||
if key not in normalized_keys:
|
||
normalized_keys.append(key)
|
||
|
||
free_keys = {
|
||
'detect_register',
|
||
'detect_baidu_site',
|
||
'detect_360_site',
|
||
'detect_chinaz',
|
||
'detect_aizhan',
|
||
'detect_wayback',
|
||
}
|
||
|
||
for index, key in enumerate(normalized_keys, start=1):
|
||
name = option_names[key]
|
||
if key in detect_options and detect_options[key]:
|
||
item_text = f"{index}.{name}"
|
||
if key in free_keys:
|
||
free_options_display.append(item_text)
|
||
else:
|
||
paid_options_display.append(item_text)
|
||
|
||
free_text = " | ".join(free_options_display) if free_options_display else "无"
|
||
paid_text = " | ".join(paid_options_display) if paid_options_display else "无"
|
||
detect_options_text = (
|
||
"当前检测选项:\n"
|
||
f"免费优先: {free_text}\n"
|
||
f"付费后置: {paid_text}"
|
||
)
|
||
|
||
# 准备代理配置文本
|
||
proxy_enable = self.worker.proxy_config.get('proxy_enable', False)
|
||
allow_direct = self.worker.proxy_config.get('allow_direct', False)
|
||
proxy_urls = self.worker.proxy_config.get('proxy_urls') or []
|
||
if not proxy_urls and self.worker.proxy_config.get('proxy_url'):
|
||
proxy_urls = [self.worker.proxy_config.get('proxy_url', '')]
|
||
with self.worker.proxy_pool_lock:
|
||
proxy_pool_count = len(self.worker.proxy_pool)
|
||
last_refresh_time = (
|
||
self.worker.proxy_last_refresh_time.strftime('%Y-%m-%d %H:%M:%S')
|
||
if self.worker.proxy_last_refresh_time else '未刷新'
|
||
)
|
||
proxy_display = (
|
||
f"启用: {'是' if proxy_enable else '否'} | "
|
||
f"允许直连: {'是' if allow_direct else '否'} | "
|
||
f"代理池链接数: {len([url for url in proxy_urls if url])} | "
|
||
f"当前可用代理数: {proxy_pool_count}\n"
|
||
f"最近刷新: {last_refresh_time} | "
|
||
f"最近结果: {self.worker.proxy_last_refresh_status} | "
|
||
f"原始返回数: {self.worker.proxy_last_refresh_total_items}"
|
||
)
|
||
proxy_text = f"代理配置: {proxy_display}"
|
||
|
||
# 准备线程数文本
|
||
thread_count_text = f"检测线程数: {self.worker.thread_count}"
|
||
|
||
logger.debug(f"准备更新标签: {detect_options_text}, {proxy_text}, {thread_count_text}")
|
||
|
||
# 检查配置标签是否存在
|
||
if hasattr(self, 'config_labels'):
|
||
# 使用QMetaObject.invokeMethod确保在主线程中更新UI
|
||
from PySide6.QtCore import QMetaObject, Qt, Q_ARG
|
||
|
||
# 更新检测选项标签
|
||
if 'detect_options' in self.config_labels:
|
||
QMetaObject.invokeMethod(self.config_labels['detect_options'], "setText",
|
||
Qt.QueuedConnection,
|
||
Q_ARG(str, detect_options_text))
|
||
|
||
# 更新代理配置标签
|
||
if 'proxy_config' in self.config_labels:
|
||
QMetaObject.invokeMethod(self.config_labels['proxy_config'], "setText",
|
||
Qt.QueuedConnection,
|
||
Q_ARG(str, proxy_text))
|
||
|
||
# 更新线程数标签
|
||
if 'thread_count' in self.config_labels:
|
||
QMetaObject.invokeMethod(self.config_labels['thread_count'], "setText",
|
||
Qt.QueuedConnection,
|
||
Q_ARG(str, thread_count_text))
|
||
|
||
logger.debug("GUI配置标签更新完成")
|
||
else:
|
||
logger.debug("配置标签不存在，跳过更新")
|
||
except Exception as e:
|
||
logger.error(f"更新GUI标签失败: {e}")
|
||
import traceback
|
||
logger.error(traceback.format_exc())
|
||
else:
|
||
logger.debug("worker对象不存在")
|
||
except Exception as e:
|
||
logger.error(f"执行update_config_labels失败: {e}")
|
||
import traceback
|
||
logger.error(traceback.format_exc())
|
||
|
||
from PySide6.QtCore import QObject, Signal, QEvent
|
||
|
||
class ConfigUpdateEvent(QEvent):
|
||
"""
|
||
配置更新事件类
|
||
"""
|
||
Type = QEvent.Type(QEvent.User + 1)
|
||
|
||
def __init__(self):
|
||
super().__init__(ConfigUpdateEvent.Type)
|
||
|
||
class ConfigUpdateSignal(QObject):
|
||
"""
|
||
配置更新信号类
|
||
"""
|
||
config_updated = Signal()
|
||
|
||
class DetectWorker:
|
||
"""
|
||
域名检测端工作类
|
||
"""
|
||
def __init__(self, main_window=None):
|
||
"""
|
||
初始化检测端
|
||
|
||
:param main_window: 主窗口实例，用于更新GUI
|
||
"""
|
||
self.main_window = main_window
|
||
self.running = False
|
||
self.stop_requested = False
|
||
self.detecting = False
|
||
self.detect_threads = []
|
||
self.detect_thread = None # 检测线程实例
|
||
self.detect_command_thread = None
|
||
self.detect_lock = threading.Lock()
|
||
self.runtime_heartbeat_interval = 30
|
||
self._runtime_heartbeat_stop = threading.Event()
|
||
self._last_runtime_phase = "idle"
|
||
self._last_runtime_detail = "Worker 初始化中"
|
||
self._last_runtime_extra = {}
|
||
self.current_cycle_token = ""
|
||
self.current_job_id = None
|
||
self.current_job_code = ""
|
||
self.runtime_settings = {
|
||
"worker_log_sync_enabled": False,
|
||
"worker_log_sync_mode": "key",
|
||
}
|
||
self._last_synced_worker_log = ""
|
||
|
||
# 初始化数据库连接
|
||
self.db = Database()
|
||
try:
|
||
self.db.ensure_cluster_runtime_tables()
|
||
self.db.register_cluster_node(
|
||
config.NODE_CODE,
|
||
config.NODE_REGION,
|
||
config.NODE_ROLE,
|
||
status="online",
|
||
current_load=0,
|
||
metadata={"service": "detect-worker", "worker_mode": os.environ.get("WORKER_MODE", "")},
|
||
)
|
||
except Exception as e:
|
||
logger.warning(f"初始化多机运行库骨架失败: {e}")
|
||
try:
|
||
recycled = self.db.recycle_running_domains(DETECT_STATUS_FAILED)
|
||
if recycled:
|
||
logger.warning(f"Worker 启动时回收了 {recycled} 个遗留的检测中域名，已标记为失败待重试")
|
||
released = self.db.release_detect_job_items_for_node(config.NODE_CODE)
|
||
if released:
|
||
logger.warning(f"Worker 启动时释放了 {released} 个当前节点遗留任务项，已重新回到 pending")
|
||
except Exception as e:
|
||
logger.warning(f"Worker 启动时清理遗留运行态失败: {e}")
|
||
|
||
# 初始化Redis连接
|
||
try:
|
||
self.redis_client = redis.Redis(
|
||
host=config.REDIS_HOST,
|
||
port=config.REDIS_PORT,
|
||
password=config.REDIS_PASSWORD,
|
||
db=config.REDIS_DB,
|
||
decode_responses=True,
|
||
socket_connect_timeout=10,
|
||
socket_timeout=10,
|
||
retry_on_timeout=True,
|
||
health_check_interval=30
|
||
)
|
||
# 测试连接
|
||
self.redis_client.ping()
|
||
logger.info(f"Redis 连接成功: {config.REDIS_HOST}:{config.REDIS_PORT}")
|
||
self.use_redis = True
|
||
except Exception as e:
|
||
logger.warning(f"Redis 连接失败: {e}，将使用本地配置")
|
||
self.redis_client = None
|
||
self.use_redis = False
|
||
|
||
# 初始化配置更新信号
|
||
self.config_signal = ConfigUpdateSignal()
|
||
|
||
# 加载配置
|
||
self.detect_options = self.load_detect_options()
|
||
self.proxy_config = self.load_proxy_config()
|
||
self.thread_count = self.load_thread_count() # 从配置文件加载线程数
|
||
self.runtime_settings = self.load_runtime_settings()
|
||
|
||
# 初始化代理池
|
||
self.proxy_pool = []
|
||
self.proxy_pool_lock = threading.Lock()
|
||
self.proxy_refresh_lock = threading.Lock()
|
||
self.proxy_last_refresh_time = None
|
||
self.proxy_last_refresh_status = "未刷新"
|
||
self.proxy_last_refresh_source_count = 0
|
||
self.proxy_last_refresh_total_items = 0
|
||
self.proxy_last_validated_count = 0
|
||
self.proxy_last_available_count = 0
|
||
self.proxy_last_source_stats = []
|
||
self.proxy_refresh_cooldown_seconds = 30
|
||
self.proxy_next_refresh_time = 0.0
|
||
self.proxy_max_reuse_count = max(8, self.thread_count * 4)
|
||
self.proxy_failure_lock = threading.Lock()
|
||
self.proxy_failure_counts = {}
|
||
self.proxy_quarantine_until = {}
|
||
|
||
# 加载敏感词（只加载一次，所有线程共享）
|
||
try:
|
||
self.sensitive_words = self.db.get_all_sensitive_words()
|
||
logger.info(f"加载敏感词完成，共 {len(self.sensitive_words)} 个敏感词")
|
||
except Exception as e:
|
||
logger.error(f"加载敏感词失败: {e}")
|
||
import traceback
|
||
logger.error(traceback.format_exc())
|
||
self.sensitive_words = []
|
||
|
||
# 初始化运行状态
|
||
self.running = False
|
||
|
||
# 加载cookies
|
||
self.load_cookies_from_remote()
|
||
|
||
# 付费检测器按需初始化，避免在默认关闭时产生无效启动告警。
|
||
self.jc_instance = None
|
||
self.juziseo_instance = None
|
||
self._init_optional_detectors()
|
||
self.wayback_detector = WaybackDetector()
|
||
|
||
# 显示所有配置信息
|
||
logger.info("检测端配置信息:")
|
||
logger.info(f"检测选项: {self.detect_options}")
|
||
logger.info(f"代理配置: {self.proxy_config}")
|
||
logger.info(f"检测线程数: {self.thread_count}")
|
||
|
||
# Redis订阅线程将在start方法中启动
|
||
self._update_runtime_state("idle", "Worker 已启动，等待检测指令")
|
||
self._start_runtime_heartbeat_loop()
|
||
|
||
def _start_runtime_heartbeat_loop(self):
|
||
existing = getattr(self, "_runtime_heartbeat_thread", None)
|
||
if existing and existing.is_alive():
|
||
return
|
||
self._runtime_heartbeat_stop.clear()
|
||
self._runtime_heartbeat_thread = threading.Thread(
|
||
target=self._runtime_heartbeat_loop,
|
||
name="RuntimeHeartbeatThread",
|
||
daemon=True,
|
||
)
|
||
self._runtime_heartbeat_thread.start()
|
||
|
||
def _runtime_heartbeat_loop(self):
|
||
while not self._runtime_heartbeat_stop.wait(self.runtime_heartbeat_interval):
|
||
try:
|
||
self._update_runtime_state(
|
||
self._last_runtime_phase,
|
||
self._last_runtime_detail,
|
||
**dict(self._last_runtime_extra or {}),
|
||
)
|
||
except Exception as e:
|
||
logger.debug(f"运行态心跳续期失败: {e}")
|
||
|
||
def _consume_pending_control_command(self):
|
||
if not self.use_redis or self.redis_client is None:
|
||
return
|
||
try:
|
||
payload = self.redis_client.getdel(PENDING_CONTROL_KEY)
|
||
except Exception:
|
||
try:
|
||
payload = self.redis_client.get(PENDING_CONTROL_KEY)
|
||
if payload:
|
||
self.redis_client.delete(PENDING_CONTROL_KEY)
|
||
except Exception as e:
|
||
logger.debug(f"读取待执行控制指令失败: {e}")
|
||
return
|
||
if not payload:
|
||
return
|
||
logger.info(f"发现待执行 Worker 控制指令: {payload}")
|
||
self._handle_control_message(payload)
|
||
|
||
def _update_runtime_state(self, phase: str, detail: str, **extra):
|
||
"""
|
||
更新 Redis 中的检测运行态，供 Web 后台读取。
|
||
"""
|
||
self._last_runtime_phase = phase
|
||
self._last_runtime_detail = detail
|
||
self._last_runtime_extra = dict(extra or {})
|
||
if not self.use_redis or self.redis_client is None:
|
||
return
|
||
with self.proxy_pool_lock:
|
||
available_proxy_count = len(self.proxy_pool)
|
||
payload_warning = ""
|
||
if self.proxy_config.get('proxy_enable', False) and available_proxy_count <= 0:
|
||
if (
|
||
self.proxy_last_refresh_status
|
||
and self.proxy_last_refresh_status not in {"未刷新", "代理未启用", "未配置代理池链接"}
|
||
and "刷新成功" not in self.proxy_last_refresh_status
|
||
):
|
||
payload_warning = self.proxy_last_refresh_status
|
||
payload = {
|
||
"phase": phase,
|
||
"detail": detail,
|
||
"service_running": bool(self.running),
|
||
"detecting": bool(self.detecting),
|
||
"stop_requested": bool(self.stop_requested),
|
||
"available_proxy_count": available_proxy_count,
|
||
"proxy_last_refresh_status": self.proxy_last_refresh_status,
|
||
"proxy_last_refresh_time": self.proxy_last_refresh_time.strftime("%Y-%m-%d %H:%M:%S") if self.proxy_last_refresh_time else "",
|
||
"proxy_last_refresh_source_count": int(self.proxy_last_refresh_source_count or 0),
|
||
"proxy_last_refresh_total_items": int(self.proxy_last_refresh_total_items or 0),
|
||
"proxy_last_validated_count": int(self.proxy_last_validated_count or 0),
|
||
"proxy_last_available_count": int(self.proxy_last_available_count or 0),
|
||
"proxy_last_source_stats": list(self.proxy_last_source_stats or []),
|
||
"recent_warning": payload_warning,
|
||
"updated_at": datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
|
||
}
|
||
payload.update(extra)
|
||
if self.current_cycle_token and not payload.get("cycle_token"):
|
||
payload["cycle_token"] = self.current_cycle_token
|
||
if self.current_job_id and not payload.get("job_id"):
|
||
payload["job_id"] = self.current_job_id
|
||
if self.current_job_code and not payload.get("job_code"):
|
||
payload["job_code"] = self.current_job_code
|
||
try:
|
||
self.redis_client.set(RUNTIME_STATE_KEY, json.dumps(payload, ensure_ascii=False))
|
||
except Exception as e:
|
||
logger.debug(f"写入检测运行态失败: {e}")
|
||
try:
|
||
current_load = int(extra.get("active_threads", payload.get("active_threads", 0)) or 0)
|
||
if bool(payload.get("detecting")) and current_load <= 0:
|
||
current_load = 1
|
||
self.db.register_cluster_node(
|
||
config.NODE_CODE,
|
||
config.NODE_REGION,
|
||
config.NODE_ROLE,
|
||
status="busy" if payload.get("detecting") else "online",
|
||
current_load=current_load,
|
||
metadata={
|
||
"phase": phase,
|
||
"detail": detail,
|
||
"detecting": bool(payload.get("detecting")),
|
||
"available_proxy_count": int(payload.get("available_proxy_count", 0) or 0),
|
||
"active_threads": current_load,
|
||
"cycle_token": str(payload.get("cycle_token") or ""),
|
||
"job_id": payload.get("job_id"),
|
||
"job_code": str(payload.get("job_code") or ""),
|
||
"proxy_runtime": {
|
||
"sources": int(payload.get("proxy_last_refresh_source_count", 0) or 0),
|
||
"raw_items": int(payload.get("proxy_last_refresh_total_items", 0) or 0),
|
||
"validated": int(payload.get("proxy_last_validated_count", 0) or 0),
|
||
"available": int(payload.get("proxy_last_available_count", 0) or 0),
|
||
},
|
||
},
|
||
)
|
||
except Exception as e:
|
||
logger.debug(f"更新检测节点心跳失败: {e}")
|
||
|
||
def _mark_detection_phase(self, phase: str, detail: str, **extra):
|
||
self._update_runtime_state(phase, detail, **extra)
|
||
|
||
def _set_active_cycle_context(self, control_payload=None):
|
||
payload = control_payload or {}
|
||
self._last_synced_worker_log = ""
|
||
self.current_cycle_token = str(payload.get("cycle_token") or "").strip()
|
||
job_id = payload.get("job_id")
|
||
try:
|
||
self.current_job_id = int(job_id) if job_id not in (None, "", 0, "0") else None
|
||
except Exception:
|
||
self.current_job_id = None
|
||
self.current_job_code = str(payload.get("job_code") or "").strip()
|
||
|
||
def _clear_active_cycle_context(self):
|
||
self._last_synced_worker_log = ""
|
||
self.current_cycle_token = ""
|
||
self.current_job_id = None
|
||
self.current_job_code = ""
|
||
|
||
def start_detection_async(self, source: str = "remote", control_payload=None):
|
||
"""
|
||
异步启动一次检测任务，避免阻塞 Redis 订阅线程。
|
||
"""
|
||
with self.detect_lock:
|
||
if self.detect_command_thread and self.detect_command_thread.is_alive():
|
||
logger.warning("收到启动检测指令，但检测任务已在运行，忽略重复启动")
|
||
self._update_runtime_state("running", "检测任务已在运行，忽略重复启动")
|
||
return False
|
||
|
||
self.stop_requested = False
|
||
self.detect_command_thread = threading.Thread(
|
||
target=self._run_detection_session,
|
||
kwargs={"source": source, "control_payload": dict(control_payload or {})},
|
||
name="RemoteDetectCommandThread",
|
||
daemon=True,
|
||
)
|
||
self.detect_command_thread.start()
|
||
logger.info(f"已接受检测启动指令，来源: {source}")
|
||
self._set_active_cycle_context(control_payload)
|
||
self._update_runtime_state(
|
||
"starting",
|
||
f"已接受检测启动指令，来源: {source}",
|
||
source=source,
|
||
cycle_token=self.current_cycle_token,
|
||
job_id=self.current_job_id,
|
||
job_code=self.current_job_code,
|
||
)
|
||
return True
|
||
|
||
def _run_detection_session(self, source: str = "remote", control_payload=None):
|
||
self._set_active_cycle_context(control_payload)
|
||
with self.detect_lock:
|
||
self.detecting = True
|
||
try:
|
||
logger.info(f"开始执行远程检测任务，来源: {source}")
|
||
self._sync_worker_log_event(f"开始执行检测任务，来源: {source}", payload={"source": source}, mode='key')
|
||
self._update_runtime_state(
|
||
"starting",
|
||
f"开始执行检测任务，来源: {source}",
|
||
source=source,
|
||
cycle_token=self.current_cycle_token,
|
||
job_id=self.current_job_id,
|
||
job_code=self.current_job_code,
|
||
)
|
||
self.start_detection()
|
||
except Exception as e:
|
||
logger.error(f"远程检测任务执行失败: {e}")
|
||
import traceback
|
||
logger.error(traceback.format_exc())
|
||
self._update_runtime_state(
|
||
"failed",
|
||
f"检测任务执行失败: {e}",
|
||
source=source,
|
||
cycle_token=self.current_cycle_token,
|
||
job_id=self.current_job_id,
|
||
job_code=self.current_job_code,
|
||
)
|
||
finally:
|
||
with self.detect_lock:
|
||
self.detecting = False
|
||
self.detect_command_thread = None
|
||
if self.running and not self.stop_requested:
|
||
self._update_runtime_state(
|
||
"idle",
|
||
"检测任务结束，Worker 保持待命",
|
||
source=source,
|
||
cycle_token=self.current_cycle_token,
|
||
job_id=self.current_job_id,
|
||
job_code=self.current_job_code,
|
||
)
|
||
self._clear_active_cycle_context()
|
||
|
||
def request_stop_detection(self, source: str = "remote"):
|
||
"""
|
||
请求停止当前检测任务，但保留 Worker 进程存活。
|
||
"""
|
||
if not self.detecting and not (self.detect_command_thread and self.detect_command_thread.is_alive()):
|
||
logger.info("收到停止检测指令，但当前没有运行中的检测任务")
|
||
self._update_runtime_state("idle", "当前没有运行中的检测任务")
|
||
return False
|
||
logger.info(f"收到停止检测指令，来源: {source}")
|
||
self._sync_worker_log_event(f"收到停止检测指令，来源: {source}", payload={"source": source}, mode='key')
|
||
self.stop_requested = True
|
||
self._update_runtime_state(
|
||
"stopping",
|
||
f"收到停止检测指令，等待当前批次安全退出，来源: {source}",
|
||
cycle_token=self.current_cycle_token,
|
||
job_id=self.current_job_id,
|
||
job_code=self.current_job_code,
|
||
)
|
||
return True
|
||
|
||
def _handle_control_message(self, payload):
|
||
try:
|
||
control_payload = json.loads(payload) if isinstance(payload, str) else payload
|
||
except Exception:
|
||
control_payload = {"action": str(payload)}
|
||
action = str(control_payload.get("action", "")).strip()
|
||
if action == "start_detection":
|
||
self.start_detection_async(source="redis-control", control_payload=control_payload)
|
||
elif action == "stop_detection":
|
||
self.request_stop_detection(source="redis-control")
|
||
elif action:
|
||
logger.warning(f"收到未知 Worker 控制动作: {action}")
|
||
|
||
def connect_signals(self):
|
||
"""
|
||
连接信号到槽函数
|
||
"""
|
||
if self.main_window:
|
||
try:
|
||
# 连接信号到槽函数
|
||
self.config_signal.config_updated.connect(self.main_window.update_config_labels)
|
||
logger.debug("信号连接成功")
|
||
# 连接后立即更新一次配置
|
||
self.update_config_labels()
|
||
except Exception as e:
|
||
logger.error(f"连接信号失败: {e}")
|
||
|
||
def _init_optional_detectors(self):
|
||
if self.detect_options.get('detect_jucha') and self.jc_instance is None:
|
||
self.jc_instance = jucha.JC(proxies=None)
|
||
self.jc_instance.load_juming_cookies()
|
||
self.jc_instance.load_cookies()
|
||
if self.detect_options.get('detect_juziseo') and self.juziseo_instance is None:
|
||
self.juziseo_instance = juziseo.Juziseo(proxies=None)
|
||
self.juziseo_instance.load_cookies()
|
||
|
||
def load_detect_options(self):
|
||
"""
|
||
加载检测选项
|
||
"""
|
||
default_order = [
|
||
'detect_register',
|
||
'detect_baidu_site',
|
||
'detect_360_site',
|
||
'detect_chinaz',
|
||
'detect_aizhan',
|
||
'detect_wayback',
|
||
'detect_jucha',
|
||
'detect_juziseo',
|
||
]
|
||
default_options = {
|
||
'detect_register': True,
|
||
'detect_wayback': True,
|
||
'detect_chinaz': True,
|
||
'detect_aizhan': True,
|
||
'detect_baidu_site': True,
|
||
'detect_360_site': True,
|
||
'detect_jucha': False,
|
||
'detect_juziseo': False,
|
||
'detect_order': default_order,
|
||
}
|
||
try:
|
||
# 从Redis获取配置
|
||
if self.use_redis:
|
||
detect_options_str = self.redis_client.get('domain_tool:detect_options')
|
||
if detect_options_str:
|
||
detect_options = default_options.copy()
|
||
detect_options.update(json.loads(detect_options_str))
|
||
if detect_options.get('detect_whois') or detect_options.get('detect_beian') or detect_options.get('detect_intercept'):
|
||
detect_options['detect_jucha'] = True
|
||
if detect_options.get('detect_juziseo_outlink'):
|
||
detect_options['detect_juziseo'] = True
|
||
order = detect_options.get('detect_order') or []
|
||
detect_options['detect_order'] = [key for key in order if key in default_order]
|
||
for key in default_order:
|
||
if key not in detect_options['detect_order']:
|
||
detect_options['detect_order'].append(key)
|
||
logger.info(f"从Redis加载检测选项成功: {detect_options}")
|
||
return detect_options
|
||
|
||
# 从本地文件获取配置
|
||
if os.path.exists('detect_options.json'):
|
||
with open('detect_options.json', 'r', encoding='utf-8') as f:
|
||
detect_options = default_options.copy()
|
||
detect_options.update(json.load(f))
|
||
if detect_options.get('detect_whois') or detect_options.get('detect_beian') or detect_options.get('detect_intercept'):
|
||
detect_options['detect_jucha'] = True
|
||
if detect_options.get('detect_juziseo_outlink'):
|
||
detect_options['detect_juziseo'] = True
|
||
order = detect_options.get('detect_order') or []
|
||
detect_options['detect_order'] = [key for key in order if key in default_order]
|
||
for key in default_order:
|
||
if key not in detect_options['detect_order']:
|
||
detect_options['detect_order'].append(key)
|
||
logger.info(f"从本地文件加载检测选项成功: {detect_options}")
|
||
return detect_options
|
||
else:
|
||
logger.info(f"使用默认检测选项: {default_options}")
|
||
return default_options
|
||
except Exception as e:
|
||
logger.error(f"加载检测选项失败: {e}")
|
||
logger.info(f"使用默认检测选项: {default_options}")
|
||
return default_options
|
||
|
||
def load_proxy_config(self):
|
||
"""
|
||
加载代理配置
|
||
"""
|
||
default_config = {'proxy_enable': False, 'proxy_url': '', 'proxy_urls': [], 'allow_direct': False}
|
||
try:
|
||
# 从Redis获取配置
|
||
if self.use_redis:
|
||
proxy_config_str = self.redis_client.get('domain_tool:proxy_config')
|
||
if proxy_config_str:
|
||
proxy_config = default_config.copy()
|
||
proxy_config.update(json.loads(proxy_config_str))
|
||
proxy_urls = proxy_config.get('proxy_urls') or []
|
||
if not proxy_urls and proxy_config.get('proxy_url'):
|
||
proxy_urls = [proxy_config.get('proxy_url', '')]
|
||
proxy_config['proxy_urls'] = [url.strip() for url in proxy_urls if url and str(url).strip()]
|
||
proxy_config['proxy_url'] = proxy_config['proxy_urls'][0] if proxy_config['proxy_urls'] else ''
|
||
proxy_config['allow_direct'] = bool(proxy_config.get('allow_direct', False))
|
||
logger.info(f"从Redis加载代理配置成功: {proxy_config}")
|
||
return proxy_config
|
||
|
||
# 从本地文件获取配置
|
||
if os.path.exists('proxy_config.json'):
|
||
with open('proxy_config.json', 'r', encoding='utf-8') as f:
|
||
proxy_config = default_config.copy()
|
||
proxy_config.update(json.load(f))
|
||
logger.info(f"从本地文件加载代理配置成功: {proxy_config}")
|
||
proxy_urls = proxy_config.get('proxy_urls') or []
|
||
if not proxy_urls and proxy_config.get('proxy_url'):
|
||
proxy_urls = [proxy_config.get('proxy_url', '')]
|
||
proxy_config['proxy_urls'] = [url.strip() for url in proxy_urls if url and str(url).strip()]
|
||
proxy_config['proxy_url'] = proxy_config['proxy_urls'][0] if proxy_config['proxy_urls'] else ''
|
||
proxy_config['allow_direct'] = bool(proxy_config.get('allow_direct', False))
|
||
return proxy_config
|
||
else:
|
||
logger.info(f"使用默认代理配置: {default_config}")
|
||
return default_config
|
||
except Exception as e:
|
||
logger.error(f"加载代理配置失败: {e}")
|
||
logger.info(f"使用默认代理配置: {default_config}")
|
||
return default_config
|
||
|
||
def load_thread_count(self):
|
||
"""
|
||
加载检测线程数
|
||
"""
|
||
max_recommended_threads = 20
|
||
try:
|
||
node_code = str(getattr(config, "NODE_CODE", "") or "").strip()
|
||
node_thread_counts = {}
|
||
|
||
# 从Redis获取配置
|
||
if self.use_redis:
|
||
node_thread_counts_raw = self.redis_client.get('domain_tool:node_thread_counts')
|
||
if node_thread_counts_raw:
|
||
try:
|
||
node_thread_counts = json.loads(node_thread_counts_raw)
|
||
except Exception as e:
|
||
logger.warning(f"解析 Redis 节点线程覆盖配置失败: {e}")
|
||
|
||
if node_code and isinstance(node_thread_counts, dict):
|
||
node_thread_count_str = node_thread_counts.get(node_code)
|
||
if node_thread_count_str is not None:
|
||
thread_count = min(max_recommended_threads, max(1, int(node_thread_count_str)))
|
||
logger.info(f"从Redis加载节点专属检测线程数成功: {node_code} -> {thread_count}")
|
||
return thread_count
|
||
|
||
thread_count_str = self.redis_client.get('domain_tool:thread_count')
|
||
if thread_count_str:
|
||
thread_count = min(max_recommended_threads, max(1, int(thread_count_str)))
|
||
logger.info(f"从Redis加载检测线程数成功: {thread_count}")
|
||
return thread_count
|
||
|
||
# 从本地文件获取配置
|
||
if os.path.exists('node_thread_counts.json'):
|
||
with open('node_thread_counts.json', 'r', encoding='utf-8') as f:
|
||
node_thread_counts = json.load(f)
|
||
if node_code and isinstance(node_thread_counts, dict):
|
||
node_thread_count = node_thread_counts.get(node_code)
|
||
if node_thread_count is not None:
|
||
thread_count = min(max_recommended_threads, max(1, int(node_thread_count)))
|
||
logger.info(f"从本地文件加载节点专属检测线程数成功: {node_code} -> {thread_count}")
|
||
return thread_count
|
||
|
||
if os.path.exists('thread_count.json'):
|
||
with open('thread_count.json', 'r', encoding='utf-8') as f:
|
||
thread_config = json.load(f)
|
||
thread_count = thread_config.get('thread_count', '10')
|
||
thread_count = min(max_recommended_threads, max(1, int(thread_count)))
|
||
logger.info(f"从本地文件加载检测线程数成功: {thread_count}")
|
||
return thread_count
|
||
else:
|
||
default_thread_count = 4 # 默认线程数，降低线程数量以减轻系统负担
|
||
logger.info(f"使用默认检测线程数: {default_thread_count}")
|
||
return default_thread_count
|
||
except Exception as e:
|
||
logger.error(f"加载检测线程数失败: {e}")
|
||
default_thread_count = 4 # 默认线程数，降低线程数量以减轻系统负担
|
||
logger.info(f"使用默认检测线程数: {default_thread_count}")
|
||
return default_thread_count
|
||
|
||
def load_runtime_settings(self):
|
||
default_settings = {
|
||
"worker_log_sync_enabled": False,
|
||
"worker_log_sync_mode": "key",
|
||
}
|
||
try:
|
||
if self.use_redis:
|
||
runtime_settings_raw = self.redis_client.get(RUNTIME_SETTINGS_KEY)
|
||
if runtime_settings_raw:
|
||
runtime_settings = default_settings.copy()
|
||
runtime_settings.update(json.loads(runtime_settings_raw))
|
||
runtime_settings["worker_log_sync_enabled"] = bool(runtime_settings.get("worker_log_sync_enabled", False))
|
||
runtime_settings["worker_log_sync_mode"] = "full" if str(runtime_settings.get("worker_log_sync_mode", "key")).strip().lower() == "full" else "key"
|
||
logger.info(f"从Redis加载运行时设置成功: {runtime_settings}")
|
||
return runtime_settings
|
||
|
||
candidate_paths = [
|
||
'runtime_settings.json',
|
||
os.path.join('runtime', 'runtime_settings.json'),
|
||
]
|
||
for candidate_path in candidate_paths:
|
||
if os.path.exists(candidate_path):
|
||
with open(candidate_path, 'r', encoding='utf-8') as f:
|
||
runtime_settings = default_settings.copy()
|
||
runtime_settings.update(json.load(f))
|
||
runtime_settings["worker_log_sync_enabled"] = bool(runtime_settings.get("worker_log_sync_enabled", False))
|
||
runtime_settings["worker_log_sync_mode"] = "full" if str(runtime_settings.get("worker_log_sync_mode", "key")).strip().lower() == "full" else "key"
|
||
logger.info(f"从本地文件加载运行时设置成功: {runtime_settings}")
|
||
return runtime_settings
|
||
except Exception as e:
|
||
logger.error(f"加载运行时设置失败: {e}")
|
||
return default_settings
|
||
|
||
def _worker_log_sync_mode(self):
|
||
if not bool((self.runtime_settings or {}).get("worker_log_sync_enabled", False)):
|
||
return "off"
|
||
return "full" if str((self.runtime_settings or {}).get("worker_log_sync_mode", "key")).strip().lower() == "full" else "key"
|
||
|
||
def _sync_worker_log_event(self, message, level='info', payload=None, *, mode='key', job_item_id=None):
|
||
current_mode = self._worker_log_sync_mode()
|
||
if current_mode == "off":
|
||
return
|
||
if mode == 'full' and current_mode != "full":
|
||
return
|
||
normalized_message = str(message or '').strip()
|
||
if not normalized_message:
|
||
return
|
||
if normalized_message == self._last_synced_worker_log and mode != 'full':
|
||
return
|
||
job_id = self.current_job_id
|
||
if not job_id:
|
||
return
|
||
event_payload = dict(payload or {})
|
||
if self.current_cycle_token and not event_payload.get("cycle_token"):
|
||
event_payload["cycle_token"] = self.current_cycle_token
|
||
if self.current_job_code and not event_payload.get("job_code"):
|
||
event_payload["job_code"] = self.current_job_code
|
||
event_payload["log_mode"] = mode
|
||
event_payload["synced_by"] = "worker_log_callback"
|
||
try:
|
||
self.db.append_detect_run_event(
|
||
job_id,
|
||
job_item_id,
|
||
config.NODE_CODE,
|
||
event_type='worker_log',
|
||
message=normalized_message,
|
||
level=level,
|
||
payload=event_payload,
|
||
)
|
||
self._last_synced_worker_log = normalized_message
|
||
except Exception as e:
|
||
logger.debug(f"回传Worker日志事件失败: {e}")
|
||
|
||
def test_proxy(self, proxy_item, result_queue):
|
||
"""
|
||
测试单个代理的可用性
|
||
|
||
:param proxy_item: 代理信息
|
||
:param result_queue: 结果队列
|
||
"""
|
||
result_payload = {
|
||
"ok": False,
|
||
"proxy": None,
|
||
"reason": "invalid_proxy_item",
|
||
}
|
||
try:
|
||
if 'ip' in proxy_item and 'port' in proxy_item:
|
||
ip = proxy_item['ip']
|
||
port = proxy_item['port']
|
||
username = proxy_item.get('username', '')
|
||
password = proxy_item.get('password', '')
|
||
|
||
if username and password:
|
||
proxy_url = f"http://{username}:{password}@{ip}:{port}"
|
||
else:
|
||
proxy_url = f"http://{ip}:{port}"
|
||
|
||
import requests
|
||
test_proxies = {
|
||
'http': proxy_url,
|
||
'https': proxy_url
|
||
}
|
||
result_payload["proxy"] = test_proxies
|
||
|
||
test_targets = [
|
||
("http://www.baidu.com", 8),
|
||
("https://m.baidu.com", 8),
|
||
]
|
||
last_reason = "unknown"
|
||
session = requests.Session()
|
||
session.headers.update(
|
||
{
|
||
"User-Agent": (
|
||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
|
||
"AppleWebKit/537.36 (KHTML, like Gecko) "
|
||
"Chrome/123.0.0.0 Safari/537.36"
|
||
)
|
||
}
|
||
)
|
||
for target_url, timeout_seconds in test_targets:
|
||
try:
|
||
test_response = session.get(
|
||
target_url,
|
||
proxies=test_proxies,
|
||
timeout=timeout_seconds,
|
||
allow_redirects=True,
|
||
verify=False if target_url.startswith("https://") else True,
|
||
)
|
||
if test_response.status_code in (200, 301, 302):
|
||
result_payload["ok"] = True
|
||
result_payload["reason"] = f"ok:{target_url}:{test_response.status_code}"
|
||
break
|
||
last_reason = f"http_status_{test_response.status_code}@{target_url}"
|
||
except Exception as target_error:
|
||
error_text = str(target_error).strip() or target_error.__class__.__name__
|
||
last_reason = f"{target_error.__class__.__name__}@{target_url}:{error_text[:160]}"
|
||
session.close()
|
||
if not result_payload["ok"]:
|
||
result_payload["reason"] = last_reason
|
||
except Exception as e:
|
||
result_payload["reason"] = f"{e.__class__.__name__}:{str(e)[:160]}"
|
||
finally:
|
||
result_queue.put(result_payload)
|
||
result_queue.put(None)
|
||
|
||
def _extract_proxy_items(self, proxy_data):
|
||
if isinstance(proxy_data, dict):
|
||
if 'list' in proxy_data and isinstance(proxy_data['list'], list):
|
||
return proxy_data['list']
|
||
if 'ip' in proxy_data and 'port' in proxy_data:
|
||
return [proxy_data]
|
||
if isinstance(proxy_data, list):
|
||
return proxy_data
|
||
return []
|
||
|
||
def _proxy_key(self, proxy):
|
||
if isinstance(proxy, dict):
|
||
return json.dumps(proxy, sort_keys=True, ensure_ascii=False)
|
||
return str(proxy)
|
||
|
||
def _is_proxy_quarantined(self, proxy):
|
||
proxy_key = self._proxy_key(proxy)
|
||
now_ts = time.time()
|
||
with self.proxy_failure_lock:
|
||
retry_after = float(self.proxy_quarantine_until.get(proxy_key, 0) or 0)
|
||
if retry_after <= now_ts:
|
||
if proxy_key in self.proxy_quarantine_until:
|
||
self.proxy_quarantine_until.pop(proxy_key, None)
|
||
self.proxy_failure_counts.pop(proxy_key, None)
|
||
return False
|
||
return True
|
||
|
||
def _mark_proxy_failure(self, proxy):
|
||
proxy_key = self._proxy_key(proxy)
|
||
with self.proxy_failure_lock:
|
||
failure_count = int(self.proxy_failure_counts.get(proxy_key, 0) or 0) + 1
|
||
self.proxy_failure_counts[proxy_key] = failure_count
|
||
cooldown_seconds = min(180, 20 * failure_count)
|
||
retry_after = time.time() + cooldown_seconds
|
||
self.proxy_quarantine_until[proxy_key] = retry_after
|
||
logger.info(f"代理临时隔离 {cooldown_seconds}s: {proxy}")
|
||
return failure_count, cooldown_seconds
|
||
|
||
def _clear_proxy_failure(self, proxy):
|
||
proxy_key = self._proxy_key(proxy)
|
||
with self.proxy_failure_lock:
|
||
self.proxy_failure_counts.pop(proxy_key, None)
|
||
self.proxy_quarantine_until.pop(proxy_key, None)
|
||
|
||
def _schedule_proxy_refresh_if_needed(self, available_count=None):
|
||
with self.proxy_pool_lock:
|
||
current_count = len(self.proxy_pool) if available_count is None else available_count
|
||
threshold = max(2, self.thread_count // 2)
|
||
if current_count < threshold and not self.proxy_refresh_lock.locked():
|
||
threading.Thread(target=self.refresh_proxy_pool, daemon=True).start()
|
||
|
||
def refresh_proxy_pool(self):
|
||
"""
|
||
刷新代理池
|
||
"""
|
||
if not self.proxy_refresh_lock.acquire(blocking=False):
|
||
logger.debug("代理池刷新已在进行中，跳过本次重复刷新")
|
||
return
|
||
|
||
if not self.proxy_config.get('proxy_enable', False):
|
||
self.proxy_last_refresh_time = datetime.now()
|
||
self.proxy_last_refresh_status = "代理未启用"
|
||
self.proxy_last_refresh_source_count = 0
|
||
self.proxy_last_refresh_total_items = 0
|
||
self.proxy_last_validated_count = 0
|
||
self.proxy_last_available_count = 0
|
||
self.proxy_last_source_stats = []
|
||
self.proxy_next_refresh_time = 0.0
|
||
self._update_runtime_state(
|
||
"idle" if not self.detecting else "running",
|
||
"代理未启用，使用直接连接",
|
||
)
|
||
self._sync_worker_log_event("代理未启用，使用直接连接", mode='key')
|
||
self.proxy_refresh_lock.release()
|
||
return
|
||
|
||
try:
|
||
now_ts = time.time()
|
||
with self.proxy_pool_lock:
|
||
has_cached_proxies = bool(self.proxy_pool)
|
||
if not has_cached_proxies and self.proxy_next_refresh_time and now_ts < self.proxy_next_refresh_time:
|
||
wait_seconds = int(max(1, self.proxy_next_refresh_time - now_ts))
|
||
self.proxy_last_refresh_status = f"冷却中，{wait_seconds} 秒后再试"
|
||
logger.info(f"代理池刷新冷却中，{wait_seconds} 秒后再试")
|
||
self._update_runtime_state(
|
||
"refreshing_proxy" if self.detecting else "idle",
|
||
f"代理池刷新冷却中，{wait_seconds} 秒后再试",
|
||
)
|
||
self._sync_worker_log_event(f"代理池刷新冷却中，{wait_seconds} 秒后再试", mode='full')
|
||
return
|
||
|
||
proxy_api_urls = self.proxy_config.get('proxy_urls') or []
|
||
if not proxy_api_urls and self.proxy_config.get('proxy_url'):
|
||
proxy_api_urls = [self.proxy_config.get('proxy_url', '')]
|
||
proxy_api_urls = [url for url in proxy_api_urls if url]
|
||
self.proxy_last_refresh_source_count = len(proxy_api_urls)
|
||
self.proxy_last_source_stats = []
|
||
if proxy_api_urls:
|
||
import requests
|
||
import threading
|
||
import random
|
||
from queue import Queue
|
||
|
||
proxy_list = []
|
||
for proxy_api_url in proxy_api_urls:
|
||
source_stat = {
|
||
"url": proxy_api_url,
|
||
"status": "unknown",
|
||
"http_status": 0,
|
||
"raw_items": 0,
|
||
"error": "",
|
||
}
|
||
try:
|
||
response = requests.get(proxy_api_url, timeout=10)
|
||
source_stat["http_status"] = int(response.status_code or 0)
|
||
if response.status_code == 200:
|
||
proxy_data = response.json()
|
||
current_items = self._extract_proxy_items(proxy_data)
|
||
proxy_list.extend(current_items)
|
||
source_stat["raw_items"] = len(current_items)
|
||
source_stat["status"] = "ok"
|
||
logger.info(f"代理池链接拉取成功: {proxy_api_url}, 原始代理数: {len(current_items)}")
|
||
else:
|
||
source_stat["status"] = "http_error"
|
||
source_stat["error"] = f"HTTP {response.status_code}"
|
||
logger.warning(f"代理池链接响应异常: {proxy_api_url}, 状态码: {response.status_code}")
|
||
except Exception as api_error:
|
||
source_stat["status"] = "request_error"
|
||
source_stat["error"] = str(api_error)
|
||
logger.warning(f"代理池链接拉取失败: {proxy_api_url}, 错误: {api_error}")
|
||
self.proxy_last_source_stats.append(source_stat)
|
||
|
||
if proxy_list:
|
||
self.proxy_last_refresh_total_items = len(proxy_list)
|
||
max_validate_count = max(24, self.thread_count * 3)
|
||
if len(proxy_list) > max_validate_count:
|
||
proxy_list = random.sample(proxy_list, max_validate_count)
|
||
logger.info(f"代理池验证已启用抽样模式，本次抽样 {len(proxy_list)} 个代理进行可用性验证")
|
||
self.proxy_last_validated_count = len(proxy_list)
|
||
result_queue = Queue()
|
||
threads = []
|
||
|
||
for proxy_item in proxy_list:
|
||
thread = threading.Thread(target=self.test_proxy, args=(proxy_item, result_queue))
|
||
thread.daemon = True
|
||
thread.start()
|
||
threads.append(thread)
|
||
|
||
new_proxies = []
|
||
seen_proxy_strings = set()
|
||
completed = 0
|
||
start_time = time.time()
|
||
timeout = 30
|
||
failure_reason_stats = collections.Counter()
|
||
failure_samples = []
|
||
|
||
while completed < len(threads) and time.time() - start_time < timeout:
|
||
try:
|
||
result = result_queue.get(timeout=1)
|
||
if result is not None:
|
||
if isinstance(result, dict) and result.get("ok"):
|
||
proxy_value = result.get("proxy")
|
||
proxy_signature = json.dumps(proxy_value, sort_keys=True)
|
||
if proxy_signature not in seen_proxy_strings:
|
||
seen_proxy_strings.add(proxy_signature)
|
||
new_proxies.append({'proxy': proxy_value, 'usage_count': 0})
|
||
else:
|
||
reason = ""
|
||
if isinstance(result, dict):
|
||
reason = str(result.get("reason", "")).strip()
|
||
proxy_value = result.get("proxy")
|
||
if proxy_value and len(failure_samples) < 3:
|
||
failure_samples.append(f"{reason} -> {proxy_value}")
|
||
failure_reason_stats[reason or "unknown"] += 1
|
||
completed += 1
|
||
except Exception:
|
||
pass
|
||
|
||
for thread in threads:
|
||
try:
|
||
thread.join(timeout=5)
|
||
except Exception:
|
||
pass
|
||
|
||
with self.proxy_pool_lock:
|
||
self.proxy_pool = new_proxies
|
||
self.proxy_last_refresh_time = datetime.now()
|
||
self.proxy_last_available_count = len(new_proxies)
|
||
self.proxy_last_refresh_status = f"刷新成功，可用 {len(new_proxies)} 个"
|
||
self.proxy_next_refresh_time = 0.0 if new_proxies else time.time() + self.proxy_refresh_cooldown_seconds
|
||
logger.info(f"代理池刷新完成，共 {len(new_proxies)} 个可用代理，来源链接 {len(proxy_api_urls)} 个")
|
||
if failure_reason_stats:
|
||
logger.warning(
|
||
f"代理校验失败汇总: {dict(failure_reason_stats.most_common(5))}"
|
||
)
|
||
if failure_samples:
|
||
logger.warning(f"代理校验失败样例: {' | '.join(failure_samples)}")
|
||
self._update_runtime_state(
|
||
"running" if self.detecting else "idle",
|
||
f"代理池刷新完成，共 {len(new_proxies)} 个可用代理，来源链接 {len(proxy_api_urls)} 个，原始 {self.proxy_last_refresh_total_items} 个，验证 {self.proxy_last_validated_count} 个",
|
||
)
|
||
self._sync_worker_log_event(
|
||
f"代理池刷新完成，共 {len(new_proxies)} 个可用代理，来源链接 {len(proxy_api_urls)} 个，原始 {self.proxy_last_refresh_total_items} 个，验证 {self.proxy_last_validated_count} 个",
|
||
payload={
|
||
"available_proxy_count": len(new_proxies),
|
||
"source_count": len(proxy_api_urls),
|
||
"raw_items": self.proxy_last_refresh_total_items,
|
||
"validated_count": self.proxy_last_validated_count,
|
||
},
|
||
mode='key',
|
||
)
|
||
else:
|
||
with self.proxy_pool_lock:
|
||
self.proxy_pool = []
|
||
self.proxy_last_refresh_time = datetime.now()
|
||
self.proxy_last_refresh_total_items = 0
|
||
self.proxy_last_validated_count = 0
|
||
self.proxy_last_available_count = 0
|
||
self.proxy_last_refresh_status = "未取到可用代理数据"
|
||
self.proxy_next_refresh_time = time.time() + self.proxy_refresh_cooldown_seconds
|
||
logger.warning("所有代理池链接均未返回可用代理数据")
|
||
self._update_runtime_state(
|
||
"refreshing_proxy" if self.detecting else "idle",
|
||
"所有代理池链接均未返回可用代理数据",
|
||
)
|
||
self._sync_worker_log_event("所有代理池链接均未返回可用代理数据", level='warning', mode='key')
|
||
else:
|
||
with self.proxy_pool_lock:
|
||
self.proxy_pool = []
|
||
self.proxy_last_refresh_time = datetime.now()
|
||
self.proxy_last_refresh_total_items = 0
|
||
self.proxy_last_validated_count = 0
|
||
self.proxy_last_available_count = 0
|
||
self.proxy_last_refresh_status = "未配置代理池链接"
|
||
self.proxy_next_refresh_time = time.time() + self.proxy_refresh_cooldown_seconds
|
||
self._update_runtime_state(
|
||
"idle" if not self.detecting else "running",
|
||
"未配置代理池链接",
|
||
)
|
||
self._sync_worker_log_event("未配置代理池链接", level='warning', mode='key')
|
||
except Exception as e:
|
||
with self.proxy_pool_lock:
|
||
self.proxy_pool = []
|
||
self.proxy_last_refresh_time = datetime.now()
|
||
self.proxy_last_refresh_total_items = 0
|
||
self.proxy_last_validated_count = 0
|
||
self.proxy_last_available_count = 0
|
||
self.proxy_last_refresh_status = f"刷新失败: {e}"
|
||
self.proxy_next_refresh_time = time.time() + self.proxy_refresh_cooldown_seconds
|
||
logger.error(f"刷新代理池失败: {e}")
|
||
self._update_runtime_state(
|
||
"refreshing_proxy" if self.detecting else "failed",
|
||
f"刷新代理池失败: {e}",
|
||
)
|
||
self._sync_worker_log_event(f"刷新代理池失败: {e}", level='error', mode='key')
|
||
finally:
|
||
self.proxy_refresh_lock.release()
|
||
|
||
def remove_proxy(self, proxy):
|
||
"""
|
||
从代理池中移除失效的代理
|
||
|
||
:param proxy: 失效的代理
|
||
"""
|
||
failure_count, cooldown_seconds = self._mark_proxy_failure(proxy)
|
||
with self.proxy_pool_lock:
|
||
remaining_count = len(self.proxy_pool)
|
||
# 检查代理池中的代理结构
|
||
if self.proxy_pool and isinstance(self.proxy_pool[0], dict) and 'proxy' in self.proxy_pool[0]:
|
||
# 新的代理池结构
|
||
for i, proxy_item in enumerate(self.proxy_pool):
|
||
if proxy_item['proxy'] == proxy:
|
||
proxy_item['usage_count'] = 0
|
||
self.proxy_pool.append(self.proxy_pool.pop(i))
|
||
remaining_count = len(self.proxy_pool)
|
||
logger.info(f"代理进入隔离观察，暂不硬删除: {proxy}")
|
||
break
|
||
else:
|
||
# 旧的代理池结构
|
||
if proxy in self.proxy_pool:
|
||
remaining_count = len(self.proxy_pool)
|
||
logger.info(f"代理进入隔离观察，暂不硬删除: {proxy}")
|
||
|
||
logger.info(
|
||
f"代理失败计数已更新: 第 {failure_count} 次失败，隔离 {cooldown_seconds}s，当前池内代理 {remaining_count} 个"
|
||
)
|
||
self._schedule_proxy_refresh_if_needed(remaining_count)
|
||
|
||
def get_proxies(self):
|
||
"""
|
||
获取代理配置
|
||
"""
|
||
if self.proxy_config.get('proxy_enable', False):
|
||
with self.proxy_pool_lock:
|
||
current_pool_size = len(self.proxy_pool)
|
||
is_empty = current_pool_size == 0
|
||
is_insufficient = current_pool_size < max(2, self.thread_count // 2)
|
||
|
||
if is_empty or is_insufficient:
|
||
if is_empty:
|
||
logger.info("代理池为空，刷新代理池")
|
||
self.refresh_proxy_pool()
|
||
if is_empty:
|
||
wait_deadline = time.time() + 2.0
|
||
while time.time() < wait_deadline:
|
||
with self.proxy_pool_lock:
|
||
if self.proxy_pool:
|
||
break
|
||
if not self.proxy_refresh_lock.locked():
|
||
break
|
||
time.sleep(0.1)
|
||
|
||
with self.proxy_pool_lock:
|
||
if self.proxy_pool:
|
||
pool_size = len(self.proxy_pool)
|
||
for _ in range(pool_size):
|
||
if isinstance(self.proxy_pool[0], dict) and 'proxy' in self.proxy_pool[0]:
|
||
proxy_item = self.proxy_pool.pop(0)
|
||
proxy = proxy_item['proxy']
|
||
if self._is_proxy_quarantined(proxy):
|
||
self.proxy_pool.append(proxy_item)
|
||
continue
|
||
|
||
usage_count = int(proxy_item.get('usage_count', 0)) + 1
|
||
if usage_count >= self.proxy_max_reuse_count:
|
||
proxy_item['usage_count'] = 0
|
||
else:
|
||
proxy_item['usage_count'] = usage_count
|
||
self.proxy_pool.append(proxy_item)
|
||
self._clear_proxy_failure(proxy)
|
||
self._schedule_proxy_refresh_if_needed(len(self.proxy_pool))
|
||
return proxy
|
||
|
||
proxy = self.proxy_pool.pop(0)
|
||
if self._is_proxy_quarantined(proxy):
|
||
self.proxy_pool.append(proxy)
|
||
continue
|
||
logger.info(f"从代理池选择代理: {proxy}")
|
||
self.proxy_pool.append(proxy)
|
||
self._clear_proxy_failure(proxy)
|
||
self._schedule_proxy_refresh_if_needed(len(self.proxy_pool))
|
||
return proxy
|
||
if self.proxy_config.get('proxy_enable', False):
|
||
logger.warning("代理已启用，但当前无可用代理")
|
||
fallback_detail = (
|
||
f"代理已启用，但当前无可用代理；已允许直连兜底，最近代理状态：{self.proxy_last_refresh_status}"
|
||
if self.allow_direct_connection()
|
||
else f"代理已启用，但当前无可用代理；且未允许直连，最近代理状态：{self.proxy_last_refresh_status}"
|
||
)
|
||
self._update_runtime_state(
|
||
"running" if self.detecting else "idle",
|
||
fallback_detail,
|
||
)
|
||
else:
|
||
logger.info("未启用代理，使用直接连接")
|
||
return None
|
||
|
||
def allow_direct_connection(self):
|
||
return bool(self.proxy_config.get('allow_direct', False))
|
||
|
||
def _get_proxy_for_step(self, domain_id, domain_name, step_name):
|
||
proxy = self.get_proxies()
|
||
if proxy:
|
||
return proxy
|
||
if self.proxy_config.get('proxy_enable', False) and not self.allow_direct_connection():
|
||
detail = self.proxy_last_refresh_status
|
||
logger.error(f"{step_name} 无可用代理，且当前不允许直连兜底: {domain_name}，最近代理状态: {detail}")
|
||
self.db.update_domain_detect_status(domain_id, DETECT_STATUS_FAILED)
|
||
return '__NO_PROXY__'
|
||
return None
|
||
|
||
def _mark_blacklisted(self, domain_id, domain_name, reason):
|
||
self.db.update_domain_detect_status(domain_id, DETECT_STATUS_BLACKLISTED)
|
||
self.db.add_to_blacklist(domain_name, reason)
|
||
logger.info(f"域名已加入黑名单: {domain_name}, 原因: {reason}")
|
||
|
||
def _step_result_payload(self, *, ok, state, message="", **extra):
|
||
payload = {
|
||
"status": bool(ok),
|
||
"state": state,
|
||
"message": message or "",
|
||
"checked_at": datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
|
||
}
|
||
payload.update(extra)
|
||
return payload
|
||
|
||
def _record_step_result(self, domain_id, field_name, *, ok, state, message="", **extra):
|
||
if not field_name:
|
||
return
|
||
self._upsert_json_detection(domain_id, **{field_name: self._step_result_payload(ok=ok, state=state, message=message, **extra)})
|
||
|
||
def _should_blacklist_result(self, message):
|
||
if not message:
|
||
return False
|
||
lowered = str(message).lower()
|
||
keywords = [
|
||
"检测到敏感词",
|
||
"黑名单分类",
|
||
"风险网站",
|
||
"高危网站",
|
||
"clienthold",
|
||
"serverhold",
|
||
"拦截",
|
||
"敏感内容",
|
||
]
|
||
return any(keyword.lower() in lowered for keyword in keywords)
|
||
|
||
def _mark_detection_failed(self, domain_id, domain_name, step_name, reason, field_name=None):
|
||
reason = reason or "未知错误"
|
||
self.db.update_domain_detect_status(domain_id, DETECT_STATUS_FAILED)
|
||
if field_name:
|
||
self._record_step_result(domain_id, field_name, ok=False, state="failed", message=reason, step=step_name)
|
||
logger.warning(f"{step_name} 技术失败，域名标记为检测失败: {domain_name}, 原因: {reason}")
|
||
|
||
def _mark_detection_degraded(self, domain_id, domain_name, step_name, reason, field_name=None, review_required=True):
|
||
reason = reason or "外部依赖异常"
|
||
if field_name:
|
||
self._record_step_result(domain_id, field_name, ok=False, state="degraded", message=reason, step=step_name)
|
||
if review_required:
|
||
try:
|
||
self.db.update_domain_review_status(domain_id, REVIEW_STATUS_PENDING)
|
||
except Exception as review_error:
|
||
logger.warning(f"{step_name} 降级后设置待人工复核失败: {domain_name}, 错误: {review_error}")
|
||
logger.warning(f"{step_name} 外部依赖异常，步骤降级继续执行: {domain_name}, 原因: {reason}")
|
||
|
||
def _is_external_dependency_issue(self, message):
|
||
if not message:
|
||
return False
|
||
lowered = str(message).lower()
|
||
keywords = [
|
||
"timeout",
|
||
"timed out",
|
||
"connection",
|
||
"connection reset",
|
||
"connection refused",
|
||
"max retries exceeded",
|
||
"httpsconnectionpool",
|
||
"proxyerror",
|
||
"remotedisconnected",
|
||
"temporarily unavailable",
|
||
"502 bad gateway",
|
||
"503 service unavailable",
|
||
"504 gateway timeout",
|
||
"network",
|
||
"ssl",
|
||
"read timeout",
|
||
"connecttimeout",
|
||
]
|
||
return any(keyword in lowered for keyword in keywords)
|
||
|
||
def _complete_detection(self, domain_id, domain_name):
|
||
self.db.update_domain_detect_status(domain_id, DETECT_STATUS_COMPLETED)
|
||
domain_info = self.db.get_domain_by_id(domain_id)
|
||
if not domain_info:
|
||
return
|
||
use_status = domain_info.get('use_status', 0)
|
||
register_status = domain_info.get('register_status', 0)
|
||
expire_date = domain_info.get('expire_date')
|
||
if use_status == 0 and register_status == REGISTER_STATUS_AVAILABLE and expire_date:
|
||
self.db.update_domain_expire_date(domain_id, None)
|
||
logger.info(f"域名 {domain_name} 满足条件，已将expire_date置空")
|
||
if register_status == REGISTER_STATUS_AVAILABLE:
|
||
self.db.update_domain_review_status(domain_id, REVIEW_STATUS_PENDING)
|
||
logger.info(f"域名 {domain_name} 满足条件，已设置为待人工复核")
|
||
|
||
def _should_run_jucha(self, domain):
|
||
return self.detect_options.get('detect_jucha', False) and domain.get('jucha_status', 0) != THIRD_PARTY_STATUS_DONE
|
||
|
||
def _should_run_juziseo(self, domain):
|
||
return self.detect_options.get('detect_juziseo', False) and domain.get('juziseo_status', 0) != THIRD_PARTY_STATUS_DONE
|
||
|
||
def _get_detect_execution_order(self):
|
||
default_order = [
|
||
'detect_register',
|
||
'detect_baidu_site',
|
||
'detect_360_site',
|
||
'detect_chinaz',
|
||
'detect_aizhan',
|
||
'detect_wayback',
|
||
'detect_jucha',
|
||
'detect_juziseo',
|
||
]
|
||
configured_order = self.detect_options.get('detect_order') or []
|
||
normalized_order = [key for key in configured_order if key in default_order]
|
||
for key in default_order:
|
||
if key not in normalized_order:
|
||
normalized_order.append(key)
|
||
free_keys = {
|
||
'detect_register',
|
||
'detect_baidu_site',
|
||
'detect_360_site',
|
||
'detect_chinaz',
|
||
'detect_aizhan',
|
||
'detect_wayback',
|
||
}
|
||
paid_keys = {'detect_jucha', 'detect_juziseo'}
|
||
enabled_order = [key for key in normalized_order if self.detect_options.get(key, False)]
|
||
free_order = [key for key in enabled_order if key in free_keys]
|
||
paid_order = [key for key in enabled_order if key in paid_keys]
|
||
return free_order + paid_order
|
||
|
||
def _run_detect_register(self, domain_id, domain, domain_name):
|
||
is_ykj = domain.get('source_type', 0) == 1
|
||
if is_ykj:
|
||
logger.info(f"一口价域名跳过注册状态检测: {domain_name}")
|
||
return True
|
||
logger.info(f"检测注册状态: {domain_name}")
|
||
proxy = self._get_proxy_for_step(domain_id, domain_name, '注册状态检测')
|
||
if proxy == '__NO_PROXY__':
|
||
return False
|
||
try:
|
||
tld = domain_name.split('.')[-1]
|
||
status, expire_date = register.check_register(domain_name, tld, proxy if proxy else None)
|
||
if status != -1:
|
||
self.db.update_domain_register_status(domain_id, status)
|
||
if expire_date:
|
||
from datetime import datetime, timedelta
|
||
try:
|
||
date_part = expire_date.split(' ')[0]
|
||
expire_date_obj = datetime.strptime(date_part, "%Y-%m-%d")
|
||
new_expire_date = expire_date_obj + timedelta(days=75)
|
||
self.db.update_domain_expire_date(domain_id, new_expire_date.strftime("%Y-%m-%d"))
|
||
except Exception as e:
|
||
logger.error(f"处理过期日期失败: {e}")
|
||
self.db.update_domain_expire_date(domain_id, expire_date)
|
||
logger.info(f"注册状态检测完成: {domain_name}, 状态: {status}, 过期日期: {expire_date}")
|
||
else:
|
||
logger.warning(f"注册状态检测失败，不更新数据: {domain_name}")
|
||
if proxy:
|
||
self.remove_proxy(proxy)
|
||
self._mark_detection_failed(domain_id, domain_name, '注册状态检测', '注册状态检测返回未知状态')
|
||
return False
|
||
except Exception as e:
|
||
logger.error(f"注册状态检测失败: {domain_name}, 错误: {e}")
|
||
if proxy:
|
||
self.remove_proxy(proxy)
|
||
self._mark_detection_failed(domain_id, domain_name, '注册状态检测', str(e))
|
||
return False
|
||
return True
|
||
|
||
def _run_detect_wayback(self, domain_id, domain_name, sensitive_words):
|
||
logger.info(f"检测时光机: {domain_name}")
|
||
try:
|
||
wayback_result = self.wayback_detector.scan_snapshots(domain_name, sensitive_words=sensitive_words)
|
||
snapshot_years = wayback_result.get('snapshot_years') or []
|
||
if snapshot_years:
|
||
self.db.update_domain_snapshot_years(domain_id, ",".join(str(year) for year in snapshot_years))
|
||
backlink_count = int(wayback_result.get('backlink_count', 0) or 0)
|
||
self.db.execute("UPDATE domains SET backlink_count = %s WHERE id = %s", (backlink_count, domain_id))
|
||
self._upsert_json_detection(
|
||
domain_id,
|
||
backlink_count_gt_10=bool(wayback_result.get('backlink_count_gt_10')),
|
||
)
|
||
if wayback_result.get('has_sensitive_content'):
|
||
reason = f"时光机快照命中敏感词: {wayback_result.get('matched_word', '')}".strip()
|
||
self._record_step_result(domain_id, 'wayback_info', ok=False, state="blacklisted", message=reason)
|
||
self._mark_blacklisted(domain_id, domain_name, reason)
|
||
return False
|
||
if int(wayback_result.get('request_error_count', 0) or 0) > 0 and int(wayback_result.get('checked_snapshot_count', 0) or 0) == 0:
|
||
reason = "时光机检测失败，快照索引请求异常: " + " | ".join(
|
||
(wayback_result.get('request_errors') or [])[:3]
|
||
)
|
||
self._mark_detection_degraded(domain_id, domain_name, '时光机检测', reason, 'wayback_info')
|
||
return True
|
||
if int(wayback_result.get('failed_snapshot_count', 0) or 0) > 0 and int(wayback_result.get('fetched_snapshot_count', 0) or 0) == 0:
|
||
reason = (
|
||
f"时光机检测失败，未成功抓取任何快照；失败次数 {wayback_result.get('failed_snapshot_count', 0)}"
|
||
)
|
||
self._mark_detection_degraded(domain_id, domain_name, '时光机检测', reason, 'wayback_info')
|
||
return True
|
||
self._record_step_result(
|
||
domain_id,
|
||
'wayback_info',
|
||
ok=True,
|
||
state="passed",
|
||
message="success",
|
||
checked_snapshot_count=int(wayback_result.get('checked_snapshot_count', 0) or 0),
|
||
fetched_snapshot_count=int(wayback_result.get('fetched_snapshot_count', 0) or 0),
|
||
failed_snapshot_count=int(wayback_result.get('failed_snapshot_count', 0) or 0),
|
||
backlink_count=int(wayback_result.get('backlink_count', 0) or 0),
|
||
)
|
||
logger.info(
|
||
"时光机检测完成: %s, 检查快照 %s 个, 成功抓取 %s 个, 失败 %s 个, 最大友链 %s, 耗时 %ss"
|
||
% (
|
||
domain_name,
|
||
wayback_result.get('checked_snapshot_count', 0),
|
||
wayback_result.get('fetched_snapshot_count', 0),
|
||
wayback_result.get('failed_snapshot_count', 0),
|
||
wayback_result.get('backlink_count', 0),
|
||
wayback_result.get('elapsed_seconds', 0),
|
||
)
|
||
)
|
||
except Exception as e:
|
||
logger.error(f"时光机检测失败: {domain_name}, 错误: {e}")
|
||
self._mark_detection_degraded(domain_id, domain_name, '时光机检测', str(e), 'wayback_info')
|
||
return True
|
||
return True
|
||
|
||
def _run_detect_chinaz(self, domain_id, domain_name, sensitive_words):
|
||
logger.info(f"检测站长之家: {domain_name}")
|
||
proxy = self._get_proxy_for_step(domain_id, domain_name, '站长之家检测')
|
||
if proxy == '__NO_PROXY__':
|
||
return False
|
||
try:
|
||
success, message, seo_data = chinaz.check_title(domain_name, sensitive_words, proxy)
|
||
if not success:
|
||
logger.warning(f"站长之家检测未通过: {domain_name}, 原因: {message}")
|
||
if self._should_blacklist_result(message):
|
||
self._record_step_result(domain_id, 'chinaz_info', ok=False, state="blacklisted", message=message)
|
||
self._mark_blacklisted(domain_id, domain_name, message)
|
||
return False
|
||
if self._is_external_dependency_issue(message):
|
||
self._mark_detection_degraded(domain_id, domain_name, '站长之家检测', message or 'external dependency issue', 'chinaz_info')
|
||
return True
|
||
self._mark_detection_failed(domain_id, domain_name, '站长之家检测', message or 'failure', 'chinaz_info')
|
||
return False
|
||
self._record_step_result(domain_id, 'chinaz_info', ok=True, state="passed", message=message or "success", seo=seo_data)
|
||
logger.info(f"站长之家检测完成: {domain_name}")
|
||
except Exception as e:
|
||
logger.error(f"站长之家检测失败: {domain_name}, 错误: {e}")
|
||
if proxy:
|
||
self.remove_proxy(proxy)
|
||
if self._is_external_dependency_issue(e):
|
||
self._mark_detection_degraded(domain_id, domain_name, '站长之家检测', str(e), 'chinaz_info')
|
||
return True
|
||
self._mark_detection_failed(domain_id, domain_name, '站长之家检测', str(e), 'chinaz_info')
|
||
return False
|
||
return True
|
||
|
||
def _run_detect_aizhan(self, domain_id, domain_name, sensitive_words):
|
||
logger.info(f"检测爱站网: {domain_name}")
|
||
proxy = self._get_proxy_for_step(domain_id, domain_name, '爱站网检测')
|
||
if proxy == '__NO_PROXY__':
|
||
return False
|
||
try:
|
||
success, message = aizhan.check_aizhan(domain_name, sensitive_words, proxy)
|
||
if not success:
|
||
logger.warning(f"爱站网检测未通过: {domain_name}, 原因: {message}")
|
||
if self._should_blacklist_result(message):
|
||
self._record_step_result(domain_id, 'aizhan_info', ok=False, state="blacklisted", message=message)
|
||
self._mark_blacklisted(domain_id, domain_name, message)
|
||
return False
|
||
if self._is_external_dependency_issue(message):
|
||
self._mark_detection_degraded(domain_id, domain_name, '爱站网检测', message or 'external dependency issue', 'aizhan_info')
|
||
return True
|
||
self._mark_detection_failed(domain_id, domain_name, '爱站网检测', message or 'failure', 'aizhan_info')
|
||
return False
|
||
self._record_step_result(domain_id, 'aizhan_info', ok=True, state="passed", message=message or "success")
|
||
logger.info(f"爱站网检测完成: {domain_name}")
|
||
except Exception as e:
|
||
logger.error(f"爱站网检测失败: {domain_name}, 错误: {e}")
|
||
if proxy:
|
||
self.remove_proxy(proxy)
|
||
if self._is_external_dependency_issue(e):
|
||
self._mark_detection_degraded(domain_id, domain_name, '爱站网检测', str(e), 'aizhan_info')
|
||
return True
|
||
self._mark_detection_failed(domain_id, domain_name, '爱站网检测', str(e), 'aizhan_info')
|
||
return False
|
||
return True
|
||
|
||
def _run_detect_baidu(self, domain_id, domain_name, sensitive_words):
|
||
logger.info(f"检测百度: {domain_name}")
|
||
proxy = self._get_proxy_for_step(domain_id, domain_name, '百度site检测')
|
||
if proxy == '__NO_PROXY__':
|
||
return False
|
||
try:
|
||
success, message = baidu.check_site(domain_name, sensitive_words, proxy)
|
||
if not success:
|
||
logger.warning(f"百度site检测未通过: {domain_name}, 原因: {message}")
|
||
if proxy and ('timeout' in message.lower() or 'connection' in message.lower()) and self.allow_direct_connection():
|
||
logger.info(f"代理检测失败，尝试不使用代理重新检测: {domain_name}")
|
||
success, message = baidu.check_site(domain_name, sensitive_words, None)
|
||
if success:
|
||
logger.info(f"不使用代理检测百度成功: {domain_name}")
|
||
else:
|
||
logger.warning(f"不使用代理检测百度仍未通过: {domain_name}, 原因: {message}")
|
||
if self._should_blacklist_result(message):
|
||
self._record_step_result(domain_id, 'baidu_site', ok=False, state="blacklisted", message=message)
|
||
self._mark_blacklisted(domain_id, domain_name, message)
|
||
elif self._is_external_dependency_issue(message):
|
||
self._mark_detection_degraded(domain_id, domain_name, '百度site检测', message, 'baidu_site')
|
||
return True
|
||
else:
|
||
self._mark_detection_failed(domain_id, domain_name, '百度site检测', message, 'baidu_site')
|
||
return False
|
||
else:
|
||
if self._should_blacklist_result(message):
|
||
self._record_step_result(domain_id, 'baidu_site', ok=False, state="blacklisted", message=message)
|
||
self._mark_blacklisted(domain_id, domain_name, message)
|
||
elif self._is_external_dependency_issue(message):
|
||
self._mark_detection_degraded(domain_id, domain_name, '百度site检测', message, 'baidu_site')
|
||
return True
|
||
else:
|
||
self._mark_detection_failed(domain_id, domain_name, '百度site检测', message, 'baidu_site')
|
||
return False
|
||
self._record_step_result(domain_id, 'baidu_site', ok=True, state="passed", message=message or "success")
|
||
logger.info(f"百度site检测完成: {domain_name}")
|
||
except Exception as e:
|
||
logger.error(f"百度site检测失败: {domain_name}, 错误: {e}")
|
||
if proxy:
|
||
self.remove_proxy(proxy)
|
||
if self.allow_direct_connection():
|
||
try:
|
||
logger.info(f"尝试不使用代理重新检测百度: {domain_name}")
|
||
success, message = baidu.check_site(domain_name, sensitive_words, None)
|
||
if success:
|
||
logger.info(f"不使用代理检测百度成功: {domain_name}")
|
||
else:
|
||
logger.warning(f"不使用代理检测百度仍未通过: {domain_name}, 原因: {message}")
|
||
if self._should_blacklist_result(message):
|
||
self._record_step_result(domain_id, 'baidu_site', ok=False, state="blacklisted", message=message)
|
||
self._mark_blacklisted(domain_id, domain_name, message)
|
||
elif self._is_external_dependency_issue(message):
|
||
self._mark_detection_degraded(domain_id, domain_name, '百度site检测', message, 'baidu_site')
|
||
return True
|
||
else:
|
||
self._mark_detection_failed(domain_id, domain_name, '百度site检测', message, 'baidu_site')
|
||
return False
|
||
except Exception as e2:
|
||
logger.error(f"不使用代理检测百度也失败: {domain_name}, 错误: {e2}")
|
||
if self._is_external_dependency_issue(e2):
|
||
self._mark_detection_degraded(domain_id, domain_name, '百度site检测', str(e2), 'baidu_site')
|
||
return True
|
||
self._mark_detection_failed(domain_id, domain_name, '百度site检测', str(e2), 'baidu_site')
|
||
return False
|
||
else:
|
||
if self._is_external_dependency_issue(e):
|
||
self._mark_detection_degraded(domain_id, domain_name, '百度site检测', str(e), 'baidu_site')
|
||
return True
|
||
self._mark_detection_failed(domain_id, domain_name, '百度site检测', str(e), 'baidu_site')
|
||
return False
|
||
return True
|
||
|
||
def _run_detect_360(self, domain_id, domain_name, sensitive_words):
|
||
logger.info(f"检测360: {domain_name}")
|
||
proxy = self._get_proxy_for_step(domain_id, domain_name, '360检测')
|
||
if proxy == '__NO_PROXY__':
|
||
return False
|
||
try:
|
||
passed, message = c360.check_domain(domain_name, sensitive_words, proxy)
|
||
if not passed:
|
||
logger.warning(f"360检测未通过: {domain_name}, 原因: {message}")
|
||
if self._should_blacklist_result(message):
|
||
self._record_step_result(domain_id, 'qihu360_site', ok=False, state="blacklisted", message=message)
|
||
self._mark_blacklisted(domain_id, domain_name, message)
|
||
return False
|
||
if self._is_external_dependency_issue(message):
|
||
self._mark_detection_degraded(domain_id, domain_name, '360检测', message or 'external dependency issue', 'qihu360_site')
|
||
return True
|
||
self._mark_detection_failed(domain_id, domain_name, '360检测', message or 'failure', 'qihu360_site')
|
||
return False
|
||
self._record_step_result(domain_id, 'qihu360_site', ok=True, state="passed", message=message or "success")
|
||
logger.info(f"360检测完成: {domain_name}")
|
||
except Exception as e:
|
||
logger.error(f"360检测失败: {domain_name}, 错误: {e}")
|
||
if self._is_external_dependency_issue(e):
|
||
self._mark_detection_degraded(domain_id, domain_name, '360检测', str(e), 'qihu360_site')
|
||
return True
|
||
self._mark_detection_failed(domain_id, domain_name, '360检测', str(e), 'qihu360_site')
|
||
return False
|
||
return True
|
||
|
||
def _run_detect_jucha(self, domain_id, domain, domain_name):
|
||
if not self._should_run_jucha(domain):
|
||
return True
|
||
self._init_optional_detectors()
|
||
if self.jc_instance is None:
|
||
logger.warning("聚查检测器未初始化，跳过本次聚查检测")
|
||
return True
|
||
proxy = self._get_proxy_for_step(domain_id, domain_name, '聚查检测')
|
||
if proxy == '__NO_PROXY__':
|
||
return False
|
||
self.jc_instance.session.proxies = proxy or {}
|
||
logger.info(f"检测聚查WHOIS: {domain_name}")
|
||
try:
|
||
success, message, whois_info = self.jc_instance.check_whois_domain(domain_name)
|
||
if not success:
|
||
logger.warning(f"聚查WHOIS查询失败: {domain_name}, 原因: {message}")
|
||
else:
|
||
logger.info(f"WHOIS查询结果: {success}, {message}, {whois_info}")
|
||
if 'clientHold' in whois_info or 'serverHold' in whois_info:
|
||
logger.warning(f"聚查WHOIS检测未通过: {domain_name}, 原因: 域名状态包含clientHold或serverHold")
|
||
self._mark_blacklisted(domain_id, domain_name, '域名状态包含clientHold或serverHold')
|
||
return False
|
||
logger.info(f"聚查WHOIS检测完成: {domain_name}")
|
||
except Exception as e:
|
||
logger.error(f"聚查WHOIS检测失败: {domain_name}, 错误: {e}")
|
||
|
||
logger.info(f"检测聚查备案: {domain_name}")
|
||
try:
|
||
success, message, beian_info = self.jc_instance.beian_check_domain(domain_name)
|
||
if not success:
|
||
logger.warning(f"聚查备案查询失败: {domain_name}, 原因: {message}")
|
||
else:
|
||
logger.info(f"备案查询结果: {success}, {message}, {beian_info}")
|
||
if isinstance(beian_info, tuple) and len(beian_info) == 4:
|
||
beian_time, company_type, website_url, has_beian = beian_info
|
||
has_beian_flag = 2 if has_beian == '当前存在' else 3
|
||
beian_year = None
|
||
if beian_time:
|
||
try:
|
||
beian_year = beian_time.split('-')[0]
|
||
except Exception:
|
||
pass
|
||
self.db.update_domain_beian_info(domain_id, company_type, website_url, has_beian_flag, beian_year)
|
||
logger.info(f"聚查备案检测完成: {domain_name}")
|
||
except Exception as e:
|
||
logger.error(f"聚查备案检测失败: {domain_name}, 错误: {e}")
|
||
|
||
logger.info(f"检测聚查拦截: {domain_name}")
|
||
try:
|
||
success, message, safe_info = self.jc_instance.safe_check_domain(domain_name)
|
||
if not success:
|
||
logger.warning(f"聚查安全检测失败: {domain_name}, 原因: {message}")
|
||
else:
|
||
logger.info(f"安全检测结果: {success}, {message}, {safe_info}")
|
||
check_items = ['QQ检测', '微信检测', '抖音检测', '被墙检测', '百度检测', '谷歌检测', '火狐检测']
|
||
blacklist_reason = []
|
||
if isinstance(safe_info, tuple):
|
||
for i, item in enumerate(safe_info):
|
||
if isinstance(item, tuple):
|
||
if len(item) >= 1 and item[0] == 3 and i < len(check_items):
|
||
blacklist_reason.append(f"{check_items[i]}: 拦截")
|
||
elif len(item) >= 2 and item[0] == 2 and '拦截' in item[1] and i < len(check_items):
|
||
blacklist_reason.append(f"{check_items[i]}: 拦截")
|
||
if blacklist_reason:
|
||
reason_str = "；".join(blacklist_reason)
|
||
logger.warning(f"聚查安全检测未通过: {domain_name}, 原因: {reason_str}")
|
||
self._mark_blacklisted(domain_id, domain_name, reason_str)
|
||
return False
|
||
logger.info(f"聚查拦截检测完成: {domain_name}")
|
||
except Exception as e:
|
||
logger.error(f"聚查拦截检测失败: {domain_name}, 错误: {e}")
|
||
finally:
|
||
self.db.mark_jucha_detected(domain_id)
|
||
return True
|
||
|
||
def _run_detect_juziseo(self, domain_id, domain, domain_name, sensitive_words):
|
||
if not self._should_run_juziseo(domain):
|
||
return True
|
||
self._init_optional_detectors()
|
||
if self.juziseo_instance is None:
|
||
logger.warning("桔子SEO检测器未初始化，跳过本次桔子检测")
|
||
return True
|
||
proxy = self._get_proxy_for_step(domain_id, domain_name, '桔子检测')
|
||
if proxy == '__NO_PROXY__':
|
||
return False
|
||
self.juziseo_instance.session.proxies = proxy or {}
|
||
logger.info(f"检测桔子历史: {domain_name}")
|
||
try:
|
||
success, message = self.juziseo_instance.check_history(domain_name, sensitive_words)
|
||
if not success:
|
||
logger.warning(f"桔子历史检测未通过: {domain_name}, 原因: {message}")
|
||
self._mark_blacklisted(domain_id, domain_name, message)
|
||
return False
|
||
logger.info(f"桔子历史检测完成: {domain_name}")
|
||
except Exception as e:
|
||
logger.error(f"桔子历史检测失败: {domain_name}, 错误: {e}")
|
||
|
||
logger.info(f"检测桔子外链: {domain_name}")
|
||
try:
|
||
success, message = self.juziseo_instance.check_external_link(domain_name, sensitive_words)
|
||
if not success:
|
||
logger.warning(f"桔子外链检测未通过: {domain_name}, 原因: {message}")
|
||
self._mark_blacklisted(domain_id, domain_name, message)
|
||
return False
|
||
logger.info(f"桔子外链检测完成: {domain_name}")
|
||
except Exception as e:
|
||
logger.error(f"桔子外链检测失败: {domain_name}, 错误: {e}")
|
||
finally:
|
||
self.db.mark_juziseo_detected(domain_id)
|
||
return True
|
||
|
||
def _upsert_json_detection(self, domain_id, **payload):
|
||
existing = self.db.fetch_one("SELECT id FROM domain_detections WHERE domain_id = %s", (domain_id,))
|
||
columns = []
|
||
params = []
|
||
for key, value in payload.items():
|
||
columns.append(key)
|
||
params.append(json.dumps(value) if isinstance(value, (dict, list, tuple)) else value)
|
||
if not columns:
|
||
return True
|
||
if existing:
|
||
assignments = ", ".join([f"{column} = %s" for column in columns] + ["update_time = CURRENT_TIMESTAMP"])
|
||
params.append(domain_id)
|
||
sql = f"UPDATE domain_detections SET {assignments} WHERE domain_id = %s"
|
||
else:
|
||
sql = f"INSERT INTO domain_detections (domain_id, {', '.join(columns)}) VALUES (%s, {', '.join(['%s'] * len(columns))})"
|
||
params = [domain_id] + params
|
||
return self.db.execute(sql, tuple(params))
|
||
|
||
def detect_domain(self, domain_id, domain, job_context=None):
|
||
"""
|
||
检测单个域名
|
||
|
||
:param domain_id: 域名ID
|
||
:param domain: 域名对象（包含id、domain、source_type等字段）
|
||
"""
|
||
import threading
|
||
thread_id = threading.current_thread().ident
|
||
|
||
# 从域名对象中获取域名字符串
|
||
domain_name = domain.get('domain', '')
|
||
logger.info(f"线程 {thread_id} 开始检测域名: {domain_name}")
|
||
|
||
job_context = job_context or {}
|
||
job_item_id = job_context.get('job_item_id')
|
||
job_id = job_context.get('job_id')
|
||
claim_token = job_context.get('claim_token')
|
||
cycle_token = str(job_context.get('cycle_token') or self.current_cycle_token or '').strip()
|
||
job_code = str(job_context.get('job_code') or self.current_job_code or '').strip()
|
||
|
||
def finalize_job_item(final_status, message=""):
|
||
if not job_item_id or not claim_token:
|
||
return
|
||
try:
|
||
if final_status in {'completed', 'blacklisted'}:
|
||
self.db.complete_detect_job_item(job_item_id, claim_token, final_status)
|
||
else:
|
||
self.db.fail_detect_job_item(job_item_id, claim_token, message or final_status)
|
||
self.db.append_detect_run_event(
|
||
job_id,
|
||
job_item_id,
|
||
config.NODE_CODE,
|
||
event_type=f"domain_{final_status}",
|
||
message=message or f"{domain_name} -> {final_status}",
|
||
level='error' if final_status == 'failed' else 'info',
|
||
payload={
|
||
"domain_id": domain_id,
|
||
"domain": domain_name,
|
||
"status": final_status,
|
||
"cycle_token": cycle_token,
|
||
"job_code": job_code,
|
||
},
|
||
)
|
||
except Exception as finalize_error:
|
||
logger.warning(f"回写任务项状态失败: {domain_name}, 错误: {finalize_error}")
|
||
|
||
def renew_job_item_lease(detail=""):
|
||
if not job_item_id or not claim_token:
|
||
return
|
||
try:
|
||
self.db.renew_detect_job_item_lease(
|
||
job_item_id,
|
||
claim_token,
|
||
lease_seconds=max(1800, self.thread_count * 600),
|
||
)
|
||
except Exception as renew_error:
|
||
logger.warning(f"续租任务项失败: {domain_name}, 错误: {renew_error}")
|
||
|
||
try:
|
||
# 检查是否需要停止
|
||
if not self.running:
|
||
logger.info(f"检测已停止，跳过域名: {domain_name}")
|
||
finalize_job_item('failed', 'Worker 已停止，任务未执行')
|
||
return
|
||
|
||
logger.info(f"开始检测域名: {domain_name}")
|
||
self.db.update_domain_detect_status(domain_id, DETECT_STATUS_RUNNING)
|
||
if job_item_id and claim_token:
|
||
self.db.mark_detect_job_item_running(job_item_id, claim_token)
|
||
renew_job_item_lease("domain_started")
|
||
self.db.append_detect_run_event(
|
||
job_id,
|
||
job_item_id,
|
||
config.NODE_CODE,
|
||
event_type='domain_started',
|
||
message=f"开始检测域名: {domain_name}",
|
||
payload={
|
||
"domain_id": domain_id,
|
||
"domain": domain_name,
|
||
"cycle_token": cycle_token,
|
||
"job_code": job_code,
|
||
},
|
||
)
|
||
self._sync_worker_log_event(
|
||
f"开始检测域名: {domain_name}",
|
||
payload={
|
||
"domain_id": domain_id,
|
||
"domain": domain_name,
|
||
},
|
||
mode='full',
|
||
job_item_id=job_item_id,
|
||
)
|
||
|
||
# 使用共享的敏感词列表
|
||
sensitive_words = self.sensitive_words
|
||
logger.debug(f"线程 {thread_id} 使用共享敏感词，共 {len(sensitive_words)} 个敏感词")
|
||
for detect_key in self._get_detect_execution_order():
|
||
renew_job_item_lease(detect_key)
|
||
if detect_key == 'detect_register':
|
||
if not self._run_detect_register(domain_id, domain, domain_name):
|
||
domain_snapshot = self.db.get_domain_by_id(domain_id) or {}
|
||
finalize_job_item('blacklisted' if int(domain_snapshot.get('detect_status', 0) or 0) == DETECT_STATUS_BLACKLISTED else 'failed', '注册状态检测未通过')
|
||
return
|
||
elif detect_key == 'detect_baidu_site':
|
||
if not self._run_detect_baidu(domain_id, domain_name, sensitive_words):
|
||
domain_snapshot = self.db.get_domain_by_id(domain_id) or {}
|
||
finalize_job_item('blacklisted' if int(domain_snapshot.get('detect_status', 0) or 0) == DETECT_STATUS_BLACKLISTED else 'failed', '百度检测未通过')
|
||
return
|
||
elif detect_key == 'detect_360_site':
|
||
if not self._run_detect_360(domain_id, domain_name, sensitive_words):
|
||
domain_snapshot = self.db.get_domain_by_id(domain_id) or {}
|
||
finalize_job_item('blacklisted' if int(domain_snapshot.get('detect_status', 0) or 0) == DETECT_STATUS_BLACKLISTED else 'failed', '360检测未通过')
|
||
return
|
||
elif detect_key == 'detect_chinaz':
|
||
if not self._run_detect_chinaz(domain_id, domain_name, sensitive_words):
|
||
domain_snapshot = self.db.get_domain_by_id(domain_id) or {}
|
||
finalize_job_item('blacklisted' if int(domain_snapshot.get('detect_status', 0) or 0) == DETECT_STATUS_BLACKLISTED else 'failed', '站长之家检测未通过')
|
||
return
|
||
elif detect_key == 'detect_aizhan':
|
||
if not self._run_detect_aizhan(domain_id, domain_name, sensitive_words):
|
||
domain_snapshot = self.db.get_domain_by_id(domain_id) or {}
|
||
finalize_job_item('blacklisted' if int(domain_snapshot.get('detect_status', 0) or 0) == DETECT_STATUS_BLACKLISTED else 'failed', '爱站检测未通过')
|
||
return
|
||
elif detect_key == 'detect_wayback':
|
||
if not self._run_detect_wayback(domain_id, domain_name, sensitive_words):
|
||
domain_snapshot = self.db.get_domain_by_id(domain_id) or {}
|
||
finalize_job_item('blacklisted' if int(domain_snapshot.get('detect_status', 0) or 0) == DETECT_STATUS_BLACKLISTED else 'failed', '时光机检测未通过')
|
||
return
|
||
elif detect_key == 'detect_jucha':
|
||
if not self._run_detect_jucha(domain_id, domain, domain_name):
|
||
domain_snapshot = self.db.get_domain_by_id(domain_id) or {}
|
||
finalize_job_item('blacklisted' if int(domain_snapshot.get('detect_status', 0) or 0) == DETECT_STATUS_BLACKLISTED else 'failed', '聚查检测未通过')
|
||
return
|
||
elif detect_key == 'detect_juziseo':
|
||
if not self._run_detect_juziseo(domain_id, domain, domain_name, sensitive_words):
|
||
domain_snapshot = self.db.get_domain_by_id(domain_id) or {}
|
||
finalize_job_item('blacklisted' if int(domain_snapshot.get('detect_status', 0) or 0) == DETECT_STATUS_BLACKLISTED else 'failed', '桔子检测未通过')
|
||
return
|
||
|
||
self._complete_detection(domain_id, domain_name)
|
||
finalize_job_item('completed', f"域名检测完成: {domain_name}")
|
||
logger.info(f"域名检测完成: {domain_name}")
|
||
self._sync_worker_log_event(
|
||
f"域名检测完成: {domain_name}",
|
||
payload={
|
||
"domain_id": domain_id,
|
||
"domain": domain_name,
|
||
"status": "completed",
|
||
},
|
||
mode='full',
|
||
job_item_id=job_item_id,
|
||
)
|
||
|
||
except Exception as e:
|
||
logger.error(f"检测域名出错: {domain_name}, 错误: {e}")
|
||
self.db.update_domain_detect_status(domain_id, DETECT_STATUS_FAILED)
|
||
finalize_job_item('failed', str(e))
|
||
self._sync_worker_log_event(
|
||
f"检测域名出错: {domain_name}, 错误: {e}",
|
||
level='error',
|
||
payload={
|
||
"domain_id": domain_id,
|
||
"domain": domain_name,
|
||
"status": "failed",
|
||
},
|
||
mode='full',
|
||
job_item_id=job_item_id,
|
||
)
|
||
|
||
def start_detection(self):
|
||
"""
|
||
开始检测
|
||
"""
|
||
logger.info("开始执行域名检测任务")
|
||
self.detecting = True
|
||
self.stop_requested = False
|
||
self._mark_detection_phase("preparing", "开始执行域名检测任务，正在加载配置")
|
||
self._sync_worker_log_event("开始执行域名检测任务，正在加载配置", mode='key')
|
||
|
||
# 重新加载配置，确保获取最新的配置
|
||
try:
|
||
self.detect_options = self.load_detect_options()
|
||
self.proxy_config = self.load_proxy_config()
|
||
self.thread_count = self.load_thread_count()
|
||
self.proxy_max_reuse_count = max(8, self.thread_count * 4)
|
||
logger.info(f"检测线程数设置为: {self.thread_count}")
|
||
self.load_cookies_from_remote()
|
||
self._init_optional_detectors()
|
||
recycled = self.db.recycle_running_domains(DETECT_STATUS_FAILED)
|
||
if recycled:
|
||
logger.warning(f"检测启动前回收了 {recycled} 个遗留的检测中任务，已标记为检测失败待重试")
|
||
released_job_items = self.db.release_detect_job_items_for_node(config.NODE_CODE)
|
||
if released_job_items:
|
||
logger.warning(f"检测启动前释放了 {released_job_items} 个当前节点遗留任务项，已重新回到 pending")
|
||
except Exception as e:
|
||
logger.error(f"加载配置失败: {e}")
|
||
import traceback
|
||
logger.error(traceback.format_exc())
|
||
self._mark_detection_phase("failed", f"加载配置失败: {e}")
|
||
self._sync_worker_log_event(f"加载配置失败: {e}", level='error', mode='key')
|
||
return
|
||
|
||
# 重新加载cookies
|
||
self.load_cookies_from_remote()
|
||
|
||
# 重新加载JC实例的cookies
|
||
if self.detect_options.get('detect_jucha') and self.jc_instance is not None:
|
||
self.jc_instance.load_cookies()
|
||
self.jc_instance.load_juming_cookies()
|
||
|
||
# 重新加载Juziseo实例的cookies
|
||
if self.detect_options.get('detect_juziseo') and self.juziseo_instance is not None:
|
||
self.juziseo_instance.load_cookies()
|
||
|
||
# 刷新代理池
|
||
if self.proxy_config.get('proxy_enable', False):
|
||
logger.info("开始检测，刷新代理池")
|
||
self._mark_detection_phase("refreshing_proxy", "开始检测，正在刷新代理池")
|
||
self._sync_worker_log_event("开始检测，正在刷新代理池", mode='key')
|
||
self.refresh_proxy_pool()
|
||
|
||
# 更新JC和Juziseo实例的代理设置
|
||
if self.detect_options.get('detect_jucha') and self.jc_instance is not None:
|
||
self.jc_instance.proxies = self.get_proxies()
|
||
if self.detect_options.get('detect_juziseo') and self.juziseo_instance is not None:
|
||
self.juziseo_instance.proxies = self.get_proxies()
|
||
|
||
# 重新加载敏感词，确保获取最新的敏感词列表
|
||
try:
|
||
self.sensitive_words = self.db.get_all_sensitive_words()
|
||
logger.info(f"重新加载敏感词完成，共 {len(self.sensitive_words)} 个敏感词")
|
||
except Exception as e:
|
||
logger.error(f"重新加载敏感词失败: {e}")
|
||
import traceback
|
||
logger.error(traceback.format_exc())
|
||
self.sensitive_words = []
|
||
|
||
# 更新配置标签
|
||
self.update_config_labels()
|
||
|
||
try:
|
||
batch_size = 1000
|
||
total_processed = 0
|
||
claim_batch_size = min(batch_size, max(50, self.thread_count * 8))
|
||
|
||
while self.running:
|
||
if self.stop_requested:
|
||
logger.info("检测任务收到停止请求，停止继续领取任务")
|
||
self._mark_detection_phase("stopping", "检测任务收到停止请求，准备安全退出")
|
||
self._sync_worker_log_event("检测任务收到停止请求，准备安全退出", level='warning', mode='key')
|
||
break
|
||
recycled_job_items = self.db.recycle_expired_detect_job_items()
|
||
if recycled_job_items:
|
||
logger.warning(f"已回收 {recycled_job_items} 个租约过期的任务项，重新回到 pending")
|
||
# 获取需要检测的域名
|
||
domains = self.db.claim_detect_job_items(
|
||
config.NODE_CODE,
|
||
limit=claim_batch_size,
|
||
lease_seconds=max(1800, self.thread_count * 600),
|
||
)
|
||
using_job_queue = bool(domains)
|
||
if not domains:
|
||
domains = self.db.get_domains_to_detect(limit=batch_size, detect_options=self.detect_options)
|
||
current_batch_size = len(domains)
|
||
queue_label = "任务队列" if using_job_queue else "兼容旧链路"
|
||
logger.info(f"从{queue_label}获取到 {current_batch_size} 个需要检测的域名")
|
||
self._mark_detection_phase(
|
||
"fetching",
|
||
f"从{queue_label}获取到 {current_batch_size} 个需要检测的域名",
|
||
batch_size=current_batch_size,
|
||
queue_source="detect_job_items" if using_job_queue else "domains",
|
||
)
|
||
self._sync_worker_log_event(
|
||
f"从{queue_label}获取到 {current_batch_size} 个需要检测的域名",
|
||
payload={
|
||
"batch_size": current_batch_size,
|
||
"queue_source": "detect_job_items" if using_job_queue else "domains",
|
||
},
|
||
mode='full',
|
||
)
|
||
|
||
if not domains:
|
||
logger.info("没有需要检测的域名")
|
||
self._mark_detection_phase("idle", "当前没有需要检测的域名，Worker 等待下一次启动")
|
||
self._sync_worker_log_event("当前没有需要检测的域名，Worker 等待下一次启动", mode='key')
|
||
break
|
||
|
||
# 旧链路直接扫 domains 表时，少于 batch_size 代表已接近尾批；
|
||
# 任务队列模式会按小批量持续领取，不能把“小批量正常领取”误判为整轮结束。
|
||
should_stop = (not using_job_queue) and current_batch_size < batch_size
|
||
|
||
# 创建线程池，限制同时运行的线程数量
|
||
active_threads = []
|
||
max_threads = self.thread_count
|
||
|
||
try:
|
||
logger.info(f"开始创建线程，当前批次域名数: {current_batch_size}，最大线程数: {max_threads}")
|
||
self._mark_detection_phase(
|
||
"creating_threads",
|
||
f"开始创建线程，当前批次域名数: {current_batch_size}，最大线程数: {max_threads}",
|
||
batch_size=current_batch_size,
|
||
max_threads=max_threads,
|
||
)
|
||
self._sync_worker_log_event(
|
||
f"开始创建线程，当前批次域名数: {current_batch_size}，最大线程数: {max_threads}",
|
||
payload={"batch_size": current_batch_size, "max_threads": max_threads},
|
||
mode='full',
|
||
)
|
||
for i, domain in enumerate(domains):
|
||
# 检查是否需要停止
|
||
if not self.running or self.stop_requested:
|
||
logger.info("检测已停止，停止创建新线程")
|
||
self._mark_detection_phase("stopping", "检测已停止，停止创建新线程")
|
||
break
|
||
|
||
# 等待线程数量降到最大值以下
|
||
while len([t for t in active_threads if t.is_alive()]) >= max_threads:
|
||
# 清理已完成的线程
|
||
active_threads = [t for t in active_threads if t.is_alive()]
|
||
# 短暂休眠，避免CPU占用过高
|
||
import time
|
||
time.sleep(0.1)
|
||
|
||
domain_id = domain['id']
|
||
domain_name = domain['domain']
|
||
|
||
# 更新进度
|
||
if self.detect_thread:
|
||
try:
|
||
# 通过信号发送进度更新
|
||
self.detect_thread.progress_signal.emit(total_processed + i, total_processed + current_batch_size)
|
||
except Exception as e:
|
||
logger.error(f"发送进度更新信号失败: {e}")
|
||
import traceback
|
||
logger.error(traceback.format_exc())
|
||
|
||
# 创建线程
|
||
try:
|
||
job_context = {
|
||
'job_item_id': domain.get('job_item_id'),
|
||
'job_id': domain.get('job_id'),
|
||
'claim_token': domain.get('claim_token'),
|
||
'cycle_token': self.current_cycle_token,
|
||
'job_code': self.current_job_code,
|
||
} if using_job_queue else None
|
||
thread = threading.Thread(target=self.detect_domain, args=(domain_id, domain, job_context))
|
||
active_threads.append(thread)
|
||
logger.debug(f"创建线程 {i+1} 成功，域名: {domain_name}")
|
||
except Exception as e:
|
||
logger.error(f"创建线程失败: {e}")
|
||
import traceback
|
||
logger.error(traceback.format_exc())
|
||
continue
|
||
|
||
# 启动线程
|
||
try:
|
||
thread.start()
|
||
logger.debug(f"启动线程 {i+1} 成功，域名: {domain_name}")
|
||
except Exception as e:
|
||
logger.error(f"启动线程失败: {e}")
|
||
import traceback
|
||
logger.error(traceback.format_exc())
|
||
active_threads.remove(thread)
|
||
continue
|
||
|
||
# 显示当前实际线程数量
|
||
current_active = len([t for t in active_threads if t.is_alive()])
|
||
logger.info(f"当前实际线程数量: {current_active}/{max_threads}")
|
||
self._mark_detection_phase(
|
||
"running",
|
||
f"当前实际线程数量: {current_active}/{max_threads}",
|
||
active_threads=current_active,
|
||
max_threads=max_threads,
|
||
batch_size=current_batch_size,
|
||
)
|
||
self._sync_worker_log_event(
|
||
f"当前实际线程数量: {current_active}/{max_threads}",
|
||
payload={
|
||
"active_threads": current_active,
|
||
"max_threads": max_threads,
|
||
"batch_size": current_batch_size,
|
||
},
|
||
mode='full',
|
||
)
|
||
|
||
# 更新GUI显示
|
||
if self.detect_thread:
|
||
try:
|
||
# 通过信号发送线程数量更新
|
||
self.detect_thread.thread_count_signal.emit(current_active, max_threads)
|
||
except Exception as e:
|
||
logger.error(f"发送线程数量更新信号失败: {e}")
|
||
import traceback
|
||
logger.error(traceback.format_exc())
|
||
except Exception as e:
|
||
logger.error(f"创建或启动线程失败: {e}")
|
||
import traceback
|
||
logger.error(traceback.format_exc())
|
||
|
||
# 等待所有线程完成
|
||
logger.info(f"等待剩余 {len(active_threads)} 个线程完成")
|
||
for i, thread in enumerate(active_threads):
|
||
# 检查是否需要停止
|
||
if not self.running or self.stop_requested:
|
||
logger.info("检测已停止，停止等待线程完成")
|
||
self._mark_detection_phase("stopping", "检测已停止，停止等待线程完成")
|
||
break
|
||
try:
|
||
thread.join(timeout=30) # 添加超时，避免线程阻塞
|
||
current_active = len([t for t in active_threads if t.is_alive()])
|
||
self._mark_detection_phase(
|
||
"running",
|
||
f"当前实际线程数量: {current_active}/{max_threads}",
|
||
active_threads=current_active,
|
||
max_threads=max_threads,
|
||
batch_size=current_batch_size,
|
||
processed=total_processed + i,
|
||
)
|
||
# 更新进度
|
||
if self.detect_thread:
|
||
try:
|
||
# 通过信号发送进度更新
|
||
self.detect_thread.progress_signal.emit(total_processed + i + 1, total_processed + current_batch_size)
|
||
except Exception as e:
|
||
logger.error(f"发送进度更新信号失败: {e}")
|
||
import traceback
|
||
logger.error(traceback.format_exc())
|
||
except Exception as e:
|
||
logger.error(f"等待线程完成失败: {e}")
|
||
import traceback
|
||
logger.error(traceback.format_exc())
|
||
|
||
# 发送线程数量更新信号
|
||
if self.detect_thread:
|
||
try:
|
||
active_count = threading.active_count()
|
||
self.detect_thread.thread_count_signal.emit(active_count, max_threads)
|
||
except Exception as e:
|
||
logger.error(f"发送线程数量更新信号失败: {e}")
|
||
import traceback
|
||
logger.error(traceback.format_exc())
|
||
|
||
# 完成当前批次进度
|
||
if self.detect_thread:
|
||
try:
|
||
# 通过信号发送进度更新
|
||
self.detect_thread.progress_signal.emit(total_processed + current_batch_size, total_processed + current_batch_size)
|
||
except Exception as e:
|
||
logger.error(f"发送进度更新信号失败: {e}")
|
||
import traceback
|
||
logger.error(traceback.format_exc())
|
||
|
||
total_processed += current_batch_size
|
||
|
||
# 如果当前批次数量少于batch_size，检测完成后停止获取域名
|
||
if should_stop:
|
||
logger.info(f"当前批次域名数量 ({current_batch_size}) 少于 {batch_size}，停止获取域名")
|
||
self._mark_detection_phase(
|
||
"completing",
|
||
f"当前批次域名数量 ({current_batch_size}) 少于 {batch_size}，本轮检测即将完成",
|
||
processed=total_processed,
|
||
)
|
||
self._sync_worker_log_event(
|
||
f"当前批次域名数量 ({current_batch_size}) 少于 {batch_size}，本轮检测即将完成",
|
||
payload={"processed": total_processed},
|
||
mode='key',
|
||
)
|
||
break
|
||
logger.info(f"当前批次检测完成，累计处理 {total_processed} 个域名")
|
||
self._mark_detection_phase("batch_completed", f"当前批次检测完成，累计处理 {total_processed} 个域名", processed=total_processed)
|
||
self._sync_worker_log_event(
|
||
f"当前批次检测完成，累计处理 {total_processed} 个域名",
|
||
payload={"processed": total_processed},
|
||
mode='key',
|
||
)
|
||
|
||
# 完成进度
|
||
if self.detect_thread:
|
||
try:
|
||
# 通过信号发送进度更新
|
||
self.detect_thread.progress_signal.emit(100, 100)
|
||
except Exception as e:
|
||
logger.error(f"发送进度更新信号失败: {e}")
|
||
import traceback
|
||
logger.error(traceback.format_exc())
|
||
|
||
logger.info("域名检测任务完成")
|
||
self._mark_detection_phase("completed", "域名检测任务完成")
|
||
self._sync_worker_log_event("域名检测任务完成", payload={"processed": total_processed}, mode='key')
|
||
|
||
except Exception as e:
|
||
logger.error(f"执行检测任务出错: {e}")
|
||
self._mark_detection_phase("failed", f"执行检测任务出错: {e}")
|
||
self._sync_worker_log_event(f"执行检测任务出错: {e}", level='error', mode='key')
|
||
finally:
|
||
self.detecting = False
|
||
|
||
def run_daily_task(self):
|
||
"""
|
||
执行每日检测任务
|
||
"""
|
||
logger.info(f"[{datetime.now()}] 开始每日检测任务")
|
||
self.start_detection()
|
||
logger.info(f"[{datetime.now()}] 每日检测任务完成")
|
||
|
||
def start_scheduler(self):
|
||
"""
|
||
启动定时任务
|
||
"""
|
||
# 避免重复注册相同定时任务
|
||
schedule.clear('daily_detection')
|
||
# 每天凌晨1点执行检测；启动程序时不自动跑一轮，避免和人工启动混淆
|
||
schedule.every().day.at("01:00").do(self.run_daily_task).tag('daily_detection')
|
||
logger.info("定时任务已启动，每天凌晨1点执行检测；程序启动后不会立即自动执行")
|
||
|
||
# 循环执行定时任务
|
||
while self.running:
|
||
schedule.run_pending()
|
||
time.sleep(60)
|
||
|
||
def start(self):
|
||
"""
|
||
启动检测端
|
||
"""
|
||
logger.info("启动域名检测端")
|
||
self.running = True
|
||
|
||
# 启动定时任务
|
||
scheduler_thread = threading.Thread(target=self.start_scheduler)
|
||
scheduler_thread.daemon = True
|
||
scheduler_thread.start()
|
||
|
||
# 保持程序运行
|
||
try:
|
||
while self.running:
|
||
time.sleep(1)
|
||
except KeyboardInterrupt:
|
||
logger.info("检测端已停止")
|
||
self.running = False
|
||
|
||
def load_cookies_from_remote(self):
|
||
"""
|
||
从远程加载cookies
|
||
"""
|
||
try:
|
||
if self.use_redis:
|
||
# 加载聚名cookies
|
||
juming_cookies_str = self.redis_client.get('domain_tool:juming_cookies')
|
||
if juming_cookies_str:
|
||
# 将cookies保存到本地文件
|
||
import pickle
|
||
from requests.cookies import RequestsCookieJar
|
||
|
||
# 创建cookie jar
|
||
cookie_jar = RequestsCookieJar()
|
||
|
||
# 解析cookies字符串
|
||
try:
|
||
import ast
|
||
cookies_dict = ast.literal_eval(juming_cookies_str)
|
||
for name, value in cookies_dict.items():
|
||
cookie_jar.set(name, value)
|
||
except Exception as e:
|
||
logger.error(f"解析聚名cookies失败: {e}")
|
||
|
||
# 保存到本地文件
|
||
try:
|
||
with open('juming_cookies.pkl', 'wb') as f:
|
||
pickle.dump(cookie_jar, f)
|
||
logger.info("从Redis加载聚名cookies成功并保存到本地")
|
||
except Exception as e:
|
||
logger.error(f"保存聚名cookies到本地失败: {e}")
|
||
|
||
# 加载聚查cookies
|
||
jucha_cookies_str = self.redis_client.get('domain_tool:jucha_cookies')
|
||
if jucha_cookies_str:
|
||
# 将cookies保存到本地文件
|
||
import pickle
|
||
from requests.cookies import RequestsCookieJar
|
||
|
||
# 创建cookie jar
|
||
cookie_jar = RequestsCookieJar()
|
||
|
||
# 解析cookies字符串
|
||
try:
|
||
import ast
|
||
cookies_dict = ast.literal_eval(jucha_cookies_str)
|
||
for name, value in cookies_dict.items():
|
||
cookie_jar.set(name, value)
|
||
except Exception as e:
|
||
logger.error(f"解析聚查cookies失败: {e}")
|
||
|
||
# 保存到本地文件
|
||
try:
|
||
with open('jucha_cookies.pkl', 'wb') as f:
|
||
pickle.dump(cookie_jar, f)
|
||
logger.info("从Redis加载聚查cookies成功并保存到本地")
|
||
except Exception as e:
|
||
logger.error(f"保存聚查cookies到本地失败: {e}")
|
||
|
||
# 加载桔子SEO cookies
|
||
juziseo_cookies_str = self.redis_client.get('domain_tool:juziseo_cookies')
|
||
if juziseo_cookies_str:
|
||
# 将cookies保存到本地文件
|
||
import pickle
|
||
from requests.cookies import RequestsCookieJar
|
||
|
||
# 创建cookie jar
|
||
cookie_jar = RequestsCookieJar()
|
||
|
||
# 解析cookies字符串
|
||
try:
|
||
import ast
|
||
cookies_dict = ast.literal_eval(juziseo_cookies_str)
|
||
for name, value in cookies_dict.items():
|
||
cookie_jar.set(name, value)
|
||
except Exception as e:
|
||
logger.error(f"解析桔子SEO cookies失败: {e}")
|
||
|
||
# 保存到本地文件
|
||
try:
|
||
with open('juziseo_cookies.pkl', 'wb') as f:
|
||
pickle.dump(cookie_jar, f)
|
||
logger.info("从Redis加载桔子SEO cookies成功并保存到本地")
|
||
except Exception as e:
|
||
logger.error(f"保存桔子SEO cookies到本地失败: {e}")
|
||
except Exception as e:
|
||
logger.error(f"从远程加载cookies失败: {e}")
|
||
|
||
def update_config_labels(self):
|
||
"""
|
||
更新GUI配置标签
|
||
"""
|
||
# 使用QCoreApplication.postEvent发送配置更新事件，这是最安全的方法
|
||
try:
|
||
logger.debug("发送配置更新事件")
|
||
# 确保主窗口存在
|
||
if self.main_window:
|
||
from PySide6.QtCore import QCoreApplication
|
||
event = ConfigUpdateEvent()
|
||
QCoreApplication.postEvent(self.main_window, event)
|
||
logger.debug("配置更新事件已发送")
|
||
else:
|
||
logger.debug("主窗口不存在，跳过配置更新")
|
||
except Exception as e:
|
||
logger.error(f"发送配置更新事件失败: {e}")
|
||
|
||
|
||
|
||
def start_redis_subscription(self):
|
||
"""
|
||
启动Redis订阅，监听配置更新
|
||
"""
|
||
while self.running:
|
||
pubsub = None
|
||
try:
|
||
# 创建新的Redis客户端用于订阅
|
||
redis_sub_client = redis.Redis(
|
||
host=config.REDIS_HOST,
|
||
port=config.REDIS_PORT,
|
||
password=config.REDIS_PASSWORD,
|
||
db=config.REDIS_DB,
|
||
decode_responses=True,
|
||
socket_connect_timeout=30, # 增加连接超时时间
|
||
socket_timeout=60, # 增加读取超时时间
|
||
retry_on_timeout=True,
|
||
health_check_interval=30
|
||
)
|
||
|
||
# 订阅配置更新频道
|
||
pubsub = redis_sub_client.pubsub()
|
||
pubsub.subscribe(CONFIG_UPDATE_CHANNEL, CONTROL_CHANNEL)
|
||
|
||
# 使用logger.debug输出到文件日志，不输出到GUI日志框
|
||
logger.debug("开始监听配置更新...")
|
||
self._consume_pending_control_command()
|
||
|
||
# 循环监听消息
|
||
while self.running:
|
||
message = pubsub.get_message(ignore_subscribe_messages=True, timeout=1.0)
|
||
if not message:
|
||
continue
|
||
if message['type'] == 'message':
|
||
channel = message.get('channel', '')
|
||
payload = message['data']
|
||
if channel == CONFIG_UPDATE_CHANNEL:
|
||
config_type = payload
|
||
logger.debug(f"收到配置更新消息: {config_type}")
|
||
self.detect_options = self.load_detect_options()
|
||
self.proxy_config = self.load_proxy_config()
|
||
self.thread_count = self.load_thread_count()
|
||
self.runtime_settings = self.load_runtime_settings()
|
||
self.load_cookies_from_remote()
|
||
self.update_config_labels()
|
||
logger.debug("配置已更新")
|
||
continue
|
||
|
||
if channel == CONTROL_CHANNEL:
|
||
logger.info(f"收到 Worker 控制消息: {payload}")
|
||
self._handle_control_message(payload)
|
||
except redis.exceptions.TimeoutError:
|
||
# Pub/Sub 空闲超时在 Linux 服务态下属于常见现象，不需要中断订阅。
|
||
continue
|
||
except Exception as e:
|
||
logger.debug(f"Redis订阅失败: {e}")
|
||
logger.debug("3秒后尝试重新连接...")
|
||
time.sleep(3) # 增加重试间隔
|
||
finally:
|
||
if pubsub is not None:
|
||
try:
|
||
pubsub.close()
|
||
except Exception:
|
||
pass
|
||
|
||
def stop(self):
|
||
"""
|
||
停止检测端
|
||
"""
|
||
logger.info("停止域名检测端")
|
||
self._runtime_heartbeat_stop.set()
|
||
self.stop_requested = True
|
||
self.running = False
|
||
self.detecting = False
|
||
self._update_runtime_state("stopped", "Worker 已停止")
|
||
|
||
if __name__ == "__main__":
|
||
try:
|
||
# 配置日志
|
||
logger.add("detect_worker.log", rotation="1 day", level="DEBUG")
|
||
logger.info("程序开始运行")
|
||
|
||
# 创建应用程序
|
||
app = QApplication([])
|
||
logger.info("创建应用程序成功")
|
||
|
||
# 创建并显示主窗口
|
||
window = DetectMainWindow()
|
||
logger.info("创建主窗口成功")
|
||
window.show()
|
||
logger.info("显示主窗口成功")
|
||
|
||
# 运行应用程序
|
||
logger.info("开始运行应用程序")
|
||
app.exec()
|
||
logger.info("应用程序运行结束")
|
||
|
||
# 当窗口关闭时，停止检测端
|
||
if window.worker:
|
||
logger.info("停止检测端")
|
||
window.worker.stop()
|
||
except Exception as e:
|
||
logger.error(f"程序运行出错: {e}")
|
||
import traceback
|
||
logger.error(traceback.format_exc())
|
||
import time
|
||
time.sleep(10) # 等待10秒，以便查看错误信息
|