2182 lines
95 KiB
Python
2182 lines
95 KiB
Python
# -*- coding: UTF-8 -*-
|
||
'''
|
||
@Project :domainScanDemo
|
||
@File :detect_worker.py
|
||
@IDE :PyCharm
|
||
@Author :梦伴
|
||
@Date :2026/4/10 22:40
|
||
@explain : 域名检测端程序
|
||
'''
|
||
|
||
import os
|
||
import sys
|
||
import json
|
||
import time
|
||
import threading
|
||
import schedule
|
||
from datetime import datetime
|
||
from loguru import logger
|
||
from queue import Queue
|
||
from PySide6.QtWidgets import QApplication, QMainWindow, QTextEdit, QPushButton, QVBoxLayout, QHBoxLayout, QWidget, QLabel, QProgressBar, QGroupBox, QScrollArea, QFrame
|
||
from PySide6.QtCore import Qt, QThread, Signal, QMetaObject, Q_ARG, QCoreApplication, QEvent, QTimer
|
||
from PySide6.QtGui import QIcon
|
||
|
||
from app.utils.database import Database
|
||
from app.config import config
|
||
from app.detectors.wayback_detector import WaybackDetector
|
||
from app.utils.status_codes import (
|
||
DETECT_STATUS_BLACKLISTED,
|
||
DETECT_STATUS_COMPLETED,
|
||
DETECT_STATUS_FAILED,
|
||
DETECT_STATUS_RUNNING,
|
||
REGISTER_STATUS_AVAILABLE,
|
||
REVIEW_STATUS_PENDING,
|
||
THIRD_PARTY_STATUS_DONE,
|
||
)
|
||
import redis
|
||
from detect import aizhan, baidu, c360, chinaz, register, jucha, juziseo
|
||
|
||
class DetectThread(QThread):
|
||
"""
|
||
检测线程
|
||
"""
|
||
log_signal = Signal(str)
|
||
progress_signal = Signal(int, int)
|
||
finished_signal = Signal()
|
||
thread_count_signal = Signal(int, int)
|
||
|
||
def __init__(self, worker):
|
||
super().__init__()
|
||
self.worker = worker
|
||
# 设置线程名称
|
||
self.setObjectName("DetectThread")
|
||
|
||
def run(self):
|
||
# 将信号传递给worker
|
||
self.worker.detect_thread = self
|
||
self.worker.start_detection()
|
||
self.finished_signal.emit()
|
||
|
||
class DetectMainWindow(QMainWindow):
|
||
"""
|
||
检测端主窗口
|
||
"""
|
||
|
||
def __init__(self):
|
||
super().__init__()
|
||
self.setWindowTitle("域名检测端")
|
||
self.setGeometry(100, 100, 980, 820)
|
||
self.setMinimumSize(860, 680)
|
||
|
||
# 设置窗口图标
|
||
# 打印当前文件路径以便诊断
|
||
current_file = os.path.abspath(__file__)
|
||
current_dir = os.path.dirname(current_file)
|
||
logger.info(f"当前文件路径: {current_file}")
|
||
logger.info(f"当前目录: {current_dir}")
|
||
|
||
# 获取PyInstaller打包后的临时目录
|
||
if hasattr(sys, '_MEIPASS'):
|
||
# 打包后运行
|
||
base_dir = sys._MEIPASS
|
||
logger.info(f"PyInstaller临时目录: {base_dir}")
|
||
else:
|
||
# 开发环境运行
|
||
base_dir = os.path.dirname(os.path.dirname(current_dir))
|
||
logger.info(f"开发环境目录: {base_dir}")
|
||
|
||
# 构建图标路径
|
||
icon_path = os.path.join(base_dir, "new_logo.svg")
|
||
logger.info(f"SVG图标路径: {icon_path}")
|
||
|
||
if os.path.exists(icon_path):
|
||
logger.info(f"SVG图标文件存在: {icon_path}")
|
||
try:
|
||
icon = QIcon(icon_path)
|
||
if icon.isNull():
|
||
logger.warning(f"SVG图标加载失败，图标为空: {icon_path}")
|
||
else:
|
||
self.setWindowIcon(icon)
|
||
logger.info(f"设置SVG窗口图标成功: {icon_path}")
|
||
except Exception as e:
|
||
logger.error(f"加载SVG图标时出错: {e}")
|
||
else:
|
||
logger.warning(f"SVG图标文件不存在: {icon_path}")
|
||
|
||
# 如果SVG加载失败，使用ico文件作为备用
|
||
ico_icon_path = os.path.join(base_dir, "favicon2.ico")
|
||
logger.info(f"ICO图标路径: {ico_icon_path}")
|
||
|
||
if os.path.exists(ico_icon_path):
|
||
logger.info(f"ICO图标文件存在: {ico_icon_path}")
|
||
try:
|
||
icon = QIcon(ico_icon_path)
|
||
if icon.isNull():
|
||
logger.warning(f"ICO图标加载失败，图标为空: {ico_icon_path}")
|
||
else:
|
||
self.setWindowIcon(icon)
|
||
logger.info(f"设置ICO窗口图标成功: {ico_icon_path}")
|
||
except Exception as e:
|
||
logger.error(f"加载ICO图标时出错: {e}")
|
||
else:
|
||
logger.warning(f"ICO图标文件不存在: {ico_icon_path}")
|
||
|
||
# 设置窗口样式
|
||
self.setStyleSheet(""
|
||
"QMainWindow {"
|
||
" background-color: #f5f5f5;"
|
||
" font-family: 'Microsoft YaHei', Arial, sans-serif;"
|
||
"}"
|
||
"QLabel {"
|
||
" font-size: 14px;"
|
||
" color: #333;"
|
||
" padding: 5px 0;"
|
||
"}"
|
||
"QGroupBox {"
|
||
" font-size: 16px;"
|
||
" font-weight: bold;"
|
||
" border: 1px solid #ddd;"
|
||
" border-radius: 8px;"
|
||
" margin-top: 15px;"
|
||
" padding: 15px;"
|
||
" background-color: #ffffff;"
|
||
"}"
|
||
"QGroupBox::title {"
|
||
" subcontrol-origin: margin;"
|
||
" subcontrol-position: top left;"
|
||
" padding: 0 15px;"
|
||
" background-color: #4CAF50;"
|
||
" color: white;"
|
||
" border-radius: 4px;"
|
||
" font-size: 14px;"
|
||
"}"
|
||
"QProgressBar {"
|
||
" border: 1px solid #ddd;"
|
||
" border-radius: 6px;"
|
||
" text-align: center;"
|
||
" background-color: #f0f0f0;"
|
||
" height: 25px;"
|
||
"}"
|
||
"QProgressBar::chunk {"
|
||
" background-color: #4CAF50;"
|
||
" border-radius: 6px;"
|
||
"}"
|
||
"QTextEdit {"
|
||
" font-family: Consolas, 'Courier New', monospace;"
|
||
" font-size: 12px;"
|
||
" border: 1px solid #ddd;"
|
||
" border-radius: 6px;"
|
||
" background-color: #2d2d2d;"
|
||
" color: #e0e0e0;"
|
||
" padding: 10px;"
|
||
"}"
|
||
"QPushButton {"
|
||
" font-family: 'PingFang SC', 'Helvetica Neue', 'Microsoft YaHei', sans-serif;"
|
||
" font-size: 12px;"
|
||
" font-weight: 600;"
|
||
" padding: 6px 12px;"
|
||
" border: none;"
|
||
" border-radius: 4px;"
|
||
" color: white;"
|
||
" min-width: 80px;"
|
||
" min-height: 28px;"
|
||
" text-align: center;"
|
||
"}"
|
||
"QPushButton:hover {"
|
||
" opacity: 0.9;"
|
||
"}"
|
||
"QPushButton:disabled {"
|
||
" opacity: 0.5;"
|
||
"}"
|
||
"QPushButton#start_button {"
|
||
" background-color: #4CAF50;"
|
||
"}"
|
||
"QPushButton#stop_button {"
|
||
" background-color: #f44336;"
|
||
"}"
|
||
"QPushButton#exit_button {"
|
||
" background-color: #2196F3;"
|
||
"}"
|
||
"")
|
||
|
||
# 创建中心部件 + 内部滚动区域
|
||
scroll_area = QScrollArea()
|
||
scroll_area.setWidgetResizable(True)
|
||
scroll_area.setFrameShape(QFrame.NoFrame)
|
||
scroll_area.setVerticalScrollBarPolicy(Qt.ScrollBarAsNeeded)
|
||
scroll_area.setHorizontalScrollBarPolicy(Qt.ScrollBarAsNeeded)
|
||
self.setCentralWidget(scroll_area)
|
||
|
||
central_widget = QWidget()
|
||
central_widget.setMinimumWidth(920)
|
||
scroll_area.setWidget(central_widget)
|
||
|
||
# 创建布局
|
||
layout = QVBoxLayout(central_widget)
|
||
layout.setSpacing(10)
|
||
layout.setContentsMargins(20, 20, 20, 20)
|
||
|
||
# 创建状态标签
|
||
self.status_label = QLabel("状态: 就绪")
|
||
self.status_label.setStyleSheet("font-family: 'PingFang SC', 'Helvetica Neue', 'Microsoft YaHei', sans-serif; font-size: 14px; font-weight: 600; color: #2c3e50;")
|
||
layout.addWidget(self.status_label)
|
||
|
||
# 创建配置信息显示区域
|
||
config_group = QGroupBox("配置信息")
|
||
config_group.setStyleSheet("""
|
||
QGroupBox {
|
||
font-size: 16px;
|
||
font-weight: 600;
|
||
color: #2c3e50;
|
||
border: 1px solid #e2e8f0;
|
||
border-radius: 10px;
|
||
margin-top: 20px;
|
||
padding: 0;
|
||
background-color: #ffffff;
|
||
}
|
||
QGroupBox::title {
|
||
subcontrol-origin: margin;
|
||
subcontrol-position: top left;
|
||
padding: 8px 20px;
|
||
background: linear-gradient(135deg, #667eea 0%, #764ba2 100%);
|
||
color: white;
|
||
border-radius: 10px 10px 0 0;
|
||
font-family: 'PingFang SC', 'Helvetica Neue', 'Microsoft YaHei', sans-serif;
|
||
font-size: 14px;
|
||
font-weight: 600;
|
||
}
|
||
""")
|
||
config_layout = QVBoxLayout()
|
||
config_layout.setSpacing(12)
|
||
config_layout.setContentsMargins(20, 20, 20, 20)
|
||
|
||
# 创建配置标签
|
||
self.config_labels = {
|
||
'detect_options': QLabel("检测选项: 加载中..."),
|
||
'proxy_config': QLabel("代理配置: 加载中..."),
|
||
'thread_count': QLabel("检测线程数: 加载中...")
|
||
}
|
||
|
||
# 添加配置标签到布局
|
||
for label in self.config_labels.values():
|
||
label.setStyleSheet("""
|
||
QLabel {
|
||
font-family: 'PingFang SC', 'Helvetica Neue', 'Microsoft YaHei', sans-serif;
|
||
font-size: 13px;
|
||
font-weight: 400;
|
||
color: #4a5568;
|
||
background-color: #f7fafc;
|
||
padding: 12px 18px;
|
||
border-radius: 8px;
|
||
border-left: 4px solid #667eea;
|
||
margin: 0;
|
||
min-height: 64px;
|
||
white-space: normal;
|
||
border: 1px solid #e2e8f0;
|
||
}
|
||
QLabel:hover {
|
||
background-color: #edf2f7;
|
||
border-left-color: #764ba2;
|
||
}
|
||
""")
|
||
label.setWordWrap(True) # 启用自动换行
|
||
label.setAlignment(Qt.AlignLeft | Qt.AlignTop)
|
||
config_layout.addWidget(label)
|
||
|
||
config_group.setLayout(config_layout)
|
||
layout.addWidget(config_group)
|
||
|
||
# 创建进度条
|
||
self.progress_bar = QProgressBar()
|
||
self.progress_bar.setValue(0)
|
||
layout.addWidget(self.progress_bar)
|
||
|
||
# 创建日志文本框
|
||
self.log_text = QTextEdit()
|
||
self.log_text.setReadOnly(True)
|
||
self.log_text.setStyleSheet("font-family: Consolas, monospace; font-size: 12px; border: 1px solid #ddd; border-radius: 5px; background-color: #2d2d2d; color: #e0e0e0;")
|
||
self.log_text.setMinimumHeight(320)
|
||
layout.addWidget(self.log_text, 1)
|
||
|
||
# 创建按钮布局
|
||
button_layout = QHBoxLayout()
|
||
button_layout.setSpacing(10)
|
||
|
||
# 按钮样式表
|
||
start_button_style = """
|
||
QPushButton {
|
||
background-color: #28a745;
|
||
color: white;
|
||
border: none;
|
||
border-radius: 8px;
|
||
font-size: 16px;
|
||
font-weight: bold;
|
||
padding: 10px 20px;
|
||
min-width: 120px;
|
||
transition: all 0.3s ease;
|
||
}
|
||
QPushButton:hover {
|
||
background-color: #218838;
|
||
transform: translateY(-2px);
|
||
box-shadow: 0 4px 8px rgba(0, 0, 0, 0.2);
|
||
}
|
||
QPushButton:pressed {
|
||
background-color: #1e7e34;
|
||
transform: translateY(1px);
|
||
box-shadow: 0 2px 4px rgba(0, 0, 0, 0.2);
|
||
}
|
||
QPushButton:disabled {
|
||
background-color: #6c757d;
|
||
color: #adb5bd;
|
||
transform: none;
|
||
box-shadow: none;
|
||
}
|
||
"""
|
||
|
||
refresh_button_style = """
|
||
QPushButton {
|
||
background-color: #6f42c1;
|
||
color: white;
|
||
border: none;
|
||
border-radius: 8px;
|
||
font-size: 16px;
|
||
font-weight: bold;
|
||
padding: 10px 20px;
|
||
min-width: 120px;
|
||
}
|
||
QPushButton:hover {
|
||
background-color: #5a32a3;
|
||
}
|
||
QPushButton:disabled {
|
||
background-color: #6c757d;
|
||
color: #adb5bd;
|
||
}
|
||
"""
|
||
|
||
stop_button_style = """
|
||
QPushButton {
|
||
background-color: #dc3545;
|
||
color: white;
|
||
border: none;
|
||
border-radius: 8px;
|
||
font-size: 16px;
|
||
font-weight: bold;
|
||
padding: 10px 20px;
|
||
min-width: 120px;
|
||
transition: all 0.3s ease;
|
||
}
|
||
QPushButton:hover {
|
||
background-color: #c82333;
|
||
transform: translateY(-2px);
|
||
box-shadow: 0 4px 8px rgba(0, 0, 0, 0.2);
|
||
}
|
||
QPushButton:pressed {
|
||
background-color: #a71e2a;
|
||
transform: translateY(1px);
|
||
box-shadow: 0 2px 4px rgba(0, 0, 0, 0.2);
|
||
}
|
||
QPushButton:disabled {
|
||
background-color: #6c757d;
|
||
color: #adb5bd;
|
||
transform: none;
|
||
box-shadow: none;
|
||
}
|
||
"""
|
||
|
||
exit_button_style = """
|
||
QPushButton {
|
||
background-color: #007bff;
|
||
color: white;
|
||
border: none;
|
||
border-radius: 8px;
|
||
font-size: 16px;
|
||
font-weight: bold;
|
||
padding: 10px 20px;
|
||
min-width: 120px;
|
||
transition: all 0.3s ease;
|
||
}
|
||
QPushButton:hover {
|
||
background-color: #0069d9;
|
||
transform: translateY(-2px);
|
||
box-shadow: 0 4px 8px rgba(0, 0, 0, 0.2);
|
||
}
|
||
QPushButton:pressed {
|
||
background-color: #0056b3;
|
||
transform: translateY(1px);
|
||
box-shadow: 0 2px 4px rgba(0, 0, 0, 0.2);
|
||
}
|
||
QPushButton:disabled {
|
||
background-color: #6c757d;
|
||
color: #adb5bd;
|
||
transform: none;
|
||
box-shadow: none;
|
||
}
|
||
"""
|
||
|
||
# 创建开始检测按钮
|
||
self.start_button = QPushButton("开始检测")
|
||
self.start_button.setObjectName("start_button")
|
||
self.start_button.clicked.connect(self.start_detection)
|
||
self.start_button.setFixedHeight(50)
|
||
self.start_button.setStyleSheet(start_button_style)
|
||
button_layout.addWidget(self.start_button)
|
||
|
||
# 创建停止按钮
|
||
self.stop_button = QPushButton("停止检测")
|
||
self.stop_button.setObjectName("stop_button")
|
||
self.stop_button.clicked.connect(self.stop_detection)
|
||
self.stop_button.setEnabled(False)
|
||
self.stop_button.setFixedHeight(50)
|
||
self.stop_button.setStyleSheet(stop_button_style)
|
||
button_layout.addWidget(self.stop_button)
|
||
|
||
self.refresh_proxy_button = QPushButton("刷新代理池")
|
||
self.refresh_proxy_button.clicked.connect(self.refresh_proxy_pool_manually)
|
||
self.refresh_proxy_button.setFixedHeight(50)
|
||
self.refresh_proxy_button.setStyleSheet(refresh_button_style)
|
||
button_layout.addWidget(self.refresh_proxy_button)
|
||
|
||
# 创建退出按钮
|
||
self.exit_button = QPushButton("退出")
|
||
self.exit_button.setObjectName("exit_button")
|
||
self.exit_button.clicked.connect(self.close)
|
||
self.exit_button.setFixedHeight(50)
|
||
self.exit_button.setStyleSheet(exit_button_style)
|
||
button_layout.addWidget(self.exit_button)
|
||
|
||
layout.addLayout(button_layout)
|
||
|
||
# 初始化检测端
|
||
self.worker = DetectWorker(self)
|
||
|
||
# 启动Redis订阅线程（如果使用Redis）
|
||
if self.worker.use_redis:
|
||
self.worker.running = True
|
||
self.worker.redis_sub_thread = threading.Thread(target=self.worker.start_redis_subscription)
|
||
self.worker.redis_sub_thread.daemon = True
|
||
self.worker.redis_sub_thread.start()
|
||
logger.debug("Redis配置更新订阅已启动")
|
||
|
||
# 连接信号
|
||
self.worker.connect_signals()
|
||
|
||
# 初始化检测线程
|
||
self.detect_thread = None
|
||
|
||
# 初始化日志队列和处理线程
|
||
self.log_queue = Queue()
|
||
self.log_processing = True
|
||
# 启动日志处理线程
|
||
self.log_thread = threading.Thread(target=self.process_log_queue)
|
||
self.log_thread.daemon = True
|
||
self.log_thread.start()
|
||
|
||
# 配置日志
|
||
logger.add(self.enqueue_log_message, level="INFO")
|
||
|
||
# 启动定时任务
|
||
self.scheduler_thread = threading.Thread(target=self.start_scheduler)
|
||
self.scheduler_thread.daemon = True
|
||
self.scheduler_thread.start()
|
||
|
||
def enqueue_log_message(self, message):
|
||
"""
|
||
日志回调函数，将日志消息放入队列
|
||
"""
|
||
self.log_queue.put(message)
|
||
|
||
def process_log_queue(self):
|
||
"""
|
||
处理日志队列中的消息
|
||
"""
|
||
while self.log_processing:
|
||
try:
|
||
message = self.log_queue.get(timeout=1)
|
||
# 提取时间和消息部分
|
||
# 日志格式: 2026-04-13 00:22:05.365 | INFO | __main__:load_thread_count:846 - 从Redis加载检测线程数成功: 500
|
||
import re
|
||
match = re.match(r'(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d{3}) \| .*? \| .*? - (.*)', message)
|
||
if match:
|
||
time_str = match.group(1)
|
||
msg_str = match.group(2)
|
||
log_text = f"{time_str} | {msg_str}"
|
||
else:
|
||
log_text = message.replace("\n", "")
|
||
# 使用QMetaObject.invokeMethod确保线程安全的GUI更新
|
||
QMetaObject.invokeMethod(self.log_text, "append", Qt.QueuedConnection, Q_ARG(str, log_text))
|
||
QMetaObject.invokeMethod(self.log_text.verticalScrollBar(), "setValue", Qt.QueuedConnection, Q_ARG(int, self.log_text.verticalScrollBar().maximum()))
|
||
except Exception as e:
|
||
pass
|
||
|
||
def log_message(self, message):
|
||
"""
|
||
日志回调函数（保留用于信号连接）
|
||
"""
|
||
self.enqueue_log_message(message)
|
||
|
||
def start_detection(self):
|
||
"""
|
||
开始检测
|
||
"""
|
||
self.status_label.setText("状态: 检测中...")
|
||
self.start_button.setEnabled(False)
|
||
self.stop_button.setEnabled(True)
|
||
|
||
# 创建并启动检测线程
|
||
self.detect_thread = DetectThread(self.worker)
|
||
self.detect_thread.log_signal.connect(self.log_message)
|
||
self.detect_thread.progress_signal.connect(self.update_progress)
|
||
self.detect_thread.thread_count_signal.connect(self.update_thread_count_display)
|
||
self.detect_thread.finished_signal.connect(self.detection_finished)
|
||
self.detect_thread.start()
|
||
|
||
def stop_detection(self):
|
||
"""
|
||
停止检测
|
||
"""
|
||
self.status_label.setText("状态: 停止中...")
|
||
self.worker.stop()
|
||
self.stop_button.setEnabled(False)
|
||
|
||
def detection_finished(self):
|
||
"""
|
||
检测完成
|
||
"""
|
||
self.status_label.setText("状态: 检测完成")
|
||
self.start_button.setEnabled(True)
|
||
self.stop_button.setEnabled(False)
|
||
|
||
def update_progress(self, current, total):
|
||
"""
|
||
更新进度条
|
||
"""
|
||
# 直接在当前线程执行，避免使用QMetaObject.invokeMethod
|
||
try:
|
||
if total > 0:
|
||
progress = int((current / total) * 100)
|
||
self.progress_bar.setValue(progress)
|
||
except Exception as e:
|
||
print(f"更新进度失败: {e}")
|
||
|
||
def update_thread_count_display(self, active_count, total_count):
|
||
"""
|
||
更新线程数量显示
|
||
|
||
:param active_count: 当前活跃线程数量
|
||
:param total_count: 总线程数量
|
||
"""
|
||
try:
|
||
# 检查是否有配置标签
|
||
if hasattr(self, 'config_labels') and 'thread_count' in self.config_labels:
|
||
self.config_labels['thread_count'].setText(f"检测线程数: {total_count} (当前: {active_count})")
|
||
except Exception as e:
|
||
print(f"更新线程数量显示失败: {e}")
|
||
|
||
def refresh_proxy_pool_manually(self):
|
||
self.status_label.setText("状态: 正在刷新代理池...")
|
||
self.refresh_proxy_button.setEnabled(False)
|
||
|
||
def run_refresh():
|
||
try:
|
||
self.worker.refresh_proxy_pool()
|
||
with self.worker.proxy_pool_lock:
|
||
proxy_count = len(self.worker.proxy_pool)
|
||
self.worker.update_config_labels()
|
||
QMetaObject.invokeMethod(
|
||
self.status_label,
|
||
"setText",
|
||
Qt.QueuedConnection,
|
||
Q_ARG(str, f"状态: 代理池刷新完成，可用代理 {proxy_count} 个")
|
||
)
|
||
except Exception as e:
|
||
QMetaObject.invokeMethod(
|
||
self.status_label,
|
||
"setText",
|
||
Qt.QueuedConnection,
|
||
Q_ARG(str, f"状态: 刷新代理池失败: {e}")
|
||
)
|
||
finally:
|
||
QMetaObject.invokeMethod(
|
||
self.refresh_proxy_button,
|
||
"setEnabled",
|
||
Qt.QueuedConnection,
|
||
Q_ARG(bool, True)
|
||
)
|
||
|
||
threading.Thread(target=run_refresh, daemon=True).start()
|
||
|
||
def start_scheduler(self):
|
||
"""
|
||
启动定时任务
|
||
"""
|
||
# 每天凌晨1点执行检测
|
||
schedule.every().day.at("01:00").do(self.scheduled_detection)
|
||
|
||
# 循环执行定时任务
|
||
while True:
|
||
schedule.run_pending()
|
||
time.sleep(60)
|
||
|
||
def scheduled_detection(self):
|
||
"""
|
||
定时检测任务
|
||
"""
|
||
self.log_message(f"[{datetime.now()}] 开始定时检测任务")
|
||
self.start_detection()
|
||
|
||
def refresh_config(self):
|
||
"""
|
||
刷新配置
|
||
"""
|
||
try:
|
||
# 重新加载配置
|
||
new_detect_options = self.worker.load_detect_options()
|
||
new_proxy_config = self.worker.load_proxy_config()
|
||
new_thread_count = self.worker.load_thread_count()
|
||
self.worker.load_cookies_from_remote()
|
||
|
||
# 检查配置是否发生变化
|
||
config_changed = False
|
||
if new_detect_options != self.worker.detect_options:
|
||
self.worker.detect_options = new_detect_options
|
||
config_changed = True
|
||
if new_proxy_config != self.worker.proxy_config:
|
||
self.worker.proxy_config = new_proxy_config
|
||
config_changed = True
|
||
if new_thread_count != self.worker.thread_count:
|
||
self.worker.thread_count = new_thread_count
|
||
config_changed = True
|
||
|
||
# 如果配置发生变化，更新标签
|
||
if config_changed:
|
||
self.worker.update_config_labels()
|
||
self.log_message(f"[{datetime.now()}] 配置已更新")
|
||
except Exception as e:
|
||
self.log_message(f"[{datetime.now()}] 刷新配置失败: {e}")
|
||
|
||
def event(self, event):
|
||
"""
|
||
处理事件
|
||
"""
|
||
if event.type() == ConfigUpdateEvent.Type:
|
||
self.update_config_labels()
|
||
return True
|
||
return super().event(event)
|
||
|
||
def update_config_labels(self):
|
||
"""
|
||
更新GUI配置标签
|
||
"""
|
||
logger.debug("开始更新GUI配置标签")
|
||
try:
|
||
if self.worker:
|
||
try:
|
||
logger.debug(f"worker对象存在，当前配置: {self.worker.detect_options}, {self.worker.proxy_config}, {self.worker.thread_count}")
|
||
# 准备更新数据
|
||
detect_options = self.worker.detect_options
|
||
free_options_display = []
|
||
paid_options_display = []
|
||
option_names = {
|
||
'detect_register': '检查注册',
|
||
'detect_baidu_site': '百度site查询',
|
||
'detect_360_site': '360的site查询',
|
||
'detect_chinaz': '站长之家查询',
|
||
'detect_aizhan': '爱站网查询',
|
||
'detect_wayback': '时光机检测',
|
||
'detect_jucha': '聚查查询',
|
||
'detect_juziseo': '桔子查询'
|
||
}
|
||
|
||
ordered_keys = detect_options.get('detect_order') or list(option_names.keys())
|
||
normalized_keys = [key for key in ordered_keys if key in option_names]
|
||
for key in option_names:
|
||
if key not in normalized_keys:
|
||
normalized_keys.append(key)
|
||
|
||
free_keys = {
|
||
'detect_register',
|
||
'detect_baidu_site',
|
||
'detect_360_site',
|
||
'detect_chinaz',
|
||
'detect_aizhan',
|
||
'detect_wayback',
|
||
}
|
||
|
||
for index, key in enumerate(normalized_keys, start=1):
|
||
name = option_names[key]
|
||
if key in detect_options and detect_options[key]:
|
||
item_text = f"{index}.{name}"
|
||
if key in free_keys:
|
||
free_options_display.append(item_text)
|
||
else:
|
||
paid_options_display.append(item_text)
|
||
|
||
free_text = " | ".join(free_options_display) if free_options_display else "无"
|
||
paid_text = " | ".join(paid_options_display) if paid_options_display else "无"
|
||
detect_options_text = (
|
||
"当前检测选项:\n"
|
||
f"免费优先: {free_text}\n"
|
||
f"付费后置: {paid_text}"
|
||
)
|
||
|
||
# 准备代理配置文本
|
||
proxy_enable = self.worker.proxy_config.get('proxy_enable', False)
|
||
allow_direct = self.worker.proxy_config.get('allow_direct', False)
|
||
proxy_urls = self.worker.proxy_config.get('proxy_urls') or []
|
||
if not proxy_urls and self.worker.proxy_config.get('proxy_url'):
|
||
proxy_urls = [self.worker.proxy_config.get('proxy_url', '')]
|
||
with self.worker.proxy_pool_lock:
|
||
proxy_pool_count = len(self.worker.proxy_pool)
|
||
last_refresh_time = (
|
||
self.worker.proxy_last_refresh_time.strftime('%Y-%m-%d %H:%M:%S')
|
||
if self.worker.proxy_last_refresh_time else '未刷新'
|
||
)
|
||
proxy_display = (
|
||
f"启用: {'是' if proxy_enable else '否'} | "
|
||
f"允许直连: {'是' if allow_direct else '否'} | "
|
||
f"代理池链接数: {len([url for url in proxy_urls if url])} | "
|
||
f"当前可用代理数: {proxy_pool_count}\n"
|
||
f"最近刷新: {last_refresh_time} | "
|
||
f"最近结果: {self.worker.proxy_last_refresh_status} | "
|
||
f"原始返回数: {self.worker.proxy_last_refresh_total_items}"
|
||
)
|
||
proxy_text = f"代理配置: {proxy_display}"
|
||
|
||
# 准备线程数文本
|
||
thread_count_text = f"检测线程数: {self.worker.thread_count}"
|
||
|
||
logger.debug(f"准备更新标签: {detect_options_text}, {proxy_text}, {thread_count_text}")
|
||
|
||
# 检查配置标签是否存在
|
||
if hasattr(self, 'config_labels'):
|
||
# 使用QMetaObject.invokeMethod确保在主线程中更新UI
|
||
from PySide6.QtCore import QMetaObject, Qt, Q_ARG
|
||
|
||
# 更新检测选项标签
|
||
if 'detect_options' in self.config_labels:
|
||
QMetaObject.invokeMethod(self.config_labels['detect_options'], "setText",
|
||
Qt.QueuedConnection,
|
||
Q_ARG(str, detect_options_text))
|
||
|
||
# 更新代理配置标签
|
||
if 'proxy_config' in self.config_labels:
|
||
QMetaObject.invokeMethod(self.config_labels['proxy_config'], "setText",
|
||
Qt.QueuedConnection,
|
||
Q_ARG(str, proxy_text))
|
||
|
||
# 更新线程数标签
|
||
if 'thread_count' in self.config_labels:
|
||
QMetaObject.invokeMethod(self.config_labels['thread_count'], "setText",
|
||
Qt.QueuedConnection,
|
||
Q_ARG(str, thread_count_text))
|
||
|
||
logger.debug("GUI配置标签更新完成")
|
||
else:
|
||
logger.debug("配置标签不存在，跳过更新")
|
||
except Exception as e:
|
||
logger.error(f"更新GUI标签失败: {e}")
|
||
import traceback
|
||
logger.error(traceback.format_exc())
|
||
else:
|
||
logger.debug("worker对象不存在")
|
||
except Exception as e:
|
||
logger.error(f"执行update_config_labels失败: {e}")
|
||
import traceback
|
||
logger.error(traceback.format_exc())
|
||
|
||
from PySide6.QtCore import QObject, Signal, QEvent
|
||
|
||
class ConfigUpdateEvent(QEvent):
|
||
"""
|
||
配置更新事件类
|
||
"""
|
||
Type = QEvent.Type(QEvent.User + 1)
|
||
|
||
def __init__(self):
|
||
super().__init__(ConfigUpdateEvent.Type)
|
||
|
||
class ConfigUpdateSignal(QObject):
|
||
"""
|
||
配置更新信号类
|
||
"""
|
||
config_updated = Signal()
|
||
|
||
class DetectWorker:
|
||
"""
|
||
域名检测端工作类
|
||
"""
|
||
def __init__(self, main_window=None):
|
||
"""
|
||
初始化检测端
|
||
|
||
:param main_window: 主窗口实例，用于更新GUI
|
||
"""
|
||
self.main_window = main_window
|
||
self.running = False
|
||
self.detect_threads = []
|
||
self.detect_thread = None # 检测线程实例
|
||
|
||
# 初始化数据库连接
|
||
self.db = Database()
|
||
|
||
# 初始化Redis连接
|
||
try:
|
||
self.redis_client = redis.Redis(
|
||
host=config.REDIS_HOST,
|
||
port=config.REDIS_PORT,
|
||
password=config.REDIS_PASSWORD,
|
||
db=config.REDIS_DB,
|
||
decode_responses=True,
|
||
socket_connect_timeout=10,
|
||
socket_timeout=10,
|
||
retry_on_timeout=True,
|
||
health_check_interval=30
|
||
)
|
||
# 测试连接
|
||
self.redis_client.ping()
|
||
logger.info(f"Redis 连接成功: {config.REDIS_HOST}:{config.REDIS_PORT}")
|
||
self.use_redis = True
|
||
except Exception as e:
|
||
logger.warning(f"Redis 连接失败: {e}，将使用本地配置")
|
||
self.redis_client = None
|
||
self.use_redis = False
|
||
|
||
# 初始化配置更新信号
|
||
self.config_signal = ConfigUpdateSignal()
|
||
|
||
# 加载配置
|
||
self.detect_options = self.load_detect_options()
|
||
self.proxy_config = self.load_proxy_config()
|
||
self.thread_count = self.load_thread_count() # 从配置文件加载线程数
|
||
|
||
# 初始化代理池
|
||
self.proxy_pool = []
|
||
self.proxy_pool_lock = threading.Lock()
|
||
self.proxy_refresh_lock = threading.Lock()
|
||
self.proxy_last_refresh_time = None
|
||
self.proxy_last_refresh_status = "未刷新"
|
||
self.proxy_last_refresh_source_count = 0
|
||
self.proxy_last_refresh_total_items = 0
|
||
self.proxy_refresh_cooldown_seconds = 30
|
||
self.proxy_next_refresh_time = 0.0
|
||
|
||
# 加载敏感词（只加载一次，所有线程共享）
|
||
try:
|
||
self.sensitive_words = self.db.get_all_sensitive_words()
|
||
logger.info(f"加载敏感词完成，共 {len(self.sensitive_words)} 个敏感词")
|
||
except Exception as e:
|
||
logger.error(f"加载敏感词失败: {e}")
|
||
import traceback
|
||
logger.error(traceback.format_exc())
|
||
self.sensitive_words = []
|
||
|
||
# 初始化运行状态
|
||
self.running = False
|
||
|
||
# 加载cookies
|
||
self.load_cookies_from_remote()
|
||
|
||
# 初始化JC实例（暂不设置代理，在开始检测时再设置）
|
||
self.jc_instance = jucha.JC(proxies=None)
|
||
self.jc_instance.load_juming_cookies()
|
||
self.jc_instance.load_cookies()
|
||
|
||
# 初始化Juziseo实例（暂不设置代理，在开始检测时再设置）
|
||
self.juziseo_instance = juziseo.Juziseo(proxies=None)
|
||
self.juziseo_instance.load_cookies()
|
||
self.wayback_detector = WaybackDetector()
|
||
|
||
# 显示所有配置信息
|
||
logger.info("检测端配置信息:")
|
||
logger.info(f"检测选项: {self.detect_options}")
|
||
logger.info(f"代理配置: {self.proxy_config}")
|
||
logger.info(f"检测线程数: {self.thread_count}")
|
||
|
||
# Redis订阅线程将在start方法中启动
|
||
|
||
def connect_signals(self):
|
||
"""
|
||
连接信号到槽函数
|
||
"""
|
||
if self.main_window:
|
||
try:
|
||
# 连接信号到槽函数
|
||
self.config_signal.config_updated.connect(self.main_window.update_config_labels)
|
||
logger.debug("信号连接成功")
|
||
# 连接后立即更新一次配置
|
||
self.update_config_labels()
|
||
except Exception as e:
|
||
logger.error(f"连接信号失败: {e}")
|
||
|
||
def load_detect_options(self):
|
||
"""
|
||
加载检测选项
|
||
"""
|
||
default_order = [
|
||
'detect_register',
|
||
'detect_baidu_site',
|
||
'detect_360_site',
|
||
'detect_chinaz',
|
||
'detect_aizhan',
|
||
'detect_wayback',
|
||
'detect_jucha',
|
||
'detect_juziseo',
|
||
]
|
||
default_options = {
|
||
'detect_register': True,
|
||
'detect_wayback': True,
|
||
'detect_chinaz': True,
|
||
'detect_aizhan': True,
|
||
'detect_baidu_site': True,
|
||
'detect_360_site': True,
|
||
'detect_jucha': False,
|
||
'detect_juziseo': False,
|
||
'detect_order': default_order,
|
||
}
|
||
try:
|
||
# 从Redis获取配置
|
||
if self.use_redis:
|
||
detect_options_str = self.redis_client.get('domain_tool:detect_options')
|
||
if detect_options_str:
|
||
detect_options = default_options.copy()
|
||
detect_options.update(json.loads(detect_options_str))
|
||
if detect_options.get('detect_whois') or detect_options.get('detect_beian') or detect_options.get('detect_intercept'):
|
||
detect_options['detect_jucha'] = True
|
||
if detect_options.get('detect_juziseo_outlink'):
|
||
detect_options['detect_juziseo'] = True
|
||
order = detect_options.get('detect_order') or []
|
||
detect_options['detect_order'] = [key for key in order if key in default_order]
|
||
for key in default_order:
|
||
if key not in detect_options['detect_order']:
|
||
detect_options['detect_order'].append(key)
|
||
logger.info(f"从Redis加载检测选项成功: {detect_options}")
|
||
return detect_options
|
||
|
||
# 从本地文件获取配置
|
||
if os.path.exists('detect_options.json'):
|
||
with open('detect_options.json', 'r', encoding='utf-8') as f:
|
||
detect_options = default_options.copy()
|
||
detect_options.update(json.load(f))
|
||
if detect_options.get('detect_whois') or detect_options.get('detect_beian') or detect_options.get('detect_intercept'):
|
||
detect_options['detect_jucha'] = True
|
||
if detect_options.get('detect_juziseo_outlink'):
|
||
detect_options['detect_juziseo'] = True
|
||
order = detect_options.get('detect_order') or []
|
||
detect_options['detect_order'] = [key for key in order if key in default_order]
|
||
for key in default_order:
|
||
if key not in detect_options['detect_order']:
|
||
detect_options['detect_order'].append(key)
|
||
logger.info(f"从本地文件加载检测选项成功: {detect_options}")
|
||
return detect_options
|
||
else:
|
||
logger.info(f"使用默认检测选项: {default_options}")
|
||
return default_options
|
||
except Exception as e:
|
||
logger.error(f"加载检测选项失败: {e}")
|
||
logger.info(f"使用默认检测选项: {default_options}")
|
||
return default_options
|
||
|
||
def load_proxy_config(self):
|
||
"""
|
||
加载代理配置
|
||
"""
|
||
default_config = {'proxy_enable': False, 'proxy_url': '', 'proxy_urls': [], 'allow_direct': False}
|
||
try:
|
||
# 从Redis获取配置
|
||
if self.use_redis:
|
||
proxy_config_str = self.redis_client.get('domain_tool:proxy_config')
|
||
if proxy_config_str:
|
||
proxy_config = default_config.copy()
|
||
proxy_config.update(json.loads(proxy_config_str))
|
||
proxy_urls = proxy_config.get('proxy_urls') or []
|
||
if not proxy_urls and proxy_config.get('proxy_url'):
|
||
proxy_urls = [proxy_config.get('proxy_url', '')]
|
||
proxy_config['proxy_urls'] = [url.strip() for url in proxy_urls if url and str(url).strip()]
|
||
proxy_config['proxy_url'] = proxy_config['proxy_urls'][0] if proxy_config['proxy_urls'] else ''
|
||
proxy_config['allow_direct'] = bool(proxy_config.get('allow_direct', False))
|
||
logger.info(f"从Redis加载代理配置成功: {proxy_config}")
|
||
return proxy_config
|
||
|
||
# 从本地文件获取配置
|
||
if os.path.exists('proxy_config.json'):
|
||
with open('proxy_config.json', 'r', encoding='utf-8') as f:
|
||
proxy_config = default_config.copy()
|
||
proxy_config.update(json.load(f))
|
||
logger.info(f"从本地文件加载代理配置成功: {proxy_config}")
|
||
proxy_urls = proxy_config.get('proxy_urls') or []
|
||
if not proxy_urls and proxy_config.get('proxy_url'):
|
||
proxy_urls = [proxy_config.get('proxy_url', '')]
|
||
proxy_config['proxy_urls'] = [url.strip() for url in proxy_urls if url and str(url).strip()]
|
||
proxy_config['proxy_url'] = proxy_config['proxy_urls'][0] if proxy_config['proxy_urls'] else ''
|
||
proxy_config['allow_direct'] = bool(proxy_config.get('allow_direct', False))
|
||
return proxy_config
|
||
else:
|
||
logger.info(f"使用默认代理配置: {default_config}")
|
||
return default_config
|
||
except Exception as e:
|
||
logger.error(f"加载代理配置失败: {e}")
|
||
logger.info(f"使用默认代理配置: {default_config}")
|
||
return default_config
|
||
|
||
def load_thread_count(self):
|
||
"""
|
||
加载检测线程数
|
||
"""
|
||
max_recommended_threads = 20
|
||
try:
|
||
# 从Redis获取配置
|
||
if self.use_redis:
|
||
thread_count_str = self.redis_client.get('domain_tool:thread_count')
|
||
if thread_count_str:
|
||
thread_count = min(max_recommended_threads, max(1, int(thread_count_str)))
|
||
logger.info(f"从Redis加载检测线程数成功: {thread_count}")
|
||
return thread_count
|
||
|
||
# 从本地文件获取配置
|
||
if os.path.exists('thread_count.json'):
|
||
with open('thread_count.json', 'r', encoding='utf-8') as f:
|
||
thread_config = json.load(f)
|
||
thread_count = thread_config.get('thread_count', '10')
|
||
thread_count = min(max_recommended_threads, max(1, int(thread_count)))
|
||
logger.info(f"从本地文件加载检测线程数成功: {thread_count}")
|
||
return thread_count
|
||
else:
|
||
default_thread_count = 4 # 默认线程数，降低线程数量以减轻系统负担
|
||
logger.info(f"使用默认检测线程数: {default_thread_count}")
|
||
return default_thread_count
|
||
except Exception as e:
|
||
logger.error(f"加载检测线程数失败: {e}")
|
||
default_thread_count = 4 # 默认线程数，降低线程数量以减轻系统负担
|
||
logger.info(f"使用默认检测线程数: {default_thread_count}")
|
||
return default_thread_count
|
||
|
||
def test_proxy(self, proxy_item, result_queue):
|
||
"""
|
||
测试单个代理的可用性
|
||
|
||
:param proxy_item: 代理信息
|
||
:param result_queue: 结果队列
|
||
"""
|
||
try:
|
||
if 'ip' in proxy_item and 'port' in proxy_item:
|
||
ip = proxy_item['ip']
|
||
port = proxy_item['port']
|
||
username = proxy_item.get('username', '')
|
||
password = proxy_item.get('password', '')
|
||
|
||
if username and password:
|
||
proxy_url = f"http://{username}:{password}@{ip}:{port}"
|
||
else:
|
||
proxy_url = f"http://{ip}:{port}"
|
||
|
||
# 检查代理是否可用
|
||
import requests
|
||
test_proxies = {
|
||
'http': proxy_url,
|
||
'https': proxy_url
|
||
}
|
||
test_response = requests.get('https://m.baidu.com', proxies=test_proxies, timeout=5)
|
||
if test_response.status_code == 200:
|
||
# logger.info(f"代理可用性检查通过: {proxy_url}")
|
||
result_queue.put(test_proxies)
|
||
else:
|
||
logger.warning(f"代理可用性检查失败，状态码: {test_response.status_code}, 代理: {proxy_url}")
|
||
except Exception as e:
|
||
# logger.error(f"代理可用性检查失败: {e}, 代理: {proxy_item}")
|
||
pass
|
||
finally:
|
||
# 确保队列中添加一个标记，表示该线程已完成
|
||
result_queue.put(None)
|
||
|
||
def _extract_proxy_items(self, proxy_data):
|
||
if isinstance(proxy_data, dict):
|
||
if 'list' in proxy_data and isinstance(proxy_data['list'], list):
|
||
return proxy_data['list']
|
||
if 'ip' in proxy_data and 'port' in proxy_data:
|
||
return [proxy_data]
|
||
if isinstance(proxy_data, list):
|
||
return proxy_data
|
||
return []
|
||
|
||
def refresh_proxy_pool(self):
|
||
"""
|
||
刷新代理池
|
||
"""
|
||
if not self.proxy_refresh_lock.acquire(blocking=False):
|
||
logger.debug("代理池刷新已在进行中，跳过本次重复刷新")
|
||
return
|
||
|
||
if not self.proxy_config.get('proxy_enable', False):
|
||
self.proxy_last_refresh_time = datetime.now()
|
||
self.proxy_last_refresh_status = "代理未启用"
|
||
self.proxy_last_refresh_source_count = 0
|
||
self.proxy_last_refresh_total_items = 0
|
||
self.proxy_next_refresh_time = 0.0
|
||
self.proxy_refresh_lock.release()
|
||
return
|
||
|
||
try:
|
||
now_ts = time.time()
|
||
with self.proxy_pool_lock:
|
||
has_cached_proxies = bool(self.proxy_pool)
|
||
if not has_cached_proxies and self.proxy_next_refresh_time and now_ts < self.proxy_next_refresh_time:
|
||
wait_seconds = int(max(1, self.proxy_next_refresh_time - now_ts))
|
||
self.proxy_last_refresh_status = f"冷却中，{wait_seconds} 秒后再试"
|
||
logger.info(f"代理池刷新冷却中，{wait_seconds} 秒后再试")
|
||
return
|
||
|
||
proxy_api_urls = self.proxy_config.get('proxy_urls') or []
|
||
if not proxy_api_urls and self.proxy_config.get('proxy_url'):
|
||
proxy_api_urls = [self.proxy_config.get('proxy_url', '')]
|
||
proxy_api_urls = [url for url in proxy_api_urls if url]
|
||
self.proxy_last_refresh_source_count = len(proxy_api_urls)
|
||
if proxy_api_urls:
|
||
import requests
|
||
import threading
|
||
import random
|
||
from queue import Queue
|
||
|
||
proxy_list = []
|
||
for proxy_api_url in proxy_api_urls:
|
||
try:
|
||
response = requests.get(proxy_api_url, timeout=10)
|
||
if response.status_code == 200:
|
||
proxy_data = response.json()
|
||
current_items = self._extract_proxy_items(proxy_data)
|
||
proxy_list.extend(current_items)
|
||
logger.info(f"代理池链接拉取成功: {proxy_api_url}, 原始代理数: {len(current_items)}")
|
||
else:
|
||
logger.warning(f"代理池链接响应异常: {proxy_api_url}, 状态码: {response.status_code}")
|
||
except Exception as api_error:
|
||
logger.warning(f"代理池链接拉取失败: {proxy_api_url}, 错误: {api_error}")
|
||
|
||
if proxy_list:
|
||
self.proxy_last_refresh_total_items = len(proxy_list)
|
||
max_validate_count = max(24, self.thread_count * 3)
|
||
if len(proxy_list) > max_validate_count:
|
||
proxy_list = random.sample(proxy_list, max_validate_count)
|
||
logger.info(f"代理池验证已启用抽样模式，本次抽样 {len(proxy_list)} 个代理进行可用性验证")
|
||
result_queue = Queue()
|
||
threads = []
|
||
|
||
for proxy_item in proxy_list:
|
||
thread = threading.Thread(target=self.test_proxy, args=(proxy_item, result_queue))
|
||
thread.daemon = True
|
||
thread.start()
|
||
threads.append(thread)
|
||
|
||
new_proxies = []
|
||
seen_proxy_strings = set()
|
||
completed = 0
|
||
import time
|
||
start_time = time.time()
|
||
timeout = 30
|
||
|
||
while completed < len(threads) and time.time() - start_time < timeout:
|
||
try:
|
||
result = result_queue.get(timeout=1)
|
||
if result is not None:
|
||
proxy_signature = json.dumps(result, sort_keys=True)
|
||
if proxy_signature not in seen_proxy_strings:
|
||
seen_proxy_strings.add(proxy_signature)
|
||
new_proxies.append({'proxy': result, 'usage_count': 0})
|
||
completed += 1
|
||
except Exception:
|
||
pass
|
||
|
||
for thread in threads:
|
||
try:
|
||
thread.join(timeout=5)
|
||
except Exception:
|
||
pass
|
||
|
||
with self.proxy_pool_lock:
|
||
self.proxy_pool = new_proxies
|
||
self.proxy_last_refresh_time = datetime.now()
|
||
self.proxy_last_refresh_status = f"刷新成功，可用 {len(new_proxies)} 个"
|
||
self.proxy_next_refresh_time = 0.0 if new_proxies else time.time() + self.proxy_refresh_cooldown_seconds
|
||
logger.info(f"代理池刷新完成，共 {len(new_proxies)} 个可用代理，来源链接 {len(proxy_api_urls)} 个")
|
||
else:
|
||
with self.proxy_pool_lock:
|
||
self.proxy_pool = []
|
||
self.proxy_last_refresh_time = datetime.now()
|
||
self.proxy_last_refresh_total_items = 0
|
||
self.proxy_last_refresh_status = "未取到可用代理数据"
|
||
self.proxy_next_refresh_time = time.time() + self.proxy_refresh_cooldown_seconds
|
||
logger.warning("所有代理池链接均未返回可用代理数据")
|
||
else:
|
||
with self.proxy_pool_lock:
|
||
self.proxy_pool = []
|
||
self.proxy_last_refresh_time = datetime.now()
|
||
self.proxy_last_refresh_total_items = 0
|
||
self.proxy_last_refresh_status = "未配置代理池链接"
|
||
self.proxy_next_refresh_time = time.time() + self.proxy_refresh_cooldown_seconds
|
||
except Exception as e:
|
||
with self.proxy_pool_lock:
|
||
self.proxy_pool = []
|
||
self.proxy_last_refresh_time = datetime.now()
|
||
self.proxy_last_refresh_total_items = 0
|
||
self.proxy_last_refresh_status = f"刷新失败: {e}"
|
||
self.proxy_next_refresh_time = time.time() + self.proxy_refresh_cooldown_seconds
|
||
logger.error(f"刷新代理池失败: {e}")
|
||
finally:
|
||
self.proxy_refresh_lock.release()
|
||
|
||
def remove_proxy(self, proxy):
|
||
"""
|
||
从代理池中移除失效的代理
|
||
|
||
:param proxy: 失效的代理
|
||
"""
|
||
with self.proxy_pool_lock:
|
||
# 检查代理池中的代理结构
|
||
if self.proxy_pool and isinstance(self.proxy_pool[0], dict) and 'proxy' in self.proxy_pool[0]:
|
||
# 新的代理池结构
|
||
for i, proxy_item in enumerate(self.proxy_pool):
|
||
if proxy_item['proxy'] == proxy:
|
||
self.proxy_pool.pop(i)
|
||
logger.info(f"从代理池移除失效代理: {proxy}")
|
||
break
|
||
else:
|
||
# 旧的代理池结构
|
||
if proxy in self.proxy_pool:
|
||
self.proxy_pool.remove(proxy)
|
||
logger.info(f"从代理池移除失效代理: {proxy}")
|
||
|
||
# 如果代理池中的代理数量少于当前设定的线程数量，自动刷新代理池
|
||
if len(self.proxy_pool) < self.thread_count:
|
||
# logger.info(f"代理池代理数量不足（当前 {len(self.proxy_pool)} 个，需要至少 {self.thread_count} 个），刷新代理池")
|
||
# 释放锁后再刷新代理池，避免死锁
|
||
import threading
|
||
threading.Thread(target=self.refresh_proxy_pool, daemon=True).start()
|
||
|
||
def get_proxies(self):
|
||
"""
|
||
获取代理配置
|
||
"""
|
||
if self.proxy_config.get('proxy_enable', False):
|
||
# 检查代理池是否为空或代理数量不足，如果是则刷新
|
||
with self.proxy_pool_lock:
|
||
is_empty = not self.proxy_pool
|
||
is_insufficient = len(self.proxy_pool) < self.thread_count
|
||
|
||
if is_empty or is_insufficient:
|
||
if is_empty:
|
||
logger.info("代理池为空，刷新代理池")
|
||
else:
|
||
# logger.info(f"代理池代理数量不足（当前 {len(self.proxy_pool)} 个，需要至少 {self.thread_count} 个），刷新代理池")
|
||
pass
|
||
# 在锁外刷新代理池，避免死锁；刷新逻辑内部带并发锁和冷却时间
|
||
self.refresh_proxy_pool()
|
||
|
||
# 再次获取锁，检查代理池并选择代理
|
||
with self.proxy_pool_lock:
|
||
# 从代理池中选择一个代理（FIFO，优先使用最早进入池的IP）
|
||
if self.proxy_pool:
|
||
# 检查代理池中的代理结构
|
||
if isinstance(self.proxy_pool[0], dict) and 'proxy' in self.proxy_pool[0]:
|
||
# 从开头取出代理，实现FIFO
|
||
proxy_item = self.proxy_pool.pop(0)
|
||
proxy = proxy_item['proxy']
|
||
usage_count = proxy_item.get('usage_count', 0)
|
||
|
||
# logger.info(f"从代理池选择代理: {proxy}，已使用次数: {usage_count}")
|
||
|
||
# 增加使用次数
|
||
usage_count += 1
|
||
|
||
# 如果使用次数小于3次，将代理放回池尾
|
||
if usage_count < 3:
|
||
proxy_item['usage_count'] = usage_count
|
||
self.proxy_pool.append(proxy_item)
|
||
# logger.info(f"代理使用次数更新为: {usage_count}，放回代理池")
|
||
else:
|
||
# logger.info(f"代理使用次数达到3次，丢弃代理: {proxy}")
|
||
pass
|
||
else:
|
||
# 兼容旧的代理池结构
|
||
proxy = self.proxy_pool.pop(0)
|
||
logger.info(f"从代理池选择代理: {proxy}")
|
||
# 将代理放回池尾
|
||
self.proxy_pool.append(proxy)
|
||
|
||
return proxy
|
||
if self.proxy_config.get('proxy_enable', False):
|
||
logger.warning("代理已启用，但当前无可用代理")
|
||
else:
|
||
logger.info("未启用代理，使用直接连接")
|
||
return None
|
||
|
||
def allow_direct_connection(self):
|
||
return bool(self.proxy_config.get('allow_direct', False))
|
||
|
||
def _get_proxy_for_step(self, domain_id, domain_name, step_name):
|
||
proxy = self.get_proxies()
|
||
if proxy:
|
||
return proxy
|
||
if self.proxy_config.get('proxy_enable', False) and not self.allow_direct_connection():
|
||
detail = self.proxy_last_refresh_status
|
||
logger.error(f"{step_name} 无可用代理，且当前不允许直连兜底: {domain_name}，最近代理状态: {detail}")
|
||
self.db.update_domain_detect_status(domain_id, DETECT_STATUS_FAILED)
|
||
return '__NO_PROXY__'
|
||
return None
|
||
|
||
def _mark_blacklisted(self, domain_id, domain_name, reason):
|
||
self.db.update_domain_detect_status(domain_id, DETECT_STATUS_BLACKLISTED)
|
||
self.db.add_to_blacklist(domain_name, reason)
|
||
logger.info(f"域名已加入黑名单: {domain_name}, 原因: {reason}")
|
||
|
||
def _complete_detection(self, domain_id, domain_name):
|
||
self.db.update_domain_detect_status(domain_id, DETECT_STATUS_COMPLETED)
|
||
domain_info = self.db.get_domain_by_id(domain_id)
|
||
if not domain_info:
|
||
return
|
||
use_status = domain_info.get('use_status', 0)
|
||
register_status = domain_info.get('register_status', 0)
|
||
expire_date = domain_info.get('expire_date')
|
||
if use_status == 0 and register_status == REGISTER_STATUS_AVAILABLE and expire_date:
|
||
self.db.update_domain_expire_date(domain_id, None)
|
||
logger.info(f"域名 {domain_name} 满足条件，已将expire_date置空")
|
||
if register_status == REGISTER_STATUS_AVAILABLE:
|
||
self.db.update_domain_review_status(domain_id, REVIEW_STATUS_PENDING)
|
||
logger.info(f"域名 {domain_name} 满足条件，已设置为待人工复核")
|
||
|
||
def _should_run_jucha(self, domain):
|
||
return self.detect_options.get('detect_jucha', False) and domain.get('jucha_status', 0) != THIRD_PARTY_STATUS_DONE
|
||
|
||
def _should_run_juziseo(self, domain):
|
||
return self.detect_options.get('detect_juziseo', False) and domain.get('juziseo_status', 0) != THIRD_PARTY_STATUS_DONE
|
||
|
||
def _get_detect_execution_order(self):
|
||
default_order = [
|
||
'detect_register',
|
||
'detect_baidu_site',
|
||
'detect_360_site',
|
||
'detect_chinaz',
|
||
'detect_aizhan',
|
||
'detect_wayback',
|
||
'detect_jucha',
|
||
'detect_juziseo',
|
||
]
|
||
configured_order = self.detect_options.get('detect_order') or []
|
||
normalized_order = [key for key in configured_order if key in default_order]
|
||
for key in default_order:
|
||
if key not in normalized_order:
|
||
normalized_order.append(key)
|
||
free_keys = {
|
||
'detect_register',
|
||
'detect_baidu_site',
|
||
'detect_360_site',
|
||
'detect_chinaz',
|
||
'detect_aizhan',
|
||
'detect_wayback',
|
||
}
|
||
paid_keys = {'detect_jucha', 'detect_juziseo'}
|
||
enabled_order = [key for key in normalized_order if self.detect_options.get(key, False)]
|
||
free_order = [key for key in enabled_order if key in free_keys]
|
||
paid_order = [key for key in enabled_order if key in paid_keys]
|
||
return free_order + paid_order
|
||
|
||
def _run_detect_register(self, domain_id, domain, domain_name):
|
||
is_ykj = domain.get('source_type', 0) == 1
|
||
if is_ykj:
|
||
logger.info(f"一口价域名跳过注册状态检测: {domain_name}")
|
||
return True
|
||
logger.info(f"检测注册状态: {domain_name}")
|
||
proxy = self._get_proxy_for_step(domain_id, domain_name, '注册状态检测')
|
||
if proxy == '__NO_PROXY__':
|
||
return False
|
||
try:
|
||
tld = domain_name.split('.')[-1]
|
||
status, expire_date = register.check_register(domain_name, tld, proxy if proxy else None)
|
||
if status != -1:
|
||
self.db.update_domain_register_status(domain_id, status)
|
||
if expire_date:
|
||
from datetime import datetime, timedelta
|
||
try:
|
||
date_part = expire_date.split(' ')[0]
|
||
expire_date_obj = datetime.strptime(date_part, "%Y-%m-%d")
|
||
new_expire_date = expire_date_obj + timedelta(days=75)
|
||
self.db.update_domain_expire_date(domain_id, new_expire_date.strftime("%Y-%m-%d"))
|
||
except Exception as e:
|
||
logger.error(f"处理过期日期失败: {e}")
|
||
self.db.update_domain_expire_date(domain_id, expire_date)
|
||
logger.info(f"注册状态检测完成: {domain_name}, 状态: {status}, 过期日期: {expire_date}")
|
||
else:
|
||
logger.warning(f"注册状态检测失败，不更新数据: {domain_name}")
|
||
if proxy:
|
||
self.remove_proxy(proxy)
|
||
except Exception as e:
|
||
logger.error(f"注册状态检测失败: {domain_name}, 错误: {e}")
|
||
if proxy:
|
||
self.remove_proxy(proxy)
|
||
return True
|
||
|
||
def _run_detect_wayback(self, domain_id, domain_name, sensitive_words):
|
||
logger.info(f"检测时光机: {domain_name}")
|
||
try:
|
||
wayback_result = self.wayback_detector.scan_snapshots(domain_name, sensitive_words=sensitive_words)
|
||
snapshot_years = wayback_result.get('snapshot_years') or []
|
||
if snapshot_years:
|
||
self.db.update_domain_snapshot_years(domain_id, ",".join(str(year) for year in snapshot_years))
|
||
backlink_count = int(wayback_result.get('backlink_count', 0) or 0)
|
||
self.db.execute("UPDATE domains SET backlink_count = %s WHERE id = %s", (backlink_count, domain_id))
|
||
self._upsert_json_detection(
|
||
domain_id,
|
||
backlink_count_gt_10=bool(wayback_result.get('backlink_count_gt_10')),
|
||
)
|
||
if wayback_result.get('has_sensitive_content'):
|
||
reason = f"时光机快照命中敏感词: {wayback_result.get('matched_word', '')}".strip()
|
||
self._mark_blacklisted(domain_id, domain_name, reason)
|
||
return False
|
||
logger.info(
|
||
"时光机检测完成: %s, 检查快照 %s 个, 成功抓取 %s 个, 失败 %s 个, 最大友链 %s, 耗时 %ss"
|
||
% (
|
||
domain_name,
|
||
wayback_result.get('checked_snapshot_count', 0),
|
||
wayback_result.get('fetched_snapshot_count', 0),
|
||
wayback_result.get('failed_snapshot_count', 0),
|
||
wayback_result.get('backlink_count', 0),
|
||
wayback_result.get('elapsed_seconds', 0),
|
||
)
|
||
)
|
||
except Exception as e:
|
||
logger.error(f"时光机检测失败: {domain_name}, 错误: {e}")
|
||
return True
|
||
|
||
def _run_detect_chinaz(self, domain_id, domain_name, sensitive_words):
|
||
logger.info(f"检测站长之家: {domain_name}")
|
||
proxy = self._get_proxy_for_step(domain_id, domain_name, '站长之家检测')
|
||
if proxy == '__NO_PROXY__':
|
||
return False
|
||
try:
|
||
success, message, seo_data = chinaz.check_title(domain_name, sensitive_words, proxy)
|
||
if not success:
|
||
logger.warning(f"站长之家检测未通过: {domain_name}, 原因: {message}")
|
||
if message != 'failure':
|
||
self._mark_blacklisted(domain_id, domain_name, message)
|
||
return False
|
||
logger.info(f"站长之家检测返回 failure，不拉黑域名: {domain_name}")
|
||
logger.info(f"站长之家检测完成: {domain_name}")
|
||
except Exception as e:
|
||
logger.error(f"站长之家检测失败: {domain_name}, 错误: {e}")
|
||
if proxy:
|
||
self.remove_proxy(proxy)
|
||
return True
|
||
|
||
def _run_detect_aizhan(self, domain_id, domain_name, sensitive_words):
|
||
logger.info(f"检测爱站网: {domain_name}")
|
||
proxy = self._get_proxy_for_step(domain_id, domain_name, '爱站网检测')
|
||
if proxy == '__NO_PROXY__':
|
||
return False
|
||
try:
|
||
success, message = aizhan.check_aizhan(domain_name, sensitive_words, proxy)
|
||
if not success:
|
||
logger.warning(f"爱站网检测未通过: {domain_name}, 原因: {message}")
|
||
if message != 'failure':
|
||
self._mark_blacklisted(domain_id, domain_name, message)
|
||
return False
|
||
logger.info(f"爱站网检测返回 failure，不拉黑域名: {domain_name}")
|
||
logger.info(f"爱站网检测完成: {domain_name}")
|
||
except Exception as e:
|
||
logger.error(f"爱站网检测失败: {domain_name}, 错误: {e}")
|
||
if proxy:
|
||
self.remove_proxy(proxy)
|
||
return True
|
||
|
||
def _run_detect_baidu(self, domain_id, domain_name, sensitive_words):
|
||
logger.info(f"检测百度: {domain_name}")
|
||
proxy = self._get_proxy_for_step(domain_id, domain_name, '百度site检测')
|
||
if proxy == '__NO_PROXY__':
|
||
return False
|
||
try:
|
||
success, message = baidu.check_site(domain_name, sensitive_words, proxy)
|
||
if not success:
|
||
logger.warning(f"百度site检测未通过: {domain_name}, 原因: {message}")
|
||
if proxy and ('timeout' in message.lower() or 'connection' in message.lower()) and self.allow_direct_connection():
|
||
logger.info(f"代理检测失败，尝试不使用代理重新检测: {domain_name}")
|
||
success, message = baidu.check_site(domain_name, sensitive_words, None)
|
||
if success:
|
||
logger.info(f"不使用代理检测百度成功: {domain_name}")
|
||
else:
|
||
logger.warning(f"不使用代理检测百度仍未通过: {domain_name}, 原因: {message}")
|
||
self._mark_blacklisted(domain_id, domain_name, message)
|
||
return False
|
||
else:
|
||
self._mark_blacklisted(domain_id, domain_name, message)
|
||
return False
|
||
logger.info(f"百度site检测完成: {domain_name}")
|
||
except Exception as e:
|
||
logger.error(f"百度site检测失败: {domain_name}, 错误: {e}")
|
||
if proxy:
|
||
self.remove_proxy(proxy)
|
||
if self.allow_direct_connection():
|
||
try:
|
||
logger.info(f"尝试不使用代理重新检测百度: {domain_name}")
|
||
success, message = baidu.check_site(domain_name, sensitive_words, None)
|
||
if success:
|
||
logger.info(f"不使用代理检测百度成功: {domain_name}")
|
||
else:
|
||
logger.warning(f"不使用代理检测百度仍未通过: {domain_name}, 原因: {message}")
|
||
self._mark_blacklisted(domain_id, domain_name, message)
|
||
return False
|
||
except Exception as e2:
|
||
logger.error(f"不使用代理检测百度也失败: {domain_name}, 错误: {e2}")
|
||
self._mark_blacklisted(domain_id, domain_name, str(e2))
|
||
return False
|
||
else:
|
||
self.db.update_domain_detect_status(domain_id, DETECT_STATUS_FAILED)
|
||
return False
|
||
return True
|
||
|
||
def _run_detect_360(self, domain_id, domain_name, sensitive_words):
|
||
logger.info(f"检测360: {domain_name}")
|
||
proxy = self._get_proxy_for_step(domain_id, domain_name, '360检测')
|
||
if proxy == '__NO_PROXY__':
|
||
return False
|
||
try:
|
||
passed, message = c360.check_domain(domain_name, sensitive_words, proxy)
|
||
if not passed:
|
||
logger.warning(f"360检测未通过: {domain_name}, 原因: {message}")
|
||
if message != 'failure':
|
||
self._mark_blacklisted(domain_id, domain_name, message)
|
||
return False
|
||
logger.info(f"360检测返回 failure，不拉黑域名: {domain_name}")
|
||
logger.info(f"360检测完成: {domain_name}")
|
||
except Exception as e:
|
||
logger.error(f"360检测失败: {domain_name}, 错误: {e}")
|
||
return True
|
||
|
||
def _run_detect_jucha(self, domain_id, domain, domain_name):
|
||
if not self._should_run_jucha(domain):
|
||
return True
|
||
proxy = self._get_proxy_for_step(domain_id, domain_name, '聚查检测')
|
||
if proxy == '__NO_PROXY__':
|
||
return False
|
||
self.jc_instance.session.proxies = proxy or {}
|
||
logger.info(f"检测聚查WHOIS: {domain_name}")
|
||
try:
|
||
success, message, whois_info = self.jc_instance.check_whois_domain(domain_name)
|
||
if not success:
|
||
logger.warning(f"聚查WHOIS查询失败: {domain_name}, 原因: {message}")
|
||
else:
|
||
logger.info(f"WHOIS查询结果: {success}, {message}, {whois_info}")
|
||
if 'clientHold' in whois_info or 'serverHold' in whois_info:
|
||
logger.warning(f"聚查WHOIS检测未通过: {domain_name}, 原因: 域名状态包含clientHold或serverHold")
|
||
self._mark_blacklisted(domain_id, domain_name, '域名状态包含clientHold或serverHold')
|
||
return False
|
||
logger.info(f"聚查WHOIS检测完成: {domain_name}")
|
||
except Exception as e:
|
||
logger.error(f"聚查WHOIS检测失败: {domain_name}, 错误: {e}")
|
||
|
||
logger.info(f"检测聚查备案: {domain_name}")
|
||
try:
|
||
success, message, beian_info = self.jc_instance.beian_check_domain(domain_name)
|
||
if not success:
|
||
logger.warning(f"聚查备案查询失败: {domain_name}, 原因: {message}")
|
||
else:
|
||
logger.info(f"备案查询结果: {success}, {message}, {beian_info}")
|
||
if isinstance(beian_info, tuple) and len(beian_info) == 4:
|
||
beian_time, company_type, website_url, has_beian = beian_info
|
||
has_beian_flag = 2 if has_beian == '当前存在' else 3
|
||
beian_year = None
|
||
if beian_time:
|
||
try:
|
||
beian_year = beian_time.split('-')[0]
|
||
except Exception:
|
||
pass
|
||
self.db.update_domain_beian_info(domain_id, company_type, website_url, has_beian_flag, beian_year)
|
||
logger.info(f"聚查备案检测完成: {domain_name}")
|
||
except Exception as e:
|
||
logger.error(f"聚查备案检测失败: {domain_name}, 错误: {e}")
|
||
|
||
logger.info(f"检测聚查拦截: {domain_name}")
|
||
try:
|
||
success, message, safe_info = self.jc_instance.safe_check_domain(domain_name)
|
||
if not success:
|
||
logger.warning(f"聚查安全检测失败: {domain_name}, 原因: {message}")
|
||
else:
|
||
logger.info(f"安全检测结果: {success}, {message}, {safe_info}")
|
||
check_items = ['QQ检测', '微信检测', '抖音检测', '被墙检测', '百度检测', '谷歌检测', '火狐检测']
|
||
blacklist_reason = []
|
||
if isinstance(safe_info, tuple):
|
||
for i, item in enumerate(safe_info):
|
||
if isinstance(item, tuple):
|
||
if len(item) >= 1 and item[0] == 3 and i < len(check_items):
|
||
blacklist_reason.append(f"{check_items[i]}: 拦截")
|
||
elif len(item) >= 2 and item[0] == 2 and '拦截' in item[1] and i < len(check_items):
|
||
blacklist_reason.append(f"{check_items[i]}: 拦截")
|
||
if blacklist_reason:
|
||
reason_str = "；".join(blacklist_reason)
|
||
logger.warning(f"聚查安全检测未通过: {domain_name}, 原因: {reason_str}")
|
||
self._mark_blacklisted(domain_id, domain_name, reason_str)
|
||
return False
|
||
logger.info(f"聚查拦截检测完成: {domain_name}")
|
||
except Exception as e:
|
||
logger.error(f"聚查拦截检测失败: {domain_name}, 错误: {e}")
|
||
finally:
|
||
self.db.mark_jucha_detected(domain_id)
|
||
return True
|
||
|
||
def _run_detect_juziseo(self, domain_id, domain, domain_name, sensitive_words):
|
||
if not self._should_run_juziseo(domain):
|
||
return True
|
||
proxy = self._get_proxy_for_step(domain_id, domain_name, '桔子检测')
|
||
if proxy == '__NO_PROXY__':
|
||
return False
|
||
self.juziseo_instance.session.proxies = proxy or {}
|
||
logger.info(f"检测桔子历史: {domain_name}")
|
||
try:
|
||
success, message = self.juziseo_instance.check_history(domain_name, sensitive_words)
|
||
if not success:
|
||
logger.warning(f"桔子历史检测未通过: {domain_name}, 原因: {message}")
|
||
self._mark_blacklisted(domain_id, domain_name, message)
|
||
return False
|
||
logger.info(f"桔子历史检测完成: {domain_name}")
|
||
except Exception as e:
|
||
logger.error(f"桔子历史检测失败: {domain_name}, 错误: {e}")
|
||
|
||
logger.info(f"检测桔子外链: {domain_name}")
|
||
try:
|
||
success, message = self.juziseo_instance.check_external_link(domain_name, sensitive_words)
|
||
if not success:
|
||
logger.warning(f"桔子外链检测未通过: {domain_name}, 原因: {message}")
|
||
self._mark_blacklisted(domain_id, domain_name, message)
|
||
return False
|
||
logger.info(f"桔子外链检测完成: {domain_name}")
|
||
except Exception as e:
|
||
logger.error(f"桔子外链检测失败: {domain_name}, 错误: {e}")
|
||
finally:
|
||
self.db.mark_juziseo_detected(domain_id)
|
||
return True
|
||
|
||
def _upsert_json_detection(self, domain_id, **payload):
|
||
existing = self.db.fetch_one("SELECT id FROM domain_detections WHERE domain_id = %s", (domain_id,))
|
||
columns = []
|
||
params = []
|
||
for key, value in payload.items():
|
||
columns.append(key)
|
||
params.append(json.dumps(value) if isinstance(value, (dict, list, tuple)) else value)
|
||
if not columns:
|
||
return True
|
||
if existing:
|
||
assignments = ", ".join([f"{column} = %s" for column in columns] + ["update_time = CURRENT_TIMESTAMP"])
|
||
params.append(domain_id)
|
||
sql = f"UPDATE domain_detections SET {assignments} WHERE domain_id = %s"
|
||
else:
|
||
sql = f"INSERT INTO domain_detections (domain_id, {', '.join(columns)}) VALUES (%s, {', '.join(['%s'] * len(columns))})"
|
||
params = [domain_id] + params
|
||
return self.db.execute(sql, tuple(params))
|
||
|
||
def detect_domain(self, domain_id, domain):
|
||
"""
|
||
检测单个域名
|
||
|
||
:param domain_id: 域名ID
|
||
:param domain: 域名对象（包含id、domain、source_type等字段）
|
||
"""
|
||
import threading
|
||
thread_id = threading.current_thread().ident
|
||
|
||
# 从域名对象中获取域名字符串
|
||
domain_name = domain.get('domain', '')
|
||
logger.info(f"线程 {thread_id} 开始检测域名: {domain_name}")
|
||
|
||
try:
|
||
# 检查是否需要停止
|
||
if not self.running:
|
||
logger.info(f"检测已停止，跳过域名: {domain_name}")
|
||
return
|
||
|
||
logger.info(f"开始检测域名: {domain_name}")
|
||
self.db.update_domain_detect_status(domain_id, DETECT_STATUS_RUNNING)
|
||
|
||
# 使用共享的敏感词列表
|
||
sensitive_words = self.sensitive_words
|
||
logger.debug(f"线程 {thread_id} 使用共享敏感词，共 {len(sensitive_words)} 个敏感词")
|
||
for detect_key in self._get_detect_execution_order():
|
||
if detect_key == 'detect_register':
|
||
if not self._run_detect_register(domain_id, domain, domain_name):
|
||
return
|
||
elif detect_key == 'detect_baidu_site':
|
||
if not self._run_detect_baidu(domain_id, domain_name, sensitive_words):
|
||
return
|
||
elif detect_key == 'detect_360_site':
|
||
if not self._run_detect_360(domain_id, domain_name, sensitive_words):
|
||
return
|
||
elif detect_key == 'detect_chinaz':
|
||
if not self._run_detect_chinaz(domain_id, domain_name, sensitive_words):
|
||
return
|
||
elif detect_key == 'detect_aizhan':
|
||
if not self._run_detect_aizhan(domain_id, domain_name, sensitive_words):
|
||
return
|
||
elif detect_key == 'detect_wayback':
|
||
if not self._run_detect_wayback(domain_id, domain_name, sensitive_words):
|
||
return
|
||
elif detect_key == 'detect_jucha':
|
||
if not self._run_detect_jucha(domain_id, domain, domain_name):
|
||
return
|
||
elif detect_key == 'detect_juziseo':
|
||
if not self._run_detect_juziseo(domain_id, domain, domain_name, sensitive_words):
|
||
return
|
||
|
||
self._complete_detection(domain_id, domain_name)
|
||
logger.info(f"域名检测完成: {domain_name}")
|
||
|
||
except Exception as e:
|
||
logger.error(f"检测域名出错: {domain_name}, 错误: {e}")
|
||
self.db.update_domain_detect_status(domain_id, DETECT_STATUS_FAILED)
|
||
|
||
def start_detection(self):
|
||
"""
|
||
开始检测
|
||
"""
|
||
logger.info("开始执行域名检测任务")
|
||
|
||
# 重新加载配置，确保获取最新的配置
|
||
try:
|
||
self.detect_options = self.load_detect_options()
|
||
self.proxy_config = self.load_proxy_config()
|
||
self.thread_count = self.load_thread_count()
|
||
logger.info(f"检测线程数设置为: {self.thread_count}")
|
||
self.load_cookies_from_remote()
|
||
except Exception as e:
|
||
logger.error(f"加载配置失败: {e}")
|
||
import traceback
|
||
logger.error(traceback.format_exc())
|
||
return
|
||
|
||
# 重新加载cookies
|
||
self.load_cookies_from_remote()
|
||
|
||
# 重新加载JC实例的cookies
|
||
if hasattr(self, 'jc_instance'):
|
||
self.jc_instance.load_cookies()
|
||
self.jc_instance.load_juming_cookies()
|
||
|
||
# 重新加载Juziseo实例的cookies
|
||
if hasattr(self, 'juziseo_instance'):
|
||
self.juziseo_instance.load_cookies()
|
||
|
||
# 刷新代理池
|
||
if self.proxy_config.get('proxy_enable', False):
|
||
logger.info("开始检测，刷新代理池")
|
||
self.refresh_proxy_pool()
|
||
|
||
# 更新JC和Juziseo实例的代理设置
|
||
if hasattr(self, 'jc_instance'):
|
||
self.jc_instance.proxies = self.get_proxies()
|
||
if hasattr(self, 'juziseo_instance'):
|
||
self.juziseo_instance.proxies = self.get_proxies()
|
||
|
||
# 重新加载敏感词，确保获取最新的敏感词列表
|
||
try:
|
||
self.sensitive_words = self.db.get_all_sensitive_words()
|
||
logger.info(f"重新加载敏感词完成，共 {len(self.sensitive_words)} 个敏感词")
|
||
except Exception as e:
|
||
logger.error(f"重新加载敏感词失败: {e}")
|
||
import traceback
|
||
logger.error(traceback.format_exc())
|
||
self.sensitive_words = []
|
||
|
||
# 更新配置标签
|
||
self.update_config_labels()
|
||
|
||
try:
|
||
batch_size = 1000
|
||
total_processed = 0
|
||
|
||
while self.running:
|
||
# 获取需要检测的域名
|
||
domains = self.db.get_domains_to_detect(limit=batch_size, detect_options=self.detect_options)
|
||
current_batch_size = len(domains)
|
||
logger.info(f"获取到 {current_batch_size} 个需要检测的域名")
|
||
|
||
if not domains:
|
||
logger.info("没有需要检测的域名")
|
||
break
|
||
|
||
# 检查是否需要停止获取域名（当获取的数量少于1000时）
|
||
should_stop = current_batch_size < batch_size
|
||
|
||
# 创建线程池，限制同时运行的线程数量
|
||
active_threads = []
|
||
max_threads = self.thread_count
|
||
|
||
try:
|
||
logger.info(f"开始创建线程，当前批次域名数: {current_batch_size}，最大线程数: {max_threads}")
|
||
for i, domain in enumerate(domains):
|
||
# 检查是否需要停止
|
||
if not self.running:
|
||
logger.info("检测已停止，停止创建新线程")
|
||
break
|
||
|
||
# 等待线程数量降到最大值以下
|
||
while len([t for t in active_threads if t.is_alive()]) >= max_threads:
|
||
# 清理已完成的线程
|
||
active_threads = [t for t in active_threads if t.is_alive()]
|
||
# 短暂休眠，避免CPU占用过高
|
||
import time
|
||
time.sleep(0.1)
|
||
|
||
domain_id = domain['id']
|
||
domain_name = domain['domain']
|
||
|
||
# 更新进度
|
||
if self.detect_thread:
|
||
try:
|
||
# 通过信号发送进度更新
|
||
self.detect_thread.progress_signal.emit(total_processed + i, total_processed + current_batch_size)
|
||
except Exception as e:
|
||
logger.error(f"发送进度更新信号失败: {e}")
|
||
import traceback
|
||
logger.error(traceback.format_exc())
|
||
|
||
# 创建线程
|
||
try:
|
||
thread = threading.Thread(target=self.detect_domain, args=(domain_id, domain))
|
||
active_threads.append(thread)
|
||
logger.debug(f"创建线程 {i+1} 成功，域名: {domain_name}")
|
||
except Exception as e:
|
||
logger.error(f"创建线程失败: {e}")
|
||
import traceback
|
||
logger.error(traceback.format_exc())
|
||
continue
|
||
|
||
# 启动线程
|
||
try:
|
||
thread.start()
|
||
logger.debug(f"启动线程 {i+1} 成功，域名: {domain_name}")
|
||
except Exception as e:
|
||
logger.error(f"启动线程失败: {e}")
|
||
import traceback
|
||
logger.error(traceback.format_exc())
|
||
active_threads.remove(thread)
|
||
continue
|
||
|
||
# 显示当前实际线程数量
|
||
current_active = len([t for t in active_threads if t.is_alive()])
|
||
logger.info(f"当前实际线程数量: {current_active}/{max_threads}")
|
||
|
||
# 更新GUI显示
|
||
if self.detect_thread:
|
||
try:
|
||
# 通过信号发送线程数量更新
|
||
self.detect_thread.thread_count_signal.emit(current_active, max_threads)
|
||
except Exception as e:
|
||
logger.error(f"发送线程数量更新信号失败: {e}")
|
||
import traceback
|
||
logger.error(traceback.format_exc())
|
||
except Exception as e:
|
||
logger.error(f"创建或启动线程失败: {e}")
|
||
import traceback
|
||
logger.error(traceback.format_exc())
|
||
|
||
# 等待所有线程完成
|
||
logger.info(f"等待剩余 {len(active_threads)} 个线程完成")
|
||
for i, thread in enumerate(active_threads):
|
||
# 检查是否需要停止
|
||
if not self.running:
|
||
logger.info("检测已停止，停止等待线程完成")
|
||
break
|
||
try:
|
||
thread.join(timeout=30) # 添加超时，避免线程阻塞
|
||
# 更新进度
|
||
if self.detect_thread:
|
||
try:
|
||
# 通过信号发送进度更新
|
||
self.detect_thread.progress_signal.emit(total_processed + i + 1, total_processed + current_batch_size)
|
||
except Exception as e:
|
||
logger.error(f"发送进度更新信号失败: {e}")
|
||
import traceback
|
||
logger.error(traceback.format_exc())
|
||
except Exception as e:
|
||
logger.error(f"等待线程完成失败: {e}")
|
||
import traceback
|
||
logger.error(traceback.format_exc())
|
||
|
||
# 发送线程数量更新信号
|
||
if self.detect_thread:
|
||
try:
|
||
active_count = threading.active_count()
|
||
self.detect_thread.thread_count_signal.emit(active_count, max_threads)
|
||
except Exception as e:
|
||
logger.error(f"发送线程数量更新信号失败: {e}")
|
||
import traceback
|
||
logger.error(traceback.format_exc())
|
||
|
||
# 完成当前批次进度
|
||
if self.detect_thread:
|
||
try:
|
||
# 通过信号发送进度更新
|
||
self.detect_thread.progress_signal.emit(total_processed + current_batch_size, total_processed + current_batch_size)
|
||
except Exception as e:
|
||
logger.error(f"发送进度更新信号失败: {e}")
|
||
import traceback
|
||
logger.error(traceback.format_exc())
|
||
|
||
total_processed += current_batch_size
|
||
|
||
# 如果当前批次数量少于batch_size，检测完成后停止获取域名
|
||
if should_stop:
|
||
logger.info(f"当前批次域名数量 ({current_batch_size}) 少于 {batch_size}，停止获取域名")
|
||
break
|
||
logger.info(f"当前批次检测完成，累计处理 {total_processed} 个域名")
|
||
|
||
# 完成进度
|
||
if self.detect_thread:
|
||
try:
|
||
# 通过信号发送进度更新
|
||
self.detect_thread.progress_signal.emit(100, 100)
|
||
except Exception as e:
|
||
logger.error(f"发送进度更新信号失败: {e}")
|
||
import traceback
|
||
logger.error(traceback.format_exc())
|
||
|
||
logger.info("域名检测任务完成")
|
||
|
||
except Exception as e:
|
||
logger.error(f"执行检测任务出错: {e}")
|
||
|
||
def run_daily_task(self):
|
||
"""
|
||
执行每日检测任务
|
||
"""
|
||
logger.info(f"[{datetime.now()}] 开始每日检测任务")
|
||
self.start_detection()
|
||
logger.info(f"[{datetime.now()}] 每日检测任务完成")
|
||
|
||
def start_scheduler(self):
|
||
"""
|
||
启动定时任务
|
||
"""
|
||
# 避免重复注册相同定时任务
|
||
schedule.clear('daily_detection')
|
||
# 每天凌晨1点执行检测；启动程序时不自动跑一轮，避免和人工启动混淆
|
||
schedule.every().day.at("01:00").do(self.run_daily_task).tag('daily_detection')
|
||
logger.info("定时任务已启动，每天凌晨1点执行检测；程序启动后不会立即自动执行")
|
||
|
||
# 循环执行定时任务
|
||
while self.running:
|
||
schedule.run_pending()
|
||
time.sleep(60)
|
||
|
||
def start(self):
|
||
"""
|
||
启动检测端
|
||
"""
|
||
logger.info("启动域名检测端")
|
||
self.running = True
|
||
|
||
# 启动定时任务
|
||
scheduler_thread = threading.Thread(target=self.start_scheduler)
|
||
scheduler_thread.daemon = True
|
||
scheduler_thread.start()
|
||
|
||
# 保持程序运行
|
||
try:
|
||
while self.running:
|
||
time.sleep(1)
|
||
except KeyboardInterrupt:
|
||
logger.info("检测端已停止")
|
||
self.running = False
|
||
|
||
def load_cookies_from_remote(self):
|
||
"""
|
||
从远程加载cookies
|
||
"""
|
||
try:
|
||
if self.use_redis:
|
||
# 加载聚名cookies
|
||
juming_cookies_str = self.redis_client.get('domain_tool:juming_cookies')
|
||
if juming_cookies_str:
|
||
# 将cookies保存到本地文件
|
||
import pickle
|
||
from requests.cookies import RequestsCookieJar
|
||
|
||
# 创建cookie jar
|
||
cookie_jar = RequestsCookieJar()
|
||
|
||
# 解析cookies字符串
|
||
try:
|
||
import ast
|
||
cookies_dict = ast.literal_eval(juming_cookies_str)
|
||
for name, value in cookies_dict.items():
|
||
cookie_jar.set(name, value)
|
||
except Exception as e:
|
||
logger.error(f"解析聚名cookies失败: {e}")
|
||
|
||
# 保存到本地文件
|
||
try:
|
||
with open('juming_cookies.pkl', 'wb') as f:
|
||
pickle.dump(cookie_jar, f)
|
||
logger.info("从Redis加载聚名cookies成功并保存到本地")
|
||
except Exception as e:
|
||
logger.error(f"保存聚名cookies到本地失败: {e}")
|
||
|
||
# 加载聚查cookies
|
||
jucha_cookies_str = self.redis_client.get('domain_tool:jucha_cookies')
|
||
if jucha_cookies_str:
|
||
# 将cookies保存到本地文件
|
||
import pickle
|
||
from requests.cookies import RequestsCookieJar
|
||
|
||
# 创建cookie jar
|
||
cookie_jar = RequestsCookieJar()
|
||
|
||
# 解析cookies字符串
|
||
try:
|
||
import ast
|
||
cookies_dict = ast.literal_eval(jucha_cookies_str)
|
||
for name, value in cookies_dict.items():
|
||
cookie_jar.set(name, value)
|
||
except Exception as e:
|
||
logger.error(f"解析聚查cookies失败: {e}")
|
||
|
||
# 保存到本地文件
|
||
try:
|
||
with open('jucha_cookies.pkl', 'wb') as f:
|
||
pickle.dump(cookie_jar, f)
|
||
logger.info("从Redis加载聚查cookies成功并保存到本地")
|
||
except Exception as e:
|
||
logger.error(f"保存聚查cookies到本地失败: {e}")
|
||
|
||
# 加载桔子SEO cookies
|
||
juziseo_cookies_str = self.redis_client.get('domain_tool:juziseo_cookies')
|
||
if juziseo_cookies_str:
|
||
# 将cookies保存到本地文件
|
||
import pickle
|
||
from requests.cookies import RequestsCookieJar
|
||
|
||
# 创建cookie jar
|
||
cookie_jar = RequestsCookieJar()
|
||
|
||
# 解析cookies字符串
|
||
try:
|
||
import ast
|
||
cookies_dict = ast.literal_eval(juziseo_cookies_str)
|
||
for name, value in cookies_dict.items():
|
||
cookie_jar.set(name, value)
|
||
except Exception as e:
|
||
logger.error(f"解析桔子SEO cookies失败: {e}")
|
||
|
||
# 保存到本地文件
|
||
try:
|
||
with open('juziseo_cookies.pkl', 'wb') as f:
|
||
pickle.dump(cookie_jar, f)
|
||
logger.info("从Redis加载桔子SEO cookies成功并保存到本地")
|
||
except Exception as e:
|
||
logger.error(f"保存桔子SEO cookies到本地失败: {e}")
|
||
except Exception as e:
|
||
logger.error(f"从远程加载cookies失败: {e}")
|
||
|
||
def update_config_labels(self):
|
||
"""
|
||
更新GUI配置标签
|
||
"""
|
||
# 使用QCoreApplication.postEvent发送配置更新事件，这是最安全的方法
|
||
try:
|
||
logger.debug("发送配置更新事件")
|
||
# 确保主窗口存在
|
||
if self.main_window:
|
||
from PySide6.QtCore import QCoreApplication
|
||
event = ConfigUpdateEvent()
|
||
QCoreApplication.postEvent(self.main_window, event)
|
||
logger.debug("配置更新事件已发送")
|
||
else:
|
||
logger.debug("主窗口不存在，跳过配置更新")
|
||
except Exception as e:
|
||
logger.error(f"发送配置更新事件失败: {e}")
|
||
|
||
|
||
|
||
def start_redis_subscription(self):
|
||
"""
|
||
启动Redis订阅，监听配置更新
|
||
"""
|
||
while self.running:
|
||
try:
|
||
# 创建新的Redis客户端用于订阅
|
||
redis_sub_client = redis.Redis(
|
||
host=config.REDIS_HOST,
|
||
port=config.REDIS_PORT,
|
||
password=config.REDIS_PASSWORD,
|
||
db=config.REDIS_DB,
|
||
decode_responses=True,
|
||
socket_connect_timeout=30, # 增加连接超时时间
|
||
socket_timeout=60, # 增加读取超时时间
|
||
retry_on_timeout=True,
|
||
health_check_interval=30
|
||
)
|
||
|
||
# 订阅配置更新频道
|
||
pubsub = redis_sub_client.pubsub()
|
||
pubsub.subscribe('domain_tool:config_update')
|
||
|
||
# 使用logger.debug输出到文件日志，不输出到GUI日志框
|
||
logger.debug("开始监听配置更新...")
|
||
|
||
# 循环监听消息
|
||
for message in pubsub.listen():
|
||
if not self.running:
|
||
break
|
||
if message['type'] == 'message':
|
||
config_type = message['data']
|
||
# 使用logger.debug输出到文件日志，不输出到GUI日志框
|
||
logger.debug(f"收到配置更新消息: {config_type}")
|
||
|
||
# 重新加载配置
|
||
self.detect_options = self.load_detect_options()
|
||
self.proxy_config = self.load_proxy_config()
|
||
self.thread_count = self.load_thread_count()
|
||
self.load_cookies_from_remote()
|
||
|
||
# 更新GUI标签
|
||
self.update_config_labels()
|
||
|
||
# 使用logger.debug输出到文件日志，不输出到GUI日志框
|
||
logger.debug("配置已更新")
|
||
except Exception as e:
|
||
logger.debug(f"Redis订阅失败: {e}")
|
||
logger.debug("3秒后尝试重新连接...")
|
||
time.sleep(3) # 增加重试间隔
|
||
|
||
def stop(self):
|
||
"""
|
||
停止检测端
|
||
"""
|
||
logger.info("停止域名检测端")
|
||
self.running = False
|
||
|
||
if __name__ == "__main__":
|
||
try:
|
||
# 配置日志
|
||
logger.add("detect_worker.log", rotation="1 day", level="DEBUG")
|
||
logger.info("程序开始运行")
|
||
|
||
# 创建应用程序
|
||
app = QApplication([])
|
||
logger.info("创建应用程序成功")
|
||
|
||
# 创建并显示主窗口
|
||
window = DetectMainWindow()
|
||
logger.info("创建主窗口成功")
|
||
window.show()
|
||
logger.info("显示主窗口成功")
|
||
|
||
# 运行应用程序
|
||
logger.info("开始运行应用程序")
|
||
app.exec()
|
||
logger.info("应用程序运行结束")
|
||
|
||
# 当窗口关闭时，停止检测端
|
||
if window.worker:
|
||
logger.info("停止检测端")
|
||
window.worker.stop()
|
||
except Exception as e:
|
||
logger.error(f"程序运行出错: {e}")
|
||
import traceback
|
||
logger.error(traceback.format_exc())
|
||
import time
|
||
time.sleep(10) # 等待10秒，以便查看错误信息
|