577 lines
22 KiB
Python
577 lines
22 KiB
Python
# -*- coding: UTF-8 -*-
|
||
'''
|
||
@Project :domainScanDemo
|
||
@File :juming_crawler.py
|
||
@IDE :PyCharm
|
||
@Author :梦伴
|
||
@Date :2026/4/9 15:00
|
||
@explain : 聚名网爬取页面
|
||
'''
|
||
|
||
from PySide6.QtWidgets import QWidget, QVBoxLayout, QHBoxLayout, QPushButton, QLabel, QTextEdit, QProgressBar, QLineEdit, QComboBox, QDateEdit, QCheckBox
|
||
from PySide6.QtCore import Qt, QThread, Signal, QDate
|
||
from PySide6.QtGui import QIntValidator
|
||
from loguru import logger
|
||
import re
|
||
import datetime
|
||
import time
|
||
|
||
from app.core.domain_collector import DomainCollector
|
||
from detect.juming import JM
|
||
|
||
|
||
class JumingCrawlThread(QThread):
|
||
"""
|
||
聚名爬取线程
|
||
"""
|
||
progress_updated = Signal(int)
|
||
status_updated = Signal(str)
|
||
finished = Signal(bool, str)
|
||
|
||
def __init__(self, crawl_type, page_start=1, page_size=50, crawl_date=None, auto_date=True):
|
||
"""
|
||
初始化聚名爬取线程
|
||
|
||
:param crawl_type: 爬取类型 (1: 一口价, 2: 删除列表)
|
||
:param page_start: 起始页码
|
||
:param page_size: 每页数量
|
||
:param crawl_date: 爬取日期（删除列表用）
|
||
:param auto_date: 是否自动新增日期
|
||
"""
|
||
super().__init__()
|
||
self.crawl_type = crawl_type
|
||
self.page_start = page_start
|
||
self.page_size = page_size
|
||
self.crawl_date = crawl_date
|
||
self.auto_date = auto_date
|
||
self.is_paused = False
|
||
self.is_stopped = False
|
||
self.current_page = 0
|
||
self.total_count = 0
|
||
|
||
def run(self):
|
||
"""
|
||
运行聚名爬取线程
|
||
"""
|
||
try:
|
||
# 初始化聚名客户端
|
||
jm = JM()
|
||
|
||
# 加载 Cookie
|
||
jm.load_cookies()
|
||
logger.info("已加载 Cookie")
|
||
self.status_updated.emit("已加载 Cookie")
|
||
|
||
# 直接开始爬取，不需要登录，因为 Cookie 已经在系统设置页面加载了
|
||
self.progress_updated.emit(10)
|
||
|
||
if self.crawl_type == 1: # 一口价
|
||
self.progress_updated.emit(30)
|
||
logger.info("开始获取一口价域名")
|
||
self.status_updated.emit("开始获取一口价域名")
|
||
|
||
# 自动爬取多页
|
||
page = self.page_start
|
||
while not self.is_stopped:
|
||
if self.is_paused:
|
||
time.sleep(0.1)
|
||
continue
|
||
|
||
self.current_page = page
|
||
self.status_updated.emit(f"正在爬取第 {page} 页")
|
||
logger.info(f"正在爬取第 {page} 页")
|
||
|
||
# 获取当前页
|
||
success, html = jm.ykj_get_list(page=page, page_size=self.page_size)
|
||
if success:
|
||
pattern_ym = r"<a class='yda1 ydz' ym='([^']*)'"
|
||
results = re.findall(pattern_ym, html)
|
||
domains = [domain.strip() for domain in results if domain.strip()]
|
||
domain_count = len(domains)
|
||
self.total_count += domain_count
|
||
|
||
self.status_updated.emit(f"第 {page} 页找到 {domain_count} 个域名，累计 {self.total_count} 个")
|
||
logger.info(f"第 {page} 页找到 {domain_count} 个域名，累计 {self.total_count} 个")
|
||
|
||
# 自动入库
|
||
if domains:
|
||
collector = DomainCollector()
|
||
stats = collector.add_domains_batch(domains, 1) # 1 表示一口价
|
||
logger.info(f"自动入库完成: {stats}")
|
||
self.status_updated.emit(f"自动入库完成: 成功添加 {stats['added']} 个域名")
|
||
|
||
# 如果返回的数量小于指定的数量，停止爬取
|
||
if domain_count < self.page_size:
|
||
logger.info(f"返回数量小于指定数量，停止爬取")
|
||
self.status_updated.emit("返回数量小于指定数量，停止爬取")
|
||
break
|
||
|
||
# 增加页码
|
||
page += 1
|
||
|
||
# 模拟网络延迟
|
||
time.sleep(1)
|
||
else:
|
||
logger.error(f"获取一口价域名失败: {html}")
|
||
self.status_updated.emit(f"获取一口价域名失败: {html}")
|
||
break
|
||
|
||
elif self.crawl_type == 2: # 删除列表
|
||
self.progress_updated.emit(30)
|
||
logger.info("开始获取删除域名列表")
|
||
self.status_updated.emit("开始获取删除域名列表")
|
||
|
||
# 使用传入的日期或默认今天
|
||
start_date_str = self.crawl_date if self.crawl_date else datetime.date.today().strftime("%Y-%m-%d")
|
||
start_date = datetime.datetime.strptime(start_date_str, "%Y-%m-%d").date()
|
||
# 计算结束日期：今天 + 4天
|
||
end_date = datetime.date.today() + datetime.timedelta(days=4)
|
||
|
||
if self.auto_date:
|
||
# 自动新增日期，从起始日期到今天+4天
|
||
current_date = start_date
|
||
while current_date <= end_date and not self.is_stopped:
|
||
if self.is_paused:
|
||
time.sleep(0.1)
|
||
continue
|
||
|
||
crawl_date = current_date.strftime("%Y-%m-%d")
|
||
self.status_updated.emit(f"正在爬取 {crawl_date} 的删除域名")
|
||
logger.info(f"正在爬取 {crawl_date} 的删除域名")
|
||
|
||
deleted_domains = jm.new_cha_del(crawl_date)
|
||
domains = [domain.strip() for domain in deleted_domains if domain.strip()]
|
||
domain_count = len(domains)
|
||
self.total_count += domain_count
|
||
|
||
self.status_updated.emit(f"{crawl_date} 找到 {domain_count} 个删除域名，累计 {self.total_count} 个")
|
||
logger.info(f"{crawl_date} 找到 {domain_count} 个删除域名，累计 {self.total_count} 个")
|
||
|
||
# 自动入库
|
||
if domains:
|
||
collector = DomainCollector()
|
||
stats = collector.add_domains_batch(domains, 2) # 2 表示删除列表
|
||
logger.info(f"{crawl_date} 自动入库完成: {stats}")
|
||
self.status_updated.emit(f"{crawl_date} 自动入库完成: 成功添加 {stats['added']} 个域名")
|
||
|
||
# 增加日期
|
||
current_date = current_date + datetime.timedelta(days=1)
|
||
|
||
# 模拟网络延迟
|
||
time.sleep(1)
|
||
else:
|
||
# 只爬取指定日期
|
||
crawl_date = start_date_str
|
||
self.status_updated.emit(f"正在爬取 {crawl_date} 的删除域名")
|
||
logger.info(f"正在爬取 {crawl_date} 的删除域名")
|
||
|
||
deleted_domains = jm.new_cha_del(crawl_date)
|
||
domains = [domain.strip() for domain in deleted_domains if domain.strip()]
|
||
domain_count = len(domains)
|
||
self.total_count = domain_count
|
||
|
||
self.status_updated.emit(f"找到 {domain_count} 个删除域名")
|
||
logger.info(f"找到 {domain_count} 个删除域名")
|
||
|
||
# 自动入库
|
||
if domains:
|
||
collector = DomainCollector()
|
||
stats = collector.add_domains_batch(domains, 2) # 2 表示删除列表
|
||
logger.info(f"自动入库完成: {stats}")
|
||
self.status_updated.emit(f"自动入库完成: 成功添加 {stats['added']} 个域名")
|
||
|
||
self.progress_updated.emit(90)
|
||
|
||
self.progress_updated.emit(100)
|
||
self.finished.emit(True, f"成功获取 {self.total_count} 个域名")
|
||
except Exception as e:
|
||
logger.error(f"从聚名网爬取失败: {e}")
|
||
self.status_updated.emit(f"爬取失败: {str(e)}")
|
||
self.finished.emit(False, f"爬取失败: {str(e)}")
|
||
|
||
def pause(self):
|
||
"""
|
||
暂停爬取
|
||
"""
|
||
self.is_paused = True
|
||
logger.info("爬取已暂停")
|
||
self.status_updated.emit("爬取已暂停")
|
||
|
||
def resume(self):
|
||
"""
|
||
恢复爬取
|
||
"""
|
||
self.is_paused = False
|
||
logger.info("爬取已恢复")
|
||
self.status_updated.emit("爬取已恢复")
|
||
|
||
def stop(self):
|
||
"""
|
||
停止爬取
|
||
"""
|
||
self.is_stopped = True
|
||
logger.info("爬取已停止")
|
||
self.status_updated.emit("爬取已停止")
|
||
|
||
|
||
class JumingCrawlerWidget(QWidget):
|
||
"""
|
||
聚名网爬取页面
|
||
"""
|
||
|
||
def __init__(self):
|
||
"""
|
||
初始化聚名网爬取页面
|
||
"""
|
||
super().__init__()
|
||
|
||
# 创建布局
|
||
layout = QVBoxLayout(self)
|
||
|
||
|
||
|
||
# 爬取类型选择
|
||
type_layout = QHBoxLayout()
|
||
type_label = QLabel("爬取类型:")
|
||
type_label.setStyleSheet("font-size: 14px; color: #666; min-width: 80px;")
|
||
self.type_combo = QComboBox()
|
||
self.type_combo.addItem("一口价域名", 1)
|
||
self.type_combo.addItem("删除列表域名", 2)
|
||
self.type_combo.setStyleSheet("font-size: 14px; padding: 5px; border: 1px solid #ddd; border-radius: 4px;")
|
||
# 监听类型变化
|
||
self.type_combo.currentIndexChanged.connect(self.on_type_changed)
|
||
type_layout.addWidget(type_label)
|
||
type_layout.addWidget(self.type_combo)
|
||
type_layout.setContentsMargins(0, 0, 0, 15)
|
||
layout.addLayout(type_layout)
|
||
|
||
# 页码和每页数量设置
|
||
page_layout = QHBoxLayout()
|
||
|
||
# 起始页码
|
||
page_start_label = QLabel("起始页码:")
|
||
page_start_label.setStyleSheet("font-size: 14px; color: #666; min-width: 80px;")
|
||
self.page_start_edit = QLineEdit("1")
|
||
# 移除所有限制，允许输入任意正整数
|
||
self.page_start_edit.setStyleSheet("font-size: 14px; padding: 5px; border: 1px solid #ddd; border-radius: 4px; width: 120px;")
|
||
page_layout.addWidget(page_start_label)
|
||
page_layout.addWidget(self.page_start_edit)
|
||
|
||
# 每页数量
|
||
page_size_label = QLabel("每页数量:")
|
||
page_size_label.setStyleSheet("font-size: 14px; color: #666; min-width: 80px; margin-left: 20px;")
|
||
self.page_size_edit = QLineEdit("500")
|
||
self.page_size_edit.setValidator(QIntValidator(1, 1000))
|
||
self.page_size_edit.setStyleSheet("font-size: 14px; padding: 5px; border: 1px solid #ddd; border-radius: 4px; width: 80px;")
|
||
page_layout.addWidget(page_size_label)
|
||
page_layout.addWidget(self.page_size_edit)
|
||
page_layout.setContentsMargins(0, 0, 0, 20)
|
||
layout.addLayout(page_layout)
|
||
|
||
# 日期设置（删除列表用）
|
||
date_container = QWidget()
|
||
date_layout = QHBoxLayout(date_container)
|
||
|
||
# 起始日期
|
||
date_label = QLabel("起始日期:")
|
||
date_label.setStyleSheet("font-size: 14px; color: #666; min-width: 80px;")
|
||
self.date_edit = QDateEdit()
|
||
self.date_edit.setDate(QDate.currentDate())
|
||
self.date_edit.setCalendarPopup(True)
|
||
self.date_edit.setStyleSheet("font-size: 14px; padding: 5px; border: 1px solid #ddd; border-radius: 4px; width: 150px;")
|
||
# 设置最大日期为今天+4天，最小日期为今天的前4天
|
||
max_date = QDate.currentDate().addDays(4)
|
||
min_date = QDate.currentDate().addDays(-4)
|
||
self.date_edit.setMinimumDate(min_date)
|
||
self.date_edit.setMaximumDate(max_date)
|
||
date_layout.addWidget(date_label)
|
||
date_layout.addWidget(self.date_edit)
|
||
|
||
# 自动新增日期选项
|
||
auto_date_checkbox = QCheckBox("自动新增日期")
|
||
auto_date_checkbox.setChecked(True)
|
||
auto_date_checkbox.setStyleSheet("font-size: 14px; margin-left: 20px;")
|
||
self.auto_date_checkbox = auto_date_checkbox
|
||
date_layout.addWidget(auto_date_checkbox)
|
||
|
||
date_layout.setContentsMargins(0, 0, 0, 20)
|
||
self.date_container = date_container
|
||
layout.addWidget(date_container)
|
||
# 默认隐藏日期输入框
|
||
self.date_container.setVisible(False)
|
||
|
||
# 按钮布局
|
||
button_layout = QHBoxLayout()
|
||
|
||
# 创建开始按钮
|
||
self.start_btn = QPushButton("开始爬取")
|
||
self.start_btn.clicked.connect(self.start_crawl)
|
||
self.start_btn.setStyleSheet("""
|
||
QPushButton {
|
||
font-size: 14px;
|
||
padding: 8px 16px;
|
||
background-color: #4CAF50;
|
||
color: white;
|
||
border: none;
|
||
border-radius: 4px;
|
||
}
|
||
QPushButton:hover {
|
||
background-color: #45a049;
|
||
}
|
||
QPushButton:disabled {
|
||
background-color: #cccccc;
|
||
}
|
||
""")
|
||
button_layout.addWidget(self.start_btn)
|
||
|
||
# 创建暂停按钮
|
||
self.pause_btn = QPushButton("暂停爬取")
|
||
self.pause_btn.clicked.connect(self.pause_crawl)
|
||
self.pause_btn.setEnabled(False)
|
||
self.pause_btn.setStyleSheet("""
|
||
QPushButton {
|
||
font-size: 14px;
|
||
padding: 8px 16px;
|
||
background-color: #ff9800;
|
||
color: white;
|
||
border: none;
|
||
border-radius: 4px;
|
||
margin-left: 10px;
|
||
}
|
||
QPushButton:hover {
|
||
background-color: #f57c00;
|
||
}
|
||
QPushButton:disabled {
|
||
background-color: #cccccc;
|
||
}
|
||
""")
|
||
button_layout.addWidget(self.pause_btn)
|
||
|
||
# 创建停止按钮
|
||
self.stop_btn = QPushButton("停止爬取")
|
||
self.stop_btn.clicked.connect(self.stop_crawl)
|
||
self.stop_btn.setEnabled(False)
|
||
self.stop_btn.setStyleSheet("""
|
||
QPushButton {
|
||
font-size: 14px;
|
||
padding: 8px 16px;
|
||
background-color: #f44336;
|
||
color: white;
|
||
border: none;
|
||
border-radius: 4px;
|
||
margin-left: 10px;
|
||
}
|
||
QPushButton:hover {
|
||
background-color: #d32f2f;
|
||
}
|
||
QPushButton:disabled {
|
||
background-color: #cccccc;
|
||
}
|
||
""")
|
||
button_layout.addWidget(self.stop_btn)
|
||
button_layout.setContentsMargins(0, 0, 0, 20)
|
||
layout.addLayout(button_layout)
|
||
|
||
# 创建日志显示区域
|
||
self.log_edit = QTextEdit()
|
||
self.log_edit.setPlaceholderText("爬取日志将显示在这里")
|
||
self.log_edit.setReadOnly(True)
|
||
self.log_edit.setStyleSheet("""
|
||
QTextEdit {
|
||
font-size: 13px;
|
||
font-family: Consolas, Monaco, monospace;
|
||
padding: 10px;
|
||
border: 1px solid #ddd;
|
||
border-radius: 4px;
|
||
background-color: #f9f9f9;
|
||
}
|
||
""")
|
||
layout.addWidget(self.log_edit)
|
||
|
||
# 创建进度条
|
||
self.progress_bar = QProgressBar()
|
||
self.progress_bar.setVisible(False)
|
||
self.progress_bar.setStyleSheet("""
|
||
QProgressBar {
|
||
height: 20px;
|
||
border: 1px solid #ddd;
|
||
border-radius: 10px;
|
||
background-color: #f0f0f0;
|
||
margin-top: 10px;
|
||
margin-bottom: 10px;
|
||
}
|
||
QProgressBar::chunk {
|
||
background-color: #4CAF50;
|
||
border-radius: 10px;
|
||
}
|
||
""")
|
||
layout.addWidget(self.progress_bar)
|
||
|
||
# 创建状态标签
|
||
self.status_label = QLabel("")
|
||
self.status_label.setAlignment(Qt.AlignCenter)
|
||
self.status_label.setStyleSheet("font-size: 14px; color: #333; margin-top: 10px; padding: 8px; background-color: #f0f8ff; border-radius: 4px;")
|
||
layout.addWidget(self.status_label)
|
||
|
||
# 存储爬取线程
|
||
self.crawl_thread = None
|
||
|
||
logger.info("聚名网爬取页面创建完成")
|
||
|
||
def start_crawl(self):
|
||
"""
|
||
开始爬取
|
||
"""
|
||
crawl_type = self.type_combo.currentData()
|
||
|
||
# 获取起始页码和每页数量
|
||
try:
|
||
page_start = int(self.page_start_edit.text())
|
||
page_size = int(self.page_size_edit.text())
|
||
if page_start < 1:
|
||
self.status_label.setText("起始页码必须大于0")
|
||
return
|
||
if page_size < 1:
|
||
self.status_label.setText("每页数量必须大于0")
|
||
return
|
||
except ValueError:
|
||
self.status_label.setText("请输入有效的页码和每页数量")
|
||
return
|
||
|
||
# 显示进度条
|
||
self.progress_bar.setVisible(True)
|
||
self.progress_bar.setValue(0)
|
||
self.status_label.setText("正在爬取域名...")
|
||
|
||
# 清空日志
|
||
self.log_edit.clear()
|
||
|
||
# 启用/禁用按钮
|
||
self.start_btn.setEnabled(False)
|
||
self.pause_btn.setEnabled(True)
|
||
self.stop_btn.setEnabled(True)
|
||
|
||
# 获取爬取日期和自动新增日期选项（删除列表用）
|
||
crawl_date = None
|
||
auto_date = False
|
||
if crawl_type == 2: # 删除列表
|
||
crawl_date = self.date_edit.date().toString("yyyy-MM-dd")
|
||
auto_date = self.auto_date_checkbox.isChecked()
|
||
|
||
# 创建并启动爬取线程
|
||
self.crawl_thread = JumingCrawlThread(crawl_type, page_start, page_size, crawl_date, auto_date)
|
||
self.crawl_thread.progress_updated.connect(self.update_progress)
|
||
self.crawl_thread.status_updated.connect(self.update_status)
|
||
self.crawl_thread.finished.connect(self.crawl_finished)
|
||
self.crawl_thread.start()
|
||
|
||
if crawl_type == 2: # 删除列表
|
||
auto_date_str = "是" if auto_date else "否"
|
||
logger.info(f"开始爬取聚名网域名, 类型: {crawl_type}, 爬取日期: {crawl_date}, 自动新增日期: {auto_date_str}")
|
||
self.log_edit.append(f"开始爬取聚名网域名, 类型: {crawl_type}, 爬取日期: {crawl_date}, 自动新增日期: {auto_date_str}")
|
||
else: # 一口价
|
||
logger.info(f"开始爬取聚名网域名, 类型: {crawl_type}, 起始页码: {page_start}, 每页数量: {page_size}")
|
||
self.log_edit.append(f"开始爬取聚名网域名, 类型: {crawl_type}, 起始页码: {page_start}, 每页数量: {page_size}")
|
||
|
||
def pause_crawl(self):
|
||
"""
|
||
暂停爬取
|
||
"""
|
||
if self.crawl_thread:
|
||
if self.crawl_thread.is_paused:
|
||
self.crawl_thread.resume()
|
||
self.pause_btn.setText("暂停爬取")
|
||
else:
|
||
self.crawl_thread.pause()
|
||
self.pause_btn.setText("恢复爬取")
|
||
|
||
def stop_crawl(self):
|
||
"""
|
||
停止爬取
|
||
"""
|
||
if self.crawl_thread:
|
||
self.crawl_thread.stop()
|
||
|
||
def update_progress(self, progress):
|
||
"""
|
||
更新进度
|
||
|
||
:param progress: 进度值
|
||
"""
|
||
self.progress_bar.setValue(progress)
|
||
|
||
def update_status(self, status):
|
||
"""
|
||
更新状态
|
||
|
||
:param status: 状态消息
|
||
"""
|
||
self.status_label.setText(status)
|
||
self.log_edit.append(status)
|
||
|
||
def crawl_finished(self, success, message):
|
||
"""
|
||
爬取完成
|
||
|
||
:param success: 是否成功
|
||
:param message: 消息
|
||
"""
|
||
self.status_label.setText(message)
|
||
self.log_edit.append(message)
|
||
|
||
# 启用/禁用按钮
|
||
self.start_btn.setEnabled(True)
|
||
self.pause_btn.setEnabled(False)
|
||
self.pause_btn.setText("暂停爬取")
|
||
self.stop_btn.setEnabled(False)
|
||
|
||
self.progress_bar.setVisible(False)
|
||
|
||
logger.info(f"聚名网爬取完成: {message}")
|
||
|
||
def on_type_changed(self, index):
|
||
"""
|
||
爬取类型变化时的处理
|
||
|
||
:param index: 选择的索引
|
||
"""
|
||
crawl_type = self.type_combo.currentData()
|
||
if crawl_type == 2: # 删除列表
|
||
self.date_container.setVisible(True)
|
||
else: # 一口价
|
||
self.date_container.setVisible(False)
|
||
|
||
|
||
class ImportThread(QThread):
|
||
"""
|
||
导入线程
|
||
"""
|
||
progress_updated = Signal(int)
|
||
finished = Signal(bool, str)
|
||
|
||
def __init__(self, domain_list, source_type):
|
||
"""
|
||
初始化导入线程
|
||
|
||
:param domain_list: 域名列表
|
||
:param source_type: 来源类型
|
||
"""
|
||
super().__init__()
|
||
self.domain_list = domain_list
|
||
self.source_type = source_type
|
||
|
||
def run(self):
|
||
"""
|
||
运行导入线程
|
||
"""
|
||
try:
|
||
collector = DomainCollector()
|
||
total = len(self.domain_list)
|
||
for i, domain in enumerate(self.domain_list):
|
||
collector.add_domain(domain, self.source_type)
|
||
progress = int((i + 1) / total * 100)
|
||
self.progress_updated.emit(progress)
|
||
self.finished.emit(True, f"成功导入 {total} 个域名")
|
||
except Exception as e:
|
||
logger.error(f"导入失败: {e}")
|
||
self.finished.emit(False, f"导入失败: {str(e)}") |