'.'
This commit is contained in:
1001
app/10.csv
Normal file
1001
app/10.csv
Normal file
File diff suppressed because it is too large
Load Diff
10001
app/100.csv
Normal file
10001
app/100.csv
Normal file
File diff suppressed because it is too large
Load Diff
200001
app/2000.csv
Normal file
200001
app/2000.csv
Normal file
File diff suppressed because it is too large
Load Diff
@@ -1,13 +1,13 @@
|
||||
{
|
||||
"detect_register": false,
|
||||
"detect_chinaz": false,
|
||||
"detect_aizhan": false,
|
||||
"detect_baidu_site": false,
|
||||
"detect_360_site": false,
|
||||
"detect_baidu_security": false,
|
||||
"detect_whois": false,
|
||||
"detect_register": true,
|
||||
"detect_chinaz": true,
|
||||
"detect_aizhan": true,
|
||||
"detect_baidu_site": true,
|
||||
"detect_360_site": true,
|
||||
"detect_baidu_security": true,
|
||||
"detect_whois": true,
|
||||
"detect_beian": true,
|
||||
"detect_intercept": false,
|
||||
"detect_juziseo": false,
|
||||
"detect_juziseo_outlink": false
|
||||
"detect_intercept": true,
|
||||
"detect_juziseo": true,
|
||||
"detect_juziseo_outlink": true
|
||||
}
|
||||
Binary file not shown.
Binary file not shown.
@@ -1,3 +1,3 @@
|
||||
{
|
||||
"thread_count": "1"
|
||||
"thread_count": "100"
|
||||
}
|
||||
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
Binary file not shown.
@@ -295,6 +295,13 @@ class QueryThread(QThread):
|
||||
domains = db.get_domains_with_details(self.conditions, page=self.page, page_size=self.page_size)
|
||||
# 获取符合条件的域名总数
|
||||
total_count = db.get_domains_count(self.conditions)
|
||||
|
||||
# 一次性获取所有域名的检测状态,避免在UI线程中执行数据库操作
|
||||
for domain in domains:
|
||||
domain_id = domain['id']
|
||||
detect_statuses = db.get_detection_statuses(domain_id)
|
||||
domain['detect_statuses'] = detect_statuses
|
||||
|
||||
self.finished.emit(domains, self.conditions, total_count)
|
||||
except Exception as e:
|
||||
self.error.emit(str(e))
|
||||
@@ -875,8 +882,8 @@ class DomainFilterWidget(QWidget):
|
||||
|
||||
# 创建表格
|
||||
self.table_widget = QTableWidget()
|
||||
self.table_widget.setColumnCount(18)
|
||||
self.table_widget.setHorizontalHeaderLabels(["域名", "注册状态", "使用状态", "检测状态", "复核状态", "域名过期时间", "单位性质", "首页地址", "检测时间", "备案历史", "备案年份", "快照年份", "百度历史收录状态", "百度site收录状态", "title是否有中文", "360site收录", "Google site收录状态", "友情链接数量"])
|
||||
self.table_widget.setColumnCount(23)
|
||||
self.table_widget.setHorizontalHeaderLabels(["域名", "注册状态", "使用状态", "检测状态", "复核状态", "域名过期时间", "单位性质", "首页地址", "检测时间", "备案历史", "备案年份", "快照年份", "百度历史收录状态", "百度site收录状态", "title是否有中文", "360site收录", "Google site收录状态", "友情链接数量", "WHOIS检测", "备案检测", "拦截检测", "桔子历史", "桔子外链"])
|
||||
# 设置表格水平滚动
|
||||
self.table_widget.setHorizontalScrollBarPolicy(Qt.ScrollBarAlwaysOn)
|
||||
# 设置列宽调整模式
|
||||
@@ -917,6 +924,7 @@ class DomainFilterWidget(QWidget):
|
||||
beian_year = self.beian_year_input.text().strip()
|
||||
snapshot_year = self.snapshot_year_input.text().strip()
|
||||
backlink = self.backlink_checkbox.isChecked()
|
||||
beian = self.beian_checkbox.isChecked()
|
||||
search_keyword = self.search_input.text().strip()
|
||||
page = self.page_spinbox.value()
|
||||
page_size = self.page_size_spinbox.value()
|
||||
@@ -968,6 +976,7 @@ class DomainFilterWidget(QWidget):
|
||||
'beian_year': beian_year,
|
||||
'snapshot_year': snapshot_year,
|
||||
'backlink_gt_10': backlink if backlink else None,
|
||||
'has_beian': 2 if beian else None,
|
||||
'search_keyword': search_keyword,
|
||||
'website_url': website_url if website_url else None
|
||||
}
|
||||
@@ -985,18 +994,15 @@ class DomainFilterWidget(QWidget):
|
||||
"""
|
||||
查询完成回调
|
||||
"""
|
||||
# 清空表格
|
||||
self.table_widget.setRowCount(0)
|
||||
|
||||
# 添加数据到表格
|
||||
for domain in domains:
|
||||
row = self.table_widget.rowCount()
|
||||
self.table_widget.insertRow(row)
|
||||
try:
|
||||
# 清空表格
|
||||
self.table_widget.setRowCount(0)
|
||||
|
||||
# 域名
|
||||
self.table_widget.setItem(row, 0, QTableWidgetItem(domain['domain']))
|
||||
# 预设置表格行数,减少UI更新次数
|
||||
num_rows = len(domains)
|
||||
self.table_widget.setRowCount(num_rows)
|
||||
|
||||
# 注册状态
|
||||
# 定义状态映射字典,避免在循环中重复创建
|
||||
register_status_mapping = {
|
||||
0: '待检测',
|
||||
1: '已注册',
|
||||
@@ -1005,20 +1011,14 @@ class DomainFilterWidget(QWidget):
|
||||
4: '赎回期',
|
||||
5: '删除期'
|
||||
}
|
||||
register_status = register_status_mapping.get(domain['register_status'], '未知')
|
||||
self.table_widget.setItem(row, 1, QTableWidgetItem(register_status))
|
||||
|
||||
# 使用状态
|
||||
use_status_mapping = {
|
||||
0: '未使用',
|
||||
1: '已经使用',
|
||||
2: '已经卖出',
|
||||
3: '已经预定'
|
||||
}
|
||||
use_status = use_status_mapping.get(domain['use_status'], '未知')
|
||||
self.table_widget.setItem(row, 2, QTableWidgetItem(use_status))
|
||||
|
||||
# 检测状态
|
||||
detect_status_mapping = {
|
||||
0: '待检测',
|
||||
1: '检测完成',
|
||||
@@ -1026,121 +1026,164 @@ class DomainFilterWidget(QWidget):
|
||||
3: '黑名单',
|
||||
4: '检测失败'
|
||||
}
|
||||
detect_status = detect_status_mapping.get(domain['detect_status'], '未知')
|
||||
self.table_widget.setItem(row, 3, QTableWidgetItem(detect_status))
|
||||
|
||||
# 复核状态
|
||||
review_status_mapping = {
|
||||
0: '无需复核',
|
||||
1: '待人工复核',
|
||||
2: '人工通过',
|
||||
3: '人工拒绝'
|
||||
}
|
||||
review_status = review_status_mapping.get(domain['review_status'], '未知')
|
||||
self.table_widget.setItem(row, 4, QTableWidgetItem(review_status))
|
||||
|
||||
# 域名过期时间
|
||||
expire_date = domain.get('expire_date', '')
|
||||
self.table_widget.setItem(row, 5, QTableWidgetItem(str(expire_date) if expire_date else ''))
|
||||
|
||||
# 单位性质
|
||||
company_type = domain.get('company_type', '')
|
||||
self.table_widget.setItem(row, 6, QTableWidgetItem(company_type))
|
||||
|
||||
# 首页地址
|
||||
home_url = domain.get('website_url', '')
|
||||
self.table_widget.setItem(row, 7, QTableWidgetItem(home_url))
|
||||
|
||||
# 检测时间
|
||||
detect_time = domain.get('detect_time', '')
|
||||
self.table_widget.setItem(row, 8, QTableWidgetItem(str(detect_time) if detect_time else ''))
|
||||
|
||||
# 备案历史
|
||||
has_beian_mapping = {
|
||||
1: '待检测',
|
||||
2: '有备案记录',
|
||||
3: '没有备案记录'
|
||||
}
|
||||
has_beian = has_beian_mapping.get(domain.get('has_beian'), '未知')
|
||||
self.table_widget.setItem(row, 9, QTableWidgetItem(has_beian))
|
||||
|
||||
# 备案年份
|
||||
beian_year = domain.get('beian_year', '')
|
||||
self.table_widget.setItem(row, 10, QTableWidgetItem(str(beian_year) if beian_year else ''))
|
||||
|
||||
# 快照年份
|
||||
snapshot_years = domain.get('snapshot_years', '')
|
||||
self.table_widget.setItem(row, 11, QTableWidgetItem(snapshot_years))
|
||||
|
||||
# 百度历史收录状态
|
||||
baidu_history = domain.get('baidu_history')
|
||||
if baidu_history is None:
|
||||
baidu_history = {}
|
||||
elif isinstance(baidu_history, str):
|
||||
import json
|
||||
try:
|
||||
baidu_history = json.loads(baidu_history)
|
||||
except:
|
||||
# 添加数据到表格
|
||||
for row, domain in enumerate(domains):
|
||||
# 域名
|
||||
self.table_widget.setItem(row, 0, QTableWidgetItem(domain['domain']))
|
||||
|
||||
# 注册状态
|
||||
register_status = register_status_mapping.get(domain['register_status'], '未知')
|
||||
self.table_widget.setItem(row, 1, QTableWidgetItem(register_status))
|
||||
|
||||
# 使用状态
|
||||
use_status = use_status_mapping.get(domain['use_status'], '未知')
|
||||
self.table_widget.setItem(row, 2, QTableWidgetItem(use_status))
|
||||
|
||||
# 检测状态
|
||||
detect_status = detect_status_mapping.get(domain['detect_status'], '未知')
|
||||
self.table_widget.setItem(row, 3, QTableWidgetItem(detect_status))
|
||||
|
||||
# 复核状态
|
||||
review_status = review_status_mapping.get(domain['review_status'], '未知')
|
||||
self.table_widget.setItem(row, 4, QTableWidgetItem(review_status))
|
||||
|
||||
# 域名过期时间
|
||||
expire_date = domain.get('expire_date', '')
|
||||
self.table_widget.setItem(row, 5, QTableWidgetItem(str(expire_date) if expire_date else ''))
|
||||
|
||||
# 单位性质
|
||||
company_type = domain.get('company_type', '')
|
||||
self.table_widget.setItem(row, 6, QTableWidgetItem(company_type))
|
||||
|
||||
# 首页地址
|
||||
home_url = domain.get('website_url', '')
|
||||
self.table_widget.setItem(row, 7, QTableWidgetItem(home_url))
|
||||
|
||||
# 检测时间
|
||||
detect_time = domain.get('detect_time', '')
|
||||
self.table_widget.setItem(row, 8, QTableWidgetItem(str(detect_time) if detect_time else ''))
|
||||
|
||||
# 备案历史
|
||||
has_beian = has_beian_mapping.get(domain.get('has_beian'), '未知')
|
||||
self.table_widget.setItem(row, 9, QTableWidgetItem(has_beian))
|
||||
|
||||
# 备案年份
|
||||
beian_year = domain.get('beian_year', '')
|
||||
self.table_widget.setItem(row, 10, QTableWidgetItem(str(beian_year) if beian_year else ''))
|
||||
|
||||
# 快照年份
|
||||
snapshot_years = domain.get('snapshot_years', '')
|
||||
self.table_widget.setItem(row, 11, QTableWidgetItem(snapshot_years))
|
||||
|
||||
# 百度历史收录状态
|
||||
baidu_history = domain.get('baidu_history')
|
||||
if baidu_history is None:
|
||||
baidu_history = {}
|
||||
baidu_history_status = '是' if baidu_history.get('status') else '否'
|
||||
self.table_widget.setItem(row, 12, QTableWidgetItem(baidu_history_status))
|
||||
|
||||
# 百度site收录状态
|
||||
baidu_site = domain.get('baidu_site')
|
||||
if baidu_site is None:
|
||||
baidu_site = {}
|
||||
elif isinstance(baidu_site, str):
|
||||
import json
|
||||
try:
|
||||
baidu_site = json.loads(baidu_site)
|
||||
except:
|
||||
elif isinstance(baidu_history, str):
|
||||
import json
|
||||
try:
|
||||
baidu_history = json.loads(baidu_history)
|
||||
except:
|
||||
baidu_history = {}
|
||||
baidu_history_status = '是' if baidu_history.get('status') else '否'
|
||||
self.table_widget.setItem(row, 12, QTableWidgetItem(baidu_history_status))
|
||||
|
||||
# 百度site收录状态
|
||||
baidu_site = domain.get('baidu_site')
|
||||
if baidu_site is None:
|
||||
baidu_site = {}
|
||||
baidu_site_status = '是' if baidu_site.get('status') else '否'
|
||||
self.table_widget.setItem(row, 13, QTableWidgetItem(baidu_site_status))
|
||||
|
||||
# title是否有中文
|
||||
is_chinese_title = domain.get('is_chinese_title', False)
|
||||
is_chinese_title_text = '是' if is_chinese_title else '否'
|
||||
self.table_widget.setItem(row, 14, QTableWidgetItem(is_chinese_title_text))
|
||||
|
||||
# 360site收录
|
||||
qihu360_site = domain.get('qihu360_site')
|
||||
if qihu360_site is None:
|
||||
qihu360_site = {}
|
||||
elif isinstance(qihu360_site, str):
|
||||
import json
|
||||
try:
|
||||
qihu360_site = json.loads(qihu360_site)
|
||||
except:
|
||||
elif isinstance(baidu_site, str):
|
||||
import json
|
||||
try:
|
||||
baidu_site = json.loads(baidu_site)
|
||||
except:
|
||||
baidu_site = {}
|
||||
baidu_site_status = '是' if baidu_site.get('status') else '否'
|
||||
self.table_widget.setItem(row, 13, QTableWidgetItem(baidu_site_status))
|
||||
|
||||
# title是否有中文
|
||||
is_chinese_title = domain.get('is_chinese_title', False)
|
||||
is_chinese_title_text = '是' if is_chinese_title else '否'
|
||||
self.table_widget.setItem(row, 14, QTableWidgetItem(is_chinese_title_text))
|
||||
|
||||
# 360site收录
|
||||
qihu360_site = domain.get('qihu360_site')
|
||||
if qihu360_site is None:
|
||||
qihu360_site = {}
|
||||
qihu360_site_status = '是' if qihu360_site.get('status') else '否'
|
||||
self.table_widget.setItem(row, 15, QTableWidgetItem(qihu360_site_status))
|
||||
|
||||
# Google site收录状态
|
||||
google_site = domain.get('google_site')
|
||||
if google_site is None:
|
||||
google_site = {}
|
||||
elif isinstance(google_site, str):
|
||||
import json
|
||||
try:
|
||||
google_site = json.loads(google_site)
|
||||
except:
|
||||
elif isinstance(qihu360_site, str):
|
||||
import json
|
||||
try:
|
||||
qihu360_site = json.loads(qihu360_site)
|
||||
except:
|
||||
qihu360_site = {}
|
||||
qihu360_site_status = '是' if qihu360_site.get('status') else '否'
|
||||
self.table_widget.setItem(row, 15, QTableWidgetItem(qihu360_site_status))
|
||||
|
||||
# Google site收录状态
|
||||
google_site = domain.get('google_site')
|
||||
if google_site is None:
|
||||
google_site = {}
|
||||
google_site_status = '是' if google_site.get('status') else '否'
|
||||
self.table_widget.setItem(row, 16, QTableWidgetItem(google_site_status))
|
||||
elif isinstance(google_site, str):
|
||||
import json
|
||||
try:
|
||||
google_site = json.loads(google_site)
|
||||
except:
|
||||
google_site = {}
|
||||
google_site_status = '是' if google_site.get('status') else '否'
|
||||
self.table_widget.setItem(row, 16, QTableWidgetItem(google_site_status))
|
||||
|
||||
# 友情链接数量
|
||||
backlink_count = domain.get('backlink_count', 0)
|
||||
self.table_widget.setItem(row, 17, QTableWidgetItem(str(backlink_count)))
|
||||
|
||||
# 检测状态
|
||||
# 使用在后台线程中获取的检测状态
|
||||
detect_statuses = domain.get('detect_statuses', {})
|
||||
|
||||
# WHOIS检测状态
|
||||
whois_status = '已检测' if detect_statuses.get('whois_status') == 1 else '待检测'
|
||||
self.table_widget.setItem(row, 18, QTableWidgetItem(whois_status))
|
||||
|
||||
# 备案检测状态
|
||||
beian_status = '已检测' if detect_statuses.get('beian_status') == 1 else '待检测'
|
||||
self.table_widget.setItem(row, 19, QTableWidgetItem(beian_status))
|
||||
|
||||
# 拦截检测状态
|
||||
intercept_status = '已检测' if detect_statuses.get('intercept_status') == 1 else '待检测'
|
||||
self.table_widget.setItem(row, 20, QTableWidgetItem(intercept_status))
|
||||
|
||||
# 桔子历史检测状态
|
||||
juziseo_history_status = '已检测' if detect_statuses.get('juziseo_history_status') == 1 else '待检测'
|
||||
self.table_widget.setItem(row, 21, QTableWidgetItem(juziseo_history_status))
|
||||
|
||||
# 桔子外链检测状态
|
||||
juziseo_outlink_status = '已检测' if detect_statuses.get('juziseo_outlink_status') == 1 else '待检测'
|
||||
self.table_widget.setItem(row, 22, QTableWidgetItem(juziseo_outlink_status))
|
||||
|
||||
# 友情链接数量
|
||||
backlink_count = domain.get('backlink_count', 0)
|
||||
self.table_widget.setItem(row, 17, QTableWidgetItem(str(backlink_count)))
|
||||
|
||||
# 显示查询结果,包括总数、总页数和当前页码
|
||||
current_page_count = len(domains)
|
||||
page_size = self.page_size_spinbox.value()
|
||||
total_pages = (total_count + page_size - 1) // page_size # 向上取整计算总页数
|
||||
current_page = self.page_spinbox.value()
|
||||
self.status_label.setText(f"查询完成,共 {total_count} 条记录,当前页显示 {current_page_count} 条,共 {total_pages} 页,当前第 {current_page} 页")
|
||||
logger.info(f"查询完成,共 {total_count} 条记录,当前页显示 {current_page_count} 条,共 {total_pages} 页,当前第 {current_page} 页")
|
||||
|
||||
# 显示查询结果,包括总数、总页数和当前页码
|
||||
current_page_count = len(domains)
|
||||
page_size = self.page_size_spinbox.value()
|
||||
total_pages = (total_count + page_size - 1) // page_size # 向上取整计算总页数
|
||||
current_page = self.page_spinbox.value()
|
||||
self.status_label.setText(f"查询完成,共 {total_count} 条记录,当前页显示 {current_page_count} 条,共 {total_pages} 页,当前第 {current_page} 页")
|
||||
logger.info(f"查询完成,共 {total_count} 条记录,当前页显示 {current_page_count} 条,共 {total_pages} 页,当前第 {current_page} 页")
|
||||
except Exception as e:
|
||||
self.status_label.setText(f"处理查询结果时出错: {str(e)}")
|
||||
logger.error(f"处理查询结果时出错: {e}")
|
||||
def on_query_error(self, error):
|
||||
"""
|
||||
查询错误回调
|
||||
@@ -1152,56 +1195,288 @@ class DomainFilterWidget(QWidget):
|
||||
"""
|
||||
导出域名
|
||||
"""
|
||||
# 获取当前表格中的域名数据
|
||||
domains = []
|
||||
for row in range(self.table_widget.rowCount()):
|
||||
domain = {
|
||||
'domain': self.table_widget.item(row, 0).text(),
|
||||
'register_status': self.table_widget.item(row, 1).text(),
|
||||
'use_status': self.table_widget.item(row, 2).text(),
|
||||
'detect_status': self.table_widget.item(row, 3).text(),
|
||||
'review_status': self.table_widget.item(row, 4).text(),
|
||||
'expire_date': self.table_widget.item(row, 5).text(),
|
||||
'company_type': self.table_widget.item(row, 6).text(),
|
||||
'website_url': self.table_widget.item(row, 7).text(),
|
||||
'detect_time': self.table_widget.item(row, 8).text(),
|
||||
'has_beian': self.table_widget.item(row, 9).text(),
|
||||
'beian_year': self.table_widget.item(row, 10).text(),
|
||||
'snapshot_years': self.table_widget.item(row, 11).text(),
|
||||
'baidu_history': self.table_widget.item(row, 12).text(),
|
||||
'baidu_site': self.table_widget.item(row, 13).text(),
|
||||
'is_chinese_title': self.table_widget.item(row, 14).text(),
|
||||
'qihu360_site': self.table_widget.item(row, 15).text(),
|
||||
'google_site': self.table_widget.item(row, 16).text(),
|
||||
'backlink_count': self.table_widget.item(row, 17).text()
|
||||
}
|
||||
domains.append(domain)
|
||||
from PySide6.QtWidgets import QMessageBox
|
||||
|
||||
if not domains:
|
||||
self.status_label.setText("没有数据可导出")
|
||||
# 询问用户导出范围
|
||||
msg_box = QMessageBox()
|
||||
msg_box.setWindowTitle("导出范围")
|
||||
msg_box.setText("请选择导出范围:")
|
||||
msg_box.addButton("只导出当前页", QMessageBox.AcceptRole)
|
||||
msg_box.addButton("导出所有页", QMessageBox.AcceptRole)
|
||||
msg_box.addButton("导出指定页码范围", QMessageBox.AcceptRole)
|
||||
msg_box.addButton("取消", QMessageBox.RejectRole)
|
||||
|
||||
result = msg_box.exec()
|
||||
|
||||
# 获取按钮文本
|
||||
button_text = msg_box.clickedButton().text() if msg_box.clickedButton() else "取消"
|
||||
|
||||
if button_text == "取消":
|
||||
return
|
||||
|
||||
start_page = 1
|
||||
end_page = 1
|
||||
|
||||
if button_text == "导出指定页码范围":
|
||||
# 创建页码范围输入对话框
|
||||
from PySide6.QtWidgets import QDialog, QVBoxLayout, QLabel, QLineEdit, QPushButton, QHBoxLayout
|
||||
dialog = QDialog(self)
|
||||
dialog.setWindowTitle("导出指定页码范围")
|
||||
dialog.resize(400, 150)
|
||||
dialog.setModal(True) # 设置为模态对话框,确保获得焦点
|
||||
|
||||
layout = QVBoxLayout(dialog)
|
||||
|
||||
# 页码范围输入
|
||||
range_label = QLabel("请输入页码范围(例如:2-6):")
|
||||
layout.addWidget(range_label)
|
||||
|
||||
page_range_input = QLineEdit()
|
||||
page_range_input.setPlaceholderText("例如:2-6")
|
||||
layout.addWidget(page_range_input)
|
||||
|
||||
# 按钮布局
|
||||
button_layout = QHBoxLayout()
|
||||
|
||||
ok_button = QPushButton("确定")
|
||||
ok_button.clicked.connect(dialog.accept)
|
||||
button_layout.addWidget(ok_button)
|
||||
|
||||
cancel_button = QPushButton("取消")
|
||||
cancel_button.clicked.connect(dialog.reject)
|
||||
button_layout.addWidget(cancel_button)
|
||||
|
||||
layout.addLayout(button_layout)
|
||||
|
||||
# 显示对话框
|
||||
if dialog.exec() != QDialog.Accepted:
|
||||
return
|
||||
|
||||
# 获取页码范围
|
||||
page_range_text = page_range_input.text().strip()
|
||||
if not page_range_text:
|
||||
self.status_label.setText("页码范围不能为空")
|
||||
return
|
||||
|
||||
# 解析页码范围
|
||||
try:
|
||||
start_page, end_page = map(int, page_range_text.split('-'))
|
||||
if start_page < 1 or end_page < start_page:
|
||||
self.status_label.setText("页码范围格式错误")
|
||||
return
|
||||
except:
|
||||
self.status_label.setText("页码范围格式错误,正确格式为:2-6")
|
||||
return
|
||||
|
||||
# 选择导出文件路径
|
||||
from PySide6.QtWidgets import QFileDialog
|
||||
file_path, _ = QFileDialog.getSaveFileName(self, "导出域名", "", "Excel文件 (*.xlsx);;CSV文件 (*.csv)")
|
||||
if not file_path:
|
||||
return
|
||||
|
||||
# 导出数据
|
||||
try:
|
||||
export_manager = ExportManager()
|
||||
if file_path.endswith('.xlsx'):
|
||||
export_manager.export_to_excel(domains, file_path)
|
||||
elif file_path.endswith('.csv'):
|
||||
export_manager.export_to_csv(domains, file_path)
|
||||
else:
|
||||
self.status_label.setText("不支持的文件格式")
|
||||
if button_text == "只导出当前页":
|
||||
# 从当前表格中获取域名数据
|
||||
domains = []
|
||||
for row in range(self.table_widget.rowCount()):
|
||||
domain = {
|
||||
'domain': self.table_widget.item(row, 0).text(),
|
||||
'register_status': self.table_widget.item(row, 1).text(),
|
||||
'use_status': self.table_widget.item(row, 2).text(),
|
||||
'detect_status': self.table_widget.item(row, 3).text(),
|
||||
'review_status': self.table_widget.item(row, 4).text(),
|
||||
'expire_date': self.table_widget.item(row, 5).text(),
|
||||
'company_type': self.table_widget.item(row, 6).text(),
|
||||
'website_url': self.table_widget.item(row, 7).text(),
|
||||
'detect_time': self.table_widget.item(row, 8).text(),
|
||||
'has_beian': self.table_widget.item(row, 9).text(),
|
||||
'beian_year': self.table_widget.item(row, 10).text(),
|
||||
'snapshot_years': self.table_widget.item(row, 11).text(),
|
||||
'baidu_history': self.table_widget.item(row, 12).text(),
|
||||
'baidu_site': self.table_widget.item(row, 13).text(),
|
||||
'is_chinese_title': self.table_widget.item(row, 14).text(),
|
||||
'qihu360_site': self.table_widget.item(row, 15).text(),
|
||||
'google_site': self.table_widget.item(row, 16).text(),
|
||||
'backlink_count': self.table_widget.item(row, 17).text()
|
||||
}
|
||||
domains.append(domain)
|
||||
|
||||
if not domains:
|
||||
self.status_label.setText("没有数据可导出")
|
||||
return
|
||||
|
||||
self.status_label.setText(f"导出成功: {file_path}")
|
||||
logger.info(f"导出成功: {file_path}")
|
||||
except Exception as e:
|
||||
self.status_label.setText(f"导出失败: {str(e)}")
|
||||
logger.error(f"导出失败: {e}")
|
||||
# 导出数据
|
||||
try:
|
||||
from app.core.export_manager import ExportManager
|
||||
export_manager = ExportManager()
|
||||
if file_path.endswith('.xlsx'):
|
||||
export_manager.export_to_excel(domains, file_path)
|
||||
elif file_path.endswith('.csv'):
|
||||
export_manager.export_to_csv(domains, file_path)
|
||||
else:
|
||||
self.status_label.setText("不支持的文件格式")
|
||||
return
|
||||
|
||||
self.status_label.setText(f"导出成功: {file_path}")
|
||||
logger.info(f"导出成功: {file_path}")
|
||||
except Exception as e:
|
||||
self.status_label.setText(f"导出失败: {str(e)}")
|
||||
logger.error(f"导出失败: {e}")
|
||||
elif button_text == "导出指定页码范围":
|
||||
# 获取筛选条件
|
||||
register_status_text = self.register_status_combo.currentText()
|
||||
use_status_text = self.use_status_combo.currentText()
|
||||
detect_status_text = self.detect_status_combo.currentText()
|
||||
review_status_text = self.review_status_combo.currentText()
|
||||
beian_year = self.beian_year_input.text().strip()
|
||||
snapshot_year = self.snapshot_year_input.text().strip()
|
||||
backlink = self.backlink_checkbox.isChecked()
|
||||
search_keyword = self.search_input.text().strip()
|
||||
page_size = self.page_size_spinbox.value()
|
||||
|
||||
# 转换状态值
|
||||
status_mappings = {
|
||||
'register_status': {
|
||||
'全部': None,
|
||||
'待检测': 0,
|
||||
'可注册': 2,
|
||||
'已注册': 1,
|
||||
'宽限期': 3,
|
||||
'赎回期': 4,
|
||||
'删除期': 5
|
||||
},
|
||||
'use_status': {
|
||||
'全部': None,
|
||||
'未使用': 0,
|
||||
'已经使用': 1,
|
||||
'已经卖出': 2,
|
||||
'已经预定': 3
|
||||
},
|
||||
'detect_status': {
|
||||
'全部': None,
|
||||
'待检测': 0,
|
||||
'检测完成': 1,
|
||||
'检测中': 2,
|
||||
'黑名单': 3,
|
||||
'检测失败': 4
|
||||
},
|
||||
'review_status': {
|
||||
'全部': None,
|
||||
'无需复核': 0,
|
||||
'待人工复核': 1,
|
||||
'人工通过': 2,
|
||||
'人工拒绝': 3
|
||||
}
|
||||
}
|
||||
|
||||
# 获取首页网址
|
||||
website_url = self.website_url_input.text().strip()
|
||||
|
||||
# 构建查询条件
|
||||
conditions = {
|
||||
'register_status': status_mappings['register_status'][register_status_text],
|
||||
'use_status': status_mappings['use_status'][use_status_text],
|
||||
'detect_status': status_mappings['detect_status'][detect_status_text],
|
||||
'review_status': status_mappings['review_status'][review_status_text],
|
||||
'beian_year': beian_year,
|
||||
'snapshot_year': snapshot_year,
|
||||
'backlink_gt_10': backlink if backlink else None,
|
||||
'search_keyword': search_keyword,
|
||||
'website_url': website_url if website_url else None
|
||||
}
|
||||
|
||||
# 显示导出中状态
|
||||
self.status_label.setText("正在导出,请稍候...")
|
||||
|
||||
# 创建并启动导出线程
|
||||
self.export_thread = ExportRangeThread(conditions, start_page, end_page, page_size, file_path)
|
||||
self.export_thread.finished.connect(self.on_export_finished)
|
||||
self.export_thread.error.connect(self.on_export_error)
|
||||
self.export_thread.start()
|
||||
else:
|
||||
# 导出所有页
|
||||
# 获取筛选条件
|
||||
register_status_text = self.register_status_combo.currentText()
|
||||
use_status_text = self.use_status_combo.currentText()
|
||||
detect_status_text = self.detect_status_combo.currentText()
|
||||
review_status_text = self.review_status_combo.currentText()
|
||||
beian_year = self.beian_year_input.text().strip()
|
||||
snapshot_year = self.snapshot_year_input.text().strip()
|
||||
backlink = self.backlink_checkbox.isChecked()
|
||||
search_keyword = self.search_input.text().strip()
|
||||
|
||||
# 转换状态值
|
||||
status_mappings = {
|
||||
'register_status': {
|
||||
'全部': None,
|
||||
'待检测': 0,
|
||||
'可注册': 2,
|
||||
'已注册': 1,
|
||||
'宽限期': 3,
|
||||
'赎回期': 4,
|
||||
'删除期': 5
|
||||
},
|
||||
'use_status': {
|
||||
'全部': None,
|
||||
'未使用': 0,
|
||||
'已经使用': 1,
|
||||
'已经卖出': 2,
|
||||
'已经预定': 3
|
||||
},
|
||||
'detect_status': {
|
||||
'全部': None,
|
||||
'待检测': 0,
|
||||
'检测完成': 1,
|
||||
'检测中': 2,
|
||||
'黑名单': 3,
|
||||
'检测失败': 4
|
||||
},
|
||||
'review_status': {
|
||||
'全部': None,
|
||||
'无需复核': 0,
|
||||
'待人工复核': 1,
|
||||
'人工通过': 2,
|
||||
'人工拒绝': 3
|
||||
}
|
||||
}
|
||||
|
||||
# 获取首页网址
|
||||
website_url = self.website_url_input.text().strip()
|
||||
|
||||
# 构建查询条件
|
||||
conditions = {
|
||||
'register_status': status_mappings['register_status'][register_status_text],
|
||||
'use_status': status_mappings['use_status'][use_status_text],
|
||||
'detect_status': status_mappings['detect_status'][detect_status_text],
|
||||
'review_status': status_mappings['review_status'][review_status_text],
|
||||
'beian_year': beian_year,
|
||||
'snapshot_year': snapshot_year,
|
||||
'backlink_gt_10': backlink if backlink else None,
|
||||
'search_keyword': search_keyword,
|
||||
'website_url': website_url if website_url else None
|
||||
}
|
||||
|
||||
# 显示导出中状态
|
||||
self.status_label.setText("正在导出,请稍候...")
|
||||
|
||||
# 创建并启动导出线程
|
||||
self.export_thread = ExportThread(conditions, file_path)
|
||||
self.export_thread.finished.connect(self.on_export_finished)
|
||||
self.export_thread.error.connect(self.on_export_error)
|
||||
self.export_thread.start()
|
||||
|
||||
def on_export_finished(self, file_path):
|
||||
"""
|
||||
导出完成回调
|
||||
"""
|
||||
self.status_label.setText(f"导出成功: {file_path}")
|
||||
logger.info(f"导出成功: {file_path}")
|
||||
|
||||
def on_export_error(self, error):
|
||||
"""
|
||||
导出错误回调
|
||||
"""
|
||||
self.status_label.setText(f"导出失败: {error}")
|
||||
logger.error(f"导出失败: {error}")
|
||||
|
||||
def update_status(self):
|
||||
"""
|
||||
@@ -1332,4 +1607,251 @@ class DomainFilterWidget(QWidget):
|
||||
# 显示更新结果
|
||||
self.status_label.setText(f"成功更新 {updated_count} 个域名的信息")
|
||||
logger.info(f"成功更新 {updated_count} 个域名的信息")
|
||||
logger.info(f"更新数据: {update_data}")
|
||||
logger.info(f"更新数据: {update_data}")
|
||||
|
||||
class ExportThread(QThread):
|
||||
"""
|
||||
导出线程
|
||||
"""
|
||||
finished = Signal(str)
|
||||
error = Signal(str)
|
||||
|
||||
def __init__(self, conditions, file_path):
|
||||
super().__init__()
|
||||
self.conditions = conditions
|
||||
self.file_path = file_path
|
||||
|
||||
def run(self):
|
||||
try:
|
||||
from app.core.export_manager import ExportManager
|
||||
|
||||
# 分批获取和导出域名
|
||||
db = Database()
|
||||
export_manager = ExportManager()
|
||||
|
||||
# 先获取符合条件的域名总数
|
||||
total_count = db.get_domains_count(self.conditions)
|
||||
if total_count == 0:
|
||||
self.error.emit("没有数据可导出")
|
||||
return
|
||||
|
||||
# 分批处理,每批1000个
|
||||
batch_size = 1000
|
||||
current_page = 1
|
||||
all_domains = []
|
||||
|
||||
while True:
|
||||
# 获取当前批次的域名
|
||||
domains = db.get_domains_with_details(self.conditions, page=current_page, page_size=batch_size)
|
||||
if not domains:
|
||||
break
|
||||
|
||||
# 转换域名数据格式
|
||||
for domain in domains:
|
||||
# 注册状态映射
|
||||
register_status_mapping = {
|
||||
0: '待检测',
|
||||
1: '已注册',
|
||||
2: '可注册',
|
||||
3: '宽限期',
|
||||
4: '赎回期',
|
||||
5: '删除期'
|
||||
}
|
||||
|
||||
# 使用状态映射
|
||||
use_status_mapping = {
|
||||
0: '未使用',
|
||||
1: '已经使用',
|
||||
2: '已经卖出',
|
||||
3: '已经预定'
|
||||
}
|
||||
|
||||
# 检测状态映射
|
||||
detect_status_mapping = {
|
||||
0: '待检测',
|
||||
1: '检测完成',
|
||||
2: '检测中',
|
||||
3: '黑名单',
|
||||
4: '检测失败'
|
||||
}
|
||||
|
||||
# 人工复核状态映射
|
||||
review_status_mapping = {
|
||||
0: '无需复核',
|
||||
1: '待人工复核',
|
||||
2: '人工通过',
|
||||
3: '人工拒绝'
|
||||
}
|
||||
|
||||
# 备案状态映射
|
||||
has_beian_mapping = {
|
||||
0: '待检测',
|
||||
1: '有备案记录',
|
||||
2: '没有备案记录'
|
||||
}
|
||||
|
||||
# 单位性质映射
|
||||
company_type_mapping = {
|
||||
0: '个人',
|
||||
1: '企业',
|
||||
2: '其他'
|
||||
}
|
||||
|
||||
# 构建导出数据
|
||||
export_domain = {
|
||||
'domain': domain.get('domain', ''),
|
||||
'register_status': register_status_mapping.get(domain.get('register_status'), '未知'),
|
||||
'use_status': use_status_mapping.get(domain.get('use_status'), '未知'),
|
||||
'detect_status': detect_status_mapping.get(domain.get('detect_status'), '未知'),
|
||||
'review_status': review_status_mapping.get(domain.get('review_status'), '未知'),
|
||||
'expire_date': domain.get('expire_date', ''),
|
||||
'company_type': company_type_mapping.get(domain.get('company_type'), '未知'),
|
||||
'website_url': domain.get('website_url', ''),
|
||||
'detect_time': domain.get('detect_time', ''),
|
||||
'has_beian': has_beian_mapping.get(domain.get('has_beian'), '未知'),
|
||||
'beian_year': domain.get('beian_year', ''),
|
||||
'snapshot_years': domain.get('snapshot_years', ''),
|
||||
'baidu_history': '是' if domain.get('baidu_history') else '否',
|
||||
'baidu_site': '是' if domain.get('baidu_site') else '否',
|
||||
'is_chinese_title': '是' if domain.get('is_chinese_title') else '否',
|
||||
'qihu360_site': '是' if domain.get('qihu360_site') else '否',
|
||||
'google_site': '是' if domain.get('google_site') else '否',
|
||||
'backlink_count': '>10' if domain.get('backlink_count_gt_10') else '<=10'
|
||||
}
|
||||
all_domains.append(export_domain)
|
||||
|
||||
# 处理下一批
|
||||
current_page += 1
|
||||
|
||||
# 导出数据
|
||||
if self.file_path.endswith('.xlsx'):
|
||||
export_manager.export_to_excel(all_domains, self.file_path)
|
||||
elif self.file_path.endswith('.csv'):
|
||||
export_manager.export_to_csv(all_domains, self.file_path)
|
||||
else:
|
||||
self.error.emit("不支持的文件格式")
|
||||
return
|
||||
|
||||
self.finished.emit(self.file_path)
|
||||
except Exception as e:
|
||||
self.error.emit(str(e))
|
||||
|
||||
class ExportRangeThread(QThread):
|
||||
"""
|
||||
导出指定页码范围线程
|
||||
"""
|
||||
finished = Signal(str)
|
||||
error = Signal(str)
|
||||
|
||||
def __init__(self, conditions, start_page, end_page, page_size, file_path):
|
||||
super().__init__()
|
||||
self.conditions = conditions
|
||||
self.start_page = start_page
|
||||
self.end_page = end_page
|
||||
self.page_size = page_size
|
||||
self.file_path = file_path
|
||||
|
||||
def run(self):
|
||||
try:
|
||||
from app.core.export_manager import ExportManager
|
||||
|
||||
# 获取指定页码范围的域名
|
||||
db = Database()
|
||||
export_domains = []
|
||||
|
||||
# 遍历指定的页码范围
|
||||
for page in range(self.start_page, self.end_page + 1):
|
||||
# 获取当前页的数据
|
||||
domains = db.get_domains_with_details(self.conditions, page=page, page_size=self.page_size)
|
||||
|
||||
# 转换域名数据格式并添加到导出列表
|
||||
for domain in domains:
|
||||
# 注册状态映射
|
||||
register_status_mapping = {
|
||||
0: '待检测',
|
||||
1: '已注册',
|
||||
2: '可注册',
|
||||
3: '宽限期',
|
||||
4: '赎回期',
|
||||
5: '删除期'
|
||||
}
|
||||
|
||||
# 使用状态映射
|
||||
use_status_mapping = {
|
||||
0: '未使用',
|
||||
1: '已经使用',
|
||||
2: '已经卖出',
|
||||
3: '已经预定'
|
||||
}
|
||||
|
||||
# 检测状态映射
|
||||
detect_status_mapping = {
|
||||
0: '待检测',
|
||||
1: '检测完成',
|
||||
2: '检测中',
|
||||
3: '黑名单',
|
||||
4: '检测失败'
|
||||
}
|
||||
|
||||
# 人工复核状态映射
|
||||
review_status_mapping = {
|
||||
0: '无需复核',
|
||||
1: '待人工复核',
|
||||
2: '人工通过',
|
||||
3: '人工拒绝'
|
||||
}
|
||||
|
||||
# 备案状态映射
|
||||
has_beian_mapping = {
|
||||
0: '待检测',
|
||||
1: '有备案记录',
|
||||
2: '没有备案记录'
|
||||
}
|
||||
|
||||
# 单位性质映射
|
||||
company_type_mapping = {
|
||||
0: '个人',
|
||||
1: '企业',
|
||||
2: '其他'
|
||||
}
|
||||
|
||||
# 构建导出数据
|
||||
export_domain = {
|
||||
'domain': domain.get('domain', ''),
|
||||
'register_status': register_status_mapping.get(domain.get('register_status'), '未知'),
|
||||
'use_status': use_status_mapping.get(domain.get('use_status'), '未知'),
|
||||
'detect_status': detect_status_mapping.get(domain.get('detect_status'), '未知'),
|
||||
'review_status': review_status_mapping.get(domain.get('review_status'), '未知'),
|
||||
'expire_date': domain.get('expire_date', ''),
|
||||
'company_type': company_type_mapping.get(domain.get('company_type'), '未知'),
|
||||
'website_url': domain.get('website_url', ''),
|
||||
'detect_time': domain.get('detect_time', ''),
|
||||
'has_beian': has_beian_mapping.get(domain.get('has_beian'), '未知'),
|
||||
'beian_year': domain.get('beian_year', ''),
|
||||
'snapshot_years': domain.get('snapshot_years', ''),
|
||||
'baidu_history': '是' if domain.get('baidu_history') else '否',
|
||||
'baidu_site': '是' if domain.get('baidu_site') else '否',
|
||||
'is_chinese_title': '是' if domain.get('is_chinese_title') else '否',
|
||||
'qihu360_site': '是' if domain.get('qihu360_site') else '否',
|
||||
'google_site': '是' if domain.get('google_site') else '否',
|
||||
'backlink_count': '>10' if domain.get('backlink_count_gt_10') else '<=10'
|
||||
}
|
||||
export_domains.append(export_domain)
|
||||
|
||||
if not export_domains:
|
||||
self.error.emit("没有数据可导出")
|
||||
return
|
||||
|
||||
# 导出数据
|
||||
export_manager = ExportManager()
|
||||
if self.file_path.endswith('.xlsx'):
|
||||
export_manager.export_to_excel(export_domains, self.file_path)
|
||||
elif self.file_path.endswith('.csv'):
|
||||
export_manager.export_to_csv(export_domains, self.file_path)
|
||||
else:
|
||||
self.error.emit("不支持的文件格式")
|
||||
return
|
||||
|
||||
self.finished.emit(self.file_path)
|
||||
except Exception as e:
|
||||
self.error.emit(str(e))
|
||||
@@ -246,6 +246,216 @@ class ImportFileThread(QThread):
|
||||
self.finished.emit(False, f"导入失败: {str(e)}")
|
||||
|
||||
|
||||
class ImportMultiFileThread(QThread):
|
||||
"""
|
||||
多文件导入线程,用于处理多个文件的导入
|
||||
"""
|
||||
progress_updated = Signal(int)
|
||||
finished = Signal(bool, str)
|
||||
|
||||
def __init__(self, file_paths, source_type):
|
||||
"""
|
||||
初始化多文件导入线程
|
||||
|
||||
:param file_paths: 文件路径列表
|
||||
:param source_type: 来源类型
|
||||
"""
|
||||
super().__init__()
|
||||
self.file_paths = file_paths
|
||||
self.source_type = source_type
|
||||
|
||||
def run(self):
|
||||
"""
|
||||
运行多文件导入线程
|
||||
"""
|
||||
try:
|
||||
collector = DomainCollector()
|
||||
|
||||
# 首先计算所有文件中的域名总数
|
||||
total = 0
|
||||
encodings = ['utf-8', 'utf-8-sig', 'gbk', 'gb2312', 'cp936', 'latin-1', 'ascii']
|
||||
|
||||
# 计算所有文件的总域名数
|
||||
for file_path in self.file_paths:
|
||||
file_total = 0
|
||||
# 尝试不同的编码格式计算域名数量
|
||||
for enc in encodings:
|
||||
try:
|
||||
with open(file_path, 'r', encoding=enc) as f:
|
||||
file_total = sum(1 for line in f if line.strip())
|
||||
break
|
||||
except UnicodeDecodeError:
|
||||
continue
|
||||
|
||||
if file_total == 0:
|
||||
# 尝试使用二进制模式读取
|
||||
try:
|
||||
import chardet
|
||||
with open(file_path, 'rb') as f:
|
||||
raw_data = f.read()
|
||||
result = chardet.detect(raw_data)
|
||||
encoding = result['encoding']
|
||||
if encoding:
|
||||
content = raw_data.decode(encoding)
|
||||
file_total = sum(1 for line in content.split('\n') if line.strip())
|
||||
else:
|
||||
# 最后尝试使用 replace 模式读取
|
||||
with open(file_path, 'r', encoding='utf-8', errors='replace') as f:
|
||||
file_total = sum(1 for line in f if line.strip())
|
||||
except Exception:
|
||||
with open(file_path, 'r', encoding='utf-8', errors='replace') as f:
|
||||
file_total = sum(1 for line in f if line.strip())
|
||||
|
||||
total += file_total
|
||||
|
||||
# 实时更新进度:开始
|
||||
self.progress_updated.emit(0)
|
||||
|
||||
# 读取并处理所有文件的域名
|
||||
normalized_domains = []
|
||||
processed = 0
|
||||
|
||||
# 处理每个文件
|
||||
for file_path in self.file_paths:
|
||||
# 尝试不同的编码读取文件
|
||||
for enc in encodings:
|
||||
try:
|
||||
with open(file_path, 'r', encoding=enc) as f:
|
||||
for line in f:
|
||||
domain = line.strip()
|
||||
if domain:
|
||||
from app.utils.domain_utils import normalize_domain
|
||||
import tldextract
|
||||
normalized = normalize_domain(domain)
|
||||
if normalized:
|
||||
# 提取顶级域名
|
||||
ext = tldextract.extract(normalized)
|
||||
tld = ext.suffix
|
||||
normalized_domains.append((normalized, tld))
|
||||
|
||||
processed += 1
|
||||
# 更新进度(占30%)
|
||||
progress = int(processed / total * 30)
|
||||
self.progress_updated.emit(progress)
|
||||
break
|
||||
except UnicodeDecodeError:
|
||||
continue
|
||||
|
||||
# 如果仍然失败,尝试使用二进制模式读取
|
||||
if processed == 0 or (processed > 0 and not normalized_domains):
|
||||
try:
|
||||
import chardet
|
||||
with open(file_path, 'rb') as f:
|
||||
raw_data = f.read()
|
||||
result = chardet.detect(raw_data)
|
||||
encoding = result['encoding']
|
||||
if encoding:
|
||||
content = raw_data.decode(encoding)
|
||||
lines = content.split('\n')
|
||||
for line in lines:
|
||||
domain = line.strip()
|
||||
if domain:
|
||||
from app.utils.domain_utils import normalize_domain
|
||||
import tldextract
|
||||
normalized = normalize_domain(domain)
|
||||
if normalized:
|
||||
# 提取顶级域名
|
||||
ext = tldextract.extract(normalized)
|
||||
tld = ext.suffix
|
||||
normalized_domains.append((normalized, tld))
|
||||
|
||||
processed += 1
|
||||
# 更新进度(占30%)
|
||||
progress = int(processed / total * 30)
|
||||
self.progress_updated.emit(progress)
|
||||
else:
|
||||
# 最后尝试使用 replace 模式读取
|
||||
with open(file_path, 'r', encoding='utf-8', errors='replace') as f:
|
||||
for line in f:
|
||||
domain = line.strip()
|
||||
if domain:
|
||||
from app.utils.domain_utils import normalize_domain
|
||||
import tldextract
|
||||
normalized = normalize_domain(domain)
|
||||
if normalized:
|
||||
# 提取顶级域名
|
||||
ext = tldextract.extract(normalized)
|
||||
tld = ext.suffix
|
||||
normalized_domains.append((normalized, tld))
|
||||
|
||||
processed += 1
|
||||
# 更新进度(占30%)
|
||||
progress = int(processed / total * 30)
|
||||
self.progress_updated.emit(progress)
|
||||
except Exception:
|
||||
with open(file_path, 'r', encoding='utf-8', errors='replace') as f:
|
||||
for line in f:
|
||||
domain = line.strip()
|
||||
if domain:
|
||||
from app.utils.domain_utils import normalize_domain
|
||||
import tldextract
|
||||
normalized = normalize_domain(domain)
|
||||
if normalized:
|
||||
# 提取顶级域名
|
||||
ext = tldextract.extract(normalized)
|
||||
tld = ext.suffix
|
||||
normalized_domains.append((normalized, tld))
|
||||
|
||||
processed += 1
|
||||
# 更新进度(占30%)
|
||||
progress = int(processed / total * 30)
|
||||
self.progress_updated.emit(progress)
|
||||
|
||||
# 批量检查域名是否存在
|
||||
batch_data = []
|
||||
existing_domains = []
|
||||
if normalized_domains:
|
||||
all_domains = [domain for domain, tld in normalized_domains]
|
||||
existing_domains = collector.db.check_domains_exist(all_domains)
|
||||
existing_set = set(existing_domains)
|
||||
|
||||
# 准备批量添加数据
|
||||
for domain, tld in normalized_domains:
|
||||
if domain not in existing_set:
|
||||
batch_data.append((domain, tld, self.source_type))
|
||||
|
||||
# 分批次添加域名(占70%进度)
|
||||
batch_size = 1000
|
||||
total_batches = len(batch_data)
|
||||
for i in range(0, len(batch_data), batch_size):
|
||||
batch = batch_data[i:i+batch_size]
|
||||
collector.db.add_domains_batch(batch)
|
||||
|
||||
# 更新进度
|
||||
processed_batches = min(i + len(batch), total_batches)
|
||||
progress = 30 + int(processed_batches / total_batches * 70)
|
||||
self.progress_updated.emit(progress)
|
||||
|
||||
# 完成导入
|
||||
self.progress_updated.emit(100)
|
||||
|
||||
# 计算统计信息
|
||||
stats = {
|
||||
'total': total,
|
||||
'valid': len(normalized_domains),
|
||||
'added': len(batch_data),
|
||||
'exists': len(existing_domains),
|
||||
'invalid': total - len(normalized_domains),
|
||||
'failed': 0
|
||||
}
|
||||
|
||||
# 根据统计信息生成消息
|
||||
if stats['added'] > 0:
|
||||
message = f"导入完成: 总域名数 {stats['total']}, 有效域名数 {stats['valid']}, 新增域名数 {stats['added']}, 已存在域名数 {stats['exists']}, 无效域名数 {stats['invalid']}"
|
||||
else:
|
||||
message = f"导入完成: 所有域名已存在,未添加新域名"
|
||||
|
||||
self.finished.emit(True, message)
|
||||
except Exception as e:
|
||||
logger.error(f"导入失败: {e}")
|
||||
self.finished.emit(False, f"导入失败: {str(e)}")
|
||||
|
||||
|
||||
class DomainImportWidget(QWidget):
|
||||
"""
|
||||
域名导入界面
|
||||
@@ -355,60 +565,33 @@ class DomainImportWidget(QWidget):
|
||||
"""
|
||||
导入文件
|
||||
"""
|
||||
file_path, _ = QFileDialog.getOpenFileName(self, "选择文件", "", "文本文件 (*.txt)")
|
||||
if file_path:
|
||||
file_paths, _ = QFileDialog.getOpenFileNames(self, "选择文件", "", "文本文件 (*.txt)")
|
||||
if file_paths:
|
||||
try:
|
||||
# 尝试不同的编码格式
|
||||
encodings = ['utf-8', 'utf-8-sig', 'gbk', 'gb2312', 'cp936', 'latin-1', 'ascii']
|
||||
domain_count = 0
|
||||
total_domain_count = 0
|
||||
all_domains = []
|
||||
|
||||
# 尝试使用不同编码读取并计数
|
||||
for encoding in encodings:
|
||||
try:
|
||||
with open(file_path, 'r', encoding=encoding) as f:
|
||||
domain_count = sum(1 for line in f if line.strip())
|
||||
logger.info(f"使用编码 {encoding} 成功读取文件")
|
||||
break
|
||||
except UnicodeDecodeError:
|
||||
continue
|
||||
|
||||
# 如果仍然失败,尝试使用二进制模式读取并猜测编码
|
||||
if domain_count == 0:
|
||||
try:
|
||||
import chardet
|
||||
with open(file_path, 'rb') as f:
|
||||
raw_data = f.read()
|
||||
result = chardet.detect(raw_data)
|
||||
encoding = result['encoding']
|
||||
if encoding:
|
||||
domain_count = sum(1 for line in raw_data.decode(encoding).split('\n') if line.strip())
|
||||
logger.info(f"使用 chardet 检测到编码 {encoding} 并成功读取文件")
|
||||
else:
|
||||
raise Exception("无法识别文件编码")
|
||||
except Exception as e:
|
||||
logger.warning(f"chardet 检测失败: {e}")
|
||||
# 最后尝试使用 replace 模式读取
|
||||
with open(file_path, 'r', encoding='utf-8', errors='replace') as f:
|
||||
domain_count = sum(1 for line in f if line.strip())
|
||||
logger.info("使用 utf-8 replace 模式读取文件")
|
||||
|
||||
# 对于大文件,不显示所有域名,只显示文件路径和域名数量
|
||||
if domain_count > 1000:
|
||||
self.text_edit.setText(f"文件路径: {file_path}\n域名数量: {domain_count}\n\n提示: 由于文件较大,仅显示文件信息,不显示具体域名。")
|
||||
# 保存文件路径,用于后续导入
|
||||
self.imported_file_path = file_path
|
||||
else:
|
||||
# 对于小文件,显示所有域名
|
||||
domains = []
|
||||
# 处理每个选择的文件
|
||||
for file_path in file_paths:
|
||||
domain_count = 0
|
||||
file_domains = []
|
||||
|
||||
# 尝试使用不同编码读取并计数
|
||||
for encoding in encodings:
|
||||
try:
|
||||
with open(file_path, 'r', encoding=encoding) as f:
|
||||
domains = f.readlines()
|
||||
lines = f.readlines()
|
||||
domain_count = sum(1 for line in lines if line.strip())
|
||||
file_domains = [line.strip() for line in lines if line.strip()]
|
||||
logger.info(f"使用编码 {encoding} 成功读取文件: {file_path}")
|
||||
break
|
||||
except UnicodeDecodeError:
|
||||
continue
|
||||
if not domains:
|
||||
# 尝试使用二进制模式读取
|
||||
|
||||
# 如果仍然失败,尝试使用二进制模式读取并猜测编码
|
||||
if domain_count == 0:
|
||||
try:
|
||||
import chardet
|
||||
with open(file_path, 'rb') as f:
|
||||
@@ -416,20 +599,42 @@ class DomainImportWidget(QWidget):
|
||||
result = chardet.detect(raw_data)
|
||||
encoding = result['encoding']
|
||||
if encoding:
|
||||
domains = raw_data.decode(encoding).split('\n')
|
||||
content = raw_data.decode(encoding)
|
||||
lines = content.split('\n')
|
||||
domain_count = sum(1 for line in lines if line.strip())
|
||||
file_domains = [line.strip() for line in lines if line.strip()]
|
||||
logger.info(f"使用 chardet 检测到编码 {encoding} 并成功读取文件: {file_path}")
|
||||
else:
|
||||
with open(file_path, 'r', encoding='utf-8', errors='replace') as f:
|
||||
domains = f.readlines()
|
||||
except Exception:
|
||||
raise Exception("无法识别文件编码")
|
||||
except Exception as e:
|
||||
logger.warning(f"chardet 检测失败: {e}")
|
||||
# 最后尝试使用 replace 模式读取
|
||||
with open(file_path, 'r', encoding='utf-8', errors='replace') as f:
|
||||
domains = f.readlines()
|
||||
domains = [domain.strip() for domain in domains if domain.strip()]
|
||||
self.text_edit.setText('\n'.join(domains))
|
||||
lines = f.readlines()
|
||||
domain_count = sum(1 for line in lines if line.strip())
|
||||
file_domains = [line.strip() for line in lines if line.strip()]
|
||||
logger.info(f"使用 utf-8 replace 模式读取文件: {file_path}")
|
||||
|
||||
total_domain_count += domain_count
|
||||
all_domains.extend(file_domains)
|
||||
|
||||
# 检查总域名数量
|
||||
if total_domain_count > 1000:
|
||||
# 对于大文件,不显示所有域名,只显示文件路径和域名数量
|
||||
file_info = "\n".join([f"- {file_path}" for file_path in file_paths])
|
||||
self.text_edit.setText(f"文件路径:\n{file_info}\n\n总域名数量: {total_domain_count}\n\n提示: 由于文件较大,仅显示文件信息,不显示具体域名。")
|
||||
# 保存文件路径列表,用于后续导入
|
||||
self.imported_file_paths = file_paths
|
||||
self.imported_file_path = None # 清除单个文件路径
|
||||
else:
|
||||
# 对于小文件,显示所有域名
|
||||
self.text_edit.setText('\n'.join(all_domains))
|
||||
# 清除文件路径,使用文本框中的域名
|
||||
self.imported_file_paths = None
|
||||
self.imported_file_path = None
|
||||
|
||||
self.status_label.setText(f"成功读取 {domain_count} 个域名")
|
||||
logger.info(f"成功读取文件: {file_path}, 共 {domain_count} 个域名")
|
||||
self.status_label.setText(f"成功读取 {len(file_paths)} 个文件,共 {total_domain_count} 个域名")
|
||||
logger.info(f"成功读取 {len(file_paths)} 个文件,共 {total_domain_count} 个域名")
|
||||
except Exception as e:
|
||||
self.status_label.setText(f"读取文件失败: {str(e)}")
|
||||
logger.error(f"读取文件失败: {e}")
|
||||
@@ -438,8 +643,29 @@ class DomainImportWidget(QWidget):
|
||||
"""
|
||||
开始导入
|
||||
"""
|
||||
# 检查是否有导入的文件路径
|
||||
if hasattr(self, 'imported_file_path') and self.imported_file_path:
|
||||
# 检查是否有导入的文件路径列表(多个文件)
|
||||
if hasattr(self, 'imported_file_paths') and self.imported_file_paths:
|
||||
# 多文件导入,使用文件路径列表
|
||||
file_paths = self.imported_file_paths
|
||||
|
||||
# 显示进度条
|
||||
self.progress_bar.setVisible(True)
|
||||
self.progress_bar.setValue(0)
|
||||
self.status_label.setText("正在导入...")
|
||||
|
||||
# 禁用按钮
|
||||
self.import_file_btn.setEnabled(False)
|
||||
self.start_import_btn.setEnabled(False)
|
||||
|
||||
# 创建并启动多文件导入线程
|
||||
self.import_thread = ImportMultiFileThread(file_paths, 7) # 7 表示 TXT 导入
|
||||
self.import_thread.progress_updated.connect(self.update_progress)
|
||||
self.import_thread.finished.connect(self.import_finished)
|
||||
self.import_thread.start()
|
||||
|
||||
logger.info(f"开始从 {len(file_paths)} 个文件导入")
|
||||
# 检查是否有导入的单个文件路径
|
||||
elif hasattr(self, 'imported_file_path') and self.imported_file_path:
|
||||
# 大文件导入,使用文件路径
|
||||
file_path = self.imported_file_path
|
||||
|
||||
|
||||
@@ -279,11 +279,7 @@ class JumingCrawlerWidget(QWidget):
|
||||
self.date_edit.setDate(QDate.currentDate())
|
||||
self.date_edit.setCalendarPopup(True)
|
||||
self.date_edit.setStyleSheet("font-size: 14px; padding: 5px; border: 1px solid #ddd; border-radius: 4px; width: 150px;")
|
||||
# 设置最大日期为今天+4天,最小日期为今天的前4天
|
||||
max_date = QDate.currentDate().addDays(4)
|
||||
min_date = QDate.currentDate().addDays(-4)
|
||||
self.date_edit.setMinimumDate(min_date)
|
||||
self.date_edit.setMaximumDate(max_date)
|
||||
# 移除日期选择限制,允许选择任意时间
|
||||
date_layout.addWidget(date_label)
|
||||
date_layout.addWidget(self.date_edit)
|
||||
|
||||
|
||||
Binary file not shown.
Binary file not shown.
@@ -791,13 +791,135 @@ class Database:
|
||||
sql = "UPDATE domains SET expire_date = %s WHERE id = %s"
|
||||
return self.execute(sql, (expire_date, domain_id))
|
||||
|
||||
def get_domains_to_detect(self, limit=1000):
|
||||
def update_domain_detect_time(self, domain_id, detect_time):
|
||||
"""
|
||||
更新域名检测时间
|
||||
|
||||
:param domain_id: 域名ID
|
||||
:param detect_time: 检测时间
|
||||
:return: bool - 是否更新成功
|
||||
"""
|
||||
sql = "UPDATE domains SET detect_time = %s WHERE id = %s"
|
||||
return self.execute(sql, (detect_time, domain_id))
|
||||
|
||||
def update_detection_status(self, domain_id, status_type, status):
|
||||
"""
|
||||
更新检测状态
|
||||
|
||||
:param domain_id: 域名ID
|
||||
:param status_type: 状态类型 (whois, beian, intercept, juziseo_history, juziseo_outlink)
|
||||
:param status: 状态值 (0: 待检测, 1: 已检测)
|
||||
:return: bool - 是否更新成功
|
||||
"""
|
||||
status_field_map = {
|
||||
'whois': 'whois_status',
|
||||
'beian': 'beian_status',
|
||||
'intercept': 'intercept_status',
|
||||
'juziseo_history': 'juziseo_history_status',
|
||||
'juziseo_outlink': 'juziseo_outlink_status'
|
||||
}
|
||||
|
||||
if status_type not in status_field_map:
|
||||
return False
|
||||
|
||||
# 先检查domain_detections表中是否存在对应记录
|
||||
check_sql = "SELECT id FROM domain_detections WHERE domain_id = %s"
|
||||
existing_record = self.fetch_one(check_sql, (domain_id,))
|
||||
|
||||
if not existing_record:
|
||||
# 如果不存在,先创建一条记录
|
||||
insert_sql = "INSERT INTO domain_detections (domain_id, create_time, update_time) VALUES (%s, NOW(), NOW())"
|
||||
if not self.execute(insert_sql, (domain_id,)):
|
||||
return False
|
||||
|
||||
# 更新状态
|
||||
field_name = status_field_map[status_type]
|
||||
sql = f"UPDATE domain_detections SET {field_name} = %s, update_time = NOW() WHERE domain_id = %s"
|
||||
return self.execute(sql, (status, domain_id))
|
||||
|
||||
def get_detection_statuses(self, domain_id):
|
||||
"""
|
||||
获取域名的检测状态
|
||||
|
||||
:param domain_id: 域名ID
|
||||
:return: dict - 检测状态字典
|
||||
"""
|
||||
try:
|
||||
sql = "SELECT whois_status, beian_status, intercept_status, juziseo_history_status, juziseo_outlink_status FROM domain_detections WHERE domain_id = %s"
|
||||
result = self.fetch_one(sql, (domain_id,))
|
||||
|
||||
if result:
|
||||
return {
|
||||
'whois_status': result.get('whois_status', 0) if result.get('whois_status') is not None else 0,
|
||||
'beian_status': result.get('beian_status', 0) if result.get('beian_status') is not None else 0,
|
||||
'intercept_status': result.get('intercept_status', 0) if result.get('intercept_status') is not None else 0,
|
||||
'juziseo_history_status': result.get('juziseo_history_status', 0) if result.get('juziseo_history_status') is not None else 0,
|
||||
'juziseo_outlink_status': result.get('juziseo_outlink_status', 0) if result.get('juziseo_outlink_status') is not None else 0
|
||||
}
|
||||
else:
|
||||
return {
|
||||
'whois_status': 0,
|
||||
'beian_status': 0,
|
||||
'intercept_status': 0,
|
||||
'juziseo_history_status': 0,
|
||||
'juziseo_outlink_status': 0
|
||||
}
|
||||
except Exception as e:
|
||||
logger.error(f"获取检测状态失败: {e}")
|
||||
return {
|
||||
'whois_status': 0,
|
||||
'beian_status': 0,
|
||||
'intercept_status': 0,
|
||||
'juziseo_history_status': 0,
|
||||
'juziseo_outlink_status': 0
|
||||
}
|
||||
|
||||
def get_domains_to_detect(self, limit=1000, detect_options=None):
|
||||
"""
|
||||
获取需要检测的域名
|
||||
|
||||
:param limit: 限制数量
|
||||
:param detect_options: 检测选项字典,如 {'detect_whois': True, 'detect_beian': True, ...}
|
||||
:return: list - 域名列表
|
||||
"""
|
||||
if detect_options:
|
||||
# 构建检测状态条件
|
||||
status_conditions = []
|
||||
params = []
|
||||
|
||||
# 检查每个检测选项
|
||||
if detect_options.get('detect_whois', False):
|
||||
status_conditions.append("(dd.whois_status = 0 OR dd.whois_status IS NULL)")
|
||||
if detect_options.get('detect_beian', False):
|
||||
status_conditions.append("(dd.beian_status = 0 OR dd.beian_status IS NULL)")
|
||||
if detect_options.get('detect_intercept', False):
|
||||
status_conditions.append("(dd.intercept_status = 0 OR dd.intercept_status IS NULL)")
|
||||
if detect_options.get('detect_juziseo', False):
|
||||
status_conditions.append("(dd.juziseo_history_status = 0 OR dd.juziseo_history_status IS NULL)")
|
||||
if detect_options.get('detect_juziseo_outlink', False):
|
||||
status_conditions.append("(dd.juziseo_outlink_status = 0 OR dd.juziseo_outlink_status IS NULL)")
|
||||
|
||||
# 构建SQL查询
|
||||
if status_conditions:
|
||||
status_condition = " OR ".join(status_conditions)
|
||||
sql = f"""
|
||||
SELECT DISTINCT d.id, d.domain, d.source_type FROM domains d
|
||||
LEFT JOIN domain_detections dd ON d.id = dd.domain_id
|
||||
WHERE d.detect_status != 3 AND (d.detect_status = 0 OR ({status_condition}))
|
||||
ORDER BY d.id ASC
|
||||
LIMIT %s
|
||||
"""
|
||||
else:
|
||||
# 如果没有检测选项,只获取detect_status=0的记录
|
||||
sql = """
|
||||
SELECT id, domain, source_type FROM domains
|
||||
WHERE detect_status = 0
|
||||
ORDER BY id ASC
|
||||
LIMIT %s
|
||||
"""
|
||||
return self.fetch_all(sql, (limit,))
|
||||
|
||||
# 默认查询逻辑
|
||||
sql = """
|
||||
SELECT id, domain, source_type FROM domains
|
||||
WHERE detect_status IN (0, 3) OR (use_status = 0 AND detect_status = 1 AND register_status = 3 AND expire_date < CURRENT_DATE)
|
||||
@@ -1147,6 +1269,10 @@ class Database:
|
||||
sql += " AND d.review_status = %s"
|
||||
params.append(conditions['review_status'])
|
||||
|
||||
if 'has_beian' in conditions and conditions['has_beian'] is not None:
|
||||
sql += " AND d.has_beian = %s"
|
||||
params.append(conditions['has_beian'])
|
||||
|
||||
# 其他条件保持不变
|
||||
if conditions.get('beian_year'):
|
||||
sql += " AND d.beian_year = %s"
|
||||
@@ -1212,6 +1338,10 @@ class Database:
|
||||
sql += " AND d.review_status = %s"
|
||||
params.append(conditions['review_status'])
|
||||
|
||||
if 'has_beian' in conditions and conditions['has_beian'] is not None:
|
||||
sql += " AND d.has_beian = %s"
|
||||
params.append(conditions['has_beian'])
|
||||
|
||||
# 其他条件保持不变
|
||||
if conditions.get('beian_year'):
|
||||
sql += " AND d.beian_year = %s"
|
||||
|
||||
Reference in New Issue
Block a user