convert domainCheck to regular directory
This commit is contained in:
6
domainCheck/app/__init__.py
Normal file
6
domainCheck/app/__init__.py
Normal file
@@ -0,0 +1,6 @@
|
||||
# -*- coding: UTF-8 -*-
|
||||
'''
|
||||
域名库系统
|
||||
'''
|
||||
|
||||
__version__ = "1.0.0"
|
||||
1257
domainCheck/app/chinaz.js
Normal file
1257
domainCheck/app/chinaz.js
Normal file
File diff suppressed because one or more lines are too long
265
domainCheck/app/config.py
Normal file
265
domainCheck/app/config.py
Normal file
@@ -0,0 +1,265 @@
|
||||
# -*- coding: UTF-8 -*-
|
||||
'''
|
||||
@Project :domainScanDemo
|
||||
@File :config.py
|
||||
@IDE :PyCharm
|
||||
@Author :梦伴
|
||||
@Date :2026/4/9 0:08
|
||||
@explain : 系统配置
|
||||
'''
|
||||
|
||||
import os
|
||||
import sys
|
||||
from dotenv import load_dotenv
|
||||
|
||||
|
||||
def _safe_echo(message):
|
||||
"""
|
||||
在不同 Windows 控制台编码下安全输出文本,避免导入阶段因中文打印失败。
|
||||
"""
|
||||
try:
|
||||
print(message)
|
||||
except UnicodeEncodeError:
|
||||
try:
|
||||
encoding = sys.stdout.encoding or 'utf-8'
|
||||
sys.stdout.buffer.write((message + '\n').encode(encoding, errors='replace'))
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
# 确定基础目录
|
||||
if hasattr(sys, '_MEIPASS'):
|
||||
# PyInstaller 打包后的临时目录
|
||||
BASE_DIR = sys._MEIPASS
|
||||
else:
|
||||
# 开发环境目录
|
||||
BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
|
||||
# 加载环境变量
|
||||
env_path = os.path.join(BASE_DIR, '.env')
|
||||
if os.path.exists(env_path):
|
||||
load_dotenv(env_path)
|
||||
_safe_echo(f"成功加载环境变量文件: {env_path}")
|
||||
else:
|
||||
load_dotenv()
|
||||
_safe_echo(f"环境变量文件不存在: {env_path},使用默认环境变量")
|
||||
|
||||
|
||||
class Config:
|
||||
"""
|
||||
系统配置
|
||||
"""
|
||||
|
||||
# 数据库配置
|
||||
DB_HOST = os.getenv('DB_HOST', 'localhost')
|
||||
DB_PORT = int(os.getenv('DB_PORT', 5432))
|
||||
DB_DATABASE = os.getenv('DB_DATABASE', 'domain_scan_db')
|
||||
DB_USER = os.getenv('DB_USER', 'postgres')
|
||||
DB_PASSWORD = os.getenv('DB_PASSWORD', 'postgres')
|
||||
DB_POOL_SIZE = int(os.getenv('DB_POOL_SIZE', 5))
|
||||
|
||||
# 消息队列配置
|
||||
RABBITMQ_HOST = os.getenv('RABBITMQ_HOST', 'localhost')
|
||||
RABBITMQ_PORT = int(os.getenv('RABBITMQ_PORT', 5672))
|
||||
RABBITMQ_USER = os.getenv('RABBITMQ_USER', 'guest')
|
||||
RABBITMQ_PASSWORD = os.getenv('RABBITMQ_PASSWORD', 'guest')
|
||||
RABBITMQ_VHOST = os.getenv('RABBITMQ_VHOST', '/')
|
||||
|
||||
# Redis配置
|
||||
REDIS_HOST = os.getenv('REDIS_HOST', 'localhost')
|
||||
REDIS_PORT = int(os.getenv('REDIS_PORT', 6379))
|
||||
REDIS_PASSWORD = os.getenv('REDIS_PASSWORD', '')
|
||||
REDIS_DB = int(os.getenv('REDIS_DB', 0))
|
||||
|
||||
# 聚名网配置
|
||||
JUMING_COOKIE = os.getenv('JUMING_COOKIE', '')
|
||||
JUMING_REFERER = os.getenv('JUMING_REFERER', 'https://www.juming.com/')
|
||||
JUMING_USER_AGENT = os.getenv('JUMING_USER_AGENT', 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/146.0.0.0 Safari/537.36')
|
||||
|
||||
# 代理配置
|
||||
PROXY_ENABLED = os.getenv('PROXY_ENABLED', 'false').lower() == 'true'
|
||||
PROXY_URL = os.getenv('PROXY_URL', '')
|
||||
|
||||
# 检测配置
|
||||
DETECT_TIMEOUT = int(os.getenv('DETECT_TIMEOUT', 30))
|
||||
DETECT_RETRY_COUNT = int(os.getenv('DETECT_RETRY_COUNT', 3))
|
||||
DETECT_CONCURRENCY = int(os.getenv('DETECT_CONCURRENCY', 10))
|
||||
WAYBACK_CDX_TIMEOUT = int(os.getenv('WAYBACK_CDX_TIMEOUT', 15))
|
||||
WAYBACK_SNAPSHOT_TIMEOUT = int(os.getenv('WAYBACK_SNAPSHOT_TIMEOUT', 12))
|
||||
WAYBACK_RETRY_COUNT = int(os.getenv('WAYBACK_RETRY_COUNT', 2))
|
||||
WAYBACK_REQUEST_DELAY = float(os.getenv('WAYBACK_REQUEST_DELAY', 0))
|
||||
WAYBACK_PROGRESS_INTERVAL = int(os.getenv('WAYBACK_PROGRESS_INTERVAL', 500))
|
||||
WAYBACK_DOMAIN_CONCURRENCY = int(os.getenv('WAYBACK_DOMAIN_CONCURRENCY', 3))
|
||||
WAYBACK_TITLE_MAX_BYTES = int(os.getenv('WAYBACK_TITLE_MAX_BYTES', 65536))
|
||||
WAYBACK_TIMESTAMP_CACHE_TTL = int(os.getenv('WAYBACK_TIMESTAMP_CACHE_TTL', 86400))
|
||||
WAYBACK_TITLE_CACHE_TTL = int(os.getenv('WAYBACK_TITLE_CACHE_TTL', 2592000))
|
||||
WAYBACK_USER_AGENT = os.getenv(
|
||||
'WAYBACK_USER_AGENT',
|
||||
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/146.0.0.0 Safari/537.36'
|
||||
)
|
||||
|
||||
# 域名配置
|
||||
DOMAIN_TLDS = ['com', 'net']
|
||||
DOMAIN_BATCH_SIZE = int(os.getenv('DOMAIN_BATCH_SIZE', 1000))
|
||||
|
||||
# 日志配置
|
||||
LOG_LEVEL = os.getenv('LOG_LEVEL', 'INFO')
|
||||
LOG_FILE = os.getenv('LOG_FILE', 'app.log')
|
||||
|
||||
# 任务配置
|
||||
TASK_PRIORITY = {
|
||||
'rdap': 10,
|
||||
'wayback': 8,
|
||||
'baidu': 6,
|
||||
'qihu360': 5,
|
||||
'google': 5,
|
||||
'chinaz': 4,
|
||||
'aizhan': 4,
|
||||
'juziseo': 3,
|
||||
'jucha': 3
|
||||
}
|
||||
|
||||
# 敏感词配置 - 现在从数据库加载
|
||||
SENSITIVE_WORDS = []
|
||||
|
||||
# 检测项配置
|
||||
DETECT_ITEMS = {
|
||||
'rdap': True,
|
||||
'wayback': True,
|
||||
'baidu': True,
|
||||
'qihu360': True,
|
||||
'google': True,
|
||||
'chinaz': True,
|
||||
'aizhan': True,
|
||||
'juziseo': True,
|
||||
'jucha': True
|
||||
}
|
||||
|
||||
# 目录配置
|
||||
BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||||
DATA_DIR = os.path.join(BASE_DIR, 'data')
|
||||
LOG_DIR = os.path.join(BASE_DIR, 'logs')
|
||||
|
||||
# 确保目录存在
|
||||
os.makedirs(DATA_DIR, exist_ok=True)
|
||||
os.makedirs(LOG_DIR, exist_ok=True)
|
||||
|
||||
|
||||
# 从数据库加载敏感词
|
||||
def load_sensitive_words():
|
||||
"""
|
||||
从数据库加载敏感词
|
||||
"""
|
||||
try:
|
||||
from app.utils.database import Database
|
||||
db = Database()
|
||||
sensitive_words = db.get_sensitive_words()
|
||||
words = [word['word'] for word in sensitive_words]
|
||||
db.close()
|
||||
return words
|
||||
except Exception as e:
|
||||
_safe_echo(f"加载敏感词失败: {e}")
|
||||
return []
|
||||
|
||||
# 从文件加载检测选项
|
||||
def load_detect_options():
|
||||
"""
|
||||
从文件加载检测选项
|
||||
"""
|
||||
try:
|
||||
default_order = [
|
||||
'detect_register',
|
||||
'detect_baidu_site',
|
||||
'detect_360_site',
|
||||
'detect_chinaz',
|
||||
'detect_aizhan',
|
||||
'detect_wayback',
|
||||
'detect_jucha',
|
||||
'detect_juziseo',
|
||||
]
|
||||
import json
|
||||
import os
|
||||
defaults = {
|
||||
'detect_register': True,
|
||||
'detect_wayback': True,
|
||||
'detect_chinaz': True,
|
||||
'detect_aizhan': True,
|
||||
'detect_baidu_site': True,
|
||||
'detect_360_site': True,
|
||||
'detect_jucha': False,
|
||||
'detect_juziseo': False,
|
||||
'detect_order': default_order,
|
||||
}
|
||||
if os.path.exists('detect_options.json'):
|
||||
with open('detect_options.json', 'r', encoding='utf-8') as f:
|
||||
detect_options = json.load(f)
|
||||
defaults.update(detect_options)
|
||||
if defaults.get('detect_whois') or defaults.get('detect_beian') or defaults.get('detect_intercept'):
|
||||
defaults['detect_jucha'] = True
|
||||
if defaults.get('detect_juziseo_outlink'):
|
||||
defaults['detect_juziseo'] = True
|
||||
order = defaults.get('detect_order') or []
|
||||
normalized_order = [key for key in order if key in default_order]
|
||||
for key in default_order:
|
||||
if key not in normalized_order:
|
||||
normalized_order.append(key)
|
||||
defaults['detect_order'] = normalized_order
|
||||
return defaults
|
||||
except Exception as e:
|
||||
_safe_echo(f"加载检测选项失败: {e}")
|
||||
return {
|
||||
'detect_register': True,
|
||||
'detect_wayback': True,
|
||||
'detect_chinaz': True,
|
||||
'detect_aizhan': True,
|
||||
'detect_baidu_site': True,
|
||||
'detect_360_site': True,
|
||||
'detect_jucha': False,
|
||||
'detect_juziseo': False,
|
||||
}
|
||||
|
||||
# 导出配置
|
||||
config = Config()
|
||||
# 加载检测选项
|
||||
config.DETECT_OPTIONS = load_detect_options()
|
||||
|
||||
# 延迟加载敏感词,避免循环导入
|
||||
def load_sensitive_words_lazy():
|
||||
"""
|
||||
延迟加载敏感词
|
||||
"""
|
||||
return load_sensitive_words()
|
||||
|
||||
# 设置敏感词属性为延迟加载
|
||||
config.load_sensitive_words = load_sensitive_words_lazy
|
||||
|
||||
# 检查检测类型是否应该执行
|
||||
def should_detect(detect_type):
|
||||
"""
|
||||
检查检测类型是否应该执行
|
||||
|
||||
:param detect_type: 检测类型
|
||||
:return: bool - 是否应该执行
|
||||
"""
|
||||
# 检测类型映射
|
||||
type_mapping = {
|
||||
'register': 'detect_register',
|
||||
'wayback': 'detect_wayback',
|
||||
'chinaz': 'detect_chinaz',
|
||||
'aizhan': 'detect_aizhan',
|
||||
'baidu_site': 'detect_baidu_site',
|
||||
'360_site': 'detect_360_site',
|
||||
'whois': 'detect_jucha',
|
||||
'beian': 'detect_jucha',
|
||||
'intercept': 'detect_jucha',
|
||||
'jucha': 'detect_jucha',
|
||||
'juziseo': 'detect_juziseo',
|
||||
'juziseo_outlink': 'detect_juziseo'
|
||||
}
|
||||
|
||||
# 获取对应的配置键
|
||||
config_key = type_mapping.get(detect_type)
|
||||
if not config_key:
|
||||
return True # 默认执行
|
||||
|
||||
# 检查是否在配置中,默认为True
|
||||
return config.DETECT_OPTIONS.get(config_key, True)
|
||||
0
domainCheck/app/config/sensitive_words.txt
Normal file
0
domainCheck/app/config/sensitive_words.txt
Normal file
4
domainCheck/app/core/__init__.py
Normal file
4
domainCheck/app/core/__init__.py
Normal file
@@ -0,0 +1,4 @@
|
||||
# -*- coding: UTF-8 -*-
|
||||
'''
|
||||
核心功能模块
|
||||
'''
|
||||
286
domainCheck/app/core/detect_engine.py
Normal file
286
domainCheck/app/core/detect_engine.py
Normal file
@@ -0,0 +1,286 @@
|
||||
# -*- coding: UTF-8 -*-
|
||||
'''
|
||||
@Project :domainScanDemo
|
||||
@File :detect_engine.py
|
||||
@IDE :PyCharm
|
||||
@Author :梦伴
|
||||
@Date :2026/4/8 23:52
|
||||
@explain : 检测引擎
|
||||
'''
|
||||
|
||||
import time
|
||||
from loguru import logger
|
||||
from app.utils.database import Database
|
||||
from app.detectors.rdap_detector import RDAPDetector
|
||||
from app.detectors.wayback_detector import WaybackDetector
|
||||
from app.detectors.baidu_detector import BaiduDetector
|
||||
from app.detectors.qihu360_detector import Qihu360Detector
|
||||
from app.detectors.google_detector import GoogleDetector
|
||||
from app.detectors.chinaz_detector import ChinazDetector
|
||||
from app.detectors.aizhan_detector import AizhanDetector
|
||||
from app.detectors.juziseo_detector import JuziseoDetector
|
||||
from app.detectors.jucha_detector import JuchaDetector
|
||||
from app.utils.status_codes import (
|
||||
DETECT_STATUS_BLACKLISTED,
|
||||
DETECT_STATUS_COMPLETED,
|
||||
DETECT_STATUS_FAILED,
|
||||
DETECT_STATUS_RUNNING,
|
||||
)
|
||||
|
||||
|
||||
class DetectEngine:
|
||||
"""
|
||||
检测引擎
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
"""
|
||||
初始化检测引擎
|
||||
"""
|
||||
self.db = Database()
|
||||
self.rdap_detector = RDAPDetector()
|
||||
self.wayback_detector = WaybackDetector()
|
||||
self.baidu_detector = BaiduDetector()
|
||||
self.qihu360_detector = Qihu360Detector()
|
||||
self.google_detector = GoogleDetector()
|
||||
self.chinaz_detector = ChinazDetector()
|
||||
self.aizhan_detector = AizhanDetector()
|
||||
self.juziseo_detector = JuziseoDetector()
|
||||
self.jucha_detector = JuchaDetector()
|
||||
|
||||
def detect_domain(self, domain_id):
|
||||
"""
|
||||
检测域名
|
||||
|
||||
:param domain_id: 域名ID
|
||||
:return: bool - 是否检测成功
|
||||
"""
|
||||
try:
|
||||
# 获取域名信息
|
||||
domain_info = self.db.get_domain_by_id(domain_id)
|
||||
if not domain_info:
|
||||
logger.error(f"域名不存在: {domain_id}")
|
||||
return False
|
||||
|
||||
domain = domain_info['domain']
|
||||
logger.info(f"开始检测域名: {domain}")
|
||||
|
||||
# 更新检测状态为检测中
|
||||
self.db.update_domain_detect_status(domain_id, DETECT_STATUS_RUNNING)
|
||||
|
||||
# 1. 基础检测
|
||||
if not self._basic_detect(domain_id, domain):
|
||||
logger.info(f"基础检测失败,停止后续检测: {domain}")
|
||||
return False
|
||||
|
||||
# 2. 深度检测
|
||||
if not self._deep_detect(domain_id, domain):
|
||||
logger.info(f"深度检测失败: {domain}")
|
||||
return False
|
||||
|
||||
# 更新检测状态为正常
|
||||
self.db.update_domain_detect_status(domain_id, DETECT_STATUS_COMPLETED)
|
||||
logger.info(f"域名检测完成: {domain}")
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.error(f"检测域名出错: {e}")
|
||||
# 更新检测状态为检测失败
|
||||
self.db.update_domain_detect_status(domain_id, DETECT_STATUS_FAILED)
|
||||
return False
|
||||
|
||||
def _basic_detect(self, domain_id, domain):
|
||||
"""
|
||||
基础检测
|
||||
|
||||
:param domain_id: 域名ID
|
||||
:param domain: 域名
|
||||
:return: bool - 是否检测通过
|
||||
"""
|
||||
# 1. 检查是否为一口价域名
|
||||
is_ykj = self.db.is_ykj_domain(domain_id)
|
||||
|
||||
# 2. 注册状态检测(一口价域名跳过)
|
||||
if not is_ykj:
|
||||
register_status = self.rdap_detector.check_register_status(domain)
|
||||
self.db.update_domain_register_status(domain_id, register_status)
|
||||
|
||||
# 3. 黑名单缓存检查
|
||||
if self.db.is_blacklisted(domain):
|
||||
logger.info(f"域名在黑名单中: {domain}")
|
||||
self.db.update_domain_detect_status(domain_id, DETECT_STATUS_BLACKLISTED)
|
||||
return False
|
||||
|
||||
# 4. 时光机快照年份采集
|
||||
snapshot_years = self.wayback_detector.get_snapshot_years(domain)
|
||||
if snapshot_years:
|
||||
self.db.update_domain_snapshot_years(domain_id, ','.join(map(str, snapshot_years)))
|
||||
|
||||
# 5. 时光机正文抽样与敏感词匹配
|
||||
if self.wayback_detector.has_sensitive_content(domain):
|
||||
logger.info(f"域名包含敏感词: {domain}")
|
||||
self.db.update_domain_detect_status(domain_id, DETECT_STATUS_BLACKLISTED)
|
||||
self.db.add_to_blacklist(domain, "快照包含敏感词")
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
def _deep_detect(self, domain_id, domain):
|
||||
"""
|
||||
深度检测
|
||||
|
||||
:param domain_id: 域名ID
|
||||
:param domain: 域名
|
||||
:return: bool - 是否检测通过
|
||||
"""
|
||||
# 1. 百度历史/Site
|
||||
baidu_history = self.baidu_detector.check_history(domain)
|
||||
baidu_site = self.baidu_detector.check_site(domain)
|
||||
|
||||
# 2. 360 Site
|
||||
qihu360_site = self.qihu360_detector.check_site(domain)
|
||||
|
||||
# 3. Google Site
|
||||
google_site = self.google_detector.check_site(domain)
|
||||
|
||||
# 4. 站长之家
|
||||
chinaz_info = self.chinaz_detector.check_domain(domain)
|
||||
|
||||
# 5. 爱站网
|
||||
aizhan_info = self.aizhan_detector.check_domain(domain)
|
||||
|
||||
# 6. 桔子SEO
|
||||
juziseo_info = self.juziseo_detector.check_domain(domain)
|
||||
|
||||
# 7. 聚查
|
||||
jucha_info = self.jucha_detector.check_domain(domain)
|
||||
|
||||
# 检查是否有风险
|
||||
if self._check_risk(domain_id, domain, baidu_history, baidu_site, qihu360_site, google_site, chinaz_info, aizhan_info, juziseo_info, jucha_info):
|
||||
return False
|
||||
|
||||
# 保存检测结果
|
||||
self.db.add_detection_result(domain_id, baidu_history, baidu_site, qihu360_site, google_site, chinaz_info, aizhan_info, juziseo_info, jucha_info)
|
||||
|
||||
return True
|
||||
|
||||
def _check_risk(self, domain_id, domain, baidu_history, baidu_site, qihu360_site, google_site, chinaz_info, aizhan_info, juziseo_info, jucha_info):
|
||||
"""
|
||||
检查风险
|
||||
|
||||
:param domain_id: 域名ID
|
||||
:param domain: 域名
|
||||
:param baidu_history: 百度历史
|
||||
:param baidu_site: 百度site
|
||||
:param qihu360_site: 360 site
|
||||
:param google_site: Google site
|
||||
:param chinaz_info: 站长之家信息
|
||||
:param aizhan_info: 爱站网信息
|
||||
:param juziseo_info: 桔子SEO信息
|
||||
:param jucha_info: 聚查信息
|
||||
:return: bool - 是否有风险
|
||||
"""
|
||||
# 检查百度历史过灰
|
||||
if baidu_history and '灰' in str(baidu_history):
|
||||
logger.info(f"百度历史过灰: {domain}")
|
||||
self.db.update_domain_detect_status(domain_id, DETECT_STATUS_BLACKLISTED)
|
||||
self.db.add_to_blacklist(domain, "百度历史过灰")
|
||||
return True
|
||||
|
||||
# 检查标题敏感词
|
||||
if chinaz_info and 'title' in chinaz_info:
|
||||
if self._contains_sensitive_words(chinaz_info['title']):
|
||||
logger.info(f"标题包含敏感词: {domain}")
|
||||
self.db.update_domain_detect_status(domain_id, DETECT_STATUS_BLACKLISTED)
|
||||
self.db.add_to_blacklist(domain, "标题包含敏感词")
|
||||
return True
|
||||
|
||||
# 检查子域名
|
||||
if baidu_site and 'subdomains' in baidu_site:
|
||||
subdomains = baidu_site['subdomains']
|
||||
# 排除 www, @, m
|
||||
valid_subdomains = [sub for sub in subdomains if sub not in ['www', '@', 'm']]
|
||||
if valid_subdomains:
|
||||
logger.info(f"存在子域名: {domain}")
|
||||
self.db.update_domain_detect_status(domain_id, DETECT_STATUS_BLACKLISTED)
|
||||
self.db.add_to_blacklist(domain, "存在子域名")
|
||||
return True
|
||||
|
||||
# 检查风险提示
|
||||
if aizhan_info and 'risk' in aizhan_info:
|
||||
if aizhan_info['risk'] in ['低风险', '疑似色情博彩风险', '严重影响权重']:
|
||||
logger.info(f"风险提示: {domain}")
|
||||
self.db.update_domain_detect_status(domain_id, DETECT_STATUS_BLACKLISTED)
|
||||
self.db.add_to_blacklist(domain, f"风险提示: {aizhan_info['risk']}")
|
||||
return True
|
||||
|
||||
# 检查WHOIS状态
|
||||
if jucha_info and 'whois' in jucha_info:
|
||||
if jucha_info['whois'].get('status') in ['clientHold', 'serverHold']:
|
||||
logger.info(f"WHOIS状态异常: {domain}")
|
||||
self.db.update_domain_detect_status(domain_id, DETECT_STATUS_BLACKLISTED)
|
||||
self.db.add_to_blacklist(domain, "WHOIS状态异常")
|
||||
return True
|
||||
|
||||
# 检查拦截检测
|
||||
if jucha_info and 'intercept' in jucha_info:
|
||||
if not jucha_info['intercept'].get('normal', True):
|
||||
logger.info(f"拦截检测异常: {domain}")
|
||||
self.db.update_domain_detect_status(domain_id, 4) # 4 表示黑名单
|
||||
self.db.add_to_blacklist(domain, "拦截检测异常")
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
def _contains_sensitive_words(self, text):
|
||||
"""
|
||||
检查文本是否包含敏感词
|
||||
|
||||
:param text: 文本
|
||||
:return: bool - 是否包含敏感词
|
||||
"""
|
||||
sensitive_words = self.db.get_all_sensitive_words()
|
||||
for word in sensitive_words:
|
||||
if word in text:
|
||||
return True
|
||||
return False
|
||||
|
||||
def process_task(self, task_id):
|
||||
"""
|
||||
处理检测任务
|
||||
|
||||
:param task_id: 任务ID
|
||||
:return: bool - 是否处理成功
|
||||
"""
|
||||
try:
|
||||
# 获取任务信息
|
||||
task = self.db.get_task_by_id(task_id)
|
||||
if not task:
|
||||
logger.error(f"任务不存在: {task_id}")
|
||||
return False
|
||||
|
||||
domain_id = task['domain_id']
|
||||
|
||||
# 更新任务状态为执行中
|
||||
self.db.update_task_status(task_id, 1) # 1 表示执行中
|
||||
|
||||
# 执行检测
|
||||
success = self.detect_domain(domain_id)
|
||||
|
||||
# 更新任务状态
|
||||
if success:
|
||||
self.db.update_task_status(task_id, 2) # 2 表示完成
|
||||
else:
|
||||
# 增加重试次数
|
||||
retry_count = task.get('retry_count', 0) + 1
|
||||
if retry_count < 3:
|
||||
self.db.update_task_retry_count(task_id, retry_count)
|
||||
self.db.update_task_status(task_id, 0) # 0 表示待执行
|
||||
else:
|
||||
self.db.update_task_status(task_id, 3) # 3 表示失败
|
||||
|
||||
return success
|
||||
except Exception as e:
|
||||
logger.error(f"处理任务出错: {e}")
|
||||
# 更新任务状态为失败
|
||||
self.db.update_task_status(task_id, 3) # 3 表示失败
|
||||
return False
|
||||
261
domainCheck/app/core/domain_collector.py
Normal file
261
domainCheck/app/core/domain_collector.py
Normal file
@@ -0,0 +1,261 @@
|
||||
# -*- coding: UTF-8 -*-
|
||||
'''
|
||||
@Project :domainScanDemo
|
||||
@File :domain_collector.py
|
||||
@IDE :PyCharm
|
||||
@Author :梦伴
|
||||
@Date :2026/4/8 23:50
|
||||
@explain : 域名收集器
|
||||
'''
|
||||
|
||||
import re
|
||||
import tldextract
|
||||
from loguru import logger
|
||||
from app.utils.database import Database
|
||||
from app.utils.domain_utils import normalize_domain
|
||||
from app.config import config
|
||||
|
||||
|
||||
class DomainCollector:
|
||||
"""
|
||||
域名收集器
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
"""
|
||||
初始化域名收集器
|
||||
"""
|
||||
self.db = Database(
|
||||
host=config.DB_HOST,
|
||||
port=config.DB_PORT,
|
||||
database=config.DB_DATABASE,
|
||||
user=config.DB_USER,
|
||||
password=config.DB_PASSWORD
|
||||
)
|
||||
|
||||
def add_domain(self, domain, source_type):
|
||||
"""
|
||||
添加域名
|
||||
|
||||
:param domain: 域名
|
||||
:param source_type: 来源类型
|
||||
:return: bool - 是否添加成功
|
||||
"""
|
||||
try:
|
||||
# 标准化域名
|
||||
normalized_domain = normalize_domain(domain)
|
||||
if not normalized_domain:
|
||||
logger.warning(f"无效域名: {domain}")
|
||||
return False
|
||||
|
||||
# 提取顶级域名
|
||||
ext = tldextract.extract(normalized_domain)
|
||||
tld = ext.suffix
|
||||
|
||||
# 只保留 .com 和 .net
|
||||
if tld not in ['com', 'net']:
|
||||
logger.warning(f"不支持的顶级域名: {tld}")
|
||||
return False
|
||||
|
||||
# 检查是否已存在
|
||||
if self.db.domain_exists(normalized_domain):
|
||||
logger.info(f"域名已存在: {normalized_domain}")
|
||||
return False
|
||||
|
||||
# 添加域名
|
||||
domain_id = self.db.add_domain(normalized_domain, tld, source_type)
|
||||
if domain_id:
|
||||
# 创建检测任务
|
||||
self.db.create_detect_task(domain_id, 1) # 1 表示基础检测
|
||||
logger.info(f"成功添加域名: {normalized_domain}")
|
||||
return True
|
||||
else:
|
||||
logger.error(f"添加域名失败: {normalized_domain}")
|
||||
return False
|
||||
except Exception as e:
|
||||
logger.error(f"添加域名出错: {e}")
|
||||
return False
|
||||
|
||||
def add_domains_batch(self, domains, source_type, batch_size=1000, dry_run=False):
|
||||
"""
|
||||
批量添加域名
|
||||
|
||||
:param domains: 域名列表
|
||||
:param source_type: 来源类型
|
||||
:param batch_size: 批量大小
|
||||
:param dry_run: 是否仅进行干运行(不实际添加域名)
|
||||
:return: dict - 统计信息
|
||||
"""
|
||||
try:
|
||||
# 统计信息
|
||||
stats = {
|
||||
'total': len(domains),
|
||||
'valid': 0,
|
||||
'added': 0,
|
||||
'exists': 0,
|
||||
'invalid': 0,
|
||||
'failed': 0
|
||||
}
|
||||
|
||||
# 处理大规模数据时,分批进行标准化和过滤
|
||||
normalized_domains = []
|
||||
batch_domains = []
|
||||
|
||||
for i, domain in enumerate(domains):
|
||||
normalized = normalize_domain(domain)
|
||||
if not normalized:
|
||||
stats['invalid'] += 1
|
||||
continue
|
||||
|
||||
# 提取顶级域名
|
||||
ext = tldextract.extract(normalized)
|
||||
tld = ext.suffix
|
||||
|
||||
# 只保留 .com 和 .net
|
||||
if tld not in ['com', 'net']:
|
||||
stats['invalid'] += 1
|
||||
continue
|
||||
|
||||
normalized_domains.append((normalized, tld))
|
||||
batch_domains.append(normalized)
|
||||
stats['valid'] += 1
|
||||
|
||||
# 每1000个域名检查一次,避免内存占用过高
|
||||
if (i + 1) % 1000 == 0:
|
||||
logger.info(f"已处理 {i + 1}/{len(domains)} 个域名")
|
||||
|
||||
logger.info(f"域名标准化完成,有效域名: {stats['valid']}")
|
||||
|
||||
# 提取所有域名
|
||||
all_domains = [domain for domain, tld in normalized_domains]
|
||||
|
||||
# 批量检查域名是否存在
|
||||
existing_domains = self.db.check_domains_exist(all_domains)
|
||||
existing_set = set(existing_domains)
|
||||
|
||||
# 准备批量添加数据
|
||||
batch_data = []
|
||||
for domain, tld in normalized_domains:
|
||||
if domain not in existing_set:
|
||||
batch_data.append((domain, tld, source_type))
|
||||
|
||||
stats['exists'] = len(existing_domains)
|
||||
stats['valid'] = len(normalized_domains)
|
||||
|
||||
# 干运行模式下直接返回统计信息
|
||||
if dry_run:
|
||||
stats['added'] = len(batch_data)
|
||||
logger.info(f"干运行模式:准备添加 {len(batch_data)} 个新域名")
|
||||
return stats
|
||||
|
||||
logger.info(f"准备添加 {len(batch_data)} 个新域名")
|
||||
|
||||
# 分批次添加
|
||||
for i in range(0, len(batch_data), batch_size):
|
||||
batch = batch_data[i:i+batch_size]
|
||||
added_count = self.db.add_domains_batch(batch)
|
||||
stats['added'] += added_count
|
||||
stats['failed'] += len(batch) - added_count
|
||||
|
||||
# 每处理一批,记录一次进度
|
||||
if (i + len(batch)) % (batch_size * 10) == 0:
|
||||
logger.info(f"已添加 {i + len(batch)}/{len(batch_data)} 个域名")
|
||||
|
||||
logger.info(f"批量添加域名完成: {stats}")
|
||||
return stats
|
||||
except Exception as e:
|
||||
logger.error(f"批量添加域名出错: {e}")
|
||||
return {
|
||||
'total': len(domains),
|
||||
'valid': 0,
|
||||
'added': 0,
|
||||
'exists': 0,
|
||||
'invalid': 0,
|
||||
'failed': len(domains)
|
||||
}
|
||||
|
||||
def import_from_file(self, file_path, source_type, batch_size=1000):
|
||||
"""
|
||||
从文件导入域名
|
||||
|
||||
:param file_path: 文件路径
|
||||
:param source_type: 来源类型
|
||||
:param batch_size: 批量大小
|
||||
:return: dict - 导入统计信息
|
||||
"""
|
||||
try:
|
||||
with open(file_path, 'r', encoding='utf-8') as f:
|
||||
domains = f.readlines()
|
||||
|
||||
# 提取域名
|
||||
domain_list = []
|
||||
for domain in domains:
|
||||
domain = domain.strip()
|
||||
if domain:
|
||||
domain_list.append(domain)
|
||||
|
||||
# 批量添加域名
|
||||
stats = self.add_domains_batch(domain_list, source_type, batch_size)
|
||||
|
||||
logger.info(f"从文件导入完成: {stats}")
|
||||
return stats
|
||||
except Exception as e:
|
||||
logger.error(f"从文件导入出错: {e}")
|
||||
return {
|
||||
'total': 0,
|
||||
'valid': 0,
|
||||
'added': 0,
|
||||
'exists': 0,
|
||||
'invalid': 0,
|
||||
'failed': 0
|
||||
}
|
||||
|
||||
def collect_from_juming(self, type_='一口价'):
|
||||
"""
|
||||
从聚名网收集域名
|
||||
|
||||
:param type_: 类型,一口价或过期删除
|
||||
:return: int - 收集到的域名数量
|
||||
"""
|
||||
# 这里可以添加从聚名网收集域名的逻辑
|
||||
logger.info(f"从聚名网收集 {type_} 域名")
|
||||
# 模拟收集结果
|
||||
return 0
|
||||
|
||||
def collect_from_search_engine(self, keyword, limit=100):
|
||||
"""
|
||||
从搜索引擎收集域名
|
||||
|
||||
:param keyword: 关键词
|
||||
:param limit: 限制数量
|
||||
:return: int - 收集到的域名数量
|
||||
"""
|
||||
# 这里可以添加从搜索引擎收集域名的逻辑
|
||||
logger.info(f"从搜索引擎收集域名,关键词: {keyword}, 限制: {limit}")
|
||||
# 模拟收集结果
|
||||
return 0
|
||||
|
||||
def collect_from_enterprise_directory(self, url, limit=100):
|
||||
"""
|
||||
从企业目录收集域名
|
||||
|
||||
:param url: 企业目录URL
|
||||
:param limit: 限制数量
|
||||
:return: int - 收集到的域名数量
|
||||
"""
|
||||
# 这里可以添加从企业目录收集域名的逻辑
|
||||
logger.info(f"从企业目录收集域名,URL: {url}, 限制: {limit}")
|
||||
# 模拟收集结果
|
||||
return 0
|
||||
|
||||
def collect_from_zone_file(self, file_path):
|
||||
"""
|
||||
从Zone File收集域名
|
||||
|
||||
:param file_path: Zone File路径
|
||||
:return: int - 收集到的域名数量
|
||||
"""
|
||||
# 这里可以添加从Zone File收集域名的逻辑
|
||||
logger.info(f"从Zone File收集域名,文件: {file_path}")
|
||||
# 模拟收集结果
|
||||
return 0
|
||||
150
domainCheck/app/core/domain_processor.py
Normal file
150
domainCheck/app/core/domain_processor.py
Normal file
@@ -0,0 +1,150 @@
|
||||
# -*- coding: UTF-8 -*-
|
||||
'''
|
||||
@Project :domainScanDemo
|
||||
@File :domain_processor.py
|
||||
@IDE :PyCharm
|
||||
@Author :梦伴
|
||||
@Date :2026/4/8 23:51
|
||||
@explain : 域名处理器
|
||||
'''
|
||||
|
||||
import re
|
||||
import tldextract
|
||||
from loguru import logger
|
||||
from app.utils.database import Database
|
||||
from app.utils.domain_utils import normalize_domain
|
||||
|
||||
|
||||
class DomainProcessor:
|
||||
"""
|
||||
域名处理器
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
"""
|
||||
初始化域名处理器
|
||||
"""
|
||||
self.db = Database()
|
||||
|
||||
def process_domain(self, domain):
|
||||
"""
|
||||
处理域名
|
||||
|
||||
:param domain: 域名
|
||||
:return: dict - 处理结果
|
||||
"""
|
||||
result = {
|
||||
'original': domain,
|
||||
'normalized': None,
|
||||
'tld': None,
|
||||
'valid': False,
|
||||
'reason': ''
|
||||
}
|
||||
|
||||
try:
|
||||
# 标准化域名
|
||||
normalized = normalize_domain(domain)
|
||||
if not normalized:
|
||||
result['reason'] = '无效域名格式'
|
||||
return result
|
||||
|
||||
# 提取顶级域名
|
||||
ext = tldextract.extract(normalized)
|
||||
tld = ext.suffix
|
||||
|
||||
# 检查顶级域名
|
||||
if tld not in ['com', 'net']:
|
||||
result['reason'] = '不支持的顶级域名'
|
||||
return result
|
||||
|
||||
result['normalized'] = normalized
|
||||
result['tld'] = tld
|
||||
result['valid'] = True
|
||||
|
||||
return result
|
||||
except Exception as e:
|
||||
logger.error(f"处理域名出错: {e}")
|
||||
result['reason'] = f"处理出错: {str(e)}"
|
||||
return result
|
||||
|
||||
def batch_process(self, domains):
|
||||
"""
|
||||
批量处理域名
|
||||
|
||||
:param domains: 域名列表
|
||||
:return: list - 处理结果列表
|
||||
"""
|
||||
results = []
|
||||
for domain in domains:
|
||||
result = self.process_domain(domain)
|
||||
results.append(result)
|
||||
return results
|
||||
|
||||
def filter_valid_domains(self, domains):
|
||||
"""
|
||||
过滤有效的域名
|
||||
|
||||
:param domains: 域名列表
|
||||
:return: list - 有效的域名列表
|
||||
"""
|
||||
valid_domains = []
|
||||
for domain in domains:
|
||||
result = self.process_domain(domain)
|
||||
if result['valid']:
|
||||
valid_domains.append(result['normalized'])
|
||||
return valid_domains
|
||||
|
||||
def update_domain_status(self, domain_id, status_type, status_value):
|
||||
"""
|
||||
更新域名状态
|
||||
|
||||
:param domain_id: 域名ID
|
||||
:param status_type: 状态类型
|
||||
:param status_value: 状态值
|
||||
:return: bool - 是否更新成功
|
||||
"""
|
||||
try:
|
||||
if status_type == 'use_status':
|
||||
return self.db.update_domain_use_status(domain_id, status_value)
|
||||
elif status_type == 'detect_status':
|
||||
return self.db.update_domain_detect_status(domain_id, status_value)
|
||||
elif status_type == 'register_status':
|
||||
return self.db.update_domain_register_status(domain_id, status_value)
|
||||
else:
|
||||
logger.error(f"未知的状态类型: {status_type}")
|
||||
return False
|
||||
except Exception as e:
|
||||
logger.error(f"更新域名状态出错: {e}")
|
||||
return False
|
||||
|
||||
def blacklist_domain(self, domain_id, reason):
|
||||
"""
|
||||
将域名加入黑名单
|
||||
|
||||
:param domain_id: 域名ID
|
||||
:param reason: 黑名单原因
|
||||
:return: bool - 是否操作成功
|
||||
"""
|
||||
try:
|
||||
# 更新检测状态为黑名单
|
||||
if self.db.update_domain_detect_status(domain_id, 4): # 4 表示黑名单
|
||||
# 添加到黑名单表
|
||||
domain = self.db.get_domain_by_id(domain_id)
|
||||
if domain:
|
||||
return self.db.add_to_blacklist(domain['domain'], reason)
|
||||
return False
|
||||
except Exception as e:
|
||||
logger.error(f"将域名加入黑名单出错: {e}")
|
||||
return False
|
||||
|
||||
def get_domain_statistics(self):
|
||||
"""
|
||||
获取域名统计信息
|
||||
|
||||
:return: dict - 统计信息
|
||||
"""
|
||||
try:
|
||||
return self.db.get_domain_statistics()
|
||||
except Exception as e:
|
||||
logger.error(f"获取域名统计信息出错: {e}")
|
||||
return {}
|
||||
289
domainCheck/app/core/export_manager.py
Normal file
289
domainCheck/app/core/export_manager.py
Normal file
@@ -0,0 +1,289 @@
|
||||
# -*- coding: UTF-8 -*-
|
||||
'''
|
||||
@Project :domainScanDemo
|
||||
@File :export_manager.py
|
||||
@IDE :PyCharm
|
||||
@Author :梦伴
|
||||
@Date :2026/4/8 23:54
|
||||
@explain : 导出管理器
|
||||
'''
|
||||
|
||||
import os
|
||||
from loguru import logger
|
||||
from app.utils.database import Database
|
||||
|
||||
|
||||
class ExportManager:
|
||||
"""
|
||||
导出管理器
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
"""
|
||||
初始化导出管理器
|
||||
"""
|
||||
self.db = Database()
|
||||
|
||||
def export_domains(self, filter_conditions, output_file):
|
||||
"""
|
||||
导出域名
|
||||
|
||||
:param filter_conditions: 筛选条件
|
||||
:param output_file: 输出文件路径
|
||||
:return: int - 导出的域名数量
|
||||
"""
|
||||
try:
|
||||
# 查询符合条件的域名
|
||||
domains = self.db.get_domains_by_conditions(filter_conditions)
|
||||
|
||||
if not domains:
|
||||
logger.warning("没有符合条件的域名")
|
||||
return 0
|
||||
|
||||
# 导出到文件
|
||||
with open(output_file, 'w', encoding='utf-8') as f:
|
||||
for domain in domains:
|
||||
f.write(domain['domain'] + '\n')
|
||||
|
||||
logger.info(f"成功导出 {len(domains)} 个域名到 {output_file}")
|
||||
return len(domains)
|
||||
except Exception as e:
|
||||
logger.error(f"导出域名出错: {e}")
|
||||
return 0
|
||||
|
||||
def export_with_details(self, filter_conditions, output_file):
|
||||
"""
|
||||
导出域名及其详细信息
|
||||
|
||||
:param filter_conditions: 筛选条件
|
||||
:param output_file: 输出文件路径
|
||||
:return: int - 导出的域名数量
|
||||
"""
|
||||
try:
|
||||
# 查询符合条件的域名及其详细信息
|
||||
domains = self.db.get_domains_with_details(filter_conditions)
|
||||
|
||||
if not domains:
|
||||
logger.warning("没有符合条件的域名")
|
||||
return 0
|
||||
|
||||
# 导出到文件
|
||||
with open(output_file, 'w', encoding='utf-8') as f:
|
||||
# 写入表头
|
||||
f.write('域名,注册状态,使用状态,检测状态,备案历史,备案年份,快照年份,友情链接数量\n')
|
||||
|
||||
# 写入数据
|
||||
for domain in domains:
|
||||
line = f"{domain['domain']},{domain['register_status']},{domain['use_status']},{domain['detect_status']},{domain['has_beian']},{domain['beian_year']},{domain['snapshot_years']},{domain['backlink_count']}\n"
|
||||
f.write(line)
|
||||
|
||||
logger.info(f"成功导出 {len(domains)} 个域名及其详细信息到 {output_file}")
|
||||
return len(domains)
|
||||
except Exception as e:
|
||||
logger.error(f"导出域名详细信息出错: {e}")
|
||||
return 0
|
||||
|
||||
def batch_update_status(self, domain_ids, status_type, status_value):
|
||||
"""
|
||||
批量更新域名状态
|
||||
|
||||
:param domain_ids: 域名ID列表
|
||||
:param status_type: 状态类型
|
||||
:param status_value: 状态值
|
||||
:return: int - 更新成功的域名数量
|
||||
"""
|
||||
try:
|
||||
success_count = 0
|
||||
for domain_id in domain_ids:
|
||||
if self.db.update_domain_status(domain_id, status_type, status_value):
|
||||
success_count += 1
|
||||
|
||||
logger.info(f"成功更新 {success_count} 个域名的状态")
|
||||
return success_count
|
||||
except Exception as e:
|
||||
logger.error(f"批量更新域名状态出错: {e}")
|
||||
return 0
|
||||
|
||||
def get_filtered_domains(self, filter_conditions, limit=1000):
|
||||
"""
|
||||
获取符合条件的域名
|
||||
|
||||
:param filter_conditions: 筛选条件
|
||||
:param limit: 限制数量
|
||||
:return: list - 域名列表
|
||||
"""
|
||||
try:
|
||||
domains = self.db.get_domains_by_conditions(filter_conditions, limit)
|
||||
logger.info(f"获取到 {len(domains)} 个符合条件的域名")
|
||||
return domains
|
||||
except Exception as e:
|
||||
logger.error(f"获取符合条件的域名出错: {e}")
|
||||
return []
|
||||
|
||||
def generate_report(self, output_file):
|
||||
"""
|
||||
生成统计报告
|
||||
|
||||
:param output_file: 输出文件路径
|
||||
:return: bool - 是否生成成功
|
||||
"""
|
||||
try:
|
||||
# 获取域名统计信息
|
||||
domain_stats = self.db.get_domain_statistics()
|
||||
|
||||
# 获取任务统计信息
|
||||
task_stats = self.db.get_task_statistics()
|
||||
|
||||
# 生成报告
|
||||
with open(output_file, 'w', encoding='utf-8') as f:
|
||||
f.write('域名库系统统计报告\n')
|
||||
f.write('=' * 50 + '\n')
|
||||
|
||||
f.write('\n域名统计:\n')
|
||||
f.write(f'总域名数: {domain_stats.get("total", 0)}\n')
|
||||
f.write(f'可注册域名: {domain_stats.get("available", 0)}\n')
|
||||
f.write(f'已注册域名: {domain_stats.get("registered", 0)}\n')
|
||||
f.write(f'黑名单域名: {domain_stats.get("blacklisted", 0)}\n')
|
||||
|
||||
f.write('\n任务统计:\n')
|
||||
f.write(f'总任务数: {task_stats.get("total", 0)}\n')
|
||||
f.write(f'待执行任务: {task_stats.get("pending", 0)}\n')
|
||||
f.write(f'执行中任务: {task_stats.get("running", 0)}\n')
|
||||
f.write(f'已完成任务: {task_stats.get("completed", 0)}\n')
|
||||
f.write(f'失败任务: {task_stats.get("failed", 0)}\n')
|
||||
|
||||
logger.info(f"成功生成统计报告到 {output_file}")
|
||||
return True
|
||||
except Exception as e:
|
||||
logger.error(f"生成统计报告出错: {e}")
|
||||
return False
|
||||
|
||||
def export_to_excel(self, domains, output_file):
|
||||
"""
|
||||
导出域名到Excel文件
|
||||
|
||||
:param domains: 域名列表
|
||||
:param output_file: 输出文件路径
|
||||
:return: int - 导出的域名数量
|
||||
"""
|
||||
try:
|
||||
# 尝试导入openpyxl
|
||||
try:
|
||||
from openpyxl import Workbook
|
||||
except ImportError:
|
||||
# 如果没有安装openpyxl,使用CSV格式作为替代
|
||||
logger.warning("openpyxl库未安装,将使用CSV格式导出")
|
||||
return self.export_to_csv(domains, output_file.replace('.xlsx', '.csv'))
|
||||
|
||||
# 创建工作簿
|
||||
wb = Workbook()
|
||||
ws = wb.active
|
||||
|
||||
# 写入表头
|
||||
headers = ['域名', '注册状态', '使用状态', '检测状态', '人工复核状态', '过期时间', '单位性质', '网站首页网址', '检测时间', '备案历史', '备案年份', '快照年份', '百度历史', '百度Site', '是否中文标题', '360 Site', 'Google Site', '友情链接数量']
|
||||
ws.append(headers)
|
||||
|
||||
# 写入数据
|
||||
for domain in domains:
|
||||
row = [
|
||||
domain.get('domain', ''),
|
||||
domain.get('register_status', ''),
|
||||
domain.get('use_status', ''),
|
||||
domain.get('detect_status', ''),
|
||||
domain.get('review_status', ''),
|
||||
domain.get('expire_date', ''),
|
||||
domain.get('company_type', ''),
|
||||
domain.get('website_url', ''),
|
||||
domain.get('detect_time', ''),
|
||||
domain.get('has_beian', ''),
|
||||
domain.get('beian_year', ''),
|
||||
domain.get('snapshot_years', ''),
|
||||
domain.get('baidu_history', ''),
|
||||
domain.get('baidu_site', ''),
|
||||
domain.get('is_chinese_title', ''),
|
||||
domain.get('qihu360_site', ''),
|
||||
domain.get('google_site', ''),
|
||||
domain.get('backlink_count', '')
|
||||
]
|
||||
ws.append(row)
|
||||
|
||||
# 保存文件
|
||||
wb.save(output_file)
|
||||
|
||||
logger.info(f"成功导出 {len(domains)} 个域名到Excel文件: {output_file}")
|
||||
return len(domains)
|
||||
except Exception as e:
|
||||
logger.error(f"导出Excel文件出错: {e}")
|
||||
# 尝试使用CSV格式作为替代
|
||||
try:
|
||||
csv_file = output_file.replace('.xlsx', '.csv')
|
||||
logger.info(f"尝试使用CSV格式导出到: {csv_file}")
|
||||
return self.export_to_csv(domains, csv_file)
|
||||
except Exception as e2:
|
||||
logger.error(f"导出CSV文件也失败: {e2}")
|
||||
raise
|
||||
|
||||
def export_to_csv(self, domains, output_file):
|
||||
"""
|
||||
导出域名到CSV文件
|
||||
|
||||
:param domains: 域名列表
|
||||
:param output_file: 输出文件路径
|
||||
:return: int - 导出的域名数量
|
||||
"""
|
||||
try:
|
||||
import csv
|
||||
|
||||
# 写入文件
|
||||
with open(output_file, 'w', encoding='utf-8', newline='') as f:
|
||||
writer = csv.writer(f)
|
||||
|
||||
# 写入表头
|
||||
headers = ['域名', '注册状态', '使用状态', '检测状态', '人工复核状态', '过期时间', '单位性质', '网站首页网址', '检测时间', '备案历史', '备案年份', '快照年份', '百度历史', '百度Site', '是否中文标题', '360 Site', 'Google Site', '友情链接数量']
|
||||
writer.writerow(headers)
|
||||
|
||||
# 写入数据
|
||||
for domain in domains:
|
||||
row = [
|
||||
domain.get('domain', ''),
|
||||
domain.get('register_status', ''),
|
||||
domain.get('use_status', ''),
|
||||
domain.get('detect_status', ''),
|
||||
domain.get('review_status', ''),
|
||||
domain.get('expire_date', ''),
|
||||
domain.get('company_type', ''),
|
||||
domain.get('website_url', ''),
|
||||
domain.get('detect_time', ''),
|
||||
domain.get('has_beian', ''),
|
||||
domain.get('beian_year', ''),
|
||||
domain.get('snapshot_years', ''),
|
||||
domain.get('baidu_history', ''),
|
||||
domain.get('baidu_site', ''),
|
||||
domain.get('is_chinese_title', ''),
|
||||
domain.get('qihu360_site', ''),
|
||||
domain.get('google_site', ''),
|
||||
domain.get('backlink_count', '')
|
||||
]
|
||||
writer.writerow(row)
|
||||
|
||||
logger.info(f"成功导出 {len(domains)} 个域名到CSV文件: {output_file}")
|
||||
return len(domains)
|
||||
except Exception as e:
|
||||
logger.error(f"导出CSV文件出错: {e}")
|
||||
raise
|
||||
|
||||
def export_to_txt(self, domains, output_file):
|
||||
"""
|
||||
导出域名到TXT文件,一行一个域名。
|
||||
"""
|
||||
try:
|
||||
with open(output_file, 'w', encoding='utf-8') as f:
|
||||
for domain in domains:
|
||||
value = domain.get('domain', '').strip()
|
||||
if value:
|
||||
f.write(value + '\n')
|
||||
logger.info(f"成功导出 {len(domains)} 个域名到TXT文件: {output_file}")
|
||||
return len(domains)
|
||||
except Exception as e:
|
||||
logger.error(f"导出TXT文件出错: {e}")
|
||||
raise
|
||||
162
domainCheck/app/core/task_scheduler.py
Normal file
162
domainCheck/app/core/task_scheduler.py
Normal file
@@ -0,0 +1,162 @@
|
||||
# -*- coding: UTF-8 -*-
|
||||
'''
|
||||
@Project :domainScanDemo
|
||||
@File :task_scheduler.py
|
||||
@IDE :PyCharm
|
||||
@Author :梦伴
|
||||
@Date :2026/4/8 23:53
|
||||
@explain : 任务调度器
|
||||
'''
|
||||
|
||||
import time
|
||||
import threading
|
||||
from loguru import logger
|
||||
from app.utils.database import Database
|
||||
from app.core.detect_engine import DetectEngine
|
||||
|
||||
|
||||
class TaskScheduler:
|
||||
"""
|
||||
任务调度器
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
"""
|
||||
初始化任务调度器
|
||||
"""
|
||||
self.db = Database()
|
||||
self.detect_engine = DetectEngine()
|
||||
self.running = False
|
||||
self.threads = []
|
||||
self.max_threads = 10
|
||||
|
||||
def start(self):
|
||||
"""
|
||||
启动任务调度器
|
||||
"""
|
||||
if self.running:
|
||||
logger.info("任务调度器已经在运行中")
|
||||
return
|
||||
|
||||
self.running = True
|
||||
logger.info("启动任务调度器")
|
||||
|
||||
# 启动多个线程处理任务
|
||||
for i in range(self.max_threads):
|
||||
thread = threading.Thread(target=self._process_tasks, daemon=True)
|
||||
thread.start()
|
||||
self.threads.append(thread)
|
||||
logger.info(f"启动任务处理线程 {i+1}")
|
||||
|
||||
def stop(self):
|
||||
"""
|
||||
停止任务调度器
|
||||
"""
|
||||
self.running = False
|
||||
logger.info("停止任务调度器")
|
||||
|
||||
# 等待线程结束
|
||||
for thread in self.threads:
|
||||
thread.join(timeout=5)
|
||||
|
||||
self.threads.clear()
|
||||
logger.info("任务调度器已停止")
|
||||
|
||||
def _process_tasks(self):
|
||||
"""
|
||||
处理任务
|
||||
"""
|
||||
while self.running:
|
||||
try:
|
||||
# 获取待执行的任务
|
||||
task = self.db.get_pending_task()
|
||||
if not task:
|
||||
# 没有任务,休眠一段时间
|
||||
time.sleep(1)
|
||||
continue
|
||||
|
||||
task_id = task['id']
|
||||
domain_id = task['domain_id']
|
||||
|
||||
logger.info(f"处理任务: {task_id}, 域名ID: {domain_id}")
|
||||
|
||||
# 执行任务
|
||||
success = self.detect_engine.process_task(task_id)
|
||||
|
||||
if success:
|
||||
logger.info(f"任务处理成功: {task_id}")
|
||||
else:
|
||||
logger.warning(f"任务处理失败: {task_id}")
|
||||
|
||||
# 短暂休眠,避免过于频繁的数据库操作
|
||||
time.sleep(0.1)
|
||||
except Exception as e:
|
||||
logger.error(f"处理任务出错: {e}")
|
||||
# 休眠一段时间,避免出错后无限循环
|
||||
time.sleep(5)
|
||||
|
||||
def add_task(self, domain_id, task_type=1, priority=0):
|
||||
"""
|
||||
添加任务
|
||||
|
||||
:param domain_id: 域名ID
|
||||
:param task_type: 任务类型,1-基础检测,2-深度检测
|
||||
:param priority: 优先级,0-低,1-中,2-高
|
||||
:return: int - 任务ID
|
||||
"""
|
||||
try:
|
||||
task_id = self.db.create_detect_task(domain_id, task_type, priority)
|
||||
logger.info(f"添加任务成功: {task_id}, 域名ID: {domain_id}")
|
||||
return task_id
|
||||
except Exception as e:
|
||||
logger.error(f"添加任务失败: {e}")
|
||||
return None
|
||||
|
||||
def get_task_stats(self):
|
||||
"""
|
||||
获取任务统计信息
|
||||
|
||||
:return: dict - 任务统计信息
|
||||
"""
|
||||
try:
|
||||
return self.db.get_task_statistics()
|
||||
except Exception as e:
|
||||
logger.error(f"获取任务统计信息出错: {e}")
|
||||
return {}
|
||||
|
||||
def retry_failed_tasks(self):
|
||||
"""
|
||||
重试失败的任务
|
||||
|
||||
:return: int - 重试的任务数量
|
||||
"""
|
||||
try:
|
||||
tasks = self.db.get_failed_tasks()
|
||||
retry_count = 0
|
||||
|
||||
for task in tasks:
|
||||
task_id = task['id']
|
||||
self.db.update_task_status(task_id, 0) # 0 表示待执行
|
||||
self.db.update_task_retry_count(task_id, 0) # 重置重试次数
|
||||
retry_count += 1
|
||||
|
||||
logger.info(f"重试 {retry_count} 个失败的任务")
|
||||
return retry_count
|
||||
except Exception as e:
|
||||
logger.error(f"重试失败任务出错: {e}")
|
||||
return 0
|
||||
|
||||
def clear_completed_tasks(self, days=7):
|
||||
"""
|
||||
清理已完成的任务
|
||||
|
||||
:param days: 保留天数
|
||||
:return: int - 清理的任务数量
|
||||
"""
|
||||
try:
|
||||
count = self.db.clear_completed_tasks(days)
|
||||
logger.info(f"清理 {count} 个已完成的任务")
|
||||
return count
|
||||
except Exception as e:
|
||||
logger.error(f"清理已完成任务出错: {e}")
|
||||
return 0
|
||||
10
domainCheck/app/credentials.json
Normal file
10
domainCheck/app/credentials.json
Normal file
@@ -0,0 +1,10 @@
|
||||
{
|
||||
"juming": {
|
||||
"email": "chaofanai1998@gmail.com",
|
||||
"password": "llzz123,./"
|
||||
},
|
||||
"juziseo": {
|
||||
"email": "mamian",
|
||||
"password": "Abc123456"
|
||||
}
|
||||
}
|
||||
20
domainCheck/app/detect_options.json
Normal file
20
domainCheck/app/detect_options.json
Normal file
@@ -0,0 +1,20 @@
|
||||
{
|
||||
"detect_register": true,
|
||||
"detect_wayback": true,
|
||||
"detect_chinaz": true,
|
||||
"detect_aizhan": true,
|
||||
"detect_baidu_site": true,
|
||||
"detect_360_site": true,
|
||||
"detect_jucha": false,
|
||||
"detect_juziseo": false,
|
||||
"detect_order": [
|
||||
"detect_register",
|
||||
"detect_baidu_site",
|
||||
"detect_360_site",
|
||||
"detect_chinaz",
|
||||
"detect_aizhan",
|
||||
"detect_wayback",
|
||||
"detect_jucha",
|
||||
"detect_juziseo"
|
||||
]
|
||||
}
|
||||
4
domainCheck/app/detectors/__init__.py
Normal file
4
domainCheck/app/detectors/__init__.py
Normal file
@@ -0,0 +1,4 @@
|
||||
# -*- coding: UTF-8 -*-
|
||||
'''
|
||||
检测插件模块
|
||||
'''
|
||||
130
domainCheck/app/detectors/aizhan_detector.py
Normal file
130
domainCheck/app/detectors/aizhan_detector.py
Normal file
@@ -0,0 +1,130 @@
|
||||
# -*- coding: UTF-8 -*-
|
||||
'''
|
||||
@Project :domainScanDemo
|
||||
@File :aizhan_detector.py
|
||||
@IDE :PyCharm
|
||||
@Author :梦伴
|
||||
@Date :2026/4/9 0:02
|
||||
@explain : 爱站网检测器
|
||||
'''
|
||||
|
||||
import requests
|
||||
from curl_cffi import requests as curl_requests
|
||||
import re
|
||||
from app.detectors.base import BaseDetector
|
||||
|
||||
|
||||
class AizhanDetector(BaseDetector):
|
||||
"""
|
||||
爱站网检测器
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
"""
|
||||
初始化爱站网检测器
|
||||
"""
|
||||
super().__init__()
|
||||
self.url = 'https://www.aizhan.com'
|
||||
self.query_url = 'https://www.aizhan.com/cha/{domain}'
|
||||
|
||||
def check_domain(self, domain):
|
||||
"""
|
||||
检测域名
|
||||
|
||||
:param domain: 域名
|
||||
:return: dict - 检测结果
|
||||
"""
|
||||
try:
|
||||
# 构建查询URL
|
||||
query_url = self.query_url.format(domain=domain)
|
||||
|
||||
headers = {
|
||||
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/146.0.0.0 Safari/537.36'
|
||||
}
|
||||
|
||||
# 使用curl_cffi模拟浏览器
|
||||
response = curl_requests.get(query_url, headers=headers, impersonate='chrome', timeout=10)
|
||||
|
||||
if response.status_code == 200:
|
||||
content = response.text
|
||||
|
||||
# 提取标题
|
||||
title = self._extract_title(content)
|
||||
|
||||
# 提取风险信息
|
||||
risk = self._extract_risk(content)
|
||||
|
||||
# 检查是否包含敏感词
|
||||
has_sensitive = self._check_sensitive(title, risk)
|
||||
|
||||
return {
|
||||
'title': title,
|
||||
'risk': risk,
|
||||
'has_sensitive': has_sensitive
|
||||
}
|
||||
else:
|
||||
self._log_warning(f"爱站网查询失败: {response.status_code}")
|
||||
return {'title': '', 'risk': '', 'has_sensitive': False}
|
||||
except Exception as e:
|
||||
return self._handle_exception(e, domain)
|
||||
|
||||
def _extract_title(self, content):
|
||||
"""
|
||||
提取标题
|
||||
|
||||
:param content: 页面内容
|
||||
:return: str - 标题
|
||||
"""
|
||||
try:
|
||||
pattern = r'<title>(.*?)</title>'
|
||||
match = re.search(pattern, content)
|
||||
if match:
|
||||
return match.group(1).strip()
|
||||
return ''
|
||||
except Exception as e:
|
||||
self._handle_exception(e, 'extract_title')
|
||||
return ''
|
||||
|
||||
def _extract_risk(self, content):
|
||||
"""
|
||||
提取风险信息
|
||||
|
||||
:param content: 页面内容
|
||||
:return: str - 风险信息
|
||||
"""
|
||||
try:
|
||||
# 这里需要根据实际页面结构调整正则表达式
|
||||
pattern = r'百度网址检测:<span[^>]+>(.*?)</span>'
|
||||
match = re.search(pattern, content)
|
||||
if match:
|
||||
return match.group(1).strip()
|
||||
return ''
|
||||
except Exception as e:
|
||||
self._handle_exception(e, 'extract_risk')
|
||||
return ''
|
||||
|
||||
def _check_sensitive(self, title, risk):
|
||||
"""
|
||||
检查是否包含敏感词
|
||||
|
||||
:param title: 标题
|
||||
:param risk: 风险信息
|
||||
:return: bool - 是否包含敏感词
|
||||
"""
|
||||
# 风险类型
|
||||
sensitive_risks = ['低风险', '疑似色情博彩风险', '严重影响权重']
|
||||
|
||||
# 敏感词
|
||||
sensitive_words = ['色情', '赌博', '博彩', '毒品', '暴力', '诈骗']
|
||||
|
||||
# 检查风险
|
||||
for r in sensitive_risks:
|
||||
if r in risk:
|
||||
return True
|
||||
|
||||
# 检查标题
|
||||
for word in sensitive_words:
|
||||
if word in title:
|
||||
return True
|
||||
|
||||
return False
|
||||
151
domainCheck/app/detectors/baidu_detector.py
Normal file
151
domainCheck/app/detectors/baidu_detector.py
Normal file
@@ -0,0 +1,151 @@
|
||||
# -*- coding: UTF-8 -*-
|
||||
'''
|
||||
@Project :domainScanDemo
|
||||
@File :baidu_detector.py
|
||||
@IDE :PyCharm
|
||||
@Author :梦伴
|
||||
@Date :2026/4/8 23:58
|
||||
@explain : 百度检测器
|
||||
'''
|
||||
|
||||
import requests
|
||||
from curl_cffi import requests as curl_requests
|
||||
import re
|
||||
from app.detectors.base import BaseDetector
|
||||
|
||||
|
||||
class BaiduDetector(BaseDetector):
|
||||
"""
|
||||
百度检测器
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
"""
|
||||
初始化百度检测器
|
||||
"""
|
||||
super().__init__()
|
||||
self.site_url = 'https://www.baidu.com/s'
|
||||
self.history_url = 'https://www.baidu.com/s'
|
||||
|
||||
def check_domain(self, domain):
|
||||
"""
|
||||
检测域名
|
||||
|
||||
:param domain: 域名
|
||||
:return: dict - 检测结果
|
||||
"""
|
||||
try:
|
||||
# 检查百度site
|
||||
site_result = self.check_site(domain)
|
||||
|
||||
# 检查百度历史
|
||||
history_result = self.check_history(domain)
|
||||
|
||||
return {
|
||||
'site': site_result,
|
||||
'history': history_result
|
||||
}
|
||||
except Exception as e:
|
||||
return self._handle_exception(e, domain)
|
||||
|
||||
def check_site(self, domain):
|
||||
"""
|
||||
检查百度site收录
|
||||
|
||||
:param domain: 域名
|
||||
:return: dict - 检测结果
|
||||
"""
|
||||
try:
|
||||
params = {
|
||||
'wd': f'site:{domain}',
|
||||
'rn': '50'
|
||||
}
|
||||
|
||||
headers = {
|
||||
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/146.0.0.0 Safari/537.36'
|
||||
}
|
||||
|
||||
# 使用curl_cffi模拟浏览器
|
||||
response = curl_requests.get(self.site_url, params=params, headers=headers, impersonate='chrome', timeout=10)
|
||||
|
||||
if response.status_code == 200:
|
||||
content = response.text
|
||||
|
||||
# 提取子域名
|
||||
subdomains = self._extract_subdomains(content, domain)
|
||||
|
||||
# 检查是否有收录
|
||||
has_收录 = '没有找到相关结果' not in content
|
||||
|
||||
return {
|
||||
'has_收录': has_收录,
|
||||
'subdomains': subdomains
|
||||
}
|
||||
else:
|
||||
self._log_warning(f"百度site查询失败: {response.status_code}")
|
||||
return {'has_收录': False, 'subdomains': []}
|
||||
except Exception as e:
|
||||
self._handle_exception(e, domain)
|
||||
return {'has_收录': False, 'subdomains': []}
|
||||
|
||||
def check_history(self, domain):
|
||||
"""
|
||||
检查百度历史收录
|
||||
|
||||
:param domain: 域名
|
||||
:return: dict - 检测结果
|
||||
"""
|
||||
try:
|
||||
params = {
|
||||
'wd': f'cache:{domain}',
|
||||
'rn': '50'
|
||||
}
|
||||
|
||||
headers = {
|
||||
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/146.0.0.0 Safari/537.36'
|
||||
}
|
||||
|
||||
# 使用curl_cffi模拟浏览器
|
||||
response = curl_requests.get(self.history_url, params=params, headers=headers, impersonate='chrome', timeout=10)
|
||||
|
||||
if response.status_code == 200:
|
||||
content = response.text
|
||||
|
||||
# 检查是否有历史收录
|
||||
has_history = '百度快照' in content
|
||||
|
||||
# 检查是否有灰色内容
|
||||
has_gray = '风险提示' in content or '安全警告' in content
|
||||
|
||||
return {
|
||||
'has_history': has_history,
|
||||
'has_gray': has_gray
|
||||
}
|
||||
else:
|
||||
self._log_warning(f"百度历史查询失败: {response.status_code}")
|
||||
return {'has_history': False, 'has_gray': False}
|
||||
except Exception as e:
|
||||
self._handle_exception(e, domain)
|
||||
return {'has_history': False, 'has_gray': False}
|
||||
|
||||
def _extract_subdomains(self, content, domain):
|
||||
"""
|
||||
提取子域名
|
||||
|
||||
:param content: 搜索结果内容
|
||||
:param domain: 主域名
|
||||
:return: list - 子域名列表
|
||||
"""
|
||||
try:
|
||||
# 提取所有包含域名的链接
|
||||
pattern = r'https?://([a-zA-Z0-9-]+)\.' + re.escape(domain)
|
||||
matches = re.findall(pattern, content)
|
||||
|
||||
# 去重并过滤空值
|
||||
subdomains = list(set(matches))
|
||||
subdomains = [sub for sub in subdomains if sub]
|
||||
|
||||
return subdomains
|
||||
except Exception as e:
|
||||
self._handle_exception(e, domain)
|
||||
return []
|
||||
70
domainCheck/app/detectors/base.py
Normal file
70
domainCheck/app/detectors/base.py
Normal file
@@ -0,0 +1,70 @@
|
||||
# -*- coding: UTF-8 -*-
|
||||
'''
|
||||
@Project :domainScanDemo
|
||||
@File :base.py
|
||||
@IDE :PyCharm
|
||||
@Author :梦伴
|
||||
@Date :2026/4/8 23:55
|
||||
@explain : 基础检测类
|
||||
'''
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
from loguru import logger
|
||||
|
||||
|
||||
class BaseDetector(ABC):
|
||||
"""
|
||||
基础检测类
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
"""
|
||||
初始化检测类
|
||||
"""
|
||||
self.name = self.__class__.__name__
|
||||
logger.info(f"初始化检测器: {self.name}")
|
||||
|
||||
@abstractmethod
|
||||
def check_domain(self, domain):
|
||||
"""
|
||||
检测域名
|
||||
|
||||
:param domain: 域名
|
||||
:return: dict - 检测结果
|
||||
"""
|
||||
pass
|
||||
|
||||
def _log_info(self, message):
|
||||
"""
|
||||
记录信息日志
|
||||
|
||||
:param message: 消息
|
||||
"""
|
||||
logger.info(f"[{self.name}] {message}")
|
||||
|
||||
def _log_warning(self, message):
|
||||
"""
|
||||
记录警告日志
|
||||
|
||||
:param message: 消息
|
||||
"""
|
||||
logger.warning(f"[{self.name}] {message}")
|
||||
|
||||
def _log_error(self, message):
|
||||
"""
|
||||
记录错误日志
|
||||
|
||||
:param message: 消息
|
||||
"""
|
||||
logger.error(f"[{self.name}] {message}")
|
||||
|
||||
def _handle_exception(self, e, domain):
|
||||
"""
|
||||
处理异常
|
||||
|
||||
:param e: 异常
|
||||
:param domain: 域名
|
||||
:return: dict - 错误结果
|
||||
"""
|
||||
self._log_error(f"检测域名 {domain} 出错: {e}")
|
||||
return {'error': str(e)}
|
||||
130
domainCheck/app/detectors/chinaz_detector.py
Normal file
130
domainCheck/app/detectors/chinaz_detector.py
Normal file
@@ -0,0 +1,130 @@
|
||||
# -*- coding: UTF-8 -*-
|
||||
'''
|
||||
@Project :domainScanDemo
|
||||
@File :chinaz_detector.py
|
||||
@IDE :PyCharm
|
||||
@Author :梦伴
|
||||
@Date :2026/4/9 0:01
|
||||
@explain : 站长之家检测器
|
||||
'''
|
||||
|
||||
import requests
|
||||
from curl_cffi import requests as curl_requests
|
||||
import re
|
||||
from app.detectors.base import BaseDetector
|
||||
|
||||
|
||||
class ChinazDetector(BaseDetector):
|
||||
"""
|
||||
站长之家检测器
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
"""
|
||||
初始化站长之家检测器
|
||||
"""
|
||||
super().__init__()
|
||||
self.url = 'https://seo.chinaz.com'
|
||||
self.query_url = 'https://seo.chinaz.com/{domain}'
|
||||
|
||||
def check_domain(self, domain):
|
||||
"""
|
||||
检测域名
|
||||
|
||||
:param domain: 域名
|
||||
:return: dict - 检测结果
|
||||
"""
|
||||
try:
|
||||
# 构建查询URL
|
||||
query_url = self.query_url.format(domain=domain)
|
||||
|
||||
headers = {
|
||||
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/146.0.0.0 Safari/537.36'
|
||||
}
|
||||
|
||||
# 使用curl_cffi模拟浏览器
|
||||
response = curl_requests.get(query_url, headers=headers, impersonate='chrome', timeout=10)
|
||||
|
||||
if response.status_code == 200:
|
||||
content = response.text
|
||||
|
||||
# 提取标题
|
||||
title = self._extract_title(content)
|
||||
|
||||
# 提取网站分类
|
||||
category = self._extract_category(content)
|
||||
|
||||
# 检查是否包含敏感词
|
||||
has_sensitive = self._check_sensitive(title, category)
|
||||
|
||||
return {
|
||||
'title': title,
|
||||
'category': category,
|
||||
'has_sensitive': has_sensitive
|
||||
}
|
||||
else:
|
||||
self._log_warning(f"站长之家查询失败: {response.status_code}")
|
||||
return {'title': '', 'category': '', 'has_sensitive': False}
|
||||
except Exception as e:
|
||||
return self._handle_exception(e, domain)
|
||||
|
||||
def _extract_title(self, content):
|
||||
"""
|
||||
提取标题
|
||||
|
||||
:param content: 页面内容
|
||||
:return: str - 标题
|
||||
"""
|
||||
try:
|
||||
pattern = r'<title>(.*?)</title>'
|
||||
match = re.search(pattern, content)
|
||||
if match:
|
||||
return match.group(1).strip()
|
||||
return ''
|
||||
except Exception as e:
|
||||
self._handle_exception(e, 'extract_title')
|
||||
return ''
|
||||
|
||||
def _extract_category(self, content):
|
||||
"""
|
||||
提取网站分类
|
||||
|
||||
:param content: 页面内容
|
||||
:return: str - 分类
|
||||
"""
|
||||
try:
|
||||
# 这里需要根据实际页面结构调整正则表达式
|
||||
pattern = r'网站分类:<a[^>]+>(.*?)</a>'
|
||||
match = re.search(pattern, content)
|
||||
if match:
|
||||
return match.group(1).strip()
|
||||
return ''
|
||||
except Exception as e:
|
||||
self._handle_exception(e, 'extract_category')
|
||||
return ''
|
||||
|
||||
def _check_sensitive(self, title, category):
|
||||
"""
|
||||
检查是否包含敏感词
|
||||
|
||||
:param title: 标题
|
||||
:param category: 分类
|
||||
:return: bool - 是否包含敏感词
|
||||
"""
|
||||
# 敏感分类
|
||||
sensitive_categories = ['视频电影', '体育运动', '常用查询']
|
||||
|
||||
# 敏感词
|
||||
sensitive_words = ['色情', '赌博', '博彩', '毒品', '暴力', '诈骗']
|
||||
|
||||
# 检查分类
|
||||
for cat in sensitive_categories:
|
||||
if cat in category:
|
||||
return True
|
||||
|
||||
# 检查标题
|
||||
for word in sensitive_words:
|
||||
if word in title:
|
||||
return True
|
||||
|
||||
return False
|
||||
80
domainCheck/app/detectors/google_detector.py
Normal file
80
domainCheck/app/detectors/google_detector.py
Normal file
@@ -0,0 +1,80 @@
|
||||
# -*- coding: UTF-8 -*-
|
||||
'''
|
||||
@Project :domainScanDemo
|
||||
@File :google_detector.py
|
||||
@IDE :PyCharm
|
||||
@Author :梦伴
|
||||
@Date :2026/4/9 0:00
|
||||
@explain : Google检测器
|
||||
'''
|
||||
|
||||
import requests
|
||||
from curl_cffi import requests as curl_requests
|
||||
import re
|
||||
from app.detectors.base import BaseDetector
|
||||
|
||||
|
||||
class GoogleDetector(BaseDetector):
|
||||
"""
|
||||
Google检测器
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
"""
|
||||
初始化Google检测器
|
||||
"""
|
||||
super().__init__()
|
||||
self.site_url = 'https://www.google.com/search'
|
||||
|
||||
def check_domain(self, domain):
|
||||
"""
|
||||
检测域名
|
||||
|
||||
:param domain: 域名
|
||||
:return: dict - 检测结果
|
||||
"""
|
||||
try:
|
||||
# 检查Google site
|
||||
site_result = self.check_site(domain)
|
||||
|
||||
return {
|
||||
'site': site_result
|
||||
}
|
||||
except Exception as e:
|
||||
return self._handle_exception(e, domain)
|
||||
|
||||
def check_site(self, domain):
|
||||
"""
|
||||
检查Google site收录
|
||||
|
||||
:param domain: 域名
|
||||
:return: dict - 检测结果
|
||||
"""
|
||||
try:
|
||||
params = {
|
||||
'q': f'site:{domain}',
|
||||
'num': '50'
|
||||
}
|
||||
|
||||
headers = {
|
||||
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/146.0.0.0 Safari/537.36'
|
||||
}
|
||||
|
||||
# 使用curl_cffi模拟浏览器
|
||||
response = curl_requests.get(self.site_url, params=params, headers=headers, impersonate='chrome', timeout=10)
|
||||
|
||||
if response.status_code == 200:
|
||||
content = response.text
|
||||
|
||||
# 检查是否有收录
|
||||
has_收录 = 'No results found for' not in content
|
||||
|
||||
return {
|
||||
'has_收录': has_收录
|
||||
}
|
||||
else:
|
||||
self._log_warning(f"Google site查询失败: {response.status_code}")
|
||||
return {'has_收录': False}
|
||||
except Exception as e:
|
||||
self._handle_exception(e, domain)
|
||||
return {'has_收录': False}
|
||||
228
domainCheck/app/detectors/jucha_detector.py
Normal file
228
domainCheck/app/detectors/jucha_detector.py
Normal file
@@ -0,0 +1,228 @@
|
||||
# -*- coding: UTF-8 -*-
|
||||
'''
|
||||
@Project :domainScanDemo
|
||||
@File :jucha_detector.py
|
||||
@IDE :PyCharm
|
||||
@Author :梦伴
|
||||
@Date :2026/4/9 0:04
|
||||
@explain : 聚查检测器
|
||||
'''
|
||||
|
||||
import requests
|
||||
from curl_cffi import requests as curl_requests
|
||||
import re
|
||||
from app.detectors.base import BaseDetector
|
||||
|
||||
|
||||
class JuchaDetector(BaseDetector):
|
||||
"""
|
||||
聚查检测器
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
"""
|
||||
初始化聚查检测器
|
||||
"""
|
||||
super().__init__()
|
||||
self.url = 'https://www.jucha.com'
|
||||
self.whois_url = 'https://www.jucha.com/whois/{domain}'
|
||||
self.beian_url = 'https://www.jucha.com/beian/{domain}'
|
||||
self.intercept_url = 'https://www.jucha.com/intercept/{domain}'
|
||||
|
||||
def check_domain(self, domain):
|
||||
"""
|
||||
检测域名
|
||||
|
||||
:param domain: 域名
|
||||
:return: dict - 检测结果
|
||||
"""
|
||||
try:
|
||||
# 检查WHOIS
|
||||
whois_result = self.check_whois(domain)
|
||||
|
||||
# 检查备案
|
||||
beian_result = self.check_beian(domain)
|
||||
|
||||
# 检查拦截
|
||||
intercept_result = self.check_intercept(domain)
|
||||
|
||||
return {
|
||||
'whois': whois_result,
|
||||
'beian': beian_result,
|
||||
'intercept': intercept_result
|
||||
}
|
||||
except Exception as e:
|
||||
return self._handle_exception(e, domain)
|
||||
|
||||
def check_whois(self, domain):
|
||||
"""
|
||||
检查WHOIS
|
||||
|
||||
:param domain: 域名
|
||||
:return: dict - 检测结果
|
||||
"""
|
||||
try:
|
||||
# 构建查询URL
|
||||
query_url = self.whois_url.format(domain=domain)
|
||||
|
||||
headers = {
|
||||
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/146.0.0.0 Safari/537.36'
|
||||
}
|
||||
|
||||
# 使用curl_cffi模拟浏览器
|
||||
response = curl_requests.get(query_url, headers=headers, impersonate='chrome', timeout=10)
|
||||
|
||||
if response.status_code == 200:
|
||||
content = response.text
|
||||
|
||||
# 提取WHOIS信息
|
||||
whois_info = self._extract_whois_info(content)
|
||||
|
||||
return whois_info
|
||||
else:
|
||||
self._log_warning(f"聚查WHOIS查询失败: {response.status_code}")
|
||||
return {'status': ''}
|
||||
except Exception as e:
|
||||
self._handle_exception(e, domain)
|
||||
return {'status': ''}
|
||||
|
||||
def check_beian(self, domain):
|
||||
"""
|
||||
检查备案
|
||||
|
||||
:param domain: 域名
|
||||
:return: dict - 检测结果
|
||||
"""
|
||||
try:
|
||||
# 构建查询URL
|
||||
query_url = self.beian_url.format(domain=domain)
|
||||
|
||||
headers = {
|
||||
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/146.0.0.0 Safari/537.36'
|
||||
}
|
||||
|
||||
# 使用curl_cffi模拟浏览器
|
||||
response = curl_requests.get(query_url, headers=headers, impersonate='chrome', timeout=10)
|
||||
|
||||
if response.status_code == 200:
|
||||
content = response.text
|
||||
|
||||
# 提取备案信息
|
||||
beian_info = self._extract_beian_info(content)
|
||||
|
||||
return beian_info
|
||||
else:
|
||||
self._log_warning(f"聚查备案查询失败: {response.status_code}")
|
||||
return {'has_beian': False, 'beian_year': '', 'is_enterprise': False, 'beian_match': False}
|
||||
except Exception as e:
|
||||
self._handle_exception(e, domain)
|
||||
return {'has_beian': False, 'beian_year': '', 'is_enterprise': False, 'beian_match': False}
|
||||
|
||||
def check_intercept(self, domain):
|
||||
"""
|
||||
检查拦截
|
||||
|
||||
:param domain: 域名
|
||||
:return: dict - 检测结果
|
||||
"""
|
||||
try:
|
||||
# 构建查询URL
|
||||
query_url = self.intercept_url.format(domain=domain)
|
||||
|
||||
headers = {
|
||||
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/146.0.0.0 Safari/537.36'
|
||||
}
|
||||
|
||||
# 使用curl_cffi模拟浏览器
|
||||
response = curl_requests.get(query_url, headers=headers, impersonate='chrome', timeout=10)
|
||||
|
||||
if response.status_code == 200:
|
||||
content = response.text
|
||||
|
||||
# 检查是否被拦截
|
||||
is_normal = self._check_intercept_status(content)
|
||||
|
||||
return {
|
||||
'normal': is_normal
|
||||
}
|
||||
else:
|
||||
self._log_warning(f"聚查拦截查询失败: {response.status_code}")
|
||||
return {'normal': False}
|
||||
except Exception as e:
|
||||
self._handle_exception(e, domain)
|
||||
return {'normal': False}
|
||||
|
||||
def _extract_whois_info(self, content):
|
||||
"""
|
||||
提取WHOIS信息
|
||||
|
||||
:param content: 页面内容
|
||||
:return: dict - WHOIS信息
|
||||
"""
|
||||
try:
|
||||
# 提取状态信息
|
||||
pattern = r'域名状态:<span[^>]+>(.*?)</span>'
|
||||
match = re.search(pattern, content)
|
||||
status = match.group(1).strip() if match else ''
|
||||
|
||||
return {
|
||||
'status': status
|
||||
}
|
||||
except Exception as e:
|
||||
self._handle_exception(e, 'extract_whois_info')
|
||||
return {'status': ''}
|
||||
|
||||
def _extract_beian_info(self, content):
|
||||
"""
|
||||
提取备案信息
|
||||
|
||||
:param content: 页面内容
|
||||
:return: dict - 备案信息
|
||||
"""
|
||||
try:
|
||||
# 检查是否有备案
|
||||
has_beian = '备案信息' in content
|
||||
|
||||
# 提取备案年份
|
||||
beian_year = ''
|
||||
pattern = r'审核时间:(\d{4})-\d{2}-\d{2}'
|
||||
match = re.search(pattern, content)
|
||||
if match:
|
||||
beian_year = match.group(1)
|
||||
|
||||
# 提取单位性质
|
||||
is_enterprise = '企业' in content
|
||||
|
||||
# 检查首网址和备案网址是否一致
|
||||
beian_match = '网站首页网址' in content
|
||||
|
||||
return {
|
||||
'has_beian': has_beian,
|
||||
'beian_year': beian_year,
|
||||
'is_enterprise': is_enterprise,
|
||||
'beian_match': beian_match
|
||||
}
|
||||
except Exception as e:
|
||||
self._handle_exception(e, 'extract_beian_info')
|
||||
return {'has_beian': False, 'beian_year': '', 'is_enterprise': False, 'beian_match': False}
|
||||
|
||||
def _check_intercept_status(self, content):
|
||||
"""
|
||||
检查拦截状态
|
||||
|
||||
:param content: 页面内容
|
||||
:return: bool - 是否正常
|
||||
"""
|
||||
try:
|
||||
# 检查是否包含正常标识
|
||||
if '正常' in content:
|
||||
return True
|
||||
|
||||
# 检查是否包含拦截标识
|
||||
if '拦截' in content:
|
||||
return False
|
||||
|
||||
return False
|
||||
except Exception as e:
|
||||
self._handle_exception(e, 'check_intercept_status')
|
||||
return False
|
||||
214
domainCheck/app/detectors/juziseo_detector.py
Normal file
214
domainCheck/app/detectors/juziseo_detector.py
Normal file
@@ -0,0 +1,214 @@
|
||||
# -*- coding: UTF-8 -*-
|
||||
'''
|
||||
@Project :domainScanDemo
|
||||
@File :juziseo_detector.py
|
||||
@IDE :PyCharm
|
||||
@Author :梦伴
|
||||
@Date :2026/4/9 0:03
|
||||
@explain : 桔子SEO检测器
|
||||
'''
|
||||
|
||||
import requests
|
||||
from curl_cffi import requests as curl_requests
|
||||
import re
|
||||
from app.detectors.base import BaseDetector
|
||||
|
||||
|
||||
class JuziseoDetector(BaseDetector):
|
||||
"""
|
||||
桔子SEO检测器
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
"""
|
||||
初始化桔子SEO检测器
|
||||
"""
|
||||
super().__init__()
|
||||
self.url = 'https://seo.juziseo.com'
|
||||
self.history_url = 'https://seo.juziseo.com/history/{domain}'
|
||||
self.backlink_url = 'https://seo.juziseo.com/backlink/{domain}'
|
||||
|
||||
def check_domain(self, domain):
|
||||
"""
|
||||
检测域名
|
||||
|
||||
:param domain: 域名
|
||||
:return: dict - 检测结果
|
||||
"""
|
||||
try:
|
||||
# 检查历史信息
|
||||
history_result = self.check_history(domain)
|
||||
|
||||
# 检查外链
|
||||
backlink_result = self.check_backlink(domain)
|
||||
|
||||
return {
|
||||
'history': history_result,
|
||||
'backlink': backlink_result
|
||||
}
|
||||
except Exception as e:
|
||||
return self._handle_exception(e, domain)
|
||||
|
||||
def check_history(self, domain):
|
||||
"""
|
||||
检查历史信息
|
||||
|
||||
:param domain: 域名
|
||||
:return: dict - 检测结果
|
||||
"""
|
||||
try:
|
||||
# 构建查询URL
|
||||
query_url = self.history_url.format(domain=domain)
|
||||
|
||||
headers = {
|
||||
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/146.0.0.0 Safari/537.36'
|
||||
}
|
||||
|
||||
# 使用curl_cffi模拟浏览器
|
||||
response = curl_requests.get(query_url, headers=headers, impersonate='chrome', timeout=10)
|
||||
|
||||
if response.status_code == 200:
|
||||
content = response.text
|
||||
|
||||
# 提取历史信息
|
||||
history_info = self._extract_history_info(content)
|
||||
|
||||
# 检查是否包含敏感词
|
||||
has_sensitive = self._check_sensitive(history_info)
|
||||
|
||||
# 检查是否有百度历史收录
|
||||
has_baidu_history = '百度历史收录' in content
|
||||
|
||||
# 检查是否有子域名
|
||||
has_subdomains = '子域名' in content
|
||||
|
||||
# 检查是否为简体中文
|
||||
is_simplified = self._check_simplified(content)
|
||||
|
||||
return {
|
||||
'has_sensitive': has_sensitive,
|
||||
'has_baidu_history': has_baidu_history,
|
||||
'has_subdomains': has_subdomains,
|
||||
'is_simplified': is_simplified
|
||||
}
|
||||
else:
|
||||
self._log_warning(f"桔子SEO历史查询失败: {response.status_code}")
|
||||
return {'has_sensitive': False, 'has_baidu_history': False, 'has_subdomains': False, 'is_simplified': True}
|
||||
except Exception as e:
|
||||
self._handle_exception(e, domain)
|
||||
return {'has_sensitive': False, 'has_baidu_history': False, 'has_subdomains': False, 'is_simplified': True}
|
||||
|
||||
def check_backlink(self, domain):
|
||||
"""
|
||||
检查外链
|
||||
|
||||
:param domain: 域名
|
||||
:return: dict - 检测结果
|
||||
"""
|
||||
try:
|
||||
# 构建查询URL
|
||||
query_url = self.backlink_url.format(domain=domain)
|
||||
|
||||
headers = {
|
||||
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/146.0.0.0 Safari/537.36'
|
||||
}
|
||||
|
||||
# 使用curl_cffi模拟浏览器
|
||||
response = curl_requests.get(query_url, headers=headers, impersonate='chrome', timeout=10)
|
||||
|
||||
if response.status_code == 200:
|
||||
content = response.text
|
||||
|
||||
# 检查是否包含敏感词
|
||||
has_sensitive = self._check_backlink_sensitive(content)
|
||||
|
||||
# 检查是否有子域名
|
||||
has_subdomains = '子域名' in content
|
||||
|
||||
return {
|
||||
'has_sensitive': has_sensitive,
|
||||
'has_subdomains': has_subdomains
|
||||
}
|
||||
else:
|
||||
self._log_warning(f"桔子SEO外链查询失败: {response.status_code}")
|
||||
return {'has_sensitive': False, 'has_subdomains': False}
|
||||
except Exception as e:
|
||||
self._handle_exception(e, domain)
|
||||
return {'has_sensitive': False, 'has_subdomains': False}
|
||||
|
||||
def _extract_history_info(self, content):
|
||||
"""
|
||||
提取历史信息
|
||||
|
||||
:param content: 页面内容
|
||||
:return: str - 历史信息
|
||||
"""
|
||||
try:
|
||||
# 这里需要根据实际页面结构调整正则表达式
|
||||
pattern = r'<div class="history-info">(.*?)</div>'
|
||||
match = re.search(pattern, content, re.DOTALL)
|
||||
if match:
|
||||
return match.group(1).strip()
|
||||
return ''
|
||||
except Exception as e:
|
||||
self._handle_exception(e, 'extract_history_info')
|
||||
return ''
|
||||
|
||||
def _check_sensitive(self, history_info):
|
||||
"""
|
||||
检查是否包含敏感词
|
||||
|
||||
:param history_info: 历史信息
|
||||
:return: bool - 是否包含敏感词
|
||||
"""
|
||||
# 敏感词
|
||||
sensitive_words = [
|
||||
'色情', '赌博', '博彩', '毒品', '暴力', '诈骗',
|
||||
'足球', '直播', '证券', '配资', '软件',
|
||||
'体育', '商行', '下载', '影视', '网络',
|
||||
'计算', 'app', 'HTML SiteMap', '模拟器', '传媒',
|
||||
'二次元', '成人', '米乐', '小说', '凯发',
|
||||
'人才', '华体', '娱乐', '开户'
|
||||
]
|
||||
|
||||
for word in sensitive_words:
|
||||
if word in history_info:
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
def _check_simplified(self, content):
|
||||
"""
|
||||
检查是否为简体中文
|
||||
|
||||
:param content: 页面内容
|
||||
:return: bool - 是否为简体中文
|
||||
"""
|
||||
# 检查是否包含简体中文标识
|
||||
if '简体中文' in content:
|
||||
return True
|
||||
|
||||
# 检查是否包含繁体中文标识
|
||||
if '繁体中文' in content:
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
def _check_backlink_sensitive(self, content):
|
||||
"""
|
||||
检查外链是否包含敏感词
|
||||
|
||||
:param content: 页面内容
|
||||
:return: bool - 是否包含敏感词
|
||||
"""
|
||||
# 敏感词
|
||||
sensitive_words = [
|
||||
'内幕', '猛料', '精料', '高手', '绝杀',
|
||||
'权威', '澳门', '色情', '赌博', '博彩'
|
||||
]
|
||||
|
||||
for word in sensitive_words:
|
||||
if word in content:
|
||||
return True
|
||||
|
||||
return False
|
||||
107
domainCheck/app/detectors/qihu360_detector.py
Normal file
107
domainCheck/app/detectors/qihu360_detector.py
Normal file
@@ -0,0 +1,107 @@
|
||||
# -*- coding: UTF-8 -*-
|
||||
'''
|
||||
@Project :domainScanDemo
|
||||
@File :qihu360_detector.py
|
||||
@IDE :PyCharm
|
||||
@Author :梦伴
|
||||
@Date :2026/4/8 23:59
|
||||
@explain : 360检测器
|
||||
'''
|
||||
|
||||
import requests
|
||||
from curl_cffi import requests as curl_requests
|
||||
import re
|
||||
from app.detectors.base import BaseDetector
|
||||
|
||||
|
||||
class Qihu360Detector(BaseDetector):
|
||||
"""
|
||||
360检测器
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
"""
|
||||
初始化360检测器
|
||||
"""
|
||||
super().__init__()
|
||||
self.site_url = 'https://www.so.com/s'
|
||||
|
||||
def check_domain(self, domain):
|
||||
"""
|
||||
检测域名
|
||||
|
||||
:param domain: 域名
|
||||
:return: dict - 检测结果
|
||||
"""
|
||||
try:
|
||||
# 检查360 site
|
||||
site_result = self.check_site(domain)
|
||||
|
||||
return {
|
||||
'site': site_result
|
||||
}
|
||||
except Exception as e:
|
||||
return self._handle_exception(e, domain)
|
||||
|
||||
def check_site(self, domain):
|
||||
"""
|
||||
检查360 site收录
|
||||
|
||||
:param domain: 域名
|
||||
:return: dict - 检测结果
|
||||
"""
|
||||
try:
|
||||
params = {
|
||||
'q': f'site:{domain}',
|
||||
'pn': '1',
|
||||
'rn': '50'
|
||||
}
|
||||
|
||||
headers = {
|
||||
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/146.0.0.0 Safari/537.36'
|
||||
}
|
||||
|
||||
# 使用curl_cffi模拟浏览器
|
||||
response = curl_requests.get(self.site_url, params=params, headers=headers, impersonate='chrome', timeout=10)
|
||||
|
||||
if response.status_code == 200:
|
||||
content = response.text
|
||||
|
||||
# 提取子域名
|
||||
subdomains = self._extract_subdomains(content, domain)
|
||||
|
||||
# 检查是否有收录
|
||||
has_收录 = '没有找到相关结果' not in content
|
||||
|
||||
return {
|
||||
'has_收录': has_收录,
|
||||
'subdomains': subdomains
|
||||
}
|
||||
else:
|
||||
self._log_warning(f"360 site查询失败: {response.status_code}")
|
||||
return {'has_收录': False, 'subdomains': []}
|
||||
except Exception as e:
|
||||
self._handle_exception(e, domain)
|
||||
return {'has_收录': False, 'subdomains': []}
|
||||
|
||||
def _extract_subdomains(self, content, domain):
|
||||
"""
|
||||
提取子域名
|
||||
|
||||
:param content: 搜索结果内容
|
||||
:param domain: 主域名
|
||||
:return: list - 子域名列表
|
||||
"""
|
||||
try:
|
||||
# 提取所有包含域名的链接
|
||||
pattern = r'https?://([a-zA-Z0-9-]+)\.' + re.escape(domain)
|
||||
matches = re.findall(pattern, content)
|
||||
|
||||
# 去重并过滤空值
|
||||
subdomains = list(set(matches))
|
||||
subdomains = [sub for sub in subdomains if sub]
|
||||
|
||||
return subdomains
|
||||
except Exception as e:
|
||||
self._handle_exception(e, domain)
|
||||
return []
|
||||
128
domainCheck/app/detectors/rdap_detector.py
Normal file
128
domainCheck/app/detectors/rdap_detector.py
Normal file
@@ -0,0 +1,128 @@
|
||||
# -*- coding: UTF-8 -*-
|
||||
'''
|
||||
@Project :domainScanDemo
|
||||
@File :rdap_detector.py
|
||||
@IDE :PyCharm
|
||||
@Author :梦伴
|
||||
@Date :2026/4/8 23:56
|
||||
@explain : RDAP检测器
|
||||
'''
|
||||
|
||||
import requests
|
||||
from app.detectors.base import BaseDetector
|
||||
|
||||
|
||||
class RDAPDetector(BaseDetector):
|
||||
"""
|
||||
RDAP检测器
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
"""
|
||||
初始化RDAP检测器
|
||||
"""
|
||||
super().__init__()
|
||||
self.rdap_urls = {
|
||||
'com': 'https://rdap.verisign.com/com/v1/domain/',
|
||||
'net': 'https://rdap.verisign.com/net/v1/domain/'
|
||||
}
|
||||
|
||||
def check_domain(self, domain):
|
||||
"""
|
||||
检测域名
|
||||
|
||||
:param domain: 域名
|
||||
:return: dict - 检测结果
|
||||
"""
|
||||
try:
|
||||
# 提取顶级域名
|
||||
tld = domain.split('.')[-1]
|
||||
if tld not in self.rdap_urls:
|
||||
return {'error': '不支持的顶级域名'}
|
||||
|
||||
# 构建RDAP查询URL
|
||||
url = self.rdap_urls[tld] + domain
|
||||
|
||||
# 发送请求
|
||||
response = requests.get(url, timeout=10)
|
||||
|
||||
if response.status_code == 200:
|
||||
data = response.json()
|
||||
return self._parse_rdap_response(data)
|
||||
elif response.status_code == 404:
|
||||
return {'status': 'available'}
|
||||
else:
|
||||
return {'error': f'RDAP查询失败: {response.status_code}'}
|
||||
except Exception as e:
|
||||
return self._handle_exception(e, domain)
|
||||
|
||||
def check_register_status(self, domain):
|
||||
"""
|
||||
检查注册状态
|
||||
|
||||
:param domain: 域名
|
||||
:return: int - 注册状态码
|
||||
"""
|
||||
try:
|
||||
result = self.check_domain(domain)
|
||||
|
||||
if 'error' in result:
|
||||
return 9 # 状态未知
|
||||
|
||||
if result.get('status') == 'available':
|
||||
return 2 # 可注册
|
||||
|
||||
# 检查域名状态
|
||||
statuses = result.get('status', [])
|
||||
if 'clientHold' in statuses:
|
||||
return 7 # clientHold
|
||||
elif 'serverHold' in statuses:
|
||||
return 8 # serverHold
|
||||
elif 'autoRenewPeriod' in statuses:
|
||||
return 4 # 宽限期
|
||||
elif 'redemptionPeriod' in statuses:
|
||||
return 5 # 赎回期
|
||||
elif 'pendingDelete' in statuses:
|
||||
return 6 # 删除期
|
||||
else:
|
||||
return 3 # 已注册
|
||||
except Exception as e:
|
||||
self._handle_exception(e, domain)
|
||||
return 10 # 检测失败
|
||||
|
||||
def _parse_rdap_response(self, data):
|
||||
"""
|
||||
解析RDAP响应
|
||||
|
||||
:param data: RDAP响应数据
|
||||
:return: dict - 解析结果
|
||||
"""
|
||||
result = {
|
||||
'status': 'registered',
|
||||
'domain': data.get('ldhName'),
|
||||
'statuses': data.get('status', []),
|
||||
'registrar': None,
|
||||
'creation_date': None,
|
||||
'expiration_date': None,
|
||||
'last_update': None
|
||||
}
|
||||
|
||||
# 解析注册商信息
|
||||
for entity in data.get('entities', []):
|
||||
if 'registrar' in entity.get('roles', []):
|
||||
result['registrar'] = entity.get('vcardArray', [[], []])[1][1][3]
|
||||
break
|
||||
|
||||
# 解析时间信息
|
||||
for event in data.get('events', []):
|
||||
event_action = event.get('eventAction')
|
||||
event_date = event.get('eventDate')
|
||||
|
||||
if event_action == 'registration':
|
||||
result['creation_date'] = event_date
|
||||
elif event_action == 'expiration':
|
||||
result['expiration_date'] = event_date
|
||||
elif event_action == 'last update':
|
||||
result['last_update'] = event_date
|
||||
|
||||
return result
|
||||
562
domainCheck/app/detectors/wayback_detector.py
Normal file
562
domainCheck/app/detectors/wayback_detector.py
Normal file
@@ -0,0 +1,562 @@
|
||||
# -*- coding: UTF-8 -*-
|
||||
'''
|
||||
@Project :domainScanDemo
|
||||
@File :wayback_detector.py
|
||||
@IDE :PyCharm
|
||||
@Author :梦伴
|
||||
@Date :2026/4/8 23:57
|
||||
@explain : Wayback检测器
|
||||
'''
|
||||
|
||||
import html
|
||||
import json
|
||||
import re
|
||||
import threading
|
||||
import time
|
||||
import zlib
|
||||
from base64 import b64decode, b64encode
|
||||
from concurrent.futures import FIRST_COMPLETED, ThreadPoolExecutor, wait
|
||||
|
||||
import redis
|
||||
import requests
|
||||
from requests.adapters import HTTPAdapter
|
||||
from urllib3.util.retry import Retry
|
||||
|
||||
from app.config import config
|
||||
from app.detectors.base import BaseDetector
|
||||
|
||||
|
||||
class WaybackDetector(BaseDetector):
|
||||
"""
|
||||
Wayback检测器
|
||||
"""
|
||||
|
||||
TITLE_PATTERN = re.compile(r'<title[^>]*>(.*?)</title>', re.IGNORECASE | re.DOTALL)
|
||||
|
||||
def __init__(self):
|
||||
"""
|
||||
初始化Wayback检测器
|
||||
"""
|
||||
super().__init__()
|
||||
self.cdx_api_url = 'https://web.archive.org/cdx/search/cdx'
|
||||
# 使用 id_ 直接取快照内容,避免回放页面额外壳层干扰正文匹配。
|
||||
self.snapshot_url = 'https://web.archive.org/web/{timestamp}id_/{domain}'
|
||||
self._timestamp_cache = {}
|
||||
self._title_cache = {}
|
||||
self._cache_lock = threading.Lock()
|
||||
self.session = self._build_session()
|
||||
self.redis_client = self._build_redis_client()
|
||||
|
||||
def _build_session(self):
|
||||
session = requests.Session()
|
||||
retry = Retry(
|
||||
total=max(0, config.WAYBACK_RETRY_COUNT),
|
||||
backoff_factor=0.5,
|
||||
status_forcelist=(429, 500, 502, 503, 504),
|
||||
allowed_methods=frozenset(["GET"]),
|
||||
raise_on_status=False,
|
||||
)
|
||||
adapter = HTTPAdapter(max_retries=retry, pool_connections=10, pool_maxsize=10)
|
||||
session.mount('http://', adapter)
|
||||
session.mount('https://', adapter)
|
||||
session.headers.update({
|
||||
'User-Agent': config.WAYBACK_USER_AGENT,
|
||||
'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8',
|
||||
})
|
||||
return session
|
||||
|
||||
def _build_redis_client(self):
|
||||
try:
|
||||
client = redis.Redis(
|
||||
host=config.REDIS_HOST,
|
||||
port=config.REDIS_PORT,
|
||||
password=config.REDIS_PASSWORD,
|
||||
db=config.REDIS_DB,
|
||||
decode_responses=True,
|
||||
socket_connect_timeout=5,
|
||||
socket_timeout=5,
|
||||
)
|
||||
client.ping()
|
||||
return client
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
def _title_cache_key(self, domain, timestamp):
|
||||
return f"domain_tool:wayback_title:{domain}:{timestamp}"
|
||||
|
||||
def _timestamp_cache_key(self, domain):
|
||||
return f"domain_tool:wayback_timestamps:{domain}"
|
||||
|
||||
def _record_cache_key(self, domain):
|
||||
return f"domain_tool:wayback_records:{domain}"
|
||||
|
||||
def _normalize_title(self, title):
|
||||
normalized = html.unescape(title or '')
|
||||
normalized = re.sub(r'\s+', ' ', normalized, flags=re.DOTALL).strip().lower()
|
||||
return normalized
|
||||
|
||||
def _extract_title(self, content):
|
||||
match = self.TITLE_PATTERN.search(content or '')
|
||||
if not match:
|
||||
return ''
|
||||
return html.unescape(match.group(1)).strip()
|
||||
|
||||
def _load_cached_title(self, domain, timestamp):
|
||||
cache_key = self._title_cache_key(domain, timestamp)
|
||||
with self._cache_lock:
|
||||
if cache_key in self._title_cache:
|
||||
return self._title_cache[cache_key]
|
||||
if self.redis_client:
|
||||
try:
|
||||
raw_value = self.redis_client.get(cache_key)
|
||||
if raw_value:
|
||||
data = json.loads(raw_value)
|
||||
with self._cache_lock:
|
||||
self._title_cache[cache_key] = data
|
||||
return data
|
||||
except Exception:
|
||||
pass
|
||||
return None
|
||||
|
||||
def _save_cached_title(self, domain, timestamp, data):
|
||||
cache_key = self._title_cache_key(domain, timestamp)
|
||||
with self._cache_lock:
|
||||
self._title_cache[cache_key] = data
|
||||
if self.redis_client:
|
||||
try:
|
||||
self.redis_client.set(
|
||||
cache_key,
|
||||
json.dumps(data, ensure_ascii=False),
|
||||
ex=max(0, config.WAYBACK_TITLE_CACHE_TTL) or None,
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
def _load_cached_timestamps(self, domain):
|
||||
if domain in self._timestamp_cache:
|
||||
return list(self._timestamp_cache[domain])
|
||||
if not self.redis_client:
|
||||
return None
|
||||
try:
|
||||
raw_value = self.redis_client.get(self._timestamp_cache_key(domain))
|
||||
if not raw_value:
|
||||
return None
|
||||
compressed = b64decode(raw_value.encode('ascii'))
|
||||
timestamps = json.loads(zlib.decompress(compressed).decode('utf-8'))
|
||||
if isinstance(timestamps, list):
|
||||
self._timestamp_cache[domain] = tuple(timestamps)
|
||||
return list(timestamps)
|
||||
except Exception:
|
||||
return None
|
||||
return None
|
||||
|
||||
def _save_cached_timestamps(self, domain, timestamps):
|
||||
self._timestamp_cache[domain] = tuple(timestamps)
|
||||
if not self.redis_client:
|
||||
return
|
||||
try:
|
||||
payload = json.dumps(timestamps, separators=(',', ':')).encode('utf-8')
|
||||
compressed = zlib.compress(payload, level=6)
|
||||
self.redis_client.set(
|
||||
self._timestamp_cache_key(domain),
|
||||
b64encode(compressed).decode('ascii'),
|
||||
ex=max(0, config.WAYBACK_TIMESTAMP_CACHE_TTL) or None,
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
def _load_cached_records(self, domain):
|
||||
cache_key = self._record_cache_key(domain)
|
||||
with self._cache_lock:
|
||||
if cache_key in self._timestamp_cache:
|
||||
return [
|
||||
{'timestamp': item[0], 'digest': item[1]}
|
||||
for item in self._timestamp_cache[cache_key]
|
||||
]
|
||||
if not self.redis_client:
|
||||
return None
|
||||
try:
|
||||
raw_value = self.redis_client.get(cache_key)
|
||||
if not raw_value:
|
||||
return None
|
||||
compressed = b64decode(raw_value.encode('ascii'))
|
||||
records = json.loads(zlib.decompress(compressed).decode('utf-8'))
|
||||
if isinstance(records, list):
|
||||
normalized = tuple(
|
||||
(item.get('timestamp', ''), item.get('digest', ''))
|
||||
for item in records if isinstance(item, dict)
|
||||
)
|
||||
with self._cache_lock:
|
||||
self._timestamp_cache[cache_key] = normalized
|
||||
return [
|
||||
{'timestamp': item[0], 'digest': item[1]}
|
||||
for item in normalized if item[0]
|
||||
]
|
||||
except Exception:
|
||||
return None
|
||||
return None
|
||||
|
||||
def _save_cached_records(self, domain, records):
|
||||
cache_key = self._record_cache_key(domain)
|
||||
normalized = tuple(
|
||||
(item.get('timestamp', ''), item.get('digest', ''))
|
||||
for item in (records or [])
|
||||
if item and item.get('timestamp')
|
||||
)
|
||||
with self._cache_lock:
|
||||
self._timestamp_cache[cache_key] = normalized
|
||||
if not self.redis_client:
|
||||
return
|
||||
try:
|
||||
payload = json.dumps(
|
||||
[{'timestamp': item[0], 'digest': item[1]} for item in normalized],
|
||||
separators=(',', ':')
|
||||
).encode('utf-8')
|
||||
compressed = zlib.compress(payload, level=6)
|
||||
self.redis_client.set(
|
||||
cache_key,
|
||||
b64encode(compressed).decode('ascii'),
|
||||
ex=max(0, config.WAYBACK_TIMESTAMP_CACHE_TTL) or None,
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
def _fetch_cdx_records(self, domain, limit=None, fast_latest=False):
|
||||
response = None
|
||||
try:
|
||||
params = {
|
||||
'url': domain,
|
||||
'output': 'txt',
|
||||
'fl': 'timestamp,digest',
|
||||
'filter': ['statuscode:200', 'mimetype:text/html'],
|
||||
}
|
||||
if limit is not None:
|
||||
params['limit'] = str(limit)
|
||||
if fast_latest:
|
||||
params['fastLatest'] = 'true'
|
||||
response = self.session.get(
|
||||
self.cdx_api_url,
|
||||
params=params,
|
||||
timeout=config.WAYBACK_CDX_TIMEOUT,
|
||||
stream=True,
|
||||
)
|
||||
if response.status_code != 200:
|
||||
self._log_warning(f"获取快照记录失败: {response.status_code}")
|
||||
return []
|
||||
records = []
|
||||
seen = set()
|
||||
for raw_line in response.iter_lines(decode_unicode=True):
|
||||
line = (raw_line or '').strip()
|
||||
if not line:
|
||||
continue
|
||||
parts = line.split()
|
||||
timestamp = parts[0].strip() if parts else ''
|
||||
digest = parts[1].strip() if len(parts) > 1 else ''
|
||||
if not timestamp or timestamp in seen:
|
||||
continue
|
||||
seen.add(timestamp)
|
||||
records.append({'timestamp': timestamp, 'digest': digest})
|
||||
return records
|
||||
except Exception as e:
|
||||
self._handle_exception(e, domain)
|
||||
return []
|
||||
finally:
|
||||
try:
|
||||
if response is not None:
|
||||
response.close()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
def get_latest_snapshot_record(self, domain):
|
||||
records = self._fetch_cdx_records(domain, limit=-1, fast_latest=True)
|
||||
return records[0] if records else None
|
||||
|
||||
def get_snapshot_records(self, domain):
|
||||
cached_records = self._load_cached_records(domain)
|
||||
if cached_records is not None:
|
||||
return cached_records
|
||||
records = self._fetch_cdx_records(domain)
|
||||
if records:
|
||||
self._save_cached_records(domain, records)
|
||||
self._save_cached_timestamps(domain, [item['timestamp'] for item in records])
|
||||
return records
|
||||
|
||||
def check_domain(self, domain):
|
||||
"""
|
||||
检测域名
|
||||
|
||||
:param domain: 域名
|
||||
:return: dict - 检测结果
|
||||
"""
|
||||
try:
|
||||
return self.scan_snapshots(domain)
|
||||
except Exception as e:
|
||||
return self._handle_exception(e, domain)
|
||||
|
||||
def get_snapshot_timestamps(self, domain):
|
||||
cached_records = self._load_cached_records(domain)
|
||||
if cached_records is not None:
|
||||
return [item['timestamp'] for item in cached_records if item.get('timestamp')]
|
||||
cached_timestamps = self._load_cached_timestamps(domain)
|
||||
if cached_timestamps is not None:
|
||||
return cached_timestamps
|
||||
records = self.get_snapshot_records(domain)
|
||||
return [item['timestamp'] for item in records if item.get('timestamp')]
|
||||
|
||||
def _fetch_snapshot_title(self, domain, timestamp):
|
||||
cached = self._load_cached_title(domain, timestamp)
|
||||
if cached is not None:
|
||||
return cached
|
||||
|
||||
snapshot_url = self.snapshot_url.format(timestamp=timestamp, domain=domain)
|
||||
response = None
|
||||
try:
|
||||
response = self.session.get(
|
||||
snapshot_url,
|
||||
timeout=config.WAYBACK_SNAPSHOT_TIMEOUT,
|
||||
stream=True,
|
||||
)
|
||||
if response.status_code != 200:
|
||||
data = {'timestamp': timestamp, 'title': '', 'ok': False}
|
||||
self._save_cached_title(domain, timestamp, data)
|
||||
return data
|
||||
|
||||
content_type = (response.headers.get('Content-Type') or '').lower()
|
||||
if content_type and 'text/html' not in content_type and 'application/xhtml+xml' not in content_type:
|
||||
data = {'timestamp': timestamp, 'title': '', 'ok': False}
|
||||
self._save_cached_title(domain, timestamp, data)
|
||||
return data
|
||||
|
||||
chunks = []
|
||||
total_bytes = 0
|
||||
found_title = False
|
||||
for chunk in response.iter_content(chunk_size=4096, decode_unicode=True):
|
||||
if not chunk:
|
||||
continue
|
||||
chunks.append(chunk)
|
||||
total_bytes += len(chunk.encode('utf-8', errors='ignore'))
|
||||
current_text = ''.join(chunks)
|
||||
if '</title>' in current_text.lower():
|
||||
found_title = True
|
||||
break
|
||||
if total_bytes >= config.WAYBACK_TITLE_MAX_BYTES:
|
||||
break
|
||||
|
||||
content = ''.join(chunks)
|
||||
title = self._extract_title(content) if found_title or content else ''
|
||||
data = {'timestamp': timestamp, 'title': title, 'ok': True}
|
||||
self._save_cached_title(domain, timestamp, data)
|
||||
return data
|
||||
except Exception:
|
||||
data = {'timestamp': timestamp, 'title': '', 'ok': False}
|
||||
self._save_cached_title(domain, timestamp, data)
|
||||
return data
|
||||
finally:
|
||||
try:
|
||||
if response is not None:
|
||||
response.close()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
def get_snapshot_years(self, domain):
|
||||
"""
|
||||
获取快照年份
|
||||
|
||||
:param domain: 域名
|
||||
:return: list - 快照年份列表
|
||||
"""
|
||||
try:
|
||||
years = set()
|
||||
for timestamp in self.get_snapshot_timestamps(domain):
|
||||
if len(timestamp) >= 4:
|
||||
years.add(int(timestamp[:4]))
|
||||
return sorted(years)
|
||||
except Exception as e:
|
||||
self._handle_exception(e, domain)
|
||||
return []
|
||||
|
||||
def has_sensitive_content(self, domain):
|
||||
"""
|
||||
检查是否包含敏感内容
|
||||
|
||||
:param domain: 域名
|
||||
:return: bool - 是否包含敏感内容
|
||||
"""
|
||||
try:
|
||||
result = self.scan_snapshots(domain)
|
||||
return result.get('has_sensitive_content', False)
|
||||
except Exception as e:
|
||||
self._handle_exception(e, domain)
|
||||
return False
|
||||
|
||||
def scan_snapshots(self, domain, sensitive_words=None, stop_on_first_hit=True):
|
||||
sensitive_words = sensitive_words or config.load_sensitive_words()
|
||||
latest_record = self.get_latest_snapshot_record(domain)
|
||||
latest_timestamp = (latest_record or {}).get('timestamp')
|
||||
latest_digest = (latest_record or {}).get('digest', '')
|
||||
matched_word = None
|
||||
matched_timestamp = None
|
||||
matched_title = None
|
||||
fetched_snapshot_count = 0
|
||||
failed_snapshot_count = 0
|
||||
unique_title_count = 0
|
||||
duplicate_title_skipped = 0
|
||||
digest_duplicate_skipped = 0
|
||||
started_at = time.time()
|
||||
progress_interval = max(1, config.WAYBACK_PROGRESS_INTERVAL)
|
||||
title_seen = set()
|
||||
digest_seen = set()
|
||||
checked_snapshot_count = 0
|
||||
domain_concurrency = max(1, config.WAYBACK_DOMAIN_CONCURRENCY)
|
||||
|
||||
if latest_timestamp:
|
||||
latest_result = self._fetch_snapshot_title(domain, latest_timestamp)
|
||||
checked_snapshot_count = 1
|
||||
if latest_result and latest_result.get('ok'):
|
||||
fetched_snapshot_count = 1
|
||||
title = latest_result.get('title', '')
|
||||
normalized_title = self._normalize_title(title)
|
||||
if latest_digest:
|
||||
digest_seen.add(latest_digest)
|
||||
if normalized_title:
|
||||
title_seen.add(normalized_title)
|
||||
unique_title_count = 1
|
||||
matched_word = self._find_sensitive_word(title, sensitive_words)
|
||||
if matched_word and stop_on_first_hit:
|
||||
matched_timestamp = latest_timestamp
|
||||
matched_title = title
|
||||
return {
|
||||
'snapshot_years': [int(latest_timestamp[:4])] if len(latest_timestamp) >= 4 else [],
|
||||
'has_sensitive_content': True,
|
||||
'matched_word': matched_word,
|
||||
'matched_timestamp': matched_timestamp,
|
||||
'matched_title': matched_title,
|
||||
'backlink_count': 0,
|
||||
'backlink_count_gt_10': False,
|
||||
'checked_snapshot_count': checked_snapshot_count,
|
||||
'fetched_snapshot_count': fetched_snapshot_count,
|
||||
'failed_snapshot_count': failed_snapshot_count,
|
||||
'unique_title_count': unique_title_count,
|
||||
'duplicate_title_skipped': duplicate_title_skipped,
|
||||
'digest_duplicate_skipped': digest_duplicate_skipped,
|
||||
'elapsed_seconds': round(time.time() - started_at, 2),
|
||||
}
|
||||
else:
|
||||
failed_snapshot_count += 1
|
||||
|
||||
records = sorted(self.get_snapshot_records(domain), key=lambda item: item.get('timestamp', ''), reverse=True)
|
||||
years = sorted({int(item['timestamp'][:4]) for item in records if len(item.get('timestamp', '')) >= 4})
|
||||
checked_snapshot_count = len(records)
|
||||
pending_records = []
|
||||
for item in records:
|
||||
timestamp = item.get('timestamp', '')
|
||||
digest = item.get('digest', '')
|
||||
if not timestamp:
|
||||
continue
|
||||
if latest_timestamp and timestamp == latest_timestamp:
|
||||
continue
|
||||
if digest and digest in digest_seen:
|
||||
digest_duplicate_skipped += 1
|
||||
continue
|
||||
if digest:
|
||||
digest_seen.add(digest)
|
||||
pending_records.append(item)
|
||||
|
||||
with ThreadPoolExecutor(max_workers=domain_concurrency) as executor:
|
||||
pending = {}
|
||||
index = 0
|
||||
finished_count = 1 if latest_timestamp else 0
|
||||
stop_requested = False
|
||||
|
||||
while (index < len(pending_records) or pending) and not stop_requested:
|
||||
while index < len(pending_records) and len(pending) < domain_concurrency and not stop_requested:
|
||||
timestamp = pending_records[index]['timestamp']
|
||||
future = executor.submit(self._fetch_snapshot_title, domain, timestamp)
|
||||
pending[future] = timestamp
|
||||
index += 1
|
||||
|
||||
if not pending:
|
||||
break
|
||||
|
||||
done, _ = wait(list(pending.keys()), return_when=FIRST_COMPLETED)
|
||||
for future in done:
|
||||
timestamp = pending.pop(future, None)
|
||||
finished_count += 1
|
||||
try:
|
||||
result = future.result()
|
||||
except Exception as e:
|
||||
failed_snapshot_count += 1
|
||||
self._handle_exception(e, domain)
|
||||
continue
|
||||
|
||||
if not result or not result.get('ok'):
|
||||
failed_snapshot_count += 1
|
||||
continue
|
||||
|
||||
fetched_snapshot_count += 1
|
||||
title = result.get('title', '')
|
||||
normalized_title = self._normalize_title(title)
|
||||
if normalized_title:
|
||||
if normalized_title in title_seen:
|
||||
duplicate_title_skipped += 1
|
||||
else:
|
||||
title_seen.add(normalized_title)
|
||||
unique_title_count += 1
|
||||
matched_word = self._find_sensitive_word(title, sensitive_words)
|
||||
if matched_word:
|
||||
matched_timestamp = timestamp
|
||||
matched_title = title
|
||||
if stop_on_first_hit:
|
||||
stop_requested = True
|
||||
if finished_count % progress_interval == 0:
|
||||
elapsed = round(time.time() - started_at, 2)
|
||||
self._log_info(
|
||||
f"{domain} 时光机进度: {finished_count}/{checked_snapshot_count},成功 {fetched_snapshot_count},失败 {failed_snapshot_count},唯一标题 {unique_title_count},标题重复跳过 {duplicate_title_skipped},digest 重复跳过 {digest_duplicate_skipped},耗时 {elapsed}s"
|
||||
)
|
||||
if config.WAYBACK_REQUEST_DELAY > 0:
|
||||
time.sleep(config.WAYBACK_REQUEST_DELAY)
|
||||
|
||||
if stop_requested:
|
||||
for future in pending:
|
||||
future.cancel()
|
||||
|
||||
return {
|
||||
'snapshot_years': years,
|
||||
'has_sensitive_content': matched_word is not None,
|
||||
'matched_word': matched_word,
|
||||
'matched_timestamp': matched_timestamp,
|
||||
'matched_title': matched_title,
|
||||
'backlink_count': 0,
|
||||
'backlink_count_gt_10': False,
|
||||
'checked_snapshot_count': checked_snapshot_count,
|
||||
'fetched_snapshot_count': fetched_snapshot_count,
|
||||
'failed_snapshot_count': failed_snapshot_count,
|
||||
'unique_title_count': unique_title_count,
|
||||
'duplicate_title_skipped': duplicate_title_skipped,
|
||||
'digest_duplicate_skipped': digest_duplicate_skipped,
|
||||
'elapsed_seconds': round(time.time() - started_at, 2),
|
||||
}
|
||||
|
||||
def _check_sensitive_words(self, content):
|
||||
"""
|
||||
检查敏感词
|
||||
|
||||
:param content: 内容
|
||||
:return: bool - 是否包含敏感词
|
||||
"""
|
||||
return self._find_sensitive_word(content, config.load_sensitive_words()) is not None
|
||||
|
||||
def _find_sensitive_word(self, content, sensitive_words):
|
||||
for word in sensitive_words or []:
|
||||
if word and word in (content or ''):
|
||||
return word
|
||||
return None
|
||||
|
||||
def get_backlink_count(self, domain):
|
||||
"""
|
||||
当前策略仅扫描标题,不再抓取正文,友链数量默认返回 0。
|
||||
"""
|
||||
return 0
|
||||
|
||||
def _count_backlinks(self, content):
|
||||
return 0
|
||||
3
domainCheck/app/domain_suffixes.json
Normal file
3
domainCheck/app/domain_suffixes.json
Normal file
@@ -0,0 +1,3 @@
|
||||
{
|
||||
"suffixes": ".com,.net"
|
||||
}
|
||||
BIN
domainCheck/app/jucha_cookies.pkl
Normal file
BIN
domainCheck/app/jucha_cookies.pkl
Normal file
Binary file not shown.
BIN
domainCheck/app/juming_cookies.pkl
Normal file
BIN
domainCheck/app/juming_cookies.pkl
Normal file
Binary file not shown.
BIN
domainCheck/app/juziseo_cookies.pkl
Normal file
BIN
domainCheck/app/juziseo_cookies.pkl
Normal file
Binary file not shown.
68
domainCheck/app/main.py
Normal file
68
domainCheck/app/main.py
Normal file
@@ -0,0 +1,68 @@
|
||||
# -*- coding: UTF-8 -*-
|
||||
'''
|
||||
@Project :domainScanDemo
|
||||
@File :main.py
|
||||
@IDE :PyCharm
|
||||
@Author :梦伴
|
||||
@Date :2026/4/9 0:09
|
||||
@explain : 系统主入口
|
||||
'''
|
||||
|
||||
import sys
|
||||
import os
|
||||
|
||||
# 添加项目根目录到 sys.path
|
||||
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
|
||||
|
||||
from loguru import logger
|
||||
from PySide6.QtWidgets import QApplication
|
||||
from app.ui.main_window import MainWindow
|
||||
from app.utils.database import Database
|
||||
from app.config import config
|
||||
|
||||
# 配置日志
|
||||
logger.add(
|
||||
os.path.join(config.LOG_DIR, config.LOG_FILE),
|
||||
level=config.LOG_LEVEL,
|
||||
rotation="10 MB",
|
||||
compression="zip"
|
||||
)
|
||||
|
||||
|
||||
def main():
|
||||
"""
|
||||
主函数
|
||||
"""
|
||||
try:
|
||||
# 初始化数据库连接
|
||||
db = Database(
|
||||
host=config.DB_HOST,
|
||||
port=config.DB_PORT,
|
||||
database=config.DB_DATABASE,
|
||||
user=config.DB_USER,
|
||||
password=config.DB_PASSWORD
|
||||
)
|
||||
|
||||
# 测试数据库连接
|
||||
db.execute("SELECT 1")
|
||||
logger.info("数据库连接成功")
|
||||
|
||||
# 初始化应用程序
|
||||
app = QApplication(sys.argv)
|
||||
window = MainWindow()
|
||||
window.show()
|
||||
|
||||
# 运行应用程序
|
||||
sys.exit(app.exec())
|
||||
except Exception as e:
|
||||
logger.error(f"启动应用程序失败: {e}")
|
||||
sys.exit(1)
|
||||
finally:
|
||||
# 关闭数据库连接
|
||||
if 'db' in locals():
|
||||
db.close()
|
||||
|
||||
|
||||
if __name__ == "__main__":
|
||||
main()
|
||||
|
||||
4
domainCheck/app/proxy_config.json
Normal file
4
domainCheck/app/proxy_config.json
Normal file
@@ -0,0 +1,4 @@
|
||||
{
|
||||
"proxy_enable": false,
|
||||
"proxy_url": ""
|
||||
}
|
||||
15108
domainCheck/app/sdk_leg.js
Normal file
15108
domainCheck/app/sdk_leg.js
Normal file
File diff suppressed because one or more lines are too long
897
domainCheck/app/sdk_leg_env.js
Normal file
897
domainCheck/app/sdk_leg_env.js
Normal file
@@ -0,0 +1,897 @@
|
||||
process_ = process;
|
||||
require_ = require;
|
||||
delete Buffer;
|
||||
// delete process;
|
||||
delete require;
|
||||
delete global;
|
||||
delete module;
|
||||
delete exports;
|
||||
delete __filename;
|
||||
delete __dirname;
|
||||
delete SharedArrayBuffer;
|
||||
|
||||
AsObj = {
|
||||
// print: console.log,
|
||||
print: function () { },
|
||||
// print_:console.log,
|
||||
}
|
||||
|
||||
no_print = ['Boolean','String','parseFloat','Array','Object','prepareStackTrace_'];
|
||||
function watch(object, WatchName) {
|
||||
const handler = {
|
||||
get(target, property, receiver) {
|
||||
if (
|
||||
property !== 'isNaN' &&
|
||||
property !== 'encodeURI' &&
|
||||
property !== "Uint8Array" &&
|
||||
property !== 'undefined' &&
|
||||
property !== 'JSON' &&
|
||||
property !== 'Number' &&
|
||||
!no_print.includes(property) &&
|
||||
property !== Symbol.for('nodejs.util.inspect.custom') &&
|
||||
typeof property !== 'symbol'
|
||||
) {
|
||||
|
||||
if (property === 'global') {
|
||||
return undefined;
|
||||
}
|
||||
if (property === 'Buffer') {
|
||||
return undefined;
|
||||
}
|
||||
if (property === 'process') {
|
||||
return undefined;
|
||||
}
|
||||
if (WatchName === 'config_data') {
|
||||
debugger
|
||||
}
|
||||
if (WatchName.indexOf('.prototype') != -1 && target[property] != undefined) {
|
||||
return Reflect.get(target, property, receiver);
|
||||
}
|
||||
|
||||
AsObj.print(
|
||||
"方法:", "get",
|
||||
"对象:", WatchName,
|
||||
"属性:", property,
|
||||
"属性类型:", typeof property,
|
||||
"属性值:", typeof target[property] == 'object' ? "object" : target[property],
|
||||
"属性值类型:", typeof target[property]
|
||||
);
|
||||
}
|
||||
|
||||
if (WatchName === 'top') {
|
||||
return window;
|
||||
}
|
||||
|
||||
return Reflect.get(target, property, receiver);
|
||||
},
|
||||
|
||||
set(target, property, value, receiver) {
|
||||
if (WatchName.indexOf('.prototype') != -1 && value != undefined) {
|
||||
return Reflect.set(target, property, value, receiver);
|
||||
}
|
||||
AsObj.print(
|
||||
"方法:", "set",
|
||||
"对象:", WatchName,
|
||||
"属性:", property,
|
||||
"属性类型:", typeof property,
|
||||
"属性值:", typeof value == 'object' ? "object" : value,
|
||||
"属性值类型:", typeof target[property]
|
||||
);
|
||||
return Reflect.set(target, property, value, receiver);
|
||||
},
|
||||
// in操作 检测
|
||||
has(target, property) {
|
||||
AsObj.print(
|
||||
"代理对象:", WatchName,
|
||||
"方法:", "has",
|
||||
"检查属性:", property,
|
||||
"结果:", typeof target[property] == 'object' ? "object" : target[property],
|
||||
);
|
||||
return Reflect.has(target, property);
|
||||
},
|
||||
// Object.key 检测
|
||||
ownKeys(target) {
|
||||
AsObj.print(
|
||||
"方法:", "ownKeys",
|
||||
"对象:", target+''
|
||||
);
|
||||
return Reflect.ownKeys(target);
|
||||
}
|
||||
};
|
||||
|
||||
return new Proxy(object, handler);
|
||||
}
|
||||
// function watch(object, WatchName) {
|
||||
// return object
|
||||
// }
|
||||
|
||||
// 保护函数,toString检测
|
||||
const safeFunction = function safeFunction(func) {
|
||||
//处理安全函数
|
||||
Function.prototype.$call = Function.prototype.call;
|
||||
const $toString = Function.toString;
|
||||
const myFunction_toString_symbol = Symbol('('.concat('', ')'));
|
||||
|
||||
const myToString = function myToString() {
|
||||
return typeof this === 'function' && this[myFunction_toString_symbol] || $toString.$call(this);
|
||||
}
|
||||
|
||||
const set_native = function set_native(func, key, value) {
|
||||
Object.defineProperty(func, key, {
|
||||
"enumerable": false,
|
||||
"configurable": true,
|
||||
"writable": true,
|
||||
"value": value
|
||||
});
|
||||
}
|
||||
|
||||
delete Function.prototype['toString'];
|
||||
set_native(Function.prototype, "toString", myToString);
|
||||
set_native(Function.prototype.toString, myFunction_toString_symbol, "function toString() { [native code] }");
|
||||
|
||||
const safe_Function = function safe_Function(func) {
|
||||
set_native(func, myFunction_toString_symbol, "function" + (func.name ? " " + func.name : "") + "() { [native code] }");
|
||||
}
|
||||
|
||||
return safe_Function(func)
|
||||
}
|
||||
|
||||
//创建函数,并代理上
|
||||
const makeFunction = function makeFunction(name) {
|
||||
v_log = AsObj.print;
|
||||
// 使用 Function 保留函数名
|
||||
func = new Function("v_log", `
|
||||
return function ${name}() {
|
||||
v_log('函数${name}传参-->', arguments);
|
||||
};
|
||||
`)(v_log); // 传递 v_log 到动态函数
|
||||
|
||||
safeFunction(func);
|
||||
func = watch(func,`${name}`);
|
||||
func.prototype = watch(func.prototype, `${name}.prototype`);
|
||||
return func;
|
||||
}
|
||||
|
||||
!(function () {
|
||||
"use strict";
|
||||
const $toString = Function.toString;
|
||||
const myFunction_toString_symbol = Symbol('('.concat('', ')_', (Math.random() + '').toString(36)));
|
||||
const mytoString = function () {
|
||||
return typeof this == 'function' && this[myFunction_toString_symbol] || $toString.call(this);
|
||||
};
|
||||
|
||||
function set_native(func, key, value) {
|
||||
Object.defineProperty(func, key, {
|
||||
"enumerable": false,
|
||||
"configurable": true,
|
||||
"writable": true,
|
||||
"value": value
|
||||
})
|
||||
};
|
||||
delete Function.prototype['toString'];
|
||||
set_native(Function.prototype, "toString", mytoString);
|
||||
set_native(Function.prototype.toString, myFunction_toString_symbol, "function toString() { [native code] }");
|
||||
this.func_set_native = function (func) {
|
||||
set_native(func, myFunction_toString_symbol, `function ${myFunction_toString_symbol, func.name || ''}() { [native code] }`)
|
||||
}
|
||||
}).call(globalThis);
|
||||
|
||||
// 重写全局对象原型链
|
||||
function setTostringAndstringTag(obj) {
|
||||
Object.defineProperties(obj.prototype, {
|
||||
[Symbol.toStringTag]: {
|
||||
configurable: true,
|
||||
value: obj.name
|
||||
}
|
||||
});
|
||||
safeFunction(obj);
|
||||
};
|
||||
|
||||
// 创建标签原型
|
||||
function createTagProto(propObj,portotypeObj) {
|
||||
let res = propObj + ' = ' + 'function ' + propObj + '() { throw new TypeError("Illegal constructor"); };\n';
|
||||
res += 'setTostringAndstringTag(' + propObj + ',null);\n';
|
||||
if (portotypeObj) {
|
||||
for (let key in portotypeObj) {
|
||||
res += propObj + '.prototype.' + portotypeObj[key] + '= function ' + portotypeObj[key] + '() {AsObj.print("'+propObj+'.prototype.' + portotypeObj[key] + '原型方法(需在实例对象上补该方法)::",arguments)};\n';
|
||||
res += 'globalThis.func_set_native(' + propObj + '.prototype.' + portotypeObj[key] + ');\n';
|
||||
}
|
||||
}
|
||||
eval(res);
|
||||
}
|
||||
|
||||
Object.defineProperties(globalThis, {
|
||||
[Symbol.toStringTag]: {
|
||||
configurable: true,
|
||||
value: 'Window'
|
||||
}
|
||||
});
|
||||
|
||||
for (let key in globalThis) {
|
||||
if (typeof globalThis[key] === 'function') {
|
||||
safeFunction(globalThis[key])
|
||||
}
|
||||
}
|
||||
for (let key in console) {
|
||||
if (typeof console[key] === 'function') {
|
||||
safeFunction(console[key])
|
||||
}
|
||||
}
|
||||
|
||||
createTagProto('EventTarget',['addEventListener']);
|
||||
createTagProto('WindowProperties');
|
||||
createTagProto('Window');
|
||||
|
||||
window = globalThis;
|
||||
window.__proto__ = Window.prototype;
|
||||
window.__proto__.__proto__ = WindowProperties.prototype;
|
||||
window.__proto__.__proto__.__proto__ = EventTarget.prototype;
|
||||
Window.__proto__ = EventTarget;
|
||||
|
||||
Object.defineProperty(window, 'WindowProperties', {
|
||||
get: function () {
|
||||
return undefined;
|
||||
}
|
||||
})
|
||||
|
||||
function randoms(min, max) {
|
||||
return Math.floor(Math.random() * (max - min + 1) + min)
|
||||
}
|
||||
|
||||
function getRandomValues(buf) {
|
||||
var min = 0,
|
||||
max = 255;
|
||||
if (buf instanceof Uint16Array) {
|
||||
max = 65535;
|
||||
} else if (buf instanceof Uint32Array) {
|
||||
max = 4294967295;
|
||||
}
|
||||
for (var element in buf) {
|
||||
buf[element] = randoms(min, max);
|
||||
}
|
||||
return buf;
|
||||
}
|
||||
|
||||
|
||||
self = window.self = window;
|
||||
frames = window.frames = window;
|
||||
top = window.top = window;
|
||||
parent = window.parent = window;
|
||||
global = window.global = window;
|
||||
|
||||
Object.defineProperty(window, "global", {
|
||||
configurable:false,
|
||||
enumerable: true,
|
||||
set: undefined,
|
||||
get: function global(){
|
||||
return window
|
||||
}
|
||||
})
|
||||
Object.defineProperty(window, "top", {
|
||||
configurable:false,
|
||||
enumerable: true,
|
||||
set: undefined,
|
||||
get: function top(){
|
||||
return window
|
||||
}
|
||||
})
|
||||
Object.defineProperty(window, "self", {
|
||||
configurable:false,
|
||||
enumerable: true,
|
||||
set: undefined,
|
||||
get: function self(){
|
||||
return window
|
||||
}
|
||||
})
|
||||
Object.defineProperty(window, "parent", {
|
||||
configurable:false,
|
||||
enumerable: true,
|
||||
set: undefined,
|
||||
get: function parent(){
|
||||
return window
|
||||
}
|
||||
})
|
||||
Object.defineProperty(window, "frames", {
|
||||
configurable:false,
|
||||
enumerable: true,
|
||||
set: undefined,
|
||||
get: function frames(){
|
||||
return window
|
||||
}
|
||||
})
|
||||
|
||||
innerWidth = 1536
|
||||
innerHeight = 715
|
||||
outerWidth = 1536
|
||||
outerHeight = 824
|
||||
devicePixelRatio = 1.25;
|
||||
screenLeft = 0;
|
||||
screenX = 0;
|
||||
screenTop = 0;
|
||||
screenY = 0;
|
||||
opener = null;
|
||||
isSecureContext = true;
|
||||
crypto = {
|
||||
getRandomValues:getRandomValues
|
||||
};
|
||||
|
||||
createTagProto('DOMStringMap')
|
||||
createTagProto('HTMLHeadElement',['insertBefore','removeChild'])
|
||||
createTagProto('HTMLBodyElement',['addEventListener','appendChild','removeChild'])
|
||||
createTagProto('HTMLHtmlElement',['getAttribute'])
|
||||
createTagProto('HTMLDocument')
|
||||
createTagProto('Document',['browsingTopics','appendChild','querySelector','evaluate','querySelectorAll','removeChild','requestStorageAccess','requestStorageAccessFor','hasStorageAccess','getElementsByTagName','hasPrivateToken','createElement','hasRedemptionRecord','hasFocus'])
|
||||
createTagProto('Node')
|
||||
document = {};
|
||||
document.__proto__ = HTMLDocument.prototype;
|
||||
document.__proto__.__proto__ = Document.prototype;
|
||||
document.__proto__.__proto__.__proto__ = Node.prototype;
|
||||
document.__proto__.__proto__.__proto__.__proto__ = EventTarget.prototype;
|
||||
HTMLDocument.__proto__ = Document;
|
||||
HTMLDocument.__proto__.__proto__ = Node;
|
||||
HTMLDocument.__proto__.__proto__.__proto__ = EventTarget;
|
||||
Document.__proto__ = Node;
|
||||
Document.__proto__.__proto__ = EventTarget;
|
||||
Node.__proto__ = EventTarget;
|
||||
|
||||
createTagProto('Plugin');
|
||||
createTagProto('PluginArray');
|
||||
plugins0 = {
|
||||
name: 'PDF Viewer',
|
||||
filename: 'internal-pdf-viewer',
|
||||
description:'Portable Document Format',
|
||||
length: 2,
|
||||
'0': {
|
||||
type: 'application/pdf',
|
||||
},
|
||||
'1':{
|
||||
type:'text/pdf'
|
||||
}
|
||||
}
|
||||
plugins0['0'].enabledPlugin = plugins0;
|
||||
plugins0['1'].enabledPlugin = plugins0;
|
||||
plugins1 = {
|
||||
name: 'Chrome PDF Viewer',
|
||||
filename: 'internal-pdf-viewer',
|
||||
description:'Portable Document Format',
|
||||
length: 2,
|
||||
'0': {
|
||||
type:'application/pdf'
|
||||
},
|
||||
'1': {
|
||||
type:'text/pdf'
|
||||
}
|
||||
}
|
||||
plugins1['0'].enabledPlugin = plugins1;
|
||||
plugins1['1'].enabledPlugin = plugins1;
|
||||
plugins2 = {
|
||||
name: 'Chromium PDF Viewer',
|
||||
filename: 'internal-pdf-viewer',
|
||||
description:'Portable Document Format',
|
||||
length: 2,
|
||||
'0': {
|
||||
type:'application/pdf'
|
||||
},
|
||||
'1': {
|
||||
type:'text/pdf'
|
||||
}
|
||||
}
|
||||
plugins2['0'].enabledPlugin = plugins2;
|
||||
plugins2['1'].enabledPlugin = plugins2;
|
||||
plugins3 = {
|
||||
name: 'Microsoft Edge PDF Viewer',
|
||||
filename: 'internal-pdf-viewer',
|
||||
description:'Portable Document Format',
|
||||
length: 2,
|
||||
'0':{
|
||||
type:'application/pdf'
|
||||
},
|
||||
'1': {
|
||||
type:'text/pdf'
|
||||
}
|
||||
}
|
||||
plugins3['0'].enabledPlugin = plugins3;
|
||||
plugins3['1'].enabledPlugin = plugins3;
|
||||
plugins4 = {
|
||||
name: 'WebKit built-in PDF',
|
||||
filename: 'internal-pdf-viewer',
|
||||
description:'Portable Document Format',
|
||||
length: 2,
|
||||
'0': {
|
||||
type:'application/pdf'
|
||||
},
|
||||
'1':{
|
||||
type:'text/pdf'
|
||||
}
|
||||
}
|
||||
plugins4['0'].enabledPlugin = plugins4;
|
||||
plugins4['1'].enabledPlugin = plugins4;
|
||||
plugins = {
|
||||
length: 5,
|
||||
'0': plugins0,
|
||||
'1': plugins1,
|
||||
'2': plugins2,
|
||||
'3': plugins3,
|
||||
'4': plugins4,
|
||||
namedItem : function (name) {
|
||||
AsObj.print('Plugin-namedItem:', name)
|
||||
},
|
||||
item: function (index) {
|
||||
AsObj.print('Plugin-item:', index)
|
||||
return watch(plugins0,'item-'+index);
|
||||
},
|
||||
refresh: function () {
|
||||
AsObj.print('Plugin-refresh:',arguments)
|
||||
},
|
||||
}
|
||||
plugins.__proto__ = PluginArray.prototype;
|
||||
|
||||
MimeTypeArray = function MimeTypeArray() {
|
||||
this.length = 2;
|
||||
this['0'] = {
|
||||
suffixes: 'pdf',
|
||||
type: 'application/pdf',
|
||||
description:"Portable Document Format",
|
||||
enabledPlugin: plugins0
|
||||
};
|
||||
this['1'] = {
|
||||
suffixes: 'pdf',
|
||||
type: 'text/pdf',
|
||||
description:"Portable Document Format",
|
||||
enabledPlugin: plugins0
|
||||
};
|
||||
};
|
||||
MimeTypeArray.prototype.toString = function () { return '[object MimeTypeArray]'; }
|
||||
MimeTypeArray.toString = function () { return 'function MimeTypeArray() { [native code] }'; }
|
||||
Object.defineProperties(MimeTypeArray.prototype, { [Symbol.toStringTag]: { value: 'MimeTypeArray' } })
|
||||
MimeTypeArrayc = new MimeTypeArray();
|
||||
MimeTypeArrayc[Symbol.iterator] = function* () {
|
||||
for (let key in this) {
|
||||
yield this[key];
|
||||
}
|
||||
}
|
||||
|
||||
// 创建电池管理器对象原型
|
||||
const BatteryManager = {
|
||||
level: 1,
|
||||
charging: true,
|
||||
chargingTime: 0,
|
||||
dischargingTime: null,
|
||||
onchargingchange: null,
|
||||
onlevelchange: null,
|
||||
toString: function toString() {
|
||||
return `BatteryManager {
|
||||
charging: ${this.charging},
|
||||
level: ${this.level},
|
||||
chargingTime: ${this.chargingTime},
|
||||
dischargingTime: ${this.dischargingTime}
|
||||
}`
|
||||
}
|
||||
}
|
||||
window.BatteryManager = BatteryManager;
|
||||
|
||||
Promise2 = {
|
||||
then: function () {
|
||||
return this;
|
||||
},
|
||||
catch: function (){},
|
||||
};
|
||||
|
||||
createTagProto('Bluetooth');
|
||||
createTagProto('Navigator');
|
||||
Navigator.prototype.hardwareConcurrency = 8;
|
||||
Navigator.prototype.userAgent = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/139.0.0.0 Safari/537.36';
|
||||
Navigator.prototype.appVersion = '5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/139.0.0.0 Safari/537.36'
|
||||
Navigator.prototype.appName = 'Netscape';
|
||||
Navigator.prototype.appCodeName = 'Mozilla';
|
||||
Navigator.prototype.vendor = 'Google Inc.';
|
||||
Navigator.prototype.maxTouchPoints = 10;
|
||||
Navigator.prototype.platform = 'Win32';
|
||||
Navigator.prototype.adAuctionComponents = function adAuctionComponents() {
|
||||
AsObj.print('adAuctionComponents:::', arguments)
|
||||
}
|
||||
safeFunction(Navigator.prototype.adAuctionComponents)
|
||||
Navigator.prototype.runAdAuction = function runAdAuction() {
|
||||
AsObj.print('runAdAuction:::', arguments)
|
||||
}
|
||||
safeFunction(Navigator.prototype.runAdAuction)
|
||||
Navigator.prototype.canLoadAdAuctionFencedFrame = makeFunction('canLoadAdAuctionFencedFrame')
|
||||
Navigator.prototype.deprecatedReplaceInURN = makeFunction('deprecatedReplaceInURN')
|
||||
Navigator.prototype.deprecatedURNToURL = makeFunction('deprecatedURNToURL')
|
||||
Navigator.prototype.joinAdInterestGroup = makeFunction('joinAdInterestGroup')
|
||||
Navigator.prototype.leaveAdInterestGroup = makeFunction('leaveAdInterestGroup')
|
||||
Navigator.prototype.updateAdInterestGroups = makeFunction('updateAdInterestGroups')
|
||||
Navigator.prototype.connection = watch({
|
||||
downlink: 9.1,
|
||||
effectiveType: '4g',
|
||||
rtt: 0,
|
||||
saveData: false,
|
||||
},'connection')
|
||||
Navigator.prototype.language = 'zh-CN';
|
||||
Navigator.prototype.languages = ["zh-CN"];
|
||||
Navigator.prototype.plugins = plugins;
|
||||
Navigator.prototype.webdriver = false;
|
||||
Navigator.prototype.cookieEnabled = true;
|
||||
Navigator.prototype.onLine = true;
|
||||
Navigator.prototype.doNotTrack = null;
|
||||
Navigator.prototype.bluetooth = {};
|
||||
Navigator.prototype.product = 'Gecko'
|
||||
Navigator.prototype.deviceMemory = 8
|
||||
Navigator.prototype.mediaDevices = watch({
|
||||
enumerateDevices: function enumerateDevices() {
|
||||
return new Promise((resolve, reject) => {
|
||||
const offer = [
|
||||
{deviceId: '', kind: 'audioinput', label: '', groupId: ''},
|
||||
{deviceId: '', kind: 'videoinput', label: '', groupId: ''},
|
||||
{deviceId: '', kind: 'audiooutput', label: '', groupId: ''},
|
||||
]
|
||||
resolve(offer);
|
||||
});
|
||||
},
|
||||
getUserMedia: function getUserMedia() {
|
||||
AsObj.print('getUserMedia:::', arguments)
|
||||
}
|
||||
},'mediaDevices')
|
||||
Navigator.prototype.storage = {
|
||||
estimate: function estimate() {
|
||||
AsObj.print('estimate:::', arguments)
|
||||
return new Promise((resolve, reject) => {
|
||||
const offer = {
|
||||
usage: 0, // 1GB
|
||||
quota: 2147483648, // 1GB,
|
||||
usageDetails: {caches: 512, indexedDB: 2855}
|
||||
};
|
||||
resolve(offer);
|
||||
});
|
||||
}
|
||||
}
|
||||
Navigator.prototype.webkitPersistentStorage = watch({},'webkitPersistentStorage')
|
||||
Navigator.prototype.webkitTemporaryStorage = watch({
|
||||
queryUsageAndQuota: function queryUsageAndQuota() {
|
||||
AsObj.print('queryUsageAndQuota:::', arguments)
|
||||
return new Promise((resolve, reject) => {
|
||||
const offer = {
|
||||
usage: 1024 * 1024 * 1024, // 1GB
|
||||
quota: 1024 * 1024 * 1024, // 1GB
|
||||
};
|
||||
resolve(offer);
|
||||
});
|
||||
}
|
||||
},'webkitTemporaryStorage')
|
||||
Navigator.prototype.bluetooth.__proto__ = Bluetooth.prototype;
|
||||
Navigator.prototype.javaEnabled = function javaEnabled() {
|
||||
return false
|
||||
};
|
||||
safeFunction(Navigator.prototype.javaEnabled)
|
||||
Navigator.prototype.getBattery = function getBattery() {
|
||||
AsObj.print('getBattery:::', arguments)
|
||||
return Promise.resolve({
|
||||
__proto__: BatteryManager,
|
||||
// 动态参数配置(示例值)
|
||||
level: 1,
|
||||
charging: true,
|
||||
dischargingTime: null // 2小时放电时间
|
||||
})
|
||||
}
|
||||
safeFunction(Navigator.prototype.getBattery)
|
||||
Navigator.prototype.registerProtocolHandler = function registerProtocolHandler() {
|
||||
AsObj.print('registerProtocolHandler:::',arguments)
|
||||
}
|
||||
safeFunction(Navigator.prototype.registerProtocolHandler)
|
||||
Navigator.prototype.mimeTypes = watch(MimeTypeArrayc,'mimeTypes');
|
||||
Navigator.prototype.geolocation = {
|
||||
getCurrentPosition: function getCurrentPosition() {
|
||||
return Promise2;
|
||||
}
|
||||
}
|
||||
Navigator.prototype.pdfViewerEnabled = true;
|
||||
Navigator.prototype.doNotTrack = null;
|
||||
Navigator.prototype.keyboard = watch({
|
||||
getLayoutMap: function getLayoutMap() {
|
||||
AsObj.print('Navigator.prototype.keyboard:', arguments)
|
||||
return {
|
||||
then: function () {
|
||||
// arguments[0](watch({
|
||||
// size: 48,
|
||||
// values: function () {
|
||||
// return ['k', 'g', '2', '0', 'v', 'a', '`', 'l', '\\', "'", 'w', '8', 'm', 'h', '.', '7', '1', 'p', 'd', 'f', 'o', 'q', 'c', 'n', '[', 'z', 'y', '3', '6', '5', 'x', '/', '\\', ',', '-', '4', 'b', 't', '9', 's', 'i', 'u', '=', 'j', ';', 'r', ']', 'e']
|
||||
// }
|
||||
// }, 'navigator.keyboard.getLayoutMap.then'))
|
||||
return {
|
||||
catch: function () {
|
||||
arguments[0]({
|
||||
message:'getLayoutMap() must be called from a top-level browsing context or allowed by the permission policy.'
|
||||
})
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
}
|
||||
},'navigator.keyboard')
|
||||
Navigator.prototype.permissions = watch({
|
||||
query: function query() {
|
||||
arg_obj = arguments[0];
|
||||
if (arg_obj.name === 'audio_capture') {
|
||||
return new Promise((resolve, reject) => {
|
||||
const offer = {
|
||||
state: 'prompt',
|
||||
onchange: null,
|
||||
name:arg_obj.name
|
||||
};
|
||||
resolve(offer);
|
||||
});
|
||||
}
|
||||
if (arg_obj.name === 'microphone') {
|
||||
return {
|
||||
then: function () {
|
||||
arguments[0](watch({
|
||||
state: 'denied',
|
||||
onchange: null,
|
||||
name: 'audio_capture'
|
||||
},'permissions.query.microphone'));
|
||||
return {catch:function(){}}
|
||||
},
|
||||
catch:function(){}
|
||||
}
|
||||
}
|
||||
if (arg_obj.name === 'camera') {
|
||||
return {
|
||||
then: function () {
|
||||
arguments[0](watch({
|
||||
state: 'prompt',
|
||||
onchange: null,
|
||||
name: 'video_capture'
|
||||
},'permissions.query.camera'));
|
||||
return {catch:function(){}}
|
||||
},
|
||||
catch:function(){}
|
||||
}
|
||||
}
|
||||
AsObj.print('permissions.query:::', arguments)
|
||||
|
||||
}
|
||||
},'permissions')
|
||||
Navigator.prototype.productSub = '20030107'
|
||||
Navigator.prototype.getGamepads = function getGamepads() {
|
||||
AsObj.print('getGamepads:::', arguments)
|
||||
return [null,null,null,null]
|
||||
}
|
||||
safeFunction(Navigator.prototype.getGamepads)
|
||||
|
||||
Navigator.prototype.sendBeacon = makeFunction('sendBeacon')
|
||||
|
||||
Navigator.prototype.deprecatedRunAdAuctionEnforcesKAnonymity = false
|
||||
Navigator.prototype.gpu = watch({
|
||||
getPreferredCanvasFormat: function getPreferredCanvasFormat() {
|
||||
AsObj.print('gpu.getPreferredCanvasFormat:', arguments)
|
||||
return 'bgra8unorm'
|
||||
},
|
||||
wgslLanguageFeatures: watch({
|
||||
size: 7,
|
||||
values: function values() {
|
||||
debugger
|
||||
AsObj.print('wgslLanguageFeatures.values')
|
||||
return ['packed_4x8_integer_dot_product', 'unrestricted_pointer_parameters', 'subgroup_uniformity', 'subgroup_id', 'pointer_composite_access', 'readonly_and_readwrite_storage_textures', 'uniform_buffer_standard_layout']
|
||||
},
|
||||
}, 'gpu.wgslLanguageFeatures'),
|
||||
requestAdapter: function requestAdapter() {
|
||||
AsObj.print('gpu.requestAdapter:', arguments)
|
||||
return {
|
||||
then: function () {
|
||||
arguments[0](watch({
|
||||
features: watch({
|
||||
size: 19,
|
||||
values: function () {
|
||||
return ['depth32float-stencil8', 'rg11b10ufloat-renderable', 'bgra8unorm-storage', 'texture-formats-tier1', 'texture-compression-bc', 'dual-source-blending', 'core-features-and-limits', 'float32-filterable', 'indirect-first-instance', 'float32-blendable', 'depth-clip-control', 'texture-compression-bc-sliced-3d', 'timestamp-query', 'texture-formats-tier2', 'clip-distances', 'shader-f16', 'primitive-index', 'texture-component-swizzle', 'subgroups']
|
||||
}
|
||||
}, 'gpu.requestAdapter.features'),
|
||||
info: watch({ vendor: 'intel', architecture: 'gen-11', device: '', description: '', subgroupMinSize: 16 }, 'gpu.requestAdapter.info'),
|
||||
limits: watch({
|
||||
maxBufferSize: 2147483648,
|
||||
maxStorageBufferBindingSize:2147483644
|
||||
}, 'gpu.requestAdapter.limits'),
|
||||
catch:function(){}
|
||||
}, 'gpu.requestAdapter'));
|
||||
return {
|
||||
catch: function () {
|
||||
return {
|
||||
then: function () {
|
||||
arguments[0]()
|
||||
return {catch:function(){}}
|
||||
}
|
||||
}
|
||||
}
|
||||
};
|
||||
},
|
||||
catch: function () {
|
||||
}
|
||||
}
|
||||
}
|
||||
},'navigator.gpu')
|
||||
Navigator.prototype.userAgentData = watch({
|
||||
brands:[
|
||||
{
|
||||
"brand": "Google Chrome",
|
||||
"version": "143"
|
||||
},
|
||||
{
|
||||
"brand": "Chromium",
|
||||
"version": "143"
|
||||
},
|
||||
{
|
||||
"brand": "Not A(Brand",
|
||||
"version": "24"
|
||||
}
|
||||
],
|
||||
mobile: false,
|
||||
platform: "Windows",
|
||||
getHighEntropyValues: function getHighEntropyValues() {
|
||||
if (arguments[0] + '' === 'architecture,bitness,model,platformVersion,uaFullVersion,wow64') {
|
||||
return new Promise((resolve, reject) => {
|
||||
const offer = {
|
||||
"architecture": "x86",
|
||||
"bitness": "64",
|
||||
"brands": [
|
||||
{
|
||||
"brand": "Not:A-Brand",
|
||||
"version": "99"
|
||||
},
|
||||
{
|
||||
"brand": "Google Chrome",
|
||||
"version": "145"
|
||||
},
|
||||
{
|
||||
"brand": "Chromium",
|
||||
"version": "145"
|
||||
}
|
||||
],
|
||||
"mobile": false,
|
||||
"model": "",
|
||||
"platform": "Windows",
|
||||
"platformVersion": "10.0.0",
|
||||
"uaFullVersion": "145.0.7632.117",
|
||||
"wow64": false
|
||||
};
|
||||
resolve(offer);
|
||||
});
|
||||
}
|
||||
AsObj.print('getHighEntropyValues:::', arguments)
|
||||
}
|
||||
},'userAgentData')
|
||||
|
||||
navigator = {};
|
||||
navigator.__proto__ = Navigator.prototype;
|
||||
|
||||
createTagProto('Location');
|
||||
location = {
|
||||
"ancestorOrigins": {},
|
||||
"href": "https://www.neimanmarcus.com/",
|
||||
"origin": "https://www.neimanmarcus.com",
|
||||
"protocol": "https:",
|
||||
"host": "www.neimanmarcus.com",
|
||||
"hostname": "www.neimanmarcus.com",
|
||||
"port": "",
|
||||
"pathname": "/",
|
||||
"search": "",
|
||||
"hash": ""
|
||||
};
|
||||
|
||||
location.__proto__ = Location.prototype;
|
||||
location.toString = function toString() {
|
||||
return this.href;
|
||||
}
|
||||
|
||||
createTagProto('Screen');
|
||||
Screen.prototype = Object.assign(Screen.prototype, {
|
||||
availHeight: 824,
|
||||
availLeft: 0,
|
||||
availTop: 0,
|
||||
availWidth: 1536,
|
||||
colorDepth: 32,
|
||||
height: 864,
|
||||
isExtended: true,
|
||||
onchange: null,
|
||||
pixelDepth: 24,
|
||||
width: 1536,
|
||||
orientation: {
|
||||
angle: 0,
|
||||
type: "landscape-primary",
|
||||
onchange: null
|
||||
}
|
||||
})
|
||||
screen = {};
|
||||
screen.__proto__ = Screen.prototype;
|
||||
|
||||
createTagProto('History',['replaceState']);
|
||||
history = {};
|
||||
history.__proto__ = History.prototype;
|
||||
|
||||
chrome = {
|
||||
loadTimes: function loadTimes() { },
|
||||
csi: function csi() { },
|
||||
app: {
|
||||
InstallState: { DISABLED: 'disabled', INSTALLED: 'installed', NOT_INSTALLED: 'not_installed' },
|
||||
RunningState: { CANNOT_RUN: 'cannot_run', READY_TO_RUN: 'ready_to_run', RUNNING: 'running' },
|
||||
getDetails:function getDetails(){},
|
||||
getIsInstalled:function getIsInstalled(){},
|
||||
installState:function installState(){},
|
||||
isInstalled: false,
|
||||
runningState: function runningState(){}
|
||||
},
|
||||
}
|
||||
|
||||
createTagProto('Storage');
|
||||
local = {
|
||||
};
|
||||
localStorage = {
|
||||
getItem: function getItem(key) {
|
||||
AsObj.print("localStorage.getItem::", arguments);
|
||||
if (!local[key]) {
|
||||
return null;
|
||||
}
|
||||
return local[key];
|
||||
},
|
||||
setItem: function setItem(key, value) {
|
||||
AsObj.print("localStorage.setItem::", arguments);
|
||||
local[key] = value;
|
||||
},
|
||||
clear: function clear() {
|
||||
local = {};
|
||||
},
|
||||
removeItem: function removeItem(key) {
|
||||
AsObj.print("localStorage.removeItem::", arguments);
|
||||
delete local[key];
|
||||
}
|
||||
}
|
||||
localStorage.__proto__ = Storage.prototype;
|
||||
sessionStorage = {
|
||||
getItem: function getItem(key) {
|
||||
AsObj.print("sessionStorage.getItem::", arguments);
|
||||
if (!local[key]) {
|
||||
return null;
|
||||
}
|
||||
return local[key];
|
||||
},
|
||||
setItem: function setItem(key, value) {
|
||||
AsObj.print("sessionStorage.setItem::", arguments);
|
||||
local[key] = value;
|
||||
},
|
||||
clear: function clear() {
|
||||
local = {};
|
||||
},
|
||||
removeItem: function removeItem(key) {
|
||||
AsObj.print("sessionStorage.removeItem::", arguments);
|
||||
delete local[key];
|
||||
}
|
||||
}
|
||||
sessionStorage.__proto__ = Storage.prototype;
|
||||
|
||||
// window = watch(window, 'window');
|
||||
// global = watch(global, 'global');
|
||||
// globalThis = watch(globalThis, 'globalThis');
|
||||
// self = watch(self, 'self');
|
||||
// crypto = watch(crypto, 'crypto');
|
||||
// performance = watch(performance, 'performance');
|
||||
// document = watch(document, 'document');
|
||||
// navigator = watch(navigator, 'navigator');
|
||||
// location = watch(location, 'location');
|
||||
// screen = watch(screen, 'screen');
|
||||
// history = watch(history, 'history');
|
||||
// localStorage = watch(localStorage, 'localStorage');
|
||||
// sessionStorage = watch(sessionStorage, 'sessionStorage');
|
||||
// chrome = watch(chrome, 'chrome');
|
||||
|
||||
require_('./sdk_leg.js');
|
||||
|
||||
let input = '';
|
||||
// 收集数据
|
||||
process.stdin.on('data', chunk => {
|
||||
input += chunk;
|
||||
});
|
||||
process.stdin.on('end', async () => {
|
||||
var config_data = JSON.parse(input);
|
||||
var cryptoManager = await CaptchaSDKCorecc();
|
||||
var encryptData = await buildEncryptedVerifyRequestcc(config_data, cryptoManager);
|
||||
console.log(JSON.stringify(encryptData));
|
||||
process.exit(0);
|
||||
})
|
||||
3
domainCheck/app/thread_count.json
Normal file
3
domainCheck/app/thread_count.json
Normal file
@@ -0,0 +1,3 @@
|
||||
{
|
||||
"thread_count": "1"
|
||||
}
|
||||
4
domainCheck/app/ui/__init__.py
Normal file
4
domainCheck/app/ui/__init__.py
Normal file
@@ -0,0 +1,4 @@
|
||||
# -*- coding: UTF-8 -*-
|
||||
'''
|
||||
UI 模块
|
||||
'''
|
||||
1423
domainCheck/app/ui/domain_filter.py
Normal file
1423
domainCheck/app/ui/domain_filter.py
Normal file
File diff suppressed because it is too large
Load Diff
510
domainCheck/app/ui/domain_import.py
Normal file
510
domainCheck/app/ui/domain_import.py
Normal file
@@ -0,0 +1,510 @@
|
||||
# -*- coding: UTF-8 -*-
|
||||
'''
|
||||
@Project :domainScanDemo
|
||||
@File :domain_import.py
|
||||
@IDE :PyCharm
|
||||
@Author :梦伴
|
||||
@Date :2026/4/8 23:47
|
||||
@explain : 域名导入界面
|
||||
'''
|
||||
|
||||
from PySide6.QtWidgets import QWidget, QVBoxLayout, QHBoxLayout, QPushButton, QLabel, QTextEdit, QFileDialog, QProgressBar
|
||||
from PySide6.QtCore import Qt, QThread, Signal
|
||||
from loguru import logger
|
||||
|
||||
from app.core.domain_collector import DomainCollector
|
||||
|
||||
|
||||
class ImportThread(QThread):
|
||||
"""
|
||||
导入线程
|
||||
"""
|
||||
progress_updated = Signal(int)
|
||||
finished = Signal(bool, str)
|
||||
|
||||
def __init__(self, domain_list, source_type):
|
||||
"""
|
||||
初始化导入线程
|
||||
|
||||
:param domain_list: 域名列表
|
||||
:param source_type: 来源类型
|
||||
"""
|
||||
super().__init__()
|
||||
self.domain_list = domain_list
|
||||
self.source_type = source_type
|
||||
|
||||
def run(self):
|
||||
"""
|
||||
运行导入线程
|
||||
"""
|
||||
try:
|
||||
collector = DomainCollector()
|
||||
total = len(self.domain_list)
|
||||
|
||||
# 实时更新进度:开始
|
||||
self.progress_updated.emit(0)
|
||||
|
||||
# 标准化域名和检查是否存在(占30%进度)
|
||||
normalized_domains = []
|
||||
for i, domain in enumerate(self.domain_list):
|
||||
from app.utils.domain_utils import normalize_domain
|
||||
import tldextract
|
||||
normalized = normalize_domain(domain)
|
||||
if normalized:
|
||||
# 提取顶级域名
|
||||
ext = tldextract.extract(normalized)
|
||||
tld = ext.suffix
|
||||
normalized_domains.append((normalized, tld))
|
||||
|
||||
# 更新进度
|
||||
progress = int((i + 1) / total * 30)
|
||||
self.progress_updated.emit(progress)
|
||||
|
||||
# 批量检查域名是否存在
|
||||
batch_data = []
|
||||
existing_domains = []
|
||||
if normalized_domains:
|
||||
all_domains = [domain for domain, tld in normalized_domains]
|
||||
existing_domains = collector.db.check_domains_exist(all_domains)
|
||||
existing_set = set(existing_domains)
|
||||
|
||||
# 准备批量添加数据
|
||||
for domain, tld in normalized_domains:
|
||||
if domain not in existing_set:
|
||||
batch_data.append((domain, tld, self.source_type))
|
||||
|
||||
# 分批次添加域名(占70%进度)
|
||||
batch_size = 1000
|
||||
total_batches = len(batch_data)
|
||||
for i in range(0, len(batch_data), batch_size):
|
||||
batch = batch_data[i:i+batch_size]
|
||||
collector.db.add_domains_batch(batch)
|
||||
|
||||
# 更新进度
|
||||
processed = min(i + len(batch), total_batches)
|
||||
progress = 30 + int(processed / total_batches * 70)
|
||||
self.progress_updated.emit(progress)
|
||||
|
||||
# 完成导入
|
||||
self.progress_updated.emit(100)
|
||||
|
||||
# 计算统计信息
|
||||
stats = {
|
||||
'total': total,
|
||||
'valid': len(normalized_domains),
|
||||
'added': len(batch_data),
|
||||
'exists': len(existing_domains),
|
||||
'invalid': total - len(normalized_domains),
|
||||
'failed': 0
|
||||
}
|
||||
|
||||
# 根据统计信息生成消息
|
||||
if stats['added'] > 0:
|
||||
message = f"导入完成: 总域名数 {stats['total']}, 有效域名数 {stats['valid']}, 新增域名数 {stats['added']}, 已存在域名数 {stats['exists']}, 无效域名数 {stats['invalid']}"
|
||||
else:
|
||||
message = f"导入完成: 所有域名已存在,未添加新域名"
|
||||
|
||||
self.finished.emit(True, message)
|
||||
except Exception as e:
|
||||
logger.error(f"导入失败: {e}")
|
||||
self.finished.emit(False, f"导入失败: {str(e)}")
|
||||
|
||||
|
||||
class ImportFileThread(QThread):
|
||||
"""
|
||||
文件导入线程,用于处理大文件
|
||||
"""
|
||||
progress_updated = Signal(int)
|
||||
finished = Signal(bool, str)
|
||||
|
||||
def __init__(self, file_path, source_type):
|
||||
"""
|
||||
初始化文件导入线程
|
||||
|
||||
:param file_path: 文件路径
|
||||
:param source_type: 来源类型
|
||||
"""
|
||||
super().__init__()
|
||||
self.file_path = file_path
|
||||
self.source_type = source_type
|
||||
|
||||
def run(self):
|
||||
"""
|
||||
运行文件导入线程
|
||||
"""
|
||||
try:
|
||||
collector = DomainCollector()
|
||||
|
||||
# 首先计算文件中的域名数量
|
||||
total = 0
|
||||
encodings = ['utf-8', 'utf-8-sig', 'gbk', 'gb2312', 'cp936', 'latin-1', 'ascii']
|
||||
encoding = 'utf-8' # 默认编码
|
||||
|
||||
# 尝试不同的编码格式计算域名数量
|
||||
for enc in encodings:
|
||||
try:
|
||||
with open(self.file_path, 'r', encoding=enc) as f:
|
||||
total = sum(1 for line in f if line.strip())
|
||||
encoding = enc
|
||||
break
|
||||
except UnicodeDecodeError:
|
||||
continue
|
||||
|
||||
if total == 0:
|
||||
# 尝试使用二进制模式读取
|
||||
try:
|
||||
import chardet
|
||||
with open(self.file_path, 'rb') as f:
|
||||
raw_data = f.read()
|
||||
result = chardet.detect(raw_data)
|
||||
encoding = result['encoding']
|
||||
if encoding:
|
||||
total = sum(1 for line in raw_data.decode(encoding).split('\n') if line.strip())
|
||||
else:
|
||||
# 最后尝试使用 replace 模式读取
|
||||
with open(self.file_path, 'r', encoding='utf-8', errors='replace') as f:
|
||||
total = sum(1 for line in f if line.strip())
|
||||
encoding = 'utf-8'
|
||||
except Exception:
|
||||
# 最后尝试使用 replace 模式读取
|
||||
with open(self.file_path, 'r', encoding='utf-8', errors='replace') as f:
|
||||
total = sum(1 for line in f if line.strip())
|
||||
encoding = 'utf-8'
|
||||
|
||||
# 实时更新进度:开始
|
||||
self.progress_updated.emit(0)
|
||||
|
||||
# 逐行读取文件并处理域名
|
||||
normalized_domains = []
|
||||
processed = 0
|
||||
|
||||
with open(self.file_path, 'r', encoding=encoding, errors='replace') as f:
|
||||
for line in f:
|
||||
domain = line.strip()
|
||||
if domain:
|
||||
from app.utils.domain_utils import normalize_domain
|
||||
import tldextract
|
||||
normalized = normalize_domain(domain)
|
||||
if normalized:
|
||||
# 提取顶级域名
|
||||
ext = tldextract.extract(normalized)
|
||||
tld = ext.suffix
|
||||
normalized_domains.append((normalized, tld))
|
||||
|
||||
processed += 1
|
||||
# 更新进度(占30%)
|
||||
progress = int(processed / total * 30)
|
||||
self.progress_updated.emit(progress)
|
||||
|
||||
# 批量检查域名是否存在
|
||||
batch_data = []
|
||||
existing_domains = []
|
||||
if normalized_domains:
|
||||
all_domains = [domain for domain, tld in normalized_domains]
|
||||
existing_domains = collector.db.check_domains_exist(all_domains)
|
||||
existing_set = set(existing_domains)
|
||||
|
||||
# 准备批量添加数据
|
||||
for domain, tld in normalized_domains:
|
||||
if domain not in existing_set:
|
||||
batch_data.append((domain, tld, self.source_type))
|
||||
|
||||
# 分批次添加域名(占70%进度)
|
||||
batch_size = 1000
|
||||
total_batches = len(batch_data)
|
||||
for i in range(0, len(batch_data), batch_size):
|
||||
batch = batch_data[i:i+batch_size]
|
||||
collector.db.add_domains_batch(batch)
|
||||
|
||||
# 更新进度
|
||||
processed_batches = min(i + len(batch), total_batches)
|
||||
progress = 30 + int(processed_batches / total_batches * 70)
|
||||
self.progress_updated.emit(progress)
|
||||
|
||||
# 完成导入
|
||||
self.progress_updated.emit(100)
|
||||
|
||||
# 计算统计信息
|
||||
stats = {
|
||||
'total': total,
|
||||
'valid': len(normalized_domains),
|
||||
'added': len(batch_data),
|
||||
'exists': len(existing_domains),
|
||||
'invalid': total - len(normalized_domains),
|
||||
'failed': 0
|
||||
}
|
||||
|
||||
# 根据统计信息生成消息
|
||||
if stats['added'] > 0:
|
||||
message = f"导入完成: 总域名数 {stats['total']}, 有效域名数 {stats['valid']}, 新增域名数 {stats['added']}, 已存在域名数 {stats['exists']}, 无效域名数 {stats['invalid']}"
|
||||
else:
|
||||
message = f"导入完成: 所有域名已存在,未添加新域名"
|
||||
|
||||
self.finished.emit(True, message)
|
||||
except Exception as e:
|
||||
logger.error(f"导入失败: {e}")
|
||||
self.finished.emit(False, f"导入失败: {str(e)}")
|
||||
|
||||
|
||||
class DomainImportWidget(QWidget):
|
||||
"""
|
||||
域名导入界面
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
"""
|
||||
初始化域名导入界面
|
||||
"""
|
||||
super().__init__()
|
||||
|
||||
# 创建布局
|
||||
layout = QVBoxLayout(self)
|
||||
|
||||
|
||||
|
||||
# 创建文本编辑框
|
||||
self.text_edit = QTextEdit()
|
||||
self.text_edit.setPlaceholderText("请输入域名,一行一个")
|
||||
self.text_edit.setStyleSheet("""
|
||||
QTextEdit {
|
||||
font-size: 14px;
|
||||
padding: 10px;
|
||||
border: 1px solid #ddd;
|
||||
border-radius: 4px;
|
||||
background-color: #f9f9f9;
|
||||
min-height: 300px;
|
||||
}
|
||||
""")
|
||||
layout.addWidget(self.text_edit)
|
||||
|
||||
# 创建按钮布局
|
||||
button_layout = QHBoxLayout()
|
||||
|
||||
# 导入文件按钮
|
||||
self.import_file_btn = QPushButton("导入文件")
|
||||
self.import_file_btn.clicked.connect(self.import_file)
|
||||
self.import_file_btn.setStyleSheet("""
|
||||
QPushButton {
|
||||
font-size: 14px;
|
||||
padding: 8px 16px;
|
||||
background-color: #2196F3;
|
||||
color: white;
|
||||
border: none;
|
||||
border-radius: 4px;
|
||||
}
|
||||
QPushButton:hover {
|
||||
background-color: #0b7dda;
|
||||
}
|
||||
QPushButton:disabled {
|
||||
background-color: #cccccc;
|
||||
}
|
||||
""")
|
||||
button_layout.addWidget(self.import_file_btn)
|
||||
|
||||
# 开始导入按钮
|
||||
self.start_import_btn = QPushButton("开始导入")
|
||||
self.start_import_btn.clicked.connect(self.start_import)
|
||||
self.start_import_btn.setStyleSheet("""
|
||||
QPushButton {
|
||||
font-size: 14px;
|
||||
padding: 8px 16px;
|
||||
background-color: #4CAF50;
|
||||
color: white;
|
||||
border: none;
|
||||
border-radius: 4px;
|
||||
margin-left: 10px;
|
||||
}
|
||||
QPushButton:hover {
|
||||
background-color: #45a049;
|
||||
}
|
||||
QPushButton:disabled {
|
||||
background-color: #cccccc;
|
||||
}
|
||||
""")
|
||||
button_layout.addWidget(self.start_import_btn)
|
||||
button_layout.setContentsMargins(0, 15, 0, 15)
|
||||
layout.addLayout(button_layout)
|
||||
|
||||
# 创建进度条
|
||||
self.progress_bar = QProgressBar()
|
||||
self.progress_bar.setVisible(False)
|
||||
self.progress_bar.setStyleSheet("""
|
||||
QProgressBar {
|
||||
height: 20px;
|
||||
border: 1px solid #ddd;
|
||||
border-radius: 10px;
|
||||
background-color: #f0f0f0;
|
||||
margin-bottom: 10px;
|
||||
}
|
||||
QProgressBar::chunk {
|
||||
background-color: #4CAF50;
|
||||
border-radius: 10px;
|
||||
}
|
||||
""")
|
||||
layout.addWidget(self.progress_bar)
|
||||
|
||||
# 创建状态标签
|
||||
self.status_label = QLabel("")
|
||||
self.status_label.setAlignment(Qt.AlignCenter)
|
||||
self.status_label.setStyleSheet("font-size: 14px; color: #333; padding: 10px; background-color: #f0f8ff; border-radius: 4px;")
|
||||
layout.addWidget(self.status_label)
|
||||
|
||||
logger.info("域名导入界面创建完成")
|
||||
|
||||
def import_file(self):
|
||||
"""
|
||||
导入文件
|
||||
"""
|
||||
file_path, _ = QFileDialog.getOpenFileName(self, "选择文件", "", "文本文件 (*.txt)")
|
||||
if file_path:
|
||||
try:
|
||||
# 尝试不同的编码格式
|
||||
encodings = ['utf-8', 'utf-8-sig', 'gbk', 'gb2312', 'cp936', 'latin-1', 'ascii']
|
||||
domain_count = 0
|
||||
|
||||
# 尝试使用不同编码读取并计数
|
||||
for encoding in encodings:
|
||||
try:
|
||||
with open(file_path, 'r', encoding=encoding) as f:
|
||||
domain_count = sum(1 for line in f if line.strip())
|
||||
logger.info(f"使用编码 {encoding} 成功读取文件")
|
||||
break
|
||||
except UnicodeDecodeError:
|
||||
continue
|
||||
|
||||
# 如果仍然失败,尝试使用二进制模式读取并猜测编码
|
||||
if domain_count == 0:
|
||||
try:
|
||||
import chardet
|
||||
with open(file_path, 'rb') as f:
|
||||
raw_data = f.read()
|
||||
result = chardet.detect(raw_data)
|
||||
encoding = result['encoding']
|
||||
if encoding:
|
||||
domain_count = sum(1 for line in raw_data.decode(encoding).split('\n') if line.strip())
|
||||
logger.info(f"使用 chardet 检测到编码 {encoding} 并成功读取文件")
|
||||
else:
|
||||
raise Exception("无法识别文件编码")
|
||||
except Exception as e:
|
||||
logger.warning(f"chardet 检测失败: {e}")
|
||||
# 最后尝试使用 replace 模式读取
|
||||
with open(file_path, 'r', encoding='utf-8', errors='replace') as f:
|
||||
domain_count = sum(1 for line in f if line.strip())
|
||||
logger.info("使用 utf-8 replace 模式读取文件")
|
||||
|
||||
# 对于大文件,不显示所有域名,只显示文件路径和域名数量
|
||||
if domain_count > 1000:
|
||||
self.text_edit.setText(f"文件路径: {file_path}\n域名数量: {domain_count}\n\n提示: 由于文件较大,仅显示文件信息,不显示具体域名。")
|
||||
# 保存文件路径,用于后续导入
|
||||
self.imported_file_path = file_path
|
||||
else:
|
||||
# 对于小文件,显示所有域名
|
||||
domains = []
|
||||
for encoding in encodings:
|
||||
try:
|
||||
with open(file_path, 'r', encoding=encoding) as f:
|
||||
domains = f.readlines()
|
||||
break
|
||||
except UnicodeDecodeError:
|
||||
continue
|
||||
if not domains:
|
||||
# 尝试使用二进制模式读取
|
||||
try:
|
||||
import chardet
|
||||
with open(file_path, 'rb') as f:
|
||||
raw_data = f.read()
|
||||
result = chardet.detect(raw_data)
|
||||
encoding = result['encoding']
|
||||
if encoding:
|
||||
domains = raw_data.decode(encoding).split('\n')
|
||||
else:
|
||||
with open(file_path, 'r', encoding='utf-8', errors='replace') as f:
|
||||
domains = f.readlines()
|
||||
except Exception:
|
||||
with open(file_path, 'r', encoding='utf-8', errors='replace') as f:
|
||||
domains = f.readlines()
|
||||
domains = [domain.strip() for domain in domains if domain.strip()]
|
||||
self.text_edit.setText('\n'.join(domains))
|
||||
# 清除文件路径,使用文本框中的域名
|
||||
self.imported_file_path = None
|
||||
|
||||
self.status_label.setText(f"成功读取 {domain_count} 个域名")
|
||||
logger.info(f"成功读取文件: {file_path}, 共 {domain_count} 个域名")
|
||||
except Exception as e:
|
||||
self.status_label.setText(f"读取文件失败: {str(e)}")
|
||||
logger.error(f"读取文件失败: {e}")
|
||||
|
||||
def start_import(self):
|
||||
"""
|
||||
开始导入
|
||||
"""
|
||||
# 检查是否有导入的文件路径
|
||||
if hasattr(self, 'imported_file_path') and self.imported_file_path:
|
||||
# 大文件导入,使用文件路径
|
||||
file_path = self.imported_file_path
|
||||
|
||||
# 显示进度条
|
||||
self.progress_bar.setVisible(True)
|
||||
self.progress_bar.setValue(0)
|
||||
self.status_label.setText("正在导入...")
|
||||
|
||||
# 禁用按钮
|
||||
self.import_file_btn.setEnabled(False)
|
||||
self.start_import_btn.setEnabled(False)
|
||||
|
||||
# 创建并启动导入线程
|
||||
self.import_thread = ImportFileThread(file_path, 7) # 7 表示 TXT 导入
|
||||
self.import_thread.progress_updated.connect(self.update_progress)
|
||||
self.import_thread.finished.connect(self.import_finished)
|
||||
self.import_thread.start()
|
||||
|
||||
logger.info(f"开始从文件导入: {file_path}")
|
||||
else:
|
||||
# 小文件或手动输入的域名
|
||||
domains = self.text_edit.toPlainText().split('\n')
|
||||
domains = [domain.strip() for domain in domains if domain.strip()]
|
||||
|
||||
if not domains:
|
||||
self.status_label.setText("请输入域名")
|
||||
return
|
||||
|
||||
# 显示进度条
|
||||
self.progress_bar.setVisible(True)
|
||||
self.progress_bar.setValue(0)
|
||||
self.status_label.setText("正在导入...")
|
||||
|
||||
# 禁用按钮
|
||||
self.import_file_btn.setEnabled(False)
|
||||
self.start_import_btn.setEnabled(False)
|
||||
|
||||
# 创建并启动导入线程
|
||||
self.import_thread = ImportThread(domains, 7) # 7 表示 TXT 导入
|
||||
self.import_thread.progress_updated.connect(self.update_progress)
|
||||
self.import_thread.finished.connect(self.import_finished)
|
||||
self.import_thread.start()
|
||||
|
||||
logger.info(f"开始导入 {len(domains)} 个域名")
|
||||
|
||||
def update_progress(self, progress):
|
||||
"""
|
||||
更新进度
|
||||
|
||||
:param progress: 进度值
|
||||
"""
|
||||
self.progress_bar.setValue(progress)
|
||||
|
||||
def import_finished(self, success, message):
|
||||
"""
|
||||
导入完成
|
||||
|
||||
:param success: 是否成功
|
||||
:param message: 消息
|
||||
"""
|
||||
self.status_label.setText(message)
|
||||
self.progress_bar.setVisible(False)
|
||||
|
||||
# 启用按钮
|
||||
self.import_file_btn.setEnabled(True)
|
||||
self.start_import_btn.setEnabled(True)
|
||||
|
||||
logger.info(f"导入完成: {message}")
|
||||
577
domainCheck/app/ui/juming_crawler.py
Normal file
577
domainCheck/app/ui/juming_crawler.py
Normal file
@@ -0,0 +1,577 @@
|
||||
# -*- coding: UTF-8 -*-
|
||||
'''
|
||||
@Project :domainScanDemo
|
||||
@File :juming_crawler.py
|
||||
@IDE :PyCharm
|
||||
@Author :梦伴
|
||||
@Date :2026/4/9 15:00
|
||||
@explain : 聚名网爬取页面
|
||||
'''
|
||||
|
||||
from PySide6.QtWidgets import QWidget, QVBoxLayout, QHBoxLayout, QPushButton, QLabel, QTextEdit, QProgressBar, QLineEdit, QComboBox, QDateEdit, QCheckBox
|
||||
from PySide6.QtCore import Qt, QThread, Signal, QDate
|
||||
from PySide6.QtGui import QIntValidator
|
||||
from loguru import logger
|
||||
import re
|
||||
import datetime
|
||||
import time
|
||||
|
||||
from app.core.domain_collector import DomainCollector
|
||||
from detect.juming import JM
|
||||
|
||||
|
||||
class JumingCrawlThread(QThread):
|
||||
"""
|
||||
聚名爬取线程
|
||||
"""
|
||||
progress_updated = Signal(int)
|
||||
status_updated = Signal(str)
|
||||
finished = Signal(bool, str)
|
||||
|
||||
def __init__(self, crawl_type, page_start=1, page_size=50, crawl_date=None, auto_date=True):
|
||||
"""
|
||||
初始化聚名爬取线程
|
||||
|
||||
:param crawl_type: 爬取类型 (1: 一口价, 2: 删除列表)
|
||||
:param page_start: 起始页码
|
||||
:param page_size: 每页数量
|
||||
:param crawl_date: 爬取日期(删除列表用)
|
||||
:param auto_date: 是否自动新增日期
|
||||
"""
|
||||
super().__init__()
|
||||
self.crawl_type = crawl_type
|
||||
self.page_start = page_start
|
||||
self.page_size = page_size
|
||||
self.crawl_date = crawl_date
|
||||
self.auto_date = auto_date
|
||||
self.is_paused = False
|
||||
self.is_stopped = False
|
||||
self.current_page = 0
|
||||
self.total_count = 0
|
||||
|
||||
def run(self):
|
||||
"""
|
||||
运行聚名爬取线程
|
||||
"""
|
||||
try:
|
||||
# 初始化聚名客户端
|
||||
jm = JM()
|
||||
|
||||
# 加载 Cookie
|
||||
jm.load_cookies()
|
||||
logger.info("已加载 Cookie")
|
||||
self.status_updated.emit("已加载 Cookie")
|
||||
|
||||
# 直接开始爬取,不需要登录,因为 Cookie 已经在系统设置页面加载了
|
||||
self.progress_updated.emit(10)
|
||||
|
||||
if self.crawl_type == 1: # 一口价
|
||||
self.progress_updated.emit(30)
|
||||
logger.info("开始获取一口价域名")
|
||||
self.status_updated.emit("开始获取一口价域名")
|
||||
|
||||
# 自动爬取多页
|
||||
page = self.page_start
|
||||
while not self.is_stopped:
|
||||
if self.is_paused:
|
||||
time.sleep(0.1)
|
||||
continue
|
||||
|
||||
self.current_page = page
|
||||
self.status_updated.emit(f"正在爬取第 {page} 页")
|
||||
logger.info(f"正在爬取第 {page} 页")
|
||||
|
||||
# 获取当前页
|
||||
success, html = jm.ykj_get_list(page=page, page_size=self.page_size)
|
||||
if success:
|
||||
pattern_ym = r"<a class='yda1 ydz' ym='([^']*)'"
|
||||
results = re.findall(pattern_ym, html)
|
||||
domains = [domain.strip() for domain in results if domain.strip()]
|
||||
domain_count = len(domains)
|
||||
self.total_count += domain_count
|
||||
|
||||
self.status_updated.emit(f"第 {page} 页找到 {domain_count} 个域名,累计 {self.total_count} 个")
|
||||
logger.info(f"第 {page} 页找到 {domain_count} 个域名,累计 {self.total_count} 个")
|
||||
|
||||
# 自动入库
|
||||
if domains:
|
||||
collector = DomainCollector()
|
||||
stats = collector.add_domains_batch(domains, 1) # 1 表示一口价
|
||||
logger.info(f"自动入库完成: {stats}")
|
||||
self.status_updated.emit(f"自动入库完成: 成功添加 {stats['added']} 个域名")
|
||||
|
||||
# 如果返回的数量小于指定的数量,停止爬取
|
||||
if domain_count < self.page_size:
|
||||
logger.info(f"返回数量小于指定数量,停止爬取")
|
||||
self.status_updated.emit("返回数量小于指定数量,停止爬取")
|
||||
break
|
||||
|
||||
# 增加页码
|
||||
page += 1
|
||||
|
||||
# 模拟网络延迟
|
||||
time.sleep(1)
|
||||
else:
|
||||
logger.error(f"获取一口价域名失败: {html}")
|
||||
self.status_updated.emit(f"获取一口价域名失败: {html}")
|
||||
break
|
||||
|
||||
elif self.crawl_type == 2: # 删除列表
|
||||
self.progress_updated.emit(30)
|
||||
logger.info("开始获取删除域名列表")
|
||||
self.status_updated.emit("开始获取删除域名列表")
|
||||
|
||||
# 使用传入的日期或默认今天
|
||||
start_date_str = self.crawl_date if self.crawl_date else datetime.date.today().strftime("%Y-%m-%d")
|
||||
start_date = datetime.datetime.strptime(start_date_str, "%Y-%m-%d").date()
|
||||
# 计算结束日期:今天 + 4天
|
||||
end_date = datetime.date.today() + datetime.timedelta(days=4)
|
||||
|
||||
if self.auto_date:
|
||||
# 自动新增日期,从起始日期到今天+4天
|
||||
current_date = start_date
|
||||
while current_date <= end_date and not self.is_stopped:
|
||||
if self.is_paused:
|
||||
time.sleep(0.1)
|
||||
continue
|
||||
|
||||
crawl_date = current_date.strftime("%Y-%m-%d")
|
||||
self.status_updated.emit(f"正在爬取 {crawl_date} 的删除域名")
|
||||
logger.info(f"正在爬取 {crawl_date} 的删除域名")
|
||||
|
||||
deleted_domains = jm.new_cha_del(crawl_date)
|
||||
domains = [domain.strip() for domain in deleted_domains if domain.strip()]
|
||||
domain_count = len(domains)
|
||||
self.total_count += domain_count
|
||||
|
||||
self.status_updated.emit(f"{crawl_date} 找到 {domain_count} 个删除域名,累计 {self.total_count} 个")
|
||||
logger.info(f"{crawl_date} 找到 {domain_count} 个删除域名,累计 {self.total_count} 个")
|
||||
|
||||
# 自动入库
|
||||
if domains:
|
||||
collector = DomainCollector()
|
||||
stats = collector.add_domains_batch(domains, 2) # 2 表示删除列表
|
||||
logger.info(f"{crawl_date} 自动入库完成: {stats}")
|
||||
self.status_updated.emit(f"{crawl_date} 自动入库完成: 成功添加 {stats['added']} 个域名")
|
||||
|
||||
# 增加日期
|
||||
current_date = current_date + datetime.timedelta(days=1)
|
||||
|
||||
# 模拟网络延迟
|
||||
time.sleep(1)
|
||||
else:
|
||||
# 只爬取指定日期
|
||||
crawl_date = start_date_str
|
||||
self.status_updated.emit(f"正在爬取 {crawl_date} 的删除域名")
|
||||
logger.info(f"正在爬取 {crawl_date} 的删除域名")
|
||||
|
||||
deleted_domains = jm.new_cha_del(crawl_date)
|
||||
domains = [domain.strip() for domain in deleted_domains if domain.strip()]
|
||||
domain_count = len(domains)
|
||||
self.total_count = domain_count
|
||||
|
||||
self.status_updated.emit(f"找到 {domain_count} 个删除域名")
|
||||
logger.info(f"找到 {domain_count} 个删除域名")
|
||||
|
||||
# 自动入库
|
||||
if domains:
|
||||
collector = DomainCollector()
|
||||
stats = collector.add_domains_batch(domains, 2) # 2 表示删除列表
|
||||
logger.info(f"自动入库完成: {stats}")
|
||||
self.status_updated.emit(f"自动入库完成: 成功添加 {stats['added']} 个域名")
|
||||
|
||||
self.progress_updated.emit(90)
|
||||
|
||||
self.progress_updated.emit(100)
|
||||
self.finished.emit(True, f"成功获取 {self.total_count} 个域名")
|
||||
except Exception as e:
|
||||
logger.error(f"从聚名网爬取失败: {e}")
|
||||
self.status_updated.emit(f"爬取失败: {str(e)}")
|
||||
self.finished.emit(False, f"爬取失败: {str(e)}")
|
||||
|
||||
def pause(self):
|
||||
"""
|
||||
暂停爬取
|
||||
"""
|
||||
self.is_paused = True
|
||||
logger.info("爬取已暂停")
|
||||
self.status_updated.emit("爬取已暂停")
|
||||
|
||||
def resume(self):
|
||||
"""
|
||||
恢复爬取
|
||||
"""
|
||||
self.is_paused = False
|
||||
logger.info("爬取已恢复")
|
||||
self.status_updated.emit("爬取已恢复")
|
||||
|
||||
def stop(self):
|
||||
"""
|
||||
停止爬取
|
||||
"""
|
||||
self.is_stopped = True
|
||||
logger.info("爬取已停止")
|
||||
self.status_updated.emit("爬取已停止")
|
||||
|
||||
|
||||
class JumingCrawlerWidget(QWidget):
|
||||
"""
|
||||
聚名网爬取页面
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
"""
|
||||
初始化聚名网爬取页面
|
||||
"""
|
||||
super().__init__()
|
||||
|
||||
# 创建布局
|
||||
layout = QVBoxLayout(self)
|
||||
|
||||
|
||||
|
||||
# 爬取类型选择
|
||||
type_layout = QHBoxLayout()
|
||||
type_label = QLabel("爬取类型:")
|
||||
type_label.setStyleSheet("font-size: 14px; color: #666; min-width: 80px;")
|
||||
self.type_combo = QComboBox()
|
||||
self.type_combo.addItem("一口价域名", 1)
|
||||
self.type_combo.addItem("删除列表域名", 2)
|
||||
self.type_combo.setStyleSheet("font-size: 14px; padding: 5px; border: 1px solid #ddd; border-radius: 4px;")
|
||||
# 监听类型变化
|
||||
self.type_combo.currentIndexChanged.connect(self.on_type_changed)
|
||||
type_layout.addWidget(type_label)
|
||||
type_layout.addWidget(self.type_combo)
|
||||
type_layout.setContentsMargins(0, 0, 0, 15)
|
||||
layout.addLayout(type_layout)
|
||||
|
||||
# 页码和每页数量设置
|
||||
page_layout = QHBoxLayout()
|
||||
|
||||
# 起始页码
|
||||
page_start_label = QLabel("起始页码:")
|
||||
page_start_label.setStyleSheet("font-size: 14px; color: #666; min-width: 80px;")
|
||||
self.page_start_edit = QLineEdit("1")
|
||||
# 移除所有限制,允许输入任意正整数
|
||||
self.page_start_edit.setStyleSheet("font-size: 14px; padding: 5px; border: 1px solid #ddd; border-radius: 4px; width: 120px;")
|
||||
page_layout.addWidget(page_start_label)
|
||||
page_layout.addWidget(self.page_start_edit)
|
||||
|
||||
# 每页数量
|
||||
page_size_label = QLabel("每页数量:")
|
||||
page_size_label.setStyleSheet("font-size: 14px; color: #666; min-width: 80px; margin-left: 20px;")
|
||||
self.page_size_edit = QLineEdit("500")
|
||||
self.page_size_edit.setValidator(QIntValidator(1, 1000))
|
||||
self.page_size_edit.setStyleSheet("font-size: 14px; padding: 5px; border: 1px solid #ddd; border-radius: 4px; width: 80px;")
|
||||
page_layout.addWidget(page_size_label)
|
||||
page_layout.addWidget(self.page_size_edit)
|
||||
page_layout.setContentsMargins(0, 0, 0, 20)
|
||||
layout.addLayout(page_layout)
|
||||
|
||||
# 日期设置(删除列表用)
|
||||
date_container = QWidget()
|
||||
date_layout = QHBoxLayout(date_container)
|
||||
|
||||
# 起始日期
|
||||
date_label = QLabel("起始日期:")
|
||||
date_label.setStyleSheet("font-size: 14px; color: #666; min-width: 80px;")
|
||||
self.date_edit = QDateEdit()
|
||||
self.date_edit.setDate(QDate.currentDate())
|
||||
self.date_edit.setCalendarPopup(True)
|
||||
self.date_edit.setStyleSheet("font-size: 14px; padding: 5px; border: 1px solid #ddd; border-radius: 4px; width: 150px;")
|
||||
# 设置最大日期为今天+4天,最小日期为今天的前4天
|
||||
max_date = QDate.currentDate().addDays(4)
|
||||
min_date = QDate.currentDate().addDays(-4)
|
||||
self.date_edit.setMinimumDate(min_date)
|
||||
self.date_edit.setMaximumDate(max_date)
|
||||
date_layout.addWidget(date_label)
|
||||
date_layout.addWidget(self.date_edit)
|
||||
|
||||
# 自动新增日期选项
|
||||
auto_date_checkbox = QCheckBox("自动新增日期")
|
||||
auto_date_checkbox.setChecked(True)
|
||||
auto_date_checkbox.setStyleSheet("font-size: 14px; margin-left: 20px;")
|
||||
self.auto_date_checkbox = auto_date_checkbox
|
||||
date_layout.addWidget(auto_date_checkbox)
|
||||
|
||||
date_layout.setContentsMargins(0, 0, 0, 20)
|
||||
self.date_container = date_container
|
||||
layout.addWidget(date_container)
|
||||
# 默认隐藏日期输入框
|
||||
self.date_container.setVisible(False)
|
||||
|
||||
# 按钮布局
|
||||
button_layout = QHBoxLayout()
|
||||
|
||||
# 创建开始按钮
|
||||
self.start_btn = QPushButton("开始爬取")
|
||||
self.start_btn.clicked.connect(self.start_crawl)
|
||||
self.start_btn.setStyleSheet("""
|
||||
QPushButton {
|
||||
font-size: 14px;
|
||||
padding: 8px 16px;
|
||||
background-color: #4CAF50;
|
||||
color: white;
|
||||
border: none;
|
||||
border-radius: 4px;
|
||||
}
|
||||
QPushButton:hover {
|
||||
background-color: #45a049;
|
||||
}
|
||||
QPushButton:disabled {
|
||||
background-color: #cccccc;
|
||||
}
|
||||
""")
|
||||
button_layout.addWidget(self.start_btn)
|
||||
|
||||
# 创建暂停按钮
|
||||
self.pause_btn = QPushButton("暂停爬取")
|
||||
self.pause_btn.clicked.connect(self.pause_crawl)
|
||||
self.pause_btn.setEnabled(False)
|
||||
self.pause_btn.setStyleSheet("""
|
||||
QPushButton {
|
||||
font-size: 14px;
|
||||
padding: 8px 16px;
|
||||
background-color: #ff9800;
|
||||
color: white;
|
||||
border: none;
|
||||
border-radius: 4px;
|
||||
margin-left: 10px;
|
||||
}
|
||||
QPushButton:hover {
|
||||
background-color: #f57c00;
|
||||
}
|
||||
QPushButton:disabled {
|
||||
background-color: #cccccc;
|
||||
}
|
||||
""")
|
||||
button_layout.addWidget(self.pause_btn)
|
||||
|
||||
# 创建停止按钮
|
||||
self.stop_btn = QPushButton("停止爬取")
|
||||
self.stop_btn.clicked.connect(self.stop_crawl)
|
||||
self.stop_btn.setEnabled(False)
|
||||
self.stop_btn.setStyleSheet("""
|
||||
QPushButton {
|
||||
font-size: 14px;
|
||||
padding: 8px 16px;
|
||||
background-color: #f44336;
|
||||
color: white;
|
||||
border: none;
|
||||
border-radius: 4px;
|
||||
margin-left: 10px;
|
||||
}
|
||||
QPushButton:hover {
|
||||
background-color: #d32f2f;
|
||||
}
|
||||
QPushButton:disabled {
|
||||
background-color: #cccccc;
|
||||
}
|
||||
""")
|
||||
button_layout.addWidget(self.stop_btn)
|
||||
button_layout.setContentsMargins(0, 0, 0, 20)
|
||||
layout.addLayout(button_layout)
|
||||
|
||||
# 创建日志显示区域
|
||||
self.log_edit = QTextEdit()
|
||||
self.log_edit.setPlaceholderText("爬取日志将显示在这里")
|
||||
self.log_edit.setReadOnly(True)
|
||||
self.log_edit.setStyleSheet("""
|
||||
QTextEdit {
|
||||
font-size: 13px;
|
||||
font-family: Consolas, Monaco, monospace;
|
||||
padding: 10px;
|
||||
border: 1px solid #ddd;
|
||||
border-radius: 4px;
|
||||
background-color: #f9f9f9;
|
||||
}
|
||||
""")
|
||||
layout.addWidget(self.log_edit)
|
||||
|
||||
# 创建进度条
|
||||
self.progress_bar = QProgressBar()
|
||||
self.progress_bar.setVisible(False)
|
||||
self.progress_bar.setStyleSheet("""
|
||||
QProgressBar {
|
||||
height: 20px;
|
||||
border: 1px solid #ddd;
|
||||
border-radius: 10px;
|
||||
background-color: #f0f0f0;
|
||||
margin-top: 10px;
|
||||
margin-bottom: 10px;
|
||||
}
|
||||
QProgressBar::chunk {
|
||||
background-color: #4CAF50;
|
||||
border-radius: 10px;
|
||||
}
|
||||
""")
|
||||
layout.addWidget(self.progress_bar)
|
||||
|
||||
# 创建状态标签
|
||||
self.status_label = QLabel("")
|
||||
self.status_label.setAlignment(Qt.AlignCenter)
|
||||
self.status_label.setStyleSheet("font-size: 14px; color: #333; margin-top: 10px; padding: 8px; background-color: #f0f8ff; border-radius: 4px;")
|
||||
layout.addWidget(self.status_label)
|
||||
|
||||
# 存储爬取线程
|
||||
self.crawl_thread = None
|
||||
|
||||
logger.info("聚名网爬取页面创建完成")
|
||||
|
||||
def start_crawl(self):
|
||||
"""
|
||||
开始爬取
|
||||
"""
|
||||
crawl_type = self.type_combo.currentData()
|
||||
|
||||
# 获取起始页码和每页数量
|
||||
try:
|
||||
page_start = int(self.page_start_edit.text())
|
||||
page_size = int(self.page_size_edit.text())
|
||||
if page_start < 1:
|
||||
self.status_label.setText("起始页码必须大于0")
|
||||
return
|
||||
if page_size < 1:
|
||||
self.status_label.setText("每页数量必须大于0")
|
||||
return
|
||||
except ValueError:
|
||||
self.status_label.setText("请输入有效的页码和每页数量")
|
||||
return
|
||||
|
||||
# 显示进度条
|
||||
self.progress_bar.setVisible(True)
|
||||
self.progress_bar.setValue(0)
|
||||
self.status_label.setText("正在爬取域名...")
|
||||
|
||||
# 清空日志
|
||||
self.log_edit.clear()
|
||||
|
||||
# 启用/禁用按钮
|
||||
self.start_btn.setEnabled(False)
|
||||
self.pause_btn.setEnabled(True)
|
||||
self.stop_btn.setEnabled(True)
|
||||
|
||||
# 获取爬取日期和自动新增日期选项(删除列表用)
|
||||
crawl_date = None
|
||||
auto_date = False
|
||||
if crawl_type == 2: # 删除列表
|
||||
crawl_date = self.date_edit.date().toString("yyyy-MM-dd")
|
||||
auto_date = self.auto_date_checkbox.isChecked()
|
||||
|
||||
# 创建并启动爬取线程
|
||||
self.crawl_thread = JumingCrawlThread(crawl_type, page_start, page_size, crawl_date, auto_date)
|
||||
self.crawl_thread.progress_updated.connect(self.update_progress)
|
||||
self.crawl_thread.status_updated.connect(self.update_status)
|
||||
self.crawl_thread.finished.connect(self.crawl_finished)
|
||||
self.crawl_thread.start()
|
||||
|
||||
if crawl_type == 2: # 删除列表
|
||||
auto_date_str = "是" if auto_date else "否"
|
||||
logger.info(f"开始爬取聚名网域名, 类型: {crawl_type}, 爬取日期: {crawl_date}, 自动新增日期: {auto_date_str}")
|
||||
self.log_edit.append(f"开始爬取聚名网域名, 类型: {crawl_type}, 爬取日期: {crawl_date}, 自动新增日期: {auto_date_str}")
|
||||
else: # 一口价
|
||||
logger.info(f"开始爬取聚名网域名, 类型: {crawl_type}, 起始页码: {page_start}, 每页数量: {page_size}")
|
||||
self.log_edit.append(f"开始爬取聚名网域名, 类型: {crawl_type}, 起始页码: {page_start}, 每页数量: {page_size}")
|
||||
|
||||
def pause_crawl(self):
|
||||
"""
|
||||
暂停爬取
|
||||
"""
|
||||
if self.crawl_thread:
|
||||
if self.crawl_thread.is_paused:
|
||||
self.crawl_thread.resume()
|
||||
self.pause_btn.setText("暂停爬取")
|
||||
else:
|
||||
self.crawl_thread.pause()
|
||||
self.pause_btn.setText("恢复爬取")
|
||||
|
||||
def stop_crawl(self):
|
||||
"""
|
||||
停止爬取
|
||||
"""
|
||||
if self.crawl_thread:
|
||||
self.crawl_thread.stop()
|
||||
|
||||
def update_progress(self, progress):
|
||||
"""
|
||||
更新进度
|
||||
|
||||
:param progress: 进度值
|
||||
"""
|
||||
self.progress_bar.setValue(progress)
|
||||
|
||||
def update_status(self, status):
|
||||
"""
|
||||
更新状态
|
||||
|
||||
:param status: 状态消息
|
||||
"""
|
||||
self.status_label.setText(status)
|
||||
self.log_edit.append(status)
|
||||
|
||||
def crawl_finished(self, success, message):
|
||||
"""
|
||||
爬取完成
|
||||
|
||||
:param success: 是否成功
|
||||
:param message: 消息
|
||||
"""
|
||||
self.status_label.setText(message)
|
||||
self.log_edit.append(message)
|
||||
|
||||
# 启用/禁用按钮
|
||||
self.start_btn.setEnabled(True)
|
||||
self.pause_btn.setEnabled(False)
|
||||
self.pause_btn.setText("暂停爬取")
|
||||
self.stop_btn.setEnabled(False)
|
||||
|
||||
self.progress_bar.setVisible(False)
|
||||
|
||||
logger.info(f"聚名网爬取完成: {message}")
|
||||
|
||||
def on_type_changed(self, index):
|
||||
"""
|
||||
爬取类型变化时的处理
|
||||
|
||||
:param index: 选择的索引
|
||||
"""
|
||||
crawl_type = self.type_combo.currentData()
|
||||
if crawl_type == 2: # 删除列表
|
||||
self.date_container.setVisible(True)
|
||||
else: # 一口价
|
||||
self.date_container.setVisible(False)
|
||||
|
||||
|
||||
class ImportThread(QThread):
|
||||
"""
|
||||
导入线程
|
||||
"""
|
||||
progress_updated = Signal(int)
|
||||
finished = Signal(bool, str)
|
||||
|
||||
def __init__(self, domain_list, source_type):
|
||||
"""
|
||||
初始化导入线程
|
||||
|
||||
:param domain_list: 域名列表
|
||||
:param source_type: 来源类型
|
||||
"""
|
||||
super().__init__()
|
||||
self.domain_list = domain_list
|
||||
self.source_type = source_type
|
||||
|
||||
def run(self):
|
||||
"""
|
||||
运行导入线程
|
||||
"""
|
||||
try:
|
||||
collector = DomainCollector()
|
||||
total = len(self.domain_list)
|
||||
for i, domain in enumerate(self.domain_list):
|
||||
collector.add_domain(domain, self.source_type)
|
||||
progress = int((i + 1) / total * 100)
|
||||
self.progress_updated.emit(progress)
|
||||
self.finished.emit(True, f"成功导入 {total} 个域名")
|
||||
except Exception as e:
|
||||
logger.error(f"导入失败: {e}")
|
||||
self.finished.emit(False, f"导入失败: {str(e)}")
|
||||
137
domainCheck/app/ui/main_window.py
Normal file
137
domainCheck/app/ui/main_window.py
Normal file
@@ -0,0 +1,137 @@
|
||||
# -*- coding: UTF-8 -*-
|
||||
'''
|
||||
@Project :domainScanDemo
|
||||
@File :main_window.py
|
||||
@IDE :PyCharm
|
||||
@Author :梦伴
|
||||
@Date :2026/4/8 23:46
|
||||
@explain : 主窗口
|
||||
'''
|
||||
|
||||
from PySide6.QtWidgets import QMainWindow, QWidget, QVBoxLayout, QHBoxLayout, QPushButton, QTabWidget, QLabel, QScrollArea, QFrame
|
||||
from PySide6.QtCore import Qt
|
||||
from PySide6.QtGui import QIcon
|
||||
import os
|
||||
from loguru import logger
|
||||
|
||||
from app.ui.domain_import import DomainImportWidget
|
||||
from app.ui.domain_filter import DomainFilterWidget
|
||||
from app.ui.sensitive_words import SensitiveWordsWidget
|
||||
from app.ui.juming_crawler import JumingCrawlerWidget
|
||||
from app.ui.system_settings import SystemSettingsWidget
|
||||
|
||||
|
||||
class MainWindow(QMainWindow):
|
||||
"""
|
||||
主窗口
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
"""
|
||||
初始化主窗口
|
||||
"""
|
||||
super().__init__()
|
||||
|
||||
# 设置窗口标题和大小
|
||||
self.setWindowTitle("域名工具")
|
||||
self.setGeometry(100, 100, 2000, 800)
|
||||
self.setMinimumSize(1200, 720)
|
||||
|
||||
# 设置窗口图标
|
||||
icon_path = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(__file__))), "favicon.ico")
|
||||
if os.path.exists(icon_path):
|
||||
self.setWindowIcon(QIcon(icon_path))
|
||||
logger.info(f"设置窗口图标成功: {icon_path}")
|
||||
else:
|
||||
logger.warning(f"窗口图标文件不存在: {icon_path}")
|
||||
|
||||
# 创建中央部件
|
||||
central_widget = QWidget()
|
||||
self.setCentralWidget(central_widget)
|
||||
|
||||
# 创建主布局
|
||||
main_layout = QVBoxLayout(central_widget)
|
||||
|
||||
# 创建标签页
|
||||
self.tab_widget = QTabWidget()
|
||||
self.tab_widget.setStyleSheet("""
|
||||
QTabWidget {
|
||||
font-size: 14px;
|
||||
}
|
||||
QTabBar::tab {
|
||||
padding: 10px 20px;
|
||||
background-color: #f0f0f0;
|
||||
border: 1px solid #ddd;
|
||||
border-bottom: none;
|
||||
border-top-left-radius: 4px;
|
||||
border-top-right-radius: 4px;
|
||||
margin-right: 2px;
|
||||
}
|
||||
QTabBar::tab:hover {
|
||||
background-color: #e0e0e0;
|
||||
}
|
||||
QTabBar::tab:selected {
|
||||
background-color: white;
|
||||
color: #4CAF50;
|
||||
font-weight: bold;
|
||||
border-color: #4CAF50;
|
||||
}
|
||||
QTabWidget::pane {
|
||||
border: 1px solid #ddd;
|
||||
border-top: none;
|
||||
border-radius: 0 0 4px 4px;
|
||||
padding: 10px;
|
||||
}
|
||||
""")
|
||||
main_layout.addWidget(self.tab_widget)
|
||||
|
||||
# 创建标签页内容
|
||||
self.create_tabs()
|
||||
|
||||
# 记录日志
|
||||
logger.info("主窗口创建完成")
|
||||
|
||||
def create_tabs(self):
|
||||
"""
|
||||
创建标签页
|
||||
"""
|
||||
# 聚名爬取标签页
|
||||
juming_widget = JumingCrawlerWidget()
|
||||
self.tab_widget.addTab(self.wrap_scrollable_tab(juming_widget), "聚名爬取")
|
||||
|
||||
# 域名筛选标签页
|
||||
filter_widget = DomainFilterWidget()
|
||||
self.tab_widget.addTab(self.wrap_scrollable_tab(filter_widget), "域名筛选")
|
||||
|
||||
# 域名导入标签页
|
||||
import_widget = DomainImportWidget()
|
||||
self.tab_widget.addTab(self.wrap_scrollable_tab(import_widget), "域名导入")
|
||||
|
||||
# 敏感词配置标签页
|
||||
sensitive_widget = SensitiveWordsWidget()
|
||||
self.tab_widget.addTab(self.wrap_scrollable_tab(sensitive_widget), "敏感词配置")
|
||||
|
||||
# 系统设置标签页
|
||||
settings_widget = SystemSettingsWidget()
|
||||
self.tab_widget.addTab(self.wrap_scrollable_tab(settings_widget), "系统设置")
|
||||
|
||||
logger.info("标签页创建完成")
|
||||
|
||||
def wrap_scrollable_tab(self, widget):
|
||||
"""
|
||||
给标签页统一包一层滚动区域,保证右侧滚动条始终可用
|
||||
"""
|
||||
container = QWidget()
|
||||
container_layout = QVBoxLayout(container)
|
||||
container_layout.setContentsMargins(0, 0, 0, 0)
|
||||
container_layout.setSpacing(0)
|
||||
|
||||
scroll_area = QScrollArea()
|
||||
scroll_area.setWidgetResizable(True)
|
||||
scroll_area.setFrameShape(QFrame.NoFrame)
|
||||
scroll_area.setVerticalScrollBarPolicy(Qt.ScrollBarAsNeeded)
|
||||
scroll_area.setHorizontalScrollBarPolicy(Qt.ScrollBarAsNeeded)
|
||||
scroll_area.setWidget(widget)
|
||||
|
||||
container_layout.addWidget(scroll_area)
|
||||
return container
|
||||
284
domainCheck/app/ui/sensitive_words.py
Normal file
284
domainCheck/app/ui/sensitive_words.py
Normal file
@@ -0,0 +1,284 @@
|
||||
# -*- coding: UTF-8 -*-
|
||||
'''
|
||||
@Project :domainScanDemo
|
||||
@File :sensitive_words.py
|
||||
@IDE :PyCharm
|
||||
@Author :梦伴
|
||||
@Date :2026/4/8 23:49
|
||||
@explain : 敏感词配置界面
|
||||
'''
|
||||
|
||||
from PySide6.QtWidgets import QWidget, QVBoxLayout, QHBoxLayout, QPushButton, QLabel, QTextEdit, QFileDialog
|
||||
from PySide6.QtCore import Qt, QThread, Signal
|
||||
from loguru import logger
|
||||
from app.utils.database import Database
|
||||
|
||||
|
||||
class SaveWordsThread(QThread):
|
||||
"""
|
||||
保存敏感词线程
|
||||
"""
|
||||
finished = Signal(bool, str, int)
|
||||
|
||||
def __init__(self, words):
|
||||
super().__init__()
|
||||
self.words = words
|
||||
|
||||
def run(self):
|
||||
try:
|
||||
db = Database()
|
||||
|
||||
# 先清空现有敏感词
|
||||
db.execute("DELETE FROM sensitive_words")
|
||||
|
||||
# 批量添加敏感词
|
||||
word_tuples = [(word, 'default', 1) for word in self.words]
|
||||
if word_tuples:
|
||||
db.batch_add_sensitive_words(word_tuples)
|
||||
|
||||
db.close()
|
||||
self.finished.emit(True, "成功保存敏感词", len(self.words))
|
||||
except Exception as e:
|
||||
self.finished.emit(False, str(e), 0)
|
||||
|
||||
|
||||
class LoadWordsThread(QThread):
|
||||
"""
|
||||
加载敏感词线程
|
||||
"""
|
||||
finished = Signal(bool, list, str)
|
||||
|
||||
def run(self):
|
||||
try:
|
||||
db = Database()
|
||||
sensitive_words = db.get_sensitive_words()
|
||||
words = [word['word'] for word in sensitive_words]
|
||||
db.close()
|
||||
self.finished.emit(True, words, f"成功加载 {len(words)} 个敏感词")
|
||||
except Exception as e:
|
||||
self.finished.emit(False, [], str(e))
|
||||
|
||||
|
||||
class SensitiveWordsWidget(QWidget):
|
||||
"""
|
||||
敏感词配置界面
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
"""
|
||||
初始化敏感词配置界面
|
||||
"""
|
||||
super().__init__()
|
||||
|
||||
# 创建布局
|
||||
layout = QVBoxLayout(self)
|
||||
|
||||
|
||||
|
||||
# 创建文本编辑框
|
||||
self.text_edit = QTextEdit()
|
||||
self.text_edit.setPlaceholderText("请输入敏感词,一行一个")
|
||||
self.text_edit.setStyleSheet("""
|
||||
QTextEdit {
|
||||
font-size: 14px;
|
||||
padding: 10px;
|
||||
border: 1px solid #ddd;
|
||||
border-radius: 4px;
|
||||
background-color: #f9f9f9;
|
||||
min-height: 300px;
|
||||
}
|
||||
""")
|
||||
layout.addWidget(self.text_edit)
|
||||
|
||||
# 创建按钮布局
|
||||
button_layout = QHBoxLayout()
|
||||
|
||||
# 导入按钮
|
||||
self.import_btn = QPushButton("导入")
|
||||
self.import_btn.clicked.connect(self.import_words)
|
||||
self.import_btn.setStyleSheet("""
|
||||
QPushButton {
|
||||
font-size: 14px;
|
||||
padding: 8px 16px;
|
||||
background-color: #2196F3;
|
||||
color: white;
|
||||
border: none;
|
||||
border-radius: 4px;
|
||||
}
|
||||
QPushButton:hover {
|
||||
background-color: #0b7dda;
|
||||
}
|
||||
""")
|
||||
button_layout.addWidget(self.import_btn)
|
||||
|
||||
# 导出按钮
|
||||
self.export_btn = QPushButton("导出")
|
||||
self.export_btn.clicked.connect(self.export_words)
|
||||
self.export_btn.setStyleSheet("""
|
||||
QPushButton {
|
||||
font-size: 14px;
|
||||
padding: 8px 16px;
|
||||
background-color: #ff9800;
|
||||
color: white;
|
||||
border: none;
|
||||
border-radius: 4px;
|
||||
margin-left: 10px;
|
||||
}
|
||||
QPushButton:hover {
|
||||
background-color: #f57c00;
|
||||
}
|
||||
""")
|
||||
button_layout.addWidget(self.export_btn)
|
||||
|
||||
# 保存按钮
|
||||
self.save_btn = QPushButton("保存")
|
||||
self.save_btn.clicked.connect(self.save_words)
|
||||
self.save_btn.setStyleSheet("""
|
||||
QPushButton {
|
||||
font-size: 14px;
|
||||
padding: 8px 16px;
|
||||
background-color: #4CAF50;
|
||||
color: white;
|
||||
border: none;
|
||||
border-radius: 4px;
|
||||
margin-left: 10px;
|
||||
}
|
||||
QPushButton:hover {
|
||||
background-color: #45a049;
|
||||
}
|
||||
""")
|
||||
button_layout.addWidget(self.save_btn)
|
||||
|
||||
# 加载按钮
|
||||
self.load_btn = QPushButton("加载")
|
||||
self.load_btn.clicked.connect(self.load_words)
|
||||
self.load_btn.setStyleSheet("""
|
||||
QPushButton {
|
||||
font-size: 14px;
|
||||
padding: 8px 16px;
|
||||
background-color: #9c27b0;
|
||||
color: white;
|
||||
border: none;
|
||||
border-radius: 4px;
|
||||
margin-left: 10px;
|
||||
}
|
||||
QPushButton:hover {
|
||||
background-color: #7b1fa2;
|
||||
}
|
||||
""")
|
||||
button_layout.addWidget(self.load_btn)
|
||||
button_layout.setContentsMargins(0, 15, 0, 15)
|
||||
layout.addLayout(button_layout)
|
||||
|
||||
# 创建状态标签
|
||||
self.status_label = QLabel("")
|
||||
self.status_label.setAlignment(Qt.AlignCenter)
|
||||
self.status_label.setStyleSheet("font-size: 14px; color: #333; padding: 10px; background-color: #f0f8ff; border-radius: 4px;")
|
||||
layout.addWidget(self.status_label)
|
||||
|
||||
# 初始化线程
|
||||
self.save_thread = None
|
||||
self.load_thread = None
|
||||
|
||||
# 加载敏感词
|
||||
self.load_words()
|
||||
|
||||
logger.info("敏感词配置界面创建完成")
|
||||
|
||||
def import_words(self):
|
||||
"""
|
||||
导入敏感词
|
||||
"""
|
||||
file_path, _ = QFileDialog.getOpenFileName(self, "选择文件", "", "文本文件 (*.txt)")
|
||||
if file_path:
|
||||
try:
|
||||
with open(file_path, 'r', encoding='utf-8') as f:
|
||||
words = f.readlines()
|
||||
words = [word.strip() for word in words if word.strip()]
|
||||
self.text_edit.setText('\n'.join(words))
|
||||
self.status_label.setText(f"成功导入 {len(words)} 个敏感词")
|
||||
logger.info(f"成功导入敏感词文件: {file_path}, 共 {len(words)} 个敏感词")
|
||||
except Exception as e:
|
||||
self.status_label.setText(f"导入失败: {str(e)}")
|
||||
logger.error(f"导入敏感词失败: {e}")
|
||||
|
||||
def export_words(self):
|
||||
"""
|
||||
导出敏感词
|
||||
"""
|
||||
words = self.text_edit.toPlainText().split('\n')
|
||||
words = [word.strip() for word in words if word.strip()]
|
||||
|
||||
if not words:
|
||||
self.status_label.setText("没有敏感词可导出")
|
||||
return
|
||||
|
||||
file_path, _ = QFileDialog.getSaveFileName(self, "保存文件", "", "文本文件 (*.txt)")
|
||||
if file_path:
|
||||
try:
|
||||
with open(file_path, 'w', encoding='utf-8') as f:
|
||||
for word in words:
|
||||
f.write(word + '\n')
|
||||
self.status_label.setText(f"成功导出 {len(words)} 个敏感词")
|
||||
logger.info(f"成功导出 {len(words)} 个敏感词到 {file_path}")
|
||||
except Exception as e:
|
||||
self.status_label.setText(f"导出失败: {str(e)}")
|
||||
logger.error(f"导出敏感词失败: {e}")
|
||||
|
||||
def save_words(self):
|
||||
"""
|
||||
保存敏感词
|
||||
"""
|
||||
words = self.text_edit.toPlainText().split('\n')
|
||||
words = [word.strip() for word in words if word.strip()]
|
||||
|
||||
# 禁用按钮,防止重复点击
|
||||
self.save_btn.setEnabled(False)
|
||||
self.status_label.setText("正在保存敏感词...")
|
||||
|
||||
# 创建并启动保存线程
|
||||
self.save_thread = SaveWordsThread(words)
|
||||
self.save_thread.finished.connect(self.on_save_finished)
|
||||
self.save_thread.start()
|
||||
|
||||
def on_save_finished(self, success, message, count):
|
||||
"""
|
||||
保存完成的回调函数
|
||||
"""
|
||||
if success:
|
||||
self.status_label.setText(f"成功保存 {count} 个敏感词")
|
||||
logger.info(f"成功保存 {count} 个敏感词到数据库")
|
||||
else:
|
||||
self.status_label.setText(f"保存失败: {message}")
|
||||
logger.error(f"保存敏感词失败: {message}")
|
||||
|
||||
# 重新启用按钮
|
||||
self.save_btn.setEnabled(True)
|
||||
|
||||
def load_words(self):
|
||||
"""
|
||||
加载敏感词
|
||||
"""
|
||||
# 禁用按钮,防止重复点击
|
||||
self.load_btn.setEnabled(False)
|
||||
self.status_label.setText("正在加载敏感词...")
|
||||
|
||||
# 创建并启动加载线程
|
||||
self.load_thread = LoadWordsThread()
|
||||
self.load_thread.finished.connect(self.on_load_finished)
|
||||
self.load_thread.start()
|
||||
|
||||
def on_load_finished(self, success, words, message):
|
||||
"""
|
||||
加载完成的回调函数
|
||||
"""
|
||||
if success:
|
||||
self.text_edit.setText('\n'.join(words))
|
||||
self.status_label.setText(message)
|
||||
logger.info(message)
|
||||
else:
|
||||
self.status_label.setText(f"加载失败: {message}")
|
||||
logger.error(f"加载敏感词失败: {message}")
|
||||
|
||||
# 重新启用按钮
|
||||
self.load_btn.setEnabled(True)
|
||||
1402
domainCheck/app/ui/system_settings.py
Normal file
1402
domainCheck/app/ui/system_settings.py
Normal file
File diff suppressed because it is too large
Load Diff
4
domainCheck/app/utils/__init__.py
Normal file
4
domainCheck/app/utils/__init__.py
Normal file
@@ -0,0 +1,4 @@
|
||||
# -*- coding: UTF-8 -*-
|
||||
'''
|
||||
工具类模块
|
||||
'''
|
||||
1499
domainCheck/app/utils/database.py
Normal file
1499
domainCheck/app/utils/database.py
Normal file
File diff suppressed because it is too large
Load Diff
195
domainCheck/app/utils/domain_utils.py
Normal file
195
domainCheck/app/utils/domain_utils.py
Normal file
@@ -0,0 +1,195 @@
|
||||
# -*- coding: UTF-8 -*-
|
||||
'''
|
||||
@Project :domainScanDemo
|
||||
@File :domain_utils.py
|
||||
@IDE :PyCharm
|
||||
@Author :梦伴
|
||||
@Date :2026/4/9 0:05
|
||||
@explain : 域名工具类
|
||||
'''
|
||||
|
||||
import re
|
||||
import tldextract
|
||||
from loguru import logger
|
||||
from app.utils.status_codes import DETECT_STATUS_LABELS, REGISTER_STATUS_LABELS, USE_STATUS_LABELS
|
||||
|
||||
|
||||
def normalize_domain(domain):
|
||||
"""
|
||||
标准化域名
|
||||
|
||||
:param domain: 域名
|
||||
:return: str - 标准化后的域名
|
||||
"""
|
||||
try:
|
||||
# 转换为小写
|
||||
domain = domain.lower()
|
||||
|
||||
# 去除空格
|
||||
domain = domain.strip()
|
||||
|
||||
# 去除协议
|
||||
domain = re.sub(r'^https?://', '', domain)
|
||||
|
||||
# 去除路径和查询参数
|
||||
domain = domain.split('/')[0]
|
||||
domain = domain.split('?')[0]
|
||||
|
||||
# 去除端口
|
||||
domain = domain.split(':')[0]
|
||||
|
||||
# 只保留主域
|
||||
ext = tldextract.extract(domain)
|
||||
if ext.domain and ext.suffix:
|
||||
domain = f"{ext.domain}.{ext.suffix}"
|
||||
|
||||
# 验证域名格式
|
||||
if not is_valid_domain(domain):
|
||||
return None
|
||||
|
||||
return domain
|
||||
except Exception as e:
|
||||
logger.error(f"标准化域名出错: {e}")
|
||||
return None
|
||||
|
||||
|
||||
def is_valid_domain(domain):
|
||||
"""
|
||||
验证域名格式
|
||||
|
||||
:param domain: 域名
|
||||
:return: bool - 是否有效
|
||||
"""
|
||||
try:
|
||||
# 域名格式正则
|
||||
pattern = r'^[a-zA-Z0-9][a-zA-Z0-9-]{0,61}[a-zA-Z0-9]\.[a-zA-Z]{2,}$'
|
||||
return bool(re.match(pattern, domain))
|
||||
except Exception as e:
|
||||
logger.error(f"验证域名格式出错: {e}")
|
||||
return False
|
||||
|
||||
|
||||
def extract_tld(domain):
|
||||
"""
|
||||
提取顶级域名
|
||||
|
||||
:param domain: 域名
|
||||
:return: str - 顶级域名
|
||||
"""
|
||||
try:
|
||||
ext = tldextract.extract(domain)
|
||||
return ext.suffix
|
||||
except Exception as e:
|
||||
logger.error(f"提取顶级域名出错: {e}")
|
||||
return ''
|
||||
|
||||
|
||||
def extract_domain(domain):
|
||||
"""
|
||||
提取主域名
|
||||
|
||||
:param domain: 域名
|
||||
:return: str - 主域名
|
||||
"""
|
||||
try:
|
||||
ext = tldextract.extract(domain)
|
||||
if ext.domain and ext.suffix:
|
||||
return f"{ext.domain}.{ext.suffix}"
|
||||
return domain
|
||||
except Exception as e:
|
||||
logger.error(f"提取主域名出错: {e}")
|
||||
return domain
|
||||
|
||||
|
||||
def is_com_or_net(domain):
|
||||
"""
|
||||
检查是否为 .com 或 .net 域名
|
||||
|
||||
:param domain: 域名
|
||||
:return: bool - 是否为 .com 或 .net 域名
|
||||
"""
|
||||
try:
|
||||
tld = extract_tld(domain)
|
||||
return tld in ['com', 'net']
|
||||
except Exception as e:
|
||||
logger.error(f"检查域名后缀出错: {e}")
|
||||
return False
|
||||
|
||||
|
||||
def generate_domain_variants(domain):
|
||||
"""
|
||||
生成域名变体
|
||||
|
||||
:param domain: 域名
|
||||
:return: list - 域名变体列表
|
||||
"""
|
||||
try:
|
||||
variants = []
|
||||
|
||||
# 原始域名
|
||||
variants.append(domain)
|
||||
|
||||
# 添加 www
|
||||
if not domain.startswith('www.'):
|
||||
variants.append(f"www.{domain}")
|
||||
|
||||
# 移除 www
|
||||
if domain.startswith('www.'):
|
||||
variants.append(domain[4:])
|
||||
|
||||
return variants
|
||||
except Exception as e:
|
||||
logger.error(f"生成域名变体出错: {e}")
|
||||
return [domain]
|
||||
|
||||
|
||||
def parse_domain_status(status_code):
|
||||
"""
|
||||
解析域名状态码
|
||||
|
||||
:param status_code: 状态码
|
||||
:return: str - 状态描述
|
||||
"""
|
||||
return REGISTER_STATUS_LABELS.get(status_code, '未知')
|
||||
|
||||
|
||||
def parse_use_status(status_code):
|
||||
"""
|
||||
解析使用状态码
|
||||
|
||||
:param status_code: 状态码
|
||||
:return: str - 状态描述
|
||||
"""
|
||||
return USE_STATUS_LABELS.get(status_code, '未知')
|
||||
|
||||
|
||||
def parse_detect_status(status_code):
|
||||
"""
|
||||
解析检测状态码
|
||||
|
||||
:param status_code: 状态码
|
||||
:return: str - 状态描述
|
||||
"""
|
||||
return DETECT_STATUS_LABELS.get(status_code, '未知')
|
||||
|
||||
|
||||
def parse_source_type(source_type):
|
||||
"""
|
||||
解析来源类型
|
||||
|
||||
:param source_type: 来源类型
|
||||
:return: str - 来源描述
|
||||
"""
|
||||
source_map = {
|
||||
1: '聚名一口价',
|
||||
2: '聚名过期删除',
|
||||
3: 'zone file',
|
||||
4: '搜索引擎采集',
|
||||
5: '企业目录采集',
|
||||
6: '手工录入',
|
||||
7: 'TXT 导入',
|
||||
8: '第三方接口',
|
||||
9: '其它'
|
||||
}
|
||||
|
||||
return source_map.get(source_type, '未知')
|
||||
157
domainCheck/app/utils/http_utils.py
Normal file
157
domainCheck/app/utils/http_utils.py
Normal file
@@ -0,0 +1,157 @@
|
||||
# -*- coding: UTF-8 -*-
|
||||
'''
|
||||
@Project :domainScanDemo
|
||||
@File :http_utils.py
|
||||
@IDE :PyCharm
|
||||
@Author :梦伴
|
||||
@Date :2026/4/9 0:06
|
||||
@explain : HTTP工具类
|
||||
'''
|
||||
|
||||
import requests
|
||||
from curl_cffi import requests as curl_requests
|
||||
from loguru import logger
|
||||
|
||||
|
||||
class HTTPUtils:
|
||||
"""
|
||||
HTTP工具类
|
||||
"""
|
||||
|
||||
@staticmethod
|
||||
def get(url, headers=None, params=None, timeout=10, proxies=None, use_curl=False):
|
||||
"""
|
||||
发送GET请求
|
||||
|
||||
:param url: 请求URL
|
||||
:param headers: 请求头
|
||||
:param params: 查询参数
|
||||
:param timeout: 超时时间
|
||||
:param proxies: 代理
|
||||
:param use_curl: 是否使用curl_cffi
|
||||
:return: requests.Response - 响应对象
|
||||
"""
|
||||
try:
|
||||
if use_curl:
|
||||
# 使用curl_cffi模拟浏览器
|
||||
response = curl_requests.get(url, headers=headers, params=params, timeout=timeout, proxies=proxies, impersonate='chrome')
|
||||
else:
|
||||
# 使用requests
|
||||
response = requests.get(url, headers=headers, params=params, timeout=timeout, proxies=proxies)
|
||||
|
||||
response.raise_for_status() # 检查状态码
|
||||
return response
|
||||
except Exception as e:
|
||||
logger.error(f"GET请求失败: {url}, 错误: {e}")
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def post(url, headers=None, data=None, json=None, timeout=10, proxies=None, use_curl=False):
|
||||
"""
|
||||
发送POST请求
|
||||
|
||||
:param url: 请求URL
|
||||
:param headers: 请求头
|
||||
:param data: 表单数据
|
||||
:param json: JSON数据
|
||||
:param timeout: 超时时间
|
||||
:param proxies: 代理
|
||||
:param use_curl: 是否使用curl_cffi
|
||||
:return: requests.Response - 响应对象
|
||||
"""
|
||||
try:
|
||||
if use_curl:
|
||||
# 使用curl_cffi模拟浏览器
|
||||
response = curl_requests.post(url, headers=headers, data=data, json=json, timeout=timeout, proxies=proxies, impersonate='chrome')
|
||||
else:
|
||||
# 使用requests
|
||||
response = requests.post(url, headers=headers, data=data, json=json, timeout=timeout, proxies=proxies)
|
||||
|
||||
response.raise_for_status() # 检查状态码
|
||||
return response
|
||||
except Exception as e:
|
||||
logger.error(f"POST请求失败: {url}, 错误: {e}")
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def get_random_user_agent():
|
||||
"""
|
||||
获取随机用户代理
|
||||
|
||||
:return: str - 用户代理
|
||||
"""
|
||||
user_agents = [
|
||||
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/146.0.0.0 Safari/537.36',
|
||||
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/145.0.0.0 Safari/537.36',
|
||||
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Firefox/138.0',
|
||||
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Edge/146.0.0.0',
|
||||
'Mozilla/5.0 (Macintosh; Intel Mac OS X 14_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.6 Safari/605.1.15'
|
||||
]
|
||||
|
||||
import random
|
||||
return random.choice(user_agents)
|
||||
|
||||
@staticmethod
|
||||
def get_default_headers():
|
||||
"""
|
||||
获取默认请求头
|
||||
|
||||
:return: dict - 请求头
|
||||
"""
|
||||
return {
|
||||
'User-Agent': HTTPUtils.get_random_user_agent(),
|
||||
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8',
|
||||
'Accept-Language': 'zh-CN,zh;q=0.9',
|
||||
'Accept-Encoding': 'gzip, deflate, br',
|
||||
'Connection': 'keep-alive',
|
||||
'Upgrade-Insecure-Requests': '1',
|
||||
'Sec-Fetch-Dest': 'document',
|
||||
'Sec-Fetch-Mode': 'navigate',
|
||||
'Sec-Fetch-Site': 'none',
|
||||
'Sec-Fetch-User': '?1'
|
||||
}
|
||||
|
||||
@staticmethod
|
||||
def retry_request(func, max_retries=3, delay=1):
|
||||
"""
|
||||
重试请求
|
||||
|
||||
:param func: 请求函数
|
||||
:param max_retries: 最大重试次数
|
||||
:param delay: 重试延迟
|
||||
:return: 函数返回值
|
||||
"""
|
||||
import time
|
||||
|
||||
for i in range(max_retries):
|
||||
try:
|
||||
result = func()
|
||||
if result:
|
||||
return result
|
||||
except Exception as e:
|
||||
logger.warning(f"请求失败,第 {i+1} 次重试: {e}")
|
||||
|
||||
if i < max_retries - 1:
|
||||
time.sleep(delay)
|
||||
|
||||
return None
|
||||
|
||||
@staticmethod
|
||||
def check_proxy(proxy):
|
||||
"""
|
||||
检查代理是否可用
|
||||
|
||||
:param proxy: 代理URL
|
||||
:return: bool - 是否可用
|
||||
"""
|
||||
try:
|
||||
proxies = {
|
||||
'http': proxy,
|
||||
'https': proxy
|
||||
}
|
||||
|
||||
response = requests.get('https://www.baidu.com', proxies=proxies, timeout=5)
|
||||
return response.status_code == 200
|
||||
except Exception as e:
|
||||
logger.error(f"代理检查失败: {proxy}, 错误: {e}")
|
||||
return False
|
||||
66
domainCheck/app/utils/status_codes.py
Normal file
66
domainCheck/app/utils/status_codes.py
Normal file
@@ -0,0 +1,66 @@
|
||||
"""Centralized status codes and labels used across detection, database and UI."""
|
||||
|
||||
REGISTER_STATUS_PENDING = 0
|
||||
REGISTER_STATUS_AVAILABLE = 2
|
||||
REGISTER_STATUS_REGISTERED = 3
|
||||
REGISTER_STATUS_GRACE = 4
|
||||
REGISTER_STATUS_REDEMPTION = 5
|
||||
REGISTER_STATUS_PENDING_DELETE = 6
|
||||
REGISTER_STATUS_CLIENT_HOLD = 7
|
||||
REGISTER_STATUS_SERVER_HOLD = 8
|
||||
REGISTER_STATUS_UNKNOWN = 9
|
||||
REGISTER_STATUS_FAILED = 10
|
||||
|
||||
DETECT_STATUS_PENDING = 0
|
||||
DETECT_STATUS_COMPLETED = 1
|
||||
DETECT_STATUS_RUNNING = 2
|
||||
DETECT_STATUS_BLACKLISTED = 3
|
||||
DETECT_STATUS_FAILED = 4
|
||||
|
||||
USE_STATUS_UNUSED = 0
|
||||
USE_STATUS_USED = 1
|
||||
USE_STATUS_SOLD = 2
|
||||
USE_STATUS_RESERVED = 3
|
||||
|
||||
REVIEW_STATUS_NONE = 0
|
||||
REVIEW_STATUS_PENDING = 1
|
||||
REVIEW_STATUS_APPROVED = 2
|
||||
REVIEW_STATUS_REJECTED = 3
|
||||
|
||||
THIRD_PARTY_STATUS_PENDING = 0
|
||||
THIRD_PARTY_STATUS_DONE = 1
|
||||
|
||||
REGISTER_STATUS_LABELS = {
|
||||
REGISTER_STATUS_PENDING: '待检测',
|
||||
REGISTER_STATUS_AVAILABLE: '可注册',
|
||||
REGISTER_STATUS_REGISTERED: '已注册',
|
||||
REGISTER_STATUS_GRACE: '宽限期',
|
||||
REGISTER_STATUS_REDEMPTION: '赎回期',
|
||||
REGISTER_STATUS_PENDING_DELETE: '删除期',
|
||||
REGISTER_STATUS_CLIENT_HOLD: 'clientHold',
|
||||
REGISTER_STATUS_SERVER_HOLD: 'serverHold',
|
||||
REGISTER_STATUS_UNKNOWN: '状态未知',
|
||||
REGISTER_STATUS_FAILED: '检测失败',
|
||||
}
|
||||
|
||||
DETECT_STATUS_LABELS = {
|
||||
DETECT_STATUS_PENDING: '待检测',
|
||||
DETECT_STATUS_COMPLETED: '检测完成',
|
||||
DETECT_STATUS_RUNNING: '检测中',
|
||||
DETECT_STATUS_BLACKLISTED: '黑名单',
|
||||
DETECT_STATUS_FAILED: '检测失败',
|
||||
}
|
||||
|
||||
USE_STATUS_LABELS = {
|
||||
USE_STATUS_UNUSED: '未使用',
|
||||
USE_STATUS_USED: '已经使用',
|
||||
USE_STATUS_SOLD: '已经卖出',
|
||||
USE_STATUS_RESERVED: '已经预定',
|
||||
}
|
||||
|
||||
REVIEW_STATUS_LABELS = {
|
||||
REVIEW_STATUS_NONE: '无需复核',
|
||||
REVIEW_STATUS_PENDING: '待人工复核',
|
||||
REVIEW_STATUS_APPROVED: '人工通过',
|
||||
REVIEW_STATUS_REJECTED: '人工拒绝',
|
||||
}
|
||||
Reference in New Issue
Block a user