convert domainCheck to regular directory

This commit is contained in:
Your Name
2026-04-16 13:33:06 +08:00
parent e406d73334
commit d37c444929
107 changed files with 254185 additions and 1 deletions

View File

@@ -0,0 +1,6 @@
# -*- coding: UTF-8 -*-
'''
域名库系统
'''
__version__ = "1.0.0"

1257
domainCheck/app/chinaz.js Normal file

File diff suppressed because one or more lines are too long

265
domainCheck/app/config.py Normal file
View File

@@ -0,0 +1,265 @@
# -*- coding: UTF-8 -*-
'''
@Project :domainScanDemo
@File :config.py
@IDE :PyCharm
@Author :梦伴
@Date :2026/4/9 0:08
@explain : 系统配置
'''
import os
import sys
from dotenv import load_dotenv
def _safe_echo(message):
"""
在不同 Windows 控制台编码下安全输出文本,避免导入阶段因中文打印失败。
"""
try:
print(message)
except UnicodeEncodeError:
try:
encoding = sys.stdout.encoding or 'utf-8'
sys.stdout.buffer.write((message + '\n').encode(encoding, errors='replace'))
except Exception:
pass
# 确定基础目录
if hasattr(sys, '_MEIPASS'):
# PyInstaller 打包后的临时目录
BASE_DIR = sys._MEIPASS
else:
# 开发环境目录
BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
# 加载环境变量
env_path = os.path.join(BASE_DIR, '.env')
if os.path.exists(env_path):
load_dotenv(env_path)
_safe_echo(f"成功加载环境变量文件: {env_path}")
else:
load_dotenv()
_safe_echo(f"环境变量文件不存在: {env_path},使用默认环境变量")
class Config:
"""
系统配置
"""
# 数据库配置
DB_HOST = os.getenv('DB_HOST', 'localhost')
DB_PORT = int(os.getenv('DB_PORT', 5432))
DB_DATABASE = os.getenv('DB_DATABASE', 'domain_scan_db')
DB_USER = os.getenv('DB_USER', 'postgres')
DB_PASSWORD = os.getenv('DB_PASSWORD', 'postgres')
DB_POOL_SIZE = int(os.getenv('DB_POOL_SIZE', 5))
# 消息队列配置
RABBITMQ_HOST = os.getenv('RABBITMQ_HOST', 'localhost')
RABBITMQ_PORT = int(os.getenv('RABBITMQ_PORT', 5672))
RABBITMQ_USER = os.getenv('RABBITMQ_USER', 'guest')
RABBITMQ_PASSWORD = os.getenv('RABBITMQ_PASSWORD', 'guest')
RABBITMQ_VHOST = os.getenv('RABBITMQ_VHOST', '/')
# Redis配置
REDIS_HOST = os.getenv('REDIS_HOST', 'localhost')
REDIS_PORT = int(os.getenv('REDIS_PORT', 6379))
REDIS_PASSWORD = os.getenv('REDIS_PASSWORD', '')
REDIS_DB = int(os.getenv('REDIS_DB', 0))
# 聚名网配置
JUMING_COOKIE = os.getenv('JUMING_COOKIE', '')
JUMING_REFERER = os.getenv('JUMING_REFERER', 'https://www.juming.com/')
JUMING_USER_AGENT = os.getenv('JUMING_USER_AGENT', 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/146.0.0.0 Safari/537.36')
# 代理配置
PROXY_ENABLED = os.getenv('PROXY_ENABLED', 'false').lower() == 'true'
PROXY_URL = os.getenv('PROXY_URL', '')
# 检测配置
DETECT_TIMEOUT = int(os.getenv('DETECT_TIMEOUT', 30))
DETECT_RETRY_COUNT = int(os.getenv('DETECT_RETRY_COUNT', 3))
DETECT_CONCURRENCY = int(os.getenv('DETECT_CONCURRENCY', 10))
WAYBACK_CDX_TIMEOUT = int(os.getenv('WAYBACK_CDX_TIMEOUT', 15))
WAYBACK_SNAPSHOT_TIMEOUT = int(os.getenv('WAYBACK_SNAPSHOT_TIMEOUT', 12))
WAYBACK_RETRY_COUNT = int(os.getenv('WAYBACK_RETRY_COUNT', 2))
WAYBACK_REQUEST_DELAY = float(os.getenv('WAYBACK_REQUEST_DELAY', 0))
WAYBACK_PROGRESS_INTERVAL = int(os.getenv('WAYBACK_PROGRESS_INTERVAL', 500))
WAYBACK_DOMAIN_CONCURRENCY = int(os.getenv('WAYBACK_DOMAIN_CONCURRENCY', 3))
WAYBACK_TITLE_MAX_BYTES = int(os.getenv('WAYBACK_TITLE_MAX_BYTES', 65536))
WAYBACK_TIMESTAMP_CACHE_TTL = int(os.getenv('WAYBACK_TIMESTAMP_CACHE_TTL', 86400))
WAYBACK_TITLE_CACHE_TTL = int(os.getenv('WAYBACK_TITLE_CACHE_TTL', 2592000))
WAYBACK_USER_AGENT = os.getenv(
'WAYBACK_USER_AGENT',
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/146.0.0.0 Safari/537.36'
)
# 域名配置
DOMAIN_TLDS = ['com', 'net']
DOMAIN_BATCH_SIZE = int(os.getenv('DOMAIN_BATCH_SIZE', 1000))
# 日志配置
LOG_LEVEL = os.getenv('LOG_LEVEL', 'INFO')
LOG_FILE = os.getenv('LOG_FILE', 'app.log')
# 任务配置
TASK_PRIORITY = {
'rdap': 10,
'wayback': 8,
'baidu': 6,
'qihu360': 5,
'google': 5,
'chinaz': 4,
'aizhan': 4,
'juziseo': 3,
'jucha': 3
}
# 敏感词配置 - 现在从数据库加载
SENSITIVE_WORDS = []
# 检测项配置
DETECT_ITEMS = {
'rdap': True,
'wayback': True,
'baidu': True,
'qihu360': True,
'google': True,
'chinaz': True,
'aizhan': True,
'juziseo': True,
'jucha': True
}
# 目录配置
BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
DATA_DIR = os.path.join(BASE_DIR, 'data')
LOG_DIR = os.path.join(BASE_DIR, 'logs')
# 确保目录存在
os.makedirs(DATA_DIR, exist_ok=True)
os.makedirs(LOG_DIR, exist_ok=True)
# 从数据库加载敏感词
def load_sensitive_words():
"""
从数据库加载敏感词
"""
try:
from app.utils.database import Database
db = Database()
sensitive_words = db.get_sensitive_words()
words = [word['word'] for word in sensitive_words]
db.close()
return words
except Exception as e:
_safe_echo(f"加载敏感词失败: {e}")
return []
# 从文件加载检测选项
def load_detect_options():
"""
从文件加载检测选项
"""
try:
default_order = [
'detect_register',
'detect_baidu_site',
'detect_360_site',
'detect_chinaz',
'detect_aizhan',
'detect_wayback',
'detect_jucha',
'detect_juziseo',
]
import json
import os
defaults = {
'detect_register': True,
'detect_wayback': True,
'detect_chinaz': True,
'detect_aizhan': True,
'detect_baidu_site': True,
'detect_360_site': True,
'detect_jucha': False,
'detect_juziseo': False,
'detect_order': default_order,
}
if os.path.exists('detect_options.json'):
with open('detect_options.json', 'r', encoding='utf-8') as f:
detect_options = json.load(f)
defaults.update(detect_options)
if defaults.get('detect_whois') or defaults.get('detect_beian') or defaults.get('detect_intercept'):
defaults['detect_jucha'] = True
if defaults.get('detect_juziseo_outlink'):
defaults['detect_juziseo'] = True
order = defaults.get('detect_order') or []
normalized_order = [key for key in order if key in default_order]
for key in default_order:
if key not in normalized_order:
normalized_order.append(key)
defaults['detect_order'] = normalized_order
return defaults
except Exception as e:
_safe_echo(f"加载检测选项失败: {e}")
return {
'detect_register': True,
'detect_wayback': True,
'detect_chinaz': True,
'detect_aizhan': True,
'detect_baidu_site': True,
'detect_360_site': True,
'detect_jucha': False,
'detect_juziseo': False,
}
# 导出配置
config = Config()
# 加载检测选项
config.DETECT_OPTIONS = load_detect_options()
# 延迟加载敏感词,避免循环导入
def load_sensitive_words_lazy():
"""
延迟加载敏感词
"""
return load_sensitive_words()
# 设置敏感词属性为延迟加载
config.load_sensitive_words = load_sensitive_words_lazy
# 检查检测类型是否应该执行
def should_detect(detect_type):
"""
检查检测类型是否应该执行
:param detect_type: 检测类型
:return: bool - 是否应该执行
"""
# 检测类型映射
type_mapping = {
'register': 'detect_register',
'wayback': 'detect_wayback',
'chinaz': 'detect_chinaz',
'aizhan': 'detect_aizhan',
'baidu_site': 'detect_baidu_site',
'360_site': 'detect_360_site',
'whois': 'detect_jucha',
'beian': 'detect_jucha',
'intercept': 'detect_jucha',
'jucha': 'detect_jucha',
'juziseo': 'detect_juziseo',
'juziseo_outlink': 'detect_juziseo'
}
# 获取对应的配置键
config_key = type_mapping.get(detect_type)
if not config_key:
return True # 默认执行
# 检查是否在配置中默认为True
return config.DETECT_OPTIONS.get(config_key, True)

View File

@@ -0,0 +1,4 @@
# -*- coding: UTF-8 -*-
'''
核心功能模块
'''

View File

@@ -0,0 +1,286 @@
# -*- coding: UTF-8 -*-
'''
@Project :domainScanDemo
@File :detect_engine.py
@IDE :PyCharm
@Author :梦伴
@Date :2026/4/8 23:52
@explain : 检测引擎
'''
import time
from loguru import logger
from app.utils.database import Database
from app.detectors.rdap_detector import RDAPDetector
from app.detectors.wayback_detector import WaybackDetector
from app.detectors.baidu_detector import BaiduDetector
from app.detectors.qihu360_detector import Qihu360Detector
from app.detectors.google_detector import GoogleDetector
from app.detectors.chinaz_detector import ChinazDetector
from app.detectors.aizhan_detector import AizhanDetector
from app.detectors.juziseo_detector import JuziseoDetector
from app.detectors.jucha_detector import JuchaDetector
from app.utils.status_codes import (
DETECT_STATUS_BLACKLISTED,
DETECT_STATUS_COMPLETED,
DETECT_STATUS_FAILED,
DETECT_STATUS_RUNNING,
)
class DetectEngine:
"""
检测引擎
"""
def __init__(self):
"""
初始化检测引擎
"""
self.db = Database()
self.rdap_detector = RDAPDetector()
self.wayback_detector = WaybackDetector()
self.baidu_detector = BaiduDetector()
self.qihu360_detector = Qihu360Detector()
self.google_detector = GoogleDetector()
self.chinaz_detector = ChinazDetector()
self.aizhan_detector = AizhanDetector()
self.juziseo_detector = JuziseoDetector()
self.jucha_detector = JuchaDetector()
def detect_domain(self, domain_id):
"""
检测域名
:param domain_id: 域名ID
:return: bool - 是否检测成功
"""
try:
# 获取域名信息
domain_info = self.db.get_domain_by_id(domain_id)
if not domain_info:
logger.error(f"域名不存在: {domain_id}")
return False
domain = domain_info['domain']
logger.info(f"开始检测域名: {domain}")
# 更新检测状态为检测中
self.db.update_domain_detect_status(domain_id, DETECT_STATUS_RUNNING)
# 1. 基础检测
if not self._basic_detect(domain_id, domain):
logger.info(f"基础检测失败,停止后续检测: {domain}")
return False
# 2. 深度检测
if not self._deep_detect(domain_id, domain):
logger.info(f"深度检测失败: {domain}")
return False
# 更新检测状态为正常
self.db.update_domain_detect_status(domain_id, DETECT_STATUS_COMPLETED)
logger.info(f"域名检测完成: {domain}")
return True
except Exception as e:
logger.error(f"检测域名出错: {e}")
# 更新检测状态为检测失败
self.db.update_domain_detect_status(domain_id, DETECT_STATUS_FAILED)
return False
def _basic_detect(self, domain_id, domain):
"""
基础检测
:param domain_id: 域名ID
:param domain: 域名
:return: bool - 是否检测通过
"""
# 1. 检查是否为一口价域名
is_ykj = self.db.is_ykj_domain(domain_id)
# 2. 注册状态检测(一口价域名跳过)
if not is_ykj:
register_status = self.rdap_detector.check_register_status(domain)
self.db.update_domain_register_status(domain_id, register_status)
# 3. 黑名单缓存检查
if self.db.is_blacklisted(domain):
logger.info(f"域名在黑名单中: {domain}")
self.db.update_domain_detect_status(domain_id, DETECT_STATUS_BLACKLISTED)
return False
# 4. 时光机快照年份采集
snapshot_years = self.wayback_detector.get_snapshot_years(domain)
if snapshot_years:
self.db.update_domain_snapshot_years(domain_id, ','.join(map(str, snapshot_years)))
# 5. 时光机正文抽样与敏感词匹配
if self.wayback_detector.has_sensitive_content(domain):
logger.info(f"域名包含敏感词: {domain}")
self.db.update_domain_detect_status(domain_id, DETECT_STATUS_BLACKLISTED)
self.db.add_to_blacklist(domain, "快照包含敏感词")
return False
return True
def _deep_detect(self, domain_id, domain):
"""
深度检测
:param domain_id: 域名ID
:param domain: 域名
:return: bool - 是否检测通过
"""
# 1. 百度历史/Site
baidu_history = self.baidu_detector.check_history(domain)
baidu_site = self.baidu_detector.check_site(domain)
# 2. 360 Site
qihu360_site = self.qihu360_detector.check_site(domain)
# 3. Google Site
google_site = self.google_detector.check_site(domain)
# 4. 站长之家
chinaz_info = self.chinaz_detector.check_domain(domain)
# 5. 爱站网
aizhan_info = self.aizhan_detector.check_domain(domain)
# 6. 桔子SEO
juziseo_info = self.juziseo_detector.check_domain(domain)
# 7. 聚查
jucha_info = self.jucha_detector.check_domain(domain)
# 检查是否有风险
if self._check_risk(domain_id, domain, baidu_history, baidu_site, qihu360_site, google_site, chinaz_info, aizhan_info, juziseo_info, jucha_info):
return False
# 保存检测结果
self.db.add_detection_result(domain_id, baidu_history, baidu_site, qihu360_site, google_site, chinaz_info, aizhan_info, juziseo_info, jucha_info)
return True
def _check_risk(self, domain_id, domain, baidu_history, baidu_site, qihu360_site, google_site, chinaz_info, aizhan_info, juziseo_info, jucha_info):
"""
检查风险
:param domain_id: 域名ID
:param domain: 域名
:param baidu_history: 百度历史
:param baidu_site: 百度site
:param qihu360_site: 360 site
:param google_site: Google site
:param chinaz_info: 站长之家信息
:param aizhan_info: 爱站网信息
:param juziseo_info: 桔子SEO信息
:param jucha_info: 聚查信息
:return: bool - 是否有风险
"""
# 检查百度历史过灰
if baidu_history and '' in str(baidu_history):
logger.info(f"百度历史过灰: {domain}")
self.db.update_domain_detect_status(domain_id, DETECT_STATUS_BLACKLISTED)
self.db.add_to_blacklist(domain, "百度历史过灰")
return True
# 检查标题敏感词
if chinaz_info and 'title' in chinaz_info:
if self._contains_sensitive_words(chinaz_info['title']):
logger.info(f"标题包含敏感词: {domain}")
self.db.update_domain_detect_status(domain_id, DETECT_STATUS_BLACKLISTED)
self.db.add_to_blacklist(domain, "标题包含敏感词")
return True
# 检查子域名
if baidu_site and 'subdomains' in baidu_site:
subdomains = baidu_site['subdomains']
# 排除 www, @, m
valid_subdomains = [sub for sub in subdomains if sub not in ['www', '@', 'm']]
if valid_subdomains:
logger.info(f"存在子域名: {domain}")
self.db.update_domain_detect_status(domain_id, DETECT_STATUS_BLACKLISTED)
self.db.add_to_blacklist(domain, "存在子域名")
return True
# 检查风险提示
if aizhan_info and 'risk' in aizhan_info:
if aizhan_info['risk'] in ['低风险', '疑似色情博彩风险', '严重影响权重']:
logger.info(f"风险提示: {domain}")
self.db.update_domain_detect_status(domain_id, DETECT_STATUS_BLACKLISTED)
self.db.add_to_blacklist(domain, f"风险提示: {aizhan_info['risk']}")
return True
# 检查WHOIS状态
if jucha_info and 'whois' in jucha_info:
if jucha_info['whois'].get('status') in ['clientHold', 'serverHold']:
logger.info(f"WHOIS状态异常: {domain}")
self.db.update_domain_detect_status(domain_id, DETECT_STATUS_BLACKLISTED)
self.db.add_to_blacklist(domain, "WHOIS状态异常")
return True
# 检查拦截检测
if jucha_info and 'intercept' in jucha_info:
if not jucha_info['intercept'].get('normal', True):
logger.info(f"拦截检测异常: {domain}")
self.db.update_domain_detect_status(domain_id, 4) # 4 表示黑名单
self.db.add_to_blacklist(domain, "拦截检测异常")
return True
return False
def _contains_sensitive_words(self, text):
"""
检查文本是否包含敏感词
:param text: 文本
:return: bool - 是否包含敏感词
"""
sensitive_words = self.db.get_all_sensitive_words()
for word in sensitive_words:
if word in text:
return True
return False
def process_task(self, task_id):
"""
处理检测任务
:param task_id: 任务ID
:return: bool - 是否处理成功
"""
try:
# 获取任务信息
task = self.db.get_task_by_id(task_id)
if not task:
logger.error(f"任务不存在: {task_id}")
return False
domain_id = task['domain_id']
# 更新任务状态为执行中
self.db.update_task_status(task_id, 1) # 1 表示执行中
# 执行检测
success = self.detect_domain(domain_id)
# 更新任务状态
if success:
self.db.update_task_status(task_id, 2) # 2 表示完成
else:
# 增加重试次数
retry_count = task.get('retry_count', 0) + 1
if retry_count < 3:
self.db.update_task_retry_count(task_id, retry_count)
self.db.update_task_status(task_id, 0) # 0 表示待执行
else:
self.db.update_task_status(task_id, 3) # 3 表示失败
return success
except Exception as e:
logger.error(f"处理任务出错: {e}")
# 更新任务状态为失败
self.db.update_task_status(task_id, 3) # 3 表示失败
return False

View File

@@ -0,0 +1,261 @@
# -*- coding: UTF-8 -*-
'''
@Project :domainScanDemo
@File :domain_collector.py
@IDE :PyCharm
@Author :梦伴
@Date :2026/4/8 23:50
@explain : 域名收集器
'''
import re
import tldextract
from loguru import logger
from app.utils.database import Database
from app.utils.domain_utils import normalize_domain
from app.config import config
class DomainCollector:
"""
域名收集器
"""
def __init__(self):
"""
初始化域名收集器
"""
self.db = Database(
host=config.DB_HOST,
port=config.DB_PORT,
database=config.DB_DATABASE,
user=config.DB_USER,
password=config.DB_PASSWORD
)
def add_domain(self, domain, source_type):
"""
添加域名
:param domain: 域名
:param source_type: 来源类型
:return: bool - 是否添加成功
"""
try:
# 标准化域名
normalized_domain = normalize_domain(domain)
if not normalized_domain:
logger.warning(f"无效域名: {domain}")
return False
# 提取顶级域名
ext = tldextract.extract(normalized_domain)
tld = ext.suffix
# 只保留 .com 和 .net
if tld not in ['com', 'net']:
logger.warning(f"不支持的顶级域名: {tld}")
return False
# 检查是否已存在
if self.db.domain_exists(normalized_domain):
logger.info(f"域名已存在: {normalized_domain}")
return False
# 添加域名
domain_id = self.db.add_domain(normalized_domain, tld, source_type)
if domain_id:
# 创建检测任务
self.db.create_detect_task(domain_id, 1) # 1 表示基础检测
logger.info(f"成功添加域名: {normalized_domain}")
return True
else:
logger.error(f"添加域名失败: {normalized_domain}")
return False
except Exception as e:
logger.error(f"添加域名出错: {e}")
return False
def add_domains_batch(self, domains, source_type, batch_size=1000, dry_run=False):
"""
批量添加域名
:param domains: 域名列表
:param source_type: 来源类型
:param batch_size: 批量大小
:param dry_run: 是否仅进行干运行(不实际添加域名)
:return: dict - 统计信息
"""
try:
# 统计信息
stats = {
'total': len(domains),
'valid': 0,
'added': 0,
'exists': 0,
'invalid': 0,
'failed': 0
}
# 处理大规模数据时,分批进行标准化和过滤
normalized_domains = []
batch_domains = []
for i, domain in enumerate(domains):
normalized = normalize_domain(domain)
if not normalized:
stats['invalid'] += 1
continue
# 提取顶级域名
ext = tldextract.extract(normalized)
tld = ext.suffix
# 只保留 .com 和 .net
if tld not in ['com', 'net']:
stats['invalid'] += 1
continue
normalized_domains.append((normalized, tld))
batch_domains.append(normalized)
stats['valid'] += 1
# 每1000个域名检查一次避免内存占用过高
if (i + 1) % 1000 == 0:
logger.info(f"已处理 {i + 1}/{len(domains)} 个域名")
logger.info(f"域名标准化完成,有效域名: {stats['valid']}")
# 提取所有域名
all_domains = [domain for domain, tld in normalized_domains]
# 批量检查域名是否存在
existing_domains = self.db.check_domains_exist(all_domains)
existing_set = set(existing_domains)
# 准备批量添加数据
batch_data = []
for domain, tld in normalized_domains:
if domain not in existing_set:
batch_data.append((domain, tld, source_type))
stats['exists'] = len(existing_domains)
stats['valid'] = len(normalized_domains)
# 干运行模式下直接返回统计信息
if dry_run:
stats['added'] = len(batch_data)
logger.info(f"干运行模式:准备添加 {len(batch_data)} 个新域名")
return stats
logger.info(f"准备添加 {len(batch_data)} 个新域名")
# 分批次添加
for i in range(0, len(batch_data), batch_size):
batch = batch_data[i:i+batch_size]
added_count = self.db.add_domains_batch(batch)
stats['added'] += added_count
stats['failed'] += len(batch) - added_count
# 每处理一批,记录一次进度
if (i + len(batch)) % (batch_size * 10) == 0:
logger.info(f"已添加 {i + len(batch)}/{len(batch_data)} 个域名")
logger.info(f"批量添加域名完成: {stats}")
return stats
except Exception as e:
logger.error(f"批量添加域名出错: {e}")
return {
'total': len(domains),
'valid': 0,
'added': 0,
'exists': 0,
'invalid': 0,
'failed': len(domains)
}
def import_from_file(self, file_path, source_type, batch_size=1000):
"""
从文件导入域名
:param file_path: 文件路径
:param source_type: 来源类型
:param batch_size: 批量大小
:return: dict - 导入统计信息
"""
try:
with open(file_path, 'r', encoding='utf-8') as f:
domains = f.readlines()
# 提取域名
domain_list = []
for domain in domains:
domain = domain.strip()
if domain:
domain_list.append(domain)
# 批量添加域名
stats = self.add_domains_batch(domain_list, source_type, batch_size)
logger.info(f"从文件导入完成: {stats}")
return stats
except Exception as e:
logger.error(f"从文件导入出错: {e}")
return {
'total': 0,
'valid': 0,
'added': 0,
'exists': 0,
'invalid': 0,
'failed': 0
}
def collect_from_juming(self, type_='一口价'):
"""
从聚名网收集域名
:param type_: 类型,一口价或过期删除
:return: int - 收集到的域名数量
"""
# 这里可以添加从聚名网收集域名的逻辑
logger.info(f"从聚名网收集 {type_} 域名")
# 模拟收集结果
return 0
def collect_from_search_engine(self, keyword, limit=100):
"""
从搜索引擎收集域名
:param keyword: 关键词
:param limit: 限制数量
:return: int - 收集到的域名数量
"""
# 这里可以添加从搜索引擎收集域名的逻辑
logger.info(f"从搜索引擎收集域名,关键词: {keyword}, 限制: {limit}")
# 模拟收集结果
return 0
def collect_from_enterprise_directory(self, url, limit=100):
"""
从企业目录收集域名
:param url: 企业目录URL
:param limit: 限制数量
:return: int - 收集到的域名数量
"""
# 这里可以添加从企业目录收集域名的逻辑
logger.info(f"从企业目录收集域名URL: {url}, 限制: {limit}")
# 模拟收集结果
return 0
def collect_from_zone_file(self, file_path):
"""
从Zone File收集域名
:param file_path: Zone File路径
:return: int - 收集到的域名数量
"""
# 这里可以添加从Zone File收集域名的逻辑
logger.info(f"从Zone File收集域名文件: {file_path}")
# 模拟收集结果
return 0

View File

@@ -0,0 +1,150 @@
# -*- coding: UTF-8 -*-
'''
@Project :domainScanDemo
@File :domain_processor.py
@IDE :PyCharm
@Author :梦伴
@Date :2026/4/8 23:51
@explain : 域名处理器
'''
import re
import tldextract
from loguru import logger
from app.utils.database import Database
from app.utils.domain_utils import normalize_domain
class DomainProcessor:
"""
域名处理器
"""
def __init__(self):
"""
初始化域名处理器
"""
self.db = Database()
def process_domain(self, domain):
"""
处理域名
:param domain: 域名
:return: dict - 处理结果
"""
result = {
'original': domain,
'normalized': None,
'tld': None,
'valid': False,
'reason': ''
}
try:
# 标准化域名
normalized = normalize_domain(domain)
if not normalized:
result['reason'] = '无效域名格式'
return result
# 提取顶级域名
ext = tldextract.extract(normalized)
tld = ext.suffix
# 检查顶级域名
if tld not in ['com', 'net']:
result['reason'] = '不支持的顶级域名'
return result
result['normalized'] = normalized
result['tld'] = tld
result['valid'] = True
return result
except Exception as e:
logger.error(f"处理域名出错: {e}")
result['reason'] = f"处理出错: {str(e)}"
return result
def batch_process(self, domains):
"""
批量处理域名
:param domains: 域名列表
:return: list - 处理结果列表
"""
results = []
for domain in domains:
result = self.process_domain(domain)
results.append(result)
return results
def filter_valid_domains(self, domains):
"""
过滤有效的域名
:param domains: 域名列表
:return: list - 有效的域名列表
"""
valid_domains = []
for domain in domains:
result = self.process_domain(domain)
if result['valid']:
valid_domains.append(result['normalized'])
return valid_domains
def update_domain_status(self, domain_id, status_type, status_value):
"""
更新域名状态
:param domain_id: 域名ID
:param status_type: 状态类型
:param status_value: 状态值
:return: bool - 是否更新成功
"""
try:
if status_type == 'use_status':
return self.db.update_domain_use_status(domain_id, status_value)
elif status_type == 'detect_status':
return self.db.update_domain_detect_status(domain_id, status_value)
elif status_type == 'register_status':
return self.db.update_domain_register_status(domain_id, status_value)
else:
logger.error(f"未知的状态类型: {status_type}")
return False
except Exception as e:
logger.error(f"更新域名状态出错: {e}")
return False
def blacklist_domain(self, domain_id, reason):
"""
将域名加入黑名单
:param domain_id: 域名ID
:param reason: 黑名单原因
:return: bool - 是否操作成功
"""
try:
# 更新检测状态为黑名单
if self.db.update_domain_detect_status(domain_id, 4): # 4 表示黑名单
# 添加到黑名单表
domain = self.db.get_domain_by_id(domain_id)
if domain:
return self.db.add_to_blacklist(domain['domain'], reason)
return False
except Exception as e:
logger.error(f"将域名加入黑名单出错: {e}")
return False
def get_domain_statistics(self):
"""
获取域名统计信息
:return: dict - 统计信息
"""
try:
return self.db.get_domain_statistics()
except Exception as e:
logger.error(f"获取域名统计信息出错: {e}")
return {}

View File

@@ -0,0 +1,289 @@
# -*- coding: UTF-8 -*-
'''
@Project :domainScanDemo
@File :export_manager.py
@IDE :PyCharm
@Author :梦伴
@Date :2026/4/8 23:54
@explain : 导出管理器
'''
import os
from loguru import logger
from app.utils.database import Database
class ExportManager:
"""
导出管理器
"""
def __init__(self):
"""
初始化导出管理器
"""
self.db = Database()
def export_domains(self, filter_conditions, output_file):
"""
导出域名
:param filter_conditions: 筛选条件
:param output_file: 输出文件路径
:return: int - 导出的域名数量
"""
try:
# 查询符合条件的域名
domains = self.db.get_domains_by_conditions(filter_conditions)
if not domains:
logger.warning("没有符合条件的域名")
return 0
# 导出到文件
with open(output_file, 'w', encoding='utf-8') as f:
for domain in domains:
f.write(domain['domain'] + '\n')
logger.info(f"成功导出 {len(domains)} 个域名到 {output_file}")
return len(domains)
except Exception as e:
logger.error(f"导出域名出错: {e}")
return 0
def export_with_details(self, filter_conditions, output_file):
"""
导出域名及其详细信息
:param filter_conditions: 筛选条件
:param output_file: 输出文件路径
:return: int - 导出的域名数量
"""
try:
# 查询符合条件的域名及其详细信息
domains = self.db.get_domains_with_details(filter_conditions)
if not domains:
logger.warning("没有符合条件的域名")
return 0
# 导出到文件
with open(output_file, 'w', encoding='utf-8') as f:
# 写入表头
f.write('域名,注册状态,使用状态,检测状态,备案历史,备案年份,快照年份,友情链接数量\n')
# 写入数据
for domain in domains:
line = f"{domain['domain']},{domain['register_status']},{domain['use_status']},{domain['detect_status']},{domain['has_beian']},{domain['beian_year']},{domain['snapshot_years']},{domain['backlink_count']}\n"
f.write(line)
logger.info(f"成功导出 {len(domains)} 个域名及其详细信息到 {output_file}")
return len(domains)
except Exception as e:
logger.error(f"导出域名详细信息出错: {e}")
return 0
def batch_update_status(self, domain_ids, status_type, status_value):
"""
批量更新域名状态
:param domain_ids: 域名ID列表
:param status_type: 状态类型
:param status_value: 状态值
:return: int - 更新成功的域名数量
"""
try:
success_count = 0
for domain_id in domain_ids:
if self.db.update_domain_status(domain_id, status_type, status_value):
success_count += 1
logger.info(f"成功更新 {success_count} 个域名的状态")
return success_count
except Exception as e:
logger.error(f"批量更新域名状态出错: {e}")
return 0
def get_filtered_domains(self, filter_conditions, limit=1000):
"""
获取符合条件的域名
:param filter_conditions: 筛选条件
:param limit: 限制数量
:return: list - 域名列表
"""
try:
domains = self.db.get_domains_by_conditions(filter_conditions, limit)
logger.info(f"获取到 {len(domains)} 个符合条件的域名")
return domains
except Exception as e:
logger.error(f"获取符合条件的域名出错: {e}")
return []
def generate_report(self, output_file):
"""
生成统计报告
:param output_file: 输出文件路径
:return: bool - 是否生成成功
"""
try:
# 获取域名统计信息
domain_stats = self.db.get_domain_statistics()
# 获取任务统计信息
task_stats = self.db.get_task_statistics()
# 生成报告
with open(output_file, 'w', encoding='utf-8') as f:
f.write('域名库系统统计报告\n')
f.write('=' * 50 + '\n')
f.write('\n域名统计:\n')
f.write(f'总域名数: {domain_stats.get("total", 0)}\n')
f.write(f'可注册域名: {domain_stats.get("available", 0)}\n')
f.write(f'已注册域名: {domain_stats.get("registered", 0)}\n')
f.write(f'黑名单域名: {domain_stats.get("blacklisted", 0)}\n')
f.write('\n任务统计:\n')
f.write(f'总任务数: {task_stats.get("total", 0)}\n')
f.write(f'待执行任务: {task_stats.get("pending", 0)}\n')
f.write(f'执行中任务: {task_stats.get("running", 0)}\n')
f.write(f'已完成任务: {task_stats.get("completed", 0)}\n')
f.write(f'失败任务: {task_stats.get("failed", 0)}\n')
logger.info(f"成功生成统计报告到 {output_file}")
return True
except Exception as e:
logger.error(f"生成统计报告出错: {e}")
return False
def export_to_excel(self, domains, output_file):
"""
导出域名到Excel文件
:param domains: 域名列表
:param output_file: 输出文件路径
:return: int - 导出的域名数量
"""
try:
# 尝试导入openpyxl
try:
from openpyxl import Workbook
except ImportError:
# 如果没有安装openpyxl使用CSV格式作为替代
logger.warning("openpyxl库未安装将使用CSV格式导出")
return self.export_to_csv(domains, output_file.replace('.xlsx', '.csv'))
# 创建工作簿
wb = Workbook()
ws = wb.active
# 写入表头
headers = ['域名', '注册状态', '使用状态', '检测状态', '人工复核状态', '过期时间', '单位性质', '网站首页网址', '检测时间', '备案历史', '备案年份', '快照年份', '百度历史', '百度Site', '是否中文标题', '360 Site', 'Google Site', '友情链接数量']
ws.append(headers)
# 写入数据
for domain in domains:
row = [
domain.get('domain', ''),
domain.get('register_status', ''),
domain.get('use_status', ''),
domain.get('detect_status', ''),
domain.get('review_status', ''),
domain.get('expire_date', ''),
domain.get('company_type', ''),
domain.get('website_url', ''),
domain.get('detect_time', ''),
domain.get('has_beian', ''),
domain.get('beian_year', ''),
domain.get('snapshot_years', ''),
domain.get('baidu_history', ''),
domain.get('baidu_site', ''),
domain.get('is_chinese_title', ''),
domain.get('qihu360_site', ''),
domain.get('google_site', ''),
domain.get('backlink_count', '')
]
ws.append(row)
# 保存文件
wb.save(output_file)
logger.info(f"成功导出 {len(domains)} 个域名到Excel文件: {output_file}")
return len(domains)
except Exception as e:
logger.error(f"导出Excel文件出错: {e}")
# 尝试使用CSV格式作为替代
try:
csv_file = output_file.replace('.xlsx', '.csv')
logger.info(f"尝试使用CSV格式导出到: {csv_file}")
return self.export_to_csv(domains, csv_file)
except Exception as e2:
logger.error(f"导出CSV文件也失败: {e2}")
raise
def export_to_csv(self, domains, output_file):
"""
导出域名到CSV文件
:param domains: 域名列表
:param output_file: 输出文件路径
:return: int - 导出的域名数量
"""
try:
import csv
# 写入文件
with open(output_file, 'w', encoding='utf-8', newline='') as f:
writer = csv.writer(f)
# 写入表头
headers = ['域名', '注册状态', '使用状态', '检测状态', '人工复核状态', '过期时间', '单位性质', '网站首页网址', '检测时间', '备案历史', '备案年份', '快照年份', '百度历史', '百度Site', '是否中文标题', '360 Site', 'Google Site', '友情链接数量']
writer.writerow(headers)
# 写入数据
for domain in domains:
row = [
domain.get('domain', ''),
domain.get('register_status', ''),
domain.get('use_status', ''),
domain.get('detect_status', ''),
domain.get('review_status', ''),
domain.get('expire_date', ''),
domain.get('company_type', ''),
domain.get('website_url', ''),
domain.get('detect_time', ''),
domain.get('has_beian', ''),
domain.get('beian_year', ''),
domain.get('snapshot_years', ''),
domain.get('baidu_history', ''),
domain.get('baidu_site', ''),
domain.get('is_chinese_title', ''),
domain.get('qihu360_site', ''),
domain.get('google_site', ''),
domain.get('backlink_count', '')
]
writer.writerow(row)
logger.info(f"成功导出 {len(domains)} 个域名到CSV文件: {output_file}")
return len(domains)
except Exception as e:
logger.error(f"导出CSV文件出错: {e}")
raise
def export_to_txt(self, domains, output_file):
"""
导出域名到TXT文件一行一个域名。
"""
try:
with open(output_file, 'w', encoding='utf-8') as f:
for domain in domains:
value = domain.get('domain', '').strip()
if value:
f.write(value + '\n')
logger.info(f"成功导出 {len(domains)} 个域名到TXT文件: {output_file}")
return len(domains)
except Exception as e:
logger.error(f"导出TXT文件出错: {e}")
raise

View File

@@ -0,0 +1,162 @@
# -*- coding: UTF-8 -*-
'''
@Project :domainScanDemo
@File :task_scheduler.py
@IDE :PyCharm
@Author :梦伴
@Date :2026/4/8 23:53
@explain : 任务调度器
'''
import time
import threading
from loguru import logger
from app.utils.database import Database
from app.core.detect_engine import DetectEngine
class TaskScheduler:
"""
任务调度器
"""
def __init__(self):
"""
初始化任务调度器
"""
self.db = Database()
self.detect_engine = DetectEngine()
self.running = False
self.threads = []
self.max_threads = 10
def start(self):
"""
启动任务调度器
"""
if self.running:
logger.info("任务调度器已经在运行中")
return
self.running = True
logger.info("启动任务调度器")
# 启动多个线程处理任务
for i in range(self.max_threads):
thread = threading.Thread(target=self._process_tasks, daemon=True)
thread.start()
self.threads.append(thread)
logger.info(f"启动任务处理线程 {i+1}")
def stop(self):
"""
停止任务调度器
"""
self.running = False
logger.info("停止任务调度器")
# 等待线程结束
for thread in self.threads:
thread.join(timeout=5)
self.threads.clear()
logger.info("任务调度器已停止")
def _process_tasks(self):
"""
处理任务
"""
while self.running:
try:
# 获取待执行的任务
task = self.db.get_pending_task()
if not task:
# 没有任务,休眠一段时间
time.sleep(1)
continue
task_id = task['id']
domain_id = task['domain_id']
logger.info(f"处理任务: {task_id}, 域名ID: {domain_id}")
# 执行任务
success = self.detect_engine.process_task(task_id)
if success:
logger.info(f"任务处理成功: {task_id}")
else:
logger.warning(f"任务处理失败: {task_id}")
# 短暂休眠,避免过于频繁的数据库操作
time.sleep(0.1)
except Exception as e:
logger.error(f"处理任务出错: {e}")
# 休眠一段时间,避免出错后无限循环
time.sleep(5)
def add_task(self, domain_id, task_type=1, priority=0):
"""
添加任务
:param domain_id: 域名ID
:param task_type: 任务类型1-基础检测2-深度检测
:param priority: 优先级0-低1-中2-高
:return: int - 任务ID
"""
try:
task_id = self.db.create_detect_task(domain_id, task_type, priority)
logger.info(f"添加任务成功: {task_id}, 域名ID: {domain_id}")
return task_id
except Exception as e:
logger.error(f"添加任务失败: {e}")
return None
def get_task_stats(self):
"""
获取任务统计信息
:return: dict - 任务统计信息
"""
try:
return self.db.get_task_statistics()
except Exception as e:
logger.error(f"获取任务统计信息出错: {e}")
return {}
def retry_failed_tasks(self):
"""
重试失败的任务
:return: int - 重试的任务数量
"""
try:
tasks = self.db.get_failed_tasks()
retry_count = 0
for task in tasks:
task_id = task['id']
self.db.update_task_status(task_id, 0) # 0 表示待执行
self.db.update_task_retry_count(task_id, 0) # 重置重试次数
retry_count += 1
logger.info(f"重试 {retry_count} 个失败的任务")
return retry_count
except Exception as e:
logger.error(f"重试失败任务出错: {e}")
return 0
def clear_completed_tasks(self, days=7):
"""
清理已完成的任务
:param days: 保留天数
:return: int - 清理的任务数量
"""
try:
count = self.db.clear_completed_tasks(days)
logger.info(f"清理 {count} 个已完成的任务")
return count
except Exception as e:
logger.error(f"清理已完成任务出错: {e}")
return 0

View File

@@ -0,0 +1,10 @@
{
"juming": {
"email": "chaofanai1998@gmail.com",
"password": "llzz123,./"
},
"juziseo": {
"email": "mamian",
"password": "Abc123456"
}
}

View File

@@ -0,0 +1,20 @@
{
"detect_register": true,
"detect_wayback": true,
"detect_chinaz": true,
"detect_aizhan": true,
"detect_baidu_site": true,
"detect_360_site": true,
"detect_jucha": false,
"detect_juziseo": false,
"detect_order": [
"detect_register",
"detect_baidu_site",
"detect_360_site",
"detect_chinaz",
"detect_aizhan",
"detect_wayback",
"detect_jucha",
"detect_juziseo"
]
}

View File

@@ -0,0 +1,4 @@
# -*- coding: UTF-8 -*-
'''
检测插件模块
'''

View File

@@ -0,0 +1,130 @@
# -*- coding: UTF-8 -*-
'''
@Project :domainScanDemo
@File :aizhan_detector.py
@IDE :PyCharm
@Author :梦伴
@Date :2026/4/9 0:02
@explain : 爱站网检测器
'''
import requests
from curl_cffi import requests as curl_requests
import re
from app.detectors.base import BaseDetector
class AizhanDetector(BaseDetector):
"""
爱站网检测器
"""
def __init__(self):
"""
初始化爱站网检测器
"""
super().__init__()
self.url = 'https://www.aizhan.com'
self.query_url = 'https://www.aizhan.com/cha/{domain}'
def check_domain(self, domain):
"""
检测域名
:param domain: 域名
:return: dict - 检测结果
"""
try:
# 构建查询URL
query_url = self.query_url.format(domain=domain)
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/146.0.0.0 Safari/537.36'
}
# 使用curl_cffi模拟浏览器
response = curl_requests.get(query_url, headers=headers, impersonate='chrome', timeout=10)
if response.status_code == 200:
content = response.text
# 提取标题
title = self._extract_title(content)
# 提取风险信息
risk = self._extract_risk(content)
# 检查是否包含敏感词
has_sensitive = self._check_sensitive(title, risk)
return {
'title': title,
'risk': risk,
'has_sensitive': has_sensitive
}
else:
self._log_warning(f"爱站网查询失败: {response.status_code}")
return {'title': '', 'risk': '', 'has_sensitive': False}
except Exception as e:
return self._handle_exception(e, domain)
def _extract_title(self, content):
"""
提取标题
:param content: 页面内容
:return: str - 标题
"""
try:
pattern = r'<title>(.*?)</title>'
match = re.search(pattern, content)
if match:
return match.group(1).strip()
return ''
except Exception as e:
self._handle_exception(e, 'extract_title')
return ''
def _extract_risk(self, content):
"""
提取风险信息
:param content: 页面内容
:return: str - 风险信息
"""
try:
# 这里需要根据实际页面结构调整正则表达式
pattern = r'百度网址检测:<span[^>]+>(.*?)</span>'
match = re.search(pattern, content)
if match:
return match.group(1).strip()
return ''
except Exception as e:
self._handle_exception(e, 'extract_risk')
return ''
def _check_sensitive(self, title, risk):
"""
检查是否包含敏感词
:param title: 标题
:param risk: 风险信息
:return: bool - 是否包含敏感词
"""
# 风险类型
sensitive_risks = ['低风险', '疑似色情博彩风险', '严重影响权重']
# 敏感词
sensitive_words = ['色情', '赌博', '博彩', '毒品', '暴力', '诈骗']
# 检查风险
for r in sensitive_risks:
if r in risk:
return True
# 检查标题
for word in sensitive_words:
if word in title:
return True
return False

View File

@@ -0,0 +1,151 @@
# -*- coding: UTF-8 -*-
'''
@Project :domainScanDemo
@File :baidu_detector.py
@IDE :PyCharm
@Author :梦伴
@Date :2026/4/8 23:58
@explain : 百度检测器
'''
import requests
from curl_cffi import requests as curl_requests
import re
from app.detectors.base import BaseDetector
class BaiduDetector(BaseDetector):
"""
百度检测器
"""
def __init__(self):
"""
初始化百度检测器
"""
super().__init__()
self.site_url = 'https://www.baidu.com/s'
self.history_url = 'https://www.baidu.com/s'
def check_domain(self, domain):
"""
检测域名
:param domain: 域名
:return: dict - 检测结果
"""
try:
# 检查百度site
site_result = self.check_site(domain)
# 检查百度历史
history_result = self.check_history(domain)
return {
'site': site_result,
'history': history_result
}
except Exception as e:
return self._handle_exception(e, domain)
def check_site(self, domain):
"""
检查百度site收录
:param domain: 域名
:return: dict - 检测结果
"""
try:
params = {
'wd': f'site:{domain}',
'rn': '50'
}
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/146.0.0.0 Safari/537.36'
}
# 使用curl_cffi模拟浏览器
response = curl_requests.get(self.site_url, params=params, headers=headers, impersonate='chrome', timeout=10)
if response.status_code == 200:
content = response.text
# 提取子域名
subdomains = self._extract_subdomains(content, domain)
# 检查是否有收录
has_收录 = '没有找到相关结果' not in content
return {
'has_收录': has_收录,
'subdomains': subdomains
}
else:
self._log_warning(f"百度site查询失败: {response.status_code}")
return {'has_收录': False, 'subdomains': []}
except Exception as e:
self._handle_exception(e, domain)
return {'has_收录': False, 'subdomains': []}
def check_history(self, domain):
"""
检查百度历史收录
:param domain: 域名
:return: dict - 检测结果
"""
try:
params = {
'wd': f'cache:{domain}',
'rn': '50'
}
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/146.0.0.0 Safari/537.36'
}
# 使用curl_cffi模拟浏览器
response = curl_requests.get(self.history_url, params=params, headers=headers, impersonate='chrome', timeout=10)
if response.status_code == 200:
content = response.text
# 检查是否有历史收录
has_history = '百度快照' in content
# 检查是否有灰色内容
has_gray = '风险提示' in content or '安全警告' in content
return {
'has_history': has_history,
'has_gray': has_gray
}
else:
self._log_warning(f"百度历史查询失败: {response.status_code}")
return {'has_history': False, 'has_gray': False}
except Exception as e:
self._handle_exception(e, domain)
return {'has_history': False, 'has_gray': False}
def _extract_subdomains(self, content, domain):
"""
提取子域名
:param content: 搜索结果内容
:param domain: 主域名
:return: list - 子域名列表
"""
try:
# 提取所有包含域名的链接
pattern = r'https?://([a-zA-Z0-9-]+)\.' + re.escape(domain)
matches = re.findall(pattern, content)
# 去重并过滤空值
subdomains = list(set(matches))
subdomains = [sub for sub in subdomains if sub]
return subdomains
except Exception as e:
self._handle_exception(e, domain)
return []

View File

@@ -0,0 +1,70 @@
# -*- coding: UTF-8 -*-
'''
@Project :domainScanDemo
@File :base.py
@IDE :PyCharm
@Author :梦伴
@Date :2026/4/8 23:55
@explain : 基础检测类
'''
from abc import ABC, abstractmethod
from loguru import logger
class BaseDetector(ABC):
"""
基础检测类
"""
def __init__(self):
"""
初始化检测类
"""
self.name = self.__class__.__name__
logger.info(f"初始化检测器: {self.name}")
@abstractmethod
def check_domain(self, domain):
"""
检测域名
:param domain: 域名
:return: dict - 检测结果
"""
pass
def _log_info(self, message):
"""
记录信息日志
:param message: 消息
"""
logger.info(f"[{self.name}] {message}")
def _log_warning(self, message):
"""
记录警告日志
:param message: 消息
"""
logger.warning(f"[{self.name}] {message}")
def _log_error(self, message):
"""
记录错误日志
:param message: 消息
"""
logger.error(f"[{self.name}] {message}")
def _handle_exception(self, e, domain):
"""
处理异常
:param e: 异常
:param domain: 域名
:return: dict - 错误结果
"""
self._log_error(f"检测域名 {domain} 出错: {e}")
return {'error': str(e)}

View File

@@ -0,0 +1,130 @@
# -*- coding: UTF-8 -*-
'''
@Project :domainScanDemo
@File :chinaz_detector.py
@IDE :PyCharm
@Author :梦伴
@Date :2026/4/9 0:01
@explain : 站长之家检测器
'''
import requests
from curl_cffi import requests as curl_requests
import re
from app.detectors.base import BaseDetector
class ChinazDetector(BaseDetector):
"""
站长之家检测器
"""
def __init__(self):
"""
初始化站长之家检测器
"""
super().__init__()
self.url = 'https://seo.chinaz.com'
self.query_url = 'https://seo.chinaz.com/{domain}'
def check_domain(self, domain):
"""
检测域名
:param domain: 域名
:return: dict - 检测结果
"""
try:
# 构建查询URL
query_url = self.query_url.format(domain=domain)
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/146.0.0.0 Safari/537.36'
}
# 使用curl_cffi模拟浏览器
response = curl_requests.get(query_url, headers=headers, impersonate='chrome', timeout=10)
if response.status_code == 200:
content = response.text
# 提取标题
title = self._extract_title(content)
# 提取网站分类
category = self._extract_category(content)
# 检查是否包含敏感词
has_sensitive = self._check_sensitive(title, category)
return {
'title': title,
'category': category,
'has_sensitive': has_sensitive
}
else:
self._log_warning(f"站长之家查询失败: {response.status_code}")
return {'title': '', 'category': '', 'has_sensitive': False}
except Exception as e:
return self._handle_exception(e, domain)
def _extract_title(self, content):
"""
提取标题
:param content: 页面内容
:return: str - 标题
"""
try:
pattern = r'<title>(.*?)</title>'
match = re.search(pattern, content)
if match:
return match.group(1).strip()
return ''
except Exception as e:
self._handle_exception(e, 'extract_title')
return ''
def _extract_category(self, content):
"""
提取网站分类
:param content: 页面内容
:return: str - 分类
"""
try:
# 这里需要根据实际页面结构调整正则表达式
pattern = r'网站分类:<a[^>]+>(.*?)</a>'
match = re.search(pattern, content)
if match:
return match.group(1).strip()
return ''
except Exception as e:
self._handle_exception(e, 'extract_category')
return ''
def _check_sensitive(self, title, category):
"""
检查是否包含敏感词
:param title: 标题
:param category: 分类
:return: bool - 是否包含敏感词
"""
# 敏感分类
sensitive_categories = ['视频电影', '体育运动', '常用查询']
# 敏感词
sensitive_words = ['色情', '赌博', '博彩', '毒品', '暴力', '诈骗']
# 检查分类
for cat in sensitive_categories:
if cat in category:
return True
# 检查标题
for word in sensitive_words:
if word in title:
return True
return False

View File

@@ -0,0 +1,80 @@
# -*- coding: UTF-8 -*-
'''
@Project :domainScanDemo
@File :google_detector.py
@IDE :PyCharm
@Author :梦伴
@Date :2026/4/9 0:00
@explain : Google检测器
'''
import requests
from curl_cffi import requests as curl_requests
import re
from app.detectors.base import BaseDetector
class GoogleDetector(BaseDetector):
"""
Google检测器
"""
def __init__(self):
"""
初始化Google检测器
"""
super().__init__()
self.site_url = 'https://www.google.com/search'
def check_domain(self, domain):
"""
检测域名
:param domain: 域名
:return: dict - 检测结果
"""
try:
# 检查Google site
site_result = self.check_site(domain)
return {
'site': site_result
}
except Exception as e:
return self._handle_exception(e, domain)
def check_site(self, domain):
"""
检查Google site收录
:param domain: 域名
:return: dict - 检测结果
"""
try:
params = {
'q': f'site:{domain}',
'num': '50'
}
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/146.0.0.0 Safari/537.36'
}
# 使用curl_cffi模拟浏览器
response = curl_requests.get(self.site_url, params=params, headers=headers, impersonate='chrome', timeout=10)
if response.status_code == 200:
content = response.text
# 检查是否有收录
has_收录 = 'No results found for' not in content
return {
'has_收录': has_收录
}
else:
self._log_warning(f"Google site查询失败: {response.status_code}")
return {'has_收录': False}
except Exception as e:
self._handle_exception(e, domain)
return {'has_收录': False}

View File

@@ -0,0 +1,228 @@
# -*- coding: UTF-8 -*-
'''
@Project :domainScanDemo
@File :jucha_detector.py
@IDE :PyCharm
@Author :梦伴
@Date :2026/4/9 0:04
@explain : 聚查检测器
'''
import requests
from curl_cffi import requests as curl_requests
import re
from app.detectors.base import BaseDetector
class JuchaDetector(BaseDetector):
"""
聚查检测器
"""
def __init__(self):
"""
初始化聚查检测器
"""
super().__init__()
self.url = 'https://www.jucha.com'
self.whois_url = 'https://www.jucha.com/whois/{domain}'
self.beian_url = 'https://www.jucha.com/beian/{domain}'
self.intercept_url = 'https://www.jucha.com/intercept/{domain}'
def check_domain(self, domain):
"""
检测域名
:param domain: 域名
:return: dict - 检测结果
"""
try:
# 检查WHOIS
whois_result = self.check_whois(domain)
# 检查备案
beian_result = self.check_beian(domain)
# 检查拦截
intercept_result = self.check_intercept(domain)
return {
'whois': whois_result,
'beian': beian_result,
'intercept': intercept_result
}
except Exception as e:
return self._handle_exception(e, domain)
def check_whois(self, domain):
"""
检查WHOIS
:param domain: 域名
:return: dict - 检测结果
"""
try:
# 构建查询URL
query_url = self.whois_url.format(domain=domain)
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/146.0.0.0 Safari/537.36'
}
# 使用curl_cffi模拟浏览器
response = curl_requests.get(query_url, headers=headers, impersonate='chrome', timeout=10)
if response.status_code == 200:
content = response.text
# 提取WHOIS信息
whois_info = self._extract_whois_info(content)
return whois_info
else:
self._log_warning(f"聚查WHOIS查询失败: {response.status_code}")
return {'status': ''}
except Exception as e:
self._handle_exception(e, domain)
return {'status': ''}
def check_beian(self, domain):
"""
检查备案
:param domain: 域名
:return: dict - 检测结果
"""
try:
# 构建查询URL
query_url = self.beian_url.format(domain=domain)
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/146.0.0.0 Safari/537.36'
}
# 使用curl_cffi模拟浏览器
response = curl_requests.get(query_url, headers=headers, impersonate='chrome', timeout=10)
if response.status_code == 200:
content = response.text
# 提取备案信息
beian_info = self._extract_beian_info(content)
return beian_info
else:
self._log_warning(f"聚查备案查询失败: {response.status_code}")
return {'has_beian': False, 'beian_year': '', 'is_enterprise': False, 'beian_match': False}
except Exception as e:
self._handle_exception(e, domain)
return {'has_beian': False, 'beian_year': '', 'is_enterprise': False, 'beian_match': False}
def check_intercept(self, domain):
"""
检查拦截
:param domain: 域名
:return: dict - 检测结果
"""
try:
# 构建查询URL
query_url = self.intercept_url.format(domain=domain)
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/146.0.0.0 Safari/537.36'
}
# 使用curl_cffi模拟浏览器
response = curl_requests.get(query_url, headers=headers, impersonate='chrome', timeout=10)
if response.status_code == 200:
content = response.text
# 检查是否被拦截
is_normal = self._check_intercept_status(content)
return {
'normal': is_normal
}
else:
self._log_warning(f"聚查拦截查询失败: {response.status_code}")
return {'normal': False}
except Exception as e:
self._handle_exception(e, domain)
return {'normal': False}
def _extract_whois_info(self, content):
"""
提取WHOIS信息
:param content: 页面内容
:return: dict - WHOIS信息
"""
try:
# 提取状态信息
pattern = r'域名状态:<span[^>]+>(.*?)</span>'
match = re.search(pattern, content)
status = match.group(1).strip() if match else ''
return {
'status': status
}
except Exception as e:
self._handle_exception(e, 'extract_whois_info')
return {'status': ''}
def _extract_beian_info(self, content):
"""
提取备案信息
:param content: 页面内容
:return: dict - 备案信息
"""
try:
# 检查是否有备案
has_beian = '备案信息' in content
# 提取备案年份
beian_year = ''
pattern = r'审核时间:(\d{4})-\d{2}-\d{2}'
match = re.search(pattern, content)
if match:
beian_year = match.group(1)
# 提取单位性质
is_enterprise = '企业' in content
# 检查首网址和备案网址是否一致
beian_match = '网站首页网址' in content
return {
'has_beian': has_beian,
'beian_year': beian_year,
'is_enterprise': is_enterprise,
'beian_match': beian_match
}
except Exception as e:
self._handle_exception(e, 'extract_beian_info')
return {'has_beian': False, 'beian_year': '', 'is_enterprise': False, 'beian_match': False}
def _check_intercept_status(self, content):
"""
检查拦截状态
:param content: 页面内容
:return: bool - 是否正常
"""
try:
# 检查是否包含正常标识
if '正常' in content:
return True
# 检查是否包含拦截标识
if '拦截' in content:
return False
return False
except Exception as e:
self._handle_exception(e, 'check_intercept_status')
return False

View File

@@ -0,0 +1,214 @@
# -*- coding: UTF-8 -*-
'''
@Project :domainScanDemo
@File :juziseo_detector.py
@IDE :PyCharm
@Author :梦伴
@Date :2026/4/9 0:03
@explain : 桔子SEO检测器
'''
import requests
from curl_cffi import requests as curl_requests
import re
from app.detectors.base import BaseDetector
class JuziseoDetector(BaseDetector):
"""
桔子SEO检测器
"""
def __init__(self):
"""
初始化桔子SEO检测器
"""
super().__init__()
self.url = 'https://seo.juziseo.com'
self.history_url = 'https://seo.juziseo.com/history/{domain}'
self.backlink_url = 'https://seo.juziseo.com/backlink/{domain}'
def check_domain(self, domain):
"""
检测域名
:param domain: 域名
:return: dict - 检测结果
"""
try:
# 检查历史信息
history_result = self.check_history(domain)
# 检查外链
backlink_result = self.check_backlink(domain)
return {
'history': history_result,
'backlink': backlink_result
}
except Exception as e:
return self._handle_exception(e, domain)
def check_history(self, domain):
"""
检查历史信息
:param domain: 域名
:return: dict - 检测结果
"""
try:
# 构建查询URL
query_url = self.history_url.format(domain=domain)
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/146.0.0.0 Safari/537.36'
}
# 使用curl_cffi模拟浏览器
response = curl_requests.get(query_url, headers=headers, impersonate='chrome', timeout=10)
if response.status_code == 200:
content = response.text
# 提取历史信息
history_info = self._extract_history_info(content)
# 检查是否包含敏感词
has_sensitive = self._check_sensitive(history_info)
# 检查是否有百度历史收录
has_baidu_history = '百度历史收录' in content
# 检查是否有子域名
has_subdomains = '子域名' in content
# 检查是否为简体中文
is_simplified = self._check_simplified(content)
return {
'has_sensitive': has_sensitive,
'has_baidu_history': has_baidu_history,
'has_subdomains': has_subdomains,
'is_simplified': is_simplified
}
else:
self._log_warning(f"桔子SEO历史查询失败: {response.status_code}")
return {'has_sensitive': False, 'has_baidu_history': False, 'has_subdomains': False, 'is_simplified': True}
except Exception as e:
self._handle_exception(e, domain)
return {'has_sensitive': False, 'has_baidu_history': False, 'has_subdomains': False, 'is_simplified': True}
def check_backlink(self, domain):
"""
检查外链
:param domain: 域名
:return: dict - 检测结果
"""
try:
# 构建查询URL
query_url = self.backlink_url.format(domain=domain)
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/146.0.0.0 Safari/537.36'
}
# 使用curl_cffi模拟浏览器
response = curl_requests.get(query_url, headers=headers, impersonate='chrome', timeout=10)
if response.status_code == 200:
content = response.text
# 检查是否包含敏感词
has_sensitive = self._check_backlink_sensitive(content)
# 检查是否有子域名
has_subdomains = '子域名' in content
return {
'has_sensitive': has_sensitive,
'has_subdomains': has_subdomains
}
else:
self._log_warning(f"桔子SEO外链查询失败: {response.status_code}")
return {'has_sensitive': False, 'has_subdomains': False}
except Exception as e:
self._handle_exception(e, domain)
return {'has_sensitive': False, 'has_subdomains': False}
def _extract_history_info(self, content):
"""
提取历史信息
:param content: 页面内容
:return: str - 历史信息
"""
try:
# 这里需要根据实际页面结构调整正则表达式
pattern = r'<div class="history-info">(.*?)</div>'
match = re.search(pattern, content, re.DOTALL)
if match:
return match.group(1).strip()
return ''
except Exception as e:
self._handle_exception(e, 'extract_history_info')
return ''
def _check_sensitive(self, history_info):
"""
检查是否包含敏感词
:param history_info: 历史信息
:return: bool - 是否包含敏感词
"""
# 敏感词
sensitive_words = [
'色情', '赌博', '博彩', '毒品', '暴力', '诈骗',
'足球', '直播', '证券', '配资', '软件',
'体育', '商行', '下载', '影视', '网络',
'计算', 'app', 'HTML SiteMap', '模拟器', '传媒',
'二次元', '成人', '米乐', '小说', '凯发',
'人才', '华体', '娱乐', '开户'
]
for word in sensitive_words:
if word in history_info:
return True
return False
def _check_simplified(self, content):
"""
检查是否为简体中文
:param content: 页面内容
:return: bool - 是否为简体中文
"""
# 检查是否包含简体中文标识
if '简体中文' in content:
return True
# 检查是否包含繁体中文标识
if '繁体中文' in content:
return False
return True
def _check_backlink_sensitive(self, content):
"""
检查外链是否包含敏感词
:param content: 页面内容
:return: bool - 是否包含敏感词
"""
# 敏感词
sensitive_words = [
'内幕', '猛料', '精料', '高手', '绝杀',
'权威', '澳门', '色情', '赌博', '博彩'
]
for word in sensitive_words:
if word in content:
return True
return False

View File

@@ -0,0 +1,107 @@
# -*- coding: UTF-8 -*-
'''
@Project :domainScanDemo
@File :qihu360_detector.py
@IDE :PyCharm
@Author :梦伴
@Date :2026/4/8 23:59
@explain : 360检测器
'''
import requests
from curl_cffi import requests as curl_requests
import re
from app.detectors.base import BaseDetector
class Qihu360Detector(BaseDetector):
"""
360检测器
"""
def __init__(self):
"""
初始化360检测器
"""
super().__init__()
self.site_url = 'https://www.so.com/s'
def check_domain(self, domain):
"""
检测域名
:param domain: 域名
:return: dict - 检测结果
"""
try:
# 检查360 site
site_result = self.check_site(domain)
return {
'site': site_result
}
except Exception as e:
return self._handle_exception(e, domain)
def check_site(self, domain):
"""
检查360 site收录
:param domain: 域名
:return: dict - 检测结果
"""
try:
params = {
'q': f'site:{domain}',
'pn': '1',
'rn': '50'
}
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/146.0.0.0 Safari/537.36'
}
# 使用curl_cffi模拟浏览器
response = curl_requests.get(self.site_url, params=params, headers=headers, impersonate='chrome', timeout=10)
if response.status_code == 200:
content = response.text
# 提取子域名
subdomains = self._extract_subdomains(content, domain)
# 检查是否有收录
has_收录 = '没有找到相关结果' not in content
return {
'has_收录': has_收录,
'subdomains': subdomains
}
else:
self._log_warning(f"360 site查询失败: {response.status_code}")
return {'has_收录': False, 'subdomains': []}
except Exception as e:
self._handle_exception(e, domain)
return {'has_收录': False, 'subdomains': []}
def _extract_subdomains(self, content, domain):
"""
提取子域名
:param content: 搜索结果内容
:param domain: 主域名
:return: list - 子域名列表
"""
try:
# 提取所有包含域名的链接
pattern = r'https?://([a-zA-Z0-9-]+)\.' + re.escape(domain)
matches = re.findall(pattern, content)
# 去重并过滤空值
subdomains = list(set(matches))
subdomains = [sub for sub in subdomains if sub]
return subdomains
except Exception as e:
self._handle_exception(e, domain)
return []

View File

@@ -0,0 +1,128 @@
# -*- coding: UTF-8 -*-
'''
@Project :domainScanDemo
@File :rdap_detector.py
@IDE :PyCharm
@Author :梦伴
@Date :2026/4/8 23:56
@explain : RDAP检测器
'''
import requests
from app.detectors.base import BaseDetector
class RDAPDetector(BaseDetector):
"""
RDAP检测器
"""
def __init__(self):
"""
初始化RDAP检测器
"""
super().__init__()
self.rdap_urls = {
'com': 'https://rdap.verisign.com/com/v1/domain/',
'net': 'https://rdap.verisign.com/net/v1/domain/'
}
def check_domain(self, domain):
"""
检测域名
:param domain: 域名
:return: dict - 检测结果
"""
try:
# 提取顶级域名
tld = domain.split('.')[-1]
if tld not in self.rdap_urls:
return {'error': '不支持的顶级域名'}
# 构建RDAP查询URL
url = self.rdap_urls[tld] + domain
# 发送请求
response = requests.get(url, timeout=10)
if response.status_code == 200:
data = response.json()
return self._parse_rdap_response(data)
elif response.status_code == 404:
return {'status': 'available'}
else:
return {'error': f'RDAP查询失败: {response.status_code}'}
except Exception as e:
return self._handle_exception(e, domain)
def check_register_status(self, domain):
"""
检查注册状态
:param domain: 域名
:return: int - 注册状态码
"""
try:
result = self.check_domain(domain)
if 'error' in result:
return 9 # 状态未知
if result.get('status') == 'available':
return 2 # 可注册
# 检查域名状态
statuses = result.get('status', [])
if 'clientHold' in statuses:
return 7 # clientHold
elif 'serverHold' in statuses:
return 8 # serverHold
elif 'autoRenewPeriod' in statuses:
return 4 # 宽限期
elif 'redemptionPeriod' in statuses:
return 5 # 赎回期
elif 'pendingDelete' in statuses:
return 6 # 删除期
else:
return 3 # 已注册
except Exception as e:
self._handle_exception(e, domain)
return 10 # 检测失败
def _parse_rdap_response(self, data):
"""
解析RDAP响应
:param data: RDAP响应数据
:return: dict - 解析结果
"""
result = {
'status': 'registered',
'domain': data.get('ldhName'),
'statuses': data.get('status', []),
'registrar': None,
'creation_date': None,
'expiration_date': None,
'last_update': None
}
# 解析注册商信息
for entity in data.get('entities', []):
if 'registrar' in entity.get('roles', []):
result['registrar'] = entity.get('vcardArray', [[], []])[1][1][3]
break
# 解析时间信息
for event in data.get('events', []):
event_action = event.get('eventAction')
event_date = event.get('eventDate')
if event_action == 'registration':
result['creation_date'] = event_date
elif event_action == 'expiration':
result['expiration_date'] = event_date
elif event_action == 'last update':
result['last_update'] = event_date
return result

View File

@@ -0,0 +1,562 @@
# -*- coding: UTF-8 -*-
'''
@Project :domainScanDemo
@File :wayback_detector.py
@IDE :PyCharm
@Author :梦伴
@Date :2026/4/8 23:57
@explain : Wayback检测器
'''
import html
import json
import re
import threading
import time
import zlib
from base64 import b64decode, b64encode
from concurrent.futures import FIRST_COMPLETED, ThreadPoolExecutor, wait
import redis
import requests
from requests.adapters import HTTPAdapter
from urllib3.util.retry import Retry
from app.config import config
from app.detectors.base import BaseDetector
class WaybackDetector(BaseDetector):
"""
Wayback检测器
"""
TITLE_PATTERN = re.compile(r'<title[^>]*>(.*?)</title>', re.IGNORECASE | re.DOTALL)
def __init__(self):
"""
初始化Wayback检测器
"""
super().__init__()
self.cdx_api_url = 'https://web.archive.org/cdx/search/cdx'
# 使用 id_ 直接取快照内容,避免回放页面额外壳层干扰正文匹配。
self.snapshot_url = 'https://web.archive.org/web/{timestamp}id_/{domain}'
self._timestamp_cache = {}
self._title_cache = {}
self._cache_lock = threading.Lock()
self.session = self._build_session()
self.redis_client = self._build_redis_client()
def _build_session(self):
session = requests.Session()
retry = Retry(
total=max(0, config.WAYBACK_RETRY_COUNT),
backoff_factor=0.5,
status_forcelist=(429, 500, 502, 503, 504),
allowed_methods=frozenset(["GET"]),
raise_on_status=False,
)
adapter = HTTPAdapter(max_retries=retry, pool_connections=10, pool_maxsize=10)
session.mount('http://', adapter)
session.mount('https://', adapter)
session.headers.update({
'User-Agent': config.WAYBACK_USER_AGENT,
'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8',
})
return session
def _build_redis_client(self):
try:
client = redis.Redis(
host=config.REDIS_HOST,
port=config.REDIS_PORT,
password=config.REDIS_PASSWORD,
db=config.REDIS_DB,
decode_responses=True,
socket_connect_timeout=5,
socket_timeout=5,
)
client.ping()
return client
except Exception:
return None
def _title_cache_key(self, domain, timestamp):
return f"domain_tool:wayback_title:{domain}:{timestamp}"
def _timestamp_cache_key(self, domain):
return f"domain_tool:wayback_timestamps:{domain}"
def _record_cache_key(self, domain):
return f"domain_tool:wayback_records:{domain}"
def _normalize_title(self, title):
normalized = html.unescape(title or '')
normalized = re.sub(r'\s+', ' ', normalized, flags=re.DOTALL).strip().lower()
return normalized
def _extract_title(self, content):
match = self.TITLE_PATTERN.search(content or '')
if not match:
return ''
return html.unescape(match.group(1)).strip()
def _load_cached_title(self, domain, timestamp):
cache_key = self._title_cache_key(domain, timestamp)
with self._cache_lock:
if cache_key in self._title_cache:
return self._title_cache[cache_key]
if self.redis_client:
try:
raw_value = self.redis_client.get(cache_key)
if raw_value:
data = json.loads(raw_value)
with self._cache_lock:
self._title_cache[cache_key] = data
return data
except Exception:
pass
return None
def _save_cached_title(self, domain, timestamp, data):
cache_key = self._title_cache_key(domain, timestamp)
with self._cache_lock:
self._title_cache[cache_key] = data
if self.redis_client:
try:
self.redis_client.set(
cache_key,
json.dumps(data, ensure_ascii=False),
ex=max(0, config.WAYBACK_TITLE_CACHE_TTL) or None,
)
except Exception:
pass
def _load_cached_timestamps(self, domain):
if domain in self._timestamp_cache:
return list(self._timestamp_cache[domain])
if not self.redis_client:
return None
try:
raw_value = self.redis_client.get(self._timestamp_cache_key(domain))
if not raw_value:
return None
compressed = b64decode(raw_value.encode('ascii'))
timestamps = json.loads(zlib.decompress(compressed).decode('utf-8'))
if isinstance(timestamps, list):
self._timestamp_cache[domain] = tuple(timestamps)
return list(timestamps)
except Exception:
return None
return None
def _save_cached_timestamps(self, domain, timestamps):
self._timestamp_cache[domain] = tuple(timestamps)
if not self.redis_client:
return
try:
payload = json.dumps(timestamps, separators=(',', ':')).encode('utf-8')
compressed = zlib.compress(payload, level=6)
self.redis_client.set(
self._timestamp_cache_key(domain),
b64encode(compressed).decode('ascii'),
ex=max(0, config.WAYBACK_TIMESTAMP_CACHE_TTL) or None,
)
except Exception:
pass
def _load_cached_records(self, domain):
cache_key = self._record_cache_key(domain)
with self._cache_lock:
if cache_key in self._timestamp_cache:
return [
{'timestamp': item[0], 'digest': item[1]}
for item in self._timestamp_cache[cache_key]
]
if not self.redis_client:
return None
try:
raw_value = self.redis_client.get(cache_key)
if not raw_value:
return None
compressed = b64decode(raw_value.encode('ascii'))
records = json.loads(zlib.decompress(compressed).decode('utf-8'))
if isinstance(records, list):
normalized = tuple(
(item.get('timestamp', ''), item.get('digest', ''))
for item in records if isinstance(item, dict)
)
with self._cache_lock:
self._timestamp_cache[cache_key] = normalized
return [
{'timestamp': item[0], 'digest': item[1]}
for item in normalized if item[0]
]
except Exception:
return None
return None
def _save_cached_records(self, domain, records):
cache_key = self._record_cache_key(domain)
normalized = tuple(
(item.get('timestamp', ''), item.get('digest', ''))
for item in (records or [])
if item and item.get('timestamp')
)
with self._cache_lock:
self._timestamp_cache[cache_key] = normalized
if not self.redis_client:
return
try:
payload = json.dumps(
[{'timestamp': item[0], 'digest': item[1]} for item in normalized],
separators=(',', ':')
).encode('utf-8')
compressed = zlib.compress(payload, level=6)
self.redis_client.set(
cache_key,
b64encode(compressed).decode('ascii'),
ex=max(0, config.WAYBACK_TIMESTAMP_CACHE_TTL) or None,
)
except Exception:
pass
def _fetch_cdx_records(self, domain, limit=None, fast_latest=False):
response = None
try:
params = {
'url': domain,
'output': 'txt',
'fl': 'timestamp,digest',
'filter': ['statuscode:200', 'mimetype:text/html'],
}
if limit is not None:
params['limit'] = str(limit)
if fast_latest:
params['fastLatest'] = 'true'
response = self.session.get(
self.cdx_api_url,
params=params,
timeout=config.WAYBACK_CDX_TIMEOUT,
stream=True,
)
if response.status_code != 200:
self._log_warning(f"获取快照记录失败: {response.status_code}")
return []
records = []
seen = set()
for raw_line in response.iter_lines(decode_unicode=True):
line = (raw_line or '').strip()
if not line:
continue
parts = line.split()
timestamp = parts[0].strip() if parts else ''
digest = parts[1].strip() if len(parts) > 1 else ''
if not timestamp or timestamp in seen:
continue
seen.add(timestamp)
records.append({'timestamp': timestamp, 'digest': digest})
return records
except Exception as e:
self._handle_exception(e, domain)
return []
finally:
try:
if response is not None:
response.close()
except Exception:
pass
def get_latest_snapshot_record(self, domain):
records = self._fetch_cdx_records(domain, limit=-1, fast_latest=True)
return records[0] if records else None
def get_snapshot_records(self, domain):
cached_records = self._load_cached_records(domain)
if cached_records is not None:
return cached_records
records = self._fetch_cdx_records(domain)
if records:
self._save_cached_records(domain, records)
self._save_cached_timestamps(domain, [item['timestamp'] for item in records])
return records
def check_domain(self, domain):
"""
检测域名
:param domain: 域名
:return: dict - 检测结果
"""
try:
return self.scan_snapshots(domain)
except Exception as e:
return self._handle_exception(e, domain)
def get_snapshot_timestamps(self, domain):
cached_records = self._load_cached_records(domain)
if cached_records is not None:
return [item['timestamp'] for item in cached_records if item.get('timestamp')]
cached_timestamps = self._load_cached_timestamps(domain)
if cached_timestamps is not None:
return cached_timestamps
records = self.get_snapshot_records(domain)
return [item['timestamp'] for item in records if item.get('timestamp')]
def _fetch_snapshot_title(self, domain, timestamp):
cached = self._load_cached_title(domain, timestamp)
if cached is not None:
return cached
snapshot_url = self.snapshot_url.format(timestamp=timestamp, domain=domain)
response = None
try:
response = self.session.get(
snapshot_url,
timeout=config.WAYBACK_SNAPSHOT_TIMEOUT,
stream=True,
)
if response.status_code != 200:
data = {'timestamp': timestamp, 'title': '', 'ok': False}
self._save_cached_title(domain, timestamp, data)
return data
content_type = (response.headers.get('Content-Type') or '').lower()
if content_type and 'text/html' not in content_type and 'application/xhtml+xml' not in content_type:
data = {'timestamp': timestamp, 'title': '', 'ok': False}
self._save_cached_title(domain, timestamp, data)
return data
chunks = []
total_bytes = 0
found_title = False
for chunk in response.iter_content(chunk_size=4096, decode_unicode=True):
if not chunk:
continue
chunks.append(chunk)
total_bytes += len(chunk.encode('utf-8', errors='ignore'))
current_text = ''.join(chunks)
if '</title>' in current_text.lower():
found_title = True
break
if total_bytes >= config.WAYBACK_TITLE_MAX_BYTES:
break
content = ''.join(chunks)
title = self._extract_title(content) if found_title or content else ''
data = {'timestamp': timestamp, 'title': title, 'ok': True}
self._save_cached_title(domain, timestamp, data)
return data
except Exception:
data = {'timestamp': timestamp, 'title': '', 'ok': False}
self._save_cached_title(domain, timestamp, data)
return data
finally:
try:
if response is not None:
response.close()
except Exception:
pass
def get_snapshot_years(self, domain):
"""
获取快照年份
:param domain: 域名
:return: list - 快照年份列表
"""
try:
years = set()
for timestamp in self.get_snapshot_timestamps(domain):
if len(timestamp) >= 4:
years.add(int(timestamp[:4]))
return sorted(years)
except Exception as e:
self._handle_exception(e, domain)
return []
def has_sensitive_content(self, domain):
"""
检查是否包含敏感内容
:param domain: 域名
:return: bool - 是否包含敏感内容
"""
try:
result = self.scan_snapshots(domain)
return result.get('has_sensitive_content', False)
except Exception as e:
self._handle_exception(e, domain)
return False
def scan_snapshots(self, domain, sensitive_words=None, stop_on_first_hit=True):
sensitive_words = sensitive_words or config.load_sensitive_words()
latest_record = self.get_latest_snapshot_record(domain)
latest_timestamp = (latest_record or {}).get('timestamp')
latest_digest = (latest_record or {}).get('digest', '')
matched_word = None
matched_timestamp = None
matched_title = None
fetched_snapshot_count = 0
failed_snapshot_count = 0
unique_title_count = 0
duplicate_title_skipped = 0
digest_duplicate_skipped = 0
started_at = time.time()
progress_interval = max(1, config.WAYBACK_PROGRESS_INTERVAL)
title_seen = set()
digest_seen = set()
checked_snapshot_count = 0
domain_concurrency = max(1, config.WAYBACK_DOMAIN_CONCURRENCY)
if latest_timestamp:
latest_result = self._fetch_snapshot_title(domain, latest_timestamp)
checked_snapshot_count = 1
if latest_result and latest_result.get('ok'):
fetched_snapshot_count = 1
title = latest_result.get('title', '')
normalized_title = self._normalize_title(title)
if latest_digest:
digest_seen.add(latest_digest)
if normalized_title:
title_seen.add(normalized_title)
unique_title_count = 1
matched_word = self._find_sensitive_word(title, sensitive_words)
if matched_word and stop_on_first_hit:
matched_timestamp = latest_timestamp
matched_title = title
return {
'snapshot_years': [int(latest_timestamp[:4])] if len(latest_timestamp) >= 4 else [],
'has_sensitive_content': True,
'matched_word': matched_word,
'matched_timestamp': matched_timestamp,
'matched_title': matched_title,
'backlink_count': 0,
'backlink_count_gt_10': False,
'checked_snapshot_count': checked_snapshot_count,
'fetched_snapshot_count': fetched_snapshot_count,
'failed_snapshot_count': failed_snapshot_count,
'unique_title_count': unique_title_count,
'duplicate_title_skipped': duplicate_title_skipped,
'digest_duplicate_skipped': digest_duplicate_skipped,
'elapsed_seconds': round(time.time() - started_at, 2),
}
else:
failed_snapshot_count += 1
records = sorted(self.get_snapshot_records(domain), key=lambda item: item.get('timestamp', ''), reverse=True)
years = sorted({int(item['timestamp'][:4]) for item in records if len(item.get('timestamp', '')) >= 4})
checked_snapshot_count = len(records)
pending_records = []
for item in records:
timestamp = item.get('timestamp', '')
digest = item.get('digest', '')
if not timestamp:
continue
if latest_timestamp and timestamp == latest_timestamp:
continue
if digest and digest in digest_seen:
digest_duplicate_skipped += 1
continue
if digest:
digest_seen.add(digest)
pending_records.append(item)
with ThreadPoolExecutor(max_workers=domain_concurrency) as executor:
pending = {}
index = 0
finished_count = 1 if latest_timestamp else 0
stop_requested = False
while (index < len(pending_records) or pending) and not stop_requested:
while index < len(pending_records) and len(pending) < domain_concurrency and not stop_requested:
timestamp = pending_records[index]['timestamp']
future = executor.submit(self._fetch_snapshot_title, domain, timestamp)
pending[future] = timestamp
index += 1
if not pending:
break
done, _ = wait(list(pending.keys()), return_when=FIRST_COMPLETED)
for future in done:
timestamp = pending.pop(future, None)
finished_count += 1
try:
result = future.result()
except Exception as e:
failed_snapshot_count += 1
self._handle_exception(e, domain)
continue
if not result or not result.get('ok'):
failed_snapshot_count += 1
continue
fetched_snapshot_count += 1
title = result.get('title', '')
normalized_title = self._normalize_title(title)
if normalized_title:
if normalized_title in title_seen:
duplicate_title_skipped += 1
else:
title_seen.add(normalized_title)
unique_title_count += 1
matched_word = self._find_sensitive_word(title, sensitive_words)
if matched_word:
matched_timestamp = timestamp
matched_title = title
if stop_on_first_hit:
stop_requested = True
if finished_count % progress_interval == 0:
elapsed = round(time.time() - started_at, 2)
self._log_info(
f"{domain} 时光机进度: {finished_count}/{checked_snapshot_count},成功 {fetched_snapshot_count},失败 {failed_snapshot_count},唯一标题 {unique_title_count},标题重复跳过 {duplicate_title_skipped}digest 重复跳过 {digest_duplicate_skipped},耗时 {elapsed}s"
)
if config.WAYBACK_REQUEST_DELAY > 0:
time.sleep(config.WAYBACK_REQUEST_DELAY)
if stop_requested:
for future in pending:
future.cancel()
return {
'snapshot_years': years,
'has_sensitive_content': matched_word is not None,
'matched_word': matched_word,
'matched_timestamp': matched_timestamp,
'matched_title': matched_title,
'backlink_count': 0,
'backlink_count_gt_10': False,
'checked_snapshot_count': checked_snapshot_count,
'fetched_snapshot_count': fetched_snapshot_count,
'failed_snapshot_count': failed_snapshot_count,
'unique_title_count': unique_title_count,
'duplicate_title_skipped': duplicate_title_skipped,
'digest_duplicate_skipped': digest_duplicate_skipped,
'elapsed_seconds': round(time.time() - started_at, 2),
}
def _check_sensitive_words(self, content):
"""
检查敏感词
:param content: 内容
:return: bool - 是否包含敏感词
"""
return self._find_sensitive_word(content, config.load_sensitive_words()) is not None
def _find_sensitive_word(self, content, sensitive_words):
for word in sensitive_words or []:
if word and word in (content or ''):
return word
return None
def get_backlink_count(self, domain):
"""
当前策略仅扫描标题,不再抓取正文,友链数量默认返回 0。
"""
return 0
def _count_backlinks(self, content):
return 0

View File

@@ -0,0 +1,3 @@
{
"suffixes": ".com,.net"
}

Binary file not shown.

Binary file not shown.

Binary file not shown.

68
domainCheck/app/main.py Normal file
View File

@@ -0,0 +1,68 @@
# -*- coding: UTF-8 -*-
'''
@Project :domainScanDemo
@File :main.py
@IDE :PyCharm
@Author :梦伴
@Date :2026/4/9 0:09
@explain : 系统主入口
'''
import sys
import os
# 添加项目根目录到 sys.path
sys.path.insert(0, os.path.dirname(os.path.dirname(os.path.abspath(__file__))))
from loguru import logger
from PySide6.QtWidgets import QApplication
from app.ui.main_window import MainWindow
from app.utils.database import Database
from app.config import config
# 配置日志
logger.add(
os.path.join(config.LOG_DIR, config.LOG_FILE),
level=config.LOG_LEVEL,
rotation="10 MB",
compression="zip"
)
def main():
"""
主函数
"""
try:
# 初始化数据库连接
db = Database(
host=config.DB_HOST,
port=config.DB_PORT,
database=config.DB_DATABASE,
user=config.DB_USER,
password=config.DB_PASSWORD
)
# 测试数据库连接
db.execute("SELECT 1")
logger.info("数据库连接成功")
# 初始化应用程序
app = QApplication(sys.argv)
window = MainWindow()
window.show()
# 运行应用程序
sys.exit(app.exec())
except Exception as e:
logger.error(f"启动应用程序失败: {e}")
sys.exit(1)
finally:
# 关闭数据库连接
if 'db' in locals():
db.close()
if __name__ == "__main__":
main()

View File

@@ -0,0 +1,4 @@
{
"proxy_enable": false,
"proxy_url": ""
}

15108
domainCheck/app/sdk_leg.js Normal file

File diff suppressed because one or more lines are too long

View File

@@ -0,0 +1,897 @@
process_ = process;
require_ = require;
delete Buffer;
// delete process;
delete require;
delete global;
delete module;
delete exports;
delete __filename;
delete __dirname;
delete SharedArrayBuffer;
AsObj = {
// print: console.log,
print: function () { },
// print_:console.log,
}
no_print = ['Boolean','String','parseFloat','Array','Object','prepareStackTrace_'];
function watch(object, WatchName) {
const handler = {
get(target, property, receiver) {
if (
property !== 'isNaN' &&
property !== 'encodeURI' &&
property !== "Uint8Array" &&
property !== 'undefined' &&
property !== 'JSON' &&
property !== 'Number' &&
!no_print.includes(property) &&
property !== Symbol.for('nodejs.util.inspect.custom') &&
typeof property !== 'symbol'
) {
if (property === 'global') {
return undefined;
}
if (property === 'Buffer') {
return undefined;
}
if (property === 'process') {
return undefined;
}
if (WatchName === 'config_data') {
debugger
}
if (WatchName.indexOf('.prototype') != -1 && target[property] != undefined) {
return Reflect.get(target, property, receiver);
}
AsObj.print(
"方法:", "get",
"对象:", WatchName,
"属性:", property,
"属性类型:", typeof property,
"属性值:", typeof target[property] == 'object' ? "object" : target[property],
"属性值类型:", typeof target[property]
);
}
if (WatchName === 'top') {
return window;
}
return Reflect.get(target, property, receiver);
},
set(target, property, value, receiver) {
if (WatchName.indexOf('.prototype') != -1 && value != undefined) {
return Reflect.set(target, property, value, receiver);
}
AsObj.print(
"方法:", "set",
"对象:", WatchName,
"属性:", property,
"属性类型:", typeof property,
"属性值:", typeof value == 'object' ? "object" : value,
"属性值类型:", typeof target[property]
);
return Reflect.set(target, property, value, receiver);
},
// in操作 检测
has(target, property) {
AsObj.print(
"代理对象:", WatchName,
"方法:", "has",
"检查属性:", property,
"结果:", typeof target[property] == 'object' ? "object" : target[property],
);
return Reflect.has(target, property);
},
// Object.key 检测
ownKeys(target) {
AsObj.print(
"方法:", "ownKeys",
"对象:", target+''
);
return Reflect.ownKeys(target);
}
};
return new Proxy(object, handler);
}
// function watch(object, WatchName) {
// return object
// }
// 保护函数toString检测
const safeFunction = function safeFunction(func) {
//处理安全函数
Function.prototype.$call = Function.prototype.call;
const $toString = Function.toString;
const myFunction_toString_symbol = Symbol('('.concat('', ')'));
const myToString = function myToString() {
return typeof this === 'function' && this[myFunction_toString_symbol] || $toString.$call(this);
}
const set_native = function set_native(func, key, value) {
Object.defineProperty(func, key, {
"enumerable": false,
"configurable": true,
"writable": true,
"value": value
});
}
delete Function.prototype['toString'];
set_native(Function.prototype, "toString", myToString);
set_native(Function.prototype.toString, myFunction_toString_symbol, "function toString() { [native code] }");
const safe_Function = function safe_Function(func) {
set_native(func, myFunction_toString_symbol, "function" + (func.name ? " " + func.name : "") + "() { [native code] }");
}
return safe_Function(func)
}
//创建函数,并代理上
const makeFunction = function makeFunction(name) {
v_log = AsObj.print;
// 使用 Function 保留函数名
func = new Function("v_log", `
return function ${name}() {
v_log('函数${name}传参-->', arguments);
};
`)(v_log); // 传递 v_log 到动态函数
safeFunction(func);
func = watch(func,`${name}`);
func.prototype = watch(func.prototype, `${name}.prototype`);
return func;
}
!(function () {
"use strict";
const $toString = Function.toString;
const myFunction_toString_symbol = Symbol('('.concat('', ')_', (Math.random() + '').toString(36)));
const mytoString = function () {
return typeof this == 'function' && this[myFunction_toString_symbol] || $toString.call(this);
};
function set_native(func, key, value) {
Object.defineProperty(func, key, {
"enumerable": false,
"configurable": true,
"writable": true,
"value": value
})
};
delete Function.prototype['toString'];
set_native(Function.prototype, "toString", mytoString);
set_native(Function.prototype.toString, myFunction_toString_symbol, "function toString() { [native code] }");
this.func_set_native = function (func) {
set_native(func, myFunction_toString_symbol, `function ${myFunction_toString_symbol, func.name || ''}() { [native code] }`)
}
}).call(globalThis);
// 重写全局对象原型链
function setTostringAndstringTag(obj) {
Object.defineProperties(obj.prototype, {
[Symbol.toStringTag]: {
configurable: true,
value: obj.name
}
});
safeFunction(obj);
};
// 创建标签原型
function createTagProto(propObj,portotypeObj) {
let res = propObj + ' = ' + 'function ' + propObj + '() { throw new TypeError("Illegal constructor"); };\n';
res += 'setTostringAndstringTag(' + propObj + ',null);\n';
if (portotypeObj) {
for (let key in portotypeObj) {
res += propObj + '.prototype.' + portotypeObj[key] + '= function ' + portotypeObj[key] + '() {AsObj.print("'+propObj+'.prototype.' + portotypeObj[key] + '原型方法(需在实例对象上补该方法)::",arguments)};\n';
res += 'globalThis.func_set_native(' + propObj + '.prototype.' + portotypeObj[key] + ');\n';
}
}
eval(res);
}
Object.defineProperties(globalThis, {
[Symbol.toStringTag]: {
configurable: true,
value: 'Window'
}
});
for (let key in globalThis) {
if (typeof globalThis[key] === 'function') {
safeFunction(globalThis[key])
}
}
for (let key in console) {
if (typeof console[key] === 'function') {
safeFunction(console[key])
}
}
createTagProto('EventTarget',['addEventListener']);
createTagProto('WindowProperties');
createTagProto('Window');
window = globalThis;
window.__proto__ = Window.prototype;
window.__proto__.__proto__ = WindowProperties.prototype;
window.__proto__.__proto__.__proto__ = EventTarget.prototype;
Window.__proto__ = EventTarget;
Object.defineProperty(window, 'WindowProperties', {
get: function () {
return undefined;
}
})
function randoms(min, max) {
return Math.floor(Math.random() * (max - min + 1) + min)
}
function getRandomValues(buf) {
var min = 0,
max = 255;
if (buf instanceof Uint16Array) {
max = 65535;
} else if (buf instanceof Uint32Array) {
max = 4294967295;
}
for (var element in buf) {
buf[element] = randoms(min, max);
}
return buf;
}
self = window.self = window;
frames = window.frames = window;
top = window.top = window;
parent = window.parent = window;
global = window.global = window;
Object.defineProperty(window, "global", {
configurable:false,
enumerable: true,
set: undefined,
get: function global(){
return window
}
})
Object.defineProperty(window, "top", {
configurable:false,
enumerable: true,
set: undefined,
get: function top(){
return window
}
})
Object.defineProperty(window, "self", {
configurable:false,
enumerable: true,
set: undefined,
get: function self(){
return window
}
})
Object.defineProperty(window, "parent", {
configurable:false,
enumerable: true,
set: undefined,
get: function parent(){
return window
}
})
Object.defineProperty(window, "frames", {
configurable:false,
enumerable: true,
set: undefined,
get: function frames(){
return window
}
})
innerWidth = 1536
innerHeight = 715
outerWidth = 1536
outerHeight = 824
devicePixelRatio = 1.25;
screenLeft = 0;
screenX = 0;
screenTop = 0;
screenY = 0;
opener = null;
isSecureContext = true;
crypto = {
getRandomValues:getRandomValues
};
createTagProto('DOMStringMap')
createTagProto('HTMLHeadElement',['insertBefore','removeChild'])
createTagProto('HTMLBodyElement',['addEventListener','appendChild','removeChild'])
createTagProto('HTMLHtmlElement',['getAttribute'])
createTagProto('HTMLDocument')
createTagProto('Document',['browsingTopics','appendChild','querySelector','evaluate','querySelectorAll','removeChild','requestStorageAccess','requestStorageAccessFor','hasStorageAccess','getElementsByTagName','hasPrivateToken','createElement','hasRedemptionRecord','hasFocus'])
createTagProto('Node')
document = {};
document.__proto__ = HTMLDocument.prototype;
document.__proto__.__proto__ = Document.prototype;
document.__proto__.__proto__.__proto__ = Node.prototype;
document.__proto__.__proto__.__proto__.__proto__ = EventTarget.prototype;
HTMLDocument.__proto__ = Document;
HTMLDocument.__proto__.__proto__ = Node;
HTMLDocument.__proto__.__proto__.__proto__ = EventTarget;
Document.__proto__ = Node;
Document.__proto__.__proto__ = EventTarget;
Node.__proto__ = EventTarget;
createTagProto('Plugin');
createTagProto('PluginArray');
plugins0 = {
name: 'PDF Viewer',
filename: 'internal-pdf-viewer',
description:'Portable Document Format',
length: 2,
'0': {
type: 'application/pdf',
},
'1':{
type:'text/pdf'
}
}
plugins0['0'].enabledPlugin = plugins0;
plugins0['1'].enabledPlugin = plugins0;
plugins1 = {
name: 'Chrome PDF Viewer',
filename: 'internal-pdf-viewer',
description:'Portable Document Format',
length: 2,
'0': {
type:'application/pdf'
},
'1': {
type:'text/pdf'
}
}
plugins1['0'].enabledPlugin = plugins1;
plugins1['1'].enabledPlugin = plugins1;
plugins2 = {
name: 'Chromium PDF Viewer',
filename: 'internal-pdf-viewer',
description:'Portable Document Format',
length: 2,
'0': {
type:'application/pdf'
},
'1': {
type:'text/pdf'
}
}
plugins2['0'].enabledPlugin = plugins2;
plugins2['1'].enabledPlugin = plugins2;
plugins3 = {
name: 'Microsoft Edge PDF Viewer',
filename: 'internal-pdf-viewer',
description:'Portable Document Format',
length: 2,
'0':{
type:'application/pdf'
},
'1': {
type:'text/pdf'
}
}
plugins3['0'].enabledPlugin = plugins3;
plugins3['1'].enabledPlugin = plugins3;
plugins4 = {
name: 'WebKit built-in PDF',
filename: 'internal-pdf-viewer',
description:'Portable Document Format',
length: 2,
'0': {
type:'application/pdf'
},
'1':{
type:'text/pdf'
}
}
plugins4['0'].enabledPlugin = plugins4;
plugins4['1'].enabledPlugin = plugins4;
plugins = {
length: 5,
'0': plugins0,
'1': plugins1,
'2': plugins2,
'3': plugins3,
'4': plugins4,
namedItem : function (name) {
AsObj.print('Plugin-namedItem:', name)
},
item: function (index) {
AsObj.print('Plugin-item:', index)
return watch(plugins0,'item-'+index);
},
refresh: function () {
AsObj.print('Plugin-refresh:',arguments)
},
}
plugins.__proto__ = PluginArray.prototype;
MimeTypeArray = function MimeTypeArray() {
this.length = 2;
this['0'] = {
suffixes: 'pdf',
type: 'application/pdf',
description:"Portable Document Format",
enabledPlugin: plugins0
};
this['1'] = {
suffixes: 'pdf',
type: 'text/pdf',
description:"Portable Document Format",
enabledPlugin: plugins0
};
};
MimeTypeArray.prototype.toString = function () { return '[object MimeTypeArray]'; }
MimeTypeArray.toString = function () { return 'function MimeTypeArray() { [native code] }'; }
Object.defineProperties(MimeTypeArray.prototype, { [Symbol.toStringTag]: { value: 'MimeTypeArray' } })
MimeTypeArrayc = new MimeTypeArray();
MimeTypeArrayc[Symbol.iterator] = function* () {
for (let key in this) {
yield this[key];
}
}
// 创建电池管理器对象原型
const BatteryManager = {
level: 1,
charging: true,
chargingTime: 0,
dischargingTime: null,
onchargingchange: null,
onlevelchange: null,
toString: function toString() {
return `BatteryManager {
charging: ${this.charging},
level: ${this.level},
chargingTime: ${this.chargingTime},
dischargingTime: ${this.dischargingTime}
}`
}
}
window.BatteryManager = BatteryManager;
Promise2 = {
then: function () {
return this;
},
catch: function (){},
};
createTagProto('Bluetooth');
createTagProto('Navigator');
Navigator.prototype.hardwareConcurrency = 8;
Navigator.prototype.userAgent = 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/139.0.0.0 Safari/537.36';
Navigator.prototype.appVersion = '5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/139.0.0.0 Safari/537.36'
Navigator.prototype.appName = 'Netscape';
Navigator.prototype.appCodeName = 'Mozilla';
Navigator.prototype.vendor = 'Google Inc.';
Navigator.prototype.maxTouchPoints = 10;
Navigator.prototype.platform = 'Win32';
Navigator.prototype.adAuctionComponents = function adAuctionComponents() {
AsObj.print('adAuctionComponents:::', arguments)
}
safeFunction(Navigator.prototype.adAuctionComponents)
Navigator.prototype.runAdAuction = function runAdAuction() {
AsObj.print('runAdAuction:::', arguments)
}
safeFunction(Navigator.prototype.runAdAuction)
Navigator.prototype.canLoadAdAuctionFencedFrame = makeFunction('canLoadAdAuctionFencedFrame')
Navigator.prototype.deprecatedReplaceInURN = makeFunction('deprecatedReplaceInURN')
Navigator.prototype.deprecatedURNToURL = makeFunction('deprecatedURNToURL')
Navigator.prototype.joinAdInterestGroup = makeFunction('joinAdInterestGroup')
Navigator.prototype.leaveAdInterestGroup = makeFunction('leaveAdInterestGroup')
Navigator.prototype.updateAdInterestGroups = makeFunction('updateAdInterestGroups')
Navigator.prototype.connection = watch({
downlink: 9.1,
effectiveType: '4g',
rtt: 0,
saveData: false,
},'connection')
Navigator.prototype.language = 'zh-CN';
Navigator.prototype.languages = ["zh-CN"];
Navigator.prototype.plugins = plugins;
Navigator.prototype.webdriver = false;
Navigator.prototype.cookieEnabled = true;
Navigator.prototype.onLine = true;
Navigator.prototype.doNotTrack = null;
Navigator.prototype.bluetooth = {};
Navigator.prototype.product = 'Gecko'
Navigator.prototype.deviceMemory = 8
Navigator.prototype.mediaDevices = watch({
enumerateDevices: function enumerateDevices() {
return new Promise((resolve, reject) => {
const offer = [
{deviceId: '', kind: 'audioinput', label: '', groupId: ''},
{deviceId: '', kind: 'videoinput', label: '', groupId: ''},
{deviceId: '', kind: 'audiooutput', label: '', groupId: ''},
]
resolve(offer);
});
},
getUserMedia: function getUserMedia() {
AsObj.print('getUserMedia:::', arguments)
}
},'mediaDevices')
Navigator.prototype.storage = {
estimate: function estimate() {
AsObj.print('estimate:::', arguments)
return new Promise((resolve, reject) => {
const offer = {
usage: 0, // 1GB
quota: 2147483648, // 1GB,
usageDetails: {caches: 512, indexedDB: 2855}
};
resolve(offer);
});
}
}
Navigator.prototype.webkitPersistentStorage = watch({},'webkitPersistentStorage')
Navigator.prototype.webkitTemporaryStorage = watch({
queryUsageAndQuota: function queryUsageAndQuota() {
AsObj.print('queryUsageAndQuota:::', arguments)
return new Promise((resolve, reject) => {
const offer = {
usage: 1024 * 1024 * 1024, // 1GB
quota: 1024 * 1024 * 1024, // 1GB
};
resolve(offer);
});
}
},'webkitTemporaryStorage')
Navigator.prototype.bluetooth.__proto__ = Bluetooth.prototype;
Navigator.prototype.javaEnabled = function javaEnabled() {
return false
};
safeFunction(Navigator.prototype.javaEnabled)
Navigator.prototype.getBattery = function getBattery() {
AsObj.print('getBattery:::', arguments)
return Promise.resolve({
__proto__: BatteryManager,
// 动态参数配置(示例值)
level: 1,
charging: true,
dischargingTime: null // 2小时放电时间
})
}
safeFunction(Navigator.prototype.getBattery)
Navigator.prototype.registerProtocolHandler = function registerProtocolHandler() {
AsObj.print('registerProtocolHandler:::',arguments)
}
safeFunction(Navigator.prototype.registerProtocolHandler)
Navigator.prototype.mimeTypes = watch(MimeTypeArrayc,'mimeTypes');
Navigator.prototype.geolocation = {
getCurrentPosition: function getCurrentPosition() {
return Promise2;
}
}
Navigator.prototype.pdfViewerEnabled = true;
Navigator.prototype.doNotTrack = null;
Navigator.prototype.keyboard = watch({
getLayoutMap: function getLayoutMap() {
AsObj.print('Navigator.prototype.keyboard:', arguments)
return {
then: function () {
// arguments[0](watch({
// size: 48,
// values: function () {
// return ['k', 'g', '2', '0', 'v', 'a', '`', 'l', '\\', "'", 'w', '8', 'm', 'h', '.', '7', '1', 'p', 'd', 'f', 'o', 'q', 'c', 'n', '[', 'z', 'y', '3', '6', '5', 'x', '/', '\\', ',', '-', '4', 'b', 't', '9', 's', 'i', 'u', '=', 'j', ';', 'r', ']', 'e']
// }
// }, 'navigator.keyboard.getLayoutMap.then'))
return {
catch: function () {
arguments[0]({
message:'getLayoutMap() must be called from a top-level browsing context or allowed by the permission policy.'
})
}
}
}
}
}
},'navigator.keyboard')
Navigator.prototype.permissions = watch({
query: function query() {
arg_obj = arguments[0];
if (arg_obj.name === 'audio_capture') {
return new Promise((resolve, reject) => {
const offer = {
state: 'prompt',
onchange: null,
name:arg_obj.name
};
resolve(offer);
});
}
if (arg_obj.name === 'microphone') {
return {
then: function () {
arguments[0](watch({
state: 'denied',
onchange: null,
name: 'audio_capture'
},'permissions.query.microphone'));
return {catch:function(){}}
},
catch:function(){}
}
}
if (arg_obj.name === 'camera') {
return {
then: function () {
arguments[0](watch({
state: 'prompt',
onchange: null,
name: 'video_capture'
},'permissions.query.camera'));
return {catch:function(){}}
},
catch:function(){}
}
}
AsObj.print('permissions.query:::', arguments)
}
},'permissions')
Navigator.prototype.productSub = '20030107'
Navigator.prototype.getGamepads = function getGamepads() {
AsObj.print('getGamepads:::', arguments)
return [null,null,null,null]
}
safeFunction(Navigator.prototype.getGamepads)
Navigator.prototype.sendBeacon = makeFunction('sendBeacon')
Navigator.prototype.deprecatedRunAdAuctionEnforcesKAnonymity = false
Navigator.prototype.gpu = watch({
getPreferredCanvasFormat: function getPreferredCanvasFormat() {
AsObj.print('gpu.getPreferredCanvasFormat:', arguments)
return 'bgra8unorm'
},
wgslLanguageFeatures: watch({
size: 7,
values: function values() {
debugger
AsObj.print('wgslLanguageFeatures.values')
return ['packed_4x8_integer_dot_product', 'unrestricted_pointer_parameters', 'subgroup_uniformity', 'subgroup_id', 'pointer_composite_access', 'readonly_and_readwrite_storage_textures', 'uniform_buffer_standard_layout']
},
}, 'gpu.wgslLanguageFeatures'),
requestAdapter: function requestAdapter() {
AsObj.print('gpu.requestAdapter:', arguments)
return {
then: function () {
arguments[0](watch({
features: watch({
size: 19,
values: function () {
return ['depth32float-stencil8', 'rg11b10ufloat-renderable', 'bgra8unorm-storage', 'texture-formats-tier1', 'texture-compression-bc', 'dual-source-blending', 'core-features-and-limits', 'float32-filterable', 'indirect-first-instance', 'float32-blendable', 'depth-clip-control', 'texture-compression-bc-sliced-3d', 'timestamp-query', 'texture-formats-tier2', 'clip-distances', 'shader-f16', 'primitive-index', 'texture-component-swizzle', 'subgroups']
}
}, 'gpu.requestAdapter.features'),
info: watch({ vendor: 'intel', architecture: 'gen-11', device: '', description: '', subgroupMinSize: 16 }, 'gpu.requestAdapter.info'),
limits: watch({
maxBufferSize: 2147483648,
maxStorageBufferBindingSize:2147483644
}, 'gpu.requestAdapter.limits'),
catch:function(){}
}, 'gpu.requestAdapter'));
return {
catch: function () {
return {
then: function () {
arguments[0]()
return {catch:function(){}}
}
}
}
};
},
catch: function () {
}
}
}
},'navigator.gpu')
Navigator.prototype.userAgentData = watch({
brands:[
{
"brand": "Google Chrome",
"version": "143"
},
{
"brand": "Chromium",
"version": "143"
},
{
"brand": "Not A(Brand",
"version": "24"
}
],
mobile: false,
platform: "Windows",
getHighEntropyValues: function getHighEntropyValues() {
if (arguments[0] + '' === 'architecture,bitness,model,platformVersion,uaFullVersion,wow64') {
return new Promise((resolve, reject) => {
const offer = {
"architecture": "x86",
"bitness": "64",
"brands": [
{
"brand": "Not:A-Brand",
"version": "99"
},
{
"brand": "Google Chrome",
"version": "145"
},
{
"brand": "Chromium",
"version": "145"
}
],
"mobile": false,
"model": "",
"platform": "Windows",
"platformVersion": "10.0.0",
"uaFullVersion": "145.0.7632.117",
"wow64": false
};
resolve(offer);
});
}
AsObj.print('getHighEntropyValues:::', arguments)
}
},'userAgentData')
navigator = {};
navigator.__proto__ = Navigator.prototype;
createTagProto('Location');
location = {
"ancestorOrigins": {},
"href": "https://www.neimanmarcus.com/",
"origin": "https://www.neimanmarcus.com",
"protocol": "https:",
"host": "www.neimanmarcus.com",
"hostname": "www.neimanmarcus.com",
"port": "",
"pathname": "/",
"search": "",
"hash": ""
};
location.__proto__ = Location.prototype;
location.toString = function toString() {
return this.href;
}
createTagProto('Screen');
Screen.prototype = Object.assign(Screen.prototype, {
availHeight: 824,
availLeft: 0,
availTop: 0,
availWidth: 1536,
colorDepth: 32,
height: 864,
isExtended: true,
onchange: null,
pixelDepth: 24,
width: 1536,
orientation: {
angle: 0,
type: "landscape-primary",
onchange: null
}
})
screen = {};
screen.__proto__ = Screen.prototype;
createTagProto('History',['replaceState']);
history = {};
history.__proto__ = History.prototype;
chrome = {
loadTimes: function loadTimes() { },
csi: function csi() { },
app: {
InstallState: { DISABLED: 'disabled', INSTALLED: 'installed', NOT_INSTALLED: 'not_installed' },
RunningState: { CANNOT_RUN: 'cannot_run', READY_TO_RUN: 'ready_to_run', RUNNING: 'running' },
getDetails:function getDetails(){},
getIsInstalled:function getIsInstalled(){},
installState:function installState(){},
isInstalled: false,
runningState: function runningState(){}
},
}
createTagProto('Storage');
local = {
};
localStorage = {
getItem: function getItem(key) {
AsObj.print("localStorage.getItem::", arguments);
if (!local[key]) {
return null;
}
return local[key];
},
setItem: function setItem(key, value) {
AsObj.print("localStorage.setItem::", arguments);
local[key] = value;
},
clear: function clear() {
local = {};
},
removeItem: function removeItem(key) {
AsObj.print("localStorage.removeItem::", arguments);
delete local[key];
}
}
localStorage.__proto__ = Storage.prototype;
sessionStorage = {
getItem: function getItem(key) {
AsObj.print("sessionStorage.getItem::", arguments);
if (!local[key]) {
return null;
}
return local[key];
},
setItem: function setItem(key, value) {
AsObj.print("sessionStorage.setItem::", arguments);
local[key] = value;
},
clear: function clear() {
local = {};
},
removeItem: function removeItem(key) {
AsObj.print("sessionStorage.removeItem::", arguments);
delete local[key];
}
}
sessionStorage.__proto__ = Storage.prototype;
// window = watch(window, 'window');
// global = watch(global, 'global');
// globalThis = watch(globalThis, 'globalThis');
// self = watch(self, 'self');
// crypto = watch(crypto, 'crypto');
// performance = watch(performance, 'performance');
// document = watch(document, 'document');
// navigator = watch(navigator, 'navigator');
// location = watch(location, 'location');
// screen = watch(screen, 'screen');
// history = watch(history, 'history');
// localStorage = watch(localStorage, 'localStorage');
// sessionStorage = watch(sessionStorage, 'sessionStorage');
// chrome = watch(chrome, 'chrome');
require_('./sdk_leg.js');
let input = '';
// 收集数据
process.stdin.on('data', chunk => {
input += chunk;
});
process.stdin.on('end', async () => {
var config_data = JSON.parse(input);
var cryptoManager = await CaptchaSDKCorecc();
var encryptData = await buildEncryptedVerifyRequestcc(config_data, cryptoManager);
console.log(JSON.stringify(encryptData));
process.exit(0);
})

View File

@@ -0,0 +1,3 @@
{
"thread_count": "1"
}

View File

@@ -0,0 +1,4 @@
# -*- coding: UTF-8 -*-
'''
UI 模块
'''

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,510 @@
# -*- coding: UTF-8 -*-
'''
@Project :domainScanDemo
@File :domain_import.py
@IDE :PyCharm
@Author :梦伴
@Date :2026/4/8 23:47
@explain : 域名导入界面
'''
from PySide6.QtWidgets import QWidget, QVBoxLayout, QHBoxLayout, QPushButton, QLabel, QTextEdit, QFileDialog, QProgressBar
from PySide6.QtCore import Qt, QThread, Signal
from loguru import logger
from app.core.domain_collector import DomainCollector
class ImportThread(QThread):
"""
导入线程
"""
progress_updated = Signal(int)
finished = Signal(bool, str)
def __init__(self, domain_list, source_type):
"""
初始化导入线程
:param domain_list: 域名列表
:param source_type: 来源类型
"""
super().__init__()
self.domain_list = domain_list
self.source_type = source_type
def run(self):
"""
运行导入线程
"""
try:
collector = DomainCollector()
total = len(self.domain_list)
# 实时更新进度:开始
self.progress_updated.emit(0)
# 标准化域名和检查是否存在占30%进度)
normalized_domains = []
for i, domain in enumerate(self.domain_list):
from app.utils.domain_utils import normalize_domain
import tldextract
normalized = normalize_domain(domain)
if normalized:
# 提取顶级域名
ext = tldextract.extract(normalized)
tld = ext.suffix
normalized_domains.append((normalized, tld))
# 更新进度
progress = int((i + 1) / total * 30)
self.progress_updated.emit(progress)
# 批量检查域名是否存在
batch_data = []
existing_domains = []
if normalized_domains:
all_domains = [domain for domain, tld in normalized_domains]
existing_domains = collector.db.check_domains_exist(all_domains)
existing_set = set(existing_domains)
# 准备批量添加数据
for domain, tld in normalized_domains:
if domain not in existing_set:
batch_data.append((domain, tld, self.source_type))
# 分批次添加域名占70%进度)
batch_size = 1000
total_batches = len(batch_data)
for i in range(0, len(batch_data), batch_size):
batch = batch_data[i:i+batch_size]
collector.db.add_domains_batch(batch)
# 更新进度
processed = min(i + len(batch), total_batches)
progress = 30 + int(processed / total_batches * 70)
self.progress_updated.emit(progress)
# 完成导入
self.progress_updated.emit(100)
# 计算统计信息
stats = {
'total': total,
'valid': len(normalized_domains),
'added': len(batch_data),
'exists': len(existing_domains),
'invalid': total - len(normalized_domains),
'failed': 0
}
# 根据统计信息生成消息
if stats['added'] > 0:
message = f"导入完成: 总域名数 {stats['total']}, 有效域名数 {stats['valid']}, 新增域名数 {stats['added']}, 已存在域名数 {stats['exists']}, 无效域名数 {stats['invalid']}"
else:
message = f"导入完成: 所有域名已存在,未添加新域名"
self.finished.emit(True, message)
except Exception as e:
logger.error(f"导入失败: {e}")
self.finished.emit(False, f"导入失败: {str(e)}")
class ImportFileThread(QThread):
"""
文件导入线程,用于处理大文件
"""
progress_updated = Signal(int)
finished = Signal(bool, str)
def __init__(self, file_path, source_type):
"""
初始化文件导入线程
:param file_path: 文件路径
:param source_type: 来源类型
"""
super().__init__()
self.file_path = file_path
self.source_type = source_type
def run(self):
"""
运行文件导入线程
"""
try:
collector = DomainCollector()
# 首先计算文件中的域名数量
total = 0
encodings = ['utf-8', 'utf-8-sig', 'gbk', 'gb2312', 'cp936', 'latin-1', 'ascii']
encoding = 'utf-8' # 默认编码
# 尝试不同的编码格式计算域名数量
for enc in encodings:
try:
with open(self.file_path, 'r', encoding=enc) as f:
total = sum(1 for line in f if line.strip())
encoding = enc
break
except UnicodeDecodeError:
continue
if total == 0:
# 尝试使用二进制模式读取
try:
import chardet
with open(self.file_path, 'rb') as f:
raw_data = f.read()
result = chardet.detect(raw_data)
encoding = result['encoding']
if encoding:
total = sum(1 for line in raw_data.decode(encoding).split('\n') if line.strip())
else:
# 最后尝试使用 replace 模式读取
with open(self.file_path, 'r', encoding='utf-8', errors='replace') as f:
total = sum(1 for line in f if line.strip())
encoding = 'utf-8'
except Exception:
# 最后尝试使用 replace 模式读取
with open(self.file_path, 'r', encoding='utf-8', errors='replace') as f:
total = sum(1 for line in f if line.strip())
encoding = 'utf-8'
# 实时更新进度:开始
self.progress_updated.emit(0)
# 逐行读取文件并处理域名
normalized_domains = []
processed = 0
with open(self.file_path, 'r', encoding=encoding, errors='replace') as f:
for line in f:
domain = line.strip()
if domain:
from app.utils.domain_utils import normalize_domain
import tldextract
normalized = normalize_domain(domain)
if normalized:
# 提取顶级域名
ext = tldextract.extract(normalized)
tld = ext.suffix
normalized_domains.append((normalized, tld))
processed += 1
# 更新进度占30%
progress = int(processed / total * 30)
self.progress_updated.emit(progress)
# 批量检查域名是否存在
batch_data = []
existing_domains = []
if normalized_domains:
all_domains = [domain for domain, tld in normalized_domains]
existing_domains = collector.db.check_domains_exist(all_domains)
existing_set = set(existing_domains)
# 准备批量添加数据
for domain, tld in normalized_domains:
if domain not in existing_set:
batch_data.append((domain, tld, self.source_type))
# 分批次添加域名占70%进度)
batch_size = 1000
total_batches = len(batch_data)
for i in range(0, len(batch_data), batch_size):
batch = batch_data[i:i+batch_size]
collector.db.add_domains_batch(batch)
# 更新进度
processed_batches = min(i + len(batch), total_batches)
progress = 30 + int(processed_batches / total_batches * 70)
self.progress_updated.emit(progress)
# 完成导入
self.progress_updated.emit(100)
# 计算统计信息
stats = {
'total': total,
'valid': len(normalized_domains),
'added': len(batch_data),
'exists': len(existing_domains),
'invalid': total - len(normalized_domains),
'failed': 0
}
# 根据统计信息生成消息
if stats['added'] > 0:
message = f"导入完成: 总域名数 {stats['total']}, 有效域名数 {stats['valid']}, 新增域名数 {stats['added']}, 已存在域名数 {stats['exists']}, 无效域名数 {stats['invalid']}"
else:
message = f"导入完成: 所有域名已存在,未添加新域名"
self.finished.emit(True, message)
except Exception as e:
logger.error(f"导入失败: {e}")
self.finished.emit(False, f"导入失败: {str(e)}")
class DomainImportWidget(QWidget):
"""
域名导入界面
"""
def __init__(self):
"""
初始化域名导入界面
"""
super().__init__()
# 创建布局
layout = QVBoxLayout(self)
# 创建文本编辑框
self.text_edit = QTextEdit()
self.text_edit.setPlaceholderText("请输入域名,一行一个")
self.text_edit.setStyleSheet("""
QTextEdit {
font-size: 14px;
padding: 10px;
border: 1px solid #ddd;
border-radius: 4px;
background-color: #f9f9f9;
min-height: 300px;
}
""")
layout.addWidget(self.text_edit)
# 创建按钮布局
button_layout = QHBoxLayout()
# 导入文件按钮
self.import_file_btn = QPushButton("导入文件")
self.import_file_btn.clicked.connect(self.import_file)
self.import_file_btn.setStyleSheet("""
QPushButton {
font-size: 14px;
padding: 8px 16px;
background-color: #2196F3;
color: white;
border: none;
border-radius: 4px;
}
QPushButton:hover {
background-color: #0b7dda;
}
QPushButton:disabled {
background-color: #cccccc;
}
""")
button_layout.addWidget(self.import_file_btn)
# 开始导入按钮
self.start_import_btn = QPushButton("开始导入")
self.start_import_btn.clicked.connect(self.start_import)
self.start_import_btn.setStyleSheet("""
QPushButton {
font-size: 14px;
padding: 8px 16px;
background-color: #4CAF50;
color: white;
border: none;
border-radius: 4px;
margin-left: 10px;
}
QPushButton:hover {
background-color: #45a049;
}
QPushButton:disabled {
background-color: #cccccc;
}
""")
button_layout.addWidget(self.start_import_btn)
button_layout.setContentsMargins(0, 15, 0, 15)
layout.addLayout(button_layout)
# 创建进度条
self.progress_bar = QProgressBar()
self.progress_bar.setVisible(False)
self.progress_bar.setStyleSheet("""
QProgressBar {
height: 20px;
border: 1px solid #ddd;
border-radius: 10px;
background-color: #f0f0f0;
margin-bottom: 10px;
}
QProgressBar::chunk {
background-color: #4CAF50;
border-radius: 10px;
}
""")
layout.addWidget(self.progress_bar)
# 创建状态标签
self.status_label = QLabel("")
self.status_label.setAlignment(Qt.AlignCenter)
self.status_label.setStyleSheet("font-size: 14px; color: #333; padding: 10px; background-color: #f0f8ff; border-radius: 4px;")
layout.addWidget(self.status_label)
logger.info("域名导入界面创建完成")
def import_file(self):
"""
导入文件
"""
file_path, _ = QFileDialog.getOpenFileName(self, "选择文件", "", "文本文件 (*.txt)")
if file_path:
try:
# 尝试不同的编码格式
encodings = ['utf-8', 'utf-8-sig', 'gbk', 'gb2312', 'cp936', 'latin-1', 'ascii']
domain_count = 0
# 尝试使用不同编码读取并计数
for encoding in encodings:
try:
with open(file_path, 'r', encoding=encoding) as f:
domain_count = sum(1 for line in f if line.strip())
logger.info(f"使用编码 {encoding} 成功读取文件")
break
except UnicodeDecodeError:
continue
# 如果仍然失败,尝试使用二进制模式读取并猜测编码
if domain_count == 0:
try:
import chardet
with open(file_path, 'rb') as f:
raw_data = f.read()
result = chardet.detect(raw_data)
encoding = result['encoding']
if encoding:
domain_count = sum(1 for line in raw_data.decode(encoding).split('\n') if line.strip())
logger.info(f"使用 chardet 检测到编码 {encoding} 并成功读取文件")
else:
raise Exception("无法识别文件编码")
except Exception as e:
logger.warning(f"chardet 检测失败: {e}")
# 最后尝试使用 replace 模式读取
with open(file_path, 'r', encoding='utf-8', errors='replace') as f:
domain_count = sum(1 for line in f if line.strip())
logger.info("使用 utf-8 replace 模式读取文件")
# 对于大文件,不显示所有域名,只显示文件路径和域名数量
if domain_count > 1000:
self.text_edit.setText(f"文件路径: {file_path}\n域名数量: {domain_count}\n\n提示: 由于文件较大,仅显示文件信息,不显示具体域名。")
# 保存文件路径,用于后续导入
self.imported_file_path = file_path
else:
# 对于小文件,显示所有域名
domains = []
for encoding in encodings:
try:
with open(file_path, 'r', encoding=encoding) as f:
domains = f.readlines()
break
except UnicodeDecodeError:
continue
if not domains:
# 尝试使用二进制模式读取
try:
import chardet
with open(file_path, 'rb') as f:
raw_data = f.read()
result = chardet.detect(raw_data)
encoding = result['encoding']
if encoding:
domains = raw_data.decode(encoding).split('\n')
else:
with open(file_path, 'r', encoding='utf-8', errors='replace') as f:
domains = f.readlines()
except Exception:
with open(file_path, 'r', encoding='utf-8', errors='replace') as f:
domains = f.readlines()
domains = [domain.strip() for domain in domains if domain.strip()]
self.text_edit.setText('\n'.join(domains))
# 清除文件路径,使用文本框中的域名
self.imported_file_path = None
self.status_label.setText(f"成功读取 {domain_count} 个域名")
logger.info(f"成功读取文件: {file_path}, 共 {domain_count} 个域名")
except Exception as e:
self.status_label.setText(f"读取文件失败: {str(e)}")
logger.error(f"读取文件失败: {e}")
def start_import(self):
"""
开始导入
"""
# 检查是否有导入的文件路径
if hasattr(self, 'imported_file_path') and self.imported_file_path:
# 大文件导入,使用文件路径
file_path = self.imported_file_path
# 显示进度条
self.progress_bar.setVisible(True)
self.progress_bar.setValue(0)
self.status_label.setText("正在导入...")
# 禁用按钮
self.import_file_btn.setEnabled(False)
self.start_import_btn.setEnabled(False)
# 创建并启动导入线程
self.import_thread = ImportFileThread(file_path, 7) # 7 表示 TXT 导入
self.import_thread.progress_updated.connect(self.update_progress)
self.import_thread.finished.connect(self.import_finished)
self.import_thread.start()
logger.info(f"开始从文件导入: {file_path}")
else:
# 小文件或手动输入的域名
domains = self.text_edit.toPlainText().split('\n')
domains = [domain.strip() for domain in domains if domain.strip()]
if not domains:
self.status_label.setText("请输入域名")
return
# 显示进度条
self.progress_bar.setVisible(True)
self.progress_bar.setValue(0)
self.status_label.setText("正在导入...")
# 禁用按钮
self.import_file_btn.setEnabled(False)
self.start_import_btn.setEnabled(False)
# 创建并启动导入线程
self.import_thread = ImportThread(domains, 7) # 7 表示 TXT 导入
self.import_thread.progress_updated.connect(self.update_progress)
self.import_thread.finished.connect(self.import_finished)
self.import_thread.start()
logger.info(f"开始导入 {len(domains)} 个域名")
def update_progress(self, progress):
"""
更新进度
:param progress: 进度值
"""
self.progress_bar.setValue(progress)
def import_finished(self, success, message):
"""
导入完成
:param success: 是否成功
:param message: 消息
"""
self.status_label.setText(message)
self.progress_bar.setVisible(False)
# 启用按钮
self.import_file_btn.setEnabled(True)
self.start_import_btn.setEnabled(True)
logger.info(f"导入完成: {message}")

View File

@@ -0,0 +1,577 @@
# -*- coding: UTF-8 -*-
'''
@Project :domainScanDemo
@File :juming_crawler.py
@IDE :PyCharm
@Author :梦伴
@Date :2026/4/9 15:00
@explain : 聚名网爬取页面
'''
from PySide6.QtWidgets import QWidget, QVBoxLayout, QHBoxLayout, QPushButton, QLabel, QTextEdit, QProgressBar, QLineEdit, QComboBox, QDateEdit, QCheckBox
from PySide6.QtCore import Qt, QThread, Signal, QDate
from PySide6.QtGui import QIntValidator
from loguru import logger
import re
import datetime
import time
from app.core.domain_collector import DomainCollector
from detect.juming import JM
class JumingCrawlThread(QThread):
"""
聚名爬取线程
"""
progress_updated = Signal(int)
status_updated = Signal(str)
finished = Signal(bool, str)
def __init__(self, crawl_type, page_start=1, page_size=50, crawl_date=None, auto_date=True):
"""
初始化聚名爬取线程
:param crawl_type: 爬取类型 (1: 一口价, 2: 删除列表)
:param page_start: 起始页码
:param page_size: 每页数量
:param crawl_date: 爬取日期(删除列表用)
:param auto_date: 是否自动新增日期
"""
super().__init__()
self.crawl_type = crawl_type
self.page_start = page_start
self.page_size = page_size
self.crawl_date = crawl_date
self.auto_date = auto_date
self.is_paused = False
self.is_stopped = False
self.current_page = 0
self.total_count = 0
def run(self):
"""
运行聚名爬取线程
"""
try:
# 初始化聚名客户端
jm = JM()
# 加载 Cookie
jm.load_cookies()
logger.info("已加载 Cookie")
self.status_updated.emit("已加载 Cookie")
# 直接开始爬取,不需要登录,因为 Cookie 已经在系统设置页面加载了
self.progress_updated.emit(10)
if self.crawl_type == 1: # 一口价
self.progress_updated.emit(30)
logger.info("开始获取一口价域名")
self.status_updated.emit("开始获取一口价域名")
# 自动爬取多页
page = self.page_start
while not self.is_stopped:
if self.is_paused:
time.sleep(0.1)
continue
self.current_page = page
self.status_updated.emit(f"正在爬取第 {page}")
logger.info(f"正在爬取第 {page}")
# 获取当前页
success, html = jm.ykj_get_list(page=page, page_size=self.page_size)
if success:
pattern_ym = r"<a class='yda1 ydz' ym='([^']*)'"
results = re.findall(pattern_ym, html)
domains = [domain.strip() for domain in results if domain.strip()]
domain_count = len(domains)
self.total_count += domain_count
self.status_updated.emit(f"{page} 页找到 {domain_count} 个域名,累计 {self.total_count}")
logger.info(f"{page} 页找到 {domain_count} 个域名,累计 {self.total_count}")
# 自动入库
if domains:
collector = DomainCollector()
stats = collector.add_domains_batch(domains, 1) # 1 表示一口价
logger.info(f"自动入库完成: {stats}")
self.status_updated.emit(f"自动入库完成: 成功添加 {stats['added']} 个域名")
# 如果返回的数量小于指定的数量,停止爬取
if domain_count < self.page_size:
logger.info(f"返回数量小于指定数量,停止爬取")
self.status_updated.emit("返回数量小于指定数量,停止爬取")
break
# 增加页码
page += 1
# 模拟网络延迟
time.sleep(1)
else:
logger.error(f"获取一口价域名失败: {html}")
self.status_updated.emit(f"获取一口价域名失败: {html}")
break
elif self.crawl_type == 2: # 删除列表
self.progress_updated.emit(30)
logger.info("开始获取删除域名列表")
self.status_updated.emit("开始获取删除域名列表")
# 使用传入的日期或默认今天
start_date_str = self.crawl_date if self.crawl_date else datetime.date.today().strftime("%Y-%m-%d")
start_date = datetime.datetime.strptime(start_date_str, "%Y-%m-%d").date()
# 计算结束日期:今天 + 4天
end_date = datetime.date.today() + datetime.timedelta(days=4)
if self.auto_date:
# 自动新增日期,从起始日期到今天+4天
current_date = start_date
while current_date <= end_date and not self.is_stopped:
if self.is_paused:
time.sleep(0.1)
continue
crawl_date = current_date.strftime("%Y-%m-%d")
self.status_updated.emit(f"正在爬取 {crawl_date} 的删除域名")
logger.info(f"正在爬取 {crawl_date} 的删除域名")
deleted_domains = jm.new_cha_del(crawl_date)
domains = [domain.strip() for domain in deleted_domains if domain.strip()]
domain_count = len(domains)
self.total_count += domain_count
self.status_updated.emit(f"{crawl_date} 找到 {domain_count} 个删除域名,累计 {self.total_count}")
logger.info(f"{crawl_date} 找到 {domain_count} 个删除域名,累计 {self.total_count}")
# 自动入库
if domains:
collector = DomainCollector()
stats = collector.add_domains_batch(domains, 2) # 2 表示删除列表
logger.info(f"{crawl_date} 自动入库完成: {stats}")
self.status_updated.emit(f"{crawl_date} 自动入库完成: 成功添加 {stats['added']} 个域名")
# 增加日期
current_date = current_date + datetime.timedelta(days=1)
# 模拟网络延迟
time.sleep(1)
else:
# 只爬取指定日期
crawl_date = start_date_str
self.status_updated.emit(f"正在爬取 {crawl_date} 的删除域名")
logger.info(f"正在爬取 {crawl_date} 的删除域名")
deleted_domains = jm.new_cha_del(crawl_date)
domains = [domain.strip() for domain in deleted_domains if domain.strip()]
domain_count = len(domains)
self.total_count = domain_count
self.status_updated.emit(f"找到 {domain_count} 个删除域名")
logger.info(f"找到 {domain_count} 个删除域名")
# 自动入库
if domains:
collector = DomainCollector()
stats = collector.add_domains_batch(domains, 2) # 2 表示删除列表
logger.info(f"自动入库完成: {stats}")
self.status_updated.emit(f"自动入库完成: 成功添加 {stats['added']} 个域名")
self.progress_updated.emit(90)
self.progress_updated.emit(100)
self.finished.emit(True, f"成功获取 {self.total_count} 个域名")
except Exception as e:
logger.error(f"从聚名网爬取失败: {e}")
self.status_updated.emit(f"爬取失败: {str(e)}")
self.finished.emit(False, f"爬取失败: {str(e)}")
def pause(self):
"""
暂停爬取
"""
self.is_paused = True
logger.info("爬取已暂停")
self.status_updated.emit("爬取已暂停")
def resume(self):
"""
恢复爬取
"""
self.is_paused = False
logger.info("爬取已恢复")
self.status_updated.emit("爬取已恢复")
def stop(self):
"""
停止爬取
"""
self.is_stopped = True
logger.info("爬取已停止")
self.status_updated.emit("爬取已停止")
class JumingCrawlerWidget(QWidget):
"""
聚名网爬取页面
"""
def __init__(self):
"""
初始化聚名网爬取页面
"""
super().__init__()
# 创建布局
layout = QVBoxLayout(self)
# 爬取类型选择
type_layout = QHBoxLayout()
type_label = QLabel("爬取类型:")
type_label.setStyleSheet("font-size: 14px; color: #666; min-width: 80px;")
self.type_combo = QComboBox()
self.type_combo.addItem("一口价域名", 1)
self.type_combo.addItem("删除列表域名", 2)
self.type_combo.setStyleSheet("font-size: 14px; padding: 5px; border: 1px solid #ddd; border-radius: 4px;")
# 监听类型变化
self.type_combo.currentIndexChanged.connect(self.on_type_changed)
type_layout.addWidget(type_label)
type_layout.addWidget(self.type_combo)
type_layout.setContentsMargins(0, 0, 0, 15)
layout.addLayout(type_layout)
# 页码和每页数量设置
page_layout = QHBoxLayout()
# 起始页码
page_start_label = QLabel("起始页码:")
page_start_label.setStyleSheet("font-size: 14px; color: #666; min-width: 80px;")
self.page_start_edit = QLineEdit("1")
# 移除所有限制,允许输入任意正整数
self.page_start_edit.setStyleSheet("font-size: 14px; padding: 5px; border: 1px solid #ddd; border-radius: 4px; width: 120px;")
page_layout.addWidget(page_start_label)
page_layout.addWidget(self.page_start_edit)
# 每页数量
page_size_label = QLabel("每页数量:")
page_size_label.setStyleSheet("font-size: 14px; color: #666; min-width: 80px; margin-left: 20px;")
self.page_size_edit = QLineEdit("500")
self.page_size_edit.setValidator(QIntValidator(1, 1000))
self.page_size_edit.setStyleSheet("font-size: 14px; padding: 5px; border: 1px solid #ddd; border-radius: 4px; width: 80px;")
page_layout.addWidget(page_size_label)
page_layout.addWidget(self.page_size_edit)
page_layout.setContentsMargins(0, 0, 0, 20)
layout.addLayout(page_layout)
# 日期设置(删除列表用)
date_container = QWidget()
date_layout = QHBoxLayout(date_container)
# 起始日期
date_label = QLabel("起始日期:")
date_label.setStyleSheet("font-size: 14px; color: #666; min-width: 80px;")
self.date_edit = QDateEdit()
self.date_edit.setDate(QDate.currentDate())
self.date_edit.setCalendarPopup(True)
self.date_edit.setStyleSheet("font-size: 14px; padding: 5px; border: 1px solid #ddd; border-radius: 4px; width: 150px;")
# 设置最大日期为今天+4天最小日期为今天的前4天
max_date = QDate.currentDate().addDays(4)
min_date = QDate.currentDate().addDays(-4)
self.date_edit.setMinimumDate(min_date)
self.date_edit.setMaximumDate(max_date)
date_layout.addWidget(date_label)
date_layout.addWidget(self.date_edit)
# 自动新增日期选项
auto_date_checkbox = QCheckBox("自动新增日期")
auto_date_checkbox.setChecked(True)
auto_date_checkbox.setStyleSheet("font-size: 14px; margin-left: 20px;")
self.auto_date_checkbox = auto_date_checkbox
date_layout.addWidget(auto_date_checkbox)
date_layout.setContentsMargins(0, 0, 0, 20)
self.date_container = date_container
layout.addWidget(date_container)
# 默认隐藏日期输入框
self.date_container.setVisible(False)
# 按钮布局
button_layout = QHBoxLayout()
# 创建开始按钮
self.start_btn = QPushButton("开始爬取")
self.start_btn.clicked.connect(self.start_crawl)
self.start_btn.setStyleSheet("""
QPushButton {
font-size: 14px;
padding: 8px 16px;
background-color: #4CAF50;
color: white;
border: none;
border-radius: 4px;
}
QPushButton:hover {
background-color: #45a049;
}
QPushButton:disabled {
background-color: #cccccc;
}
""")
button_layout.addWidget(self.start_btn)
# 创建暂停按钮
self.pause_btn = QPushButton("暂停爬取")
self.pause_btn.clicked.connect(self.pause_crawl)
self.pause_btn.setEnabled(False)
self.pause_btn.setStyleSheet("""
QPushButton {
font-size: 14px;
padding: 8px 16px;
background-color: #ff9800;
color: white;
border: none;
border-radius: 4px;
margin-left: 10px;
}
QPushButton:hover {
background-color: #f57c00;
}
QPushButton:disabled {
background-color: #cccccc;
}
""")
button_layout.addWidget(self.pause_btn)
# 创建停止按钮
self.stop_btn = QPushButton("停止爬取")
self.stop_btn.clicked.connect(self.stop_crawl)
self.stop_btn.setEnabled(False)
self.stop_btn.setStyleSheet("""
QPushButton {
font-size: 14px;
padding: 8px 16px;
background-color: #f44336;
color: white;
border: none;
border-radius: 4px;
margin-left: 10px;
}
QPushButton:hover {
background-color: #d32f2f;
}
QPushButton:disabled {
background-color: #cccccc;
}
""")
button_layout.addWidget(self.stop_btn)
button_layout.setContentsMargins(0, 0, 0, 20)
layout.addLayout(button_layout)
# 创建日志显示区域
self.log_edit = QTextEdit()
self.log_edit.setPlaceholderText("爬取日志将显示在这里")
self.log_edit.setReadOnly(True)
self.log_edit.setStyleSheet("""
QTextEdit {
font-size: 13px;
font-family: Consolas, Monaco, monospace;
padding: 10px;
border: 1px solid #ddd;
border-radius: 4px;
background-color: #f9f9f9;
}
""")
layout.addWidget(self.log_edit)
# 创建进度条
self.progress_bar = QProgressBar()
self.progress_bar.setVisible(False)
self.progress_bar.setStyleSheet("""
QProgressBar {
height: 20px;
border: 1px solid #ddd;
border-radius: 10px;
background-color: #f0f0f0;
margin-top: 10px;
margin-bottom: 10px;
}
QProgressBar::chunk {
background-color: #4CAF50;
border-radius: 10px;
}
""")
layout.addWidget(self.progress_bar)
# 创建状态标签
self.status_label = QLabel("")
self.status_label.setAlignment(Qt.AlignCenter)
self.status_label.setStyleSheet("font-size: 14px; color: #333; margin-top: 10px; padding: 8px; background-color: #f0f8ff; border-radius: 4px;")
layout.addWidget(self.status_label)
# 存储爬取线程
self.crawl_thread = None
logger.info("聚名网爬取页面创建完成")
def start_crawl(self):
"""
开始爬取
"""
crawl_type = self.type_combo.currentData()
# 获取起始页码和每页数量
try:
page_start = int(self.page_start_edit.text())
page_size = int(self.page_size_edit.text())
if page_start < 1:
self.status_label.setText("起始页码必须大于0")
return
if page_size < 1:
self.status_label.setText("每页数量必须大于0")
return
except ValueError:
self.status_label.setText("请输入有效的页码和每页数量")
return
# 显示进度条
self.progress_bar.setVisible(True)
self.progress_bar.setValue(0)
self.status_label.setText("正在爬取域名...")
# 清空日志
self.log_edit.clear()
# 启用/禁用按钮
self.start_btn.setEnabled(False)
self.pause_btn.setEnabled(True)
self.stop_btn.setEnabled(True)
# 获取爬取日期和自动新增日期选项(删除列表用)
crawl_date = None
auto_date = False
if crawl_type == 2: # 删除列表
crawl_date = self.date_edit.date().toString("yyyy-MM-dd")
auto_date = self.auto_date_checkbox.isChecked()
# 创建并启动爬取线程
self.crawl_thread = JumingCrawlThread(crawl_type, page_start, page_size, crawl_date, auto_date)
self.crawl_thread.progress_updated.connect(self.update_progress)
self.crawl_thread.status_updated.connect(self.update_status)
self.crawl_thread.finished.connect(self.crawl_finished)
self.crawl_thread.start()
if crawl_type == 2: # 删除列表
auto_date_str = "" if auto_date else ""
logger.info(f"开始爬取聚名网域名, 类型: {crawl_type}, 爬取日期: {crawl_date}, 自动新增日期: {auto_date_str}")
self.log_edit.append(f"开始爬取聚名网域名, 类型: {crawl_type}, 爬取日期: {crawl_date}, 自动新增日期: {auto_date_str}")
else: # 一口价
logger.info(f"开始爬取聚名网域名, 类型: {crawl_type}, 起始页码: {page_start}, 每页数量: {page_size}")
self.log_edit.append(f"开始爬取聚名网域名, 类型: {crawl_type}, 起始页码: {page_start}, 每页数量: {page_size}")
def pause_crawl(self):
"""
暂停爬取
"""
if self.crawl_thread:
if self.crawl_thread.is_paused:
self.crawl_thread.resume()
self.pause_btn.setText("暂停爬取")
else:
self.crawl_thread.pause()
self.pause_btn.setText("恢复爬取")
def stop_crawl(self):
"""
停止爬取
"""
if self.crawl_thread:
self.crawl_thread.stop()
def update_progress(self, progress):
"""
更新进度
:param progress: 进度值
"""
self.progress_bar.setValue(progress)
def update_status(self, status):
"""
更新状态
:param status: 状态消息
"""
self.status_label.setText(status)
self.log_edit.append(status)
def crawl_finished(self, success, message):
"""
爬取完成
:param success: 是否成功
:param message: 消息
"""
self.status_label.setText(message)
self.log_edit.append(message)
# 启用/禁用按钮
self.start_btn.setEnabled(True)
self.pause_btn.setEnabled(False)
self.pause_btn.setText("暂停爬取")
self.stop_btn.setEnabled(False)
self.progress_bar.setVisible(False)
logger.info(f"聚名网爬取完成: {message}")
def on_type_changed(self, index):
"""
爬取类型变化时的处理
:param index: 选择的索引
"""
crawl_type = self.type_combo.currentData()
if crawl_type == 2: # 删除列表
self.date_container.setVisible(True)
else: # 一口价
self.date_container.setVisible(False)
class ImportThread(QThread):
"""
导入线程
"""
progress_updated = Signal(int)
finished = Signal(bool, str)
def __init__(self, domain_list, source_type):
"""
初始化导入线程
:param domain_list: 域名列表
:param source_type: 来源类型
"""
super().__init__()
self.domain_list = domain_list
self.source_type = source_type
def run(self):
"""
运行导入线程
"""
try:
collector = DomainCollector()
total = len(self.domain_list)
for i, domain in enumerate(self.domain_list):
collector.add_domain(domain, self.source_type)
progress = int((i + 1) / total * 100)
self.progress_updated.emit(progress)
self.finished.emit(True, f"成功导入 {total} 个域名")
except Exception as e:
logger.error(f"导入失败: {e}")
self.finished.emit(False, f"导入失败: {str(e)}")

View File

@@ -0,0 +1,137 @@
# -*- coding: UTF-8 -*-
'''
@Project :domainScanDemo
@File :main_window.py
@IDE :PyCharm
@Author :梦伴
@Date :2026/4/8 23:46
@explain : 主窗口
'''
from PySide6.QtWidgets import QMainWindow, QWidget, QVBoxLayout, QHBoxLayout, QPushButton, QTabWidget, QLabel, QScrollArea, QFrame
from PySide6.QtCore import Qt
from PySide6.QtGui import QIcon
import os
from loguru import logger
from app.ui.domain_import import DomainImportWidget
from app.ui.domain_filter import DomainFilterWidget
from app.ui.sensitive_words import SensitiveWordsWidget
from app.ui.juming_crawler import JumingCrawlerWidget
from app.ui.system_settings import SystemSettingsWidget
class MainWindow(QMainWindow):
"""
主窗口
"""
def __init__(self):
"""
初始化主窗口
"""
super().__init__()
# 设置窗口标题和大小
self.setWindowTitle("域名工具")
self.setGeometry(100, 100, 2000, 800)
self.setMinimumSize(1200, 720)
# 设置窗口图标
icon_path = os.path.join(os.path.dirname(os.path.dirname(os.path.dirname(__file__))), "favicon.ico")
if os.path.exists(icon_path):
self.setWindowIcon(QIcon(icon_path))
logger.info(f"设置窗口图标成功: {icon_path}")
else:
logger.warning(f"窗口图标文件不存在: {icon_path}")
# 创建中央部件
central_widget = QWidget()
self.setCentralWidget(central_widget)
# 创建主布局
main_layout = QVBoxLayout(central_widget)
# 创建标签页
self.tab_widget = QTabWidget()
self.tab_widget.setStyleSheet("""
QTabWidget {
font-size: 14px;
}
QTabBar::tab {
padding: 10px 20px;
background-color: #f0f0f0;
border: 1px solid #ddd;
border-bottom: none;
border-top-left-radius: 4px;
border-top-right-radius: 4px;
margin-right: 2px;
}
QTabBar::tab:hover {
background-color: #e0e0e0;
}
QTabBar::tab:selected {
background-color: white;
color: #4CAF50;
font-weight: bold;
border-color: #4CAF50;
}
QTabWidget::pane {
border: 1px solid #ddd;
border-top: none;
border-radius: 0 0 4px 4px;
padding: 10px;
}
""")
main_layout.addWidget(self.tab_widget)
# 创建标签页内容
self.create_tabs()
# 记录日志
logger.info("主窗口创建完成")
def create_tabs(self):
"""
创建标签页
"""
# 聚名爬取标签页
juming_widget = JumingCrawlerWidget()
self.tab_widget.addTab(self.wrap_scrollable_tab(juming_widget), "聚名爬取")
# 域名筛选标签页
filter_widget = DomainFilterWidget()
self.tab_widget.addTab(self.wrap_scrollable_tab(filter_widget), "域名筛选")
# 域名导入标签页
import_widget = DomainImportWidget()
self.tab_widget.addTab(self.wrap_scrollable_tab(import_widget), "域名导入")
# 敏感词配置标签页
sensitive_widget = SensitiveWordsWidget()
self.tab_widget.addTab(self.wrap_scrollable_tab(sensitive_widget), "敏感词配置")
# 系统设置标签页
settings_widget = SystemSettingsWidget()
self.tab_widget.addTab(self.wrap_scrollable_tab(settings_widget), "系统设置")
logger.info("标签页创建完成")
def wrap_scrollable_tab(self, widget):
"""
给标签页统一包一层滚动区域,保证右侧滚动条始终可用
"""
container = QWidget()
container_layout = QVBoxLayout(container)
container_layout.setContentsMargins(0, 0, 0, 0)
container_layout.setSpacing(0)
scroll_area = QScrollArea()
scroll_area.setWidgetResizable(True)
scroll_area.setFrameShape(QFrame.NoFrame)
scroll_area.setVerticalScrollBarPolicy(Qt.ScrollBarAsNeeded)
scroll_area.setHorizontalScrollBarPolicy(Qt.ScrollBarAsNeeded)
scroll_area.setWidget(widget)
container_layout.addWidget(scroll_area)
return container

View File

@@ -0,0 +1,284 @@
# -*- coding: UTF-8 -*-
'''
@Project :domainScanDemo
@File :sensitive_words.py
@IDE :PyCharm
@Author :梦伴
@Date :2026/4/8 23:49
@explain : 敏感词配置界面
'''
from PySide6.QtWidgets import QWidget, QVBoxLayout, QHBoxLayout, QPushButton, QLabel, QTextEdit, QFileDialog
from PySide6.QtCore import Qt, QThread, Signal
from loguru import logger
from app.utils.database import Database
class SaveWordsThread(QThread):
"""
保存敏感词线程
"""
finished = Signal(bool, str, int)
def __init__(self, words):
super().__init__()
self.words = words
def run(self):
try:
db = Database()
# 先清空现有敏感词
db.execute("DELETE FROM sensitive_words")
# 批量添加敏感词
word_tuples = [(word, 'default', 1) for word in self.words]
if word_tuples:
db.batch_add_sensitive_words(word_tuples)
db.close()
self.finished.emit(True, "成功保存敏感词", len(self.words))
except Exception as e:
self.finished.emit(False, str(e), 0)
class LoadWordsThread(QThread):
"""
加载敏感词线程
"""
finished = Signal(bool, list, str)
def run(self):
try:
db = Database()
sensitive_words = db.get_sensitive_words()
words = [word['word'] for word in sensitive_words]
db.close()
self.finished.emit(True, words, f"成功加载 {len(words)} 个敏感词")
except Exception as e:
self.finished.emit(False, [], str(e))
class SensitiveWordsWidget(QWidget):
"""
敏感词配置界面
"""
def __init__(self):
"""
初始化敏感词配置界面
"""
super().__init__()
# 创建布局
layout = QVBoxLayout(self)
# 创建文本编辑框
self.text_edit = QTextEdit()
self.text_edit.setPlaceholderText("请输入敏感词,一行一个")
self.text_edit.setStyleSheet("""
QTextEdit {
font-size: 14px;
padding: 10px;
border: 1px solid #ddd;
border-radius: 4px;
background-color: #f9f9f9;
min-height: 300px;
}
""")
layout.addWidget(self.text_edit)
# 创建按钮布局
button_layout = QHBoxLayout()
# 导入按钮
self.import_btn = QPushButton("导入")
self.import_btn.clicked.connect(self.import_words)
self.import_btn.setStyleSheet("""
QPushButton {
font-size: 14px;
padding: 8px 16px;
background-color: #2196F3;
color: white;
border: none;
border-radius: 4px;
}
QPushButton:hover {
background-color: #0b7dda;
}
""")
button_layout.addWidget(self.import_btn)
# 导出按钮
self.export_btn = QPushButton("导出")
self.export_btn.clicked.connect(self.export_words)
self.export_btn.setStyleSheet("""
QPushButton {
font-size: 14px;
padding: 8px 16px;
background-color: #ff9800;
color: white;
border: none;
border-radius: 4px;
margin-left: 10px;
}
QPushButton:hover {
background-color: #f57c00;
}
""")
button_layout.addWidget(self.export_btn)
# 保存按钮
self.save_btn = QPushButton("保存")
self.save_btn.clicked.connect(self.save_words)
self.save_btn.setStyleSheet("""
QPushButton {
font-size: 14px;
padding: 8px 16px;
background-color: #4CAF50;
color: white;
border: none;
border-radius: 4px;
margin-left: 10px;
}
QPushButton:hover {
background-color: #45a049;
}
""")
button_layout.addWidget(self.save_btn)
# 加载按钮
self.load_btn = QPushButton("加载")
self.load_btn.clicked.connect(self.load_words)
self.load_btn.setStyleSheet("""
QPushButton {
font-size: 14px;
padding: 8px 16px;
background-color: #9c27b0;
color: white;
border: none;
border-radius: 4px;
margin-left: 10px;
}
QPushButton:hover {
background-color: #7b1fa2;
}
""")
button_layout.addWidget(self.load_btn)
button_layout.setContentsMargins(0, 15, 0, 15)
layout.addLayout(button_layout)
# 创建状态标签
self.status_label = QLabel("")
self.status_label.setAlignment(Qt.AlignCenter)
self.status_label.setStyleSheet("font-size: 14px; color: #333; padding: 10px; background-color: #f0f8ff; border-radius: 4px;")
layout.addWidget(self.status_label)
# 初始化线程
self.save_thread = None
self.load_thread = None
# 加载敏感词
self.load_words()
logger.info("敏感词配置界面创建完成")
def import_words(self):
"""
导入敏感词
"""
file_path, _ = QFileDialog.getOpenFileName(self, "选择文件", "", "文本文件 (*.txt)")
if file_path:
try:
with open(file_path, 'r', encoding='utf-8') as f:
words = f.readlines()
words = [word.strip() for word in words if word.strip()]
self.text_edit.setText('\n'.join(words))
self.status_label.setText(f"成功导入 {len(words)} 个敏感词")
logger.info(f"成功导入敏感词文件: {file_path}, 共 {len(words)} 个敏感词")
except Exception as e:
self.status_label.setText(f"导入失败: {str(e)}")
logger.error(f"导入敏感词失败: {e}")
def export_words(self):
"""
导出敏感词
"""
words = self.text_edit.toPlainText().split('\n')
words = [word.strip() for word in words if word.strip()]
if not words:
self.status_label.setText("没有敏感词可导出")
return
file_path, _ = QFileDialog.getSaveFileName(self, "保存文件", "", "文本文件 (*.txt)")
if file_path:
try:
with open(file_path, 'w', encoding='utf-8') as f:
for word in words:
f.write(word + '\n')
self.status_label.setText(f"成功导出 {len(words)} 个敏感词")
logger.info(f"成功导出 {len(words)} 个敏感词到 {file_path}")
except Exception as e:
self.status_label.setText(f"导出失败: {str(e)}")
logger.error(f"导出敏感词失败: {e}")
def save_words(self):
"""
保存敏感词
"""
words = self.text_edit.toPlainText().split('\n')
words = [word.strip() for word in words if word.strip()]
# 禁用按钮,防止重复点击
self.save_btn.setEnabled(False)
self.status_label.setText("正在保存敏感词...")
# 创建并启动保存线程
self.save_thread = SaveWordsThread(words)
self.save_thread.finished.connect(self.on_save_finished)
self.save_thread.start()
def on_save_finished(self, success, message, count):
"""
保存完成的回调函数
"""
if success:
self.status_label.setText(f"成功保存 {count} 个敏感词")
logger.info(f"成功保存 {count} 个敏感词到数据库")
else:
self.status_label.setText(f"保存失败: {message}")
logger.error(f"保存敏感词失败: {message}")
# 重新启用按钮
self.save_btn.setEnabled(True)
def load_words(self):
"""
加载敏感词
"""
# 禁用按钮,防止重复点击
self.load_btn.setEnabled(False)
self.status_label.setText("正在加载敏感词...")
# 创建并启动加载线程
self.load_thread = LoadWordsThread()
self.load_thread.finished.connect(self.on_load_finished)
self.load_thread.start()
def on_load_finished(self, success, words, message):
"""
加载完成的回调函数
"""
if success:
self.text_edit.setText('\n'.join(words))
self.status_label.setText(message)
logger.info(message)
else:
self.status_label.setText(f"加载失败: {message}")
logger.error(f"加载敏感词失败: {message}")
# 重新启用按钮
self.load_btn.setEnabled(True)

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,4 @@
# -*- coding: UTF-8 -*-
'''
工具类模块
'''

File diff suppressed because it is too large Load Diff

View File

@@ -0,0 +1,195 @@
# -*- coding: UTF-8 -*-
'''
@Project :domainScanDemo
@File :domain_utils.py
@IDE :PyCharm
@Author :梦伴
@Date :2026/4/9 0:05
@explain : 域名工具类
'''
import re
import tldextract
from loguru import logger
from app.utils.status_codes import DETECT_STATUS_LABELS, REGISTER_STATUS_LABELS, USE_STATUS_LABELS
def normalize_domain(domain):
"""
标准化域名
:param domain: 域名
:return: str - 标准化后的域名
"""
try:
# 转换为小写
domain = domain.lower()
# 去除空格
domain = domain.strip()
# 去除协议
domain = re.sub(r'^https?://', '', domain)
# 去除路径和查询参数
domain = domain.split('/')[0]
domain = domain.split('?')[0]
# 去除端口
domain = domain.split(':')[0]
# 只保留主域
ext = tldextract.extract(domain)
if ext.domain and ext.suffix:
domain = f"{ext.domain}.{ext.suffix}"
# 验证域名格式
if not is_valid_domain(domain):
return None
return domain
except Exception as e:
logger.error(f"标准化域名出错: {e}")
return None
def is_valid_domain(domain):
"""
验证域名格式
:param domain: 域名
:return: bool - 是否有效
"""
try:
# 域名格式正则
pattern = r'^[a-zA-Z0-9][a-zA-Z0-9-]{0,61}[a-zA-Z0-9]\.[a-zA-Z]{2,}$'
return bool(re.match(pattern, domain))
except Exception as e:
logger.error(f"验证域名格式出错: {e}")
return False
def extract_tld(domain):
"""
提取顶级域名
:param domain: 域名
:return: str - 顶级域名
"""
try:
ext = tldextract.extract(domain)
return ext.suffix
except Exception as e:
logger.error(f"提取顶级域名出错: {e}")
return ''
def extract_domain(domain):
"""
提取主域名
:param domain: 域名
:return: str - 主域名
"""
try:
ext = tldextract.extract(domain)
if ext.domain and ext.suffix:
return f"{ext.domain}.{ext.suffix}"
return domain
except Exception as e:
logger.error(f"提取主域名出错: {e}")
return domain
def is_com_or_net(domain):
"""
检查是否为 .com 或 .net 域名
:param domain: 域名
:return: bool - 是否为 .com 或 .net 域名
"""
try:
tld = extract_tld(domain)
return tld in ['com', 'net']
except Exception as e:
logger.error(f"检查域名后缀出错: {e}")
return False
def generate_domain_variants(domain):
"""
生成域名变体
:param domain: 域名
:return: list - 域名变体列表
"""
try:
variants = []
# 原始域名
variants.append(domain)
# 添加 www
if not domain.startswith('www.'):
variants.append(f"www.{domain}")
# 移除 www
if domain.startswith('www.'):
variants.append(domain[4:])
return variants
except Exception as e:
logger.error(f"生成域名变体出错: {e}")
return [domain]
def parse_domain_status(status_code):
"""
解析域名状态码
:param status_code: 状态码
:return: str - 状态描述
"""
return REGISTER_STATUS_LABELS.get(status_code, '未知')
def parse_use_status(status_code):
"""
解析使用状态码
:param status_code: 状态码
:return: str - 状态描述
"""
return USE_STATUS_LABELS.get(status_code, '未知')
def parse_detect_status(status_code):
"""
解析检测状态码
:param status_code: 状态码
:return: str - 状态描述
"""
return DETECT_STATUS_LABELS.get(status_code, '未知')
def parse_source_type(source_type):
"""
解析来源类型
:param source_type: 来源类型
:return: str - 来源描述
"""
source_map = {
1: '聚名一口价',
2: '聚名过期删除',
3: 'zone file',
4: '搜索引擎采集',
5: '企业目录采集',
6: '手工录入',
7: 'TXT 导入',
8: '第三方接口',
9: '其它'
}
return source_map.get(source_type, '未知')

View File

@@ -0,0 +1,157 @@
# -*- coding: UTF-8 -*-
'''
@Project :domainScanDemo
@File :http_utils.py
@IDE :PyCharm
@Author :梦伴
@Date :2026/4/9 0:06
@explain : HTTP工具类
'''
import requests
from curl_cffi import requests as curl_requests
from loguru import logger
class HTTPUtils:
"""
HTTP工具类
"""
@staticmethod
def get(url, headers=None, params=None, timeout=10, proxies=None, use_curl=False):
"""
发送GET请求
:param url: 请求URL
:param headers: 请求头
:param params: 查询参数
:param timeout: 超时时间
:param proxies: 代理
:param use_curl: 是否使用curl_cffi
:return: requests.Response - 响应对象
"""
try:
if use_curl:
# 使用curl_cffi模拟浏览器
response = curl_requests.get(url, headers=headers, params=params, timeout=timeout, proxies=proxies, impersonate='chrome')
else:
# 使用requests
response = requests.get(url, headers=headers, params=params, timeout=timeout, proxies=proxies)
response.raise_for_status() # 检查状态码
return response
except Exception as e:
logger.error(f"GET请求失败: {url}, 错误: {e}")
return None
@staticmethod
def post(url, headers=None, data=None, json=None, timeout=10, proxies=None, use_curl=False):
"""
发送POST请求
:param url: 请求URL
:param headers: 请求头
:param data: 表单数据
:param json: JSON数据
:param timeout: 超时时间
:param proxies: 代理
:param use_curl: 是否使用curl_cffi
:return: requests.Response - 响应对象
"""
try:
if use_curl:
# 使用curl_cffi模拟浏览器
response = curl_requests.post(url, headers=headers, data=data, json=json, timeout=timeout, proxies=proxies, impersonate='chrome')
else:
# 使用requests
response = requests.post(url, headers=headers, data=data, json=json, timeout=timeout, proxies=proxies)
response.raise_for_status() # 检查状态码
return response
except Exception as e:
logger.error(f"POST请求失败: {url}, 错误: {e}")
return None
@staticmethod
def get_random_user_agent():
"""
获取随机用户代理
:return: str - 用户代理
"""
user_agents = [
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/146.0.0.0 Safari/537.36',
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/145.0.0.0 Safari/537.36',
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Firefox/138.0',
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Edge/146.0.0.0',
'Mozilla/5.0 (Macintosh; Intel Mac OS X 14_7) AppleWebKit/605.1.15 (KHTML, like Gecko) Version/17.6 Safari/605.1.15'
]
import random
return random.choice(user_agents)
@staticmethod
def get_default_headers():
"""
获取默认请求头
:return: dict - 请求头
"""
return {
'User-Agent': HTTPUtils.get_random_user_agent(),
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,*/*;q=0.8',
'Accept-Language': 'zh-CN,zh;q=0.9',
'Accept-Encoding': 'gzip, deflate, br',
'Connection': 'keep-alive',
'Upgrade-Insecure-Requests': '1',
'Sec-Fetch-Dest': 'document',
'Sec-Fetch-Mode': 'navigate',
'Sec-Fetch-Site': 'none',
'Sec-Fetch-User': '?1'
}
@staticmethod
def retry_request(func, max_retries=3, delay=1):
"""
重试请求
:param func: 请求函数
:param max_retries: 最大重试次数
:param delay: 重试延迟
:return: 函数返回值
"""
import time
for i in range(max_retries):
try:
result = func()
if result:
return result
except Exception as e:
logger.warning(f"请求失败,第 {i+1} 次重试: {e}")
if i < max_retries - 1:
time.sleep(delay)
return None
@staticmethod
def check_proxy(proxy):
"""
检查代理是否可用
:param proxy: 代理URL
:return: bool - 是否可用
"""
try:
proxies = {
'http': proxy,
'https': proxy
}
response = requests.get('https://www.baidu.com', proxies=proxies, timeout=5)
return response.status_code == 200
except Exception as e:
logger.error(f"代理检查失败: {proxy}, 错误: {e}")
return False

View File

@@ -0,0 +1,66 @@
"""Centralized status codes and labels used across detection, database and UI."""
REGISTER_STATUS_PENDING = 0
REGISTER_STATUS_AVAILABLE = 2
REGISTER_STATUS_REGISTERED = 3
REGISTER_STATUS_GRACE = 4
REGISTER_STATUS_REDEMPTION = 5
REGISTER_STATUS_PENDING_DELETE = 6
REGISTER_STATUS_CLIENT_HOLD = 7
REGISTER_STATUS_SERVER_HOLD = 8
REGISTER_STATUS_UNKNOWN = 9
REGISTER_STATUS_FAILED = 10
DETECT_STATUS_PENDING = 0
DETECT_STATUS_COMPLETED = 1
DETECT_STATUS_RUNNING = 2
DETECT_STATUS_BLACKLISTED = 3
DETECT_STATUS_FAILED = 4
USE_STATUS_UNUSED = 0
USE_STATUS_USED = 1
USE_STATUS_SOLD = 2
USE_STATUS_RESERVED = 3
REVIEW_STATUS_NONE = 0
REVIEW_STATUS_PENDING = 1
REVIEW_STATUS_APPROVED = 2
REVIEW_STATUS_REJECTED = 3
THIRD_PARTY_STATUS_PENDING = 0
THIRD_PARTY_STATUS_DONE = 1
REGISTER_STATUS_LABELS = {
REGISTER_STATUS_PENDING: '待检测',
REGISTER_STATUS_AVAILABLE: '可注册',
REGISTER_STATUS_REGISTERED: '已注册',
REGISTER_STATUS_GRACE: '宽限期',
REGISTER_STATUS_REDEMPTION: '赎回期',
REGISTER_STATUS_PENDING_DELETE: '删除期',
REGISTER_STATUS_CLIENT_HOLD: 'clientHold',
REGISTER_STATUS_SERVER_HOLD: 'serverHold',
REGISTER_STATUS_UNKNOWN: '状态未知',
REGISTER_STATUS_FAILED: '检测失败',
}
DETECT_STATUS_LABELS = {
DETECT_STATUS_PENDING: '待检测',
DETECT_STATUS_COMPLETED: '检测完成',
DETECT_STATUS_RUNNING: '检测中',
DETECT_STATUS_BLACKLISTED: '黑名单',
DETECT_STATUS_FAILED: '检测失败',
}
USE_STATUS_LABELS = {
USE_STATUS_UNUSED: '未使用',
USE_STATUS_USED: '已经使用',
USE_STATUS_SOLD: '已经卖出',
USE_STATUS_RESERVED: '已经预定',
}
REVIEW_STATUS_LABELS = {
REVIEW_STATUS_NONE: '无需复核',
REVIEW_STATUS_PENDING: '待人工复核',
REVIEW_STATUS_APPROVED: '人工通过',
REVIEW_STATUS_REJECTED: '人工拒绝',
}