convert domainCheck to regular directory

This commit is contained in:
Your Name
2026-04-16 13:33:06 +08:00
parent e406d73334
commit d37c444929
107 changed files with 254185 additions and 1 deletions

View File

@@ -0,0 +1,4 @@
# -*- coding: UTF-8 -*-
'''
检测插件模块
'''

View File

@@ -0,0 +1,130 @@
# -*- coding: UTF-8 -*-
'''
@Project :domainScanDemo
@File :aizhan_detector.py
@IDE :PyCharm
@Author :梦伴
@Date :2026/4/9 0:02
@explain : 爱站网检测器
'''
import requests
from curl_cffi import requests as curl_requests
import re
from app.detectors.base import BaseDetector
class AizhanDetector(BaseDetector):
"""
爱站网检测器
"""
def __init__(self):
"""
初始化爱站网检测器
"""
super().__init__()
self.url = 'https://www.aizhan.com'
self.query_url = 'https://www.aizhan.com/cha/{domain}'
def check_domain(self, domain):
"""
检测域名
:param domain: 域名
:return: dict - 检测结果
"""
try:
# 构建查询URL
query_url = self.query_url.format(domain=domain)
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/146.0.0.0 Safari/537.36'
}
# 使用curl_cffi模拟浏览器
response = curl_requests.get(query_url, headers=headers, impersonate='chrome', timeout=10)
if response.status_code == 200:
content = response.text
# 提取标题
title = self._extract_title(content)
# 提取风险信息
risk = self._extract_risk(content)
# 检查是否包含敏感词
has_sensitive = self._check_sensitive(title, risk)
return {
'title': title,
'risk': risk,
'has_sensitive': has_sensitive
}
else:
self._log_warning(f"爱站网查询失败: {response.status_code}")
return {'title': '', 'risk': '', 'has_sensitive': False}
except Exception as e:
return self._handle_exception(e, domain)
def _extract_title(self, content):
"""
提取标题
:param content: 页面内容
:return: str - 标题
"""
try:
pattern = r'<title>(.*?)</title>'
match = re.search(pattern, content)
if match:
return match.group(1).strip()
return ''
except Exception as e:
self._handle_exception(e, 'extract_title')
return ''
def _extract_risk(self, content):
"""
提取风险信息
:param content: 页面内容
:return: str - 风险信息
"""
try:
# 这里需要根据实际页面结构调整正则表达式
pattern = r'百度网址检测:<span[^>]+>(.*?)</span>'
match = re.search(pattern, content)
if match:
return match.group(1).strip()
return ''
except Exception as e:
self._handle_exception(e, 'extract_risk')
return ''
def _check_sensitive(self, title, risk):
"""
检查是否包含敏感词
:param title: 标题
:param risk: 风险信息
:return: bool - 是否包含敏感词
"""
# 风险类型
sensitive_risks = ['低风险', '疑似色情博彩风险', '严重影响权重']
# 敏感词
sensitive_words = ['色情', '赌博', '博彩', '毒品', '暴力', '诈骗']
# 检查风险
for r in sensitive_risks:
if r in risk:
return True
# 检查标题
for word in sensitive_words:
if word in title:
return True
return False

View File

@@ -0,0 +1,151 @@
# -*- coding: UTF-8 -*-
'''
@Project :domainScanDemo
@File :baidu_detector.py
@IDE :PyCharm
@Author :梦伴
@Date :2026/4/8 23:58
@explain : 百度检测器
'''
import requests
from curl_cffi import requests as curl_requests
import re
from app.detectors.base import BaseDetector
class BaiduDetector(BaseDetector):
"""
百度检测器
"""
def __init__(self):
"""
初始化百度检测器
"""
super().__init__()
self.site_url = 'https://www.baidu.com/s'
self.history_url = 'https://www.baidu.com/s'
def check_domain(self, domain):
"""
检测域名
:param domain: 域名
:return: dict - 检测结果
"""
try:
# 检查百度site
site_result = self.check_site(domain)
# 检查百度历史
history_result = self.check_history(domain)
return {
'site': site_result,
'history': history_result
}
except Exception as e:
return self._handle_exception(e, domain)
def check_site(self, domain):
"""
检查百度site收录
:param domain: 域名
:return: dict - 检测结果
"""
try:
params = {
'wd': f'site:{domain}',
'rn': '50'
}
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/146.0.0.0 Safari/537.36'
}
# 使用curl_cffi模拟浏览器
response = curl_requests.get(self.site_url, params=params, headers=headers, impersonate='chrome', timeout=10)
if response.status_code == 200:
content = response.text
# 提取子域名
subdomains = self._extract_subdomains(content, domain)
# 检查是否有收录
has_收录 = '没有找到相关结果' not in content
return {
'has_收录': has_收录,
'subdomains': subdomains
}
else:
self._log_warning(f"百度site查询失败: {response.status_code}")
return {'has_收录': False, 'subdomains': []}
except Exception as e:
self._handle_exception(e, domain)
return {'has_收录': False, 'subdomains': []}
def check_history(self, domain):
"""
检查百度历史收录
:param domain: 域名
:return: dict - 检测结果
"""
try:
params = {
'wd': f'cache:{domain}',
'rn': '50'
}
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/146.0.0.0 Safari/537.36'
}
# 使用curl_cffi模拟浏览器
response = curl_requests.get(self.history_url, params=params, headers=headers, impersonate='chrome', timeout=10)
if response.status_code == 200:
content = response.text
# 检查是否有历史收录
has_history = '百度快照' in content
# 检查是否有灰色内容
has_gray = '风险提示' in content or '安全警告' in content
return {
'has_history': has_history,
'has_gray': has_gray
}
else:
self._log_warning(f"百度历史查询失败: {response.status_code}")
return {'has_history': False, 'has_gray': False}
except Exception as e:
self._handle_exception(e, domain)
return {'has_history': False, 'has_gray': False}
def _extract_subdomains(self, content, domain):
"""
提取子域名
:param content: 搜索结果内容
:param domain: 主域名
:return: list - 子域名列表
"""
try:
# 提取所有包含域名的链接
pattern = r'https?://([a-zA-Z0-9-]+)\.' + re.escape(domain)
matches = re.findall(pattern, content)
# 去重并过滤空值
subdomains = list(set(matches))
subdomains = [sub for sub in subdomains if sub]
return subdomains
except Exception as e:
self._handle_exception(e, domain)
return []

View File

@@ -0,0 +1,70 @@
# -*- coding: UTF-8 -*-
'''
@Project :domainScanDemo
@File :base.py
@IDE :PyCharm
@Author :梦伴
@Date :2026/4/8 23:55
@explain : 基础检测类
'''
from abc import ABC, abstractmethod
from loguru import logger
class BaseDetector(ABC):
"""
基础检测类
"""
def __init__(self):
"""
初始化检测类
"""
self.name = self.__class__.__name__
logger.info(f"初始化检测器: {self.name}")
@abstractmethod
def check_domain(self, domain):
"""
检测域名
:param domain: 域名
:return: dict - 检测结果
"""
pass
def _log_info(self, message):
"""
记录信息日志
:param message: 消息
"""
logger.info(f"[{self.name}] {message}")
def _log_warning(self, message):
"""
记录警告日志
:param message: 消息
"""
logger.warning(f"[{self.name}] {message}")
def _log_error(self, message):
"""
记录错误日志
:param message: 消息
"""
logger.error(f"[{self.name}] {message}")
def _handle_exception(self, e, domain):
"""
处理异常
:param e: 异常
:param domain: 域名
:return: dict - 错误结果
"""
self._log_error(f"检测域名 {domain} 出错: {e}")
return {'error': str(e)}

View File

@@ -0,0 +1,130 @@
# -*- coding: UTF-8 -*-
'''
@Project :domainScanDemo
@File :chinaz_detector.py
@IDE :PyCharm
@Author :梦伴
@Date :2026/4/9 0:01
@explain : 站长之家检测器
'''
import requests
from curl_cffi import requests as curl_requests
import re
from app.detectors.base import BaseDetector
class ChinazDetector(BaseDetector):
"""
站长之家检测器
"""
def __init__(self):
"""
初始化站长之家检测器
"""
super().__init__()
self.url = 'https://seo.chinaz.com'
self.query_url = 'https://seo.chinaz.com/{domain}'
def check_domain(self, domain):
"""
检测域名
:param domain: 域名
:return: dict - 检测结果
"""
try:
# 构建查询URL
query_url = self.query_url.format(domain=domain)
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/146.0.0.0 Safari/537.36'
}
# 使用curl_cffi模拟浏览器
response = curl_requests.get(query_url, headers=headers, impersonate='chrome', timeout=10)
if response.status_code == 200:
content = response.text
# 提取标题
title = self._extract_title(content)
# 提取网站分类
category = self._extract_category(content)
# 检查是否包含敏感词
has_sensitive = self._check_sensitive(title, category)
return {
'title': title,
'category': category,
'has_sensitive': has_sensitive
}
else:
self._log_warning(f"站长之家查询失败: {response.status_code}")
return {'title': '', 'category': '', 'has_sensitive': False}
except Exception as e:
return self._handle_exception(e, domain)
def _extract_title(self, content):
"""
提取标题
:param content: 页面内容
:return: str - 标题
"""
try:
pattern = r'<title>(.*?)</title>'
match = re.search(pattern, content)
if match:
return match.group(1).strip()
return ''
except Exception as e:
self._handle_exception(e, 'extract_title')
return ''
def _extract_category(self, content):
"""
提取网站分类
:param content: 页面内容
:return: str - 分类
"""
try:
# 这里需要根据实际页面结构调整正则表达式
pattern = r'网站分类:<a[^>]+>(.*?)</a>'
match = re.search(pattern, content)
if match:
return match.group(1).strip()
return ''
except Exception as e:
self._handle_exception(e, 'extract_category')
return ''
def _check_sensitive(self, title, category):
"""
检查是否包含敏感词
:param title: 标题
:param category: 分类
:return: bool - 是否包含敏感词
"""
# 敏感分类
sensitive_categories = ['视频电影', '体育运动', '常用查询']
# 敏感词
sensitive_words = ['色情', '赌博', '博彩', '毒品', '暴力', '诈骗']
# 检查分类
for cat in sensitive_categories:
if cat in category:
return True
# 检查标题
for word in sensitive_words:
if word in title:
return True
return False

View File

@@ -0,0 +1,80 @@
# -*- coding: UTF-8 -*-
'''
@Project :domainScanDemo
@File :google_detector.py
@IDE :PyCharm
@Author :梦伴
@Date :2026/4/9 0:00
@explain : Google检测器
'''
import requests
from curl_cffi import requests as curl_requests
import re
from app.detectors.base import BaseDetector
class GoogleDetector(BaseDetector):
"""
Google检测器
"""
def __init__(self):
"""
初始化Google检测器
"""
super().__init__()
self.site_url = 'https://www.google.com/search'
def check_domain(self, domain):
"""
检测域名
:param domain: 域名
:return: dict - 检测结果
"""
try:
# 检查Google site
site_result = self.check_site(domain)
return {
'site': site_result
}
except Exception as e:
return self._handle_exception(e, domain)
def check_site(self, domain):
"""
检查Google site收录
:param domain: 域名
:return: dict - 检测结果
"""
try:
params = {
'q': f'site:{domain}',
'num': '50'
}
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/146.0.0.0 Safari/537.36'
}
# 使用curl_cffi模拟浏览器
response = curl_requests.get(self.site_url, params=params, headers=headers, impersonate='chrome', timeout=10)
if response.status_code == 200:
content = response.text
# 检查是否有收录
has_收录 = 'No results found for' not in content
return {
'has_收录': has_收录
}
else:
self._log_warning(f"Google site查询失败: {response.status_code}")
return {'has_收录': False}
except Exception as e:
self._handle_exception(e, domain)
return {'has_收录': False}

View File

@@ -0,0 +1,228 @@
# -*- coding: UTF-8 -*-
'''
@Project :domainScanDemo
@File :jucha_detector.py
@IDE :PyCharm
@Author :梦伴
@Date :2026/4/9 0:04
@explain : 聚查检测器
'''
import requests
from curl_cffi import requests as curl_requests
import re
from app.detectors.base import BaseDetector
class JuchaDetector(BaseDetector):
"""
聚查检测器
"""
def __init__(self):
"""
初始化聚查检测器
"""
super().__init__()
self.url = 'https://www.jucha.com'
self.whois_url = 'https://www.jucha.com/whois/{domain}'
self.beian_url = 'https://www.jucha.com/beian/{domain}'
self.intercept_url = 'https://www.jucha.com/intercept/{domain}'
def check_domain(self, domain):
"""
检测域名
:param domain: 域名
:return: dict - 检测结果
"""
try:
# 检查WHOIS
whois_result = self.check_whois(domain)
# 检查备案
beian_result = self.check_beian(domain)
# 检查拦截
intercept_result = self.check_intercept(domain)
return {
'whois': whois_result,
'beian': beian_result,
'intercept': intercept_result
}
except Exception as e:
return self._handle_exception(e, domain)
def check_whois(self, domain):
"""
检查WHOIS
:param domain: 域名
:return: dict - 检测结果
"""
try:
# 构建查询URL
query_url = self.whois_url.format(domain=domain)
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/146.0.0.0 Safari/537.36'
}
# 使用curl_cffi模拟浏览器
response = curl_requests.get(query_url, headers=headers, impersonate='chrome', timeout=10)
if response.status_code == 200:
content = response.text
# 提取WHOIS信息
whois_info = self._extract_whois_info(content)
return whois_info
else:
self._log_warning(f"聚查WHOIS查询失败: {response.status_code}")
return {'status': ''}
except Exception as e:
self._handle_exception(e, domain)
return {'status': ''}
def check_beian(self, domain):
"""
检查备案
:param domain: 域名
:return: dict - 检测结果
"""
try:
# 构建查询URL
query_url = self.beian_url.format(domain=domain)
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/146.0.0.0 Safari/537.36'
}
# 使用curl_cffi模拟浏览器
response = curl_requests.get(query_url, headers=headers, impersonate='chrome', timeout=10)
if response.status_code == 200:
content = response.text
# 提取备案信息
beian_info = self._extract_beian_info(content)
return beian_info
else:
self._log_warning(f"聚查备案查询失败: {response.status_code}")
return {'has_beian': False, 'beian_year': '', 'is_enterprise': False, 'beian_match': False}
except Exception as e:
self._handle_exception(e, domain)
return {'has_beian': False, 'beian_year': '', 'is_enterprise': False, 'beian_match': False}
def check_intercept(self, domain):
"""
检查拦截
:param domain: 域名
:return: dict - 检测结果
"""
try:
# 构建查询URL
query_url = self.intercept_url.format(domain=domain)
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/146.0.0.0 Safari/537.36'
}
# 使用curl_cffi模拟浏览器
response = curl_requests.get(query_url, headers=headers, impersonate='chrome', timeout=10)
if response.status_code == 200:
content = response.text
# 检查是否被拦截
is_normal = self._check_intercept_status(content)
return {
'normal': is_normal
}
else:
self._log_warning(f"聚查拦截查询失败: {response.status_code}")
return {'normal': False}
except Exception as e:
self._handle_exception(e, domain)
return {'normal': False}
def _extract_whois_info(self, content):
"""
提取WHOIS信息
:param content: 页面内容
:return: dict - WHOIS信息
"""
try:
# 提取状态信息
pattern = r'域名状态:<span[^>]+>(.*?)</span>'
match = re.search(pattern, content)
status = match.group(1).strip() if match else ''
return {
'status': status
}
except Exception as e:
self._handle_exception(e, 'extract_whois_info')
return {'status': ''}
def _extract_beian_info(self, content):
"""
提取备案信息
:param content: 页面内容
:return: dict - 备案信息
"""
try:
# 检查是否有备案
has_beian = '备案信息' in content
# 提取备案年份
beian_year = ''
pattern = r'审核时间:(\d{4})-\d{2}-\d{2}'
match = re.search(pattern, content)
if match:
beian_year = match.group(1)
# 提取单位性质
is_enterprise = '企业' in content
# 检查首网址和备案网址是否一致
beian_match = '网站首页网址' in content
return {
'has_beian': has_beian,
'beian_year': beian_year,
'is_enterprise': is_enterprise,
'beian_match': beian_match
}
except Exception as e:
self._handle_exception(e, 'extract_beian_info')
return {'has_beian': False, 'beian_year': '', 'is_enterprise': False, 'beian_match': False}
def _check_intercept_status(self, content):
"""
检查拦截状态
:param content: 页面内容
:return: bool - 是否正常
"""
try:
# 检查是否包含正常标识
if '正常' in content:
return True
# 检查是否包含拦截标识
if '拦截' in content:
return False
return False
except Exception as e:
self._handle_exception(e, 'check_intercept_status')
return False

View File

@@ -0,0 +1,214 @@
# -*- coding: UTF-8 -*-
'''
@Project :domainScanDemo
@File :juziseo_detector.py
@IDE :PyCharm
@Author :梦伴
@Date :2026/4/9 0:03
@explain : 桔子SEO检测器
'''
import requests
from curl_cffi import requests as curl_requests
import re
from app.detectors.base import BaseDetector
class JuziseoDetector(BaseDetector):
"""
桔子SEO检测器
"""
def __init__(self):
"""
初始化桔子SEO检测器
"""
super().__init__()
self.url = 'https://seo.juziseo.com'
self.history_url = 'https://seo.juziseo.com/history/{domain}'
self.backlink_url = 'https://seo.juziseo.com/backlink/{domain}'
def check_domain(self, domain):
"""
检测域名
:param domain: 域名
:return: dict - 检测结果
"""
try:
# 检查历史信息
history_result = self.check_history(domain)
# 检查外链
backlink_result = self.check_backlink(domain)
return {
'history': history_result,
'backlink': backlink_result
}
except Exception as e:
return self._handle_exception(e, domain)
def check_history(self, domain):
"""
检查历史信息
:param domain: 域名
:return: dict - 检测结果
"""
try:
# 构建查询URL
query_url = self.history_url.format(domain=domain)
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/146.0.0.0 Safari/537.36'
}
# 使用curl_cffi模拟浏览器
response = curl_requests.get(query_url, headers=headers, impersonate='chrome', timeout=10)
if response.status_code == 200:
content = response.text
# 提取历史信息
history_info = self._extract_history_info(content)
# 检查是否包含敏感词
has_sensitive = self._check_sensitive(history_info)
# 检查是否有百度历史收录
has_baidu_history = '百度历史收录' in content
# 检查是否有子域名
has_subdomains = '子域名' in content
# 检查是否为简体中文
is_simplified = self._check_simplified(content)
return {
'has_sensitive': has_sensitive,
'has_baidu_history': has_baidu_history,
'has_subdomains': has_subdomains,
'is_simplified': is_simplified
}
else:
self._log_warning(f"桔子SEO历史查询失败: {response.status_code}")
return {'has_sensitive': False, 'has_baidu_history': False, 'has_subdomains': False, 'is_simplified': True}
except Exception as e:
self._handle_exception(e, domain)
return {'has_sensitive': False, 'has_baidu_history': False, 'has_subdomains': False, 'is_simplified': True}
def check_backlink(self, domain):
"""
检查外链
:param domain: 域名
:return: dict - 检测结果
"""
try:
# 构建查询URL
query_url = self.backlink_url.format(domain=domain)
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/146.0.0.0 Safari/537.36'
}
# 使用curl_cffi模拟浏览器
response = curl_requests.get(query_url, headers=headers, impersonate='chrome', timeout=10)
if response.status_code == 200:
content = response.text
# 检查是否包含敏感词
has_sensitive = self._check_backlink_sensitive(content)
# 检查是否有子域名
has_subdomains = '子域名' in content
return {
'has_sensitive': has_sensitive,
'has_subdomains': has_subdomains
}
else:
self._log_warning(f"桔子SEO外链查询失败: {response.status_code}")
return {'has_sensitive': False, 'has_subdomains': False}
except Exception as e:
self._handle_exception(e, domain)
return {'has_sensitive': False, 'has_subdomains': False}
def _extract_history_info(self, content):
"""
提取历史信息
:param content: 页面内容
:return: str - 历史信息
"""
try:
# 这里需要根据实际页面结构调整正则表达式
pattern = r'<div class="history-info">(.*?)</div>'
match = re.search(pattern, content, re.DOTALL)
if match:
return match.group(1).strip()
return ''
except Exception as e:
self._handle_exception(e, 'extract_history_info')
return ''
def _check_sensitive(self, history_info):
"""
检查是否包含敏感词
:param history_info: 历史信息
:return: bool - 是否包含敏感词
"""
# 敏感词
sensitive_words = [
'色情', '赌博', '博彩', '毒品', '暴力', '诈骗',
'足球', '直播', '证券', '配资', '软件',
'体育', '商行', '下载', '影视', '网络',
'计算', 'app', 'HTML SiteMap', '模拟器', '传媒',
'二次元', '成人', '米乐', '小说', '凯发',
'人才', '华体', '娱乐', '开户'
]
for word in sensitive_words:
if word in history_info:
return True
return False
def _check_simplified(self, content):
"""
检查是否为简体中文
:param content: 页面内容
:return: bool - 是否为简体中文
"""
# 检查是否包含简体中文标识
if '简体中文' in content:
return True
# 检查是否包含繁体中文标识
if '繁体中文' in content:
return False
return True
def _check_backlink_sensitive(self, content):
"""
检查外链是否包含敏感词
:param content: 页面内容
:return: bool - 是否包含敏感词
"""
# 敏感词
sensitive_words = [
'内幕', '猛料', '精料', '高手', '绝杀',
'权威', '澳门', '色情', '赌博', '博彩'
]
for word in sensitive_words:
if word in content:
return True
return False

View File

@@ -0,0 +1,107 @@
# -*- coding: UTF-8 -*-
'''
@Project :domainScanDemo
@File :qihu360_detector.py
@IDE :PyCharm
@Author :梦伴
@Date :2026/4/8 23:59
@explain : 360检测器
'''
import requests
from curl_cffi import requests as curl_requests
import re
from app.detectors.base import BaseDetector
class Qihu360Detector(BaseDetector):
"""
360检测器
"""
def __init__(self):
"""
初始化360检测器
"""
super().__init__()
self.site_url = 'https://www.so.com/s'
def check_domain(self, domain):
"""
检测域名
:param domain: 域名
:return: dict - 检测结果
"""
try:
# 检查360 site
site_result = self.check_site(domain)
return {
'site': site_result
}
except Exception as e:
return self._handle_exception(e, domain)
def check_site(self, domain):
"""
检查360 site收录
:param domain: 域名
:return: dict - 检测结果
"""
try:
params = {
'q': f'site:{domain}',
'pn': '1',
'rn': '50'
}
headers = {
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/146.0.0.0 Safari/537.36'
}
# 使用curl_cffi模拟浏览器
response = curl_requests.get(self.site_url, params=params, headers=headers, impersonate='chrome', timeout=10)
if response.status_code == 200:
content = response.text
# 提取子域名
subdomains = self._extract_subdomains(content, domain)
# 检查是否有收录
has_收录 = '没有找到相关结果' not in content
return {
'has_收录': has_收录,
'subdomains': subdomains
}
else:
self._log_warning(f"360 site查询失败: {response.status_code}")
return {'has_收录': False, 'subdomains': []}
except Exception as e:
self._handle_exception(e, domain)
return {'has_收录': False, 'subdomains': []}
def _extract_subdomains(self, content, domain):
"""
提取子域名
:param content: 搜索结果内容
:param domain: 主域名
:return: list - 子域名列表
"""
try:
# 提取所有包含域名的链接
pattern = r'https?://([a-zA-Z0-9-]+)\.' + re.escape(domain)
matches = re.findall(pattern, content)
# 去重并过滤空值
subdomains = list(set(matches))
subdomains = [sub for sub in subdomains if sub]
return subdomains
except Exception as e:
self._handle_exception(e, domain)
return []

View File

@@ -0,0 +1,128 @@
# -*- coding: UTF-8 -*-
'''
@Project :domainScanDemo
@File :rdap_detector.py
@IDE :PyCharm
@Author :梦伴
@Date :2026/4/8 23:56
@explain : RDAP检测器
'''
import requests
from app.detectors.base import BaseDetector
class RDAPDetector(BaseDetector):
"""
RDAP检测器
"""
def __init__(self):
"""
初始化RDAP检测器
"""
super().__init__()
self.rdap_urls = {
'com': 'https://rdap.verisign.com/com/v1/domain/',
'net': 'https://rdap.verisign.com/net/v1/domain/'
}
def check_domain(self, domain):
"""
检测域名
:param domain: 域名
:return: dict - 检测结果
"""
try:
# 提取顶级域名
tld = domain.split('.')[-1]
if tld not in self.rdap_urls:
return {'error': '不支持的顶级域名'}
# 构建RDAP查询URL
url = self.rdap_urls[tld] + domain
# 发送请求
response = requests.get(url, timeout=10)
if response.status_code == 200:
data = response.json()
return self._parse_rdap_response(data)
elif response.status_code == 404:
return {'status': 'available'}
else:
return {'error': f'RDAP查询失败: {response.status_code}'}
except Exception as e:
return self._handle_exception(e, domain)
def check_register_status(self, domain):
"""
检查注册状态
:param domain: 域名
:return: int - 注册状态码
"""
try:
result = self.check_domain(domain)
if 'error' in result:
return 9 # 状态未知
if result.get('status') == 'available':
return 2 # 可注册
# 检查域名状态
statuses = result.get('status', [])
if 'clientHold' in statuses:
return 7 # clientHold
elif 'serverHold' in statuses:
return 8 # serverHold
elif 'autoRenewPeriod' in statuses:
return 4 # 宽限期
elif 'redemptionPeriod' in statuses:
return 5 # 赎回期
elif 'pendingDelete' in statuses:
return 6 # 删除期
else:
return 3 # 已注册
except Exception as e:
self._handle_exception(e, domain)
return 10 # 检测失败
def _parse_rdap_response(self, data):
"""
解析RDAP响应
:param data: RDAP响应数据
:return: dict - 解析结果
"""
result = {
'status': 'registered',
'domain': data.get('ldhName'),
'statuses': data.get('status', []),
'registrar': None,
'creation_date': None,
'expiration_date': None,
'last_update': None
}
# 解析注册商信息
for entity in data.get('entities', []):
if 'registrar' in entity.get('roles', []):
result['registrar'] = entity.get('vcardArray', [[], []])[1][1][3]
break
# 解析时间信息
for event in data.get('events', []):
event_action = event.get('eventAction')
event_date = event.get('eventDate')
if event_action == 'registration':
result['creation_date'] = event_date
elif event_action == 'expiration':
result['expiration_date'] = event_date
elif event_action == 'last update':
result['last_update'] = event_date
return result

View File

@@ -0,0 +1,562 @@
# -*- coding: UTF-8 -*-
'''
@Project :domainScanDemo
@File :wayback_detector.py
@IDE :PyCharm
@Author :梦伴
@Date :2026/4/8 23:57
@explain : Wayback检测器
'''
import html
import json
import re
import threading
import time
import zlib
from base64 import b64decode, b64encode
from concurrent.futures import FIRST_COMPLETED, ThreadPoolExecutor, wait
import redis
import requests
from requests.adapters import HTTPAdapter
from urllib3.util.retry import Retry
from app.config import config
from app.detectors.base import BaseDetector
class WaybackDetector(BaseDetector):
"""
Wayback检测器
"""
TITLE_PATTERN = re.compile(r'<title[^>]*>(.*?)</title>', re.IGNORECASE | re.DOTALL)
def __init__(self):
"""
初始化Wayback检测器
"""
super().__init__()
self.cdx_api_url = 'https://web.archive.org/cdx/search/cdx'
# 使用 id_ 直接取快照内容,避免回放页面额外壳层干扰正文匹配。
self.snapshot_url = 'https://web.archive.org/web/{timestamp}id_/{domain}'
self._timestamp_cache = {}
self._title_cache = {}
self._cache_lock = threading.Lock()
self.session = self._build_session()
self.redis_client = self._build_redis_client()
def _build_session(self):
session = requests.Session()
retry = Retry(
total=max(0, config.WAYBACK_RETRY_COUNT),
backoff_factor=0.5,
status_forcelist=(429, 500, 502, 503, 504),
allowed_methods=frozenset(["GET"]),
raise_on_status=False,
)
adapter = HTTPAdapter(max_retries=retry, pool_connections=10, pool_maxsize=10)
session.mount('http://', adapter)
session.mount('https://', adapter)
session.headers.update({
'User-Agent': config.WAYBACK_USER_AGENT,
'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8',
})
return session
def _build_redis_client(self):
try:
client = redis.Redis(
host=config.REDIS_HOST,
port=config.REDIS_PORT,
password=config.REDIS_PASSWORD,
db=config.REDIS_DB,
decode_responses=True,
socket_connect_timeout=5,
socket_timeout=5,
)
client.ping()
return client
except Exception:
return None
def _title_cache_key(self, domain, timestamp):
return f"domain_tool:wayback_title:{domain}:{timestamp}"
def _timestamp_cache_key(self, domain):
return f"domain_tool:wayback_timestamps:{domain}"
def _record_cache_key(self, domain):
return f"domain_tool:wayback_records:{domain}"
def _normalize_title(self, title):
normalized = html.unescape(title or '')
normalized = re.sub(r'\s+', ' ', normalized, flags=re.DOTALL).strip().lower()
return normalized
def _extract_title(self, content):
match = self.TITLE_PATTERN.search(content or '')
if not match:
return ''
return html.unescape(match.group(1)).strip()
def _load_cached_title(self, domain, timestamp):
cache_key = self._title_cache_key(domain, timestamp)
with self._cache_lock:
if cache_key in self._title_cache:
return self._title_cache[cache_key]
if self.redis_client:
try:
raw_value = self.redis_client.get(cache_key)
if raw_value:
data = json.loads(raw_value)
with self._cache_lock:
self._title_cache[cache_key] = data
return data
except Exception:
pass
return None
def _save_cached_title(self, domain, timestamp, data):
cache_key = self._title_cache_key(domain, timestamp)
with self._cache_lock:
self._title_cache[cache_key] = data
if self.redis_client:
try:
self.redis_client.set(
cache_key,
json.dumps(data, ensure_ascii=False),
ex=max(0, config.WAYBACK_TITLE_CACHE_TTL) or None,
)
except Exception:
pass
def _load_cached_timestamps(self, domain):
if domain in self._timestamp_cache:
return list(self._timestamp_cache[domain])
if not self.redis_client:
return None
try:
raw_value = self.redis_client.get(self._timestamp_cache_key(domain))
if not raw_value:
return None
compressed = b64decode(raw_value.encode('ascii'))
timestamps = json.loads(zlib.decompress(compressed).decode('utf-8'))
if isinstance(timestamps, list):
self._timestamp_cache[domain] = tuple(timestamps)
return list(timestamps)
except Exception:
return None
return None
def _save_cached_timestamps(self, domain, timestamps):
self._timestamp_cache[domain] = tuple(timestamps)
if not self.redis_client:
return
try:
payload = json.dumps(timestamps, separators=(',', ':')).encode('utf-8')
compressed = zlib.compress(payload, level=6)
self.redis_client.set(
self._timestamp_cache_key(domain),
b64encode(compressed).decode('ascii'),
ex=max(0, config.WAYBACK_TIMESTAMP_CACHE_TTL) or None,
)
except Exception:
pass
def _load_cached_records(self, domain):
cache_key = self._record_cache_key(domain)
with self._cache_lock:
if cache_key in self._timestamp_cache:
return [
{'timestamp': item[0], 'digest': item[1]}
for item in self._timestamp_cache[cache_key]
]
if not self.redis_client:
return None
try:
raw_value = self.redis_client.get(cache_key)
if not raw_value:
return None
compressed = b64decode(raw_value.encode('ascii'))
records = json.loads(zlib.decompress(compressed).decode('utf-8'))
if isinstance(records, list):
normalized = tuple(
(item.get('timestamp', ''), item.get('digest', ''))
for item in records if isinstance(item, dict)
)
with self._cache_lock:
self._timestamp_cache[cache_key] = normalized
return [
{'timestamp': item[0], 'digest': item[1]}
for item in normalized if item[0]
]
except Exception:
return None
return None
def _save_cached_records(self, domain, records):
cache_key = self._record_cache_key(domain)
normalized = tuple(
(item.get('timestamp', ''), item.get('digest', ''))
for item in (records or [])
if item and item.get('timestamp')
)
with self._cache_lock:
self._timestamp_cache[cache_key] = normalized
if not self.redis_client:
return
try:
payload = json.dumps(
[{'timestamp': item[0], 'digest': item[1]} for item in normalized],
separators=(',', ':')
).encode('utf-8')
compressed = zlib.compress(payload, level=6)
self.redis_client.set(
cache_key,
b64encode(compressed).decode('ascii'),
ex=max(0, config.WAYBACK_TIMESTAMP_CACHE_TTL) or None,
)
except Exception:
pass
def _fetch_cdx_records(self, domain, limit=None, fast_latest=False):
response = None
try:
params = {
'url': domain,
'output': 'txt',
'fl': 'timestamp,digest',
'filter': ['statuscode:200', 'mimetype:text/html'],
}
if limit is not None:
params['limit'] = str(limit)
if fast_latest:
params['fastLatest'] = 'true'
response = self.session.get(
self.cdx_api_url,
params=params,
timeout=config.WAYBACK_CDX_TIMEOUT,
stream=True,
)
if response.status_code != 200:
self._log_warning(f"获取快照记录失败: {response.status_code}")
return []
records = []
seen = set()
for raw_line in response.iter_lines(decode_unicode=True):
line = (raw_line or '').strip()
if not line:
continue
parts = line.split()
timestamp = parts[0].strip() if parts else ''
digest = parts[1].strip() if len(parts) > 1 else ''
if not timestamp or timestamp in seen:
continue
seen.add(timestamp)
records.append({'timestamp': timestamp, 'digest': digest})
return records
except Exception as e:
self._handle_exception(e, domain)
return []
finally:
try:
if response is not None:
response.close()
except Exception:
pass
def get_latest_snapshot_record(self, domain):
records = self._fetch_cdx_records(domain, limit=-1, fast_latest=True)
return records[0] if records else None
def get_snapshot_records(self, domain):
cached_records = self._load_cached_records(domain)
if cached_records is not None:
return cached_records
records = self._fetch_cdx_records(domain)
if records:
self._save_cached_records(domain, records)
self._save_cached_timestamps(domain, [item['timestamp'] for item in records])
return records
def check_domain(self, domain):
"""
检测域名
:param domain: 域名
:return: dict - 检测结果
"""
try:
return self.scan_snapshots(domain)
except Exception as e:
return self._handle_exception(e, domain)
def get_snapshot_timestamps(self, domain):
cached_records = self._load_cached_records(domain)
if cached_records is not None:
return [item['timestamp'] for item in cached_records if item.get('timestamp')]
cached_timestamps = self._load_cached_timestamps(domain)
if cached_timestamps is not None:
return cached_timestamps
records = self.get_snapshot_records(domain)
return [item['timestamp'] for item in records if item.get('timestamp')]
def _fetch_snapshot_title(self, domain, timestamp):
cached = self._load_cached_title(domain, timestamp)
if cached is not None:
return cached
snapshot_url = self.snapshot_url.format(timestamp=timestamp, domain=domain)
response = None
try:
response = self.session.get(
snapshot_url,
timeout=config.WAYBACK_SNAPSHOT_TIMEOUT,
stream=True,
)
if response.status_code != 200:
data = {'timestamp': timestamp, 'title': '', 'ok': False}
self._save_cached_title(domain, timestamp, data)
return data
content_type = (response.headers.get('Content-Type') or '').lower()
if content_type and 'text/html' not in content_type and 'application/xhtml+xml' not in content_type:
data = {'timestamp': timestamp, 'title': '', 'ok': False}
self._save_cached_title(domain, timestamp, data)
return data
chunks = []
total_bytes = 0
found_title = False
for chunk in response.iter_content(chunk_size=4096, decode_unicode=True):
if not chunk:
continue
chunks.append(chunk)
total_bytes += len(chunk.encode('utf-8', errors='ignore'))
current_text = ''.join(chunks)
if '</title>' in current_text.lower():
found_title = True
break
if total_bytes >= config.WAYBACK_TITLE_MAX_BYTES:
break
content = ''.join(chunks)
title = self._extract_title(content) if found_title or content else ''
data = {'timestamp': timestamp, 'title': title, 'ok': True}
self._save_cached_title(domain, timestamp, data)
return data
except Exception:
data = {'timestamp': timestamp, 'title': '', 'ok': False}
self._save_cached_title(domain, timestamp, data)
return data
finally:
try:
if response is not None:
response.close()
except Exception:
pass
def get_snapshot_years(self, domain):
"""
获取快照年份
:param domain: 域名
:return: list - 快照年份列表
"""
try:
years = set()
for timestamp in self.get_snapshot_timestamps(domain):
if len(timestamp) >= 4:
years.add(int(timestamp[:4]))
return sorted(years)
except Exception as e:
self._handle_exception(e, domain)
return []
def has_sensitive_content(self, domain):
"""
检查是否包含敏感内容
:param domain: 域名
:return: bool - 是否包含敏感内容
"""
try:
result = self.scan_snapshots(domain)
return result.get('has_sensitive_content', False)
except Exception as e:
self._handle_exception(e, domain)
return False
def scan_snapshots(self, domain, sensitive_words=None, stop_on_first_hit=True):
sensitive_words = sensitive_words or config.load_sensitive_words()
latest_record = self.get_latest_snapshot_record(domain)
latest_timestamp = (latest_record or {}).get('timestamp')
latest_digest = (latest_record or {}).get('digest', '')
matched_word = None
matched_timestamp = None
matched_title = None
fetched_snapshot_count = 0
failed_snapshot_count = 0
unique_title_count = 0
duplicate_title_skipped = 0
digest_duplicate_skipped = 0
started_at = time.time()
progress_interval = max(1, config.WAYBACK_PROGRESS_INTERVAL)
title_seen = set()
digest_seen = set()
checked_snapshot_count = 0
domain_concurrency = max(1, config.WAYBACK_DOMAIN_CONCURRENCY)
if latest_timestamp:
latest_result = self._fetch_snapshot_title(domain, latest_timestamp)
checked_snapshot_count = 1
if latest_result and latest_result.get('ok'):
fetched_snapshot_count = 1
title = latest_result.get('title', '')
normalized_title = self._normalize_title(title)
if latest_digest:
digest_seen.add(latest_digest)
if normalized_title:
title_seen.add(normalized_title)
unique_title_count = 1
matched_word = self._find_sensitive_word(title, sensitive_words)
if matched_word and stop_on_first_hit:
matched_timestamp = latest_timestamp
matched_title = title
return {
'snapshot_years': [int(latest_timestamp[:4])] if len(latest_timestamp) >= 4 else [],
'has_sensitive_content': True,
'matched_word': matched_word,
'matched_timestamp': matched_timestamp,
'matched_title': matched_title,
'backlink_count': 0,
'backlink_count_gt_10': False,
'checked_snapshot_count': checked_snapshot_count,
'fetched_snapshot_count': fetched_snapshot_count,
'failed_snapshot_count': failed_snapshot_count,
'unique_title_count': unique_title_count,
'duplicate_title_skipped': duplicate_title_skipped,
'digest_duplicate_skipped': digest_duplicate_skipped,
'elapsed_seconds': round(time.time() - started_at, 2),
}
else:
failed_snapshot_count += 1
records = sorted(self.get_snapshot_records(domain), key=lambda item: item.get('timestamp', ''), reverse=True)
years = sorted({int(item['timestamp'][:4]) for item in records if len(item.get('timestamp', '')) >= 4})
checked_snapshot_count = len(records)
pending_records = []
for item in records:
timestamp = item.get('timestamp', '')
digest = item.get('digest', '')
if not timestamp:
continue
if latest_timestamp and timestamp == latest_timestamp:
continue
if digest and digest in digest_seen:
digest_duplicate_skipped += 1
continue
if digest:
digest_seen.add(digest)
pending_records.append(item)
with ThreadPoolExecutor(max_workers=domain_concurrency) as executor:
pending = {}
index = 0
finished_count = 1 if latest_timestamp else 0
stop_requested = False
while (index < len(pending_records) or pending) and not stop_requested:
while index < len(pending_records) and len(pending) < domain_concurrency and not stop_requested:
timestamp = pending_records[index]['timestamp']
future = executor.submit(self._fetch_snapshot_title, domain, timestamp)
pending[future] = timestamp
index += 1
if not pending:
break
done, _ = wait(list(pending.keys()), return_when=FIRST_COMPLETED)
for future in done:
timestamp = pending.pop(future, None)
finished_count += 1
try:
result = future.result()
except Exception as e:
failed_snapshot_count += 1
self._handle_exception(e, domain)
continue
if not result or not result.get('ok'):
failed_snapshot_count += 1
continue
fetched_snapshot_count += 1
title = result.get('title', '')
normalized_title = self._normalize_title(title)
if normalized_title:
if normalized_title in title_seen:
duplicate_title_skipped += 1
else:
title_seen.add(normalized_title)
unique_title_count += 1
matched_word = self._find_sensitive_word(title, sensitive_words)
if matched_word:
matched_timestamp = timestamp
matched_title = title
if stop_on_first_hit:
stop_requested = True
if finished_count % progress_interval == 0:
elapsed = round(time.time() - started_at, 2)
self._log_info(
f"{domain} 时光机进度: {finished_count}/{checked_snapshot_count},成功 {fetched_snapshot_count},失败 {failed_snapshot_count},唯一标题 {unique_title_count},标题重复跳过 {duplicate_title_skipped}digest 重复跳过 {digest_duplicate_skipped},耗时 {elapsed}s"
)
if config.WAYBACK_REQUEST_DELAY > 0:
time.sleep(config.WAYBACK_REQUEST_DELAY)
if stop_requested:
for future in pending:
future.cancel()
return {
'snapshot_years': years,
'has_sensitive_content': matched_word is not None,
'matched_word': matched_word,
'matched_timestamp': matched_timestamp,
'matched_title': matched_title,
'backlink_count': 0,
'backlink_count_gt_10': False,
'checked_snapshot_count': checked_snapshot_count,
'fetched_snapshot_count': fetched_snapshot_count,
'failed_snapshot_count': failed_snapshot_count,
'unique_title_count': unique_title_count,
'duplicate_title_skipped': duplicate_title_skipped,
'digest_duplicate_skipped': digest_duplicate_skipped,
'elapsed_seconds': round(time.time() - started_at, 2),
}
def _check_sensitive_words(self, content):
"""
检查敏感词
:param content: 内容
:return: bool - 是否包含敏感词
"""
return self._find_sensitive_word(content, config.load_sensitive_words()) is not None
def _find_sensitive_word(self, content, sensitive_words):
for word in sensitive_words or []:
if word and word in (content or ''):
return word
return None
def get_backlink_count(self, domain):
"""
当前策略仅扫描标题,不再抓取正文,友链数量默认返回 0。
"""
return 0
def _count_backlinks(self, content):
return 0