convert domainCheck to regular directory
This commit is contained in:
4
domainCheck/app/detectors/__init__.py
Normal file
4
domainCheck/app/detectors/__init__.py
Normal file
@@ -0,0 +1,4 @@
|
||||
# -*- coding: UTF-8 -*-
|
||||
'''
|
||||
检测插件模块
|
||||
'''
|
||||
130
domainCheck/app/detectors/aizhan_detector.py
Normal file
130
domainCheck/app/detectors/aizhan_detector.py
Normal file
@@ -0,0 +1,130 @@
|
||||
# -*- coding: UTF-8 -*-
|
||||
'''
|
||||
@Project :domainScanDemo
|
||||
@File :aizhan_detector.py
|
||||
@IDE :PyCharm
|
||||
@Author :梦伴
|
||||
@Date :2026/4/9 0:02
|
||||
@explain : 爱站网检测器
|
||||
'''
|
||||
|
||||
import requests
|
||||
from curl_cffi import requests as curl_requests
|
||||
import re
|
||||
from app.detectors.base import BaseDetector
|
||||
|
||||
|
||||
class AizhanDetector(BaseDetector):
|
||||
"""
|
||||
爱站网检测器
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
"""
|
||||
初始化爱站网检测器
|
||||
"""
|
||||
super().__init__()
|
||||
self.url = 'https://www.aizhan.com'
|
||||
self.query_url = 'https://www.aizhan.com/cha/{domain}'
|
||||
|
||||
def check_domain(self, domain):
|
||||
"""
|
||||
检测域名
|
||||
|
||||
:param domain: 域名
|
||||
:return: dict - 检测结果
|
||||
"""
|
||||
try:
|
||||
# 构建查询URL
|
||||
query_url = self.query_url.format(domain=domain)
|
||||
|
||||
headers = {
|
||||
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/146.0.0.0 Safari/537.36'
|
||||
}
|
||||
|
||||
# 使用curl_cffi模拟浏览器
|
||||
response = curl_requests.get(query_url, headers=headers, impersonate='chrome', timeout=10)
|
||||
|
||||
if response.status_code == 200:
|
||||
content = response.text
|
||||
|
||||
# 提取标题
|
||||
title = self._extract_title(content)
|
||||
|
||||
# 提取风险信息
|
||||
risk = self._extract_risk(content)
|
||||
|
||||
# 检查是否包含敏感词
|
||||
has_sensitive = self._check_sensitive(title, risk)
|
||||
|
||||
return {
|
||||
'title': title,
|
||||
'risk': risk,
|
||||
'has_sensitive': has_sensitive
|
||||
}
|
||||
else:
|
||||
self._log_warning(f"爱站网查询失败: {response.status_code}")
|
||||
return {'title': '', 'risk': '', 'has_sensitive': False}
|
||||
except Exception as e:
|
||||
return self._handle_exception(e, domain)
|
||||
|
||||
def _extract_title(self, content):
|
||||
"""
|
||||
提取标题
|
||||
|
||||
:param content: 页面内容
|
||||
:return: str - 标题
|
||||
"""
|
||||
try:
|
||||
pattern = r'<title>(.*?)</title>'
|
||||
match = re.search(pattern, content)
|
||||
if match:
|
||||
return match.group(1).strip()
|
||||
return ''
|
||||
except Exception as e:
|
||||
self._handle_exception(e, 'extract_title')
|
||||
return ''
|
||||
|
||||
def _extract_risk(self, content):
|
||||
"""
|
||||
提取风险信息
|
||||
|
||||
:param content: 页面内容
|
||||
:return: str - 风险信息
|
||||
"""
|
||||
try:
|
||||
# 这里需要根据实际页面结构调整正则表达式
|
||||
pattern = r'百度网址检测:<span[^>]+>(.*?)</span>'
|
||||
match = re.search(pattern, content)
|
||||
if match:
|
||||
return match.group(1).strip()
|
||||
return ''
|
||||
except Exception as e:
|
||||
self._handle_exception(e, 'extract_risk')
|
||||
return ''
|
||||
|
||||
def _check_sensitive(self, title, risk):
|
||||
"""
|
||||
检查是否包含敏感词
|
||||
|
||||
:param title: 标题
|
||||
:param risk: 风险信息
|
||||
:return: bool - 是否包含敏感词
|
||||
"""
|
||||
# 风险类型
|
||||
sensitive_risks = ['低风险', '疑似色情博彩风险', '严重影响权重']
|
||||
|
||||
# 敏感词
|
||||
sensitive_words = ['色情', '赌博', '博彩', '毒品', '暴力', '诈骗']
|
||||
|
||||
# 检查风险
|
||||
for r in sensitive_risks:
|
||||
if r in risk:
|
||||
return True
|
||||
|
||||
# 检查标题
|
||||
for word in sensitive_words:
|
||||
if word in title:
|
||||
return True
|
||||
|
||||
return False
|
||||
151
domainCheck/app/detectors/baidu_detector.py
Normal file
151
domainCheck/app/detectors/baidu_detector.py
Normal file
@@ -0,0 +1,151 @@
|
||||
# -*- coding: UTF-8 -*-
|
||||
'''
|
||||
@Project :domainScanDemo
|
||||
@File :baidu_detector.py
|
||||
@IDE :PyCharm
|
||||
@Author :梦伴
|
||||
@Date :2026/4/8 23:58
|
||||
@explain : 百度检测器
|
||||
'''
|
||||
|
||||
import requests
|
||||
from curl_cffi import requests as curl_requests
|
||||
import re
|
||||
from app.detectors.base import BaseDetector
|
||||
|
||||
|
||||
class BaiduDetector(BaseDetector):
|
||||
"""
|
||||
百度检测器
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
"""
|
||||
初始化百度检测器
|
||||
"""
|
||||
super().__init__()
|
||||
self.site_url = 'https://www.baidu.com/s'
|
||||
self.history_url = 'https://www.baidu.com/s'
|
||||
|
||||
def check_domain(self, domain):
|
||||
"""
|
||||
检测域名
|
||||
|
||||
:param domain: 域名
|
||||
:return: dict - 检测结果
|
||||
"""
|
||||
try:
|
||||
# 检查百度site
|
||||
site_result = self.check_site(domain)
|
||||
|
||||
# 检查百度历史
|
||||
history_result = self.check_history(domain)
|
||||
|
||||
return {
|
||||
'site': site_result,
|
||||
'history': history_result
|
||||
}
|
||||
except Exception as e:
|
||||
return self._handle_exception(e, domain)
|
||||
|
||||
def check_site(self, domain):
|
||||
"""
|
||||
检查百度site收录
|
||||
|
||||
:param domain: 域名
|
||||
:return: dict - 检测结果
|
||||
"""
|
||||
try:
|
||||
params = {
|
||||
'wd': f'site:{domain}',
|
||||
'rn': '50'
|
||||
}
|
||||
|
||||
headers = {
|
||||
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/146.0.0.0 Safari/537.36'
|
||||
}
|
||||
|
||||
# 使用curl_cffi模拟浏览器
|
||||
response = curl_requests.get(self.site_url, params=params, headers=headers, impersonate='chrome', timeout=10)
|
||||
|
||||
if response.status_code == 200:
|
||||
content = response.text
|
||||
|
||||
# 提取子域名
|
||||
subdomains = self._extract_subdomains(content, domain)
|
||||
|
||||
# 检查是否有收录
|
||||
has_收录 = '没有找到相关结果' not in content
|
||||
|
||||
return {
|
||||
'has_收录': has_收录,
|
||||
'subdomains': subdomains
|
||||
}
|
||||
else:
|
||||
self._log_warning(f"百度site查询失败: {response.status_code}")
|
||||
return {'has_收录': False, 'subdomains': []}
|
||||
except Exception as e:
|
||||
self._handle_exception(e, domain)
|
||||
return {'has_收录': False, 'subdomains': []}
|
||||
|
||||
def check_history(self, domain):
|
||||
"""
|
||||
检查百度历史收录
|
||||
|
||||
:param domain: 域名
|
||||
:return: dict - 检测结果
|
||||
"""
|
||||
try:
|
||||
params = {
|
||||
'wd': f'cache:{domain}',
|
||||
'rn': '50'
|
||||
}
|
||||
|
||||
headers = {
|
||||
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/146.0.0.0 Safari/537.36'
|
||||
}
|
||||
|
||||
# 使用curl_cffi模拟浏览器
|
||||
response = curl_requests.get(self.history_url, params=params, headers=headers, impersonate='chrome', timeout=10)
|
||||
|
||||
if response.status_code == 200:
|
||||
content = response.text
|
||||
|
||||
# 检查是否有历史收录
|
||||
has_history = '百度快照' in content
|
||||
|
||||
# 检查是否有灰色内容
|
||||
has_gray = '风险提示' in content or '安全警告' in content
|
||||
|
||||
return {
|
||||
'has_history': has_history,
|
||||
'has_gray': has_gray
|
||||
}
|
||||
else:
|
||||
self._log_warning(f"百度历史查询失败: {response.status_code}")
|
||||
return {'has_history': False, 'has_gray': False}
|
||||
except Exception as e:
|
||||
self._handle_exception(e, domain)
|
||||
return {'has_history': False, 'has_gray': False}
|
||||
|
||||
def _extract_subdomains(self, content, domain):
|
||||
"""
|
||||
提取子域名
|
||||
|
||||
:param content: 搜索结果内容
|
||||
:param domain: 主域名
|
||||
:return: list - 子域名列表
|
||||
"""
|
||||
try:
|
||||
# 提取所有包含域名的链接
|
||||
pattern = r'https?://([a-zA-Z0-9-]+)\.' + re.escape(domain)
|
||||
matches = re.findall(pattern, content)
|
||||
|
||||
# 去重并过滤空值
|
||||
subdomains = list(set(matches))
|
||||
subdomains = [sub for sub in subdomains if sub]
|
||||
|
||||
return subdomains
|
||||
except Exception as e:
|
||||
self._handle_exception(e, domain)
|
||||
return []
|
||||
70
domainCheck/app/detectors/base.py
Normal file
70
domainCheck/app/detectors/base.py
Normal file
@@ -0,0 +1,70 @@
|
||||
# -*- coding: UTF-8 -*-
|
||||
'''
|
||||
@Project :domainScanDemo
|
||||
@File :base.py
|
||||
@IDE :PyCharm
|
||||
@Author :梦伴
|
||||
@Date :2026/4/8 23:55
|
||||
@explain : 基础检测类
|
||||
'''
|
||||
|
||||
from abc import ABC, abstractmethod
|
||||
from loguru import logger
|
||||
|
||||
|
||||
class BaseDetector(ABC):
|
||||
"""
|
||||
基础检测类
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
"""
|
||||
初始化检测类
|
||||
"""
|
||||
self.name = self.__class__.__name__
|
||||
logger.info(f"初始化检测器: {self.name}")
|
||||
|
||||
@abstractmethod
|
||||
def check_domain(self, domain):
|
||||
"""
|
||||
检测域名
|
||||
|
||||
:param domain: 域名
|
||||
:return: dict - 检测结果
|
||||
"""
|
||||
pass
|
||||
|
||||
def _log_info(self, message):
|
||||
"""
|
||||
记录信息日志
|
||||
|
||||
:param message: 消息
|
||||
"""
|
||||
logger.info(f"[{self.name}] {message}")
|
||||
|
||||
def _log_warning(self, message):
|
||||
"""
|
||||
记录警告日志
|
||||
|
||||
:param message: 消息
|
||||
"""
|
||||
logger.warning(f"[{self.name}] {message}")
|
||||
|
||||
def _log_error(self, message):
|
||||
"""
|
||||
记录错误日志
|
||||
|
||||
:param message: 消息
|
||||
"""
|
||||
logger.error(f"[{self.name}] {message}")
|
||||
|
||||
def _handle_exception(self, e, domain):
|
||||
"""
|
||||
处理异常
|
||||
|
||||
:param e: 异常
|
||||
:param domain: 域名
|
||||
:return: dict - 错误结果
|
||||
"""
|
||||
self._log_error(f"检测域名 {domain} 出错: {e}")
|
||||
return {'error': str(e)}
|
||||
130
domainCheck/app/detectors/chinaz_detector.py
Normal file
130
domainCheck/app/detectors/chinaz_detector.py
Normal file
@@ -0,0 +1,130 @@
|
||||
# -*- coding: UTF-8 -*-
|
||||
'''
|
||||
@Project :domainScanDemo
|
||||
@File :chinaz_detector.py
|
||||
@IDE :PyCharm
|
||||
@Author :梦伴
|
||||
@Date :2026/4/9 0:01
|
||||
@explain : 站长之家检测器
|
||||
'''
|
||||
|
||||
import requests
|
||||
from curl_cffi import requests as curl_requests
|
||||
import re
|
||||
from app.detectors.base import BaseDetector
|
||||
|
||||
|
||||
class ChinazDetector(BaseDetector):
|
||||
"""
|
||||
站长之家检测器
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
"""
|
||||
初始化站长之家检测器
|
||||
"""
|
||||
super().__init__()
|
||||
self.url = 'https://seo.chinaz.com'
|
||||
self.query_url = 'https://seo.chinaz.com/{domain}'
|
||||
|
||||
def check_domain(self, domain):
|
||||
"""
|
||||
检测域名
|
||||
|
||||
:param domain: 域名
|
||||
:return: dict - 检测结果
|
||||
"""
|
||||
try:
|
||||
# 构建查询URL
|
||||
query_url = self.query_url.format(domain=domain)
|
||||
|
||||
headers = {
|
||||
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/146.0.0.0 Safari/537.36'
|
||||
}
|
||||
|
||||
# 使用curl_cffi模拟浏览器
|
||||
response = curl_requests.get(query_url, headers=headers, impersonate='chrome', timeout=10)
|
||||
|
||||
if response.status_code == 200:
|
||||
content = response.text
|
||||
|
||||
# 提取标题
|
||||
title = self._extract_title(content)
|
||||
|
||||
# 提取网站分类
|
||||
category = self._extract_category(content)
|
||||
|
||||
# 检查是否包含敏感词
|
||||
has_sensitive = self._check_sensitive(title, category)
|
||||
|
||||
return {
|
||||
'title': title,
|
||||
'category': category,
|
||||
'has_sensitive': has_sensitive
|
||||
}
|
||||
else:
|
||||
self._log_warning(f"站长之家查询失败: {response.status_code}")
|
||||
return {'title': '', 'category': '', 'has_sensitive': False}
|
||||
except Exception as e:
|
||||
return self._handle_exception(e, domain)
|
||||
|
||||
def _extract_title(self, content):
|
||||
"""
|
||||
提取标题
|
||||
|
||||
:param content: 页面内容
|
||||
:return: str - 标题
|
||||
"""
|
||||
try:
|
||||
pattern = r'<title>(.*?)</title>'
|
||||
match = re.search(pattern, content)
|
||||
if match:
|
||||
return match.group(1).strip()
|
||||
return ''
|
||||
except Exception as e:
|
||||
self._handle_exception(e, 'extract_title')
|
||||
return ''
|
||||
|
||||
def _extract_category(self, content):
|
||||
"""
|
||||
提取网站分类
|
||||
|
||||
:param content: 页面内容
|
||||
:return: str - 分类
|
||||
"""
|
||||
try:
|
||||
# 这里需要根据实际页面结构调整正则表达式
|
||||
pattern = r'网站分类:<a[^>]+>(.*?)</a>'
|
||||
match = re.search(pattern, content)
|
||||
if match:
|
||||
return match.group(1).strip()
|
||||
return ''
|
||||
except Exception as e:
|
||||
self._handle_exception(e, 'extract_category')
|
||||
return ''
|
||||
|
||||
def _check_sensitive(self, title, category):
|
||||
"""
|
||||
检查是否包含敏感词
|
||||
|
||||
:param title: 标题
|
||||
:param category: 分类
|
||||
:return: bool - 是否包含敏感词
|
||||
"""
|
||||
# 敏感分类
|
||||
sensitive_categories = ['视频电影', '体育运动', '常用查询']
|
||||
|
||||
# 敏感词
|
||||
sensitive_words = ['色情', '赌博', '博彩', '毒品', '暴力', '诈骗']
|
||||
|
||||
# 检查分类
|
||||
for cat in sensitive_categories:
|
||||
if cat in category:
|
||||
return True
|
||||
|
||||
# 检查标题
|
||||
for word in sensitive_words:
|
||||
if word in title:
|
||||
return True
|
||||
|
||||
return False
|
||||
80
domainCheck/app/detectors/google_detector.py
Normal file
80
domainCheck/app/detectors/google_detector.py
Normal file
@@ -0,0 +1,80 @@
|
||||
# -*- coding: UTF-8 -*-
|
||||
'''
|
||||
@Project :domainScanDemo
|
||||
@File :google_detector.py
|
||||
@IDE :PyCharm
|
||||
@Author :梦伴
|
||||
@Date :2026/4/9 0:00
|
||||
@explain : Google检测器
|
||||
'''
|
||||
|
||||
import requests
|
||||
from curl_cffi import requests as curl_requests
|
||||
import re
|
||||
from app.detectors.base import BaseDetector
|
||||
|
||||
|
||||
class GoogleDetector(BaseDetector):
|
||||
"""
|
||||
Google检测器
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
"""
|
||||
初始化Google检测器
|
||||
"""
|
||||
super().__init__()
|
||||
self.site_url = 'https://www.google.com/search'
|
||||
|
||||
def check_domain(self, domain):
|
||||
"""
|
||||
检测域名
|
||||
|
||||
:param domain: 域名
|
||||
:return: dict - 检测结果
|
||||
"""
|
||||
try:
|
||||
# 检查Google site
|
||||
site_result = self.check_site(domain)
|
||||
|
||||
return {
|
||||
'site': site_result
|
||||
}
|
||||
except Exception as e:
|
||||
return self._handle_exception(e, domain)
|
||||
|
||||
def check_site(self, domain):
|
||||
"""
|
||||
检查Google site收录
|
||||
|
||||
:param domain: 域名
|
||||
:return: dict - 检测结果
|
||||
"""
|
||||
try:
|
||||
params = {
|
||||
'q': f'site:{domain}',
|
||||
'num': '50'
|
||||
}
|
||||
|
||||
headers = {
|
||||
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/146.0.0.0 Safari/537.36'
|
||||
}
|
||||
|
||||
# 使用curl_cffi模拟浏览器
|
||||
response = curl_requests.get(self.site_url, params=params, headers=headers, impersonate='chrome', timeout=10)
|
||||
|
||||
if response.status_code == 200:
|
||||
content = response.text
|
||||
|
||||
# 检查是否有收录
|
||||
has_收录 = 'No results found for' not in content
|
||||
|
||||
return {
|
||||
'has_收录': has_收录
|
||||
}
|
||||
else:
|
||||
self._log_warning(f"Google site查询失败: {response.status_code}")
|
||||
return {'has_收录': False}
|
||||
except Exception as e:
|
||||
self._handle_exception(e, domain)
|
||||
return {'has_收录': False}
|
||||
228
domainCheck/app/detectors/jucha_detector.py
Normal file
228
domainCheck/app/detectors/jucha_detector.py
Normal file
@@ -0,0 +1,228 @@
|
||||
# -*- coding: UTF-8 -*-
|
||||
'''
|
||||
@Project :domainScanDemo
|
||||
@File :jucha_detector.py
|
||||
@IDE :PyCharm
|
||||
@Author :梦伴
|
||||
@Date :2026/4/9 0:04
|
||||
@explain : 聚查检测器
|
||||
'''
|
||||
|
||||
import requests
|
||||
from curl_cffi import requests as curl_requests
|
||||
import re
|
||||
from app.detectors.base import BaseDetector
|
||||
|
||||
|
||||
class JuchaDetector(BaseDetector):
|
||||
"""
|
||||
聚查检测器
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
"""
|
||||
初始化聚查检测器
|
||||
"""
|
||||
super().__init__()
|
||||
self.url = 'https://www.jucha.com'
|
||||
self.whois_url = 'https://www.jucha.com/whois/{domain}'
|
||||
self.beian_url = 'https://www.jucha.com/beian/{domain}'
|
||||
self.intercept_url = 'https://www.jucha.com/intercept/{domain}'
|
||||
|
||||
def check_domain(self, domain):
|
||||
"""
|
||||
检测域名
|
||||
|
||||
:param domain: 域名
|
||||
:return: dict - 检测结果
|
||||
"""
|
||||
try:
|
||||
# 检查WHOIS
|
||||
whois_result = self.check_whois(domain)
|
||||
|
||||
# 检查备案
|
||||
beian_result = self.check_beian(domain)
|
||||
|
||||
# 检查拦截
|
||||
intercept_result = self.check_intercept(domain)
|
||||
|
||||
return {
|
||||
'whois': whois_result,
|
||||
'beian': beian_result,
|
||||
'intercept': intercept_result
|
||||
}
|
||||
except Exception as e:
|
||||
return self._handle_exception(e, domain)
|
||||
|
||||
def check_whois(self, domain):
|
||||
"""
|
||||
检查WHOIS
|
||||
|
||||
:param domain: 域名
|
||||
:return: dict - 检测结果
|
||||
"""
|
||||
try:
|
||||
# 构建查询URL
|
||||
query_url = self.whois_url.format(domain=domain)
|
||||
|
||||
headers = {
|
||||
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/146.0.0.0 Safari/537.36'
|
||||
}
|
||||
|
||||
# 使用curl_cffi模拟浏览器
|
||||
response = curl_requests.get(query_url, headers=headers, impersonate='chrome', timeout=10)
|
||||
|
||||
if response.status_code == 200:
|
||||
content = response.text
|
||||
|
||||
# 提取WHOIS信息
|
||||
whois_info = self._extract_whois_info(content)
|
||||
|
||||
return whois_info
|
||||
else:
|
||||
self._log_warning(f"聚查WHOIS查询失败: {response.status_code}")
|
||||
return {'status': ''}
|
||||
except Exception as e:
|
||||
self._handle_exception(e, domain)
|
||||
return {'status': ''}
|
||||
|
||||
def check_beian(self, domain):
|
||||
"""
|
||||
检查备案
|
||||
|
||||
:param domain: 域名
|
||||
:return: dict - 检测结果
|
||||
"""
|
||||
try:
|
||||
# 构建查询URL
|
||||
query_url = self.beian_url.format(domain=domain)
|
||||
|
||||
headers = {
|
||||
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/146.0.0.0 Safari/537.36'
|
||||
}
|
||||
|
||||
# 使用curl_cffi模拟浏览器
|
||||
response = curl_requests.get(query_url, headers=headers, impersonate='chrome', timeout=10)
|
||||
|
||||
if response.status_code == 200:
|
||||
content = response.text
|
||||
|
||||
# 提取备案信息
|
||||
beian_info = self._extract_beian_info(content)
|
||||
|
||||
return beian_info
|
||||
else:
|
||||
self._log_warning(f"聚查备案查询失败: {response.status_code}")
|
||||
return {'has_beian': False, 'beian_year': '', 'is_enterprise': False, 'beian_match': False}
|
||||
except Exception as e:
|
||||
self._handle_exception(e, domain)
|
||||
return {'has_beian': False, 'beian_year': '', 'is_enterprise': False, 'beian_match': False}
|
||||
|
||||
def check_intercept(self, domain):
|
||||
"""
|
||||
检查拦截
|
||||
|
||||
:param domain: 域名
|
||||
:return: dict - 检测结果
|
||||
"""
|
||||
try:
|
||||
# 构建查询URL
|
||||
query_url = self.intercept_url.format(domain=domain)
|
||||
|
||||
headers = {
|
||||
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/146.0.0.0 Safari/537.36'
|
||||
}
|
||||
|
||||
# 使用curl_cffi模拟浏览器
|
||||
response = curl_requests.get(query_url, headers=headers, impersonate='chrome', timeout=10)
|
||||
|
||||
if response.status_code == 200:
|
||||
content = response.text
|
||||
|
||||
# 检查是否被拦截
|
||||
is_normal = self._check_intercept_status(content)
|
||||
|
||||
return {
|
||||
'normal': is_normal
|
||||
}
|
||||
else:
|
||||
self._log_warning(f"聚查拦截查询失败: {response.status_code}")
|
||||
return {'normal': False}
|
||||
except Exception as e:
|
||||
self._handle_exception(e, domain)
|
||||
return {'normal': False}
|
||||
|
||||
def _extract_whois_info(self, content):
|
||||
"""
|
||||
提取WHOIS信息
|
||||
|
||||
:param content: 页面内容
|
||||
:return: dict - WHOIS信息
|
||||
"""
|
||||
try:
|
||||
# 提取状态信息
|
||||
pattern = r'域名状态:<span[^>]+>(.*?)</span>'
|
||||
match = re.search(pattern, content)
|
||||
status = match.group(1).strip() if match else ''
|
||||
|
||||
return {
|
||||
'status': status
|
||||
}
|
||||
except Exception as e:
|
||||
self._handle_exception(e, 'extract_whois_info')
|
||||
return {'status': ''}
|
||||
|
||||
def _extract_beian_info(self, content):
|
||||
"""
|
||||
提取备案信息
|
||||
|
||||
:param content: 页面内容
|
||||
:return: dict - 备案信息
|
||||
"""
|
||||
try:
|
||||
# 检查是否有备案
|
||||
has_beian = '备案信息' in content
|
||||
|
||||
# 提取备案年份
|
||||
beian_year = ''
|
||||
pattern = r'审核时间:(\d{4})-\d{2}-\d{2}'
|
||||
match = re.search(pattern, content)
|
||||
if match:
|
||||
beian_year = match.group(1)
|
||||
|
||||
# 提取单位性质
|
||||
is_enterprise = '企业' in content
|
||||
|
||||
# 检查首网址和备案网址是否一致
|
||||
beian_match = '网站首页网址' in content
|
||||
|
||||
return {
|
||||
'has_beian': has_beian,
|
||||
'beian_year': beian_year,
|
||||
'is_enterprise': is_enterprise,
|
||||
'beian_match': beian_match
|
||||
}
|
||||
except Exception as e:
|
||||
self._handle_exception(e, 'extract_beian_info')
|
||||
return {'has_beian': False, 'beian_year': '', 'is_enterprise': False, 'beian_match': False}
|
||||
|
||||
def _check_intercept_status(self, content):
|
||||
"""
|
||||
检查拦截状态
|
||||
|
||||
:param content: 页面内容
|
||||
:return: bool - 是否正常
|
||||
"""
|
||||
try:
|
||||
# 检查是否包含正常标识
|
||||
if '正常' in content:
|
||||
return True
|
||||
|
||||
# 检查是否包含拦截标识
|
||||
if '拦截' in content:
|
||||
return False
|
||||
|
||||
return False
|
||||
except Exception as e:
|
||||
self._handle_exception(e, 'check_intercept_status')
|
||||
return False
|
||||
214
domainCheck/app/detectors/juziseo_detector.py
Normal file
214
domainCheck/app/detectors/juziseo_detector.py
Normal file
@@ -0,0 +1,214 @@
|
||||
# -*- coding: UTF-8 -*-
|
||||
'''
|
||||
@Project :domainScanDemo
|
||||
@File :juziseo_detector.py
|
||||
@IDE :PyCharm
|
||||
@Author :梦伴
|
||||
@Date :2026/4/9 0:03
|
||||
@explain : 桔子SEO检测器
|
||||
'''
|
||||
|
||||
import requests
|
||||
from curl_cffi import requests as curl_requests
|
||||
import re
|
||||
from app.detectors.base import BaseDetector
|
||||
|
||||
|
||||
class JuziseoDetector(BaseDetector):
|
||||
"""
|
||||
桔子SEO检测器
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
"""
|
||||
初始化桔子SEO检测器
|
||||
"""
|
||||
super().__init__()
|
||||
self.url = 'https://seo.juziseo.com'
|
||||
self.history_url = 'https://seo.juziseo.com/history/{domain}'
|
||||
self.backlink_url = 'https://seo.juziseo.com/backlink/{domain}'
|
||||
|
||||
def check_domain(self, domain):
|
||||
"""
|
||||
检测域名
|
||||
|
||||
:param domain: 域名
|
||||
:return: dict - 检测结果
|
||||
"""
|
||||
try:
|
||||
# 检查历史信息
|
||||
history_result = self.check_history(domain)
|
||||
|
||||
# 检查外链
|
||||
backlink_result = self.check_backlink(domain)
|
||||
|
||||
return {
|
||||
'history': history_result,
|
||||
'backlink': backlink_result
|
||||
}
|
||||
except Exception as e:
|
||||
return self._handle_exception(e, domain)
|
||||
|
||||
def check_history(self, domain):
|
||||
"""
|
||||
检查历史信息
|
||||
|
||||
:param domain: 域名
|
||||
:return: dict - 检测结果
|
||||
"""
|
||||
try:
|
||||
# 构建查询URL
|
||||
query_url = self.history_url.format(domain=domain)
|
||||
|
||||
headers = {
|
||||
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/146.0.0.0 Safari/537.36'
|
||||
}
|
||||
|
||||
# 使用curl_cffi模拟浏览器
|
||||
response = curl_requests.get(query_url, headers=headers, impersonate='chrome', timeout=10)
|
||||
|
||||
if response.status_code == 200:
|
||||
content = response.text
|
||||
|
||||
# 提取历史信息
|
||||
history_info = self._extract_history_info(content)
|
||||
|
||||
# 检查是否包含敏感词
|
||||
has_sensitive = self._check_sensitive(history_info)
|
||||
|
||||
# 检查是否有百度历史收录
|
||||
has_baidu_history = '百度历史收录' in content
|
||||
|
||||
# 检查是否有子域名
|
||||
has_subdomains = '子域名' in content
|
||||
|
||||
# 检查是否为简体中文
|
||||
is_simplified = self._check_simplified(content)
|
||||
|
||||
return {
|
||||
'has_sensitive': has_sensitive,
|
||||
'has_baidu_history': has_baidu_history,
|
||||
'has_subdomains': has_subdomains,
|
||||
'is_simplified': is_simplified
|
||||
}
|
||||
else:
|
||||
self._log_warning(f"桔子SEO历史查询失败: {response.status_code}")
|
||||
return {'has_sensitive': False, 'has_baidu_history': False, 'has_subdomains': False, 'is_simplified': True}
|
||||
except Exception as e:
|
||||
self._handle_exception(e, domain)
|
||||
return {'has_sensitive': False, 'has_baidu_history': False, 'has_subdomains': False, 'is_simplified': True}
|
||||
|
||||
def check_backlink(self, domain):
|
||||
"""
|
||||
检查外链
|
||||
|
||||
:param domain: 域名
|
||||
:return: dict - 检测结果
|
||||
"""
|
||||
try:
|
||||
# 构建查询URL
|
||||
query_url = self.backlink_url.format(domain=domain)
|
||||
|
||||
headers = {
|
||||
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/146.0.0.0 Safari/537.36'
|
||||
}
|
||||
|
||||
# 使用curl_cffi模拟浏览器
|
||||
response = curl_requests.get(query_url, headers=headers, impersonate='chrome', timeout=10)
|
||||
|
||||
if response.status_code == 200:
|
||||
content = response.text
|
||||
|
||||
# 检查是否包含敏感词
|
||||
has_sensitive = self._check_backlink_sensitive(content)
|
||||
|
||||
# 检查是否有子域名
|
||||
has_subdomains = '子域名' in content
|
||||
|
||||
return {
|
||||
'has_sensitive': has_sensitive,
|
||||
'has_subdomains': has_subdomains
|
||||
}
|
||||
else:
|
||||
self._log_warning(f"桔子SEO外链查询失败: {response.status_code}")
|
||||
return {'has_sensitive': False, 'has_subdomains': False}
|
||||
except Exception as e:
|
||||
self._handle_exception(e, domain)
|
||||
return {'has_sensitive': False, 'has_subdomains': False}
|
||||
|
||||
def _extract_history_info(self, content):
|
||||
"""
|
||||
提取历史信息
|
||||
|
||||
:param content: 页面内容
|
||||
:return: str - 历史信息
|
||||
"""
|
||||
try:
|
||||
# 这里需要根据实际页面结构调整正则表达式
|
||||
pattern = r'<div class="history-info">(.*?)</div>'
|
||||
match = re.search(pattern, content, re.DOTALL)
|
||||
if match:
|
||||
return match.group(1).strip()
|
||||
return ''
|
||||
except Exception as e:
|
||||
self._handle_exception(e, 'extract_history_info')
|
||||
return ''
|
||||
|
||||
def _check_sensitive(self, history_info):
|
||||
"""
|
||||
检查是否包含敏感词
|
||||
|
||||
:param history_info: 历史信息
|
||||
:return: bool - 是否包含敏感词
|
||||
"""
|
||||
# 敏感词
|
||||
sensitive_words = [
|
||||
'色情', '赌博', '博彩', '毒品', '暴力', '诈骗',
|
||||
'足球', '直播', '证券', '配资', '软件',
|
||||
'体育', '商行', '下载', '影视', '网络',
|
||||
'计算', 'app', 'HTML SiteMap', '模拟器', '传媒',
|
||||
'二次元', '成人', '米乐', '小说', '凯发',
|
||||
'人才', '华体', '娱乐', '开户'
|
||||
]
|
||||
|
||||
for word in sensitive_words:
|
||||
if word in history_info:
|
||||
return True
|
||||
|
||||
return False
|
||||
|
||||
def _check_simplified(self, content):
|
||||
"""
|
||||
检查是否为简体中文
|
||||
|
||||
:param content: 页面内容
|
||||
:return: bool - 是否为简体中文
|
||||
"""
|
||||
# 检查是否包含简体中文标识
|
||||
if '简体中文' in content:
|
||||
return True
|
||||
|
||||
# 检查是否包含繁体中文标识
|
||||
if '繁体中文' in content:
|
||||
return False
|
||||
|
||||
return True
|
||||
|
||||
def _check_backlink_sensitive(self, content):
|
||||
"""
|
||||
检查外链是否包含敏感词
|
||||
|
||||
:param content: 页面内容
|
||||
:return: bool - 是否包含敏感词
|
||||
"""
|
||||
# 敏感词
|
||||
sensitive_words = [
|
||||
'内幕', '猛料', '精料', '高手', '绝杀',
|
||||
'权威', '澳门', '色情', '赌博', '博彩'
|
||||
]
|
||||
|
||||
for word in sensitive_words:
|
||||
if word in content:
|
||||
return True
|
||||
|
||||
return False
|
||||
107
domainCheck/app/detectors/qihu360_detector.py
Normal file
107
domainCheck/app/detectors/qihu360_detector.py
Normal file
@@ -0,0 +1,107 @@
|
||||
# -*- coding: UTF-8 -*-
|
||||
'''
|
||||
@Project :domainScanDemo
|
||||
@File :qihu360_detector.py
|
||||
@IDE :PyCharm
|
||||
@Author :梦伴
|
||||
@Date :2026/4/8 23:59
|
||||
@explain : 360检测器
|
||||
'''
|
||||
|
||||
import requests
|
||||
from curl_cffi import requests as curl_requests
|
||||
import re
|
||||
from app.detectors.base import BaseDetector
|
||||
|
||||
|
||||
class Qihu360Detector(BaseDetector):
|
||||
"""
|
||||
360检测器
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
"""
|
||||
初始化360检测器
|
||||
"""
|
||||
super().__init__()
|
||||
self.site_url = 'https://www.so.com/s'
|
||||
|
||||
def check_domain(self, domain):
|
||||
"""
|
||||
检测域名
|
||||
|
||||
:param domain: 域名
|
||||
:return: dict - 检测结果
|
||||
"""
|
||||
try:
|
||||
# 检查360 site
|
||||
site_result = self.check_site(domain)
|
||||
|
||||
return {
|
||||
'site': site_result
|
||||
}
|
||||
except Exception as e:
|
||||
return self._handle_exception(e, domain)
|
||||
|
||||
def check_site(self, domain):
|
||||
"""
|
||||
检查360 site收录
|
||||
|
||||
:param domain: 域名
|
||||
:return: dict - 检测结果
|
||||
"""
|
||||
try:
|
||||
params = {
|
||||
'q': f'site:{domain}',
|
||||
'pn': '1',
|
||||
'rn': '50'
|
||||
}
|
||||
|
||||
headers = {
|
||||
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/146.0.0.0 Safari/537.36'
|
||||
}
|
||||
|
||||
# 使用curl_cffi模拟浏览器
|
||||
response = curl_requests.get(self.site_url, params=params, headers=headers, impersonate='chrome', timeout=10)
|
||||
|
||||
if response.status_code == 200:
|
||||
content = response.text
|
||||
|
||||
# 提取子域名
|
||||
subdomains = self._extract_subdomains(content, domain)
|
||||
|
||||
# 检查是否有收录
|
||||
has_收录 = '没有找到相关结果' not in content
|
||||
|
||||
return {
|
||||
'has_收录': has_收录,
|
||||
'subdomains': subdomains
|
||||
}
|
||||
else:
|
||||
self._log_warning(f"360 site查询失败: {response.status_code}")
|
||||
return {'has_收录': False, 'subdomains': []}
|
||||
except Exception as e:
|
||||
self._handle_exception(e, domain)
|
||||
return {'has_收录': False, 'subdomains': []}
|
||||
|
||||
def _extract_subdomains(self, content, domain):
|
||||
"""
|
||||
提取子域名
|
||||
|
||||
:param content: 搜索结果内容
|
||||
:param domain: 主域名
|
||||
:return: list - 子域名列表
|
||||
"""
|
||||
try:
|
||||
# 提取所有包含域名的链接
|
||||
pattern = r'https?://([a-zA-Z0-9-]+)\.' + re.escape(domain)
|
||||
matches = re.findall(pattern, content)
|
||||
|
||||
# 去重并过滤空值
|
||||
subdomains = list(set(matches))
|
||||
subdomains = [sub for sub in subdomains if sub]
|
||||
|
||||
return subdomains
|
||||
except Exception as e:
|
||||
self._handle_exception(e, domain)
|
||||
return []
|
||||
128
domainCheck/app/detectors/rdap_detector.py
Normal file
128
domainCheck/app/detectors/rdap_detector.py
Normal file
@@ -0,0 +1,128 @@
|
||||
# -*- coding: UTF-8 -*-
|
||||
'''
|
||||
@Project :domainScanDemo
|
||||
@File :rdap_detector.py
|
||||
@IDE :PyCharm
|
||||
@Author :梦伴
|
||||
@Date :2026/4/8 23:56
|
||||
@explain : RDAP检测器
|
||||
'''
|
||||
|
||||
import requests
|
||||
from app.detectors.base import BaseDetector
|
||||
|
||||
|
||||
class RDAPDetector(BaseDetector):
|
||||
"""
|
||||
RDAP检测器
|
||||
"""
|
||||
|
||||
def __init__(self):
|
||||
"""
|
||||
初始化RDAP检测器
|
||||
"""
|
||||
super().__init__()
|
||||
self.rdap_urls = {
|
||||
'com': 'https://rdap.verisign.com/com/v1/domain/',
|
||||
'net': 'https://rdap.verisign.com/net/v1/domain/'
|
||||
}
|
||||
|
||||
def check_domain(self, domain):
|
||||
"""
|
||||
检测域名
|
||||
|
||||
:param domain: 域名
|
||||
:return: dict - 检测结果
|
||||
"""
|
||||
try:
|
||||
# 提取顶级域名
|
||||
tld = domain.split('.')[-1]
|
||||
if tld not in self.rdap_urls:
|
||||
return {'error': '不支持的顶级域名'}
|
||||
|
||||
# 构建RDAP查询URL
|
||||
url = self.rdap_urls[tld] + domain
|
||||
|
||||
# 发送请求
|
||||
response = requests.get(url, timeout=10)
|
||||
|
||||
if response.status_code == 200:
|
||||
data = response.json()
|
||||
return self._parse_rdap_response(data)
|
||||
elif response.status_code == 404:
|
||||
return {'status': 'available'}
|
||||
else:
|
||||
return {'error': f'RDAP查询失败: {response.status_code}'}
|
||||
except Exception as e:
|
||||
return self._handle_exception(e, domain)
|
||||
|
||||
def check_register_status(self, domain):
|
||||
"""
|
||||
检查注册状态
|
||||
|
||||
:param domain: 域名
|
||||
:return: int - 注册状态码
|
||||
"""
|
||||
try:
|
||||
result = self.check_domain(domain)
|
||||
|
||||
if 'error' in result:
|
||||
return 9 # 状态未知
|
||||
|
||||
if result.get('status') == 'available':
|
||||
return 2 # 可注册
|
||||
|
||||
# 检查域名状态
|
||||
statuses = result.get('status', [])
|
||||
if 'clientHold' in statuses:
|
||||
return 7 # clientHold
|
||||
elif 'serverHold' in statuses:
|
||||
return 8 # serverHold
|
||||
elif 'autoRenewPeriod' in statuses:
|
||||
return 4 # 宽限期
|
||||
elif 'redemptionPeriod' in statuses:
|
||||
return 5 # 赎回期
|
||||
elif 'pendingDelete' in statuses:
|
||||
return 6 # 删除期
|
||||
else:
|
||||
return 3 # 已注册
|
||||
except Exception as e:
|
||||
self._handle_exception(e, domain)
|
||||
return 10 # 检测失败
|
||||
|
||||
def _parse_rdap_response(self, data):
|
||||
"""
|
||||
解析RDAP响应
|
||||
|
||||
:param data: RDAP响应数据
|
||||
:return: dict - 解析结果
|
||||
"""
|
||||
result = {
|
||||
'status': 'registered',
|
||||
'domain': data.get('ldhName'),
|
||||
'statuses': data.get('status', []),
|
||||
'registrar': None,
|
||||
'creation_date': None,
|
||||
'expiration_date': None,
|
||||
'last_update': None
|
||||
}
|
||||
|
||||
# 解析注册商信息
|
||||
for entity in data.get('entities', []):
|
||||
if 'registrar' in entity.get('roles', []):
|
||||
result['registrar'] = entity.get('vcardArray', [[], []])[1][1][3]
|
||||
break
|
||||
|
||||
# 解析时间信息
|
||||
for event in data.get('events', []):
|
||||
event_action = event.get('eventAction')
|
||||
event_date = event.get('eventDate')
|
||||
|
||||
if event_action == 'registration':
|
||||
result['creation_date'] = event_date
|
||||
elif event_action == 'expiration':
|
||||
result['expiration_date'] = event_date
|
||||
elif event_action == 'last update':
|
||||
result['last_update'] = event_date
|
||||
|
||||
return result
|
||||
562
domainCheck/app/detectors/wayback_detector.py
Normal file
562
domainCheck/app/detectors/wayback_detector.py
Normal file
@@ -0,0 +1,562 @@
|
||||
# -*- coding: UTF-8 -*-
|
||||
'''
|
||||
@Project :domainScanDemo
|
||||
@File :wayback_detector.py
|
||||
@IDE :PyCharm
|
||||
@Author :梦伴
|
||||
@Date :2026/4/8 23:57
|
||||
@explain : Wayback检测器
|
||||
'''
|
||||
|
||||
import html
|
||||
import json
|
||||
import re
|
||||
import threading
|
||||
import time
|
||||
import zlib
|
||||
from base64 import b64decode, b64encode
|
||||
from concurrent.futures import FIRST_COMPLETED, ThreadPoolExecutor, wait
|
||||
|
||||
import redis
|
||||
import requests
|
||||
from requests.adapters import HTTPAdapter
|
||||
from urllib3.util.retry import Retry
|
||||
|
||||
from app.config import config
|
||||
from app.detectors.base import BaseDetector
|
||||
|
||||
|
||||
class WaybackDetector(BaseDetector):
|
||||
"""
|
||||
Wayback检测器
|
||||
"""
|
||||
|
||||
TITLE_PATTERN = re.compile(r'<title[^>]*>(.*?)</title>', re.IGNORECASE | re.DOTALL)
|
||||
|
||||
def __init__(self):
|
||||
"""
|
||||
初始化Wayback检测器
|
||||
"""
|
||||
super().__init__()
|
||||
self.cdx_api_url = 'https://web.archive.org/cdx/search/cdx'
|
||||
# 使用 id_ 直接取快照内容,避免回放页面额外壳层干扰正文匹配。
|
||||
self.snapshot_url = 'https://web.archive.org/web/{timestamp}id_/{domain}'
|
||||
self._timestamp_cache = {}
|
||||
self._title_cache = {}
|
||||
self._cache_lock = threading.Lock()
|
||||
self.session = self._build_session()
|
||||
self.redis_client = self._build_redis_client()
|
||||
|
||||
def _build_session(self):
|
||||
session = requests.Session()
|
||||
retry = Retry(
|
||||
total=max(0, config.WAYBACK_RETRY_COUNT),
|
||||
backoff_factor=0.5,
|
||||
status_forcelist=(429, 500, 502, 503, 504),
|
||||
allowed_methods=frozenset(["GET"]),
|
||||
raise_on_status=False,
|
||||
)
|
||||
adapter = HTTPAdapter(max_retries=retry, pool_connections=10, pool_maxsize=10)
|
||||
session.mount('http://', adapter)
|
||||
session.mount('https://', adapter)
|
||||
session.headers.update({
|
||||
'User-Agent': config.WAYBACK_USER_AGENT,
|
||||
'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8',
|
||||
})
|
||||
return session
|
||||
|
||||
def _build_redis_client(self):
|
||||
try:
|
||||
client = redis.Redis(
|
||||
host=config.REDIS_HOST,
|
||||
port=config.REDIS_PORT,
|
||||
password=config.REDIS_PASSWORD,
|
||||
db=config.REDIS_DB,
|
||||
decode_responses=True,
|
||||
socket_connect_timeout=5,
|
||||
socket_timeout=5,
|
||||
)
|
||||
client.ping()
|
||||
return client
|
||||
except Exception:
|
||||
return None
|
||||
|
||||
def _title_cache_key(self, domain, timestamp):
|
||||
return f"domain_tool:wayback_title:{domain}:{timestamp}"
|
||||
|
||||
def _timestamp_cache_key(self, domain):
|
||||
return f"domain_tool:wayback_timestamps:{domain}"
|
||||
|
||||
def _record_cache_key(self, domain):
|
||||
return f"domain_tool:wayback_records:{domain}"
|
||||
|
||||
def _normalize_title(self, title):
|
||||
normalized = html.unescape(title or '')
|
||||
normalized = re.sub(r'\s+', ' ', normalized, flags=re.DOTALL).strip().lower()
|
||||
return normalized
|
||||
|
||||
def _extract_title(self, content):
|
||||
match = self.TITLE_PATTERN.search(content or '')
|
||||
if not match:
|
||||
return ''
|
||||
return html.unescape(match.group(1)).strip()
|
||||
|
||||
def _load_cached_title(self, domain, timestamp):
|
||||
cache_key = self._title_cache_key(domain, timestamp)
|
||||
with self._cache_lock:
|
||||
if cache_key in self._title_cache:
|
||||
return self._title_cache[cache_key]
|
||||
if self.redis_client:
|
||||
try:
|
||||
raw_value = self.redis_client.get(cache_key)
|
||||
if raw_value:
|
||||
data = json.loads(raw_value)
|
||||
with self._cache_lock:
|
||||
self._title_cache[cache_key] = data
|
||||
return data
|
||||
except Exception:
|
||||
pass
|
||||
return None
|
||||
|
||||
def _save_cached_title(self, domain, timestamp, data):
|
||||
cache_key = self._title_cache_key(domain, timestamp)
|
||||
with self._cache_lock:
|
||||
self._title_cache[cache_key] = data
|
||||
if self.redis_client:
|
||||
try:
|
||||
self.redis_client.set(
|
||||
cache_key,
|
||||
json.dumps(data, ensure_ascii=False),
|
||||
ex=max(0, config.WAYBACK_TITLE_CACHE_TTL) or None,
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
def _load_cached_timestamps(self, domain):
|
||||
if domain in self._timestamp_cache:
|
||||
return list(self._timestamp_cache[domain])
|
||||
if not self.redis_client:
|
||||
return None
|
||||
try:
|
||||
raw_value = self.redis_client.get(self._timestamp_cache_key(domain))
|
||||
if not raw_value:
|
||||
return None
|
||||
compressed = b64decode(raw_value.encode('ascii'))
|
||||
timestamps = json.loads(zlib.decompress(compressed).decode('utf-8'))
|
||||
if isinstance(timestamps, list):
|
||||
self._timestamp_cache[domain] = tuple(timestamps)
|
||||
return list(timestamps)
|
||||
except Exception:
|
||||
return None
|
||||
return None
|
||||
|
||||
def _save_cached_timestamps(self, domain, timestamps):
|
||||
self._timestamp_cache[domain] = tuple(timestamps)
|
||||
if not self.redis_client:
|
||||
return
|
||||
try:
|
||||
payload = json.dumps(timestamps, separators=(',', ':')).encode('utf-8')
|
||||
compressed = zlib.compress(payload, level=6)
|
||||
self.redis_client.set(
|
||||
self._timestamp_cache_key(domain),
|
||||
b64encode(compressed).decode('ascii'),
|
||||
ex=max(0, config.WAYBACK_TIMESTAMP_CACHE_TTL) or None,
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
def _load_cached_records(self, domain):
|
||||
cache_key = self._record_cache_key(domain)
|
||||
with self._cache_lock:
|
||||
if cache_key in self._timestamp_cache:
|
||||
return [
|
||||
{'timestamp': item[0], 'digest': item[1]}
|
||||
for item in self._timestamp_cache[cache_key]
|
||||
]
|
||||
if not self.redis_client:
|
||||
return None
|
||||
try:
|
||||
raw_value = self.redis_client.get(cache_key)
|
||||
if not raw_value:
|
||||
return None
|
||||
compressed = b64decode(raw_value.encode('ascii'))
|
||||
records = json.loads(zlib.decompress(compressed).decode('utf-8'))
|
||||
if isinstance(records, list):
|
||||
normalized = tuple(
|
||||
(item.get('timestamp', ''), item.get('digest', ''))
|
||||
for item in records if isinstance(item, dict)
|
||||
)
|
||||
with self._cache_lock:
|
||||
self._timestamp_cache[cache_key] = normalized
|
||||
return [
|
||||
{'timestamp': item[0], 'digest': item[1]}
|
||||
for item in normalized if item[0]
|
||||
]
|
||||
except Exception:
|
||||
return None
|
||||
return None
|
||||
|
||||
def _save_cached_records(self, domain, records):
|
||||
cache_key = self._record_cache_key(domain)
|
||||
normalized = tuple(
|
||||
(item.get('timestamp', ''), item.get('digest', ''))
|
||||
for item in (records or [])
|
||||
if item and item.get('timestamp')
|
||||
)
|
||||
with self._cache_lock:
|
||||
self._timestamp_cache[cache_key] = normalized
|
||||
if not self.redis_client:
|
||||
return
|
||||
try:
|
||||
payload = json.dumps(
|
||||
[{'timestamp': item[0], 'digest': item[1]} for item in normalized],
|
||||
separators=(',', ':')
|
||||
).encode('utf-8')
|
||||
compressed = zlib.compress(payload, level=6)
|
||||
self.redis_client.set(
|
||||
cache_key,
|
||||
b64encode(compressed).decode('ascii'),
|
||||
ex=max(0, config.WAYBACK_TIMESTAMP_CACHE_TTL) or None,
|
||||
)
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
def _fetch_cdx_records(self, domain, limit=None, fast_latest=False):
|
||||
response = None
|
||||
try:
|
||||
params = {
|
||||
'url': domain,
|
||||
'output': 'txt',
|
||||
'fl': 'timestamp,digest',
|
||||
'filter': ['statuscode:200', 'mimetype:text/html'],
|
||||
}
|
||||
if limit is not None:
|
||||
params['limit'] = str(limit)
|
||||
if fast_latest:
|
||||
params['fastLatest'] = 'true'
|
||||
response = self.session.get(
|
||||
self.cdx_api_url,
|
||||
params=params,
|
||||
timeout=config.WAYBACK_CDX_TIMEOUT,
|
||||
stream=True,
|
||||
)
|
||||
if response.status_code != 200:
|
||||
self._log_warning(f"获取快照记录失败: {response.status_code}")
|
||||
return []
|
||||
records = []
|
||||
seen = set()
|
||||
for raw_line in response.iter_lines(decode_unicode=True):
|
||||
line = (raw_line or '').strip()
|
||||
if not line:
|
||||
continue
|
||||
parts = line.split()
|
||||
timestamp = parts[0].strip() if parts else ''
|
||||
digest = parts[1].strip() if len(parts) > 1 else ''
|
||||
if not timestamp or timestamp in seen:
|
||||
continue
|
||||
seen.add(timestamp)
|
||||
records.append({'timestamp': timestamp, 'digest': digest})
|
||||
return records
|
||||
except Exception as e:
|
||||
self._handle_exception(e, domain)
|
||||
return []
|
||||
finally:
|
||||
try:
|
||||
if response is not None:
|
||||
response.close()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
def get_latest_snapshot_record(self, domain):
|
||||
records = self._fetch_cdx_records(domain, limit=-1, fast_latest=True)
|
||||
return records[0] if records else None
|
||||
|
||||
def get_snapshot_records(self, domain):
|
||||
cached_records = self._load_cached_records(domain)
|
||||
if cached_records is not None:
|
||||
return cached_records
|
||||
records = self._fetch_cdx_records(domain)
|
||||
if records:
|
||||
self._save_cached_records(domain, records)
|
||||
self._save_cached_timestamps(domain, [item['timestamp'] for item in records])
|
||||
return records
|
||||
|
||||
def check_domain(self, domain):
|
||||
"""
|
||||
检测域名
|
||||
|
||||
:param domain: 域名
|
||||
:return: dict - 检测结果
|
||||
"""
|
||||
try:
|
||||
return self.scan_snapshots(domain)
|
||||
except Exception as e:
|
||||
return self._handle_exception(e, domain)
|
||||
|
||||
def get_snapshot_timestamps(self, domain):
|
||||
cached_records = self._load_cached_records(domain)
|
||||
if cached_records is not None:
|
||||
return [item['timestamp'] for item in cached_records if item.get('timestamp')]
|
||||
cached_timestamps = self._load_cached_timestamps(domain)
|
||||
if cached_timestamps is not None:
|
||||
return cached_timestamps
|
||||
records = self.get_snapshot_records(domain)
|
||||
return [item['timestamp'] for item in records if item.get('timestamp')]
|
||||
|
||||
def _fetch_snapshot_title(self, domain, timestamp):
|
||||
cached = self._load_cached_title(domain, timestamp)
|
||||
if cached is not None:
|
||||
return cached
|
||||
|
||||
snapshot_url = self.snapshot_url.format(timestamp=timestamp, domain=domain)
|
||||
response = None
|
||||
try:
|
||||
response = self.session.get(
|
||||
snapshot_url,
|
||||
timeout=config.WAYBACK_SNAPSHOT_TIMEOUT,
|
||||
stream=True,
|
||||
)
|
||||
if response.status_code != 200:
|
||||
data = {'timestamp': timestamp, 'title': '', 'ok': False}
|
||||
self._save_cached_title(domain, timestamp, data)
|
||||
return data
|
||||
|
||||
content_type = (response.headers.get('Content-Type') or '').lower()
|
||||
if content_type and 'text/html' not in content_type and 'application/xhtml+xml' not in content_type:
|
||||
data = {'timestamp': timestamp, 'title': '', 'ok': False}
|
||||
self._save_cached_title(domain, timestamp, data)
|
||||
return data
|
||||
|
||||
chunks = []
|
||||
total_bytes = 0
|
||||
found_title = False
|
||||
for chunk in response.iter_content(chunk_size=4096, decode_unicode=True):
|
||||
if not chunk:
|
||||
continue
|
||||
chunks.append(chunk)
|
||||
total_bytes += len(chunk.encode('utf-8', errors='ignore'))
|
||||
current_text = ''.join(chunks)
|
||||
if '</title>' in current_text.lower():
|
||||
found_title = True
|
||||
break
|
||||
if total_bytes >= config.WAYBACK_TITLE_MAX_BYTES:
|
||||
break
|
||||
|
||||
content = ''.join(chunks)
|
||||
title = self._extract_title(content) if found_title or content else ''
|
||||
data = {'timestamp': timestamp, 'title': title, 'ok': True}
|
||||
self._save_cached_title(domain, timestamp, data)
|
||||
return data
|
||||
except Exception:
|
||||
data = {'timestamp': timestamp, 'title': '', 'ok': False}
|
||||
self._save_cached_title(domain, timestamp, data)
|
||||
return data
|
||||
finally:
|
||||
try:
|
||||
if response is not None:
|
||||
response.close()
|
||||
except Exception:
|
||||
pass
|
||||
|
||||
def get_snapshot_years(self, domain):
|
||||
"""
|
||||
获取快照年份
|
||||
|
||||
:param domain: 域名
|
||||
:return: list - 快照年份列表
|
||||
"""
|
||||
try:
|
||||
years = set()
|
||||
for timestamp in self.get_snapshot_timestamps(domain):
|
||||
if len(timestamp) >= 4:
|
||||
years.add(int(timestamp[:4]))
|
||||
return sorted(years)
|
||||
except Exception as e:
|
||||
self._handle_exception(e, domain)
|
||||
return []
|
||||
|
||||
def has_sensitive_content(self, domain):
|
||||
"""
|
||||
检查是否包含敏感内容
|
||||
|
||||
:param domain: 域名
|
||||
:return: bool - 是否包含敏感内容
|
||||
"""
|
||||
try:
|
||||
result = self.scan_snapshots(domain)
|
||||
return result.get('has_sensitive_content', False)
|
||||
except Exception as e:
|
||||
self._handle_exception(e, domain)
|
||||
return False
|
||||
|
||||
def scan_snapshots(self, domain, sensitive_words=None, stop_on_first_hit=True):
|
||||
sensitive_words = sensitive_words or config.load_sensitive_words()
|
||||
latest_record = self.get_latest_snapshot_record(domain)
|
||||
latest_timestamp = (latest_record or {}).get('timestamp')
|
||||
latest_digest = (latest_record or {}).get('digest', '')
|
||||
matched_word = None
|
||||
matched_timestamp = None
|
||||
matched_title = None
|
||||
fetched_snapshot_count = 0
|
||||
failed_snapshot_count = 0
|
||||
unique_title_count = 0
|
||||
duplicate_title_skipped = 0
|
||||
digest_duplicate_skipped = 0
|
||||
started_at = time.time()
|
||||
progress_interval = max(1, config.WAYBACK_PROGRESS_INTERVAL)
|
||||
title_seen = set()
|
||||
digest_seen = set()
|
||||
checked_snapshot_count = 0
|
||||
domain_concurrency = max(1, config.WAYBACK_DOMAIN_CONCURRENCY)
|
||||
|
||||
if latest_timestamp:
|
||||
latest_result = self._fetch_snapshot_title(domain, latest_timestamp)
|
||||
checked_snapshot_count = 1
|
||||
if latest_result and latest_result.get('ok'):
|
||||
fetched_snapshot_count = 1
|
||||
title = latest_result.get('title', '')
|
||||
normalized_title = self._normalize_title(title)
|
||||
if latest_digest:
|
||||
digest_seen.add(latest_digest)
|
||||
if normalized_title:
|
||||
title_seen.add(normalized_title)
|
||||
unique_title_count = 1
|
||||
matched_word = self._find_sensitive_word(title, sensitive_words)
|
||||
if matched_word and stop_on_first_hit:
|
||||
matched_timestamp = latest_timestamp
|
||||
matched_title = title
|
||||
return {
|
||||
'snapshot_years': [int(latest_timestamp[:4])] if len(latest_timestamp) >= 4 else [],
|
||||
'has_sensitive_content': True,
|
||||
'matched_word': matched_word,
|
||||
'matched_timestamp': matched_timestamp,
|
||||
'matched_title': matched_title,
|
||||
'backlink_count': 0,
|
||||
'backlink_count_gt_10': False,
|
||||
'checked_snapshot_count': checked_snapshot_count,
|
||||
'fetched_snapshot_count': fetched_snapshot_count,
|
||||
'failed_snapshot_count': failed_snapshot_count,
|
||||
'unique_title_count': unique_title_count,
|
||||
'duplicate_title_skipped': duplicate_title_skipped,
|
||||
'digest_duplicate_skipped': digest_duplicate_skipped,
|
||||
'elapsed_seconds': round(time.time() - started_at, 2),
|
||||
}
|
||||
else:
|
||||
failed_snapshot_count += 1
|
||||
|
||||
records = sorted(self.get_snapshot_records(domain), key=lambda item: item.get('timestamp', ''), reverse=True)
|
||||
years = sorted({int(item['timestamp'][:4]) for item in records if len(item.get('timestamp', '')) >= 4})
|
||||
checked_snapshot_count = len(records)
|
||||
pending_records = []
|
||||
for item in records:
|
||||
timestamp = item.get('timestamp', '')
|
||||
digest = item.get('digest', '')
|
||||
if not timestamp:
|
||||
continue
|
||||
if latest_timestamp and timestamp == latest_timestamp:
|
||||
continue
|
||||
if digest and digest in digest_seen:
|
||||
digest_duplicate_skipped += 1
|
||||
continue
|
||||
if digest:
|
||||
digest_seen.add(digest)
|
||||
pending_records.append(item)
|
||||
|
||||
with ThreadPoolExecutor(max_workers=domain_concurrency) as executor:
|
||||
pending = {}
|
||||
index = 0
|
||||
finished_count = 1 if latest_timestamp else 0
|
||||
stop_requested = False
|
||||
|
||||
while (index < len(pending_records) or pending) and not stop_requested:
|
||||
while index < len(pending_records) and len(pending) < domain_concurrency and not stop_requested:
|
||||
timestamp = pending_records[index]['timestamp']
|
||||
future = executor.submit(self._fetch_snapshot_title, domain, timestamp)
|
||||
pending[future] = timestamp
|
||||
index += 1
|
||||
|
||||
if not pending:
|
||||
break
|
||||
|
||||
done, _ = wait(list(pending.keys()), return_when=FIRST_COMPLETED)
|
||||
for future in done:
|
||||
timestamp = pending.pop(future, None)
|
||||
finished_count += 1
|
||||
try:
|
||||
result = future.result()
|
||||
except Exception as e:
|
||||
failed_snapshot_count += 1
|
||||
self._handle_exception(e, domain)
|
||||
continue
|
||||
|
||||
if not result or not result.get('ok'):
|
||||
failed_snapshot_count += 1
|
||||
continue
|
||||
|
||||
fetched_snapshot_count += 1
|
||||
title = result.get('title', '')
|
||||
normalized_title = self._normalize_title(title)
|
||||
if normalized_title:
|
||||
if normalized_title in title_seen:
|
||||
duplicate_title_skipped += 1
|
||||
else:
|
||||
title_seen.add(normalized_title)
|
||||
unique_title_count += 1
|
||||
matched_word = self._find_sensitive_word(title, sensitive_words)
|
||||
if matched_word:
|
||||
matched_timestamp = timestamp
|
||||
matched_title = title
|
||||
if stop_on_first_hit:
|
||||
stop_requested = True
|
||||
if finished_count % progress_interval == 0:
|
||||
elapsed = round(time.time() - started_at, 2)
|
||||
self._log_info(
|
||||
f"{domain} 时光机进度: {finished_count}/{checked_snapshot_count},成功 {fetched_snapshot_count},失败 {failed_snapshot_count},唯一标题 {unique_title_count},标题重复跳过 {duplicate_title_skipped},digest 重复跳过 {digest_duplicate_skipped},耗时 {elapsed}s"
|
||||
)
|
||||
if config.WAYBACK_REQUEST_DELAY > 0:
|
||||
time.sleep(config.WAYBACK_REQUEST_DELAY)
|
||||
|
||||
if stop_requested:
|
||||
for future in pending:
|
||||
future.cancel()
|
||||
|
||||
return {
|
||||
'snapshot_years': years,
|
||||
'has_sensitive_content': matched_word is not None,
|
||||
'matched_word': matched_word,
|
||||
'matched_timestamp': matched_timestamp,
|
||||
'matched_title': matched_title,
|
||||
'backlink_count': 0,
|
||||
'backlink_count_gt_10': False,
|
||||
'checked_snapshot_count': checked_snapshot_count,
|
||||
'fetched_snapshot_count': fetched_snapshot_count,
|
||||
'failed_snapshot_count': failed_snapshot_count,
|
||||
'unique_title_count': unique_title_count,
|
||||
'duplicate_title_skipped': duplicate_title_skipped,
|
||||
'digest_duplicate_skipped': digest_duplicate_skipped,
|
||||
'elapsed_seconds': round(time.time() - started_at, 2),
|
||||
}
|
||||
|
||||
def _check_sensitive_words(self, content):
|
||||
"""
|
||||
检查敏感词
|
||||
|
||||
:param content: 内容
|
||||
:return: bool - 是否包含敏感词
|
||||
"""
|
||||
return self._find_sensitive_word(content, config.load_sensitive_words()) is not None
|
||||
|
||||
def _find_sensitive_word(self, content, sensitive_words):
|
||||
for word in sensitive_words or []:
|
||||
if word and word in (content or ''):
|
||||
return word
|
||||
return None
|
||||
|
||||
def get_backlink_count(self, domain):
|
||||
"""
|
||||
当前策略仅扫描标题,不再抓取正文,友链数量默认返回 0。
|
||||
"""
|
||||
return 0
|
||||
|
||||
def _count_backlinks(self, content):
|
||||
return 0
|
||||
Reference in New Issue
Block a user