admin/getDomain
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

d

Browse Source
This commit is contained in:
Your Name
2026-04-22 14:13:21 +08:00
parent e0406b5d0e
commit 7cbde2aa78
145 changed files with 23086 additions and 2243 deletions
Download Patch File Download Diff File Expand all files Collapse all files

77
domainCheck/app/config.py
View File

@@ -34,14 +34,56 @@ else:
# 开发环境目录
BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
# 加载环境变量
env_path = os.path.join(BASE_DIR, '.env')
if os.path.exists(env_path):
load_dotenv(env_path)
_safe_echo(f"成功加载环境变量文件: {env_path}")
else:
def _detect_install_root(base_dir):
normalized = os.path.abspath(str(base_dir or ""))
marker = f"{os.sep}releases{os.sep}"
if marker in normalized:
return normalized.split(marker, 1)[0]
return ""
def _resolve_runtime_root(base_dir):
explicit_root = str(os.getenv('DOMAINCHECK_RUNTIME_ROOT', '') or '').strip()
if explicit_root:
return os.path.abspath(explicit_root)
install_root = _detect_install_root(base_dir)
if install_root:
return os.path.join(install_root, 'runtime', 'domainCheck')
return os.path.abspath(base_dir)
def _load_environment(base_dir):
install_root = _detect_install_root(base_dir)
runtime_root = _resolve_runtime_root(base_dir)
candidate_paths = [
str(os.getenv('DOMAINCHECK_ENV_FILE', '') or '').strip(),
os.path.join(base_dir, '.env'),
os.path.join(runtime_root, '.env'),
os.path.join(os.getcwd(), '.env'),
]
if install_root:
candidate_paths.append(os.path.join(install_root, 'domainCheck', '.env'))
seen = set()
for path in candidate_paths:
normalized = os.path.abspath(path) if path else ''
if not normalized or normalized in seen:
continue
seen.add(normalized)
if os.path.exists(normalized):
load_dotenv(normalized)
_safe_echo(f"成功加载环境变量文件: {normalized}")
return normalized
load_dotenv()
_safe_echo(f"环境变量文件不存在: {env_path}，使用默认环境变量")
fallback_path = os.path.join(base_dir, '.env')
_safe_echo(f"环境变量文件不存在: {fallback_path}，使用默认环境变量")
return ""
# 加载环境变量
env_path = _load_environment(BASE_DIR)
RUNTIME_ROOT = _resolve_runtime_root(BASE_DIR)
class Config:
@@ -55,7 +97,11 @@ class Config:
DB_DATABASE = os.getenv('DB_DATABASE', 'domain_scan_db')
DB_USER = os.getenv('DB_USER', 'postgres')
DB_PASSWORD = os.getenv('DB_PASSWORD', 'postgres')
DB_POOL_SIZE = int(os.getenv('DB_POOL_SIZE', 5))
DB_POOL_SIZE = int(os.getenv('DB_POOL_SIZE', 64))
DB_POOL_WARM_SIZE = int(os.getenv('DB_POOL_WARM_SIZE', 8))
DB_POOL_IDLE_KEEP_MAX = int(os.getenv('DB_POOL_IDLE_KEEP_MAX', DB_POOL_SIZE))
DB_POOL_HEALTHCHECK_INTERVAL = float(os.getenv('DB_POOL_HEALTHCHECK_INTERVAL', 30))
DB_POOL_ACQUIRE_TIMEOUT = float(os.getenv('DB_POOL_ACQUIRE_TIMEOUT', 20))
# 消息队列配置
RABBITMQ_HOST = os.getenv('RABBITMQ_HOST', 'localhost')
@@ -86,12 +132,14 @@ class Config:
DETECT_TIMEOUT = int(os.getenv('DETECT_TIMEOUT', 30))
DETECT_RETRY_COUNT = int(os.getenv('DETECT_RETRY_COUNT', 3))
DETECT_CONCURRENCY = int(os.getenv('DETECT_CONCURRENCY', 10))
WAYBACK_CDX_TIMEOUT = int(os.getenv('WAYBACK_CDX_TIMEOUT', 15))
WAYBACK_SNAPSHOT_TIMEOUT = int(os.getenv('WAYBACK_SNAPSHOT_TIMEOUT', 12))
WAYBACK_RETRY_COUNT = int(os.getenv('WAYBACK_RETRY_COUNT', 2))
WAYBACK_CDX_TIMEOUT = int(os.getenv('WAYBACK_CDX_TIMEOUT', 2))
WAYBACK_SNAPSHOT_TIMEOUT = int(os.getenv('WAYBACK_SNAPSHOT_TIMEOUT', 2))
WAYBACK_RETRY_COUNT = int(os.getenv('WAYBACK_RETRY_COUNT', 0))
WAYBACK_REQUEST_DELAY = float(os.getenv('WAYBACK_REQUEST_DELAY', 0))
WAYBACK_PROGRESS_INTERVAL = int(os.getenv('WAYBACK_PROGRESS_INTERVAL', 500))
WAYBACK_DOMAIN_CONCURRENCY = int(os.getenv('WAYBACK_DOMAIN_CONCURRENCY', 3))
WAYBACK_DOMAIN_CONCURRENCY = int(os.getenv('WAYBACK_DOMAIN_CONCURRENCY', 2))
WAYBACK_MAX_RECORDS = int(os.getenv('WAYBACK_MAX_RECORDS', 4))
WAYBACK_TRANSIENT_BACKOFF_SECONDS = float(os.getenv('WAYBACK_TRANSIENT_BACKOFF_SECONDS', 8))
WAYBACK_TITLE_MAX_BYTES = int(os.getenv('WAYBACK_TITLE_MAX_BYTES', 65536))
WAYBACK_TIMESTAMP_CACHE_TTL = int(os.getenv('WAYBACK_TIMESTAMP_CACHE_TTL', 86400))
WAYBACK_TITLE_CACHE_TTL = int(os.getenv('WAYBACK_TITLE_CACHE_TTL', 2592000))
@@ -139,8 +187,9 @@ class Config:
# 目录配置
BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
DATA_DIR = os.path.join(BASE_DIR, 'data')
LOG_DIR = os.path.join(BASE_DIR, 'logs')
RUNTIME_ROOT = RUNTIME_ROOT
DATA_DIR = os.path.join(RUNTIME_ROOT, 'data')
LOG_DIR = os.path.join(RUNTIME_ROOT, 'logs')
# 确保目录存在
os.makedirs(DATA_DIR, exist_ok=True)

194
domainCheck/app/detectors/wayback_detector.py
View File

@@ -32,6 +32,8 @@ class WaybackDetector(BaseDetector):
"""
TITLE_PATTERN = re.compile(r'<title[^>]*>(.*?)</title>', re.IGNORECASE | re.DOTALL)
_transient_backoff_until = 0.0
_transient_backoff_lock = threading.Lock()
def __init__(self):
"""
@@ -56,7 +58,7 @@ class WaybackDetector(BaseDetector):
allowed_methods=frozenset(["GET"]),
raise_on_status=False,
)
adapter = HTTPAdapter(max_retries=retry, pool_connections=10, pool_maxsize=10)
adapter = HTTPAdapter(max_retries=retry, pool_connections=128, pool_maxsize=256)
session.mount('http://', adapter)
session.mount('https://', adapter)
session.headers.update({
@@ -65,6 +67,43 @@ class WaybackDetector(BaseDetector):
})
return session
@staticmethod
def _resolve_timeout(total_timeout, connect_cap=3):
total = max(1.0, float(total_timeout or 0))
return (min(float(connect_cap), total), total)
@staticmethod
def _is_transient_request_error(message):
lowered = str(message or '').lower()
keywords = (
'timeout',
'timed out',
'connecttimeout',
'read timeout',
'connection reset',
'connection refused',
'connection aborted',
'name or service not known',
'temporary failure',
'max retries exceeded',
'ssl',
)
return any(keyword in lowered for keyword in keywords)
@classmethod
def _transient_backoff_remaining_seconds(cls):
with cls._transient_backoff_lock:
remaining = float(cls._transient_backoff_until or 0) - time.time()
return max(0.0, remaining)
@classmethod
def _trip_transient_backoff(cls):
backoff_seconds = max(0.0, float(getattr(config, "WAYBACK_TRANSIENT_BACKOFF_SECONDS", 0) or 0))
if backoff_seconds <= 0:
return
with cls._transient_backoff_lock:
cls._transient_backoff_until = max(float(cls._transient_backoff_until or 0), time.time() + backoff_seconds)
def _build_redis_client(self):
try:
client = redis.Redis(
@@ -237,7 +276,7 @@ class WaybackDetector(BaseDetector):
response = self.session.get(
self.cdx_api_url,
params=params,
timeout=config.WAYBACK_CDX_TIMEOUT,
timeout=self._resolve_timeout(config.WAYBACK_CDX_TIMEOUT),
stream=True,
)
if response.status_code != 200:
@@ -328,17 +367,27 @@ class WaybackDetector(BaseDetector):
try:
response = self.session.get(
snapshot_url,
timeout=config.WAYBACK_SNAPSHOT_TIMEOUT,
timeout=self._resolve_timeout(config.WAYBACK_SNAPSHOT_TIMEOUT),
stream=True,
)
if response.status_code != 200:
data = {'timestamp': timestamp, 'title': '', 'ok': False}
data = {
'timestamp': timestamp,
'title': '',
'ok': False,
'error': f'HTTP {response.status_code}',
}
self._save_cached_title(domain, timestamp, data)
return data
content_type = (response.headers.get('Content-Type') or '').lower()
if content_type and 'text/html' not in content_type and 'application/xhtml+xml' not in content_type:
data = {'timestamp': timestamp, 'title': '', 'ok': False}
data = {
'timestamp': timestamp,
'title': '',
'ok': False,
'error': f'content_type:{content_type}',
}
self._save_cached_title(domain, timestamp, data)
return data
@@ -359,11 +408,11 @@ class WaybackDetector(BaseDetector):
content = ''.join(chunks)
title = self._extract_title(content) if found_title or content else ''
data = {'timestamp': timestamp, 'title': title, 'ok': True}
data = {'timestamp': timestamp, 'title': title, 'ok': True, 'error': ''}
self._save_cached_title(domain, timestamp, data)
return data
except Exception:
data = {'timestamp': timestamp, 'title': '', 'ok': False}
except Exception as exc:
data = {'timestamp': timestamp, 'title': '', 'ok': False, 'error': str(exc)}
self._save_cached_title(domain, timestamp, data)
return data
finally:
@@ -404,15 +453,83 @@ class WaybackDetector(BaseDetector):
self._handle_exception(e, domain)
return False
def scan_snapshots(self, domain, sensitive_words=None, stop_on_first_hit=True):
@staticmethod
def _resolve_recent_year_cutoff(recent_years):
try:
normalized_recent_years = int(recent_years or 0)
except (TypeError, ValueError):
return None
if normalized_recent_years <= 0:
return None
return time.gmtime().tm_year - normalized_recent_years + 1
def _filter_records_recent_years(self, records, recent_years=None):
cutoff_year = self._resolve_recent_year_cutoff(recent_years)
if cutoff_year is None:
return list(records or [])
filtered = []
for item in list(records or []):
timestamp = str((item or {}).get('timestamp') or '').strip()
if len(timestamp) < 4:
continue
try:
year = int(timestamp[:4])
except (TypeError, ValueError):
continue
if year >= cutoff_year:
filtered.append(item)
return filtered
def _limit_records_for_scan(self, records):
max_records = max(1, int(getattr(config, "WAYBACK_MAX_RECORDS", 8) or 8))
trimmed = []
for item in list(records or []):
if len(trimmed) >= max_records:
break
trimmed.append(item)
return trimmed
def scan_snapshots(self, domain, sensitive_words=None, stop_on_first_hit=True, recent_years=None):
sensitive_words = sensitive_words or config.load_sensitive_words()
request_errors = []
transient_backoff_remaining = self._transient_backoff_remaining_seconds()
if transient_backoff_remaining > 0:
return {
'snapshot_years': [],
'has_sensitive_content': False,
'matched_word': None,
'matched_timestamp': None,
'matched_title': None,
'backlink_count': 0,
'backlink_count_gt_10': False,
'checked_snapshot_count': 0,
'fetched_snapshot_count': 0,
'failed_snapshot_count': 1,
'unique_title_count': 0,
'duplicate_title_skipped': 0,
'digest_duplicate_skipped': 0,
'request_error_count': 1,
'request_errors': [f"wayback_backoff_active:{round(transient_backoff_remaining, 2)}s"],
'elapsed_seconds': 0.0,
}
latest_fetch = self._fetch_cdx_records_with_meta(domain, limit=-1, fast_latest=True)
latest_record = (latest_fetch.get('records') or [None])[0]
if latest_fetch.get('error'):
request_errors.append(f"latest_cdx: {latest_fetch.get('error')}")
if self._is_transient_request_error(latest_fetch.get('error')):
self._trip_transient_backoff()
latest_timestamp = (latest_record or {}).get('timestamp')
latest_digest = (latest_record or {}).get('digest', '')
cutoff_year = self._resolve_recent_year_cutoff(recent_years)
if latest_timestamp and cutoff_year is not None:
try:
latest_year = int(str(latest_timestamp)[:4])
except (TypeError, ValueError):
latest_year = 0
if latest_year < cutoff_year:
latest_record = None
latest_timestamp = None
latest_digest = ''
matched_word = None
matched_timestamp = None
matched_title = None
@@ -427,6 +544,28 @@ class WaybackDetector(BaseDetector):
digest_seen = set()
checked_snapshot_count = 0
domain_concurrency = max(1, config.WAYBACK_DOMAIN_CONCURRENCY)
transient_snapshot_failures = 0
transient_snapshot_failure_threshold = max(2, domain_concurrency)
if (not latest_timestamp) and latest_fetch.get('error') and self._is_transient_request_error(latest_fetch.get('error')):
return {
'snapshot_years': [],
'has_sensitive_content': False,
'matched_word': None,
'matched_timestamp': None,
'matched_title': None,
'backlink_count': 0,
'backlink_count_gt_10': False,
'checked_snapshot_count': 0,
'fetched_snapshot_count': 0,
'failed_snapshot_count': max(1, failed_snapshot_count),
'unique_title_count': 0,
'duplicate_title_skipped': 0,
'digest_duplicate_skipped': 0,
'request_error_count': len(request_errors),
'request_errors': request_errors,
'elapsed_seconds': round(time.time() - started_at, 2),
}
if latest_timestamp:
latest_result = self._fetch_snapshot_title(domain, latest_timestamp)
@@ -462,6 +601,30 @@ class WaybackDetector(BaseDetector):
}
else:
failed_snapshot_count += 1
latest_error = str((latest_result or {}).get('error') or '').strip()
if latest_error:
request_errors.append(f"latest_snapshot: {latest_error}")
if latest_error and self._is_transient_request_error(latest_error):
transient_snapshot_failures += 1
self._trip_transient_backoff()
return {
'snapshot_years': [],
'has_sensitive_content': False,
'matched_word': None,
'matched_timestamp': None,
'matched_title': None,
'backlink_count': 0,
'backlink_count_gt_10': False,
'checked_snapshot_count': checked_snapshot_count,
'fetched_snapshot_count': fetched_snapshot_count,
'failed_snapshot_count': failed_snapshot_count,
'unique_title_count': unique_title_count,
'duplicate_title_skipped': duplicate_title_skipped,
'digest_duplicate_skipped': digest_duplicate_skipped,
'request_error_count': len(request_errors),
'request_errors': request_errors,
'elapsed_seconds': round(time.time() - started_at, 2),
}
cached_records = self._load_cached_records(domain)
if cached_records is not None:
@@ -470,12 +633,16 @@ class WaybackDetector(BaseDetector):
records_fetch = self._fetch_cdx_records_with_meta(domain)
if records_fetch.get('error'):
request_errors.append(f"records_cdx: {records_fetch.get('error')}")
if self._is_transient_request_error(records_fetch.get('error')):
self._trip_transient_backoff()
records = records_fetch.get('records') or []
if records:
self._save_cached_records(domain, records)
self._save_cached_timestamps(domain, [item['timestamp'] for item in records])
records = self._filter_records_recent_years(records, recent_years=recent_years)
records = sorted(records, key=lambda item: item.get('timestamp', ''), reverse=True)
records = self._limit_records_for_scan(records)
years = sorted({int(item['timestamp'][:4]) for item in records if len(item.get('timestamp', '')) >= 4})
checked_snapshot_count = len(records)
pending_records = []
@@ -522,9 +689,18 @@ class WaybackDetector(BaseDetector):
if not result or not result.get('ok'):
failed_snapshot_count += 1
error_message = str((result or {}).get('error') or '').strip()
if error_message:
request_errors.append(f"snapshot:{timestamp}: {error_message}")
if error_message and self._is_transient_request_error(error_message):
transient_snapshot_failures += 1
self._trip_transient_backoff()
if transient_snapshot_failures >= transient_snapshot_failure_threshold:
stop_requested = True
continue
fetched_snapshot_count += 1
transient_snapshot_failures = 0
title = result.get('title', '')
normalized_title = self._normalize_title(title)
if normalized_title:

4
domainCheck/app/ui/system_settings.py
View File

@@ -1355,7 +1355,7 @@ class SystemSettingsWidget(QWidget):
return
raw_value = self.thread_count_input.text().strip()
try:
thread_count = str(min(20, max(1, int(raw_value or '10'))))
thread_count = str(max(1, int(raw_value or '10')))
except Exception:
thread_count = '10'
if self.thread_count_input.text().strip() != thread_count:
@@ -1390,7 +1390,7 @@ class SystemSettingsWidget(QWidget):
if os.path.exists('thread_count.json'):
with open('thread_count.json', 'r', encoding='utf-8') as f:
thread_config = json.load(f)
thread_count = str(min(20, max(1, int(thread_config.get('thread_count', '10') or '10'))))
thread_count = str(max(1, int(thread_config.get('thread_count', '10') or '10')))
self.thread_count_input.setText(thread_count)
logger.info("检测线程数配置加载成功")
else:

1011
domainCheck/app/utils/database.py
View File

File diff suppressed because it is too large Load Diff

140
domainCheck/detect/aizhan.py
View File

@@ -9,14 +9,41 @@
'''
import re # 正则表达式模块，用于从HTML中提取网站标题
import os
import requests # HTTP请求库，用于发送网络请求
from loguru import logger # 日志记录模块，用于记录程序运行日志
from requests.adapters import HTTPAdapter
from typing import List, Optional # 类型提示
def check_aizhan(domain: str, sensitive_words: Optional[List[str]] = None, proxies: dict = None):
_SESSION = requests.Session()
_SESSION.mount("http://", HTTPAdapter(pool_connections=256, pool_maxsize=512, max_retries=0))
_SESSION.mount("https://", HTTPAdapter(pool_connections=256, pool_maxsize=512, max_retries=0))
AIZHAN_TIMEOUT_PROXY = max(
0.8,
float(os.getenv("DOMAINCHECK_AIZHAN_TIMEOUT_PROXY", "1.6") or 1.6),
)
AIZHAN_TIMEOUT_DIRECT = max(
1.0,
float(os.getenv("DOMAINCHECK_AIZHAN_TIMEOUT_DIRECT", "2.2") or 2.2),
)
def _resolve_aizhan_timeout(proxies: dict = None, budget_seconds: Optional[float] = None) -> float:
timeout = float(AIZHAN_TIMEOUT_PROXY if proxies else AIZHAN_TIMEOUT_DIRECT)
if budget_seconds not in (None, "", 0, "0"):
timeout = min(timeout, max(0.6, float(budget_seconds or 0.0)))
return max(0.6, timeout)
def check_aizhan(
domain: str,
sensitive_words: Optional[List[str]] = None,
proxies: dict = None,
budget_seconds: Optional[float] = None,
):
'''
查询域名的网站标题信息是否存在敏感词
@@ -43,86 +70,43 @@ def check_aizhan(domain: str, sensitive_words: Optional[List[str]] = None, proxi
'host': 'www.aizhan.com', # 目标主机地址
}
# 添加重试机制
max_retries = 3
for retry in range(max_retries):
try:
# 发送GET请求获取页面内容
# url: 请求URL
# headers: HTTP请求头
# timeout=10: 设置超时时间10秒
# proxies: 代理配置（如需使用代理）
response = requests.get(url, headers=headers, timeout=10, proxies=proxies)
try:
# 发送GET请求获取页面内容
# url: 请求URL
# headers: HTTP请求头
# timeout=10: 设置超时时间10秒
# proxies: 代理配置（如需使用代理）
response = _SESSION.get(
url,
headers=headers,
timeout=_resolve_aizhan_timeout(proxies, budget_seconds=budget_seconds),
proxies=proxies,
)
# 判断请求是否成功（HTTP 200表示成功）
if response.status_code == 200:
# 使用正则表达式提取网站标题
# 正则解释：匹配 id="webpage_title"> 开头，</div> 结尾，中间的内容
# 匹配模式：id="webpage_title">标题内容</div>
# ([^<]+) 表示匹配一个或多个非<字符，作为捕获组
match = re.search(r'id="webpage_title">([^<]+)</div>', response.text)
# 判断请求是否成功（HTTP 200表示成功）
if response.status_code == 200:
# 使用正则表达式提取网站标题
# 正则解释：匹配 id="webpage_title"> 开头，</div> 结尾，中间的内容
# 匹配模式：id="webpage_title">标题内容</div>
# ([^<]+) 表示匹配一个或多个非<字符，作为捕获组
match = re.search(r'id="webpage_title">([^<]+)</div>', response.text)
# 如果找到匹配则返回标题文本，否则返回空字符串
title = match.group(1) if match else ''
if title:
for sensitive_word in sensitive_words:
# 使用正则表达式检查是否包含敏感词
if re.search(sensitive_word, title, re.IGNORECASE):
logger.warning(f'检测到敏感词: {sensitive_word} 在结果: {title}')
return False, f'检测到敏感词: {sensitive_word} 在结果: {title}'
return True, title
# 如果找到匹配则返回标题文本，否则返回空字符串
title = match.group(1) if match else ''
if title:
for sensitive_word in sensitive_words:
# 使用正则表达式检查是否包含敏感词
if re.search(sensitive_word, title, re.IGNORECASE):
logger.warning(f'检测到敏感词: {sensitive_word} 在结果: {title}')
return False, f'检测到敏感词: {sensitive_word} 在结果: {title}'
return True, title
# 请求失败，继续重试
logger.warning(f"爱站网检测第{retry+1}次失败: {domain}, 状态码: {response.status_code}")
if retry < max_retries - 1:
import time
time.sleep(2) # 等待2秒后重试
else:
# 达到最大重试次数，尝试不使用代理
if proxies:
logger.info('爱站网检测失败，尝试不使用代理')
try:
response = requests.get(url, headers=headers, timeout=10, proxies=None)
if response.status_code == 200:
match = re.search(r'id="webpage_title">([^<]+)</div>', response.text)
title = match.group(1) if match else ''
if title:
for sensitive_word in sensitive_words:
if re.search(sensitive_word, title, re.IGNORECASE):
logger.warning(f'检测到敏感词: {sensitive_word} 在结果: {title}')
return False, f'检测到敏感词: {sensitive_word} 在结果: {title}'
return True, title
return True, ''
except Exception as e2:
logger.error(f"不使用代理的爱站网检测也失败: {domain}, 错误: {e2}")
return False, str(e2)
return True, ''
logger.debug(f"爱站网检测失败: {domain}, 状态码: {response.status_code}")
return True, ''
except Exception as e: # 捕获所有异常（网络错误、超时等）
logger.warning(f"爱站网检测第{retry+1}次失败: {domain}, 错误: {e}")
if retry < max_retries - 1:
import time
time.sleep(2) # 等待2秒后重试
else:
# 达到最大重试次数，尝试不使用代理
if proxies:
logger.info('爱站网检测失败，尝试不使用代理')
try:
response = requests.get(url, headers=headers, timeout=10, proxies=None)
if response.status_code == 200:
match = re.search(r'id="webpage_title">([^<]+)</div>', response.text)
title = match.group(1) if match else ''
if title:
for sensitive_word in sensitive_words:
if re.search(sensitive_word, title, re.IGNORECASE):
logger.warning(f'检测到敏感词: {sensitive_word} 在结果: {title}')
return False, f'检测到敏感词: {sensitive_word} 在结果: {title}'
return True, title
return True, ''
except Exception as e2:
logger.error(f"不使用代理的爱站网检测也失败: {domain}, 错误: {e2}")
return False, str(e2)
return False, str(e)
except Exception as e: # 捕获所有异常（网络错误、超时等）
logger.debug(f"爱站网检测失败: {domain}, 错误: {e}")
return False, str(e)
# # ========================= 【测试代码】 =========================
# if __name__ == '__main__':
@@ -152,4 +136,4 @@ def check_aizhan(domain: str, sensitive_words: Optional[List[str]] = None, proxi
# logger.info(f"查询域名: {test_domain_4}")
#
# result_4 = check_aizhan(test_domain_4)
# logger.info(f"网站标题: {result_4 if result_4 else '(无结果)'}")
# logger.info(f"网站标题: {result_4 if result_4 else '(无结果)'}")

282
domainCheck/detect/baidu.py
View File

@@ -8,15 +8,121 @@
@explain : 百度域名检测工具 - 包含百度搜索结果查询和百度安全API检测功能
'''
import os
import random # 随机数生成模块，用于随机选择浏览器类型
import re # 正则表达式模块，用于从HTML中提取搜索结果
import threading
import time
from loguru import logger # 日志记录模块，用于记录程序运行日志
from curl_cffi import requests # curl_cffi库，支持浏览器模拟的HTTP请求
from typing import List, Optional, Union, Any # 类型提示
def check_site(domain: str, sensitive_words: Optional[List[str]] = None, proxies: dict = None) -> Union[
BAIDU_SITE_TIMEOUT_PROXY = max(
0.8,
float(os.getenv("DOMAINCHECK_BAIDU_TIMEOUT_PROXY", "1.4") or 1.4),
)
BAIDU_SITE_TIMEOUT_DIRECT = max(
1.0,
float(os.getenv("DOMAINCHECK_BAIDU_TIMEOUT_DIRECT", "1.8") or 1.8),
)
BAIDU_SCAN_TIMEOUT_PROXY = max(
1.0,
float(os.getenv("DOMAINCHECK_BAIDU_SCAN_TIMEOUT_PROXY", "1.6") or 1.6),
)
BAIDU_SCAN_TIMEOUT_DIRECT = max(
1.2,
float(os.getenv("DOMAINCHECK_BAIDU_SCAN_TIMEOUT_DIRECT", "2.0") or 2.0),
)
BAIDU_SITE_URL = 'https://m.baidu.com/s'
BAIDU_SAFE_SCAN_URL = 'https://mobsec-sec.baidu.com/3.1/scanurl'
BAIDU_SITE_PATTERN = r'<!--s-text-->([^<]+)<!--/s-text-->'
_THREAD_LOCAL = threading.local()
def _resolve_baidu_timeout(proxies: dict = None, budget_seconds: Optional[float] = None, *, scan: bool = False) -> float:
if scan:
timeout = BAIDU_SCAN_TIMEOUT_PROXY if proxies else BAIDU_SCAN_TIMEOUT_DIRECT
else:
timeout = BAIDU_SITE_TIMEOUT_PROXY if proxies else BAIDU_SITE_TIMEOUT_DIRECT
timeout = float(timeout or 0.0)
if budget_seconds not in (None, "", 0, "0"):
timeout = min(timeout, max(0.6, float(budget_seconds or 0.0)))
return max(0.6, timeout)
BAIDU_SITE_COOKIES = {
'BIDUPSID': '053DBE4D820C6EFB729DC7B13B1F82B2',
'PSTM': '1775657119',
'H_PS_PSSID': '63148_67862_67986_68002_68142_68148_68152_68141_68165_68189_68226_68267_68296_68336_68369_68453_68438_68464_68541_68546_68558_68520_68589_68621_68615_68606_68601_68682_68671_68735_68544_68733_68766_68807_68901_68918_68836_68921_68955_68976_68997_69007_69010_69018_69024_69014',
'BAIDUID': '053DBE4D820C6EFB729DC7B13B1F82B2:FG=1',
'BDORZ': 'B490B5EBF6F3CD402E515D22BCDA1598',
'BAIDUID_BFESS': '053DBE4D820C6EFB729DC7B13B1F82B2:FG=1',
'BA_HECTOR': 'ah0l24210la4050l0g0k0g2g0h20071ktcocj27',
'ZFY': 'Qxyr75Xm7o8zUxYzuFnYoW7cnS:AnVp:BpnrNVkr3usno:C',
'delPer': '0',
'BAIDUID_REF': '053DBE4D820C6EFB729DC7B13B1F82B2:FG=1',
'H_WISE_SIDS': '110085_661771_667681_673676_683089_682564_685373_660925_687556_686285_690306_690303_690478_690576_690334_691545_685595_690883_692754_692777_692118_692910_693221_693364_693403_693656_693941_694124_693392_694171_694236_693996_694316_694324_693886_694409_694379_694696_694780_694783_692006_694884_694918_694996_694991_695032_695118_694840_695140_695193_695110_695262_688610_694933_695278_694865_695214_695374_690651_694577_692378_695452_695457_695480_695603_695642_695631_695668_695824_695843_695866_695888_695892_695897_695975_694177_695900_695939_696145_696150_696153_696069_696078_696066_694359_696288_696308_694985_696297_695715_696128_696313_696457_696431_696473_696492_696590_693385_696610_696682_696679_696662_696651_696655_696658_696673_696643_696110_696729_696733_696772_8000116_8000133_8000138_8000159_8000163_8000167_8000176_8000186_8000190_8000204',
'H_WISE_SIDS_BFESS': '110085_661771_667681_673676_683089_682564_685373_660925_687556_686285_690306_690303_690478_690576_690334_691545_685595_690883_692754_692777_692118_692910_693221_693364_693403_693656_693941_694124_693392_694171_694236_693996_694316_694324_693886_694409_694379_694696_694780_694783_692006_694884_694918_694996_694991_695032_695118_694840_695140_695193_695110_695262_688610_694933_695278_694865_695214_695374_690651_694577_692378_695452_695457_695480_695603_695642_695631_695668_695824_695843_695866_695888_695892_695897_695975_694177_695900_695939_696145_696150_696153_696069_696078_696066_694359_696288_696308_694985_696297_695715_696128_696313_696457_696431_696473_696492_696590_693385_696610_696682_696679_696662_696651_696655_696658_696673_696643_696110_696729_696733_696772_8000116_8000133_8000138_8000159_8000163_8000167_8000176_8000186_8000190_8000204',
'MSA_PBT': '147',
'MSA_ZOOM': '1000',
'wpr': '0',
'COOKIE_SESSION': '0_0_0_0_0_0_0_0_0_0_0_0_0_1775657396%7C1%230_0_0_0_0_0_0_0_1775657396%7C1',
'MSA_PHY_WH': '1440_3440',
'POLYFILL': '0',
'MSA_WH': '1254_940',
'kleck': '7ce2abac229d2e8ba435a8bcc6256f57f53e9d08954443bf',
'PSCBD': '16%3A1%3A3',
'SE_LAUNCH': '5%3A1775657396_16%3A29594323%3A3',
'BDSVRTM': '3',
'PSINO': '6',
'__bsi': '17976785662214065461_00_7_R_R_6_0303_c02f_Y',
}
BAIDU_SITE_HEADERS = {
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7',
'Accept-Language': 'zh-CN,zh;q=0.9',
'Connection': 'keep-alive',
'Sec-Fetch-Dest': 'document',
'Sec-Fetch-Mode': 'navigate',
'Sec-Fetch-Site': 'none',
'Sec-Fetch-User': '?1',
'Upgrade-Insecure-Requests': '1',
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/146.0.0.0 Safari/537.36 Edg/146.0.0.0',
'sec-ch-ua': '"Chromium";v="146", "Not-A.Brand";v="24", "Microsoft Edge";v="146"',
'sec-ch-ua-mobile': '?0',
'sec-ch-ua-platform': '"Windows"',
}
BAIDU_SAFE_SCAN_HEADERS = {
'Cache-Control': 'no-cache',
'Content-Type': 'application/json; charset=utf-8',
'Host': 'mobsec-sec.baidu.com',
'User-Agent': 'okhttp/3.12.12',
}
BAIDU_SAFE_SCAN_PARAMS = {
'auth_ver': '2',
'appkey': '4665fd2c6b0922a551e8ae74',
'nonce': '1751309255340',
'lc': '77qHTv4VtmRiXYtd',
'pkg': 'com.baidu.searchbox',
'vc': '-1',
'cuid': 'CF0E6FCCD824D146605C16AC1C8BAC95%7CVFWAO56TC',
'tk': '',
'type': '3',
'vn': '2.6.0',
's': 'fdfef2ce96c1c7193f9b80425020f63e',
}
def _get_thread_session():
session = getattr(_THREAD_LOCAL, "session", None)
if session is None:
session = requests.Session()
_THREAD_LOCAL.session = session
return session
def check_site(domain: str, sensitive_words: Optional[List[str]] = None, proxies: dict = None, budget_seconds: Optional[float] = None) -> Union[
None, tuple[bool, str], list[Any]]:
'''
通过百度搜索查询域名的相关信息
@@ -31,80 +137,41 @@ def check_site(domain: str, sensitive_words: Optional[List[str]] = None, proxies
if sensitive_words is None:
sensitive_words = []
# 添加重试机制
max_retries = 3
for retry in range(max_retries):
try: # try-except块，用于捕获网络请求中的异常
try: # try-except块，用于捕获网络请求中的异常
# 构建搜索查询参数
# wd: 搜索关键词，格式为site:域名
params = {
'wd': f'site:{domain}' # 搜索site:domain，限制搜索结果为指定域名
}
cookies = {
'BIDUPSID': '053DBE4D820C6EFB729DC7B13B1F82B2',
'PSTM': '1775657119',
'H_PS_PSSID': '63148_67862_67986_68002_68142_68148_68152_68141_68165_68189_68226_68267_68296_68336_68369_68453_68438_68464_68541_68546_68558_68520_68589_68621_68615_68606_68601_68682_68671_68735_68544_68733_68766_68807_68901_68918_68836_68921_68955_68976_68997_69007_69010_69018_69024_69014',
'BAIDUID': '053DBE4D820C6EFB729DC7B13B1F82B2:FG=1',
'BDORZ': 'B490B5EBF6F3CD402E515D22BCDA1598',
'BAIDUID_BFESS': '053DBE4D820C6EFB729DC7B13B1F82B2:FG=1',
'BA_HECTOR': 'ah0l24210la4050l0g0k0g2g0h20071ktcocj27',
'ZFY': 'Qxyr75Xm7o8zUxYzuFnYoW7cnS:AnVp:BpnrNVkr3usno:C',
'delPer': '0',
'BAIDUID_REF': '053DBE4D820C6EFB729DC7B13B1F82B2:FG=1',
'H_WISE_SIDS': '110085_661771_667681_673676_683089_682564_685373_660925_687556_686285_690306_690303_690478_690576_690334_691545_685595_690883_692754_692777_692118_692910_693221_693364_693403_693656_693941_694124_693392_694171_694236_693996_694316_694324_693886_694409_694379_694696_694780_694783_692006_694884_694918_694996_694991_695032_695118_694840_695140_695193_695110_695262_688610_694933_695278_694865_695214_695374_690651_694577_692378_695452_695457_695480_695603_695642_695631_695668_695824_695843_695866_695888_695892_695897_695975_694177_695900_695939_696145_696150_696153_696069_696078_696066_694359_696288_696308_694985_696297_695715_696128_696313_696457_696431_696473_696492_696590_693385_696610_696682_696679_696662_696651_696655_696658_696673_696643_696110_696729_696733_696772_8000116_8000133_8000138_8000159_8000163_8000167_8000176_8000186_8000190_8000204',
'H_WISE_SIDS_BFESS': '110085_661771_667681_673676_683089_682564_685373_660925_687556_686285_690306_690303_690478_690576_690334_691545_685595_690883_692754_692777_692118_692910_693221_693364_693403_693656_693941_694124_693392_694171_694236_693996_694316_694324_693886_694409_694379_694696_694780_694783_692006_694884_694918_694996_694991_695032_695118_694840_695140_695193_695110_695262_688610_694933_695278_694865_695214_695374_690651_694577_692378_695452_695457_695480_695603_695642_695631_695668_695824_695843_695866_695888_695892_695897_695975_694177_695900_695939_696145_696150_696153_696069_696078_696066_694359_696288_696308_694985_696297_695715_696128_696313_696457_696431_696473_696492_696590_693385_696610_696682_696679_696662_696651_696655_696658_696673_696643_696110_696729_696733_696772_8000116_8000133_8000138_8000159_8000163_8000167_8000176_8000186_8000190_8000204',
'MSA_PBT': '147',
'MSA_ZOOM': '1000',
'wpr': '0',
'COOKIE_SESSION': '0_0_0_0_0_0_0_0_0_0_0_0_0_1775657396%7C1%230_0_0_0_0_0_0_0_1775657396%7C1',
'MSA_PHY_WH': '1440_3440',
'POLYFILL': '0',
'MSA_WH': '1254_940',
'kleck': '7ce2abac229d2e8ba435a8bcc6256f57f53e9d08954443bf',
'PSCBD': '16%3A1%3A3',
'SE_LAUNCH': '5%3A1775657396_16%3A29594323%3A3',
'BDSVRTM': '3',
'PSINO': '6',
'__bsi': '17976785662214065461_00_7_R_R_6_0303_c02f_Y',
}
headers = {
'Accept': 'text/html,application/xhtml+xml,application/xml;q=0.9,image/avif,image/webp,image/apng,*/*;q=0.8,application/signed-exchange;v=b3;q=0.7',
'Accept-Language': 'zh-CN,zh;q=0.9',
'Connection': 'keep-alive',
'Sec-Fetch-Dest': 'document',
'Sec-Fetch-Mode': 'navigate',
'Sec-Fetch-Site': 'none',
'Sec-Fetch-User': '?1',
'Upgrade-Insecure-Requests': '1',
'User-Agent': 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/146.0.0.0 Safari/537.36 Edg/146.0.0.0',
'sec-ch-ua': '"Chromium";v="146", "Not-A.Brand";v="24", "Microsoft Edge";v="146"',
'sec-ch-ua-mobile': '?0',
'sec-ch-ua-platform': '"Windows"',
}
# 百度搜索URL
url = 'https://m.baidu.com/s'
# 发送GET请求获取搜索结果
# url: 请求URL
# params: 查询参数
# headers: HTTP请求头
# timeout=15: 超时时间15秒
# proxies: 代理配置
# impersonate: 模拟的浏览器类型
# .content: 获取响应内容（字节流）
# .decode('utf-8'): 解码为UTF-8字符串
response_text = requests.get(url, params=params, headers=headers, cookies=cookies, timeout=15,
proxies=proxies).content.decode('utf-8')
session = _get_thread_session()
request_started_at = time.perf_counter()
logger.info(
f"百度site阶段: domain={domain} | stage=request_start | proxy={'yes' if proxies else 'no'}"
)
response = session.get(
BAIDU_SITE_URL,
params=params,
headers=BAIDU_SITE_HEADERS,
cookies=BAIDU_SITE_COOKIES,
timeout=_resolve_baidu_timeout(proxies, budget_seconds=budget_seconds, scan=False),
proxies=proxies,
)
logger.info(
f"百度site阶段: domain={domain} | stage=request_done | status={response.status_code} "
f"| elapsed_ms={int((time.perf_counter() - request_started_at) * 1000)}"
)
response_text = response.content.decode('utf-8', errors='ignore')
# 使用正则表达式提取搜索结果标题
# re.findall()返回所有匹配的结果列表
# 正则解释：匹配 <!--s-text--> 和 <!--/s-text--> 之间的内容
match = re.findall(r'<!--s-text-->([^<]+)<!--/s-text-->', response_text,re.S)
match = re.findall(BAIDU_SITE_PATTERN, response_text, re.S)
# 返回匹配结果列表，如果没有匹配则返回空列表
search_results= match if match else []
logger.info(
f"百度site阶段: domain={domain} | stage=parsed | result_count={len(search_results)}"
)
# 如果没有搜索结果，直接返回通过
if not search_results:
@@ -120,19 +187,12 @@ def check_site(domain: str, sensitive_words: Optional[List[str]] = None, proxies
# 所有搜索结果都不包含敏感词，返回通过
return True, ''
except Exception as e: # 捕获所有异常
# logger.warning(f"百度site检测第{retry+1}次失败: {domain}, 错误: {e}")
if retry < max_retries - 1:
import time
time.sleep(2) # 等待2秒后重试
proxies=None
else:
# 达到最大重试次数，返回检测失败
logger.error(f"百度site检测失败: {domain}, 已达到最大重试次数")
return False, str(e) # 返回错误信息
except Exception as e: # 捕获所有异常
logger.error(f"百度site检测失败: {domain}, 错误: {e}")
return False, str(e)
def baidu(domain: str, proxies: dict = None) -> str:
def baidu(domain: str, proxies: dict = None, budget_seconds: Optional[float] = None) -> str:
'''
通过百度安全API检测域名安全状态
@@ -142,29 +202,6 @@ def baidu(domain: str, proxies: dict = None) -> str:
:param proxies: 代理配置（如：{'http': 'http://127.0.0.1:7890', 'https': 'http://127.0.0.1:7890'}
:return: str - 安全状态描述（"风险网站提示"、"高危网站提示"或"正常"），如果检测失败则返回错误信息
'''
# 构建HTTP请求头
headers = {
'Cache-Control': 'no-cache', # 不使用缓存
'Content-Type': 'application/json; charset=utf-8', # 内容类型为JSON
'Host': 'mobsec-sec.baidu.com', # 目标主机
'User-Agent': 'okhttp/3.12.12', # 模拟OKHttp客户端
}
# 构建请求参数
params = {
'auth_ver': '2', # 认证版本
'appkey': '4665fd2c6b0922a551e8ae74', # API应用密钥
'nonce': '1751309255340', # 随机数（注意：实际使用时应该动态生成）
'lc': '77qHTv4VtmRiXYtd', # 本地配置参数
'pkg': 'com.baidu.searchbox', # 应用包名
'vc': '-1', # 版本码
'cuid': 'CF0E6FCCD824D146605C16AC1C8BAC95%7CVFWAO56TC', # 设备ID（注意：实际使用时应该动态生成）
'tk': '', # 令牌（空）
'type': '3', # 检测类型
'vn': '2.6.0', # 版本号
's': 'fdfef2ce96c1c7193f9b80425020f63e', # 签名（注意：实际使用时应该动态生成）
}
# 构建JSON请求数据
json_data = {
'url': f'https://{domain}/', # 待检测的URL（添加https协议头）
@@ -173,10 +210,7 @@ def baidu(domain: str, proxies: dict = None) -> str:
# 提取URL作为后续获取结果的键
key = json_data['url']
# 添加重试机制
max_retries = 3
for retry in range(max_retries):
try: # try-except块，用于捕获API调用中的异常
try: # try-except块，用于捕获API调用中的异常
# 发送POST请求到百度安全API
# url: API接口地址
# params: 查询参数
@@ -184,8 +218,24 @@ def baidu(domain: str, proxies: dict = None) -> str:
# json: JSON请求数据
# proxies: 代理配置
# .json(): 解析JSON响应
response = requests.post('https://mobsec-sec.baidu.com/3.1/scanurl', params=params, headers=headers,
json=json_data, proxies=proxies, timeout=15).json()
session = _get_thread_session()
request_started_at = time.perf_counter()
logger.info(
f"百度安全阶段: domain={domain} | stage=request_start | proxy={'yes' if proxies else 'no'}"
)
response = session.post(
BAIDU_SAFE_SCAN_URL,
params=BAIDU_SAFE_SCAN_PARAMS,
headers=BAIDU_SAFE_SCAN_HEADERS,
json=json_data,
proxies=proxies,
timeout=_resolve_baidu_timeout(proxies, budget_seconds=budget_seconds, scan=True),
)
logger.info(
f"百度安全阶段: domain={domain} | stage=request_done | status={response.status_code} "
f"| elapsed_ms={int((time.perf_counter() - request_started_at) * 1000)}"
)
response = response.json()
# # 记录API响应（调试用）
# logger.info(response)
@@ -202,27 +252,9 @@ def baidu(domain: str, proxies: dict = None) -> str:
# msgs.get(): 根据安全等级获取对应的状态描述，默认值为'正常'
return msgs.get(response['response']['datas'][key].get('grand_level', 'level'), '正常')
except Exception as e: # 捕获所有异常
logger.warning(f"百度网页安全中心检测第{retry+1}次失败: {domain}, 错误: {e}")
if retry < max_retries - 1:
time.sleep(2) # 等待2秒后重试
else:
# 达到最大重试次数，尝试不使用代理
if proxies:
logger.info('百度网页安全中心检测失败，尝试不使用代理')
try:
response = requests.post('https://mobsec-sec.baidu.com/3.1/scanurl', params=params, headers=headers,
json=json_data, proxies=None, timeout=15).json()
msgs = {
2193: '风险网站提示', # 风险网站
2243: '高危网站提示', # 高危网站
'_': '正常' # 默认正常
}
return msgs.get(response['response']['datas'][key].get('grand_level', 'level'), '正常')
except Exception as e2:
logger.error(f"不使用代理的百度网页安全中心检测也失败: {domain}, 错误: {e2}")
return str(e2) # 返回错误信息
return str(e) # 返回错误信息
except Exception as e: # 捕获所有异常
logger.error(f"百度网页安全中心检测失败: {domain}, 错误: {e}")
return str(e)
# # ========================= 【测试代码】 =========================
@@ -260,4 +292,4 @@ def baidu(domain: str, proxies: dict = None) -> str:
# logger.info(f"搜索结果标题数: {len(result_1)}")
# for i, title in enumerate(result_1, 1):
# logger.info(f" 结果{i}: {title}")
# time.sleep(1)
# time.sleep(1)

202
domainCheck/detect/c360.py
View File

@@ -9,8 +9,12 @@
'''
import re # 正则表达式模块
import os
import threading
import time
import requests # HTTP请求库
from loguru import logger # 日志记录
from requests.adapters import HTTPAdapter
from typing import List, Tuple, Optional, Dict # 类型提示
@@ -19,12 +23,40 @@ SO_SEARCH_URL = 'https://www.so.com/s' # 360搜索URL
SO_REFERER_TEMPLATE = 'https://www.so.com/s?ie=utf-8&q=site%3A{domain}' # Referer模板
SEARCH_PATTERN = r'target="_blank">([^<]+)</a></h3>' # 搜索结果匹配模式
BLOCKED_CODE = 3 # 拦截状态码
_SESSION_LOCAL = threading.local()
SO_TIMEOUT_PROXY = max(
1.0,
float(os.getenv("DOMAINCHECK_360_TIMEOUT_PROXY", "1.4") or 1.4),
)
SO_TIMEOUT_DIRECT = max(
1.2,
float(os.getenv("DOMAINCHECK_360_TIMEOUT_DIRECT", "1.8") or 1.8),
)
def _resolve_360_timeout(proxies: Optional[Dict] = None, budget_seconds: Optional[float] = None) -> float:
timeout = float(SO_TIMEOUT_PROXY if proxies else SO_TIMEOUT_DIRECT)
if budget_seconds not in (None, "", 0, "0"):
timeout = min(timeout, max(0.6, float(budget_seconds or 0.0)))
return max(0.6, timeout)
def _get_session():
session = getattr(_SESSION_LOCAL, "session", None)
if session is not None:
return session
session = requests.Session()
session.mount("http://", HTTPAdapter(pool_connections=64, pool_maxsize=128, max_retries=0))
session.mount("https://", HTTPAdapter(pool_connections=64, pool_maxsize=128, max_retries=0))
_SESSION_LOCAL.session = session
return session
def check_domain(
domain: str,
sensitive_words: Optional[List[str]] = None,
proxies: Optional[Dict] = None
proxies: Optional[Dict] = None,
budget_seconds: Optional[float] = None,
) -> Tuple[bool, str]:
"""
检查域名是否包含敏感词
@@ -60,125 +92,65 @@ def check_domain(
'q': f'site:{domain}', # 搜索查询
}
# 添加重试机制
max_retries = 3
for retry in range(max_retries):
try:
# 发送GET请求获取搜索结果
response = requests.get(
SO_SEARCH_URL,
params=params,
headers=headers,
proxies=proxies,
timeout=15 # 增加超时时间到15秒
)
try:
session_started_at = time.perf_counter()
logger.info(f'360阶段: domain={domain} | stage=session_init_start')
session = _get_session()
logger.info(
f'360阶段: domain={domain} | stage=session_init_done '
f'| elapsed_ms={int((time.perf_counter() - session_started_at) * 1000)}'
)
# 发送GET请求获取搜索结果
request_started_at = time.perf_counter()
logger.info(f'360阶段: domain={domain} | stage=request_start | proxy={"yes" if proxies else "no"}')
response = session.get(
SO_SEARCH_URL,
params=params,
headers=headers,
proxies=proxies,
timeout=_resolve_360_timeout(proxies, budget_seconds=budget_seconds),
)
logger.info(
f'360阶段: domain={domain} | stage=request_done | status={response.status_code} '
f'| elapsed_ms={int((time.perf_counter() - request_started_at) * 1000)}'
)
# 解析响应内容
response_html = response.content.decode('utf-8', errors='ignore')
# 解析响应内容
response_html = response.content.decode('utf-8', errors='ignore')
# 使用正则表达式提取搜索结果
search_results = re.findall(SEARCH_PATTERN, response_html)
# 使用正则表达式提取搜索结果
search_results = re.findall(SEARCH_PATTERN, response_html)
logger.info(f'360阶段: domain={domain} | stage=parsed | result_count={len(search_results)}')
# 记录搜索结果数量
logger.info(f'360搜索结果数量: {len(search_results)}')
# 记录搜索结果数量
logger.info(f'360搜索结果数量: {len(search_results)}')
# 如果没有搜索结果，直接返回通过
if not search_results:
return True, ''
# 检查每个搜索结果是否包含敏感词
for result in search_results:
for sensitive_word in sensitive_words:
# 使用正则表达式检查是否包含敏感词
if re.search(sensitive_word, result, re.IGNORECASE):
logger.warning(f'检测到敏感词: {sensitive_word} 在结果: {result}')
return False, f'检测到敏感词: {sensitive_word} 在结果: {result}'
# 所有搜索结果都不包含敏感词，返回通过
# 如果没有搜索结果，直接返回通过
if not search_results:
return True, ''
except requests.exceptions.Timeout:
logger.warning(f'360搜索请求第{retry+1}次超时')
if retry < max_retries - 1:
import time
time.sleep(2) # 等待2秒后重试
else:
# 达到最大重试次数，尝试不使用代理
if proxies:
logger.info('360搜索请求超时，尝试不使用代理')
try:
response = requests.get(
SO_SEARCH_URL,
params=params,
headers=headers,
proxies=None,
timeout=15
)
response_html = response.content.decode('utf-8', errors='ignore')
search_results = re.findall(SEARCH_PATTERN, response_html)
logger.info(f'不使用代理的360搜索结果数量: {len(search_results)}')
if not search_results:
return True, ''
for result in search_results:
for sensitive_word in sensitive_words:
if re.search(sensitive_word, result, re.IGNORECASE):
logger.warning(f'检测到敏感词: {sensitive_word} 在结果: {result}')
return False, f'检测到敏感词: {sensitive_word} 在结果: {result}'
return True, ''
except Exception as e:
logger.error(f'不使用代理的360搜索请求也失败: {str(e)}')
return False, f'360搜索请求超时'
return False, '360搜索请求超时'
except requests.exceptions.RequestException as e:
error_str = str(e)
logger.warning(f'360搜索请求第{retry+1}次失败: {error_str}')
# 检查是否是"Too many open files"错误
if 'Too many open files' in error_str:
if retry < max_retries - 1:
import time
time.sleep(2) # 等待2秒后重试
else:
# 达到最大重试次数，返回failure标记，不拉黑域名
logger.error(f'360搜索请求失败，已达到最大重试次数: {error_str}')
return True, 'failure'
else:
if retry < max_retries - 1:
import time
time.sleep(2) # 等待2秒后重试
else:
# 达到最大重试次数，尝试不使用代理
if proxies:
logger.info('360搜索请求失败，尝试不使用代理')
try:
response = requests.get(
SO_SEARCH_URL,
params=params,
headers=headers,
proxies=None,
timeout=15
)
response_html = response.content.decode('utf-8', errors='ignore')
search_results = re.findall(SEARCH_PATTERN, response_html)
logger.info(f'不使用代理的360搜索结果数量: {len(search_results)}')
if not search_results:
return True, ''
for result in search_results:
for sensitive_word in sensitive_words:
if re.search(sensitive_word, result, re.IGNORECASE):
logger.warning(f'检测到敏感词: {sensitive_word} 在结果: {result}')
return False, f'检测到敏感词: {sensitive_word} 在结果: {result}'
return True, ''
except Exception as e2:
logger.error(f'不使用代理的360搜索请求也失败: {str(e2)}')
return False, f'360搜索请求失败: {str(e)}'
return False, f'360搜索请求失败: {str(e)}'
except Exception as e:
logger.warning(f'360搜索第{retry+1}次未知错误: {str(e)}')
if retry < max_retries - 1:
import time
time.sleep(2) # 等待2秒后重试
else:
return False, f'未知错误: {str(e)}'
# 检查每个搜索结果是否包含敏感词
for result in search_results:
for sensitive_word in sensitive_words:
# 使用正则表达式检查是否包含敏感词
if re.search(sensitive_word, result, re.IGNORECASE):
logger.warning(f'检测到敏感词: {sensitive_word} 在结果: {result}')
return False, f'检测到敏感词: {sensitive_word} 在结果: {result}'
# 所有搜索结果都不包含敏感词，返回通过
return True, ''
except requests.exceptions.Timeout:
return False, '360搜索请求超时'
except requests.exceptions.RequestException as e:
error_str = str(e)
logger.debug(f'360搜索请求失败: {error_str}')
# 检查是否是"Too many open files"错误
if 'Too many open files' in error_str:
return True, 'failure'
return False, f'360搜索请求失败: {error_str}'
except Exception as e:
return False, f'未知错误: {str(e)}'
# if __name__ == '__main__':
@@ -188,4 +160,4 @@ def check_domain(
#
# logger.info(f'开始检测域名: {test_domain}')
# result = check_domain(test_domain, test_sensitive_words)
# logger.info(f'检测结果: {result}')
# logger.info(f'检测结果: {result}')

287
domainCheck/detect/chinaz.py
View File

@@ -11,15 +11,50 @@
import json # JSON处理模块，用于解析API响应数据
import os # 操作系统接口模块，用于获取脚本所在目录路径
import re # 正则表达式模块，用于提取网页中的标题和加密密钥
import threading
import time # 时间处理模块，用于生成时间戳
from typing import List, Optional # 类型提示
import quickjs # JavaScript执行引擎，用于执行JS加密代码生成签名
import requests # HTTP请求库，用于发送网络请求
from loguru import logger # 日志记录模块，用于记录程序运行日志
from requests.adapters import HTTPAdapter
def check_title(domain: str,sensitive_words: Optional[List[str]] = None, proxies: dict = None):
_CHINAZ_JS_CACHE = None
_SESSION_LOCAL = threading.local()
def _resolve_chinaz_timeout(proxies: dict = None, budget_seconds: Optional[float] = None) -> float:
timeout = float(1.8 if proxies else 2.8)
if budget_seconds not in (None, "", 0, "0"):
timeout = min(timeout, max(0.6, float(budget_seconds or 0.0)))
return max(0.6, timeout)
def _get_session():
session = getattr(_SESSION_LOCAL, "session", None)
if session is not None:
return session
session = requests.Session()
session.mount("http://", HTTPAdapter(pool_connections=32, pool_maxsize=64, max_retries=0))
session.mount("https://", HTTPAdapter(pool_connections=32, pool_maxsize=64, max_retries=0))
_SESSION_LOCAL.session = session
return session
def _load_chinaz_js():
global _CHINAZ_JS_CACHE
if _CHINAZ_JS_CACHE is not None:
return _CHINAZ_JS_CACHE
current_dir = os.path.dirname(__file__)
js_path = os.path.join(current_dir, "chinaz.js")
with open(js_path, "r", encoding="utf-8") as f:
_CHINAZ_JS_CACHE = f.read()
return _CHINAZ_JS_CACHE
def check_title(domain: str, sensitive_words: Optional[List[str]] = None, proxies: dict = None, budget_seconds: Optional[float] = None):
'''
检测域名标题并获取SEO数据
@@ -31,105 +66,133 @@ def check_title(domain: str,sensitive_words: Optional[List[str]] = None, proxies
if sensitive_words is None:
sensitive_words = []
# 添加重试机制
max_retries = 3
for retry in range(max_retries):
try:
# 构建SEO查询URL（站长工具网站）
url = f'https://seo.chinaz.com/{domain}'
# 发送GET请求获取页面内容，设置超时10秒
response = requests.get(url, timeout=10, proxies=proxies)
# 判断请求是否成功（HTTP 200表示成功）
if response.status_code == 200:
# 使用正则表达式提取网站标题（从id="site_title"的div标签中提取文本内容）
# 正则解释：匹配 id="site_title"> 开头，</div> 结尾，中间的内容
match = re.search(r'id="site_title">([^<]+)</div>', response.text)
# 使用正则表达式提取加密密钥enkey（用于后续API请求的身份验证）
# 正则解释：匹配 var enkey = '...' 中的单引号内容
pattern = r"var enkey = '([^']+)''"
try:
step_started_at = time.perf_counter()
print(
f"[chinaz-entry] domain={domain} proxy={'yes' if proxies else 'no'}",
flush=True,
)
session_started_at = time.perf_counter()
logger.info(f"站长之家阶段: domain={domain} | stage=session_init_start")
session = _get_session()
logger.info(
f"站长之家阶段: domain={domain} | stage=session_init_done "
f"| elapsed_ms={int((time.perf_counter() - session_started_at) * 1000)}"
)
# 构建SEO查询URL（站长工具网站）
url = f'https://seo.chinaz.com/{domain}'
page_started_at = time.perf_counter()
logger.info(
f"站长之家阶段: domain={domain} | stage=page_request_start | proxy={'yes' if proxies else 'no'}"
)
# 发送GET请求获取页面内容，设置超时并收口到步骤剩余预算内
response = session.get(
url,
timeout=_resolve_chinaz_timeout(proxies, budget_seconds=budget_seconds),
proxies=proxies,
)
logger.info(
f"站长之家阶段: domain={domain} | stage=page_request_done | status={response.status_code} "
f"| elapsed_ms={int((time.perf_counter() - page_started_at) * 1000)}"
)
# 判断请求是否成功（HTTP 200表示成功）
if response.status_code == 200:
# 使用正则表达式提取网站标题（从id="site_title"的div标签中提取文本内容）
# 正则解释：匹配 id="site_title"> 开头，</div> 结尾，中间的内容
match = re.search(r'id="site_title">([^<]+)</div>', response.text)
# 使用正则表达式提取加密密钥enkey（用于后续API请求的身份验证）
# 正则解释：匹配 var enkey = '...' 中的单引号内容
pattern = r"var enkey = '([^']+)''"
match_enkey = re.search(pattern, response.text)
# 如果上面的正则匹配失败，尝试匹配正确的单引号版本
if not match_enkey:
pattern = r"var enkey = '([^']+)" # 匹配 var enkey = '...' 格式
match_enkey = re.search(pattern, response.text)
# 如果上面的正则匹配失败，尝试匹配正确的单引号版本
if not match_enkey:
pattern = r"var enkey = '([^']+)" # 匹配 var enkey = '...' 格式
match_enkey = re.search(pattern, response.text)
# 如果找到enkey则提取并去除首尾空格，否则为空字符串
enkey = match_enkey.group(1).strip() if match_enkey else ''
# 提取网站标题，如果找到则返回，否则返回空字符串
title = match.group(1).strip() if match else ''
# 如果找到enkey则提取并去除首尾空格，否则为空字符串
enkey = match_enkey.group(1).strip() if match_enkey else ''
if title:
for sensitive_word in sensitive_words:
# 使用正则表达式检查是否包含敏感词
if re.search(sensitive_word, title, re.IGNORECASE):
logger.warning(f'检测到敏感词: {sensitive_word} 在结果: {title}')
return False, f'检测到敏感词: {sensitive_word} 在结果: {title}', ()
# 提取网站标题，如果找到则返回，否则返回空字符串
title = match.group(1).strip() if match else ''
logger.info(
f"站长之家阶段: domain={domain} | stage=page_parsed | title={'yes' if bool(title) else 'no'} "
f"| enkey={'yes' if bool(enkey) else 'no'}"
)
# 调用get_site_data获取详细SEO数据
success, result = get_site_data(domain, enkey, proxies=proxies)
# 检查网站分类
if success and result:
# 定义需要拉黑的分类
blacklist_categories = ['视频电影', '体育运动', '常用查询', '游戏网站', '游戏', '视频', '体育']
# 检查分类是否在黑名单中
if isinstance(result, dict):
# 检查可能的分类字段
category_fields = ['Category', 'category', '分类', '网站分类']
for field in category_fields:
if field in result:
category = result[field]
if isinstance(category, str):
for blacklist_category in blacklist_categories:
if blacklist_category in category:
logger.warning(f'检测到黑名单分类: {category} 在域名: {domain}')
return False, f'网站分类: {category} 属于黑名单分类', ()
# 检查Result是否为列表
elif isinstance(result, list):
for item in result:
if isinstance(item, dict):
category_fields = ['Category', 'category', '分类', '网站分类']
for field in category_fields:
if field in item:
category = item[field]
if isinstance(category, str):
for blacklist_category in blacklist_categories:
if blacklist_category in category:
logger.warning(f'检测到黑名单分类: {category} 在域名: {domain}')
return False, f'网站分类: {category} 属于黑名单分类', ()
return True, 'success', (success, result)
# 请求失败返回空标题和空元组
return True, '', ()
except Exception as e: # 捕获所有异常
error_str = str(e)
# 检查是否是需要重试的错误类型
if any(error_type in error_str for error_type in ['Too many open files', 'EOF occurred in violation of protocol', 'Max retries exceeded', 'Read timed out']):
logger.warning(f"站长工具检测第{retry+1}次失败: {domain}, 错误: {e}")
if retry < max_retries - 1:
time.sleep(2) # 等待2秒后重试
continue
else:
# 达到最大重试次数，返回失败但不拉黑
logger.error(f"站长工具检测失败，已达到最大重试次数: {domain}, 错误: {e}")
return True, 'failure', (False, str(e))
else:
# 其他错误直接返回
logger.error(f"站长工具检测失败: {domain}, 错误: {e}")
return False, str(e), ()
if title:
for sensitive_word in sensitive_words:
# 使用正则表达式检查是否包含敏感词
if re.search(sensitive_word, title, re.IGNORECASE):
logger.warning(f'检测到敏感词: {sensitive_word} 在结果: {title}')
return False, f'检测到敏感词: {sensitive_word} 在结果: {title}', ()
# 调用get_site_data获取详细SEO数据
seo_started_at = time.perf_counter()
logger.info(f"站长之家阶段: domain={domain} | stage=seo_request_start")
remaining_budget_seconds = None
if budget_seconds not in (None, "", 0, "0"):
remaining_budget_seconds = max(0.6, float(budget_seconds or 0.0) - (time.perf_counter() - step_started_at))
success, result = get_site_data(domain, enkey, proxies=proxies, budget_seconds=remaining_budget_seconds)
logger.info(
f"站长之家阶段: domain={domain} | stage=seo_request_done | success={success} "
f"| elapsed_ms={int((time.perf_counter() - seo_started_at) * 1000)}"
)
if not success:
return False, str(result or '站长工具 SEO 数据获取失败'), ()
# 检查网站分类
if success and result:
# 定义需要拉黑的分类
blacklist_categories = ['视频电影', '体育运动', '常用查询', '游戏网站', '游戏', '视频', '体育']
# 检查分类是否在黑名单中
if isinstance(result, dict):
# 检查可能的分类字段
category_fields = ['Category', 'category', '分类', '网站分类']
for field in category_fields:
if field in result:
category = result[field]
if isinstance(category, str):
for blacklist_category in blacklist_categories:
if blacklist_category in category:
logger.warning(f'检测到黑名单分类: {category} 在域名: {domain}')
return False, f'网站分类: {category} 属于黑名单分类', ()
# 检查Result是否为列表
elif isinstance(result, list):
for item in result:
if isinstance(item, dict):
category_fields = ['Category', 'category', '分类', '网站分类']
for field in category_fields:
if field in item:
category = item[field]
if isinstance(category, str):
for blacklist_category in blacklist_categories:
if blacklist_category in category:
logger.warning(f'检测到黑名单分类: {category} 在域名: {domain}')
return False, f'网站分类: {category} 属于黑名单分类', ()
return True, 'success', result
logger.debug(f"站长工具检测失败: {domain}, 状态码: {response.status_code}")
return False, f"站长工具页面返回状态码: {response.status_code}", ()
except Exception as e: # 捕获所有异常
error_str = str(e)
# 检查是否是需要重试的错误类型
if any(error_type in error_str for error_type in ['Too many open files', 'EOF occurred in violation of protocol', 'Max retries exceeded', 'Read timed out']):
logger.error(f"站长工具检测失败，重试链路已耗尽: {domain}, 错误: {e}")
return True, 'failure', (False, str(e))
logger.error(f"站长工具检测失败: {domain}, 错误: {e}")
return False, str(e), ()
# ========================= 【最终请求】 =========================
def get_site_data(domain, enkey: str, proxies: dict = None):
def get_site_data(domain, enkey: str, proxies: dict = None, budget_seconds: Optional[float] = None):
'''
获取站点的详细SEO数据
@@ -143,15 +206,10 @@ def get_site_data(domain, enkey: str, proxies: dict = None):
- (False, 错误信息): 查询失败
'''
try:
# 获取当前脚本所在目录（用于构建JS文件的完整路径）
current_dir = os.path.dirname(__file__)
# 构建JS加密代码文件的完整路径
js_path = os.path.join(current_dir, "chinaz.js")
# 读取JS加密代码文件（包含生成签名的函数）
with open(js_path, "r", encoding="utf-8") as f:
js_code = f.read()
session = _get_session()
js_code = _load_chinaz_js()
js_started_at = time.perf_counter()
logger.info(f"站长之家阶段: domain={domain} | stage=js_context_start")
# 创建JavaScript执行上下文（QuickJS引擎）
ctx = quickjs.Context()
@@ -166,6 +224,10 @@ def get_site_data(domain, enkey: str, proxies: dict = None):
# 获取当前时间戳（毫秒），用于防止缓存和重放攻击
ts = str(int(time.time() * 1000))
logger.info(
f"站长之家阶段: domain={domain} | stage=js_context_done "
f"| elapsed_ms={int((time.perf_counter() - js_started_at) * 1000)}"
)
# 构建请求参数字典（包含API所需的全部参数）
params = {
@@ -198,28 +260,43 @@ def get_site_data(domain, enkey: str, proxies: dict = None):
url = 'https://othertool.chinaz.com/GetTopRanked.ashx'
# 发送GET请求获取SEO数据
resp = requests.get(
api_started_at = time.perf_counter()
logger.info(
f"站长之家阶段: domain={domain} | stage=seo_api_request_start | proxy={'yes' if proxies else 'no'}"
)
resp = session.get(
url, # 请求URL
params=params, # 查询参数（自动拼接为URL参数）
headers=headers, # 请求头
timeout=10, # 超时时间10秒
timeout=_resolve_chinaz_timeout(proxies, budget_seconds=budget_seconds), # 进一步缩短超时，避免长时间卡住线程
proxies=proxies # 代理配置
)
logger.info(
f"站长之家阶段: domain={domain} | stage=seo_api_request_done | status={resp.status_code} "
f"| elapsed_ms={int((time.perf_counter() - api_started_at) * 1000)}"
)
# 判断请求是否成功
if resp.status_code == 200:
# 使用正则表达式提取JSON数据（API返回格式：callback(json_data)）
# 正则解释：匹配括号内的JSON内容
match = re.search(r'\((.*?)\)', resp.text)
if not match:
return False, "站长工具接口返回格式异常"
# 提取JSON字符串
json_str = match.group(1)
# 解析JSON数据为Python字典
data = json.loads(json_str)
logger.info(
f"站长之家阶段: domain={domain} | stage=seo_api_parsed "
f"| state_code={data.get('StateCode')}"
)
# 返回状态码和结果（StateCode为1表示成功）
return data['StateCode'] == 1, data['Result']
return False, f"站长工具接口返回状态码: {resp.status_code}"
except Exception as e: # 捕获所有异常
logger.error(e) # 记录错误日志
@@ -239,4 +316,4 @@ def get_site_data(domain, enkey: str, proxies: dict = None):
#
# title_3, seo_data_3 = check_title(test_domain_3)
# logger.info(f"域名标题: {title_3}")
# logger.info(f"SEO数据: {seo_data_3}")
# logger.info(f"SEO数据: {seo_data_3}")

122
domainCheck/detect/register.py
View File

@@ -8,13 +8,78 @@
@explain : 域名注册状态检测工具
'''
import json
import os
import threading
from datetime import datetime, timezone, timedelta # 日期时间处理，用于时区转换
from typing import Optional
import requests # HTTP请求库
from loguru import logger # 日志记录
import requests
import urllib3
_DIRECT_HTTP = requests.Session()
_DIRECT_HTTP.trust_env = False
_DIRECT_HTTP.mount("http://", requests.adapters.HTTPAdapter(pool_connections=256, pool_maxsize=512, max_retries=0))
_DIRECT_HTTP.mount("https://", requests.adapters.HTTPAdapter(pool_connections=256, pool_maxsize=512, max_retries=0))
_PROXY_MANAGERS = {}
_PROXY_MANAGER_LOCK = threading.Lock()
def check_register(domain: str, postfix: str = 'com',proxies: dict = None):
def _get_http_manager(proxies: Optional[dict] = None):
proxy_url = ""
if proxies:
proxy_url = str(proxies.get("https") or proxies.get("http") or "").strip()
if not proxy_url:
return _DIRECT_HTTP
with _PROXY_MANAGER_LOCK:
manager = _PROXY_MANAGERS.get(proxy_url)
if manager is None:
manager = requests.Session()
manager.trust_env = False
manager.mount("http://", requests.adapters.HTTPAdapter(pool_connections=128, pool_maxsize=256, max_retries=0))
manager.mount("https://", requests.adapters.HTTPAdapter(pool_connections=128, pool_maxsize=256, max_retries=0))
_PROXY_MANAGERS[proxy_url] = manager
return manager
def _request_register(url: str, *, proxies: dict = None, timeout=None):
manager = _get_http_manager(proxies)
request_timeout = timeout
if isinstance(timeout, urllib3.Timeout):
request_timeout = (timeout.connect_timeout, timeout.read_timeout)
return manager.get(url, timeout=request_timeout, proxies=proxies, allow_redirects=True)
def _resolve_register_timeout(proxies: Optional[dict], budget_seconds: Optional[float] = None):
if proxies:
connect_timeout = float(os.getenv("DOMAINCHECK_REGISTER_PROXY_CONNECT_TIMEOUT", "1.0") or 1.0)
read_timeout = float(os.getenv("DOMAINCHECK_REGISTER_PROXY_READ_TIMEOUT", "1.5") or 1.5)
total_timeout = float(os.getenv("DOMAINCHECK_REGISTER_PROXY_TOTAL_TIMEOUT", "2.2") or 2.2)
else:
connect_timeout = float(os.getenv("DOMAINCHECK_REGISTER_DIRECT_CONNECT_TIMEOUT", "1.2") or 1.2)
read_timeout = float(os.getenv("DOMAINCHECK_REGISTER_DIRECT_READ_TIMEOUT", "1.8") or 1.8)
total_timeout = float(os.getenv("DOMAINCHECK_REGISTER_DIRECT_TOTAL_TIMEOUT", "2.6") or 2.6)
connect_timeout = max(0.2, connect_timeout)
read_timeout = max(0.3, read_timeout)
total_timeout = max(max(connect_timeout, read_timeout), float(total_timeout or 0.0))
if budget_seconds not in (None, "", 0, "0"):
remaining_budget = max(0.6, float(budget_seconds or 0.0))
total_timeout = min(total_timeout, remaining_budget)
# requests 只原生支持 (connect, read)，这里把剩余预算重新切成更短的
# connect/read，避免单次 RDAP 请求把整个步骤预算一次性吃掉。
if total_timeout <= 1.0:
connect_timeout = min(connect_timeout, 0.35)
read_timeout = min(read_timeout, max(0.3, total_timeout - 0.2))
else:
connect_timeout = min(connect_timeout, max(0.35, total_timeout * 0.35))
read_timeout = min(read_timeout, max(0.45, total_timeout - connect_timeout))
connect_timeout = max(0.2, min(connect_timeout, total_timeout))
read_timeout = max(0.3, min(read_timeout, total_timeout))
return urllib3.Timeout(connect=connect_timeout, read=read_timeout, total=total_timeout)
def check_register(domain: str, postfix: str = 'com', proxies: dict = None, budget_seconds: Optional[float] = None):
'''
检测注册状态
:param domain: 待检测域名（不包含后缀）
@@ -25,34 +90,31 @@ def check_register(domain: str, postfix: str = 'com',proxies: dict = None):
url = f'https://rdap.verisign.com/{postfix}/v1/domain/{domain}' # 构建RDAP查询URL
# url = f'https://www.baidu.com' # 构建RDAP查询URL
# 添加重试机制
max_retries = 3
for retry in range(max_retries):
try:
response = requests.get(url, timeout=20, proxies=proxies) # 发送GET请求，设置超时5秒
if response.status_code == 200: # HTTP 200表示域名已注册
json_data = response.json() # 解析JSON响应
for item in json_data['events']: # 遍历事件列表
if item['eventAction'] == 'expiration': # 如果是过期时间事件
utc_time = datetime.fromisoformat(item['eventDate'].replace('Z', '+00:00')) # 解析UTC时间
beijing_tz = timezone(timedelta(hours=8)) # 创建北京时区（UTC+8）
beijing_time = utc_time.astimezone(beijing_tz) # 将UTC时间转换为北京时间
return 3, beijing_time.strftime("%Y-%m-%d %H:%M:%S") # 返回已注册状态和过期时间
elif response.status_code == 404: # HTTP 404表示域名不存在，可注册
# 可注册状态
return 2, '' # 返回可注册状态，过期时间为空
logger.error(response.status_code)
except requests.exceptions.RequestException as e:
# logger.warning(f"注册状态检测第{retry+1}次失败: {domain}, 错误: {e}")
if retry < max_retries - 1:
import time
time.sleep(2) # 等待2秒后重试
proxies=None
else:
# 达到最大重试次数，返回检测失败
logger.error(f"注册状态检测失败: {domain}, 已达到最大重试次数")
return -1, '' # -1表示检测失败，过期时间为空
mode = "proxy" if proxies else "direct"
timeout = _resolve_register_timeout(proxies, budget_seconds=budget_seconds)
response = None
try:
response = _request_register(url, proxies=proxies, timeout=timeout)
except Exception as e:
logger.warning(f"注册状态检测请求异常: {domain}, 模式: {mode}, 错误: {e}")
raise
if int(response.status_code) == 200: # HTTP 200表示域名已注册
json_data = response.json() if response.content else {} # 解析JSON响应
for item in json_data['events']: # 遍历事件列表
if item['eventAction'] == 'expiration': # 如果是过期时间事件
utc_time = datetime.fromisoformat(item['eventDate'].replace('Z', '+00:00')) # 解析UTC时间
beijing_tz = timezone(timedelta(hours=8)) # 创建北京时区（UTC+8）
beijing_time = utc_time.astimezone(beijing_tz) # 将UTC时间转换为北京时间
return 3, beijing_time.strftime("%Y-%m-%d %H:%M:%S") # 返回已注册状态和过期时间
return 3, ''
if int(response.status_code) == 404: # HTTP 404表示域名不存在，可注册
return 2, '' # 返回可注册状态，过期时间为空
error_message = f"rdap unexpected status {response.status_code}"
logger.warning(f"注册状态检测返回异常状态码: {domain}, 模式: {mode}, 状态码: {response.status_code}")
raise RuntimeError(error_message)

4372
domainCheck/detect_worker.py
View File

File diff suppressed because it is too large Load Diff

4
domainCheck/node_thread_counts.json
View File

@@ -1,4 +1,4 @@
{
"mainland-controller-01": 100,
"mainland-worker-01": 50
"mainland-controller-01": 2000,
"mainland-worker-01": 1200
}

14
domainCheck/proxy_config.json
View File

@@ -1,13 +1,13 @@
{
"proxy_enable": true,
"allow_direct": true,
"proxy_url": "http://211.101.244.154:18003/getProxy_batch.php?group=B&count=50",
"proxy_url": "http://211.101.244.154:18003/getProxy_batch.php?group=B&count=140",
"proxy_urls": [
"http://211.101.244.154:18003/getProxy_batch.php?group=B&count=50",
"http://211.101.244.154:18003/getProxy_batch.php?group=C&count=50",
"http://211.101.244.154:18003/getProxy_batch.php?group=D&count=50",
"http://211.101.244.154:18003/getProxy_batch.php?group=E&count=50",
"http://211.101.244.154:18008/getProxy_batch.php?group=F&count=50",
"http://211.101.244.154:18003/getProxy_batch.php?group=G&count=50"
"http://211.101.244.154:18003/getProxy_batch.php?group=B&count=140",
"http://211.101.244.154:18003/getProxy_batch.php?group=C&count=140",
"http://211.101.244.154:18003/getProxy_batch.php?group=D&count=140",
"http://211.101.244.154:18003/getProxy_batch.php?group=E&count=140",
"http://211.101.244.154:18003/getProxy_batch.php?group=F&count=140",
"http://211.101.244.154:18003/getProxy_batch.php?group=G&count=140"
]
}

8
domainCheck/runtime/runtime_settings.json Normal file
View File

@@ -0,0 +1,8 @@
{
"worker_mode": "linux-systemd",
"worker_service_name": "domaincheck-worker",
"api_service_name": "domaincheck-api",
"sync_agent_service_name": "domaincheck-sync-agent",
"worker_log_sync_enabled": true,
"worker_log_sync_mode": "full"
}

2156
domainCheck/runtime/sensitive_words.json Normal file
View File

File diff suppressed because it is too large Load Diff

8
domainCheck/runtime_settings.json Normal file
View File

@@ -0,0 +1,8 @@
{
"worker_mode": "linux-systemd",
"worker_service_name": "domaincheck-worker",
"api_service_name": "domaincheck-api",
"sync_agent_service_name": "domaincheck-sync-agent",
"worker_log_sync_enabled": true,
"worker_log_sync_mode": "full"
}

55
domainCheck/tests/test_database_claim_priority.py Normal file
View File

@@ -0,0 +1,55 @@
import unittest
import sys
sys.path.insert(0, "/www/wwwroot/getDomain/domainCheck")
from app.utils.database import _detect_job_item_step_priority
from app.utils.database import _ordered_step_claim_codes
from app.utils.database import _resolve_step_claim_quota
class DetectJobItemClaimPriorityTestCase(unittest.TestCase):
def test_ordered_step_claim_codes_prioritize_deeper_steps_first(self):
ordered = _ordered_step_claim_codes()
self.assertLess(ordered.index("detect_360_site"), ordered.index("detect_baidu_site"))
self.assertLess(ordered.index("detect_chinaz"), ordered.index("detect_360_site"))
self.assertGreater(ordered.index("detect_register"), ordered.index("detect_baidu_site"))
def test_later_pipeline_steps_are_prioritized_ahead_of_earlier_steps(self):
self.assertLess(
_detect_job_item_step_priority("detect_360_site"),
_detect_job_item_step_priority("detect_baidu_site"),
)
self.assertLess(
_detect_job_item_step_priority("detect_chinaz"),
_detect_job_item_step_priority("detect_360_site"),
)
self.assertLess(
_detect_job_item_step_priority("detect_aizhan"),
_detect_job_item_step_priority("detect_chinaz"),
)
def test_non_register_steps_are_prioritized_ahead_of_register(self):
self.assertLess(
_detect_job_item_step_priority("detect_baidu_site"),
_detect_job_item_step_priority("detect_register"),
)
self.assertLess(
_detect_job_item_step_priority("detect_wayback"),
_detect_job_item_step_priority("detect_register"),
)
def test_blank_step_code_has_lowest_priority(self):
self.assertGreater(
_detect_job_item_step_priority(""),
_detect_job_item_step_priority("detect_register"),
)
def test_step_claim_quota_defaults_to_quarter_window_with_floor(self):
self.assertEqual(64, _resolve_step_claim_quota(120))
self.assertEqual(300, _resolve_step_claim_quota(1200))
if __name__ == "__main__":
unittest.main()

225
domainCheck/tests/test_detect_worker_flush_retry.py Normal file
View File

@@ -0,0 +1,225 @@
import collections
import threading
import unittest
import sys
from unittest.mock import MagicMock
from unittest.mock import patch
sys.path.insert(0, "/www/wwwroot/getDomain/domainCheck")
from detect_worker import DetectWorker # noqa: E402
class _FakeDb:
def __init__(self, method_name: str):
self.method_name = method_name
self.calls = 0
def _run(self, batch):
self.calls += 1
return -1 if self.calls == 1 else len(batch)
def mark_detect_job_items_running_batch(self, batch):
return self._run(batch)
def finalize_detect_job_items_batch(self, batch):
return self._run(batch)
class DetectWorkerFlushRetryTest(unittest.TestCase):
def _build_worker(self, db):
worker = DetectWorker.__new__(DetectWorker)
worker.db = db
worker._task_local = threading.local()
worker._running_mark_lock = threading.Lock()
worker._pending_running_marks = collections.deque()
worker._last_running_mark_flush_at = 0.0
worker._job_finalize_lock = threading.Lock()
worker._pending_job_finalizations = collections.deque()
worker._last_job_finalize_flush_at = 0.0
worker.proxy_pool = []
worker.proxy_pool_lock = threading.Lock()
worker.proxy_failure_lock = threading.Lock()
worker.proxy_failure_counts = {}
worker.proxy_quarantine_until = {}
worker.proxy_source_by_key = {}
worker.proxy_source_failure_counts = {}
worker.proxy_source_quarantine_until = {}
worker.proxy_config = {"proxy_enable": True, "allow_direct": True}
worker.proxy_last_refresh_status = "test"
worker.trigger_proxy_refresh = MagicMock()
worker.runtime_settings = {}
worker.thread_count = 600
worker.detecting = True
worker._get_active_domain_threads = MagicMock(return_value=0)
return worker
def test_running_mark_flush_requeues_batch_when_db_write_fails(self):
worker = self._build_worker(_FakeDb("running"))
worker._pending_running_marks.extend([(101, "token-a"), (102, "token-b")])
updated = worker._flush_pending_running_marks(force=True, batch_limit=10)
self.assertEqual(0, updated)
self.assertEqual([(101, "token-a"), (102, "token-b")], list(worker._pending_running_marks))
updated = worker._flush_pending_running_marks(force=True, batch_limit=10)
self.assertEqual(2, updated)
self.assertEqual([], list(worker._pending_running_marks))
def test_job_finalization_flush_requeues_batch_when_db_write_fails(self):
worker = self._build_worker(_FakeDb("finalize"))
worker._pending_job_finalizations.extend(
[
{"job_item_id": 201, "claim_token": "token-a", "final_status": "completed"},
{"job_item_id": 202, "claim_token": "token-b", "final_status": "failed"},
]
)
updated = worker._flush_pending_job_finalizations(force=True, batch_limit=10)
self.assertEqual(0, updated)
self.assertEqual(2, len(worker._pending_job_finalizations))
updated = worker._flush_pending_job_finalizations(force=True, batch_limit=10)
self.assertEqual(2, updated)
self.assertEqual(0, len(worker._pending_job_finalizations))
def test_proxy_auth_failure_quarantines_whole_source_group(self):
worker = self._build_worker(_FakeDb("noop"))
bad_proxy = {"http": "http://bad-proxy", "https": "http://bad-proxy"}
good_proxy = {"http": "http://good-proxy", "https": "http://good-proxy"}
worker.proxy_pool = [
{"proxy": bad_proxy, "usage_count": 0, "source_tag": "D"},
{"proxy": good_proxy, "usage_count": 0, "source_tag": "E"},
]
worker.proxy_source_by_key[worker._proxy_key(bad_proxy)] = "D"
worker.proxy_source_by_key[worker._proxy_key(good_proxy)] = "E"
source_tag, failure_count, cooldown_seconds, removed = worker._mark_proxy_source_failure(
bad_proxy,
"Tunnel connection failed: 407 Proxy Authentication Required",
)
self.assertEqual("D", source_tag)
self.assertEqual(1, failure_count)
self.assertGreaterEqual(cooldown_seconds, 90)
self.assertEqual(1, removed)
self.assertTrue(worker._is_proxy_source_quarantined("D"))
self.assertEqual(
[{"proxy": good_proxy, "usage_count": 0, "source_tag": "E"}],
worker.proxy_pool,
)
def test_prepare_proxy_fetch_plan_respects_pool_shortage_instead_of_overfetching(self):
worker = self._build_worker(_FakeDb("noop"))
plan, meta = worker._prepare_proxy_fetch_plan(
[
"http://provider.local/get?group=B",
"http://provider.local/get?group=C",
"http://provider.local/get?group=D",
"http://provider.local/get?group=E",
"http://provider.local/get?group=F",
"http://provider.local/get?group=G",
],
current_pool_size=1000,
)
self.assertEqual(1, meta["rounds"])
self.assertGreaterEqual(meta["target_total"], 840)
self.assertLessEqual(meta["target_total"], 1200)
self.assertEqual(6, len(plan))
def test_prepare_proxy_fetch_plan_caps_active_rounds(self):
worker = self._build_worker(_FakeDb("noop"))
plan, meta = worker._prepare_proxy_fetch_plan(
[
"http://provider.local/get?group=B",
"http://provider.local/get?group=C",
"http://provider.local/get?group=D",
"http://provider.local/get?group=E",
"http://provider.local/get?group=F",
"http://provider.local/get?group=G",
],
current_pool_size=0,
)
self.assertLessEqual(meta["rounds"], 2)
self.assertEqual(meta["rounds"] * 6, len(plan))
def test_submit_backlog_limit_defaults_to_thread_count_window(self):
worker = self._build_worker(_FakeDb("noop"))
self.assertEqual(2000, worker._resolve_submit_backlog_limit(2000, 0))
self.assertEqual(2000, worker._resolve_submit_backlog_limit(2000, 1200))
def test_dispatch_capacity_can_ramp_to_full_thread_limit(self):
worker = self._build_worker(_FakeDb("noop"))
self.assertEqual(2000, worker._resolve_dispatch_capacity(2000, 0, 2000))
self.assertEqual(2000, worker._resolve_dispatch_capacity(2000, 1000, 1000))
def test_claim_batch_size_defaults_to_thread_count_window(self):
worker = self._build_worker(_FakeDb("noop"))
self.assertEqual(2000, worker._resolve_claim_batch_size(2000, 2000, live_active=0))
self.assertEqual(1500, worker._resolve_claim_batch_size(2000, 1500, live_active=800))
def test_get_proxy_for_step_can_force_single_direct_fallback_once(self):
worker = self._build_worker(_FakeDb("noop"))
worker._set_current_task_context(domain_id=1, is_step_task=True, task_mode="single_step")
worker._set_step_force_direct_once("百度site检测", True)
proxy = worker._get_proxy_for_step(1, "example.com", "百度site检测")
self.assertIsNone(proxy)
self.assertFalse(worker._consume_step_force_direct_once("百度site检测"))
def test_retry_external_issue_with_proxy_forces_next_attempt_direct_when_allowed(self):
worker = self._build_worker(_FakeDb("noop"))
worker._set_current_task_context(domain_id=1, is_step_task=True, task_mode="single_step")
worker.remove_proxy = MagicMock()
proxy = {"http": "http://bad-proxy", "https": "http://bad-proxy"}
should_retry = worker._should_retry_external_issue(
proxy,
"curl: (28) Connection timed out after 552 milliseconds",
"百度site检测",
"example.com",
)
self.assertTrue(should_retry)
worker.remove_proxy.assert_called_once_with(proxy)
worker.trigger_proxy_refresh.assert_called_once()
self.assertTrue(worker._consume_step_force_direct_once("百度site检测"))
def test_external_dependency_issue_recognizes_common_chinese_timeout_messages(self):
worker = self._build_worker(_FakeDb("noop"))
self.assertTrue(worker._is_external_dependency_issue("360搜索请求超时"))
self.assertTrue(worker._is_external_dependency_issue("百度site检测失败: 连接超时"))
self.assertTrue(worker._is_external_dependency_issue("站长之家检测失败: 网络异常"))
def test_run_detect_aizhan_passes_remaining_budget_to_detector(self):
worker = self._build_worker(_FakeDb("noop"))
worker._check_proxy_step_retry_budget = MagicMock(return_value=(False, ""))
worker._remaining_proxy_step_retry_budget_seconds = MagicMock(return_value=1.7)
worker._get_proxy_for_step = MagicMock(return_value=None)
worker._mark_detection_degraded = MagicMock()
worker._should_blacklist_result = MagicMock(return_value=False)
worker._record_step_result = MagicMock()
worker._should_rotate_proxy_on_error = MagicMock(return_value=False)
worker._is_external_dependency_issue = MagicMock(return_value=False)
worker._mark_detection_rejected = MagicMock()
worker._mark_detection_failed = MagicMock()
with patch("detect_worker.aizhan.check_aizhan", return_value=(True, "")) as mock_check:
result = worker._run_detect_aizhan(1, "example.com", [])
self.assertTrue(result)
self.assertEqual(1.7, mock_check.call_args.kwargs["budget_seconds"])
if __name__ == "__main__":
unittest.main()

64
domainCheck/tests/test_register_timeout_config.py Normal file
View File

@@ -0,0 +1,64 @@
import unittest
from unittest.mock import MagicMock, patch
import urllib3
from domainCheck.detect import register
class RegisterTimeoutConfigTests(unittest.TestCase):
def test_proxy_timeout_uses_bounded_total(self):
with patch.dict("os.environ", {}, clear=False):
timeout = register._resolve_register_timeout({"http": "http://127.0.0.1:8080"})
self.assertIsInstance(timeout, urllib3.Timeout)
self.assertEqual(timeout.connect_timeout, 1.0)
self.assertEqual(timeout.read_timeout, 1.5)
self.assertEqual(timeout.total, 2.2)
def test_direct_timeout_respects_override(self):
with patch.dict(
"os.environ",
{
"DOMAINCHECK_REGISTER_DIRECT_CONNECT_TIMEOUT": "0.9",
"DOMAINCHECK_REGISTER_DIRECT_READ_TIMEOUT": "1.4",
"DOMAINCHECK_REGISTER_DIRECT_TOTAL_TIMEOUT": "1.7",
},
clear=False,
):
timeout = register._resolve_register_timeout(None)
self.assertEqual(timeout.connect_timeout, 0.9)
self.assertEqual(timeout.read_timeout, 1.4)
self.assertEqual(timeout.total, 1.7)
def test_budget_clamps_proxy_timeout_into_remaining_window(self):
with patch.dict("os.environ", {}, clear=False):
timeout = register._resolve_register_timeout(
{"http": "http://127.0.0.1:8080"},
budget_seconds=0.8,
)
self.assertEqual(timeout.total, 0.8)
self.assertLessEqual(timeout.connect_timeout, 0.35)
self.assertLessEqual(timeout.read_timeout, 0.61)
def test_request_register_converts_urllib3_timeout_for_requests(self):
session = MagicMock()
response = MagicMock()
session.get.return_value = response
timeout = urllib3.Timeout(connect=1.1, read=1.7, total=2.4)
with patch("domainCheck.detect.register._get_http_manager", return_value=session):
result = register._request_register(
"https://rdap.verisign.com/com/v1/domain/example",
proxies={"http": "http://127.0.0.1:8080", "https": "http://127.0.0.1:8080"},
timeout=timeout,
)
self.assertIs(result, response)
session.get.assert_called_once_with(
"https://rdap.verisign.com/com/v1/domain/example",
timeout=(1.1, 1.7),
proxies={"http": "http://127.0.0.1:8080", "https": "http://127.0.0.1:8080"},
allow_redirects=True,
)
if __name__ == "__main__":
unittest.main()

33
domainCheck/tests/test_step_timeout_budgets.py Normal file
View File

@@ -0,0 +1,33 @@
import unittest
from unittest.mock import patch
from domainCheck.detect import aizhan, baidu, c360, chinaz
class StepTimeoutBudgetTests(unittest.TestCase):
def test_baidu_timeout_respects_remaining_budget(self):
with patch.dict("os.environ", {}, clear=False):
timeout = baidu._resolve_baidu_timeout({"http": "http://127.0.0.1:8080"}, budget_seconds=1.1)
self.assertLessEqual(timeout, 1.1)
self.assertGreaterEqual(timeout, 0.6)
def test_360_timeout_respects_remaining_budget(self):
with patch.dict("os.environ", {}, clear=False):
timeout = c360._resolve_360_timeout({"http": "http://127.0.0.1:8080"}, budget_seconds=0.9)
self.assertLessEqual(timeout, 0.9)
self.assertGreaterEqual(timeout, 0.6)
def test_chinaz_timeout_respects_remaining_budget(self):
timeout = chinaz._resolve_chinaz_timeout({"http": "http://127.0.0.1:8080"}, budget_seconds=1.3)
self.assertLessEqual(timeout, 1.3)
self.assertGreaterEqual(timeout, 0.6)
def test_aizhan_timeout_respects_remaining_budget(self):
with patch.dict("os.environ", {}, clear=False):
timeout = aizhan._resolve_aizhan_timeout({"http": "http://127.0.0.1:8080"}, budget_seconds=1.0)
self.assertLessEqual(timeout, 1.0)
self.assertGreaterEqual(timeout, 0.6)
if __name__ == "__main__":
unittest.main()

173
domainCheck/tests/test_wayback_detector_recent_years.py Normal file
View File

@@ -0,0 +1,173 @@
import sys
import time
import unittest
from unittest.mock import patch
sys.path.insert(0, "/www/wwwroot/getDomain/domainCheck")
from app.detectors.wayback_detector import WaybackDetector # noqa: E402
class WaybackDetectorRecentYearsTest(unittest.TestCase):
def setUp(self):
WaybackDetector._transient_backoff_until = 0.0
def test_filter_records_recent_years_drops_old_snapshots(self):
detector = WaybackDetector.__new__(WaybackDetector)
current_year = time.gmtime().tm_year
records = [
{"timestamp": f"{current_year}0101000000", "digest": "new"},
{"timestamp": f"{current_year - 2}0101000000", "digest": "mid"},
{"timestamp": f"{current_year - 6}0101000000", "digest": "old"},
]
filtered = detector._filter_records_recent_years(records, recent_years=5)
self.assertEqual(
[
f"{current_year}0101000000",
f"{current_year - 2}0101000000",
],
[item["timestamp"] for item in filtered],
)
def test_scan_snapshots_only_counts_recent_years_window(self):
detector = WaybackDetector.__new__(WaybackDetector)
current_year = time.gmtime().tm_year
detector._fetch_cdx_records_with_meta = lambda domain, limit=None, fast_latest=False: {
"records": [
{"timestamp": f"{current_year}0101000000", "digest": "latest"},
{"timestamp": f"{current_year - 1}0101000000", "digest": "prev"},
{"timestamp": f"{current_year - 6}0101000000", "digest": "old"},
]
}
detector._load_cached_records = lambda domain: None
detector._save_cached_records = lambda domain, records: None
detector._save_cached_timestamps = lambda domain, timestamps: None
detector._fetch_snapshot_title = lambda domain, timestamp: {
"timestamp": timestamp,
"title": f"title-{timestamp}",
"ok": True,
}
detector._normalize_title = lambda title: title
detector._find_sensitive_word = lambda title, words: None
detector._log_info = lambda message: None
detector._handle_exception = lambda exc, domain: None
result = detector.scan_snapshots("example.com", sensitive_words=[], recent_years=5)
self.assertEqual(2, result["checked_snapshot_count"])
self.assertEqual(2, result["fetched_snapshot_count"])
self.assertEqual(sorted([current_year - 1, current_year]), result["snapshot_years"])
def test_scan_snapshots_limits_records_per_domain(self):
detector = WaybackDetector.__new__(WaybackDetector)
current_year = time.gmtime().tm_year
records = [
{"timestamp": f"{current_year}01010{i}0000", "digest": f"d{i}"}
for i in range(6)
]
detector._fetch_cdx_records_with_meta = lambda domain, limit=None, fast_latest=False: {
"records": records,
"error": None,
}
detector._load_cached_records = lambda domain: None
detector._save_cached_records = lambda domain, records: None
detector._save_cached_timestamps = lambda domain, timestamps: None
detector._fetch_snapshot_title = lambda domain, timestamp: {
"timestamp": timestamp,
"title": f"title-{timestamp}",
"ok": True,
}
detector._normalize_title = lambda title: title
detector._find_sensitive_word = lambda title, words: None
detector._log_info = lambda message: None
detector._handle_exception = lambda exc, domain: None
with patch("app.detectors.wayback_detector.config.WAYBACK_MAX_RECORDS", 3):
result = detector.scan_snapshots("example.com", sensitive_words=[], recent_years=5)
self.assertEqual(3, result["checked_snapshot_count"])
# 最新快照会先独立尝试一次，再进入裁剪后的扫描窗口。
self.assertEqual(4, result["fetched_snapshot_count"])
def test_scan_snapshots_fast_degrades_when_latest_cdx_is_transient_failure(self):
detector = WaybackDetector.__new__(WaybackDetector)
detector._fetch_cdx_records_with_meta = lambda domain, limit=None, fast_latest=False: {
"records": [],
"error": "HTTPSConnectionPool(host='web.archive.org', port=443): Max retries exceeded",
}
detector._load_cached_records = lambda domain: [
{"timestamp": "20260101000000", "digest": "d1"},
{"timestamp": "20250101000000", "digest": "d2"},
]
detector._save_cached_records = lambda domain, records: None
detector._save_cached_timestamps = lambda domain, timestamps: None
detector._fetch_snapshot_title = lambda domain, timestamp: self.fail("should not fetch snapshot titles")
detector._normalize_title = lambda title: title
detector._find_sensitive_word = lambda title, words: None
detector._log_info = lambda message: None
detector._handle_exception = lambda exc, domain: None
result = detector.scan_snapshots("example.com", sensitive_words=[], recent_years=5)
self.assertEqual(0, result["checked_snapshot_count"])
self.assertGreaterEqual(result["failed_snapshot_count"], 1)
self.assertGreaterEqual(result["request_error_count"], 1)
def test_scan_snapshots_fast_degrades_when_latest_snapshot_is_transient_failure(self):
detector = WaybackDetector.__new__(WaybackDetector)
detector._fetch_cdx_records_with_meta = lambda domain, limit=None, fast_latest=False: {
"records": [{"timestamp": "20260101000000", "digest": "latest"}] if fast_latest else [
{"timestamp": "20260101000000", "digest": "latest"},
{"timestamp": "20250101000000", "digest": "older"},
],
"error": None,
}
detector._load_cached_records = lambda domain: None
detector._save_cached_records = lambda domain, records: None
detector._save_cached_timestamps = lambda domain, timestamps: None
detector._fetch_snapshot_title = lambda domain, timestamp: {
"timestamp": timestamp,
"title": "",
"ok": False,
"error": "HTTPSConnectionPool(host='web.archive.org', port=443): Max retries exceeded",
}
detector._normalize_title = lambda title: title
detector._find_sensitive_word = lambda title, words: None
detector._log_info = lambda message: None
detector._handle_exception = lambda exc, domain: None
result = detector.scan_snapshots("example.com", sensitive_words=[], recent_years=5)
self.assertEqual(1, result["checked_snapshot_count"])
self.assertEqual(0, result["fetched_snapshot_count"])
self.assertGreaterEqual(result["failed_snapshot_count"], 1)
self.assertTrue(
any("latest_snapshot:" in item for item in result["request_errors"])
)
def test_scan_snapshots_short_circuits_when_transient_backoff_active(self):
detector = WaybackDetector.__new__(WaybackDetector)
WaybackDetector._transient_backoff_until = time.time() + 5
detector._fetch_cdx_records_with_meta = lambda *args, **kwargs: self.fail("should not request cdx during backoff")
detector._load_cached_records = lambda domain: None
detector._save_cached_records = lambda domain, records: None
detector._save_cached_timestamps = lambda domain, timestamps: None
detector._fetch_snapshot_title = lambda domain, timestamp: self.fail("should not request snapshot during backoff")
detector._normalize_title = lambda title: title
detector._find_sensitive_word = lambda title, words: None
detector._log_info = lambda message: None
detector._handle_exception = lambda exc, domain: None
result = detector.scan_snapshots("example.com", sensitive_words=[], recent_years=5)
self.assertEqual(0, result["checked_snapshot_count"])
self.assertEqual(0, result["fetched_snapshot_count"])
self.assertEqual(1, result["request_error_count"])
self.assertTrue(any("wayback_backoff_active:" in item for item in result["request_errors"]))
if __name__ == "__main__":
unittest.main()

2
domainCheck/thread_count.json
View File

@@ -1,3 +1,3 @@
{
"thread_count": "5"
"thread_count": "100"
}