318 lines
10 KiB
Python
318 lines
10 KiB
Python
# -*- coding: UTF-8 -*-
|
||
'''
|
||
@Project :domainScanDemo
|
||
@File :config.py
|
||
@IDE :PyCharm
|
||
@Author :梦伴
|
||
@Date :2026/4/9 0:08
|
||
@explain : 系统配置
|
||
'''
|
||
|
||
import os
|
||
import sys
|
||
from dotenv import load_dotenv
|
||
|
||
|
||
def _safe_echo(message):
|
||
"""
|
||
在不同 Windows 控制台编码下安全输出文本，避免导入阶段因中文打印失败。
|
||
"""
|
||
try:
|
||
print(message)
|
||
except UnicodeEncodeError:
|
||
try:
|
||
encoding = sys.stdout.encoding or 'utf-8'
|
||
sys.stdout.buffer.write((message + '\n').encode(encoding, errors='replace'))
|
||
except Exception:
|
||
pass
|
||
|
||
# 确定基础目录
|
||
if hasattr(sys, '_MEIPASS'):
|
||
# PyInstaller 打包后的临时目录
|
||
BASE_DIR = sys._MEIPASS
|
||
else:
|
||
# 开发环境目录
|
||
BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||
|
||
|
||
def _detect_install_root(base_dir):
|
||
normalized = os.path.abspath(str(base_dir or ""))
|
||
marker = f"{os.sep}releases{os.sep}"
|
||
if marker in normalized:
|
||
return normalized.split(marker, 1)[0]
|
||
return ""
|
||
|
||
|
||
def _resolve_runtime_root(base_dir):
|
||
explicit_root = str(os.getenv('DOMAINCHECK_RUNTIME_ROOT', '') or '').strip()
|
||
if explicit_root:
|
||
return os.path.abspath(explicit_root)
|
||
install_root = _detect_install_root(base_dir)
|
||
if install_root:
|
||
return os.path.join(install_root, 'runtime', 'domainCheck')
|
||
return os.path.abspath(base_dir)
|
||
|
||
|
||
def _load_environment(base_dir):
|
||
install_root = _detect_install_root(base_dir)
|
||
runtime_root = _resolve_runtime_root(base_dir)
|
||
candidate_paths = [
|
||
str(os.getenv('DOMAINCHECK_ENV_FILE', '') or '').strip(),
|
||
os.path.join(base_dir, '.env'),
|
||
os.path.join(runtime_root, '.env'),
|
||
os.path.join(os.getcwd(), '.env'),
|
||
]
|
||
if install_root:
|
||
candidate_paths.append(os.path.join(install_root, 'domainCheck', '.env'))
|
||
|
||
seen = set()
|
||
for path in candidate_paths:
|
||
normalized = os.path.abspath(path) if path else ''
|
||
if not normalized or normalized in seen:
|
||
continue
|
||
seen.add(normalized)
|
||
if os.path.exists(normalized):
|
||
load_dotenv(normalized)
|
||
_safe_echo(f"成功加载环境变量文件: {normalized}")
|
||
return normalized
|
||
|
||
load_dotenv()
|
||
fallback_path = os.path.join(base_dir, '.env')
|
||
_safe_echo(f"环境变量文件不存在: {fallback_path}，使用默认环境变量")
|
||
return ""
|
||
|
||
# 加载环境变量
|
||
env_path = _load_environment(BASE_DIR)
|
||
RUNTIME_ROOT = _resolve_runtime_root(BASE_DIR)
|
||
|
||
|
||
class Config:
|
||
"""
|
||
系统配置
|
||
"""
|
||
|
||
# 数据库配置
|
||
DB_HOST = os.getenv('DB_HOST', 'localhost')
|
||
DB_PORT = int(os.getenv('DB_PORT', 5432))
|
||
DB_DATABASE = os.getenv('DB_DATABASE', 'domain_scan_db')
|
||
DB_USER = os.getenv('DB_USER', 'postgres')
|
||
DB_PASSWORD = os.getenv('DB_PASSWORD', 'postgres')
|
||
DB_POOL_SIZE = int(os.getenv('DB_POOL_SIZE', 64))
|
||
DB_POOL_WARM_SIZE = int(os.getenv('DB_POOL_WARM_SIZE', 8))
|
||
DB_POOL_IDLE_KEEP_MAX = int(os.getenv('DB_POOL_IDLE_KEEP_MAX', DB_POOL_SIZE))
|
||
DB_POOL_HEALTHCHECK_INTERVAL = float(os.getenv('DB_POOL_HEALTHCHECK_INTERVAL', 30))
|
||
DB_POOL_ACQUIRE_TIMEOUT = float(os.getenv('DB_POOL_ACQUIRE_TIMEOUT', 20))
|
||
|
||
# 消息队列配置
|
||
RABBITMQ_HOST = os.getenv('RABBITMQ_HOST', 'localhost')
|
||
RABBITMQ_PORT = int(os.getenv('RABBITMQ_PORT', 5672))
|
||
RABBITMQ_USER = os.getenv('RABBITMQ_USER', 'guest')
|
||
RABBITMQ_PASSWORD = os.getenv('RABBITMQ_PASSWORD', 'guest')
|
||
RABBITMQ_VHOST = os.getenv('RABBITMQ_VHOST', '/')
|
||
|
||
# Redis配置
|
||
REDIS_HOST = os.getenv('REDIS_HOST', 'localhost')
|
||
REDIS_PORT = int(os.getenv('REDIS_PORT', 6379))
|
||
REDIS_PASSWORD = os.getenv('REDIS_PASSWORD', '')
|
||
REDIS_DB = int(os.getenv('REDIS_DB', 0))
|
||
NODE_CODE = os.getenv('NODE_CODE', 'mainland-worker-01')
|
||
NODE_REGION = os.getenv('NODE_REGION', 'mainland')
|
||
NODE_ROLE = os.getenv('NODE_ROLE', 'worker')
|
||
|
||
# 聚名网配置
|
||
JUMING_COOKIE = os.getenv('JUMING_COOKIE', '')
|
||
JUMING_REFERER = os.getenv('JUMING_REFERER', 'https://www.juming.com/')
|
||
JUMING_USER_AGENT = os.getenv('JUMING_USER_AGENT', 'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/146.0.0.0 Safari/537.36')
|
||
|
||
# 代理配置
|
||
PROXY_ENABLED = os.getenv('PROXY_ENABLED', 'false').lower() == 'true'
|
||
PROXY_URL = os.getenv('PROXY_URL', '')
|
||
|
||
# 检测配置
|
||
DETECT_TIMEOUT = int(os.getenv('DETECT_TIMEOUT', 30))
|
||
DETECT_RETRY_COUNT = int(os.getenv('DETECT_RETRY_COUNT', 3))
|
||
DETECT_CONCURRENCY = int(os.getenv('DETECT_CONCURRENCY', 10))
|
||
WAYBACK_CDX_TIMEOUT = int(os.getenv('WAYBACK_CDX_TIMEOUT', 2))
|
||
WAYBACK_SNAPSHOT_TIMEOUT = int(os.getenv('WAYBACK_SNAPSHOT_TIMEOUT', 2))
|
||
WAYBACK_RETRY_COUNT = int(os.getenv('WAYBACK_RETRY_COUNT', 0))
|
||
WAYBACK_REQUEST_DELAY = float(os.getenv('WAYBACK_REQUEST_DELAY', 0))
|
||
WAYBACK_PROGRESS_INTERVAL = int(os.getenv('WAYBACK_PROGRESS_INTERVAL', 500))
|
||
WAYBACK_DOMAIN_CONCURRENCY = int(os.getenv('WAYBACK_DOMAIN_CONCURRENCY', 2))
|
||
WAYBACK_MAX_RECORDS = int(os.getenv('WAYBACK_MAX_RECORDS', 4))
|
||
WAYBACK_TRANSIENT_BACKOFF_SECONDS = float(os.getenv('WAYBACK_TRANSIENT_BACKOFF_SECONDS', 8))
|
||
WAYBACK_TITLE_MAX_BYTES = int(os.getenv('WAYBACK_TITLE_MAX_BYTES', 65536))
|
||
WAYBACK_TIMESTAMP_CACHE_TTL = int(os.getenv('WAYBACK_TIMESTAMP_CACHE_TTL', 86400))
|
||
WAYBACK_TITLE_CACHE_TTL = int(os.getenv('WAYBACK_TITLE_CACHE_TTL', 2592000))
|
||
WAYBACK_USER_AGENT = os.getenv(
|
||
'WAYBACK_USER_AGENT',
|
||
'Mozilla/5.0 (Windows NT 10.0; Win64; x64) AppleWebKit/537.36 (KHTML, like Gecko) Chrome/146.0.0.0 Safari/537.36'
|
||
)
|
||
|
||
# 域名配置
|
||
DOMAIN_TLDS = ['com', 'net']
|
||
DOMAIN_BATCH_SIZE = int(os.getenv('DOMAIN_BATCH_SIZE', 1000))
|
||
|
||
# 日志配置
|
||
LOG_LEVEL = os.getenv('LOG_LEVEL', 'INFO')
|
||
LOG_FILE = os.getenv('LOG_FILE', 'app.log')
|
||
|
||
# 任务配置
|
||
TASK_PRIORITY = {
|
||
'rdap': 10,
|
||
'wayback': 8,
|
||
'baidu': 6,
|
||
'qihu360': 5,
|
||
'google': 5,
|
||
'chinaz': 4,
|
||
'aizhan': 4,
|
||
'juziseo': 3,
|
||
'jucha': 3
|
||
}
|
||
|
||
# 敏感词配置 - 现在从数据库加载
|
||
SENSITIVE_WORDS = []
|
||
|
||
# 检测项配置
|
||
DETECT_ITEMS = {
|
||
'rdap': True,
|
||
'wayback': True,
|
||
'baidu': True,
|
||
'qihu360': True,
|
||
'google': True,
|
||
'chinaz': True,
|
||
'aizhan': True,
|
||
'juziseo': True,
|
||
'jucha': True
|
||
}
|
||
|
||
# 目录配置
|
||
BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
|
||
RUNTIME_ROOT = RUNTIME_ROOT
|
||
DATA_DIR = os.path.join(RUNTIME_ROOT, 'data')
|
||
LOG_DIR = os.path.join(RUNTIME_ROOT, 'logs')
|
||
|
||
# 确保目录存在
|
||
os.makedirs(DATA_DIR, exist_ok=True)
|
||
os.makedirs(LOG_DIR, exist_ok=True)
|
||
|
||
|
||
# 从数据库加载敏感词
|
||
def load_sensitive_words():
|
||
"""
|
||
从数据库加载敏感词
|
||
"""
|
||
try:
|
||
from app.utils.database import Database
|
||
db = Database()
|
||
sensitive_words = db.get_sensitive_words()
|
||
words = [word['word'] for word in sensitive_words]
|
||
db.close()
|
||
return words
|
||
except Exception as e:
|
||
_safe_echo(f"加载敏感词失败: {e}")
|
||
return []
|
||
|
||
# 从文件加载检测选项
|
||
def load_detect_options():
|
||
"""
|
||
从文件加载检测选项
|
||
"""
|
||
try:
|
||
default_order = [
|
||
'detect_register',
|
||
'detect_baidu_site',
|
||
'detect_360_site',
|
||
'detect_chinaz',
|
||
'detect_aizhan',
|
||
'detect_wayback',
|
||
'detect_jucha',
|
||
'detect_juziseo',
|
||
]
|
||
import json
|
||
import os
|
||
defaults = {
|
||
'detect_register': True,
|
||
'detect_wayback': True,
|
||
'detect_chinaz': True,
|
||
'detect_aizhan': True,
|
||
'detect_baidu_site': True,
|
||
'detect_360_site': True,
|
||
'detect_jucha': False,
|
||
'detect_juziseo': False,
|
||
'detect_order': default_order,
|
||
}
|
||
if os.path.exists('detect_options.json'):
|
||
with open('detect_options.json', 'r', encoding='utf-8') as f:
|
||
detect_options = json.load(f)
|
||
defaults.update(detect_options)
|
||
if defaults.get('detect_whois') or defaults.get('detect_beian') or defaults.get('detect_intercept'):
|
||
defaults['detect_jucha'] = True
|
||
if defaults.get('detect_juziseo_outlink'):
|
||
defaults['detect_juziseo'] = True
|
||
order = defaults.get('detect_order') or []
|
||
normalized_order = [key for key in order if key in default_order]
|
||
for key in default_order:
|
||
if key not in normalized_order:
|
||
normalized_order.append(key)
|
||
defaults['detect_order'] = normalized_order
|
||
return defaults
|
||
except Exception as e:
|
||
_safe_echo(f"加载检测选项失败: {e}")
|
||
return {
|
||
'detect_register': True,
|
||
'detect_wayback': True,
|
||
'detect_chinaz': True,
|
||
'detect_aizhan': True,
|
||
'detect_baidu_site': True,
|
||
'detect_360_site': True,
|
||
'detect_jucha': False,
|
||
'detect_juziseo': False,
|
||
}
|
||
|
||
# 导出配置
|
||
config = Config()
|
||
# 加载检测选项
|
||
config.DETECT_OPTIONS = load_detect_options()
|
||
|
||
# 延迟加载敏感词，避免循环导入
|
||
def load_sensitive_words_lazy():
|
||
"""
|
||
延迟加载敏感词
|
||
"""
|
||
return load_sensitive_words()
|
||
|
||
# 设置敏感词属性为延迟加载
|
||
config.load_sensitive_words = load_sensitive_words_lazy
|
||
|
||
# 检查检测类型是否应该执行
|
||
def should_detect(detect_type):
|
||
"""
|
||
检查检测类型是否应该执行
|
||
|
||
:param detect_type: 检测类型
|
||
:return: bool - 是否应该执行
|
||
"""
|
||
# 检测类型映射
|
||
type_mapping = {
|
||
'register': 'detect_register',
|
||
'wayback': 'detect_wayback',
|
||
'chinaz': 'detect_chinaz',
|
||
'aizhan': 'detect_aizhan',
|
||
'baidu_site': 'detect_baidu_site',
|
||
'360_site': 'detect_360_site',
|
||
'whois': 'detect_jucha',
|
||
'beian': 'detect_jucha',
|
||
'intercept': 'detect_jucha',
|
||
'jucha': 'detect_jucha',
|
||
'juziseo': 'detect_juziseo',
|
||
'juziseo_outlink': 'detect_juziseo'
|
||
}
|
||
|
||
# 获取对应的配置键
|
||
config_key = type_mapping.get(detect_type)
|
||
if not config_key:
|
||
return True # 默认执行
|
||
|
||
# 检查是否在配置中，默认为True
|
||
return config.DETECT_OPTIONS.get(config_key, True)
|