admin/getDomain
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

d

Browse Source
This commit is contained in:
Your Name
2026-04-22 14:13:21 +08:00
parent e0406b5d0e
commit 7cbde2aa78
145 changed files with 23086 additions and 2243 deletions
Download Patch File Download Diff File Expand all files Collapse all files

77
domainCheck/app/config.py
View File

@@ -34,14 +34,56 @@ else:
# 开发环境目录
BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
# 加载环境变量
env_path = os.path.join(BASE_DIR, '.env')
if os.path.exists(env_path):
load_dotenv(env_path)
_safe_echo(f"成功加载环境变量文件: {env_path}")
else:
def _detect_install_root(base_dir):
normalized = os.path.abspath(str(base_dir or ""))
marker = f"{os.sep}releases{os.sep}"
if marker in normalized:
return normalized.split(marker, 1)[0]
return ""
def _resolve_runtime_root(base_dir):
explicit_root = str(os.getenv('DOMAINCHECK_RUNTIME_ROOT', '') or '').strip()
if explicit_root:
return os.path.abspath(explicit_root)
install_root = _detect_install_root(base_dir)
if install_root:
return os.path.join(install_root, 'runtime', 'domainCheck')
return os.path.abspath(base_dir)
def _load_environment(base_dir):
install_root = _detect_install_root(base_dir)
runtime_root = _resolve_runtime_root(base_dir)
candidate_paths = [
str(os.getenv('DOMAINCHECK_ENV_FILE', '') or '').strip(),
os.path.join(base_dir, '.env'),
os.path.join(runtime_root, '.env'),
os.path.join(os.getcwd(), '.env'),
]
if install_root:
candidate_paths.append(os.path.join(install_root, 'domainCheck', '.env'))
seen = set()
for path in candidate_paths:
normalized = os.path.abspath(path) if path else ''
if not normalized or normalized in seen:
continue
seen.add(normalized)
if os.path.exists(normalized):
load_dotenv(normalized)
_safe_echo(f"成功加载环境变量文件: {normalized}")
return normalized
load_dotenv()
_safe_echo(f"环境变量文件不存在: {env_path}，使用默认环境变量")
fallback_path = os.path.join(base_dir, '.env')
_safe_echo(f"环境变量文件不存在: {fallback_path}，使用默认环境变量")
return ""
# 加载环境变量
env_path = _load_environment(BASE_DIR)
RUNTIME_ROOT = _resolve_runtime_root(BASE_DIR)
class Config:
@@ -55,7 +97,11 @@ class Config:
DB_DATABASE = os.getenv('DB_DATABASE', 'domain_scan_db')
DB_USER = os.getenv('DB_USER', 'postgres')
DB_PASSWORD = os.getenv('DB_PASSWORD', 'postgres')
DB_POOL_SIZE = int(os.getenv('DB_POOL_SIZE', 5))
DB_POOL_SIZE = int(os.getenv('DB_POOL_SIZE', 64))
DB_POOL_WARM_SIZE = int(os.getenv('DB_POOL_WARM_SIZE', 8))
DB_POOL_IDLE_KEEP_MAX = int(os.getenv('DB_POOL_IDLE_KEEP_MAX', DB_POOL_SIZE))
DB_POOL_HEALTHCHECK_INTERVAL = float(os.getenv('DB_POOL_HEALTHCHECK_INTERVAL', 30))
DB_POOL_ACQUIRE_TIMEOUT = float(os.getenv('DB_POOL_ACQUIRE_TIMEOUT', 20))
# 消息队列配置
RABBITMQ_HOST = os.getenv('RABBITMQ_HOST', 'localhost')
@@ -86,12 +132,14 @@ class Config:
DETECT_TIMEOUT = int(os.getenv('DETECT_TIMEOUT', 30))
DETECT_RETRY_COUNT = int(os.getenv('DETECT_RETRY_COUNT', 3))
DETECT_CONCURRENCY = int(os.getenv('DETECT_CONCURRENCY', 10))
WAYBACK_CDX_TIMEOUT = int(os.getenv('WAYBACK_CDX_TIMEOUT', 15))
WAYBACK_SNAPSHOT_TIMEOUT = int(os.getenv('WAYBACK_SNAPSHOT_TIMEOUT', 12))
WAYBACK_RETRY_COUNT = int(os.getenv('WAYBACK_RETRY_COUNT', 2))
WAYBACK_CDX_TIMEOUT = int(os.getenv('WAYBACK_CDX_TIMEOUT', 2))
WAYBACK_SNAPSHOT_TIMEOUT = int(os.getenv('WAYBACK_SNAPSHOT_TIMEOUT', 2))
WAYBACK_RETRY_COUNT = int(os.getenv('WAYBACK_RETRY_COUNT', 0))
WAYBACK_REQUEST_DELAY = float(os.getenv('WAYBACK_REQUEST_DELAY', 0))
WAYBACK_PROGRESS_INTERVAL = int(os.getenv('WAYBACK_PROGRESS_INTERVAL', 500))
WAYBACK_DOMAIN_CONCURRENCY = int(os.getenv('WAYBACK_DOMAIN_CONCURRENCY', 3))
WAYBACK_DOMAIN_CONCURRENCY = int(os.getenv('WAYBACK_DOMAIN_CONCURRENCY', 2))
WAYBACK_MAX_RECORDS = int(os.getenv('WAYBACK_MAX_RECORDS', 4))
WAYBACK_TRANSIENT_BACKOFF_SECONDS = float(os.getenv('WAYBACK_TRANSIENT_BACKOFF_SECONDS', 8))
WAYBACK_TITLE_MAX_BYTES = int(os.getenv('WAYBACK_TITLE_MAX_BYTES', 65536))
WAYBACK_TIMESTAMP_CACHE_TTL = int(os.getenv('WAYBACK_TIMESTAMP_CACHE_TTL', 86400))
WAYBACK_TITLE_CACHE_TTL = int(os.getenv('WAYBACK_TITLE_CACHE_TTL', 2592000))
@@ -139,8 +187,9 @@ class Config:
# 目录配置
BASE_DIR = os.path.dirname(os.path.dirname(os.path.abspath(__file__)))
DATA_DIR = os.path.join(BASE_DIR, 'data')
LOG_DIR = os.path.join(BASE_DIR, 'logs')
RUNTIME_ROOT = RUNTIME_ROOT
DATA_DIR = os.path.join(RUNTIME_ROOT, 'data')
LOG_DIR = os.path.join(RUNTIME_ROOT, 'logs')
# 确保目录存在
os.makedirs(DATA_DIR, exist_ok=True)

194
domainCheck/app/detectors/wayback_detector.py
View File

@@ -32,6 +32,8 @@ class WaybackDetector(BaseDetector):
"""
TITLE_PATTERN = re.compile(r'<title[^>]*>(.*?)</title>', re.IGNORECASE | re.DOTALL)
_transient_backoff_until = 0.0
_transient_backoff_lock = threading.Lock()
def __init__(self):
"""
@@ -56,7 +58,7 @@ class WaybackDetector(BaseDetector):
allowed_methods=frozenset(["GET"]),
raise_on_status=False,
)
adapter = HTTPAdapter(max_retries=retry, pool_connections=10, pool_maxsize=10)
adapter = HTTPAdapter(max_retries=retry, pool_connections=128, pool_maxsize=256)
session.mount('http://', adapter)
session.mount('https://', adapter)
session.headers.update({
@@ -65,6 +67,43 @@ class WaybackDetector(BaseDetector):
})
return session
@staticmethod
def _resolve_timeout(total_timeout, connect_cap=3):
total = max(1.0, float(total_timeout or 0))
return (min(float(connect_cap), total), total)
@staticmethod
def _is_transient_request_error(message):
lowered = str(message or '').lower()
keywords = (
'timeout',
'timed out',
'connecttimeout',
'read timeout',
'connection reset',
'connection refused',
'connection aborted',
'name or service not known',
'temporary failure',
'max retries exceeded',
'ssl',
)
return any(keyword in lowered for keyword in keywords)
@classmethod
def _transient_backoff_remaining_seconds(cls):
with cls._transient_backoff_lock:
remaining = float(cls._transient_backoff_until or 0) - time.time()
return max(0.0, remaining)
@classmethod
def _trip_transient_backoff(cls):
backoff_seconds = max(0.0, float(getattr(config, "WAYBACK_TRANSIENT_BACKOFF_SECONDS", 0) or 0))
if backoff_seconds <= 0:
return
with cls._transient_backoff_lock:
cls._transient_backoff_until = max(float(cls._transient_backoff_until or 0), time.time() + backoff_seconds)
def _build_redis_client(self):
try:
client = redis.Redis(
@@ -237,7 +276,7 @@ class WaybackDetector(BaseDetector):
response = self.session.get(
self.cdx_api_url,
params=params,
timeout=config.WAYBACK_CDX_TIMEOUT,
timeout=self._resolve_timeout(config.WAYBACK_CDX_TIMEOUT),
stream=True,
)
if response.status_code != 200:
@@ -328,17 +367,27 @@ class WaybackDetector(BaseDetector):
try:
response = self.session.get(
snapshot_url,
timeout=config.WAYBACK_SNAPSHOT_TIMEOUT,
timeout=self._resolve_timeout(config.WAYBACK_SNAPSHOT_TIMEOUT),
stream=True,
)
if response.status_code != 200:
data = {'timestamp': timestamp, 'title': '', 'ok': False}
data = {
'timestamp': timestamp,
'title': '',
'ok': False,
'error': f'HTTP {response.status_code}',
}
self._save_cached_title(domain, timestamp, data)
return data
content_type = (response.headers.get('Content-Type') or '').lower()
if content_type and 'text/html' not in content_type and 'application/xhtml+xml' not in content_type:
data = {'timestamp': timestamp, 'title': '', 'ok': False}
data = {
'timestamp': timestamp,
'title': '',
'ok': False,
'error': f'content_type:{content_type}',
}
self._save_cached_title(domain, timestamp, data)
return data
@@ -359,11 +408,11 @@ class WaybackDetector(BaseDetector):
content = ''.join(chunks)
title = self._extract_title(content) if found_title or content else ''
data = {'timestamp': timestamp, 'title': title, 'ok': True}
data = {'timestamp': timestamp, 'title': title, 'ok': True, 'error': ''}
self._save_cached_title(domain, timestamp, data)
return data
except Exception:
data = {'timestamp': timestamp, 'title': '', 'ok': False}
except Exception as exc:
data = {'timestamp': timestamp, 'title': '', 'ok': False, 'error': str(exc)}
self._save_cached_title(domain, timestamp, data)
return data
finally:
@@ -404,15 +453,83 @@ class WaybackDetector(BaseDetector):
self._handle_exception(e, domain)
return False
def scan_snapshots(self, domain, sensitive_words=None, stop_on_first_hit=True):
@staticmethod
def _resolve_recent_year_cutoff(recent_years):
try:
normalized_recent_years = int(recent_years or 0)
except (TypeError, ValueError):
return None
if normalized_recent_years <= 0:
return None
return time.gmtime().tm_year - normalized_recent_years + 1
def _filter_records_recent_years(self, records, recent_years=None):
cutoff_year = self._resolve_recent_year_cutoff(recent_years)
if cutoff_year is None:
return list(records or [])
filtered = []
for item in list(records or []):
timestamp = str((item or {}).get('timestamp') or '').strip()
if len(timestamp) < 4:
continue
try:
year = int(timestamp[:4])
except (TypeError, ValueError):
continue
if year >= cutoff_year:
filtered.append(item)
return filtered
def _limit_records_for_scan(self, records):
max_records = max(1, int(getattr(config, "WAYBACK_MAX_RECORDS", 8) or 8))
trimmed = []
for item in list(records or []):
if len(trimmed) >= max_records:
break
trimmed.append(item)
return trimmed
def scan_snapshots(self, domain, sensitive_words=None, stop_on_first_hit=True, recent_years=None):
sensitive_words = sensitive_words or config.load_sensitive_words()
request_errors = []
transient_backoff_remaining = self._transient_backoff_remaining_seconds()
if transient_backoff_remaining > 0:
return {
'snapshot_years': [],
'has_sensitive_content': False,
'matched_word': None,
'matched_timestamp': None,
'matched_title': None,
'backlink_count': 0,
'backlink_count_gt_10': False,
'checked_snapshot_count': 0,
'fetched_snapshot_count': 0,
'failed_snapshot_count': 1,
'unique_title_count': 0,
'duplicate_title_skipped': 0,
'digest_duplicate_skipped': 0,
'request_error_count': 1,
'request_errors': [f"wayback_backoff_active:{round(transient_backoff_remaining, 2)}s"],
'elapsed_seconds': 0.0,
}
latest_fetch = self._fetch_cdx_records_with_meta(domain, limit=-1, fast_latest=True)
latest_record = (latest_fetch.get('records') or [None])[0]
if latest_fetch.get('error'):
request_errors.append(f"latest_cdx: {latest_fetch.get('error')}")
if self._is_transient_request_error(latest_fetch.get('error')):
self._trip_transient_backoff()
latest_timestamp = (latest_record or {}).get('timestamp')
latest_digest = (latest_record or {}).get('digest', '')
cutoff_year = self._resolve_recent_year_cutoff(recent_years)
if latest_timestamp and cutoff_year is not None:
try:
latest_year = int(str(latest_timestamp)[:4])
except (TypeError, ValueError):
latest_year = 0
if latest_year < cutoff_year:
latest_record = None
latest_timestamp = None
latest_digest = ''
matched_word = None
matched_timestamp = None
matched_title = None
@@ -427,6 +544,28 @@ class WaybackDetector(BaseDetector):
digest_seen = set()
checked_snapshot_count = 0
domain_concurrency = max(1, config.WAYBACK_DOMAIN_CONCURRENCY)
transient_snapshot_failures = 0
transient_snapshot_failure_threshold = max(2, domain_concurrency)
if (not latest_timestamp) and latest_fetch.get('error') and self._is_transient_request_error(latest_fetch.get('error')):
return {
'snapshot_years': [],
'has_sensitive_content': False,
'matched_word': None,
'matched_timestamp': None,
'matched_title': None,
'backlink_count': 0,
'backlink_count_gt_10': False,
'checked_snapshot_count': 0,
'fetched_snapshot_count': 0,
'failed_snapshot_count': max(1, failed_snapshot_count),
'unique_title_count': 0,
'duplicate_title_skipped': 0,
'digest_duplicate_skipped': 0,
'request_error_count': len(request_errors),
'request_errors': request_errors,
'elapsed_seconds': round(time.time() - started_at, 2),
}
if latest_timestamp:
latest_result = self._fetch_snapshot_title(domain, latest_timestamp)
@@ -462,6 +601,30 @@ class WaybackDetector(BaseDetector):
}
else:
failed_snapshot_count += 1
latest_error = str((latest_result or {}).get('error') or '').strip()
if latest_error:
request_errors.append(f"latest_snapshot: {latest_error}")
if latest_error and self._is_transient_request_error(latest_error):
transient_snapshot_failures += 1
self._trip_transient_backoff()
return {
'snapshot_years': [],
'has_sensitive_content': False,
'matched_word': None,
'matched_timestamp': None,
'matched_title': None,
'backlink_count': 0,
'backlink_count_gt_10': False,
'checked_snapshot_count': checked_snapshot_count,
'fetched_snapshot_count': fetched_snapshot_count,
'failed_snapshot_count': failed_snapshot_count,
'unique_title_count': unique_title_count,
'duplicate_title_skipped': duplicate_title_skipped,
'digest_duplicate_skipped': digest_duplicate_skipped,
'request_error_count': len(request_errors),
'request_errors': request_errors,
'elapsed_seconds': round(time.time() - started_at, 2),
}
cached_records = self._load_cached_records(domain)
if cached_records is not None:
@@ -470,12 +633,16 @@ class WaybackDetector(BaseDetector):
records_fetch = self._fetch_cdx_records_with_meta(domain)
if records_fetch.get('error'):
request_errors.append(f"records_cdx: {records_fetch.get('error')}")
if self._is_transient_request_error(records_fetch.get('error')):
self._trip_transient_backoff()
records = records_fetch.get('records') or []
if records:
self._save_cached_records(domain, records)
self._save_cached_timestamps(domain, [item['timestamp'] for item in records])
records = self._filter_records_recent_years(records, recent_years=recent_years)
records = sorted(records, key=lambda item: item.get('timestamp', ''), reverse=True)
records = self._limit_records_for_scan(records)
years = sorted({int(item['timestamp'][:4]) for item in records if len(item.get('timestamp', '')) >= 4})
checked_snapshot_count = len(records)
pending_records = []
@@ -522,9 +689,18 @@ class WaybackDetector(BaseDetector):
if not result or not result.get('ok'):
failed_snapshot_count += 1
error_message = str((result or {}).get('error') or '').strip()
if error_message:
request_errors.append(f"snapshot:{timestamp}: {error_message}")
if error_message and self._is_transient_request_error(error_message):
transient_snapshot_failures += 1
self._trip_transient_backoff()
if transient_snapshot_failures >= transient_snapshot_failure_threshold:
stop_requested = True
continue
fetched_snapshot_count += 1
transient_snapshot_failures = 0
title = result.get('title', '')
normalized_title = self._normalize_title(title)
if normalized_title:

4
domainCheck/app/ui/system_settings.py
View File

@@ -1355,7 +1355,7 @@ class SystemSettingsWidget(QWidget):
return
raw_value = self.thread_count_input.text().strip()
try:
thread_count = str(min(20, max(1, int(raw_value or '10'))))
thread_count = str(max(1, int(raw_value or '10')))
except Exception:
thread_count = '10'
if self.thread_count_input.text().strip() != thread_count:
@@ -1390,7 +1390,7 @@ class SystemSettingsWidget(QWidget):
if os.path.exists('thread_count.json'):
with open('thread_count.json', 'r', encoding='utf-8') as f:
thread_config = json.load(f)
thread_count = str(min(20, max(1, int(thread_config.get('thread_count', '10') or '10'))))
thread_count = str(max(1, int(thread_config.get('thread_count', '10') or '10')))
self.thread_count_input.setText(thread_count)
logger.info("检测线程数配置加载成功")
else:

1011
domainCheck/app/utils/database.py
View File

File diff suppressed because it is too large Load Diff