admin/getDomain
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

d

Browse Source
This commit is contained in:
Your Name
2026-04-22 14:13:21 +08:00
parent e0406b5d0e
commit 7cbde2aa78
145 changed files with 23086 additions and 2243 deletions
Download Patch File Download Diff File Expand all files Collapse all files

194
domainCheck/app/detectors/wayback_detector.py
View File

@@ -32,6 +32,8 @@ class WaybackDetector(BaseDetector):
"""
TITLE_PATTERN = re.compile(r'<title[^>]*>(.*?)</title>', re.IGNORECASE | re.DOTALL)
_transient_backoff_until = 0.0
_transient_backoff_lock = threading.Lock()
def __init__(self):
"""
@@ -56,7 +58,7 @@ class WaybackDetector(BaseDetector):
allowed_methods=frozenset(["GET"]),
raise_on_status=False,
)
adapter = HTTPAdapter(max_retries=retry, pool_connections=10, pool_maxsize=10)
adapter = HTTPAdapter(max_retries=retry, pool_connections=128, pool_maxsize=256)
session.mount('http://', adapter)
session.mount('https://', adapter)
session.headers.update({
@@ -65,6 +67,43 @@ class WaybackDetector(BaseDetector):
})
return session
@staticmethod
def _resolve_timeout(total_timeout, connect_cap=3):
total = max(1.0, float(total_timeout or 0))
return (min(float(connect_cap), total), total)
@staticmethod
def _is_transient_request_error(message):
lowered = str(message or '').lower()
keywords = (
'timeout',
'timed out',
'connecttimeout',
'read timeout',
'connection reset',
'connection refused',
'connection aborted',
'name or service not known',
'temporary failure',
'max retries exceeded',
'ssl',
)
return any(keyword in lowered for keyword in keywords)
@classmethod
def _transient_backoff_remaining_seconds(cls):
with cls._transient_backoff_lock:
remaining = float(cls._transient_backoff_until or 0) - time.time()
return max(0.0, remaining)
@classmethod
def _trip_transient_backoff(cls):
backoff_seconds = max(0.0, float(getattr(config, "WAYBACK_TRANSIENT_BACKOFF_SECONDS", 0) or 0))
if backoff_seconds <= 0:
return
with cls._transient_backoff_lock:
cls._transient_backoff_until = max(float(cls._transient_backoff_until or 0), time.time() + backoff_seconds)
def _build_redis_client(self):
try:
client = redis.Redis(
@@ -237,7 +276,7 @@ class WaybackDetector(BaseDetector):
response = self.session.get(
self.cdx_api_url,
params=params,
timeout=config.WAYBACK_CDX_TIMEOUT,
timeout=self._resolve_timeout(config.WAYBACK_CDX_TIMEOUT),
stream=True,
)
if response.status_code != 200:
@@ -328,17 +367,27 @@ class WaybackDetector(BaseDetector):
try:
response = self.session.get(
snapshot_url,
timeout=config.WAYBACK_SNAPSHOT_TIMEOUT,
timeout=self._resolve_timeout(config.WAYBACK_SNAPSHOT_TIMEOUT),
stream=True,
)
if response.status_code != 200:
data = {'timestamp': timestamp, 'title': '', 'ok': False}
data = {
'timestamp': timestamp,
'title': '',
'ok': False,
'error': f'HTTP {response.status_code}',
}
self._save_cached_title(domain, timestamp, data)
return data
content_type = (response.headers.get('Content-Type') or '').lower()
if content_type and 'text/html' not in content_type and 'application/xhtml+xml' not in content_type:
data = {'timestamp': timestamp, 'title': '', 'ok': False}
data = {
'timestamp': timestamp,
'title': '',
'ok': False,
'error': f'content_type:{content_type}',
}
self._save_cached_title(domain, timestamp, data)
return data
@@ -359,11 +408,11 @@ class WaybackDetector(BaseDetector):
content = ''.join(chunks)
title = self._extract_title(content) if found_title or content else ''
data = {'timestamp': timestamp, 'title': title, 'ok': True}
data = {'timestamp': timestamp, 'title': title, 'ok': True, 'error': ''}
self._save_cached_title(domain, timestamp, data)
return data
except Exception:
data = {'timestamp': timestamp, 'title': '', 'ok': False}
except Exception as exc:
data = {'timestamp': timestamp, 'title': '', 'ok': False, 'error': str(exc)}
self._save_cached_title(domain, timestamp, data)
return data
finally:
@@ -404,15 +453,83 @@ class WaybackDetector(BaseDetector):
self._handle_exception(e, domain)
return False
def scan_snapshots(self, domain, sensitive_words=None, stop_on_first_hit=True):
@staticmethod
def _resolve_recent_year_cutoff(recent_years):
try:
normalized_recent_years = int(recent_years or 0)
except (TypeError, ValueError):
return None
if normalized_recent_years <= 0:
return None
return time.gmtime().tm_year - normalized_recent_years + 1
def _filter_records_recent_years(self, records, recent_years=None):
cutoff_year = self._resolve_recent_year_cutoff(recent_years)
if cutoff_year is None:
return list(records or [])
filtered = []
for item in list(records or []):
timestamp = str((item or {}).get('timestamp') or '').strip()
if len(timestamp) < 4:
continue
try:
year = int(timestamp[:4])
except (TypeError, ValueError):
continue
if year >= cutoff_year:
filtered.append(item)
return filtered
def _limit_records_for_scan(self, records):
max_records = max(1, int(getattr(config, "WAYBACK_MAX_RECORDS", 8) or 8))
trimmed = []
for item in list(records or []):
if len(trimmed) >= max_records:
break
trimmed.append(item)
return trimmed
def scan_snapshots(self, domain, sensitive_words=None, stop_on_first_hit=True, recent_years=None):
sensitive_words = sensitive_words or config.load_sensitive_words()
request_errors = []
transient_backoff_remaining = self._transient_backoff_remaining_seconds()
if transient_backoff_remaining > 0:
return {
'snapshot_years': [],
'has_sensitive_content': False,
'matched_word': None,
'matched_timestamp': None,
'matched_title': None,
'backlink_count': 0,
'backlink_count_gt_10': False,
'checked_snapshot_count': 0,
'fetched_snapshot_count': 0,
'failed_snapshot_count': 1,
'unique_title_count': 0,
'duplicate_title_skipped': 0,
'digest_duplicate_skipped': 0,
'request_error_count': 1,
'request_errors': [f"wayback_backoff_active:{round(transient_backoff_remaining, 2)}s"],
'elapsed_seconds': 0.0,
}
latest_fetch = self._fetch_cdx_records_with_meta(domain, limit=-1, fast_latest=True)
latest_record = (latest_fetch.get('records') or [None])[0]
if latest_fetch.get('error'):
request_errors.append(f"latest_cdx: {latest_fetch.get('error')}")
if self._is_transient_request_error(latest_fetch.get('error')):
self._trip_transient_backoff()
latest_timestamp = (latest_record or {}).get('timestamp')
latest_digest = (latest_record or {}).get('digest', '')
cutoff_year = self._resolve_recent_year_cutoff(recent_years)
if latest_timestamp and cutoff_year is not None:
try:
latest_year = int(str(latest_timestamp)[:4])
except (TypeError, ValueError):
latest_year = 0
if latest_year < cutoff_year:
latest_record = None
latest_timestamp = None
latest_digest = ''
matched_word = None
matched_timestamp = None
matched_title = None
@@ -427,6 +544,28 @@ class WaybackDetector(BaseDetector):
digest_seen = set()
checked_snapshot_count = 0
domain_concurrency = max(1, config.WAYBACK_DOMAIN_CONCURRENCY)
transient_snapshot_failures = 0
transient_snapshot_failure_threshold = max(2, domain_concurrency)
if (not latest_timestamp) and latest_fetch.get('error') and self._is_transient_request_error(latest_fetch.get('error')):
return {
'snapshot_years': [],
'has_sensitive_content': False,
'matched_word': None,
'matched_timestamp': None,
'matched_title': None,
'backlink_count': 0,
'backlink_count_gt_10': False,
'checked_snapshot_count': 0,
'fetched_snapshot_count': 0,
'failed_snapshot_count': max(1, failed_snapshot_count),
'unique_title_count': 0,
'duplicate_title_skipped': 0,
'digest_duplicate_skipped': 0,
'request_error_count': len(request_errors),
'request_errors': request_errors,
'elapsed_seconds': round(time.time() - started_at, 2),
}
if latest_timestamp:
latest_result = self._fetch_snapshot_title(domain, latest_timestamp)
@@ -462,6 +601,30 @@ class WaybackDetector(BaseDetector):
}
else:
failed_snapshot_count += 1
latest_error = str((latest_result or {}).get('error') or '').strip()
if latest_error:
request_errors.append(f"latest_snapshot: {latest_error}")
if latest_error and self._is_transient_request_error(latest_error):
transient_snapshot_failures += 1
self._trip_transient_backoff()
return {
'snapshot_years': [],
'has_sensitive_content': False,
'matched_word': None,
'matched_timestamp': None,
'matched_title': None,
'backlink_count': 0,
'backlink_count_gt_10': False,
'checked_snapshot_count': checked_snapshot_count,
'fetched_snapshot_count': fetched_snapshot_count,
'failed_snapshot_count': failed_snapshot_count,
'unique_title_count': unique_title_count,
'duplicate_title_skipped': duplicate_title_skipped,
'digest_duplicate_skipped': digest_duplicate_skipped,
'request_error_count': len(request_errors),
'request_errors': request_errors,
'elapsed_seconds': round(time.time() - started_at, 2),
}
cached_records = self._load_cached_records(domain)
if cached_records is not None:
@@ -470,12 +633,16 @@ class WaybackDetector(BaseDetector):
records_fetch = self._fetch_cdx_records_with_meta(domain)
if records_fetch.get('error'):
request_errors.append(f"records_cdx: {records_fetch.get('error')}")
if self._is_transient_request_error(records_fetch.get('error')):
self._trip_transient_backoff()
records = records_fetch.get('records') or []
if records:
self._save_cached_records(domain, records)
self._save_cached_timestamps(domain, [item['timestamp'] for item in records])
records = self._filter_records_recent_years(records, recent_years=recent_years)
records = sorted(records, key=lambda item: item.get('timestamp', ''), reverse=True)
records = self._limit_records_for_scan(records)
years = sorted({int(item['timestamp'][:4]) for item in records if len(item.get('timestamp', '')) >= 4})
checked_snapshot_count = len(records)
pending_records = []
@@ -522,9 +689,18 @@ class WaybackDetector(BaseDetector):
if not result or not result.get('ok'):
failed_snapshot_count += 1
error_message = str((result or {}).get('error') or '').strip()
if error_message:
request_errors.append(f"snapshot:{timestamp}: {error_message}")
if error_message and self._is_transient_request_error(error_message):
transient_snapshot_failures += 1
self._trip_transient_backoff()
if transient_snapshot_failures >= transient_snapshot_failure_threshold:
stop_requested = True
continue
fetched_snapshot_count += 1
transient_snapshot_failures = 0
title = result.get('title', '')
normalized_title = self._normalize_title(title)
if normalized_title: