772 lines
31 KiB
Python
772 lines
31 KiB
Python
# -*- coding: UTF-8 -*-
|
||
'''
|
||
@Project :domainScanDemo
|
||
@File :wayback_detector.py
|
||
@IDE :PyCharm
|
||
@Author :梦伴
|
||
@Date :2026/4/8 23:57
|
||
@explain : Wayback检测器
|
||
'''
|
||
|
||
import html
|
||
import json
|
||
import re
|
||
import threading
|
||
import time
|
||
import zlib
|
||
from base64 import b64decode, b64encode
|
||
from concurrent.futures import FIRST_COMPLETED, ThreadPoolExecutor, wait
|
||
|
||
import redis
|
||
import requests
|
||
from requests.adapters import HTTPAdapter
|
||
from urllib3.util.retry import Retry
|
||
|
||
from app.config import config
|
||
from app.detectors.base import BaseDetector
|
||
|
||
|
||
class WaybackDetector(BaseDetector):
|
||
"""
|
||
Wayback检测器
|
||
"""
|
||
|
||
TITLE_PATTERN = re.compile(r'<title[^>]*>(.*?)</title>', re.IGNORECASE | re.DOTALL)
|
||
_transient_backoff_until = 0.0
|
||
_transient_backoff_lock = threading.Lock()
|
||
|
||
def __init__(self):
|
||
"""
|
||
初始化Wayback检测器
|
||
"""
|
||
super().__init__()
|
||
self.cdx_api_url = 'https://web.archive.org/cdx/search/cdx'
|
||
# 使用 id_ 直接取快照内容，避免回放页面额外壳层干扰正文匹配。
|
||
self.snapshot_url = 'https://web.archive.org/web/{timestamp}id_/{domain}'
|
||
self._timestamp_cache = {}
|
||
self._title_cache = {}
|
||
self._cache_lock = threading.Lock()
|
||
self.session = self._build_session()
|
||
self.redis_client = self._build_redis_client()
|
||
|
||
def _build_session(self):
|
||
session = requests.Session()
|
||
retry = Retry(
|
||
total=max(0, config.WAYBACK_RETRY_COUNT),
|
||
backoff_factor=0.5,
|
||
status_forcelist=(429, 500, 502, 503, 504),
|
||
allowed_methods=frozenset(["GET"]),
|
||
raise_on_status=False,
|
||
)
|
||
adapter = HTTPAdapter(max_retries=retry, pool_connections=128, pool_maxsize=256)
|
||
session.mount('http://', adapter)
|
||
session.mount('https://', adapter)
|
||
session.headers.update({
|
||
'User-Agent': config.WAYBACK_USER_AGENT,
|
||
'Accept-Language': 'zh-CN,zh;q=0.9,en;q=0.8',
|
||
})
|
||
return session
|
||
|
||
@staticmethod
|
||
def _resolve_timeout(total_timeout, connect_cap=3):
|
||
total = max(1.0, float(total_timeout or 0))
|
||
return (min(float(connect_cap), total), total)
|
||
|
||
@staticmethod
|
||
def _is_transient_request_error(message):
|
||
lowered = str(message or '').lower()
|
||
keywords = (
|
||
'timeout',
|
||
'timed out',
|
||
'connecttimeout',
|
||
'read timeout',
|
||
'connection reset',
|
||
'connection refused',
|
||
'connection aborted',
|
||
'name or service not known',
|
||
'temporary failure',
|
||
'max retries exceeded',
|
||
'ssl',
|
||
)
|
||
return any(keyword in lowered for keyword in keywords)
|
||
|
||
@classmethod
|
||
def _transient_backoff_remaining_seconds(cls):
|
||
with cls._transient_backoff_lock:
|
||
remaining = float(cls._transient_backoff_until or 0) - time.time()
|
||
return max(0.0, remaining)
|
||
|
||
@classmethod
|
||
def _trip_transient_backoff(cls):
|
||
backoff_seconds = max(0.0, float(getattr(config, "WAYBACK_TRANSIENT_BACKOFF_SECONDS", 0) or 0))
|
||
if backoff_seconds <= 0:
|
||
return
|
||
with cls._transient_backoff_lock:
|
||
cls._transient_backoff_until = max(float(cls._transient_backoff_until or 0), time.time() + backoff_seconds)
|
||
|
||
def _build_redis_client(self):
|
||
try:
|
||
client = redis.Redis(
|
||
host=config.REDIS_HOST,
|
||
port=config.REDIS_PORT,
|
||
password=config.REDIS_PASSWORD,
|
||
db=config.REDIS_DB,
|
||
decode_responses=True,
|
||
socket_connect_timeout=5,
|
||
socket_timeout=5,
|
||
)
|
||
client.ping()
|
||
return client
|
||
except Exception:
|
||
return None
|
||
|
||
def _title_cache_key(self, domain, timestamp):
|
||
return f"domain_tool:wayback_title:{domain}:{timestamp}"
|
||
|
||
def _timestamp_cache_key(self, domain):
|
||
return f"domain_tool:wayback_timestamps:{domain}"
|
||
|
||
def _record_cache_key(self, domain):
|
||
return f"domain_tool:wayback_records:{domain}"
|
||
|
||
def _normalize_title(self, title):
|
||
normalized = html.unescape(title or '')
|
||
normalized = re.sub(r'\s+', ' ', normalized, flags=re.DOTALL).strip().lower()
|
||
return normalized
|
||
|
||
def _extract_title(self, content):
|
||
match = self.TITLE_PATTERN.search(content or '')
|
||
if not match:
|
||
return ''
|
||
return html.unescape(match.group(1)).strip()
|
||
|
||
def _load_cached_title(self, domain, timestamp):
|
||
cache_key = self._title_cache_key(domain, timestamp)
|
||
with self._cache_lock:
|
||
if cache_key in self._title_cache:
|
||
return self._title_cache[cache_key]
|
||
if self.redis_client:
|
||
try:
|
||
raw_value = self.redis_client.get(cache_key)
|
||
if raw_value:
|
||
data = json.loads(raw_value)
|
||
with self._cache_lock:
|
||
self._title_cache[cache_key] = data
|
||
return data
|
||
except Exception:
|
||
pass
|
||
return None
|
||
|
||
def _save_cached_title(self, domain, timestamp, data):
|
||
cache_key = self._title_cache_key(domain, timestamp)
|
||
with self._cache_lock:
|
||
self._title_cache[cache_key] = data
|
||
if self.redis_client:
|
||
try:
|
||
self.redis_client.set(
|
||
cache_key,
|
||
json.dumps(data, ensure_ascii=False),
|
||
ex=max(0, config.WAYBACK_TITLE_CACHE_TTL) or None,
|
||
)
|
||
except Exception:
|
||
pass
|
||
|
||
def _load_cached_timestamps(self, domain):
|
||
if domain in self._timestamp_cache:
|
||
return list(self._timestamp_cache[domain])
|
||
if not self.redis_client:
|
||
return None
|
||
try:
|
||
raw_value = self.redis_client.get(self._timestamp_cache_key(domain))
|
||
if not raw_value:
|
||
return None
|
||
compressed = b64decode(raw_value.encode('ascii'))
|
||
timestamps = json.loads(zlib.decompress(compressed).decode('utf-8'))
|
||
if isinstance(timestamps, list):
|
||
self._timestamp_cache[domain] = tuple(timestamps)
|
||
return list(timestamps)
|
||
except Exception:
|
||
return None
|
||
return None
|
||
|
||
def _save_cached_timestamps(self, domain, timestamps):
|
||
self._timestamp_cache[domain] = tuple(timestamps)
|
||
if not self.redis_client:
|
||
return
|
||
try:
|
||
payload = json.dumps(timestamps, separators=(',', ':')).encode('utf-8')
|
||
compressed = zlib.compress(payload, level=6)
|
||
self.redis_client.set(
|
||
self._timestamp_cache_key(domain),
|
||
b64encode(compressed).decode('ascii'),
|
||
ex=max(0, config.WAYBACK_TIMESTAMP_CACHE_TTL) or None,
|
||
)
|
||
except Exception:
|
||
pass
|
||
|
||
def _load_cached_records(self, domain):
|
||
cache_key = self._record_cache_key(domain)
|
||
with self._cache_lock:
|
||
if cache_key in self._timestamp_cache:
|
||
return [
|
||
{'timestamp': item[0], 'digest': item[1]}
|
||
for item in self._timestamp_cache[cache_key]
|
||
]
|
||
if not self.redis_client:
|
||
return None
|
||
try:
|
||
raw_value = self.redis_client.get(cache_key)
|
||
if not raw_value:
|
||
return None
|
||
compressed = b64decode(raw_value.encode('ascii'))
|
||
records = json.loads(zlib.decompress(compressed).decode('utf-8'))
|
||
if isinstance(records, list):
|
||
normalized = tuple(
|
||
(item.get('timestamp', ''), item.get('digest', ''))
|
||
for item in records if isinstance(item, dict)
|
||
)
|
||
with self._cache_lock:
|
||
self._timestamp_cache[cache_key] = normalized
|
||
return [
|
||
{'timestamp': item[0], 'digest': item[1]}
|
||
for item in normalized if item[0]
|
||
]
|
||
except Exception:
|
||
return None
|
||
return None
|
||
|
||
def _save_cached_records(self, domain, records):
|
||
cache_key = self._record_cache_key(domain)
|
||
normalized = tuple(
|
||
(item.get('timestamp', ''), item.get('digest', ''))
|
||
for item in (records or [])
|
||
if item and item.get('timestamp')
|
||
)
|
||
with self._cache_lock:
|
||
self._timestamp_cache[cache_key] = normalized
|
||
if not self.redis_client:
|
||
return
|
||
try:
|
||
payload = json.dumps(
|
||
[{'timestamp': item[0], 'digest': item[1]} for item in normalized],
|
||
separators=(',', ':')
|
||
).encode('utf-8')
|
||
compressed = zlib.compress(payload, level=6)
|
||
self.redis_client.set(
|
||
cache_key,
|
||
b64encode(compressed).decode('ascii'),
|
||
ex=max(0, config.WAYBACK_TIMESTAMP_CACHE_TTL) or None,
|
||
)
|
||
except Exception:
|
||
pass
|
||
|
||
def _fetch_cdx_records_with_meta(self, domain, limit=None, fast_latest=False):
|
||
response = None
|
||
try:
|
||
params = {
|
||
'url': domain,
|
||
'output': 'txt',
|
||
'fl': 'timestamp,digest',
|
||
'filter': ['statuscode:200', 'mimetype:text/html'],
|
||
}
|
||
if limit is not None:
|
||
params['limit'] = str(limit)
|
||
if fast_latest:
|
||
params['fastLatest'] = 'true'
|
||
response = self.session.get(
|
||
self.cdx_api_url,
|
||
params=params,
|
||
timeout=self._resolve_timeout(config.WAYBACK_CDX_TIMEOUT),
|
||
stream=True,
|
||
)
|
||
if response.status_code != 200:
|
||
self._log_warning(f"获取快照记录失败: {response.status_code}")
|
||
return {
|
||
'records': [],
|
||
'error': f"HTTP {response.status_code}",
|
||
'status_code': response.status_code,
|
||
}
|
||
records = []
|
||
seen = set()
|
||
for raw_line in response.iter_lines(decode_unicode=True):
|
||
line = (raw_line or '').strip()
|
||
if not line:
|
||
continue
|
||
parts = line.split()
|
||
timestamp = parts[0].strip() if parts else ''
|
||
digest = parts[1].strip() if len(parts) > 1 else ''
|
||
if not timestamp or timestamp in seen:
|
||
continue
|
||
seen.add(timestamp)
|
||
records.append({'timestamp': timestamp, 'digest': digest})
|
||
return {
|
||
'records': records,
|
||
'error': None,
|
||
'status_code': response.status_code,
|
||
}
|
||
except Exception as e:
|
||
self._handle_exception(e, domain)
|
||
return {
|
||
'records': [],
|
||
'error': str(e),
|
||
'status_code': None,
|
||
}
|
||
finally:
|
||
try:
|
||
if response is not None:
|
||
response.close()
|
||
except Exception:
|
||
pass
|
||
|
||
def _fetch_cdx_records(self, domain, limit=None, fast_latest=False):
|
||
return self._fetch_cdx_records_with_meta(domain, limit=limit, fast_latest=fast_latest).get('records', [])
|
||
|
||
def get_latest_snapshot_record(self, domain):
|
||
records = self._fetch_cdx_records(domain, limit=-1, fast_latest=True)
|
||
return records[0] if records else None
|
||
|
||
def get_snapshot_records(self, domain):
|
||
cached_records = self._load_cached_records(domain)
|
||
if cached_records is not None:
|
||
return cached_records
|
||
records = self._fetch_cdx_records(domain)
|
||
if records:
|
||
self._save_cached_records(domain, records)
|
||
self._save_cached_timestamps(domain, [item['timestamp'] for item in records])
|
||
return records
|
||
|
||
def check_domain(self, domain):
|
||
"""
|
||
检测域名
|
||
|
||
:param domain: 域名
|
||
:return: dict - 检测结果
|
||
"""
|
||
try:
|
||
return self.scan_snapshots(domain)
|
||
except Exception as e:
|
||
return self._handle_exception(e, domain)
|
||
|
||
def get_snapshot_timestamps(self, domain):
|
||
cached_records = self._load_cached_records(domain)
|
||
if cached_records is not None:
|
||
return [item['timestamp'] for item in cached_records if item.get('timestamp')]
|
||
cached_timestamps = self._load_cached_timestamps(domain)
|
||
if cached_timestamps is not None:
|
||
return cached_timestamps
|
||
records = self.get_snapshot_records(domain)
|
||
return [item['timestamp'] for item in records if item.get('timestamp')]
|
||
|
||
def _fetch_snapshot_title(self, domain, timestamp):
|
||
cached = self._load_cached_title(domain, timestamp)
|
||
if cached is not None:
|
||
return cached
|
||
|
||
snapshot_url = self.snapshot_url.format(timestamp=timestamp, domain=domain)
|
||
response = None
|
||
try:
|
||
response = self.session.get(
|
||
snapshot_url,
|
||
timeout=self._resolve_timeout(config.WAYBACK_SNAPSHOT_TIMEOUT),
|
||
stream=True,
|
||
)
|
||
if response.status_code != 200:
|
||
data = {
|
||
'timestamp': timestamp,
|
||
'title': '',
|
||
'ok': False,
|
||
'error': f'HTTP {response.status_code}',
|
||
}
|
||
self._save_cached_title(domain, timestamp, data)
|
||
return data
|
||
|
||
content_type = (response.headers.get('Content-Type') or '').lower()
|
||
if content_type and 'text/html' not in content_type and 'application/xhtml+xml' not in content_type:
|
||
data = {
|
||
'timestamp': timestamp,
|
||
'title': '',
|
||
'ok': False,
|
||
'error': f'content_type:{content_type}',
|
||
}
|
||
self._save_cached_title(domain, timestamp, data)
|
||
return data
|
||
|
||
chunks = []
|
||
total_bytes = 0
|
||
found_title = False
|
||
for chunk in response.iter_content(chunk_size=4096, decode_unicode=True):
|
||
if not chunk:
|
||
continue
|
||
chunks.append(chunk)
|
||
total_bytes += len(chunk.encode('utf-8', errors='ignore'))
|
||
current_text = ''.join(chunks)
|
||
if '</title>' in current_text.lower():
|
||
found_title = True
|
||
break
|
||
if total_bytes >= config.WAYBACK_TITLE_MAX_BYTES:
|
||
break
|
||
|
||
content = ''.join(chunks)
|
||
title = self._extract_title(content) if found_title or content else ''
|
||
data = {'timestamp': timestamp, 'title': title, 'ok': True, 'error': ''}
|
||
self._save_cached_title(domain, timestamp, data)
|
||
return data
|
||
except Exception as exc:
|
||
data = {'timestamp': timestamp, 'title': '', 'ok': False, 'error': str(exc)}
|
||
self._save_cached_title(domain, timestamp, data)
|
||
return data
|
||
finally:
|
||
try:
|
||
if response is not None:
|
||
response.close()
|
||
except Exception:
|
||
pass
|
||
|
||
def get_snapshot_years(self, domain):
|
||
"""
|
||
获取快照年份
|
||
|
||
:param domain: 域名
|
||
:return: list - 快照年份列表
|
||
"""
|
||
try:
|
||
years = set()
|
||
for timestamp in self.get_snapshot_timestamps(domain):
|
||
if len(timestamp) >= 4:
|
||
years.add(int(timestamp[:4]))
|
||
return sorted(years)
|
||
except Exception as e:
|
||
self._handle_exception(e, domain)
|
||
return []
|
||
|
||
def has_sensitive_content(self, domain):
|
||
"""
|
||
检查是否包含敏感内容
|
||
|
||
:param domain: 域名
|
||
:return: bool - 是否包含敏感内容
|
||
"""
|
||
try:
|
||
result = self.scan_snapshots(domain)
|
||
return result.get('has_sensitive_content', False)
|
||
except Exception as e:
|
||
self._handle_exception(e, domain)
|
||
return False
|
||
|
||
@staticmethod
|
||
def _resolve_recent_year_cutoff(recent_years):
|
||
try:
|
||
normalized_recent_years = int(recent_years or 0)
|
||
except (TypeError, ValueError):
|
||
return None
|
||
if normalized_recent_years <= 0:
|
||
return None
|
||
return time.gmtime().tm_year - normalized_recent_years + 1
|
||
|
||
def _filter_records_recent_years(self, records, recent_years=None):
|
||
cutoff_year = self._resolve_recent_year_cutoff(recent_years)
|
||
if cutoff_year is None:
|
||
return list(records or [])
|
||
filtered = []
|
||
for item in list(records or []):
|
||
timestamp = str((item or {}).get('timestamp') or '').strip()
|
||
if len(timestamp) < 4:
|
||
continue
|
||
try:
|
||
year = int(timestamp[:4])
|
||
except (TypeError, ValueError):
|
||
continue
|
||
if year >= cutoff_year:
|
||
filtered.append(item)
|
||
return filtered
|
||
|
||
def _limit_records_for_scan(self, records):
|
||
max_records = max(1, int(getattr(config, "WAYBACK_MAX_RECORDS", 8) or 8))
|
||
trimmed = []
|
||
for item in list(records or []):
|
||
if len(trimmed) >= max_records:
|
||
break
|
||
trimmed.append(item)
|
||
return trimmed
|
||
|
||
def scan_snapshots(self, domain, sensitive_words=None, stop_on_first_hit=True, recent_years=None):
|
||
sensitive_words = sensitive_words or config.load_sensitive_words()
|
||
request_errors = []
|
||
transient_backoff_remaining = self._transient_backoff_remaining_seconds()
|
||
if transient_backoff_remaining > 0:
|
||
return {
|
||
'snapshot_years': [],
|
||
'has_sensitive_content': False,
|
||
'matched_word': None,
|
||
'matched_timestamp': None,
|
||
'matched_title': None,
|
||
'backlink_count': 0,
|
||
'backlink_count_gt_10': False,
|
||
'checked_snapshot_count': 0,
|
||
'fetched_snapshot_count': 0,
|
||
'failed_snapshot_count': 1,
|
||
'unique_title_count': 0,
|
||
'duplicate_title_skipped': 0,
|
||
'digest_duplicate_skipped': 0,
|
||
'request_error_count': 1,
|
||
'request_errors': [f"wayback_backoff_active:{round(transient_backoff_remaining, 2)}s"],
|
||
'elapsed_seconds': 0.0,
|
||
}
|
||
latest_fetch = self._fetch_cdx_records_with_meta(domain, limit=-1, fast_latest=True)
|
||
latest_record = (latest_fetch.get('records') or [None])[0]
|
||
if latest_fetch.get('error'):
|
||
request_errors.append(f"latest_cdx: {latest_fetch.get('error')}")
|
||
if self._is_transient_request_error(latest_fetch.get('error')):
|
||
self._trip_transient_backoff()
|
||
latest_timestamp = (latest_record or {}).get('timestamp')
|
||
latest_digest = (latest_record or {}).get('digest', '')
|
||
cutoff_year = self._resolve_recent_year_cutoff(recent_years)
|
||
if latest_timestamp and cutoff_year is not None:
|
||
try:
|
||
latest_year = int(str(latest_timestamp)[:4])
|
||
except (TypeError, ValueError):
|
||
latest_year = 0
|
||
if latest_year < cutoff_year:
|
||
latest_record = None
|
||
latest_timestamp = None
|
||
latest_digest = ''
|
||
matched_word = None
|
||
matched_timestamp = None
|
||
matched_title = None
|
||
fetched_snapshot_count = 0
|
||
failed_snapshot_count = 0
|
||
unique_title_count = 0
|
||
duplicate_title_skipped = 0
|
||
digest_duplicate_skipped = 0
|
||
started_at = time.time()
|
||
progress_interval = max(1, config.WAYBACK_PROGRESS_INTERVAL)
|
||
title_seen = set()
|
||
digest_seen = set()
|
||
checked_snapshot_count = 0
|
||
domain_concurrency = max(1, config.WAYBACK_DOMAIN_CONCURRENCY)
|
||
transient_snapshot_failures = 0
|
||
transient_snapshot_failure_threshold = max(2, domain_concurrency)
|
||
|
||
if (not latest_timestamp) and latest_fetch.get('error') and self._is_transient_request_error(latest_fetch.get('error')):
|
||
return {
|
||
'snapshot_years': [],
|
||
'has_sensitive_content': False,
|
||
'matched_word': None,
|
||
'matched_timestamp': None,
|
||
'matched_title': None,
|
||
'backlink_count': 0,
|
||
'backlink_count_gt_10': False,
|
||
'checked_snapshot_count': 0,
|
||
'fetched_snapshot_count': 0,
|
||
'failed_snapshot_count': max(1, failed_snapshot_count),
|
||
'unique_title_count': 0,
|
||
'duplicate_title_skipped': 0,
|
||
'digest_duplicate_skipped': 0,
|
||
'request_error_count': len(request_errors),
|
||
'request_errors': request_errors,
|
||
'elapsed_seconds': round(time.time() - started_at, 2),
|
||
}
|
||
|
||
if latest_timestamp:
|
||
latest_result = self._fetch_snapshot_title(domain, latest_timestamp)
|
||
checked_snapshot_count = 1
|
||
if latest_result and latest_result.get('ok'):
|
||
fetched_snapshot_count = 1
|
||
title = latest_result.get('title', '')
|
||
normalized_title = self._normalize_title(title)
|
||
if latest_digest:
|
||
digest_seen.add(latest_digest)
|
||
if normalized_title:
|
||
title_seen.add(normalized_title)
|
||
unique_title_count = 1
|
||
matched_word = self._find_sensitive_word(title, sensitive_words)
|
||
if matched_word and stop_on_first_hit:
|
||
matched_timestamp = latest_timestamp
|
||
matched_title = title
|
||
return {
|
||
'snapshot_years': [int(latest_timestamp[:4])] if len(latest_timestamp) >= 4 else [],
|
||
'has_sensitive_content': True,
|
||
'matched_word': matched_word,
|
||
'matched_timestamp': matched_timestamp,
|
||
'matched_title': matched_title,
|
||
'backlink_count': 0,
|
||
'backlink_count_gt_10': False,
|
||
'checked_snapshot_count': checked_snapshot_count,
|
||
'fetched_snapshot_count': fetched_snapshot_count,
|
||
'failed_snapshot_count': failed_snapshot_count,
|
||
'unique_title_count': unique_title_count,
|
||
'duplicate_title_skipped': duplicate_title_skipped,
|
||
'digest_duplicate_skipped': digest_duplicate_skipped,
|
||
'elapsed_seconds': round(time.time() - started_at, 2),
|
||
}
|
||
else:
|
||
failed_snapshot_count += 1
|
||
latest_error = str((latest_result or {}).get('error') or '').strip()
|
||
if latest_error:
|
||
request_errors.append(f"latest_snapshot: {latest_error}")
|
||
if latest_error and self._is_transient_request_error(latest_error):
|
||
transient_snapshot_failures += 1
|
||
self._trip_transient_backoff()
|
||
return {
|
||
'snapshot_years': [],
|
||
'has_sensitive_content': False,
|
||
'matched_word': None,
|
||
'matched_timestamp': None,
|
||
'matched_title': None,
|
||
'backlink_count': 0,
|
||
'backlink_count_gt_10': False,
|
||
'checked_snapshot_count': checked_snapshot_count,
|
||
'fetched_snapshot_count': fetched_snapshot_count,
|
||
'failed_snapshot_count': failed_snapshot_count,
|
||
'unique_title_count': unique_title_count,
|
||
'duplicate_title_skipped': duplicate_title_skipped,
|
||
'digest_duplicate_skipped': digest_duplicate_skipped,
|
||
'request_error_count': len(request_errors),
|
||
'request_errors': request_errors,
|
||
'elapsed_seconds': round(time.time() - started_at, 2),
|
||
}
|
||
|
||
cached_records = self._load_cached_records(domain)
|
||
if cached_records is not None:
|
||
records = cached_records
|
||
else:
|
||
records_fetch = self._fetch_cdx_records_with_meta(domain)
|
||
if records_fetch.get('error'):
|
||
request_errors.append(f"records_cdx: {records_fetch.get('error')}")
|
||
if self._is_transient_request_error(records_fetch.get('error')):
|
||
self._trip_transient_backoff()
|
||
records = records_fetch.get('records') or []
|
||
if records:
|
||
self._save_cached_records(domain, records)
|
||
self._save_cached_timestamps(domain, [item['timestamp'] for item in records])
|
||
|
||
records = self._filter_records_recent_years(records, recent_years=recent_years)
|
||
records = sorted(records, key=lambda item: item.get('timestamp', ''), reverse=True)
|
||
records = self._limit_records_for_scan(records)
|
||
years = sorted({int(item['timestamp'][:4]) for item in records if len(item.get('timestamp', '')) >= 4})
|
||
checked_snapshot_count = len(records)
|
||
pending_records = []
|
||
for item in records:
|
||
timestamp = item.get('timestamp', '')
|
||
digest = item.get('digest', '')
|
||
if not timestamp:
|
||
continue
|
||
if latest_timestamp and timestamp == latest_timestamp:
|
||
continue
|
||
if digest and digest in digest_seen:
|
||
digest_duplicate_skipped += 1
|
||
continue
|
||
if digest:
|
||
digest_seen.add(digest)
|
||
pending_records.append(item)
|
||
|
||
with ThreadPoolExecutor(max_workers=domain_concurrency) as executor:
|
||
pending = {}
|
||
index = 0
|
||
finished_count = 1 if latest_timestamp else 0
|
||
stop_requested = False
|
||
|
||
while (index < len(pending_records) or pending) and not stop_requested:
|
||
while index < len(pending_records) and len(pending) < domain_concurrency and not stop_requested:
|
||
timestamp = pending_records[index]['timestamp']
|
||
future = executor.submit(self._fetch_snapshot_title, domain, timestamp)
|
||
pending[future] = timestamp
|
||
index += 1
|
||
|
||
if not pending:
|
||
break
|
||
|
||
done, _ = wait(list(pending.keys()), return_when=FIRST_COMPLETED)
|
||
for future in done:
|
||
timestamp = pending.pop(future, None)
|
||
finished_count += 1
|
||
try:
|
||
result = future.result()
|
||
except Exception as e:
|
||
failed_snapshot_count += 1
|
||
self._handle_exception(e, domain)
|
||
continue
|
||
|
||
if not result or not result.get('ok'):
|
||
failed_snapshot_count += 1
|
||
error_message = str((result or {}).get('error') or '').strip()
|
||
if error_message:
|
||
request_errors.append(f"snapshot:{timestamp}: {error_message}")
|
||
if error_message and self._is_transient_request_error(error_message):
|
||
transient_snapshot_failures += 1
|
||
self._trip_transient_backoff()
|
||
if transient_snapshot_failures >= transient_snapshot_failure_threshold:
|
||
stop_requested = True
|
||
continue
|
||
|
||
fetched_snapshot_count += 1
|
||
transient_snapshot_failures = 0
|
||
title = result.get('title', '')
|
||
normalized_title = self._normalize_title(title)
|
||
if normalized_title:
|
||
if normalized_title in title_seen:
|
||
duplicate_title_skipped += 1
|
||
else:
|
||
title_seen.add(normalized_title)
|
||
unique_title_count += 1
|
||
matched_word = self._find_sensitive_word(title, sensitive_words)
|
||
if matched_word:
|
||
matched_timestamp = timestamp
|
||
matched_title = title
|
||
if stop_on_first_hit:
|
||
stop_requested = True
|
||
if finished_count % progress_interval == 0:
|
||
elapsed = round(time.time() - started_at, 2)
|
||
self._log_info(
|
||
f"{domain} 时光机进度: {finished_count}/{checked_snapshot_count}，成功 {fetched_snapshot_count}，失败 {failed_snapshot_count}，唯一标题 {unique_title_count}，标题重复跳过 {duplicate_title_skipped}，digest 重复跳过 {digest_duplicate_skipped}，耗时 {elapsed}s"
|
||
)
|
||
if config.WAYBACK_REQUEST_DELAY > 0:
|
||
time.sleep(config.WAYBACK_REQUEST_DELAY)
|
||
|
||
if stop_requested:
|
||
for future in pending:
|
||
future.cancel()
|
||
|
||
return {
|
||
'snapshot_years': years,
|
||
'has_sensitive_content': matched_word is not None,
|
||
'matched_word': matched_word,
|
||
'matched_timestamp': matched_timestamp,
|
||
'matched_title': matched_title,
|
||
'backlink_count': 0,
|
||
'backlink_count_gt_10': False,
|
||
'checked_snapshot_count': checked_snapshot_count,
|
||
'fetched_snapshot_count': fetched_snapshot_count,
|
||
'failed_snapshot_count': failed_snapshot_count,
|
||
'unique_title_count': unique_title_count,
|
||
'duplicate_title_skipped': duplicate_title_skipped,
|
||
'digest_duplicate_skipped': digest_duplicate_skipped,
|
||
'request_error_count': len(request_errors),
|
||
'request_errors': request_errors,
|
||
'elapsed_seconds': round(time.time() - started_at, 2),
|
||
}
|
||
|
||
def _check_sensitive_words(self, content):
|
||
"""
|
||
检查敏感词
|
||
|
||
:param content: 内容
|
||
:return: bool - 是否包含敏感词
|
||
"""
|
||
return self._find_sensitive_word(content, config.load_sensitive_words()) is not None
|
||
|
||
def _find_sensitive_word(self, content, sensitive_words):
|
||
for word in sensitive_words or []:
|
||
if word and word in (content or ''):
|
||
return word
|
||
return None
|
||
|
||
def get_backlink_count(self, domain):
|
||
"""
|
||
当前策略仅扫描标题，不再抓取正文，友链数量默认返回 0。
|
||
"""
|
||
return 0
|
||
|
||
def _count_backlinks(self, content):
|
||
return 0
|