admin/getDomain
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

first

Browse Source
This commit is contained in:
Your Name
2026-04-16 21:35:47 +08:00
parent ff32aa50bf
commit ebf632e651
86 changed files with 14097 additions and 585 deletions
Download Patch File Download Diff File Expand all files Collapse all files

45
domainCheck/app/detectors/wayback_detector.py
View File

@@ -221,7 +221,7 @@ class WaybackDetector(BaseDetector):
except Exception:
pass
def _fetch_cdx_records(self, domain, limit=None, fast_latest=False):
def _fetch_cdx_records_with_meta(self, domain, limit=None, fast_latest=False):
response = None
try:
params = {
@@ -242,7 +242,11 @@ class WaybackDetector(BaseDetector):
)
if response.status_code != 200:
self._log_warning(f"获取快照记录失败: {response.status_code}")
return []
return {
'records': [],
'error': f"HTTP {response.status_code}",
'status_code': response.status_code,
}
records = []
seen = set()
for raw_line in response.iter_lines(decode_unicode=True):
@@ -256,10 +260,18 @@ class WaybackDetector(BaseDetector):
continue
seen.add(timestamp)
records.append({'timestamp': timestamp, 'digest': digest})
return records
return {
'records': records,
'error': None,
'status_code': response.status_code,
}
except Exception as e:
self._handle_exception(e, domain)
return []
return {
'records': [],
'error': str(e),
'status_code': None,
}
finally:
try:
if response is not None:
@@ -267,6 +279,9 @@ class WaybackDetector(BaseDetector):
except Exception:
pass
def _fetch_cdx_records(self, domain, limit=None, fast_latest=False):
return self._fetch_cdx_records_with_meta(domain, limit=limit, fast_latest=fast_latest).get('records', [])
def get_latest_snapshot_record(self, domain):
records = self._fetch_cdx_records(domain, limit=-1, fast_latest=True)
return records[0] if records else None
@@ -391,7 +406,11 @@ class WaybackDetector(BaseDetector):
def scan_snapshots(self, domain, sensitive_words=None, stop_on_first_hit=True):
sensitive_words = sensitive_words or config.load_sensitive_words()
latest_record = self.get_latest_snapshot_record(domain)
request_errors = []
latest_fetch = self._fetch_cdx_records_with_meta(domain, limit=-1, fast_latest=True)
latest_record = (latest_fetch.get('records') or [None])[0]
if latest_fetch.get('error'):
request_errors.append(f"latest_cdx: {latest_fetch.get('error')}")
latest_timestamp = (latest_record or {}).get('timestamp')
latest_digest = (latest_record or {}).get('digest', '')
matched_word = None
@@ -444,7 +463,19 @@ class WaybackDetector(BaseDetector):
else:
failed_snapshot_count += 1
records = sorted(self.get_snapshot_records(domain), key=lambda item: item.get('timestamp', ''), reverse=True)
cached_records = self._load_cached_records(domain)
if cached_records is not None:
records = cached_records
else:
records_fetch = self._fetch_cdx_records_with_meta(domain)
if records_fetch.get('error'):
request_errors.append(f"records_cdx: {records_fetch.get('error')}")
records = records_fetch.get('records') or []
if records:
self._save_cached_records(domain, records)
self._save_cached_timestamps(domain, [item['timestamp'] for item in records])
records = sorted(records, key=lambda item: item.get('timestamp', ''), reverse=True)
years = sorted({int(item['timestamp'][:4]) for item in records if len(item.get('timestamp', '')) >= 4})
checked_snapshot_count = len(records)
pending_records = []
@@ -534,6 +565,8 @@ class WaybackDetector(BaseDetector):
'unique_title_count': unique_title_count,
'duplicate_title_skipped': duplicate_title_skipped,
'digest_duplicate_skipped': digest_duplicate_skipped,
'request_error_count': len(request_errors),
'request_errors': request_errors,
'elapsed_seconds': round(time.time() - started_at, 2),
}