341 lines
16 KiB
Python
341 lines
16 KiB
Python
import sys
|
|
import time
|
|
import unittest
|
|
from unittest.mock import patch
|
|
|
|
|
|
sys.path.insert(0, "/www/wwwroot/getDomain/domainCheck")
|
|
|
|
from app.detectors.wayback_detector import WaybackDetector # noqa: E402
|
|
|
|
|
|
class WaybackDetectorRecentYearsTest(unittest.TestCase):
|
|
def setUp(self):
|
|
WaybackDetector._transient_backoff_until = 0.0
|
|
|
|
def test_filter_records_recent_years_drops_old_snapshots(self):
|
|
detector = WaybackDetector.__new__(WaybackDetector)
|
|
current_year = time.gmtime().tm_year
|
|
records = [
|
|
{"timestamp": f"{current_year}0101000000", "digest": "new"},
|
|
{"timestamp": f"{current_year - 2}0101000000", "digest": "mid"},
|
|
{"timestamp": f"{current_year - 6}0101000000", "digest": "old"},
|
|
]
|
|
|
|
filtered = detector._filter_records_recent_years(records, recent_years=5)
|
|
|
|
self.assertEqual(
|
|
[
|
|
f"{current_year}0101000000",
|
|
f"{current_year - 2}0101000000",
|
|
],
|
|
[item["timestamp"] for item in filtered],
|
|
)
|
|
|
|
def test_scan_snapshots_only_counts_recent_years_window(self):
|
|
detector = WaybackDetector.__new__(WaybackDetector)
|
|
current_year = time.gmtime().tm_year
|
|
detector._fetch_cdx_records_with_meta = lambda domain, limit=None, fast_latest=False: {
|
|
"records": [
|
|
{"timestamp": f"{current_year}0101000000", "digest": "latest"},
|
|
{"timestamp": f"{current_year - 1}0101000000", "digest": "prev"},
|
|
{"timestamp": f"{current_year - 6}0101000000", "digest": "old"},
|
|
]
|
|
}
|
|
detector._load_cached_records = lambda domain: None
|
|
detector._save_cached_records = lambda domain, records: None
|
|
detector._save_cached_timestamps = lambda domain, timestamps: None
|
|
detector._fetch_snapshot_title = lambda domain, timestamp: {
|
|
"timestamp": timestamp,
|
|
"title": f"title-{timestamp}",
|
|
"ok": True,
|
|
}
|
|
detector._normalize_title = lambda title: title
|
|
detector._find_sensitive_word = lambda title, words: None
|
|
detector._log_info = lambda message: None
|
|
detector._handle_exception = lambda exc, domain: None
|
|
|
|
result = detector.scan_snapshots("example.com", sensitive_words=[], recent_years=5)
|
|
|
|
self.assertEqual(2, result["checked_snapshot_count"])
|
|
self.assertEqual(2, result["fetched_snapshot_count"])
|
|
self.assertEqual(sorted([current_year - 1, current_year]), result["snapshot_years"])
|
|
|
|
def test_scan_snapshots_limits_records_per_domain(self):
|
|
detector = WaybackDetector.__new__(WaybackDetector)
|
|
current_year = time.gmtime().tm_year
|
|
records = [
|
|
{"timestamp": f"{current_year}01010{i}0000", "digest": f"d{i}"}
|
|
for i in range(6)
|
|
]
|
|
detector._fetch_cdx_records_with_meta = lambda domain, limit=None, fast_latest=False: {
|
|
"records": records,
|
|
"error": None,
|
|
}
|
|
detector._load_cached_records = lambda domain: None
|
|
detector._save_cached_records = lambda domain, records: None
|
|
detector._save_cached_timestamps = lambda domain, timestamps: None
|
|
detector._fetch_snapshot_title = lambda domain, timestamp: {
|
|
"timestamp": timestamp,
|
|
"title": f"title-{timestamp}",
|
|
"ok": True,
|
|
}
|
|
detector._normalize_title = lambda title: title
|
|
detector._find_sensitive_word = lambda title, words: None
|
|
detector._log_info = lambda message: None
|
|
detector._handle_exception = lambda exc, domain: None
|
|
|
|
with patch("app.detectors.wayback_detector.config.WAYBACK_MAX_RECORDS", 3):
|
|
result = detector.scan_snapshots("example.com", sensitive_words=[], recent_years=5)
|
|
|
|
self.assertEqual(3, result["checked_snapshot_count"])
|
|
# 最新快照会先独立尝试一次，再进入裁剪后的扫描窗口。
|
|
self.assertEqual(4, result["fetched_snapshot_count"])
|
|
|
|
def test_scan_snapshots_fetches_recent_cdx_window_instead_of_full_history(self):
|
|
detector = WaybackDetector.__new__(WaybackDetector)
|
|
fetch_limits = []
|
|
|
|
def fake_fetch(domain, limit=None, fast_latest=False):
|
|
fetch_limits.append((limit, fast_latest))
|
|
if fast_latest:
|
|
return {
|
|
"records": [{"timestamp": "20260101000000", "digest": "latest"}],
|
|
"error": None,
|
|
}
|
|
return {
|
|
"records": [
|
|
{"timestamp": "20260101000000", "digest": "latest"},
|
|
{"timestamp": "20251201000000", "digest": "older-1"},
|
|
{"timestamp": "20251101000000", "digest": "older-2"},
|
|
{"timestamp": "20251001000000", "digest": "older-3"},
|
|
],
|
|
"error": None,
|
|
}
|
|
|
|
detector._fetch_cdx_records_with_meta = fake_fetch
|
|
detector._load_cached_records = lambda domain: None
|
|
detector._save_cached_records = lambda domain, records: None
|
|
detector._save_cached_timestamps = lambda domain, timestamps: None
|
|
detector._fetch_snapshot_title = lambda domain, timestamp: {
|
|
"timestamp": timestamp,
|
|
"title": f"title-{timestamp}",
|
|
"ok": True,
|
|
}
|
|
detector._normalize_title = lambda title: title
|
|
detector._find_sensitive_word = lambda title, words: None
|
|
detector._log_info = lambda message: None
|
|
detector._handle_exception = lambda exc, domain: None
|
|
|
|
with patch("app.detectors.wayback_detector.config.WAYBACK_MAX_RECORDS", 3):
|
|
detector.scan_snapshots("example.com", sensitive_words=[], recent_years=5)
|
|
|
|
self.assertEqual((-18, False), fetch_limits[1])
|
|
|
|
def test_scan_snapshots_continues_with_cached_records_when_latest_cdx_is_transient_failure(self):
|
|
detector = WaybackDetector.__new__(WaybackDetector)
|
|
detector._fetch_cdx_records_with_meta = lambda domain, limit=None, fast_latest=False: {
|
|
"records": [],
|
|
"error": "HTTPSConnectionPool(host='web.archive.org', port=443): Max retries exceeded",
|
|
}
|
|
detector._load_cached_records = lambda domain: [
|
|
{"timestamp": "20260101000000", "digest": "d1"},
|
|
{"timestamp": "20250101000000", "digest": "d2"},
|
|
]
|
|
detector._save_cached_records = lambda domain, records: None
|
|
detector._save_cached_timestamps = lambda domain, timestamps: None
|
|
detector._fetch_snapshot_title = lambda domain, timestamp: {
|
|
"timestamp": timestamp,
|
|
"title": f"title-{timestamp}",
|
|
"ok": True,
|
|
}
|
|
detector._normalize_title = lambda title: title
|
|
detector._find_sensitive_word = lambda title, words: None
|
|
detector._log_info = lambda message: None
|
|
detector._handle_exception = lambda exc, domain: None
|
|
detector._trip_transient_backoff = lambda: self.fail("single latest_cdx timeout should not trigger global backoff")
|
|
|
|
result = detector.scan_snapshots("example.com", sensitive_words=[], recent_years=5)
|
|
|
|
self.assertEqual(2, result["checked_snapshot_count"])
|
|
self.assertEqual(2, result["fetched_snapshot_count"])
|
|
self.assertGreaterEqual(result["request_error_count"], 1)
|
|
|
|
def test_scan_snapshots_skips_records_cdx_when_latest_cdx_transient_fails_without_cache(self):
|
|
detector = WaybackDetector.__new__(WaybackDetector)
|
|
fetch_calls = []
|
|
|
|
def fake_fetch(domain, limit=None, fast_latest=False):
|
|
fetch_calls.append((limit, fast_latest))
|
|
return {
|
|
"records": [],
|
|
"error": "HTTPSConnectionPool(host='web.archive.org', port=443): Max retries exceeded",
|
|
}
|
|
|
|
detector._fetch_cdx_records_with_meta = fake_fetch
|
|
detector._load_cached_records = lambda domain: None
|
|
detector._save_cached_records = lambda domain, records: None
|
|
detector._save_cached_timestamps = lambda domain, timestamps: None
|
|
detector._fetch_snapshot_title = lambda domain, timestamp: self.fail("should not fetch snapshot titles")
|
|
detector._normalize_title = lambda title: title
|
|
detector._find_sensitive_word = lambda title, words: None
|
|
detector._log_info = lambda message: None
|
|
detector._handle_exception = lambda exc, domain: None
|
|
trip_calls = []
|
|
detector._trip_transient_backoff = lambda: trip_calls.append("trip")
|
|
|
|
result = detector.scan_snapshots("example.com", sensitive_words=[], recent_years=5)
|
|
|
|
self.assertEqual([(-1, True)], fetch_calls)
|
|
self.assertEqual(1, len(trip_calls))
|
|
self.assertEqual(0, result["checked_snapshot_count"])
|
|
self.assertEqual(0, result["fetched_snapshot_count"])
|
|
self.assertGreaterEqual(result["request_error_count"], 1)
|
|
|
|
def test_scan_snapshots_continues_when_latest_snapshot_is_transient_failure(self):
|
|
detector = WaybackDetector.__new__(WaybackDetector)
|
|
detector._fetch_cdx_records_with_meta = lambda domain, limit=None, fast_latest=False: {
|
|
"records": [{"timestamp": "20260101000000", "digest": "latest"}] if fast_latest else [
|
|
{"timestamp": "20260101000000", "digest": "latest"},
|
|
{"timestamp": "20250101000000", "digest": "older"},
|
|
],
|
|
"error": None,
|
|
}
|
|
detector._load_cached_records = lambda domain: None
|
|
detector._save_cached_records = lambda domain, records: None
|
|
detector._save_cached_timestamps = lambda domain, timestamps: None
|
|
def fake_fetch(domain, timestamp):
|
|
if timestamp == "20260101000000":
|
|
return {
|
|
"timestamp": timestamp,
|
|
"title": "",
|
|
"ok": False,
|
|
"error": "HTTPSConnectionPool(host='web.archive.org', port=443): Max retries exceeded",
|
|
}
|
|
return {
|
|
"timestamp": timestamp,
|
|
"title": "older-title",
|
|
"ok": True,
|
|
}
|
|
|
|
detector._fetch_snapshot_title = fake_fetch
|
|
detector._normalize_title = lambda title: title
|
|
detector._find_sensitive_word = lambda title, words: None
|
|
detector._log_info = lambda message: None
|
|
detector._handle_exception = lambda exc, domain: None
|
|
detector._trip_transient_backoff = lambda: self.fail("single latest_snapshot timeout should not trigger global backoff")
|
|
|
|
result = detector.scan_snapshots("example.com", sensitive_words=[], recent_years=5)
|
|
|
|
self.assertEqual(2, result["checked_snapshot_count"])
|
|
self.assertEqual(1, result["fetched_snapshot_count"])
|
|
self.assertGreaterEqual(result["failed_snapshot_count"], 1)
|
|
self.assertTrue(
|
|
any("latest_snapshot:" in item for item in result["request_errors"])
|
|
)
|
|
|
|
def test_scan_snapshots_short_circuits_when_transient_backoff_active(self):
|
|
detector = WaybackDetector.__new__(WaybackDetector)
|
|
WaybackDetector._transient_backoff_until = time.time() + 5
|
|
detector._fetch_cdx_records_with_meta = lambda *args, **kwargs: self.fail("should not request cdx during backoff")
|
|
detector._load_cached_records = lambda domain: None
|
|
detector._save_cached_records = lambda domain, records: None
|
|
detector._save_cached_timestamps = lambda domain, timestamps: None
|
|
detector._fetch_snapshot_title = lambda domain, timestamp: self.fail("should not request snapshot during backoff")
|
|
detector._normalize_title = lambda title: title
|
|
detector._find_sensitive_word = lambda title, words: None
|
|
detector._log_info = lambda message: None
|
|
detector._handle_exception = lambda exc, domain: None
|
|
|
|
result = detector.scan_snapshots("example.com", sensitive_words=[], recent_years=5)
|
|
|
|
self.assertEqual(0, result["checked_snapshot_count"])
|
|
self.assertEqual(0, result["fetched_snapshot_count"])
|
|
self.assertEqual(1, result["request_error_count"])
|
|
self.assertTrue(any("wayback_backoff_active:" in item for item in result["request_errors"]))
|
|
|
|
def test_scan_snapshots_does_not_trip_global_backoff_on_single_snapshot_timeout(self):
|
|
detector = WaybackDetector.__new__(WaybackDetector)
|
|
detector._fetch_cdx_records_with_meta = lambda domain, limit=None, fast_latest=False: {
|
|
"records": [{"timestamp": "20260101000000", "digest": "latest"}] if fast_latest else [
|
|
{"timestamp": "20260101000000", "digest": "latest"},
|
|
{"timestamp": "20250101000000", "digest": "older"},
|
|
],
|
|
"error": None,
|
|
}
|
|
detector._load_cached_records = lambda domain: None
|
|
detector._save_cached_records = lambda domain, records: None
|
|
detector._save_cached_timestamps = lambda domain, timestamps: None
|
|
|
|
def fake_fetch(domain, timestamp):
|
|
if timestamp == "20260101000000":
|
|
return {"timestamp": timestamp, "title": "latest", "ok": True}
|
|
return {
|
|
"timestamp": timestamp,
|
|
"title": "",
|
|
"ok": False,
|
|
"error": "HTTPSConnectionPool(host='web.archive.org', port=443): Max retries exceeded",
|
|
}
|
|
|
|
detector._fetch_snapshot_title = fake_fetch
|
|
detector._normalize_title = lambda title: title
|
|
detector._find_sensitive_word = lambda title, words: None
|
|
detector._log_info = lambda message: None
|
|
detector._handle_exception = lambda exc, domain: None
|
|
detector._trip_transient_backoff = lambda: self.fail("single snapshot timeout should not trigger global backoff")
|
|
|
|
result = detector.scan_snapshots("example.com", sensitive_words=[], recent_years=5)
|
|
|
|
self.assertEqual(2, result["checked_snapshot_count"])
|
|
self.assertEqual(1, result["fetched_snapshot_count"])
|
|
self.assertEqual(1, result["failed_snapshot_count"])
|
|
|
|
def test_scan_snapshots_trips_global_backoff_after_threshold_transient_failures(self):
|
|
detector = WaybackDetector.__new__(WaybackDetector)
|
|
detector._fetch_cdx_records_with_meta = lambda domain, limit=None, fast_latest=False: {
|
|
"records": [{"timestamp": "20260101000000", "digest": "latest"}] if fast_latest else [
|
|
{"timestamp": "20260101000000", "digest": "latest"},
|
|
{"timestamp": "20250101000000", "digest": "older-1"},
|
|
{"timestamp": "20240101000000", "digest": "older-2"},
|
|
],
|
|
"error": None,
|
|
}
|
|
detector._load_cached_records = lambda domain: None
|
|
detector._save_cached_records = lambda domain, records: None
|
|
detector._save_cached_timestamps = lambda domain, timestamps: None
|
|
|
|
def fake_fetch(domain, timestamp):
|
|
if timestamp == "20260101000000":
|
|
return {"timestamp": timestamp, "title": "latest", "ok": True}
|
|
return {
|
|
"timestamp": timestamp,
|
|
"title": "",
|
|
"ok": False,
|
|
"error": "HTTPSConnectionPool(host='web.archive.org', port=443): Max retries exceeded",
|
|
}
|
|
|
|
detector._fetch_snapshot_title = fake_fetch
|
|
detector._normalize_title = lambda title: title
|
|
detector._find_sensitive_word = lambda title, words: None
|
|
detector._log_info = lambda message: None
|
|
detector._handle_exception = lambda exc, domain: None
|
|
trip_calls = []
|
|
detector._trip_transient_backoff = lambda: trip_calls.append("trip")
|
|
|
|
result = detector.scan_snapshots("example.com", sensitive_words=[], recent_years=5)
|
|
|
|
self.assertEqual(1, len(trip_calls))
|
|
self.assertEqual(3, result["checked_snapshot_count"])
|
|
self.assertEqual(2, result["failed_snapshot_count"])
|
|
|
|
def test_build_session_disables_env_proxy(self):
|
|
detector = WaybackDetector.__new__(WaybackDetector)
|
|
|
|
session = detector._build_session()
|
|
|
|
self.assertFalse(session.trust_env)
|
|
|
|
|
|
if __name__ == "__main__":
|
|
unittest.main()
|