admin/getDomain
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

feat: stabilize multi-region runtime sync and worker orchestration

Browse Source
This commit is contained in:
root
2026-04-27 15:48:12 +08:00
parent 7cbde2aa78
commit 215a364891
137 changed files with 31931 additions and 1943 deletions
Download Patch File Download Diff File Expand all files Collapse all files

189
domainCheck/tests/test_wayback_detector_recent_years.py
View File

@@ -92,7 +92,47 @@ class WaybackDetectorRecentYearsTest(unittest.TestCase):
# 最新快照会先独立尝试一次，再进入裁剪后的扫描窗口。
self.assertEqual(4, result["fetched_snapshot_count"])
def test_scan_snapshots_fast_degrades_when_latest_cdx_is_transient_failure(self):
def test_scan_snapshots_fetches_recent_cdx_window_instead_of_full_history(self):
detector = WaybackDetector.__new__(WaybackDetector)
fetch_limits = []
def fake_fetch(domain, limit=None, fast_latest=False):
fetch_limits.append((limit, fast_latest))
if fast_latest:
return {
"records": [{"timestamp": "20260101000000", "digest": "latest"}],
"error": None,
}
return {
"records": [
{"timestamp": "20260101000000", "digest": "latest"},
{"timestamp": "20251201000000", "digest": "older-1"},
{"timestamp": "20251101000000", "digest": "older-2"},
{"timestamp": "20251001000000", "digest": "older-3"},
],
"error": None,
}
detector._fetch_cdx_records_with_meta = fake_fetch
detector._load_cached_records = lambda domain: None
detector._save_cached_records = lambda domain, records: None
detector._save_cached_timestamps = lambda domain, timestamps: None
detector._fetch_snapshot_title = lambda domain, timestamp: {
"timestamp": timestamp,
"title": f"title-{timestamp}",
"ok": True,
}
detector._normalize_title = lambda title: title
detector._find_sensitive_word = lambda title, words: None
detector._log_info = lambda message: None
detector._handle_exception = lambda exc, domain: None
with patch("app.detectors.wayback_detector.config.WAYBACK_MAX_RECORDS", 3):
detector.scan_snapshots("example.com", sensitive_words=[], recent_years=5)
self.assertEqual((-18, False), fetch_limits[1])
def test_scan_snapshots_continues_with_cached_records_when_latest_cdx_is_transient_failure(self):
detector = WaybackDetector.__new__(WaybackDetector)
detector._fetch_cdx_records_with_meta = lambda domain, limit=None, fast_latest=False: {
"records": [],
@@ -104,19 +144,55 @@ class WaybackDetectorRecentYearsTest(unittest.TestCase):
]
detector._save_cached_records = lambda domain, records: None
detector._save_cached_timestamps = lambda domain, timestamps: None
detector._fetch_snapshot_title = lambda domain, timestamp: {
"timestamp": timestamp,
"title": f"title-{timestamp}",
"ok": True,
}
detector._normalize_title = lambda title: title
detector._find_sensitive_word = lambda title, words: None
detector._log_info = lambda message: None
detector._handle_exception = lambda exc, domain: None
detector._trip_transient_backoff = lambda: self.fail("single latest_cdx timeout should not trigger global backoff")
result = detector.scan_snapshots("example.com", sensitive_words=[], recent_years=5)
self.assertEqual(2, result["checked_snapshot_count"])
self.assertEqual(2, result["fetched_snapshot_count"])
self.assertGreaterEqual(result["request_error_count"], 1)
def test_scan_snapshots_skips_records_cdx_when_latest_cdx_transient_fails_without_cache(self):
detector = WaybackDetector.__new__(WaybackDetector)
fetch_calls = []
def fake_fetch(domain, limit=None, fast_latest=False):
fetch_calls.append((limit, fast_latest))
return {
"records": [],
"error": "HTTPSConnectionPool(host='web.archive.org', port=443): Max retries exceeded",
}
detector._fetch_cdx_records_with_meta = fake_fetch
detector._load_cached_records = lambda domain: None
detector._save_cached_records = lambda domain, records: None
detector._save_cached_timestamps = lambda domain, timestamps: None
detector._fetch_snapshot_title = lambda domain, timestamp: self.fail("should not fetch snapshot titles")
detector._normalize_title = lambda title: title
detector._find_sensitive_word = lambda title, words: None
detector._log_info = lambda message: None
detector._handle_exception = lambda exc, domain: None
trip_calls = []
detector._trip_transient_backoff = lambda: trip_calls.append("trip")
result = detector.scan_snapshots("example.com", sensitive_words=[], recent_years=5)
self.assertEqual([(-1, True)], fetch_calls)
self.assertEqual(1, len(trip_calls))
self.assertEqual(0, result["checked_snapshot_count"])
self.assertGreaterEqual(result["failed_snapshot_count"], 1)
self.assertEqual(0, result["fetched_snapshot_count"])
self.assertGreaterEqual(result["request_error_count"], 1)
def test_scan_snapshots_fast_degrades_when_latest_snapshot_is_transient_failure(self):
def test_scan_snapshots_continues_when_latest_snapshot_is_transient_failure(self):
detector = WaybackDetector.__new__(WaybackDetector)
detector._fetch_cdx_records_with_meta = lambda domain, limit=None, fast_latest=False: {
"records": [{"timestamp": "20260101000000", "digest": "latest"}] if fast_latest else [
@@ -128,21 +204,31 @@ class WaybackDetectorRecentYearsTest(unittest.TestCase):
detector._load_cached_records = lambda domain: None
detector._save_cached_records = lambda domain, records: None
detector._save_cached_timestamps = lambda domain, timestamps: None
detector._fetch_snapshot_title = lambda domain, timestamp: {
"timestamp": timestamp,
"title": "",
"ok": False,
"error": "HTTPSConnectionPool(host='web.archive.org', port=443): Max retries exceeded",
}
def fake_fetch(domain, timestamp):
if timestamp == "20260101000000":
return {
"timestamp": timestamp,
"title": "",
"ok": False,
"error": "HTTPSConnectionPool(host='web.archive.org', port=443): Max retries exceeded",
}
return {
"timestamp": timestamp,
"title": "older-title",
"ok": True,
}
detector._fetch_snapshot_title = fake_fetch
detector._normalize_title = lambda title: title
detector._find_sensitive_word = lambda title, words: None
detector._log_info = lambda message: None
detector._handle_exception = lambda exc, domain: None
detector._trip_transient_backoff = lambda: self.fail("single latest_snapshot timeout should not trigger global backoff")
result = detector.scan_snapshots("example.com", sensitive_words=[], recent_years=5)
self.assertEqual(1, result["checked_snapshot_count"])
self.assertEqual(0, result["fetched_snapshot_count"])
self.assertEqual(2, result["checked_snapshot_count"])
self.assertEqual(1, result["fetched_snapshot_count"])
self.assertGreaterEqual(result["failed_snapshot_count"], 1)
self.assertTrue(
any("latest_snapshot:" in item for item in result["request_errors"])
@@ -168,6 +254,87 @@ class WaybackDetectorRecentYearsTest(unittest.TestCase):
self.assertEqual(1, result["request_error_count"])
self.assertTrue(any("wayback_backoff_active:" in item for item in result["request_errors"]))
def test_scan_snapshots_does_not_trip_global_backoff_on_single_snapshot_timeout(self):
detector = WaybackDetector.__new__(WaybackDetector)
detector._fetch_cdx_records_with_meta = lambda domain, limit=None, fast_latest=False: {
"records": [{"timestamp": "20260101000000", "digest": "latest"}] if fast_latest else [
{"timestamp": "20260101000000", "digest": "latest"},
{"timestamp": "20250101000000", "digest": "older"},
],
"error": None,
}
detector._load_cached_records = lambda domain: None
detector._save_cached_records = lambda domain, records: None
detector._save_cached_timestamps = lambda domain, timestamps: None
def fake_fetch(domain, timestamp):
if timestamp == "20260101000000":
return {"timestamp": timestamp, "title": "latest", "ok": True}
return {
"timestamp": timestamp,
"title": "",
"ok": False,
"error": "HTTPSConnectionPool(host='web.archive.org', port=443): Max retries exceeded",
}
detector._fetch_snapshot_title = fake_fetch
detector._normalize_title = lambda title: title
detector._find_sensitive_word = lambda title, words: None
detector._log_info = lambda message: None
detector._handle_exception = lambda exc, domain: None
detector._trip_transient_backoff = lambda: self.fail("single snapshot timeout should not trigger global backoff")
result = detector.scan_snapshots("example.com", sensitive_words=[], recent_years=5)
self.assertEqual(2, result["checked_snapshot_count"])
self.assertEqual(1, result["fetched_snapshot_count"])
self.assertEqual(1, result["failed_snapshot_count"])
def test_scan_snapshots_trips_global_backoff_after_threshold_transient_failures(self):
detector = WaybackDetector.__new__(WaybackDetector)
detector._fetch_cdx_records_with_meta = lambda domain, limit=None, fast_latest=False: {
"records": [{"timestamp": "20260101000000", "digest": "latest"}] if fast_latest else [
{"timestamp": "20260101000000", "digest": "latest"},
{"timestamp": "20250101000000", "digest": "older-1"},
{"timestamp": "20240101000000", "digest": "older-2"},
],
"error": None,
}
detector._load_cached_records = lambda domain: None
detector._save_cached_records = lambda domain, records: None
detector._save_cached_timestamps = lambda domain, timestamps: None
def fake_fetch(domain, timestamp):
if timestamp == "20260101000000":
return {"timestamp": timestamp, "title": "latest", "ok": True}
return {
"timestamp": timestamp,
"title": "",
"ok": False,
"error": "HTTPSConnectionPool(host='web.archive.org', port=443): Max retries exceeded",
}
detector._fetch_snapshot_title = fake_fetch
detector._normalize_title = lambda title: title
detector._find_sensitive_word = lambda title, words: None
detector._log_info = lambda message: None
detector._handle_exception = lambda exc, domain: None
trip_calls = []
detector._trip_transient_backoff = lambda: trip_calls.append("trip")
result = detector.scan_snapshots("example.com", sensitive_words=[], recent_years=5)
self.assertEqual(1, len(trip_calls))
self.assertEqual(3, result["checked_snapshot_count"])
self.assertEqual(2, result["failed_snapshot_count"])
def test_build_session_disables_env_proxy(self):
detector = WaybackDetector.__new__(WaybackDetector)
session = detector._build_session()
self.assertFalse(session.trust_env)
if __name__ == "__main__":
unittest.main()