import sys import time import unittest from unittest.mock import patch sys.path.insert(0, "/www/wwwroot/getDomain/domainCheck") from app.detectors.wayback_detector import WaybackDetector # noqa: E402 class WaybackDetectorRecentYearsTest(unittest.TestCase): def setUp(self): WaybackDetector._transient_backoff_until = 0.0 def test_filter_records_recent_years_drops_old_snapshots(self): detector = WaybackDetector.__new__(WaybackDetector) current_year = time.gmtime().tm_year records = [ {"timestamp": f"{current_year}0101000000", "digest": "new"}, {"timestamp": f"{current_year - 2}0101000000", "digest": "mid"}, {"timestamp": f"{current_year - 6}0101000000", "digest": "old"}, ] filtered = detector._filter_records_recent_years(records, recent_years=5) self.assertEqual( [ f"{current_year}0101000000", f"{current_year - 2}0101000000", ], [item["timestamp"] for item in filtered], ) def test_scan_snapshots_only_counts_recent_years_window(self): detector = WaybackDetector.__new__(WaybackDetector) current_year = time.gmtime().tm_year detector._fetch_cdx_records_with_meta = lambda domain, limit=None, fast_latest=False: { "records": [ {"timestamp": f"{current_year}0101000000", "digest": "latest"}, {"timestamp": f"{current_year - 1}0101000000", "digest": "prev"}, {"timestamp": f"{current_year - 6}0101000000", "digest": "old"}, ] } detector._load_cached_records = lambda domain: None detector._save_cached_records = lambda domain, records: None detector._save_cached_timestamps = lambda domain, timestamps: None detector._fetch_snapshot_title = lambda domain, timestamp: { "timestamp": timestamp, "title": f"title-{timestamp}", "ok": True, } detector._normalize_title = lambda title: title detector._find_sensitive_word = lambda title, words: None detector._log_info = lambda message: None detector._handle_exception = lambda exc, domain: None result = detector.scan_snapshots("example.com", sensitive_words=[], recent_years=5) self.assertEqual(2, result["checked_snapshot_count"]) self.assertEqual(2, result["fetched_snapshot_count"]) self.assertEqual(sorted([current_year - 1, current_year]), result["snapshot_years"]) def test_scan_snapshots_limits_records_per_domain(self): detector = WaybackDetector.__new__(WaybackDetector) current_year = time.gmtime().tm_year records = [ {"timestamp": f"{current_year}01010{i}0000", "digest": f"d{i}"} for i in range(6) ] detector._fetch_cdx_records_with_meta = lambda domain, limit=None, fast_latest=False: { "records": records, "error": None, } detector._load_cached_records = lambda domain: None detector._save_cached_records = lambda domain, records: None detector._save_cached_timestamps = lambda domain, timestamps: None detector._fetch_snapshot_title = lambda domain, timestamp: { "timestamp": timestamp, "title": f"title-{timestamp}", "ok": True, } detector._normalize_title = lambda title: title detector._find_sensitive_word = lambda title, words: None detector._log_info = lambda message: None detector._handle_exception = lambda exc, domain: None with patch("app.detectors.wayback_detector.config.WAYBACK_MAX_RECORDS", 3): result = detector.scan_snapshots("example.com", sensitive_words=[], recent_years=5) self.assertEqual(3, result["checked_snapshot_count"]) # 最新快照会先独立尝试一次，再进入裁剪后的扫描窗口。 self.assertEqual(4, result["fetched_snapshot_count"]) def test_scan_snapshots_fast_degrades_when_latest_cdx_is_transient_failure(self): detector = WaybackDetector.__new__(WaybackDetector) detector._fetch_cdx_records_with_meta = lambda domain, limit=None, fast_latest=False: { "records": [], "error": "HTTPSConnectionPool(host='web.archive.org', port=443): Max retries exceeded", } detector._load_cached_records = lambda domain: [ {"timestamp": "20260101000000", "digest": "d1"}, {"timestamp": "20250101000000", "digest": "d2"}, ] detector._save_cached_records = lambda domain, records: None detector._save_cached_timestamps = lambda domain, timestamps: None detector._fetch_snapshot_title = lambda domain, timestamp: self.fail("should not fetch snapshot titles") detector._normalize_title = lambda title: title detector._find_sensitive_word = lambda title, words: None detector._log_info = lambda message: None detector._handle_exception = lambda exc, domain: None result = detector.scan_snapshots("example.com", sensitive_words=[], recent_years=5) self.assertEqual(0, result["checked_snapshot_count"]) self.assertGreaterEqual(result["failed_snapshot_count"], 1) self.assertGreaterEqual(result["request_error_count"], 1) def test_scan_snapshots_fast_degrades_when_latest_snapshot_is_transient_failure(self): detector = WaybackDetector.__new__(WaybackDetector) detector._fetch_cdx_records_with_meta = lambda domain, limit=None, fast_latest=False: { "records": [{"timestamp": "20260101000000", "digest": "latest"}] if fast_latest else [ {"timestamp": "20260101000000", "digest": "latest"}, {"timestamp": "20250101000000", "digest": "older"}, ], "error": None, } detector._load_cached_records = lambda domain: None detector._save_cached_records = lambda domain, records: None detector._save_cached_timestamps = lambda domain, timestamps: None detector._fetch_snapshot_title = lambda domain, timestamp: { "timestamp": timestamp, "title": "", "ok": False, "error": "HTTPSConnectionPool(host='web.archive.org', port=443): Max retries exceeded", } detector._normalize_title = lambda title: title detector._find_sensitive_word = lambda title, words: None detector._log_info = lambda message: None detector._handle_exception = lambda exc, domain: None result = detector.scan_snapshots("example.com", sensitive_words=[], recent_years=5) self.assertEqual(1, result["checked_snapshot_count"]) self.assertEqual(0, result["fetched_snapshot_count"]) self.assertGreaterEqual(result["failed_snapshot_count"], 1) self.assertTrue( any("latest_snapshot:" in item for item in result["request_errors"]) ) def test_scan_snapshots_short_circuits_when_transient_backoff_active(self): detector = WaybackDetector.__new__(WaybackDetector) WaybackDetector._transient_backoff_until = time.time() + 5 detector._fetch_cdx_records_with_meta = lambda *args, **kwargs: self.fail("should not request cdx during backoff") detector._load_cached_records = lambda domain: None detector._save_cached_records = lambda domain, records: None detector._save_cached_timestamps = lambda domain, timestamps: None detector._fetch_snapshot_title = lambda domain, timestamp: self.fail("should not request snapshot during backoff") detector._normalize_title = lambda title: title detector._find_sensitive_word = lambda title, words: None detector._log_info = lambda message: None detector._handle_exception = lambda exc, domain: None result = detector.scan_snapshots("example.com", sensitive_words=[], recent_years=5) self.assertEqual(0, result["checked_snapshot_count"]) self.assertEqual(0, result["fetched_snapshot_count"]) self.assertEqual(1, result["request_error_count"]) self.assertTrue(any("wayback_backoff_active:" in item for item in result["request_errors"])) if __name__ == "__main__": unittest.main()