d

domainCheck/tests/test_wayback_detector_recent_years.py

@@ -0,0 +1,173 @@
+import sys
+import time
+import unittest
+from unittest.mock import patch
+
+
+sys.path.insert(0, "/www/wwwroot/getDomain/domainCheck")
+
+from app.detectors.wayback_detector import WaybackDetector  # noqa: E402
+
+
+class WaybackDetectorRecentYearsTest(unittest.TestCase):
+    def setUp(self):
+        WaybackDetector._transient_backoff_until = 0.0
+
+    def test_filter_records_recent_years_drops_old_snapshots(self):
+        detector = WaybackDetector.__new__(WaybackDetector)
+        current_year = time.gmtime().tm_year
+        records = [
+            {"timestamp": f"{current_year}0101000000", "digest": "new"},
+            {"timestamp": f"{current_year - 2}0101000000", "digest": "mid"},
+            {"timestamp": f"{current_year - 6}0101000000", "digest": "old"},
+        ]
+
+        filtered = detector._filter_records_recent_years(records, recent_years=5)
+
+        self.assertEqual(
+            [
+                f"{current_year}0101000000",
+                f"{current_year - 2}0101000000",
+            ],
+            [item["timestamp"] for item in filtered],
+        )
+
+    def test_scan_snapshots_only_counts_recent_years_window(self):
+        detector = WaybackDetector.__new__(WaybackDetector)
+        current_year = time.gmtime().tm_year
+        detector._fetch_cdx_records_with_meta = lambda domain, limit=None, fast_latest=False: {
+            "records": [
+                {"timestamp": f"{current_year}0101000000", "digest": "latest"},
+                {"timestamp": f"{current_year - 1}0101000000", "digest": "prev"},
+                {"timestamp": f"{current_year - 6}0101000000", "digest": "old"},
+            ]
+        }
+        detector._load_cached_records = lambda domain: None
+        detector._save_cached_records = lambda domain, records: None
+        detector._save_cached_timestamps = lambda domain, timestamps: None
+        detector._fetch_snapshot_title = lambda domain, timestamp: {
+            "timestamp": timestamp,
+            "title": f"title-{timestamp}",
+            "ok": True,
+        }
+        detector._normalize_title = lambda title: title
+        detector._find_sensitive_word = lambda title, words: None
+        detector._log_info = lambda message: None
+        detector._handle_exception = lambda exc, domain: None
+
+        result = detector.scan_snapshots("example.com", sensitive_words=[], recent_years=5)
+
+        self.assertEqual(2, result["checked_snapshot_count"])
+        self.assertEqual(2, result["fetched_snapshot_count"])
+        self.assertEqual(sorted([current_year - 1, current_year]), result["snapshot_years"])
+
+    def test_scan_snapshots_limits_records_per_domain(self):
+        detector = WaybackDetector.__new__(WaybackDetector)
+        current_year = time.gmtime().tm_year
+        records = [
+            {"timestamp": f"{current_year}01010{i}0000", "digest": f"d{i}"}
+            for i in range(6)
+        ]
+        detector._fetch_cdx_records_with_meta = lambda domain, limit=None, fast_latest=False: {
+            "records": records,
+            "error": None,
+        }
+        detector._load_cached_records = lambda domain: None
+        detector._save_cached_records = lambda domain, records: None
+        detector._save_cached_timestamps = lambda domain, timestamps: None
+        detector._fetch_snapshot_title = lambda domain, timestamp: {
+            "timestamp": timestamp,
+            "title": f"title-{timestamp}",
+            "ok": True,
+        }
+        detector._normalize_title = lambda title: title
+        detector._find_sensitive_word = lambda title, words: None
+        detector._log_info = lambda message: None
+        detector._handle_exception = lambda exc, domain: None
+
+        with patch("app.detectors.wayback_detector.config.WAYBACK_MAX_RECORDS", 3):
+            result = detector.scan_snapshots("example.com", sensitive_words=[], recent_years=5)
+
+        self.assertEqual(3, result["checked_snapshot_count"])
+        # 最新快照会先独立尝试一次，再进入裁剪后的扫描窗口。
+        self.assertEqual(4, result["fetched_snapshot_count"])
+
+    def test_scan_snapshots_fast_degrades_when_latest_cdx_is_transient_failure(self):
+        detector = WaybackDetector.__new__(WaybackDetector)
+        detector._fetch_cdx_records_with_meta = lambda domain, limit=None, fast_latest=False: {
+            "records": [],
+            "error": "HTTPSConnectionPool(host='web.archive.org', port=443): Max retries exceeded",
+        }
+        detector._load_cached_records = lambda domain: [
+            {"timestamp": "20260101000000", "digest": "d1"},
+            {"timestamp": "20250101000000", "digest": "d2"},
+        ]
+        detector._save_cached_records = lambda domain, records: None
+        detector._save_cached_timestamps = lambda domain, timestamps: None
+        detector._fetch_snapshot_title = lambda domain, timestamp: self.fail("should not fetch snapshot titles")
+        detector._normalize_title = lambda title: title
+        detector._find_sensitive_word = lambda title, words: None
+        detector._log_info = lambda message: None
+        detector._handle_exception = lambda exc, domain: None
+
+        result = detector.scan_snapshots("example.com", sensitive_words=[], recent_years=5)
+
+        self.assertEqual(0, result["checked_snapshot_count"])
+        self.assertGreaterEqual(result["failed_snapshot_count"], 1)
+        self.assertGreaterEqual(result["request_error_count"], 1)
+
+    def test_scan_snapshots_fast_degrades_when_latest_snapshot_is_transient_failure(self):
+        detector = WaybackDetector.__new__(WaybackDetector)
+        detector._fetch_cdx_records_with_meta = lambda domain, limit=None, fast_latest=False: {
+            "records": [{"timestamp": "20260101000000", "digest": "latest"}] if fast_latest else [
+                {"timestamp": "20260101000000", "digest": "latest"},
+                {"timestamp": "20250101000000", "digest": "older"},
+            ],
+            "error": None,
+        }
+        detector._load_cached_records = lambda domain: None
+        detector._save_cached_records = lambda domain, records: None
+        detector._save_cached_timestamps = lambda domain, timestamps: None
+        detector._fetch_snapshot_title = lambda domain, timestamp: {
+            "timestamp": timestamp,
+            "title": "",
+            "ok": False,
+            "error": "HTTPSConnectionPool(host='web.archive.org', port=443): Max retries exceeded",
+        }
+        detector._normalize_title = lambda title: title
+        detector._find_sensitive_word = lambda title, words: None
+        detector._log_info = lambda message: None
+        detector._handle_exception = lambda exc, domain: None
+
+        result = detector.scan_snapshots("example.com", sensitive_words=[], recent_years=5)
+
+        self.assertEqual(1, result["checked_snapshot_count"])
+        self.assertEqual(0, result["fetched_snapshot_count"])
+        self.assertGreaterEqual(result["failed_snapshot_count"], 1)
+        self.assertTrue(
+            any("latest_snapshot:" in item for item in result["request_errors"])
+        )
+
+    def test_scan_snapshots_short_circuits_when_transient_backoff_active(self):
+        detector = WaybackDetector.__new__(WaybackDetector)
+        WaybackDetector._transient_backoff_until = time.time() + 5
+        detector._fetch_cdx_records_with_meta = lambda *args, **kwargs: self.fail("should not request cdx during backoff")
+        detector._load_cached_records = lambda domain: None
+        detector._save_cached_records = lambda domain, records: None
+        detector._save_cached_timestamps = lambda domain, timestamps: None
+        detector._fetch_snapshot_title = lambda domain, timestamp: self.fail("should not request snapshot during backoff")
+        detector._normalize_title = lambda title: title
+        detector._find_sensitive_word = lambda title, words: None
+        detector._log_info = lambda message: None
+        detector._handle_exception = lambda exc, domain: None
+
+        result = detector.scan_snapshots("example.com", sensitive_words=[], recent_years=5)
+
+        self.assertEqual(0, result["checked_snapshot_count"])
+        self.assertEqual(0, result["fetched_snapshot_count"])
+        self.assertEqual(1, result["request_error_count"])
+        self.assertTrue(any("wayback_backoff_active:" in item for item in result["request_errors"]))
+
+
+if __name__ == "__main__":
+    unittest.main()