admin/getDomain
1
0
Fork 0
Code Issues Pull Requests Actions Packages Projects Releases Wiki Activity

d

Browse Source
This commit is contained in:
Your Name
2026-04-22 14:13:21 +08:00
parent e0406b5d0e
commit 7cbde2aa78
145 changed files with 23086 additions and 2243 deletions
Download Patch File Download Diff File Expand all files Collapse all files

55
domainCheck/tests/test_database_claim_priority.py Normal file
View File

@@ -0,0 +1,55 @@
import unittest
import sys
sys.path.insert(0, "/www/wwwroot/getDomain/domainCheck")
from app.utils.database import _detect_job_item_step_priority
from app.utils.database import _ordered_step_claim_codes
from app.utils.database import _resolve_step_claim_quota
class DetectJobItemClaimPriorityTestCase(unittest.TestCase):
def test_ordered_step_claim_codes_prioritize_deeper_steps_first(self):
ordered = _ordered_step_claim_codes()
self.assertLess(ordered.index("detect_360_site"), ordered.index("detect_baidu_site"))
self.assertLess(ordered.index("detect_chinaz"), ordered.index("detect_360_site"))
self.assertGreater(ordered.index("detect_register"), ordered.index("detect_baidu_site"))
def test_later_pipeline_steps_are_prioritized_ahead_of_earlier_steps(self):
self.assertLess(
_detect_job_item_step_priority("detect_360_site"),
_detect_job_item_step_priority("detect_baidu_site"),
)
self.assertLess(
_detect_job_item_step_priority("detect_chinaz"),
_detect_job_item_step_priority("detect_360_site"),
)
self.assertLess(
_detect_job_item_step_priority("detect_aizhan"),
_detect_job_item_step_priority("detect_chinaz"),
)
def test_non_register_steps_are_prioritized_ahead_of_register(self):
self.assertLess(
_detect_job_item_step_priority("detect_baidu_site"),
_detect_job_item_step_priority("detect_register"),
)
self.assertLess(
_detect_job_item_step_priority("detect_wayback"),
_detect_job_item_step_priority("detect_register"),
)
def test_blank_step_code_has_lowest_priority(self):
self.assertGreater(
_detect_job_item_step_priority(""),
_detect_job_item_step_priority("detect_register"),
)
def test_step_claim_quota_defaults_to_quarter_window_with_floor(self):
self.assertEqual(64, _resolve_step_claim_quota(120))
self.assertEqual(300, _resolve_step_claim_quota(1200))
if __name__ == "__main__":
unittest.main()

225
domainCheck/tests/test_detect_worker_flush_retry.py Normal file
View File

@@ -0,0 +1,225 @@
import collections
import threading
import unittest
import sys
from unittest.mock import MagicMock
from unittest.mock import patch
sys.path.insert(0, "/www/wwwroot/getDomain/domainCheck")
from detect_worker import DetectWorker # noqa: E402
class _FakeDb:
def __init__(self, method_name: str):
self.method_name = method_name
self.calls = 0
def _run(self, batch):
self.calls += 1
return -1 if self.calls == 1 else len(batch)
def mark_detect_job_items_running_batch(self, batch):
return self._run(batch)
def finalize_detect_job_items_batch(self, batch):
return self._run(batch)
class DetectWorkerFlushRetryTest(unittest.TestCase):
def _build_worker(self, db):
worker = DetectWorker.__new__(DetectWorker)
worker.db = db
worker._task_local = threading.local()
worker._running_mark_lock = threading.Lock()
worker._pending_running_marks = collections.deque()
worker._last_running_mark_flush_at = 0.0
worker._job_finalize_lock = threading.Lock()
worker._pending_job_finalizations = collections.deque()
worker._last_job_finalize_flush_at = 0.0
worker.proxy_pool = []
worker.proxy_pool_lock = threading.Lock()
worker.proxy_failure_lock = threading.Lock()
worker.proxy_failure_counts = {}
worker.proxy_quarantine_until = {}
worker.proxy_source_by_key = {}
worker.proxy_source_failure_counts = {}
worker.proxy_source_quarantine_until = {}
worker.proxy_config = {"proxy_enable": True, "allow_direct": True}
worker.proxy_last_refresh_status = "test"
worker.trigger_proxy_refresh = MagicMock()
worker.runtime_settings = {}
worker.thread_count = 600
worker.detecting = True
worker._get_active_domain_threads = MagicMock(return_value=0)
return worker
def test_running_mark_flush_requeues_batch_when_db_write_fails(self):
worker = self._build_worker(_FakeDb("running"))
worker._pending_running_marks.extend([(101, "token-a"), (102, "token-b")])
updated = worker._flush_pending_running_marks(force=True, batch_limit=10)
self.assertEqual(0, updated)
self.assertEqual([(101, "token-a"), (102, "token-b")], list(worker._pending_running_marks))
updated = worker._flush_pending_running_marks(force=True, batch_limit=10)
self.assertEqual(2, updated)
self.assertEqual([], list(worker._pending_running_marks))
def test_job_finalization_flush_requeues_batch_when_db_write_fails(self):
worker = self._build_worker(_FakeDb("finalize"))
worker._pending_job_finalizations.extend(
[
{"job_item_id": 201, "claim_token": "token-a", "final_status": "completed"},
{"job_item_id": 202, "claim_token": "token-b", "final_status": "failed"},
]
)
updated = worker._flush_pending_job_finalizations(force=True, batch_limit=10)
self.assertEqual(0, updated)
self.assertEqual(2, len(worker._pending_job_finalizations))
updated = worker._flush_pending_job_finalizations(force=True, batch_limit=10)
self.assertEqual(2, updated)
self.assertEqual(0, len(worker._pending_job_finalizations))
def test_proxy_auth_failure_quarantines_whole_source_group(self):
worker = self._build_worker(_FakeDb("noop"))
bad_proxy = {"http": "http://bad-proxy", "https": "http://bad-proxy"}
good_proxy = {"http": "http://good-proxy", "https": "http://good-proxy"}
worker.proxy_pool = [
{"proxy": bad_proxy, "usage_count": 0, "source_tag": "D"},
{"proxy": good_proxy, "usage_count": 0, "source_tag": "E"},
]
worker.proxy_source_by_key[worker._proxy_key(bad_proxy)] = "D"
worker.proxy_source_by_key[worker._proxy_key(good_proxy)] = "E"
source_tag, failure_count, cooldown_seconds, removed = worker._mark_proxy_source_failure(
bad_proxy,
"Tunnel connection failed: 407 Proxy Authentication Required",
)
self.assertEqual("D", source_tag)
self.assertEqual(1, failure_count)
self.assertGreaterEqual(cooldown_seconds, 90)
self.assertEqual(1, removed)
self.assertTrue(worker._is_proxy_source_quarantined("D"))
self.assertEqual(
[{"proxy": good_proxy, "usage_count": 0, "source_tag": "E"}],
worker.proxy_pool,
)
def test_prepare_proxy_fetch_plan_respects_pool_shortage_instead_of_overfetching(self):
worker = self._build_worker(_FakeDb("noop"))
plan, meta = worker._prepare_proxy_fetch_plan(
[
"http://provider.local/get?group=B",
"http://provider.local/get?group=C",
"http://provider.local/get?group=D",
"http://provider.local/get?group=E",
"http://provider.local/get?group=F",
"http://provider.local/get?group=G",
],
current_pool_size=1000,
)
self.assertEqual(1, meta["rounds"])
self.assertGreaterEqual(meta["target_total"], 840)
self.assertLessEqual(meta["target_total"], 1200)
self.assertEqual(6, len(plan))
def test_prepare_proxy_fetch_plan_caps_active_rounds(self):
worker = self._build_worker(_FakeDb("noop"))
plan, meta = worker._prepare_proxy_fetch_plan(
[
"http://provider.local/get?group=B",
"http://provider.local/get?group=C",
"http://provider.local/get?group=D",
"http://provider.local/get?group=E",
"http://provider.local/get?group=F",
"http://provider.local/get?group=G",
],
current_pool_size=0,
)
self.assertLessEqual(meta["rounds"], 2)
self.assertEqual(meta["rounds"] * 6, len(plan))
def test_submit_backlog_limit_defaults_to_thread_count_window(self):
worker = self._build_worker(_FakeDb("noop"))
self.assertEqual(2000, worker._resolve_submit_backlog_limit(2000, 0))
self.assertEqual(2000, worker._resolve_submit_backlog_limit(2000, 1200))
def test_dispatch_capacity_can_ramp_to_full_thread_limit(self):
worker = self._build_worker(_FakeDb("noop"))
self.assertEqual(2000, worker._resolve_dispatch_capacity(2000, 0, 2000))
self.assertEqual(2000, worker._resolve_dispatch_capacity(2000, 1000, 1000))
def test_claim_batch_size_defaults_to_thread_count_window(self):
worker = self._build_worker(_FakeDb("noop"))
self.assertEqual(2000, worker._resolve_claim_batch_size(2000, 2000, live_active=0))
self.assertEqual(1500, worker._resolve_claim_batch_size(2000, 1500, live_active=800))
def test_get_proxy_for_step_can_force_single_direct_fallback_once(self):
worker = self._build_worker(_FakeDb("noop"))
worker._set_current_task_context(domain_id=1, is_step_task=True, task_mode="single_step")
worker._set_step_force_direct_once("百度site检测", True)
proxy = worker._get_proxy_for_step(1, "example.com", "百度site检测")
self.assertIsNone(proxy)
self.assertFalse(worker._consume_step_force_direct_once("百度site检测"))
def test_retry_external_issue_with_proxy_forces_next_attempt_direct_when_allowed(self):
worker = self._build_worker(_FakeDb("noop"))
worker._set_current_task_context(domain_id=1, is_step_task=True, task_mode="single_step")
worker.remove_proxy = MagicMock()
proxy = {"http": "http://bad-proxy", "https": "http://bad-proxy"}
should_retry = worker._should_retry_external_issue(
proxy,
"curl: (28) Connection timed out after 552 milliseconds",
"百度site检测",
"example.com",
)
self.assertTrue(should_retry)
worker.remove_proxy.assert_called_once_with(proxy)
worker.trigger_proxy_refresh.assert_called_once()
self.assertTrue(worker._consume_step_force_direct_once("百度site检测"))
def test_external_dependency_issue_recognizes_common_chinese_timeout_messages(self):
worker = self._build_worker(_FakeDb("noop"))
self.assertTrue(worker._is_external_dependency_issue("360搜索请求超时"))
self.assertTrue(worker._is_external_dependency_issue("百度site检测失败: 连接超时"))
self.assertTrue(worker._is_external_dependency_issue("站长之家检测失败: 网络异常"))
def test_run_detect_aizhan_passes_remaining_budget_to_detector(self):
worker = self._build_worker(_FakeDb("noop"))
worker._check_proxy_step_retry_budget = MagicMock(return_value=(False, ""))
worker._remaining_proxy_step_retry_budget_seconds = MagicMock(return_value=1.7)
worker._get_proxy_for_step = MagicMock(return_value=None)
worker._mark_detection_degraded = MagicMock()
worker._should_blacklist_result = MagicMock(return_value=False)
worker._record_step_result = MagicMock()
worker._should_rotate_proxy_on_error = MagicMock(return_value=False)
worker._is_external_dependency_issue = MagicMock(return_value=False)
worker._mark_detection_rejected = MagicMock()
worker._mark_detection_failed = MagicMock()
with patch("detect_worker.aizhan.check_aizhan", return_value=(True, "")) as mock_check:
result = worker._run_detect_aizhan(1, "example.com", [])
self.assertTrue(result)
self.assertEqual(1.7, mock_check.call_args.kwargs["budget_seconds"])
if __name__ == "__main__":
unittest.main()

64
domainCheck/tests/test_register_timeout_config.py Normal file
View File

@@ -0,0 +1,64 @@
import unittest
from unittest.mock import MagicMock, patch
import urllib3
from domainCheck.detect import register
class RegisterTimeoutConfigTests(unittest.TestCase):
def test_proxy_timeout_uses_bounded_total(self):
with patch.dict("os.environ", {}, clear=False):
timeout = register._resolve_register_timeout({"http": "http://127.0.0.1:8080"})
self.assertIsInstance(timeout, urllib3.Timeout)
self.assertEqual(timeout.connect_timeout, 1.0)
self.assertEqual(timeout.read_timeout, 1.5)
self.assertEqual(timeout.total, 2.2)
def test_direct_timeout_respects_override(self):
with patch.dict(
"os.environ",
{
"DOMAINCHECK_REGISTER_DIRECT_CONNECT_TIMEOUT": "0.9",
"DOMAINCHECK_REGISTER_DIRECT_READ_TIMEOUT": "1.4",
"DOMAINCHECK_REGISTER_DIRECT_TOTAL_TIMEOUT": "1.7",
},
clear=False,
):
timeout = register._resolve_register_timeout(None)
self.assertEqual(timeout.connect_timeout, 0.9)
self.assertEqual(timeout.read_timeout, 1.4)
self.assertEqual(timeout.total, 1.7)
def test_budget_clamps_proxy_timeout_into_remaining_window(self):
with patch.dict("os.environ", {}, clear=False):
timeout = register._resolve_register_timeout(
{"http": "http://127.0.0.1:8080"},
budget_seconds=0.8,
)
self.assertEqual(timeout.total, 0.8)
self.assertLessEqual(timeout.connect_timeout, 0.35)
self.assertLessEqual(timeout.read_timeout, 0.61)
def test_request_register_converts_urllib3_timeout_for_requests(self):
session = MagicMock()
response = MagicMock()
session.get.return_value = response
timeout = urllib3.Timeout(connect=1.1, read=1.7, total=2.4)
with patch("domainCheck.detect.register._get_http_manager", return_value=session):
result = register._request_register(
"https://rdap.verisign.com/com/v1/domain/example",
proxies={"http": "http://127.0.0.1:8080", "https": "http://127.0.0.1:8080"},
timeout=timeout,
)
self.assertIs(result, response)
session.get.assert_called_once_with(
"https://rdap.verisign.com/com/v1/domain/example",
timeout=(1.1, 1.7),
proxies={"http": "http://127.0.0.1:8080", "https": "http://127.0.0.1:8080"},
allow_redirects=True,
)
if __name__ == "__main__":
unittest.main()

33
domainCheck/tests/test_step_timeout_budgets.py Normal file
View File

@@ -0,0 +1,33 @@
import unittest
from unittest.mock import patch
from domainCheck.detect import aizhan, baidu, c360, chinaz
class StepTimeoutBudgetTests(unittest.TestCase):
def test_baidu_timeout_respects_remaining_budget(self):
with patch.dict("os.environ", {}, clear=False):
timeout = baidu._resolve_baidu_timeout({"http": "http://127.0.0.1:8080"}, budget_seconds=1.1)
self.assertLessEqual(timeout, 1.1)
self.assertGreaterEqual(timeout, 0.6)
def test_360_timeout_respects_remaining_budget(self):
with patch.dict("os.environ", {}, clear=False):
timeout = c360._resolve_360_timeout({"http": "http://127.0.0.1:8080"}, budget_seconds=0.9)
self.assertLessEqual(timeout, 0.9)
self.assertGreaterEqual(timeout, 0.6)
def test_chinaz_timeout_respects_remaining_budget(self):
timeout = chinaz._resolve_chinaz_timeout({"http": "http://127.0.0.1:8080"}, budget_seconds=1.3)
self.assertLessEqual(timeout, 1.3)
self.assertGreaterEqual(timeout, 0.6)
def test_aizhan_timeout_respects_remaining_budget(self):
with patch.dict("os.environ", {}, clear=False):
timeout = aizhan._resolve_aizhan_timeout({"http": "http://127.0.0.1:8080"}, budget_seconds=1.0)
self.assertLessEqual(timeout, 1.0)
self.assertGreaterEqual(timeout, 0.6)
if __name__ == "__main__":
unittest.main()

173
domainCheck/tests/test_wayback_detector_recent_years.py Normal file
View File

@@ -0,0 +1,173 @@
import sys
import time
import unittest
from unittest.mock import patch
sys.path.insert(0, "/www/wwwroot/getDomain/domainCheck")
from app.detectors.wayback_detector import WaybackDetector # noqa: E402
class WaybackDetectorRecentYearsTest(unittest.TestCase):
def setUp(self):
WaybackDetector._transient_backoff_until = 0.0
def test_filter_records_recent_years_drops_old_snapshots(self):
detector = WaybackDetector.__new__(WaybackDetector)
current_year = time.gmtime().tm_year
records = [
{"timestamp": f"{current_year}0101000000", "digest": "new"},
{"timestamp": f"{current_year - 2}0101000000", "digest": "mid"},
{"timestamp": f"{current_year - 6}0101000000", "digest": "old"},
]
filtered = detector._filter_records_recent_years(records, recent_years=5)
self.assertEqual(
[
f"{current_year}0101000000",
f"{current_year - 2}0101000000",
],
[item["timestamp"] for item in filtered],
)
def test_scan_snapshots_only_counts_recent_years_window(self):
detector = WaybackDetector.__new__(WaybackDetector)
current_year = time.gmtime().tm_year
detector._fetch_cdx_records_with_meta = lambda domain, limit=None, fast_latest=False: {
"records": [
{"timestamp": f"{current_year}0101000000", "digest": "latest"},
{"timestamp": f"{current_year - 1}0101000000", "digest": "prev"},
{"timestamp": f"{current_year - 6}0101000000", "digest": "old"},
]
}
detector._load_cached_records = lambda domain: None
detector._save_cached_records = lambda domain, records: None
detector._save_cached_timestamps = lambda domain, timestamps: None
detector._fetch_snapshot_title = lambda domain, timestamp: {
"timestamp": timestamp,
"title": f"title-{timestamp}",
"ok": True,
}
detector._normalize_title = lambda title: title
detector._find_sensitive_word = lambda title, words: None
detector._log_info = lambda message: None
detector._handle_exception = lambda exc, domain: None
result = detector.scan_snapshots("example.com", sensitive_words=[], recent_years=5)
self.assertEqual(2, result["checked_snapshot_count"])
self.assertEqual(2, result["fetched_snapshot_count"])
self.assertEqual(sorted([current_year - 1, current_year]), result["snapshot_years"])
def test_scan_snapshots_limits_records_per_domain(self):
detector = WaybackDetector.__new__(WaybackDetector)
current_year = time.gmtime().tm_year
records = [
{"timestamp": f"{current_year}01010{i}0000", "digest": f"d{i}"}
for i in range(6)
]
detector._fetch_cdx_records_with_meta = lambda domain, limit=None, fast_latest=False: {
"records": records,
"error": None,
}
detector._load_cached_records = lambda domain: None
detector._save_cached_records = lambda domain, records: None
detector._save_cached_timestamps = lambda domain, timestamps: None
detector._fetch_snapshot_title = lambda domain, timestamp: {
"timestamp": timestamp,
"title": f"title-{timestamp}",
"ok": True,
}
detector._normalize_title = lambda title: title
detector._find_sensitive_word = lambda title, words: None
detector._log_info = lambda message: None
detector._handle_exception = lambda exc, domain: None
with patch("app.detectors.wayback_detector.config.WAYBACK_MAX_RECORDS", 3):
result = detector.scan_snapshots("example.com", sensitive_words=[], recent_years=5)
self.assertEqual(3, result["checked_snapshot_count"])
# 最新快照会先独立尝试一次，再进入裁剪后的扫描窗口。
self.assertEqual(4, result["fetched_snapshot_count"])
def test_scan_snapshots_fast_degrades_when_latest_cdx_is_transient_failure(self):
detector = WaybackDetector.__new__(WaybackDetector)
detector._fetch_cdx_records_with_meta = lambda domain, limit=None, fast_latest=False: {
"records": [],
"error": "HTTPSConnectionPool(host='web.archive.org', port=443): Max retries exceeded",
}
detector._load_cached_records = lambda domain: [
{"timestamp": "20260101000000", "digest": "d1"},
{"timestamp": "20250101000000", "digest": "d2"},
]
detector._save_cached_records = lambda domain, records: None
detector._save_cached_timestamps = lambda domain, timestamps: None
detector._fetch_snapshot_title = lambda domain, timestamp: self.fail("should not fetch snapshot titles")
detector._normalize_title = lambda title: title
detector._find_sensitive_word = lambda title, words: None
detector._log_info = lambda message: None
detector._handle_exception = lambda exc, domain: None
result = detector.scan_snapshots("example.com", sensitive_words=[], recent_years=5)
self.assertEqual(0, result["checked_snapshot_count"])
self.assertGreaterEqual(result["failed_snapshot_count"], 1)
self.assertGreaterEqual(result["request_error_count"], 1)
def test_scan_snapshots_fast_degrades_when_latest_snapshot_is_transient_failure(self):
detector = WaybackDetector.__new__(WaybackDetector)
detector._fetch_cdx_records_with_meta = lambda domain, limit=None, fast_latest=False: {
"records": [{"timestamp": "20260101000000", "digest": "latest"}] if fast_latest else [
{"timestamp": "20260101000000", "digest": "latest"},
{"timestamp": "20250101000000", "digest": "older"},
],
"error": None,
}
detector._load_cached_records = lambda domain: None
detector._save_cached_records = lambda domain, records: None
detector._save_cached_timestamps = lambda domain, timestamps: None
detector._fetch_snapshot_title = lambda domain, timestamp: {
"timestamp": timestamp,
"title": "",
"ok": False,
"error": "HTTPSConnectionPool(host='web.archive.org', port=443): Max retries exceeded",
}
detector._normalize_title = lambda title: title
detector._find_sensitive_word = lambda title, words: None
detector._log_info = lambda message: None
detector._handle_exception = lambda exc, domain: None
result = detector.scan_snapshots("example.com", sensitive_words=[], recent_years=5)
self.assertEqual(1, result["checked_snapshot_count"])
self.assertEqual(0, result["fetched_snapshot_count"])
self.assertGreaterEqual(result["failed_snapshot_count"], 1)
self.assertTrue(
any("latest_snapshot:" in item for item in result["request_errors"])
)
def test_scan_snapshots_short_circuits_when_transient_backoff_active(self):
detector = WaybackDetector.__new__(WaybackDetector)
WaybackDetector._transient_backoff_until = time.time() + 5
detector._fetch_cdx_records_with_meta = lambda *args, **kwargs: self.fail("should not request cdx during backoff")
detector._load_cached_records = lambda domain: None
detector._save_cached_records = lambda domain, records: None
detector._save_cached_timestamps = lambda domain, timestamps: None
detector._fetch_snapshot_title = lambda domain, timestamp: self.fail("should not request snapshot during backoff")
detector._normalize_title = lambda title: title
detector._find_sensitive_word = lambda title, words: None
detector._log_info = lambda message: None
detector._handle_exception = lambda exc, domain: None
result = detector.scan_snapshots("example.com", sensitive_words=[], recent_years=5)
self.assertEqual(0, result["checked_snapshot_count"])
self.assertEqual(0, result["fetched_snapshot_count"])
self.assertEqual(1, result["request_error_count"])
self.assertTrue(any("wayback_backoff_active:" in item for item in result["request_errors"]))
if __name__ == "__main__":
unittest.main()