793 lines
31 KiB
Python
793 lines
31 KiB
Python
import unittest
|
||
from unittest.mock import MagicMock, patch
|
||
|
||
from psycopg2 import errors
|
||
|
||
from app.services.detect_job_service import (
|
||
append_detect_job_event,
|
||
_build_step_payload,
|
||
_classify_pipeline_item_outcome,
|
||
_classify_runtime_debug_event,
|
||
_enrich_active_job_summary_with_runtime,
|
||
_build_display_summary,
|
||
_build_runtime_display_bucket,
|
||
normalize_detect_step_code,
|
||
get_detect_queue_health,
|
||
process_detect_pipeline_now,
|
||
resolve_initial_domain_pipeline_item,
|
||
resolve_domain_pipeline_step,
|
||
resolve_detect_job_definition,
|
||
)
|
||
|
||
|
||
class DetectJobServiceTests(unittest.TestCase):
|
||
def test_append_detect_job_event_skips_missing_job_fk(self) -> None:
|
||
class FakeCursor:
|
||
def __init__(self) -> None:
|
||
self.exec_calls = []
|
||
self.selects = 0
|
||
|
||
def __enter__(self):
|
||
return self
|
||
|
||
def __exit__(self, exc_type, exc, tb):
|
||
return False
|
||
|
||
def execute(self, sql, params=None):
|
||
self.exec_calls.append((sql, params))
|
||
if "INSERT INTO detect_run_events" in sql:
|
||
raise AssertionError("should not insert detect_run_events when job row is missing")
|
||
if "SELECT 1 FROM detect_jobs" in sql:
|
||
self.selects += 1
|
||
|
||
def fetchone(self):
|
||
return None
|
||
|
||
class FakeConn:
|
||
def __init__(self) -> None:
|
||
self.cursor_obj = FakeCursor()
|
||
self.commit_calls = 0
|
||
|
||
def __enter__(self):
|
||
return self
|
||
|
||
def __exit__(self, exc_type, exc, tb):
|
||
return False
|
||
|
||
def cursor(self):
|
||
return self.cursor_obj
|
||
|
||
def commit(self):
|
||
self.commit_calls += 1
|
||
|
||
fake_conn = FakeConn()
|
||
|
||
with patch("app.services.detect_job_service.get_db", return_value=fake_conn):
|
||
with patch("app.services.detect_job_service.push_debug_event") as mock_push:
|
||
append_detect_job_event(
|
||
5,
|
||
event_type="job_dispatch_requested",
|
||
message="控制面已发送检测启动请求",
|
||
)
|
||
|
||
self.assertEqual(1, fake_conn.cursor_obj.selects)
|
||
self.assertEqual(0, fake_conn.commit_calls)
|
||
mock_push.assert_called_once()
|
||
|
||
def test_process_detect_pipeline_now_retries_deadlock_once(self) -> None:
|
||
calls = {"count": 0}
|
||
|
||
def _run_pipeline(*, limit, job_id):
|
||
calls["count"] += 1
|
||
if calls["count"] == 1:
|
||
raise errors.DeadlockDetected()
|
||
return {
|
||
"processed_items": 7,
|
||
"advanced_items": 3,
|
||
"retried_items": 1,
|
||
}
|
||
|
||
with patch("app.services.detect_job_service.process_detect_pipeline", side_effect=_run_pipeline):
|
||
with patch("app.services.detect_job_service.time.sleep") as mock_sleep:
|
||
ok, message, data = process_detect_pipeline_now(limit=123, job_id=45)
|
||
|
||
self.assertTrue(ok)
|
||
self.assertIn("deadlock 自动重试 1 次后成功", message)
|
||
self.assertEqual(1, data["retry_attempts"])
|
||
self.assertEqual(7, data["processed_items"])
|
||
self.assertEqual(2, calls["count"])
|
||
mock_sleep.assert_called_once()
|
||
|
||
def test_classify_runtime_debug_event_maps_single_step_finalized_completed(self) -> None:
|
||
classified = _classify_runtime_debug_event(
|
||
event_type="worker_log",
|
||
message="检测步骤跟踪: domain=example.com | step=注册状态检测 | stage=single_step_finalized | elapsed_ms=2700 | ok=1 | detect_key=detect_register | final_status=completed | result_state=passed",
|
||
payload={"job_code": "sync-overseas-27456", "node_code": "mainland-worker-01"},
|
||
)
|
||
|
||
self.assertEqual("sync-overseas-27456", classified["job_code"])
|
||
self.assertTrue(classified["terminal"])
|
||
self.assertEqual("completed", classified["terminal_status"])
|
||
self.assertEqual("detect_register", classified["step_code"])
|
||
|
||
def test_build_runtime_display_bucket_prefers_real_active_threads_over_raw_current_load(self) -> None:
|
||
bucket = _build_runtime_display_bucket(
|
||
(
|
||
"mainland-worker-01",
|
||
"mainland",
|
||
"worker",
|
||
"busy",
|
||
753,
|
||
{
|
||
"job_items_total": 1000,
|
||
"job_items_claimed": 0,
|
||
"job_items_running": 0,
|
||
"job_items_completed": 342,
|
||
"active_threads": 61,
|
||
"max_threads": 400,
|
||
"detect_participating": True,
|
||
},
|
||
None,
|
||
)
|
||
)
|
||
|
||
self.assertIsNotNone(bucket)
|
||
self.assertEqual(61, bucket["current_load"])
|
||
self.assertEqual(61, bucket["display_running"])
|
||
self.assertEqual(61, bucket["active_threads"])
|
||
|
||
def test_build_runtime_display_bucket_skips_local_overseas_control_plane_load(self) -> None:
|
||
with patch("app.services.detect_job_service.settings.node_region", "overseas"):
|
||
with patch("app.services.detect_job_service.settings.node_role", "control"):
|
||
with patch("app.services.detect_job_service.settings.node_code", "overseas-control-01"):
|
||
bucket = _build_runtime_display_bucket(
|
||
(
|
||
"overseas-control-01",
|
||
"overseas",
|
||
"control",
|
||
"busy",
|
||
371,
|
||
{
|
||
"active_threads": 371,
|
||
"max_threads": 3200,
|
||
"detect_participating": False,
|
||
},
|
||
None,
|
||
)
|
||
)
|
||
|
||
self.assertIsNone(bucket)
|
||
|
||
def test_build_runtime_display_bucket_skips_any_control_node_without_worker_support(self) -> None:
|
||
bucket = _build_runtime_display_bucket(
|
||
(
|
||
"overseas-control-01",
|
||
"overseas",
|
||
"control",
|
||
"busy",
|
||
378,
|
||
{
|
||
"active_threads": 378,
|
||
"max_threads": 3200,
|
||
"detect_participating": False,
|
||
"worker_online": False,
|
||
},
|
||
None,
|
||
)
|
||
)
|
||
|
||
self.assertIsNone(bucket)
|
||
|
||
def test_build_display_summary_ignores_inflated_raw_current_load(self) -> None:
|
||
summary = _build_display_summary(
|
||
[
|
||
{
|
||
"node_code": "mainland-controller-01",
|
||
"items_claimed": 0,
|
||
"items_running": 0,
|
||
"display_running": 729,
|
||
"current_load": 729,
|
||
"active_threads": 729,
|
||
"max_threads": 800,
|
||
"items_completed": 196,
|
||
"items_failed": 0,
|
||
},
|
||
{
|
||
"node_code": "mainland-worker-01",
|
||
"items_claimed": 0,
|
||
"items_running": 0,
|
||
"display_running": 753,
|
||
"current_load": 753,
|
||
"active_threads": 61,
|
||
"max_threads": 400,
|
||
"items_completed": 146,
|
||
"items_failed": 0,
|
||
},
|
||
]
|
||
)
|
||
|
||
self.assertEqual(790, summary["display_running"])
|
||
self.assertEqual(790, summary["current_load"])
|
||
self.assertEqual(790, summary["active_threads"])
|
||
|
||
def test_enrich_active_job_summary_with_runtime_adds_runtime_job_code_and_recent_events(self) -> None:
|
||
summary = _enrich_active_job_summary_with_runtime(
|
||
{
|
||
"job_id": 275,
|
||
"job_code": "sync-overseas-5297",
|
||
"processed_recent": 0,
|
||
"processed_per_minute": 0,
|
||
"completed_recent": 0,
|
||
"failed_recent": 0,
|
||
"blacklisted_recent": 0,
|
||
"recent_domain_events": [],
|
||
},
|
||
event_limit=20,
|
||
window_minutes=15,
|
||
runtime_activity={
|
||
"focus_job_code": "sync-overseas-9506",
|
||
"job_codes": ["sync-overseas-9506", "sync-overseas-27456"],
|
||
"processed_recent": 120,
|
||
"completed_recent": 118,
|
||
"failed_recent": 1,
|
||
"blacklisted_recent": 1,
|
||
},
|
||
runtime_snapshot={
|
||
"job": {
|
||
"job_id": 1835,
|
||
"job_code": "sync-overseas-9506",
|
||
"progress_percent": 44.2,
|
||
},
|
||
"queue_health": {
|
||
"queue": {
|
||
"items_total": 1000,
|
||
"pending": 176,
|
||
"claimed": 223,
|
||
"running": 159,
|
||
"completed": 442,
|
||
"display_running": 220,
|
||
}
|
||
},
|
||
},
|
||
recent_domain_events=[
|
||
{
|
||
"event_type": "domain_completed",
|
||
"message": "域名检测完成: example.com",
|
||
"created_at": "2026-04-21 04:10:00",
|
||
}
|
||
],
|
||
)
|
||
|
||
self.assertEqual("sync-overseas-9506", summary["runtime_job_code"])
|
||
self.assertEqual(["sync-overseas-9506", "sync-overseas-27456"], summary["runtime_job_codes"])
|
||
self.assertEqual(120, summary["processed_recent"])
|
||
self.assertEqual(8.0, summary["processed_per_minute"])
|
||
self.assertEqual(118, summary["completed_recent"])
|
||
self.assertEqual(1, summary["failed_recent"])
|
||
self.assertEqual(1, summary["blacklisted_recent"])
|
||
self.assertEqual(1, len(summary["recent_domain_events"]))
|
||
self.assertEqual("sync-overseas-9506", summary["runtime_snapshot_job_code"])
|
||
self.assertEqual(1000, summary["runtime_snapshot_queue"]["items_total"])
|
||
self.assertEqual(1000, summary["items_total"])
|
||
self.assertEqual(176, summary["items_pending"])
|
||
self.assertEqual(223, summary["items_claimed"])
|
||
self.assertEqual(159, summary["items_running"])
|
||
self.assertEqual(442, summary["items_completed"])
|
||
self.assertEqual(220, summary["display_items_running"])
|
||
|
||
def test_enrich_active_job_summary_with_runtime_prefers_snapshot_identity_and_events(self) -> None:
|
||
summary = _enrich_active_job_summary_with_runtime(
|
||
{
|
||
"job_id": 275,
|
||
"job_code": "sync-overseas-5297",
|
||
"status": "running",
|
||
"recent_events": [
|
||
{
|
||
"node_code": "mainland-worker-01",
|
||
"event_type": "domain_started",
|
||
"message": "开始检测域名: stale.com",
|
||
"payload": {"job_code": "sync-overseas-5297"},
|
||
"created_at": "2026-04-21 11:58:00",
|
||
}
|
||
],
|
||
"current_cycle_events": [],
|
||
"latest_event": None,
|
||
"display_items_running": 0,
|
||
"display_current_load": 0,
|
||
"display_active_threads": 0,
|
||
"display_max_threads": 0,
|
||
"display_active_node_codes": [],
|
||
},
|
||
event_limit=10,
|
||
window_minutes=15,
|
||
runtime_activity={
|
||
"focus_job_code": "sync-overseas-31437",
|
||
"job_codes": ["sync-overseas-31437"],
|
||
"processed_recent": 10,
|
||
"completed_recent": 10,
|
||
"failed_recent": 0,
|
||
"blacklisted_recent": 0,
|
||
},
|
||
runtime_snapshot={
|
||
"job": {
|
||
"job_id": 1902,
|
||
"job_code": "sync-overseas-31437",
|
||
"status": "running",
|
||
"progress_percent": 34.3,
|
||
"node_stats": [
|
||
{
|
||
"node_code": "mainland-controller-01",
|
||
"items_claimed": 230,
|
||
"items_running": 67,
|
||
"items_completed": 343,
|
||
"items_failed": 0,
|
||
"active_threads": 170,
|
||
"max_threads": 2000,
|
||
},
|
||
{
|
||
"node_code": "mainland-worker-01",
|
||
"items_claimed": 75,
|
||
"items_running": 0,
|
||
"items_completed": 0,
|
||
"items_failed": 0,
|
||
"active_threads": 19,
|
||
"max_threads": 1200,
|
||
},
|
||
],
|
||
},
|
||
"queue_health": {
|
||
"queue": {
|
||
"items_total": 1000,
|
||
"pending": 285,
|
||
"claimed": 305,
|
||
"running": 67,
|
||
"completed": 343,
|
||
"display_running": 189,
|
||
}
|
||
},
|
||
"recent_events": [
|
||
{
|
||
"job_id": 2137,
|
||
"node_code": "mainland-controller-01",
|
||
"event_type": "job_created",
|
||
"message": "同步拉取待检测批次 sync-overseas-35461，共 1000 个任务项",
|
||
"payload": {"source_record_id": 35461},
|
||
"created_at": "2026-04-22 01:21:29",
|
||
},
|
||
{
|
||
"job_id": 1902,
|
||
"node_code": "mainland-controller-01",
|
||
"event_type": "worker_log",
|
||
"message": "从任务队列获取到 125 个需要检测的域名",
|
||
"payload": {"job_code": "sync-overseas-31437"},
|
||
"created_at": "2026-04-22 01:19:25",
|
||
},
|
||
],
|
||
},
|
||
)
|
||
|
||
self.assertEqual(1902, summary["job_id"])
|
||
self.assertEqual("sync-overseas-31437", summary["job_code"])
|
||
self.assertEqual("sync-overseas-31437", summary["runtime_job_code"])
|
||
self.assertEqual(189, summary["display_items_running"])
|
||
self.assertEqual(189, summary["display_current_load"])
|
||
self.assertEqual(189, summary["display_active_threads"])
|
||
self.assertEqual(3200, summary["display_max_threads"])
|
||
self.assertEqual(["mainland-controller-01", "mainland-worker-01"], summary["display_active_node_codes"])
|
||
self.assertEqual(1, len(summary["recent_events"]))
|
||
self.assertEqual("worker_log", summary["latest_event"]["event_type"])
|
||
self.assertEqual("sync-overseas-31437", summary["recent_events"][0]["payload"]["job_code"])
|
||
|
||
def test_get_detect_queue_health_preserves_runtime_node_throughput_after_snapshot_override(self) -> None:
|
||
lease_row = (None, None, 0, 0)
|
||
throughput_rows = []
|
||
step_throughput_rows = []
|
||
runtime_display_rows = [
|
||
(
|
||
"mainland-controller-01",
|
||
"mainland",
|
||
"control",
|
||
"busy",
|
||
170,
|
||
{
|
||
"job_items_total": 265,
|
||
"job_items_claimed": 85,
|
||
"job_items_running": 0,
|
||
"job_items_completed": 105,
|
||
"active_threads": 170,
|
||
"max_threads": 2000,
|
||
"detect_participating": True,
|
||
},
|
||
None,
|
||
),
|
||
(
|
||
"mainland-worker-01",
|
||
"mainland",
|
||
"worker",
|
||
"busy",
|
||
19,
|
||
{
|
||
"job_items_total": 139,
|
||
"job_items_claimed": 64,
|
||
"job_items_running": 0,
|
||
"job_items_completed": 4,
|
||
"active_threads": 19,
|
||
"max_threads": 1200,
|
||
"detect_participating": True,
|
||
},
|
||
None,
|
||
),
|
||
(
|
||
"overseas-control-01",
|
||
"overseas",
|
||
"control",
|
||
"busy",
|
||
371,
|
||
{
|
||
"active_threads": 371,
|
||
"max_threads": 3200,
|
||
"detect_participating": False,
|
||
},
|
||
None,
|
||
),
|
||
]
|
||
|
||
conn = MagicMock()
|
||
cursor_cm = MagicMock()
|
||
cursor = MagicMock()
|
||
conn.cursor.return_value = cursor_cm
|
||
cursor_cm.__enter__.return_value = cursor
|
||
db_cm = MagicMock()
|
||
db_cm.__enter__.return_value = conn
|
||
cursor.fetchone.return_value = lease_row
|
||
cursor.fetchall.side_effect = [throughput_rows, step_throughput_rows, runtime_display_rows]
|
||
|
||
active_job = {
|
||
"job_id": 1937,
|
||
"job_code": "sync-overseas-31987",
|
||
"status": "running",
|
||
"items_total": 1000,
|
||
"items_pending": 596,
|
||
"items_claimed": 149,
|
||
"items_running": 146,
|
||
"items_completed": 109,
|
||
"items_blacklisted": 0,
|
||
"items_failed": 0,
|
||
"progress_percent": 10.9,
|
||
"node_stats": [],
|
||
"distributed_node_stats": [
|
||
{
|
||
"node_code": "mainland-controller-01",
|
||
"items_total": 265,
|
||
"items_pending": 0,
|
||
"items_claimed": 85,
|
||
"items_running": 75,
|
||
"items_completed": 105,
|
||
"items_blacklisted": 0,
|
||
"items_failed": 0,
|
||
"metrics_source": "runtime",
|
||
},
|
||
{
|
||
"node_code": "mainland-worker-01",
|
||
"items_total": 139,
|
||
"items_pending": 0,
|
||
"items_claimed": 64,
|
||
"items_running": 71,
|
||
"items_completed": 4,
|
||
"items_blacklisted": 0,
|
||
"items_failed": 0,
|
||
"metrics_source": "runtime",
|
||
},
|
||
],
|
||
"step_stats": [
|
||
{
|
||
"step_code": "detect_register",
|
||
"items_total": 1000,
|
||
"items_pending": 449,
|
||
"items_claimed": 131,
|
||
"items_running": 273,
|
||
"items_completed": 147,
|
||
"items_blacklisted": 0,
|
||
"items_failed": 0,
|
||
}
|
||
],
|
||
}
|
||
runtime_activity = {
|
||
"processed_recent": 133,
|
||
"completed_recent": 113,
|
||
"failed_recent": 20,
|
||
"blacklisted_recent": 0,
|
||
"focus_job_code": "sync-overseas-31563",
|
||
"job_codes": ["sync-overseas-31563"],
|
||
"step_code": "detect_register",
|
||
"nodes": {
|
||
"mainland-controller-01": {
|
||
"node_code": "mainland-controller-01",
|
||
"processed_recent": 74,
|
||
"completed_recent": 54,
|
||
"failed_recent": 20,
|
||
"blacklisted_recent": 0,
|
||
},
|
||
"mainland-worker-01": {
|
||
"node_code": "mainland-worker-01",
|
||
"processed_recent": 59,
|
||
"completed_recent": 59,
|
||
"failed_recent": 0,
|
||
"blacklisted_recent": 0,
|
||
},
|
||
},
|
||
}
|
||
runtime_snapshot = {
|
||
"job": {
|
||
"job_id": 1937,
|
||
"job_code": "sync-overseas-31987",
|
||
"progress_percent": 10.9,
|
||
},
|
||
"queue_health": {
|
||
"queue": {
|
||
"items_total": 1000,
|
||
"pending": 596,
|
||
"claimed": 149,
|
||
"running": 146,
|
||
"completed": 109,
|
||
"blacklisted": 0,
|
||
"failed": 0,
|
||
"display_claimed": 149,
|
||
"display_running": 403,
|
||
},
|
||
"nodes": [
|
||
{
|
||
"node_code": "mainland-controller-01",
|
||
"items_total": 265,
|
||
"items_pending": 0,
|
||
"items_claimed": 85,
|
||
"items_running": 75,
|
||
"items_completed": 105,
|
||
"items_blacklisted": 0,
|
||
"items_failed": 0,
|
||
"processed_recent": 0,
|
||
"processed_per_minute": 0,
|
||
"completed_recent": 0,
|
||
"blacklisted_recent": 0,
|
||
"failed_recent": 0,
|
||
"metrics_source": "runtime",
|
||
},
|
||
{
|
||
"node_code": "mainland-worker-01",
|
||
"items_total": 139,
|
||
"items_pending": 0,
|
||
"items_claimed": 64,
|
||
"items_running": 71,
|
||
"items_completed": 4,
|
||
"items_blacklisted": 0,
|
||
"items_failed": 0,
|
||
"processed_recent": 0,
|
||
"processed_per_minute": 0,
|
||
"completed_recent": 0,
|
||
"blacklisted_recent": 0,
|
||
"failed_recent": 0,
|
||
"metrics_source": "runtime",
|
||
},
|
||
],
|
||
"steps": [],
|
||
},
|
||
}
|
||
|
||
with patch("app.services.detect_job_service.settings.node_region", "overseas"):
|
||
with patch("app.services.detect_job_service.settings.node_role", "control"):
|
||
with patch("app.services.detect_job_service.settings.node_code", "overseas-control-01"):
|
||
with patch("app.services.detect_job_service.get_active_detect_job_summary", return_value=active_job):
|
||
with patch("app.services.detect_job_service._load_runtime_activity_snapshot", return_value=runtime_activity):
|
||
with patch("app.services.detect_job_service._load_latest_runtime_active_job_snapshot", return_value=runtime_snapshot):
|
||
with patch("app.services.detect_job_service.get_db", return_value=db_cm):
|
||
health = get_detect_queue_health(window_minutes=15)
|
||
|
||
node_map = {item["node_code"]: item for item in health["nodes"]}
|
||
self.assertEqual(74, node_map["mainland-controller-01"]["processed_recent"])
|
||
self.assertEqual(54, node_map["mainland-controller-01"]["completed_recent"])
|
||
self.assertEqual(20, node_map["mainland-controller-01"]["failed_recent"])
|
||
self.assertEqual(170, node_map["mainland-controller-01"]["items_running"])
|
||
self.assertEqual(59, node_map["mainland-worker-01"]["processed_recent"])
|
||
self.assertEqual(59, node_map["mainland-worker-01"]["completed_recent"])
|
||
self.assertEqual(19, node_map["mainland-worker-01"]["items_running"])
|
||
self.assertEqual(113, health["throughput"]["completed_recent"])
|
||
self.assertEqual(189, health["queue"]["display_running"])
|
||
self.assertNotIn("overseas-control-01", node_map)
|
||
|
||
def test_normalize_detect_step_code_accepts_supported_single_step(self) -> None:
|
||
self.assertEqual("detect_baidu_site", normalize_detect_step_code("detect_baidu_site"))
|
||
self.assertEqual("detect_wayback", normalize_detect_step_code("detect_wayback"))
|
||
|
||
def test_normalize_detect_step_code_rejects_unknown_step(self) -> None:
|
||
self.assertEqual("", normalize_detect_step_code("detect_unknown"))
|
||
|
||
def test_resolve_detect_job_definition_builds_single_step_job(self) -> None:
|
||
definition = resolve_detect_job_definition("detect_baidu_site")
|
||
|
||
self.assertTrue(definition["is_single_step"])
|
||
self.assertEqual("single_step", definition["task_mode"])
|
||
self.assertEqual("detect_baidu_site", definition["step_code"])
|
||
|
||
def test_resolve_detect_job_definition_builds_wayback_single_step_job(self) -> None:
|
||
definition = resolve_detect_job_definition("detect_wayback")
|
||
|
||
self.assertTrue(definition["is_single_step"])
|
||
self.assertEqual("single_step", definition["task_mode"])
|
||
self.assertEqual("detect_wayback", definition["step_code"])
|
||
|
||
def test_build_step_payload_adds_wayback_recent_years_strategy(self) -> None:
|
||
payload = _build_step_payload(
|
||
step_code="detect_wayback",
|
||
domain_snapshot={"domain": "example.com", "source_type": 2},
|
||
settings_payload={"detect_options": {"detect_wayback": True}},
|
||
)
|
||
|
||
self.assertEqual("detect_wayback", payload["step_code"])
|
||
self.assertEqual("recent_years", payload["wayback_strategy"])
|
||
self.assertEqual(5, payload["wayback_recent_years"])
|
||
self.assertTrue(payload["wayback_stop_on_first_hit"])
|
||
|
||
def test_resolve_detect_job_definition_defaults_to_domain_pipeline(self) -> None:
|
||
definition = resolve_detect_job_definition(None)
|
||
|
||
self.assertFalse(definition["is_single_step"])
|
||
self.assertEqual("domain_pipeline", definition["task_mode"])
|
||
self.assertEqual("", definition["step_code"])
|
||
|
||
def test_resolve_domain_pipeline_step_skips_yikoujia_register(self) -> None:
|
||
step_code = resolve_domain_pipeline_step(
|
||
{
|
||
"source_type": 1,
|
||
"register_status": 0,
|
||
"baidu_site": {},
|
||
"qihu360_site": {},
|
||
"chinaz_info": {},
|
||
"aizhan_info": {},
|
||
"wayback_info": {},
|
||
"jucha_info": {},
|
||
"juziseo_info": {},
|
||
},
|
||
settings_payload={
|
||
"detect_options": {
|
||
"detect_register": True,
|
||
"detect_baidu_site": True,
|
||
"detect_360_site": False,
|
||
"detect_chinaz": False,
|
||
"detect_aizhan": False,
|
||
"detect_wayback": False,
|
||
"detect_jucha": False,
|
||
"detect_juziseo": False,
|
||
"detect_order": ["detect_register", "detect_baidu_site"],
|
||
}
|
||
},
|
||
)
|
||
|
||
self.assertEqual("detect_baidu_site", step_code)
|
||
|
||
def test_resolve_domain_pipeline_step_moves_to_next_incomplete_step(self) -> None:
|
||
step_code = resolve_domain_pipeline_step(
|
||
{
|
||
"source_type": 2,
|
||
"register_status": 3,
|
||
"baidu_site": {"state": "passed"},
|
||
"qihu360_site": {},
|
||
"chinaz_info": {},
|
||
"aizhan_info": {},
|
||
"wayback_info": {},
|
||
"jucha_info": {},
|
||
"juziseo_info": {},
|
||
},
|
||
settings_payload={
|
||
"detect_options": {
|
||
"detect_register": True,
|
||
"detect_baidu_site": True,
|
||
"detect_360_site": True,
|
||
"detect_chinaz": False,
|
||
"detect_aizhan": False,
|
||
"detect_wayback": False,
|
||
"detect_jucha": False,
|
||
"detect_juziseo": False,
|
||
"detect_order": ["detect_register", "detect_baidu_site", "detect_360_site"],
|
||
}
|
||
},
|
||
after_step_code="detect_baidu_site",
|
||
)
|
||
|
||
self.assertEqual("detect_360_site", step_code)
|
||
|
||
def test_resolve_initial_domain_pipeline_item_builds_first_step_payload(self) -> None:
|
||
step_code, payload = resolve_initial_domain_pipeline_item(
|
||
{
|
||
"id": 10,
|
||
"domain": "example.com",
|
||
"source_type": 2,
|
||
"register_status": 0,
|
||
"baidu_site": {},
|
||
"qihu360_site": {},
|
||
"chinaz_info": {},
|
||
"aizhan_info": {},
|
||
"wayback_info": {},
|
||
"jucha_info": {},
|
||
"juziseo_info": {},
|
||
},
|
||
settings_payload={
|
||
"detect_options": {
|
||
"detect_register": True,
|
||
"detect_baidu_site": True,
|
||
"detect_order": ["detect_register", "detect_baidu_site"],
|
||
}
|
||
},
|
||
)
|
||
|
||
self.assertEqual("detect_register", step_code)
|
||
self.assertIsNotNone(payload)
|
||
self.assertEqual("detect_register", payload["step_code"])
|
||
self.assertEqual("example.com", payload["domain"])
|
||
|
||
def test_resolve_initial_domain_pipeline_item_returns_empty_when_pipeline_already_done(self) -> None:
|
||
step_code, payload = resolve_initial_domain_pipeline_item(
|
||
{
|
||
"id": 11,
|
||
"domain": "done.com",
|
||
"source_type": 2,
|
||
"register_status": 3,
|
||
"baidu_site": {"state": "passed"},
|
||
"qihu360_site": {},
|
||
"chinaz_info": {},
|
||
"aizhan_info": {},
|
||
"wayback_info": {},
|
||
"jucha_info": {},
|
||
"juziseo_info": {},
|
||
},
|
||
settings_payload={
|
||
"detect_options": {
|
||
"detect_register": True,
|
||
"detect_baidu_site": True,
|
||
"detect_order": ["detect_register", "detect_baidu_site"],
|
||
}
|
||
},
|
||
)
|
||
|
||
self.assertEqual("", step_code)
|
||
self.assertIsNone(payload)
|
||
|
||
def test_classify_pipeline_item_outcome_retries_external_failure(self) -> None:
|
||
outcome = _classify_pipeline_item_outcome(
|
||
item_status="failed",
|
||
result_payload={"state": "degraded", "message": "timeout", "retry_recommended": True},
|
||
step_code="detect_baidu_site",
|
||
attempt_count=0,
|
||
)
|
||
|
||
self.assertEqual("retry", outcome["action"])
|
||
self.assertTrue(outcome["should_retry"])
|
||
self.assertEqual("external_retry", outcome["reason_code"])
|
||
|
||
def test_classify_pipeline_item_outcome_rejects_business_failure(self) -> None:
|
||
outcome = _classify_pipeline_item_outcome(
|
||
item_status="failed",
|
||
result_payload={"state": "rejected", "message": "title contains forbidden keyword"},
|
||
step_code="detect_chinaz",
|
||
attempt_count=0,
|
||
)
|
||
|
||
self.assertEqual("reject", outcome["action"])
|
||
self.assertFalse(outcome["should_retry"])
|
||
self.assertEqual("business_reject", outcome["reason_code"])
|
||
|
||
def test_classify_pipeline_item_outcome_marks_blacklisted_terminal(self) -> None:
|
||
outcome = _classify_pipeline_item_outcome(
|
||
item_status="blacklisted",
|
||
result_payload={"state": "blacklisted", "message": "risk hit"},
|
||
step_code="detect_baidu_site",
|
||
attempt_count=0,
|
||
)
|
||
|
||
self.assertEqual("black_hit", outcome["action"])
|
||
self.assertFalse(outcome["should_retry"])
|
||
self.assertEqual("blacklisted", outcome["reason_code"])
|
||
|
||
|
||
if __name__ == "__main__":
|
||
unittest.main()
|