9440 lines
443 KiB
Python
9440 lines
443 KiB
Python
# -*- coding: UTF-8 -*-
|
||
'''
|
||
@Project :domainScanDemo
|
||
@File :detect_worker.py
|
||
@IDE :PyCharm
|
||
@Author :梦伴
|
||
@Date :2026/4/10 22:40
|
||
@explain : 域名检测端程序
|
||
'''
|
||
|
||
import os
|
||
import signal
|
||
import sys
|
||
import json
|
||
import gc
|
||
import math
|
||
import time
|
||
import socket
|
||
import threading
|
||
import collections
|
||
import resource
|
||
import schedule
|
||
import urllib.error
|
||
import urllib.parse
|
||
import urllib.request
|
||
from pathlib import Path
|
||
from concurrent.futures import ThreadPoolExecutor, as_completed
|
||
from datetime import datetime
|
||
from loguru import logger
|
||
from queue import Empty, Full, Queue
|
||
from PySide6.QtWidgets import QApplication, QMainWindow, QTextEdit, QPushButton, QVBoxLayout, QHBoxLayout, QWidget, QLabel, QProgressBar, QGroupBox, QScrollArea, QFrame
|
||
from PySide6.QtCore import Qt, QThread, Signal, QMetaObject, Q_ARG, QCoreApplication, QEvent, QTimer
|
||
from PySide6.QtGui import QIcon
|
||
|
||
from app.utils.database import Database
|
||
from app.config import config
|
||
from app.detectors.wayback_detector import WaybackDetector
|
||
from app.utils.redis_client import get_redis_client
|
||
from app.utils.status_codes import (
|
||
DETECT_STATUS_BLACKLISTED,
|
||
DETECT_STATUS_COMPLETED,
|
||
DETECT_STATUS_FAILED,
|
||
DETECT_STATUS_RUNNING,
|
||
REGISTER_STATUS_AVAILABLE,
|
||
REGISTER_STATUS_CLIENT_HOLD,
|
||
REGISTER_STATUS_SERVER_HOLD,
|
||
REVIEW_STATUS_PENDING,
|
||
THIRD_PARTY_STATUS_DONE,
|
||
)
|
||
import redis
|
||
from detect import aizhan, baidu, c360, chinaz, register, jucha, juziseo
|
||
|
||
CONFIG_UPDATE_CHANNEL = "domain_tool:config_update"
|
||
CONTROL_CHANNEL = "domain_tool:worker_control"
|
||
RUNTIME_STATE_KEY = "domain_tool:detect_runtime_state"
|
||
PENDING_CONTROL_KEY = "domain_tool:worker_pending_command"
|
||
RUNTIME_SETTINGS_KEY = "domain_tool:runtime_settings"
|
||
SHARED_PROXY_POOL_KEY_PREFIX = "domain_tool:shared_proxy_pool"
|
||
SHARED_PROXY_REFRESH_LOCK_KEY_PREFIX = "domain_tool:shared_proxy_refresh_lock"
|
||
|
||
|
||
def _raise_nofile_soft_limit():
|
||
try:
|
||
soft_limit, hard_limit = resource.getrlimit(resource.RLIMIT_NOFILE)
|
||
target_limit = hard_limit if hard_limit not in (-1, resource.RLIM_INFINITY) else soft_limit
|
||
if target_limit and soft_limit < target_limit:
|
||
resource.setrlimit(resource.RLIMIT_NOFILE, (target_limit, hard_limit))
|
||
logger.info(f"已提升 NOFILE 软限制: {soft_limit} -> {target_limit}")
|
||
except Exception as exc:
|
||
logger.warning(f"提升 NOFILE 软限制失败: {exc}")
|
||
|
||
|
||
def _resolve_worker_thread_stack_size_bytes() -> int:
|
||
worker_mode = str(os.getenv("WORKER_MODE", "") or "").strip()
|
||
raw_override = str(os.getenv("DOMAINCHECK_THREAD_STACK_SIZE_KB", "") or "").strip()
|
||
if not raw_override and worker_mode != "linux-systemd":
|
||
return 0
|
||
default_kb = "1024" if worker_mode == "linux-systemd" else "0"
|
||
try:
|
||
target_kb = int(raw_override or default_kb or 0)
|
||
except Exception:
|
||
target_kb = int(default_kb or 0)
|
||
if target_kb <= 0:
|
||
return 0
|
||
return max(256 * 1024, target_kb * 1024)
|
||
|
||
|
||
def _configure_worker_thread_stack_size() -> int:
|
||
target_bytes = int(_resolve_worker_thread_stack_size_bytes() or 0)
|
||
try:
|
||
current_bytes = int(threading.stack_size() or 0)
|
||
except Exception:
|
||
current_bytes = 0
|
||
if target_bytes <= 0:
|
||
return current_bytes
|
||
if current_bytes == target_bytes:
|
||
return current_bytes
|
||
try:
|
||
threading.stack_size(target_bytes)
|
||
applied_bytes = int(threading.stack_size() or 0)
|
||
logger.info(f"已设置检测线程默认栈大小: {current_bytes} -> {applied_bytes}")
|
||
return applied_bytes
|
||
except Exception as exc:
|
||
logger.warning(f"设置检测线程默认栈大小失败: target={target_bytes}, error={exc}")
|
||
return current_bytes
|
||
|
||
|
||
def _resolve_worker_log_file() -> str:
|
||
candidate_dirs = [
|
||
str(getattr(config, "LOG_DIR", "") or "").strip(),
|
||
os.path.join("/opt/domaincheck", "runtime", "domainCheck", "logs"),
|
||
os.path.join("/tmp", "domaincheck", "logs"),
|
||
]
|
||
seen = set()
|
||
for raw_dir in candidate_dirs:
|
||
log_dir = os.path.abspath(raw_dir) if raw_dir else ""
|
||
if not log_dir or log_dir in seen:
|
||
continue
|
||
seen.add(log_dir)
|
||
try:
|
||
os.makedirs(log_dir, exist_ok=True)
|
||
log_file_path = os.path.join(log_dir, "detect_worker.log")
|
||
with open(log_file_path, "a", encoding="utf-8"):
|
||
pass
|
||
return log_file_path
|
||
except Exception:
|
||
continue
|
||
return os.path.join("/tmp", "detect_worker.log")
|
||
|
||
|
||
def _worker_local_config_roots() -> list[Path]:
|
||
seen = set()
|
||
roots: list[Path] = []
|
||
|
||
def append_root(raw_path) -> None:
|
||
text = str(raw_path or "").strip()
|
||
if not text:
|
||
return
|
||
path = Path(text).expanduser().resolve()
|
||
key = str(path)
|
||
if key in seen:
|
||
return
|
||
seen.add(key)
|
||
roots.append(path)
|
||
|
||
append_root(str(os.getenv("DOMAINCHECK_CONFIG_ROOT", "") or "").strip())
|
||
append_root(Path.cwd())
|
||
|
||
script_root = Path(__file__).resolve().parent
|
||
append_root(script_root)
|
||
|
||
for base_root in list(roots):
|
||
normalized = str(base_root)
|
||
if f"{os.sep}releases{os.sep}" in normalized:
|
||
install_root = normalized.split(f"{os.sep}releases{os.sep}", 1)[0]
|
||
append_root(Path(install_root) / "current" / "domainCheck")
|
||
append_root(Path(install_root) / "domainCheck")
|
||
elif f"{os.sep}current{os.sep}" in normalized:
|
||
install_root = normalized.split(f"{os.sep}current{os.sep}", 1)[0]
|
||
append_root(Path(install_root) / "current" / "domainCheck")
|
||
append_root(Path(install_root) / "domainCheck")
|
||
elif base_root.name == "domainCheck":
|
||
append_root(base_root.parent / "current" / "domainCheck")
|
||
append_root(base_root.parent / "domainCheck")
|
||
|
||
return roots
|
||
|
||
|
||
def _read_worker_local_json_config(filename: str):
|
||
normalized_name = str(filename or "").strip()
|
||
if not normalized_name:
|
||
return None
|
||
for root in _worker_local_config_roots():
|
||
candidate = root / normalized_name
|
||
try:
|
||
if candidate.exists():
|
||
with candidate.open("r", encoding="utf-8") as handle:
|
||
return json.load(handle)
|
||
except Exception:
|
||
continue
|
||
return None
|
||
|
||
|
||
def pending_control_key(node_code=None):
|
||
normalized_node_code = str(node_code or config.NODE_CODE or "").strip()
|
||
if not normalized_node_code:
|
||
return PENDING_CONTROL_KEY
|
||
return f"{PENDING_CONTROL_KEY}:{normalized_node_code}"
|
||
|
||
|
||
def control_target_node_codes(control_payload) -> list[str]:
|
||
if not isinstance(control_payload, dict):
|
||
return []
|
||
|
||
normalized_targets: list[str] = []
|
||
|
||
def append_target(raw_value):
|
||
normalized_value = str(raw_value or "").strip()
|
||
if normalized_value and normalized_value not in normalized_targets:
|
||
normalized_targets.append(normalized_value)
|
||
|
||
list_keys = ("target_node_codes", "node_codes")
|
||
scalar_keys = ("target_node_code", "node_code")
|
||
|
||
for key in list_keys:
|
||
raw_value = control_payload.get(key)
|
||
if isinstance(raw_value, (list, tuple, set)):
|
||
for item in raw_value:
|
||
append_target(item)
|
||
elif isinstance(raw_value, str) and raw_value.strip():
|
||
for item in raw_value.split(","):
|
||
append_target(item)
|
||
if normalized_targets:
|
||
return normalized_targets
|
||
|
||
for key in scalar_keys:
|
||
raw_value = control_payload.get(key)
|
||
if raw_value not in (None, ""):
|
||
append_target(raw_value)
|
||
if normalized_targets:
|
||
return normalized_targets
|
||
|
||
return normalized_targets
|
||
|
||
|
||
def control_targets_current_worker(control_payload, *, node_code=None) -> bool:
|
||
normalized_node_code = str(node_code or config.NODE_CODE or "").strip()
|
||
if not normalized_node_code:
|
||
return True
|
||
target_node_codes = control_target_node_codes(control_payload)
|
||
if not target_node_codes:
|
||
return True
|
||
return normalized_node_code in target_node_codes
|
||
|
||
class DetectThread(QThread):
|
||
"""
|
||
检测线程
|
||
"""
|
||
log_signal = Signal(str)
|
||
progress_signal = Signal(int, int)
|
||
finished_signal = Signal()
|
||
thread_count_signal = Signal(int, int)
|
||
|
||
def __init__(self, worker):
|
||
super().__init__()
|
||
self.worker = worker
|
||
# 设置线程名称
|
||
self.setObjectName("DetectThread")
|
||
|
||
def run(self):
|
||
# 将信号传递给worker
|
||
self.worker.detect_thread = self
|
||
self.worker.start_detection()
|
||
self.finished_signal.emit()
|
||
|
||
class DetectMainWindow(QMainWindow):
|
||
"""
|
||
检测端主窗口
|
||
"""
|
||
|
||
@staticmethod
|
||
def _button_style(
|
||
normal_color: str,
|
||
hover_color: str,
|
||
pressed_color: str,
|
||
disabled_color: str = "#6c757d",
|
||
*,
|
||
fancy_effects: bool = True,
|
||
) -> str:
|
||
hover_extra = ""
|
||
pressed_extra = ""
|
||
disabled_extra = ""
|
||
if fancy_effects:
|
||
hover_extra = "\n transform: translateY(-2px);\n box-shadow: 0 4px 8px rgba(0, 0, 0, 0.2);"
|
||
pressed_extra = "\n transform: translateY(1px);\n box-shadow: 0 2px 4px rgba(0, 0, 0, 0.2);"
|
||
disabled_extra = "\n transform: none;\n box-shadow: none;"
|
||
|
||
return f"""
|
||
QPushButton {{
|
||
background-color: {normal_color};
|
||
color: white;
|
||
border: none;
|
||
border-radius: 8px;
|
||
font-size: 16px;
|
||
font-weight: bold;
|
||
padding: 10px 20px;
|
||
min-width: 120px;
|
||
}}
|
||
QPushButton:hover {{
|
||
background-color: {hover_color};{hover_extra}
|
||
}}
|
||
QPushButton:pressed {{
|
||
background-color: {pressed_color};{pressed_extra}
|
||
}}
|
||
QPushButton:disabled {{
|
||
background-color: {disabled_color};
|
||
color: #adb5bd;{disabled_extra}
|
||
}}
|
||
"""
|
||
|
||
def __init__(self):
|
||
super().__init__()
|
||
self.setWindowTitle("域名检测端")
|
||
self.setGeometry(100, 100, 980, 820)
|
||
self.setMinimumSize(860, 680)
|
||
|
||
# 设置窗口图标
|
||
# 打印当前文件路径以便诊断
|
||
current_file = os.path.abspath(__file__)
|
||
current_dir = os.path.dirname(current_file)
|
||
logger.info(f"当前文件路径: {current_file}")
|
||
logger.info(f"当前目录: {current_dir}")
|
||
|
||
# 获取PyInstaller打包后的临时目录；开发/部署态优先使用脚本所在目录。
|
||
base_dir = current_dir
|
||
if hasattr(sys, '_MEIPASS'):
|
||
base_dir = sys._MEIPASS
|
||
logger.info(f"PyInstaller临时目录: {base_dir}")
|
||
else:
|
||
logger.info(f"开发环境目录: {base_dir}")
|
||
|
||
def resolve_asset_path(filename: str) -> str:
|
||
candidates = [
|
||
os.path.join(base_dir, filename),
|
||
os.path.join(current_dir, filename),
|
||
os.path.join(os.path.dirname(current_dir), filename),
|
||
]
|
||
for candidate in candidates:
|
||
if os.path.exists(candidate):
|
||
return candidate
|
||
return candidates[0]
|
||
|
||
# 构建图标路径
|
||
icon_path = resolve_asset_path("new_logo.svg")
|
||
logger.info(f"SVG图标路径: {icon_path}")
|
||
|
||
if os.path.exists(icon_path):
|
||
logger.info(f"SVG图标文件存在: {icon_path}")
|
||
try:
|
||
icon = QIcon(icon_path)
|
||
if icon.isNull():
|
||
logger.warning(f"SVG图标加载失败，图标为空: {icon_path}")
|
||
else:
|
||
self.setWindowIcon(icon)
|
||
logger.info(f"设置SVG窗口图标成功: {icon_path}")
|
||
except Exception as e:
|
||
logger.error(f"加载SVG图标时出错: {e}")
|
||
else:
|
||
logger.warning(f"SVG图标文件不存在: {icon_path}")
|
||
|
||
# 如果SVG加载失败，使用ico文件作为备用
|
||
ico_icon_path = resolve_asset_path("favicon2.ico")
|
||
logger.info(f"ICO图标路径: {ico_icon_path}")
|
||
|
||
if os.path.exists(ico_icon_path):
|
||
logger.info(f"ICO图标文件存在: {ico_icon_path}")
|
||
try:
|
||
icon = QIcon(ico_icon_path)
|
||
if icon.isNull():
|
||
logger.warning(f"ICO图标加载失败，图标为空: {ico_icon_path}")
|
||
else:
|
||
self.setWindowIcon(icon)
|
||
logger.info(f"设置ICO窗口图标成功: {ico_icon_path}")
|
||
except Exception as e:
|
||
logger.error(f"加载ICO图标时出错: {e}")
|
||
else:
|
||
logger.warning(f"ICO图标文件不存在: {ico_icon_path}")
|
||
|
||
# 设置窗口样式
|
||
self.setStyleSheet(""
|
||
"QMainWindow {"
|
||
" background-color: #f5f5f5;"
|
||
" font-family: 'Microsoft YaHei', Arial, sans-serif;"
|
||
"}"
|
||
"QLabel {"
|
||
" font-size: 14px;"
|
||
" color: #333;"
|
||
" padding: 5px 0;"
|
||
"}"
|
||
"QGroupBox {"
|
||
" font-size: 16px;"
|
||
" font-weight: bold;"
|
||
" border: 1px solid #ddd;"
|
||
" border-radius: 8px;"
|
||
" margin-top: 15px;"
|
||
" padding: 15px;"
|
||
" background-color: #ffffff;"
|
||
"}"
|
||
"QGroupBox::title {"
|
||
" subcontrol-origin: margin;"
|
||
" subcontrol-position: top left;"
|
||
" padding: 0 15px;"
|
||
" background-color: #4CAF50;"
|
||
" color: white;"
|
||
" border-radius: 4px;"
|
||
" font-size: 14px;"
|
||
"}"
|
||
"QProgressBar {"
|
||
" border: 1px solid #ddd;"
|
||
" border-radius: 6px;"
|
||
" text-align: center;"
|
||
" background-color: #f0f0f0;"
|
||
" height: 25px;"
|
||
"}"
|
||
"QProgressBar::chunk {"
|
||
" background-color: #4CAF50;"
|
||
" border-radius: 6px;"
|
||
"}"
|
||
"QTextEdit {"
|
||
" font-family: Consolas, 'Courier New', monospace;"
|
||
" font-size: 12px;"
|
||
" border: 1px solid #ddd;"
|
||
" border-radius: 6px;"
|
||
" background-color: #2d2d2d;"
|
||
" color: #e0e0e0;"
|
||
" padding: 10px;"
|
||
"}"
|
||
"QPushButton {"
|
||
" font-family: 'PingFang SC', 'Helvetica Neue', 'Microsoft YaHei', sans-serif;"
|
||
" font-size: 12px;"
|
||
" font-weight: 600;"
|
||
" padding: 6px 12px;"
|
||
" border: none;"
|
||
" border-radius: 4px;"
|
||
" color: white;"
|
||
" min-width: 80px;"
|
||
" min-height: 28px;"
|
||
" text-align: center;"
|
||
"}"
|
||
"QPushButton:hover {"
|
||
" opacity: 0.9;"
|
||
"}"
|
||
"QPushButton:disabled {"
|
||
" opacity: 0.5;"
|
||
"}"
|
||
"QPushButton#start_button {"
|
||
" background-color: #4CAF50;"
|
||
"}"
|
||
"QPushButton#stop_button {"
|
||
" background-color: #f44336;"
|
||
"}"
|
||
"QPushButton#exit_button {"
|
||
" background-color: #2196F3;"
|
||
"}"
|
||
"")
|
||
|
||
# 创建中心部件 + 内部滚动区域
|
||
scroll_area = QScrollArea()
|
||
scroll_area.setWidgetResizable(True)
|
||
scroll_area.setFrameShape(QFrame.NoFrame)
|
||
scroll_area.setVerticalScrollBarPolicy(Qt.ScrollBarAsNeeded)
|
||
scroll_area.setHorizontalScrollBarPolicy(Qt.ScrollBarAsNeeded)
|
||
self.setCentralWidget(scroll_area)
|
||
|
||
central_widget = QWidget()
|
||
central_widget.setMinimumWidth(920)
|
||
scroll_area.setWidget(central_widget)
|
||
|
||
# 创建布局
|
||
layout = QVBoxLayout(central_widget)
|
||
layout.setSpacing(10)
|
||
layout.setContentsMargins(20, 20, 20, 20)
|
||
|
||
# 创建状态标签
|
||
self.status_label = QLabel("状态: 就绪")
|
||
self.status_label.setStyleSheet("font-family: 'PingFang SC', 'Helvetica Neue', 'Microsoft YaHei', sans-serif; font-size: 14px; font-weight: 600; color: #2c3e50;")
|
||
layout.addWidget(self.status_label)
|
||
|
||
# 创建配置信息显示区域
|
||
config_group = QGroupBox("配置信息")
|
||
config_group.setStyleSheet("""
|
||
QGroupBox {
|
||
font-size: 16px;
|
||
font-weight: 600;
|
||
color: #2c3e50;
|
||
border: 1px solid #e2e8f0;
|
||
border-radius: 10px;
|
||
margin-top: 20px;
|
||
padding: 0;
|
||
background-color: #ffffff;
|
||
}
|
||
QGroupBox::title {
|
||
subcontrol-origin: margin;
|
||
subcontrol-position: top left;
|
||
padding: 8px 20px;
|
||
background: linear-gradient(135deg, #667eea 0%, #764ba2 100%);
|
||
color: white;
|
||
border-radius: 10px 10px 0 0;
|
||
font-family: 'PingFang SC', 'Helvetica Neue', 'Microsoft YaHei', sans-serif;
|
||
font-size: 14px;
|
||
font-weight: 600;
|
||
}
|
||
""")
|
||
config_layout = QVBoxLayout()
|
||
config_layout.setSpacing(12)
|
||
config_layout.setContentsMargins(20, 20, 20, 20)
|
||
|
||
# 创建配置标签
|
||
self.config_labels = {
|
||
'detect_options': QLabel("检测选项: 加载中..."),
|
||
'proxy_config': QLabel("代理配置: 加载中..."),
|
||
'thread_count': QLabel("检测线程数: 加载中...")
|
||
}
|
||
|
||
# 添加配置标签到布局
|
||
for label in self.config_labels.values():
|
||
label.setStyleSheet("""
|
||
QLabel {
|
||
font-family: 'PingFang SC', 'Helvetica Neue', 'Microsoft YaHei', sans-serif;
|
||
font-size: 13px;
|
||
font-weight: 400;
|
||
color: #4a5568;
|
||
background-color: #f7fafc;
|
||
padding: 12px 18px;
|
||
border-radius: 8px;
|
||
border-left: 4px solid #667eea;
|
||
margin: 0;
|
||
min-height: 64px;
|
||
white-space: normal;
|
||
border: 1px solid #e2e8f0;
|
||
}
|
||
QLabel:hover {
|
||
background-color: #edf2f7;
|
||
border-left-color: #764ba2;
|
||
}
|
||
""")
|
||
label.setWordWrap(True) # 启用自动换行
|
||
label.setAlignment(Qt.AlignLeft | Qt.AlignTop)
|
||
config_layout.addWidget(label)
|
||
|
||
config_group.setLayout(config_layout)
|
||
layout.addWidget(config_group)
|
||
|
||
# 创建进度条
|
||
self.progress_bar = QProgressBar()
|
||
self.progress_bar.setValue(0)
|
||
layout.addWidget(self.progress_bar)
|
||
|
||
# 创建日志文本框
|
||
self.log_text = QTextEdit()
|
||
self.log_text.setReadOnly(True)
|
||
self.log_text.setStyleSheet("font-family: Consolas, monospace; font-size: 12px; border: 1px solid #ddd; border-radius: 5px; background-color: #2d2d2d; color: #e0e0e0;")
|
||
self.log_text.setMinimumHeight(320)
|
||
layout.addWidget(self.log_text, 1)
|
||
|
||
# 创建按钮布局
|
||
button_layout = QHBoxLayout()
|
||
button_layout.setSpacing(10)
|
||
|
||
use_fancy_button_effects = os.environ.get("QT_QPA_PLATFORM", "").lower() != "offscreen"
|
||
|
||
# 按钮样式表
|
||
start_button_style = self._button_style(
|
||
"#28a745",
|
||
"#218838",
|
||
"#1e7e34",
|
||
fancy_effects=use_fancy_button_effects,
|
||
)
|
||
|
||
refresh_button_style = """
|
||
QPushButton {
|
||
background-color: #6f42c1;
|
||
color: white;
|
||
border: none;
|
||
border-radius: 8px;
|
||
font-size: 16px;
|
||
font-weight: bold;
|
||
padding: 10px 20px;
|
||
min-width: 120px;
|
||
}
|
||
QPushButton:hover {
|
||
background-color: #5a32a3;
|
||
}
|
||
QPushButton:disabled {
|
||
background-color: #6c757d;
|
||
color: #adb5bd;
|
||
}
|
||
"""
|
||
stop_button_style = self._button_style(
|
||
"#dc3545",
|
||
"#c82333",
|
||
"#a71e2a",
|
||
fancy_effects=use_fancy_button_effects,
|
||
)
|
||
|
||
exit_button_style = self._button_style(
|
||
"#007bff",
|
||
"#0069d9",
|
||
"#0056b3",
|
||
fancy_effects=use_fancy_button_effects,
|
||
)
|
||
|
||
# 创建开始检测按钮
|
||
self.start_button = QPushButton("开始检测")
|
||
self.start_button.setObjectName("start_button")
|
||
self.start_button.clicked.connect(self.start_detection)
|
||
self.start_button.setFixedHeight(50)
|
||
self.start_button.setStyleSheet(start_button_style)
|
||
button_layout.addWidget(self.start_button)
|
||
|
||
# 创建停止按钮
|
||
self.stop_button = QPushButton("停止检测")
|
||
self.stop_button.setObjectName("stop_button")
|
||
self.stop_button.clicked.connect(self.stop_detection)
|
||
self.stop_button.setEnabled(False)
|
||
self.stop_button.setFixedHeight(50)
|
||
self.stop_button.setStyleSheet(stop_button_style)
|
||
button_layout.addWidget(self.stop_button)
|
||
|
||
self.refresh_proxy_button = QPushButton("刷新代理池")
|
||
self.refresh_proxy_button.clicked.connect(self.refresh_proxy_pool_manually)
|
||
self.refresh_proxy_button.setFixedHeight(50)
|
||
self.refresh_proxy_button.setStyleSheet(refresh_button_style)
|
||
button_layout.addWidget(self.refresh_proxy_button)
|
||
|
||
# 创建退出按钮
|
||
self.exit_button = QPushButton("退出")
|
||
self.exit_button.setObjectName("exit_button")
|
||
self.exit_button.clicked.connect(self.close)
|
||
self.exit_button.setFixedHeight(50)
|
||
self.exit_button.setStyleSheet(exit_button_style)
|
||
button_layout.addWidget(self.exit_button)
|
||
|
||
layout.addLayout(button_layout)
|
||
|
||
# 初始化检测端
|
||
self.worker = DetectWorker(self)
|
||
|
||
# 激活服务态运行时：启动控制链路，并在服务重启后立即尝试回挂活动检测任务。
|
||
self.worker.activate_service_runtime()
|
||
|
||
# 连接信号
|
||
self.worker.connect_signals()
|
||
|
||
# 初始化检测线程
|
||
self.detect_thread = None
|
||
|
||
# 初始化日志队列和处理线程
|
||
self.log_queue = Queue()
|
||
self.log_processing = True
|
||
# 启动日志处理线程
|
||
self.log_thread = threading.Thread(target=self.process_log_queue)
|
||
self.log_thread.daemon = True
|
||
self.log_thread.start()
|
||
|
||
# 配置日志
|
||
logger.add(self.enqueue_log_message, level="INFO")
|
||
|
||
# 启动定时任务
|
||
self.scheduler_thread = threading.Thread(target=self.start_scheduler)
|
||
self.scheduler_thread.daemon = True
|
||
self.scheduler_thread.start()
|
||
|
||
def enqueue_log_message(self, message):
|
||
"""
|
||
日志回调函数，将日志消息放入队列
|
||
"""
|
||
self.log_queue.put(message)
|
||
|
||
def process_log_queue(self):
|
||
"""
|
||
处理日志队列中的消息
|
||
"""
|
||
while self.log_processing:
|
||
try:
|
||
message = self.log_queue.get(timeout=1)
|
||
# 提取时间和消息部分
|
||
# 日志格式: 2026-04-13 00:22:05.365 | INFO | __main__:load_thread_count:846 - 从Redis加载检测线程数成功: 500
|
||
import re
|
||
match = re.match(r'(\d{4}-\d{2}-\d{2} \d{2}:\d{2}:\d{2}\.\d{3}) \| .*? \| .*? - (.*)', message)
|
||
if match:
|
||
time_str = match.group(1)
|
||
msg_str = match.group(2)
|
||
log_text = f"{time_str} | {msg_str}"
|
||
else:
|
||
log_text = message.replace("\n", "")
|
||
# 使用QMetaObject.invokeMethod确保线程安全的GUI更新
|
||
QMetaObject.invokeMethod(self.log_text, "append", Qt.QueuedConnection, Q_ARG(str, log_text))
|
||
QMetaObject.invokeMethod(self.log_text.verticalScrollBar(), "setValue", Qt.QueuedConnection, Q_ARG(int, self.log_text.verticalScrollBar().maximum()))
|
||
except Exception as e:
|
||
pass
|
||
|
||
def log_message(self, message):
|
||
"""
|
||
日志回调函数（保留用于信号连接）
|
||
"""
|
||
self.enqueue_log_message(message)
|
||
|
||
def start_detection(self):
|
||
"""
|
||
开始检测
|
||
"""
|
||
self.status_label.setText("状态: 检测中...")
|
||
self.start_button.setEnabled(False)
|
||
self.stop_button.setEnabled(True)
|
||
|
||
# 创建并启动检测线程
|
||
self.detect_thread = DetectThread(self.worker)
|
||
self.detect_thread.log_signal.connect(self.log_message)
|
||
self.detect_thread.progress_signal.connect(self.update_progress)
|
||
self.detect_thread.thread_count_signal.connect(self.update_thread_count_display)
|
||
self.detect_thread.finished_signal.connect(self.detection_finished)
|
||
self.detect_thread.start()
|
||
|
||
def stop_detection(self):
|
||
"""
|
||
停止检测
|
||
"""
|
||
self.status_label.setText("状态: 停止中...")
|
||
self.worker.stop()
|
||
self.stop_button.setEnabled(False)
|
||
|
||
def detection_finished(self):
|
||
"""
|
||
检测完成
|
||
"""
|
||
self.status_label.setText("状态: 检测完成")
|
||
self.start_button.setEnabled(True)
|
||
self.stop_button.setEnabled(False)
|
||
|
||
def update_progress(self, current, total):
|
||
"""
|
||
更新进度条
|
||
"""
|
||
# 直接在当前线程执行，避免使用QMetaObject.invokeMethod
|
||
try:
|
||
if total > 0:
|
||
progress = int((current / total) * 100)
|
||
self.progress_bar.setValue(progress)
|
||
except Exception as e:
|
||
print(f"更新进度失败: {e}")
|
||
|
||
def update_thread_count_display(self, active_count, total_count):
|
||
"""
|
||
更新线程数量显示
|
||
|
||
:param active_count: 当前活跃线程数量
|
||
:param total_count: 总线程数量
|
||
"""
|
||
try:
|
||
# 检查是否有配置标签
|
||
if hasattr(self, 'config_labels') and 'thread_count' in self.config_labels:
|
||
self.config_labels['thread_count'].setText(f"检测线程数: {total_count} (当前: {active_count})")
|
||
except Exception as e:
|
||
print(f"更新线程数量显示失败: {e}")
|
||
|
||
def refresh_proxy_pool_manually(self):
|
||
self.status_label.setText("状态: 正在刷新代理池...")
|
||
self.refresh_proxy_button.setEnabled(False)
|
||
|
||
def run_refresh():
|
||
try:
|
||
self.worker.refresh_proxy_pool()
|
||
with self.worker.proxy_pool_lock:
|
||
proxy_count = len(self.worker.proxy_pool)
|
||
self.worker.update_config_labels()
|
||
QMetaObject.invokeMethod(
|
||
self.status_label,
|
||
"setText",
|
||
Qt.QueuedConnection,
|
||
Q_ARG(str, f"状态: 代理池刷新完成，可用代理 {proxy_count} 个")
|
||
)
|
||
except Exception as e:
|
||
QMetaObject.invokeMethod(
|
||
self.status_label,
|
||
"setText",
|
||
Qt.QueuedConnection,
|
||
Q_ARG(str, f"状态: 刷新代理池失败: {e}")
|
||
)
|
||
finally:
|
||
QMetaObject.invokeMethod(
|
||
self.refresh_proxy_button,
|
||
"setEnabled",
|
||
Qt.QueuedConnection,
|
||
Q_ARG(bool, True)
|
||
)
|
||
|
||
threading.Thread(target=run_refresh, daemon=True).start()
|
||
|
||
def start_scheduler(self):
|
||
"""
|
||
启动定时任务
|
||
"""
|
||
# 每天凌晨1点执行检测
|
||
schedule.every().day.at("01:00").do(self.scheduled_detection)
|
||
|
||
# 循环执行定时任务
|
||
while True:
|
||
schedule.run_pending()
|
||
time.sleep(60)
|
||
|
||
def scheduled_detection(self):
|
||
"""
|
||
定时检测任务
|
||
"""
|
||
self.log_message(f"[{datetime.now()}] 开始定时检测任务")
|
||
self.start_detection()
|
||
|
||
def refresh_config(self):
|
||
"""
|
||
刷新配置
|
||
"""
|
||
try:
|
||
# 重新加载配置
|
||
new_detect_options = self.worker.load_detect_options()
|
||
new_proxy_config = self.worker.load_proxy_config()
|
||
new_thread_count = self.worker.load_thread_count()
|
||
new_sensitive_words = self.worker.db.get_all_sensitive_words()
|
||
self.worker.load_cookies_from_remote()
|
||
|
||
# 检查配置是否发生变化
|
||
config_changed = False
|
||
if new_detect_options != self.worker.detect_options:
|
||
self.worker.detect_options = new_detect_options
|
||
config_changed = True
|
||
if new_proxy_config != self.worker.proxy_config:
|
||
self.worker.proxy_config = new_proxy_config
|
||
config_changed = True
|
||
if new_thread_count != self.worker.thread_count:
|
||
self.worker.thread_count = new_thread_count
|
||
config_changed = True
|
||
if new_sensitive_words != self.worker.sensitive_words:
|
||
self.worker.sensitive_words = new_sensitive_words
|
||
config_changed = True
|
||
|
||
# 如果配置发生变化，更新标签
|
||
if config_changed:
|
||
self.worker.update_config_labels()
|
||
self.log_message(f"[{datetime.now()}] 配置已更新")
|
||
except Exception as e:
|
||
self.log_message(f"[{datetime.now()}] 刷新配置失败: {e}")
|
||
|
||
def event(self, event):
|
||
"""
|
||
处理事件
|
||
"""
|
||
if event.type() == ConfigUpdateEvent.Type:
|
||
self.update_config_labels()
|
||
return True
|
||
return super().event(event)
|
||
|
||
def update_config_labels(self):
|
||
"""
|
||
更新GUI配置标签
|
||
"""
|
||
logger.debug("开始更新GUI配置标签")
|
||
try:
|
||
if self.worker:
|
||
try:
|
||
logger.debug(f"worker对象存在，当前配置: {self.worker.detect_options}, {self.worker.proxy_config}, {self.worker.thread_count}")
|
||
# 准备更新数据
|
||
detect_options = self.worker.detect_options
|
||
free_options_display = []
|
||
paid_options_display = []
|
||
option_names = {
|
||
'detect_register': '检查注册',
|
||
'detect_baidu_site': '百度site查询',
|
||
'detect_360_site': '360的site查询',
|
||
'detect_chinaz': '站长之家查询',
|
||
'detect_aizhan': '爱站网查询',
|
||
'detect_wayback': '时光机检测',
|
||
'detect_jucha': '聚查查询',
|
||
'detect_juziseo': '桔子查询'
|
||
}
|
||
|
||
ordered_keys = detect_options.get('detect_order') or list(option_names.keys())
|
||
normalized_keys = [key for key in ordered_keys if key in option_names]
|
||
for key in option_names:
|
||
if key not in normalized_keys:
|
||
normalized_keys.append(key)
|
||
|
||
free_keys = {
|
||
'detect_register',
|
||
'detect_baidu_site',
|
||
'detect_360_site',
|
||
'detect_chinaz',
|
||
'detect_aizhan',
|
||
'detect_wayback',
|
||
}
|
||
|
||
for index, key in enumerate(normalized_keys, start=1):
|
||
name = option_names[key]
|
||
if key in detect_options and detect_options[key]:
|
||
item_text = f"{index}.{name}"
|
||
if key in free_keys:
|
||
free_options_display.append(item_text)
|
||
else:
|
||
paid_options_display.append(item_text)
|
||
|
||
free_text = " | ".join(free_options_display) if free_options_display else "无"
|
||
paid_text = " | ".join(paid_options_display) if paid_options_display else "无"
|
||
detect_options_text = (
|
||
"当前检测选项:\n"
|
||
f"免费优先: {free_text}\n"
|
||
f"付费后置: {paid_text}"
|
||
)
|
||
|
||
# 准备代理配置文本
|
||
proxy_enable = self.worker.proxy_config.get('proxy_enable', False)
|
||
allow_direct = self.worker.proxy_config.get('allow_direct', False)
|
||
proxy_urls = self.worker.proxy_config.get('proxy_urls') or []
|
||
if not proxy_urls and self.worker.proxy_config.get('proxy_url'):
|
||
proxy_urls = [self.worker.proxy_config.get('proxy_url', '')]
|
||
with self.worker.proxy_pool_lock:
|
||
proxy_pool_count = len(self.worker.proxy_pool)
|
||
last_refresh_time = (
|
||
self.worker.proxy_last_refresh_time.strftime('%Y-%m-%d %H:%M:%S')
|
||
if self.worker.proxy_last_refresh_time else '未刷新'
|
||
)
|
||
proxy_display = (
|
||
f"启用: {'是' if proxy_enable else '否'} | "
|
||
f"允许直连: {'是' if allow_direct else '否'} | "
|
||
f"代理池链接数: {len([url for url in proxy_urls if url])} | "
|
||
f"当前可用代理数: {proxy_pool_count}\n"
|
||
f"最近刷新: {last_refresh_time} | "
|
||
f"最近结果: {self.worker.proxy_last_refresh_status} | "
|
||
f"原始返回数: {self.worker.proxy_last_refresh_total_items}"
|
||
)
|
||
proxy_text = f"代理配置: {proxy_display}"
|
||
|
||
# 准备线程数文本
|
||
thread_count_text = f"检测线程数: {self.worker.thread_count}"
|
||
|
||
logger.debug(f"准备更新标签: {detect_options_text}, {proxy_text}, {thread_count_text}")
|
||
|
||
# 检查配置标签是否存在
|
||
if hasattr(self, 'config_labels'):
|
||
# 使用QMetaObject.invokeMethod确保在主线程中更新UI
|
||
from PySide6.QtCore import QMetaObject, Qt, Q_ARG
|
||
|
||
# 更新检测选项标签
|
||
if 'detect_options' in self.config_labels:
|
||
QMetaObject.invokeMethod(self.config_labels['detect_options'], "setText",
|
||
Qt.QueuedConnection,
|
||
Q_ARG(str, detect_options_text))
|
||
|
||
# 更新代理配置标签
|
||
if 'proxy_config' in self.config_labels:
|
||
QMetaObject.invokeMethod(self.config_labels['proxy_config'], "setText",
|
||
Qt.QueuedConnection,
|
||
Q_ARG(str, proxy_text))
|
||
|
||
# 更新线程数标签
|
||
if 'thread_count' in self.config_labels:
|
||
QMetaObject.invokeMethod(self.config_labels['thread_count'], "setText",
|
||
Qt.QueuedConnection,
|
||
Q_ARG(str, thread_count_text))
|
||
|
||
logger.debug("GUI配置标签更新完成")
|
||
else:
|
||
logger.debug("配置标签不存在，跳过更新")
|
||
except Exception as e:
|
||
logger.error(f"更新GUI标签失败: {e}")
|
||
import traceback
|
||
logger.error(traceback.format_exc())
|
||
else:
|
||
logger.debug("worker对象不存在")
|
||
except Exception as e:
|
||
logger.error(f"执行update_config_labels失败: {e}")
|
||
import traceback
|
||
logger.error(traceback.format_exc())
|
||
|
||
from PySide6.QtCore import QObject, Signal, QEvent
|
||
|
||
class ConfigUpdateEvent(QEvent):
|
||
"""
|
||
配置更新事件类
|
||
"""
|
||
Type = QEvent.Type(QEvent.User + 1)
|
||
|
||
def __init__(self):
|
||
super().__init__(ConfigUpdateEvent.Type)
|
||
|
||
class ConfigUpdateSignal(QObject):
|
||
"""
|
||
配置更新信号类
|
||
"""
|
||
config_updated = Signal()
|
||
|
||
def runtime_state_key(node_code=None):
|
||
normalized_node_code = str(node_code or config.NODE_CODE or "").strip()
|
||
if not normalized_node_code:
|
||
return RUNTIME_STATE_KEY
|
||
return f"{RUNTIME_STATE_KEY}:{normalized_node_code}"
|
||
|
||
|
||
class DetectWorker:
|
||
"""
|
||
域名检测端工作类
|
||
"""
|
||
def __init__(self, main_window=None):
|
||
"""
|
||
初始化检测端
|
||
|
||
:param main_window: 主窗口实例，用于更新GUI
|
||
"""
|
||
self.main_window = main_window
|
||
self.running = False
|
||
self.stop_requested = False
|
||
self.detecting = False
|
||
self.detect_threads = []
|
||
self.detect_thread = None # 检测线程实例
|
||
self.detect_command_thread = None
|
||
self.detect_lock = threading.Lock()
|
||
self._domain_thread_counter_lock = threading.Lock()
|
||
self._task_local = threading.local()
|
||
self._step_result_cache_lock = threading.Lock()
|
||
self._step_result_cache = {}
|
||
self._active_domain_threads = 0
|
||
self.detect_command_started_at = 0.0
|
||
self.detect_command_last_activity_at = 0.0
|
||
self._last_domain_started_at = 0.0
|
||
self._last_domain_result_at = 0.0
|
||
self._pending_restart_source = ""
|
||
self._pending_restart_payload = None
|
||
self._pending_restart_reason = ""
|
||
self._detect_session_seq = 0
|
||
self._detect_session_owner = 0
|
||
self._worker_started_at = time.time()
|
||
self._last_worker_activity_at = self._worker_started_at
|
||
self._last_explicit_start_signal_at = 0.0
|
||
self._last_explicit_start_payload = {}
|
||
self._last_detection_start_at = 0.0
|
||
self._last_detection_start_source = ""
|
||
self._last_detection_start_job_id = None
|
||
self._last_detection_start_job_code = ""
|
||
self._last_detection_start_task_mode = ""
|
||
self._cached_active_detect_job = None
|
||
self._last_active_detect_job_refresh_at = 0.0
|
||
self._explicit_claim_scope_job_id = None
|
||
self._explicit_claim_scope_job_code = ""
|
||
self._explicit_claim_scope_until = 0.0
|
||
self._explicit_claim_scope_source = ""
|
||
self._ignored_targeted_job_id = None
|
||
self._ignored_targeted_job_code = ""
|
||
self._ignored_targeted_job_until = 0.0
|
||
self._ignored_targeted_job_targets = ""
|
||
self._last_explicit_scope_claim_log_at = 0.0
|
||
self._last_explicit_scope_claim_log_key = ""
|
||
self._last_explicit_scope_probe_log_at = 0.0
|
||
self._last_job_status_refresh_probe_log_at = 0.0
|
||
self._idle_recycle_requested = False
|
||
self._last_thread_count_refresh_at = 0.0
|
||
self.runtime_heartbeat_interval = max(
|
||
3.0,
|
||
float(os.getenv("DOMAINCHECK_RUNTIME_HEARTBEAT_INTERVAL", "5") or 5),
|
||
)
|
||
self._runtime_heartbeat_stop = threading.Event()
|
||
self._last_runtime_phase = "idle"
|
||
self._last_runtime_detail = "Worker 初始化中"
|
||
self._last_runtime_extra = {}
|
||
self._last_runtime_state_push_at = 0.0
|
||
self._last_runtime_state_push_phase = ""
|
||
self.current_cycle_token = ""
|
||
self.current_job_id = None
|
||
self.current_job_code = ""
|
||
self.current_job_task_mode = ""
|
||
self._restart_release_handoff_job_id = None
|
||
self._restart_release_handoff_job_code = ""
|
||
self._restart_release_handoff_until = 0.0
|
||
self._restart_release_handoff_reason = ""
|
||
self.runtime_settings = {
|
||
"worker_log_sync_enabled": False,
|
||
"worker_log_sync_mode": "key",
|
||
"worker_step_trace_enabled": True,
|
||
"worker_step_trace_sync_full": True,
|
||
"claim_batch_floor": 0,
|
||
"claim_batch_ceil": 0,
|
||
"submit_backlog_floor": 0,
|
||
"submit_backlog_ceil": 0,
|
||
"dispatch_cap_multiplier": 1,
|
||
"pending_buffer_cap_multiplier": 1,
|
||
}
|
||
self._last_synced_worker_log = ""
|
||
self._worker_log_sync_queue = Queue(maxsize=50000)
|
||
self._worker_log_sync_stop = threading.Event()
|
||
self._worker_log_sync_drop_count = 0
|
||
self._last_worker_log_sync_drop_notice_at = 0.0
|
||
self._last_worker_log_sync_degrade_notice_at = 0.0
|
||
self._remote_debug_event_failure_streak = 0
|
||
self._remote_debug_event_cooldown_until = 0.0
|
||
self._last_remote_debug_event_notice_at = 0.0
|
||
self._running_mark_lock = threading.Lock()
|
||
self._pending_running_marks = collections.deque()
|
||
self._last_running_mark_flush_at = 0.0
|
||
self._job_release_lock = threading.Lock()
|
||
self._pending_job_releases = collections.deque()
|
||
self._pending_job_release_reasons = {}
|
||
self._last_job_release_flush_at = 0.0
|
||
self._job_finalize_lock = threading.Lock()
|
||
self._pending_job_finalizations = collections.deque()
|
||
self._last_job_finalize_flush_at = 0.0
|
||
self._domain_status_update_lock = threading.Lock()
|
||
self._pending_domain_status_updates = collections.deque()
|
||
self._last_domain_status_flush_at = 0.0
|
||
self._domain_completion_lock = threading.Lock()
|
||
self._pending_domain_completions = collections.deque()
|
||
self._last_domain_completion_flush_at = 0.0
|
||
self._review_status_update_lock = threading.Lock()
|
||
self._pending_review_status_updates = collections.deque()
|
||
self._last_review_status_flush_at = 0.0
|
||
self._completed_future_lock = threading.Lock()
|
||
self._completed_futures = collections.deque()
|
||
self._last_job_item_release_at = 0.0
|
||
|
||
# 初始化数据库连接
|
||
self.db = Database()
|
||
try:
|
||
self.db.ensure_cluster_runtime_tables()
|
||
self.db.register_cluster_node(
|
||
config.NODE_CODE,
|
||
config.NODE_REGION,
|
||
config.NODE_ROLE,
|
||
status="online",
|
||
current_load=0,
|
||
metadata={"service": "detect-worker", "worker_mode": os.environ.get("WORKER_MODE", "")},
|
||
)
|
||
except Exception as e:
|
||
logger.warning(f"初始化多机运行库骨架失败: {e}")
|
||
try:
|
||
recycled = self.db.recycle_running_domains(DETECT_STATUS_FAILED)
|
||
if recycled:
|
||
logger.warning(f"Worker 启动时回收了 {recycled} 个遗留的检测中域名，已标记为失败待重试")
|
||
released = self._maybe_release_node_job_items(reason="worker_bootstrap", min_interval_seconds=0.0)
|
||
if released:
|
||
logger.warning(f"Worker 启动时释放了 {released} 个当前节点遗留任务项，已重新回到 pending")
|
||
except Exception as e:
|
||
logger.warning(f"Worker 启动时清理遗留运行态失败: {e}")
|
||
|
||
# 初始化Redis连接
|
||
try:
|
||
self.redis_client = get_redis_client(role="standard")
|
||
# 测试连接
|
||
self.redis_client.ping()
|
||
logger.info(f"Redis 连接成功: {config.REDIS_HOST}:{config.REDIS_PORT}")
|
||
self.use_redis = True
|
||
except Exception as e:
|
||
logger.warning(f"Redis 连接失败: {e}，将使用本地配置")
|
||
self.redis_client = None
|
||
self.use_redis = False
|
||
|
||
# 初始化配置更新信号
|
||
self.config_signal = ConfigUpdateSignal()
|
||
|
||
# 加载配置
|
||
self.detect_options = self.load_detect_options()
|
||
self.proxy_config = self.load_proxy_config()
|
||
self.thread_count = self.load_thread_count() # 从配置文件加载线程数
|
||
self.process_count = self.load_process_count() # 从配置文件加载进程数
|
||
self.runtime_settings = self.load_runtime_settings()
|
||
self.register_single_machine_mode_enabled = self._env_flag_enabled(
|
||
"DOMAINCHECK_REGISTER_SINGLE_MACHINE_MODE"
|
||
)
|
||
self.single_machine_site_direct_fallback_enabled = self._env_flag_enabled(
|
||
"DOMAINCHECK_SINGLE_MACHINE_SITE_DIRECT_FALLBACK",
|
||
default=self.register_single_machine_mode_enabled,
|
||
)
|
||
self.single_machine_aizhan_direct_first_enabled = self._env_flag_enabled(
|
||
"DOMAINCHECK_SINGLE_MACHINE_AIZHAN_DIRECT_FIRST",
|
||
default=False,
|
||
)
|
||
self.single_machine_baidu_direct_first_enabled = self._env_flag_enabled(
|
||
"DOMAINCHECK_SINGLE_MACHINE_BAIDU_DIRECT_FIRST",
|
||
default=False,
|
||
)
|
||
self.aizhan_remote_disconnect_degrade_enabled = self._env_flag_enabled(
|
||
"DOMAINCHECK_AIZHAN_REMOTE_DISCONNECT_DEGRADE",
|
||
default=False,
|
||
)
|
||
self.aizhan_external_fast_degrade_enabled = self._env_flag_enabled(
|
||
"DOMAINCHECK_AIZHAN_EXTERNAL_FAST_DEGRADE",
|
||
default=False,
|
||
)
|
||
self.register_single_machine_direct_streak_attempts = max(
|
||
0,
|
||
int(os.getenv("DOMAINCHECK_REGISTER_DIRECT_STREAK_ATTEMPTS", "2") or 2),
|
||
) if self.register_single_machine_mode_enabled else 0
|
||
|
||
# 初始化代理池
|
||
self.proxy_pool = []
|
||
self.proxy_pool_lock = threading.Lock()
|
||
self.proxy_refresh_lock = threading.Lock()
|
||
self.proxy_last_refresh_time = None
|
||
self.proxy_last_refresh_status = "未刷新"
|
||
self.proxy_last_refresh_source_count = 0
|
||
self.proxy_last_refresh_total_items = 0
|
||
self.proxy_last_validated_count = 0
|
||
self.proxy_last_available_count = 0
|
||
self.proxy_last_source_stats = []
|
||
self._last_proxy_clock_skew_warning_at = 0.0
|
||
self.proxy_refresh_cooldown_seconds = 30
|
||
self.proxy_next_refresh_time = 0.0
|
||
self._last_proxy_refresh_reason_at = {}
|
||
self.proxy_max_reuse_count = max(8, self.thread_count * 4)
|
||
self.proxy_step_wait_timeout_seconds = max(
|
||
0.0,
|
||
float(os.getenv("DOMAINCHECK_PROXY_WAIT_TIMEOUT", "1.5") or 1.5),
|
||
)
|
||
self.proxy_direct_fallback_grace_seconds = max(
|
||
0.0,
|
||
float(os.getenv("DOMAINCHECK_PROXY_DIRECT_FALLBACK_GRACE", "0.35") or 0.35),
|
||
)
|
||
self.proxy_direct_fallback_grace_overrides = {
|
||
"注册状态检测": max(
|
||
0.0,
|
||
float(os.getenv("DOMAINCHECK_PROXY_DIRECT_FALLBACK_GRACE_REGISTER", "0.08") or 0.08),
|
||
),
|
||
"360检测": max(
|
||
0.0,
|
||
float(os.getenv("DOMAINCHECK_PROXY_DIRECT_FALLBACK_GRACE_360", "0.12") or 0.12),
|
||
),
|
||
"百度site检测": max(
|
||
0.0,
|
||
float(os.getenv("DOMAINCHECK_PROXY_DIRECT_FALLBACK_GRACE_BAIDU", "0.12") or 0.12),
|
||
),
|
||
"站长之家检测": max(
|
||
0.0,
|
||
float(os.getenv("DOMAINCHECK_PROXY_DIRECT_FALLBACK_GRACE_CHINAZ", "0.12") or 0.12),
|
||
),
|
||
"爱站网检测": max(
|
||
0.0,
|
||
float(os.getenv("DOMAINCHECK_PROXY_DIRECT_FALLBACK_GRACE_AIZHAN", "0.12") or 0.12),
|
||
),
|
||
"时光机检测": max(
|
||
0.0,
|
||
float(os.getenv("DOMAINCHECK_PROXY_DIRECT_FALLBACK_GRACE_WAYBACK", "0.2") or 0.2),
|
||
),
|
||
}
|
||
self.proxy_direct_retry_wait_seconds = max(
|
||
0.0,
|
||
float(os.getenv("DOMAINCHECK_DIRECT_RETRY_PROXY_WAIT", "1.2") or 1.2),
|
||
)
|
||
self.proxy_direct_retry_wait_overrides = {
|
||
"注册状态检测": max(
|
||
0.0,
|
||
float(os.getenv("DOMAINCHECK_DIRECT_RETRY_PROXY_WAIT_REGISTER", "0.2") or 0.2),
|
||
),
|
||
"360检测": max(
|
||
0.0,
|
||
float(os.getenv("DOMAINCHECK_DIRECT_RETRY_PROXY_WAIT_360", "0.25") or 0.25),
|
||
),
|
||
"百度site检测": max(
|
||
0.0,
|
||
float(os.getenv("DOMAINCHECK_DIRECT_RETRY_PROXY_WAIT_BAIDU", "0.25") or 0.25),
|
||
),
|
||
"站长之家检测": max(
|
||
0.0,
|
||
float(os.getenv("DOMAINCHECK_DIRECT_RETRY_PROXY_WAIT_CHINAZ", "0.25") or 0.25),
|
||
),
|
||
"爱站网检测": max(
|
||
0.0,
|
||
float(os.getenv("DOMAINCHECK_DIRECT_RETRY_PROXY_WAIT_AIZHAN", "0.25") or 0.25),
|
||
),
|
||
}
|
||
self.proxy_step_retry_max_attempts = max(
|
||
1,
|
||
int(os.getenv("DOMAINCHECK_PROXY_STEP_MAX_ATTEMPTS", "12") or 12),
|
||
)
|
||
self.proxy_step_retry_max_seconds = max(
|
||
5.0,
|
||
float(os.getenv("DOMAINCHECK_PROXY_STEP_MAX_SECONDS", "45") or 45),
|
||
)
|
||
self.proxy_step_retry_budget_overrides = {
|
||
"注册状态检测": {
|
||
"max_attempts": max(
|
||
1,
|
||
int(os.getenv("DOMAINCHECK_PROXY_STEP_MAX_ATTEMPTS_REGISTER", "3") or 3),
|
||
),
|
||
"max_seconds": max(
|
||
5.0,
|
||
float(os.getenv("DOMAINCHECK_PROXY_STEP_MAX_SECONDS_REGISTER", "8") or 8),
|
||
),
|
||
},
|
||
"百度site检测": {
|
||
"max_attempts": max(
|
||
1,
|
||
int(os.getenv("DOMAINCHECK_PROXY_STEP_MAX_ATTEMPTS_BAIDU", "3") or 3),
|
||
),
|
||
"max_seconds": max(
|
||
5.0,
|
||
float(os.getenv("DOMAINCHECK_PROXY_STEP_MAX_SECONDS_BAIDU", "6") or 6),
|
||
),
|
||
},
|
||
"360检测": {
|
||
"max_attempts": max(
|
||
1,
|
||
int(os.getenv("DOMAINCHECK_PROXY_STEP_MAX_ATTEMPTS_360", "2") or 2),
|
||
),
|
||
"max_seconds": max(
|
||
5.0,
|
||
float(os.getenv("DOMAINCHECK_PROXY_STEP_MAX_SECONDS_360", "5") or 5),
|
||
),
|
||
},
|
||
"站长之家检测": {
|
||
"max_attempts": max(
|
||
1,
|
||
int(os.getenv("DOMAINCHECK_PROXY_STEP_MAX_ATTEMPTS_CHINAZ", "2") or 2),
|
||
),
|
||
"max_seconds": max(
|
||
5.0,
|
||
float(os.getenv("DOMAINCHECK_PROXY_STEP_MAX_SECONDS_CHINAZ", "5") or 5),
|
||
),
|
||
},
|
||
"爱站网检测": {
|
||
"max_attempts": max(
|
||
1,
|
||
int(os.getenv("DOMAINCHECK_PROXY_STEP_MAX_ATTEMPTS_AIZHAN", "2") or 2),
|
||
),
|
||
"max_seconds": max(
|
||
5.0,
|
||
float(os.getenv("DOMAINCHECK_PROXY_STEP_MAX_SECONDS_AIZHAN", "5") or 5),
|
||
),
|
||
},
|
||
"时光机检测": {
|
||
"max_attempts": max(
|
||
1,
|
||
int(os.getenv("DOMAINCHECK_PROXY_STEP_MAX_ATTEMPTS_WAYBACK", "3") or 3),
|
||
),
|
||
"max_seconds": max(
|
||
5.0,
|
||
float(os.getenv("DOMAINCHECK_PROXY_STEP_MAX_SECONDS_WAYBACK", "12") or 12),
|
||
),
|
||
},
|
||
}
|
||
self._last_no_proxy_notice_at = 0.0
|
||
self.proxy_failure_lock = threading.Lock()
|
||
self.proxy_failure_counts = {}
|
||
self.proxy_quarantine_until = {}
|
||
self.proxy_source_by_key = {}
|
||
self.proxy_source_failure_counts = {}
|
||
self.proxy_source_quarantine_until = {}
|
||
self.proxy_active_leases = {}
|
||
self.redis_sub_thread = None
|
||
self._last_autoresume_skip_log_at = 0.0
|
||
self._last_autoresume_success_log_at = 0.0
|
||
|
||
# 加载敏感词（只加载一次，所有线程共享）
|
||
try:
|
||
self.sensitive_words = self._load_sensitive_words_runtime()
|
||
logger.info(f"加载敏感词完成，共 {len(self.sensitive_words)} 个敏感词")
|
||
except Exception as e:
|
||
logger.error(f"加载敏感词失败: {e}")
|
||
import traceback
|
||
logger.error(traceback.format_exc())
|
||
self.sensitive_words = []
|
||
|
||
# 初始化运行状态
|
||
self.running = False
|
||
|
||
# 加载cookies
|
||
self.load_cookies_from_remote()
|
||
|
||
# 付费检测器按需初始化，避免在默认关闭时产生无效启动告警。
|
||
self.jc_instance = None
|
||
self.juziseo_instance = None
|
||
self._init_optional_detectors()
|
||
self.wayback_detector = WaybackDetector()
|
||
|
||
# 显示所有配置信息
|
||
logger.info("检测端配置信息:")
|
||
logger.info(f"检测选项: {self.detect_options}")
|
||
logger.info(f"代理配置: {self.proxy_config}")
|
||
logger.info(f"检测线程数: {self.thread_count}")
|
||
logger.info(f"检测进程数: {self.process_count}")
|
||
|
||
# Redis订阅线程将在start方法中启动
|
||
self._update_runtime_state("idle", "Worker 已启动，等待检测指令")
|
||
self._start_runtime_heartbeat_loop()
|
||
self._start_worker_log_sync_loop()
|
||
self.trigger_proxy_refresh(reason="worker_startup", reset_cooldown=True)
|
||
|
||
def _start_runtime_heartbeat_loop(self):
|
||
existing = getattr(self, "_runtime_heartbeat_thread", None)
|
||
if existing and existing.is_alive():
|
||
return
|
||
self._runtime_heartbeat_stop.clear()
|
||
self._runtime_heartbeat_thread = threading.Thread(
|
||
target=self._runtime_heartbeat_loop,
|
||
name="RuntimeHeartbeatThread",
|
||
daemon=True,
|
||
)
|
||
self._runtime_heartbeat_thread.start()
|
||
|
||
def _runtime_heartbeat_loop(self):
|
||
while not self._runtime_heartbeat_stop.wait(self.runtime_heartbeat_interval):
|
||
try:
|
||
self._consume_pending_control_command()
|
||
except Exception as e:
|
||
logger.debug(f"运行态心跳补偿消费待执行控制指令失败: {e}")
|
||
try:
|
||
self._resume_active_detect_job_if_needed(reason="heartbeat_autoresume")
|
||
except Exception as e:
|
||
logger.debug(f"运行态心跳自动回挂检测任务失败: {e}")
|
||
try:
|
||
self._update_runtime_state(
|
||
self._last_runtime_phase,
|
||
self._last_runtime_detail,
|
||
**dict(self._last_runtime_extra or {}),
|
||
)
|
||
except Exception as e:
|
||
logger.debug(f"运行态心跳续期失败: {e}")
|
||
try:
|
||
self._maybe_recycle_idle_worker(trigger_reason="heartbeat")
|
||
except Exception as e:
|
||
logger.debug(f"空闲 Worker 生命周期检查失败: {e}")
|
||
|
||
def _start_worker_log_sync_loop(self):
|
||
existing = getattr(self, "_worker_log_sync_thread", None)
|
||
if existing and existing.is_alive():
|
||
return
|
||
self._worker_log_sync_stop.clear()
|
||
self._worker_log_sync_thread = threading.Thread(
|
||
target=self._worker_log_sync_loop,
|
||
name="WorkerLogSyncThread",
|
||
daemon=True,
|
||
)
|
||
self._worker_log_sync_thread.start()
|
||
|
||
def _worker_log_sync_loop(self):
|
||
while True:
|
||
if self._worker_log_sync_stop.is_set() and self._worker_log_sync_queue.empty():
|
||
break
|
||
try:
|
||
item = self._worker_log_sync_queue.get(timeout=0.5)
|
||
except Empty:
|
||
continue
|
||
if item is None:
|
||
self._worker_log_sync_queue.task_done()
|
||
if self._worker_log_sync_stop.is_set():
|
||
break
|
||
continue
|
||
|
||
if item.get("write_db", True):
|
||
try:
|
||
self.db.append_detect_run_event(
|
||
item["job_id"],
|
||
item["job_item_id"],
|
||
config.NODE_CODE,
|
||
event_type='worker_log',
|
||
message=item["message"],
|
||
level=item["level"],
|
||
payload=item["payload"],
|
||
)
|
||
except Exception as e:
|
||
logger.debug(f"Worker 日志事件写库失败: {e}")
|
||
|
||
try:
|
||
self._push_remote_debug_event(
|
||
message=item["message"],
|
||
level=item["level"],
|
||
payload=item["payload"],
|
||
event_type="worker_log",
|
||
)
|
||
except Exception as e:
|
||
logger.debug(f"Worker 日志事件远端回传失败: {e}")
|
||
finally:
|
||
self._worker_log_sync_queue.task_done()
|
||
|
||
def _enqueue_running_mark(self, job_item_id, claim_token):
|
||
normalized_claim_token = str(claim_token or "").strip()
|
||
try:
|
||
normalized_job_item_id = int(job_item_id or 0)
|
||
except Exception:
|
||
normalized_job_item_id = 0
|
||
if normalized_job_item_id <= 0 or not normalized_claim_token:
|
||
return
|
||
with self._running_mark_lock:
|
||
self._pending_running_marks.append((normalized_job_item_id, normalized_claim_token))
|
||
|
||
def _flush_pending_running_marks(self, *, force=False, batch_limit=512):
|
||
now_ts = time.time()
|
||
if not force and now_ts - float(getattr(self, "_last_running_mark_flush_at", 0.0) or 0.0) < 0.05:
|
||
return 0
|
||
batch = []
|
||
with self._running_mark_lock:
|
||
if not self._pending_running_marks:
|
||
if force:
|
||
self._last_running_mark_flush_at = now_ts
|
||
return 0
|
||
while self._pending_running_marks and len(batch) < max(1, int(batch_limit or 1)):
|
||
batch.append(self._pending_running_marks.popleft())
|
||
updated_count = 0
|
||
try:
|
||
updated_count = int(self.db.mark_detect_job_items_running_batch(batch) or 0)
|
||
except Exception as e:
|
||
logger.debug(f"批量刷新运行中任务项失败: {e}")
|
||
updated_count = -1
|
||
if updated_count < 0:
|
||
with self._running_mark_lock:
|
||
for item in reversed(batch):
|
||
self._pending_running_marks.appendleft(item)
|
||
logger.warning(
|
||
f"批量刷新运行中任务项失败，已回退待重试队列: batch={len(batch)} | force={1 if force else 0}"
|
||
)
|
||
updated_count = 0
|
||
self._last_running_mark_flush_at = now_ts
|
||
return updated_count
|
||
|
||
def _enqueue_job_finalization(
|
||
self,
|
||
*,
|
||
job_item_id,
|
||
claim_token,
|
||
final_status,
|
||
message="",
|
||
result_payload=None,
|
||
result_version="v1",
|
||
job_id=None,
|
||
node_code="",
|
||
event_type="",
|
||
event_level="info",
|
||
event_message="",
|
||
event_payload=None,
|
||
):
|
||
normalized_claim_token = str(claim_token or "").strip()
|
||
try:
|
||
normalized_job_item_id = int(job_item_id or 0)
|
||
except Exception:
|
||
normalized_job_item_id = 0
|
||
if normalized_job_item_id <= 0 or not normalized_claim_token:
|
||
return
|
||
try:
|
||
normalized_job_id = int(job_id or 0)
|
||
except Exception:
|
||
normalized_job_id = 0
|
||
with self._job_finalize_lock:
|
||
self._pending_job_finalizations.append(
|
||
{
|
||
"job_item_id": normalized_job_item_id,
|
||
"claim_token": normalized_claim_token,
|
||
"final_status": str(final_status or "").strip() or "failed",
|
||
"message": str(message or "").strip(),
|
||
"result_payload": result_payload,
|
||
"result_version": str(result_version or "v1").strip() or "v1",
|
||
"job_id": normalized_job_id,
|
||
"node_code": str(node_code or "").strip(),
|
||
"event_type": str(event_type or "").strip(),
|
||
"event_level": str(event_level or "info").strip() or "info",
|
||
"event_message": str(event_message or "").strip(),
|
||
"event_payload": dict(event_payload or {}),
|
||
}
|
||
)
|
||
|
||
def _enqueue_job_release(self, job_item_id, claim_token, reason=""):
|
||
normalized_claim_token = str(claim_token or "").strip()
|
||
try:
|
||
normalized_job_item_id = int(job_item_id or 0)
|
||
except Exception:
|
||
normalized_job_item_id = 0
|
||
if normalized_job_item_id <= 0 or not normalized_claim_token:
|
||
return False
|
||
release_key = (normalized_job_item_id, normalized_claim_token)
|
||
normalized_reason = str(reason or "").strip()[:1000]
|
||
with self._job_release_lock:
|
||
existing_reason = str(
|
||
getattr(self, "_pending_job_release_reasons", {}).get(release_key, "") or ""
|
||
).strip()
|
||
if release_key not in getattr(self, "_pending_job_release_reasons", {}):
|
||
self._pending_job_releases.append(release_key)
|
||
# Keep the newest non-empty reason, but avoid growing the queue for duplicates.
|
||
self._pending_job_release_reasons[release_key] = normalized_reason or existing_reason
|
||
return True
|
||
|
||
def _flush_pending_job_releases(self, *, force=False, batch_limit=128):
|
||
now_ts = time.time()
|
||
if not force and now_ts - float(getattr(self, "_last_job_release_flush_at", 0.0) or 0.0) < 0.20:
|
||
return 0
|
||
batch = []
|
||
with self._job_release_lock:
|
||
if not self._pending_job_releases:
|
||
if force:
|
||
self._last_job_release_flush_at = now_ts
|
||
return 0
|
||
while self._pending_job_releases and len(batch) < max(1, int(batch_limit or 1)):
|
||
release_key = self._pending_job_releases.popleft()
|
||
reason = str(self._pending_job_release_reasons.pop(release_key, "") or "").strip()
|
||
batch.append((int(release_key[0]), str(release_key[1]), reason))
|
||
updated_count = 0
|
||
flush_started_at = time.perf_counter()
|
||
try:
|
||
updated_count = int(self.db.release_detect_job_items_batch(batch) or 0)
|
||
except Exception as e:
|
||
logger.debug(f"批量释放任务项回队列失败: {e}")
|
||
updated_count = -1
|
||
if updated_count < 0:
|
||
with self._job_release_lock:
|
||
for item in reversed(batch):
|
||
release_key = (int(item[0]), str(item[1]))
|
||
self._pending_job_releases.appendleft(release_key)
|
||
self._pending_job_release_reasons[release_key] = str(item[2] or "").strip()[:1000]
|
||
logger.warning(
|
||
f"批量释放任务项回队列失败，已回退待重试队列: batch={len(batch)} | force={1 if force else 0}"
|
||
)
|
||
updated_count = 0
|
||
flush_elapsed_ms = int((time.perf_counter() - flush_started_at) * 1000)
|
||
if batch and (flush_elapsed_ms >= 20 or len(batch) >= 64 or force):
|
||
logger.info(
|
||
f"任务项回队列批量释放: batch={len(batch)} | updated={updated_count} "
|
||
f"| elapsed_ms={flush_elapsed_ms} | force={1 if force else 0}"
|
||
)
|
||
self._last_job_release_flush_at = now_ts
|
||
return updated_count
|
||
|
||
def _flush_pending_job_finalizations(self, *, force=False, batch_limit=512):
|
||
now_ts = time.time()
|
||
if not force and now_ts - float(getattr(self, "_last_job_finalize_flush_at", 0.0) or 0.0) < 0.05:
|
||
return 0
|
||
batch = []
|
||
with self._job_finalize_lock:
|
||
if not self._pending_job_finalizations:
|
||
if force:
|
||
self._last_job_finalize_flush_at = now_ts
|
||
return 0
|
||
while self._pending_job_finalizations and len(batch) < max(1, int(batch_limit or 1)):
|
||
batch.append(self._pending_job_finalizations.popleft())
|
||
updated_count = 0
|
||
flush_started_at = time.perf_counter()
|
||
try:
|
||
updated_count = int(self.db.finalize_detect_job_items_batch(batch) or 0)
|
||
except Exception as e:
|
||
logger.debug(f"批量回写任务项完成态失败: {e}")
|
||
updated_count = -1
|
||
if updated_count < 0:
|
||
with self._job_finalize_lock:
|
||
for item in reversed(batch):
|
||
self._pending_job_finalizations.appendleft(item)
|
||
logger.warning(
|
||
f"批量回写任务项完成态失败，已回退待重试队列: batch={len(batch)} | force={1 if force else 0}"
|
||
)
|
||
updated_count = 0
|
||
flush_elapsed_ms = int((time.perf_counter() - flush_started_at) * 1000)
|
||
if batch and (flush_elapsed_ms >= 20 or len(batch) >= 64 or force):
|
||
logger.info(
|
||
f"任务项完成态批量回写: batch={len(batch)} | updated={updated_count} "
|
||
f"| elapsed_ms={flush_elapsed_ms} | force={1 if force else 0}"
|
||
)
|
||
self._last_job_finalize_flush_at = now_ts
|
||
return updated_count
|
||
|
||
def _enqueue_domain_status_update(self, domain_id, status):
|
||
try:
|
||
normalized_domain_id = int(domain_id or 0)
|
||
normalized_status = int(status)
|
||
except Exception:
|
||
return
|
||
if normalized_domain_id <= 0:
|
||
return
|
||
with self._domain_status_update_lock:
|
||
self._pending_domain_status_updates.append((normalized_domain_id, normalized_status))
|
||
|
||
def _enqueue_domain_completion(self, domain_id, *, register_status, use_status, expire_date):
|
||
try:
|
||
normalized_domain_id = int(domain_id or 0)
|
||
normalized_register_status = int(register_status or 0)
|
||
normalized_use_status = int(use_status or 0)
|
||
except Exception:
|
||
return
|
||
if normalized_domain_id <= 0:
|
||
return
|
||
with self._domain_completion_lock:
|
||
self._pending_domain_completions.append(
|
||
(
|
||
normalized_domain_id,
|
||
normalized_register_status,
|
||
normalized_use_status,
|
||
bool(expire_date),
|
||
)
|
||
)
|
||
|
||
def _flush_pending_domain_completions(self, *, force=False, batch_limit=512):
|
||
now_ts = time.time()
|
||
if not force and now_ts - float(getattr(self, "_last_domain_completion_flush_at", 0.0) or 0.0) < 0.05:
|
||
return 0
|
||
batch = []
|
||
with self._domain_completion_lock:
|
||
if not self._pending_domain_completions:
|
||
if force:
|
||
self._last_domain_completion_flush_at = now_ts
|
||
return 0
|
||
while self._pending_domain_completions and len(batch) < max(1, int(batch_limit or 1)):
|
||
batch.append(self._pending_domain_completions.popleft())
|
||
updated_count = 0
|
||
flush_started_at = time.perf_counter()
|
||
try:
|
||
updated_count = int(self.db.complete_domain_detection_batch(batch) or 0)
|
||
except Exception as e:
|
||
logger.debug(f"批量刷新域名完成态失败: {e}")
|
||
updated_count = -1
|
||
if updated_count < 0:
|
||
with self._domain_completion_lock:
|
||
for item in reversed(batch):
|
||
self._pending_domain_completions.appendleft(item)
|
||
logger.warning(
|
||
f"批量刷新域名完成态失败，已回退待重试队列: batch={len(batch)} | force={1 if force else 0}"
|
||
)
|
||
updated_count = 0
|
||
flush_elapsed_ms = int((time.perf_counter() - flush_started_at) * 1000)
|
||
if batch and (flush_elapsed_ms >= 20 or len(batch) >= 64 or force):
|
||
logger.info(
|
||
f"域名完成态批量回写: batch={len(batch)} | updated={updated_count} "
|
||
f"| elapsed_ms={flush_elapsed_ms} | force={1 if force else 0}"
|
||
)
|
||
self._last_domain_completion_flush_at = now_ts
|
||
return updated_count
|
||
|
||
def _flush_pending_domain_status_updates(self, *, force=False, batch_limit=512):
|
||
now_ts = time.time()
|
||
if not force and now_ts - float(getattr(self, "_last_domain_status_flush_at", 0.0) or 0.0) < 0.05:
|
||
return 0
|
||
batch = []
|
||
with self._domain_status_update_lock:
|
||
if not self._pending_domain_status_updates:
|
||
if force:
|
||
self._last_domain_status_flush_at = now_ts
|
||
return 0
|
||
while self._pending_domain_status_updates and len(batch) < max(1, int(batch_limit or 1)):
|
||
batch.append(self._pending_domain_status_updates.popleft())
|
||
updated_count = 0
|
||
try:
|
||
updated_count = int(self.db.update_domain_detect_status_batch(batch) or 0)
|
||
except Exception as e:
|
||
logger.debug(f"批量刷新域名检测状态失败: {e}")
|
||
updated_count = -1
|
||
if updated_count < 0:
|
||
with self._domain_status_update_lock:
|
||
for item in reversed(batch):
|
||
self._pending_domain_status_updates.appendleft(item)
|
||
logger.warning(
|
||
f"批量刷新域名检测状态失败，已回退待重试队列: batch={len(batch)} | force={1 if force else 0}"
|
||
)
|
||
updated_count = 0
|
||
self._last_domain_status_flush_at = now_ts
|
||
return updated_count
|
||
|
||
def _enqueue_review_status_update(self, domain_id, review_status):
|
||
try:
|
||
normalized_domain_id = int(domain_id or 0)
|
||
normalized_review_status = int(review_status)
|
||
except Exception:
|
||
return
|
||
if normalized_domain_id <= 0:
|
||
return
|
||
with self._review_status_update_lock:
|
||
self._pending_review_status_updates.append((normalized_domain_id, normalized_review_status))
|
||
|
||
def _flush_pending_review_status_updates(self, *, force=False, batch_limit=512):
|
||
now_ts = time.time()
|
||
if not force and now_ts - float(getattr(self, "_last_review_status_flush_at", 0.0) or 0.0) < 0.05:
|
||
return 0
|
||
batch = []
|
||
with self._review_status_update_lock:
|
||
if not self._pending_review_status_updates:
|
||
if force:
|
||
self._last_review_status_flush_at = now_ts
|
||
return 0
|
||
while self._pending_review_status_updates and len(batch) < max(1, int(batch_limit or 1)):
|
||
batch.append(self._pending_review_status_updates.popleft())
|
||
updated_count = 0
|
||
try:
|
||
updated_count = int(self.db.update_domain_review_status_batch(batch) or 0)
|
||
except Exception as e:
|
||
logger.debug(f"批量刷新域名复核状态失败: {e}")
|
||
updated_count = -1
|
||
if updated_count < 0:
|
||
with self._review_status_update_lock:
|
||
for item in reversed(batch):
|
||
self._pending_review_status_updates.appendleft(item)
|
||
logger.warning(
|
||
f"批量刷新域名复核状态失败，已回退待重试队列: batch={len(batch)} | force={1 if force else 0}"
|
||
)
|
||
updated_count = 0
|
||
self._last_review_status_flush_at = now_ts
|
||
return updated_count
|
||
|
||
def _enqueue_completed_future(self, future):
|
||
if future is None:
|
||
return
|
||
with self._completed_future_lock:
|
||
self._completed_futures.append(future)
|
||
|
||
def _consume_pending_control_command(self):
|
||
if not self.use_redis or self.redis_client is None:
|
||
return
|
||
pending_keys = [pending_control_key()]
|
||
if pending_keys[0] != PENDING_CONTROL_KEY:
|
||
pending_keys.append(PENDING_CONTROL_KEY)
|
||
|
||
payload = None
|
||
payload_key = ""
|
||
try:
|
||
for key in pending_keys:
|
||
payload = self.redis_client.getdel(key)
|
||
if payload:
|
||
payload_key = key
|
||
break
|
||
except Exception:
|
||
try:
|
||
for key in pending_keys:
|
||
payload = self.redis_client.get(key)
|
||
if payload:
|
||
self.redis_client.delete(key)
|
||
payload_key = key
|
||
break
|
||
except Exception as e:
|
||
logger.debug(f"读取待执行控制指令失败: {e}")
|
||
return
|
||
if not payload:
|
||
return
|
||
logger.info(f"发现待执行 Worker 控制指令: key={payload_key or 'unknown'} payload={payload}")
|
||
self._handle_control_message(payload)
|
||
|
||
def _active_detect_job_refresh_interval_seconds(self, reason: str) -> float:
|
||
if reason == "heartbeat_autoresume":
|
||
raw_value = os.getenv("DOMAINCHECK_AUTORESUME_ACTIVE_JOB_MIN_REFRESH_SECONDS", "5")
|
||
else:
|
||
raw_value = os.getenv("DOMAINCHECK_ACTIVE_JOB_MIN_REFRESH_SECONDS", "1")
|
||
try:
|
||
return max(0.0, float(raw_value or 0.0))
|
||
except (TypeError, ValueError):
|
||
return 5.0 if reason == "heartbeat_autoresume" else 1.0
|
||
|
||
def _load_active_detect_job_snapshot(self, reason: str, now_ts: float):
|
||
refresh_interval = self._active_detect_job_refresh_interval_seconds(reason)
|
||
cached_job = getattr(self, "_cached_active_detect_job", None)
|
||
last_refresh_at = float(getattr(self, "_last_active_detect_job_refresh_at", 0.0) or 0.0)
|
||
if refresh_interval > 0 and cached_job is not None and now_ts - last_refresh_at < refresh_interval:
|
||
return dict(cached_job or {})
|
||
active_job = self.db.get_active_detect_job() or {}
|
||
self._cached_active_detect_job = dict(active_job or {})
|
||
self._last_active_detect_job_refresh_at = now_ts
|
||
return dict(active_job or {})
|
||
|
||
def _autoresume_same_job_restart_grace_seconds(self) -> float:
|
||
return max(
|
||
5.0,
|
||
float(os.getenv("DOMAINCHECK_AUTORESUME_SAME_JOB_RESTART_GRACE_SECONDS", "30") or 30),
|
||
)
|
||
|
||
def _bootstrap_autoresume_lock_ttl_seconds(self) -> int:
|
||
return max(
|
||
5,
|
||
int(os.getenv("DOMAINCHECK_BOOTSTRAP_AUTORESUME_LOCK_TTL_SECONDS", "30") or 30),
|
||
)
|
||
|
||
def _acquire_bootstrap_autoresume_lock(self, *, job_id: int, job_code: str = "") -> bool:
|
||
if not self.use_redis or self.redis_client is None:
|
||
return True
|
||
try:
|
||
normalized_job_id = int(job_id or 0)
|
||
except Exception:
|
||
normalized_job_id = 0
|
||
if normalized_job_id <= 0:
|
||
return True
|
||
lock_action = f"bootstrap-autoresume:{normalized_job_id}:{str(job_code or '').strip() or '-'}"
|
||
lock_key = self._shared_job_maintenance_lock_key(lock_action)
|
||
if not lock_key:
|
||
return True
|
||
owner_token = self._job_maintenance_owner_token(lock_action)
|
||
try:
|
||
acquired = self.redis_client.set(
|
||
lock_key,
|
||
owner_token,
|
||
nx=True,
|
||
ex=self._bootstrap_autoresume_lock_ttl_seconds(),
|
||
)
|
||
return bool(acquired)
|
||
except Exception as exc:
|
||
logger.debug(f"获取 bootstrap 自动回挂锁失败: action={lock_action}, error={exc}")
|
||
return True
|
||
|
||
def _should_skip_same_job_autoresume(self, *, active_job: dict, now_ts: float) -> tuple[bool, str]:
|
||
incoming_job_code = str(active_job.get("job_code") or "").strip()
|
||
incoming_job_id = active_job.get("id")
|
||
try:
|
||
normalized_incoming_job_id = int(incoming_job_id or 0)
|
||
except Exception:
|
||
normalized_incoming_job_id = 0
|
||
|
||
current_job_code = str(getattr(self, "current_job_code", "") or "").strip()
|
||
current_job_id = getattr(self, "current_job_id", None)
|
||
try:
|
||
normalized_current_job_id = int(current_job_id or 0)
|
||
except Exception:
|
||
normalized_current_job_id = 0
|
||
|
||
same_job = bool(
|
||
incoming_job_code
|
||
and current_job_code
|
||
and incoming_job_code == current_job_code
|
||
) or bool(
|
||
normalized_incoming_job_id > 0
|
||
and normalized_current_job_id > 0
|
||
and normalized_incoming_job_id == normalized_current_job_id
|
||
)
|
||
if not same_job:
|
||
return False, ""
|
||
|
||
live_active_threads = max(0, int(self._get_active_domain_threads() or 0))
|
||
if live_active_threads > 0:
|
||
return True, f"same_job_threads_active:{live_active_threads}"
|
||
|
||
handoff_job_id = self._restart_release_handoff_job_id_active()
|
||
if (
|
||
handoff_job_id not in (None, "", 0, "0")
|
||
and normalized_incoming_job_id > 0
|
||
and int(handoff_job_id) == normalized_incoming_job_id
|
||
):
|
||
return True, "same_job_restart_handoff_active"
|
||
|
||
recent_start_at = float(getattr(self, "_last_detection_start_at", 0.0) or 0.0)
|
||
recent_start_grace_seconds = self._autoresume_same_job_restart_grace_seconds()
|
||
recent_start_elapsed = max(0.0, now_ts - recent_start_at) if recent_start_at > 0 else 0.0
|
||
if recent_start_at > 0 and recent_start_elapsed < recent_start_grace_seconds:
|
||
return True, f"same_job_recent_start:{int(recent_start_elapsed)}s"
|
||
|
||
return False, ""
|
||
|
||
def _should_skip_explicit_scope_autoresume(self, *, active_job: dict) -> tuple[bool, str]:
|
||
explicit_job_id = self._explicit_claim_scope_job_id_active()
|
||
if explicit_job_id in (None, "", 0, "0"):
|
||
return False, ""
|
||
try:
|
||
normalized_explicit_job_id = int(explicit_job_id or 0)
|
||
except Exception:
|
||
return False, ""
|
||
try:
|
||
normalized_incoming_job_id = int(active_job.get("id") or 0)
|
||
except Exception:
|
||
normalized_incoming_job_id = 0
|
||
if normalized_explicit_job_id <= 0 or normalized_incoming_job_id <= 0:
|
||
return False, ""
|
||
if normalized_explicit_job_id != normalized_incoming_job_id:
|
||
return False, ""
|
||
explicit_job_code = str(getattr(self, "_explicit_claim_scope_job_code", "") or "").strip()
|
||
explicit_source = str(getattr(self, "_explicit_claim_scope_source", "") or "").strip() or "unknown"
|
||
detail_parts = [f"explicit_scope_active:{normalized_explicit_job_id}"]
|
||
if explicit_job_code:
|
||
detail_parts.append(explicit_job_code)
|
||
detail_parts.append(explicit_source)
|
||
return True, ":".join(detail_parts)
|
||
|
||
def _ignored_targeted_job_window_seconds(self) -> float:
|
||
return max(
|
||
30.0,
|
||
float(os.getenv("DOMAINCHECK_IGNORED_TARGETED_JOB_WINDOW_SECONDS", "180") or 180),
|
||
)
|
||
|
||
def _remember_ignored_targeted_control(self, control_payload=None) -> None:
|
||
if not isinstance(control_payload, dict):
|
||
return
|
||
if control_targets_current_worker(control_payload):
|
||
return
|
||
target_node_codes = control_target_node_codes(control_payload)
|
||
if not target_node_codes:
|
||
return
|
||
job_id = self._incoming_job_id(control_payload)
|
||
job_code = self._incoming_job_code(control_payload)
|
||
if job_id in (None, "", 0, "0") or not job_code:
|
||
return
|
||
self._ignored_targeted_job_id = int(job_id)
|
||
self._ignored_targeted_job_code = str(job_code or "").strip()
|
||
self._ignored_targeted_job_until = time.time() + self._ignored_targeted_job_window_seconds()
|
||
self._ignored_targeted_job_targets = ",".join(target_node_codes)
|
||
|
||
def _clear_ignored_targeted_job(self, *, reason: str = "") -> None:
|
||
self._ignored_targeted_job_id = None
|
||
self._ignored_targeted_job_code = ""
|
||
self._ignored_targeted_job_until = 0.0
|
||
self._ignored_targeted_job_targets = ""
|
||
|
||
def _should_skip_ignored_targeted_job_autoresume(self, *, active_job: dict) -> tuple[bool, str]:
|
||
ignored_job_id = getattr(self, "_ignored_targeted_job_id", None)
|
||
if ignored_job_id in (None, "", 0, "0"):
|
||
return False, ""
|
||
if time.time() >= float(getattr(self, "_ignored_targeted_job_until", 0.0) or 0.0):
|
||
self._clear_ignored_targeted_job(reason="expired")
|
||
return False, ""
|
||
try:
|
||
active_job_id = int(active_job.get("id") or 0)
|
||
except Exception:
|
||
active_job_id = 0
|
||
try:
|
||
normalized_ignored_job_id = int(ignored_job_id or 0)
|
||
except Exception:
|
||
normalized_ignored_job_id = 0
|
||
if active_job_id <= 0 or normalized_ignored_job_id <= 0:
|
||
return False, ""
|
||
if active_job_id != normalized_ignored_job_id:
|
||
return False, ""
|
||
ignored_job_code = str(getattr(self, "_ignored_targeted_job_code", "") or "").strip()
|
||
ignored_targets = str(getattr(self, "_ignored_targeted_job_targets", "") or "").strip() or "unknown"
|
||
detail_parts = [f"ignored_targeted_job:{normalized_ignored_job_id}"]
|
||
if ignored_job_code:
|
||
detail_parts.append(ignored_job_code)
|
||
detail_parts.append(ignored_targets)
|
||
return True, ":".join(detail_parts)
|
||
|
||
def _resume_active_detect_job_if_needed(self, reason: str = "worker_bootstrap"):
|
||
now_ts = time.time()
|
||
stale_detect = False
|
||
autoresume_reasons = {
|
||
"heartbeat_autoresume",
|
||
"service_runtime_bootstrap",
|
||
"redis_subscription_bootstrap",
|
||
}
|
||
node_role = str(getattr(config, "NODE_ROLE", "") or "").strip()
|
||
is_primary_worker = self._is_primary_job_maintenance_worker()
|
||
tail_handoff_probe_enabled = self._runtime_bool_override(
|
||
"tail_handoff_autoresume_enabled",
|
||
"DOMAINCHECK_TAIL_HANDOFF_AUTORESUME",
|
||
default=True,
|
||
)
|
||
|
||
def log_skip(detail: str):
|
||
if now_ts - float(getattr(self, "_last_autoresume_skip_log_at", 0.0) or 0.0) >= 15.0:
|
||
logger.info(f"自动回挂跳过: reason={reason}, detail={detail}")
|
||
self._last_autoresume_skip_log_at = now_ts
|
||
|
||
if (
|
||
node_role == "control"
|
||
and not self._runtime_bool_override(
|
||
"control_node_autoresume_enabled",
|
||
"DOMAINCHECK_CONTROL_NODE_AUTORESUME",
|
||
default=False,
|
||
)
|
||
):
|
||
log_skip("control_node_autoresume_disabled")
|
||
return False
|
||
if (
|
||
reason in autoresume_reasons
|
||
and node_role != "control"
|
||
and not is_primary_worker
|
||
and not tail_handoff_probe_enabled
|
||
):
|
||
log_skip("non_primary_autoresume_worker")
|
||
return False
|
||
if not self.running:
|
||
log_skip("worker_not_running")
|
||
return False
|
||
if self._consume_pending_restart_request(trigger_reason=reason):
|
||
return True
|
||
if self.stop_requested:
|
||
log_skip("stop_requested")
|
||
return False
|
||
explicit_start_grace_seconds = max(
|
||
15.0,
|
||
float(os.getenv("DOMAINCHECK_AUTORESUME_EXPLICIT_START_GRACE_SECONDS", "120") or 120),
|
||
)
|
||
last_explicit_start_at = float(getattr(self, "_last_explicit_start_signal_at", 0.0) or 0.0)
|
||
if (
|
||
last_explicit_start_at > 0
|
||
and now_ts - last_explicit_start_at < explicit_start_grace_seconds
|
||
):
|
||
log_skip(
|
||
"recent_explicit_start_signal"
|
||
)
|
||
return False
|
||
if self.detecting:
|
||
stale_detect, stale_reason = self._is_stale_detect_session()
|
||
if not stale_detect:
|
||
log_skip(stale_reason or "already_detecting")
|
||
return False
|
||
logger.warning(f"检测运行态疑似空转，允许自动回挂接管: {stale_reason}")
|
||
thread = self.detect_command_thread
|
||
if thread and thread.is_alive() and not stale_detect:
|
||
log_skip("detect_command_thread_alive")
|
||
return False
|
||
try:
|
||
active_job = self._load_active_detect_job_snapshot(reason, now_ts)
|
||
except Exception as e:
|
||
logger.debug(f"读取当前检测任务失败，跳过自动回挂: {e}")
|
||
return False
|
||
job_id = active_job.get("id")
|
||
job_code = str(active_job.get("job_code") or "").strip()
|
||
task_mode = str(active_job.get("task_mode") or "").strip()
|
||
if not job_id or not job_code:
|
||
log_skip("active_job_missing_id_or_code")
|
||
return False
|
||
items_pending = int(active_job.get("items_pending", 0) or 0)
|
||
items_claimed = int(active_job.get("items_claimed", 0) or 0)
|
||
items_running = int(active_job.get("items_running", 0) or 0)
|
||
selection_reason = str(active_job.get("selection_reason") or "").strip()
|
||
if selection_reason == "running_job_stalled" or bool(active_job.get("running_job_stalled")):
|
||
self._maybe_recycle_stalled_job_items(
|
||
job_id,
|
||
reason=f"autoresume:{reason or 'unspecified'}",
|
||
)
|
||
log_skip(
|
||
"running_job_stalled"
|
||
)
|
||
return False
|
||
tail_handoff_candidate = bool(active_job.get("tail_handoff_candidate")) or selection_reason == "tail_handoff_pending"
|
||
if items_pending <= 0 and items_claimed <= 0 and items_running <= 0:
|
||
log_skip("active_job_empty")
|
||
return False
|
||
if reason == "heartbeat_autoresume":
|
||
should_skip_targeted_job, targeted_job_detail = self._should_skip_ignored_targeted_job_autoresume(
|
||
active_job=active_job,
|
||
)
|
||
if should_skip_targeted_job:
|
||
log_skip(targeted_job_detail or "ignored_targeted_job")
|
||
return False
|
||
should_skip_explicit_scope, explicit_scope_detail = self._should_skip_explicit_scope_autoresume(
|
||
active_job=active_job,
|
||
)
|
||
if should_skip_explicit_scope:
|
||
log_skip(explicit_scope_detail or "explicit_scope_autoresume_guard")
|
||
return False
|
||
should_skip_same_job, skip_detail = self._should_skip_same_job_autoresume(
|
||
active_job=active_job,
|
||
now_ts=now_ts,
|
||
)
|
||
if should_skip_same_job:
|
||
log_skip(skip_detail or "same_job_autoresume_guard")
|
||
return False
|
||
if reason in {"service_runtime_bootstrap", "redis_subscription_bootstrap"}:
|
||
if not self._acquire_bootstrap_autoresume_lock(job_id=int(job_id), job_code=job_code):
|
||
log_skip("bootstrap_autoresume_lock_held")
|
||
return False
|
||
if (
|
||
reason in autoresume_reasons
|
||
and node_role != "control"
|
||
and not is_primary_worker
|
||
and not tail_handoff_candidate
|
||
):
|
||
log_skip("non_primary_autoresume_worker")
|
||
return False
|
||
logger.warning(
|
||
f"检测 Worker 空闲但发现活动任务，准备自动回挂: "
|
||
f"job_code={job_code}, pending={items_pending}, claimed={items_claimed}, "
|
||
f"running={items_running}, reason={reason}, selection={selection_reason or '-'}"
|
||
)
|
||
started = self.start_detection_async(
|
||
source="auto-resume",
|
||
control_payload={
|
||
"source": reason,
|
||
"job_id": int(job_id),
|
||
"job_code": job_code,
|
||
"task_mode": task_mode,
|
||
"target_job_id": int(job_id),
|
||
"target_job_code": job_code,
|
||
"target_task_mode": task_mode,
|
||
"selection_reason": selection_reason,
|
||
"tail_handoff_candidate": tail_handoff_candidate,
|
||
},
|
||
)
|
||
if started and now_ts - float(getattr(self, "_last_autoresume_success_log_at", 0.0) or 0.0) >= 5.0:
|
||
logger.warning(
|
||
f"自动回挂已触发: reason={reason}, job_code={job_code}, "
|
||
f"pending={items_pending}, claimed={items_claimed}, running={items_running}, "
|
||
f"selection={selection_reason or '-'}"
|
||
)
|
||
self._last_autoresume_success_log_at = now_ts
|
||
return started
|
||
|
||
def activate_service_runtime(self):
|
||
if not self.running:
|
||
self.running = True
|
||
logger.info(
|
||
f"激活 Worker 服务态运行时: node={config.NODE_CODE}, "
|
||
f"thread_count={self.thread_count}, proxy_enabled={1 if self.proxy_config.get('proxy_enable', False) else 0}"
|
||
)
|
||
if self.use_redis:
|
||
thread = getattr(self, "redis_sub_thread", None)
|
||
if not thread or not thread.is_alive():
|
||
self.redis_sub_thread = threading.Thread(
|
||
target=self.start_redis_subscription,
|
||
name="RedisSubscriptionThread",
|
||
daemon=True,
|
||
)
|
||
self.redis_sub_thread.start()
|
||
logger.info("Redis 配置/控制订阅线程已启动")
|
||
try:
|
||
self._consume_pending_control_command()
|
||
except Exception as e:
|
||
logger.warning(f"服务态启动时消费待执行控制指令失败: {e}")
|
||
try:
|
||
self._resume_active_detect_job_if_needed(reason="service_runtime_bootstrap")
|
||
except Exception as e:
|
||
logger.warning(f"服务态启动时自动回挂活动任务失败: {e}")
|
||
|
||
def _update_runtime_state(self, phase: str, detail: str, **extra):
|
||
"""
|
||
更新 Redis 中的检测运行态，供 Web 后台读取。
|
||
"""
|
||
volatile_keys = {
|
||
"active_threads",
|
||
"current_load",
|
||
"cycle_token",
|
||
"job_id",
|
||
"job_code",
|
||
}
|
||
previous_extra = dict(getattr(self, "_last_runtime_extra", {}) or {})
|
||
merged_extra = {key: value for key, value in previous_extra.items() if key not in volatile_keys}
|
||
merged_extra.update({key: value for key, value in dict(extra or {}).items() if key not in volatile_keys})
|
||
if "max_threads" not in merged_extra:
|
||
merged_extra["max_threads"] = int(getattr(self, "thread_count", 0) or 0)
|
||
with self._domain_thread_counter_lock:
|
||
live_active_threads = int(self._active_domain_threads or 0)
|
||
explicit_active_threads = (extra or {}).get("active_threads")
|
||
if not bool(self.detecting):
|
||
effective_active_threads = 0
|
||
elif explicit_active_threads in (None, ""):
|
||
effective_active_threads = live_active_threads
|
||
else:
|
||
effective_active_threads = max(int(explicit_active_threads or 0), live_active_threads)
|
||
merged_extra["active_threads"] = effective_active_threads
|
||
|
||
waiting_for_dispatch = bool(self.detecting and effective_active_threads <= 0)
|
||
merged_extra["waiting_for_dispatch"] = waiting_for_dispatch
|
||
explicit_current_load = (extra or {}).get("current_load")
|
||
default_current_load = effective_active_threads if effective_active_threads > 0 else 0
|
||
if explicit_current_load in (None, ""):
|
||
merged_extra["current_load"] = default_current_load
|
||
else:
|
||
merged_extra["current_load"] = max(int(explicit_current_load or 0), default_current_load)
|
||
self._last_runtime_phase = phase
|
||
self._last_runtime_detail = detail
|
||
self._last_runtime_extra = merged_extra
|
||
if not self.use_redis or self.redis_client is None:
|
||
return
|
||
with self.proxy_pool_lock:
|
||
available_proxy_count = len(self.proxy_pool)
|
||
payload_warning = ""
|
||
if self.proxy_config.get('proxy_enable', False) and available_proxy_count <= 0:
|
||
if (
|
||
self.proxy_last_refresh_status
|
||
and self.proxy_last_refresh_status not in {"未刷新", "代理未启用", "未配置代理池链接"}
|
||
and "刷新成功" not in self.proxy_last_refresh_status
|
||
):
|
||
payload_warning = self.proxy_last_refresh_status
|
||
payload = {
|
||
"node_code": config.NODE_CODE,
|
||
"phase": phase,
|
||
"detail": detail,
|
||
"service_running": bool(self.running),
|
||
"detecting": bool(self.detecting),
|
||
"stop_requested": bool(self.stop_requested),
|
||
"available_proxy_count": available_proxy_count,
|
||
"proxy_last_refresh_status": self.proxy_last_refresh_status,
|
||
"proxy_last_refresh_time": self.proxy_last_refresh_time.strftime("%Y-%m-%d %H:%M:%S") if self.proxy_last_refresh_time else "",
|
||
"proxy_last_refresh_source_count": int(self.proxy_last_refresh_source_count or 0),
|
||
"proxy_last_refresh_total_items": int(self.proxy_last_refresh_total_items or 0),
|
||
"proxy_last_validated_count": int(self.proxy_last_validated_count or 0),
|
||
"proxy_last_available_count": int(self.proxy_last_available_count or 0),
|
||
"proxy_last_source_stats": list(self.proxy_last_source_stats or []),
|
||
"recent_warning": payload_warning,
|
||
"updated_at": datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
|
||
}
|
||
payload.update(merged_extra)
|
||
if self.current_cycle_token and not payload.get("cycle_token"):
|
||
payload["cycle_token"] = self.current_cycle_token
|
||
if self.current_job_id and not payload.get("job_id"):
|
||
payload["job_id"] = self.current_job_id
|
||
if self.current_job_code and not payload.get("job_code"):
|
||
payload["job_code"] = self.current_job_code
|
||
now_ts = time.time()
|
||
runtime_state_busy_phases = {"preparing", "refreshing_proxy", "fetching", "running", "completing"}
|
||
runtime_state_min_interval = max(
|
||
0.05,
|
||
float(os.getenv("DOMAINCHECK_RUNTIME_STATE_MIN_INTERVAL", "0.25") or 0.25),
|
||
)
|
||
last_push_phase = str(getattr(self, "_last_runtime_state_push_phase", "") or "")
|
||
last_push_at = float(getattr(self, "_last_runtime_state_push_at", 0.0) or 0.0)
|
||
if (
|
||
phase in runtime_state_busy_phases
|
||
and phase == last_push_phase
|
||
and now_ts - last_push_at < runtime_state_min_interval
|
||
):
|
||
return
|
||
try:
|
||
self.redis_client.set(runtime_state_key(), json.dumps(payload, ensure_ascii=False))
|
||
except Exception as e:
|
||
logger.debug(f"写入检测运行态失败: {e}")
|
||
try:
|
||
actual_active_threads = int(payload.get("active_threads", 0) or 0)
|
||
max_threads = int(payload.get("max_threads", getattr(self, "thread_count", 0)) or 0)
|
||
current_load = actual_active_threads
|
||
self.db.register_cluster_node(
|
||
config.NODE_CODE,
|
||
config.NODE_REGION,
|
||
config.NODE_ROLE,
|
||
status="busy" if payload.get("detecting") else "online",
|
||
current_load=current_load,
|
||
metadata={
|
||
"phase": phase,
|
||
"detail": detail,
|
||
"detecting": bool(payload.get("detecting")),
|
||
"waiting_for_dispatch": bool(payload.get("waiting_for_dispatch")),
|
||
"available_proxy_count": int(payload.get("available_proxy_count", 0) or 0),
|
||
"active_threads": actual_active_threads,
|
||
"max_threads": max_threads,
|
||
"cycle_token": str(payload.get("cycle_token") or ""),
|
||
"job_id": payload.get("job_id"),
|
||
"job_code": str(payload.get("job_code") or ""),
|
||
"proxy_runtime": {
|
||
"sources": int(payload.get("proxy_last_refresh_source_count", 0) or 0),
|
||
"raw_items": int(payload.get("proxy_last_refresh_total_items", 0) or 0),
|
||
"validated": int(payload.get("proxy_last_validated_count", 0) or 0),
|
||
"available": int(payload.get("proxy_last_available_count", 0) or 0),
|
||
},
|
||
},
|
||
)
|
||
except Exception as e:
|
||
logger.debug(f"更新检测节点心跳失败: {e}")
|
||
self._last_runtime_state_push_at = now_ts
|
||
self._last_runtime_state_push_phase = phase
|
||
|
||
def _mark_detection_phase(self, phase: str, detail: str, **extra):
|
||
if phase not in {"idle", "completed", "stopped"}:
|
||
self._note_detection_activity()
|
||
self._update_runtime_state(phase, detail, **extra)
|
||
|
||
def _change_active_domain_threads(self, delta: int):
|
||
with self._domain_thread_counter_lock:
|
||
self._active_domain_threads = max(0, int(self._active_domain_threads or 0) + int(delta))
|
||
if self._active_domain_threads > 0:
|
||
self._note_detection_activity()
|
||
return self._active_domain_threads
|
||
|
||
def _get_active_domain_threads(self):
|
||
with self._domain_thread_counter_lock:
|
||
return int(self._active_domain_threads or 0)
|
||
|
||
def _note_detection_activity(self):
|
||
now_ts = time.time()
|
||
self.detect_command_last_activity_at = now_ts
|
||
self._last_worker_activity_at = now_ts
|
||
|
||
def _note_domain_started(self):
|
||
now_ts = time.time()
|
||
self._last_domain_started_at = now_ts
|
||
self.detect_command_last_activity_at = now_ts
|
||
self._last_worker_activity_at = now_ts
|
||
|
||
def _note_domain_result(self):
|
||
now_ts = time.time()
|
||
self._last_domain_result_at = now_ts
|
||
self.detect_command_last_activity_at = now_ts
|
||
self._last_worker_activity_at = now_ts
|
||
|
||
def _idle_worker_recycle_enabled(self):
|
||
return self._env_flag_enabled("DOMAINCHECK_IDLE_WORKER_RECYCLE_ENABLED", default=False)
|
||
|
||
def _idle_worker_recycle_after_seconds(self):
|
||
try:
|
||
return max(30.0, float(os.getenv("DOMAINCHECK_IDLE_WORKER_RECYCLE_AFTER_SECONDS", "180") or 180.0))
|
||
except Exception:
|
||
return 180.0
|
||
|
||
def _idle_worker_recycle_min_uptime_seconds(self):
|
||
try:
|
||
return max(30.0, float(os.getenv("DOMAINCHECK_IDLE_WORKER_RECYCLE_MIN_UPTIME_SECONDS", "300") or 300.0))
|
||
except Exception:
|
||
return 300.0
|
||
|
||
def _idle_worker_recycle_explicit_start_grace_seconds(self):
|
||
try:
|
||
return max(10.0, float(os.getenv("DOMAINCHECK_IDLE_WORKER_RECYCLE_START_GRACE_SECONDS", "90") or 90.0))
|
||
except Exception:
|
||
return 90.0
|
||
|
||
def _idle_worker_recycle_jitter_seconds(self):
|
||
try:
|
||
return max(0.0, float(os.getenv("DOMAINCHECK_IDLE_WORKER_RECYCLE_JITTER_SECONDS", "45") or 45.0))
|
||
except Exception:
|
||
return 45.0
|
||
|
||
def _idle_worker_recycle_jitter_offset(self):
|
||
jitter_window = float(self._idle_worker_recycle_jitter_seconds() or 0.0)
|
||
if jitter_window <= 0:
|
||
return 0.0
|
||
node_code = str(getattr(config, "NODE_CODE", "") or "").strip() or str(os.getpid())
|
||
spread = sum(ord(ch) for ch in node_code) % 1000
|
||
return (spread / 1000.0) * jitter_window
|
||
|
||
def _is_recyclable_idle_worker_instance(self):
|
||
if str(os.getenv("WORKER_MODE", "") or "").strip() != "linux-systemd":
|
||
return False
|
||
parent_node_code = self._worker_parent_node_code()
|
||
node_code = str(getattr(config, "NODE_CODE", "") or "").strip()
|
||
if not parent_node_code or not node_code:
|
||
return False
|
||
return node_code != parent_node_code and node_code.startswith(f"{parent_node_code}-")
|
||
|
||
def _idle_worker_waiting_for_dispatch(self):
|
||
runtime_extra = dict(getattr(self, "_last_runtime_extra", {}) or {})
|
||
if bool(runtime_extra.get("waiting_for_dispatch", False)):
|
||
return True
|
||
return bool(self.detecting and self._get_active_domain_threads() <= 0)
|
||
|
||
def _idle_worker_seconds(self):
|
||
last_activity_at = max(
|
||
float(getattr(self, "_last_worker_activity_at", 0.0) or 0.0),
|
||
float(getattr(self, "detect_command_last_activity_at", 0.0) or 0.0),
|
||
float(getattr(self, "_last_domain_started_at", 0.0) or 0.0),
|
||
float(getattr(self, "_last_domain_result_at", 0.0) or 0.0),
|
||
)
|
||
if last_activity_at <= 0:
|
||
last_activity_at = float(getattr(self, "_worker_started_at", 0.0) or time.time())
|
||
return max(0.0, time.time() - last_activity_at)
|
||
|
||
def _trigger_idle_worker_recycle(self, *, trigger_reason: str, idle_seconds: float):
|
||
if self._idle_recycle_requested:
|
||
return False
|
||
self._idle_recycle_requested = True
|
||
detail = (
|
||
f"子 Worker 空闲过久，触发生命周期回收: idle={int(idle_seconds)}s, "
|
||
f"trigger={trigger_reason or 'unknown'}"
|
||
)
|
||
logger.warning(detail)
|
||
try:
|
||
self._sync_worker_log_event(
|
||
"空闲 Worker 生命周期回收",
|
||
level='warning',
|
||
payload={
|
||
"idle_seconds": int(idle_seconds),
|
||
"trigger_reason": str(trigger_reason or "").strip() or "unknown",
|
||
"node_code": str(getattr(config, "NODE_CODE", "") or "").strip(),
|
||
},
|
||
mode='key',
|
||
)
|
||
except Exception:
|
||
pass
|
||
try:
|
||
self._update_runtime_state(
|
||
"restarting",
|
||
detail,
|
||
active_threads=0,
|
||
current_load=0,
|
||
recycle_reason=str(trigger_reason or "").strip() or "idle",
|
||
)
|
||
except Exception:
|
||
pass
|
||
try:
|
||
self._flush_pending_running_marks(force=True, batch_limit=2048)
|
||
self._flush_pending_job_releases(force=True, batch_limit=2048)
|
||
self._flush_pending_job_finalizations(force=True, batch_limit=2048)
|
||
self._flush_pending_domain_status_updates(force=True, batch_limit=2048)
|
||
self._flush_pending_domain_completions(force=True, batch_limit=2048)
|
||
self._flush_pending_review_status_updates(force=True, batch_limit=2048)
|
||
self._flush_completed_futures(force=True, batch_limit=2048)
|
||
except Exception as exc:
|
||
logger.debug(f"空闲 Worker 回收前 flush 失败: {exc}")
|
||
self.stop_requested = True
|
||
self.detecting = False
|
||
self.running = False
|
||
self._runtime_heartbeat_stop.set()
|
||
self._worker_log_sync_stop.set()
|
||
try:
|
||
self._worker_log_sync_queue.put_nowait(None)
|
||
except Exception:
|
||
pass
|
||
time.sleep(0.2)
|
||
os.kill(os.getpid(), signal.SIGTERM)
|
||
return True
|
||
|
||
def _maybe_recycle_idle_worker(self, *, trigger_reason: str = ""):
|
||
if self._idle_recycle_requested:
|
||
return False
|
||
if not self._idle_worker_recycle_enabled():
|
||
return False
|
||
if not self._is_recyclable_idle_worker_instance():
|
||
return False
|
||
if not self.running or self.stop_requested:
|
||
return False
|
||
if self._get_active_domain_threads() > 0:
|
||
return False
|
||
if not self._idle_worker_waiting_for_dispatch():
|
||
return False
|
||
if time.time() - float(getattr(self, "_worker_started_at", 0.0) or 0.0) < self._idle_worker_recycle_min_uptime_seconds():
|
||
return False
|
||
explicit_start_at = float(getattr(self, "_last_explicit_start_signal_at", 0.0) or 0.0)
|
||
if explicit_start_at > 0 and time.time() - explicit_start_at < self._idle_worker_recycle_explicit_start_grace_seconds():
|
||
return False
|
||
idle_seconds = self._idle_worker_seconds()
|
||
recycle_after = self._idle_worker_recycle_after_seconds() + self._idle_worker_recycle_jitter_offset()
|
||
if idle_seconds < recycle_after:
|
||
return False
|
||
return self._trigger_idle_worker_recycle(
|
||
trigger_reason=trigger_reason or "idle_waiting_for_dispatch",
|
||
idle_seconds=idle_seconds,
|
||
)
|
||
|
||
def _schedule_pending_restart(self, source: str, control_payload=None, reason: str = ""):
|
||
self._pending_restart_source = str(source or "deferred-restart").strip() or "deferred-restart"
|
||
self._pending_restart_payload = dict(control_payload or {})
|
||
self._pending_restart_reason = str(reason or "").strip()
|
||
|
||
def _should_force_takeover_stale_session(self, stale_reason: str = "") -> bool:
|
||
thread = self.detect_command_thread
|
||
if not thread or not thread.is_alive():
|
||
return False
|
||
with self._domain_thread_counter_lock:
|
||
live_active_threads = int(self._active_domain_threads or 0)
|
||
if live_active_threads > 0:
|
||
return False
|
||
|
||
last_activity_at = max(
|
||
float(self.detect_command_last_activity_at or 0.0),
|
||
float(self.detect_command_started_at or 0.0),
|
||
float(getattr(self, "_last_domain_started_at", 0.0) or 0.0),
|
||
float(getattr(self, "_last_domain_result_at", 0.0) or 0.0),
|
||
)
|
||
idle_seconds = max(0.0, time.time() - last_activity_at) if last_activity_at > 0 else 0.0
|
||
thread_age_seconds = max(0.0, time.time() - float(self.detect_command_started_at or 0.0))
|
||
if idle_seconds >= 10 or thread_age_seconds >= 20:
|
||
return True
|
||
return bool(stale_reason)
|
||
|
||
def _consume_pending_restart_request(self, trigger_reason: str = ""):
|
||
with self.detect_lock:
|
||
if self.detect_command_thread and self.detect_command_thread.is_alive():
|
||
return False
|
||
source = str(self._pending_restart_source or "").strip()
|
||
payload = dict(self._pending_restart_payload or {})
|
||
pending_reason = str(self._pending_restart_reason or "").strip()
|
||
if not source and not payload:
|
||
return False
|
||
self._pending_restart_source = ""
|
||
self._pending_restart_payload = None
|
||
self._pending_restart_reason = ""
|
||
logger.warning(
|
||
"检测任务执行延期重启: "
|
||
f"trigger={trigger_reason or 'unknown'}, source={source or 'deferred-restart'}, "
|
||
f"detail={pending_reason or 'none'}"
|
||
)
|
||
return self.start_detection_async(source=source or "deferred-restart", control_payload=payload)
|
||
|
||
def _detection_session_abort_reason(self, session_id: int = 0) -> str:
|
||
if not self.running:
|
||
return "worker_stopped"
|
||
normalized_session_id = int(session_id or 0)
|
||
if normalized_session_id <= 0:
|
||
return ""
|
||
current_owner = int(getattr(self, "_detect_session_owner", 0) or 0)
|
||
if current_owner > 0 and current_owner != normalized_session_id:
|
||
return f"session_replaced:{current_owner}"
|
||
return ""
|
||
|
||
def _release_job_item_for_session_abort(self, job_item_id, claim_token, *, session_id: int = 0, domain_name: str = "", detail: str = ""):
|
||
abort_reason = self._detection_session_abort_reason(session_id)
|
||
if not abort_reason:
|
||
return False
|
||
try:
|
||
normalized_job_item_id = int(job_item_id or 0)
|
||
except Exception:
|
||
normalized_job_item_id = 0
|
||
normalized_claim_token = str(claim_token or "").strip()
|
||
if normalized_job_item_id <= 0 or not normalized_claim_token:
|
||
return False
|
||
normalized_detail = str(detail or "").strip()
|
||
release_reason = f"released after session abort: {abort_reason}"
|
||
if normalized_detail:
|
||
release_reason = f"{release_reason} | {normalized_detail[:240]}"
|
||
release_reason = release_reason[:1000]
|
||
released = self._enqueue_job_release(
|
||
normalized_job_item_id,
|
||
normalized_claim_token,
|
||
reason=release_reason,
|
||
)
|
||
if released:
|
||
pending_release_count = 0
|
||
with self._job_release_lock:
|
||
pending_release_count = len(self._pending_job_releases)
|
||
if pending_release_count >= 64:
|
||
self._flush_pending_job_releases(batch_limit=512)
|
||
logger.warning(
|
||
f"检测会话已切换，任务项已登记回队列: domain={domain_name or 'unknown'}, "
|
||
f"job_item_id={normalized_job_item_id}, reason={abort_reason}, "
|
||
f"detail={str(detail or '').strip() or 'none'}"
|
||
)
|
||
return released
|
||
|
||
def _assert_detection_session_active(self, session_id: int = 0, *, domain_name: str = "", stage: str = "") -> None:
|
||
abort_reason = self._detection_session_abort_reason(session_id)
|
||
if not abort_reason:
|
||
return
|
||
normalized_stage = str(stage or "unknown").strip() or "unknown"
|
||
normalized_domain_name = str(domain_name or "").strip()
|
||
detail = f"检测会话已失效: reason={abort_reason} stage={normalized_stage}"
|
||
if normalized_domain_name:
|
||
detail = f"{detail} domain={normalized_domain_name}"
|
||
raise RuntimeError(detail)
|
||
|
||
def _incoming_job_code(self, control_payload=None) -> str:
|
||
payload = control_payload or {}
|
||
job_code = str(payload.get("job_code") or "").strip()
|
||
if not job_code:
|
||
job_code = str(payload.get("target_job_code") or "").strip()
|
||
return job_code
|
||
|
||
def _incoming_cycle_token(self, control_payload=None) -> str:
|
||
payload = control_payload or {}
|
||
return str(payload.get("cycle_token") or "").strip()
|
||
|
||
def _incoming_control_source(self, control_payload=None) -> str:
|
||
payload = control_payload or {}
|
||
return str(payload.get("source") or "").strip()
|
||
|
||
def _incoming_job_id(self, control_payload=None):
|
||
payload = control_payload or {}
|
||
# For domain_pipeline sync-pull commands, runtime/log identity may follow
|
||
# the upstream projection job_code/job_id, but local queue ownership must
|
||
# stay pinned to the ingested target_job_id. Claim/release/session scope
|
||
# logic therefore prefers target_job_id when it exists.
|
||
job_id = payload.get("target_job_id")
|
||
if job_id in (None, "", 0, "0"):
|
||
job_id = payload.get("job_id")
|
||
try:
|
||
return int(job_id) if job_id not in (None, "", 0, "0") else None
|
||
except Exception:
|
||
return None
|
||
|
||
def _is_overlap_handoff_command(self, control_payload=None, *, source: str = "") -> bool:
|
||
payload = control_payload or {}
|
||
normalized_source = str(source or self._incoming_control_source(payload) or "").strip()
|
||
if normalized_source == "overlap-handoff":
|
||
return True
|
||
return bool(payload.get("tail_handoff_candidate"))
|
||
|
||
def _matches_active_detect_session(self, control_payload=None) -> bool:
|
||
payload = control_payload or {}
|
||
incoming_job_code = self._incoming_job_code(payload)
|
||
incoming_job_id = self._incoming_job_id(payload)
|
||
incoming_cycle_token = self._incoming_cycle_token(payload)
|
||
|
||
current_job_code = str(self.current_job_code or "").strip()
|
||
current_cycle_token = str(self.current_cycle_token or "").strip()
|
||
current_job_id = getattr(self, "current_job_id", None)
|
||
|
||
matches_job_code = bool(
|
||
incoming_job_code and current_job_code and incoming_job_code == current_job_code
|
||
)
|
||
matches_job_id = bool(
|
||
incoming_job_id not in (None, 0)
|
||
and current_job_id not in (None, 0)
|
||
and int(incoming_job_id) == int(current_job_id)
|
||
)
|
||
if not (matches_job_code or matches_job_id):
|
||
return False
|
||
if incoming_cycle_token and current_cycle_token and incoming_cycle_token != current_cycle_token:
|
||
return False
|
||
return True
|
||
|
||
def _is_stale_detect_session(self, control_payload=None):
|
||
thread = self.detect_command_thread
|
||
if not thread or not thread.is_alive():
|
||
return False, ""
|
||
|
||
live_active_threads = 0
|
||
with self._domain_thread_counter_lock:
|
||
live_active_threads = int(self._active_domain_threads or 0)
|
||
|
||
last_activity_at = max(
|
||
float(self.detect_command_last_activity_at or 0.0),
|
||
float(self.detect_command_started_at or 0.0),
|
||
float(getattr(self, "_last_domain_started_at", 0.0) or 0.0),
|
||
float(getattr(self, "_last_domain_result_at", 0.0) or 0.0),
|
||
)
|
||
if last_activity_at <= 0:
|
||
return False, ""
|
||
|
||
idle_seconds = max(0.0, time.time() - last_activity_at)
|
||
thread_age_seconds = max(0.0, time.time() - float(self.detect_command_started_at or 0.0))
|
||
last_result_at = float(getattr(self, "_last_domain_result_at", 0.0) or 0.0)
|
||
result_idle_seconds = max(0.0, time.time() - last_result_at) if last_result_at > 0 else idle_seconds
|
||
incoming_job_code = self._incoming_job_code(control_payload)
|
||
incoming_cycle_token = self._incoming_cycle_token(control_payload)
|
||
incoming_task_mode = str(
|
||
(control_payload or {}).get("task_mode")
|
||
or (control_payload or {}).get("target_task_mode")
|
||
or ""
|
||
).strip()
|
||
current_job_code = str(self.current_job_code or "").strip()
|
||
current_cycle_token = str(self.current_cycle_token or "").strip()
|
||
if self._matches_active_detect_session(control_payload):
|
||
if idle_seconds < 180 and result_idle_seconds < 180:
|
||
return False, (
|
||
"同任务同周期重复唤起已忽略: "
|
||
f"job={incoming_job_code or current_job_code or 'unknown'} "
|
||
f"cycle={incoming_cycle_token or current_cycle_token or 'none'}"
|
||
)
|
||
session_scoped_to_job = bool(
|
||
self._is_single_step_session_active()
|
||
or incoming_task_mode == "single_step"
|
||
)
|
||
job_switched = bool(
|
||
session_scoped_to_job
|
||
and incoming_job_code
|
||
and current_job_code
|
||
and incoming_job_code != current_job_code
|
||
)
|
||
incoming_job_changed = bool(
|
||
session_scoped_to_job
|
||
and incoming_job_code
|
||
and incoming_job_code != current_job_code
|
||
)
|
||
low_activity_recoverable = live_active_threads <= 1
|
||
multi_thread_force_recover = incoming_job_changed and thread_age_seconds >= 30 and result_idle_seconds >= 30
|
||
if not low_activity_recoverable and not multi_thread_force_recover:
|
||
return False, ""
|
||
|
||
hold_for_proxy_shortage, hold_reason = self._should_hold_stale_detect_for_proxy_shortage(
|
||
idle_seconds=idle_seconds,
|
||
low_activity_recoverable=low_activity_recoverable,
|
||
incoming_job_changed=job_switched or incoming_job_changed,
|
||
)
|
||
if hold_for_proxy_shortage:
|
||
return False, hold_reason
|
||
|
||
if job_switched and idle_seconds >= 15:
|
||
return True, (
|
||
f"检测指令线程仍停留在旧任务 {current_job_code}，"
|
||
f"但新任务已切换为 {incoming_job_code}，且已空转 {int(idle_seconds)} 秒"
|
||
)
|
||
if incoming_job_changed and idle_seconds >= 20:
|
||
return True, (
|
||
f"检测指令线程未挂载新任务 {incoming_job_code}，"
|
||
f"当前任务上下文为 {current_job_code or 'empty'}，且已空转 {int(idle_seconds)} 秒"
|
||
)
|
||
if multi_thread_force_recover:
|
||
return True, (
|
||
f"检测指令线程已运行 {int(thread_age_seconds)} 秒，最近 {int(result_idle_seconds)} 秒无域名产出，"
|
||
f"且新任务 {incoming_job_code} 已到达，当前活跃线程 {live_active_threads}"
|
||
)
|
||
if idle_seconds >= 45:
|
||
return True, f"检测指令线程已空转 {int(idle_seconds)} 秒，当前活跃线程 {live_active_threads}"
|
||
if thread_age_seconds >= 30 and idle_seconds >= 20:
|
||
return True, (
|
||
f"检测指令线程已启动 {int(thread_age_seconds)} 秒，但最近 {int(idle_seconds)} 秒没有任何有效检测活动，当前活跃线程 {live_active_threads}"
|
||
)
|
||
return False, ""
|
||
|
||
def _set_active_cycle_context(self, control_payload=None):
|
||
payload = control_payload or {}
|
||
self._last_synced_worker_log = ""
|
||
self.current_cycle_token = str(payload.get("cycle_token") or "").strip()
|
||
job_id = payload.get("job_id")
|
||
if job_id in (None, "", 0, "0"):
|
||
job_id = payload.get("target_job_id")
|
||
try:
|
||
self.current_job_id = int(job_id) if job_id not in (None, "", 0, "0") else None
|
||
except Exception:
|
||
self.current_job_id = None
|
||
job_code = str(payload.get("job_code") or "").strip()
|
||
if not job_code:
|
||
job_code = str(payload.get("target_job_code") or "").strip()
|
||
self.current_job_code = job_code
|
||
task_mode = str(payload.get("task_mode") or payload.get("target_task_mode") or "").strip()
|
||
if not task_mode and str(job_code).startswith("step-"):
|
||
task_mode = "single_step"
|
||
self.current_job_task_mode = task_mode
|
||
|
||
def _clear_active_cycle_context(self):
|
||
self._last_synced_worker_log = ""
|
||
self.current_cycle_token = ""
|
||
self.current_job_id = None
|
||
self.current_job_code = ""
|
||
self.current_job_task_mode = ""
|
||
self._clear_restart_release_handoff(reason="clear_active_cycle")
|
||
self.detect_command_started_at = 0.0
|
||
self.detect_command_last_activity_at = 0.0
|
||
self._last_domain_started_at = 0.0
|
||
self._last_domain_result_at = 0.0
|
||
|
||
def _is_single_step_session_active(self):
|
||
task_mode = str(getattr(self, "current_job_task_mode", "") or "").strip()
|
||
if task_mode == "single_step":
|
||
return True
|
||
return str(getattr(self, "current_job_code", "") or "").strip().startswith("step-")
|
||
|
||
def _shared_job_maintenance_scope(self):
|
||
parent_node_code = str(os.getenv("WORKER_PARENT_NODE_CODE", "") or "").strip()
|
||
scope_node = parent_node_code or str(getattr(config, "NODE_CODE", "") or "").strip() or "unknown"
|
||
db_host = str(getattr(config, "DB_HOST", "") or "").strip() or "db"
|
||
db_port = str(getattr(config, "DB_PORT", "") or "").strip() or "5432"
|
||
db_name = str(getattr(config, "DB_DATABASE", "") or "").strip() or "domain"
|
||
return f"{db_host}:{db_port}:{db_name}:{scope_node}"
|
||
|
||
def _shared_job_maintenance_lock_key(self, action: str) -> str:
|
||
if not self.use_redis or self.redis_client is None:
|
||
return ""
|
||
normalized_action = str(action or "").strip()
|
||
if not normalized_action:
|
||
return ""
|
||
return f"domaincheck:job-maintenance:{normalized_action}:{self._shared_job_maintenance_scope()}"
|
||
|
||
def _job_maintenance_owner_token(self, action: str) -> str:
|
||
return (
|
||
f"{str(getattr(config, 'NODE_CODE', '') or '').strip()}:"
|
||
f"{os.getpid()}:{threading.get_ident()}:{time.time_ns()}:{str(action or '').strip()}"
|
||
)
|
||
|
||
def _acquire_shared_job_maintenance_lock(self, action: str, *, ttl_seconds: int = 15) -> str:
|
||
redis_key = self._shared_job_maintenance_lock_key(action)
|
||
if not redis_key:
|
||
return "__local__" if self._is_primary_job_maintenance_worker() else ""
|
||
owner_token = self._job_maintenance_owner_token(action)
|
||
try:
|
||
acquired = self.redis_client.set(
|
||
redis_key,
|
||
owner_token,
|
||
nx=True,
|
||
ex=max(1, int(ttl_seconds or 15)),
|
||
)
|
||
except Exception as exc:
|
||
logger.debug(f"获取共享任务维护锁失败: action={action}, error={exc}")
|
||
return ""
|
||
return owner_token if acquired else ""
|
||
|
||
def _release_shared_job_maintenance_lock(self, action: str, owner_token: str) -> None:
|
||
if not owner_token or owner_token == "__local__":
|
||
return
|
||
redis_key = self._shared_job_maintenance_lock_key(action)
|
||
if not redis_key:
|
||
return
|
||
try:
|
||
current_owner = self.redis_client.get(redis_key)
|
||
if str(current_owner or "") == str(owner_token):
|
||
self.redis_client.delete(redis_key)
|
||
except Exception as exc:
|
||
logger.debug(f"释放共享任务维护锁失败: action={action}, error={exc}")
|
||
|
||
def _stalled_job_recycle_interval_seconds(self) -> float:
|
||
return max(
|
||
15.0,
|
||
float(os.getenv("DOMAINCHECK_STALLED_JOB_RECYCLE_INTERVAL_SECONDS", "60") or 60),
|
||
)
|
||
|
||
def _stalled_job_recycle_stall_seconds(self) -> int:
|
||
return max(
|
||
300,
|
||
int(os.getenv("DOMAINCHECK_RUNNING_JOB_STALL_SECONDS", "900") or 900),
|
||
)
|
||
|
||
def _maybe_recycle_stalled_job_items(self, job_id, *, reason: str = "") -> int:
|
||
try:
|
||
normalized_job_id = int(job_id or 0)
|
||
except Exception:
|
||
normalized_job_id = 0
|
||
if normalized_job_id <= 0:
|
||
return 0
|
||
now_ts = time.time()
|
||
last_attempts = getattr(self, "_last_stalled_job_recycle_attempt_at", None)
|
||
if not isinstance(last_attempts, dict):
|
||
last_attempts = {}
|
||
self._last_stalled_job_recycle_attempt_at = last_attempts
|
||
last_attempt_at = float(last_attempts.get(normalized_job_id, 0.0) or 0.0)
|
||
if now_ts - last_attempt_at < self._stalled_job_recycle_interval_seconds():
|
||
return 0
|
||
last_attempts[normalized_job_id] = now_ts
|
||
lock_action = f"recycle-stalled-job-items:{normalized_job_id}"
|
||
lock_owner = self._acquire_shared_job_maintenance_lock(lock_action, ttl_seconds=15)
|
||
if not lock_owner:
|
||
return 0
|
||
try:
|
||
recycled = int(
|
||
self.db.recycle_stalled_detect_job_items(
|
||
normalized_job_id,
|
||
stall_seconds=self._stalled_job_recycle_stall_seconds(),
|
||
)
|
||
or 0
|
||
)
|
||
if recycled:
|
||
logger.warning(
|
||
f"已定向回收长挂任务项: job_id={normalized_job_id}, "
|
||
f"recycled={recycled}, reason={reason or 'unspecified'}"
|
||
)
|
||
return recycled
|
||
finally:
|
||
self._release_shared_job_maintenance_lock(lock_action, lock_owner)
|
||
|
||
def _is_primary_job_maintenance_worker(self) -> bool:
|
||
node_code = str(getattr(config, "NODE_CODE", "") or "").strip()
|
||
parent_node_code = str(os.getenv("WORKER_PARENT_NODE_CODE", "") or "").strip()
|
||
if parent_node_code and node_code == parent_node_code:
|
||
return True
|
||
if parent_node_code and node_code == f"{parent_node_code}-a":
|
||
return True
|
||
return bool(node_code and node_code.endswith("-a"))
|
||
|
||
def _maybe_release_node_job_items(self, *, reason: str = "", min_interval_seconds: float = 300.0) -> int:
|
||
now_ts = time.time()
|
||
last_release_at = float(getattr(self, "_last_job_item_release_at", 0.0) or 0.0)
|
||
if min_interval_seconds > 0 and now_ts - last_release_at < float(min_interval_seconds):
|
||
return 0
|
||
released = int(self.db.release_detect_job_items_for_node(config.NODE_CODE) or 0)
|
||
self._last_job_item_release_at = now_ts
|
||
if released:
|
||
self._schedule_restart_release_handoff(
|
||
released_count=released,
|
||
reason=reason or "node_item_release",
|
||
)
|
||
logger.warning(
|
||
f"释放当前节点遗留任务项: count={released}, node={config.NODE_CODE}, "
|
||
f"reason={reason or 'unspecified'}"
|
||
)
|
||
return released
|
||
|
||
def _restart_release_handoff_window_seconds(self) -> float:
|
||
return max(
|
||
30.0,
|
||
float(os.getenv("DOMAINCHECK_RESTART_RELEASE_HANDOFF_WINDOW_SECONDS", "240") or 240),
|
||
)
|
||
|
||
def _restart_release_handoff_batch_limit(self, requested_limit: int) -> int:
|
||
configured = int(os.getenv("DOMAINCHECK_RESTART_RELEASE_HANDOFF_BATCH_LIMIT", "128") or 128)
|
||
safe_requested_limit = max(1, int(requested_limit or 1))
|
||
return max(1, min(safe_requested_limit, max(1, configured)))
|
||
|
||
def _schedule_restart_release_handoff(self, *, released_count: int = 0, reason: str = "") -> None:
|
||
if int(released_count or 0) <= 0:
|
||
return
|
||
job_id = getattr(self, "current_job_id", None)
|
||
if job_id in (None, "", 0, "0"):
|
||
return
|
||
try:
|
||
normalized_job_id = int(job_id)
|
||
except Exception:
|
||
return
|
||
self._restart_release_handoff_job_id = normalized_job_id
|
||
self._restart_release_handoff_job_code = str(getattr(self, "current_job_code", "") or "").strip()
|
||
self._restart_release_handoff_until = time.time() + self._restart_release_handoff_window_seconds()
|
||
self._restart_release_handoff_reason = str(reason or "").strip()
|
||
logger.warning(
|
||
f"登记重启尾批优先回收窗口: job_id={normalized_job_id}, "
|
||
f"job_code={self._restart_release_handoff_job_code or '-'}, "
|
||
f"released={int(released_count or 0)}, reason={self._restart_release_handoff_reason or 'unspecified'}"
|
||
)
|
||
|
||
def _clear_restart_release_handoff(self, *, reason: str = "") -> None:
|
||
job_id = getattr(self, "_restart_release_handoff_job_id", None)
|
||
if job_id not in (None, "", 0, "0"):
|
||
logger.info(
|
||
f"清理重启尾批优先回收窗口: job_id={job_id}, "
|
||
f"job_code={str(getattr(self, '_restart_release_handoff_job_code', '') or '').strip() or '-'}, "
|
||
f"reason={str(reason or 'unspecified').strip() or 'unspecified'}"
|
||
)
|
||
self._restart_release_handoff_job_id = None
|
||
self._restart_release_handoff_job_code = ""
|
||
self._restart_release_handoff_until = 0.0
|
||
self._restart_release_handoff_reason = ""
|
||
|
||
def _restart_release_handoff_job_id_active(self):
|
||
handoff_job_id = getattr(self, "_restart_release_handoff_job_id", None)
|
||
if handoff_job_id in (None, "", 0, "0"):
|
||
return None
|
||
if time.time() >= float(getattr(self, "_restart_release_handoff_until", 0.0) or 0.0):
|
||
self._clear_restart_release_handoff(reason="expired")
|
||
return None
|
||
current_job_id = getattr(self, "current_job_id", None)
|
||
if current_job_id not in (None, "", 0, "0"):
|
||
try:
|
||
if int(current_job_id) != int(handoff_job_id):
|
||
self._clear_restart_release_handoff(reason="job_changed")
|
||
return None
|
||
except Exception:
|
||
self._clear_restart_release_handoff(reason="job_changed")
|
||
return None
|
||
try:
|
||
return int(handoff_job_id)
|
||
except Exception:
|
||
self._clear_restart_release_handoff(reason="invalid_job_id")
|
||
return None
|
||
|
||
def _explicit_claim_scope_window_seconds(self) -> float:
|
||
return max(
|
||
30.0,
|
||
float(os.getenv("DOMAINCHECK_EXPLICIT_CLAIM_SCOPE_WINDOW_SECONDS", "180") or 180),
|
||
)
|
||
|
||
def _schedule_explicit_claim_scope(self, control_payload=None, *, source: str = "") -> None:
|
||
payload = dict(control_payload or {})
|
||
task_mode = str(
|
||
payload.get("task_mode")
|
||
or payload.get("target_task_mode")
|
||
or ""
|
||
).strip()
|
||
if task_mode == "single_step":
|
||
return
|
||
job_id = self._incoming_job_id(payload)
|
||
if job_id in (None, "", 0, "0"):
|
||
return
|
||
try:
|
||
normalized_job_id = int(job_id)
|
||
except Exception:
|
||
return
|
||
job_code = self._incoming_job_code(payload)
|
||
self._explicit_claim_scope_job_id = normalized_job_id
|
||
self._explicit_claim_scope_job_code = str(job_code or "").strip()
|
||
self._explicit_claim_scope_until = time.time() + self._explicit_claim_scope_window_seconds()
|
||
self._explicit_claim_scope_source = str(source or "").strip()
|
||
logger.warning(
|
||
f"登记显式启动定向领取窗口: job_id={normalized_job_id}, "
|
||
f"job_code={self._explicit_claim_scope_job_code or '-'}, "
|
||
f"source={self._explicit_claim_scope_source or 'unknown'}"
|
||
)
|
||
|
||
def _clear_explicit_claim_scope(self, *, reason: str = "") -> None:
|
||
job_id = getattr(self, "_explicit_claim_scope_job_id", None)
|
||
if job_id not in (None, "", 0, "0"):
|
||
logger.info(
|
||
f"清理显式启动定向领取窗口: job_id={job_id}, "
|
||
f"job_code={str(getattr(self, '_explicit_claim_scope_job_code', '') or '').strip() or '-'}, "
|
||
f"reason={str(reason or 'unspecified').strip() or 'unspecified'}"
|
||
)
|
||
self._explicit_claim_scope_job_id = None
|
||
self._explicit_claim_scope_job_code = ""
|
||
self._explicit_claim_scope_until = 0.0
|
||
self._explicit_claim_scope_source = ""
|
||
|
||
def _explicit_claim_scope_job_id_active(self):
|
||
explicit_job_id = getattr(self, "_explicit_claim_scope_job_id", None)
|
||
if explicit_job_id in (None, "", 0, "0"):
|
||
return None
|
||
if time.time() >= float(getattr(self, "_explicit_claim_scope_until", 0.0) or 0.0):
|
||
self._clear_explicit_claim_scope(reason="expired")
|
||
return None
|
||
try:
|
||
return int(explicit_job_id)
|
||
except Exception:
|
||
self._clear_explicit_claim_scope(reason="invalid_job_id")
|
||
return None
|
||
|
||
def _should_scope_claims_to_current_job(self):
|
||
"""
|
||
single_step 会话必须严格绑定当前 job；
|
||
domain_pipeline 会话则应持续从全局队列补位，避免旧 job 占住 worker、
|
||
新 job 长时间堆在 pending。
|
||
"""
|
||
if self._is_single_step_session_active():
|
||
return True
|
||
if self._restart_release_handoff_job_id_active() not in (None, "", 0, "0"):
|
||
return True
|
||
return self._explicit_claim_scope_job_id_active() not in (None, "", 0, "0")
|
||
|
||
def _current_scoped_claim_job_id(self):
|
||
if self._is_single_step_session_active():
|
||
try:
|
||
return int(self.current_job_id) if self.current_job_id not in (None, "", 0, "0") else None
|
||
except Exception:
|
||
return None
|
||
handoff_job_id = self._restart_release_handoff_job_id_active()
|
||
if handoff_job_id not in (None, "", 0, "0"):
|
||
return handoff_job_id
|
||
explicit_job_id = self._explicit_claim_scope_job_id_active()
|
||
if explicit_job_id not in (None, "", 0, "0"):
|
||
return explicit_job_id
|
||
return None
|
||
|
||
def _log_explicit_scope_claim_result(self, *, scoped_job_id, limit, rows_count, reason=""):
|
||
try:
|
||
normalized_job_id = int(scoped_job_id or 0)
|
||
except Exception:
|
||
normalized_job_id = 0
|
||
if normalized_job_id <= 0:
|
||
return
|
||
normalized_rows = max(0, int(rows_count or 0))
|
||
normalized_limit = max(0, int(limit or 0))
|
||
detail_reason = str(reason or "").strip() or ("claimed" if normalized_rows > 0 else "empty")
|
||
log_key = f"{normalized_job_id}:{normalized_rows}:{detail_reason}"
|
||
now_ts = time.time()
|
||
min_interval_seconds = 1.0 if normalized_rows > 0 else 10.0
|
||
if (
|
||
log_key == str(getattr(self, "_last_explicit_scope_claim_log_key", "") or "")
|
||
and now_ts - float(getattr(self, "_last_explicit_scope_claim_log_at", 0.0) or 0.0) < min_interval_seconds
|
||
):
|
||
return
|
||
self._last_explicit_scope_claim_log_at = now_ts
|
||
self._last_explicit_scope_claim_log_key = log_key
|
||
log_message = (
|
||
f"显式定向领取{'命中' if normalized_rows > 0 else '未命中'}: "
|
||
f"scoped_job_id={normalized_job_id} | current_job_id={self.current_job_id or '-'} "
|
||
f"| current_job_code={self.current_job_code or '-'} | detecting={1 if self.detecting else 0} "
|
||
f"| limit={normalized_limit} | rows={normalized_rows} | reason={detail_reason}"
|
||
)
|
||
if normalized_rows > 0:
|
||
logger.warning(log_message)
|
||
else:
|
||
logger.info(log_message)
|
||
|
||
def _log_explicit_scope_probe(self, *, scoped_job_id, max_threads, refill_slots, claim_batch_size, pending_buffer):
|
||
try:
|
||
normalized_job_id = int(scoped_job_id or 0)
|
||
except Exception:
|
||
normalized_job_id = 0
|
||
if normalized_job_id <= 0:
|
||
return
|
||
now_ts = time.time()
|
||
if now_ts - float(getattr(self, "_last_explicit_scope_probe_log_at", 0.0) or 0.0) < 5.0:
|
||
return
|
||
self._last_explicit_scope_probe_log_at = now_ts
|
||
logger.warning(
|
||
f"显式定向领取准备开始: scoped_job_id={normalized_job_id} | current_job_id={self.current_job_id or '-'} "
|
||
f"| current_job_code={self.current_job_code or '-'} | detecting={1 if self.detecting else 0} "
|
||
f"| max_threads={max(0, int(max_threads or 0))} | refill_slots={max(0, int(refill_slots or 0))} "
|
||
f"| claim_batch_size={max(0, int(claim_batch_size or 0))} | pending_buffer={max(0, int(pending_buffer or 0))}"
|
||
)
|
||
|
||
def _log_job_status_refresh_probe(self, job_id):
|
||
try:
|
||
normalized_job_id = int(job_id or 0)
|
||
except Exception:
|
||
normalized_job_id = 0
|
||
if normalized_job_id <= 0:
|
||
return
|
||
now_ts = time.time()
|
||
if now_ts - float(getattr(self, "_last_job_status_refresh_probe_log_at", 0.0) or 0.0) < 10.0:
|
||
return
|
||
self._last_job_status_refresh_probe_log_at = now_ts
|
||
logger.info(
|
||
f"准备刷新检测任务聚合状态: job_id={normalized_job_id} | current_job_code={self.current_job_code or '-'} "
|
||
f"| detecting={1 if self.detecting else 0} | active_threads={self._get_active_domain_threads()}"
|
||
)
|
||
|
||
def _recent_duplicate_start_window_seconds(self) -> float:
|
||
return max(
|
||
15.0,
|
||
float(os.getenv("DOMAINCHECK_RECENT_START_DUPLICATE_WINDOW_SECONDS", "120") or 120),
|
||
)
|
||
|
||
def _record_detection_start_request(self, *, source: str = "", control_payload=None) -> None:
|
||
now_ts = time.time()
|
||
payload = dict(control_payload or {})
|
||
job_id = self._incoming_job_id(payload)
|
||
job_code = self._incoming_job_code(payload)
|
||
task_mode = str(
|
||
payload.get("task_mode")
|
||
or payload.get("target_task_mode")
|
||
or ""
|
||
).strip()
|
||
self._last_detection_start_at = now_ts
|
||
self._last_detection_start_source = str(source or "").strip()
|
||
self._last_detection_start_job_id = int(job_id) if job_id not in (None, 0) else None
|
||
self._last_detection_start_job_code = job_code
|
||
self._last_detection_start_task_mode = task_mode
|
||
|
||
def _mark_explicit_start_signal(self, control_payload=None, *, source: str = "") -> None:
|
||
self._last_explicit_start_signal_at = time.time()
|
||
self._last_explicit_start_payload = {
|
||
**dict(control_payload or {}),
|
||
"_source": str(source or "").strip(),
|
||
}
|
||
self._schedule_explicit_claim_scope(control_payload, source=source)
|
||
|
||
def _should_coalesce_recent_start(self, control_payload=None) -> tuple[bool, str]:
|
||
now_ts = time.time()
|
||
recent_start_at = float(getattr(self, "_last_detection_start_at", 0.0) or 0.0)
|
||
if recent_start_at <= 0:
|
||
return False, ""
|
||
if now_ts - recent_start_at > self._recent_duplicate_start_window_seconds():
|
||
return False, ""
|
||
if self.stop_requested:
|
||
return False, ""
|
||
thread = self.detect_command_thread
|
||
thread_alive = bool(thread and thread.is_alive())
|
||
if not bool(self.detecting) and not thread_alive:
|
||
# A recent start alone is not enough to suppress a legitimate retry.
|
||
# Once the previous session has fully stopped, repeated start_detection
|
||
# for the same job must be allowed to re-open tail work.
|
||
return False, ""
|
||
stale_detect, _ = self._is_stale_detect_session(control_payload)
|
||
if stale_detect:
|
||
# A stale session must never suppress a restart for the same job.
|
||
# Otherwise auto-resume loops forever on "already running" while
|
||
# active_threads stays at zero.
|
||
return False, ""
|
||
|
||
payload = dict(control_payload or {})
|
||
incoming_job_code = self._incoming_job_code(payload)
|
||
incoming_job_id = self._incoming_job_id(payload)
|
||
incoming_task_mode = str(
|
||
payload.get("task_mode")
|
||
or payload.get("target_task_mode")
|
||
or ""
|
||
).strip()
|
||
recent_job_code = str(getattr(self, "_last_detection_start_job_code", "") or "").strip()
|
||
recent_job_id = getattr(self, "_last_detection_start_job_id", None)
|
||
recent_task_mode = str(getattr(self, "_last_detection_start_task_mode", "") or "").strip()
|
||
|
||
if incoming_job_code and recent_job_code and incoming_job_code != recent_job_code:
|
||
return False, ""
|
||
if (
|
||
incoming_job_id not in (None, 0)
|
||
and recent_job_id not in (None, 0)
|
||
and int(incoming_job_id) != int(recent_job_id)
|
||
):
|
||
return False, ""
|
||
if incoming_task_mode == "single_step" and recent_task_mode and incoming_task_mode != recent_task_mode:
|
||
return False, ""
|
||
|
||
effective_job_code = incoming_job_code or recent_job_code or str(self.current_job_code or "").strip()
|
||
effective_job_id = incoming_job_id or recent_job_id or self.current_job_id
|
||
if not effective_job_code and effective_job_id in (None, 0):
|
||
return False, ""
|
||
return True, (
|
||
"最近已启动同一轮检测，会话仍在稳定中，忽略重复启动: "
|
||
f"job={effective_job_code or effective_job_id or 'none'} "
|
||
f"source={str(getattr(self, '_last_detection_start_source', '') or '').strip() or 'unknown'}"
|
||
)
|
||
|
||
def start_detection_async(self, source: str = "remote", control_payload=None):
|
||
"""
|
||
异步启动一次检测任务，避免阻塞 Redis 订阅线程。
|
||
"""
|
||
with self.detect_lock:
|
||
recent_duplicate_start, recent_duplicate_message = self._should_coalesce_recent_start(control_payload)
|
||
if recent_duplicate_start:
|
||
logger.info(recent_duplicate_message)
|
||
self._update_runtime_state(
|
||
"running",
|
||
recent_duplicate_message,
|
||
source=source,
|
||
cycle_token=self.current_cycle_token or self._incoming_cycle_token(control_payload),
|
||
job_id=self.current_job_id or self._incoming_job_id(control_payload),
|
||
job_code=str(self.current_job_code or "").strip() or self._incoming_job_code(control_payload),
|
||
)
|
||
return True
|
||
if self.detect_command_thread and self.detect_command_thread.is_alive():
|
||
duplicate_session_command = self._matches_active_detect_session(control_payload)
|
||
stale_detect, stale_reason = self._is_stale_detect_session(control_payload)
|
||
if not stale_detect:
|
||
incoming_job_code = self._incoming_job_code(control_payload)
|
||
incoming_cycle_token = self._incoming_cycle_token(control_payload)
|
||
incoming_job_id = self._incoming_job_id(control_payload)
|
||
overlap_handoff_requested = self._is_overlap_handoff_command(control_payload, source=source)
|
||
current_job_code = str(self.current_job_code or "").strip()
|
||
incoming_task_mode = str(
|
||
(control_payload or {}).get("task_mode")
|
||
or (control_payload or {}).get("target_task_mode")
|
||
or ""
|
||
).strip()
|
||
if duplicate_session_command:
|
||
message = (
|
||
"收到同任务同周期重复启动指令，保持当前检测会话: "
|
||
f"job={current_job_code or incoming_job_code or 'none'} "
|
||
f"cycle={self.current_cycle_token or incoming_cycle_token or 'none'}"
|
||
)
|
||
logger.info(message)
|
||
self._update_runtime_state(
|
||
"running",
|
||
message,
|
||
source=source,
|
||
cycle_token=self.current_cycle_token or incoming_cycle_token,
|
||
job_id=self.current_job_id or incoming_job_id,
|
||
job_code=current_job_code or incoming_job_code,
|
||
)
|
||
return True
|
||
if (
|
||
incoming_task_mode != "single_step"
|
||
and not self._is_single_step_session_active()
|
||
and incoming_job_code
|
||
and incoming_job_code != current_job_code
|
||
):
|
||
explicit_scope_job_id = self._explicit_claim_scope_job_id_active()
|
||
explicit_switch_requested = bool(
|
||
explicit_scope_job_id not in (None, "", 0, "0")
|
||
and incoming_job_id not in (None, "", 0, "0")
|
||
and int(explicit_scope_job_id) == int(incoming_job_id)
|
||
)
|
||
live_active_threads = max(0, int(self._get_active_domain_threads() or 0))
|
||
if overlap_handoff_requested and live_active_threads > 0:
|
||
message = (
|
||
"收到 overlap handoff 启动指令，但当前会话仍有活跃线程，忽略跨 job 抢占: "
|
||
f"current={current_job_code or 'none'} -> incoming={incoming_job_code} "
|
||
f"active_threads={live_active_threads}"
|
||
)
|
||
logger.info(message)
|
||
if explicit_switch_requested:
|
||
self._clear_explicit_claim_scope(reason="overlap_busy_ignore")
|
||
self._update_runtime_state(
|
||
"running",
|
||
message,
|
||
source=source,
|
||
cycle_token=self.current_cycle_token or incoming_cycle_token,
|
||
job_id=self.current_job_id or incoming_job_id,
|
||
job_code=current_job_code or incoming_job_code,
|
||
)
|
||
return True
|
||
if explicit_switch_requested:
|
||
message = (
|
||
"检测任务已在运行，收到显式 pipeline 切换指令，"
|
||
f"登记安全重启接棒: current={current_job_code or 'none'} -> incoming={incoming_job_code}"
|
||
)
|
||
logger.warning(message)
|
||
self.stop_requested = True
|
||
self._schedule_pending_restart(source, control_payload, message)
|
||
self._update_runtime_state(
|
||
"restarting",
|
||
message,
|
||
source=source,
|
||
cycle_token=incoming_cycle_token or self.current_cycle_token,
|
||
job_id=incoming_job_id,
|
||
job_code=incoming_job_code,
|
||
)
|
||
return True
|
||
message = (
|
||
"检测任务已在运行，已接收新的 pipeline 唤醒指令，"
|
||
f"继续由当前 worker pool 补位处理: current={current_job_code or 'none'} -> incoming={incoming_job_code}"
|
||
)
|
||
logger.info(message)
|
||
self._update_runtime_state("running", message)
|
||
return True
|
||
logger.warning("收到启动检测指令，但检测任务已在运行，忽略重复启动")
|
||
self._update_runtime_state("running", "检测任务已在运行，忽略重复启动")
|
||
return False
|
||
logger.warning(f"检测任务线程疑似卡死，准备回收后重新启动: {stale_reason}")
|
||
self.stop_requested = True
|
||
recovery_deadline = time.time() + 8
|
||
while self.detect_command_thread and self.detect_command_thread.is_alive() and time.time() < recovery_deadline:
|
||
time.sleep(0.2)
|
||
if self.detect_command_thread and self.detect_command_thread.is_alive():
|
||
if self._should_force_takeover_stale_session(stale_reason):
|
||
logger.warning(f"检测任务线程超时未退出，执行强制接管: {stale_reason}")
|
||
self.detect_command_thread = None
|
||
self.detecting = False
|
||
self._clear_active_cycle_context()
|
||
self._update_runtime_state("restarting", f"检测线程疑似卡死，执行强制接管: {stale_reason}")
|
||
else:
|
||
self._schedule_pending_restart(source, control_payload, stale_reason)
|
||
self._update_runtime_state("restarting", f"检测线程疑似卡死，已登记延期重启: {stale_reason}")
|
||
return True
|
||
|
||
self.stop_requested = False
|
||
self.detect_command_started_at = time.time()
|
||
self.detect_command_last_activity_at = self.detect_command_started_at
|
||
self._detect_session_seq = int(self._detect_session_seq or 0) + 1
|
||
session_id = int(self._detect_session_seq or 0)
|
||
self._detect_session_owner = session_id
|
||
self._record_detection_start_request(source=source, control_payload=control_payload)
|
||
self.detect_command_thread = threading.Thread(
|
||
target=self._run_detection_session,
|
||
kwargs={
|
||
"source": source,
|
||
"control_payload": dict(control_payload or {}),
|
||
"session_id": session_id,
|
||
},
|
||
name="RemoteDetectCommandThread",
|
||
daemon=True,
|
||
)
|
||
self.detect_command_thread.start()
|
||
logger.info(f"已接受检测启动指令，来源: {source}")
|
||
self._set_active_cycle_context(control_payload)
|
||
self._update_runtime_state(
|
||
"starting",
|
||
f"已接受检测启动指令，来源: {source}",
|
||
source=source,
|
||
cycle_token=self.current_cycle_token,
|
||
job_id=self.current_job_id,
|
||
job_code=self.current_job_code,
|
||
)
|
||
return True
|
||
|
||
def _run_detection_session(self, source: str = "remote", control_payload=None, session_id: int = 0):
|
||
self._set_active_cycle_context(control_payload)
|
||
with self.detect_lock:
|
||
current_owner = int(getattr(self, "_detect_session_owner", 0) or 0)
|
||
if session_id and current_owner and session_id != current_owner:
|
||
logger.warning(
|
||
f"检测任务会话启动时发现 owner 已切换，忽略旧会话: session_id={session_id}, current_owner={current_owner}"
|
||
)
|
||
return
|
||
self.detecting = True
|
||
try:
|
||
logger.info(f"开始执行远程检测任务，来源: {source}")
|
||
self._sync_worker_log_event(f"开始执行检测任务，来源: {source}", payload={"source": source}, mode='key')
|
||
self._update_runtime_state(
|
||
"starting",
|
||
f"开始执行检测任务，来源: {source}",
|
||
source=source,
|
||
cycle_token=self.current_cycle_token,
|
||
job_id=self.current_job_id,
|
||
job_code=self.current_job_code,
|
||
)
|
||
self.start_detection(session_id=session_id)
|
||
except Exception as e:
|
||
logger.error(f"远程检测任务执行失败: {e}")
|
||
import traceback
|
||
logger.error(traceback.format_exc())
|
||
self._update_runtime_state(
|
||
"failed",
|
||
f"检测任务执行失败: {e}",
|
||
source=source,
|
||
cycle_token=self.current_cycle_token,
|
||
job_id=self.current_job_id,
|
||
job_code=self.current_job_code,
|
||
)
|
||
finally:
|
||
current_thread = threading.current_thread()
|
||
with self.detect_lock:
|
||
current_owner = int(getattr(self, "_detect_session_owner", 0) or 0)
|
||
is_owner = not session_id or current_owner in {0, session_id}
|
||
if is_owner:
|
||
self.detecting = False
|
||
if session_id:
|
||
self._detect_session_owner = 0
|
||
if self.detect_command_thread is current_thread or not (
|
||
self.detect_command_thread and self.detect_command_thread.is_alive()
|
||
):
|
||
self.detect_command_thread = None
|
||
if not is_owner:
|
||
logger.warning(
|
||
f"检测任务旧会话结束，忽略清理: session_id={session_id}, current_owner={current_owner}"
|
||
)
|
||
return
|
||
restarted_from_pending = False
|
||
if self.running:
|
||
try:
|
||
restarted_from_pending = bool(
|
||
self._consume_pending_restart_request(trigger_reason="session_finalize")
|
||
)
|
||
except Exception as restart_error:
|
||
logger.warning(f"检测任务延期重启触发失败: {restart_error}")
|
||
if self.running and not self.stop_requested and not restarted_from_pending:
|
||
self._clear_active_cycle_context()
|
||
self._update_runtime_state(
|
||
"idle",
|
||
"检测任务结束，Worker 保持待命",
|
||
source=source,
|
||
)
|
||
try:
|
||
self._maybe_recycle_idle_worker(trigger_reason="session_finalize")
|
||
except Exception as recycle_error:
|
||
logger.debug(f"检测任务结束后空闲回收检查失败: {recycle_error}")
|
||
else:
|
||
self._clear_active_cycle_context()
|
||
|
||
def request_stop_detection(self, source: str = "remote"):
|
||
"""
|
||
请求停止当前检测任务，但保留 Worker 进程存活。
|
||
"""
|
||
if not self.detecting and not (self.detect_command_thread and self.detect_command_thread.is_alive()):
|
||
logger.info("收到停止检测指令，但当前没有运行中的检测任务")
|
||
self._update_runtime_state("idle", "当前没有运行中的检测任务")
|
||
return False
|
||
logger.info(f"收到停止检测指令，来源: {source}")
|
||
self._sync_worker_log_event(f"收到停止检测指令，来源: {source}", payload={"source": source}, mode='key')
|
||
self.stop_requested = True
|
||
self._update_runtime_state(
|
||
"stopping",
|
||
f"收到停止检测指令，等待当前批次安全退出，来源: {source}",
|
||
cycle_token=self.current_cycle_token,
|
||
job_id=self.current_job_id,
|
||
job_code=self.current_job_code,
|
||
)
|
||
return True
|
||
|
||
def _acknowledge_pending_control_command(self, control_payload):
|
||
if not self.use_redis or self.redis_client is None:
|
||
return
|
||
request_id = str((control_payload or {}).get("request_id", "")).strip()
|
||
if not request_id:
|
||
return
|
||
pending_keys = [pending_control_key()]
|
||
if pending_keys[0] != PENDING_CONTROL_KEY:
|
||
pending_keys.append(PENDING_CONTROL_KEY)
|
||
for key in pending_keys:
|
||
try:
|
||
raw_pending = self.redis_client.get(key)
|
||
if not raw_pending:
|
||
continue
|
||
pending_payload = json.loads(raw_pending)
|
||
pending_request_id = str((pending_payload or {}).get("request_id", "")).strip()
|
||
if pending_request_id and pending_request_id == request_id:
|
||
self.redis_client.delete(key)
|
||
except Exception as e:
|
||
logger.debug(f"确认待执行控制指令失败: key={key}, error={e}")
|
||
|
||
def _handle_control_message(self, payload):
|
||
try:
|
||
control_payload = json.loads(payload) if isinstance(payload, str) else payload
|
||
except Exception:
|
||
control_payload = {"action": str(payload)}
|
||
if not control_targets_current_worker(control_payload):
|
||
self._remember_ignored_targeted_control(control_payload)
|
||
target_node_codes = ",".join(control_target_node_codes(control_payload)) or "unknown"
|
||
logger.info(
|
||
f"忽略发往其他 Worker 实例的控制指令: current={config.NODE_CODE}, targets={target_node_codes}"
|
||
)
|
||
return
|
||
self._acknowledge_pending_control_command(control_payload)
|
||
action = str(control_payload.get("action", "")).strip()
|
||
if action == "start_detection":
|
||
self._mark_explicit_start_signal(control_payload, source="redis-control")
|
||
self.start_detection_async(source="redis-control", control_payload=control_payload)
|
||
elif action == "stop_detection":
|
||
self.request_stop_detection(source="redis-control")
|
||
elif action:
|
||
logger.warning(f"收到未知 Worker 控制动作: {action}")
|
||
|
||
def connect_signals(self):
|
||
"""
|
||
连接信号到槽函数
|
||
"""
|
||
if self.main_window:
|
||
try:
|
||
# 连接信号到槽函数
|
||
self.config_signal.config_updated.connect(self.main_window.update_config_labels)
|
||
logger.debug("信号连接成功")
|
||
# 连接后立即更新一次配置
|
||
self.update_config_labels()
|
||
except Exception as e:
|
||
logger.error(f"连接信号失败: {e}")
|
||
|
||
def _init_optional_detectors(self):
|
||
if self.detect_options.get('detect_jucha') and self.jc_instance is None:
|
||
self.jc_instance = jucha.JC(proxies=None)
|
||
self.jc_instance.load_juming_cookies()
|
||
self.jc_instance.load_cookies()
|
||
if self.detect_options.get('detect_juziseo') and self.juziseo_instance is None:
|
||
self.juziseo_instance = juziseo.Juziseo(proxies=None)
|
||
self.juziseo_instance.load_cookies()
|
||
|
||
def load_detect_options(self):
|
||
"""
|
||
加载检测选项
|
||
"""
|
||
default_order = [
|
||
'detect_register',
|
||
'detect_baidu_site',
|
||
'detect_360_site',
|
||
'detect_chinaz',
|
||
'detect_aizhan',
|
||
'detect_wayback',
|
||
'detect_jucha',
|
||
'detect_juziseo',
|
||
]
|
||
default_options = {
|
||
'detect_register': True,
|
||
'detect_wayback': True,
|
||
'detect_chinaz': True,
|
||
'detect_aizhan': True,
|
||
'detect_baidu_site': True,
|
||
'detect_360_site': True,
|
||
'detect_jucha': False,
|
||
'detect_juziseo': False,
|
||
'detect_order': default_order,
|
||
}
|
||
try:
|
||
# 从Redis获取配置
|
||
if getattr(self, "use_redis", False):
|
||
detect_options_str = self.redis_client.get('domain_tool:detect_options')
|
||
if detect_options_str:
|
||
detect_options = default_options.copy()
|
||
detect_options.update(json.loads(detect_options_str))
|
||
if detect_options.get('detect_whois') or detect_options.get('detect_beian') or detect_options.get('detect_intercept'):
|
||
detect_options['detect_jucha'] = True
|
||
if detect_options.get('detect_juziseo_outlink'):
|
||
detect_options['detect_juziseo'] = True
|
||
order = detect_options.get('detect_order') or []
|
||
detect_options['detect_order'] = [key for key in order if key in default_order]
|
||
for key in default_order:
|
||
if key not in detect_options['detect_order']:
|
||
detect_options['detect_order'].append(key)
|
||
logger.info(f"从Redis加载检测选项成功: {detect_options}")
|
||
return detect_options
|
||
|
||
# 从本地文件获取配置
|
||
if os.path.exists('detect_options.json'):
|
||
with open('detect_options.json', 'r', encoding='utf-8') as f:
|
||
detect_options = default_options.copy()
|
||
detect_options.update(json.load(f))
|
||
if detect_options.get('detect_whois') or detect_options.get('detect_beian') or detect_options.get('detect_intercept'):
|
||
detect_options['detect_jucha'] = True
|
||
if detect_options.get('detect_juziseo_outlink'):
|
||
detect_options['detect_juziseo'] = True
|
||
order = detect_options.get('detect_order') or []
|
||
detect_options['detect_order'] = [key for key in order if key in default_order]
|
||
for key in default_order:
|
||
if key not in detect_options['detect_order']:
|
||
detect_options['detect_order'].append(key)
|
||
logger.info(f"从本地文件加载检测选项成功: {detect_options}")
|
||
return detect_options
|
||
else:
|
||
logger.info(f"使用默认检测选项: {default_options}")
|
||
return default_options
|
||
except Exception as e:
|
||
logger.error(f"加载检测选项失败: {e}")
|
||
logger.info(f"使用默认检测选项: {default_options}")
|
||
return default_options
|
||
|
||
def load_proxy_config(self):
|
||
"""
|
||
加载代理配置
|
||
"""
|
||
default_config = {'proxy_enable': False, 'proxy_url': '', 'proxy_urls': [], 'allow_direct': False}
|
||
try:
|
||
# 从Redis获取配置
|
||
if self.use_redis:
|
||
proxy_config_str = self.redis_client.get('domain_tool:proxy_config')
|
||
if proxy_config_str:
|
||
proxy_config = default_config.copy()
|
||
proxy_config.update(json.loads(proxy_config_str))
|
||
proxy_urls = proxy_config.get('proxy_urls') or []
|
||
if not proxy_urls and proxy_config.get('proxy_url'):
|
||
proxy_urls = [proxy_config.get('proxy_url', '')]
|
||
proxy_config['proxy_urls'] = [url.strip() for url in proxy_urls if url and str(url).strip()]
|
||
proxy_config['proxy_url'] = proxy_config['proxy_urls'][0] if proxy_config['proxy_urls'] else ''
|
||
proxy_config['allow_direct'] = bool(proxy_config.get('allow_direct', False))
|
||
logger.info(f"从Redis加载代理配置成功: {proxy_config}")
|
||
return proxy_config
|
||
|
||
# 从本地文件获取配置
|
||
if os.path.exists('proxy_config.json'):
|
||
with open('proxy_config.json', 'r', encoding='utf-8') as f:
|
||
proxy_config = default_config.copy()
|
||
proxy_config.update(json.load(f))
|
||
logger.info(f"从本地文件加载代理配置成功: {proxy_config}")
|
||
proxy_urls = proxy_config.get('proxy_urls') or []
|
||
if not proxy_urls and proxy_config.get('proxy_url'):
|
||
proxy_urls = [proxy_config.get('proxy_url', '')]
|
||
proxy_config['proxy_urls'] = [url.strip() for url in proxy_urls if url and str(url).strip()]
|
||
proxy_config['proxy_url'] = proxy_config['proxy_urls'][0] if proxy_config['proxy_urls'] else ''
|
||
proxy_config['allow_direct'] = bool(proxy_config.get('allow_direct', False))
|
||
return proxy_config
|
||
else:
|
||
logger.info(f"使用默认代理配置: {default_config}")
|
||
return default_config
|
||
except Exception as e:
|
||
logger.error(f"加载代理配置失败: {e}")
|
||
logger.info(f"使用默认代理配置: {default_config}")
|
||
return default_config
|
||
|
||
def _worker_parent_node_code(self):
|
||
return str(os.getenv("WORKER_PARENT_NODE_CODE", "") or "").strip()
|
||
|
||
def _worker_node_code_candidates(self):
|
||
candidates = []
|
||
for candidate in (
|
||
str(getattr(config, "NODE_CODE", "") or "").strip(),
|
||
self._worker_parent_node_code(),
|
||
):
|
||
if candidate and candidate not in candidates:
|
||
candidates.append(candidate)
|
||
return candidates
|
||
|
||
def load_thread_count(self):
|
||
"""
|
||
加载检测线程数
|
||
"""
|
||
try:
|
||
node_code_candidates = self._worker_node_code_candidates()
|
||
node_code = node_code_candidates[0] if node_code_candidates else ""
|
||
node_thread_counts = {}
|
||
redis_global_thread_count = None
|
||
|
||
def _warn_missing_node_override(source_name, overrides, fallback_value):
|
||
if not node_code or not isinstance(overrides, dict) or not overrides:
|
||
return
|
||
for candidate in node_code_candidates:
|
||
if candidate in overrides:
|
||
return
|
||
available_codes = ", ".join(sorted(str(code) for code in overrides.keys() if str(code).strip()))
|
||
logger.warning(
|
||
"节点线程覆盖未命中，回退到%s通用线程数: node_code=%s, fallback=%s, available_node_codes=%s"
|
||
% (
|
||
source_name,
|
||
node_code,
|
||
fallback_value,
|
||
available_codes or "-",
|
||
)
|
||
)
|
||
|
||
def _resolve_node_override(overrides, source_name):
|
||
if not isinstance(overrides, dict):
|
||
return None
|
||
for candidate in node_code_candidates:
|
||
node_thread_count_raw = overrides.get(candidate)
|
||
if node_thread_count_raw is None:
|
||
continue
|
||
thread_count = max(1, int(node_thread_count_raw))
|
||
if candidate == node_code:
|
||
logger.info(f"从{source_name}加载节点专属检测线程数成功: {candidate} -> {thread_count}")
|
||
else:
|
||
logger.info(
|
||
f"从{source_name}加载父节点检测线程覆盖成功: {candidate} -> {thread_count} (current={node_code})"
|
||
)
|
||
return thread_count
|
||
return None
|
||
|
||
# 从Redis获取配置
|
||
if self.use_redis:
|
||
node_thread_counts_raw = self.redis_client.get('domain_tool:node_thread_counts')
|
||
if node_thread_counts_raw:
|
||
try:
|
||
node_thread_counts = json.loads(node_thread_counts_raw)
|
||
except Exception as e:
|
||
logger.warning(f"解析 Redis 节点线程覆盖配置失败: {e}")
|
||
|
||
resolved_node_thread_count = _resolve_node_override(node_thread_counts, "Redis")
|
||
if resolved_node_thread_count is not None:
|
||
return resolved_node_thread_count
|
||
|
||
thread_count_str = self.redis_client.get('domain_tool:thread_count')
|
||
if thread_count_str:
|
||
thread_count = max(1, int(thread_count_str))
|
||
redis_global_thread_count = thread_count
|
||
_warn_missing_node_override("Redis", node_thread_counts, thread_count)
|
||
logger.info(f"从Redis加载检测线程数成功: {thread_count}")
|
||
return thread_count
|
||
|
||
# 从本地文件获取配置
|
||
node_thread_counts_payload = _read_worker_local_json_config('node_thread_counts.json')
|
||
if isinstance(node_thread_counts_payload, dict):
|
||
node_thread_counts = node_thread_counts_payload
|
||
resolved_node_thread_count = _resolve_node_override(node_thread_counts, "本地文件")
|
||
if resolved_node_thread_count is not None:
|
||
return resolved_node_thread_count
|
||
|
||
thread_config = _read_worker_local_json_config('thread_count.json')
|
||
if isinstance(thread_config, dict):
|
||
thread_count = thread_config.get('thread_count', '10')
|
||
thread_count = max(1, int(thread_count))
|
||
if redis_global_thread_count is None:
|
||
_warn_missing_node_override("本地文件", node_thread_counts, thread_count)
|
||
logger.info(f"从本地文件加载检测线程数成功: {thread_count}")
|
||
return thread_count
|
||
else:
|
||
default_thread_count = 1000
|
||
logger.info(f"使用默认检测线程数: {default_thread_count}")
|
||
return default_thread_count
|
||
except Exception as e:
|
||
logger.error(f"加载检测线程数失败: {e}")
|
||
default_thread_count = 1000
|
||
logger.info(f"使用默认检测线程数: {default_thread_count}")
|
||
return default_thread_count
|
||
|
||
def load_process_count(self):
|
||
"""
|
||
加载检测进程数
|
||
"""
|
||
try:
|
||
node_code_candidates = self._worker_node_code_candidates()
|
||
node_code = node_code_candidates[0] if node_code_candidates else ""
|
||
node_process_counts = {}
|
||
redis_global_process_count = None
|
||
|
||
def _warn_missing_node_override(source_name, overrides, fallback_value):
|
||
if not node_code or not isinstance(overrides, dict) or not overrides:
|
||
return
|
||
for candidate in node_code_candidates:
|
||
if candidate in overrides:
|
||
return
|
||
available_codes = ", ".join(sorted(str(code) for code in overrides.keys() if str(code).strip()))
|
||
logger.warning(
|
||
"节点进程覆盖未命中，回退到%s通用进程数: node_code=%s, fallback=%s, available_node_codes=%s"
|
||
% (
|
||
source_name,
|
||
node_code,
|
||
fallback_value,
|
||
available_codes or "-",
|
||
)
|
||
)
|
||
|
||
def _resolve_node_override(overrides, source_name):
|
||
if not isinstance(overrides, dict):
|
||
return None
|
||
for candidate in node_code_candidates:
|
||
node_process_count_raw = overrides.get(candidate)
|
||
if node_process_count_raw is None:
|
||
continue
|
||
process_count = max(1, int(node_process_count_raw))
|
||
if candidate == node_code:
|
||
logger.info(f"从{source_name}加载节点专属检测进程数成功: {candidate} -> {process_count}")
|
||
else:
|
||
logger.info(
|
||
f"从{source_name}加载父节点检测进程覆盖成功: {candidate} -> {process_count} (current={node_code})"
|
||
)
|
||
return process_count
|
||
return None
|
||
|
||
if self.use_redis:
|
||
node_process_counts_raw = self.redis_client.get('domain_tool:node_process_counts')
|
||
if node_process_counts_raw:
|
||
try:
|
||
node_process_counts = json.loads(node_process_counts_raw)
|
||
except Exception as e:
|
||
logger.warning(f"解析 Redis 节点进程覆盖配置失败: {e}")
|
||
|
||
resolved_node_process_count = _resolve_node_override(node_process_counts, "Redis")
|
||
if resolved_node_process_count is not None:
|
||
return resolved_node_process_count
|
||
|
||
process_count_str = self.redis_client.get('domain_tool:process_count')
|
||
if process_count_str:
|
||
process_count = max(1, int(process_count_str))
|
||
redis_global_process_count = process_count
|
||
_warn_missing_node_override("Redis", node_process_counts, process_count)
|
||
logger.info(f"从Redis加载检测进程数成功: {process_count}")
|
||
return process_count
|
||
|
||
node_process_counts_payload = _read_worker_local_json_config('node_process_counts.json')
|
||
if isinstance(node_process_counts_payload, dict):
|
||
node_process_counts = node_process_counts_payload
|
||
resolved_node_process_count = _resolve_node_override(node_process_counts, "本地文件")
|
||
if resolved_node_process_count is not None:
|
||
return resolved_node_process_count
|
||
|
||
process_config = _read_worker_local_json_config('process_count.json')
|
||
if isinstance(process_config, dict):
|
||
process_count = process_config.get('process_count', '1')
|
||
process_count = max(1, int(process_count))
|
||
if redis_global_process_count is None:
|
||
_warn_missing_node_override("本地文件", node_process_counts, process_count)
|
||
logger.info(f"从本地文件加载检测进程数成功: {process_count}")
|
||
return process_count
|
||
|
||
default_process_count = max(
|
||
1,
|
||
int(os.getenv("DOMAINCHECK_NODE_PROCESS_COUNT", "1") or 1),
|
||
)
|
||
logger.info(f"使用默认检测进程数: {default_process_count}")
|
||
return default_process_count
|
||
except Exception as e:
|
||
logger.error(f"加载检测进程数失败: {e}")
|
||
default_process_count = max(
|
||
1,
|
||
int(os.getenv("DOMAINCHECK_NODE_PROCESS_COUNT", "1") or 1),
|
||
)
|
||
logger.info(f"使用默认检测进程数: {default_process_count}")
|
||
return default_process_count
|
||
|
||
def refresh_thread_count_runtime(self, *, force=False, min_interval=2.0):
|
||
"""
|
||
运行中热刷新线程数配置。
|
||
|
||
Web 后台改了节点并发后，不应该等整批任务跑完才生效。
|
||
"""
|
||
now_ts = time.time()
|
||
last_refresh_at = float(getattr(self, "_last_thread_count_refresh_at", 0.0) or 0.0)
|
||
if not force and now_ts - last_refresh_at < max(0.2, float(min_interval or 0.0)):
|
||
return max(1, int(getattr(self, "thread_count", 1) or 1))
|
||
|
||
previous = max(1, int(getattr(self, "thread_count", 1) or 1))
|
||
latest = max(1, int(self.load_thread_count() or previous))
|
||
self._last_thread_count_refresh_at = now_ts
|
||
if latest != previous:
|
||
self.thread_count = latest
|
||
self.proxy_max_reuse_count = max(8, self.thread_count * 4)
|
||
logger.info(f"检测线程数热更新: {previous} -> {latest}")
|
||
self.update_config_labels()
|
||
return max(1, int(getattr(self, "thread_count", latest) or latest))
|
||
|
||
def refresh_process_count_runtime(self, *, force=False, min_interval=5.0):
|
||
"""
|
||
运行中热刷新进程数配置。
|
||
"""
|
||
now_ts = time.time()
|
||
last_refresh_at = float(getattr(self, "_last_process_count_refresh_at", 0.0) or 0.0)
|
||
if not force and now_ts - last_refresh_at < max(0.5, float(min_interval or 0.0)):
|
||
return max(1, int(getattr(self, "process_count", 1) or 1))
|
||
|
||
previous = max(1, int(getattr(self, "process_count", 1) or 1))
|
||
latest = max(1, int(self.load_process_count() or previous))
|
||
self._last_process_count_refresh_at = now_ts
|
||
if latest != previous:
|
||
self.process_count = latest
|
||
logger.info(f"检测进程数热更新: {previous} -> {latest}")
|
||
self.update_config_labels()
|
||
return max(1, int(getattr(self, "process_count", latest) or latest))
|
||
|
||
def _proxy_runtime_process_count(self):
|
||
current = max(1, int(getattr(self, "process_count", 1) or 1))
|
||
try:
|
||
latest = self.refresh_process_count_runtime(min_interval=5.0)
|
||
except Exception:
|
||
latest = current
|
||
return max(1, int(latest or current))
|
||
|
||
def _proxy_multi_process_divisor(self):
|
||
process_count = self._proxy_runtime_process_count()
|
||
if process_count <= 1:
|
||
return 1.0
|
||
return max(1.0, math.sqrt(float(process_count)))
|
||
|
||
def _proxy_refresh_holdoff_seconds(self):
|
||
process_count = self._proxy_runtime_process_count()
|
||
if process_count < 4 or not self.detecting:
|
||
return 0.0
|
||
pid_spread = (os.getpid() % 7) * 0.02
|
||
return min(0.9, 0.12 + math.log2(float(process_count)) * 0.06 + pid_spread)
|
||
|
||
def _proxy_coordination_node_code(self):
|
||
return self._worker_parent_node_code() or str(getattr(config, "NODE_CODE", "") or "").strip()
|
||
|
||
def _proxy_should_coordinate_shared_refresh(self):
|
||
return bool(
|
||
self.use_redis
|
||
and self.redis_client is not None
|
||
and self._proxy_runtime_process_count() > 1
|
||
and self._proxy_coordination_node_code()
|
||
)
|
||
|
||
def _shared_proxy_pool_key(self):
|
||
node_code = self._proxy_coordination_node_code()
|
||
if not node_code:
|
||
return ""
|
||
return f"{SHARED_PROXY_POOL_KEY_PREFIX}:{node_code}"
|
||
|
||
def _shared_proxy_refresh_lock_key(self):
|
||
node_code = self._proxy_coordination_node_code()
|
||
if not node_code:
|
||
return ""
|
||
return f"{SHARED_PROXY_REFRESH_LOCK_KEY_PREFIX}:{node_code}"
|
||
|
||
def _shared_proxy_snapshot_ttl_seconds(self):
|
||
configured = int(os.getenv("DOMAINCHECK_SHARED_PROXY_SNAPSHOT_TTL", "18") or 18)
|
||
return max(5, configured)
|
||
|
||
def _shared_proxy_refresh_lock_seconds(self):
|
||
configured = int(os.getenv("DOMAINCHECK_SHARED_PROXY_REFRESH_LOCK_SECONDS", "8") or 8)
|
||
return max(2, configured)
|
||
|
||
def _shared_proxy_wait_timeout_seconds(self):
|
||
configured = float(os.getenv("DOMAINCHECK_SHARED_PROXY_WAIT_TIMEOUT", "0.9") or 0.9)
|
||
return max(0.2, configured)
|
||
|
||
def _shared_proxy_reuse_gap(self):
|
||
configured = int(os.getenv("DOMAINCHECK_SHARED_PROXY_REUSE_GAP", "0") or 0)
|
||
if configured > 0:
|
||
return max(8, configured)
|
||
refresh_threshold = max(1, int(self._proxy_refresh_threshold() or 1))
|
||
return max(24, min(160, max(24, refresh_threshold // 3)))
|
||
|
||
def _shared_proxy_snapshot_should_replace_local_pool(self, shared_count, local_count, refresh_threshold):
|
||
normalized_shared = max(0, int(shared_count or 0))
|
||
normalized_local = max(0, int(local_count or 0))
|
||
normalized_threshold = max(1, int(refresh_threshold or 1))
|
||
if normalized_shared <= 0:
|
||
return False
|
||
if normalized_local <= 0:
|
||
return True
|
||
if normalized_shared >= normalized_threshold:
|
||
return True
|
||
if normalized_shared <= normalized_local:
|
||
return False
|
||
return (normalized_shared - normalized_local) >= self._shared_proxy_reuse_gap()
|
||
|
||
def _config_update_proxy_refresh_debounce_seconds(self):
|
||
configured = float(os.getenv("DOMAINCHECK_PROXY_CONFIG_UPDATE_REFRESH_DEBOUNCE", "20") or 20)
|
||
return max(2.0, configured)
|
||
|
||
def _shared_proxy_snapshot_owner(self):
|
||
return f"{config.NODE_CODE}:{os.getpid()}:{int(time.time() * 1000)}"
|
||
|
||
def _rotate_proxy_entries_for_current_worker(self, proxy_entries):
|
||
entries = list(proxy_entries or [])
|
||
if len(entries) <= 1:
|
||
return entries
|
||
seed_source = str(getattr(config, "NODE_CODE", "") or "").strip() or str(os.getpid())
|
||
seed = sum(ord(ch) for ch in seed_source) + int(os.getpid() or 0)
|
||
offset = seed % len(entries)
|
||
if offset <= 0:
|
||
return entries
|
||
return entries[offset:] + entries[:offset]
|
||
|
||
def _shared_proxy_payload_from_entries(self, proxy_entries, *, source_count=0, raw_items=0):
|
||
now_ts = time.time()
|
||
shareable_entries = []
|
||
for proxy_entry in list(proxy_entries or []):
|
||
if not isinstance(proxy_entry, dict) or not proxy_entry.get("proxy"):
|
||
continue
|
||
if not self._proxy_entry_can_be_reused(proxy_entry, now_ts=now_ts):
|
||
continue
|
||
shareable_entries.append(dict(proxy_entry))
|
||
if not shareable_entries:
|
||
return None
|
||
return {
|
||
"node_code": self._proxy_coordination_node_code(),
|
||
"owner": str(getattr(config, "NODE_CODE", "") or "").strip(),
|
||
"refreshed_at_ts": now_ts,
|
||
"source_count": int(source_count or 0),
|
||
"raw_items": int(raw_items or 0),
|
||
"available_count": len(shareable_entries),
|
||
"proxy_pool": shareable_entries,
|
||
}
|
||
|
||
def _load_shared_proxy_snapshot_payload(self, *, max_age_seconds=None):
|
||
if not self._proxy_should_coordinate_shared_refresh():
|
||
return None
|
||
redis_key = self._shared_proxy_pool_key()
|
||
if not redis_key:
|
||
return None
|
||
try:
|
||
raw_payload = self.redis_client.get(redis_key)
|
||
except Exception as exc:
|
||
logger.debug(f"读取共享代理快照失败: {exc}")
|
||
return None
|
||
if not raw_payload:
|
||
return None
|
||
try:
|
||
payload = json.loads(raw_payload)
|
||
except Exception as exc:
|
||
logger.debug(f"解析共享代理快照失败: {exc}")
|
||
return None
|
||
if not isinstance(payload, dict):
|
||
return None
|
||
refreshed_at_ts = float(payload.get("refreshed_at_ts", 0.0) or 0.0)
|
||
allowed_age = max(
|
||
1.0,
|
||
float(max_age_seconds if max_age_seconds is not None else self._shared_proxy_snapshot_ttl_seconds()),
|
||
)
|
||
if refreshed_at_ts > 0 and (time.time() - refreshed_at_ts) > allowed_age:
|
||
return None
|
||
shareable_payload = self._shared_proxy_payload_from_entries(
|
||
payload.get("proxy_pool") or [],
|
||
source_count=payload.get("source_count", 0),
|
||
raw_items=payload.get("raw_items", 0),
|
||
)
|
||
if not shareable_payload:
|
||
return None
|
||
shareable_payload["owner"] = str(payload.get("owner") or "").strip()
|
||
shareable_payload["refreshed_at_ts"] = refreshed_at_ts or shareable_payload.get("refreshed_at_ts", time.time())
|
||
return shareable_payload
|
||
|
||
def _apply_shared_proxy_snapshot(self, payload, *, status_prefix="复用共享代理快照"):
|
||
if not isinstance(payload, dict):
|
||
return False
|
||
proxy_entries = payload.get("proxy_pool") or []
|
||
if not isinstance(proxy_entries, list) or not proxy_entries:
|
||
return False
|
||
rotated_entries = self._rotate_proxy_entries_for_current_worker(proxy_entries)
|
||
with self.proxy_pool_lock:
|
||
self.proxy_pool = list(rotated_entries)
|
||
pool_size = len(self.proxy_pool)
|
||
refreshed_at_ts = float(payload.get("refreshed_at_ts", 0.0) or 0.0)
|
||
self.proxy_last_refresh_time = (
|
||
datetime.fromtimestamp(refreshed_at_ts) if refreshed_at_ts > 0 else datetime.now()
|
||
)
|
||
self.proxy_last_refresh_source_count = int(payload.get("source_count", 0) or 0)
|
||
self.proxy_last_refresh_total_items = int(payload.get("raw_items", 0) or 0)
|
||
self.proxy_last_validated_count = 0
|
||
self.proxy_last_available_count = pool_size
|
||
source_owner = str(payload.get("owner") or "").strip()
|
||
if source_owner and source_owner != str(getattr(config, "NODE_CODE", "") or "").strip():
|
||
self.proxy_last_refresh_status = f"{status_prefix} {pool_size} 个（来源 {source_owner}）"
|
||
else:
|
||
self.proxy_last_refresh_status = f"{status_prefix} {pool_size} 个"
|
||
return True
|
||
|
||
def _publish_shared_proxy_snapshot(self, proxy_entries, *, source_count=0, raw_items=0):
|
||
if not self._proxy_should_coordinate_shared_refresh():
|
||
return False
|
||
payload = self._shared_proxy_payload_from_entries(
|
||
proxy_entries,
|
||
source_count=source_count,
|
||
raw_items=raw_items,
|
||
)
|
||
if not payload:
|
||
return False
|
||
redis_key = self._shared_proxy_pool_key()
|
||
if not redis_key:
|
||
return False
|
||
try:
|
||
self.redis_client.set(
|
||
redis_key,
|
||
json.dumps(payload, ensure_ascii=False),
|
||
ex=self._shared_proxy_snapshot_ttl_seconds(),
|
||
)
|
||
return True
|
||
except Exception as exc:
|
||
logger.debug(f"写入共享代理快照失败: {exc}")
|
||
return False
|
||
|
||
def _acquire_shared_proxy_refresh_lock(self):
|
||
if not self._proxy_should_coordinate_shared_refresh():
|
||
return ""
|
||
redis_key = self._shared_proxy_refresh_lock_key()
|
||
if not redis_key:
|
||
return ""
|
||
owner_token = self._shared_proxy_snapshot_owner()
|
||
try:
|
||
acquired = self.redis_client.set(
|
||
redis_key,
|
||
owner_token,
|
||
nx=True,
|
||
ex=self._shared_proxy_refresh_lock_seconds(),
|
||
)
|
||
except Exception as exc:
|
||
logger.debug(f"获取共享代理刷新锁失败: {exc}")
|
||
return ""
|
||
return owner_token if acquired else ""
|
||
|
||
def _release_shared_proxy_refresh_lock(self, owner_token):
|
||
if not owner_token or not self._proxy_should_coordinate_shared_refresh():
|
||
return
|
||
redis_key = self._shared_proxy_refresh_lock_key()
|
||
if not redis_key:
|
||
return
|
||
try:
|
||
current_owner = self.redis_client.get(redis_key)
|
||
if str(current_owner or "") == str(owner_token):
|
||
self.redis_client.delete(redis_key)
|
||
except Exception as exc:
|
||
logger.debug(f"释放共享代理刷新锁失败: {exc}")
|
||
|
||
def _wait_for_shared_proxy_snapshot(self, timeout_seconds=None):
|
||
if not self._proxy_should_coordinate_shared_refresh():
|
||
return False
|
||
deadline = time.time() + max(
|
||
0.2,
|
||
float(timeout_seconds if timeout_seconds is not None else self._shared_proxy_wait_timeout_seconds()),
|
||
)
|
||
while time.time() < deadline:
|
||
payload = self._load_shared_proxy_snapshot_payload()
|
||
if payload and self._apply_shared_proxy_snapshot(payload):
|
||
return True
|
||
time.sleep(0.05)
|
||
return False
|
||
|
||
def _should_trigger_proxy_refresh_for_config_update(
|
||
self,
|
||
*,
|
||
config_type="",
|
||
previous_proxy_config=None,
|
||
previous_thread_count=0,
|
||
current_thread_count=0,
|
||
):
|
||
previous_proxy_config = dict(previous_proxy_config or {})
|
||
previous_urls = tuple(previous_proxy_config.get('proxy_urls') or [])
|
||
current_urls = tuple((self.proxy_config or {}).get('proxy_urls') or [])
|
||
proxy_changed = (
|
||
bool(previous_proxy_config.get('proxy_enable', False)) != bool((self.proxy_config or {}).get('proxy_enable', False))
|
||
or bool(previous_proxy_config.get('allow_direct', False)) != bool((self.proxy_config or {}).get('allow_direct', False))
|
||
or previous_urls != current_urls
|
||
)
|
||
if proxy_changed:
|
||
return True, True
|
||
|
||
normalized_config_type = str(config_type or "").strip()
|
||
if normalized_config_type not in {"thread_count", "node_thread_counts"}:
|
||
return False, False
|
||
if int(current_thread_count or 0) == int(previous_thread_count or 0):
|
||
return False, False
|
||
|
||
with self.proxy_pool_lock:
|
||
local_available = len(self.proxy_pool)
|
||
shared_payload = self._load_shared_proxy_snapshot_payload()
|
||
shared_available = 0
|
||
if isinstance(shared_payload, dict):
|
||
shared_available = len(list(shared_payload.get("proxy_pool") or []))
|
||
effective_available = max(local_available, shared_available)
|
||
refresh_threshold = self._proxy_refresh_threshold()
|
||
if effective_available >= max(1, int(refresh_threshold or 1)):
|
||
return False, False
|
||
shortage = max(0, int(refresh_threshold or 0) - effective_available)
|
||
if effective_available <= 0:
|
||
return True, False
|
||
return shortage >= self._shared_proxy_reuse_gap(), False
|
||
|
||
def _pull_sync_tasks_until_available(self, *, thread_limit: int, claim_after_pull: bool = True) -> list[dict]:
|
||
"""
|
||
mainland-controller 在本地任务吃空时，不要立即退出检测循环，而是主动向海外控制面
|
||
连续补货几轮，尽量把本地队列重新喂满。
|
||
"""
|
||
if str(getattr(config, "NODE_REGION", "") or "").strip() != "mainland":
|
||
return []
|
||
if str(getattr(config, "NODE_ROLE", "") or "").strip() != "control":
|
||
return []
|
||
|
||
max_rounds = max(1, int(os.getenv("DOMAINCHECK_SYNC_PULL_BURST_ROUNDS", "4") or 4))
|
||
current_thread_limit = max(1, int(thread_limit or getattr(self, "thread_count", 1) or 1))
|
||
pull_limit = self._resolve_sync_pull_limit(current_thread_limit)
|
||
claim_batch_size = max(20, current_thread_limit)
|
||
claim_lease_seconds = max(300, min(1800, claim_batch_size * 30))
|
||
maintenance_lock_owner = self._acquire_shared_job_maintenance_lock(
|
||
"pull-sync-tasks",
|
||
ttl_seconds=max(15, max_rounds * 12),
|
||
)
|
||
if not maintenance_lock_owner:
|
||
return []
|
||
local_api_base_url = str(
|
||
os.getenv("DOMAINCHECK_LOCAL_API_BASE_URL", "http://127.0.0.1:8100/api/v1") or "http://127.0.0.1:8100/api/v1"
|
||
).strip().rstrip("/")
|
||
|
||
try:
|
||
for round_index in range(1, max_rounds + 1):
|
||
try:
|
||
request = urllib.request.Request(
|
||
f"{local_api_base_url}/runtime/actions/pull_tasks",
|
||
data=json.dumps({"limit": pull_limit}, ensure_ascii=False).encode("utf-8"),
|
||
headers={"Content-Type": "application/json"},
|
||
method="POST",
|
||
)
|
||
with urllib.request.urlopen(request, timeout=max(15, min(60, 10 + pull_limit // 100))) as response:
|
||
raw = response.read().decode("utf-8")
|
||
response_data = json.loads(raw) if raw else {}
|
||
pull_ok = int(response_data.get("code", 1) or 1) == 0
|
||
pull_message = str(response_data.get("message") or "").strip()
|
||
pull_data = response_data.get("data") or {}
|
||
except Exception as exc:
|
||
logger.warning(f"本地队列补货失败: round={round_index}/{max_rounds}, error={exc}")
|
||
self._sync_worker_log_event(
|
||
"本地队列补货失败",
|
||
level='warning',
|
||
payload={
|
||
"round": round_index,
|
||
"max_rounds": max_rounds,
|
||
"pull_limit": pull_limit,
|
||
"error": str(exc),
|
||
},
|
||
mode='full',
|
||
)
|
||
break
|
||
|
||
pull_state = str((pull_data or {}).get("pull_state") or "").strip()
|
||
queued_count = int((pull_data or {}).get("queued_count", 0) or 0)
|
||
worker_start_message = str((pull_data or {}).get("worker_start_message") or "").strip()
|
||
refill_message = (
|
||
f"本地队列主动补货: round={round_index}/{max_rounds} | "
|
||
f"pull_limit={pull_limit} | ok={1 if pull_ok else 0} | "
|
||
f"state={pull_state or 'unknown'} | queued_count={queued_count}"
|
||
)
|
||
if worker_start_message:
|
||
refill_message = f"{refill_message} | worker_start={worker_start_message}"
|
||
logger.info(f"{refill_message} | message={pull_message}")
|
||
self._sync_worker_log_event(
|
||
refill_message,
|
||
payload={
|
||
"round": round_index,
|
||
"max_rounds": max_rounds,
|
||
"pull_limit": pull_limit,
|
||
"pull_ok": bool(pull_ok),
|
||
"pull_state": pull_state,
|
||
"queued_count": queued_count,
|
||
"message": str(pull_message or "").strip(),
|
||
"worker_start_message": worker_start_message,
|
||
},
|
||
mode='full',
|
||
)
|
||
|
||
if not claim_after_pull:
|
||
if queued_count > 0:
|
||
logger.info(
|
||
f"主动补货已写入本地队列: round={round_index}/{max_rounds}, "
|
||
f"queued_count={queued_count}, claim_after_pull=0"
|
||
)
|
||
return []
|
||
else:
|
||
domains = self._claim_detect_job_items(
|
||
limit=claim_batch_size,
|
||
lease_seconds=claim_lease_seconds,
|
||
)
|
||
if domains:
|
||
logger.info(
|
||
f"主动补货后已重新领取任务: round={round_index}/{max_rounds}, "
|
||
f"claim_batch_size={claim_batch_size}, domains={len(domains)}"
|
||
)
|
||
return domains
|
||
|
||
if pull_state == "idle":
|
||
break
|
||
if not pull_ok and pull_state not in {"ack_warning", "worker_start_warning"}:
|
||
break
|
||
time.sleep(0.2)
|
||
|
||
return []
|
||
finally:
|
||
self._release_shared_job_maintenance_lock("pull-sync-tasks", maintenance_lock_owner)
|
||
|
||
def _resolve_sync_pull_limit(self, current_thread_limit: int) -> int:
|
||
configured_pull_limit = int(os.getenv("DOMAINCHECK_SYNC_PULL_LIMIT", "0") or 0)
|
||
configured_pull_cap = int(os.getenv("DOMAINCHECK_SYNC_PULL_LIMIT_CAP", "200000") or 200000)
|
||
pull_cap = max(10000, configured_pull_cap)
|
||
if configured_pull_limit > 0:
|
||
return max(1000, min(configured_pull_limit, pull_cap))
|
||
|
||
normalized_thread_limit = max(1, int(current_thread_limit or getattr(self, "thread_count", 1) or 1))
|
||
process_count = max(1, int(self._proxy_runtime_process_count() or 1))
|
||
total_thread_capacity = max(
|
||
normalized_thread_limit,
|
||
normalized_thread_limit * process_count,
|
||
)
|
||
default_pull_limit = max(
|
||
4000,
|
||
min(pull_cap, total_thread_capacity * 2),
|
||
)
|
||
return max(1000, int(default_pull_limit or 1000))
|
||
|
||
def _process_pipeline_tasks_until_available(self, *, thread_limit: int) -> list[dict]:
|
||
"""
|
||
当本地任务队列空了时，主动让 controller 处理已完成步骤，尽量把下一步任务补出来。
|
||
"""
|
||
maintenance_lock_owner = self._acquire_shared_job_maintenance_lock(
|
||
"process-pipeline-tasks",
|
||
ttl_seconds=20,
|
||
)
|
||
if not maintenance_lock_owner:
|
||
return []
|
||
local_api_base_url = str(
|
||
os.getenv("DOMAINCHECK_LOCAL_API_BASE_URL", "http://127.0.0.1:8100/api/v1") or "http://127.0.0.1:8100/api/v1"
|
||
).strip().rstrip("/")
|
||
current_thread_limit = max(1, int(thread_limit or getattr(self, "thread_count", 1) or 1))
|
||
configured_process_limit = int(os.getenv("DOMAINCHECK_PIPELINE_PROCESS_LIMIT", "0") or 0)
|
||
process_limit = max(
|
||
20,
|
||
configured_process_limit if configured_process_limit > 0 else current_thread_limit * 4,
|
||
)
|
||
claim_batch_size = max(20, current_thread_limit)
|
||
claim_lease_seconds = max(300, min(1800, claim_batch_size * 30))
|
||
|
||
try:
|
||
try:
|
||
request = urllib.request.Request(
|
||
f"{local_api_base_url}/runtime/actions/process_pipeline",
|
||
data=json.dumps({"limit": process_limit}, ensure_ascii=False).encode("utf-8"),
|
||
headers={"Content-Type": "application/json"},
|
||
method="POST",
|
||
)
|
||
with urllib.request.urlopen(request, timeout=20) as response:
|
||
raw = response.read().decode("utf-8")
|
||
response_data = json.loads(raw) if raw else {}
|
||
process_ok = int(response_data.get("code", 1) or 1) == 0
|
||
process_message = str(response_data.get("message") or "").strip()
|
||
process_data = response_data.get("data") or {}
|
||
logger.info(
|
||
f"本地 pipeline 推进: ok={1 if process_ok else 0} | "
|
||
f"processed={int(process_data.get('processed_items', 0) or 0)} | "
|
||
f"advanced={int(process_data.get('advanced_items', 0) or 0)} | "
|
||
f"retried={int(process_data.get('retried_items', 0) or 0)} | "
|
||
f"message={process_message}"
|
||
)
|
||
except Exception as exc:
|
||
logger.warning(f"本地 pipeline 推进失败: {exc}")
|
||
self._sync_worker_log_event(
|
||
"本地 pipeline 推进失败",
|
||
level='warning',
|
||
payload={"error": str(exc), "process_limit": process_limit},
|
||
mode='full',
|
||
)
|
||
return []
|
||
|
||
domains = self._claim_detect_job_items(
|
||
limit=claim_batch_size,
|
||
lease_seconds=claim_lease_seconds,
|
||
)
|
||
if domains:
|
||
self._sync_worker_log_event(
|
||
"本地 pipeline 推进后已重新领取任务",
|
||
payload={"domains": len(domains), "claim_batch_size": claim_batch_size},
|
||
mode='full',
|
||
)
|
||
return domains
|
||
finally:
|
||
self._release_shared_job_maintenance_lock("process-pipeline-tasks", maintenance_lock_owner)
|
||
|
||
def load_runtime_settings(self):
|
||
default_settings = {
|
||
"worker_log_sync_enabled": False,
|
||
"worker_log_sync_mode": "key",
|
||
"worker_step_trace_enabled": True,
|
||
"worker_step_trace_sync_full": True,
|
||
"control_node_autoresume_enabled": False,
|
||
"claim_recent_jobs_first": False,
|
||
"claim_recent_jobs_limit": 0,
|
||
"claim_recent_jobs_window_hours": 0,
|
||
"claim_batch_floor": 0,
|
||
"claim_batch_ceil": 0,
|
||
"submit_backlog_floor": 0,
|
||
"submit_backlog_ceil": 0,
|
||
"dispatch_cap_multiplier": 1,
|
||
"pending_buffer_cap_multiplier": 1,
|
||
}
|
||
|
||
def _normalize_bool(value, default=False):
|
||
if value is None:
|
||
return bool(default)
|
||
if isinstance(value, bool):
|
||
return value
|
||
return str(value or "").strip().lower() not in {"", "0", "false", "no", "off"}
|
||
|
||
def _normalize_non_negative_int(value, default=0):
|
||
try:
|
||
normalized = int(value)
|
||
except (TypeError, ValueError):
|
||
normalized = int(default)
|
||
return max(0, normalized)
|
||
|
||
def _normalize_positive_int(value, default=1):
|
||
try:
|
||
normalized = int(value)
|
||
except (TypeError, ValueError):
|
||
normalized = int(default)
|
||
return max(1, normalized)
|
||
|
||
def _normalize_runtime_settings(payload):
|
||
runtime_settings = default_settings.copy()
|
||
runtime_settings.update(payload or {})
|
||
runtime_settings["worker_log_sync_enabled"] = _normalize_bool(runtime_settings.get("worker_log_sync_enabled", False))
|
||
runtime_settings["worker_log_sync_mode"] = "full" if str(runtime_settings.get("worker_log_sync_mode", "key")).strip().lower() == "full" else "key"
|
||
runtime_settings["worker_step_trace_enabled"] = _normalize_bool(runtime_settings.get("worker_step_trace_enabled", True), default=True)
|
||
runtime_settings["worker_step_trace_sync_full"] = _normalize_bool(runtime_settings.get("worker_step_trace_sync_full", True), default=True)
|
||
runtime_settings["control_node_autoresume_enabled"] = _normalize_bool(
|
||
runtime_settings.get("control_node_autoresume_enabled", False),
|
||
default=False,
|
||
)
|
||
runtime_settings["claim_recent_jobs_first"] = _normalize_bool(
|
||
runtime_settings.get("claim_recent_jobs_first", False),
|
||
default=False,
|
||
)
|
||
for key in (
|
||
"claim_recent_jobs_limit",
|
||
"claim_recent_jobs_window_hours",
|
||
"claim_batch_floor",
|
||
"claim_batch_ceil",
|
||
"submit_backlog_floor",
|
||
"submit_backlog_ceil",
|
||
):
|
||
runtime_settings[key] = _normalize_non_negative_int(runtime_settings.get(key), default_settings[key])
|
||
for key in ("dispatch_cap_multiplier", "pending_buffer_cap_multiplier"):
|
||
runtime_settings[key] = _normalize_positive_int(runtime_settings.get(key), default_settings[key])
|
||
return runtime_settings
|
||
|
||
try:
|
||
if self.use_redis:
|
||
runtime_settings_raw = self.redis_client.get(RUNTIME_SETTINGS_KEY)
|
||
if runtime_settings_raw:
|
||
runtime_settings = _normalize_runtime_settings(json.loads(runtime_settings_raw))
|
||
logger.info(f"从Redis加载运行时设置成功: {runtime_settings}")
|
||
return runtime_settings
|
||
|
||
candidate_paths = [
|
||
'runtime_settings.json',
|
||
os.path.join('runtime', 'runtime_settings.json'),
|
||
os.path.join('..', 'domain-api', 'runtime', 'runtime_settings.json'),
|
||
]
|
||
for candidate_path in candidate_paths:
|
||
if os.path.exists(candidate_path):
|
||
with open(candidate_path, 'r', encoding='utf-8') as f:
|
||
runtime_settings = _normalize_runtime_settings(json.load(f))
|
||
logger.info(f"从本地文件加载运行时设置成功: {runtime_settings}")
|
||
return runtime_settings
|
||
except Exception as e:
|
||
logger.error(f"加载运行时设置失败: {e}")
|
||
return default_settings
|
||
|
||
def _runtime_optional_int_override(self, runtime_key, env_key):
|
||
runtime_settings = self.runtime_settings or {}
|
||
try:
|
||
runtime_value = int(runtime_settings.get(runtime_key, 0) or 0)
|
||
except (TypeError, ValueError):
|
||
runtime_value = 0
|
||
if runtime_value > 0:
|
||
return runtime_value
|
||
try:
|
||
return max(0, int(os.getenv(env_key, "0") or 0))
|
||
except (TypeError, ValueError):
|
||
return 0
|
||
|
||
def _runtime_bool_override(self, runtime_key, env_key, default=False):
|
||
env_value = os.getenv(env_key)
|
||
if env_value is not None:
|
||
return self._normalize_flag_value(env_value, default=default)
|
||
runtime_settings = self.runtime_settings or {}
|
||
if runtime_key in runtime_settings:
|
||
return self._normalize_flag_value(runtime_settings.get(runtime_key, default), default=default)
|
||
return bool(default)
|
||
|
||
def _runtime_positive_int_override(self, runtime_key, env_key, default=1):
|
||
runtime_settings = self.runtime_settings or {}
|
||
try:
|
||
runtime_value = int(runtime_settings.get(runtime_key, 0) or 0)
|
||
except (TypeError, ValueError):
|
||
runtime_value = 0
|
||
if runtime_value > 0:
|
||
return runtime_value
|
||
try:
|
||
return max(1, int(os.getenv(env_key, str(default)) or default))
|
||
except (TypeError, ValueError):
|
||
return max(1, int(default))
|
||
|
||
def _claim_recent_jobs_first_enabled(self):
|
||
return self._runtime_bool_override(
|
||
"claim_recent_jobs_first",
|
||
"DOMAINCHECK_CLAIM_RECENT_JOBS_FIRST",
|
||
default=False,
|
||
)
|
||
|
||
def _preferred_claim_job_limit(self):
|
||
return max(1, self._runtime_optional_int_override(
|
||
"claim_recent_jobs_limit",
|
||
"DOMAINCHECK_CLAIM_RECENT_JOBS_LIMIT",
|
||
) or 8)
|
||
|
||
def _preferred_claim_job_window_hours(self):
|
||
return max(1, self._runtime_optional_int_override(
|
||
"claim_recent_jobs_window_hours",
|
||
"DOMAINCHECK_CLAIM_RECENT_JOBS_WINDOW_HOURS",
|
||
) or 24)
|
||
|
||
def _claim_detect_job_items(self, *, limit, lease_seconds, job_id=None):
|
||
if job_id in (None, "", 0, "0") and not self._is_single_step_session_active():
|
||
handoff_job_id = self._restart_release_handoff_job_id_active()
|
||
if handoff_job_id:
|
||
if not self._is_primary_job_maintenance_worker():
|
||
return []
|
||
handoff_limit = self._restart_release_handoff_batch_limit(limit)
|
||
handoff_rows = self.db.claim_restart_released_detect_job_items(
|
||
config.NODE_CODE,
|
||
handoff_job_id,
|
||
limit=handoff_limit,
|
||
lease_seconds=lease_seconds,
|
||
)
|
||
if handoff_rows:
|
||
logger.warning(
|
||
f"优先回收重启尾批任务成功: job_id={handoff_job_id}, "
|
||
f"claimed={len(handoff_rows)}, reason={self._restart_release_handoff_reason or 'worker_restart'}"
|
||
)
|
||
remaining = max(0, int(limit or 0) - len(handoff_rows))
|
||
if remaining <= 0:
|
||
return handoff_rows
|
||
prefer_recent_jobs = self._claim_recent_jobs_first_enabled()
|
||
fallback_rows = self.db.claim_detect_job_items(
|
||
config.NODE_CODE,
|
||
limit=remaining,
|
||
lease_seconds=lease_seconds,
|
||
job_id=None,
|
||
prefer_recent_jobs=prefer_recent_jobs,
|
||
preferred_recent_job_limit=self._preferred_claim_job_limit(),
|
||
preferred_recent_job_window_hours=self._preferred_claim_job_window_hours(),
|
||
)
|
||
return list(handoff_rows or []) + list(fallback_rows or [])
|
||
self._clear_restart_release_handoff(reason="handoff_queue_drained")
|
||
explicit_scope_job_id = self._explicit_claim_scope_job_id_active()
|
||
if explicit_scope_job_id not in (None, "", 0, "0"):
|
||
scoped_rows = self.db.claim_detect_job_items(
|
||
config.NODE_CODE,
|
||
limit=limit,
|
||
lease_seconds=lease_seconds,
|
||
job_id=explicit_scope_job_id,
|
||
prefer_recent_jobs=False,
|
||
preferred_recent_job_limit=self._preferred_claim_job_limit(),
|
||
preferred_recent_job_window_hours=self._preferred_claim_job_window_hours(),
|
||
)
|
||
self._log_explicit_scope_claim_result(
|
||
scoped_job_id=explicit_scope_job_id,
|
||
limit=limit,
|
||
rows_count=len(scoped_rows or []),
|
||
reason="explicit_scope",
|
||
)
|
||
return scoped_rows
|
||
prefer_recent_jobs = bool(job_id in (None, "", 0, "0") and self._claim_recent_jobs_first_enabled())
|
||
return self.db.claim_detect_job_items(
|
||
config.NODE_CODE,
|
||
limit=limit,
|
||
lease_seconds=lease_seconds,
|
||
job_id=job_id,
|
||
prefer_recent_jobs=prefer_recent_jobs,
|
||
preferred_recent_job_limit=self._preferred_claim_job_limit(),
|
||
preferred_recent_job_window_hours=self._preferred_claim_job_window_hours(),
|
||
)
|
||
|
||
def _load_sensitive_words_runtime(self):
|
||
try:
|
||
if self.use_redis and self.redis_client is not None:
|
||
sensitive_words_raw = self.redis_client.get("domain_tool:sensitive_words")
|
||
if sensitive_words_raw is not None:
|
||
try:
|
||
parsed = json.loads(sensitive_words_raw)
|
||
except Exception:
|
||
parsed = sensitive_words_raw
|
||
if isinstance(parsed, list):
|
||
words = [str(item).strip() for item in parsed if str(item).strip()]
|
||
else:
|
||
words = [line.strip() for line in str(parsed or "").splitlines() if line.strip()]
|
||
logger.info(f"从Redis加载敏感词成功，共 {len(words)} 个敏感词")
|
||
return words
|
||
|
||
candidate_paths = [
|
||
'runtime/sensitive_words.json',
|
||
'sensitive_words.json',
|
||
'sensitive_words.txt',
|
||
]
|
||
for candidate_path in candidate_paths:
|
||
if not os.path.exists(candidate_path):
|
||
continue
|
||
if candidate_path.endswith('.json'):
|
||
with open(candidate_path, 'r', encoding='utf-8') as f:
|
||
payload = json.load(f)
|
||
if isinstance(payload, dict):
|
||
if isinstance(payload.get('items'), list):
|
||
words = []
|
||
for item in payload.get('items') or []:
|
||
if isinstance(item, dict):
|
||
word = str(item.get('word') or '').strip()
|
||
else:
|
||
word = str(item).strip()
|
||
if word:
|
||
words.append(word)
|
||
else:
|
||
words = [line.strip() for line in str(payload.get('text') or '').splitlines() if line.strip()]
|
||
elif isinstance(payload, list):
|
||
words = [str(item).strip() for item in payload if str(item).strip()]
|
||
else:
|
||
words = []
|
||
else:
|
||
with open(candidate_path, 'r', encoding='utf-8') as f:
|
||
words = [line.strip() for line in f.read().splitlines() if line.strip()]
|
||
logger.info(f"从本地文件加载敏感词成功: {candidate_path}，共 {len(words)} 个敏感词")
|
||
return words
|
||
except Exception as e:
|
||
logger.error(f"从运行时配置加载敏感词失败: {e}")
|
||
|
||
words = self.db.get_all_sensitive_words()
|
||
logger.info(f"从数据库加载敏感词成功，共 {len(words)} 个敏感词")
|
||
return words
|
||
|
||
@staticmethod
|
||
def _remote_debug_ingest_url():
|
||
base_url = str(os.getenv("SYNC_TARGET_API_BASE_URL", "") or "").strip().rstrip("/")
|
||
if not base_url:
|
||
return ""
|
||
if base_url.endswith("/api/v1"):
|
||
return f"{base_url}/runtime/debug-ingest"
|
||
if base_url.endswith("/api/v1/runtime"):
|
||
return f"{base_url}/debug-ingest"
|
||
return f"{base_url}/api/v1/runtime/debug-ingest"
|
||
|
||
@staticmethod
|
||
def _remote_debug_event_timeout_seconds():
|
||
configured_timeout = float(os.getenv("DOMAINCHECK_REMOTE_DEBUG_EVENT_TIMEOUT", "0") or 0.0)
|
||
if configured_timeout > 0:
|
||
return max(0.2, min(configured_timeout, 10.0))
|
||
if str(getattr(config, "NODE_ROLE", "") or "").strip() == "control":
|
||
return 1.2
|
||
return 2.0
|
||
|
||
@staticmethod
|
||
def _remote_debug_event_cooldown_seconds(failure_streak):
|
||
normalized_streak = max(1, int(failure_streak or 1))
|
||
base_seconds = max(
|
||
5.0,
|
||
float(os.getenv("DOMAINCHECK_REMOTE_DEBUG_EVENT_COOLDOWN_BASE_SECONDS", "15") or 15.0),
|
||
)
|
||
max_seconds = max(
|
||
base_seconds,
|
||
float(os.getenv("DOMAINCHECK_REMOTE_DEBUG_EVENT_COOLDOWN_MAX_SECONDS", "180") or 180.0),
|
||
)
|
||
return min(max_seconds, base_seconds * (2 ** max(0, normalized_streak - 1)))
|
||
|
||
def _push_remote_debug_event(self, *, message, level='info', payload=None, event_type='worker_log'):
|
||
ingest_url = self._remote_debug_ingest_url()
|
||
if not ingest_url:
|
||
return
|
||
now_ts = time.time()
|
||
cooldown_until = float(getattr(self, "_remote_debug_event_cooldown_until", 0.0) or 0.0)
|
||
if cooldown_until > now_ts:
|
||
return
|
||
request_payload = {
|
||
"source_region": config.NODE_REGION,
|
||
"node_code": config.NODE_CODE,
|
||
"hostname": socket.gethostname(),
|
||
"service": "worker-event",
|
||
"event_type": str(event_type or "worker_log"),
|
||
"level": str(level or "info"),
|
||
"message": str(message or "").strip()[:2000],
|
||
"payload": payload or {},
|
||
}
|
||
headers = {
|
||
"Content-Type": "application/json",
|
||
}
|
||
shared_token = str(os.getenv("SYNC_SHARED_TOKEN", "") or "").strip()
|
||
if shared_token:
|
||
headers["X-Domaincheck-Sync-Token"] = shared_token
|
||
request = urllib.request.Request(
|
||
ingest_url,
|
||
data=json.dumps(request_payload, ensure_ascii=False, sort_keys=True).encode("utf-8"),
|
||
headers=headers,
|
||
method="POST",
|
||
)
|
||
try:
|
||
timeout_seconds = self._remote_debug_event_timeout_seconds()
|
||
with urllib.request.urlopen(request, timeout=timeout_seconds) as response:
|
||
raw = response.read().decode("utf-8", errors="ignore")
|
||
if raw:
|
||
parsed = json.loads(raw)
|
||
if isinstance(parsed, dict) and parsed.get("code") not in (0, "0", None, ""):
|
||
logger.debug(f"远端调试事件返回非成功: {parsed}")
|
||
self._remote_debug_event_failure_streak = 0
|
||
self._remote_debug_event_cooldown_until = 0.0
|
||
except Exception as e:
|
||
failure_streak = int(getattr(self, "_remote_debug_event_failure_streak", 0) or 0) + 1
|
||
cooldown_seconds = self._remote_debug_event_cooldown_seconds(failure_streak)
|
||
self._remote_debug_event_failure_streak = failure_streak
|
||
self._remote_debug_event_cooldown_until = now_ts + cooldown_seconds
|
||
last_notice_at = float(getattr(self, "_last_remote_debug_event_notice_at", 0.0) or 0.0)
|
||
if failure_streak <= 2 or now_ts - last_notice_at >= max(10.0, min(cooldown_seconds, 30.0)):
|
||
logger.debug(
|
||
f"远端调试事件回传失败，进入冷却 {int(cooldown_seconds)} 秒: {e}"
|
||
)
|
||
self._last_remote_debug_event_notice_at = now_ts
|
||
|
||
def _worker_log_sync_mode(self):
|
||
if not bool((self.runtime_settings or {}).get("worker_log_sync_enabled", False)):
|
||
return "off"
|
||
configured_mode = (
|
||
"full"
|
||
if str((self.runtime_settings or {}).get("worker_log_sync_mode", "key")).strip().lower() == "full"
|
||
else "key"
|
||
)
|
||
if configured_mode != "full":
|
||
return configured_mode
|
||
degrade_reason = self._worker_log_sync_full_degrade_reason()
|
||
if not degrade_reason:
|
||
return configured_mode
|
||
now_ts = time.time()
|
||
last_notice_at = float(getattr(self, "_last_worker_log_sync_degrade_notice_at", 0.0) or 0.0)
|
||
if now_ts - last_notice_at >= 10.0:
|
||
logger.warning(f"Worker 全量日志同步已自动降级为 key: {degrade_reason}")
|
||
self._last_worker_log_sync_degrade_notice_at = now_ts
|
||
return "key"
|
||
|
||
def _worker_log_sync_full_active_thread_threshold(self):
|
||
configured_threshold = int(os.getenv("DOMAINCHECK_WORKER_LOG_SYNC_FULL_MAX_ACTIVE_THREADS", "0") or 0)
|
||
if configured_threshold > 0:
|
||
return max(32, configured_threshold)
|
||
thread_limit = max(1, int(getattr(self, "thread_count", 0) or 0))
|
||
return max(120, min(600, max(120, thread_limit // 3)))
|
||
|
||
def _worker_log_sync_full_queue_threshold(self):
|
||
configured_threshold = int(os.getenv("DOMAINCHECK_WORKER_LOG_SYNC_FULL_MAX_QUEUE", "0") or 0)
|
||
if configured_threshold > 0:
|
||
return max(256, configured_threshold)
|
||
queue_maxsize = int(getattr(getattr(self, "_worker_log_sync_queue", None), "maxsize", 0) or 0)
|
||
if queue_maxsize > 0:
|
||
return max(1000, min(15000, queue_maxsize // 5))
|
||
return 5000
|
||
|
||
def _worker_log_sync_full_degrade_reason(self):
|
||
queue_obj = getattr(self, "_worker_log_sync_queue", None)
|
||
queue_depth = 0
|
||
if queue_obj is not None:
|
||
try:
|
||
queue_depth = max(0, int(queue_obj.qsize() or 0))
|
||
except Exception:
|
||
queue_depth = 0
|
||
queue_threshold = self._worker_log_sync_full_queue_threshold()
|
||
if queue_depth >= queue_threshold:
|
||
return f"queue_backlog={queue_depth}, threshold={queue_threshold}"
|
||
|
||
live_active_threads = max(0, int(self._get_active_domain_threads() or 0))
|
||
active_threshold = self._worker_log_sync_full_active_thread_threshold()
|
||
if bool(getattr(self, "detecting", False)) and live_active_threads >= active_threshold:
|
||
return f"active_threads={live_active_threads}, threshold={active_threshold}"
|
||
|
||
return ""
|
||
|
||
def _worker_step_trace_enabled(self):
|
||
return bool((self.runtime_settings or {}).get("worker_step_trace_enabled", True))
|
||
|
||
def _worker_step_trace_sync_full_enabled(self):
|
||
return bool((self.runtime_settings or {}).get("worker_step_trace_sync_full", True))
|
||
|
||
@staticmethod
|
||
def _single_step_cache_key(domain_id, field_name):
|
||
return f"domain_tool:single_step_result:{int(domain_id or 0)}:{str(field_name or '').strip()}"
|
||
|
||
def _set_current_task_context(self, *, domain_id, is_step_task, task_mode, job_item_id=None):
|
||
self._task_local.domain_id = int(domain_id or 0)
|
||
self._task_local.is_step_task = bool(is_step_task)
|
||
self._task_local.task_mode = str(task_mode or "").strip()
|
||
self._task_local.job_item_id = int(job_item_id or 0) if job_item_id else None
|
||
self._task_local.detect_status = None
|
||
self._task_local.step_proxy_retry_waits = {}
|
||
self._task_local.step_force_direct_once = {}
|
||
|
||
def _clear_current_task_context(self):
|
||
for attr in ("domain_id", "is_step_task", "task_mode", "job_item_id", "detect_status", "step_proxy_retry_waits", "step_force_direct_once"):
|
||
try:
|
||
delattr(self._task_local, attr)
|
||
except Exception:
|
||
pass
|
||
|
||
def _is_current_single_step_task(self):
|
||
return bool(getattr(self._task_local, "is_step_task", False))
|
||
|
||
def _set_task_detect_status(self, status):
|
||
try:
|
||
normalized_status = int(status)
|
||
except Exception:
|
||
normalized_status = None
|
||
self._task_local.detect_status = normalized_status
|
||
|
||
def _get_task_detect_status(self):
|
||
try:
|
||
value = getattr(self._task_local, "detect_status", None)
|
||
return int(value) if value is not None else None
|
||
except Exception:
|
||
return None
|
||
|
||
def _resolve_current_task_final_status(self):
|
||
current_status = self._get_task_detect_status()
|
||
if current_status == DETECT_STATUS_BLACKLISTED:
|
||
return 'blacklisted'
|
||
return 'failed'
|
||
|
||
def _set_step_proxy_retry_wait(self, step_name, wait_seconds):
|
||
normalized_step_name = str(step_name or "").strip()
|
||
if not normalized_step_name:
|
||
return
|
||
try:
|
||
normalized_wait_seconds = max(0.0, float(wait_seconds or 0.0))
|
||
except Exception:
|
||
normalized_wait_seconds = 0.0
|
||
retry_waits = getattr(self._task_local, "step_proxy_retry_waits", None)
|
||
if not isinstance(retry_waits, dict):
|
||
retry_waits = {}
|
||
self._task_local.step_proxy_retry_waits = retry_waits
|
||
retry_waits[normalized_step_name] = normalized_wait_seconds
|
||
|
||
def _consume_step_proxy_retry_wait(self, step_name):
|
||
normalized_step_name = str(step_name or "").strip()
|
||
if not normalized_step_name:
|
||
return 0.0
|
||
retry_waits = getattr(self._task_local, "step_proxy_retry_waits", None)
|
||
if not isinstance(retry_waits, dict):
|
||
return 0.0
|
||
try:
|
||
return max(0.0, float(retry_waits.pop(normalized_step_name, 0.0) or 0.0))
|
||
except Exception:
|
||
return 0.0
|
||
|
||
def _set_step_force_direct_once(self, step_name, enabled=True):
|
||
normalized_step_name = str(step_name or "").strip()
|
||
if not normalized_step_name:
|
||
return
|
||
force_map = getattr(self._task_local, "step_force_direct_once", None)
|
||
if not isinstance(force_map, dict):
|
||
force_map = {}
|
||
self._task_local.step_force_direct_once = force_map
|
||
force_map[normalized_step_name] = bool(enabled)
|
||
|
||
def _consume_step_force_direct_once(self, step_name):
|
||
normalized_step_name = str(step_name or "").strip()
|
||
if not normalized_step_name:
|
||
return False
|
||
force_map = getattr(self._task_local, "step_force_direct_once", None)
|
||
if not isinstance(force_map, dict):
|
||
return False
|
||
return bool(force_map.pop(normalized_step_name, False))
|
||
|
||
def _peek_step_force_direct_once(self, step_name):
|
||
normalized_step_name = str(step_name or "").strip()
|
||
if not normalized_step_name:
|
||
return False
|
||
force_map = getattr(self._task_local, "step_force_direct_once", None)
|
||
if not isinstance(force_map, dict):
|
||
return False
|
||
return bool(force_map.get(normalized_step_name, False))
|
||
|
||
def _cache_single_step_result_payload(self, domain_id, field_name, payload):
|
||
normalized_field_name = str(field_name or "").strip()
|
||
if not normalized_field_name:
|
||
return
|
||
cache_key = self._single_step_cache_key(domain_id, normalized_field_name)
|
||
with self._step_result_cache_lock:
|
||
self._step_result_cache[(int(domain_id or 0), normalized_field_name)] = dict(payload or {})
|
||
if self.use_redis and self.redis_client is not None:
|
||
try:
|
||
self.redis_client.setex(cache_key, 600, json.dumps(payload or {}, ensure_ascii=False))
|
||
except Exception as e:
|
||
logger.debug(f"写入 Redis single_step 结果缓存失败: {e}")
|
||
|
||
def _load_cached_single_step_result_payload(self, domain_id, field_name):
|
||
normalized_field_name = str(field_name or "").strip()
|
||
if not normalized_field_name:
|
||
return None
|
||
cache_tuple_key = (int(domain_id or 0), normalized_field_name)
|
||
with self._step_result_cache_lock:
|
||
cached_payload = self._step_result_cache.pop(cache_tuple_key, None)
|
||
if isinstance(cached_payload, dict) and cached_payload:
|
||
return dict(cached_payload)
|
||
if self.use_redis and self.redis_client is not None:
|
||
try:
|
||
redis_key = self._single_step_cache_key(domain_id, normalized_field_name)
|
||
raw_payload = self.redis_client.get(redis_key)
|
||
if raw_payload:
|
||
self.redis_client.delete(redis_key)
|
||
parsed = json.loads(raw_payload)
|
||
if isinstance(parsed, dict):
|
||
return parsed
|
||
except Exception as e:
|
||
logger.debug(f"读取 Redis single_step 结果缓存失败: {e}")
|
||
return None
|
||
|
||
def _sync_worker_log_event(self, message, level='info', payload=None, *, mode='key', job_item_id=None):
|
||
current_mode = self._worker_log_sync_mode()
|
||
if current_mode == "off":
|
||
return
|
||
if mode == 'full' and current_mode != "full":
|
||
return
|
||
normalized_message = str(message or '').strip()
|
||
if not normalized_message:
|
||
return
|
||
if normalized_message == self._last_synced_worker_log and mode != 'full':
|
||
return
|
||
event_payload = dict(payload or {})
|
||
explicit_job_id = event_payload.get("job_id")
|
||
try:
|
||
job_id = int(explicit_job_id) if explicit_job_id not in (None, "", 0, "0") else self.current_job_id
|
||
except Exception:
|
||
job_id = self.current_job_id
|
||
if not job_id:
|
||
return
|
||
if self.current_cycle_token and not event_payload.get("cycle_token"):
|
||
event_payload["cycle_token"] = self.current_cycle_token
|
||
if self.current_job_code and not event_payload.get("job_code"):
|
||
event_payload["job_code"] = self.current_job_code
|
||
event_payload["log_mode"] = mode
|
||
event_payload["synced_by"] = "worker_log_callback"
|
||
try:
|
||
queue_item = {
|
||
"job_id": job_id,
|
||
"job_item_id": job_item_id,
|
||
"message": normalized_message,
|
||
"level": level,
|
||
"payload": event_payload,
|
||
"write_db": not (mode == 'full' and self._is_current_single_step_task()),
|
||
}
|
||
try:
|
||
self._worker_log_sync_queue.put_nowait(queue_item)
|
||
except Full:
|
||
if mode != 'full':
|
||
self._worker_log_sync_queue.put(queue_item, timeout=0.2)
|
||
else:
|
||
raise
|
||
self._last_synced_worker_log = normalized_message
|
||
except Exception as e:
|
||
if mode == 'full':
|
||
self._worker_log_sync_drop_count += 1
|
||
now_ts = time.time()
|
||
if now_ts - self._last_worker_log_sync_drop_notice_at >= 10:
|
||
logger.warning(
|
||
f"Worker 全量日志同步队列拥堵，已丢弃 {self._worker_log_sync_drop_count} 条 full 日志"
|
||
)
|
||
self._last_worker_log_sync_drop_notice_at = now_ts
|
||
else:
|
||
logger.debug(f"回传Worker日志事件失败: {e}")
|
||
|
||
@staticmethod
|
||
def _step_display_name(detect_key):
|
||
mapping = {
|
||
'detect_register': '注册状态检测',
|
||
'detect_baidu_site': '百度site检测',
|
||
'detect_360_site': '360 site检测',
|
||
'detect_chinaz': '站长之家检测',
|
||
'detect_aizhan': '爱站检测',
|
||
'detect_wayback': '时光机检测',
|
||
'detect_jucha': '聚查检测',
|
||
'detect_juziseo': '桔子检测',
|
||
}
|
||
return mapping.get(str(detect_key or '').strip(), str(detect_key or '').strip() or 'unknown')
|
||
|
||
@staticmethod
|
||
def _step_result_field_name(detect_key):
|
||
mapping = {
|
||
'detect_register': 'register_status',
|
||
'detect_baidu_site': 'baidu_site',
|
||
'detect_360_site': 'qihu360_site',
|
||
'detect_chinaz': 'chinaz_info',
|
||
'detect_aizhan': 'aizhan_info',
|
||
'detect_wayback': 'wayback_info',
|
||
}
|
||
return mapping.get(str(detect_key or '').strip(), '')
|
||
|
||
def _build_single_step_job_result(self, *, domain_id, detect_key, step_ok, step_name):
|
||
field_name = self._step_result_field_name(detect_key)
|
||
cached_payload = self._load_cached_single_step_result_payload(domain_id, field_name)
|
||
if isinstance(cached_payload, dict) and cached_payload:
|
||
payload = dict(cached_payload)
|
||
payload["step_code"] = str(detect_key or "").strip()
|
||
payload["step_name"] = str(step_name or "").strip()
|
||
payload["field_name"] = field_name
|
||
payload["task_mode"] = "single_step"
|
||
payload["domain_id"] = int(domain_id or 0)
|
||
if payload.get("state") == "degraded":
|
||
payload["retry_recommended"] = True
|
||
return payload
|
||
try:
|
||
domain_snapshot = self.db.get_domain_by_id(domain_id) or {}
|
||
except Exception:
|
||
domain_snapshot = {}
|
||
raw_payload = domain_snapshot.get(field_name) if field_name else None
|
||
payload = dict(raw_payload) if isinstance(raw_payload, dict) else {}
|
||
if not payload:
|
||
payload = {
|
||
"status": bool(step_ok),
|
||
"state": "passed" if step_ok else "failed",
|
||
"message": "success" if step_ok else f"{step_name} 未通过",
|
||
}
|
||
payload["step_code"] = str(detect_key or "").strip()
|
||
payload["step_name"] = str(step_name or "").strip()
|
||
payload["field_name"] = field_name
|
||
payload["task_mode"] = "single_step"
|
||
payload["domain_id"] = int(domain_id or 0)
|
||
if payload.get("state") == "degraded":
|
||
payload["retry_recommended"] = True
|
||
return payload
|
||
|
||
@staticmethod
|
||
def _resolve_single_step_finalization(result_payload):
|
||
payload = dict(result_payload or {})
|
||
state = str(payload.get("state") or "").strip().lower()
|
||
message = str(payload.get("message") or "").strip()
|
||
status = payload.get("status")
|
||
if state == "blacklisted":
|
||
return "blacklisted", message or "步骤命中黑名单"
|
||
if state == "degraded":
|
||
return "completed", message or "步骤外部依赖异常，建议重试"
|
||
if state in {"failed", "error", "rejected"}:
|
||
return "failed", message or "步骤执行失败"
|
||
if status is False:
|
||
return "failed", message or "步骤执行失败"
|
||
return "completed", message or "步骤执行完成"
|
||
|
||
def _emit_step_trace(self, domain_name, step_name, stage, *, elapsed_ms=None, ok=None, level='info', job_item_id=None, sync_mode=None, **extra):
|
||
if not self._worker_step_trace_enabled():
|
||
return
|
||
payload = {
|
||
"domain": str(domain_name or "").strip(),
|
||
"step": str(step_name or "").strip(),
|
||
"stage": str(stage or "").strip(),
|
||
}
|
||
if elapsed_ms is not None:
|
||
payload["elapsed_ms"] = int(elapsed_ms or 0)
|
||
if ok is not None:
|
||
payload["ok"] = bool(ok)
|
||
for key, value in dict(extra or {}).items():
|
||
if value in (None, ""):
|
||
continue
|
||
payload[key] = value
|
||
|
||
message_parts = [
|
||
f"domain={payload.get('domain') or '-'}",
|
||
f"step={payload.get('step') or '-'}",
|
||
f"stage={payload.get('stage') or '-'}",
|
||
]
|
||
if "elapsed_ms" in payload:
|
||
message_parts.append(f"elapsed_ms={payload['elapsed_ms']}")
|
||
if "ok" in payload:
|
||
message_parts.append(f"ok={1 if payload['ok'] else 0}")
|
||
for key in sorted(payload.keys()):
|
||
if key in {"domain", "step", "stage", "elapsed_ms", "ok"}:
|
||
continue
|
||
value = payload.get(key)
|
||
if isinstance(value, (dict, list, tuple)):
|
||
value = json.dumps(value, ensure_ascii=False, sort_keys=True)
|
||
message_parts.append(f"{key}={value}")
|
||
message = "检测步骤跟踪: " + " | ".join(message_parts)
|
||
logger.log(level.upper(), message)
|
||
|
||
should_sync_full = self._worker_step_trace_sync_full_enabled()
|
||
if sync_mode == 'off':
|
||
should_sync_full = False
|
||
if should_sync_full:
|
||
self._sync_worker_log_event(
|
||
message,
|
||
level=level,
|
||
payload=payload,
|
||
mode='full',
|
||
job_item_id=job_item_id,
|
||
)
|
||
|
||
def test_proxy(self, proxy_item, result_queue):
|
||
"""
|
||
测试单个代理的可用性
|
||
|
||
:param proxy_item: 代理信息
|
||
:param result_queue: 结果队列
|
||
"""
|
||
result_payload = {
|
||
"ok": False,
|
||
"proxy": None,
|
||
"reason": "invalid_proxy_item",
|
||
}
|
||
try:
|
||
if 'ip' in proxy_item and 'port' in proxy_item:
|
||
ip = proxy_item['ip']
|
||
port = proxy_item['port']
|
||
username = proxy_item.get('username', '')
|
||
password = proxy_item.get('password', '')
|
||
|
||
if username and password:
|
||
proxy_url = f"http://{username}:{password}@{ip}:{port}"
|
||
else:
|
||
proxy_url = f"http://{ip}:{port}"
|
||
|
||
import requests
|
||
test_proxies = {
|
||
'http': proxy_url,
|
||
'https': proxy_url
|
||
}
|
||
result_payload["proxy"] = test_proxies
|
||
|
||
test_targets = [
|
||
("http://www.baidu.com", 8),
|
||
("https://m.baidu.com", 8),
|
||
("http://www.qq.com", 8),
|
||
("http://www.360.cn", 8),
|
||
]
|
||
last_reason = "unknown"
|
||
session = requests.Session()
|
||
session.headers.update(
|
||
{
|
||
"User-Agent": (
|
||
"Mozilla/5.0 (Windows NT 10.0; Win64; x64) "
|
||
"AppleWebKit/537.36 (KHTML, like Gecko) "
|
||
"Chrome/123.0.0.0 Safari/537.36"
|
||
)
|
||
}
|
||
)
|
||
for target_url, timeout_seconds in test_targets:
|
||
try:
|
||
test_response = session.get(
|
||
target_url,
|
||
proxies=test_proxies,
|
||
timeout=timeout_seconds,
|
||
allow_redirects=True,
|
||
verify=False if target_url.startswith("https://") else True,
|
||
)
|
||
if test_response.status_code in (200, 204, 301, 302, 403):
|
||
result_payload["ok"] = True
|
||
result_payload["reason"] = f"ok:{target_url}:{test_response.status_code}"
|
||
break
|
||
last_reason = f"http_status_{test_response.status_code}@{target_url}"
|
||
except Exception as target_error:
|
||
error_text = str(target_error).strip() or target_error.__class__.__name__
|
||
last_reason = f"{target_error.__class__.__name__}@{target_url}:{error_text[:160]}"
|
||
session.close()
|
||
if not result_payload["ok"]:
|
||
result_payload["reason"] = last_reason
|
||
except Exception as e:
|
||
result_payload["reason"] = f"{e.__class__.__name__}:{str(e)[:160]}"
|
||
finally:
|
||
result_queue.put(result_payload)
|
||
result_queue.put(None)
|
||
|
||
def _extract_proxy_items(self, proxy_data):
|
||
if isinstance(proxy_data, dict):
|
||
if 'list' in proxy_data and isinstance(proxy_data['list'], list):
|
||
return proxy_data['list']
|
||
if 'ip' in proxy_data and 'port' in proxy_data:
|
||
return [proxy_data]
|
||
if isinstance(proxy_data, list):
|
||
return proxy_data
|
||
return []
|
||
|
||
def _proxy_source_tag_from_url(self, raw_url):
|
||
normalized_url = str(raw_url or "").strip()
|
||
if not normalized_url:
|
||
return "unknown"
|
||
try:
|
||
parsed = urllib.parse.urlsplit(normalized_url)
|
||
query = urllib.parse.parse_qs(parsed.query, keep_blank_values=True)
|
||
group_value = str((query.get("group") or [""])[0] or "").strip()
|
||
if group_value:
|
||
return group_value.upper()
|
||
if parsed.netloc:
|
||
return parsed.netloc
|
||
except Exception:
|
||
pass
|
||
return normalized_url
|
||
|
||
def _interleave_proxy_entries(self, proxy_entries):
|
||
if not proxy_entries:
|
||
return []
|
||
grouped_entries = collections.OrderedDict()
|
||
for proxy_entry in proxy_entries:
|
||
source_tag = str((proxy_entry or {}).get("source_tag") or "unknown").strip() or "unknown"
|
||
grouped_entries.setdefault(source_tag, collections.deque()).append(proxy_entry)
|
||
merged_entries = []
|
||
while grouped_entries:
|
||
exhausted_tags = []
|
||
for source_tag, items in grouped_entries.items():
|
||
if not items:
|
||
exhausted_tags.append(source_tag)
|
||
continue
|
||
merged_entries.append(items.popleft())
|
||
if not items:
|
||
exhausted_tags.append(source_tag)
|
||
for source_tag in exhausted_tags:
|
||
grouped_entries.pop(source_tag, None)
|
||
return merged_entries
|
||
|
||
def _merge_proxy_entries(self, new_entries, existing_entries):
|
||
merged = []
|
||
seen = set()
|
||
for proxy_entry in list(new_entries or []) + list(existing_entries or []):
|
||
if isinstance(proxy_entry, dict) and proxy_entry.get("proxy"):
|
||
signature = self._proxy_key(proxy_entry.get("proxy"))
|
||
else:
|
||
signature = self._proxy_key(proxy_entry)
|
||
if not signature or signature in seen:
|
||
continue
|
||
seen.add(signature)
|
||
merged.append(proxy_entry)
|
||
if merged and all(isinstance(item, dict) and item.get("proxy") for item in merged):
|
||
return self._interleave_proxy_entries(merged)
|
||
return merged
|
||
|
||
def _proxy_signature_from_item(self, proxy_item):
|
||
if not isinstance(proxy_item, dict):
|
||
return ""
|
||
ip = str(proxy_item.get("ip", "") or "").strip()
|
||
port = str(proxy_item.get("port", "") or "").strip()
|
||
username = str(proxy_item.get("username", "") or "").strip()
|
||
password = str(proxy_item.get("password", "") or "").strip()
|
||
if not ip or not port:
|
||
return ""
|
||
return json.dumps(
|
||
{
|
||
"ip": ip,
|
||
"port": port,
|
||
"username": username,
|
||
"password": password,
|
||
},
|
||
sort_keys=True,
|
||
ensure_ascii=False,
|
||
)
|
||
|
||
def _proxy_item_expire_timestamp(self, proxy_item):
|
||
if not isinstance(proxy_item, dict):
|
||
return 0.0
|
||
expire_at_ms = proxy_item.get("expire_at_ms")
|
||
expire_at = proxy_item.get("expire_at")
|
||
try:
|
||
if expire_at_ms not in (None, "", 0, "0"):
|
||
return float(expire_at_ms) / 1000.0
|
||
if expire_at not in (None, "", 0, "0"):
|
||
return float(expire_at)
|
||
except Exception:
|
||
return 0.0
|
||
return 0.0
|
||
|
||
def _proxy_entry_expire_timestamp(self, proxy_entry):
|
||
if isinstance(proxy_entry, dict):
|
||
expire_at_ts = proxy_entry.get("expire_at_ts")
|
||
try:
|
||
if expire_at_ts not in (None, "", 0, "0"):
|
||
return float(expire_at_ts)
|
||
except Exception:
|
||
return 0.0
|
||
raw_item = proxy_entry.get("raw_item")
|
||
if isinstance(raw_item, dict):
|
||
return float(self._proxy_item_expire_timestamp(raw_item) or 0.0)
|
||
return float(self._proxy_item_expire_timestamp(proxy_entry) or 0.0)
|
||
|
||
def _proxy_remaining_ttl_seconds(self, proxy_entry, *, now_ts=None):
|
||
expire_at_ts = float(self._proxy_entry_expire_timestamp(proxy_entry) or 0.0)
|
||
if expire_at_ts <= 0:
|
||
return float("inf")
|
||
observed_now_ts = float(now_ts or time.time())
|
||
return max(0.0, expire_at_ts - observed_now_ts)
|
||
|
||
def _proxy_min_reuse_ttl_seconds(self, step_name=""):
|
||
normalized_step_name = str(step_name or "").strip()
|
||
override_map = {
|
||
"注册状态检测": float(os.getenv("DOMAINCHECK_PROXY_REUSE_MIN_TTL_REGISTER", "6") or 6),
|
||
"百度site检测": float(os.getenv("DOMAINCHECK_PROXY_REUSE_MIN_TTL_BAIDU", "8") or 8),
|
||
"360检测": float(os.getenv("DOMAINCHECK_PROXY_REUSE_MIN_TTL_360", "8") or 8),
|
||
"站长之家检测": float(os.getenv("DOMAINCHECK_PROXY_REUSE_MIN_TTL_CHINAZ", "10") or 10),
|
||
"爱站网检测": float(os.getenv("DOMAINCHECK_PROXY_REUSE_MIN_TTL_AIZHAN", "10") or 10),
|
||
"聚查检测": float(os.getenv("DOMAINCHECK_PROXY_REUSE_MIN_TTL_JUCHA", "12") or 12),
|
||
"桔子检测": float(os.getenv("DOMAINCHECK_PROXY_REUSE_MIN_TTL_JUZISEO", "12") or 12),
|
||
}
|
||
if normalized_step_name in override_map:
|
||
return max(0.0, float(override_map.get(normalized_step_name, 0.0) or 0.0))
|
||
configured_default = float(os.getenv("DOMAINCHECK_PROXY_REUSE_MIN_TTL_SECONDS", "8") or 8)
|
||
return max(0.0, configured_default)
|
||
|
||
def _proxy_entry_can_be_reused(self, proxy_entry, step_name="", *, now_ts=None):
|
||
remaining_ttl_seconds = self._proxy_remaining_ttl_seconds(proxy_entry, now_ts=now_ts)
|
||
if remaining_ttl_seconds == float("inf"):
|
||
return True
|
||
return remaining_ttl_seconds >= self._proxy_min_reuse_ttl_seconds(step_name)
|
||
|
||
def _store_proxy_active_lease(self, proxy_entry, *, step_name="", already_locked=False):
|
||
if not isinstance(proxy_entry, dict):
|
||
return
|
||
proxy = proxy_entry.get("proxy")
|
||
proxy_key = self._proxy_key(proxy)
|
||
if not proxy or not proxy_key:
|
||
return
|
||
|
||
def _store():
|
||
active_leases = getattr(self, "proxy_active_leases", None)
|
||
if active_leases is None:
|
||
self.proxy_active_leases = {}
|
||
active_leases = self.proxy_active_leases
|
||
lease_entry = dict(proxy_entry)
|
||
lease_entry["leased_at"] = time.time()
|
||
lease_entry["leased_step_name"] = str(step_name or proxy_entry.get("leased_step_name") or "").strip()
|
||
active_leases[proxy_key] = lease_entry
|
||
|
||
if already_locked:
|
||
_store()
|
||
return
|
||
with self.proxy_pool_lock:
|
||
_store()
|
||
|
||
def _pop_proxy_active_lease(self, proxy, *, already_locked=False):
|
||
proxy_key = self._proxy_key(proxy)
|
||
if not proxy_key:
|
||
return None
|
||
|
||
def _pop():
|
||
active_leases = getattr(self, "proxy_active_leases", None)
|
||
if not isinstance(active_leases, dict):
|
||
return None
|
||
return active_leases.pop(proxy_key, None)
|
||
|
||
if already_locked:
|
||
return _pop()
|
||
with self.proxy_pool_lock:
|
||
return _pop()
|
||
|
||
def release_proxy(self, proxy, *, step_name="", discard=False, reason=""):
|
||
if not proxy:
|
||
return False
|
||
|
||
with self.proxy_pool_lock:
|
||
lease_entry = self._pop_proxy_active_lease(proxy, already_locked=True)
|
||
if not isinstance(lease_entry, dict):
|
||
return False
|
||
remaining_count = len(self.proxy_pool)
|
||
|
||
normalized_step_name = str(step_name or lease_entry.get("leased_step_name") or "").strip()
|
||
source_tag = str(lease_entry.get("source_tag") or "").strip()
|
||
reusable = (
|
||
not discard
|
||
and not self._is_proxy_quarantined(proxy)
|
||
and not self._is_proxy_source_quarantined(source_tag)
|
||
and self._proxy_entry_can_be_reused(lease_entry, normalized_step_name)
|
||
)
|
||
if reusable:
|
||
lease_entry.pop("leased_at", None)
|
||
lease_entry.pop("leased_step_name", None)
|
||
with self.proxy_pool_lock:
|
||
self.proxy_pool.append(lease_entry)
|
||
remaining_count = len(self.proxy_pool)
|
||
|
||
self._schedule_proxy_refresh_if_needed(remaining_count)
|
||
if not reusable:
|
||
remaining_ttl_seconds = self._proxy_remaining_ttl_seconds(lease_entry)
|
||
if remaining_ttl_seconds != float("inf"):
|
||
logger.debug(
|
||
f"代理归还时已丢弃: step={normalized_step_name or '-'} "
|
||
f"ttl={int(remaining_ttl_seconds)}s reason={reason or ('discard' if discard else 'ttl_low')}"
|
||
)
|
||
return reusable
|
||
|
||
def _is_proxy_item_expired(self, proxy_item, *, grace_seconds=0.0):
|
||
if not isinstance(proxy_item, dict):
|
||
return True
|
||
now_ts = time.time()
|
||
expire_at_ts = float(self._proxy_item_expire_timestamp(proxy_item) or 0.0)
|
||
if expire_at_ts <= 0:
|
||
return False
|
||
return expire_at_ts <= (now_ts - max(0.0, float(grace_seconds or 0.0)))
|
||
|
||
def _summarize_proxy_expire_clock_skew(self, proxy_items, *, now_ts=None):
|
||
if not isinstance(proxy_items, list) or not proxy_items:
|
||
return None
|
||
observed_now_ts = float(now_ts or time.time())
|
||
expire_timestamps = []
|
||
for proxy_item in proxy_items:
|
||
expire_at_ts = float(self._proxy_item_expire_timestamp(proxy_item) or 0.0)
|
||
if expire_at_ts > 0:
|
||
expire_timestamps.append(expire_at_ts)
|
||
if not expire_timestamps:
|
||
return None
|
||
|
||
latest_expire_at = max(expire_timestamps)
|
||
earliest_expire_at = min(expire_timestamps)
|
||
expired_timed_count = sum(1 for expire_at_ts in expire_timestamps if expire_at_ts <= observed_now_ts)
|
||
# 所有带过期时间的代理都比本机时间早很多时，优先提示“时钟异常/源数据陈旧”，
|
||
# 避免把问题误判成单纯的代理质量差。
|
||
if latest_expire_at > observed_now_ts - 900:
|
||
return None
|
||
if expired_timed_count < max(12, int(len(expire_timestamps) * 0.8)):
|
||
return None
|
||
|
||
return {
|
||
"observed_now_ts": observed_now_ts,
|
||
"observed_now_iso": datetime.fromtimestamp(observed_now_ts).strftime("%Y-%m-%d %H:%M:%S"),
|
||
"timed_proxy_count": len(expire_timestamps),
|
||
"expired_timed_count": expired_timed_count,
|
||
"latest_expire_at": latest_expire_at,
|
||
"latest_expire_at_iso": datetime.fromtimestamp(latest_expire_at).strftime("%Y-%m-%d %H:%M:%S"),
|
||
"earliest_expire_at": earliest_expire_at,
|
||
"earliest_expire_at_iso": datetime.fromtimestamp(earliest_expire_at).strftime("%Y-%m-%d %H:%M:%S"),
|
||
"clock_ahead_seconds": max(0.0, observed_now_ts - latest_expire_at),
|
||
}
|
||
|
||
def _proxy_recently_expired_reuse_grace_seconds(self):
|
||
configured_grace = float(os.getenv("DOMAINCHECK_PROXY_STALE_REUSE_GRACE_SECONDS", "0") or 0.0)
|
||
if configured_grace > 0:
|
||
return max(0.0, configured_grace)
|
||
if not self.detecting:
|
||
return 0.0
|
||
demand_threads = self._proxy_demand_threads()
|
||
if demand_threads >= 1600:
|
||
return 180.0
|
||
if demand_threads >= 1000:
|
||
return 120.0
|
||
return 90.0
|
||
|
||
def _proxy_recently_expired_backfill_floor(self):
|
||
configured_floor = int(os.getenv("DOMAINCHECK_PROXY_STALE_BACKFILL_FLOOR", "0") or 0)
|
||
if configured_floor > 0:
|
||
return max(8, configured_floor)
|
||
demand_threads = self._proxy_demand_threads()
|
||
return max(32, min(160, max(32, demand_threads // 12)))
|
||
|
||
def _backfill_recently_expired_proxy_entries(self, current_entries, stale_entries):
|
||
merged_entries = list(current_entries or [])
|
||
normalized_stale_entries = list(stale_entries or [])
|
||
if not self.detecting or not normalized_stale_entries:
|
||
return merged_entries, 0, 0
|
||
floor = self._proxy_recently_expired_backfill_floor()
|
||
if len(merged_entries) >= floor:
|
||
return merged_entries, 0, floor
|
||
before_count = len(merged_entries)
|
||
needed = max(0, floor - before_count)
|
||
if needed <= 0:
|
||
return merged_entries, 0, floor
|
||
merged_entries = self._merge_proxy_entries(normalized_stale_entries[:needed], merged_entries)
|
||
return merged_entries, max(0, len(merged_entries) - before_count), floor
|
||
|
||
def _set_proxy_request_count(self, raw_url, count):
|
||
normalized_url = str(raw_url or "").strip()
|
||
if not normalized_url:
|
||
return ""
|
||
try:
|
||
parsed = urllib.parse.urlsplit(normalized_url)
|
||
query = urllib.parse.parse_qs(parsed.query, keep_blank_values=True)
|
||
query["count"] = [str(max(1, int(count or 0)))]
|
||
return urllib.parse.urlunsplit(
|
||
(
|
||
parsed.scheme,
|
||
parsed.netloc,
|
||
parsed.path,
|
||
urllib.parse.urlencode(query, doseq=True),
|
||
parsed.fragment,
|
||
)
|
||
)
|
||
except Exception:
|
||
return normalized_url
|
||
|
||
def _prepare_proxy_fetch_plan(self, proxy_api_urls, current_pool_size=0):
|
||
source_urls = [str(url or "").strip() for url in proxy_api_urls if str(url or "").strip()]
|
||
if not source_urls:
|
||
return [], {
|
||
"target_total": 0,
|
||
"batch_size": 0,
|
||
"rounds": 0,
|
||
"source_count": 0,
|
||
}
|
||
|
||
demand_threads = self._proxy_demand_threads()
|
||
process_count = self._proxy_runtime_process_count()
|
||
source_count = len(source_urls)
|
||
provider_batch_cap = max(20, int(os.getenv("DOMAINCHECK_PROXY_PROVIDER_BATCH_CAP", "140") or 140))
|
||
provider_batch_floor = max(
|
||
20,
|
||
min(
|
||
provider_batch_cap,
|
||
int(round(provider_batch_cap / self._proxy_multi_process_divisor())),
|
||
),
|
||
)
|
||
current_pool_size = max(0, int(current_pool_size or 0))
|
||
refresh_threshold = self._proxy_refresh_threshold()
|
||
shortage = max(0, refresh_threshold - current_pool_size)
|
||
headroom = max(source_count * 10, int(refresh_threshold * 0.15))
|
||
active_round_cap = max(1, int(os.getenv("DOMAINCHECK_PROXY_MAX_ROUNDS_ACTIVE", "2") or 2))
|
||
idle_round_cap = max(2, int(os.getenv("DOMAINCHECK_PROXY_MAX_ROUNDS_IDLE", "4") or 4))
|
||
round_cap = active_round_cap if self.detecting else idle_round_cap
|
||
target_total = max(
|
||
source_count * provider_batch_floor,
|
||
shortage + headroom,
|
||
)
|
||
per_round_capacity = max(1, source_count * provider_batch_cap)
|
||
rounds = max(1, min(round_cap, (target_total + per_round_capacity - 1) // per_round_capacity))
|
||
batch_size = max(
|
||
20,
|
||
min(
|
||
provider_batch_cap,
|
||
(target_total + (source_count * rounds) - 1) // (source_count * rounds),
|
||
),
|
||
)
|
||
fetch_plan = []
|
||
for round_index in range(rounds):
|
||
for source_url in source_urls:
|
||
fetch_plan.append(
|
||
{
|
||
"source_url": source_url,
|
||
"request_url": self._set_proxy_request_count(source_url, batch_size),
|
||
"batch_size": batch_size,
|
||
"round": round_index + 1,
|
||
}
|
||
)
|
||
|
||
return fetch_plan, {
|
||
"target_total": target_total,
|
||
"batch_size": batch_size,
|
||
"rounds": rounds,
|
||
"source_count": source_count,
|
||
"demand_threads": demand_threads,
|
||
"process_count": process_count,
|
||
"current_pool_size": current_pool_size,
|
||
"refresh_threshold": refresh_threshold,
|
||
"shortage": shortage,
|
||
"provider_batch_floor": provider_batch_floor,
|
||
}
|
||
|
||
def _proxy_source_fetch_timeout_seconds(self):
|
||
default_timeout = "4.0" if self.detecting else "6.0"
|
||
return max(
|
||
1.5,
|
||
float(os.getenv("DOMAINCHECK_PROXY_SOURCE_FETCH_TIMEOUT", default_timeout) or default_timeout),
|
||
)
|
||
|
||
def _proxy_key(self, proxy):
|
||
if isinstance(proxy, dict):
|
||
return json.dumps(proxy, sort_keys=True, ensure_ascii=False)
|
||
return str(proxy)
|
||
|
||
def _is_proxy_quarantined(self, proxy):
|
||
proxy_key = self._proxy_key(proxy)
|
||
now_ts = time.time()
|
||
with self.proxy_failure_lock:
|
||
retry_after = float(self.proxy_quarantine_until.get(proxy_key, 0) or 0)
|
||
if retry_after <= now_ts:
|
||
if proxy_key in self.proxy_quarantine_until:
|
||
self.proxy_quarantine_until.pop(proxy_key, None)
|
||
self.proxy_failure_counts.pop(proxy_key, None)
|
||
return False
|
||
return True
|
||
|
||
def _mark_proxy_failure(self, proxy):
|
||
proxy_key = self._proxy_key(proxy)
|
||
with self.proxy_failure_lock:
|
||
failure_count = int(self.proxy_failure_counts.get(proxy_key, 0) or 0) + 1
|
||
self.proxy_failure_counts[proxy_key] = failure_count
|
||
cooldown_seconds = min(180, 20 * failure_count)
|
||
retry_after = time.time() + cooldown_seconds
|
||
self.proxy_quarantine_until[proxy_key] = retry_after
|
||
logger.info(f"代理临时隔离 {cooldown_seconds}s: {proxy}")
|
||
return failure_count, cooldown_seconds
|
||
|
||
def _clear_proxy_failure(self, proxy):
|
||
proxy_key = self._proxy_key(proxy)
|
||
with self.proxy_failure_lock:
|
||
self.proxy_failure_counts.pop(proxy_key, None)
|
||
self.proxy_quarantine_until.pop(proxy_key, None)
|
||
|
||
def _proxy_source_tag_for_proxy(self, proxy):
|
||
proxy_key = self._proxy_key(proxy)
|
||
with self.proxy_failure_lock:
|
||
source_map = getattr(self, "proxy_source_by_key", None) or {}
|
||
return str(source_map.get(proxy_key, "") or "").strip()
|
||
|
||
def _is_proxy_source_quarantined(self, source_tag):
|
||
normalized_source_tag = str(source_tag or "").strip()
|
||
if not normalized_source_tag:
|
||
return False
|
||
now_ts = time.time()
|
||
with self.proxy_failure_lock:
|
||
quarantine_map = getattr(self, "proxy_source_quarantine_until", None) or {}
|
||
retry_after = float(quarantine_map.get(normalized_source_tag, 0) or 0)
|
||
if retry_after <= now_ts:
|
||
if normalized_source_tag in quarantine_map:
|
||
quarantine_map.pop(normalized_source_tag, None)
|
||
(getattr(self, "proxy_source_failure_counts", None) or {}).pop(normalized_source_tag, None)
|
||
return False
|
||
return True
|
||
|
||
def _is_proxy_auth_issue(self, message):
|
||
lowered = str(message or "").lower()
|
||
return any(
|
||
keyword in lowered
|
||
for keyword in (
|
||
"407 proxy authentication required",
|
||
"proxy authentication required",
|
||
"tunnel connection failed: 407",
|
||
"status code 407",
|
||
)
|
||
)
|
||
|
||
def _remove_proxy_source_from_pool(self, source_tag):
|
||
normalized_source_tag = str(source_tag or "").strip()
|
||
if not normalized_source_tag:
|
||
return 0
|
||
removed = 0
|
||
with self.proxy_pool_lock:
|
||
filtered_pool = []
|
||
for proxy_item in list(getattr(self, "proxy_pool", []) or []):
|
||
item_source_tag = ""
|
||
if isinstance(proxy_item, dict):
|
||
item_source_tag = str(proxy_item.get("source_tag") or "").strip()
|
||
if item_source_tag and item_source_tag == normalized_source_tag:
|
||
removed += 1
|
||
continue
|
||
filtered_pool.append(proxy_item)
|
||
if removed:
|
||
self.proxy_pool = filtered_pool
|
||
return removed
|
||
|
||
def _mark_proxy_source_failure(self, proxy, reason=""):
|
||
source_tag = self._proxy_source_tag_for_proxy(proxy)
|
||
if not source_tag:
|
||
return "", 0, 0, 0
|
||
|
||
with self.proxy_failure_lock:
|
||
failure_counts = getattr(self, "proxy_source_failure_counts", None)
|
||
if failure_counts is None:
|
||
self.proxy_source_failure_counts = {}
|
||
failure_counts = self.proxy_source_failure_counts
|
||
quarantine_map = getattr(self, "proxy_source_quarantine_until", None)
|
||
if quarantine_map is None:
|
||
self.proxy_source_quarantine_until = {}
|
||
quarantine_map = self.proxy_source_quarantine_until
|
||
failure_count = int(failure_counts.get(source_tag, 0) or 0) + 1
|
||
failure_counts[source_tag] = failure_count
|
||
if self._is_proxy_auth_issue(reason):
|
||
cooldown_seconds = min(600, max(90, 90 * failure_count))
|
||
else:
|
||
cooldown_seconds = min(300, max(30, 30 * failure_count))
|
||
quarantine_map[source_tag] = time.time() + cooldown_seconds
|
||
|
||
removed = self._remove_proxy_source_from_pool(source_tag)
|
||
logger.warning(
|
||
f"代理来源已临时隔离 {cooldown_seconds}s: source={source_tag}, failures={failure_count}, removed={removed}, reason={reason}"
|
||
)
|
||
return source_tag, failure_count, cooldown_seconds, removed
|
||
|
||
def _clear_proxy_source_failure(self, proxy):
|
||
source_tag = self._proxy_source_tag_for_proxy(proxy)
|
||
if not source_tag:
|
||
return
|
||
with self.proxy_failure_lock:
|
||
(getattr(self, "proxy_source_failure_counts", None) or {}).pop(source_tag, None)
|
||
(getattr(self, "proxy_source_quarantine_until", None) or {}).pop(source_tag, None)
|
||
|
||
def _schedule_proxy_refresh_if_needed(self, available_count=None):
|
||
if available_count is None:
|
||
with self.proxy_pool_lock:
|
||
current_count = len(self.proxy_pool)
|
||
else:
|
||
# 调用方可能已经持有 proxy_pool_lock；避免对同一把非重入锁再次加锁。
|
||
current_count = int(available_count or 0)
|
||
threshold = self._proxy_refresh_threshold()
|
||
if float(getattr(self, "proxy_next_refresh_time", 0.0) or 0.0) > time.time():
|
||
return
|
||
if current_count < threshold and not self.proxy_refresh_lock.locked():
|
||
threading.Thread(target=self.refresh_proxy_pool, daemon=True).start()
|
||
|
||
def _has_proxy_sources(self):
|
||
proxy_urls = self.proxy_config.get('proxy_urls') or []
|
||
if not proxy_urls and self.proxy_config.get('proxy_url'):
|
||
proxy_urls = [self.proxy_config.get('proxy_url', '')]
|
||
return bool([url for url in proxy_urls if str(url or '').strip()])
|
||
|
||
def trigger_proxy_refresh(self, reason="", reset_cooldown=False, debounce_seconds=None):
|
||
"""
|
||
代理配置一旦下发，就允许在非检测态主动补一次刷新，避免页面长期停在“未刷新”。
|
||
"""
|
||
if not self.proxy_config.get('proxy_enable', False):
|
||
return False
|
||
if not self._has_proxy_sources():
|
||
return False
|
||
normalized_reason = str(reason or "manual").strip() or "manual"
|
||
if debounce_seconds is None and normalized_reason.startswith("config_update:"):
|
||
debounce_seconds = self._config_update_proxy_refresh_debounce_seconds()
|
||
debounce_seconds = max(0.0, float(debounce_seconds or 0.0))
|
||
refresh_reason_timestamps = getattr(self, "_last_proxy_refresh_reason_at", None)
|
||
if not isinstance(refresh_reason_timestamps, dict):
|
||
refresh_reason_timestamps = {}
|
||
self._last_proxy_refresh_reason_at = refresh_reason_timestamps
|
||
trigger_key = "config_update" if normalized_reason.startswith("config_update:") else normalized_reason
|
||
now_ts = time.time()
|
||
last_trigger_at = float(refresh_reason_timestamps.get(trigger_key, 0.0) or 0.0)
|
||
if debounce_seconds > 0 and (now_ts - last_trigger_at) < debounce_seconds:
|
||
logger.debug(
|
||
f"代理池刷新触发去抖，跳过主动触发: {normalized_reason}, cooldown={debounce_seconds:.1f}s"
|
||
)
|
||
return False
|
||
if reset_cooldown:
|
||
self.proxy_next_refresh_time = 0.0
|
||
elif float(getattr(self, "proxy_next_refresh_time", 0.0) or 0.0) > time.time():
|
||
logger.debug(f"代理池仍在冷却窗口，跳过主动触发: {normalized_reason}")
|
||
return False
|
||
if self.proxy_refresh_lock.locked():
|
||
logger.debug(f"代理池刷新已在进行中，跳过主动触发: {normalized_reason}")
|
||
return False
|
||
refresh_reason_timestamps[trigger_key] = now_ts
|
||
logger.info(f"主动触发代理池刷新: {normalized_reason}")
|
||
threading.Thread(target=self.refresh_proxy_pool, daemon=True).start()
|
||
return True
|
||
|
||
def _proxy_refresh_threshold(self):
|
||
"""
|
||
代理池的补货阈值只用于触发后台刷新，不应反向限制并发。
|
||
"""
|
||
demand_threads = self._proxy_demand_threads()
|
||
base_threshold = max(840, demand_threads * 3)
|
||
process_count = self._proxy_runtime_process_count()
|
||
if process_count <= 1:
|
||
return base_threshold
|
||
scaled_threshold = int(round(base_threshold / self._proxy_multi_process_divisor()))
|
||
process_floor = max(120, min(480, max(120, demand_threads // 4)))
|
||
return max(process_floor, scaled_threshold)
|
||
|
||
def _proxy_demand_threads(self):
|
||
configured_threads = max(1, int(getattr(self, "thread_count", 0) or 0))
|
||
live_active_threads = max(0, int(self._get_active_domain_threads() or 0))
|
||
if self.detecting:
|
||
warm_floor = min(configured_threads, max(400, configured_threads // 2))
|
||
demand_threads = max(live_active_threads, warm_floor)
|
||
else:
|
||
demand_threads = min(configured_threads, 64)
|
||
return max(1, demand_threads)
|
||
|
||
def _should_emit_thread_progress(self, current_active, max_threads, last_emit_count, last_emit_at):
|
||
"""
|
||
线程创建/回收阶段的活跃数上报做节流，避免每起一个线程就同步
|
||
一次远端日志和运行态，反向把并发拉低。
|
||
"""
|
||
now = time.time()
|
||
step = max(1, min(10, max_threads // 10))
|
||
if current_active == 1:
|
||
return True, now
|
||
if current_active >= max_threads:
|
||
return True, now
|
||
if current_active == 0 and last_emit_count != 0:
|
||
return True, now
|
||
if current_active - last_emit_count >= step:
|
||
return True, now
|
||
if current_active > 0 and now - last_emit_at >= 5.0:
|
||
return True, now
|
||
return False, now
|
||
|
||
def _wait_for_proxy_refresh_settle(self, timeout_seconds=2.0):
|
||
deadline = time.time() + max(0.0, float(timeout_seconds or 0.0))
|
||
while self.proxy_refresh_lock.locked() and time.time() < deadline:
|
||
with self.proxy_pool_lock:
|
||
if self.proxy_pool:
|
||
return True
|
||
time.sleep(0.05)
|
||
with self.proxy_pool_lock:
|
||
return bool(self.proxy_pool)
|
||
|
||
def _prepare_proxy_pool_for_detection_start(self):
|
||
"""
|
||
检测会话启动时，优先复用当前可用代理池/共享快照。
|
||
只有在本地和共享快照都为空时，才做同步刷新。
|
||
否则 large controller 会在每轮 auto-resume 前都卡一次 refresh_proxy_pool，
|
||
线程池还没起就先停在 refreshing_proxy。
|
||
"""
|
||
with self.proxy_pool_lock:
|
||
current_pool_size = len(self.proxy_pool)
|
||
if current_pool_size > 0:
|
||
self._schedule_proxy_refresh_if_needed(current_pool_size)
|
||
return True, current_pool_size, "local_pool_ready"
|
||
|
||
if self._proxy_should_coordinate_shared_refresh():
|
||
shared_payload = self._load_shared_proxy_snapshot_payload()
|
||
if shared_payload and self._apply_shared_proxy_snapshot(
|
||
shared_payload,
|
||
status_prefix="检测启动复用共享代理快照",
|
||
):
|
||
with self.proxy_pool_lock:
|
||
shared_pool_size = len(self.proxy_pool)
|
||
self._schedule_proxy_refresh_if_needed(shared_pool_size)
|
||
return True, shared_pool_size, "shared_snapshot_ready"
|
||
|
||
return False, 0, "proxy_pool_empty"
|
||
|
||
def refresh_proxy_pool(self):
|
||
"""
|
||
刷新代理池
|
||
"""
|
||
if not self.proxy_refresh_lock.acquire(blocking=False):
|
||
logger.debug("代理池刷新已在进行中，跳过本次重复刷新")
|
||
return
|
||
|
||
shared_refresh_owner = ""
|
||
if not self.proxy_config.get('proxy_enable', False):
|
||
self.proxy_last_refresh_time = datetime.now()
|
||
self.proxy_last_refresh_status = "代理未启用"
|
||
self.proxy_last_refresh_source_count = 0
|
||
self.proxy_last_refresh_total_items = 0
|
||
self.proxy_last_validated_count = 0
|
||
self.proxy_last_available_count = 0
|
||
self.proxy_last_source_stats = []
|
||
self.proxy_next_refresh_time = 0.0
|
||
self._update_runtime_state(
|
||
"idle" if not self.detecting else "running",
|
||
"代理未启用，使用直接连接",
|
||
)
|
||
self._sync_worker_log_event("代理未启用，使用直接连接", mode='key')
|
||
self.proxy_refresh_lock.release()
|
||
return
|
||
|
||
try:
|
||
now_ts = time.time()
|
||
retry_delay_seconds = 0.15 if self.detecting else min(self.proxy_refresh_cooldown_seconds, 10)
|
||
holdoff_seconds = self._proxy_refresh_holdoff_seconds()
|
||
refresh_threshold = self._proxy_refresh_threshold()
|
||
with self.proxy_pool_lock:
|
||
has_cached_proxies = bool(self.proxy_pool)
|
||
cached_proxy_pool = list(self.proxy_pool)
|
||
|
||
if self._proxy_should_coordinate_shared_refresh():
|
||
shared_payload = self._load_shared_proxy_snapshot_payload()
|
||
if shared_payload:
|
||
shared_entries = shared_payload.get("proxy_pool") or []
|
||
if self._shared_proxy_snapshot_should_replace_local_pool(
|
||
len(shared_entries),
|
||
len(cached_proxy_pool),
|
||
refresh_threshold,
|
||
):
|
||
if self._apply_shared_proxy_snapshot(shared_payload):
|
||
self.proxy_next_refresh_time = time.time() + holdoff_seconds
|
||
return
|
||
|
||
if not has_cached_proxies and self.proxy_next_refresh_time and now_ts < self.proxy_next_refresh_time:
|
||
wait_seconds = int(max(1, self.proxy_next_refresh_time - now_ts))
|
||
self.proxy_last_refresh_status = f"冷却中，{wait_seconds} 秒后再试"
|
||
logger.info(f"代理池刷新冷却中，{wait_seconds} 秒后再试")
|
||
self._update_runtime_state(
|
||
"refreshing_proxy" if self.detecting else "idle",
|
||
f"代理池刷新冷却中，{wait_seconds} 秒后再试",
|
||
)
|
||
self._sync_worker_log_event(f"代理池刷新冷却中，{wait_seconds} 秒后再试", mode='full')
|
||
return
|
||
|
||
if self._proxy_should_coordinate_shared_refresh():
|
||
shared_refresh_owner = self._acquire_shared_proxy_refresh_lock()
|
||
if not shared_refresh_owner:
|
||
if self._wait_for_shared_proxy_snapshot():
|
||
self.proxy_next_refresh_time = time.time() + holdoff_seconds
|
||
return
|
||
if cached_proxy_pool:
|
||
with self.proxy_pool_lock:
|
||
self.proxy_pool = cached_proxy_pool
|
||
self.proxy_last_refresh_time = datetime.now()
|
||
self.proxy_last_refresh_total_items = 0
|
||
self.proxy_last_validated_count = 0
|
||
self.proxy_last_available_count = len(self.proxy_pool)
|
||
self.proxy_last_refresh_status = f"共享刷新进行中，继续沿用缓存 {len(cached_proxy_pool)} 个"
|
||
else:
|
||
self.proxy_last_refresh_time = datetime.now()
|
||
self.proxy_last_refresh_total_items = 0
|
||
self.proxy_last_validated_count = 0
|
||
self.proxy_last_available_count = 0
|
||
self.proxy_last_refresh_status = "共享代理刷新进行中，等待补货"
|
||
self.proxy_next_refresh_time = time.time() + retry_delay_seconds + holdoff_seconds
|
||
self._update_runtime_state(
|
||
"refreshing_proxy" if self.detecting else "idle",
|
||
self.proxy_last_refresh_status,
|
||
)
|
||
self._sync_worker_log_event(self.proxy_last_refresh_status, mode='key')
|
||
return
|
||
|
||
proxy_api_urls = self.proxy_config.get('proxy_urls') or []
|
||
if not proxy_api_urls and self.proxy_config.get('proxy_url'):
|
||
proxy_api_urls = [self.proxy_config.get('proxy_url', '')]
|
||
proxy_api_urls = [url for url in proxy_api_urls if url]
|
||
proxy_fetch_plan, fetch_plan_meta = self._prepare_proxy_fetch_plan(
|
||
proxy_api_urls,
|
||
current_pool_size=len(cached_proxy_pool),
|
||
)
|
||
source_count = int(fetch_plan_meta.get("source_count", 0) or 0)
|
||
batch_size = int(fetch_plan_meta.get("batch_size", 0) or 0)
|
||
target_total = int(fetch_plan_meta.get("target_total", 0) or 0)
|
||
rounds = int(fetch_plan_meta.get("rounds", 0) or 0)
|
||
refresh_threshold = int(fetch_plan_meta.get("refresh_threshold", 0) or refresh_threshold)
|
||
self.proxy_last_refresh_source_count = source_count
|
||
self.proxy_last_source_stats = []
|
||
if proxy_fetch_plan:
|
||
import requests
|
||
proxy_list = []
|
||
cooldown_detected = False
|
||
|
||
aggressive_proxy_rotation = str(os.getenv("DOMAINCHECK_PROXY_FAST_ROTATION", "1") or "1").strip().lower() not in {
|
||
"",
|
||
"0",
|
||
"false",
|
||
"no",
|
||
"off",
|
||
}
|
||
|
||
def fetch_proxy_source(plan_item):
|
||
source_url = str(plan_item.get("source_url") or "").strip()
|
||
requested_batch_size = max(1, int(plan_item.get("batch_size", 0) or batch_size or 1))
|
||
round_index = int(plan_item.get("round", 1) or 1)
|
||
source_stat = {
|
||
"url": source_url,
|
||
"source_url": source_url,
|
||
"status": "unknown",
|
||
"http_status": 0,
|
||
"raw_items": 0,
|
||
"error": "",
|
||
"requested_batch_size": requested_batch_size,
|
||
"effective_batch_size": requested_batch_size,
|
||
"round": round_index,
|
||
}
|
||
current_items = []
|
||
batch_attempts = []
|
||
if self.detecting and aggressive_proxy_rotation:
|
||
dynamic_fallbacks = [
|
||
requested_batch_size,
|
||
max(80, requested_batch_size // 2),
|
||
80,
|
||
50,
|
||
20,
|
||
10,
|
||
]
|
||
else:
|
||
dynamic_fallbacks = [
|
||
requested_batch_size,
|
||
max(500, requested_batch_size // 2),
|
||
200,
|
||
100,
|
||
50,
|
||
20,
|
||
10,
|
||
]
|
||
for candidate in dynamic_fallbacks:
|
||
candidate = max(1, int(candidate or 0))
|
||
if candidate not in batch_attempts:
|
||
batch_attempts.append(candidate)
|
||
|
||
for candidate_batch_size in batch_attempts:
|
||
request_url = self._set_proxy_request_count(source_url, candidate_batch_size)
|
||
source_stat["effective_batch_size"] = candidate_batch_size
|
||
try:
|
||
response = requests.get(
|
||
request_url,
|
||
timeout=self._proxy_source_fetch_timeout_seconds(),
|
||
)
|
||
source_stat["http_status"] = int(response.status_code or 0)
|
||
if response.status_code != 200:
|
||
source_stat["status"] = "http_error"
|
||
source_stat["error"] = f"HTTP {response.status_code}"
|
||
logger.warning(f"代理池链接响应异常: {request_url}, 状态码: {response.status_code}")
|
||
continue
|
||
|
||
proxy_data = response.json()
|
||
current_items = self._extract_proxy_items(proxy_data)
|
||
source_tag = self._proxy_source_tag_from_url(source_url)
|
||
annotated_items = []
|
||
for proxy_item in current_items:
|
||
if isinstance(proxy_item, dict):
|
||
item_copy = dict(proxy_item)
|
||
item_copy.setdefault("source_url", source_url)
|
||
item_copy.setdefault("source_tag", source_tag)
|
||
annotated_items.append(item_copy)
|
||
current_items = annotated_items
|
||
source_stat["raw_items"] = len(current_items)
|
||
if current_items:
|
||
source_stat["status"] = "ok"
|
||
logger.info(
|
||
f"代理池链接拉取成功: {request_url}, 原始代理数: {len(current_items)}, 轮次: {round_index}"
|
||
)
|
||
break
|
||
|
||
error_message = ""
|
||
if isinstance(proxy_data, dict):
|
||
error_message = str(proxy_data.get("error") or "").strip()
|
||
source_stat["status"] = "cooldown" if "cooldown" in error_message.lower() else "empty"
|
||
source_stat["error"] = error_message or "empty"
|
||
logger.info(
|
||
f"代理池链接空载，降批重试: {request_url}, 轮次: {round_index}, 提示: {source_stat['error']}"
|
||
)
|
||
if source_stat["status"] == "cooldown":
|
||
break
|
||
except Exception as api_error:
|
||
source_stat["status"] = "request_error"
|
||
source_stat["error"] = str(api_error)
|
||
logger.warning(f"代理池链接拉取失败: {request_url}, 错误: {api_error}")
|
||
source_stat["url"] = self._set_proxy_request_count(source_url, source_stat["effective_batch_size"])
|
||
return source_stat, current_items
|
||
|
||
round_plan_map = {}
|
||
for plan_item in proxy_fetch_plan:
|
||
round_key = max(1, int(plan_item.get("round", 1) or 1))
|
||
round_plan_map.setdefault(round_key, []).append(plan_item)
|
||
|
||
round_keys = sorted(round_plan_map.keys())
|
||
per_round_workers = max(1, min(source_count or len(proxy_api_urls) or 1, 12))
|
||
cooled_source_urls = set()
|
||
for round_index in round_keys:
|
||
round_plan_items = [
|
||
item
|
||
for item in (round_plan_map.get(round_index) or [])
|
||
if str(item.get("source_url") or "").strip() not in cooled_source_urls
|
||
]
|
||
if not round_plan_items:
|
||
continue
|
||
with ThreadPoolExecutor(max_workers=min(per_round_workers, len(round_plan_items))) as executor:
|
||
futures = [executor.submit(fetch_proxy_source, plan_item) for plan_item in round_plan_items]
|
||
for future in as_completed(futures):
|
||
source_stat, current_items = future.result()
|
||
if "cooldown" in str(source_stat.get("error") or "").lower() or source_stat.get("status") == "cooldown":
|
||
cooldown_detected = True
|
||
cooled_source_urls.add(str(source_stat.get("source_url") or "").strip())
|
||
self.proxy_last_source_stats.append(source_stat)
|
||
proxy_list.extend(current_items)
|
||
if len(proxy_list) >= target_total:
|
||
break
|
||
if self.detecting and aggressive_proxy_rotation and round_index < len(round_keys):
|
||
time.sleep(0.15 if cooldown_detected else 0.05)
|
||
|
||
if proxy_list:
|
||
self.proxy_last_refresh_total_items = len(proxy_list)
|
||
self.proxy_last_validated_count = 0
|
||
|
||
new_proxies = []
|
||
fallback_stale_proxies = []
|
||
seen_proxy_strings = set()
|
||
expired_count = 0
|
||
invalid_count = 0
|
||
stale_reuse_grace_seconds = self._proxy_recently_expired_reuse_grace_seconds()
|
||
|
||
for proxy_item in proxy_list:
|
||
signature = self._proxy_signature_from_item(proxy_item)
|
||
if not signature:
|
||
invalid_count += 1
|
||
continue
|
||
if signature in seen_proxy_strings:
|
||
continue
|
||
seen_proxy_strings.add(signature)
|
||
ip = str(proxy_item.get("ip", "") or "").strip()
|
||
port = str(proxy_item.get("port", "") or "").strip()
|
||
username = str(proxy_item.get("username", "") or "").strip()
|
||
password = str(proxy_item.get("password", "") or "").strip()
|
||
proxy_url = f"http://{username}:{password}@{ip}:{port}" if username and password else f"http://{ip}:{port}"
|
||
source_tag = self._proxy_source_tag_from_url(proxy_item.get("source_url") or proxy_item.get("source_tag") or "")
|
||
if self._is_proxy_source_quarantined(source_tag):
|
||
continue
|
||
proxy_entry = {
|
||
'proxy': {
|
||
'http': proxy_url,
|
||
'https': proxy_url,
|
||
},
|
||
'usage_count': 0,
|
||
'source_tag': source_tag,
|
||
'expire_at_ts': float(self._proxy_item_expire_timestamp(proxy_item) or 0.0),
|
||
'raw_item': dict(proxy_item),
|
||
}
|
||
|
||
if self._is_proxy_item_expired(proxy_item):
|
||
expired_count += 1
|
||
if (
|
||
stale_reuse_grace_seconds > 0
|
||
and not self._is_proxy_item_expired(
|
||
proxy_item,
|
||
grace_seconds=stale_reuse_grace_seconds,
|
||
)
|
||
):
|
||
fallback_stale_proxies.append(proxy_entry)
|
||
continue
|
||
|
||
new_proxies.append(proxy_entry)
|
||
|
||
clock_skew_hint = self._summarize_proxy_expire_clock_skew(proxy_list)
|
||
if clock_skew_hint:
|
||
now_ts = time.time()
|
||
last_warning_at = float(getattr(self, "_last_proxy_clock_skew_warning_at", 0.0) or 0.0)
|
||
if now_ts - last_warning_at >= 30.0:
|
||
self._last_proxy_clock_skew_warning_at = now_ts
|
||
warning_message = (
|
||
"疑似系统时钟异常或代理源返回陈旧数据: "
|
||
f"本机时间 {clock_skew_hint['observed_now_iso']}，"
|
||
f"代理最晚过期 {clock_skew_hint['latest_expire_at_iso']}，"
|
||
f"偏移约 {int(clock_skew_hint['clock_ahead_seconds'])} 秒"
|
||
)
|
||
logger.warning(
|
||
warning_message
|
||
+ (
|
||
f"，带过期时间代理 {clock_skew_hint['timed_proxy_count']} 个，"
|
||
f"其中已过期 {clock_skew_hint['expired_timed_count']} 个"
|
||
)
|
||
)
|
||
self._sync_worker_log_event(
|
||
warning_message,
|
||
level='warning',
|
||
payload=clock_skew_hint,
|
||
mode='key',
|
||
)
|
||
|
||
fallback_stale_count = 0
|
||
stale_backfill_count = 0
|
||
stale_backfill_floor = 0
|
||
if not new_proxies and fallback_stale_proxies and not cached_proxy_pool:
|
||
new_proxies = fallback_stale_proxies
|
||
fallback_stale_count = len(new_proxies)
|
||
|
||
merged_proxy_pool = []
|
||
if new_proxies:
|
||
merged_proxy_pool = self._merge_proxy_entries(new_proxies, cached_proxy_pool)
|
||
elif cached_proxy_pool:
|
||
merged_proxy_pool = list(cached_proxy_pool)
|
||
|
||
if fallback_stale_proxies and stale_reuse_grace_seconds > 0:
|
||
(
|
||
merged_proxy_pool,
|
||
stale_backfill_count,
|
||
stale_backfill_floor,
|
||
) = self._backfill_recently_expired_proxy_entries(
|
||
merged_proxy_pool,
|
||
fallback_stale_proxies,
|
||
)
|
||
|
||
with self.proxy_pool_lock:
|
||
self.proxy_pool = merged_proxy_pool
|
||
source_map = {}
|
||
for proxy_entry in merged_proxy_pool:
|
||
if isinstance(proxy_entry, dict) and proxy_entry.get("proxy"):
|
||
source_map[self._proxy_key(proxy_entry.get("proxy"))] = str(
|
||
proxy_entry.get("source_tag") or ""
|
||
).strip()
|
||
with self.proxy_failure_lock:
|
||
self.proxy_source_by_key = source_map
|
||
self.proxy_last_refresh_time = datetime.now()
|
||
self.proxy_last_available_count = len(self.proxy_pool)
|
||
if new_proxies:
|
||
self._publish_shared_proxy_snapshot(
|
||
merged_proxy_pool,
|
||
source_count=source_count,
|
||
raw_items=self.proxy_last_refresh_total_items,
|
||
)
|
||
if new_proxies:
|
||
if fallback_stale_count > 0:
|
||
self.proxy_last_refresh_status = (
|
||
f"疑似过期回退增量入池 {len(new_proxies)} 个，当前池 {len(self.proxy_pool)} 个（跳过预验证）"
|
||
)
|
||
elif stale_backfill_count > 0:
|
||
self.proxy_last_refresh_status = (
|
||
f"最近过期代理回补 {stale_backfill_count} 个，当前池 {len(self.proxy_pool)} 个"
|
||
)
|
||
else:
|
||
self.proxy_last_refresh_status = (
|
||
f"增量入池 {len(new_proxies)} 个，当前池 {len(self.proxy_pool)} 个（跳过预验证）"
|
||
)
|
||
self.proxy_next_refresh_time = (
|
||
0.0 if len(self.proxy_pool) >= refresh_threshold else (time.time() + holdoff_seconds)
|
||
)
|
||
elif cached_proxy_pool:
|
||
self.proxy_last_refresh_status = f"本轮未拿到新代理，继续沿用缓存 {len(cached_proxy_pool)} 个"
|
||
next_retry_delay = (
|
||
0.5 if (self.detecting and aggressive_proxy_rotation and cooldown_detected)
|
||
else (2 if cooldown_detected else retry_delay_seconds)
|
||
)
|
||
next_retry_delay += holdoff_seconds
|
||
self.proxy_next_refresh_time = time.time() + next_retry_delay
|
||
else:
|
||
self.proxy_last_refresh_status = "代理源暂时冷却中，稍后继续补货" if cooldown_detected else "未取到可用代理数据"
|
||
next_retry_delay = (
|
||
0.5 if (self.detecting and aggressive_proxy_rotation and cooldown_detected)
|
||
else (2 if cooldown_detected else retry_delay_seconds)
|
||
)
|
||
next_retry_delay += holdoff_seconds
|
||
self.proxy_next_refresh_time = time.time() + next_retry_delay
|
||
logger.info(
|
||
f"代理池刷新完成，共 {len(self.proxy_pool)} 个可用代理，来源链接 {source_count} 个，"
|
||
f"原始 {self.proxy_last_refresh_total_items} 个，去过期 {expired_count} 个，非法 {invalid_count} 个，"
|
||
f"目标总量 {target_total}，单批 {batch_size}，轮次 {rounds}"
|
||
)
|
||
self._update_runtime_state(
|
||
"running" if self.detecting else "idle",
|
||
f"代理池刷新完成，共 {len(self.proxy_pool)} 个可用代理，来源链接 {source_count} 个，原始 {self.proxy_last_refresh_total_items} 个，单批 {batch_size}，轮次 {rounds}",
|
||
)
|
||
self._sync_worker_log_event(
|
||
f"代理池刷新完成，共 {len(self.proxy_pool)} 个可用代理，来源链接 {source_count} 个，原始 {self.proxy_last_refresh_total_items} 个，单批 {batch_size}，轮次 {rounds}",
|
||
payload={
|
||
"available_proxy_count": len(self.proxy_pool),
|
||
"source_count": source_count,
|
||
"raw_items": self.proxy_last_refresh_total_items,
|
||
"validated_count": 0,
|
||
"expired_count": expired_count,
|
||
"invalid_count": invalid_count,
|
||
"fallback_stale_count": fallback_stale_count,
|
||
"stale_backfill_count": stale_backfill_count,
|
||
"stale_backfill_floor": stale_backfill_floor,
|
||
"stale_reuse_grace_seconds": stale_reuse_grace_seconds,
|
||
"target_total": target_total,
|
||
"batch_size": batch_size,
|
||
"rounds": rounds,
|
||
},
|
||
mode='key',
|
||
)
|
||
else:
|
||
with self.proxy_pool_lock:
|
||
self.proxy_pool = cached_proxy_pool
|
||
self.proxy_last_refresh_time = datetime.now()
|
||
self.proxy_last_refresh_total_items = 0
|
||
self.proxy_last_validated_count = 0
|
||
self.proxy_last_available_count = len(self.proxy_pool)
|
||
if cached_proxy_pool:
|
||
self.proxy_last_refresh_status = f"未取到新代理，继续沿用缓存 {len(cached_proxy_pool)} 个"
|
||
else:
|
||
self.proxy_last_refresh_status = "代理源暂时冷却中，稍后继续补货" if cooldown_detected else "未取到可用代理数据"
|
||
next_retry_delay = (
|
||
0.5 if (self.detecting and aggressive_proxy_rotation and cooldown_detected)
|
||
else (2 if cooldown_detected else retry_delay_seconds)
|
||
)
|
||
next_retry_delay += holdoff_seconds
|
||
self.proxy_next_refresh_time = time.time() + next_retry_delay
|
||
logger.warning("所有代理池链接均未返回可用代理数据")
|
||
self._update_runtime_state(
|
||
"refreshing_proxy" if self.detecting else "idle",
|
||
self.proxy_last_refresh_status,
|
||
)
|
||
self._sync_worker_log_event(self.proxy_last_refresh_status, level='warning', mode='key')
|
||
else:
|
||
with self.proxy_pool_lock:
|
||
self.proxy_pool = cached_proxy_pool
|
||
self.proxy_last_refresh_time = datetime.now()
|
||
self.proxy_last_refresh_total_items = 0
|
||
self.proxy_last_validated_count = 0
|
||
self.proxy_last_available_count = len(self.proxy_pool)
|
||
self.proxy_last_refresh_status = "未配置代理池链接"
|
||
self.proxy_next_refresh_time = time.time() + (
|
||
(0.5 if (self.detecting and aggressive_proxy_rotation) else retry_delay_seconds)
|
||
+ holdoff_seconds
|
||
)
|
||
self._update_runtime_state(
|
||
"idle" if not self.detecting else "running",
|
||
"未配置代理池链接",
|
||
)
|
||
self._sync_worker_log_event("未配置代理池链接", level='warning', mode='key')
|
||
except Exception as e:
|
||
with self.proxy_pool_lock:
|
||
cached_proxy_pool = list(self.proxy_pool)
|
||
self.proxy_pool = cached_proxy_pool
|
||
self.proxy_last_refresh_time = datetime.now()
|
||
self.proxy_last_refresh_total_items = 0
|
||
self.proxy_last_validated_count = 0
|
||
self.proxy_last_available_count = len(self.proxy_pool)
|
||
if cached_proxy_pool:
|
||
self.proxy_last_refresh_status = f"刷新失败，继续沿用缓存 {len(cached_proxy_pool)} 个: {e}"
|
||
else:
|
||
self.proxy_last_refresh_status = f"刷新失败: {e}"
|
||
self.proxy_next_refresh_time = time.time() + retry_delay_seconds + holdoff_seconds
|
||
logger.error(f"刷新代理池失败: {e}")
|
||
self._update_runtime_state(
|
||
"refreshing_proxy" if self.detecting else "failed",
|
||
self.proxy_last_refresh_status,
|
||
)
|
||
self._sync_worker_log_event(self.proxy_last_refresh_status, level='error', mode='key')
|
||
finally:
|
||
self._release_shared_proxy_refresh_lock(shared_refresh_owner)
|
||
self.proxy_refresh_lock.release()
|
||
|
||
def remove_proxy(self, proxy):
|
||
"""
|
||
从代理池中移除失效的代理
|
||
|
||
:param proxy: 失效的代理
|
||
"""
|
||
failure_count, cooldown_seconds = self._mark_proxy_failure(proxy)
|
||
with self.proxy_pool_lock:
|
||
remaining_count = len(self.proxy_pool)
|
||
removed = 0
|
||
if self._pop_proxy_active_lease(proxy, already_locked=True):
|
||
removed += 1
|
||
if self.proxy_pool and isinstance(self.proxy_pool[0], dict) and 'proxy' in self.proxy_pool[0]:
|
||
filtered_pool = []
|
||
for proxy_item in self.proxy_pool:
|
||
if proxy_item.get('proxy') == proxy:
|
||
removed += 1
|
||
continue
|
||
filtered_pool.append(proxy_item)
|
||
if removed:
|
||
self.proxy_pool = filtered_pool
|
||
remaining_count = len(self.proxy_pool)
|
||
logger.info(f"代理失效已从当前池移除 {removed} 个: {proxy}")
|
||
else:
|
||
filtered_pool = []
|
||
for existing_proxy in self.proxy_pool:
|
||
if existing_proxy == proxy:
|
||
removed += 1
|
||
continue
|
||
filtered_pool.append(existing_proxy)
|
||
if removed:
|
||
self.proxy_pool = filtered_pool
|
||
remaining_count = len(self.proxy_pool)
|
||
logger.info(f"代理失效已从当前池移除 {removed} 个: {proxy}")
|
||
|
||
logger.info(
|
||
f"代理失败计数已更新: 第 {failure_count} 次失败，隔离 {cooldown_seconds}s，当前池内代理 {remaining_count} 个"
|
||
)
|
||
self._schedule_proxy_refresh_if_needed(remaining_count)
|
||
|
||
def _proxy_pool_scan_batch_size(self):
|
||
configured_batch_size = int(os.getenv("DOMAINCHECK_PROXY_POOL_SCAN_BATCH_SIZE", "0") or 0)
|
||
if configured_batch_size > 0:
|
||
return max(8, configured_batch_size)
|
||
demand_threads = self._proxy_demand_threads()
|
||
if demand_threads >= 1600:
|
||
return 64
|
||
if demand_threads >= 800:
|
||
return 48
|
||
return 32
|
||
|
||
def _pop_proxy_candidate_batch(self, batch_size):
|
||
normalized_batch_size = max(1, int(batch_size or 1))
|
||
with self.proxy_pool_lock:
|
||
if not self.proxy_pool:
|
||
return []
|
||
actual_batch_size = min(len(self.proxy_pool), normalized_batch_size)
|
||
candidates = []
|
||
for _ in range(actual_batch_size):
|
||
candidates.append(self.proxy_pool.pop(0))
|
||
return candidates
|
||
|
||
def _restore_proxy_candidate_batch(self, candidates, selected_entry=None, *, selected_step_name="", lease_selected=False):
|
||
restored_count = 0
|
||
with self.proxy_pool_lock:
|
||
if candidates:
|
||
self.proxy_pool.extend(candidates)
|
||
restored_count += len(candidates)
|
||
if selected_entry is not None:
|
||
if lease_selected:
|
||
self._store_proxy_active_lease(
|
||
selected_entry,
|
||
step_name=selected_step_name,
|
||
already_locked=True,
|
||
)
|
||
else:
|
||
self.proxy_pool.append(selected_entry)
|
||
restored_count += 1
|
||
remaining_count = len(self.proxy_pool)
|
||
return remaining_count, restored_count
|
||
|
||
def get_proxies(self, excluded_proxy_keys=None, step_name="", lease_selected=False):
|
||
"""
|
||
获取代理配置
|
||
"""
|
||
excluded_proxy_keys = set(excluded_proxy_keys or [])
|
||
if self.proxy_config.get('proxy_enable', False):
|
||
with self.proxy_pool_lock:
|
||
current_pool_size = len(self.proxy_pool)
|
||
is_empty = current_pool_size == 0
|
||
is_insufficient = current_pool_size < self._proxy_refresh_threshold()
|
||
|
||
if is_empty:
|
||
if self._wait_for_shared_proxy_snapshot(timeout_seconds=min(0.25, self._shared_proxy_wait_timeout_seconds())):
|
||
with self.proxy_pool_lock:
|
||
current_pool_size = len(self.proxy_pool)
|
||
is_empty = current_pool_size == 0
|
||
is_insufficient = current_pool_size < self._proxy_refresh_threshold()
|
||
# 不要在检测线程里同步阻塞等待代理刷新，否则线程数上去了也会卡死在这里。
|
||
if is_empty:
|
||
self._schedule_proxy_refresh_if_needed(current_pool_size)
|
||
elif is_insufficient:
|
||
logger.debug(
|
||
f"代理池低水位，触发后台补货: 当前 {current_pool_size}，阈值 {self._proxy_refresh_threshold()}"
|
||
)
|
||
self._schedule_proxy_refresh_if_needed(current_pool_size)
|
||
|
||
if current_pool_size > 0:
|
||
scan_batch_size = self._proxy_pool_scan_batch_size()
|
||
scanned_count = 0
|
||
while scanned_count < current_pool_size:
|
||
candidates = self._pop_proxy_candidate_batch(
|
||
min(scan_batch_size, max(1, current_pool_size - scanned_count))
|
||
)
|
||
if not candidates:
|
||
break
|
||
scanned_count += len(candidates)
|
||
deferred_candidates = []
|
||
selected_proxy = None
|
||
selected_entry = None
|
||
observed_now_ts = time.time()
|
||
|
||
for index, candidate in enumerate(candidates):
|
||
if isinstance(candidate, dict) and candidate.get('proxy'):
|
||
proxy = candidate['proxy']
|
||
proxy_key = self._proxy_key(proxy)
|
||
if self._is_proxy_quarantined(proxy):
|
||
deferred_candidates.append(candidate)
|
||
continue
|
||
source_tag = str(candidate.get("source_tag") or "").strip()
|
||
if self._is_proxy_source_quarantined(source_tag):
|
||
deferred_candidates.append(candidate)
|
||
continue
|
||
if proxy_key in excluded_proxy_keys:
|
||
deferred_candidates.append(candidate)
|
||
continue
|
||
if lease_selected and not self._proxy_entry_can_be_reused(candidate, step_name, now_ts=observed_now_ts):
|
||
continue
|
||
|
||
usage_count = int(candidate.get('usage_count', 0)) + 1
|
||
candidate['usage_count'] = 0 if usage_count >= self.proxy_max_reuse_count else usage_count
|
||
selected_proxy = proxy
|
||
selected_entry = candidate
|
||
with self.proxy_failure_lock:
|
||
source_map = getattr(self, "proxy_source_by_key", None)
|
||
if source_map is None:
|
||
self.proxy_source_by_key = {}
|
||
source_map = self.proxy_source_by_key
|
||
source_map[proxy_key] = source_tag
|
||
deferred_candidates.extend(candidates[index + 1 :])
|
||
break
|
||
|
||
proxy = candidate
|
||
proxy_key = self._proxy_key(proxy)
|
||
if self._is_proxy_quarantined(proxy):
|
||
deferred_candidates.append(candidate)
|
||
continue
|
||
if proxy_key in excluded_proxy_keys:
|
||
deferred_candidates.append(candidate)
|
||
continue
|
||
logger.info(f"从代理池选择代理: {proxy}")
|
||
selected_proxy = proxy
|
||
selected_entry = candidate if isinstance(candidate, dict) else {
|
||
"proxy": proxy,
|
||
"usage_count": 0,
|
||
"source_tag": "",
|
||
}
|
||
deferred_candidates.extend(candidates[index + 1 :])
|
||
break
|
||
|
||
if selected_proxy is None:
|
||
deferred_candidates = candidates
|
||
|
||
remaining_count, _ = self._restore_proxy_candidate_batch(
|
||
deferred_candidates,
|
||
selected_entry=selected_entry,
|
||
selected_step_name=step_name,
|
||
lease_selected=lease_selected,
|
||
)
|
||
if selected_proxy is not None:
|
||
self._schedule_proxy_refresh_if_needed(remaining_count)
|
||
return selected_proxy
|
||
if self.proxy_config.get('proxy_enable', False):
|
||
now_ts = time.time()
|
||
if now_ts - float(getattr(self, "_last_no_proxy_notice_at", 0.0) or 0.0) >= 5.0:
|
||
self._last_no_proxy_notice_at = now_ts
|
||
logger.warning("代理已启用，但当前无可用代理")
|
||
fallback_detail = (
|
||
f"代理已启用，但当前无可用代理；已允许直连兜底，最近代理状态：{self.proxy_last_refresh_status}"
|
||
if self.allow_direct_connection()
|
||
else f"代理已启用，但当前无可用代理；且未允许直连，最近代理状态：{self.proxy_last_refresh_status}"
|
||
)
|
||
self._update_runtime_state(
|
||
"running" if self.detecting else "idle",
|
||
fallback_detail,
|
||
)
|
||
else:
|
||
logger.info("未启用代理，使用直接连接")
|
||
return None
|
||
|
||
def allow_direct_connection(self):
|
||
return bool(self.proxy_config.get('allow_direct', False))
|
||
|
||
@staticmethod
|
||
def _normalize_flag_value(value, default=False):
|
||
if value is None:
|
||
return bool(default)
|
||
if isinstance(value, bool):
|
||
return value
|
||
return str(value or "").strip().lower() not in {"", "0", "false", "no", "off"}
|
||
|
||
@staticmethod
|
||
def _env_flag_enabled(env_name, default=False):
|
||
value = os.getenv(env_name)
|
||
return DetectWorker._normalize_flag_value(value, default=default)
|
||
|
||
def _register_single_machine_mode_enabled(self):
|
||
cached = getattr(self, "register_single_machine_mode_enabled", None)
|
||
if cached is not None:
|
||
return bool(cached)
|
||
return self._env_flag_enabled("DOMAINCHECK_REGISTER_SINGLE_MACHINE_MODE")
|
||
|
||
def _register_single_machine_direct_streak_attempts(self):
|
||
cached = getattr(self, "register_single_machine_direct_streak_attempts", None)
|
||
if cached is not None:
|
||
return max(0, int(cached or 0))
|
||
if not self._register_single_machine_mode_enabled():
|
||
return 0
|
||
return max(0, int(os.getenv("DOMAINCHECK_REGISTER_DIRECT_STREAK_ATTEMPTS", "2") or 2))
|
||
|
||
def _single_machine_site_direct_fallback_enabled(self):
|
||
cached = getattr(self, "single_machine_site_direct_fallback_enabled", None)
|
||
if cached is not None:
|
||
return bool(cached)
|
||
return self._env_flag_enabled(
|
||
"DOMAINCHECK_SINGLE_MACHINE_SITE_DIRECT_FALLBACK",
|
||
default=self._register_single_machine_mode_enabled(),
|
||
)
|
||
|
||
def _single_machine_aizhan_direct_first_enabled(self):
|
||
cached = getattr(self, "single_machine_aizhan_direct_first_enabled", None)
|
||
if cached is not None:
|
||
return bool(cached)
|
||
return self._env_flag_enabled(
|
||
"DOMAINCHECK_SINGLE_MACHINE_AIZHAN_DIRECT_FIRST",
|
||
default=False,
|
||
)
|
||
|
||
def _single_machine_baidu_direct_first_enabled(self):
|
||
cached = getattr(self, "single_machine_baidu_direct_first_enabled", None)
|
||
if cached is not None:
|
||
return bool(cached)
|
||
return self._env_flag_enabled(
|
||
"DOMAINCHECK_SINGLE_MACHINE_BAIDU_DIRECT_FIRST",
|
||
default=False,
|
||
)
|
||
|
||
def _single_machine_chinaz_direct_first_enabled(self):
|
||
cached = getattr(self, "single_machine_chinaz_direct_first_enabled", None)
|
||
if cached is not None:
|
||
return bool(cached)
|
||
return self._env_flag_enabled(
|
||
"DOMAINCHECK_SINGLE_MACHINE_CHINAZ_DIRECT_FIRST",
|
||
default=False,
|
||
)
|
||
|
||
def _aizhan_remote_disconnect_degrade_enabled(self):
|
||
cached = getattr(self, "aizhan_remote_disconnect_degrade_enabled", None)
|
||
if cached is not None:
|
||
return bool(cached)
|
||
return self._env_flag_enabled(
|
||
"DOMAINCHECK_AIZHAN_REMOTE_DISCONNECT_DEGRADE",
|
||
default=False,
|
||
)
|
||
|
||
def _aizhan_external_fast_degrade_enabled(self):
|
||
cached = getattr(self, "aizhan_external_fast_degrade_enabled", None)
|
||
if cached is not None:
|
||
return bool(cached)
|
||
return self._env_flag_enabled(
|
||
"DOMAINCHECK_AIZHAN_EXTERNAL_FAST_DEGRADE",
|
||
default=False,
|
||
)
|
||
|
||
def _should_force_direct_first_attempt(self, step_name, attempt_count):
|
||
if int(attempt_count or 0) != 0:
|
||
return False
|
||
if (
|
||
step_name == "爱站网检测"
|
||
and self._single_machine_aizhan_direct_first_enabled()
|
||
and self._allow_direct_connection_for_step(step_name)
|
||
):
|
||
return True
|
||
if (
|
||
step_name == "百度site检测"
|
||
and self._single_machine_baidu_direct_first_enabled()
|
||
and self._allow_direct_connection_for_step(step_name)
|
||
):
|
||
return True
|
||
if (
|
||
step_name == "站长之家检测"
|
||
and self._single_machine_chinaz_direct_first_enabled()
|
||
and self._allow_direct_connection_for_step(step_name)
|
||
):
|
||
return True
|
||
return False
|
||
|
||
def _should_degrade_remote_disconnect_immediately(self, step_name, reason):
|
||
if step_name != "爱站网检测" or not self._aizhan_remote_disconnect_degrade_enabled():
|
||
return False
|
||
lowered = str(reason or "").strip().lower()
|
||
if not lowered:
|
||
return False
|
||
return (
|
||
"remotedisconnected" in lowered
|
||
or "remote end closed connection without response" in lowered
|
||
)
|
||
|
||
def _should_degrade_external_issue_immediately(self, step_name, reason):
|
||
if step_name != "爱站网检测":
|
||
return False
|
||
if self._should_degrade_remote_disconnect_immediately(step_name, reason):
|
||
return True
|
||
if not self._aizhan_external_fast_degrade_enabled():
|
||
return False
|
||
return self._is_external_dependency_issue(reason)
|
||
|
||
def _allow_direct_connection_for_step(self, step_name):
|
||
if step_name == "注册状态检测" and self._register_single_machine_mode_enabled():
|
||
return True
|
||
if (
|
||
step_name in {"360检测", "百度site检测", "站长之家检测", "爱站网检测"}
|
||
and self._single_machine_site_direct_fallback_enabled()
|
||
):
|
||
return True
|
||
return self.allow_direct_connection()
|
||
|
||
def _proxy_status_indicates_shortage(self):
|
||
status = str(getattr(self, "proxy_last_refresh_status", "") or "").strip()
|
||
if not status:
|
||
return False
|
||
keywords = (
|
||
"冷却中",
|
||
"暂时冷却",
|
||
"无可用代理",
|
||
"未取到可用代理",
|
||
"继续沿用缓存 0",
|
||
"稍后继续补货",
|
||
)
|
||
return any(keyword in status for keyword in keywords)
|
||
|
||
def _proxy_pool_available_count(self):
|
||
try:
|
||
with self.proxy_pool_lock:
|
||
return len(list(getattr(self, "proxy_pool", []) or []))
|
||
except Exception:
|
||
return len(list(getattr(self, "proxy_pool", []) or []))
|
||
|
||
def _should_hold_stale_detect_for_proxy_shortage(
|
||
self,
|
||
*,
|
||
idle_seconds: float,
|
||
low_activity_recoverable: bool,
|
||
incoming_job_changed: bool,
|
||
):
|
||
if incoming_job_changed or not low_activity_recoverable:
|
||
return False, ""
|
||
if not bool(self.proxy_config.get('proxy_enable', False)):
|
||
return False, ""
|
||
if not self._proxy_status_indicates_shortage():
|
||
return False, ""
|
||
if self._proxy_pool_available_count() > 0:
|
||
return False, ""
|
||
grace_seconds = 120.0 if self.allow_direct_connection() else 180.0
|
||
if idle_seconds >= grace_seconds:
|
||
return False, ""
|
||
return True, (
|
||
f"代理池仍处于短缺/冷却窗口，当前空转 {int(idle_seconds)} 秒，"
|
||
f"小于保护阈值 {int(grace_seconds)} 秒，最近代理状态：{self.proxy_last_refresh_status}"
|
||
)
|
||
|
||
def _proxy_direct_fallback_grace_for_step(self, step_name):
|
||
default_grace = max(0.0, float(getattr(self, "proxy_direct_fallback_grace_seconds", 0.0) or 0.0))
|
||
override_map = getattr(self, "proxy_direct_fallback_grace_overrides", None) or {}
|
||
step_grace = max(0.0, float(override_map.get(step_name, default_grace) or 0.0))
|
||
if step_name == "注册状态检测" and self._register_single_machine_mode_enabled():
|
||
return 0.0
|
||
if not self._allow_direct_connection_for_step(step_name):
|
||
return step_grace
|
||
if (
|
||
step_name in {"注册状态检测", "360检测", "百度site检测", "站长之家检测", "爱站网检测"}
|
||
and self._proxy_status_indicates_shortage()
|
||
):
|
||
return min(step_grace, 0.05)
|
||
return step_grace
|
||
|
||
def _proxy_direct_retry_wait_for_step(self, step_name):
|
||
default_wait = max(0.0, float(getattr(self, "proxy_direct_retry_wait_seconds", 0.0) or 0.0))
|
||
override_map = getattr(self, "proxy_direct_retry_wait_overrides", None) or {}
|
||
step_wait = max(0.0, float(override_map.get(step_name, default_wait) or 0.0))
|
||
if step_name == "注册状态检测" and self._register_single_machine_mode_enabled():
|
||
return 0.0
|
||
if (
|
||
step_name in {"注册状态检测", "360检测", "百度site检测", "站长之家检测", "爱站网检测"}
|
||
and self._proxy_status_indicates_shortage()
|
||
):
|
||
return min(step_wait, 0.25)
|
||
return step_wait
|
||
|
||
def _get_proxy_for_step(self, domain_id, domain_name, step_name, excluded_proxy_keys=None):
|
||
started_at = time.perf_counter()
|
||
excluded_proxy_keys = set(excluded_proxy_keys or [])
|
||
proxy_enabled = bool(self.proxy_config.get('proxy_enable', False))
|
||
direct_allowed = self._allow_direct_connection_for_step(step_name)
|
||
force_direct_once = self._consume_step_force_direct_once(step_name)
|
||
step_direct_grace_seconds = self._proxy_direct_fallback_grace_for_step(step_name)
|
||
if proxy_enabled and direct_allowed and force_direct_once:
|
||
self._emit_step_trace(
|
||
domain_name,
|
||
step_name,
|
||
"proxy_direct_fallback",
|
||
elapsed_ms=0,
|
||
proxy_mode="direct",
|
||
recent_proxy_status=self.proxy_last_refresh_status,
|
||
retry_attempts=0,
|
||
reason="external_retry_direct_fallback",
|
||
)
|
||
return None
|
||
direct_retry_wait_seconds = self._consume_step_proxy_retry_wait(step_name)
|
||
wait_window_seconds = 0.0
|
||
if proxy_enabled and not direct_allowed:
|
||
wait_window_seconds = self.proxy_step_wait_timeout_seconds
|
||
elif proxy_enabled and direct_allowed:
|
||
wait_window_seconds = max(step_direct_grace_seconds, direct_retry_wait_seconds)
|
||
wait_deadline = time.time() + wait_window_seconds
|
||
wait_logged_at = 0.0
|
||
attempt = 0
|
||
|
||
while True:
|
||
proxy = self.get_proxies(
|
||
excluded_proxy_keys=excluded_proxy_keys,
|
||
step_name=step_name,
|
||
lease_selected=True,
|
||
)
|
||
elapsed_ms = int((time.perf_counter() - started_at) * 1000)
|
||
if proxy:
|
||
self._emit_step_trace(
|
||
domain_name,
|
||
step_name,
|
||
"proxy_selected",
|
||
elapsed_ms=elapsed_ms,
|
||
proxy_mode="proxy",
|
||
excluded_proxy_count=len(excluded_proxy_keys),
|
||
retry_attempts=attempt,
|
||
)
|
||
return proxy
|
||
|
||
if not proxy_enabled:
|
||
self._emit_step_trace(
|
||
domain_name,
|
||
step_name,
|
||
"proxy_direct_fallback",
|
||
elapsed_ms=elapsed_ms,
|
||
proxy_mode="direct",
|
||
recent_proxy_status=self.proxy_last_refresh_status,
|
||
)
|
||
return None
|
||
|
||
now_ts = time.time()
|
||
if now_ts >= wait_deadline:
|
||
break
|
||
|
||
attempt += 1
|
||
if not self.proxy_refresh_lock.locked():
|
||
self.trigger_proxy_refresh(
|
||
reason=f"step_wait:{step_name}",
|
||
reset_cooldown=bool(direct_retry_wait_seconds) or step_name == '注册状态检测',
|
||
)
|
||
if now_ts - wait_logged_at >= 1.0:
|
||
wait_logged_at = now_ts
|
||
logger.info(
|
||
f"{step_name} 暂无可用代理，继续等待补货: {domain_name}, "
|
||
f"attempt={attempt}, waited_ms={elapsed_ms}, 最近状态: {self.proxy_last_refresh_status}"
|
||
)
|
||
time.sleep(0.05 if direct_retry_wait_seconds > 0 else 0.1)
|
||
|
||
if proxy_enabled and direct_allowed:
|
||
self._emit_step_trace(
|
||
domain_name,
|
||
step_name,
|
||
"proxy_direct_fallback",
|
||
elapsed_ms=int((time.perf_counter() - started_at) * 1000),
|
||
proxy_mode="direct",
|
||
recent_proxy_status=self.proxy_last_refresh_status,
|
||
retry_attempts=attempt,
|
||
)
|
||
return None
|
||
|
||
detail = self.proxy_last_refresh_status
|
||
elapsed_ms = int((time.perf_counter() - started_at) * 1000)
|
||
logger.error(
|
||
f"{step_name} 无可用代理，且当前不允许直连兜底: {domain_name}，"
|
||
f"waited_ms={elapsed_ms}, attempts={attempt}, 最近代理状态: {detail}"
|
||
)
|
||
self._emit_step_trace(
|
||
domain_name,
|
||
step_name,
|
||
"proxy_unavailable",
|
||
elapsed_ms=elapsed_ms,
|
||
level='warning',
|
||
proxy_mode="required",
|
||
recent_proxy_status=detail,
|
||
retry_attempts=attempt,
|
||
)
|
||
if not self._is_current_single_step_task():
|
||
self.db.update_domain_detect_status(domain_id, DETECT_STATUS_FAILED)
|
||
return '__NO_PROXY__'
|
||
|
||
def _should_rotate_proxy_on_error(self, proxy, reason, step_name, domain_name):
|
||
if not proxy or not self.proxy_config.get('proxy_enable', False):
|
||
return False
|
||
normalized_reason = str(reason or "")
|
||
if not self._is_external_dependency_issue(normalized_reason):
|
||
return False
|
||
if self._is_proxy_auth_issue(normalized_reason):
|
||
self._mark_proxy_source_failure(proxy, normalized_reason)
|
||
self.remove_proxy(proxy)
|
||
try:
|
||
self.trigger_proxy_refresh(reason=f"proxy_rotate:{step_name}", reset_cooldown=True)
|
||
except Exception as refresh_error:
|
||
logger.debug(f"{step_name} 触发代理快速补货失败: {refresh_error}")
|
||
logger.warning(f"{step_name} 当前代理异常，切换下一个代理继续: {domain_name}, 原因: {normalized_reason}")
|
||
return True
|
||
|
||
def _should_retry_external_issue(self, proxy, reason, step_name, domain_name):
|
||
normalized_reason = str(reason or "")
|
||
if not self._is_external_dependency_issue(normalized_reason):
|
||
return False
|
||
if proxy and self._allow_direct_connection_for_step(step_name):
|
||
if self._is_proxy_auth_issue(normalized_reason):
|
||
self._mark_proxy_source_failure(proxy, normalized_reason)
|
||
self.remove_proxy(proxy)
|
||
self._set_step_force_direct_once(step_name, True)
|
||
try:
|
||
self.trigger_proxy_refresh(reason=f"direct_fallback:{step_name}", reset_cooldown=True)
|
||
except Exception as refresh_error:
|
||
logger.debug(f"{step_name} 触发直连兜底前代理补货失败: {refresh_error}")
|
||
logger.warning(
|
||
f"{step_name} 代理链路异常，下一次尝试直连兜底: "
|
||
f"{domain_name}, 原因: {normalized_reason}"
|
||
)
|
||
return True
|
||
if self._should_rotate_proxy_on_error(proxy, normalized_reason, step_name, domain_name):
|
||
return True
|
||
if self.proxy_config.get('proxy_enable', False):
|
||
if not proxy and self._allow_direct_connection_for_step(step_name):
|
||
retry_proxy_wait_seconds = self._proxy_direct_retry_wait_for_step(step_name)
|
||
forced_wait_seconds = max(
|
||
self._proxy_direct_fallback_grace_for_step(step_name),
|
||
min(
|
||
self.proxy_step_wait_timeout_seconds or 0.0,
|
||
retry_proxy_wait_seconds,
|
||
) if self.proxy_step_wait_timeout_seconds > 0 else retry_proxy_wait_seconds,
|
||
)
|
||
self._set_step_proxy_retry_wait(step_name, forced_wait_seconds)
|
||
try:
|
||
self.trigger_proxy_refresh(reason=f"external_retry:{step_name}", reset_cooldown=True)
|
||
except Exception as refresh_error:
|
||
logger.debug(f"{step_name} 触发代理补货失败: {refresh_error}")
|
||
logger.warning(
|
||
f"{step_name} 外部链路异常，已触发代理补货并继续重试: "
|
||
f"{domain_name}, 原因: {normalized_reason}"
|
||
)
|
||
return True
|
||
return False
|
||
|
||
def _mark_blacklisted(self, domain_id, domain_name, reason):
|
||
self._set_task_detect_status(DETECT_STATUS_BLACKLISTED)
|
||
if self._is_current_single_step_task():
|
||
logger.info(f"single_step 域名命中黑名单，跳过本地域名表/黑名单表写入: {domain_name}, 原因: {reason}")
|
||
return
|
||
self.db.mark_domain_blacklisted(domain_id, domain_name, reason)
|
||
logger.info(f"域名已加入黑名单: {domain_name}, 原因: {reason}")
|
||
|
||
def _step_result_payload(self, *, ok, state, message="", **extra):
|
||
payload = {
|
||
"status": bool(ok),
|
||
"state": state,
|
||
"message": message or "",
|
||
"checked_at": datetime.now().strftime("%Y-%m-%d %H:%M:%S"),
|
||
}
|
||
payload.update(extra)
|
||
return payload
|
||
|
||
def _record_step_result(self, domain_id, field_name, *, ok, state, message="", **extra):
|
||
if not field_name:
|
||
return
|
||
payload = self._step_result_payload(ok=ok, state=state, message=message, **extra)
|
||
if self._is_current_single_step_task():
|
||
self._cache_single_step_result_payload(domain_id, field_name, payload)
|
||
return
|
||
with self._step_result_cache_lock:
|
||
self._step_result_cache[(int(domain_id or 0), field_name)] = dict(payload)
|
||
self._upsert_json_detection(domain_id, **{field_name: payload})
|
||
|
||
def _should_blacklist_result(self, message):
|
||
if not message:
|
||
return False
|
||
lowered = str(message).lower()
|
||
keywords = [
|
||
"检测到敏感词",
|
||
"黑名单分类",
|
||
"风险网站",
|
||
"高危网站",
|
||
"clienthold",
|
||
"serverhold",
|
||
"拦截",
|
||
"敏感内容",
|
||
]
|
||
return any(keyword.lower() in lowered for keyword in keywords)
|
||
|
||
def _mark_detection_failed(self, domain_id, domain_name, step_name, reason, field_name=None):
|
||
reason = reason or "未知错误"
|
||
if self._is_current_single_step_task():
|
||
self._set_task_detect_status(DETECT_STATUS_FAILED)
|
||
if field_name:
|
||
self._record_step_result(domain_id, field_name, ok=False, state="failed", message=reason, step=step_name)
|
||
logger.warning(f"{step_name} single_step 技术失败，跳过本地域名状态写入: {domain_name}, 原因: {reason}")
|
||
return
|
||
current_status = self._get_task_detect_status() or 0
|
||
if current_status == DETECT_STATUS_BLACKLISTED:
|
||
logger.warning(f"{step_name} 技术失败，但域名已是黑名单状态，保持黑名单不回退: {domain_name}, 原因: {reason}")
|
||
else:
|
||
self._enqueue_domain_status_update(domain_id, DETECT_STATUS_FAILED)
|
||
self._set_task_detect_status(DETECT_STATUS_FAILED)
|
||
if field_name:
|
||
self._record_step_result(domain_id, field_name, ok=False, state="failed", message=reason, step=step_name)
|
||
logger.warning(f"{step_name} 技术失败，域名标记为检测失败: {domain_name}, 原因: {reason}")
|
||
|
||
def _mark_detection_rejected(self, domain_id, domain_name, step_name, reason, field_name=None):
|
||
reason = reason or "业务判定未通过"
|
||
self._set_task_detect_status(DETECT_STATUS_FAILED)
|
||
if field_name:
|
||
self._record_step_result(domain_id, field_name, ok=False, state="rejected", message=reason, step=step_name)
|
||
if self._is_current_single_step_task():
|
||
logger.warning(f"{step_name} single_step 业务不通过，跳过本地域名状态写入: {domain_name}, 原因: {reason}")
|
||
return
|
||
self._enqueue_domain_status_update(domain_id, DETECT_STATUS_FAILED)
|
||
logger.warning(f"{step_name} 业务判定未通过，终止后续步骤: {domain_name}, 原因: {reason}")
|
||
|
||
def _mark_detection_degraded(self, domain_id, domain_name, step_name, reason, field_name=None, review_required=True):
|
||
reason = reason or "外部依赖异常"
|
||
if field_name:
|
||
self._record_step_result(domain_id, field_name, ok=False, state="degraded", message=reason, step=step_name)
|
||
if self._is_current_single_step_task():
|
||
logger.warning(f"{step_name} single_step 外部依赖异常，跳过本地 review 状态写入: {domain_name}, 原因: {reason}")
|
||
return
|
||
if review_required:
|
||
try:
|
||
self._enqueue_review_status_update(domain_id, REVIEW_STATUS_PENDING)
|
||
except Exception as review_error:
|
||
logger.warning(f"{step_name} 降级后设置待人工复核失败: {domain_name}, 错误: {review_error}")
|
||
logger.warning(f"{step_name} 外部依赖异常，步骤降级继续执行: {domain_name}, 原因: {reason}")
|
||
|
||
def _is_external_dependency_issue(self, message):
|
||
if not message:
|
||
return False
|
||
lowered = str(message).lower()
|
||
keywords = [
|
||
"timeout",
|
||
"timed out",
|
||
"超时",
|
||
"请求超时",
|
||
"连接超时",
|
||
"读取超时",
|
||
"网络超时",
|
||
"请求失败",
|
||
"网络异常",
|
||
"连接失败",
|
||
"连接中断",
|
||
"连接被重置",
|
||
"服务暂不可用",
|
||
"connection",
|
||
"connection reset",
|
||
"connection refused",
|
||
"max retries exceeded",
|
||
"httpsconnectionpool",
|
||
"proxyerror",
|
||
"remotedisconnected",
|
||
"temporarily unavailable",
|
||
"502 bad gateway",
|
||
"503 service unavailable",
|
||
"504 gateway timeout",
|
||
"network",
|
||
"ssl",
|
||
"read timeout",
|
||
"connecttimeout",
|
||
"operation timed out",
|
||
"connection timed out",
|
||
"resolving timed out",
|
||
"http 403",
|
||
"http 429",
|
||
"status code 403",
|
||
"status code 429",
|
||
"状态码: 403",
|
||
"状态码: 429",
|
||
"状态码: 502",
|
||
"状态码: 503",
|
||
"状态码: 504",
|
||
"forbidden",
|
||
"empty reply from server",
|
||
"recv failure",
|
||
"curl: (28)",
|
||
"curl:(28)",
|
||
"curl: (52)",
|
||
"curl:(52)",
|
||
"响应格式异常",
|
||
"cannot unpack non-iterable nonetype object",
|
||
]
|
||
return any(keyword in lowered for keyword in keywords)
|
||
|
||
def _resolve_proxy_step_retry_budget(self, step_name):
|
||
overrides = (getattr(self, "proxy_step_retry_budget_overrides", None) or {}).get(step_name) or {}
|
||
max_attempts = max(
|
||
1,
|
||
int(overrides.get("max_attempts", getattr(self, "proxy_step_retry_max_attempts", 12)) or 1),
|
||
)
|
||
max_seconds = max(
|
||
5.0,
|
||
float(overrides.get("max_seconds", getattr(self, "proxy_step_retry_max_seconds", 45)) or 5.0),
|
||
)
|
||
return max_attempts, max_seconds
|
||
|
||
def _check_proxy_step_retry_budget(self, step_name, domain_name, started_at, attempt_count, tried_proxy_keys, last_reason=""):
|
||
elapsed_seconds = max(0.0, time.perf_counter() - float(started_at or 0.0))
|
||
max_attempts, max_seconds = self._resolve_proxy_step_retry_budget(step_name)
|
||
pending_direct_fallback = (
|
||
attempt_count < max_attempts
|
||
and self._allow_direct_connection_for_step(step_name)
|
||
and self._peek_step_force_direct_once(step_name)
|
||
)
|
||
if attempt_count < max_attempts and (elapsed_seconds < max_seconds or pending_direct_fallback):
|
||
return False, ""
|
||
|
||
reason = (
|
||
f"{step_name} 重试预算耗尽: attempts={attempt_count}, "
|
||
f"elapsed_ms={int(elapsed_seconds * 1000)}, tried_proxies={len(tried_proxy_keys)}"
|
||
)
|
||
if last_reason:
|
||
reason = f"{reason}, last_reason={str(last_reason)[:200]}"
|
||
logger.warning(f"{reason}: {domain_name}")
|
||
self._emit_step_trace(
|
||
domain_name,
|
||
step_name,
|
||
"retry_budget_exhausted",
|
||
level='warning',
|
||
elapsed_ms=int(elapsed_seconds * 1000),
|
||
retry_attempts=attempt_count,
|
||
tried_proxy_count=len(tried_proxy_keys),
|
||
last_reason=str(last_reason or "")[:200],
|
||
)
|
||
return True, reason
|
||
|
||
def _remaining_proxy_step_retry_budget_seconds(self, step_name, started_at, reserve_seconds=0.0):
|
||
elapsed_seconds = max(0.0, time.perf_counter() - float(started_at or 0.0))
|
||
_, max_seconds = self._resolve_proxy_step_retry_budget(step_name)
|
||
return max(0.0, float(max_seconds or 0.0) - elapsed_seconds - max(0.0, float(reserve_seconds or 0.0)))
|
||
|
||
def _resolve_submit_backlog_limit(self, max_threads, live_active):
|
||
normalized_threads = max(1, int(max_threads or 1))
|
||
live_active = max(0, int(live_active or 0))
|
||
configured_backlog_floor = self._runtime_optional_int_override("submit_backlog_floor", "DOMAINCHECK_SUBMIT_BACKLOG_FLOOR")
|
||
configured_backlog_ceiling = self._runtime_optional_int_override("submit_backlog_ceil", "DOMAINCHECK_SUBMIT_BACKLOG_CEIL")
|
||
# Default to a full-thread warm window. Large machines should be allowed
|
||
# to keep enough local backlog to actually fill the configured worker
|
||
# pool instead of self-throttling to a few hundred runnable items.
|
||
default_backlog_floor = normalized_threads
|
||
backlog_floor = max(16, configured_backlog_floor if configured_backlog_floor > 0 else default_backlog_floor)
|
||
backlog_ceiling = max(
|
||
backlog_floor,
|
||
configured_backlog_ceiling if configured_backlog_ceiling > 0 else normalized_threads,
|
||
)
|
||
dynamic_target = max(
|
||
backlog_floor,
|
||
live_active + max(32, normalized_threads // 4),
|
||
)
|
||
return min(backlog_ceiling, dynamic_target)
|
||
|
||
def _resolve_dispatch_capacity(self, max_threads, live_active, backlog_limit):
|
||
normalized_threads = max(1, int(max_threads or 1))
|
||
live_active = max(0, int(live_active or 0))
|
||
backlog_limit = max(1, int(backlog_limit or 1))
|
||
dispatch_cap_multiplier = self._runtime_positive_int_override(
|
||
"dispatch_cap_multiplier",
|
||
"DOMAINCHECK_DISPATCH_CAP_MULTIPLIER",
|
||
default=1,
|
||
)
|
||
dispatch_capacity_cap = max(1, normalized_threads * dispatch_cap_multiplier)
|
||
return min(dispatch_capacity_cap, max(backlog_limit, live_active + backlog_limit))
|
||
|
||
def _resolve_claim_batch_size(self, current_limit, available_slots, live_active=None):
|
||
normalized_limit = max(1, int(current_limit or 1))
|
||
available_slots = max(1, int(available_slots or 0))
|
||
live_active = max(
|
||
0,
|
||
int(self._get_active_domain_threads() or 0) if live_active is None else int(live_active or 0),
|
||
)
|
||
claim_cap_multiplier = max(
|
||
1,
|
||
int(os.getenv("DOMAINCHECK_CLAIM_CAP_MULTIPLIER", "1") or 1),
|
||
)
|
||
configured_claim_floor = self._runtime_optional_int_override("claim_batch_floor", "DOMAINCHECK_CLAIM_BATCH_FLOOR")
|
||
configured_claim_ceiling = self._runtime_optional_int_override("claim_batch_ceil", "DOMAINCHECK_CLAIM_BATCH_CEIL")
|
||
# Default to a full-thread claim window so large worker pools can
|
||
# actually fill their slots. Operators can still shrink this via env
|
||
# overrides if they explicitly want a tighter rolling buffer.
|
||
default_claim_floor = normalized_limit
|
||
claim_floor = max(4, configured_claim_floor if configured_claim_floor > 0 else default_claim_floor)
|
||
claim_ceiling = max(
|
||
claim_floor,
|
||
configured_claim_ceiling if configured_claim_ceiling > 0 else normalized_limit,
|
||
)
|
||
warm_window = max(
|
||
claim_floor,
|
||
live_active + max(32, normalized_limit // 4),
|
||
)
|
||
return max(
|
||
1,
|
||
min(
|
||
available_slots,
|
||
normalized_limit * claim_cap_multiplier,
|
||
max(claim_floor, min(claim_ceiling, warm_window)),
|
||
),
|
||
)
|
||
|
||
def _should_proactively_top_up_pipeline(self, claimed_count, refill_slots, max_threads):
|
||
claimed_count = max(0, int(claimed_count or 0))
|
||
refill_slots = max(0, int(refill_slots or 0))
|
||
normalized_threads = max(1, int(max_threads or 1))
|
||
remaining_gap = max(0, refill_slots - claimed_count)
|
||
if remaining_gap <= 0:
|
||
return False
|
||
configured_threshold = int(os.getenv("DOMAINCHECK_PIPELINE_TOPUP_THRESHOLD", "0") or 0)
|
||
threshold = max(
|
||
1,
|
||
configured_threshold if configured_threshold > 0 else max(128, normalized_threads // 4),
|
||
)
|
||
return remaining_gap >= threshold
|
||
|
||
def _should_prefetch_sync_tasks(self, pending_buffer_count, inflight_count, max_threads):
|
||
pending_buffer_count = max(0, int(pending_buffer_count or 0))
|
||
inflight_count = max(0, int(inflight_count or 0))
|
||
normalized_threads = max(1, int(max_threads or 1))
|
||
local_work = pending_buffer_count + inflight_count
|
||
configured_threshold = int(os.getenv("DOMAINCHECK_SYNC_PULL_PREFETCH_THRESHOLD", "0") or 0)
|
||
threshold = max(
|
||
32,
|
||
configured_threshold if configured_threshold > 0 else max(64, min(normalized_threads, normalized_threads // 2)),
|
||
)
|
||
return local_work <= threshold
|
||
|
||
def _complete_detection(self, domain_id, domain_name, domain):
|
||
register_status = int((domain or {}).get('register_status', 0) or 0)
|
||
use_status = int((domain or {}).get('use_status', 0) or 0)
|
||
expire_date = (domain or {}).get('expire_date')
|
||
self._enqueue_domain_completion(
|
||
domain_id,
|
||
register_status=register_status,
|
||
use_status=use_status,
|
||
expire_date=expire_date,
|
||
)
|
||
if use_status == 0 and register_status == REGISTER_STATUS_AVAILABLE and expire_date:
|
||
logger.info(f"域名 {domain_name} 满足条件，已将expire_date置空")
|
||
domain['expire_date'] = None
|
||
if register_status == REGISTER_STATUS_AVAILABLE:
|
||
logger.info(f"域名 {domain_name} 满足条件，已设置为待人工复核")
|
||
|
||
def _should_run_jucha(self, domain):
|
||
return self.detect_options.get('detect_jucha', False) and domain.get('jucha_status', 0) != THIRD_PARTY_STATUS_DONE
|
||
|
||
def _should_run_juziseo(self, domain):
|
||
return self.detect_options.get('detect_juziseo', False) and domain.get('juziseo_status', 0) != THIRD_PARTY_STATUS_DONE
|
||
|
||
def _get_detect_execution_order(self):
|
||
default_order = [
|
||
'detect_register',
|
||
'detect_baidu_site',
|
||
'detect_360_site',
|
||
'detect_chinaz',
|
||
'detect_aizhan',
|
||
'detect_wayback',
|
||
'detect_jucha',
|
||
'detect_juziseo',
|
||
]
|
||
configured_order = self.detect_options.get('detect_order') or []
|
||
normalized_order = [key for key in configured_order if key in default_order]
|
||
for key in default_order:
|
||
if key not in normalized_order:
|
||
normalized_order.append(key)
|
||
free_keys = {
|
||
'detect_register',
|
||
'detect_baidu_site',
|
||
'detect_360_site',
|
||
'detect_chinaz',
|
||
'detect_aizhan',
|
||
'detect_wayback',
|
||
}
|
||
paid_keys = {'detect_jucha', 'detect_juziseo'}
|
||
enabled_order = [key for key in normalized_order if self.detect_options.get(key, False)]
|
||
free_order = [key for key in enabled_order if key in free_keys]
|
||
paid_order = [key for key in enabled_order if key in paid_keys]
|
||
return free_order + paid_order
|
||
|
||
def _run_detect_register(self, domain_id, domain, domain_name):
|
||
is_ykj = domain.get('source_type', 0) == 1
|
||
if is_ykj:
|
||
logger.info(f"一口价域名跳过注册状态检测: {domain_name}")
|
||
if self._is_current_single_step_task():
|
||
self._cache_single_step_result_payload(
|
||
domain_id,
|
||
'register_status',
|
||
self._step_result_payload(
|
||
ok=True,
|
||
state="passed",
|
||
message="一口价域名跳过注册状态检测",
|
||
source_type=domain.get('source_type'),
|
||
skipped=True,
|
||
),
|
||
)
|
||
return True
|
||
logger.info(f"检测注册状态: {domain_name}")
|
||
tried_proxy_keys = set()
|
||
started_at = time.perf_counter()
|
||
attempt_count = 0
|
||
last_reason = ""
|
||
step_name = '注册状态检测'
|
||
direct_allowed = self._allow_direct_connection_for_step(step_name)
|
||
register_single_machine_mode = self._register_single_machine_mode_enabled()
|
||
direct_first_attempt_limit = 0
|
||
# 高并发下如果注册检测仍然直连优先，会先把 controller 本机的外连资源打爆，
|
||
# 导致大量线程卡在 direct request error，而不是尽快让代理承接流量。
|
||
# 这里默认改成“代理优先，直连兜底”；只有显式打开环境变量，或最近已明确
|
||
# 进入代理短缺/冷却态时，才允许注册检测直接优先走直连。
|
||
register_direct_first = False
|
||
if direct_allowed:
|
||
register_direct_first = (
|
||
self._env_flag_enabled("DOMAINCHECK_REGISTER_DIRECT_FIRST")
|
||
or self._proxy_status_indicates_shortage()
|
||
or register_single_machine_mode
|
||
)
|
||
if register_direct_first and direct_allowed:
|
||
direct_first_attempt_limit = 1
|
||
if register_single_machine_mode and direct_allowed:
|
||
direct_first_attempt_limit = max(
|
||
direct_first_attempt_limit,
|
||
self._register_single_machine_direct_streak_attempts(),
|
||
)
|
||
direct_attempt_count = 0
|
||
while True:
|
||
budget_exhausted, budget_reason = self._check_proxy_step_retry_budget(
|
||
step_name,
|
||
domain_name,
|
||
started_at,
|
||
attempt_count,
|
||
tried_proxy_keys,
|
||
last_reason=last_reason,
|
||
)
|
||
if budget_exhausted:
|
||
self._mark_detection_failed(domain_id, domain_name, '注册状态检测', budget_reason)
|
||
return False
|
||
|
||
proxy = None
|
||
if direct_attempt_count < direct_first_attempt_limit:
|
||
direct_attempt_count += 1
|
||
self._emit_step_trace(
|
||
domain_name,
|
||
step_name,
|
||
'proxy_direct_fallback',
|
||
proxy_mode='direct_perf' if register_single_machine_mode else 'direct_first',
|
||
retry_attempts=attempt_count,
|
||
)
|
||
else:
|
||
proxy = self._get_proxy_for_step(domain_id, domain_name, step_name, excluded_proxy_keys=tried_proxy_keys)
|
||
if proxy == '__NO_PROXY__':
|
||
return False
|
||
if proxy:
|
||
tried_proxy_keys.add(self._proxy_key(proxy))
|
||
attempt_count += 1
|
||
try:
|
||
tld = domain_name.split('.')[-1]
|
||
remaining_budget_seconds = self._remaining_proxy_step_retry_budget_seconds(
|
||
step_name,
|
||
started_at,
|
||
reserve_seconds=0.2,
|
||
)
|
||
status, expire_date = register.check_register(
|
||
domain_name,
|
||
tld,
|
||
proxy if proxy else None,
|
||
budget_seconds=remaining_budget_seconds,
|
||
)
|
||
if status != -1:
|
||
if proxy:
|
||
self._clear_proxy_failure(proxy)
|
||
self._clear_proxy_source_failure(proxy)
|
||
self.release_proxy(proxy, step_name='注册状态检测')
|
||
normalized_expire_date = expire_date
|
||
if expire_date:
|
||
from datetime import datetime, timedelta
|
||
try:
|
||
date_part = expire_date.split(' ')[0]
|
||
expire_date_obj = datetime.strptime(date_part, "%Y-%m-%d")
|
||
normalized_expire_date = (expire_date_obj + timedelta(days=75)).strftime("%Y-%m-%d")
|
||
except Exception as e:
|
||
logger.error(f"处理过期日期失败: {e}")
|
||
normalized_expire_date = expire_date
|
||
if status in {REGISTER_STATUS_CLIENT_HOLD, REGISTER_STATUS_SERVER_HOLD}:
|
||
hold_label = (
|
||
"clientHold"
|
||
if int(status) == int(REGISTER_STATUS_CLIENT_HOLD)
|
||
else "serverHold"
|
||
)
|
||
blacklist_reason = f"注册状态命中 {hold_label}"
|
||
if self._is_current_single_step_task():
|
||
self._cache_single_step_result_payload(
|
||
domain_id,
|
||
'register_status',
|
||
self._step_result_payload(
|
||
ok=False,
|
||
state="blacklisted",
|
||
message=blacklist_reason,
|
||
register_status=int(status),
|
||
expire_date=normalized_expire_date or "",
|
||
),
|
||
)
|
||
else:
|
||
self.db.update_domain_register_result(domain_id, status, normalized_expire_date)
|
||
domain['register_status'] = status
|
||
if normalized_expire_date:
|
||
domain['expire_date'] = normalized_expire_date
|
||
self._mark_blacklisted(domain_id, domain_name, blacklist_reason)
|
||
logger.warning(f"注册状态检测命中黑名单: {domain_name}, 原因: {blacklist_reason}")
|
||
return False
|
||
if self._is_current_single_step_task():
|
||
self._cache_single_step_result_payload(
|
||
domain_id,
|
||
'register_status',
|
||
self._step_result_payload(
|
||
ok=True,
|
||
state="passed",
|
||
message="success",
|
||
register_status=int(status),
|
||
expire_date=normalized_expire_date or "",
|
||
),
|
||
)
|
||
else:
|
||
self.db.update_domain_register_result(domain_id, status, normalized_expire_date)
|
||
domain['register_status'] = status
|
||
if normalized_expire_date:
|
||
domain['expire_date'] = normalized_expire_date
|
||
logger.info(f"注册状态检测完成: {domain_name}, 状态: {status}, 过期日期: {expire_date}")
|
||
return True
|
||
logger.warning(f"注册状态检测失败，不更新数据: {domain_name}")
|
||
last_reason = 'register lookup failed'
|
||
if self._should_rotate_proxy_on_error(proxy, 'register lookup failed', step_name, domain_name):
|
||
continue
|
||
if proxy:
|
||
self.release_proxy(proxy, step_name=step_name, reason='register lookup failed')
|
||
self._mark_detection_failed(domain_id, domain_name, step_name, '注册状态检测返回未知状态')
|
||
return False
|
||
except Exception as e:
|
||
logger.error(f"注册状态检测失败: {domain_name}, 错误: {e}")
|
||
last_reason = str(e)
|
||
if proxy is None and self.proxy_config.get('proxy_enable', False):
|
||
if direct_allowed and self._proxy_status_indicates_shortage():
|
||
self._set_step_force_direct_once(step_name, True)
|
||
self._set_step_proxy_retry_wait(
|
||
step_name,
|
||
self._proxy_direct_fallback_grace_for_step(step_name),
|
||
)
|
||
logger.warning(
|
||
f"注册状态检测直连失败，代理源仍短缺，下一次继续快速直连: {domain_name}, 原因: {e}"
|
||
)
|
||
else:
|
||
logger.warning(f"注册状态检测直连失败，切换代理继续: {domain_name}, 原因: {e}")
|
||
continue
|
||
if self._should_rotate_proxy_on_error(proxy, str(e), step_name, domain_name):
|
||
continue
|
||
if proxy:
|
||
self.release_proxy(proxy, step_name=step_name, reason=str(e))
|
||
self._mark_detection_failed(domain_id, domain_name, step_name, str(e))
|
||
return False
|
||
return True
|
||
|
||
def _run_detect_wayback(self, domain_id, domain_name, sensitive_words, step_payload=None):
|
||
logger.info(f"检测时光机: {domain_name}")
|
||
try:
|
||
normalized_step_payload = dict(step_payload or {}) if isinstance(step_payload, dict) else {}
|
||
recent_years = normalized_step_payload.get("wayback_recent_years")
|
||
stop_on_first_hit = normalized_step_payload.get("wayback_stop_on_first_hit")
|
||
wayback_result = self.wayback_detector.scan_snapshots(
|
||
domain_name,
|
||
sensitive_words=sensitive_words,
|
||
stop_on_first_hit=bool(True if stop_on_first_hit is None else stop_on_first_hit),
|
||
recent_years=int(recent_years) if str(recent_years or "").strip() else None,
|
||
)
|
||
snapshot_years = wayback_result.get('snapshot_years') or []
|
||
is_step_task = self._is_current_single_step_task()
|
||
backlink_count = int(wayback_result.get('backlink_count', 0) or 0)
|
||
if not is_step_task:
|
||
self.db.update_domain_wayback_summary(
|
||
domain_id,
|
||
years=",".join(str(year) for year in snapshot_years) if snapshot_years else None,
|
||
backlink_count=backlink_count,
|
||
)
|
||
self._upsert_json_detection(
|
||
domain_id,
|
||
backlink_count_gt_10=bool(wayback_result.get('backlink_count_gt_10')),
|
||
)
|
||
if wayback_result.get('has_sensitive_content'):
|
||
reason = f"时光机快照命中敏感词: {wayback_result.get('matched_word', '')}".strip()
|
||
self._record_step_result(domain_id, 'wayback_info', ok=False, state="blacklisted", message=reason)
|
||
self._mark_blacklisted(domain_id, domain_name, reason)
|
||
return False
|
||
if int(wayback_result.get('request_error_count', 0) or 0) > 0 and int(wayback_result.get('checked_snapshot_count', 0) or 0) == 0:
|
||
reason = "时光机检测失败，快照索引请求异常: " + " | ".join(
|
||
(wayback_result.get('request_errors') or [])[:3]
|
||
)
|
||
self._mark_detection_degraded(domain_id, domain_name, '时光机检测', reason, 'wayback_info')
|
||
return True
|
||
if int(wayback_result.get('failed_snapshot_count', 0) or 0) > 0 and int(wayback_result.get('fetched_snapshot_count', 0) or 0) == 0:
|
||
reason = (
|
||
f"时光机检测失败，未成功抓取任何快照；失败次数 {wayback_result.get('failed_snapshot_count', 0)}"
|
||
)
|
||
self._mark_detection_degraded(domain_id, domain_name, '时光机检测', reason, 'wayback_info')
|
||
return True
|
||
self._record_step_result(
|
||
domain_id,
|
||
'wayback_info',
|
||
ok=True,
|
||
state="passed",
|
||
message="success",
|
||
checked_snapshot_count=int(wayback_result.get('checked_snapshot_count', 0) or 0),
|
||
fetched_snapshot_count=int(wayback_result.get('fetched_snapshot_count', 0) or 0),
|
||
failed_snapshot_count=int(wayback_result.get('failed_snapshot_count', 0) or 0),
|
||
backlink_count=int(wayback_result.get('backlink_count', 0) or 0),
|
||
backlink_count_gt_10=bool(wayback_result.get('backlink_count_gt_10')),
|
||
snapshot_years=list(snapshot_years),
|
||
)
|
||
logger.info(
|
||
"时光机检测完成: %s, 检查快照 %s 个, 成功抓取 %s 个, 失败 %s 个, 最大友链 %s, 耗时 %ss"
|
||
% (
|
||
domain_name,
|
||
wayback_result.get('checked_snapshot_count', 0),
|
||
wayback_result.get('fetched_snapshot_count', 0),
|
||
wayback_result.get('failed_snapshot_count', 0),
|
||
wayback_result.get('backlink_count', 0),
|
||
wayback_result.get('elapsed_seconds', 0),
|
||
)
|
||
)
|
||
except Exception as e:
|
||
logger.error(f"时光机检测失败: {domain_name}, 错误: {e}")
|
||
self._mark_detection_degraded(domain_id, domain_name, '时光机检测', str(e), 'wayback_info')
|
||
return True
|
||
return True
|
||
|
||
def _run_detect_chinaz(self, domain_id, domain_name, sensitive_words):
|
||
logger.info(f"检测站长之家: {domain_name}")
|
||
tried_proxy_keys = set()
|
||
started_at = time.perf_counter()
|
||
attempt_count = 0
|
||
last_reason = ""
|
||
while True:
|
||
budget_exhausted, budget_reason = self._check_proxy_step_retry_budget(
|
||
'站长之家检测',
|
||
domain_name,
|
||
started_at,
|
||
attempt_count,
|
||
tried_proxy_keys,
|
||
last_reason=last_reason,
|
||
)
|
||
if budget_exhausted:
|
||
self._mark_detection_degraded(domain_id, domain_name, '站长之家检测', budget_reason, 'chinaz_info')
|
||
return True
|
||
if self._should_force_direct_first_attempt('站长之家检测', attempt_count):
|
||
self._emit_step_trace(
|
||
domain_name,
|
||
'站长之家检测',
|
||
'proxy_direct_fallback',
|
||
elapsed_ms=0,
|
||
proxy_mode='direct',
|
||
recent_proxy_status=self.proxy_last_refresh_status,
|
||
retry_attempts=int(attempt_count or 0),
|
||
reason='single_machine_direct_first',
|
||
)
|
||
proxy = None
|
||
else:
|
||
proxy = self._get_proxy_for_step(domain_id, domain_name, '站长之家检测', excluded_proxy_keys=tried_proxy_keys)
|
||
if proxy == '__NO_PROXY__':
|
||
return False
|
||
if proxy:
|
||
tried_proxy_keys.add(self._proxy_key(proxy))
|
||
attempt_count += 1
|
||
try:
|
||
remaining_budget_seconds = self._remaining_proxy_step_retry_budget_seconds(
|
||
'站长之家检测',
|
||
started_at,
|
||
reserve_seconds=0.2,
|
||
)
|
||
logger.info(
|
||
f"站长之家调用边界: domain={domain_name} | attempt={attempt_count} "
|
||
f"| module={getattr(chinaz, '__file__', 'unknown')} | proxy={'yes' if proxy else 'no'}"
|
||
)
|
||
success, message, seo_data = chinaz.check_title(
|
||
domain_name,
|
||
sensitive_words,
|
||
proxy,
|
||
budget_seconds=remaining_budget_seconds,
|
||
)
|
||
logger.info(
|
||
f"站长之家返回边界: domain={domain_name} | attempt={attempt_count} "
|
||
f"| success={success} | message={message}"
|
||
)
|
||
if not success:
|
||
logger.warning(f"站长之家检测未通过: {domain_name}, 原因: {message}")
|
||
last_reason = str(message or "")
|
||
if self._should_blacklist_result(message):
|
||
if proxy:
|
||
self.release_proxy(proxy, step_name='站长之家检测', reason=message or 'blacklisted')
|
||
self._record_step_result(domain_id, 'chinaz_info', ok=False, state="blacklisted", message=message)
|
||
self._mark_blacklisted(domain_id, domain_name, message)
|
||
return False
|
||
if self._should_rotate_proxy_on_error(proxy, message, '站长之家检测', domain_name):
|
||
continue
|
||
if self._is_external_dependency_issue(message):
|
||
if proxy:
|
||
self.release_proxy(proxy, step_name='站长之家检测', reason=message or 'external dependency issue')
|
||
self._mark_detection_degraded(domain_id, domain_name, '站长之家检测', message or 'external dependency issue', 'chinaz_info')
|
||
return True
|
||
if proxy:
|
||
self.release_proxy(proxy, step_name='站长之家检测', reason=message or 'business rejected')
|
||
self._mark_detection_rejected(domain_id, domain_name, '站长之家检测', message or 'business rejected', 'chinaz_info')
|
||
return False
|
||
if proxy:
|
||
self._clear_proxy_failure(proxy)
|
||
self._clear_proxy_source_failure(proxy)
|
||
self.release_proxy(proxy, step_name='站长之家检测')
|
||
self._record_step_result(domain_id, 'chinaz_info', ok=True, state="passed", message=message or "success", seo=seo_data)
|
||
logger.info(f"站长之家检测完成: {domain_name}")
|
||
return True
|
||
except Exception as e:
|
||
logger.error(f"站长之家检测失败: {domain_name}, 错误: {e}")
|
||
last_reason = str(e)
|
||
if self._should_rotate_proxy_on_error(proxy, str(e), '站长之家检测', domain_name):
|
||
continue
|
||
if self._is_external_dependency_issue(e):
|
||
if proxy:
|
||
self.release_proxy(proxy, step_name='站长之家检测', reason=str(e))
|
||
self._mark_detection_degraded(domain_id, domain_name, '站长之家检测', str(e), 'chinaz_info')
|
||
return True
|
||
if proxy:
|
||
self.release_proxy(proxy, step_name='站长之家检测', reason=str(e))
|
||
self._mark_detection_failed(domain_id, domain_name, '站长之家检测', str(e), 'chinaz_info')
|
||
return False
|
||
return True
|
||
|
||
def _run_detect_aizhan(self, domain_id, domain_name, sensitive_words):
|
||
logger.info(f"检测爱站网: {domain_name}")
|
||
tried_proxy_keys = set()
|
||
started_at = time.perf_counter()
|
||
attempt_count = 0
|
||
last_reason = ""
|
||
while True:
|
||
budget_exhausted, budget_reason = self._check_proxy_step_retry_budget(
|
||
'爱站网检测',
|
||
domain_name,
|
||
started_at,
|
||
attempt_count,
|
||
tried_proxy_keys,
|
||
last_reason=last_reason,
|
||
)
|
||
if budget_exhausted:
|
||
self._mark_detection_degraded(domain_id, domain_name, '爱站网检测', budget_reason, 'aizhan_info')
|
||
return True
|
||
if self._should_force_direct_first_attempt('爱站网检测', attempt_count):
|
||
self._set_step_force_direct_once('爱站网检测', True)
|
||
proxy = self._get_proxy_for_step(domain_id, domain_name, '爱站网检测', excluded_proxy_keys=tried_proxy_keys)
|
||
if proxy == '__NO_PROXY__':
|
||
return False
|
||
if proxy:
|
||
tried_proxy_keys.add(self._proxy_key(proxy))
|
||
attempt_count += 1
|
||
try:
|
||
remaining_budget_seconds = self._remaining_proxy_step_retry_budget_seconds(
|
||
'爱站网检测',
|
||
started_at,
|
||
reserve_seconds=0.2,
|
||
)
|
||
success, message = aizhan.check_aizhan(
|
||
domain_name,
|
||
sensitive_words,
|
||
proxy,
|
||
budget_seconds=remaining_budget_seconds,
|
||
)
|
||
if not success:
|
||
logger.warning(f"爱站网检测未通过: {domain_name}, 原因: {message}")
|
||
last_reason = str(message or "")
|
||
if self._should_blacklist_result(message):
|
||
if proxy:
|
||
self.release_proxy(proxy, step_name='爱站网检测', reason=message or 'blacklisted')
|
||
self._record_step_result(domain_id, 'aizhan_info', ok=False, state="blacklisted", message=message)
|
||
self._mark_blacklisted(domain_id, domain_name, message)
|
||
return False
|
||
if self._should_degrade_external_issue_immediately('爱站网检测', message):
|
||
if proxy:
|
||
self.release_proxy(proxy, step_name='爱站网检测', reason=message or 'external dependency issue')
|
||
self._mark_detection_degraded(domain_id, domain_name, '爱站网检测', message or 'external dependency issue', 'aizhan_info')
|
||
return True
|
||
if self._should_retry_external_issue(proxy, message, '爱站网检测', domain_name):
|
||
continue
|
||
if self._is_external_dependency_issue(message):
|
||
if proxy:
|
||
self.release_proxy(proxy, step_name='爱站网检测', reason=message or 'external dependency issue')
|
||
self._mark_detection_degraded(domain_id, domain_name, '爱站网检测', message or 'external dependency issue', 'aizhan_info')
|
||
return True
|
||
if proxy:
|
||
self.release_proxy(proxy, step_name='爱站网检测', reason=message or 'business rejected')
|
||
self._mark_detection_rejected(domain_id, domain_name, '爱站网检测', message or 'business rejected', 'aizhan_info')
|
||
return False
|
||
if proxy:
|
||
self._clear_proxy_failure(proxy)
|
||
self._clear_proxy_source_failure(proxy)
|
||
self.release_proxy(proxy, step_name='爱站网检测')
|
||
self._record_step_result(domain_id, 'aizhan_info', ok=True, state="passed", message=message or "success")
|
||
logger.info(f"爱站网检测完成: {domain_name}")
|
||
return True
|
||
except Exception as e:
|
||
logger.error(f"爱站网检测失败: {domain_name}, 错误: {e}")
|
||
last_reason = str(e)
|
||
if self._should_degrade_external_issue_immediately('爱站网检测', e):
|
||
if proxy:
|
||
self.release_proxy(proxy, step_name='爱站网检测', reason=str(e))
|
||
self._mark_detection_degraded(domain_id, domain_name, '爱站网检测', str(e), 'aizhan_info')
|
||
return True
|
||
if self._should_retry_external_issue(proxy, str(e), '爱站网检测', domain_name):
|
||
continue
|
||
if self._is_external_dependency_issue(e):
|
||
if proxy:
|
||
self.release_proxy(proxy, step_name='爱站网检测', reason=str(e))
|
||
self._mark_detection_degraded(domain_id, domain_name, '爱站网检测', str(e), 'aizhan_info')
|
||
return True
|
||
if proxy:
|
||
self.release_proxy(proxy, step_name='爱站网检测', reason=str(e))
|
||
self._mark_detection_failed(domain_id, domain_name, '爱站网检测', str(e), 'aizhan_info')
|
||
return False
|
||
return True
|
||
|
||
def _run_detect_baidu(self, domain_id, domain_name, sensitive_words):
|
||
logger.info(f"检测百度: {domain_name}")
|
||
tried_proxy_keys = set()
|
||
started_at = time.perf_counter()
|
||
attempt_count = 0
|
||
last_reason = ""
|
||
while True:
|
||
budget_exhausted, budget_reason = self._check_proxy_step_retry_budget(
|
||
'百度site检测',
|
||
domain_name,
|
||
started_at,
|
||
attempt_count,
|
||
tried_proxy_keys,
|
||
last_reason=last_reason,
|
||
)
|
||
if budget_exhausted:
|
||
self._mark_detection_degraded(domain_id, domain_name, '百度site检测', budget_reason, 'baidu_site')
|
||
return True
|
||
if self._should_force_direct_first_attempt('百度site检测', attempt_count):
|
||
self._set_step_force_direct_once('百度site检测', True)
|
||
proxy = self._get_proxy_for_step(domain_id, domain_name, '百度site检测', excluded_proxy_keys=tried_proxy_keys)
|
||
if proxy == '__NO_PROXY__':
|
||
return False
|
||
if proxy:
|
||
tried_proxy_keys.add(self._proxy_key(proxy))
|
||
attempt_count += 1
|
||
try:
|
||
remaining_budget_seconds = self._remaining_proxy_step_retry_budget_seconds(
|
||
'百度site检测',
|
||
started_at,
|
||
reserve_seconds=0.2,
|
||
)
|
||
success, message = baidu.check_site(
|
||
domain_name,
|
||
sensitive_words,
|
||
proxy,
|
||
budget_seconds=remaining_budget_seconds,
|
||
)
|
||
if not success:
|
||
logger.warning(f"百度site检测未通过: {domain_name}, 原因: {message}")
|
||
last_reason = str(message or "")
|
||
if self._should_blacklist_result(message):
|
||
if proxy:
|
||
self.release_proxy(proxy, step_name='百度site检测', reason=message or 'blacklisted')
|
||
self._record_step_result(domain_id, 'baidu_site', ok=False, state="blacklisted", message=message)
|
||
self._mark_blacklisted(domain_id, domain_name, message)
|
||
return False
|
||
if self._should_retry_external_issue(proxy, message, '百度site检测', domain_name):
|
||
continue
|
||
if self._is_external_dependency_issue(message):
|
||
if proxy:
|
||
self.release_proxy(proxy, step_name='百度site检测', reason=message or 'external dependency issue')
|
||
self._mark_detection_degraded(domain_id, domain_name, '百度site检测', message, 'baidu_site')
|
||
return True
|
||
if proxy:
|
||
self.release_proxy(proxy, step_name='百度site检测', reason=message or 'business rejected')
|
||
self._mark_detection_rejected(domain_id, domain_name, '百度site检测', message or 'business rejected', 'baidu_site')
|
||
return False
|
||
if proxy:
|
||
self._clear_proxy_failure(proxy)
|
||
self._clear_proxy_source_failure(proxy)
|
||
self.release_proxy(proxy, step_name='百度site检测')
|
||
self._record_step_result(domain_id, 'baidu_site', ok=True, state="passed", message=message or "success")
|
||
logger.info(f"百度site检测完成: {domain_name}")
|
||
return True
|
||
except Exception as e:
|
||
logger.error(f"百度site检测失败: {domain_name}, 错误: {e}")
|
||
last_reason = str(e)
|
||
if self._should_retry_external_issue(proxy, str(e), '百度site检测', domain_name):
|
||
continue
|
||
if self._is_external_dependency_issue(e):
|
||
if proxy:
|
||
self.release_proxy(proxy, step_name='百度site检测', reason=str(e))
|
||
self._mark_detection_degraded(domain_id, domain_name, '百度site检测', str(e), 'baidu_site')
|
||
return True
|
||
if proxy:
|
||
self.release_proxy(proxy, step_name='百度site检测', reason=str(e))
|
||
self._mark_detection_failed(domain_id, domain_name, '百度site检测', str(e), 'baidu_site')
|
||
return False
|
||
return True
|
||
|
||
def _run_detect_360(self, domain_id, domain_name, sensitive_words):
|
||
logger.info(f"检测360: {domain_name}")
|
||
tried_proxy_keys = set()
|
||
started_at = time.perf_counter()
|
||
attempt_count = 0
|
||
last_reason = ""
|
||
while True:
|
||
budget_exhausted, budget_reason = self._check_proxy_step_retry_budget(
|
||
'360检测',
|
||
domain_name,
|
||
started_at,
|
||
attempt_count,
|
||
tried_proxy_keys,
|
||
last_reason=last_reason,
|
||
)
|
||
if budget_exhausted:
|
||
self._mark_detection_degraded(domain_id, domain_name, '360检测', budget_reason, 'qihu360_site')
|
||
return True
|
||
proxy = self._get_proxy_for_step(domain_id, domain_name, '360检测', excluded_proxy_keys=tried_proxy_keys)
|
||
if proxy == '__NO_PROXY__':
|
||
return False
|
||
if proxy:
|
||
tried_proxy_keys.add(self._proxy_key(proxy))
|
||
attempt_count += 1
|
||
try:
|
||
remaining_budget_seconds = self._remaining_proxy_step_retry_budget_seconds(
|
||
'360检测',
|
||
started_at,
|
||
reserve_seconds=0.2,
|
||
)
|
||
passed, message = c360.check_domain(
|
||
domain_name,
|
||
sensitive_words,
|
||
proxy,
|
||
budget_seconds=remaining_budget_seconds,
|
||
)
|
||
if not passed:
|
||
logger.warning(f"360检测未通过: {domain_name}, 原因: {message}")
|
||
last_reason = str(message or "")
|
||
if self._should_blacklist_result(message):
|
||
if proxy:
|
||
self.release_proxy(proxy, step_name='360检测', reason=message or 'blacklisted')
|
||
self._record_step_result(domain_id, 'qihu360_site', ok=False, state="blacklisted", message=message)
|
||
self._mark_blacklisted(domain_id, domain_name, message)
|
||
return False
|
||
if self._should_retry_external_issue(proxy, message, '360检测', domain_name):
|
||
continue
|
||
if self._is_external_dependency_issue(message):
|
||
if proxy:
|
||
self.release_proxy(proxy, step_name='360检测', reason=message or 'external dependency issue')
|
||
self._mark_detection_degraded(domain_id, domain_name, '360检测', message or 'external dependency issue', 'qihu360_site')
|
||
return True
|
||
if proxy:
|
||
self.release_proxy(proxy, step_name='360检测', reason=message or 'business rejected')
|
||
self._mark_detection_rejected(domain_id, domain_name, '360检测', message or 'business rejected', 'qihu360_site')
|
||
return False
|
||
if proxy:
|
||
self._clear_proxy_failure(proxy)
|
||
self._clear_proxy_source_failure(proxy)
|
||
self.release_proxy(proxy, step_name='360检测')
|
||
self._record_step_result(domain_id, 'qihu360_site', ok=True, state="passed", message=message or "success")
|
||
logger.info(f"360检测完成: {domain_name}")
|
||
return True
|
||
except Exception as e:
|
||
logger.error(f"360检测失败: {domain_name}, 错误: {e}")
|
||
last_reason = str(e)
|
||
if self._should_retry_external_issue(proxy, str(e), '360检测', domain_name):
|
||
continue
|
||
if self._is_external_dependency_issue(e):
|
||
if proxy:
|
||
self.release_proxy(proxy, step_name='360检测', reason=str(e))
|
||
self._mark_detection_degraded(domain_id, domain_name, '360检测', str(e), 'qihu360_site')
|
||
return True
|
||
if proxy:
|
||
self.release_proxy(proxy, step_name='360检测', reason=str(e))
|
||
self._mark_detection_failed(domain_id, domain_name, '360检测', str(e), 'qihu360_site')
|
||
return False
|
||
return True
|
||
|
||
def _run_detect_jucha(self, domain_id, domain, domain_name):
|
||
if not self._should_run_jucha(domain):
|
||
return True
|
||
self._init_optional_detectors()
|
||
if self.jc_instance is None:
|
||
logger.warning("聚查检测器未初始化，跳过本次聚查检测")
|
||
return True
|
||
proxy = self._get_proxy_for_step(domain_id, domain_name, '聚查检测')
|
||
if proxy == '__NO_PROXY__':
|
||
return False
|
||
self.jc_instance.session.proxies = proxy or {}
|
||
beian_update_payload = {}
|
||
logger.info(f"检测聚查WHOIS: {domain_name}")
|
||
try:
|
||
success, message, whois_info = self.jc_instance.check_whois_domain(domain_name)
|
||
if not success:
|
||
logger.warning(f"聚查WHOIS查询失败: {domain_name}, 原因: {message}")
|
||
else:
|
||
logger.info(f"WHOIS查询结果: {success}, {message}, {whois_info}")
|
||
if 'clientHold' in whois_info or 'serverHold' in whois_info:
|
||
logger.warning(f"聚查WHOIS检测未通过: {domain_name}, 原因: 域名状态包含clientHold或serverHold")
|
||
self._mark_blacklisted(domain_id, domain_name, '域名状态包含clientHold或serverHold')
|
||
return False
|
||
logger.info(f"聚查WHOIS检测完成: {domain_name}")
|
||
except Exception as e:
|
||
logger.error(f"聚查WHOIS检测失败: {domain_name}, 错误: {e}")
|
||
|
||
logger.info(f"检测聚查备案: {domain_name}")
|
||
try:
|
||
success, message, beian_info = self.jc_instance.beian_check_domain(domain_name)
|
||
if not success:
|
||
logger.warning(f"聚查备案查询失败: {domain_name}, 原因: {message}")
|
||
else:
|
||
logger.info(f"备案查询结果: {success}, {message}, {beian_info}")
|
||
if isinstance(beian_info, tuple) and len(beian_info) == 4:
|
||
beian_time, company_type, website_url, has_beian = beian_info
|
||
has_beian_flag = 2 if has_beian == '当前存在' else 3
|
||
beian_year = None
|
||
if beian_time:
|
||
try:
|
||
beian_year = beian_time.split('-')[0]
|
||
except Exception:
|
||
pass
|
||
beian_update_payload = {
|
||
"company_type": company_type,
|
||
"website_url": website_url,
|
||
"has_beian": has_beian_flag,
|
||
"beian_year": beian_year,
|
||
}
|
||
logger.info(f"聚查备案检测完成: {domain_name}")
|
||
except Exception as e:
|
||
logger.error(f"聚查备案检测失败: {domain_name}, 错误: {e}")
|
||
|
||
logger.info(f"检测聚查拦截: {domain_name}")
|
||
try:
|
||
success, message, safe_info = self.jc_instance.safe_check_domain(domain_name)
|
||
if not success:
|
||
logger.warning(f"聚查安全检测失败: {domain_name}, 原因: {message}")
|
||
else:
|
||
logger.info(f"安全检测结果: {success}, {message}, {safe_info}")
|
||
check_items = ['QQ检测', '微信检测', '抖音检测', '被墙检测', '百度检测', '谷歌检测', '火狐检测']
|
||
blacklist_reason = []
|
||
if isinstance(safe_info, tuple):
|
||
for i, item in enumerate(safe_info):
|
||
if isinstance(item, tuple):
|
||
if len(item) >= 1 and item[0] == 3 and i < len(check_items):
|
||
blacklist_reason.append(f"{check_items[i]}: 拦截")
|
||
elif len(item) >= 2 and item[0] == 2 and '拦截' in item[1] and i < len(check_items):
|
||
blacklist_reason.append(f"{check_items[i]}: 拦截")
|
||
if blacklist_reason:
|
||
reason_str = "；".join(blacklist_reason)
|
||
logger.warning(f"聚查安全检测未通过: {domain_name}, 原因: {reason_str}")
|
||
self._mark_blacklisted(domain_id, domain_name, reason_str)
|
||
return False
|
||
logger.info(f"聚查拦截检测完成: {domain_name}")
|
||
except Exception as e:
|
||
logger.error(f"聚查拦截检测失败: {domain_name}, 错误: {e}")
|
||
finally:
|
||
self.db.update_domain_beian_info_and_mark_jucha_detected(
|
||
domain_id,
|
||
company_type=beian_update_payload.get("company_type"),
|
||
website_url=beian_update_payload.get("website_url"),
|
||
has_beian=beian_update_payload.get("has_beian"),
|
||
beian_year=beian_update_payload.get("beian_year"),
|
||
)
|
||
if proxy:
|
||
self._clear_proxy_failure(proxy)
|
||
self._clear_proxy_source_failure(proxy)
|
||
self.release_proxy(proxy, step_name='聚查检测')
|
||
return True
|
||
|
||
def _run_detect_juziseo(self, domain_id, domain, domain_name, sensitive_words):
|
||
if not self._should_run_juziseo(domain):
|
||
return True
|
||
self._init_optional_detectors()
|
||
if self.juziseo_instance is None:
|
||
logger.warning("桔子SEO检测器未初始化，跳过本次桔子检测")
|
||
return True
|
||
proxy = self._get_proxy_for_step(domain_id, domain_name, '桔子检测')
|
||
if proxy == '__NO_PROXY__':
|
||
return False
|
||
self.juziseo_instance.session.proxies = proxy or {}
|
||
logger.info(f"检测桔子历史: {domain_name}")
|
||
try:
|
||
success, message = self.juziseo_instance.check_history(domain_name, sensitive_words)
|
||
if not success:
|
||
logger.warning(f"桔子历史检测未通过: {domain_name}, 原因: {message}")
|
||
self._mark_blacklisted(domain_id, domain_name, message)
|
||
return False
|
||
logger.info(f"桔子历史检测完成: {domain_name}")
|
||
except Exception as e:
|
||
logger.error(f"桔子历史检测失败: {domain_name}, 错误: {e}")
|
||
|
||
logger.info(f"检测桔子外链: {domain_name}")
|
||
try:
|
||
success, message = self.juziseo_instance.check_external_link(domain_name, sensitive_words)
|
||
if not success:
|
||
logger.warning(f"桔子外链检测未通过: {domain_name}, 原因: {message}")
|
||
self._mark_blacklisted(domain_id, domain_name, message)
|
||
return False
|
||
logger.info(f"桔子外链检测完成: {domain_name}")
|
||
except Exception as e:
|
||
logger.error(f"桔子外链检测失败: {domain_name}, 错误: {e}")
|
||
finally:
|
||
self.db.mark_juziseo_detected(domain_id)
|
||
if proxy:
|
||
self._clear_proxy_failure(proxy)
|
||
self._clear_proxy_source_failure(proxy)
|
||
self.release_proxy(proxy, step_name='桔子检测')
|
||
return True
|
||
|
||
def _upsert_json_detection(self, domain_id, **payload):
|
||
existing = self.db.fetch_one("SELECT id FROM domain_detections WHERE domain_id = %s", (domain_id,))
|
||
columns = []
|
||
params = []
|
||
for key, value in payload.items():
|
||
columns.append(key)
|
||
params.append(json.dumps(value) if isinstance(value, (dict, list, tuple)) else value)
|
||
if not columns:
|
||
return True
|
||
if existing:
|
||
assignments = ", ".join([f"{column} = %s" for column in columns] + ["update_time = CURRENT_TIMESTAMP"])
|
||
params.append(domain_id)
|
||
sql = f"UPDATE domain_detections SET {assignments} WHERE domain_id = %s"
|
||
else:
|
||
sql = f"INSERT INTO domain_detections (domain_id, {', '.join(columns)}) VALUES (%s, {', '.join(['%s'] * len(columns))})"
|
||
params = [domain_id] + params
|
||
return self.db.execute(sql, tuple(params))
|
||
|
||
def detect_domain(self, domain_id, domain, job_context=None):
|
||
"""
|
||
检测单个域名
|
||
|
||
:param domain_id: 域名ID
|
||
:param domain: 域名对象（包含id、domain、source_type等字段）
|
||
"""
|
||
import threading
|
||
thread_id = threading.current_thread().ident
|
||
|
||
# 从域名对象中获取域名字符串
|
||
domain_name = domain.get('domain', '')
|
||
logger.info(f"线程 {thread_id} 开始检测域名: {domain_name}")
|
||
|
||
job_context = job_context or {}
|
||
job_item_id = job_context.get('job_item_id')
|
||
job_id = job_context.get('job_id')
|
||
claim_token = job_context.get('claim_token')
|
||
task_mode = str(job_context.get('task_mode') or '').strip() or 'domain_pipeline'
|
||
step_payload = job_context.get('step_payload') if isinstance(job_context.get('step_payload'), dict) else {}
|
||
requested_step_code = str(step_payload.get('step_code') or job_context.get('step_code') or '').strip()
|
||
is_step_task = bool(requested_step_code)
|
||
cycle_token = str(job_context.get('cycle_token') or self.current_cycle_token or '').strip()
|
||
job_code = str(job_context.get('job_code') or self.current_job_code or '').strip()
|
||
session_id = int(job_context.get('session_id') or 0)
|
||
domain_started_at = time.perf_counter()
|
||
self._set_current_task_context(
|
||
domain_id=domain_id,
|
||
is_step_task=is_step_task,
|
||
task_mode=task_mode,
|
||
job_item_id=job_item_id,
|
||
)
|
||
early_abort_reason = self._detection_session_abort_reason(session_id)
|
||
if early_abort_reason:
|
||
self._release_job_item_for_session_abort(
|
||
job_item_id,
|
||
claim_token,
|
||
session_id=session_id,
|
||
domain_name=domain_name,
|
||
detail="queued_before_start",
|
||
)
|
||
logger.info(
|
||
f"检测会话已切换，跳过旧域名任务派发: {domain_name}, "
|
||
f"job_item_id={job_item_id or 0}, reason={early_abort_reason}"
|
||
)
|
||
self._emit_step_trace(
|
||
domain_name,
|
||
"domain",
|
||
"aborted",
|
||
job_item_id=job_item_id,
|
||
elapsed_ms=0,
|
||
ok=False,
|
||
error=f"检测会话已失效: reason={early_abort_reason} stage=queued_before_start domain={domain_name}"[:300],
|
||
)
|
||
self._clear_current_task_context()
|
||
return
|
||
current_active_threads = self._change_active_domain_threads(1)
|
||
self._note_domain_started()
|
||
if self.detecting and (
|
||
current_active_threads <= 16
|
||
or current_active_threads % 10 == 0
|
||
or current_active_threads >= max(1, int(getattr(self, "thread_count", 1) or 1))
|
||
):
|
||
self._mark_detection_phase(
|
||
"running",
|
||
f"当前实际线程数量: {current_active_threads}/{self.thread_count}",
|
||
active_threads=current_active_threads,
|
||
max_threads=self.thread_count,
|
||
)
|
||
self._emit_step_trace(
|
||
domain_name,
|
||
"domain",
|
||
"started",
|
||
job_item_id=job_item_id,
|
||
thread_id=thread_id,
|
||
cycle_token=cycle_token,
|
||
job_code=job_code,
|
||
)
|
||
|
||
def finalize_job_item(final_status, message="", result_payload=None):
|
||
self._note_domain_result()
|
||
if not job_item_id or not claim_token:
|
||
return
|
||
abort_reason = self._detection_session_abort_reason(session_id)
|
||
if abort_reason:
|
||
self._release_job_item_for_session_abort(
|
||
job_item_id,
|
||
claim_token,
|
||
session_id=session_id,
|
||
domain_name=domain_name,
|
||
detail=f"finalize:{final_status}",
|
||
)
|
||
logger.warning(
|
||
f"检测会话已切换，跳过任务项完成态回写并释放回队列: domain={domain_name}, "
|
||
f"job_item_id={job_item_id}, status={final_status}, reason={abort_reason}"
|
||
)
|
||
return
|
||
try:
|
||
db_host = str(getattr(self.db, "host", "") or getattr(config, "DB_HOST", "") or "").strip() or "db"
|
||
db_name = str(getattr(self.db, "database", "") or getattr(config, "DB_DATABASE", "") or "").strip() or "-"
|
||
event_payload = {
|
||
"domain_id": domain_id,
|
||
"domain": domain_name,
|
||
"status": final_status,
|
||
"cycle_token": cycle_token,
|
||
"job_code": job_code,
|
||
}
|
||
self._enqueue_job_finalization(
|
||
job_item_id=job_item_id,
|
||
claim_token=claim_token,
|
||
final_status=final_status,
|
||
message=message or final_status,
|
||
result_payload=result_payload,
|
||
job_id=job_id,
|
||
node_code=config.NODE_CODE,
|
||
event_type=f"domain_{final_status}",
|
||
event_level='error' if final_status == 'failed' else 'info',
|
||
event_message=message or f"{domain_name} -> {final_status}",
|
||
event_payload=event_payload,
|
||
)
|
||
logger.info(
|
||
"登记 domain 终态事件入队: "
|
||
f"domain={domain_name} job_id={job_id or 0} job_item_id={job_item_id or 0} "
|
||
f"event_type=domain_{final_status} db={db_host}:{db_name}"
|
||
)
|
||
except Exception as finalize_error:
|
||
logger.warning(
|
||
f"回写任务项状态失败: {domain_name}, 错误: {finalize_error}, "
|
||
f"job_id={job_id or 0}, job_item_id={job_item_id or 0}, "
|
||
f"event_type=domain_{final_status}"
|
||
)
|
||
|
||
def renew_job_item_lease(detail=""):
|
||
if is_step_task or not job_item_id or not claim_token:
|
||
return
|
||
abort_reason = self._detection_session_abort_reason(session_id)
|
||
if abort_reason:
|
||
self._release_job_item_for_session_abort(
|
||
job_item_id,
|
||
claim_token,
|
||
session_id=session_id,
|
||
domain_name=domain_name,
|
||
detail=f"renew:{detail}",
|
||
)
|
||
logger.warning(
|
||
f"检测会话已切换，跳过任务项续租并释放回队列: domain={domain_name}, "
|
||
f"job_item_id={job_item_id}, detail={detail}, reason={abort_reason}"
|
||
)
|
||
return
|
||
try:
|
||
self.db.renew_detect_job_item_lease(
|
||
job_item_id,
|
||
claim_token,
|
||
# 租约续期必须和线程数解耦；线程数越高不代表单任务应该锁更久，
|
||
# 否则节点重启或线程异常后会把大量 running 项锁死数天。
|
||
lease_seconds=900,
|
||
)
|
||
except Exception as renew_error:
|
||
logger.warning(f"续租任务项失败: {domain_name}, 错误: {renew_error}")
|
||
|
||
try:
|
||
# 检查是否需要停止
|
||
if not self.running:
|
||
logger.info(f"检测已停止，跳过域名: {domain_name}")
|
||
finalize_job_item('failed', 'Worker 已停止，任务未执行')
|
||
return
|
||
self._assert_detection_session_active(session_id, domain_name=domain_name, stage="prepare")
|
||
|
||
logger.info(f"开始检测域名: {domain_name}")
|
||
if not is_step_task:
|
||
self.db.update_domain_detect_status(domain_id, DETECT_STATUS_RUNNING)
|
||
self._set_task_detect_status(DETECT_STATUS_RUNNING)
|
||
if job_item_id and claim_token:
|
||
self._enqueue_running_mark(job_item_id, claim_token)
|
||
renew_job_item_lease("domain_started")
|
||
db_host = str(getattr(self.db, "host", "") or getattr(config, "DB_HOST", "") or "").strip() or "db"
|
||
db_name = str(getattr(self.db, "database", "") or getattr(config, "DB_DATABASE", "") or "").strip() or "-"
|
||
self.db.append_detect_run_event(
|
||
job_id,
|
||
job_item_id,
|
||
config.NODE_CODE,
|
||
event_type='domain_started',
|
||
message=f"开始检测域名: {domain_name}",
|
||
payload={
|
||
"domain_id": domain_id,
|
||
"domain": domain_name,
|
||
"cycle_token": cycle_token,
|
||
"job_code": job_code,
|
||
"task_mode": task_mode,
|
||
"step_code": requested_step_code if is_step_task else "",
|
||
},
|
||
)
|
||
logger.info(
|
||
"写入 domain_started 事件成功: "
|
||
f"domain={domain_name} job_id={job_id or 0} job_item_id={job_item_id or 0} "
|
||
f"event_type=domain_started db={db_host}:{db_name} "
|
||
f"task_mode={task_mode} step_code={requested_step_code if is_step_task else '-'}"
|
||
)
|
||
self._sync_worker_log_event(
|
||
f"开始检测域名: {domain_name}",
|
||
payload={
|
||
"domain_id": domain_id,
|
||
"domain": domain_name,
|
||
},
|
||
mode='full',
|
||
job_item_id=job_item_id,
|
||
)
|
||
|
||
# 使用共享的敏感词列表
|
||
sensitive_words = self.sensitive_words
|
||
execution_order = self._get_detect_execution_order()
|
||
|
||
parallel_free_steps_enabled = bool(
|
||
int(os.getenv("DOMAINCHECK_PARALLEL_FREE_STEPS", "1") or 1)
|
||
)
|
||
parallel_free_step_keys = (
|
||
'detect_baidu_site',
|
||
'detect_360_site',
|
||
'detect_chinaz',
|
||
'detect_aizhan',
|
||
)
|
||
parallel_free_step_set = set(parallel_free_step_keys)
|
||
parallel_free_step_workers = max(
|
||
1,
|
||
int(os.getenv("DOMAINCHECK_PARALLEL_FREE_STEP_WORKERS", "1") or 1),
|
||
)
|
||
|
||
def execute_detect_step(detect_key):
|
||
step_name = self._step_display_name(detect_key)
|
||
self._assert_detection_session_active(session_id, domain_name=domain_name, stage=f"{detect_key}:prepare")
|
||
renew_job_item_lease(detect_key)
|
||
step_started_at = time.perf_counter()
|
||
self._emit_step_trace(
|
||
domain_name,
|
||
step_name,
|
||
"started",
|
||
job_item_id=job_item_id,
|
||
detect_key=detect_key,
|
||
)
|
||
step_ok = True
|
||
if detect_key == 'detect_register':
|
||
step_ok = self._run_detect_register(domain_id, domain, domain_name)
|
||
elif detect_key == 'detect_baidu_site':
|
||
step_ok = self._run_detect_baidu(domain_id, domain_name, sensitive_words)
|
||
elif detect_key == 'detect_360_site':
|
||
step_ok = self._run_detect_360(domain_id, domain_name, sensitive_words)
|
||
elif detect_key == 'detect_chinaz':
|
||
step_ok = self._run_detect_chinaz(domain_id, domain_name, sensitive_words)
|
||
elif detect_key == 'detect_aizhan':
|
||
step_ok = self._run_detect_aizhan(domain_id, domain_name, sensitive_words)
|
||
elif detect_key == 'detect_wayback':
|
||
step_ok = self._run_detect_wayback(
|
||
domain_id,
|
||
domain_name,
|
||
sensitive_words,
|
||
step_payload=step_payload,
|
||
)
|
||
elif detect_key == 'detect_jucha':
|
||
step_ok = self._run_detect_jucha(domain_id, domain, domain_name)
|
||
elif detect_key == 'detect_juziseo':
|
||
step_ok = self._run_detect_juziseo(domain_id, domain, domain_name, sensitive_words)
|
||
else:
|
||
logger.error(f"不支持的检测步骤: {detect_key}")
|
||
step_ok = False
|
||
|
||
step_elapsed_ms = int((time.perf_counter() - step_started_at) * 1000)
|
||
self._assert_detection_session_active(session_id, domain_name=domain_name, stage=f"{detect_key}:finished")
|
||
self._emit_step_trace(
|
||
domain_name,
|
||
step_name,
|
||
"finished",
|
||
job_item_id=job_item_id,
|
||
detect_key=detect_key,
|
||
elapsed_ms=step_elapsed_ms,
|
||
ok=step_ok,
|
||
)
|
||
return {
|
||
"detect_key": detect_key,
|
||
"step_name": step_name,
|
||
"step_ok": bool(step_ok),
|
||
"step_elapsed_ms": step_elapsed_ms,
|
||
}
|
||
|
||
if is_step_task:
|
||
self._emit_step_trace(
|
||
domain_name,
|
||
"domain",
|
||
"plan_ready",
|
||
job_item_id=job_item_id,
|
||
step_count=1,
|
||
detect_order=requested_step_code,
|
||
sensitive_word_count=len(sensitive_words),
|
||
task_mode=task_mode,
|
||
)
|
||
step_result = execute_detect_step(requested_step_code)
|
||
self._assert_detection_session_active(session_id, domain_name=domain_name, stage=f"{requested_step_code}:finalize")
|
||
result_payload = self._build_single_step_job_result(
|
||
domain_id=domain_id,
|
||
detect_key=requested_step_code,
|
||
step_ok=step_result["step_ok"],
|
||
step_name=step_result["step_name"],
|
||
)
|
||
final_status, final_message = self._resolve_single_step_finalization(result_payload)
|
||
self._emit_step_trace(
|
||
domain_name,
|
||
step_result["step_name"],
|
||
"single_step_finalized",
|
||
job_item_id=job_item_id,
|
||
detect_key=requested_step_code,
|
||
elapsed_ms=step_result["step_elapsed_ms"],
|
||
ok=final_status in {"completed", "blacklisted"},
|
||
final_status=final_status,
|
||
result_state=result_payload.get("state"),
|
||
)
|
||
finalize_job_item(final_status, final_message, result_payload=result_payload)
|
||
return
|
||
|
||
self._emit_step_trace(
|
||
domain_name,
|
||
"domain",
|
||
"plan_ready",
|
||
job_item_id=job_item_id,
|
||
step_count=len(execution_order),
|
||
detect_order=",".join(execution_order),
|
||
sensitive_word_count=len(sensitive_words),
|
||
task_mode=task_mode,
|
||
)
|
||
|
||
step_index = 0
|
||
while step_index < len(execution_order):
|
||
detect_key = execution_order[step_index]
|
||
if parallel_free_steps_enabled and detect_key in parallel_free_step_set:
|
||
parallel_group = []
|
||
while step_index < len(execution_order) and execution_order[step_index] in parallel_free_step_set:
|
||
parallel_group.append(execution_order[step_index])
|
||
step_index += 1
|
||
if len(parallel_group) > 1 and parallel_free_step_workers > 1:
|
||
parallel_message = (
|
||
f"域名步骤并行组启动: domain={domain_name} | "
|
||
f"steps={','.join(parallel_group)} | workers={min(parallel_free_step_workers, len(parallel_group))}"
|
||
)
|
||
logger.info(parallel_message)
|
||
self._sync_worker_log_event(
|
||
parallel_message,
|
||
payload={
|
||
"domain_id": domain_id,
|
||
"domain": domain_name,
|
||
"steps": parallel_group,
|
||
"workers": min(parallel_free_step_workers, len(parallel_group)),
|
||
},
|
||
mode='full',
|
||
job_item_id=job_item_id,
|
||
)
|
||
parallel_results = {}
|
||
with ThreadPoolExecutor(max_workers=min(parallel_free_step_workers, len(parallel_group))) as executor:
|
||
future_map = {executor.submit(execute_detect_step, key): key for key in parallel_group}
|
||
for future in as_completed(future_map):
|
||
key = future_map[future]
|
||
parallel_results[key] = future.result()
|
||
for ordered_key in parallel_group:
|
||
step_result = parallel_results.get(ordered_key) or {
|
||
"detect_key": ordered_key,
|
||
"step_name": self._step_display_name(ordered_key),
|
||
"step_ok": False,
|
||
"step_elapsed_ms": 0,
|
||
}
|
||
self._assert_detection_session_active(
|
||
session_id,
|
||
domain_name=domain_name,
|
||
stage=f"{ordered_key}:parallel_finalize",
|
||
)
|
||
if not step_result["step_ok"]:
|
||
final_status = self._resolve_current_task_final_status()
|
||
self._emit_step_trace(
|
||
domain_name,
|
||
step_result["step_name"],
|
||
"aborted",
|
||
job_item_id=job_item_id,
|
||
detect_key=ordered_key,
|
||
elapsed_ms=step_result["step_elapsed_ms"],
|
||
ok=False,
|
||
final_status=final_status,
|
||
)
|
||
failure_reason_map = {
|
||
'detect_register': '注册状态检测未通过',
|
||
'detect_baidu_site': '百度检测未通过',
|
||
'detect_360_site': '360检测未通过',
|
||
'detect_chinaz': '站长之家检测未通过',
|
||
'detect_aizhan': '爱站检测未通过',
|
||
'detect_wayback': '时光机检测未通过',
|
||
'detect_jucha': '聚查检测未通过',
|
||
'detect_juziseo': '桔子检测未通过',
|
||
}
|
||
finalize_job_item(final_status, failure_reason_map.get(ordered_key, f'{step_result["step_name"]} 未通过'))
|
||
return
|
||
continue
|
||
if len(parallel_group) > 1:
|
||
serial_message = (
|
||
f"域名步骤串行组推进: domain={domain_name} | "
|
||
f"steps={','.join(parallel_group)} | workers=1"
|
||
)
|
||
logger.info(serial_message)
|
||
self._sync_worker_log_event(
|
||
serial_message,
|
||
payload={
|
||
"domain_id": domain_id,
|
||
"domain": domain_name,
|
||
"steps": parallel_group,
|
||
"workers": 1,
|
||
},
|
||
mode='full',
|
||
job_item_id=job_item_id,
|
||
)
|
||
for ordered_key in parallel_group:
|
||
step_result = execute_detect_step(ordered_key)
|
||
self._assert_detection_session_active(
|
||
session_id,
|
||
domain_name=domain_name,
|
||
stage=f"{ordered_key}:serial_finalize",
|
||
)
|
||
if not step_result["step_ok"]:
|
||
final_status = self._resolve_current_task_final_status()
|
||
self._emit_step_trace(
|
||
domain_name,
|
||
step_result["step_name"],
|
||
"aborted",
|
||
job_item_id=job_item_id,
|
||
detect_key=ordered_key,
|
||
elapsed_ms=step_result["step_elapsed_ms"],
|
||
ok=False,
|
||
final_status=final_status,
|
||
)
|
||
failure_reason_map = {
|
||
'detect_register': '注册状态检测未通过',
|
||
'detect_baidu_site': '百度检测未通过',
|
||
'detect_360_site': '360检测未通过',
|
||
'detect_chinaz': '站长之家检测未通过',
|
||
'detect_aizhan': '爱站检测未通过',
|
||
'detect_wayback': '时光机检测未通过',
|
||
'detect_jucha': '聚查检测未通过',
|
||
'detect_juziseo': '桔子检测未通过',
|
||
}
|
||
finalize_job_item(final_status, failure_reason_map.get(ordered_key, f'{step_result["step_name"]} 未通过'))
|
||
return
|
||
continue
|
||
|
||
step_index += 1
|
||
step_result = execute_detect_step(detect_key)
|
||
self._assert_detection_session_active(session_id, domain_name=domain_name, stage=f"{detect_key}:finalize")
|
||
if not step_result["step_ok"]:
|
||
final_status = self._resolve_current_task_final_status()
|
||
self._emit_step_trace(
|
||
domain_name,
|
||
step_result["step_name"],
|
||
"aborted",
|
||
job_item_id=job_item_id,
|
||
detect_key=detect_key,
|
||
elapsed_ms=step_result["step_elapsed_ms"],
|
||
ok=False,
|
||
final_status=final_status,
|
||
)
|
||
failure_reason_map = {
|
||
'detect_register': '注册状态检测未通过',
|
||
'detect_baidu_site': '百度检测未通过',
|
||
'detect_360_site': '360检测未通过',
|
||
'detect_chinaz': '站长之家检测未通过',
|
||
'detect_aizhan': '爱站检测未通过',
|
||
'detect_wayback': '时光机检测未通过',
|
||
'detect_jucha': '聚查检测未通过',
|
||
'detect_juziseo': '桔子检测未通过',
|
||
}
|
||
finalize_job_item(final_status, failure_reason_map.get(detect_key, f'{step_result["step_name"]} 未通过'))
|
||
return
|
||
|
||
self._assert_detection_session_active(session_id, domain_name=domain_name, stage="completion")
|
||
completion_tail_started_at = time.perf_counter()
|
||
complete_detection_started_at = time.perf_counter()
|
||
self._complete_detection(domain_id, domain_name, domain)
|
||
complete_detection_elapsed_ms = int((time.perf_counter() - complete_detection_started_at) * 1000)
|
||
self._set_task_detect_status(DETECT_STATUS_COMPLETED)
|
||
finalize_enqueue_started_at = time.perf_counter()
|
||
finalize_job_item('completed', f"域名检测完成: {domain_name}")
|
||
finalize_enqueue_elapsed_ms = int((time.perf_counter() - finalize_enqueue_started_at) * 1000)
|
||
trace_emit_started_at = time.perf_counter()
|
||
if str(os.getenv("DOMAINCHECK_DOMAIN_COMPLETED_TRACE", "0") or "0").strip().lower() not in {"", "0", "false", "no", "off"}:
|
||
self._emit_step_trace(
|
||
domain_name,
|
||
"domain",
|
||
"completed",
|
||
job_item_id=job_item_id,
|
||
elapsed_ms=int((time.perf_counter() - domain_started_at) * 1000),
|
||
ok=True,
|
||
sync_mode='off',
|
||
)
|
||
trace_emit_elapsed_ms = int((time.perf_counter() - trace_emit_started_at) * 1000)
|
||
logger.info(f"域名检测完成: {domain_name}")
|
||
completion_log_sync_started_at = time.perf_counter()
|
||
self._sync_worker_log_event(
|
||
f"域名检测完成: {domain_name}",
|
||
payload={
|
||
"domain_id": domain_id,
|
||
"domain": domain_name,
|
||
"status": "completed",
|
||
},
|
||
mode='full',
|
||
job_item_id=job_item_id,
|
||
)
|
||
completion_log_sync_elapsed_ms = int((time.perf_counter() - completion_log_sync_started_at) * 1000)
|
||
completion_tail_elapsed_ms = int((time.perf_counter() - completion_tail_started_at) * 1000)
|
||
if (
|
||
completion_tail_elapsed_ms >= 20
|
||
or complete_detection_elapsed_ms >= 10
|
||
or trace_emit_elapsed_ms >= 10
|
||
or completion_log_sync_elapsed_ms >= 10
|
||
):
|
||
logger.info(
|
||
f"域名完成尾部耗时: domain={domain_name} | total_ms={completion_tail_elapsed_ms} "
|
||
f"| complete_detection_ms={complete_detection_elapsed_ms} "
|
||
f"| finalize_enqueue_ms={finalize_enqueue_elapsed_ms} "
|
||
f"| trace_emit_ms={trace_emit_elapsed_ms} "
|
||
f"| completion_log_enqueue_ms={completion_log_sync_elapsed_ms}"
|
||
)
|
||
except RuntimeError as e:
|
||
if str(e).startswith("检测会话已失效:"):
|
||
self._release_job_item_for_session_abort(
|
||
job_item_id,
|
||
claim_token,
|
||
session_id=session_id,
|
||
domain_name=domain_name,
|
||
detail=str(e),
|
||
)
|
||
logger.info(f"检测会话切换，终止旧域名任务: {domain_name}, 详情: {e}")
|
||
self._emit_step_trace(
|
||
domain_name,
|
||
"domain",
|
||
"aborted",
|
||
job_item_id=job_item_id,
|
||
elapsed_ms=int((time.perf_counter() - domain_started_at) * 1000),
|
||
ok=False,
|
||
error=str(e)[:300],
|
||
)
|
||
return
|
||
raise
|
||
except Exception as e:
|
||
logger.error(f"检测域名出错: {domain_name}, 错误: {e}")
|
||
if not is_step_task:
|
||
self.db.update_domain_detect_status(domain_id, DETECT_STATUS_FAILED)
|
||
self._set_task_detect_status(DETECT_STATUS_FAILED)
|
||
finalize_job_item('failed', str(e))
|
||
self._emit_step_trace(
|
||
domain_name,
|
||
"domain",
|
||
"failed",
|
||
job_item_id=job_item_id,
|
||
elapsed_ms=int((time.perf_counter() - domain_started_at) * 1000),
|
||
ok=False,
|
||
error=str(e)[:300],
|
||
)
|
||
self._sync_worker_log_event(
|
||
f"检测域名出错: {domain_name}, 错误: {e}",
|
||
level='error',
|
||
payload={
|
||
"domain_id": domain_id,
|
||
"domain": domain_name,
|
||
"status": "failed",
|
||
},
|
||
mode='full',
|
||
job_item_id=job_item_id,
|
||
)
|
||
finally:
|
||
self._clear_current_task_context()
|
||
remaining_active_threads = self._change_active_domain_threads(-1)
|
||
if self.detecting:
|
||
self._mark_detection_phase(
|
||
"running",
|
||
f"当前实际线程数量: {remaining_active_threads}/{self.thread_count}",
|
||
active_threads=remaining_active_threads,
|
||
max_threads=self.thread_count,
|
||
)
|
||
|
||
def start_detection(self, session_id: int = 0):
|
||
"""
|
||
开始检测
|
||
"""
|
||
logger.info("开始执行域名检测任务")
|
||
self.detecting = True
|
||
self.stop_requested = False
|
||
total_processed = 0
|
||
final_phase = "idle"
|
||
final_detail = "Worker 等待下一次启动"
|
||
self._mark_detection_phase("preparing", "开始执行域名检测任务，正在加载配置")
|
||
self._sync_worker_log_event("开始执行域名检测任务，正在加载配置", mode='key')
|
||
|
||
# 重新加载配置，确保获取最新的配置
|
||
try:
|
||
self.detect_options = self.load_detect_options()
|
||
self.proxy_config = self.load_proxy_config()
|
||
self.thread_count = self.load_thread_count()
|
||
self._last_thread_count_refresh_at = time.time()
|
||
self.proxy_max_reuse_count = max(8, self.thread_count * 4)
|
||
logger.info(f"检测线程数设置为: {self.thread_count}")
|
||
self.load_cookies_from_remote()
|
||
self._init_optional_detectors()
|
||
recycled = self.db.recycle_running_domains(DETECT_STATUS_FAILED)
|
||
if recycled:
|
||
logger.warning(f"检测启动前回收了 {recycled} 个遗留的检测中任务，已标记为检测失败待重试")
|
||
released_job_items = self._maybe_release_node_job_items(reason="start_detection")
|
||
if released_job_items:
|
||
logger.warning(f"检测启动前释放了 {released_job_items} 个当前节点遗留任务项，已重新回到 pending")
|
||
except Exception as e:
|
||
logger.error(f"加载配置失败: {e}")
|
||
import traceback
|
||
logger.error(traceback.format_exc())
|
||
self._mark_detection_phase("failed", f"加载配置失败: {e}")
|
||
self._sync_worker_log_event(f"加载配置失败: {e}", level='error', mode='key')
|
||
final_phase = "failed"
|
||
final_detail = f"加载配置失败: {e}"
|
||
return
|
||
|
||
# 重新加载cookies
|
||
self.load_cookies_from_remote()
|
||
|
||
# 重新加载JC实例的cookies
|
||
if self.detect_options.get('detect_jucha') and self.jc_instance is not None:
|
||
self.jc_instance.load_cookies()
|
||
self.jc_instance.load_juming_cookies()
|
||
|
||
# 重新加载Juziseo实例的cookies
|
||
if self.detect_options.get('detect_juziseo') and self.juziseo_instance is not None:
|
||
self.juziseo_instance.load_cookies()
|
||
|
||
# 刷新代理池
|
||
if self.proxy_config.get('proxy_enable', False):
|
||
logger.info("开始检测，刷新代理池")
|
||
self._mark_detection_phase("refreshing_proxy", "开始检测，正在刷新代理池")
|
||
self._sync_worker_log_event("开始检测，正在刷新代理池", mode='key')
|
||
proxy_ready, proxy_count, proxy_ready_reason = self._prepare_proxy_pool_for_detection_start()
|
||
if proxy_ready:
|
||
logger.info(
|
||
f"检测启动前代理池已就绪，跳过同步刷新: "
|
||
f"count={proxy_count}, reason={proxy_ready_reason}"
|
||
)
|
||
self._update_runtime_state(
|
||
"running",
|
||
f"检测启动前代理池已就绪，直接进入任务派发: count={proxy_count}, reason={proxy_ready_reason}",
|
||
)
|
||
self._sync_worker_log_event(
|
||
"检测启动前代理池已就绪，直接进入任务派发",
|
||
payload={"proxy_count": proxy_count, "reason": proxy_ready_reason},
|
||
mode='full',
|
||
)
|
||
elif self.proxy_refresh_lock.locked():
|
||
if self._wait_for_proxy_refresh_settle(timeout_seconds=2.0):
|
||
logger.info("检测启动前等待到了可用代理，直接复用当前代理池")
|
||
else:
|
||
logger.info("检测启动前代理池仍未就绪，继续主动刷新一次")
|
||
self.refresh_proxy_pool()
|
||
else:
|
||
self.refresh_proxy_pool()
|
||
|
||
# 更新JC和Juziseo实例的代理设置
|
||
if self.detect_options.get('detect_jucha') and self.jc_instance is not None:
|
||
self.jc_instance.proxies = self.get_proxies()
|
||
if self.detect_options.get('detect_juziseo') and self.juziseo_instance is not None:
|
||
self.juziseo_instance.proxies = self.get_proxies()
|
||
|
||
# 重新加载敏感词，确保获取最新的敏感词列表
|
||
try:
|
||
self.sensitive_words = self._load_sensitive_words_runtime()
|
||
logger.info(f"重新加载敏感词完成，共 {len(self.sensitive_words)} 个敏感词")
|
||
except Exception as e:
|
||
logger.error(f"重新加载敏感词失败: {e}")
|
||
import traceback
|
||
logger.error(traceback.format_exc())
|
||
self.sensitive_words = []
|
||
|
||
# 更新配置标签
|
||
self.update_config_labels()
|
||
|
||
try:
|
||
runtime_mode_summary = {
|
||
"qt_platform": str(os.environ.get("QT_QPA_PLATFORM", "") or "").strip() or "desktop",
|
||
"worker_mode": str(os.environ.get("WORKER_MODE", "") or "").strip() or "default",
|
||
"redis_control": bool(self.use_redis),
|
||
"log_sync_mode": self._worker_log_sync_mode(),
|
||
"step_trace_enabled": self._worker_step_trace_enabled(),
|
||
}
|
||
runtime_mode_message = (
|
||
"检测运行模式摘要: "
|
||
f"core=detect_worker.py | qt_platform={runtime_mode_summary['qt_platform']} | "
|
||
f"worker_mode={runtime_mode_summary['worker_mode']} | redis_control={1 if runtime_mode_summary['redis_control'] else 0} | "
|
||
f"log_sync_mode={runtime_mode_summary['log_sync_mode']} | "
|
||
f"step_trace_enabled={1 if runtime_mode_summary['step_trace_enabled'] else 0}"
|
||
)
|
||
logger.info(runtime_mode_message)
|
||
self._sync_worker_log_event(runtime_mode_message, payload=runtime_mode_summary, mode='full')
|
||
pending_domains = collections.deque()
|
||
inflight_futures = set()
|
||
legacy_tail_reached = False
|
||
last_thread_progress_emit_count = 0
|
||
last_thread_progress_emit_at = 0.0
|
||
last_job_status_refresh_at = 0.0
|
||
last_expired_job_recycle_at = 0.0
|
||
last_future_fallback_scan_at = 0.0
|
||
thread_count_refresh_interval = max(
|
||
0.5,
|
||
float(os.getenv("DOMAINCHECK_THREAD_COUNT_REFRESH_INTERVAL", "2.0") or 2.0),
|
||
)
|
||
job_status_refresh_interval = max(
|
||
5.0,
|
||
float(os.getenv("DOMAINCHECK_JOB_STATUS_REFRESH_INTERVAL", "15.0") or 15.0),
|
||
)
|
||
worker_pool_size = max(
|
||
1,
|
||
int(
|
||
self.refresh_thread_count_runtime(
|
||
force=True,
|
||
min_interval=thread_count_refresh_interval,
|
||
)
|
||
or 1
|
||
),
|
||
)
|
||
configured_thread_stack_size = _configure_worker_thread_stack_size()
|
||
worker_pool = ThreadPoolExecutor(
|
||
max_workers=worker_pool_size,
|
||
thread_name_prefix="detect-domain",
|
||
)
|
||
last_worker_pool_size = worker_pool_size
|
||
if configured_thread_stack_size > 0:
|
||
logger.info(
|
||
f"检测线程池初始化: max_workers={worker_pool_size}, thread_stack_size={configured_thread_stack_size}"
|
||
)
|
||
|
||
def prune_finished_futures():
|
||
nonlocal inflight_futures, last_future_fallback_scan_at
|
||
if not inflight_futures:
|
||
return 0
|
||
finished = []
|
||
with self._completed_future_lock:
|
||
while self._completed_futures and len(finished) < 4096:
|
||
finished.append(self._completed_futures.popleft())
|
||
if not finished:
|
||
now_ts = time.time()
|
||
if now_ts - last_future_fallback_scan_at < 0.5:
|
||
return 0
|
||
last_future_fallback_scan_at = now_ts
|
||
finished = [future for future in list(inflight_futures) if future.done()]
|
||
if not finished:
|
||
return 0
|
||
pruned_count = 0
|
||
for future in dict.fromkeys(finished):
|
||
if future not in inflight_futures:
|
||
continue
|
||
inflight_futures.discard(future)
|
||
pruned_count += 1
|
||
try:
|
||
future.result()
|
||
except Exception as future_error:
|
||
logger.error(f"检测任务 future 执行异常: {future_error}")
|
||
import traceback
|
||
logger.error(traceback.format_exc())
|
||
return pruned_count
|
||
|
||
def emit_thread_progress(max_threads, *, batch_size=0):
|
||
nonlocal last_thread_progress_emit_count, last_thread_progress_emit_at
|
||
current_active = max(len(inflight_futures), self._get_active_domain_threads())
|
||
should_emit_progress, emit_now = self._should_emit_thread_progress(
|
||
current_active,
|
||
max_threads,
|
||
last_thread_progress_emit_count,
|
||
last_thread_progress_emit_at,
|
||
)
|
||
if should_emit_progress:
|
||
logger.info(f"当前实际线程数量: {current_active}/{max_threads}")
|
||
self._mark_detection_phase(
|
||
"running",
|
||
f"当前实际线程数量: {current_active}/{max_threads}",
|
||
active_threads=current_active,
|
||
max_threads=max_threads,
|
||
batch_size=batch_size,
|
||
processed=total_processed,
|
||
pending_buffer=len(pending_domains),
|
||
)
|
||
last_thread_progress_emit_count = current_active
|
||
last_thread_progress_emit_at = emit_now
|
||
if self.detect_thread:
|
||
try:
|
||
self.detect_thread.thread_count_signal.emit(current_active, max_threads)
|
||
except Exception as e:
|
||
logger.error(f"发送线程数量更新信号失败: {e}")
|
||
import traceback
|
||
logger.error(traceback.format_exc())
|
||
|
||
def inflight_submission_metrics(max_threads):
|
||
live_active = max(0, int(self._get_active_domain_threads() or 0))
|
||
submitted_total = len(inflight_futures)
|
||
queued_backlog = max(0, submitted_total - live_active)
|
||
backlog_limit = self._resolve_submit_backlog_limit(max_threads, live_active)
|
||
dispatch_capacity = self._resolve_dispatch_capacity(max_threads, live_active, backlog_limit)
|
||
effective_available = max(0, dispatch_capacity - submitted_total)
|
||
return live_active, submitted_total, queued_backlog, backlog_limit, dispatch_capacity, effective_available
|
||
|
||
def sync_worker_pool_size(desired_threads):
|
||
nonlocal last_worker_pool_size
|
||
normalized_threads = max(1, int(desired_threads or 1))
|
||
current_pool_size = max(1, int(getattr(worker_pool, "_max_workers", normalized_threads) or normalized_threads))
|
||
if current_pool_size != normalized_threads:
|
||
worker_pool._max_workers = normalized_threads
|
||
logger.info(f"动态调整 worker_pool 大小: {current_pool_size} -> {normalized_threads}")
|
||
self._sync_worker_log_event(
|
||
f"动态调整 worker_pool 大小: {current_pool_size} -> {normalized_threads}",
|
||
payload={
|
||
"previous_max_workers": current_pool_size,
|
||
"current_max_workers": normalized_threads,
|
||
},
|
||
mode='full',
|
||
)
|
||
last_worker_pool_size = normalized_threads
|
||
elif last_worker_pool_size != normalized_threads:
|
||
last_worker_pool_size = normalized_threads
|
||
return normalized_threads
|
||
|
||
def append_pending_domains(domains, using_job_queue, queue_label, claim_elapsed_ms, legacy_fetch_elapsed_ms, thread_limit, claim_batch_size, claim_lease_seconds):
|
||
current_batch_size = len(domains)
|
||
if current_batch_size <= 0:
|
||
return
|
||
for domain in domains:
|
||
pending_domains.append(
|
||
{
|
||
"domain": domain,
|
||
"using_job_queue": bool(using_job_queue),
|
||
}
|
||
)
|
||
fetch_trace_payload = {
|
||
"queue_label": queue_label,
|
||
"batch_size": current_batch_size,
|
||
"claim_elapsed_ms": claim_elapsed_ms,
|
||
"legacy_fetch_elapsed_ms": legacy_fetch_elapsed_ms,
|
||
"thread_limit": thread_limit,
|
||
"claim_batch_size": claim_batch_size,
|
||
"claim_lease_seconds": claim_lease_seconds,
|
||
"pending_buffer": len(pending_domains),
|
||
}
|
||
fetch_trace_message = (
|
||
f"批次领取跟踪: queue={queue_label} | batch_size={current_batch_size} | "
|
||
f"claim_elapsed_ms={claim_elapsed_ms} | legacy_fetch_elapsed_ms={legacy_fetch_elapsed_ms} | "
|
||
f"thread_limit={thread_limit} | claim_batch_size={claim_batch_size} | "
|
||
f"claim_lease_seconds={claim_lease_seconds} | pending_buffer={len(pending_domains)}"
|
||
)
|
||
logger.info(fetch_trace_message)
|
||
self._sync_worker_log_event(fetch_trace_message, payload=fetch_trace_payload, mode='full')
|
||
self._mark_detection_phase(
|
||
"fetching",
|
||
f"从{queue_label}获取到 {current_batch_size} 个需要检测的域名",
|
||
batch_size=current_batch_size,
|
||
queue_source="detect_job_items" if using_job_queue else "domains",
|
||
pending_buffer=len(pending_domains),
|
||
)
|
||
self._sync_worker_log_event(
|
||
f"从{queue_label}获取到 {current_batch_size} 个需要检测的域名",
|
||
payload={
|
||
"batch_size": current_batch_size,
|
||
"queue_source": "detect_job_items" if using_job_queue else "domains",
|
||
"pending_buffer": len(pending_domains),
|
||
},
|
||
mode='full',
|
||
)
|
||
|
||
def compute_claim_batch_size(current_limit, available_slots):
|
||
return self._resolve_claim_batch_size(current_limit, available_slots)
|
||
|
||
last_dispatchable_step_items_check_at = 0.0
|
||
dispatchable_step_items_cached = False
|
||
legacy_domain_fallback_enabled = bool(
|
||
int(os.getenv("DOMAINCHECK_ENABLE_LEGACY_DOMAIN_FALLBACK", "0") or 0)
|
||
)
|
||
last_legacy_fallback_skip_log_at = 0.0
|
||
last_proactive_pipeline_topup_at = 0.0
|
||
last_proactive_sync_topup_at = 0.0
|
||
|
||
def has_dispatchable_step_items():
|
||
nonlocal last_dispatchable_step_items_check_at, dispatchable_step_items_cached
|
||
now_ts = time.time()
|
||
if now_ts - float(last_dispatchable_step_items_check_at or 0.0) < 1.0:
|
||
return dispatchable_step_items_cached
|
||
try:
|
||
dispatchable_step_items_cached = bool(self.db.has_dispatchable_detect_job_items())
|
||
except Exception as e:
|
||
logger.debug(f"检查标准步骤待派发任务失败: {e}")
|
||
last_dispatchable_step_items_check_at = now_ts
|
||
return dispatchable_step_items_cached
|
||
|
||
def start_domain_thread(domain, using_job_queue, max_threads):
|
||
nonlocal total_processed
|
||
domain_id = domain['id']
|
||
try:
|
||
job_context = {
|
||
'cycle_token': self.current_cycle_token,
|
||
'job_code': domain.get('job_code') or self.current_job_code,
|
||
'session_id': session_id,
|
||
}
|
||
if using_job_queue:
|
||
job_context.update(
|
||
{
|
||
'job_item_id': domain.get('job_item_id'),
|
||
'job_id': domain.get('job_id'),
|
||
'claim_token': domain.get('claim_token'),
|
||
'task_mode': domain.get('task_mode'),
|
||
'step_code': domain.get('step_code'),
|
||
'step_payload': domain.get('step_payload'),
|
||
}
|
||
)
|
||
future = worker_pool.submit(self.detect_domain, domain_id, domain, job_context)
|
||
future.add_done_callback(self._enqueue_completed_future)
|
||
inflight_futures.add(future)
|
||
total_processed += 1
|
||
if self.detect_thread:
|
||
try:
|
||
self.detect_thread.progress_signal.emit(total_processed, total_processed + len(pending_domains) + len(inflight_futures))
|
||
except Exception as e:
|
||
logger.error(f"发送进度更新信号失败: {e}")
|
||
import traceback
|
||
logger.error(traceback.format_exc())
|
||
emit_thread_progress(max_threads, batch_size=len(pending_domains) + len(inflight_futures))
|
||
return True
|
||
except Exception as e:
|
||
logger.error(f"创建或启动线程失败: {e}")
|
||
import traceback
|
||
logger.error(traceback.format_exc())
|
||
return False
|
||
|
||
try:
|
||
while self.running:
|
||
abort_reason = self._detection_session_abort_reason(session_id)
|
||
if abort_reason:
|
||
logger.warning(f"检测会话已切换，停止旧会话主循环: {abort_reason}")
|
||
self._mark_detection_phase("restarting", f"检测会话已切换，停止旧会话主循环: {abort_reason}")
|
||
self._sync_worker_log_event(
|
||
f"检测会话已切换，停止旧会话主循环: {abort_reason}",
|
||
level='warning',
|
||
mode='key',
|
||
)
|
||
final_phase = "restarting"
|
||
final_detail = f"检测会话已切换，停止旧会话主循环: {abort_reason}"
|
||
break
|
||
if self.stop_requested:
|
||
logger.info("检测任务收到停止请求，停止继续领取任务")
|
||
self._mark_detection_phase("stopping", "检测任务收到停止请求，准备安全退出")
|
||
self._sync_worker_log_event("检测任务收到停止请求，准备安全退出", level='warning', mode='key')
|
||
final_phase = "stopped"
|
||
final_detail = "检测任务收到停止请求，已停止继续领取任务"
|
||
break
|
||
prune_finished_futures()
|
||
self._flush_pending_running_marks()
|
||
self._flush_pending_job_releases()
|
||
self._flush_pending_job_finalizations()
|
||
self._flush_pending_domain_completions()
|
||
self._flush_pending_domain_status_updates()
|
||
self._flush_pending_review_status_updates()
|
||
current_thread_limit = self.refresh_thread_count_runtime(
|
||
min_interval=thread_count_refresh_interval
|
||
)
|
||
max_threads = sync_worker_pool_size(current_thread_limit)
|
||
live_active, submitted_total, queued_backlog, backlog_limit, dispatch_capacity, available_slots = inflight_submission_metrics(max_threads)
|
||
now_ts = time.time()
|
||
if now_ts - last_expired_job_recycle_at >= 2.0:
|
||
recycle_lock_owner = self._acquire_shared_job_maintenance_lock(
|
||
"recycle-expired-job-items",
|
||
ttl_seconds=10,
|
||
)
|
||
if recycle_lock_owner:
|
||
try:
|
||
recycled_job_items = self.db.recycle_expired_detect_job_items()
|
||
last_expired_job_recycle_at = now_ts
|
||
if recycled_job_items:
|
||
logger.warning(f"已回收 {recycled_job_items} 个租约过期的任务项，重新回到 pending")
|
||
finally:
|
||
self._release_shared_job_maintenance_lock(
|
||
"recycle-expired-job-items",
|
||
recycle_lock_owner,
|
||
)
|
||
if self.current_job_id and now_ts - last_job_status_refresh_at >= job_status_refresh_interval:
|
||
self._log_job_status_refresh_probe(self.current_job_id)
|
||
self.db.refresh_detect_job_status(self.current_job_id)
|
||
last_job_status_refresh_at = now_ts
|
||
pending_buffer_cap_multiplier = self._runtime_positive_int_override(
|
||
"pending_buffer_cap_multiplier",
|
||
"DOMAINCHECK_PENDING_BUFFER_CAP_MULTIPLIER",
|
||
default=1,
|
||
)
|
||
pending_buffer_target = min(
|
||
max(4, backlog_limit),
|
||
max(4, int(max_threads or 0) * pending_buffer_cap_multiplier),
|
||
)
|
||
single_step_session_active = self._is_single_step_session_active()
|
||
refill_slots = max(0, min(int(available_slots or 0), pending_buffer_target - len(pending_domains)))
|
||
if refill_slots > 0:
|
||
claim_batch_size = compute_claim_batch_size(max_threads, refill_slots)
|
||
claim_lease_seconds = max(300, min(1800, claim_batch_size * 30))
|
||
# single_step 必须绑定当前 job；domain_pipeline 则持续从全局任务队列补位，
|
||
# 否则旧 job 未完全结束时，新 job 会长期堆在 pending。
|
||
scoped_job_id = self._current_scoped_claim_job_id()
|
||
if scoped_job_id not in (None, "", 0, "0"):
|
||
self._log_explicit_scope_probe(
|
||
scoped_job_id=scoped_job_id,
|
||
max_threads=max_threads,
|
||
refill_slots=refill_slots,
|
||
claim_batch_size=claim_batch_size,
|
||
pending_buffer=len(pending_domains),
|
||
)
|
||
claim_started_at = time.perf_counter()
|
||
domains = self._claim_detect_job_items(
|
||
limit=claim_batch_size,
|
||
lease_seconds=claim_lease_seconds,
|
||
job_id=scoped_job_id,
|
||
)
|
||
claim_elapsed_ms = int((time.perf_counter() - claim_started_at) * 1000)
|
||
legacy_fetch_elapsed_ms = 0
|
||
proactive_topup_needed = (
|
||
not single_step_session_active
|
||
and self._should_proactively_top_up_pipeline(len(domains), refill_slots, max_threads)
|
||
)
|
||
if domains:
|
||
append_pending_domains(
|
||
domains,
|
||
True,
|
||
"任务队列",
|
||
claim_elapsed_ms,
|
||
legacy_fetch_elapsed_ms,
|
||
dispatch_capacity,
|
||
claim_batch_size,
|
||
claim_lease_seconds,
|
||
)
|
||
if proactive_topup_needed:
|
||
proactive_now = time.time()
|
||
if proactive_now - float(last_proactive_pipeline_topup_at or 0.0) >= 0.5:
|
||
pipeline_domains = self._process_pipeline_tasks_until_available(thread_limit=max_threads)
|
||
last_proactive_pipeline_topup_at = proactive_now
|
||
if pipeline_domains:
|
||
append_pending_domains(
|
||
pipeline_domains,
|
||
True,
|
||
"任务队列(pipeline推进补位)",
|
||
claim_elapsed_ms,
|
||
legacy_fetch_elapsed_ms,
|
||
dispatch_capacity,
|
||
claim_batch_size,
|
||
claim_lease_seconds,
|
||
)
|
||
elif proactive_now - float(last_proactive_sync_topup_at or 0.0) >= 1.0:
|
||
refill_domains = self._pull_sync_tasks_until_available(
|
||
thread_limit=max_threads,
|
||
claim_after_pull=False,
|
||
)
|
||
last_proactive_sync_topup_at = proactive_now
|
||
if refill_domains:
|
||
append_pending_domains(
|
||
refill_domains,
|
||
True,
|
||
"任务队列(主动补货补位)",
|
||
claim_elapsed_ms,
|
||
legacy_fetch_elapsed_ms,
|
||
dispatch_capacity,
|
||
claim_batch_size,
|
||
claim_lease_seconds,
|
||
)
|
||
if self._should_prefetch_sync_tasks(len(pending_domains), len(inflight_futures), max_threads):
|
||
proactive_now = time.time()
|
||
if proactive_now - float(last_proactive_sync_topup_at or 0.0) >= 1.0:
|
||
self._pull_sync_tasks_until_available(
|
||
thread_limit=max_threads,
|
||
claim_after_pull=False,
|
||
)
|
||
last_proactive_sync_topup_at = proactive_now
|
||
elif not single_step_session_active:
|
||
pipeline_domains = self._process_pipeline_tasks_until_available(thread_limit=max_threads)
|
||
if pipeline_domains:
|
||
append_pending_domains(
|
||
pipeline_domains,
|
||
True,
|
||
"任务队列(pipeline推进)",
|
||
claim_elapsed_ms,
|
||
legacy_fetch_elapsed_ms,
|
||
dispatch_capacity,
|
||
claim_batch_size,
|
||
claim_lease_seconds,
|
||
)
|
||
else:
|
||
refill_domains = self._pull_sync_tasks_until_available(thread_limit=max_threads)
|
||
if refill_domains:
|
||
append_pending_domains(
|
||
refill_domains,
|
||
True,
|
||
"任务队列(主动补货)",
|
||
claim_elapsed_ms,
|
||
legacy_fetch_elapsed_ms,
|
||
dispatch_capacity,
|
||
claim_batch_size,
|
||
claim_lease_seconds,
|
||
)
|
||
elif has_dispatchable_step_items():
|
||
logger.info("标准步骤队列仍有待派发任务，跳过兼容旧链路补位")
|
||
elif legacy_domain_fallback_enabled and not legacy_tail_reached:
|
||
legacy_fetch_limit = max(1, claim_batch_size)
|
||
legacy_fetch_started_at = time.perf_counter()
|
||
legacy_domains = self.db.get_domains_to_detect(limit=legacy_fetch_limit, detect_options=self.detect_options)
|
||
legacy_fetch_elapsed_ms = int((time.perf_counter() - legacy_fetch_started_at) * 1000)
|
||
if legacy_domains:
|
||
append_pending_domains(
|
||
legacy_domains,
|
||
False,
|
||
"兼容旧链路",
|
||
claim_elapsed_ms,
|
||
legacy_fetch_elapsed_ms,
|
||
dispatch_capacity,
|
||
claim_batch_size,
|
||
claim_lease_seconds,
|
||
)
|
||
if len(legacy_domains) < legacy_fetch_limit:
|
||
legacy_tail_reached = True
|
||
elif not legacy_domain_fallback_enabled:
|
||
current_ts = time.time()
|
||
if current_ts - float(last_legacy_fallback_skip_log_at or 0.0) >= 5.0:
|
||
logger.info("兼容旧链路补位已禁用，当前仅允许 detect_job_items 标准步骤补位")
|
||
last_legacy_fallback_skip_log_at = current_ts
|
||
else:
|
||
logger.info("single_step 会话进行中，跳过 pipeline 推进 / 主动补货 / 兼容旧链路补位")
|
||
|
||
dispatched_this_round = 0
|
||
while pending_domains:
|
||
live_active, submitted_total, queued_backlog, backlog_limit, dispatch_capacity, effective_available = inflight_submission_metrics(max_threads)
|
||
if effective_available <= 0:
|
||
break
|
||
if not self.running or self.stop_requested:
|
||
logger.info("检测已停止，停止创建新线程")
|
||
self._mark_detection_phase("stopping", "检测已停止，停止创建新线程")
|
||
final_phase = "stopped"
|
||
final_detail = "检测已停止，停止创建新线程"
|
||
break
|
||
pending_entry = pending_domains.popleft()
|
||
domain = pending_entry.get("domain") or {}
|
||
using_job_queue = bool(pending_entry.get("using_job_queue"))
|
||
if start_domain_thread(domain, using_job_queue, max_threads):
|
||
dispatched_this_round += 1
|
||
# 派发大批量任务时，尽早把刚启动的任务项从 claimed 刷到 running，
|
||
# 避免数据库状态长期滞后，影响 controller 对真实活跃量的判断。
|
||
if dispatched_this_round % 64 == 0:
|
||
self._flush_pending_running_marks()
|
||
|
||
if dispatched_this_round:
|
||
dispatch_summary_message = (
|
||
f"持续补位派发完成: dispatched={dispatched_this_round} | "
|
||
f"active_threads={len(inflight_futures)} | pending_buffer={len(pending_domains)} | "
|
||
f"max_threads={max_threads} | queued_backlog={queued_backlog} | backlog_limit={backlog_limit}"
|
||
)
|
||
logger.info(dispatch_summary_message)
|
||
self._sync_worker_log_event(
|
||
dispatch_summary_message,
|
||
payload={
|
||
"dispatched": dispatched_this_round,
|
||
"active_threads": len(inflight_futures),
|
||
"pending_buffer": len(pending_domains),
|
||
"max_threads": max_threads,
|
||
"queued_backlog": queued_backlog,
|
||
"backlog_limit": backlog_limit,
|
||
},
|
||
mode='full',
|
||
)
|
||
|
||
if not pending_domains and not inflight_futures:
|
||
if legacy_tail_reached:
|
||
logger.info("兼容旧链路已到尾批，且当前没有活跃线程，本轮检测完成")
|
||
self._mark_detection_phase(
|
||
"completing",
|
||
"兼容旧链路已到尾批，且当前没有活跃线程，本轮检测完成",
|
||
processed=total_processed,
|
||
)
|
||
self._sync_worker_log_event(
|
||
"兼容旧链路已到尾批，且当前没有活跃线程，本轮检测完成",
|
||
payload={"processed": total_processed},
|
||
mode='key',
|
||
)
|
||
break
|
||
logger.info("没有需要检测的域名")
|
||
self._mark_detection_phase("idle", "当前没有需要检测的域名，Worker 等待下一次启动")
|
||
self._sync_worker_log_event("当前没有需要检测的域名，Worker 等待下一次启动", mode='key')
|
||
break
|
||
|
||
emit_thread_progress(max_threads, batch_size=len(pending_domains) + len(inflight_futures))
|
||
time.sleep(0.05 if pending_domains or inflight_futures else 0.2)
|
||
finally:
|
||
self._flush_pending_running_marks(force=True, batch_limit=5000)
|
||
self._flush_pending_job_releases(force=True, batch_limit=5000)
|
||
self._flush_pending_job_finalizations(force=True, batch_limit=5000)
|
||
self._flush_pending_domain_completions(force=True, batch_limit=5000)
|
||
self._flush_pending_domain_status_updates(force=True, batch_limit=5000)
|
||
self._flush_pending_review_status_updates(force=True, batch_limit=5000)
|
||
if self.current_job_id:
|
||
self.db.refresh_detect_job_status(self.current_job_id)
|
||
can_blocking_shutdown = (
|
||
not pending_domains
|
||
and not inflight_futures
|
||
and self._get_active_domain_threads() <= 0
|
||
)
|
||
worker_pool.shutdown(wait=can_blocking_shutdown, cancel_futures=True)
|
||
if can_blocking_shutdown:
|
||
gc.collect()
|
||
|
||
# 完成进度
|
||
if self.detect_thread:
|
||
try:
|
||
# 通过信号发送进度更新
|
||
self.detect_thread.progress_signal.emit(100, 100)
|
||
except Exception as e:
|
||
logger.error(f"发送进度更新信号失败: {e}")
|
||
import traceback
|
||
logger.error(traceback.format_exc())
|
||
|
||
logger.info("域名检测任务完成")
|
||
self._mark_detection_phase("completed", "域名检测任务完成")
|
||
self._sync_worker_log_event("域名检测任务完成", payload={"processed": total_processed}, mode='key')
|
||
final_phase = "completed"
|
||
final_detail = "域名检测任务完成"
|
||
|
||
except Exception as e:
|
||
logger.error(f"执行检测任务出错: {e}")
|
||
self._mark_detection_phase("failed", f"执行检测任务出错: {e}")
|
||
self._sync_worker_log_event(f"执行检测任务出错: {e}", level='error', mode='key')
|
||
final_phase = "failed"
|
||
final_detail = f"执行检测任务出错: {e}"
|
||
finally:
|
||
self.detecting = False
|
||
if not self.running or self.stop_requested:
|
||
final_phase = "stopped"
|
||
final_detail = "检测任务已停止"
|
||
self._update_runtime_state(
|
||
final_phase,
|
||
final_detail,
|
||
active_threads=0,
|
||
max_threads=max(1, int(getattr(self, "thread_count", 1) or 1)),
|
||
processed=total_processed,
|
||
)
|
||
|
||
def run_daily_task(self):
|
||
"""
|
||
执行每日检测任务
|
||
"""
|
||
logger.info(f"[{datetime.now()}] 开始每日检测任务")
|
||
self.start_detection()
|
||
logger.info(f"[{datetime.now()}] 每日检测任务完成")
|
||
|
||
def start_scheduler(self):
|
||
"""
|
||
启动定时任务
|
||
"""
|
||
# 避免重复注册相同定时任务
|
||
schedule.clear('daily_detection')
|
||
# 每天凌晨1点执行检测；启动程序时不自动跑一轮，避免和人工启动混淆
|
||
schedule.every().day.at("01:00").do(self.run_daily_task).tag('daily_detection')
|
||
logger.info("定时任务已启动，每天凌晨1点执行检测；程序启动后不会立即自动执行")
|
||
|
||
# 循环执行定时任务
|
||
while self.running:
|
||
schedule.run_pending()
|
||
time.sleep(60)
|
||
|
||
def start(self):
|
||
"""
|
||
启动检测端
|
||
"""
|
||
logger.info("启动域名检测端")
|
||
self.running = True
|
||
|
||
# 启动定时任务
|
||
scheduler_thread = threading.Thread(target=self.start_scheduler)
|
||
scheduler_thread.daemon = True
|
||
scheduler_thread.start()
|
||
|
||
# 保持程序运行
|
||
try:
|
||
while self.running:
|
||
time.sleep(1)
|
||
except KeyboardInterrupt:
|
||
logger.info("检测端已停止")
|
||
self.running = False
|
||
|
||
def load_cookies_from_remote(self):
|
||
"""
|
||
从远程加载cookies
|
||
"""
|
||
try:
|
||
if self.use_redis:
|
||
# 加载聚名cookies
|
||
juming_cookies_str = self.redis_client.get('domain_tool:juming_cookies')
|
||
if juming_cookies_str:
|
||
# 将cookies保存到本地文件
|
||
import pickle
|
||
from requests.cookies import RequestsCookieJar
|
||
|
||
# 创建cookie jar
|
||
cookie_jar = RequestsCookieJar()
|
||
|
||
# 解析cookies字符串
|
||
try:
|
||
import ast
|
||
cookies_dict = ast.literal_eval(juming_cookies_str)
|
||
for name, value in cookies_dict.items():
|
||
cookie_jar.set(name, value)
|
||
except Exception as e:
|
||
logger.error(f"解析聚名cookies失败: {e}")
|
||
|
||
# 保存到本地文件
|
||
try:
|
||
with open('juming_cookies.pkl', 'wb') as f:
|
||
pickle.dump(cookie_jar, f)
|
||
logger.info("从Redis加载聚名cookies成功并保存到本地")
|
||
except Exception as e:
|
||
logger.error(f"保存聚名cookies到本地失败: {e}")
|
||
|
||
# 加载聚查cookies
|
||
jucha_cookies_str = self.redis_client.get('domain_tool:jucha_cookies')
|
||
if jucha_cookies_str:
|
||
# 将cookies保存到本地文件
|
||
import pickle
|
||
from requests.cookies import RequestsCookieJar
|
||
|
||
# 创建cookie jar
|
||
cookie_jar = RequestsCookieJar()
|
||
|
||
# 解析cookies字符串
|
||
try:
|
||
import ast
|
||
cookies_dict = ast.literal_eval(jucha_cookies_str)
|
||
for name, value in cookies_dict.items():
|
||
cookie_jar.set(name, value)
|
||
except Exception as e:
|
||
logger.error(f"解析聚查cookies失败: {e}")
|
||
|
||
# 保存到本地文件
|
||
try:
|
||
with open('jucha_cookies.pkl', 'wb') as f:
|
||
pickle.dump(cookie_jar, f)
|
||
logger.info("从Redis加载聚查cookies成功并保存到本地")
|
||
except Exception as e:
|
||
logger.error(f"保存聚查cookies到本地失败: {e}")
|
||
|
||
# 加载桔子SEO cookies
|
||
juziseo_cookies_str = self.redis_client.get('domain_tool:juziseo_cookies')
|
||
if juziseo_cookies_str:
|
||
# 将cookies保存到本地文件
|
||
import pickle
|
||
from requests.cookies import RequestsCookieJar
|
||
|
||
# 创建cookie jar
|
||
cookie_jar = RequestsCookieJar()
|
||
|
||
# 解析cookies字符串
|
||
try:
|
||
import ast
|
||
cookies_dict = ast.literal_eval(juziseo_cookies_str)
|
||
for name, value in cookies_dict.items():
|
||
cookie_jar.set(name, value)
|
||
except Exception as e:
|
||
logger.error(f"解析桔子SEO cookies失败: {e}")
|
||
|
||
# 保存到本地文件
|
||
try:
|
||
with open('juziseo_cookies.pkl', 'wb') as f:
|
||
pickle.dump(cookie_jar, f)
|
||
logger.info("从Redis加载桔子SEO cookies成功并保存到本地")
|
||
except Exception as e:
|
||
logger.error(f"保存桔子SEO cookies到本地失败: {e}")
|
||
except Exception as e:
|
||
logger.error(f"从远程加载cookies失败: {e}")
|
||
|
||
def update_config_labels(self):
|
||
"""
|
||
更新GUI配置标签
|
||
"""
|
||
# 使用QCoreApplication.postEvent发送配置更新事件，这是最安全的方法
|
||
try:
|
||
logger.debug("发送配置更新事件")
|
||
# 确保主窗口存在
|
||
if self.main_window:
|
||
from PySide6.QtCore import QCoreApplication
|
||
event = ConfigUpdateEvent()
|
||
QCoreApplication.postEvent(self.main_window, event)
|
||
logger.debug("配置更新事件已发送")
|
||
else:
|
||
logger.debug("主窗口不存在，跳过配置更新")
|
||
except Exception as e:
|
||
logger.error(f"发送配置更新事件失败: {e}")
|
||
|
||
|
||
|
||
def start_redis_subscription(self):
|
||
"""
|
||
启动Redis订阅，监听配置更新
|
||
"""
|
||
while self.running:
|
||
pubsub = None
|
||
try:
|
||
# 创建新的Redis客户端用于订阅
|
||
redis_sub_client = get_redis_client(role="pubsub")
|
||
|
||
# 订阅配置更新频道
|
||
pubsub = redis_sub_client.pubsub()
|
||
pubsub.subscribe(CONFIG_UPDATE_CHANNEL, CONTROL_CHANNEL)
|
||
|
||
# 使用logger.debug输出到文件日志，不输出到GUI日志框
|
||
logger.debug("开始监听配置更新...")
|
||
self._consume_pending_control_command()
|
||
self._resume_active_detect_job_if_needed(reason="redis_subscription_bootstrap")
|
||
|
||
# 循环监听消息
|
||
while self.running:
|
||
message = pubsub.get_message(ignore_subscribe_messages=True, timeout=1.0)
|
||
if not message:
|
||
continue
|
||
if message['type'] == 'message':
|
||
channel = message.get('channel', '')
|
||
payload = message['data']
|
||
if channel == CONFIG_UPDATE_CHANNEL:
|
||
config_type = payload
|
||
logger.debug(f"收到配置更新消息: {config_type}")
|
||
previous_proxy_config = dict(self.proxy_config or {})
|
||
self.detect_options = self.load_detect_options()
|
||
self.proxy_config = self.load_proxy_config()
|
||
previous_thread_count = int(getattr(self, "thread_count", 0) or 0)
|
||
self.thread_count = self.load_thread_count()
|
||
self.sensitive_words = self._load_sensitive_words_runtime()
|
||
self.proxy_max_reuse_count = max(8, self.thread_count * 4)
|
||
self.runtime_settings = self.load_runtime_settings()
|
||
self.load_cookies_from_remote()
|
||
self.update_config_labels()
|
||
should_refresh_proxy, reset_proxy_cooldown = self._should_trigger_proxy_refresh_for_config_update(
|
||
config_type=config_type,
|
||
previous_proxy_config=previous_proxy_config,
|
||
previous_thread_count=previous_thread_count,
|
||
current_thread_count=self.thread_count,
|
||
)
|
||
if should_refresh_proxy:
|
||
self.trigger_proxy_refresh(
|
||
reason=f"config_update:{config_type}",
|
||
reset_cooldown=reset_proxy_cooldown,
|
||
)
|
||
logger.debug("配置已更新")
|
||
continue
|
||
|
||
if channel == CONTROL_CHANNEL:
|
||
logger.info(f"收到 Worker 控制消息: {payload}")
|
||
self._handle_control_message(payload)
|
||
except redis.exceptions.TimeoutError:
|
||
# Pub/Sub 空闲超时在 Linux 服务态下属于常见现象，不需要中断订阅。
|
||
continue
|
||
except Exception as e:
|
||
logger.debug(f"Redis订阅失败: {e}")
|
||
logger.debug("3秒后尝试重新连接...")
|
||
time.sleep(3) # 增加重试间隔
|
||
finally:
|
||
if pubsub is not None:
|
||
try:
|
||
pubsub.close()
|
||
except Exception:
|
||
pass
|
||
|
||
def stop(self):
|
||
"""
|
||
停止检测端
|
||
"""
|
||
logger.info("停止域名检测端")
|
||
self._runtime_heartbeat_stop.set()
|
||
self._worker_log_sync_stop.set()
|
||
try:
|
||
self._worker_log_sync_queue.put_nowait(None)
|
||
except Exception:
|
||
pass
|
||
self.stop_requested = True
|
||
self.running = False
|
||
self.detecting = False
|
||
self._update_runtime_state("stopped", "Worker 已停止")
|
||
|
||
if __name__ == "__main__":
|
||
try:
|
||
# 配置日志
|
||
log_file_path = _resolve_worker_log_file()
|
||
logger.add(log_file_path, rotation="1 day", level="DEBUG")
|
||
_raise_nofile_soft_limit()
|
||
logger.info("程序开始运行")
|
||
|
||
# 创建应用程序
|
||
app = QApplication([])
|
||
logger.info("创建应用程序成功")
|
||
|
||
# 创建并显示主窗口
|
||
window = DetectMainWindow()
|
||
logger.info("创建主窗口成功")
|
||
window.show()
|
||
logger.info("显示主窗口成功")
|
||
|
||
# 运行应用程序
|
||
logger.info("开始运行应用程序")
|
||
app.exec()
|
||
logger.info("应用程序运行结束")
|
||
|
||
# 当窗口关闭时，停止检测端
|
||
if window.worker:
|
||
logger.info("停止检测端")
|
||
window.worker.stop()
|
||
except Exception as e:
|
||
logger.error(f"程序运行出错: {e}")
|
||
import traceback
|
||
logger.error(traceback.format_exc())
|
||
import time
|
||
time.sleep(10) # 等待10秒，以便查看错误信息
|